diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24701e838..fc2571f3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -757,3 +757,103 @@ jobs: with: path: .codex-artifacts/ci/timings.json key: ci-node-timings-v1-${{ github.run_id }}-${{ github.run_attempt }} + + timing-telemetry: + name: CI timing telemetry + needs: [scope, quality] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 3 + continue-on-error: true + permissions: + actions: read + contents: read + steps: + - name: Check out timing telemetry without retained credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 + with: + node-version: 22.22.1 + + - name: Download available shard reports including failures + if: always() && needs.scope.outputs.execution-tier == 'complete' && (needs.scope.outputs.daemon == 'true' || contains(needs.scope.outputs.test-packages, '"@ling-stack/core"')) + continue-on-error: true + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + pattern: ci-test-report-* + merge-multiple: true + path: .codex-artifacts/ci/reports + + - name: Collect workflow timing metadata + if: always() + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -u + set +e + mkdir -p .codex-artifacts/ci + run_status=0 + gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" > .codex-artifacts/ci/telemetry-run.json || run_status=$? + if [ "$run_status" -ne 0 ]; then + printf '{"telemetry_unavailable":true,"reason":"workflow run API request failed"}\n' > .codex-artifacts/ci/telemetry-run.json + fi + jobs_status=0 + gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs?per_page=100" > .codex-artifacts/ci/telemetry-jobs-pages.json || jobs_status=$? + if [ "$jobs_status" -ne 0 ]; then + printf '{"telemetry_unavailable":true,"reason":"workflow jobs API request failed"}\n' > .codex-artifacts/ci/telemetry-jobs-pages.json + else + node --input-type=module <<'NODE' + import { readFile, writeFile } from "node:fs/promises"; + const pages = JSON.parse(await readFile(".codex-artifacts/ci/telemetry-jobs-pages.json", "utf8")); + if (!Array.isArray(pages) || pages.some(page => !Array.isArray(page?.jobs))) throw new Error("Invalid jobs API pages"); + const jobs = pages.flatMap(page => page.jobs); + await writeFile(".codex-artifacts/ci/telemetry-jobs.json", `${JSON.stringify({ jobs }, null, 2)}\n`); + NODE + fi + if [ ! -f .codex-artifacts/ci/telemetry-jobs.json ]; then + printf '{"jobs":[],"telemetry_unavailable":true}\n' > .codex-artifacts/ci/telemetry-jobs.json + fi + exit 0 + + - name: Render CI timing telemetry + if: always() + continue-on-error: true + shell: bash + run: | + set +e + node scripts/ci/ci-telemetry.mjs \ + --run=.codex-artifacts/ci/telemetry-run.json \ + --jobs=.codex-artifacts/ci/telemetry-jobs.json \ + --reports-dir=.codex-artifacts/ci/reports \ + --output=.codex-artifacts/ci/ci-telemetry.json \ + --summary=.codex-artifacts/ci/ci-telemetry.md + telemetry_status=$? + if [ "$telemetry_status" -ne 0 ]; then + printf '{"version":1,"telemetryUnavailable":true}\n' > .codex-artifacts/ci/ci-telemetry.json + printf '## CI timing telemetry\n\nTelemetry was unavailable; this did not affect the quality gate.\n' > .codex-artifacts/ci/ci-telemetry.md + echo "Warning: CI timing telemetry was unavailable" >&2 + fi + cat .codex-artifacts/ci/ci-telemetry.md >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Preserve CI timing telemetry + if: always() + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: ci-telemetry + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 + path: | + .codex-artifacts/ci/ci-telemetry.json + .codex-artifacts/ci/ci-telemetry.md + .codex-artifacts/ci/telemetry-run.json + .codex-artifacts/ci/telemetry-jobs.json diff --git a/.gitleaksignore b/.gitleaksignore index 683c1fee7..75e324711 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -402,3 +402,113 @@ f9f73f6cb5c4e5c13cc16758fb318a69a9de7bae:packages/providers/test/media-adapter-c 1047b00500ec478c5bf4594f25a97e73b8dd34a3:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:938 1047b00500ec478c5bf4594f25a97e73b8dd34a3:output/e6-20260928/p2-cold-bootstrap-final-regression-20261023.tap:generic-api-key:88 1047b00500ec478c5bf4594f25a97e73b8dd34a3:output/e6-20260928/p2-cold-bootstrap-final-regression-20261023.tap:generic-api-key:90 + +# Exact merge-imported SHA-256 records independently verified on 2026-11-03. +# Each digest matches source blobs, public Node-API headers or test fixture inputs. +# Commit-specific fingerprints preserve scanning of other commits and credentials. +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1045 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1156 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1169 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1170 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1178 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1409 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1568 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1618 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1662 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1664 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1669 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1671 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1674 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1686 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1728 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1772 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1773 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1774 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:1775 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:304 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:437 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:453 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:528 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:667 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:818 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-c01-delta-frozen-20261024.json:generic-api-key:946 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:1084 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:1097 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:1098 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:1106 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:1337 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:1496 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:1546 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:1590 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:1592 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:1597 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:1599 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:1602 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:1614 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:1656 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:2115 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:2116 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:2117 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:2118 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:232 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:365 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:381 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:456 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:595 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:746 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:874 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-delta-frozen-20261023.json:generic-api-key:973 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1016 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1127 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1140 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1141 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1149 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1380 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1539 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1589 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1633 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1635 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1640 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1642 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1645 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1657 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:1699 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:2254 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:2255 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:2256 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:2257 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:275 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:408 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:424 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:499 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:638 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:789 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-main-delta-frozen-20261024.json:generic-api-key:917 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1037 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1148 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1161 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1162 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1170 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1403 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1562 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1612 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1656 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1658 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1663 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1665 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1668 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1680 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1722 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1885 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1886 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1887 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:1888 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:296 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:429 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:445 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:520 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:659 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:810 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/office-human-qa-p3-memory-delta-frozen-20261025.json:generic-api-key:938 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/p2-cold-bootstrap-final-regression-20261023.tap:generic-api-key:88 +7d8830a7ce57966965a75b0e0ce64fe98e746628:output/e6-20260928/p2-cold-bootstrap-final-regression-20261023.tap:generic-api-key:90 diff --git a/docs/maintenance/ci.md b/docs/maintenance/ci.md index 1dab7991d..2e0e79e4d 100644 --- a/docs/maintenance/ci.md +++ b/docs/maintenance/ci.md @@ -156,6 +156,18 @@ Linux 每日回归仅在同一提交、同一检查方案已有有效的完整 每次运行分别记录排队时间、页面总时长、各任务运行时间、Linux／macOS 累计任务分钟、测试数量与原生跳过数量。累计任务分钟是性能观察，不能在没有账单证据时称为实际扣额；失败任务已运行的部分也会消耗资源。 +独立的 `timing-telemetry` 任务在 `quality` 汇总结束后，尽力保存 +`ci-telemetry` artifact（保留 14 天）并写入 GitHub Step Summary。质量门 +不依赖该任务；采集、下载、解析、上传或观测任务失败都不改变原有门禁。 +质量门失败时仍下载可用分片报告，记录本次运行尝试的 run/job/step 观测、 +逐文件结果、Core/Daemon 最大分片文件累计耗时、分片不均衡比和可用报告中的 +timeout/leaked-process 计数。job 启动延迟包含依赖等待，不能当作纯 runner +排队时间；并发文件累计耗时不等于分片实际时长；截至已完成任务的经过时间 +不等于依赖图关键路径，且不包含仍在执行的任务。run 总时长标为采集时 +已观测值，报告缺失也不能据此断言全量测试没有超时或泄漏。先观察至少 +5～10 次完整 hosted run，再决定是否调整分片、worker 并发或增加 E2E；当前 +不据此承诺固定提速比例。 + 缓存继续用于锁定依赖和同一次构建输出复用。不得复用与提交、平台或方案不匹配的测试成功。分片缩短等待不一定减少累计费用；不在测量前承诺节省比例，也不把公开仓库、自托管 runner 或增加支出上限当作本轮方案的前提。 ## 实施顺序与验收 diff --git a/packages/daemon/test/channel-node-invocation-binding.test.mjs b/packages/daemon/test/channel-node-invocation-binding.test.ts similarity index 59% rename from packages/daemon/test/channel-node-invocation-binding.test.mjs rename to packages/daemon/test/channel-node-invocation-binding.test.ts index 64eb70d39..b5f141533 100644 --- a/packages/daemon/test/channel-node-invocation-binding.test.mjs +++ b/packages/daemon/test/channel-node-invocation-binding.test.ts @@ -1,21 +1,26 @@ import assert from "node:assert/strict"; import { mkdtemp, rm, writeFile, readFile } from "node:fs/promises"; import { join } from "node:path"; -import test from "node:test"; -import { bindChannelOperationPlan } from "../dist/features/channels/channel-operation-bindings.js"; +import test, { type TestContext } from "node:test"; +import { bindChannelOperationPlan, type ChannelOperationCandidatePlanV2, type ChannelOperationCandidateV2 } from "../dist/features/channels/channel-operation-bindings.js"; import { workflowExactCommandBindingRejection } from "../dist/features/runs/approvals.js"; -import { channelOperationPlanFingerprintV1, isChannelIntentExecutionPlanV1, nodeFunctionGoalCommand, parseNodeFunctionInvocation } from "@ling-stack/contracts"; +import { channelOperationPlanFingerprintV1, isChannelIntentExecutionPlanV1, nodeFunctionGoalCommand, parseNodeFunctionInvocation, + type ChannelIntentExecutionOperationV1, type ChannelIntentExecutionPlanV1, type UnifiedExecResult } from "@ling-stack/contracts"; import { executeUnifiedBashTool } from "../dist/features/runs/unified-exec.js"; import { createUnifiedExecFixture } from "./support/unified-exec-fixture.mjs"; +import { captureNodeGoalCommandDiagnostic, type NodeGoalCommandDiagnostic } from "./support/node-goal-command-diagnostic.ts"; import { createSqliteRunStore, DEFAULT_WORKSPACE_ID, toRuntimeAttemptFence } from "@ling-stack/core"; import { appendChannelItemFixture } from "../../core/test/channel-conversation-fixture.mjs"; import { createDelegatedStepRequirement } from "../dist/features/channels/workflow-step-requirement.js"; import { createWorkflowOperationExecutionSnapshot } from "../dist/features/runs/workflow-operation-snapshot.js"; import { verifyWorkflowStepExecutionOperations } from "../dist/features/channels/workflow-operation-evidence.js"; import { listAllExecutionActionsForRun } from "../dist/features/runs/execution-actions.js"; +import { isRecord } from "../dist/shared/validation.js"; + +type Mutable = { -readonly [Key in keyof T]: Mutable }; const body = '创建 continuity-v2-proof.js，内容是导出 formatProof，然后运行 Node 调用 formatProof("7319") 验证。'; -const owner = { kind: "host_default" }; +const owner = { kind: "host_default" } as const; function candidate() { return { schemaVersion: 2, operations: [ { id: "create_file", kind: "file_create", targetKind: "file", required: true, dependsOn: [], owner, @@ -26,9 +31,11 @@ function candidate() { sourceQuote: '然后运行 Node 调用 formatProof("7319") 验证。', commandGoal: { kind: "node_function", module: { path: "continuity-v2-proof.js", sourceQuote: "continuity-v2-proof.js" }, invocation: 'formatProof("7319")', sourceQuote: 'formatProof("7319")' } }, - ] }; + ] } satisfies Omit & { + operations: [ChannelOperationCandidateV2, ChannelOperationCandidateV2]; + }; } -async function fixture(t) { +async function fixture(t: TestContext) { const cwd = await mkdtemp("/private/tmp/ling-node-goal-binding-"); t.after(() => rm(cwd, { recursive: true, force: true })); const bind = (plan = candidate(), sourceText = body) => bindChannelOperationPlan({ candidate: plan, sourceText, @@ -40,34 +47,39 @@ async function fixture(t) { test("natural Node goal binds module, invocation and original source without inventing a user command", async t => { const { bind } = await fixture(t); const result = bind(); - assert.equal(result.status, "bound", JSON.stringify(result)); + assert.ok(result.status === "bound", JSON.stringify(result)); assert.equal(isChannelIntentExecutionPlanV1(result.execution), true); const operation = result.execution.operations[1]; + assert.ok(operation.binding?.commandGoal); assert.equal(operation.binding.commandGoal.kind, "node_function"); assert.equal(operation.binding.commandGoal.modulePath, "continuity-v2-proof.js"); assert.equal(operation.binding.commandGoal.invocation, 'formatProof("7319")'); assert.deepEqual(operation.binding.effectScope.readPaths, ["continuity-v2-proof.js"]); assert.deepEqual(operation.dependsOn, ["create_file"]); assert.ok(operation.binding.commandGoal.sourceRefs.every(ref => ref.itemId === "node-goal-user-item" && body.includes(ref.quote))); + assert.ok(operation.commandMatchPolicy?.kind === "exact_command"); const command = operation.commandMatchPolicy.command; assert.equal(body.includes(command), false, "the Host command must not masquerade as a user quote"); assert.equal(workflowExactCommandBindingRejection({ bindings: [operation], request: { toolName: "bash", input: { command } } }), undefined); for (const substitute of [command.replace("7319", "7320"), command.replace("formatProof", "otherFunction"), command.replace("continuity-v2-proof.js", "other.js"), `node -e "console.log('proof:7319')"`]) { - assert.match(workflowExactCommandBindingRejection({ bindings: [operation], request: { toolName: "bash", input: { command: substitute } } }), /rejected/); + const rejection = workflowExactCommandBindingRejection({ bindings: [operation], request: { toolName: "bash", input: { command: substitute } } }); + assert.ok(rejection); + assert.match(rejection, /rejected/); } }); test("goal source, module dependency, invocation literals and exclusive command forms are verified", async t => { const { bind } = await fixture(t); - for (const mutate of [ + const mutations: ((plan: ReturnType) => void)[] = [ plan => { plan.operations[1].commandGoal.invocation = 'formatProof("9999")'; }, plan => { plan.operations[1].commandGoal.module.path = "other.js"; }, plan => { plan.operations[1].dependsOn = []; }, - plan => { plan.operations[1].command = { text: "node --version", sourceQuote: "node --version" }; }, + plan => { Object.assign(plan.operations[1], { command: { text: "node --version", sourceQuote: "node --version" } }); }, plan => { plan.operations[1].commandGoal.sourceQuote = "not in the user item"; }, plan => { plan.operations[1].sourceQuote = 'formatProof("7319")'; }, - ]) { + ]; + for (const mutate of mutations) { const plan = candidate(); mutate(plan); assert.equal(bind(plan).status, "clarify"); } @@ -75,7 +87,7 @@ test("goal source, module dependency, invocation literals and exclusive command const plan = candidate(); plan.operations[1].sourceQuote = "运行 Node 调用 " + invocation; plan.operations[1].commandGoal.invocation = invocation; plan.operations[1].commandGoal.sourceQuote = invocation; const result = bind(plan, body + plan.operations[1].sourceQuote); - assert.equal(result.status, "clarify", invocation); + assert.ok(result.status === "clarify", invocation); assert.deepEqual(result.missingFields, ["commandGoal.invocation"], "reject the expression itself, not unrelated provenance"); } }); @@ -98,43 +110,60 @@ test("Host-generated Node commands really invoke ESM/CJS exports and retain fail const { cwd } = await fixture(t); const task = { id: "node_goal_task", title: "Node goal fixture", prompt: "Execute only temporary fixture functions" }; const f = createUnifiedExecFixture(t, cwd, task, "node_goal_run"); + const run = f.run; + assert.ok(run); let sequence = 0; - const execute = async (modulePath, invocation) => { + const diagnostics: NodeGoalCommandDiagnostic[] = []; + const execute = async (modulePath: string, invocation: string) => { const command = nodeFunctionGoalCommand({ modulePath, invocation }); assert.ok(command); - const result = await executeUnifiedBashTool({ ...f, - request: { toolCallId: `node-goal-${++sequence}`, toolName: "bash", cwd, task, input: { command, timeoutMs: 4000 } }, - appendAndMaybeStream: (type, data) => f.store.appendEvent(f.run.id, type, data) }); - assert.equal(result.details.unifiedExec.command.command, command); - assert.equal(result.details.unifiedExec.runId, f.run.id); - return result.details.unifiedExec; + const result = await captureNodeGoalCommandDiagnostic({ sequence: ++sequence, timeoutMs: 4000, + record: diagnostic => diagnostics.push(diagnostic), + execute: () => executeUnifiedBashTool({ ...f, run, + request: { toolCallId: `node-goal-${sequence}`, toolName: "bash", cwd, task, input: { command, timeoutMs: 4000 } }, + appendAndMaybeStream: (type, data) => f.store.appendEvent(run.id, type, data) }) }); + assert.ok(isRecord(result.details) && isRecord(result.details.unifiedExec)); + const unifiedExec = result.details.unifiedExec as unknown as UnifiedExecResult; + assert.ok(unifiedExec.command.kind === "shell"); + assert.equal(unifiedExec.command.command, command); + assert.equal(unifiedExec.runId, run.id); + return unifiedExec; }; - await writeFile(join(cwd, "proof.mjs"), 'export const formatProof = async value => "proof:" + value;'); - let result = await execute("proof.mjs", 'formatProof("7319")'); - assert.equal(result.status, "succeeded"); assert.equal(result.stdout.preview, "proof:7319\n"); - await writeFile(join(cwd, "proof.cjs"), 'module.exports = { formatProof: value => "common:" + value };'); - result = await execute("proof.cjs", 'formatProof("7319")'); - assert.equal(result.status, "succeeded"); assert.equal(result.stdout.preview, "common:7319\n"); - await writeFile(join(cwd, "quoted '# %.mjs"), "export const echo = value => value;"); - result = await execute("quoted '# %.mjs", `echo("'$(touch must-not-exist)")`); - assert.equal(result.status, "succeeded"); assert.equal(result.stdout.preview, "'$(touch must-not-exist)\n"); - await assert.rejects(readFile(join(cwd, "must-not-exist")), { code: "ENOENT" }); - result = await execute("proof.mjs", "missing()"); - assert.equal(result.status, "failed"); assert.notEqual(result.exitCode, 0); assert.match(result.stderr.preview, /missing export/); - await writeFile(join(cwd, "throwing.mjs"), 'export const formatProof = async () => { throw Error("fixture rejection"); };'); - result = await execute("throwing.mjs", 'formatProof("7319")'); - assert.equal(result.status, "failed"); assert.notEqual(result.exitCode, 0); assert.match(result.stderr.preview, /fixture rejection/); - assert.deepEqual(f.store.listHostCommandWriteBlockers(cwd), []); + try { + await writeFile(join(cwd, "proof.mjs"), 'export const formatProof = async value => "proof:" + value;'); + let result = await execute("proof.mjs", 'formatProof("7319")'); + assert.equal(result.status, "succeeded"); assert.equal(result.stdout.preview, "proof:7319\n"); + await writeFile(join(cwd, "proof.cjs"), 'module.exports = { formatProof: value => "common:" + value };'); + result = await execute("proof.cjs", 'formatProof("7319")'); + assert.equal(result.status, "succeeded"); assert.equal(result.stdout.preview, "common:7319\n"); + await writeFile(join(cwd, "quoted '# %.mjs"), "export const echo = value => value;"); + result = await execute("quoted '# %.mjs", `echo("'$(touch must-not-exist)")`); + assert.equal(result.status, "succeeded"); assert.equal(result.stdout.preview, "'$(touch must-not-exist)\n"); + await assert.rejects(readFile(join(cwd, "must-not-exist")), { code: "ENOENT" }); + result = await execute("proof.mjs", "missing()"); + assert.equal(result.status, "failed"); assert.notEqual(result.exitCode, 0); assert.match(result.stderr.preview, /missing export/); + await writeFile(join(cwd, "throwing.mjs"), 'export const formatProof = async () => { throw Error("fixture rejection"); };'); + result = await execute("throwing.mjs", 'formatProof("7319")'); + assert.equal(result.status, "failed"); assert.notEqual(result.exitCode, 0); assert.match(result.stderr.preview, /fixture rejection/); + assert.deepEqual(f.store.listHostCommandWriteBlockers(cwd), []); + } catch (error) { + for (const diagnostic of diagnostics) t.diagnostic(JSON.stringify(diagnostic)); + throw error; + } }); test("recomputed plan fingerprints cannot detach generated commands from their accepted goal", async t => { const { bind } = await fixture(t); - const result = bind(); assert.equal(result.status, "bound", JSON.stringify(result)); - for (const mutate of [ - operation => { operation.binding.commandGoal.invocation = 'formatProof("9999")'; }, - operation => { operation.binding.commandGoal.modulePath = "other.js"; }, - operation => { operation.commandMatchPolicy.command = "node --version"; operation.binding.effectScope.exactCommands = ["node --version"]; }, - ]) { - const plan = structuredClone(result.execution); mutate(plan.operations[1]); + const result = bind(); assert.ok(result.status === "bound", JSON.stringify(result)); + const mutations: ((operation: Mutable) => void)[] = [ + operation => { assert.ok(operation.binding?.commandGoal); operation.binding.commandGoal.invocation = 'formatProof("9999")'; }, + operation => { assert.ok(operation.binding?.commandGoal); operation.binding.commandGoal.modulePath = "other.js"; }, + operation => { + assert.ok(operation.commandMatchPolicy?.kind === "exact_command" && operation.binding); + operation.commandMatchPolicy.command = "node --version"; operation.binding.effectScope.exactCommands = ["node --version"]; + }, + ]; + for (const mutate of mutations) { + const plan = structuredClone(result.execution) as Mutable; mutate(plan.operations[1]); plan.fingerprint = channelOperationPlanFingerprintV1(plan.operations); assert.equal(isChannelIntentExecutionPlanV1(plan), false); } diff --git a/packages/daemon/test/node-goal-command-diagnostic.test.ts b/packages/daemon/test/node-goal-command-diagnostic.test.ts new file mode 100644 index 000000000..50f1dea73 --- /dev/null +++ b/packages/daemon/test/node-goal-command-diagnostic.test.ts @@ -0,0 +1,111 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { channel, hasSubscribers } from "node:diagnostics_channel"; +import { EventEmitter } from "node:events"; +import test from "node:test"; +import { captureNodeGoalCommandDiagnostic, type NodeGoalCommandDiagnostic } from "./support/node-goal-command-diagnostic.ts"; + +test("command diagnostics retain failure evidence but exclude command, environment and output payloads", async () => { + const secret = "private-diagnostic-sentinel"; + const result = { status: "timed_out", startedAt: "2026-11-02T00:00:00Z", endedAt: "2026-11-02T00:00:04Z", + durationMs: 4001, signal: "SIGTERM", timedOut: true, cancelled: false, + command: { command: secret }, cwd: secret, environmentSummary: { keys: [secret], extraEnv: { TOKEN: secret } }, + processObservation: { scope: "process_group", leaderExited: true, stdioClosed: true, + descendants: "unverified", stopReason: "timeout", settledBy: "close", signalFailures: ["SIGTERM:group:EPERM"], private: secret }, + stdout: { byteCount: 0, truncated: false, preview: secret, artifactId: secret }, + stderr: { byteCount: 12, truncated: true, preview: secret }, artifacts: [{ path: secret }], reason: secret }; + const handled = { details: { unifiedExec: result, private: secret }, content: [{ text: secret }] }; + const records: NodeGoalCommandDiagnostic[] = []; + assert.equal(await captureNodeGoalCommandDiagnostic({ sequence: 1, timeoutMs: 4000, + execute: async () => handled, record: value => records.push(value) }), handled); + assert.equal(records.length, 1); + const diagnostic = records[0]; + assert.equal(diagnostic.kind, "node_goal_command_diagnostic_v1"); + assert.equal(diagnostic.timeoutMs, 4000); + assert.equal(diagnostic.result.status, "timed_out"); + assert.equal(diagnostic.result.durationMs, 4001); + assert.equal(diagnostic.result.exitCode, null); + assert.equal(diagnostic.result.signal, "SIGTERM"); + assert.deepEqual(diagnostic.result.processObservation, { scope: "process_group", leaderExited: true, + stdioClosed: true, descendants: "unverified", stopReason: "timeout", settledBy: "close", signalFailures: ["SIGTERM:group:EPERM"] }); + assert.deepEqual(diagnostic.result.stdout, { byteCount: 0, truncated: false }); + assert.deepEqual(diagnostic.result.stderr, { byteCount: 12, truncated: true }); + assert.equal(JSON.stringify(diagnostic).includes(secret), false); +}); + +test("diagnostics distinguish no-start denial from a returned process result", async () => { + let diagnostic: NodeGoalCommandDiagnostic | undefined; + await captureNodeGoalCommandDiagnostic({ sequence: 2, timeoutMs: 4000, + execute: async () => ({ details: { unifiedExec: { status: "failed", denied: true, reason: "private denial" } } }), + record: value => { diagnostic = value; } }); + assert.ok(diagnostic); + assert.equal(diagnostic.returned, true); + assert.equal(diagnostic.result.denied, true); + assert.equal(diagnostic.result.processObservation, null); + assert.equal(diagnostic.result.durationMs, null); + assert.deepEqual(diagnostic.processes, []); + assert.equal(JSON.stringify(diagnostic).includes("private denial"), false); +}); + +test("real process lifecycle timings do not consume or alter output", async () => { + const subscribed = hasSubscribers("child_process"); + let diagnostic: NodeGoalCommandDiagnostic | undefined; + let stdout = "", stderr = ""; + await captureNodeGoalCommandDiagnostic({ sequence: 3, timeoutMs: 4000, + execute: () => new Promise>((resolve, reject) => { + const child = spawn(process.execPath, ["-e", 'process.stdout.write("fixture-out");process.stderr.write("fixture-err");']); + child.stdout.on("data", chunk => { stdout += chunk; }); + child.stderr.on("data", chunk => { stderr += chunk; }); + child.once("error", reject); + child.once("close", code => { + if (code === 0) resolve({}); + else reject(new Error(`fixture exited with code ${code}`)); + }); + }), record: value => { diagnostic = value; } }); + assert.equal(hasSubscribers("child_process"), subscribed); + assert.equal(stdout, "fixture-out"); + assert.equal(stderr, "fixture-err"); + assert.ok(diagnostic); + assert.equal(diagnostic.processes.length, 1); + const timing = diagnostic.processes[0]; + for (const key of ["createdMs", "spawnedMs", "firstStdoutMs", "firstStderrMs", "exitMs", "closeMs"] as const) { + const value = timing[key]; + assert.equal(typeof value, "number", key); + assert.ok(value !== null && value >= 0, key); + } + assert.ok(timing.spawnedMs !== null && timing.firstStdoutMs !== null && timing.firstStderrMs !== null + && timing.exitMs !== null && timing.closeMs !== null); + assert.ok(timing.createdMs <= timing.spawnedMs); + assert.ok(timing.spawnedMs <= timing.firstStdoutMs); + assert.ok(timing.spawnedMs <= timing.firstStderrMs); + assert.ok(timing.exitMs <= timing.closeMs); + assert.ok(timing.closeMs <= diagnostic.elapsedMs); + assert.equal(timing.exitCode, 0); + assert.equal(timing.signal, null); + assert.equal(JSON.stringify(diagnostic).includes("fixture-out"), false); + assert.equal(JSON.stringify(diagnostic).includes("fixture-err"), false); +}); + +test("thrown execution keeps its original error and detaches every diagnostic listener", async () => { + const subscribed = hasSubscribers("child_process"); + const child = Object.assign(new EventEmitter(), { stdout: new EventEmitter(), stderr: new EventEmitter() }); + const error = new Error("original private error"); + let diagnostic: NodeGoalCommandDiagnostic | undefined; + await assert.rejects(captureNodeGoalCommandDiagnostic({ sequence: 4, timeoutMs: 4000, + execute: async () => { + channel("child_process").publish({ process: child }); + child.emit("spawn"); + child.emit("error", { code: "ENOENT", message: "private spawn error" }); + throw error; + }, record: value => { diagnostic = value; } }), cause => cause === error); + assert.equal(hasSubscribers("child_process"), subscribed); + for (const emitter of [child, child.stdout, child.stderr]) assert.deepEqual(emitter.eventNames(), []); + assert.ok(diagnostic); + assert.equal(diagnostic.returned, false); + assert.equal(diagnostic.result.status, null); + assert.equal(diagnostic.processes[0].errorCode, "ENOENT"); + assert.equal(diagnostic.processes[0].firstStdoutMs, null); + assert.equal(diagnostic.processes[0].exitMs, null); + assert.equal(diagnostic.processes[0].closeMs, null); + assert.equal(JSON.stringify(diagnostic).includes("private"), false); +}); diff --git a/packages/daemon/test/support/node-goal-command-diagnostic.ts b/packages/daemon/test/support/node-goal-command-diagnostic.ts new file mode 100644 index 000000000..102c59f27 --- /dev/null +++ b/packages/daemon/test/support/node-goal-command-diagnostic.ts @@ -0,0 +1,113 @@ +import { subscribe, unsubscribe } from "node:diagnostics_channel"; +import type { ChildProcess } from "node:child_process"; +import type { EventEmitter } from "node:events"; +import type { UnifiedExecProcessObservation, UnifiedExecResult } from "@ling-stack/contracts"; + +interface ProcessTiming { + createdMs: number; + spawnedMs: number | null; + firstStdoutMs: number | null; + firstStderrMs: number | null; + exitMs: number | null; + closeMs: number | null; + exitCode: number | null; + signal: NodeJS.Signals | null; + errorCode: string | null; +} + +export interface NodeGoalCommandDiagnostic { + kind: "node_goal_command_diagnostic_v1"; + sequence: number; + timeoutMs: number; + nodeVersion: string; + platform: NodeJS.Platform; + arch: string; + elapsedMs: number; + returned: boolean; + processes: ProcessTiming[]; + result: { + status: UnifiedExecResult["status"] | null; + denied: boolean; + startedAt: string | null; + endedAt: string | null; + durationMs: number | null; + exitCode: number | null; + signal: string | null; + timedOut: boolean | null; + cancelled: boolean | null; + processObservation: (Omit & { + stopReason: UnifiedExecProcessObservation["stopReason"] | null; + }) | null; + stdout: { byteCount: number | null; truncated: boolean | null }; + stderr: { byteCount: number | null; truncated: boolean | null }; + }; +} + +interface DiagnosticHandledResult { + details?: { unifiedExec?: Partial & { denied?: boolean } }; +} + +/** Observe the real producer without replacing spawn or retaining command/output text. */ +export async function captureNodeGoalCommandDiagnostic({ sequence, timeoutMs, execute, record }: { + sequence: number; + timeoutMs: number; + execute: () => Promise; + record: (diagnostic: NodeGoalCommandDiagnostic) => void; +}): Promise { + const started = performance.now(); + const elapsed = () => performance.now() - started; + const processes: ProcessTiming[] = []; + const cleanup: (() => void)[] = []; + const listen = (emitter: EventEmitter | null | undefined, event: string, callback: (...args: Args) => void) => { + emitter?.once(event, callback); + if (emitter) cleanup.push(() => emitter.removeListener(event, callback)); + }; + const onChild = (message: unknown) => { + const { process: child } = message as { process: ChildProcess }; + const timing: ProcessTiming = { createdMs: elapsed(), spawnedMs: null, firstStdoutMs: null, firstStderrMs: null, + exitMs: null, closeMs: null, exitCode: null, signal: null, errorCode: null }; + processes.push(timing); + listen(child, "spawn", () => { + timing.spawnedMs = elapsed(); + listen(child.stdout, "data", () => { timing.firstStdoutMs = elapsed(); }); + listen(child.stderr, "data", () => { timing.firstStderrMs = elapsed(); }); + }); + listen(child, "exit", (code: number | null, signal: NodeJS.Signals | null) => { timing.exitMs = elapsed(); timing.exitCode = code; timing.signal = signal; }); + listen(child, "close", () => { timing.closeMs = elapsed(); }); + listen(child, "error", (error: unknown) => { + const code = error !== null && typeof error === "object" && "code" in error ? error.code : undefined; + timing.errorCode = typeof code === "string" && /^[A-Z0-9_]{1,40}$/.test(code) ? code : "unknown"; + }); + }; + let handled: T | undefined; + let returned = false; + subscribe("child_process", onChild); + try { + handled = await execute(); + returned = true; + return handled; + } finally { + const elapsedMs = elapsed(); + unsubscribe("child_process", onChild); + for (const detach of cleanup) detach(); + const result = (handled as DiagnosticHandledResult | undefined)?.details?.unifiedExec; + const observation = result?.processObservation; + record({ kind: "node_goal_command_diagnostic_v1", sequence, timeoutMs, + nodeVersion: process.version, platform: process.platform, arch: process.arch, + elapsedMs, returned, processes, + result: { + status: result?.status ?? null, denied: result?.denied === true, + startedAt: result?.startedAt ?? null, endedAt: result?.endedAt ?? null, + durationMs: result?.durationMs ?? null, exitCode: result?.exitCode ?? null, signal: result?.signal ?? null, + timedOut: result?.timedOut ?? null, cancelled: result?.cancelled ?? null, + processObservation: observation ? { + scope: observation.scope, leaderExited: observation.leaderExited, stdioClosed: observation.stdioClosed, + descendants: observation.descendants, stopReason: observation.stopReason ?? null, + settledBy: observation.settledBy, signalFailures: [...observation.signalFailures], + } : null, + stdout: { byteCount: result?.stdout?.byteCount ?? null, truncated: result?.stdout?.truncated ?? null }, + stderr: { byteCount: result?.stderr?.byteCount ?? null, truncated: result?.stderr?.truncated ?? null }, + }, + }); + } +} diff --git a/scripts/ci/README.md b/scripts/ci/README.md index a0ed05971..32c2486c5 100644 --- a/scripts/ci/README.md +++ b/scripts/ci/README.md @@ -172,6 +172,38 @@ a statistical benchmark; full CI setup, shard contention and total job time stil need hosted-runner measurement. This documentation sync does not establish that the current changes have passed complete hosted CI. +## Timing Telemetry + +The independent `timing-telemetry` job runs after `quality` and preserves a +best-effort `ci-telemetry` artifact for complete and preview runs. The mandatory +quality gate never depends on telemetry, so collection, download, parsing, +upload, or telemetry job failures cannot change that gate. Available shard +reports are downloaded even when the quality gate fails. The artifact retains +run/job/step observations, per-file results, cumulative shard file time, +imbalance by suite, and timeout/leaked-process counts from available reports. +The summary is appended to the GitHub step summary and the JSON is retained +for 14 days. Actions API jobs are restricted to the current run attempt. + +Job start delay includes dependency waits; it is not a pure runner queue +measurement. Cumulative file time is not shard wall-clock time because files +run concurrently. Elapsed time through completed jobs is not a dependency-DAG +critical path and excludes in-progress jobs. Run duration is observed at +collection while the workflow is still in progress. Missing reports do not +establish zero failures across the full run. + +Use at least 5-10 complete hosted runs to compare the observations before +changing shard counts, worker concurrency, or adding another E2E lane. The +telemetry does not establish a fixed speedup percentage by itself. + +Telemetry and workflow regression tests are TypeScript files in the existing +`scripts/ci/*.test.ts` lane and are checked by `pnpm typecheck:ci-tests`. +The Node command diagnostic helper and its two owning daemon tests are also +TypeScript; daemon discovery includes both test files, while +`packages/daemon/tsconfig.test.json` checks the tests and helper through the +required `typecheck:ci` script. Runtime type stripping alone is not validation. +The diagnostic observes real child-process lifecycle events without changing +command timeouts, invocation assertions, output capture, or native skip rules. + Generate a local plan without running tests: ```sh @@ -200,6 +232,12 @@ Missing/ambiguous history or changes to scanner configuration/ignore policy select full history. Scheduled/manual runs scan full reachable history. Checkout must retain complete history; the scanner cannot reconstruct a shallow clone. +Gitleaks fingerprints bind a commit, path, rule and line. A merge that imports +reviewed evidence can produce distinct fingerprints for identical SHA-256 +records. Verify each digest against the source bytes before adding exact +fingerprints to `.gitleaksignore`; do not exclude whole evidence paths, broaden +hash-pattern allowlists or omit merge patches. + ## Validation ```sh diff --git a/scripts/ci/ci-telemetry.mjs b/scripts/ci/ci-telemetry.mjs new file mode 100644 index 000000000..70305adad --- /dev/null +++ b/scripts/ci/ci-telemetry.mjs @@ -0,0 +1,226 @@ +import { readFile, writeFile, mkdir, readdir } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +const MAX_REASONABLE_MS = 7 * 24 * 60 * 60 * 1000; + +function parseTimestamp(value) { + const timestamp = typeof value === "string" ? Date.parse(value) : NaN; + return Number.isFinite(timestamp) ? timestamp : null; +} + +function duration(start, end) { + const started = parseTimestamp(start); + const completed = parseTimestamp(end); + if (started === null || completed === null || completed < started || completed - started > MAX_REASONABLE_MS) return null; + return completed - started; +} + +function sum(values) { + return values.reduce((total, value) => total + (Number.isFinite(value) && value >= 0 ? value : 0), 0); +} + +function percentile(values, fraction) { + const sorted = values.filter(Number.isFinite).sort((a, b) => a - b); + if (!sorted.length) return null; + return sorted[Math.max(0, Math.ceil(sorted.length * fraction) - 1)]; +} + +function normalizeJobs(input) { + const jobs = Array.isArray(input) ? input : Array.isArray(input?.jobs) ? input.jobs : []; + return jobs.filter(job => job && typeof job === "object"); +} + +function normalizeReports(input) { + if (!Array.isArray(input)) return []; + return input.filter(report => report && report.version === 1 && typeof report.suite === "string" + && report.suite.length > 0 && Number.isInteger(report.shard) && report.shard >= 0 && Array.isArray(report.results)); +} + +function parseArguments(argv, allowed) { + const args = {}; + for (let index = 0; index < argv.length; index += 1) { + const [key, ...valueParts] = argv[index].split("="); + if (!allowed.includes(key) || Object.hasOwn(args, key)) throw new Error(`Unknown or duplicate argument: ${key}`); + const value = valueParts.length ? valueParts.join("=") : argv[++index]; + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${key}`); + args[key] = value; + } + return args; +} + +async function readReportsDirectory(directory) { + try { + const files = (await readdir(directory)).filter(file => file.endsWith(".json")).sort(); + const reports = []; + for (const file of files) { + try { + reports.push(JSON.parse(await readFile(join(directory, file), "utf8"))); + } catch { + // A failed shard must not turn observation into a quality failure. + } + } + return reports; + } catch { + return []; + } +} + +function jobTelemetry(jobs, run) { + const runStarted = parseTimestamp(run.run_started_at) !== null ? run.run_started_at : run.created_at; + return jobs.map(job => { + const executionMs = duration(job.started_at, job.completed_at); + const startDelayMs = duration(runStarted, job.started_at); + const stepDurations = (Array.isArray(job.steps) ? job.steps : []).filter(step => step && typeof step === "object").map(step => ({ + name: step.name, + conclusion: step.conclusion ?? null, + durationMs: duration(step.started_at, step.completed_at), + })); + return { + name: job.name ?? "unnamed", + status: job.status ?? null, + conclusion: job.conclusion ?? null, + runner: job.runner_name ?? null, + startDelayMs, + executionMs, + stepDurations, + }; + }); +} + +function reportTelemetry(reports) { + const shards = reports.map(report => { + const files = report.results.filter(result => result && typeof result.file === "string").map(result => ({ + file: result.file, + status: typeof result.status === "string" ? result.status : null, + durationMs: Number.isFinite(result.durationMs) && result.durationMs >= 0 && result.durationMs <= MAX_REASONABLE_MS + ? result.durationMs : null, + })); + const durations = files.map(result => result.durationMs).filter(Number.isFinite); + const statuses = Object.fromEntries([...new Set(files.map(result => result.status))] + .map(status => [status, files.filter(result => result.status === status).length])); + return { + suite: report.suite, + shard: report.shard, + platform: report.platform ?? null, + complete: report.complete === true, + fileCount: files.length, + passedCount: files.filter(result => result.status === "passed").length, + statusCounts: statuses, + sumFileDurationMs: durations.length ? sum(durations) : null, + maxFileDurationMs: durations.length ? Math.max(...durations) : null, + p95FileDurationMs: percentile(durations, 0.95), + files, + }; + }); + const bySuite = Object.create(null); + for (const shard of shards) (bySuite[shard.suite] ??= []).push(shard); + const suites = Object.fromEntries(Object.entries(bySuite).map(([suite, entries]) => { + const durations = entries.map(entry => entry.sumFileDurationMs).filter(Number.isFinite); + const max = durations.length ? Math.max(...durations) : null; + const average = durations.length ? sum(durations) / durations.length : null; + return [suite, { + shardCount: entries.length, + maxShardFileDurationMs: max, + averageShardFileDurationMs: average, + imbalanceRatio: max !== null && average ? max / average : null, + }]; + })); + return { shards, suites }; +} + +/** @param {{ run?: Record | null, jobs?: unknown, reports?: unknown, capturedAt?: string }} [input] */ +export function buildTelemetry({ run = {}, jobs = [], reports = [], capturedAt = new Date().toISOString() } = {}) { + run ??= {}; + const normalizedJobs = normalizeJobs(jobs); + const normalizedReports = normalizeReports(reports); + const jobDurations = jobTelemetry(normalizedJobs, run); + const durationSource = run.status === "completed" ? "updated_at" : "capturedAt"; + const runDurationMs = duration(run.created_at, durationSource === "updated_at" ? run.updated_at : capturedAt); + const completedAt = normalizedJobs.filter(job => job.status === "completed") + .map(job => parseTimestamp(job.completed_at)).filter(Number.isFinite); + const completedJobsElapsedMs = completedAt.length + ? duration(run.created_at, new Date(Math.max(...completedAt)).toISOString()) : null; + const failedJobs = jobDurations.filter(job => job.conclusion && !["success", "skipped"].includes(job.conclusion)).length; + const reportDetails = reportTelemetry(normalizedReports); + const files = reportDetails.shards.flatMap(shard => shard.files); + const timedOutFiles = files + .filter(result => result.status === "timed_out").length; + const leakedProcessFiles = files + .filter(result => result.status === "leaked_processes").length; + const suiteSummaries = reportDetails.suites; + const finiteJobExecutions = jobDurations.map(job => job.executionMs).filter(Number.isFinite); + const finiteJobStartDelays = jobDurations.map(job => job.startDelayMs).filter(Number.isFinite); + return { + version: 1, + capturedAt, + telemetryUnavailable: run.telemetry_unavailable === true || jobs?.telemetry_unavailable === true, + run: { + id: run.id ?? null, + event: run.event ?? null, + sha: run.head_sha ?? null, + status: run.status ?? null, + conclusion: run.conclusion ?? null, + createdAt: run.created_at ?? null, + startedAt: run.run_started_at ?? null, + updatedAt: run.updated_at ?? null, + durationMs: runDurationMs, + durationSource, + completedJobsElapsedMs, + }, + summary: { + jobCount: jobDurations.length, + failedJobCount: failedJobs, + availableReportCount: normalizedReports.length, + timedOutFileCount: normalizedReports.length ? timedOutFiles : null, + leakedProcessFileCount: normalizedReports.length ? leakedProcessFiles : null, + maxJobExecutionMs: finiteJobExecutions.length ? Math.max(...finiteJobExecutions) : null, + maxJobStartDelayMs: finiteJobStartDelays.length ? Math.max(...finiteJobStartDelays) : null, + maxCoreShardFileDurationMs: suiteSummaries.core?.maxShardFileDurationMs ?? null, + maxDaemonShardFileDurationMs: suiteSummaries.daemon?.maxShardFileDurationMs ?? null, + shardImbalanceRatioBySuite: Object.fromEntries(Object.entries(suiteSummaries) + .map(([suite, summary]) => [suite, summary.imbalanceRatio])), + }, + jobs: jobDurations, + reports: reportDetails, + }; +} + +export function renderSummary(telemetry) { + const ms = value => value === null ? "n/a" : `${(value / 60000).toFixed(2)} min`; + const count = value => value === null ? "n/a" : value; + const lines = [ + "## CI timing telemetry", + "", + ...(telemetry.telemetryUnavailable ? ["- Collection status: **unavailable**; this did not affect the quality gate."] : []), + `- Run duration: **${ms(telemetry.run.durationMs)}** (source: ${telemetry.run.durationSource}); completed jobs elapsed since run creation: **${ms(telemetry.run.completedJobsElapsedMs)}** (excludes in-progress jobs; not a DAG critical path).`, + `- Jobs: **${telemetry.summary.jobCount}**; max execution: **${ms(telemetry.summary.maxJobExecutionMs)}**; max start delay: **${ms(telemetry.summary.maxJobStartDelayMs)}** (includes dependency wait; measured from run start, or run creation if unavailable).`, + `- Failed jobs: **${telemetry.summary.failedJobCount}**; timed-out files: **${count(telemetry.summary.timedOutFileCount)}**; leaked-process files: **${count(telemetry.summary.leakedProcessFileCount)}** (counts from available reports: ${telemetry.summary.availableReportCount}; missing or incomplete reports do not establish zero faults across the run).`, + ]; + for (const [suite, data] of Object.entries(telemetry.reports.suites)) { + lines.push(`- ${suite} shards: **${data.shardCount}**; max/average aggregate file time: **${ms(data.maxShardFileDurationMs)} / ${ms(data.averageShardFileDurationMs)}** (aggregate file time, not wall clock); imbalance: **${data.imbalanceRatio === null ? "n/a" : data.imbalanceRatio.toFixed(2)}x**`); + } + lines.push("", "Per-job and per-file details are retained in the `ci-telemetry` artifact."); + return `${lines.join("\n")}\n`; +} + +async function main() { + const args = parseArguments(process.argv.slice(2), ["--run", "--jobs", "--reports", "--reports-dir", "--output", "--summary"]); + if (!args["--run"] || !args["--jobs"] || !args["--output"] || !args["--summary"] + || (args["--reports"] && args["--reports-dir"])) { + throw new Error("--run, --jobs, --output and --summary are required; use only one reports input"); + } + const run = JSON.parse(await readFile(args["--run"], "utf8")); + const jobs = JSON.parse(await readFile(args["--jobs"], "utf8")); + const reports = args["--reports"] ? JSON.parse(await readFile(args["--reports"], "utf8")) + : args["--reports-dir"] ? await readReportsDirectory(args["--reports-dir"]) : []; + const telemetry = buildTelemetry({ run, jobs, reports }); + await mkdir(dirname(resolve(args["--output"])), { recursive: true }); + await writeFile(args["--output"], `${JSON.stringify(telemetry, null, 2)}\n`); + await writeFile(args["--summary"], renderSummary(telemetry)); + console.log(`Captured CI telemetry for ${telemetry.summary.jobCount} jobs and ${telemetry.reports.shards.length} shard reports.`); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { await main(); } catch (error) { console.error(error); process.exitCode = 1; } +} diff --git a/scripts/ci/ci-telemetry.test.ts b/scripts/ci/ci-telemetry.test.ts new file mode 100644 index 000000000..1f4fb651a --- /dev/null +++ b/scripts/ci/ci-telemetry.test.ts @@ -0,0 +1,249 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawn } from "node:child_process"; +import test from "node:test"; +import { buildTelemetry, renderSummary } from "./ci-telemetry.mjs"; + +const run = { + id: 42, + event: "pull_request", + head_sha: "abc123", + status: "completed", + created_at: "2026-01-01T00:00:00Z", + run_started_at: "2026-01-01T00:00:10Z", + updated_at: "2026-01-01T00:05:00Z", +}; + +test("builds run, start-delay, job, and aggregate shard file telemetry", () => { + const telemetry = buildTelemetry({ + run, + jobs: [{ name: "Daemon tests (shard 0)", status: "completed", conclusion: "success", runner_name: "macos-26", + started_at: "2026-01-01T00:01:00Z", completed_at: "2026-01-01T00:03:00Z", + steps: [{ name: "Run isolated daemon shard", started_at: "2026-01-01T00:02:00Z", completed_at: "2026-01-01T00:02:30Z", conclusion: "success" }] }], + reports: [{ version: 1, suite: "daemon", shard: 0, platform: "darwin", complete: true, results: [ + { file: "test/a.test.mjs", status: "passed", durationMs: 1000 }, + { file: "test/b.test.mjs", status: "timed_out", durationMs: 2000 }, + ] }], + }); + assert.equal(telemetry.run.durationMs, 300000); + assert.equal(telemetry.run.durationSource, "updated_at"); + assert.equal(telemetry.run.completedJobsElapsedMs, 180000); + assert.equal(telemetry.jobs[0].startDelayMs, 50000); + assert.equal(telemetry.summary.maxJobStartDelayMs, 50000); + assert.equal(telemetry.jobs[0].executionMs, 120000); + assert.equal(telemetry.jobs[0].stepDurations[0].durationMs, 30000); + assert.equal(telemetry.summary.timedOutFileCount, 1); + assert.equal(telemetry.summary.availableReportCount, 1); + assert.equal(telemetry.summary.maxDaemonShardFileDurationMs, 3000); + assert.deepEqual(telemetry.reports.shards[0].files, [ + { file: "test/a.test.mjs", status: "passed", durationMs: 1000 }, + { file: "test/b.test.mjs", status: "timed_out", durationMs: 2000 }, + ]); + assert.equal(telemetry.summary.shardImbalanceRatioBySuite.daemon, 1); + assert.equal(telemetry.reports.suites.daemon.imbalanceRatio, 1); + assert.match(renderSummary(telemetry), /timed-out files: \*\*1\*\*/u); + assert.match(renderSummary(telemetry), /includes dependency wait/u); + assert.match(renderSummary(telemetry), /aggregate file time, not wall clock/u); + assert.match(renderSummary(telemetry), /excludes in-progress jobs; not a DAG critical path/u); + assert.match(renderSummary(telemetry), /source: updated_at/u); + assert.equal(Object.hasOwn(telemetry.run, "criticalPathMs"), false); + assert.equal(Object.hasOwn(telemetry.jobs[0], "queuedMs"), false); + for (const field of ["maxJobQueueMs", "maxCoreShardDurationMs", "maxDaemonShardDurationMs"]) { + assert.equal(Object.hasOwn(telemetry.summary, field), false); + } +}); + +test("in-progress run duration uses the injected capture time rather than updated_at", () => { + const capturedAt = "2026-01-01T00:10:00Z"; + const telemetry = buildTelemetry({ + run: { ...run, status: "in_progress", updated_at: "2026-01-01T00:00:20Z" }, + capturedAt, + jobs: [ + { status: "completed", completed_at: "2026-01-01T00:02:00Z" }, + { status: "in_progress", started_at: "2026-01-01T00:03:00Z", completed_at: "2026-01-01T00:09:00Z" }, + ], + }); + assert.equal(telemetry.capturedAt, capturedAt); + assert.equal(telemetry.run.durationSource, "capturedAt"); + assert.equal(telemetry.run.durationMs, 600000); + assert.equal(telemetry.run.completedJobsElapsedMs, 120000); + assert.match(renderSummary(telemetry), /source: capturedAt/u); + for (const updated_at of ["not-a-date", "2026-01-01T00:09:59Z"]) { + assert.equal(buildTelemetry({ run: { ...run, status: "in_progress", updated_at }, capturedAt }).run.durationMs, 600000); + } + assert.equal(buildTelemetry({ run: { ...run, status: "queued" }, capturedAt }).run.durationMs, 600000); +}); + +test("completed run duration retains the updated_at source even after collection", () => { + const telemetry = buildTelemetry({ run, capturedAt: "2026-01-02T00:00:00Z" }); + assert.equal(telemetry.run.durationMs, 300000); + assert.equal(telemetry.run.durationSource, "updated_at"); + assert.equal(telemetry.run.completedJobsElapsedMs, null); +}); + +test("default capture time is a real collection clock", () => { + const before = Date.now(); + const telemetry = buildTelemetry({ run: { status: "in_progress", created_at: new Date(before - 1000).toISOString() } }); + const captured = Date.parse(telemetry.capturedAt); + assert.ok(captured >= before && captured <= Date.now()); + assert.equal(telemetry.run.durationMs, captured - (before - 1000)); +}); + +test("keeps incomplete or malformed timestamps observable without inventing durations", () => { + const telemetry = buildTelemetry({ + run: { created_at: "not-a-date", updated_at: "2026-01-01T00:00:00Z" }, + jobs: [{ name: "broken", started_at: "not-a-date", completed_at: "2026-01-01T00:01:00Z", conclusion: "failure" }], + reports: [{ version: 2, suite: "ignored", shard: 0, results: [] }], + }); + assert.equal(telemetry.run.durationMs, null); + assert.equal(telemetry.jobs[0].executionMs, null); + assert.equal(telemetry.summary.failedJobCount, 1); + assert.equal(telemetry.summary.maxJobExecutionMs, null); + assert.deepEqual(telemetry.reports.shards, []); + assert.equal(telemetry.summary.availableReportCount, 0); + assert.equal(telemetry.summary.timedOutFileCount, null); + assert.equal(telemetry.summary.leakedProcessFileCount, null); + assert.match(renderSummary(telemetry), /timed-out files: \*\*n\/a\*\*/u); + assert.match(renderSummary(telemetry), /counts from available reports: 0/u); + assert.equal(buildTelemetry({ run: { telemetry_unavailable: true } }).telemetryUnavailable, true); + assert.equal(buildTelemetry({ jobs: { telemetry_unavailable: true } }).telemetryUnavailable, true); +}); + +test("rejects invalid, reversed, and unreasonable timing intervals without clamping", () => { + for (const [started_at, completed_at] of [ + ["not-a-date", "2026-01-01T00:01:00Z"], + ["2026-01-01T00:01:00Z", "2026-01-01T00:00:00Z"], + ["2026-01-01T00:00:00Z", "2026-02-01T00:00:00Z"], + [null, "2026-01-01T00:01:00Z"], + [0, "2026-01-01T00:01:00Z"], + [{ invalid: true }, "2026-01-01T00:01:00Z"], + ]) { + const telemetry = buildTelemetry({ + run: { ...run, created_at: started_at, updated_at: completed_at }, + jobs: [{ status: "completed", started_at, completed_at, steps: [{ started_at, completed_at }] }], + }); + assert.equal(telemetry.run.durationMs, null); + assert.equal(telemetry.run.completedJobsElapsedMs, null); + assert.equal(telemetry.jobs[0].executionMs, null); + assert.equal(telemetry.jobs[0].stepDurations[0].durationMs, null); + } + for (const started_at of ["not-a-date", "2025-12-31T23:59:00Z", "2026-02-01T00:00:00Z"]) { + const telemetry = buildTelemetry({ run, jobs: [{ started_at }] }); + assert.equal(telemetry.jobs[0].startDelayMs, null); + assert.equal(telemetry.summary.maxJobStartDelayMs, null); + } + for (const capturedAt of ["bad-clock", "2025-12-31T23:59:00Z", "2026-02-01T00:00:00Z"]) { + assert.equal(buildTelemetry({ run: { ...run, status: "in_progress" }, capturedAt }).run.durationMs, null); + } + const fallback = buildTelemetry({ run: { ...run, run_started_at: "bad" }, jobs: [{ started_at: "2026-01-01T00:01:00Z" }] }); + assert.equal(fallback.jobs[0].startDelayMs, 60000); +}); + +test("retains pending and running file details and normalizes invalid file durations", () => { + const results = [ + { file: "pending.mjs", status: "pending" }, + { file: "running.mjs", status: "running" }, + { file: "timeout.mjs", status: "timed_out", durationMs: -1 }, + { file: "leak.mjs", status: "leaked_processes", durationMs: "100" }, + ...[null, NaN, Infinity, 8 * 24 * 60 * 60 * 1000].map((durationMs, index) => ({ file: `invalid-${index}.mjs`, status: "failed", durationMs })), + { file: "passed.mjs", status: "passed", durationMs: 1000 }, + ]; + const telemetry = buildTelemetry({ reports: [{ version: 1, suite: "daemon", shard: 0, complete: false, results }] }); + const shard = telemetry.reports.shards[0]; + assert.deepEqual(shard.files, results.map(result => ({ ...result, durationMs: result.status === "passed" ? 1000 : null }))); + assert.equal(shard.complete, false); + assert.equal(shard.fileCount, results.length); + assert.equal(shard.statusCounts.pending, 1); + assert.equal(shard.statusCounts.running, 1); + assert.equal(shard.passedCount, 1); + assert.equal(shard.sumFileDurationMs, 1000); + assert.equal(shard.maxFileDurationMs, 1000); + assert.equal(shard.p95FileDurationMs, 1000); + assert.equal(telemetry.summary.timedOutFileCount, 1); + assert.equal(telemetry.summary.leakedProcessFileCount, 1); + assert.match(renderSummary(telemetry), /counts from available reports: 1/u); + assert.match(renderSummary(telemetry), /missing or incomplete reports do not establish zero faults across the run/u); +}); + +test("aggregate shard durations are file sums and retain measured zero durations", () => { + const telemetry = buildTelemetry({ reports: [ + { version: 1, suite: "core", shard: 0, results: [ + { file: "a.mjs", status: "passed", durationMs: 1000 }, + { file: "b.mjs", status: "passed", durationMs: 2000 }, + ] }, + { version: 1, suite: "core", shard: 1, results: [{ file: "c.mjs", status: "passed", durationMs: 0 }] }, + { version: 1, suite: "core", shard: 2, results: [{ file: "d.mjs", status: "pending" }] }, + ] }); + assert.equal(telemetry.summary.maxCoreShardFileDurationMs, 3000); + assert.equal(telemetry.reports.shards[0].sumFileDurationMs, 3000); + assert.equal(telemetry.reports.shards[1].sumFileDurationMs, 0); + assert.equal(telemetry.reports.shards[2].sumFileDurationMs, null); + assert.deepEqual(telemetry.reports.suites.core, { + shardCount: 3, maxShardFileDurationMs: 3000, averageShardFileDurationMs: 1500, imbalanceRatio: 2, + }); +}); + +test("p95 file duration uses nearest-rank for small samples", () => { + const samples: [number[], number | null][] = [ + [[], null], [[1000], 1000], [[2000, 1000], 2000], + [[3000, 1000, 2000], 3000], [Array.from({ length: 20 }, (_, index) => index + 1), 19], + ]; + for (const [durations, expected] of samples) { + const telemetry = buildTelemetry({ reports: [{ version: 1, suite: "core", shard: 0, + results: durations.map((durationMs, index) => ({ file: `${index}.mjs`, status: "passed", durationMs })) }] }); + assert.equal(telemetry.reports.shards[0].p95FileDurationMs, expected); + } +}); + +test("tolerates invalid reports and malformed result, job, and step entries", () => { + const invalidReports = [null, false, "bad", {}, { version: 2, suite: "core", shard: 0, results: [] }, + { version: 1, suite: "core", shard: -1, results: [] }, { version: 1, suite: "core", shard: 0, results: null }]; + assert.equal(buildTelemetry({ reports: invalidReports }).summary.availableReportCount, 0); + for (const reports of [null, {}, "bad"]) assert.deepEqual(buildTelemetry({ reports }).reports.shards, []); + assert.doesNotThrow(() => buildTelemetry({ run: null, jobs: null })); + const telemetry = buildTelemetry({ + jobs: { jobs: [null, false, "bad", { steps: {} }, { steps: [null, false, "bad", {}] }] }, + reports: [...invalidReports, { version: 1, suite: "__proto__", shard: 0, + results: [null, false, "bad", {}, { file: "retained.mjs", status: "running" }] }], + }); + assert.equal(telemetry.jobs.length, 2); + assert.deepEqual(telemetry.jobs[0].stepDurations, []); + assert.equal(telemetry.jobs[1].stepDurations.length, 1); + assert.equal(telemetry.summary.availableReportCount, 1); + assert.deepEqual(telemetry.reports.shards[0].files, [{ file: "retained.mjs", status: "running", durationMs: null }]); + assert.equal(telemetry.reports.shards[0].sumFileDurationMs, null); + assert.equal(telemetry.reports.suites.__proto__.maxShardFileDurationMs, null); +}); + +test("CLI reads a report directory and writes both telemetry outputs", async () => { + const directory = await mkdtemp(join(tmpdir(), "ci-telemetry-test-")); + try { + const runPath = join(directory, "run.json"); + const jobsPath = join(directory, "jobs.json"); + const reportsPath = join(directory, "reports"); + const outputPath = join(directory, "nested", "telemetry.json"); + const summaryPath = join(directory, "nested", "telemetry.md"); + await writeFile(runPath, JSON.stringify(run)); + await writeFile(jobsPath, JSON.stringify({ jobs: [] })); + await mkdir(reportsPath); + await writeFile(join(reportsPath, "daemon-0.json"), JSON.stringify({ + version: 1, suite: "daemon", shard: 0, results: [{ file: "pending.mjs", status: "pending" }], + })); + await writeFile(join(reportsPath, "broken.json"), "{broken"); + await writeFile(join(reportsPath, "invalid.json"), JSON.stringify({ version: 2 })); + const child = spawn(process.execPath, ["scripts/ci/ci-telemetry.mjs", + `--run=${runPath}`, `--jobs=${jobsPath}`, `--reports-dir=${reportsPath}`, + `--output=${outputPath}`, `--summary=${summaryPath}`], { cwd: process.cwd() }); + const [code] = await new Promise<[number | null, NodeJS.Signals | null]>(resolve => + child.on("close", (status, signal) => resolve([status, signal]))); + assert.equal(code, 0); + const telemetry = JSON.parse(await readFile(outputPath, "utf8")); + assert.equal(telemetry.reports.shards.length, 1); + assert.deepEqual(telemetry.reports.shards[0].files, [{ file: "pending.mjs", status: "pending", durationMs: null }]); + assert.match(await readFile(summaryPath, "utf8"), /CI timing telemetry/u); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/scripts/ci/runtime-workflow.test.ts b/scripts/ci/runtime-workflow.test.ts index 02974f2a9..48cfcbebf 100644 --- a/scripts/ci/runtime-workflow.test.ts +++ b/scripts/ci/runtime-workflow.test.ts @@ -1,5 +1,9 @@ import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; import { readFileSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import test from "node:test"; const workflow = readFileSync(new URL("../../.github/workflows/ci.yml", import.meta.url), "utf8"); @@ -9,6 +13,19 @@ function job(id: string) { return match[1]; } +function telemetryStep(name: string) { + const match = new RegExp(`^ - name: ${name}\\n([\\s\\S]*?)(?=^ - name:|$(?![\\s\\S]))`, "mu") + .exec(job("timing-telemetry")); + assert.ok(match, name); + return match[1]; +} + +function stepCommand(name: string) { + const match = /^ {8}run: \|\n((?: {10}[^\n]*(?:\n|$)|\n)*)/mu.exec(telemetryStep(name)); + assert.ok(match, `Missing executable command: ${name}`); + return match[1].replace(/^ {10}/gmu, ""); +} + test("all core matrix jobs are part of the existing mandatory unit gate", () => { const source = job("unit-tests"); assert.match(source, /matrix: \$\{\{ fromJSON\(needs\.scope\.outputs\.unit-matrix\) \}\}/u); @@ -50,6 +67,82 @@ test("the aggregate verifies reports before caching only default-branch successf assert.doesNotMatch(quality, /continue-on-error/u); }); +test("timing telemetry is independent of the mandatory quality gate", () => { + const quality = job("quality"); + const telemetry = job("timing-telemetry"); + assert.doesNotMatch(quality, /timing-telemetry|ci-telemetry|continue-on-error/u); + assert.match(telemetry, /^ needs: \[scope, quality\]$/mu); + assert.match(telemetry, /^ if: always\(\)$/mu); + assert.match(telemetry, /^ timeout-minutes: 3$/mu); + assert.match(telemetry, /^ continue-on-error: true$/mu); + assert.match(telemetry, /permissions:\n actions: read\n contents: read/u); + assert.match(telemetry, /GH_TOKEN: \$\{\{ github\.token \}\}/u); + assert.match(telemetry, /actions\/runs\/\$\{GITHUB_RUN_ID\}/u); + assert.match(telemetry, /\/attempts\/\$\{GITHUB_RUN_ATTEMPT\}\/jobs\?per_page=100/u); + assert.match(telemetry, /ci-telemetry\.mjs/u); + assert.match(telemetry, /--reports-dir=\.codex-artifacts\/ci\/reports/u); + assert.match(telemetry, /cat \.codex-artifacts\/ci\/ci-telemetry\.md >> "\$GITHUB_STEP_SUMMARY"/u); + assert.match(telemetry, /name: ci-telemetry\n include-hidden-files: true/u); + for (const name of ["Collect workflow timing metadata", "Render CI timing telemetry", "Preserve CI timing telemetry"]) { + assert.match(telemetryStep(name), /^ if: always\(\)\n continue-on-error: true$/mu, name); + } + const download = telemetryStep("Download available shard reports including failures"); + assert.match(download, /if: always\(\) && needs\.scope\.outputs\.execution-tier == 'complete'/u); + assert.match(download, /continue-on-error: true/u); + assert.match(download, /pattern: ci-test-report-\*/u); +}); + +test("workflow collection preserves unavailable observations on API and JSON failures", async t => { + for (const mode of ["success", "api-failure", "invalid-json", "invalid-pages"]) { + await t.test(mode, async () => { + const directory = await mkdtemp(join(tmpdir(), "ci-telemetry-workflow-")); + try { + const bin = join(directory, "bin"); + await mkdir(bin); + await writeFile(join(bin, "gh"), `#!/usr/bin/env node +if (process.env.TELEMETRY_API_MODE === "api-failure") process.exit(1); +if (process.argv.at(-1).includes("/jobs?")) { + const mode = process.env.TELEMETRY_API_MODE; + process.stdout.write(mode === "invalid-json" ? "{" : mode === "invalid-pages" ? "[{}]" : '[{"jobs":[{"id":1}]},{"jobs":[{"id":2}]}]'); +} else process.stdout.write('{"id":42}'); +`, { mode: 0o755 }); + const result = spawnSync("bash", ["--noprofile", "--norc", "-eo", "pipefail", "-c", stepCommand("Collect workflow timing metadata")], { + cwd: directory, encoding: "utf8", timeout: 10000, + env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, TELEMETRY_API_MODE: mode, + GITHUB_REPOSITORY: "example/repo", GITHUB_RUN_ID: "42", GITHUB_RUN_ATTEMPT: "2" }, + }); + assert.equal(result.status, 0, result.stderr); + const jobs = JSON.parse(await readFile(join(directory, ".codex-artifacts/ci/telemetry-jobs.json"), "utf8")); + if (mode === "success") assert.deepEqual(jobs, { jobs: [{ id: 1 }, { id: 2 }] }); + else assert.deepEqual(jobs, { jobs: [], telemetry_unavailable: true }); + const run = JSON.parse(await readFile(join(directory, ".codex-artifacts/ci/telemetry-run.json"), "utf8")); + if (mode === "api-failure") assert.equal(run.telemetry_unavailable, true); + else assert.equal(run.id, 42); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + } +}); + +test("workflow rendering preserves an unavailable artifact when the telemetry command fails", async () => { + const directory = await mkdtemp(join(tmpdir(), "ci-telemetry-render-")); + try { + await mkdir(join(directory, ".codex-artifacts/ci"), { recursive: true }); + const summary = join(directory, "summary.md"); + const result = spawnSync("bash", ["--noprofile", "--norc", "-eo", "pipefail", "-c", stepCommand("Render CI timing telemetry")], { + cwd: directory, encoding: "utf8", timeout: 10000, + env: { ...process.env, GITHUB_STEP_SUMMARY: summary }, + }); + assert.equal(result.status, 0, result.stderr); + const telemetry = JSON.parse(await readFile(join(directory, ".codex-artifacts/ci/ci-telemetry.json"), "utf8")); + assert.equal(telemetry.telemetryUnavailable, true); + assert.match(await readFile(summary, "utf8"), /Telemetry was unavailable/u); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + test("test-complete timing evidence survives unrelated gate failures without weakening the gate", () => { const quality = job("quality"); for (const name of [