diff --git a/.github/scripts/ci-policy.mjs b/.github/scripts/ci-policy.mjs index 87f6ed389..2c93e5101 100644 --- a/.github/scripts/ci-policy.mjs +++ b/.github/scripts/ci-policy.mjs @@ -131,7 +131,35 @@ export function selectScope({ eventName, files, site, enumerationFailed = false }; } -export function validateGate(needs, eventName) { +export function admissionState(eventName, event) { + if (!['pull_request', 'merge_group', 'workflow_dispatch'].includes(eventName)) throw new Error('unsupported admission event'); + if (eventName !== 'pull_request') return 'ready'; + if (typeof event?.pull_request?.draft !== 'boolean') throw new Error('missing or invalid pull request draft metadata'); + return event.pull_request.draft ? 'deferred' : 'ready'; +} + +export class AdmissionDeferredError extends Error { + constructor() { + super('Draft admission deferred: native acceptance was not executed in this run; this PR is not admitted. The root scheduler must release it for fresh ready_for_review validation.'); + this.name = 'AdmissionDeferredError'; + } +} + +export function validateGate(needs, eventName, admission = 'ready') { + if (!['ready', 'deferred'].includes(admission)) throw new Error('invalid admission state'); + if (admission === 'deferred') { + if (eventName !== 'pull_request') throw new Error('only draft pull requests may defer admission'); + throw new AdmissionDeferredError(); + } + validateResults(needs, eventName, false); +} + +export function validatePreview(needs, eventName, admission) { + if (eventName !== 'pull_request' || admission !== 'deferred') throw new Error('preview requires a genuine draft pull request'); + validateResults(needs, eventName, true); +} + +function validateResults(needs, eventName, preview) { if (!needs || needs.scope?.result !== 'success') throw new Error('scope job did not succeed'); if (!['pull_request', 'merge_group', 'workflow_dispatch'].includes(eventName) || needs.scope.outputs?.event_name !== eventName) throw new Error('scope event does not match this run'); @@ -170,7 +198,7 @@ export function validateGate(needs, eventName) { network: flags.network_check, applications: flags.applications_check, database: flags.database_check, }; for (const [job, mustRun] of Object.entries(required)) { - const expected = mustRun ? 'success' : 'skipped'; + const expected = !preview && mustRun ? 'success' : 'skipped'; if (needs[job]?.result !== expected) throw new Error(`${job}: expected ${expected}, got ${needs[job]?.result ?? 'missing'}`); } if (needs.validation?.result !== 'success') throw new Error('validation job did not succeed'); @@ -187,10 +215,12 @@ function classify() { const root = process.cwd(); const eventName = process.env.GITHUB_EVENT_NAME; let files; + let admission = eventName === 'pull_request' ? 'unknown' : 'ready'; let enumerationFailed = false; try { const event = JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, 'utf8')); if (eventName === 'pull_request') { + admission = admissionState(eventName, event); files = changedPaths({ root, base: event.pull_request?.base?.sha, head: event.pull_request?.head?.sha }); } else if (eventName === 'merge_group') { files = changedPaths({ root, base: event.merge_group?.base_sha, head: event.merge_group?.head_sha, pullRequest: false }); @@ -206,23 +236,39 @@ function classify() { const output = Object.entries(decision).map(([key, value]) => `${key}=${value}\n`).join(''); if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, output); const summary = `CI scope: ${decision.reason}\n\n${output}\n` + + (admission === 'deferred' ? 'Admission deferred: native checks will not execute. Light checks do not admit this draft or establish native acceptance.\n' : + admission === 'unknown' ? 'Invalid draft metadata: full scope fallback; the required gate must reject admission.\n' : '') + (!decision.site_available ? (decision.docs_check ? 'The required Nimbus site is missing or invalid; its job must fail.\n' : 'Nimbus is not introduced in this checkout; site validation is explicitly not required and all native modules remain required.\n') : ''); if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); - console.log(JSON.stringify({ ...decision, file_count: files?.length ?? null }, null, 2)); + console.log(JSON.stringify({ ...decision, admission, file_count: files?.length ?? null }, null, 2)); } if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { try { if (process.argv[2] === 'classify') classify(); - else if (process.argv[2] === 'gate') { - validateGate(JSON.parse(process.env.CI_NEEDS_JSON), process.env.GITHUB_EVENT_NAME); - console.log('Every required job and validation step succeeded; only explicitly unnecessary checks were skipped.'); + else if (['gate', 'preview'].includes(process.argv[2])) { + const event = JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, 'utf8')); + const admission = admissionState(process.env.GITHUB_EVENT_NAME, event); + const needs = JSON.parse(process.env.CI_NEEDS_JSON); + if (process.argv[2] === 'preview') { + validatePreview(needs, process.env.GITHUB_EVENT_NAME, admission); + const summary = 'Draft preview only: scope and selected light checks succeeded. Native acceptance was not executed in this run; this PR is not admitted. This preview does not report CI - required success. Root scheduling must release it for fresh ready_for_review checks before admission.\n'; + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); + console.log(summary.trim()); + } else { + validateGate(needs, process.env.GITHUB_EVENT_NAME, admission); + console.log('Every required job and validation step succeeded; only explicitly unnecessary checks were skipped.'); + } } else if (process.argv[2] === 'require-site') { if (!inspectSite(process.cwd()).available) throw new Error('Required Nimbus package, lockfile or check script is unavailable'); - } else throw new Error('Expected classify, gate or require-site'); + } else throw new Error('Expected classify, gate, preview or require-site'); } catch (error) { - console.error(error.message); + if (error instanceof AdmissionDeferredError && process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, + 'Admission deferred (not a passing check). Native acceptance was not executed in this run; this draft is not admitted. Light-check results remain separate. Root scheduling must release it for fresh ready_for_review checks.\n'); + } + console.error(`${error instanceof AdmissionDeferredError ? '[DRAFT_ADMISSION_DEFERRED] ' : ''}${error.message}`); process.exitCode = 1; } } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa46901fc..2edfa096f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ run-name: CI - ${{ github.event_name }} - ${{ github.ref_name }} on: pull_request: branches: [main] + types: [opened, synchronize, reopened, ready_for_review, converted_to_draft] merge_group: types: [checks_requested] workflow_dispatch: @@ -91,7 +92,7 @@ jobs: network: name: Network - ${{ matrix.os }} - Boost - ${{ matrix.sanitizer }} needs: scope - if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.network_check != 'false') }} + if: ${{ !cancelled() && !(github.event_name == 'pull_request' && toJSON(github.event.pull_request.draft) == 'true') && (needs.scope.result != 'success' || needs.scope.outputs.network_check != 'false') }} runs-on: ${{ matrix.os }} timeout-minutes: 45 strategy: @@ -156,7 +157,7 @@ jobs: applications: name: Applications - ${{ matrix.os }} - C++20 needs: scope - if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.applications_check != 'false') }} + if: ${{ !cancelled() && !(github.event_name == 'pull_request' && toJSON(github.event.pull_request.draft) == 'true') && (needs.scope.result != 'success' || needs.scope.outputs.applications_check != 'false') }} runs-on: ${{ matrix.os }} timeout-minutes: 45 strategy: @@ -220,7 +221,7 @@ jobs: database: name: Database - ${{ matrix.os }} - ${{ matrix.asio }} needs: scope - if: ${{ !cancelled() && (needs.scope.result != 'success' || needs.scope.outputs.database_check != 'false') }} + if: ${{ !cancelled() && !(github.event_name == 'pull_request' && toJSON(github.event.pull_request.draft) == 'true') && (needs.scope.result != 'success' || needs.scope.outputs.database_check != 'false') }} runs-on: ${{ matrix.os }} timeout-minutes: 45 strategy: @@ -292,7 +293,7 @@ jobs: - name: Run local database benchmark self-tests run: cmake --build ci-database --target check-database-benchmark required: - name: CI - required + name: ${{ github.event_name == 'pull_request' && toJSON(github.event.pull_request.draft) == 'true' && 'CI - draft preview' || 'CI - required' }} if: always() needs: [scope, network, applications, database, validation] runs-on: ubuntu-latest @@ -304,7 +305,7 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 26 - - name: Require successful builds and regressions + - name: Validate acceptance or draft preview env: CI_NEEDS_JSON: ${{ toJSON(needs) }} - run: node .github/scripts/ci-policy.mjs gate + run: node .github/scripts/ci-policy.mjs ${{ github.event_name == 'pull_request' && toJSON(github.event.pull_request.draft) == 'true' && 'preview' || 'gate' }} diff --git a/AGENTS.md b/AGENTS.md index b88509f89..f462f518e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,7 +29,8 @@ branch or worktree when the existing checkout contains uncommitted changes. are resolved. Outside the merge queue, synchronize the PR head with `main` before acceptance. With the merge queue enabled, validate the queue candidate containing the latest `main` and preceding queued changes; a green PR head alone - does not validate that combination. Add eligible PRs to the queue promptly. + does not validate that combination. Admit eligible PRs in the creation order + defined below. Another maintainer's approval is not currently mandatory. Merge eligible PRs without asking for confirmation again. - Explicitly set the squash commit subject to the Conventional Commit PR title. @@ -38,11 +39,11 @@ branch or worktree when the existing checkout contains uncommitted changes. ## Verification and delivery -- When runner capacity is shared by several PRs, prioritize the oldest unblocked - PRs and their required queue candidates. Prepare newer work locally and defer - its hosted runs until older ready PRs can finish; do not repeatedly push newer - branches ahead of the backlog. Preserve active acceptance runs. A cancelled - scheduling run still needs fresh successful checks before acceptance. +- Follow the root scheduler and PR creation order defined below when runner + capacity is shared. Prepare newer work locally and defer hosted admission; + do not repeatedly push newer branches ahead of the backlog. Preserve active + acceptance runs. A cancelled scheduling run still needs fresh successful checks + before acceptance. - Run the checks relevant to the change before publishing. Report the exact tested revision, commands, results and material limitations. - Investigate failed CI logs before changing code or retrying. Do not repeatedly @@ -81,12 +82,16 @@ branch or worktree when the existing checkout contains uncommitted changes. - CI policy tests: `node --test tests/ci/*.test.mjs`. Nimbus documentation: `npm ci --prefix docs/site` followed by `npm run check --prefix docs/site`. - Keep official plugins in the ark repository and application business plugins - in application repositories. The approved layout has one public header root, - `include/ark/`, with implementation under `src/core/` and - `src/plugins//`; retain independent plugin targets and optional builds. - Verify migration status against accepted source before describing new paths - as available. Do not expand the paused hot-update design without discussing - its lifecycle and compatibility boundaries. + in application repositories. The approved next plugin layout co-locates public + headers, private declarations and implementation files inside each plugin; + do not reintroduce a mandatory per-plugin include/src split. Keep public header + mappings explicit and independent from physical paths. Concrete implementation + types may live in cpp files when registration and tests no longer need their + definitions. Public-header staging/installation and creation-macro replacements + require the agreed design before implementation; do not silently change ABI, + lifecycle or ownership. Retain independent plugin targets and optional builds. + Verify accepted source before claiming the new layout or an installed SDK is + available. Hot-update and Registry design discussions remain paused. ## Documentation and releases @@ -104,3 +109,90 @@ branch or worktree when the existing checkout contains uncommitted changes. - Keep raw measurements and build logs outside the source repository. Published performance claims must identify the machine, workload, units and tested revision; local tests do not establish production PCU capacity. + +## Task execution and CI admission + +- Use `gh` CLI for GitHub operations; do not replace it with direct REST/GraphQL + requests or `gh api`. Normal Git fetch/push commands remain appropriate. +- The root agent alone observes CI and controls PR readiness and queue admission. + Order admission by PR creation time: finish the oldest unfinished PR before + admitting newer ones. A newer PR becoming green first is not permission to + overtake an older PR. Account for shared runner capacity across ark and arknet. +- Continue independent implementation and local validation in parallel. Publish + focused modules without starting newer native matrices ahead of the backlog. + Use drafts only when the deployed workflow actually defers draft native jobs; + otherwise retain a reviewable local branch until admission is available. +- Do not enable automatic merge for held PRs. Release them in creation order; + after required checks pass, squash through the protected merge queue promptly. + Do not repeatedly ask for already-authorized merges or bypass any check. +- When a draft-preview workflow is deployed, require its fresh `ready_for_review` + run to succeed at the intended head before enabling auto-merge or queue admission. + Preview success, an absent required context, or older green checks on the same + SHA do not establish fresh admission. Verify the hosted behavior before claiming it works. +- Give jobs unique, descriptive names. Consolidate compatible jobs/steps to reduce + setup time without dropping required platforms, protocols or verification. +- Preserve active acceptance runs. If correcting scheduling requires cancellation, + record its reason and arrange fresh acceptance; cancellation never counts as + a successful test. GitHub Actions concurrency groups do not by themselves + enforce PR-creation order. +- Keep one observer per PR batch. Batch status reads, back off unchanged polling, + and let scripts collect logs and exit codes. Read summaries and relevant failure + excerpts, not entire repeated logs. Communicate completion, failures, interface + changes and decisions; do not repeatedly ask agents for unchanged progress. +- Delegate only concrete independent deliveries with a scope and acceptance + criteria. Do not wake finished agents just to keep them busy. Record a short + handoff at module completion and preserve unrelated work in dirty checkouts. +- Current-session monitoring does not authorize recurring/background automation. + Stop owned observers before ending the session unless separately authorized. + +## Keep documentation synchronized + +- Resolve legacy page aliases to their canonical source before editing. Nimbus + may skip retired Markdown files even when they remain in Git. Check the emitted + HTML and Markdown for the changed content; a successful build alone does not + prove that an edited source is published. +- Treat documentation as part of each functional change. Review both languages, + README, feature/support tables, API and client/server examples, configuration, + installation, lifecycle, navigation and affected performance/validation reports. + Update the relevant material in the same PR; do not routinely defer it. +- Describe the final supported behavior. Remove stale TODO/experimental/private + development-layer claims when accepted code supersedes them, and avoid + redundant "implemented", "retest" or "latest test" labels. Keep actual future + capabilities clearly marked TODO; do not imply they already work. +- Keep ark and arknet boundaries clear. Cross-link the other project's accepted + public APIs rather than copying them or claiming pending integration is merged. + Never attribute later main changes or unpublished results to an older release. +- Run the relevant Nimbus/link checks. After publication, verify the Pages run + succeeded for the intended source revision and inspect affected published pages. + A local preview, merged PR and deployed site are distinct evidence states. +- Use the full names AddressSanitizer, UndefinedBehaviorSanitizer and + ThreadSanitizer in explanatory prose. Preserve command flags, paths, workflow + identifiers and machine-readable fields. Call Asio/Beast dependencies rather + than backends; capitalize Standalone Asio in prose. +- Prefer SVG or Mermaid for editable diagrams and charts. Show labeled units and + readable legends, retain useful error bars, and avoid repeating chart contents + in long prose. Keep the README concise, with properly formatted dependency lists; + detailed guidance belongs in the bilingual documentation site. + +## Evidence and performance boundaries + +- Keep raw measurement JSON, packet captures, logs and credentials local, outside + version control. Do not upload them as release assets, CI artifacts or public + downloads without separate explicit authorization for those exact files. +- Extend existing doctest correctness tests rather than adding a Boost test + dependency. Prefer standard C++20 facilities over obsolete wrappers/macros when + behavior and ownership can be preserved; keep necessary platform/ABI boundaries. +- Maintain compile-and-run C++ tests and benchmarks in their project directories; + do not require Python or Ruby as the user's test entry point. Shell scripts may + orchestrate internal batches. Do not replace permanent tests with temporary ones. +- Measure performance locally before writing reports. Identify machine, OS, + compiler, dependency versions, source/binary identity, duration and units. + Compare payload sizes, in-flight batches, repeated runs, coroutine/callback + interfaces and IO/thread models, including one context/one thread and one + context/multiple threads. Report throughput, latency and CPU utilization; + 100% CPU denotes one logical core, with the measurement scope stated. +- Retain the final valid results and material limitations, not a retesting diary. + Functional checks, sanitizer checks, performance measurements and deployment + capacity are separate claims. Loopback tests do not prove production PCU or + cross-host capacity; successful controls do not establish an unresolved root + cause. Do not extend timeouts or rerun to green to conceal failures. diff --git a/tests/ci/README.md b/tests/ci/README.md index 783ea3e22..63ae089c9 100644 --- a/tests/ci/README.md +++ b/tests/ci/README.md @@ -57,13 +57,44 @@ that passed their required checks; its resulting push does not repeat CI for the same accepted tree. The separate documentation workflow still publishes on main pushes. README badges display the merge-group workflow status. +Draft PRs retain their real scope and required-check flags. Scope classification, +documentation, root README links and formatting run when selected, but genuine +boolean `draft: true` metadata prevents new native jobs from starting. Only those +draft events rename the aggregate to `CI - draft preview` and select its separate +preview validator. A successful preview requires scope success, every selected +light step's actual `success`, every unselected light step's `skipped`, and every +native job's `skipped`. Its summary states that native acceptance was not executed +in this run and the PR is not admitted. It does not emit a successful +`CI - required` check. The final gate still rejects deferred admission. + +An initial draft run therefore creates no `CI - required` context; the protection +rule keeps that exact required name. The intended Expected/pending display needs +hosted confirmation before deployment. Missing or non-boolean draft metadata +selects conservative full scope, retains the required name and final gate, and is +rejected. Light-check failures still fail the preview. + +`ready_for_review` triggers fresh source validation with the existing module +selection; opening, updating and reopening ready PRs retain their normal checks. +`converted_to_draft` applies the hold to a new run. Neither event cancels an active +run. Code merge-group candidates and manual runs still require the full native +matrix, and CI policy changes still require every source module before the full +queue candidate is accepted. The workflow does not enqueue PRs. The root agent +alone admits the oldest unfinished PR by creation time; newer work stays local or +in draft with auto-merge disabled until earlier PRs are accepted. Scheduling-only +cancellations require fresh acceptance before admission. Returning an already +validated SHA to draft does not erase old check contexts. Keep auto-merge disabled +and require the fresh `ready_for_review` run to succeed before enabling auto-merge +or queueing; an earlier success on the same SHA is insufficient. Dynamic names do +not themselves enforce this freshness or a global admission order. Full code +queue validation remains necessary. + Each existing PR/ref concurrency group permits one active run and only the most recently queued pending run. A new run waits for the active run to finish instead of automatically cancelling it. Individual running jobs retain their configured timeouts; those limits do not include pending time. Merge-group candidates use -their own refs. The required gate still rejects failed, cancelled or missing -results. Manual cancellation can still prevent the `always()` summary from -completing. See [GitHub concurrency rules](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#concurrency). +their own refs. Independent groups do not enforce PR creation order. The required +gate still rejects failed, cancelled or missing results. Manual cancellation can +still prevent the `always()` summary from completing. See [GitHub concurrency rules](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#concurrency). Within `src/`, only the exact plugin index paths `src/plugins/README.md` and `src/plugins/README_CN.md` additionally qualify as documentation. Changes confined diff --git a/tests/ci/README_CN.md b/tests/ci/README_CN.md index 6947266a8..49ecd4390 100644 --- a/tests/ci/README_CN.md +++ b/tests/ci/README_CN.md @@ -39,10 +39,31 @@ required gate 核对当前工作流的真实事件和声明的验证阶段。选 合入后的推送不再对同一代码树重复执行 CI。独立的文档工作流仍在 main 推送后发布文档。 README 徽章展示合并候选的工作流状态。 +草稿 PR 保留真实范围和必需检查标志，按范围执行分类、文档、根 README 链接和格式检查。 +只有事件元数据中的 `draft` 是布尔值 `true` 时，才阻止新原生任务启动， +并将汇总任务命名为 `CI - draft preview`，使用独立预览校验。 +预览通过要求 scope 成功、所有选中轻量步骤的实际结果为 `success`、未选中步骤为 `skipped`， +且全部原生任务为 `skipped`。摘要明确说明本轮未执行原生验收、PR 尚未获准入场。 +预览不会输出成功的 `CI - required` 检查；最终门禁仍拒绝延后的入场请求。 + +因此，首次草稿运行不会创建 `CI - required` 上下文；保护规则仍保留这一精确必需名称。 +预期的 Expected/pending 界面状态须在部署前通过托管草稿运行确认。 +缺失或非布尔值的草稿元数据会回退到保守完整范围，保留必需名称及最终门禁，并被拒绝。 +轻量检查失败仍使预览失败。 + +`ready_for_review` 触发新一轮源分支验证，沿用现有模块选择；普通 ready PR 的创建、更新和重新打开仍执行原有检查。 +`converted_to_draft` 在新一轮运行中实施延后；这两类事件均不取消已启动的运行。 +代码合并候选与手动运行仍要求完整原生矩阵；CI 策略变更仍须先通过全部源分支模块，再验收完整队列候选。 +工作流本身不将 PR 加入队列。root 代理独占入场调度，按创建时间优先放行最早尚未完成的 PR； +前序 PR 验收前，新工作保留在本地或以关闭自动合并的草稿发布。仅因调度取消的运行仍须重新完成验收，才能获准入场。 +已验证的同一 SHA 转回草稿不会清除旧检查上下文。须保持关闭自动合并，确认新一轮 `ready_for_review` 运行成功后， +才能开启自动合并或加入队列；同一 SHA 的旧成功记录不能替代新一轮入场检查。 +动态名称本身不能强制执行这一新鲜度要求或全局入场顺序，代码候选仍须完成完整队列验证。 + 每个既有的 PR/引用并发分组最多执行一轮，只保留最新进入该分组的一轮等待任务。 新运行等待旧轮结束，不再自动取消正在执行的旧轮。每个已运行的 job 保留原有超时限制， -等待时间不计入这些限制；Merge Queue 候选使用各自的引用分组。required gate 仍拒绝失败、 -取消或缺失的结果。手动取消仍可能使 `always()` 汇总无法完成。 +等待时间不计入这些限制；Merge Queue 候选使用各自的引用分组。独立分组不保证按 PR 创建顺序执行。 +required gate 仍拒绝失败、取消或缺失的结果。手动取消仍可能使 `always()` 汇总无法完成。 详见 [GitHub 并发规则](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#concurrency)。 `src/` 下仅将两个精确的插件索引路径 `src/plugins/README.md` 与 `src/plugins/README_CN.md` 归为文档。 diff --git a/tests/ci/draft-admission.test.mjs b/tests/ci/draft-admission.test.mjs new file mode 100644 index 000000000..a6873bb7d --- /dev/null +++ b/tests/ci/draft-admission.test.mjs @@ -0,0 +1,349 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { runInNewContext } from 'node:vm'; +import test from 'node:test'; +import { admissionState, AdmissionDeferredError, selectScope, validateGate, validatePreview } from '../../.github/scripts/ci-policy.mjs'; + +const script = fileURLToPath(new URL('../../.github/scripts/ci-policy.mjs', import.meta.url)); +const workflow = readFileSync(new URL('../../.github/workflows/ci.yml', import.meta.url), 'utf8').replaceAll('\r\n', '\n'); +const jobs = ['network', 'applications', 'database']; +const site = { present: true, available: true }; +const plan = (files, eventName = 'pull_request') => selectScope({ files, eventName, site }); +const results = scope => ({ + scope: { result: 'success', outputs: Object.fromEntries(Object.entries(scope).map(([key, value]) => [key, String(value)])) }, + ...Object.fromEntries(jobs.map(job => [job, { result: scope[`${job}_check`] ? 'success' : 'skipped' }])), + validation: { result: 'success', outputs: { + documentation_outcome: scope.docs_check ? 'success' : 'skipped', + readme_outcome: scope.readme_check ? 'success' : 'skipped', + formatting_outcome: scope.format_check ? 'success' : 'skipped', + } }, +}); +const jobBlock = name => workflow.split(/(?=^ [a-z][a-z0-9_-]*:$)/m).find(block => block.startsWith(` ${name}:\n`)); + +// Evaluate the actual workflow guard with its JSON conversion and the subset of +// expression operators it uses. Do not duplicate the guard as a test-only policy. +function nativeRuns(job, scope, eventName, draft, scopeResult = 'success') { + const expression = jobBlock(job).match(/^ if: \$\{\{ (.+) \}\}$/m)?.[1]; + assert.ok(expression, `${job}: explicit native admission guard`); + return runInNewContext(expression, { + cancelled: () => false, toJSON: JSON.stringify, + github: { event_name: eventName, event: { pull_request: { draft } } }, + needs: { scope: { result: scopeResult, outputs: results(scope).scope.outputs } }, + }); +} + +function fixture(t) { + const root = mkdtempSync(resolve(tmpdir(), 'ark-draft-admission-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const git = (...args) => execFileSync('git', args, { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); + const write = (file, text) => { mkdirSync(dirname(resolve(root, file)), { recursive: true }); writeFileSync(resolve(root, file), text); }; + const commit = () => { git('add', '-A'); git('-c', 'commit.gpgsign=false', 'commit', '-m', 'Draft admission fixture'); return git('rev-parse', 'HEAD'); }; + git('init', '--initial-branch=main'); + git('config', 'user.name', 'ark CI tests'); + git('config', 'user.email', 'ci-tests@example.invalid'); + git('config', 'core.hooksPath', resolve(root, 'empty-hooks')); + write('docs/site/package.json', JSON.stringify({ scripts: { check: 'node check.mjs' } })); + write('docs/site/package-lock.json', '{}'); + write('docs/guide.md', '# Guide\n'); + write('tests/network/example.cpp', 'int example;\n'); + const base = commit(); + const metadata = mkdtempSync(resolve(tmpdir(), 'ark-draft-event-')); + t.after(() => rmSync(metadata, { recursive: true, force: true })); + function invoke(command, eventName, event, needs) { + const eventPath = resolve(metadata, 'event.json'); + const summaryPath = resolve(metadata, 'summary'); + const outputPath = resolve(metadata, 'outputs'); + writeFileSync(eventPath, typeof event === 'string' ? event : JSON.stringify(event)); + writeFileSync(summaryPath, ''); + writeFileSync(outputPath, ''); + const env = { ...process.env, GITHUB_EVENT_NAME: eventName, GITHUB_EVENT_PATH: eventPath, + GITHUB_STEP_SUMMARY: summaryPath, GITHUB_OUTPUT: outputPath, CI_NEEDS_JSON: JSON.stringify(needs) }; + delete env.GITHUB_TOKEN; + delete env.GH_TOKEN; + const run = spawnSync(process.execPath, [script, command], { cwd: root, env, encoding: 'utf8' }); + assert.ifError(run.error); + return { ...run, summary: readFileSync(summaryPath, 'utf8'), output: readFileSync(outputPath, 'utf8') }; + } + return { root, write, commit, base, invoke }; +} + +const pr = (base, head, draft = false, action = 'synchronize') => ({ action, pull_request: { draft, base: { sha: base }, head: { sha: head } } }); + +test('only explicit boolean draft metadata can defer a pull request', () => { + assert.equal(admissionState('pull_request', { pull_request: { draft: true } }), 'deferred'); + assert.equal(admissionState('pull_request', { pull_request: { draft: false } }), 'ready'); + for (const draft of [undefined, null, 'true', 'false', 0, 1, {}, []]) { + assert.throws(() => admissionState('pull_request', { pull_request: { draft } }), /draft metadata/); + } + for (const event of [undefined, {}, { pull_request: null }]) assert.throws(() => admissionState('pull_request', event)); + for (const eventName of ['merge_group', 'workflow_dispatch']) { + assert.equal(admissionState(eventName, { pull_request: { draft: true } }), 'ready'); + } + assert.throws(() => admissionState('push', {}), /unsupported/); +}); + +test('workflow admits fresh ready events and keeps existing concurrency and strict gate', () => { + assert.match(workflow, /^ types: \[opened, synchronize, reopened, ready_for_review, converted_to_draft\]$/m); + assert.match(workflow, /^ cancel-in-progress: false$/m); + assert.match(jobBlock('required'), /^ if: always\(\)$/m); + assert.match(jobBlock('required'), /^ name: \$\{\{ .* && 'CI - draft preview' \|\| 'CI - required' \}\}$/m); + assert.doesNotMatch(jobBlock('required'), /continue-on-error:|if:.*draft/); + assert.match(jobBlock('validation'), /^ if: \$\{\{ !cancelled\(\) \}\}$/m, 'light validation is not draft-gated'); + assert.doesNotMatch(workflow, /pull_request_target:|gh pr |enable-auto-merge|merge-queue/); +}); + +test('actual native guards suppress drafts but preserve ready source selection and full queue matrices', () => { + for (const files of [['tests/network/example.cpp'], ['README.md'], ['.github/workflows/ci.yml'], []]) { + const scope = plan(files); + for (const job of jobs) { + assert.equal(nativeRuns(job, scope, 'pull_request', true), false, `${job}: draft ${files}`); + assert.equal(nativeRuns(job, scope, 'pull_request', true, 'failure'), false, `${job}: failed draft scope`); + assert.equal(nativeRuns(job, scope, 'pull_request', false), scope[`${job}_check`], `${job}: ready ${files}`); + for (const draft of [undefined, null, 'true', 1]) { + assert.equal(nativeRuns(job, plan([]), 'pull_request', draft), true, `${job}: invalid metadata must not suppress full fallback`); + } + assert.equal(nativeRuns(job, scope, 'pull_request', false, 'failure'), true, `${job}: failed ready scope runs conservatively`); + } + } + const queue = plan(['tests/network/example.cpp'], 'merge_group'); + assert.equal(queue.validation_stage, 'full'); + let cells = 0; + for (const job of jobs) { + assert.equal(nativeRuns(job, queue, 'merge_group', true), true, `${job}: PR metadata cannot defer queue`); + const matrices = jobBlock(job).match(/^ matrix: .* && '([^']+)' \|\| '([^']+)'\) \}\}$/m); + assert.ok(matrices); + cells += JSON.parse(matrices[2]).include.length; + } + assert.equal(cells, 12); +}); + +test('deferred gate never passes even documentation or successful-looking native results', () => { + for (const files of [['README.md'], ['tests/network/example.cpp'], ['.github/workflows/ci.yml']]) { + const scope = plan(files); + for (const skipped of [false, true]) { + const needs = results(scope); + if (skipped) for (const job of jobs) needs[job].result = 'skipped'; + assert.throws(() => validateGate(needs, 'pull_request', 'deferred'), AdmissionDeferredError); + } + validateGate(results(scope), 'pull_request', 'ready'); + } + assert.throws(() => validateGate({}, 'merge_group', 'deferred'), /only draft/); + assert.throws(() => validateGate({}, 'pull_request', 'unknown'), /invalid admission/); +}); + +test('real draft classification preserves required flags and ready_for_review requires real native success', t => { + const repo = fixture(t); + repo.write('tests/network/example.cpp', 'int changed;\n'); + const head = repo.commit(); + const draftEvent = pr(repo.base, head, true, 'opened'); + const draftRun = repo.invoke('classify', 'pull_request', draftEvent); + assert.equal(draftRun.status, 0, draftRun.stderr); + const draft = JSON.parse(draftRun.stdout); + assert.equal(draft.admission, 'deferred'); + assert.equal(draft.network_check, true); + assert.equal(draft.native_check, true); + assert.equal(draft.validation_stage, 'modules'); + assert.equal(draft.format_check, true); + assert.match(draftRun.summary, /Light checks do not admit this draft/); + const held = results(draft); + held.network.result = 'skipped'; + const gate = repo.invoke('gate', 'pull_request', draftEvent, held); + assert.equal(gate.status, 1); + assert.match(gate.stderr, /\[DRAFT_ADMISSION_DEFERRED\]/); + assert.match(gate.summary, /not a passing check/); + assert.match(gate.summary, /not admitted/); + assert.doesNotMatch(gate.stdout, /Every required job/); + + const readyEvent = pr(repo.base, head, false, 'ready_for_review'); + const ready = JSON.parse(repo.invoke('classify', 'pull_request', readyEvent).stdout); + assert.equal(ready.admission, 'ready'); + const { admission: draftAdmission, ...draftScope } = draft; + const { admission: readyAdmission, ...readyScope } = ready; + assert.deepEqual(readyScope, draftScope, 'ready event changes admission, not scope requirements'); + assert.equal(repo.invoke('gate', 'pull_request', readyEvent, held).status, 1, 'skipped native cannot admit ready PR'); + assert.equal(repo.invoke('gate', 'pull_request', readyEvent, results(ready)).status, 0); + for (const outcome of ['failure', 'cancelled', 'skipped']) { + const needs = results(ready); + needs.network.result = outcome; + assert.equal(repo.invoke('gate', 'pull_request', readyEvent, needs).status, 1); + } +}); + +test('documentation drafts stay unadmitted even when every real light step succeeds', t => { + const repo = fixture(t); + repo.write('docs/guide.md', '# Updated guide\n'); + const head = repo.commit(); + const event = pr(repo.base, head, true, 'converted_to_draft'); + const scope = JSON.parse(repo.invoke('classify', 'pull_request', event).stdout); + assert.equal(scope.native_check, false); + assert.equal(scope.docs_check, true); + const gate = repo.invoke('gate', 'pull_request', event, results(scope)); + assert.equal(gate.status, 1); + assert.match(gate.stderr, /DRAFT_ADMISSION_DEFERRED/); + const readyEvent = pr(repo.base, head, false, 'ready_for_review'); + assert.equal(repo.invoke('gate', 'pull_request', readyEvent, results(scope)).status, 0); +}); + +test('invalid draft metadata and invalid ready ranges cannot enable a lightweight bypass', t => { + const repo = fixture(t); + repo.write('docs/guide.md', '# Updated guide\n'); + const head = repo.commit(); + for (const draft of [undefined, null, 'true', 'false', 0, 1]) { + const event = pr(repo.base, head, draft); + if (draft === undefined) delete event.pull_request.draft; + const run = repo.invoke('classify', 'pull_request', event); + const scope = JSON.parse(run.stdout); + assert.equal(scope.admission, 'unknown'); + assert.equal(scope.validation_stage, 'full'); + assert.equal(scope.reason, 'enumeration-unavailable'); + for (const job of jobs) assert.equal(scope[`${job}_check`], true); + assert.match(run.summary, /Invalid draft metadata/); + const gate = repo.invoke('gate', 'pull_request', event, results(scope)); + assert.equal(gate.status, 1, 'even successful-looking results cannot replace real draft metadata'); + assert.match(gate.stderr, /draft metadata/); + } + for (const base of [undefined, 'invalid', '0'.repeat(40), 'f'.repeat(40)]) { + const event = pr(base, head, false, 'ready_for_review'); + const scope = JSON.parse(repo.invoke('classify', 'pull_request', event).stdout); + assert.equal(scope.validation_stage, 'full'); + assert.equal(scope.reason, 'enumeration-unavailable'); + const incomplete = results(scope); + incomplete.database.result = 'skipped'; + assert.equal(repo.invoke('gate', 'pull_request', event, incomplete).status, 1); + } + assert.equal(repo.invoke('gate', 'pull_request', '{not JSON', results(plan(['README.md']))).status, 1); +}); + +test('queue code remains full and cannot reuse source module acceptance', t => { + const repo = fixture(t); + repo.write('tests/network/example.cpp', 'int changed;\n'); + const head = repo.commit(); + const event = { merge_group: { base_sha: repo.base, head_sha: head }, pull_request: { draft: true } }; + const scope = JSON.parse(repo.invoke('classify', 'merge_group', event).stdout); + assert.equal(scope.admission, 'ready'); + assert.equal(scope.validation_stage, 'full'); + for (const job of jobs) assert.equal(scope[`${job}_check`], true); + assert.equal(repo.invoke('gate', 'merge_group', event, results(scope)).status, 0); + const source = plan(['tests/network/example.cpp']); + assert.equal(repo.invoke('gate', 'merge_group', event, results(source)).status, 1); + const incomplete = results(scope); + incomplete.applications.result = 'skipped'; + assert.equal(repo.invoke('gate', 'merge_group', event, incomplete).status, 1); +}); + +function aggregateFor(eventName, draft) { + const aggregate = jobBlock('required'); + const name = aggregate.match(/^ name: \$\{\{ (.+) \}\}$/m)?.[1]; + const command = aggregate.match(/^ run: node \.github\/scripts\/ci-policy\.mjs \$\{\{ (.+) \}\}$/m)?.[1]; + assert.ok(name); + assert.ok(command); + const predicate = "github.event_name == 'pull_request' && toJSON(github.event.pull_request.draft) == 'true'"; + assert.equal(name, `${predicate} && 'CI - draft preview' || 'CI - required'`); + assert.equal(command, `${predicate} && 'preview' || 'gate'`); + const context = { github: { event_name: eventName, event: { pull_request: { draft } } }, toJSON: JSON.stringify }; + // Both job name and run support github context. No step, matrix, env or + // unavailable job-output context is involved in these admission expressions. + return { name: runInNewContext(name, context), command: runInNewContext(command, context) }; +} + +const previewResults = scope => { + const needs = results(scope); + for (const job of jobs) needs[job].result = 'skipped'; + return needs; +}; + +test('actual aggregate expressions select only draft preview and leave the required name to ready or queue runs', () => { + assert.deepEqual(aggregateFor('pull_request', true), { name: 'CI - draft preview', command: 'preview' }); + for (const draft of [false, undefined, null, 'true', 'false', 0, 1]) { + assert.deepEqual(aggregateFor('pull_request', draft), { name: 'CI - required', command: 'gate' }); + } + for (const eventName of ['merge_group', 'workflow_dispatch']) { + assert.deepEqual(aggregateFor(eventName, true), { name: 'CI - required', command: 'gate' }); + } +}); + +test('preview succeeds only for genuine draft scope and real selected light successes with every native job skipped', () => { + for (const files of [['README.md'], ['tests/network/example.cpp'], ['README.md', 'tests/network/example.cpp'], []]) { + const scope = plan(files); + const needs = previewResults(scope); + validatePreview(needs, 'pull_request', 'deferred'); + assert.throws(() => validateGate(needs, 'pull_request', 'deferred'), AdmissionDeferredError); + for (const admission of ['ready', 'unknown', undefined, true]) { + assert.throws(() => validatePreview(needs, 'pull_request', admission), /genuine draft/); + } + for (const eventName of ['merge_group', 'workflow_dispatch']) { + assert.throws(() => validatePreview(needs, eventName, 'deferred'), /genuine draft/); + } + for (const job of jobs) { + for (const result of ['success', 'failure', 'cancelled', undefined, 'unknown']) { + const invalid = structuredClone(needs); + invalid[job].result = result; + assert.throws(() => validatePreview(invalid, 'pull_request', 'deferred'), /expected skipped/); + } + } + } +}); + +test('mixed preview light checks reject failures, skips, missing outcomes and contradictory scope', () => { + const scope = plan(['README.md', 'tests/network/example.cpp']); + for (const check of ['documentation', 'readme', 'formatting']) { + for (const outcome of ['failure', 'cancelled', 'skipped', undefined, 'unknown']) { + const needs = previewResults(scope); + needs.validation.outputs[`${check}_outcome`] = outcome; + assert.throws(() => validatePreview(needs, 'pull_request', 'deferred'), new RegExp(`${check}: expected success`)); + } + } + for (const job of ['scope', 'validation']) { + for (const result of ['failure', 'cancelled', 'skipped', undefined]) { + const needs = previewResults(scope); + needs[job].result = result; + assert.throws(() => validatePreview(needs, 'pull_request', 'deferred')); + } + } + for (const [key, value] of [['network_check', undefined], ['format_check', 'false'], ['validation_stage', 'documentation'], + ['event_name', 'merge_group'], ['native_check', 'false'], ['reason', 'documentation-only']]) { + const needs = previewResults(scope); + needs.scope.outputs[key] = value; + assert.throws(() => validatePreview(needs, 'pull_request', 'deferred')); + } + const unselected = previewResults(plan(['docs/guide.md'])); + unselected.validation.outputs.formatting_outcome = 'success'; + assert.throws(() => validatePreview(unselected, 'pull_request', 'deferred'), /formatting: expected skipped/); +}); + +test('actual workflow command reports preview success without reporting required acceptance and rejects real light failure', t => { + const repo = fixture(t); + repo.write('docs/guide.md', '# Updated guide\n'); + repo.write('tests/network/example.cpp', 'int changed;\n'); + const head = repo.commit(); + const event = pr(repo.base, head, true, 'opened'); + const scope = JSON.parse(repo.invoke('classify', 'pull_request', event).stdout); + const aggregate = aggregateFor('pull_request', event.pull_request.draft); + assert.equal(aggregate.name, 'CI - draft preview'); + const needs = previewResults(scope); + const preview = repo.invoke(aggregate.command, 'pull_request', event, needs); + assert.equal(preview.status, 0, preview.stderr); + assert.match(preview.summary, /Draft preview only/); + assert.match(preview.summary, /not executed in this run/); + assert.match(preview.summary, /not admitted/); + assert.match(preview.summary, /does not report CI - required success/); + assert.doesNotMatch(preview.stdout, /Every required job/); + needs.validation.outputs.documentation_outcome = 'failure'; + const failed = repo.invoke(aggregate.command, 'pull_request', event, needs); + assert.equal(failed.status, 1); + assert.match(failed.stderr, /documentation: expected success/); + assert.doesNotMatch(failed.summary, /Draft preview only/); + const ready = pr(repo.base, head, false, 'ready_for_review'); + assert.equal(repo.invoke('preview', 'pull_request', ready, previewResults(scope)).status, 1); + const full = plan(['tests/network/example.cpp'], 'merge_group'); + assert.equal(repo.invoke('preview', 'merge_group', { merge_group: {} }, previewResults(full)).status, 1); + for (const draft of [undefined, 'true', 1]) { + const invalid = pr(repo.base, head, draft); + if (draft === undefined) delete invalid.pull_request.draft; + assert.equal(repo.invoke('preview', 'pull_request', invalid, previewResults(scope)).status, 1); + } +}); diff --git a/tests/ci/module-validation.test.mjs b/tests/ci/module-validation.test.mjs index 9a169b4f9..c8dcb584b 100644 --- a/tests/ci/module-validation.test.mjs +++ b/tests/ci/module-validation.test.mjs @@ -226,7 +226,7 @@ test('validation is fatal and required alongside event-aware module results', () assert.match(jobBlock('required'), /needs: \[scope, network, applications, database, validation\]/); assert.match(jobBlock('required'), /if: always\(\)/); const script = readFileSync(new URL('../../.github/scripts/ci-policy.mjs', import.meta.url), 'utf8'); - assert.match(script, /validateGate\(JSON.parse\(process.env.CI_NEEDS_JSON\), process.env.GITHUB_EVENT_NAME\)/); + assert.match(script, /validateGate\(needs, process.env.GITHUB_EVENT_NAME, admission\)/); }); test('module flags and workflow matrix guards keep uncertain source ranges and queue candidates on full acceptance', () => { diff --git a/tests/ci/policy.test.mjs b/tests/ci/policy.test.mjs index 36e3e9f75..2a0810544 100644 --- a/tests/ci/policy.test.mjs +++ b/tests/ci/policy.test.mjs @@ -35,6 +35,9 @@ function addSite(repo) { } function classifyEvent(repo, eventName, payload) { + if (eventName === 'pull_request' && payload && typeof payload === 'object') { + payload = { ...payload, pull_request: { draft: false, ...payload.pull_request } }; + } const metadata = mkdtempSync(resolve(tmpdir(), 'ark-ci-event-')); try { const event = resolve(metadata, 'event.json'); @@ -375,7 +378,7 @@ test('the real classifier handles a fork PR from pinned local history without an repo.write('docs/guide.md', '# Fork change\n'); const head = repo.commit(); const event = resolve(repo.root, 'event.json'); - writeFileSync(event, JSON.stringify({ pull_request: { base: { sha: base }, head: { sha: head, repo: { fork: true } } } })); + writeFileSync(event, JSON.stringify({ pull_request: { draft: false, base: { sha: base }, head: { sha: head, repo: { fork: true } } } })); const env = { ...process.env, GITHUB_EVENT_NAME: 'pull_request', GITHUB_EVENT_PATH: event, GITHUB_OUTPUT: resolve(repo.root, 'outputs'), GITHUB_STEP_SUMMARY: resolve(repo.root, 'summary') }; delete env.GITHUB_TOKEN; diff --git a/tests/ci/workflow-concurrency.test.mjs b/tests/ci/workflow-concurrency.test.mjs index 8fab6d34d..b5ea8e725 100644 --- a/tests/ci/workflow-concurrency.test.mjs +++ b/tests/ci/workflow-concurrency.test.mjs @@ -30,10 +30,10 @@ test('the required gate still runs after failed needs and invokes the fatal resu const required = workflow.split(/(?=^ [a-z][a-z0-9_-]*:$)/m) .find(block => block.startsWith(' required:\n')); assert.ok(required, 'retain the required gate'); - assert.match(required, /^ name: CI - required$/m); + assert.match(required, /^ name: \$\{\{ .* && 'CI - draft preview' \|\| 'CI - required' \}\}$/m); assert.match(required, /^ if: always\(\)$/m); assert.match(required, /^ needs: \[scope, network, applications, database, validation\]$/m); assert.match(required, /^ CI_NEEDS_JSON: \$\{\{ toJSON\(needs\) \}\}$/m); - assert.match(required, /^ run: node \.github\/scripts\/ci-policy\.mjs gate$/m); + assert.match(required, /^ run: node \.github\/scripts\/ci-policy\.mjs \$\{\{ .* && 'preview' \|\| 'gate' \}\}$/m); assert.doesNotMatch(required, /^ *continue-on-error:/m, 'gate errors must fail the job'); });