diff --git a/.github/workflows/hub-nightly-post-check.yml b/.github/workflows/hub-nightly-post-check.yml new file mode 100644 index 000000000..a4d136787 --- /dev/null +++ b/.github/workflows/hub-nightly-post-check.yml @@ -0,0 +1,68 @@ +# Morning check that last night's nightly run and its triage run happened. A failed Slack step already +# turns its run red, but a run that never starts (a disabled schedule, an outage) or hangs posts nothing +# and is not red, so a missing post would go unnoticed. Posts one alert to +# #camunda-hub-nightly-test-results, pinging hub-medic, when the nightly or the triage is missing, stuck +# or ended without finishing. Silent when both ran. Deterministic: no agent. +name: Hub nightly post check + +on: + schedule: + # 04:30 UTC daily: the nightly starts at 02:00 and the triage follows it, so both have finished. + - cron: '30 4 * * *' + workflow_dispatch: + inputs: + dry_run: + description: Print the alert, post nothing. + type: boolean + default: true + +permissions: + contents: read + actions: read # list the runs of the nightly and triage workflows + id-token: write # OIDC → Vault (Slack token) + +concurrency: + group: hub-nightly-post-check + cancel-in-progress: false + +jobs: + check: + name: Did the nightly post? + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout api-test-generator + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Setup Node.js 22 + uses: actions/setup-node@v6 + with: + node-version: '22' + + - name: Fetch Slack bot token from Vault + id: slack-secrets + # A manual run defaults to a dry run that posts nothing, so it needs no token. + if: ${{ github.event_name != 'workflow_dispatch' || !inputs.dry_run }} + continue-on-error: true + uses: ./.github/actions/slack-token + with: + vault-url: ${{ secrets.VAULT_ADDR }} + vault-jwt-path: ${{ secrets.VAULT_JWT_PATH }} + vault-jwt-role: ${{ secrets.VAULT_JWT_ROLE }} + vault-jwt-audience: ${{ secrets.VAULT_JWT_AUDIENCE }} + + - name: Check the nightly and triage runs, and post if one is missing + env: + GITHUB_TOKEN: ${{ github.token }} + SLACK_TOKEN: ${{ steps.slack-secrets.outputs.SLACK_BOT_TOKEN }} + # Scheduled runs post; a manual run defaults to a dry run. + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || 'false' }} + run: node scripts/triage/hub-nightly-post-check.ts + + - name: Fail the run if the Slack alert did not go out + if: always() + uses: ./.github/actions/fail-if-slack-failed + with: + outcomes: "${{ (steps.slack-secrets.outcome == 'success' && steps.slack-secrets.outputs.SLACK_BOT_TOKEN == '') && 'failure' || steps.slack-secrets.outcome }}" diff --git a/.github/workflows/triage-camunda-hub-nightly.yml b/.github/workflows/triage-camunda-hub-nightly.yml index c856a792b..656ef01dd 100644 --- a/.github/workflows/triage-camunda-hub-nightly.yml +++ b/.github/workflows/triage-camunda-hub-nightly.yml @@ -16,6 +16,9 @@ # owner: @camunda/test-automation-team --- name: Triage — camunda-hub nightly API failures +# The run title carries the id of the nightly run it triages, so the nightly post check +# (hub-nightly-post-check.yml) can tell which nightly each triage run belongs to. +run-name: Triage of nightly ${{ github.event.workflow_run.id || inputs.nightly_run_id }} on: # Auto-run when the nightly test workflow finishes (any conclusion — a green diff --git a/AGENTS.md b/AGENTS.md index 660ed3193..6ed268865 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -878,6 +878,16 @@ warning). `do-not-close` holds a PR across the daily reset; `dry_run` previews. Fix PRs must carry the `nightly-api-fix` label to be managed (the filed issues say so). +The **nightly post check** ([hub-nightly-post-check.yml](.github/workflows/hub-nightly-post-check.yml), +`scripts/triage/hub-nightly-post-check.ts`) runs daily at 04:30 UTC, after the nightly (02:00) and its triage have +finished. A failed Slack step already turns a run red, but a run that never starts or hangs posts nothing and is not +red, so this looks at today's runs (UTC) of `nightly-camunda-hub.yml` and `triage-camunda-hub-nightly.yml` and posts one +alert to `#camunda-hub-nightly-test-results`, pinging `hub-medic`, when the scheduled nightly did not start (a manual run does not +stand in for it), is still running, or ended as anything but success or failure (a nightly with failing tests is red but still posts), or when the triage run for that +nightly (its title, set by `run-name` in the triage workflow, names the nightly's run id) did not run, is still running, or ended as anything but success (`skipped` and `neutral` mean no digest; +neither a manual triage run nor the triage of a manual nightly stands in for it). Silent when both ran. A manual run is a dry run +by default. Deterministic, no agent; covered by `tests/triage/hub-nightly-post-check.test.ts`. + The **hub known-issue re-enable check** ([hub-known-issue-reenable-check.yml](.github/workflows/hub-known-issue-reenable-check.yml), #432) is a scheduled (04:00 UTC, after the nightly + spec-bump-check) + diff --git a/scripts/triage/hub-nightly-post-check.ts b/scripts/triage/hub-nightly-post-check.ts new file mode 100644 index 000000000..b0676fe6e --- /dev/null +++ b/scripts/triage/hub-nightly-post-check.ts @@ -0,0 +1,229 @@ +// Checks, once a morning, that last night's nightly run and its triage run actually happened. +// +// The nightly (02:00 UTC) and the triage that follows it post to #camunda-hub-nightly-test-results. +// A failed Slack step turns its run red, but if a run never starts (a disabled schedule, an outage) +// or hangs, nothing posts and nothing is red, so a missing post would go unnoticed. This looks at +// the two workflows' runs for today (UTC) and posts one alert when one is missing, stuck or ended +// without finishing. Silent when both ran. +// +// Runs under plain `node` (type stripping): no enums, no parameter properties. + +import { postDigest } from './hub-gap-digest.ts'; + +export interface Run { + id: number; + /** The run title. A triage run's title ends with the id of the nightly it triages. */ + title: string; + url: string; + event: string; + status: string; + conclusion: string; + createdAt: string; +} + +export interface Problem { + /** Which post is affected, for the alert line. */ + what: string; + /** What is wrong, in a few words. */ + why: string; + /** The run to open, when there is one. */ + url: string; +} + +const NIGHTLY = 'the nightly run (02:00 UTC)'; +const TRIAGE = 'the triage digest'; + +/** The conclusions that mean a run did its job. For the nightly a plain `failure` is healthy: a + * nightly with failing tests is red but still posts. For the triage only `success` is, because its + * digest is the post: `skipped` (its job has an `if`), `neutral`, `failure`, `cancelled` and the + * rest all mean no digest. */ +const NIGHTLY_HEALTHY = new Set(['success', 'failure']); +const TRIAGE_HEALTHY = new Set(['success']); + +/** Midnight UTC of the day `now` falls on: the nightly of "today" starts after it. */ +export function startOfDayUtc(now: Date): Date { + return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())); +} + +/** True when `title` contains `id` as a whole number. */ +function titleNames(title: string, id: number): boolean { + return new RegExp(`(^|\\D)${id}(\\D|$)`).test(title); +} + +function latestSince(runs: Run[], since: Date): Run | undefined { + return runs + .filter((r) => Date.parse(r.createdAt) >= since.getTime()) + .sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt))[0]; +} + +function judge(run: Run, what: string, healthy: Set): Problem | null { + if (run.status !== 'completed') { + return { what, why: `still ${run.status.replace('_', ' ')}`, url: run.url }; + } + if (!healthy.has(run.conclusion)) { + return { what, why: `ended as ${run.conclusion || 'no result'}`, url: run.url }; + } + return null; +} + +/** The problems with today's nightly and triage runs. `nightly` and `triage` are the runs of each + * workflow: only scheduled nightly runs created since midnight UTC count (a manual run on + * some branch must not replace the scheduled one, and its triage is skipped on purpose). The triage that counts is the one + * the nightly triggered (`workflow_run`), whose title names that nightly's run id: a manual triage + * run, or the triage of a manual nightly, must not hide a missing scheduled digest. A triage is only expected + * once the nightly has finished, so a missing nightly is reported once, not twice. */ +export function evaluate(nightly: Run[], triage: Run[], now: Date): Problem[] { + const n = latestSince( + nightly.filter((r) => r.event === 'schedule'), + startOfDayUtc(now), + ); + if (!n) { + return [{ what: NIGHTLY, why: 'did not start on schedule today', url: '' }]; + } + const nightlyProblem = judge(n, NIGHTLY, NIGHTLY_HEALTHY); + if (nightlyProblem) return [nightlyProblem]; + // The triage that counts is the one whose title names this nightly: a manual nightly on + // main also triggers a triage, and its digest must not stand in for the scheduled one. + const t = latestSince( + triage.filter((r) => r.event === 'workflow_run' && titleNames(r.title, n.id)), + new Date(Date.parse(n.createdAt)), + ); + if (!t) return [{ what: TRIAGE, why: 'did not run after the nightly finished', url: n.url }]; + const triageProblem = judge(t, TRIAGE, TRIAGE_HEALTHY); + return triageProblem ? [triageProblem] : []; +} + +const HUB_MEDIC = ''; +const COOKBOOK_URL = + 'https://github.com/camunda/api-test-generator/blob/main/docs/hub-nightly-cookbook.md'; + +export function buildAlert(problems: Problem[]): string { + if (problems.length === 0) return ''; + const lines = [':warning: *No nightly post this morning*']; + for (const p of problems) { + lines.push(`• ${p.what}: ${p.why}${p.url ? ` (<${p.url}|run>)` : ''}`); + } + lines.push(`<${COOKBOOK_URL}|📖 What to check> ${HUB_MEDIC}`); + return lines.join('\n'); +} + +// ---- I/O ------------------------------------------------------------------------------- + +interface ApiRun { + id: number; + display_title: string | null; + html_url: string; + event: string; + status: string | null; + conclusion: string | null; + created_at: string; +} + +const PAGE_SIZE = 100; +/** More than this many pages of one day's runs is not plausible; a full last page then proves nothing. */ +const MAX_PAGES = 5; + +/** Reads every page of a day's runs. Every ad-hoc nightly adds a nightly run and a triage run, so a + * single short page could push the scheduled run's triage out of view and give a false "missing" + * alert. Stops at the first short page; if MAX_PAGES full pages come back, completeness cannot be + * shown, so it throws and the run is red instead of green over a partial read. `fetchPage` is + * injectable for tests. */ +export async function collectRuns(fetchPage: (page: number) => Promise): Promise { + const out: Run[] = []; + for (let page = 1; page <= MAX_PAGES; page++) { + const batch = await fetchPage(page); + out.push(...batch); + if (batch.length < PAGE_SIZE) return out; + } + throw new Error(`more than ${MAX_PAGES * PAGE_SIZE} runs today; the rest were not read`); +} + +async function fetchRunsPage( + repo: string, + file: string, + token: string, + since: Date, + page: number, +): Promise { + const day = since.toISOString().slice(0, 10); + const url = `https://api.github.com/repos/${repo}/actions/workflows/${file}/runs?per_page=${PAGE_SIZE}&page=${page}&created=%3E%3D${day}`; + const resp = await fetch(url, { + headers: { Authorization: `Bearer ${token}`, Accept: 'application/vnd.github+json' }, + signal: AbortSignal.timeout(30_000), + }); + if (!resp.ok) throw new Error(`${file}: HTTP ${resp.status}`); + // biome-ignore lint/plugin: runtime contract boundary for parsed GitHub JSON + const body = (await resp.json()) as { workflow_runs?: ApiRun[] }; + return (body.workflow_runs ?? []).map((r) => ({ + id: r.id, + title: r.display_title ?? '', + url: r.html_url, + event: r.event, + status: r.status ?? '', + conclusion: r.conclusion ?? '', + createdAt: r.created_at, + })); +} + +function listRuns(repo: string, file: string, token: string, since: Date): Promise { + return collectRuns((page) => fetchRunsPage(repo, file, token, since, page)); +} + +/** Everything `runCheck` reads or writes, so a test can drive the whole run without the network. */ +export interface CheckDeps { + dryRun: boolean; + /** Empty when the Slack token was not available. */ + slackToken: string; + now: Date; + listNightly: (since: Date) => Promise; + listTriage: (since: Date) => Promise; + post: (text: string, token: string) => Promise; + log: (line: string) => void; + error: (line: string) => void; +} + +/** Reads both workflows' runs and posts the alert when a post is missing. Throws when the alert + * is needed but cannot go out, so the run is red rather than silently green. */ +export async function runCheck(deps: CheckDeps): Promise { + const since = startOfDayUtc(deps.now); + const [nightly, triage] = await Promise.all([deps.listNightly(since), deps.listTriage(since)]); + const problems = evaluate(nightly, triage, deps.now); + const text = buildAlert(problems); + if (text === '') { + deps.log('Both the nightly and the triage ran today; staying silent.'); + return; + } + deps.log(text); + if (deps.dryRun) { + deps.log('Dry run: not posting.'); + return; + } + if (!deps.slackToken) { + deps.error('::error title=Alert not posted::a post is missing but SLACK_TOKEN is empty'); + throw new Error('A nightly post is missing and the alert could not be posted'); + } + await deps.post(text, deps.slackToken); +} + +function env(name: string): string { + return process.env[name] ?? ''; +} + +async function main(): Promise { + const repo = env('GITHUB_REPOSITORY') || 'camunda/api-test-generator'; + const token = env('GITHUB_TOKEN'); + if (!token) throw new Error('GITHUB_TOKEN is required'); + await runCheck({ + dryRun: env('DRY_RUN') === 'true', + slackToken: env('SLACK_TOKEN'), + now: new Date(), + listNightly: (since) => listRuns(repo, 'nightly-camunda-hub.yml', token, since), + listTriage: (since) => listRuns(repo, 'triage-camunda-hub-nightly.yml', token, since), + post: (text, slack) => + postDigest(text, slack, env('SLACK_CHANNEL') || '#camunda-hub-nightly-test-results'), + log: (line) => console.log(line), + error: (line) => console.error(line), + }); +} + +if (import.meta.url === `file://${process.argv[1]}`) await main(); diff --git a/tests/request-validation/slack-failure-marks-run-failed.test.ts b/tests/request-validation/slack-failure-marks-run-failed.test.ts index adf3793b0..fc65121fe 100644 --- a/tests/request-validation/slack-failure-marks-run-failed.test.ts +++ b/tests/request-validation/slack-failure-marks-run-failed.test.ts @@ -14,6 +14,7 @@ const WORKFLOWS = [ 'hub-known-issue-reenable-check', 'hub-response-coverage', 'hub-generator-gap-digest', + 'hub-nightly-post-check', ]; interface Step { @@ -150,6 +151,7 @@ describe.each(WORKFLOWS)('%s', (file) => { describe('workflows whose manual run is a dry run that posts nothing', () => { it.each([ 'hub-generator-gap-digest', + 'hub-nightly-post-check', 'hub-response-coverage', ])('%s fetches no Slack token in a dry run, so a Vault problem cannot fail it', (file) => { for (const steps of stepsOf(file)) { diff --git a/tests/triage/hub-nightly-post-check.test.ts b/tests/triage/hub-nightly-post-check.test.ts new file mode 100644 index 000000000..c32f86be1 --- /dev/null +++ b/tests/triage/hub-nightly-post-check.test.ts @@ -0,0 +1,270 @@ +import { describe, expect, it } from 'vitest'; +import { + buildAlert, + type CheckDeps, + collectRuns, + evaluate, + type Run, + runCheck, + startOfDayUtc, +} from '../../scripts/triage/hub-nightly-post-check.ts'; + +const now = new Date('2026-10-12T04:30:00Z'); + +function triageRun(over: Partial = {}): Run { + return run({ + id: 900, + title: 'Triage of nightly 100', + event: 'workflow_run', + createdAt: '2026-10-12T02:40:00Z', + ...over, + }); +} + +function run(over: Partial): Run { + return { + id: 100, + title: 'Nightly', + url: 'https://github.com/camunda/api-test-generator/actions/runs/1', + event: 'schedule', + status: 'completed', + conclusion: 'success', + createdAt: '2026-10-12T02:05:00Z', + ...over, + }; +} + +describe('startOfDayUtc', () => { + it('is midnight UTC of the same day', () => { + expect(startOfDayUtc(now).toISOString()).toBe('2026-10-12T00:00:00.000Z'); + }); +}); + +describe('evaluate', () => { + it('finds nothing when both ran and finished', () => { + expect(evaluate([run({})], [triageRun()], now)).toEqual([]); + }); + + it('reports a nightly that did not start today, and only that', () => { + const yesterday = run({ createdAt: '2026-10-11T02:05:00Z' }); + const problems = evaluate([yesterday], [yesterday], now); + expect(problems).toHaveLength(1); + expect(problems[0]?.what).toContain('nightly'); + expect(problems[0]?.why).toBe('did not start on schedule today'); + }); + + it('does not let a manual nightly run stand in for the scheduled one', () => { + const manual = run({ event: 'workflow_dispatch', createdAt: '2026-10-12T03:00:00Z' }); + const problems = evaluate([manual], [], now); + expect(problems[0]?.why).toBe('did not start on schedule today'); + }); + + it('judges the scheduled run even when a later manual run exists', () => { + const manual = run({ event: 'workflow_dispatch', createdAt: '2026-10-12T03:30:00Z' }); + expect(evaluate([run({}), manual], [triageRun()], now)).toEqual([]); + }); + + it('reports a nightly that is still running at the check time', () => { + const problems = evaluate([run({ status: 'in_progress', conclusion: '' })], [], now); + expect(problems).toHaveLength(1); + expect(problems[0]?.why).toBe('still in progress'); + expect(problems[0]?.url).toContain('/actions/runs/'); + }); + + it('reports a nightly that was cancelled or timed out', () => { + for (const conclusion of ['cancelled', 'timed_out', 'startup_failure']) { + const problems = evaluate([run({ conclusion })], [], now); + expect(problems[0]?.why).toBe(`ended as ${conclusion}`); + } + }); + + it('does not report a nightly whose tests failed: it still posts', () => { + expect(evaluate([run({ conclusion: 'failure' })], [triageRun()], now)).toEqual([]); + }); + + it('reports a triage that never ran after the nightly finished', () => { + const problems = evaluate([run({})], [], now); + expect(problems).toHaveLength(1); + expect(problems[0]?.what).toContain('triage'); + expect(problems[0]?.why).toContain('did not run'); + }); + + it('ignores yesterday’s triage run', () => { + const problems = evaluate([run({})], [triageRun({ createdAt: '2026-10-11T02:40:00Z' })], now); + expect(problems[0]?.what).toContain('triage'); + }); + + it('reports a failed triage run, because the digest is its post', () => { + const problems = evaluate([run({})], [triageRun({ conclusion: 'failure' })], now); + expect(problems[0]?.why).toBe('ended as failure'); + }); + + it('reports a skipped or neutral triage: it posted no digest', () => { + for (const conclusion of ['skipped', 'neutral']) { + const problems = evaluate([run({})], [triageRun({ conclusion })], now); + expect(problems[0]?.why).toBe(`ended as ${conclusion}`); + } + }); + + it('reports a nightly that ended skipped or neutral', () => { + for (const conclusion of ['skipped', 'neutral']) { + expect(evaluate([run({ conclusion })], [], now)[0]?.why).toBe(`ended as ${conclusion}`); + } + }); + + it('does not let a manual triage run stand in for the automatic one', () => { + const manual = triageRun({ event: 'workflow_dispatch', createdAt: '2026-10-12T03:00:00Z' }); + const problems = evaluate([run({})], [manual], now); + expect(problems[0]?.what).toContain('triage'); + }); + + it('does not let the triage of a manual nightly stand in for the scheduled one', () => { + const manualTriage = triageRun({ title: 'Triage of nightly 200' }); + const problems = evaluate([run({})], [manualTriage], now); + expect(problems[0]?.what).toContain('triage'); + }); + + it('matches the nightly id as a whole number, not as a prefix or suffix', () => { + for (const title of [ + 'Triage of nightly 1000', + 'Triage of nightly 10', + 'Triage of nightly 2100', + ]) { + const problems = evaluate([run({})], [triageRun({ title })], now); + expect(problems[0]?.what, title).toContain('triage'); + } + }); + + it('does not count a triage run from before the nightly started', () => { + const earlier = triageRun({ createdAt: '2026-10-12T01:00:00Z' }); + const problems = evaluate([run({ createdAt: '2026-10-12T02:05:00Z' })], [earlier], now); + expect(problems[0]?.what).toContain('triage'); + }); + + it('judges the newest run of the day when there was a re-run', () => { + const first = run({ conclusion: 'cancelled', createdAt: '2026-10-12T02:05:00Z' }); + const rerun = run({ createdAt: '2026-10-12T03:10:00Z' }); + expect( + evaluate([first, rerun], [triageRun({ createdAt: '2026-10-12T03:50:00Z' })], now), + ).toEqual([]); + }); +}); + +describe('buildAlert', () => { + it('is empty when nothing is wrong', () => { + expect(buildAlert([])).toBe(''); + }); + + it('names each problem, links the run and pings hub-medic', () => { + const text = buildAlert([ + { what: 'the triage digest', why: 'ended as failure', url: 'https://example.test/run/9' }, + ]); + expect(text).toContain('No nightly post this morning'); + expect(text).toContain('the triage digest: ended as failure'); + expect(text).toContain(''); + expect(text).toContain(''); + expect(text).toContain('docs/hub-nightly-cookbook.md|'); + expect(text).not.toContain('#when-a-post-is-missing'); + }); + + it('has no run link when there is none', () => { + const text = buildAlert([ + { what: 'the nightly run (02:00 UTC)', why: 'did not start on schedule today', url: '' }, + ]); + expect(text).not.toContain('|run>'); + }); +}); + +function deps(over: Partial & { nightly?: Run[]; triage?: Run[] }): { + d: CheckDeps; + posted: string[]; + logs: string[]; + errors: string[]; +} { + const posted: string[] = []; + const logs: string[] = []; + const errors: string[] = []; + const d: CheckDeps = { + dryRun: false, + slackToken: 'xoxb-test', + now, + listNightly: async () => over.nightly ?? [run({})], + listTriage: async () => over.triage ?? [triageRun()], + post: async (text) => { + posted.push(text); + }, + log: (l) => logs.push(l), + error: (e) => errors.push(e), + ...over, + }; + return { d, posted, logs, errors }; +} + +describe('runCheck', () => { + it('stays silent when both ran', async () => { + const { d, posted, logs } = deps({}); + await runCheck(d); + expect(posted).toEqual([]); + expect(logs[0]).toContain('staying silent'); + }); + + it('posts one alert when the nightly did not start', async () => { + const { d, posted } = deps({ nightly: [], triage: [] }); + await runCheck(d); + expect(posted).toHaveLength(1); + expect(posted[0]).toContain('did not start on schedule today'); + }); + + it('prints the alert but posts nothing in a dry run', async () => { + const { d, posted, logs } = deps({ nightly: [], triage: [], dryRun: true }); + await runCheck(d); + expect(posted).toEqual([]); + expect(logs.join('\n')).toContain('Dry run: not posting.'); + }); + + it('fails when the alert is needed but there is no Slack token', async () => { + const { d, posted, errors } = deps({ nightly: [], triage: [], slackToken: '' }); + await expect(runCheck(d)).rejects.toThrow('could not be posted'); + expect(posted).toEqual([]); + expect(errors[0]).toContain('::error'); + }); + + it('lets an API failure fail the run instead of reading as "all fine"', async () => { + const { d } = deps({ + listNightly: async () => { + throw new Error('nightly-camunda-hub.yml: HTTP 403'); + }, + }); + await expect(runCheck(d)).rejects.toThrow('HTTP 403'); + }); +}); + +describe('collectRuns', () => { + const full = (n: number): Run[] => + Array.from({ length: 100 }, (_, i) => run({ id: n * 1000 + i })); + + it('reads every page until a short one', async () => { + const pages = [full(1), full(2), [run({ id: 9 })]]; + const seen: number[] = []; + const runs = await collectRuns(async (page) => { + seen.push(page); + return pages[page - 1] ?? []; + }); + expect(runs).toHaveLength(201); + expect(seen).toEqual([1, 2, 3]); + }); + + it('stops after the first page when it is short', async () => { + const seen: number[] = []; + const runs = await collectRuns(async (page) => { + seen.push(page); + return [run({})]; + }); + expect(runs).toHaveLength(1); + expect(seen).toEqual([1]); + }); + + it('fails instead of reading green over a partial read when every page is full', async () => { + await expect(collectRuns(async (page) => full(page))).rejects.toThrow('were not read'); + }); +});