feat: alert when the nightly or its triage post is missing

.github/workflows/hub-nightly-post-check.yml

@@ -0,0 +1,68 @@
+# Morning check that last night's nightly run and its triage run happened. A failed Slack step already
+# turns its run red, but a run that never starts (a disabled schedule, an outage) or hangs posts nothing
+# and is not red, so a missing post would go unnoticed. Posts one alert to
+# #camunda-hub-nightly-test-results, pinging hub-medic, when the nightly or the triage is missing, stuck
+# or ended without finishing. Silent when both ran. Deterministic: no agent.
+name: Hub nightly post check
+
+on:
+  schedule:
+    # 04:30 UTC daily: the nightly starts at 02:00 and the triage follows it, so both have finished.
+    - cron: '30 4 * * *'
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: Print the alert, post nothing.
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+  actions: read # list the runs of the nightly and triage workflows
+  id-token: write # OIDC → Vault (Slack token)
+
+concurrency:
+  group: hub-nightly-post-check
+  cancel-in-progress: false
+
+jobs:
+  check:
+    name: Did the nightly post?
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout api-test-generator
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Setup Node.js 22
+        uses: actions/setup-node@v6
+        with:
+          node-version: '22'
+
+      - name: Fetch Slack bot token from Vault
+        id: slack-secrets
+        # A manual run defaults to a dry run that posts nothing, so it needs no token.
+        if: ${{ github.event_name != 'workflow_dispatch' || !inputs.dry_run }}
+        continue-on-error: true
+        uses: ./.github/actions/slack-token
+        with:
+          vault-url: ${{ secrets.VAULT_ADDR }}
+          vault-jwt-path: ${{ secrets.VAULT_JWT_PATH }}
+          vault-jwt-role: ${{ secrets.VAULT_JWT_ROLE }}
+          vault-jwt-audience: ${{ secrets.VAULT_JWT_AUDIENCE }}
+
+      - name: Check the nightly and triage runs, and post if one is missing
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          SLACK_TOKEN: ${{ steps.slack-secrets.outputs.SLACK_BOT_TOKEN }}
+          # Scheduled runs post; a manual run defaults to a dry run.
+          DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || 'false' }}
+        run: node scripts/triage/hub-nightly-post-check.ts
+
+      - name: Fail the run if the Slack alert did not go out
+        if: always()
+        uses: ./.github/actions/fail-if-slack-failed
+        with:
+          outcomes: "${{ (steps.slack-secrets.outcome == 'success' && steps.slack-secrets.outputs.SLACK_BOT_TOKEN == '') && 'failure' || steps.slack-secrets.outcome }}"

.github/workflows/triage-camunda-hub-nightly.yml

@@ -16,6 +16,9 @@
 # owner: @camunda/test-automation-team
 ---
 name: Triage — camunda-hub nightly API failures
+# The run title carries the id of the nightly run it triages, so the nightly post check
+# (hub-nightly-post-check.yml) can tell which nightly each triage run belongs to.
+run-name: Triage of nightly ${{ github.event.workflow_run.id || inputs.nightly_run_id }}
 
 on:
   # Auto-run when the nightly test workflow finishes (any conclusion — a green

AGENTS.md

@@ -878,6 +878,16 @@ warning). `do-not-close` holds a PR across the daily reset; `dry_run` previews.
 Fix PRs must carry the `nightly-api-fix` label to be managed (the filed issues say
 so).
 
+The **nightly post check** ([hub-nightly-post-check.yml](.github/workflows/hub-nightly-post-check.yml),
+`scripts/triage/hub-nightly-post-check.ts`) runs daily at 04:30 UTC, after the nightly (02:00) and its triage have
+finished. A failed Slack step already turns a run red, but a run that never starts or hangs posts nothing and is not
+red, so this looks at today's runs (UTC) of `nightly-camunda-hub.yml` and `triage-camunda-hub-nightly.yml` and posts one
+alert to `#camunda-hub-nightly-test-results`, pinging `hub-medic`, when the scheduled nightly did not start (a manual run does not
+stand in for it), is still running, or ended as anything but success or failure (a nightly with failing tests is red but still posts), or when the triage run for that
+nightly (its title, set by `run-name` in the triage workflow, names the nightly's run id) did not run, is still running, or ended as anything but success (`skipped` and `neutral` mean no digest;
+neither a manual triage run nor the triage of a manual nightly stands in for it). Silent when both ran. A manual run is a dry run
+by default. Deterministic, no agent; covered by `tests/triage/hub-nightly-post-check.test.ts`.
+
 The **hub known-issue re-enable check**
 ([hub-known-issue-reenable-check.yml](.github/workflows/hub-known-issue-reenable-check.yml),
 #432) is a scheduled (04:00 UTC, after the nightly + spec-bump-check) +

scripts/triage/hub-nightly-post-check.ts

@@ -0,0 +1,229 @@
+// Checks, once a morning, that last night's nightly run and its triage run actually happened.
+//
+// The nightly (02:00 UTC) and the triage that follows it post to #camunda-hub-nightly-test-results.
+// A failed Slack step turns its run red, but if a run never starts (a disabled schedule, an outage)
+// or hangs, nothing posts and nothing is red, so a missing post would go unnoticed. This looks at
+// the two workflows' runs for today (UTC) and posts one alert when one is missing, stuck or ended
+// without finishing. Silent when both ran.
+//
+// Runs under plain `node` (type stripping): no enums, no parameter properties.
+
+import { postDigest } from './hub-gap-digest.ts';
+
+export interface Run {
+  id: number;
+  /** The run title. A triage run's title ends with the id of the nightly it triages. */
+  title: string;
+  url: string;
+  event: string;
+  status: string;
+  conclusion: string;
+  createdAt: string;
+}
+
+export interface Problem {
+  /** Which post is affected, for the alert line. */
+  what: string;
+  /** What is wrong, in a few words. */
+  why: string;
+  /** The run to open, when there is one. */
+  url: string;
+}
+
+const NIGHTLY = 'the nightly run (02:00 UTC)';
+const TRIAGE = 'the triage digest';
+
+/** The conclusions that mean a run did its job. For the nightly a plain `failure` is healthy: a
+ * nightly with failing tests is red but still posts. For the triage only `success` is, because its
+ * digest is the post: `skipped` (its job has an `if`), `neutral`, `failure`, `cancelled` and the
+ * rest all mean no digest. */
+const NIGHTLY_HEALTHY = new Set(['success', 'failure']);
+const TRIAGE_HEALTHY = new Set(['success']);
+
+/** Midnight UTC of the day `now` falls on: the nightly of "today" starts after it. */
+export function startOfDayUtc(now: Date): Date {
+  return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()));
+}
+
+/** True when `title` contains `id` as a whole number. */
+function titleNames(title: string, id: number): boolean {
+  return new RegExp(`(^|\\D)${id}(\\D|$)`).test(title);
+}
+
+function latestSince(runs: Run[], since: Date): Run | undefined {
+  return runs
+    .filter((r) => Date.parse(r.createdAt) >= since.getTime())
+    .sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt))[0];
+}
+
+function judge(run: Run, what: string, healthy: Set<string>): Problem | null {
+  if (run.status !== 'completed') {
+    return { what, why: `still ${run.status.replace('_', ' ')}`, url: run.url };
+  }
+  if (!healthy.has(run.conclusion)) {
+    return { what, why: `ended as ${run.conclusion || 'no result'}`, url: run.url };
+  }
+  return null;
+}
+
+/** The problems with today's nightly and triage runs. `nightly` and `triage` are the runs of each
+ * workflow: only scheduled nightly runs created since midnight UTC count (a manual run on
+ * some branch must not replace the scheduled one, and its triage is skipped on purpose). The triage that counts is the one
+ * the nightly triggered (`workflow_run`), whose title names that nightly's run id: a manual triage
+ * run, or the triage of a manual nightly, must not hide a missing scheduled digest. A triage is only expected
+ * once the nightly has finished, so a missing nightly is reported once, not twice. */
+export function evaluate(nightly: Run[], triage: Run[], now: Date): Problem[] {
+  const n = latestSince(
+    nightly.filter((r) => r.event === 'schedule'),
+    startOfDayUtc(now),
+  );
+  if (!n) {
+    return [{ what: NIGHTLY, why: 'did not start on schedule today', url: '' }];
+  }
+  const nightlyProblem = judge(n, NIGHTLY, NIGHTLY_HEALTHY);
+  if (nightlyProblem) return [nightlyProblem];
+  // The triage that counts is the one whose title names this nightly: a manual nightly on
+  // main also triggers a triage, and its digest must not stand in for the scheduled one.
+  const t = latestSince(
+    triage.filter((r) => r.event === 'workflow_run' && titleNames(r.title, n.id)),
+    new Date(Date.parse(n.createdAt)),
+  );
+  if (!t) return [{ what: TRIAGE, why: 'did not run after the nightly finished', url: n.url }];
+  const triageProblem = judge(t, TRIAGE, TRIAGE_HEALTHY);
+  return triageProblem ? [triageProblem] : [];
+}
+
+const HUB_MEDIC = '<!subteam^S014VK4482H|hub-medic>';
+const COOKBOOK_URL =
+  'https://github.com/camunda/api-test-generator/blob/main/docs/hub-nightly-cookbook.md';
+
+export function buildAlert(problems: Problem[]): string {
+  if (problems.length === 0) return '';
+  const lines = [':warning: *No nightly post this morning*'];
+  for (const p of problems) {
+    lines.push(`• ${p.what}: ${p.why}${p.url ? ` (<${p.url}|run>)` : ''}`);
+  }
+  lines.push(`<${COOKBOOK_URL}|📖 What to check> ${HUB_MEDIC}`);
+  return lines.join('\n');
+}
+
+// ---- I/O -------------------------------------------------------------------------------
+
+interface ApiRun {
+  id: number;
+  display_title: string | null;
+  html_url: string;
+  event: string;
+  status: string | null;
+  conclusion: string | null;
+  created_at: string;
+}
+
+const PAGE_SIZE = 100;
+/** More than this many pages of one day's runs is not plausible; a full last page then proves nothing. */
+const MAX_PAGES = 5;
+
+/** Reads every page of a day's runs. Every ad-hoc nightly adds a nightly run and a triage run, so a
+ * single short page could push the scheduled run's triage out of view and give a false "missing"
+ * alert. Stops at the first short page; if MAX_PAGES full pages come back, completeness cannot be
+ * shown, so it throws and the run is red instead of green over a partial read. `fetchPage` is
+ * injectable for tests. */
+export async function collectRuns(fetchPage: (page: number) => Promise<Run[]>): Promise<Run[]> {
+  const out: Run[] = [];
+  for (let page = 1; page <= MAX_PAGES; page++) {
+    const batch = await fetchPage(page);
+    out.push(...batch);
+    if (batch.length < PAGE_SIZE) return out;
+  }
+  throw new Error(`more than ${MAX_PAGES * PAGE_SIZE} runs today; the rest were not read`);
+}
+
+async function fetchRunsPage(
+  repo: string,
+  file: string,
+  token: string,
+  since: Date,
+  page: number,
+): Promise<Run[]> {
+  const day = since.toISOString().slice(0, 10);
+  const url = `https://api.github.com/repos/${repo}/actions/workflows/${file}/runs?per_page=${PAGE_SIZE}&page=${page}&created=%3E%3D${day}`;
+  const resp = await fetch(url, {
+    headers: { Authorization: `Bearer ${token}`, Accept: 'application/vnd.github+json' },
+    signal: AbortSignal.timeout(30_000),
+  });
+  if (!resp.ok) throw new Error(`${file}: HTTP ${resp.status}`);
+  // biome-ignore lint/plugin: runtime contract boundary for parsed GitHub JSON
+  const body = (await resp.json()) as { workflow_runs?: ApiRun[] };
+  return (body.workflow_runs ?? []).map((r) => ({
+    id: r.id,
+    title: r.display_title ?? '',
+    url: r.html_url,
+    event: r.event,
+    status: r.status ?? '',
+    conclusion: r.conclusion ?? '',
+    createdAt: r.created_at,
+  }));
+}
+
+function listRuns(repo: string, file: string, token: string, since: Date): Promise<Run[]> {
+  return collectRuns((page) => fetchRunsPage(repo, file, token, since, page));
+}
+
+/** Everything `runCheck` reads or writes, so a test can drive the whole run without the network. */
+export interface CheckDeps {
+  dryRun: boolean;
+  /** Empty when the Slack token was not available. */
+  slackToken: string;
+  now: Date;
+  listNightly: (since: Date) => Promise<Run[]>;
+  listTriage: (since: Date) => Promise<Run[]>;
+  post: (text: string, token: string) => Promise<void>;
+  log: (line: string) => void;
+  error: (line: string) => void;
+}
+
+/** Reads both workflows' runs and posts the alert when a post is missing. Throws when the alert
+ * is needed but cannot go out, so the run is red rather than silently green. */
+export async function runCheck(deps: CheckDeps): Promise<void> {
+  const since = startOfDayUtc(deps.now);
+  const [nightly, triage] = await Promise.all([deps.listNightly(since), deps.listTriage(since)]);
+  const problems = evaluate(nightly, triage, deps.now);
+  const text = buildAlert(problems);
+  if (text === '') {
+    deps.log('Both the nightly and the triage ran today; staying silent.');
+    return;
+  }
+  deps.log(text);
+  if (deps.dryRun) {
+    deps.log('Dry run: not posting.');
+    return;
+  }
+  if (!deps.slackToken) {
+    deps.error('::error title=Alert not posted::a post is missing but SLACK_TOKEN is empty');
+    throw new Error('A nightly post is missing and the alert could not be posted');
+  }
+  await deps.post(text, deps.slackToken);
+}
+
+function env(name: string): string {
+  return process.env[name] ?? '';
+}
+
+async function main(): Promise<void> {
+  const repo = env('GITHUB_REPOSITORY') || 'camunda/api-test-generator';
+  const token = env('GITHUB_TOKEN');
+  if (!token) throw new Error('GITHUB_TOKEN is required');
+  await runCheck({
+    dryRun: env('DRY_RUN') === 'true',
+    slackToken: env('SLACK_TOKEN'),
+    now: new Date(),
+    listNightly: (since) => listRuns(repo, 'nightly-camunda-hub.yml', token, since),
+    listTriage: (since) => listRuns(repo, 'triage-camunda-hub-nightly.yml', token, since),
+    post: (text, slack) =>
+      postDigest(text, slack, env('SLACK_CHANNEL') || '#camunda-hub-nightly-test-results'),
+    log: (line) => console.log(line),
+    error: (line) => console.error(line),
+  });
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) await main();

tests/request-validation/slack-failure-marks-run-failed.test.ts

@@ -14,6 +14,7 @@ const WORKFLOWS = [
   'hub-known-issue-reenable-check',
   'hub-response-coverage',
   'hub-generator-gap-digest',
+  'hub-nightly-post-check',
 ];
 
 interface Step {
@@ -150,6 +151,7 @@ describe.each(WORKFLOWS)('%s', (file) => {
 describe('workflows whose manual run is a dry run that posts nothing', () => {
   it.each([
     'hub-generator-gap-digest',
+    'hub-nightly-post-check',
     'hub-response-coverage',
   ])('%s fetches no Slack token in a dry run, so a Vault problem cannot fail it', (file) => {
     for (const steps of stepsOf(file)) {