From 50c282d9879ee894b3e66c4a15c5e293b6eccc00 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 31 Jul 2026 17:24:54 +0000 Subject: [PATCH 1/2] ci: probe production health off-host on a schedule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Item 8 of the rollout plan. The status dashboard already runs every check this needs, but it runs on the host and is pull-only: something has to look at it. A host that is down cannot tell you it is down, which is the one failure mode that matters most and the one the current setup structurally cannot report. This runs the same suite from GitHub's infrastructure every fifteen minutes. No new service, no new vendor, no dependencies — the dashboard package has none, so the probe is a checkout and a node invocation. Escalation is labelled honestly rather than overstated. A failure fails the workflow, which notifies watchers, and posts to Slack once a webhook secret exists. Neither is a page and nobody is on call, so the Slack step is conditional on the secret being present: the workflow is useful the day it lands and gains routing later with no code change. Real escalation belongs in the organisation's Prometheus/OpsGenie stack, where an alert reaches someone who has agreed to be woken; this is the stopgap and says so. Deliberately not --strict. That escalates warnings to failures, and a third of the suite probes Internet Identity and the IC, which this team does not operate. Paging on someone else's degradation is how a channel learns to ignore its alerts. Without the flag the CLI exits non-zero exactly when the overall verdict is "fail" — the same threshold the dashboard already uses to serve 503. Three things found by running it rather than reasoning about it. The probe refuses any origin outside its SSRF allow-list, so the production host has to be named explicitly or the run reports a usage error instead of a health verdict. Report sections carry `title`, not `label`, so the summary table would have rendered every section as "?". And the Slack step was gated on failure(), which never fires here: the probe swallows its own exit code on purpose, so nothing has failed at that point in the job — the notification would have been silently dead while the run still went red and looked correct from outside. Verified against three inputs: a healthy production report, a report from an origin that answers but is not the service (eight checks fail), and a missing report standing in for an unreachable host. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01R8ZshwKmjD5fZ4Hs9dS6zh --- .github/workflows/health.yml | 165 +++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 .github/workflows/health.yml diff --git a/.github/workflows/health.yml b/.github/workflows/health.yml new file mode 100644 index 0000000..4280797 --- /dev/null +++ b/.github/workflows/health.yml @@ -0,0 +1,165 @@ +# Off-host health probe for the production MCP service. +# +# The status dashboard already runs every check this needs — MCP, the OAuth +# suite, TLS, and the Internet Identity linkage — but it runs *on the host* and +# is pull-only: something has to look at it. This runs the same probe suite from +# GitHub's infrastructure on a schedule, which is the part the host cannot do +# for itself. A host that is down cannot tell you it is down. +# +# Escalation, honestly labelled: a failure here fails the workflow, which +# notifies watchers by email, and posts to Slack when a webhook is configured. +# Neither is a page. Nobody is on call. For real escalation the probe belongs in +# the organisation's Prometheus/OpsGenie stack, where an alert reaches someone +# who has agreed to be woken — this is the stopgap, not the destination. +# +# Optional repository secret: +# SLACK_WEBHOOK_URL Incoming-webhook URL, intended for #eng-identity-imcp2. +# Absent, the Slack step is skipped and the workflow +# failure remains the only signal. Nothing else changes. +name: Production health + +on: + schedule: + # Every 15 minutes. GitHub's scheduler is best-effort and routinely drifts + # under load, so treat this as "noticed within the hour", not an SLA. Note + # that scheduled workflows are disabled automatically after 60 days without + # repository activity. + - cron: '*/15 * * * *' + # Manual run, for verifying the probe itself rather than waiting for a tick. + workflow_dispatch: + +# One probe at a time; a slow run should not overlap the next tick. +concurrency: + group: health + cancel-in-progress: false + +permissions: + contents: read + +jobs: + probe: + runs-on: ubuntu-24.04 + env: + MCP_ORIGIN: https://mcp.internetcomputer.org + # The probe refuses any origin outside its SSRF allow-list, which defaults + # to id.ai and loopback. The production host is neither, so it has to be + # named explicitly — exactly as deploy.sh does when it starts the hosted + # dashboard. Without this the run fails with a usage error rather than a + # health verdict, which is a confusing way to learn the service is fine. + MCP_STATUS_ALLOWED_HOSTS: mcp.internetcomputer.org + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + # Deliberately NOT --strict. That flag escalates warnings to failures, and + # a third of the suite probes Internet Identity and the IC — infrastructure + # this team does not operate. Alerting on someone else's degradation is how + # a channel learns to ignore the alerts. Real failures still fail: without + # --strict the CLI exits non-zero exactly when `overall` is "fail", which + # is the same threshold the dashboard uses to serve 503. + - name: Probe production + id: probe + run: | + set +e + node monitoring/mcp-status/cli.js --mcp "$MCP_ORIGIN" --json > report.json 2> probe.err + code=$? + set -e + echo "exit_code=$code" >> "$GITHUB_OUTPUT" + if [ "$code" -eq 0 ]; then + echo "healthy" + else + echo "probe exited $code" + cat probe.err || true + fi + + # Summarise which checks failed, rather than only that something did. A + # bare "the workflow failed" costs the reader a log dive at exactly the + # moment they are least inclined to do one. + - name: Summarise + id: summary + if: always() + run: | + python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json, os + try: + r = json.load(open("report.json")) + except Exception as e: + print(f"Could not parse the probe report: {e}\n") + print("The probe did not produce usable JSON — most likely it could " + "not reach the host at all, which is itself the finding.") + raise SystemExit(0) + bad = [] + for section in r.get("sections", []): + for c in section.get("checks", []): + if c.get("status") in ("fail", "warn"): + bad.append((c.get("status"), section.get("title", "?"), c.get("label", "?"))) + print(f"**overall: {r.get('overall')}**\n") + if bad: + print("| status | section | check |") + print("|---|---|---|") + for s, sec, lab in bad: + print(f"| {s} | {sec} | {lab} |") + else: + print("All checks passed.") + PY + # A one-line form for the Slack payload. + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import json + try: + r = json.load(open("report.json")) + failed = [c.get("label", "?") + for s in r.get("sections", []) + for c in s.get("checks", []) + if c.get("status") == "fail"] + line = ", ".join(failed) if failed else "no individual check reported fail" + except Exception: + line = "the probe produced no usable report (the host may be unreachable)" + # $GITHUB_OUTPUT is line-oriented, so a newline in a value would let + # the value declare further outputs. The labels are our own static + # strings today, but sanitising here costs nothing and keeps this from + # becoming a hazard if a future check ever interpolates a server value. + line = " ".join(line.split())[:400] + print(f"failed={line}") + PY + + # Skipped when the secret is absent, so this workflow is useful the moment + # it lands and gains Slack routing later with no code change. `secrets` is + # not available in a step condition, hence the job-level env mapping above. + # + # The condition reads the probe's recorded exit code rather than + # `failure()`. The probe step swallows its own failure on purpose, so at + # this point in the job nothing has failed yet and `failure()` is false — + # the notification would never have fired, while the run still went red at + # the final step and looked correct from the outside. + - name: Notify Slack + if: steps.probe.outputs.exit_code != '0' && env.SLACK_WEBHOOK_URL != '' + run: | + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + python3 - "$run_url" <<'PY' > payload.json + import json, sys, os + text = ( + ":rotating_light: *ICP MCP production health check failed*\n" + f"Failing: {os.environ.get('FAILED', 'unknown')}\n" + f"<{sys.argv[1]}|Run log> · " + "" + ) + json.dump({"text": text}, sys.stdout) + PY + curl -sS --fail-with-body -X POST -H 'Content-type: application/json' \ + --data @payload.json "$SLACK_WEBHOOK_URL" + env: + FAILED: ${{ steps.summary.outputs.failed }} + + # Re-raise the probe's verdict so the run itself is red. This is the + # baseline notification and the one that needs no configuration at all. + - name: Fail the run if unhealthy + if: steps.probe.outputs.exit_code != '0' + run: | + echo "::error::production health check failed (${{ steps.summary.outputs.failed }})" + exit 1 From a4bfb7ec8262f31b58f42bfd711206f3e3e4aa14 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 3 Aug 2026 10:08:42 +0000 Subject: [PATCH 2/2] ci: harden the health probe after review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four findings, all valid, all confirmed before changing anything. No job timeout. The default is six hours, and `concurrency` here lets one run proceed without cancelling — so a wedged run does not merely waste a slot, it queues every later tick behind itself. Six hours is twenty-four missed probes during which the monitor says nothing, and silence from a monitor is indistinguishable from health. Capped below the cadence, and the Slack request now carries --max-time so a stalled webhook cannot consume the budget either. The webhook was mapped at job scope. Every step and the checked-out probe code could read it, none of which needs it, and log masking does not stop a process reading its own environment. It is now scoped to the single step that posts it. The step condition needs to know whether the secret exists without seeing it, so the job exports the comparison rather than the value — an expression that resolves to "true" or "false", since `secrets` is unavailable in a step `if:`. Node 20 is end-of-life since April and no longer receives security fixes, which is a poor choice for a job that parses responses from an internet-facing service. Moved to 22, which is what every probe run during development already used. deploy-native.yml and the host still pin 20; that is a separate change. The summary table interpolated values that are not ours. The probed server supplies II instance names in /version and the dashboard puts them straight into check labels and section titles, so a misconfigured or compromised host could inject a pipe, a newline or Markdown and corrupt or spoof the summary this job writes. Cells are now escaped and length-capped, verified against a report carrying a name built to forge an extra table row. That last one also falsified a comment written one commit earlier, which claimed the interpolated labels were our own static strings. They are not, and the note explaining why sanitising was cheap-but-optional now says the opposite. Same sink class, two places: the output line had been hardened while the table next to it had not. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01R8ZshwKmjD5fZ4Hs9dS6zh --- .github/workflows/health.yml | 50 +++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/.github/workflows/health.yml b/.github/workflows/health.yml index 4280797..30c7700 100644 --- a/.github/workflows/health.yml +++ b/.github/workflows/health.yml @@ -39,6 +39,13 @@ permissions: jobs: probe: runs-on: ubuntu-24.04 + # Below the probe cadence, and deliberately so. `concurrency` above lets one + # run at a time without cancelling, so a wedged run does not just waste a + # slot — it queues every later tick behind itself. With the default 6-hour + # job timeout that is 24 missed probes, during which the monitor reports + # nothing and silence is indistinguishable from health. Failing fast and + # letting the next tick try again is strictly better than hanging. + timeout-minutes: 10 env: MCP_ORIGIN: https://mcp.internetcomputer.org # The probe refuses any origin outside its SSRF allow-list, which defaults @@ -47,7 +54,14 @@ jobs: # dashboard. Without this the run fails with a usage error rather than a # health verdict, which is a confusing way to learn the service is fine. MCP_STATUS_ALLOWED_HOSTS: mcp.internetcomputer.org - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + # A boolean, not the secret. The webhook itself is scoped to the one step + # that posts it (below); putting it in the job environment would hand it to + # every step and to the checked-out probe code, none of which needs it, and + # log masking does not stop a process from reading its own environment. + # This expression evaluates to "true"/"false" without exposing the value, + # which is what lets the step condition stay declarative — `secrets` is not + # available in a step `if:`. + HAS_SLACK: ${{ secrets.SLACK_WEBHOOK_URL != '' }} steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -55,7 +69,11 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version: 20 + # 20 reached end of life on 2026-04-30 and no longer gets security + # fixes, and this job parses responses from an internet-facing service. + # deploy-native.yml and the host still pin 20; bumping those is a + # separate change with its own blast radius. + node-version: 22 # Deliberately NOT --strict. That flag escalates warnings to failures, and # a third of the suite probes Internet Identity and the IC — infrastructure @@ -94,17 +112,26 @@ jobs: print("The probe did not produce usable JSON — most likely it could " "not reach the host at all, which is itself the finding.") raise SystemExit(0) + # Not every value here is ours. The probed server supplies the II + # instance names in /version, and the dashboard interpolates them + # straight into check labels and section titles. A misconfigured or + # compromised host could therefore put a pipe, a newline or Markdown + # into a table cell and corrupt or spoof this summary. Escape per cell. + def cell(v): + t = "".join(ch if ch.isprintable() else " " for ch in str(v)) + return " ".join(t.split()).replace("\\", "").replace("|", "\\|")[:120] or "?" + bad = [] for section in r.get("sections", []): for c in section.get("checks", []): if c.get("status") in ("fail", "warn"): bad.append((c.get("status"), section.get("title", "?"), c.get("label", "?"))) - print(f"**overall: {r.get('overall')}**\n") + print(f"**overall: {cell(r.get('overall'))}**\n") if bad: print("| status | section | check |") print("|---|---|---|") for s, sec, lab in bad: - print(f"| {s} | {sec} | {lab} |") + print(f"| {cell(s)} | {cell(sec)} | {cell(lab)} |") else: print("All checks passed.") PY @@ -121,9 +148,9 @@ jobs: except Exception: line = "the probe produced no usable report (the host may be unreachable)" # $GITHUB_OUTPUT is line-oriented, so a newline in a value would let - # the value declare further outputs. The labels are our own static - # strings today, but sanitising here costs nothing and keeps this from - # becoming a hazard if a future check ever interpolates a server value. + # the value declare further outputs. This is not hypothetical: check + # labels embed the II instance names the probed server advertises, so + # part of this string originates outside our control. line = " ".join(line.split())[:400] print(f"failed={line}") PY @@ -138,7 +165,7 @@ jobs: # the notification would never have fired, while the run still went red at # the final step and looked correct from the outside. - name: Notify Slack - if: steps.probe.outputs.exit_code != '0' && env.SLACK_WEBHOOK_URL != '' + if: steps.probe.outputs.exit_code != '0' && env.HAS_SLACK == 'true' run: | run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" python3 - "$run_url" <<'PY' > payload.json @@ -151,9 +178,14 @@ jobs: ) json.dump({"text": text}, sys.stdout) PY - curl -sS --fail-with-body -X POST -H 'Content-type: application/json' \ + # --max-time so a stalled webhook cannot eat the job's budget and + # queue every later tick behind this one. + curl -sS --fail-with-body --max-time 20 \ + -X POST -H 'Content-type: application/json' \ --data @payload.json "$SLACK_WEBHOOK_URL" env: + # Scoped to this step alone — the only one that needs it. + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} FAILED: ${{ steps.summary.outputs.failed }} # Re-raise the probe's verdict so the run itself is red. This is the