diff --git a/.github/workflows/health.yml b/.github/workflows/health.yml new file mode 100644 index 0000000..30c7700 --- /dev/null +++ b/.github/workflows/health.yml @@ -0,0 +1,197 @@ +# Off-host health probe for the production MCP service. +# +# The status dashboard already runs every check this needs — MCP, the OAuth +# suite, TLS, and the Internet Identity linkage — but it runs *on the host* and +# is pull-only: something has to look at it. This runs the same probe suite from +# GitHub's infrastructure on a schedule, which is the part the host cannot do +# for itself. A host that is down cannot tell you it is down. +# +# Escalation, honestly labelled: a failure here fails the workflow, which +# notifies watchers by email, and posts to Slack when a webhook is configured. +# Neither is a page. Nobody is on call. For real escalation the probe belongs in +# the organisation's Prometheus/OpsGenie stack, where an alert reaches someone +# who has agreed to be woken — this is the stopgap, not the destination. +# +# Optional repository secret: +# SLACK_WEBHOOK_URL Incoming-webhook URL, intended for #eng-identity-imcp2. +# Absent, the Slack step is skipped and the workflow +# failure remains the only signal. Nothing else changes. +name: Production health + +on: + schedule: + # Every 15 minutes. GitHub's scheduler is best-effort and routinely drifts + # under load, so treat this as "noticed within the hour", not an SLA. Note + # that scheduled workflows are disabled automatically after 60 days without + # repository activity. + - cron: '*/15 * * * *' + # Manual run, for verifying the probe itself rather than waiting for a tick. + workflow_dispatch: + +# One probe at a time; a slow run should not overlap the next tick. +concurrency: + group: health + cancel-in-progress: false + +permissions: + contents: read + +jobs: + probe: + runs-on: ubuntu-24.04 + # Below the probe cadence, and deliberately so. `concurrency` above lets one + # run at a time without cancelling, so a wedged run does not just waste a + # slot — it queues every later tick behind itself. With the default 6-hour + # job timeout that is 24 missed probes, during which the monitor reports + # nothing and silence is indistinguishable from health. Failing fast and + # letting the next tick try again is strictly better than hanging. + timeout-minutes: 10 + env: + MCP_ORIGIN: https://mcp.internetcomputer.org + # The probe refuses any origin outside its SSRF allow-list, which defaults + # to id.ai and loopback. The production host is neither, so it has to be + # named explicitly — exactly as deploy.sh does when it starts the hosted + # dashboard. Without this the run fails with a usage error rather than a + # health verdict, which is a confusing way to learn the service is fine. + MCP_STATUS_ALLOWED_HOSTS: mcp.internetcomputer.org + # A boolean, not the secret. The webhook itself is scoped to the one step + # that posts it (below); putting it in the job environment would hand it to + # every step and to the checked-out probe code, none of which needs it, and + # log masking does not stop a process from reading its own environment. + # This expression evaluates to "true"/"false" without exposing the value, + # which is what lets the step condition stay declarative — `secrets` is not + # available in a step `if:`. + HAS_SLACK: ${{ secrets.SLACK_WEBHOOK_URL != '' }} + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + # 20 reached end of life on 2026-04-30 and no longer gets security + # fixes, and this job parses responses from an internet-facing service. + # deploy-native.yml and the host still pin 20; bumping those is a + # separate change with its own blast radius. + node-version: 22 + + # Deliberately NOT --strict. That flag escalates warnings to failures, and + # a third of the suite probes Internet Identity and the IC — infrastructure + # this team does not operate. Alerting on someone else's degradation is how + # a channel learns to ignore the alerts. Real failures still fail: without + # --strict the CLI exits non-zero exactly when `overall` is "fail", which + # is the same threshold the dashboard uses to serve 503. + - name: Probe production + id: probe + run: | + set +e + node monitoring/mcp-status/cli.js --mcp "$MCP_ORIGIN" --json > report.json 2> probe.err + code=$? + set -e + echo "exit_code=$code" >> "$GITHUB_OUTPUT" + if [ "$code" -eq 0 ]; then + echo "healthy" + else + echo "probe exited $code" + cat probe.err || true + fi + + # Summarise which checks failed, rather than only that something did. A + # bare "the workflow failed" costs the reader a log dive at exactly the + # moment they are least inclined to do one. + - name: Summarise + id: summary + if: always() + run: | + python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json, os + try: + r = json.load(open("report.json")) + except Exception as e: + print(f"Could not parse the probe report: {e}\n") + print("The probe did not produce usable JSON — most likely it could " + "not reach the host at all, which is itself the finding.") + raise SystemExit(0) + # Not every value here is ours. The probed server supplies the II + # instance names in /version, and the dashboard interpolates them + # straight into check labels and section titles. A misconfigured or + # compromised host could therefore put a pipe, a newline or Markdown + # into a table cell and corrupt or spoof this summary. Escape per cell. + def cell(v): + t = "".join(ch if ch.isprintable() else " " for ch in str(v)) + return " ".join(t.split()).replace("\\", "").replace("|", "\\|")[:120] or "?" + + bad = [] + for section in r.get("sections", []): + for c in section.get("checks", []): + if c.get("status") in ("fail", "warn"): + bad.append((c.get("status"), section.get("title", "?"), c.get("label", "?"))) + print(f"**overall: {cell(r.get('overall'))}**\n") + if bad: + print("| status | section | check |") + print("|---|---|---|") + for s, sec, lab in bad: + print(f"| {cell(s)} | {cell(sec)} | {cell(lab)} |") + else: + print("All checks passed.") + PY + # A one-line form for the Slack payload. + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import json + try: + r = json.load(open("report.json")) + failed = [c.get("label", "?") + for s in r.get("sections", []) + for c in s.get("checks", []) + if c.get("status") == "fail"] + line = ", ".join(failed) if failed else "no individual check reported fail" + except Exception: + line = "the probe produced no usable report (the host may be unreachable)" + # $GITHUB_OUTPUT is line-oriented, so a newline in a value would let + # the value declare further outputs. This is not hypothetical: check + # labels embed the II instance names the probed server advertises, so + # part of this string originates outside our control. + line = " ".join(line.split())[:400] + print(f"failed={line}") + PY + + # Skipped when the secret is absent, so this workflow is useful the moment + # it lands and gains Slack routing later with no code change. `secrets` is + # not available in a step condition, hence the job-level env mapping above. + # + # The condition reads the probe's recorded exit code rather than + # `failure()`. The probe step swallows its own failure on purpose, so at + # this point in the job nothing has failed yet and `failure()` is false — + # the notification would never have fired, while the run still went red at + # the final step and looked correct from the outside. + - name: Notify Slack + if: steps.probe.outputs.exit_code != '0' && env.HAS_SLACK == 'true' + run: | + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + python3 - "$run_url" <<'PY' > payload.json + import json, sys, os + text = ( + ":rotating_light: *ICP MCP production health check failed*\n" + f"Failing: {os.environ.get('FAILED', 'unknown')}\n" + f"<{sys.argv[1]}|Run log> · " + "" + ) + json.dump({"text": text}, sys.stdout) + PY + # --max-time so a stalled webhook cannot eat the job's budget and + # queue every later tick behind this one. + curl -sS --fail-with-body --max-time 20 \ + -X POST -H 'Content-type: application/json' \ + --data @payload.json "$SLACK_WEBHOOK_URL" + env: + # Scoped to this step alone — the only one that needs it. + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + FAILED: ${{ steps.summary.outputs.failed }} + + # Re-raise the probe's verdict so the run itself is red. This is the + # baseline notification and the one that needs no configuration at all. + - name: Fail the run if unhealthy + if: steps.probe.outputs.exit_code != '0' + run: | + echo "::error::production health check failed (${{ steps.summary.outputs.failed }})" + exit 1