From fdeb12063dca00749d3322999dd42084e2b6e029 Mon Sep 17 00:00:00 2001 From: Joey Houtenbos Date: Tue, 15 Sep 2026 21:13:48 +0200 Subject: [PATCH] feat: warn before a certificate expires An uptime check notices nothing when renewal stalls. The old certificate stays valid until the hour it expires, so the monitor is green right up to the point where it is an outage. SSL_EXPIRATION sets the ssl_expiration field on every monitor the operator manages. Unset means unmanaged: the field is then left as it was, the same rule as the maintenance window. Values are checked at startup against the list Better Stack accepts, so a typo does not first surface as a rejected create call. The README also records what this does not cover: hostnames without a monitor, and that it warns about the consequence rather than the cause. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 24 ++++++ deploy/operator.yaml | 1 + src/gateway_uptime/config.py | 23 ++++++ src/gateway_uptime/model.py | 6 ++ src/gateway_uptime/providers/betterstack.py | 4 + src/gateway_uptime/selector.py | 1 + tests/test_betterstack.py | 86 +++++++++++++++++++++ 7 files changed, 145 insertions(+) diff --git a/README.md b/README.md index d05ca2d..354922d 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,7 @@ team that owns your monitors is enough — it does not need account-wide rights. | `REQUEST_TIMEOUT` | `30` | seconds; Better Stack allows 2, 3, 5, 10, 15, 30, 45, 60 | | `EXPECTED_STATUS_CODES` | `200,201,202,204` | what counts as up | | `REGIONS` | `eu` | comma-separated, from `us,eu,as,au` | +| `SSL_EXPIRATION` | — | days before certificate expiry to warn; `1,2,3,7,14,30,60` | | `MAINTENANCE_FROM` | — | start of a window in which nothing is checked, `01:00` | | `MAINTENANCE_TO` | — | end of it, `03:00` | | `MAINTENANCE_TIMEZONE` | `UTC` | any zone name, `Europe/Amsterdam` | @@ -189,6 +190,29 @@ If you set 3xx codes to assert the redirect itself, following is switched off automatically — Better Stack refuses the combination, and rightly: follow the redirect and you never observe the status you asked for. +## Certificate expiry + +An uptime check says nothing about a certificate that stopped renewing. It keeps serving +the old one, valid, right up to the hour it expires — so every check is green, and the +first signal is the outage itself. + +``` +SSL_EXPIRATION=14 +``` + +Better Stack then warns that many days ahead. Its list is `1, 2, 3, 7, 14, 30, 60`; +anything else is refused at startup rather than by a create call later. + +Unset, the operator leaves the field alone, including on monitors that already have a +value — the same rule as the maintenance window. + +Worth knowing what this does and does not cover. It watches the certificate on the +hostname, so it catches a renewal that silently stopped whatever the cause. It does not +watch hostnames with no monitor, and it warns about the consequence rather than the +cause: by the time it fires, issuance has already been failing for a while. Where you +can also alert on the cause — a cert-manager `Issuing` condition that has been true for +hours, say — that fires earlier and covers more. + ## Maintenance windows Most clusters have a period when disruption is expected and an alert says nothing. A diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 8d3b261..199b60c 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -62,6 +62,7 @@ data: CHECK_FREQUENCY: "180" REQUEST_TIMEOUT: "30" REGIONS: "eu" + SSL_EXPIRATION: "14" RESYNC_SECONDS: "900" # Start here. Flip to "false" once the log shows the plan you expected. DRY_RUN: "true" diff --git a/src/gateway_uptime/config.py b/src/gateway_uptime/config.py index 0d37a2b..a0ac478 100644 --- a/src/gateway_uptime/config.py +++ b/src/gateway_uptime/config.py @@ -65,11 +65,17 @@ class Config: maintenance_days: tuple[str, ...] = () # Optioneel. Alleen gebruikt bij het aanmaken van een monitor — zie de provider. policy_id: str | None = None + # Days before certificate expiry to warn on. None leaves the field unmanaged. + ssl_expiration: int | None = None @property def monitor_group_name(self) -> str: return self.cluster_name + @property + def manages_ssl_expiration(self) -> bool: + return self.ssl_expiration is not None + @property def manages_maintenance(self) -> bool: return bool(self.maintenance_from and self.maintenance_to) @@ -87,6 +93,10 @@ def excluded(self, hostname: str) -> bool: # create time, which is a confusing place to learn about a typo in a ConfigMap. VALID_TIMEOUTS = (2, 3, 5, 10, 15, 30, 45, 60) +# Days before expiry that Better Stack will warn on. Its own list; anything else is +# refused at create time. +VALID_SSL_EXPIRATION = (1, 2, 3, 7, 14, 30, 60) + def from_env(env: dict[str, str] | None = None) -> Config: e = os.environ if env is None else env @@ -119,6 +129,18 @@ def from_env(env: dict[str, str] | None = None) -> Config: raise ConfigError("CHECK_FREQUENCY (%d) must be at least REQUEST_TIMEOUT (%d)" % (frequency, timeout)) + raw_ssl = e.get("SSL_EXPIRATION", "").strip() + ssl_expiration: int | None = None + if raw_ssl: + try: + ssl_expiration = int(raw_ssl) + except ValueError: + raise ConfigError("SSL_EXPIRATION must be a number of days, got %r" + % raw_ssl) from None + if ssl_expiration not in VALID_SSL_EXPIRATION: + raise ConfigError("SSL_EXPIRATION must be one of %s days, got %d" + % (list(VALID_SSL_EXPIRATION), ssl_expiration)) + mf = _clock(e.get("MAINTENANCE_FROM", "")) mt = _clock(e.get("MAINTENANCE_TO", "")) if bool(mf) != bool(mt): @@ -143,6 +165,7 @@ def from_env(env: dict[str, str] | None = None) -> Config: regions=regions, resync_seconds=int(e.get("RESYNC_SECONDS", "900")), policy_id=(e.get("BETTERSTACK_POLICY_ID", "").strip() or None), + ssl_expiration=ssl_expiration, dry_run=e.get("DRY_RUN", "false").lower() in ("1", "true", "yes"), maintenance_from=mf, maintenance_to=mt, diff --git a/src/gateway_uptime/model.py b/src/gateway_uptime/model.py index ab4730f..15f20d1 100644 --- a/src/gateway_uptime/model.py +++ b/src/gateway_uptime/model.py @@ -24,6 +24,8 @@ class DesiredMonitor: # Applied when the monitor is first created and never afterwards, so that whoever # tunes alerting in Better Stack keeps their change. policy_id: str | None = None + # Days before certificate expiry to warn on. None when unmanaged. + ssl_expiration: int | None = None # A window in which Better Stack does not check. Empty when unmanaged. maintenance_from: str = "" maintenance_to: str = "" @@ -60,6 +62,7 @@ class ExistingMonitor: request_timeout: int expected_status_codes: tuple[int, ...] regions: tuple[str, ...] + ssl_expiration: int | None = None maintenance_from: str = "" maintenance_to: str = "" maintenance_timezone: str = "UTC" @@ -79,6 +82,9 @@ def differs_from(self, want: DesiredMonitor) -> list[str]: out.append("expected_status_codes") if sorted(self.regions) != sorted(want.regions): out.append("regions") + if want.ssl_expiration is not None \ + and self.ssl_expiration != want.ssl_expiration: + out.append("ssl_expiration") # Only compared when a window is configured. Without one the operator leaves # the fields alone rather than clearing whatever is already there. if want.maintenance_from: diff --git a/src/gateway_uptime/providers/betterstack.py b/src/gateway_uptime/providers/betterstack.py index 0b92bba..b3bf64e 100644 --- a/src/gateway_uptime/providers/betterstack.py +++ b/src/gateway_uptime/providers/betterstack.py @@ -104,6 +104,8 @@ def _to_existing(item: dict) -> ExistingMonitor: request_timeout=int(a.get("request_timeout") or 0), expected_status_codes=tuple(int(c) for c in codes), regions=tuple(a.get("regions") or []), + ssl_expiration=(int(a["ssl_expiration"]) + if a.get("ssl_expiration") is not None else None), maintenance_from=a.get("maintenance_from") or "", maintenance_to=a.get("maintenance_to") or "", maintenance_timezone=a.get("maintenance_timezone") or "UTC", @@ -121,6 +123,8 @@ def _payload(self, want: DesiredMonitor, creating: bool = False) -> dict: "regions": list(want.regions), "monitor_group_id": int(self._ensure_group()), } + if want.ssl_expiration is not None: + body["ssl_expiration"] = want.ssl_expiration # Sent only when a window is configured, so an operator with no window set # never clears one that was put on by hand. if want.maintenance_from: diff --git a/src/gateway_uptime/selector.py b/src/gateway_uptime/selector.py index 1895570..981ddf5 100644 --- a/src/gateway_uptime/selector.py +++ b/src/gateway_uptime/selector.py @@ -88,6 +88,7 @@ def monitors_for(route: dict, cfg: Config) -> list[DesiredMonitor]: namespace=ns, route=name, policy_id=cfg.policy_id, + ssl_expiration=cfg.ssl_expiration, maintenance_from=cfg.maintenance_from, maintenance_to=cfg.maintenance_to, maintenance_timezone=cfg.maintenance_timezone, diff --git a/tests/test_betterstack.py b/tests/test_betterstack.py index 5a9ba0a..d6cbaa6 100644 --- a/tests/test_betterstack.py +++ b/tests/test_betterstack.py @@ -342,3 +342,89 @@ def test_an_existing_window_is_ignored_when_none_is_configured(): maintenance_from="02:00:00", maintenance_to="04:00:00") p = plan([want()], [have]) assert p.empty + + +# ------------------------------------------------------------- ssl expiration +# Not an uptime property: a certificate that stopped renewing keeps serving a valid +# one until the day it expires, so every other check stays green right up to the +# outage. This is the only warning that arrives while there is still time. + +def test_a_valid_number_of_days_is_accepted(): + c = cfg_with_window(SSL_EXPIRATION="14") + assert c.ssl_expiration == 14 and c.manages_ssl_expiration + + +def test_not_configured_means_unmanaged(): + c = cfg_with_window() + assert c.ssl_expiration is None and not c.manages_ssl_expiration + + +def test_a_number_better_stack_does_not_take_is_refused(): + # 10 looks reasonable and is not on its list; better to fail at start than to + # learn about it from a rejected create call + from gateway_uptime.config import ConfigError + with pytest.raises(ConfigError): + cfg_with_window(SSL_EXPIRATION="10") + + +def test_something_that_is_not_a_number_is_refused(): + from gateway_uptime.config import ConfigError + with pytest.raises(ConfigError): + cfg_with_window(SSL_EXPIRATION="two weeks") + + +def test_the_warning_is_sent_when_configured(): + http = FakeHTTP({("GET", "/monitor-groups"): GROUP_PAGE, + ("POST", "/monitors"): {"data": {"id": "13"}}}) + w = DesiredMonitor(hostname="a.example.com", url="https://a.example.com/", + check_frequency=180, request_timeout=30, + expected_status_codes=(200,), regions=("eu",), + namespace="ns", route="r", ssl_expiration=14) + BetterStack("tok", "test-cluster", opener=http).create(w) + body = [c for c in http.calls if c[0] == "POST"][0][2] + assert body["ssl_expiration"] == 14 + + +def test_not_configured_means_the_field_is_left_alone(): + http = FakeHTTP({("GET", "/monitor-groups"): GROUP_PAGE, + ("POST", "/monitors"): {"data": {"id": "14"}}}) + BetterStack("tok", "test-cluster", opener=http).create(want()) + body = [c for c in http.calls if c[0] == "POST"][0][2] + assert "ssl_expiration" not in body + + +def test_a_monitor_without_the_warning_is_planned_as_an_update(): + from gateway_uptime.model import plan + w = DesiredMonitor(hostname="a.example.com", url="https://a.example.com/", + check_frequency=180, request_timeout=30, + expected_status_codes=(200,), regions=("eu",), + namespace="ns", route="r", ssl_expiration=14) + have = ExistingMonitor(id="1", url=w.url, display_name="a.example.com", + check_frequency=180, request_timeout=30, + expected_status_codes=(200,), regions=("eu",)) + p = plan([w], [have]) + assert p.update[0][2] == ["ssl_expiration"] + + +def test_an_existing_warning_is_ignored_when_none_is_configured(): + from gateway_uptime.model import plan + have = ExistingMonitor(id="1", url="https://shop.example.com/", + display_name="shop.example.com", check_frequency=180, + request_timeout=30, expected_status_codes=(200, 204), + regions=("eu",), ssl_expiration=30) + p = plan([want()], [have]) + assert p.empty + + +def test_the_api_value_is_read_back(): + http = FakeHTTP({("GET", "/monitor-groups"): GROUP_PAGE, + ("GET", "/monitor-groups/77/monitors"): { + "data": [{"id": "5", "attributes": { + "url": "https://a.example.com/", + "pronounceable_name": "a.example.com", + "check_frequency": 180, "request_timeout": 30, + "expected_status_codes": [200], "regions": ["eu"], + "ssl_expiration": 14}}], + "pagination": {"next": None}}}) + got = BetterStack("tok", "test-cluster", opener=http).list_managed() + assert got[0].ssl_expiration == 14