Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ team that owns your monitors is enough — it does not need account-wide rights.
| `REQUEST_TIMEOUT` | `30` | seconds; Better Stack allows 2, 3, 5, 10, 15, 30, 45, 60 |
| `EXPECTED_STATUS_CODES` | `200,201,202,204` | what counts as up |
| `REGIONS` | `eu` | comma-separated, from `us,eu,as,au` |
| `SSL_EXPIRATION` | — | days before certificate expiry to warn; `1,2,3,7,14,30,60` |
| `MAINTENANCE_FROM` | — | start of a window in which nothing is checked, `01:00` |
| `MAINTENANCE_TO` | — | end of it, `03:00` |
| `MAINTENANCE_TIMEZONE` | `UTC` | any zone name, `Europe/Amsterdam` |
Expand Down Expand Up @@ -189,6 +190,29 @@ If you set 3xx codes to assert the redirect itself, following is switched off
automatically — Better Stack refuses the combination, and rightly: follow the redirect
and you never observe the status you asked for.

## Certificate expiry

An uptime check says nothing about a certificate that stopped renewing. It keeps serving
the old one, valid, right up to the hour it expires — so every check is green, and the
first signal is the outage itself.

```
SSL_EXPIRATION=14
```

Better Stack then warns that many days ahead. Its list is `1, 2, 3, 7, 14, 30, 60`;
anything else is refused at startup rather than by a create call later.

Unset, the operator leaves the field alone, including on monitors that already have a
value — the same rule as the maintenance window.

Worth knowing what this does and does not cover. It watches the certificate on the
hostname, so it catches a renewal that silently stopped whatever the cause. It does not
watch hostnames with no monitor, and it warns about the consequence rather than the
cause: by the time it fires, issuance has already been failing for a while. Where you
can also alert on the cause — a cert-manager `Issuing` condition that has been true for
hours, say — that fires earlier and covers more.

## Maintenance windows

Most clusters have a period when disruption is expected and an alert says nothing. A
Expand Down
1 change: 1 addition & 0 deletions deploy/operator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ data:
CHECK_FREQUENCY: "180"
REQUEST_TIMEOUT: "30"
REGIONS: "eu"
SSL_EXPIRATION: "14"
RESYNC_SECONDS: "900"
# Start here. Flip to "false" once the log shows the plan you expected.
DRY_RUN: "true"
Expand Down
23 changes: 23 additions & 0 deletions src/gateway_uptime/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,17 @@ class Config:
maintenance_days: tuple[str, ...] = ()
# Optioneel. Alleen gebruikt bij het aanmaken van een monitor — zie de provider.
policy_id: str | None = None
# Days before certificate expiry to warn on. None leaves the field unmanaged.
ssl_expiration: int | None = None

@property
def monitor_group_name(self) -> str:
return self.cluster_name

@property
def manages_ssl_expiration(self) -> bool:
return self.ssl_expiration is not None

@property
def manages_maintenance(self) -> bool:
return bool(self.maintenance_from and self.maintenance_to)
Expand All @@ -87,6 +93,10 @@ def excluded(self, hostname: str) -> bool:
# create time, which is a confusing place to learn about a typo in a ConfigMap.
VALID_TIMEOUTS = (2, 3, 5, 10, 15, 30, 45, 60)

# Days before expiry that Better Stack will warn on. Its own list; anything else is
# refused at create time.
VALID_SSL_EXPIRATION = (1, 2, 3, 7, 14, 30, 60)


def from_env(env: dict[str, str] | None = None) -> Config:
e = os.environ if env is None else env
Expand Down Expand Up @@ -119,6 +129,18 @@ def from_env(env: dict[str, str] | None = None) -> Config:
raise ConfigError("CHECK_FREQUENCY (%d) must be at least REQUEST_TIMEOUT (%d)"
% (frequency, timeout))

raw_ssl = e.get("SSL_EXPIRATION", "").strip()
ssl_expiration: int | None = None
if raw_ssl:
try:
ssl_expiration = int(raw_ssl)
except ValueError:
raise ConfigError("SSL_EXPIRATION must be a number of days, got %r"
% raw_ssl) from None
if ssl_expiration not in VALID_SSL_EXPIRATION:
raise ConfigError("SSL_EXPIRATION must be one of %s days, got %d"
% (list(VALID_SSL_EXPIRATION), ssl_expiration))

mf = _clock(e.get("MAINTENANCE_FROM", ""))
mt = _clock(e.get("MAINTENANCE_TO", ""))
if bool(mf) != bool(mt):
Expand All @@ -143,6 +165,7 @@ def from_env(env: dict[str, str] | None = None) -> Config:
regions=regions,
resync_seconds=int(e.get("RESYNC_SECONDS", "900")),
policy_id=(e.get("BETTERSTACK_POLICY_ID", "").strip() or None),
ssl_expiration=ssl_expiration,
dry_run=e.get("DRY_RUN", "false").lower() in ("1", "true", "yes"),
maintenance_from=mf,
maintenance_to=mt,
Expand Down
6 changes: 6 additions & 0 deletions src/gateway_uptime/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ class DesiredMonitor:
# Applied when the monitor is first created and never afterwards, so that whoever
# tunes alerting in Better Stack keeps their change.
policy_id: str | None = None
# Days before certificate expiry to warn on. None when unmanaged.
ssl_expiration: int | None = None
# A window in which Better Stack does not check. Empty when unmanaged.
maintenance_from: str = ""
maintenance_to: str = ""
Expand Down Expand Up @@ -60,6 +62,7 @@ class ExistingMonitor:
request_timeout: int
expected_status_codes: tuple[int, ...]
regions: tuple[str, ...]
ssl_expiration: int | None = None
maintenance_from: str = ""
maintenance_to: str = ""
maintenance_timezone: str = "UTC"
Expand All @@ -79,6 +82,9 @@ def differs_from(self, want: DesiredMonitor) -> list[str]:
out.append("expected_status_codes")
if sorted(self.regions) != sorted(want.regions):
out.append("regions")
if want.ssl_expiration is not None \
and self.ssl_expiration != want.ssl_expiration:
out.append("ssl_expiration")
# Only compared when a window is configured. Without one the operator leaves
# the fields alone rather than clearing whatever is already there.
if want.maintenance_from:
Expand Down
4 changes: 4 additions & 0 deletions src/gateway_uptime/providers/betterstack.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ def _to_existing(item: dict) -> ExistingMonitor:
request_timeout=int(a.get("request_timeout") or 0),
expected_status_codes=tuple(int(c) for c in codes),
regions=tuple(a.get("regions") or []),
ssl_expiration=(int(a["ssl_expiration"])
if a.get("ssl_expiration") is not None else None),
maintenance_from=a.get("maintenance_from") or "",
maintenance_to=a.get("maintenance_to") or "",
maintenance_timezone=a.get("maintenance_timezone") or "UTC",
Expand All @@ -121,6 +123,8 @@ def _payload(self, want: DesiredMonitor, creating: bool = False) -> dict:
"regions": list(want.regions),
"monitor_group_id": int(self._ensure_group()),
}
if want.ssl_expiration is not None:
body["ssl_expiration"] = want.ssl_expiration
# Sent only when a window is configured, so an operator with no window set
# never clears one that was put on by hand.
if want.maintenance_from:
Expand Down
1 change: 1 addition & 0 deletions src/gateway_uptime/selector.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ def monitors_for(route: dict, cfg: Config) -> list[DesiredMonitor]:
namespace=ns,
route=name,
policy_id=cfg.policy_id,
ssl_expiration=cfg.ssl_expiration,
maintenance_from=cfg.maintenance_from,
maintenance_to=cfg.maintenance_to,
maintenance_timezone=cfg.maintenance_timezone,
Expand Down
86 changes: 86 additions & 0 deletions tests/test_betterstack.py
Original file line number Diff line number Diff line change
Expand Up @@ -342,3 +342,89 @@ def test_an_existing_window_is_ignored_when_none_is_configured():
maintenance_from="02:00:00", maintenance_to="04:00:00")
p = plan([want()], [have])
assert p.empty


# ------------------------------------------------------------- ssl expiration
# Not an uptime property: a certificate that stopped renewing keeps serving a valid
# one until the day it expires, so every other check stays green right up to the
# outage. This is the only warning that arrives while there is still time.

def test_a_valid_number_of_days_is_accepted():
c = cfg_with_window(SSL_EXPIRATION="14")
assert c.ssl_expiration == 14 and c.manages_ssl_expiration


def test_not_configured_means_unmanaged():
c = cfg_with_window()
assert c.ssl_expiration is None and not c.manages_ssl_expiration


def test_a_number_better_stack_does_not_take_is_refused():
# 10 looks reasonable and is not on its list; better to fail at start than to
# learn about it from a rejected create call
from gateway_uptime.config import ConfigError
with pytest.raises(ConfigError):
cfg_with_window(SSL_EXPIRATION="10")


def test_something_that_is_not_a_number_is_refused():
from gateway_uptime.config import ConfigError
with pytest.raises(ConfigError):
cfg_with_window(SSL_EXPIRATION="two weeks")


def test_the_warning_is_sent_when_configured():
http = FakeHTTP({("GET", "/monitor-groups"): GROUP_PAGE,
("POST", "/monitors"): {"data": {"id": "13"}}})
w = DesiredMonitor(hostname="a.example.com", url="https://a.example.com/",
check_frequency=180, request_timeout=30,
expected_status_codes=(200,), regions=("eu",),
namespace="ns", route="r", ssl_expiration=14)
BetterStack("tok", "test-cluster", opener=http).create(w)
body = [c for c in http.calls if c[0] == "POST"][0][2]
assert body["ssl_expiration"] == 14


def test_not_configured_means_the_field_is_left_alone():
http = FakeHTTP({("GET", "/monitor-groups"): GROUP_PAGE,
("POST", "/monitors"): {"data": {"id": "14"}}})
BetterStack("tok", "test-cluster", opener=http).create(want())
body = [c for c in http.calls if c[0] == "POST"][0][2]
assert "ssl_expiration" not in body


def test_a_monitor_without_the_warning_is_planned_as_an_update():
from gateway_uptime.model import plan
w = DesiredMonitor(hostname="a.example.com", url="https://a.example.com/",
check_frequency=180, request_timeout=30,
expected_status_codes=(200,), regions=("eu",),
namespace="ns", route="r", ssl_expiration=14)
have = ExistingMonitor(id="1", url=w.url, display_name="a.example.com",
check_frequency=180, request_timeout=30,
expected_status_codes=(200,), regions=("eu",))
p = plan([w], [have])
assert p.update[0][2] == ["ssl_expiration"]


def test_an_existing_warning_is_ignored_when_none_is_configured():
from gateway_uptime.model import plan
have = ExistingMonitor(id="1", url="https://shop.example.com/",
display_name="shop.example.com", check_frequency=180,
request_timeout=30, expected_status_codes=(200, 204),
regions=("eu",), ssl_expiration=30)
p = plan([want()], [have])
assert p.empty


def test_the_api_value_is_read_back():
http = FakeHTTP({("GET", "/monitor-groups"): GROUP_PAGE,
("GET", "/monitor-groups/77/monitors"): {
"data": [{"id": "5", "attributes": {
"url": "https://a.example.com/",
"pronounceable_name": "a.example.com",
"check_frequency": 180, "request_timeout": 30,
"expected_status_codes": [200], "regions": ["eu"],
"ssl_expiration": 14}}],
"pagination": {"next": None}}})
got = BetterStack("tok", "test-cluster", opener=http).list_managed()
assert got[0].ssl_expiration == 14
Loading