From 8676e543687ff3bd6018504aba86c1bc7394ff95 Mon Sep 17 00:00:00 2001 From: Tim Bradgate Date: Thu, 22 Oct 2026 18:29:17 +0100 Subject: [PATCH 1/6] Add Prometheus/Grafana monitoring compose overlay and docs Co-Authored-By: Claude Sonnet 5.5 --- docker-compose.monitoring.yaml | 42 ++ documentation/deployment.md | 54 ++ monitoring/grafana/dashboards/digiscript.json | 575 ++++++++++++++++++ .../provisioning/dashboards/dashboards.yml | 7 + .../provisioning/datasources/prometheus.yml | 9 + monitoring/prometheus/alerts.yml | 36 ++ monitoring/prometheus/prometheus.yml | 15 + 7 files changed, 738 insertions(+) create mode 100644 docker-compose.monitoring.yaml create mode 100644 monitoring/grafana/dashboards/digiscript.json create mode 100644 monitoring/grafana/provisioning/dashboards/dashboards.yml create mode 100644 monitoring/grafana/provisioning/datasources/prometheus.yml create mode 100644 monitoring/prometheus/alerts.yml create mode 100644 monitoring/prometheus/prometheus.yml diff --git a/docker-compose.monitoring.yaml b/docker-compose.monitoring.yaml new file mode 100644 index 00000000..ece3f82b --- /dev/null +++ b/docker-compose.monitoring.yaml @@ -0,0 +1,42 @@ +# Optional Prometheus + Grafana monitoring overlay for DigiScript. +# +# export GF_SECURITY_ADMIN_PASSWORD='choose-a-strong-password' +# docker compose -f docker-compose.yaml -f docker-compose.monitoring.yaml up -d +# +# Published ports are bound to 127.0.0.1 only. To expose them on other +# interfaces set PROMETHEUS_BIND / GRAFANA_BIND (e.g. GRAFANA_BIND=0.0.0.0), +# and put a reverse proxy with TLS/auth in front of them. +# +# The scrape path / auth for the DigiScript metrics endpoint lives in ONE place: +# monitoring/prometheus/prometheus.yml +services: + prometheus: + image: prom/prometheus:v3.15.1 + restart: unless-stopped + ports: + - "${PROMETHEUS_BIND:-127.0.0.1}:9090:9090" + volumes: + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./monitoring/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro + - prometheus_data:/prometheus + depends_on: + - server + + grafana: + image: grafana/grafana:13.2.4 + restart: unless-stopped + ports: + - "${GRAFANA_BIND:-127.0.0.1}:3000:3000" + environment: + GF_SECURITY_ADMIN_PASSWORD: "${GF_SECURITY_ADMIN_PASSWORD:?Set GF_SECURITY_ADMIN_PASSWORD to a strong Grafana admin password}" + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - grafana_data:/var/lib/grafana + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - prometheus + +volumes: + prometheus_data: + grafana_data: diff --git a/documentation/deployment.md b/documentation/deployment.md index cca564aa..350727aa 100644 --- a/documentation/deployment.md +++ b/documentation/deployment.md @@ -20,6 +20,60 @@ docker build -t digiscript:latest . docker-compose up -d ``` +`docker-compose up -d` starts the DigiScript server only. For metrics dashboards see +[Monitoring](#monitoring) below. + +### Monitoring + +An optional compose overlay adds Prometheus and Grafana, pre-provisioned with a +"DigiScript Overview" dashboard and basic alert rules: + +```shell +export GF_SECURITY_ADMIN_PASSWORD='choose-a-strong-password' +docker compose -f docker-compose.yaml -f docker-compose.monitoring.yaml up -d +``` + +`GF_SECURITY_ADMIN_PASSWORD` is required (there is no default); compose refuses to +start without it. + +| Service | URL | Notes | +|------------|-------------------------|-----------------------------------------| +| Grafana | `http://localhost:3000` | Log in as `admin` with the password set | +| Prometheus | `http://localhost:9090` | Scrapes `server:8080/debug/metrics` | + +Both ports are published on `127.0.0.1` only. To expose them set `GRAFANA_BIND` / +`PROMETHEUS_BIND` (e.g. `GRAFANA_BIND=0.0.0.0`) and put a TLS-terminating reverse proxy +in front. The scrape path is configured in `monitoring/prometheus/prometheus.yml`. + +**Security:** `/debug/metrics` is currently unauthenticated. Do not expose DigiScript's +port 8080 to the public internet without a reverse proxy that restricts this path. + +**Health check:** `/api/v1/health` returns the server health and is suitable for load +balancer or container health checks. Application logs can be viewed in-app via the log +viewer. + +#### Metrics reference + +| Metric | Type | Meaning | +|--------|------|---------| +| `python_*`, `process_*` | various | Python runtime and process stats | +| `tornado_http_requests_total` | counter | HTTP requests by method, handler and status (see note) | +| `tornado_http_request_duration_seconds` | histogram | HTTP request latency | +| `digiscript_http_request_body_bytes` | histogram | Size of HTTP request bodies, by handler and method | +| `digiscript_ws_message_bytes` | histogram | Size of inbound WebSocket messages | +| `digiscript_ws_connections` | gauge | Open WebSocket connections | +| `digiscript_ws_messages_total{op}` | counter | WebSocket messages by operation | +| `digiscript_collab_rooms` | gauge | Active collaborative editing rooms | +| `digiscript_show_session_active` | gauge | 1 if a live show session is running, else 0 | +| `digiscript_info{version}` | gauge | Always 1; carries the server version label | +| `digiscript_auth_events_total{event,result}` | counter | Authentication events by type and result | +| `digiscript_unhandled_errors_total{source}` | counter | Unhandled errors by source | + +Note: `tornado_http_*` currently has no samples until the pending "fix HTTP metrics" +change is released, and the `digiscript_*` metrics from `digiscript_ws_connections` +onwards are being added incrementally. Dashboard panels for metrics your server version +does not yet expose show "No data". + ### Pre-built Server Executables Pre-built server executables are available from [GitHub Releases](https://github.com/dreamteamprod/DigiScript/releases): diff --git a/monitoring/grafana/dashboards/digiscript.json b/monitoring/grafana/dashboards/digiscript.json new file mode 100644 index 00000000..8c2d4b4e --- /dev/null +++ b/monitoring/grafana/dashboards/digiscript.json @@ -0,0 +1,575 @@ +{ + "uid": "digiscript-overview", + "title": "DigiScript Overview", + "tags": [ + "digiscript" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Overview", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Version", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "max by (version) (digiscript_info)", + "legendFormat": "{{version}}" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "name" + } + }, + { + "id": 3, + "type": "stat", + "title": "Show session active", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "max(digiscript_show_session_active)", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "No" + }, + "1": { + "text": "Yes" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 4, + "type": "stat", + "title": "WebSocket connections", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "sum(digiscript_ws_connections)", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 5, + "type": "stat", + "title": "Collab rooms", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "sum(digiscript_collab_rooms)", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 6, + "type": "row", + "title": "HTTP", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "panels": [] + }, + { + "id": 7, + "type": "timeseries", + "title": "Request rate by status class", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "sum by (status_class) (label_replace(rate(tornado_http_requests_total[5m]), \"status_class\", \"${1}xx\", \"status\", \"(\\\\d)..\"))", + "legendFormat": "{{status_class}}" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data", + "unit": "reqps" + }, + "overrides": [] + }, + "options": {}, + "description": "No data until HTTP metrics are recorded by the server." + }, + { + "id": 8, + "type": "timeseries", + "title": "Request latency p50 / p95", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(tornado_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(tornado_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data", + "unit": "s" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 9, + "type": "timeseries", + "title": "Request body size p50 / p95", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(digiscript_http_request_body_bytes_bucket[5m])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(digiscript_http_request_body_bytes_bucket[5m])))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data", + "unit": "bytes" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 10, + "type": "timeseries", + "title": "WebSocket messages by op", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "sum by (op) (rate(digiscript_ws_messages_total[5m]))", + "legendFormat": "{{op}}" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data", + "unit": "ops" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 11, + "type": "timeseries", + "title": "WebSocket message size p95", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(digiscript_ws_message_bytes_bucket[5m])))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data", + "unit": "bytes" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 12, + "type": "row", + "title": "Auth", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "panels": [] + }, + { + "id": 13, + "type": "timeseries", + "title": "Auth events by result", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "sum by (event, result) (increase(digiscript_auth_events_total[5m]))", + "legendFormat": "{{event}} / {{result}}" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 14, + "type": "row", + "title": "Health", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "panels": [] + }, + { + "id": 15, + "type": "stat", + "title": "Server up", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 32 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "up{job=\"digiscript\"}", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 16, + "type": "timeseries", + "title": "Unhandled errors by source", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 18, + "x": 6, + "y": 32 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "sum by (source) (increase(digiscript_unhandled_errors_total[5m]))", + "legendFormat": "{{source}}" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data" + }, + "overrides": [] + }, + "options": {} + } + ], + "templating": { + "list": [] + }, + "annotations": { + "list": [] + } +} \ No newline at end of file diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 00000000..209117da --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,7 @@ +apiVersion: 1 +providers: + - name: DigiScript + type: file + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..48d8ba37 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,9 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: digiscript-prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/monitoring/prometheus/alerts.yml b/monitoring/prometheus/alerts.yml new file mode 100644 index 00000000..ca27610c --- /dev/null +++ b/monitoring/prometheus/alerts.yml @@ -0,0 +1,36 @@ +groups: + - name: digiscript + rules: + - alert: DigiScriptDown + expr: up{job="digiscript"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: DigiScript server is not being scraped successfully + - alert: DigiScriptHighLatencyP95 + expr: histogram_quantile(0.95, sum by (le) (rate(tornado_http_request_duration_seconds_bucket[5m]))) > 1 + for: 5m + labels: + severity: warning + annotations: + summary: p95 HTTP request latency above 1s for 5m + - alert: DigiScriptHigh5xxRate + expr: sum(rate(tornado_http_requests_total{status=~"5.."}[5m])) / sum(rate(tornado_http_requests_total[5m])) > 0.05 + for: 5m + labels: + severity: warning + annotations: + summary: More than 5% of HTTP requests are returning 5xx + - alert: DigiScriptUnhandledErrors + expr: increase(digiscript_unhandled_errors_total[5m]) > 0 + labels: + severity: warning + annotations: + summary: Unhandled errors raised in the last 5m + - alert: DigiScriptAuthFailureSpike + expr: sum(increase(digiscript_auth_events_total{result="failure"}[5m])) > 20 + labels: + severity: warning + annotations: + summary: More than 20 failed auth events in 5m diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 00000000..ef6dc80a --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,15 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/alerts.yml + +scrape_configs: + - job_name: digiscript + # Scrape path and any auth for the DigiScript metrics endpoint: change here only. + # Currently unauthenticated; if it is gated later add e.g. + # authorization: { credentials_file: /etc/prometheus/token } + metrics_path: /debug/metrics + static_configs: + - targets: ["server:8080"] From 4151830862113ca381fadd63f6cb935a718b7977 Mon Sep 17 00:00:00 2001 From: Tim Bradgate Date: Thu, 22 Oct 2026 18:54:27 +0100 Subject: [PATCH 2/6] Document the /api/v1/health contract in the Monitoring docs Describes the checks, 200/503 behaviour, response shape and the Docker HEALTHCHECK added in the health-check PR. Co-Authored-By: Claude Sonnet 5.5 --- documentation/deployment.md | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/documentation/deployment.md b/documentation/deployment.md index 350727aa..fc80a4ce 100644 --- a/documentation/deployment.md +++ b/documentation/deployment.md @@ -48,9 +48,37 @@ in front. The scrape path is configured in `monitoring/prometheus/prometheus.yml **Security:** `/debug/metrics` is currently unauthenticated. Do not expose DigiScript's port 8080 to the public internet without a reverse proxy that restricts this path. -**Health check:** `/api/v1/health` returns the server health and is suitable for load -balancer or container health checks. Application logs can be viewed in-app via the log -viewer. +Application logs can be viewed in-app via the log viewer. + +#### Health check + +`GET /api/v1/health` is an unauthenticated liveness/readiness probe, suitable for load +balancer or container health checks. The Docker image uses it for its built-in +`HEALTHCHECK` (every 30s, 5s timeout, 30s start period, 3 retries). Requests to it are +not written to the access log. + +It runs `SELECT 1` against the database (3s timeout) and checks that the directories +holding the log files, the draft script path and the compiled script path are writable. + +| Condition | HTTP status | `status` | +|---|---|---| +| Database and storage checks pass | `200` | `ok` | +| Either check fails or the database times out | `503` | `degraded` | + +Example healthy response: + +```json +{ + "version": "0.35.2", + "status": "ok", + "checks": { "database": "ok", "storage": "ok" }, + "uptime_seconds": 1234 +} +``` + +A failing check is reported as `"fail"` in `checks`. The response never includes error +text or file paths; the detail is written to the server log instead (`ERROR` for the +database, `WARNING` for storage), so look there to find out why a probe is failing. #### Metrics reference From 3bb8caddf257b2268957d94b4276439555a83fe1 Mon Sep 17 00:00:00 2001 From: Tim Bradgate Date: Thu, 22 Oct 2026 23:14:03 +0100 Subject: [PATCH 3/6] Address review: robust 5xx and unhandled-error alerts, simplify dashboard, update docs Co-Authored-By: Claude Sonnet 5.5 --- documentation/deployment.md | 9 ++++----- monitoring/grafana/dashboards/digiscript.json | 10 +++++----- monitoring/prometheus/alerts.yml | 18 ++++++++++++++++-- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/documentation/deployment.md b/documentation/deployment.md index fc80a4ce..323cfbd5 100644 --- a/documentation/deployment.md +++ b/documentation/deployment.md @@ -85,7 +85,7 @@ database, `WARNING` for storage), so look there to find out why a probe is faili | Metric | Type | Meaning | |--------|------|---------| | `python_*`, `process_*` | various | Python runtime and process stats | -| `tornado_http_requests_total` | counter | HTTP requests by method, handler and status (see note) | +| `tornado_http_requests_total` | counter | HTTP requests by method, handler and status | | `tornado_http_request_duration_seconds` | histogram | HTTP request latency | | `digiscript_http_request_body_bytes` | histogram | Size of HTTP request bodies, by handler and method | | `digiscript_ws_message_bytes` | histogram | Size of inbound WebSocket messages | @@ -97,10 +97,9 @@ database, `WARNING` for storage), so look there to find out why a probe is faili | `digiscript_auth_events_total{event,result}` | counter | Authentication events by type and result | | `digiscript_unhandled_errors_total{source}` | counter | Unhandled errors by source | -Note: `tornado_http_*` currently has no samples until the pending "fix HTTP metrics" -change is released, and the `digiscript_*` metrics from `digiscript_ws_connections` -onwards are being added incrementally. Dashboard panels for metrics your server version -does not yet expose show "No data". +Note: older releases do not expose all of these metrics (for example the `tornado_http_*` +series and the newer `digiscript_*` gauges and counters). Dashboard panels for metrics +your server version does not expose show "No data". ### Pre-built Server Executables diff --git a/monitoring/grafana/dashboards/digiscript.json b/monitoring/grafana/dashboards/digiscript.json index 8c2d4b4e..46998f8d 100644 --- a/monitoring/grafana/dashboards/digiscript.json +++ b/monitoring/grafana/dashboards/digiscript.json @@ -247,8 +247,8 @@ "type": "prometheus", "uid": "digiscript-prometheus" }, - "expr": "sum by (status_class) (label_replace(rate(tornado_http_requests_total[5m]), \"status_class\", \"${1}xx\", \"status\", \"(\\\\d)..\"))", - "legendFormat": "{{status_class}}" + "expr": "sum by (status) (rate(tornado_http_requests_total{handler!=\"WebSocketController\"}[5m]))", + "legendFormat": "{{status}}" } ], "fieldConfig": { @@ -259,7 +259,7 @@ "overrides": [] }, "options": {}, - "description": "No data until HTTP metrics are recorded by the server." + "description": "HTTP requests per second by status class. WebSocket upgrades (101) are excluded." }, { "id": 8, @@ -553,7 +553,7 @@ "type": "prometheus", "uid": "digiscript-prometheus" }, - "expr": "sum by (source) (increase(digiscript_unhandled_errors_total[5m]))", + "expr": "sum by (source) (increase(digiscript_unhandled_errors_total[5m])) or sum by (source) (digiscript_unhandled_errors_total > 0 unless digiscript_unhandled_errors_total offset 5m)", "legendFormat": "{{source}}" } ], @@ -572,4 +572,4 @@ "annotations": { "list": [] } -} \ No newline at end of file +} diff --git a/monitoring/prometheus/alerts.yml b/monitoring/prometheus/alerts.yml index ca27610c..ca87d02e 100644 --- a/monitoring/prometheus/alerts.yml +++ b/monitoring/prometheus/alerts.yml @@ -15,15 +15,29 @@ groups: severity: warning annotations: summary: p95 HTTP request latency above 1s for 5m + # tornado_prometheus.classify_status_code buckets every status outside + # 200-499 as "5xx", including the 101 returned by a WebSocket upgrade. The + # server-side fix keeps 101 out of the series; as defence in depth the + # WebSocket handler is also excluded here (numerator and denominator) so + # normal WS connects/reconnects can never trip this alert. - alert: DigiScriptHigh5xxRate - expr: sum(rate(tornado_http_requests_total{status=~"5.."}[5m])) / sum(rate(tornado_http_requests_total[5m])) > 0.05 + expr: >- + sum(rate(tornado_http_requests_total{status="5xx", handler!="WebSocketController"}[5m])) + / sum(rate(tornado_http_requests_total{handler!="WebSocketController"}[5m])) > 0.05 for: 5m labels: severity: warning annotations: summary: More than 5% of HTTP requests are returning 5xx + # increase() yields nothing for a series that first appears inside the + # window, so a first-ever error for a source would be missed if the series + # is created lazily. The second branch catches that: a series that is + # non-zero now but did not exist 5m ago. (The server also pre-registers the + # series at 0, in which case the first branch alone suffices.) - alert: DigiScriptUnhandledErrors - expr: increase(digiscript_unhandled_errors_total[5m]) > 0 + expr: >- + increase(digiscript_unhandled_errors_total[5m]) > 0 + or (digiscript_unhandled_errors_total > 0 unless digiscript_unhandled_errors_total offset 5m) labels: severity: warning annotations: From 523de872df5cf7baa1279cf5bb53fc39c880a19e Mon Sep 17 00:00:00 2001 From: Tim Bradgate Date: Thu, 22 Oct 2026 23:18:12 +0100 Subject: [PATCH 4/6] Update health check docs to final /api/v1/health contract Co-Authored-By: Claude Sonnet 5.5 --- documentation/deployment.md | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/documentation/deployment.md b/documentation/deployment.md index 323cfbd5..052cfdae 100644 --- a/documentation/deployment.md +++ b/documentation/deployment.md @@ -57,13 +57,20 @@ balancer or container health checks. The Docker image uses it for its built-in `HEALTHCHECK` (every 30s, 5s timeout, 30s start period, 3 retries). Requests to it are not written to the access log. -It runs `SELECT 1` against the database (3s timeout) and checks that the directories -holding the log files, the draft script path and the compiled script path are writable. +Database check: `SELECT 1 FROM sqlite_master LIMIT 1` (reads the database file +header/schema) with a 3s timeout. Probes run single-flight on a dedicated one-worker +executor: while an earlier timed-out probe is still hung, the check reports `fail` +immediately until that probe finishes, so a stuck database cannot pile up threads. + +Storage check: the parent directories of `log_path`, `db_log_path` and `client_log_path`, +plus `draft_script_path` and `compiled_script_path`, must be writable. A directory that +does not exist yet is judged by its nearest existing ancestor. | Condition | HTTP status | `status` | |---|---|---| | Database and storage checks pass | `200` | `ok` | | Either check fails or the database times out | `503` | `degraded` | +| Unexpected internal error | `503` | `degraded` (checks are `"unknown"`, no `uptime_seconds`) | Example healthy response: @@ -78,7 +85,24 @@ Example healthy response: A failing check is reported as `"fail"` in `checks`. The response never includes error text or file paths; the detail is written to the server log instead (`ERROR` for the -database, `WARNING` for storage), so look there to find out why a probe is failing. +database, `WARNING` for storage), so look there to find out why a probe is failing. On an unexpected internal error the +`checks` values are `"unknown"` and `uptime_seconds` is omitted. + +The Docker `HEALTHCHECK` calls `http://localhost:8080/api/v1/health` with Python's +`urllib`; any non-2xx response marks the container unhealthy. The port is hard-wired to +8080 because the app has no environment variable for its port. If you run the server on a +different port, override the check with `--health-cmd` or a compose `healthcheck:` block: + +```yaml +services: + server: + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9090/api/v1/health', timeout=4)"] + interval: 30s + timeout: 5s + start_period: 30s + retries: 3 +``` #### Metrics reference From 4daa3936a73bd4b14894be73d6e8a74f09f1a3a2 Mon Sep 17 00:00:00 2001 From: Tim Bradgate Date: Thu, 22 Oct 2026 23:49:06 +0100 Subject: [PATCH 5/6] Fix health check port override example and dedupe error-state docs Co-Authored-By: Claude Sonnet 5.5 --- documentation/deployment.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/documentation/deployment.md b/documentation/deployment.md index 052cfdae..a435f3d5 100644 --- a/documentation/deployment.md +++ b/documentation/deployment.md @@ -70,7 +70,7 @@ does not exist yet is judged by its nearest existing ancestor. |---|---|---| | Database and storage checks pass | `200` | `ok` | | Either check fails or the database times out | `503` | `degraded` | -| Unexpected internal error | `503` | `degraded` (checks are `"unknown"`, no `uptime_seconds`) | +| Unexpected internal error | `503` | `degraded` (`checks` are `"unknown"`, `uptime_seconds` omitted) | Example healthy response: @@ -85,25 +85,33 @@ Example healthy response: A failing check is reported as `"fail"` in `checks`. The response never includes error text or file paths; the detail is written to the server log instead (`ERROR` for the -database, `WARNING` for storage), so look there to find out why a probe is failing. On an unexpected internal error the -`checks` values are `"unknown"` and `uptime_seconds` is omitted. +database, `WARNING` for storage), so look there to find out why a probe is failing. The Docker `HEALTHCHECK` calls `http://localhost:8080/api/v1/health` with Python's `urllib`; any non-2xx response marks the container unhealthy. The port is hard-wired to -8080 because the app has no environment variable for its port. If you run the server on a -different port, override the check with `--health-cmd` or a compose `healthcheck:` block: +8080 because the app has no environment variable for its port. If you change the +listening port with the `--port` flag, override the check with `--health-cmd` or a +compose `healthcheck:` block that targets the same container port. For example, to +listen on 8081 (published on host port 8081): ```yaml services: server: + command: ["--port=8081"] + ports: + - 8081:8081 healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9090/api/v1/health', timeout=4)"] + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8081/api/v1/health', timeout=4)"] interval: 30s timeout: 5s start_period: 30s retries: 3 ``` +If you also use the monitoring overlay, change the scrape target in +`monitoring/prometheus/prometheus.yml` to `server:8081` too. Avoid host ports 3000 and +9090, which the overlay uses for Grafana and Prometheus. + #### Metrics reference | Metric | Type | Meaning | From c907f9e331b373d288b3eb9bd5ca8e788ff6c284 Mon Sep 17 00:00:00 2001 From: Tim Bradgate Date: Thu, 22 Oct 2026 23:53:33 +0100 Subject: [PATCH 6/6] Update health DB probe single-flight description Co-Authored-By: Claude Sonnet 5.5 --- documentation/deployment.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/documentation/deployment.md b/documentation/deployment.md index a435f3d5..6dde49cf 100644 --- a/documentation/deployment.md +++ b/documentation/deployment.md @@ -58,9 +58,10 @@ balancer or container health checks. The Docker image uses it for its built-in not written to the access log. Database check: `SELECT 1 FROM sqlite_master LIMIT 1` (reads the database file -header/schema) with a 3s timeout. Probes run single-flight on a dedicated one-worker -executor: while an earlier timed-out probe is still hung, the check reports `fail` -immediately until that probe finishes, so a stuck database cannot pile up threads. +header/schema). Probes run single-flight on a dedicated one-worker executor: overlapping +polls await the same in-flight probe, and every caller is bounded by a 3s timeout. While a +probe is genuinely hung, each poll waits up to 3s and then reports `fail`; no second probe +starts until the first finishes, so a stuck database cannot pile up threads. Storage check: the parent directories of `log_path`, `db_log_path` and `client_log_path`, plus `draft_script_path` and `compiled_script_path`, must be writable. A directory that