diff --git a/docker-compose.monitoring.yaml b/docker-compose.monitoring.yaml new file mode 100644 index 00000000..ece3f82b --- /dev/null +++ b/docker-compose.monitoring.yaml @@ -0,0 +1,42 @@ +# Optional Prometheus + Grafana monitoring overlay for DigiScript. +# +# export GF_SECURITY_ADMIN_PASSWORD='choose-a-strong-password' +# docker compose -f docker-compose.yaml -f docker-compose.monitoring.yaml up -d +# +# Published ports are bound to 127.0.0.1 only. To expose them on other +# interfaces set PROMETHEUS_BIND / GRAFANA_BIND (e.g. GRAFANA_BIND=0.0.0.0), +# and put a reverse proxy with TLS/auth in front of them. +# +# The scrape path / auth for the DigiScript metrics endpoint lives in ONE place: +# monitoring/prometheus/prometheus.yml +services: + prometheus: + image: prom/prometheus:v3.15.1 + restart: unless-stopped + ports: + - "${PROMETHEUS_BIND:-127.0.0.1}:9090:9090" + volumes: + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./monitoring/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro + - prometheus_data:/prometheus + depends_on: + - server + + grafana: + image: grafana/grafana:13.2.4 + restart: unless-stopped + ports: + - "${GRAFANA_BIND:-127.0.0.1}:3000:3000" + environment: + GF_SECURITY_ADMIN_PASSWORD: "${GF_SECURITY_ADMIN_PASSWORD:?Set GF_SECURITY_ADMIN_PASSWORD to a strong Grafana admin password}" + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - grafana_data:/var/lib/grafana + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - prometheus + +volumes: + prometheus_data: + grafana_data: diff --git a/documentation/deployment.md b/documentation/deployment.md index cca564aa..6dde49cf 100644 --- a/documentation/deployment.md +++ b/documentation/deployment.md @@ -20,6 +20,120 @@ docker build -t digiscript:latest . docker-compose up -d ``` +`docker-compose up -d` starts the DigiScript server only. For metrics dashboards see +[Monitoring](#monitoring) below. + +### Monitoring + +An optional compose overlay adds Prometheus and Grafana, pre-provisioned with a +"DigiScript Overview" dashboard and basic alert rules: + +```shell +export GF_SECURITY_ADMIN_PASSWORD='choose-a-strong-password' +docker compose -f docker-compose.yaml -f docker-compose.monitoring.yaml up -d +``` + +`GF_SECURITY_ADMIN_PASSWORD` is required (there is no default); compose refuses to +start without it. + +| Service | URL | Notes | +|------------|-------------------------|-----------------------------------------| +| Grafana | `http://localhost:3000` | Log in as `admin` with the password set | +| Prometheus | `http://localhost:9090` | Scrapes `server:8080/debug/metrics` | + +Both ports are published on `127.0.0.1` only. To expose them set `GRAFANA_BIND` / +`PROMETHEUS_BIND` (e.g. `GRAFANA_BIND=0.0.0.0`) and put a TLS-terminating reverse proxy +in front. The scrape path is configured in `monitoring/prometheus/prometheus.yml`. + +**Security:** `/debug/metrics` is currently unauthenticated. Do not expose DigiScript's +port 8080 to the public internet without a reverse proxy that restricts this path. + +Application logs can be viewed in-app via the log viewer. + +#### Health check + +`GET /api/v1/health` is an unauthenticated liveness/readiness probe, suitable for load +balancer or container health checks. The Docker image uses it for its built-in +`HEALTHCHECK` (every 30s, 5s timeout, 30s start period, 3 retries). Requests to it are +not written to the access log. + +Database check: `SELECT 1 FROM sqlite_master LIMIT 1` (reads the database file +header/schema). Probes run single-flight on a dedicated one-worker executor: overlapping +polls await the same in-flight probe, and every caller is bounded by a 3s timeout. While a +probe is genuinely hung, each poll waits up to 3s and then reports `fail`; no second probe +starts until the first finishes, so a stuck database cannot pile up threads. + +Storage check: the parent directories of `log_path`, `db_log_path` and `client_log_path`, +plus `draft_script_path` and `compiled_script_path`, must be writable. A directory that +does not exist yet is judged by its nearest existing ancestor. + +| Condition | HTTP status | `status` | +|---|---|---| +| Database and storage checks pass | `200` | `ok` | +| Either check fails or the database times out | `503` | `degraded` | +| Unexpected internal error | `503` | `degraded` (`checks` are `"unknown"`, `uptime_seconds` omitted) | + +Example healthy response: + +```json +{ + "version": "0.35.2", + "status": "ok", + "checks": { "database": "ok", "storage": "ok" }, + "uptime_seconds": 1234 +} +``` + +A failing check is reported as `"fail"` in `checks`. The response never includes error +text or file paths; the detail is written to the server log instead (`ERROR` for the +database, `WARNING` for storage), so look there to find out why a probe is failing. + +The Docker `HEALTHCHECK` calls `http://localhost:8080/api/v1/health` with Python's +`urllib`; any non-2xx response marks the container unhealthy. The port is hard-wired to +8080 because the app has no environment variable for its port. If you change the +listening port with the `--port` flag, override the check with `--health-cmd` or a +compose `healthcheck:` block that targets the same container port. For example, to +listen on 8081 (published on host port 8081): + +```yaml +services: + server: + command: ["--port=8081"] + ports: + - 8081:8081 + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8081/api/v1/health', timeout=4)"] + interval: 30s + timeout: 5s + start_period: 30s + retries: 3 +``` + +If you also use the monitoring overlay, change the scrape target in +`monitoring/prometheus/prometheus.yml` to `server:8081` too. Avoid host ports 3000 and +9090, which the overlay uses for Grafana and Prometheus. + +#### Metrics reference + +| Metric | Type | Meaning | +|--------|------|---------| +| `python_*`, `process_*` | various | Python runtime and process stats | +| `tornado_http_requests_total` | counter | HTTP requests by method, handler and status | +| `tornado_http_request_duration_seconds` | histogram | HTTP request latency | +| `digiscript_http_request_body_bytes` | histogram | Size of HTTP request bodies, by handler and method | +| `digiscript_ws_message_bytes` | histogram | Size of inbound WebSocket messages | +| `digiscript_ws_connections` | gauge | Open WebSocket connections | +| `digiscript_ws_messages_total{op}` | counter | WebSocket messages by operation | +| `digiscript_collab_rooms` | gauge | Active collaborative editing rooms | +| `digiscript_show_session_active` | gauge | 1 if a live show session is running, else 0 | +| `digiscript_info{version}` | gauge | Always 1; carries the server version label | +| `digiscript_auth_events_total{event,result}` | counter | Authentication events by type and result | +| `digiscript_unhandled_errors_total{source}` | counter | Unhandled errors by source | + +Note: older releases do not expose all of these metrics (for example the `tornado_http_*` +series and the newer `digiscript_*` gauges and counters). Dashboard panels for metrics +your server version does not expose show "No data". + ### Pre-built Server Executables Pre-built server executables are available from [GitHub Releases](https://github.com/dreamteamprod/DigiScript/releases): diff --git a/monitoring/grafana/dashboards/digiscript.json b/monitoring/grafana/dashboards/digiscript.json new file mode 100644 index 00000000..46998f8d --- /dev/null +++ b/monitoring/grafana/dashboards/digiscript.json @@ -0,0 +1,575 @@ +{ + "uid": "digiscript-overview", + "title": "DigiScript Overview", + "tags": [ + "digiscript" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Overview", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Version", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "max by (version) (digiscript_info)", + "legendFormat": "{{version}}" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "name" + } + }, + { + "id": 3, + "type": "stat", + "title": "Show session active", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "max(digiscript_show_session_active)", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "No" + }, + "1": { + "text": "Yes" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 4, + "type": "stat", + "title": "WebSocket connections", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "sum(digiscript_ws_connections)", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 5, + "type": "stat", + "title": "Collab rooms", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "sum(digiscript_collab_rooms)", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 6, + "type": "row", + "title": "HTTP", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "panels": [] + }, + { + "id": 7, + "type": "timeseries", + "title": "Request rate by status class", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "sum by (status) (rate(tornado_http_requests_total{handler!=\"WebSocketController\"}[5m]))", + "legendFormat": "{{status}}" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data", + "unit": "reqps" + }, + "overrides": [] + }, + "options": {}, + "description": "HTTP requests per second by status class. WebSocket upgrades (101) are excluded." + }, + { + "id": 8, + "type": "timeseries", + "title": "Request latency p50 / p95", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(tornado_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(tornado_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data", + "unit": "s" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 9, + "type": "timeseries", + "title": "Request body size p50 / p95", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(digiscript_http_request_body_bytes_bucket[5m])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(digiscript_http_request_body_bytes_bucket[5m])))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data", + "unit": "bytes" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 10, + "type": "timeseries", + "title": "WebSocket messages by op", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "sum by (op) (rate(digiscript_ws_messages_total[5m]))", + "legendFormat": "{{op}}" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data", + "unit": "ops" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 11, + "type": "timeseries", + "title": "WebSocket message size p95", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(digiscript_ws_message_bytes_bucket[5m])))", + "legendFormat": "p95" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data", + "unit": "bytes" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 12, + "type": "row", + "title": "Auth", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "panels": [] + }, + { + "id": 13, + "type": "timeseries", + "title": "Auth events by result", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "sum by (event, result) (increase(digiscript_auth_events_total[5m]))", + "legendFormat": "{{event}} / {{result}}" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 14, + "type": "row", + "title": "Health", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "panels": [] + }, + { + "id": 15, + "type": "stat", + "title": "Server up", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 32 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "up{job=\"digiscript\"}", + "legendFormat": "" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 16, + "type": "timeseries", + "title": "Unhandled errors by source", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "gridPos": { + "h": 7, + "w": 18, + "x": 6, + "y": 32 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "digiscript-prometheus" + }, + "expr": "sum by (source) (increase(digiscript_unhandled_errors_total[5m])) or sum by (source) (digiscript_unhandled_errors_total > 0 unless digiscript_unhandled_errors_total offset 5m)", + "legendFormat": "{{source}}" + } + ], + "fieldConfig": { + "defaults": { + "noValue": "No data" + }, + "overrides": [] + }, + "options": {} + } + ], + "templating": { + "list": [] + }, + "annotations": { + "list": [] + } +} diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 00000000..209117da --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,7 @@ +apiVersion: 1 +providers: + - name: DigiScript + type: file + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..48d8ba37 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,9 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: digiscript-prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/monitoring/prometheus/alerts.yml b/monitoring/prometheus/alerts.yml new file mode 100644 index 00000000..ca87d02e --- /dev/null +++ b/monitoring/prometheus/alerts.yml @@ -0,0 +1,50 @@ +groups: + - name: digiscript + rules: + - alert: DigiScriptDown + expr: up{job="digiscript"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: DigiScript server is not being scraped successfully + - alert: DigiScriptHighLatencyP95 + expr: histogram_quantile(0.95, sum by (le) (rate(tornado_http_request_duration_seconds_bucket[5m]))) > 1 + for: 5m + labels: + severity: warning + annotations: + summary: p95 HTTP request latency above 1s for 5m + # tornado_prometheus.classify_status_code buckets every status outside + # 200-499 as "5xx", including the 101 returned by a WebSocket upgrade. The + # server-side fix keeps 101 out of the series; as defence in depth the + # WebSocket handler is also excluded here (numerator and denominator) so + # normal WS connects/reconnects can never trip this alert. + - alert: DigiScriptHigh5xxRate + expr: >- + sum(rate(tornado_http_requests_total{status="5xx", handler!="WebSocketController"}[5m])) + / sum(rate(tornado_http_requests_total{handler!="WebSocketController"}[5m])) > 0.05 + for: 5m + labels: + severity: warning + annotations: + summary: More than 5% of HTTP requests are returning 5xx + # increase() yields nothing for a series that first appears inside the + # window, so a first-ever error for a source would be missed if the series + # is created lazily. The second branch catches that: a series that is + # non-zero now but did not exist 5m ago. (The server also pre-registers the + # series at 0, in which case the first branch alone suffices.) + - alert: DigiScriptUnhandledErrors + expr: >- + increase(digiscript_unhandled_errors_total[5m]) > 0 + or (digiscript_unhandled_errors_total > 0 unless digiscript_unhandled_errors_total offset 5m) + labels: + severity: warning + annotations: + summary: Unhandled errors raised in the last 5m + - alert: DigiScriptAuthFailureSpike + expr: sum(increase(digiscript_auth_events_total{result="failure"}[5m])) > 20 + labels: + severity: warning + annotations: + summary: More than 20 failed auth events in 5m diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 00000000..ef6dc80a --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,15 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/alerts.yml + +scrape_configs: + - job_name: digiscript + # Scrape path and any auth for the DigiScript metrics endpoint: change here only. + # Currently unauthenticated; if it is gated later add e.g. + # authorization: { credentials_file: /etc/prometheus/token } + metrics_path: /debug/metrics + static_configs: + - targets: ["server:8080"]