diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..3ba22b5 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,45 @@ +# Build and publish the coven-github container image to GHCR. +# Pushes to main publish `latest` + the commit SHA; version tags publish the +# semver tag. deploy/hosted/compose.yaml pins deployments to these images. +name: publish + +on: + push: + branches: [main] + tags: ["v*"] + +permissions: + contents: read + packages: write + +jobs: + image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/opencoven/coven-github + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=sha,format=long + type=semver,pattern={{version}} + + - uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Dockerfile b/Dockerfile index 6a30433..79b544e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,8 +23,9 @@ RUN cargo build --release --locked -p coven-github FROM debian:bookworm-slim AS runtime # ca-certificates: TLS to api.github.com. git: clone target repos. +# curl: container healthchecks against /healthz (deploy/hosted/compose.yaml). RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates git \ + && apt-get install -y --no-install-recommends ca-certificates git curl \ && rm -rf /var/lib/apt/lists/* # Run as an unprivileged user. Secrets are mounted read-only at runtime, never diff --git a/HOSTED.md b/HOSTED.md index bdbae45..282b21a 100644 --- a/HOSTED.md +++ b/HOSTED.md @@ -88,6 +88,11 @@ Hosted beta should wait for: 5. Usage metering by installation, repo, familiar, and task. 6. Cave oversight dashboard for task history and human intervention. +All six code gates are shipped. The operational side — provisioning, TLS +ingress, continuous store backup, monitoring, restore drills, upgrades — is +the [hosted deployment runbook](docs/hosted-deploy.md) with its stack in +[`deploy/hosted/`](deploy/hosted/). + ## Landing Page Copy Headline: diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index d164aa3..dc31cba 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -12,7 +12,8 @@ use tracing_subscriber::EnvFilter; use coven_github_config::Config; use coven_github_webhook::routes::{ - audit, handle_webhook, list_memory, list_tasks, revoke_memory, routing, usage, AppState, + audit, handle_webhook, healthz, list_memory, list_tasks, revoke_memory, routing, usage, + AppState, }; use coven_github_worker as worker; @@ -172,6 +173,7 @@ async fn main() -> Result<()> { }; let app = Router::new() + .route("/healthz", get(healthz)) .route("/webhook", post(handle_webhook)) .route("/api/github/tasks", get(list_tasks)) .route("/api/github/memory", get(list_memory)) diff --git a/crates/webhook/src/routes.rs b/crates/webhook/src/routes.rs index 6fba2e9..85af861 100644 --- a/crates/webhook/src/routes.rs +++ b/crates/webhook/src/routes.rs @@ -43,6 +43,23 @@ fn familiar_names(config: &Config) -> std::collections::HashMap .collect() } +/// GET /healthz — liveness/readiness for ingress and uptime checks. +/// Unauthenticated by design, so it must never expose tenant data: the body +/// is only `ok` plus whether the durable store answers a trivial read. +pub async fn healthz(State(state): State) -> impl IntoResponse { + match state.store.delivery_routing("__healthz__").await { + Ok(_) => (StatusCode::OK, Json(json!({"ok": true}))).into_response(), + Err(e) => { + error!("healthz store probe failed: {e:#}"); + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({"ok": false})), + ) + .into_response() + } + } +} + /// GET /api/github/tasks — task state for CovenCave polling, behind the /// tenant boundary (issue #3). `token` mode fails closed; a tenant token sees /// only its own installation (optionally narrowed to repositories); the @@ -2996,6 +3013,25 @@ mod metering_route_tests { } } +#[cfg(test)] +mod healthz_tests { + use super::tests::app_state; + use super::*; + use axum::extract::State; + + #[tokio::test] + async fn healthz_answers_ok_and_leaks_nothing() { + let state = app_state(); + let response = healthz(State(state)).await.into_response(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("body"); + let json: serde_json::Value = serde_json::from_slice(&bytes).expect("json"); + assert_eq!(json, serde_json::json!({"ok": true})); + } +} + #[cfg(test)] mod billing_route_tests { //! Marketplace plan intake: entitlement recording, plan-derived limits, diff --git a/deploy/coven-github/__pycache__/coven_github_adapter.cpython-314.pyc b/deploy/coven-github/__pycache__/coven_github_adapter.cpython-314.pyc new file mode 100644 index 0000000..675ce3c Binary files /dev/null and b/deploy/coven-github/__pycache__/coven_github_adapter.cpython-314.pyc differ diff --git a/deploy/coven-github/__pycache__/test_coven_github_adapter.cpython-314.pyc b/deploy/coven-github/__pycache__/test_coven_github_adapter.cpython-314.pyc new file mode 100644 index 0000000..3a5d4e8 Binary files /dev/null and b/deploy/coven-github/__pycache__/test_coven_github_adapter.cpython-314.pyc differ diff --git a/deploy/hosted/.env.example b/deploy/hosted/.env.example new file mode 100644 index 0000000..3617652 --- /dev/null +++ b/deploy/hosted/.env.example @@ -0,0 +1,20 @@ +# Secrets for deploy/hosted/compose.yaml. Copy to .env (mode 0600) and fill +# in. NEVER commit the filled-in file. + +# Public hostname the GitHub App webhook points at. +WEBHOOK_DOMAIN=gh.opencoven.ai + +# Image tag to run (a released ghcr.io/opencoven/coven-github tag). +COVEN_GITHUB_TAG=latest + +# Cloudflare R2 (S3-compatible) for Litestream SQLite replication. +# Create an R2 API token scoped to one bucket with object read+write. +R2_ACCOUNT_ID= +R2_BUCKET=coven-github-store +R2_ACCESS_KEY_ID= +R2_SECRET_ACCESS_KEY= + +# Host docker group id, so the unprivileged adapter user can drive the +# daemon for the container worker backend. provision.sh fills this in +# (getent group docker | cut -d: -f3). +DOCKER_GID= diff --git a/deploy/hosted/Caddyfile b/deploy/hosted/Caddyfile new file mode 100644 index 0000000..b55885e --- /dev/null +++ b/deploy/hosted/Caddyfile @@ -0,0 +1,13 @@ +# TLS ingress for hosted coven-github. Caddy provisions and renews +# Let's Encrypt certificates automatically for $WEBHOOK_DOMAIN. +{$WEBHOOK_DOMAIN} { + encode gzip + + # GitHub webhook deliveries and the tenant-scoped API. + reverse_proxy coven-github:3000 + + log { + output stdout + format console + } +} diff --git a/deploy/hosted/compose.yaml b/deploy/hosted/compose.yaml new file mode 100644 index 0000000..36a428b --- /dev/null +++ b/deploy/hosted/compose.yaml @@ -0,0 +1,77 @@ +# Production Docker Compose for hosted coven-github: adapter + TLS ingress +# (Caddy) + continuous SQLite replication (Litestream → Cloudflare R2). +# +# Runbook: docs/hosted-deploy.md. Secrets come from ./.env (never committed); +# copy .env.example and fill it in. +# +# Layout on the host (created by provision.sh): +# /opt/coven-github/ +# compose.yaml this file +# .env secrets (mode 0600) +# Caddyfile TLS ingress +# litestream.yml replication config +# config/production.toml adapter config +# keys/app.pem GitHub App private key (mode 0600) +# data/ SQLite store (replicated) + +services: + coven-github: + image: ghcr.io/opencoven/coven-github:${COVEN_GITHUB_TAG:-latest} + command: ["serve", "--config", "/config/production.toml"] + restart: unless-stopped + expose: + - "3000" + volumes: + - ./config:/config:ro + - ./keys:/keys:ro + - ./data:/data + - ./workspaces:/tmp/coven-github-tasks + # Container worker backend: the adapter launches per-task hardened + # containers through the host Docker daemon (docs/container-isolation.md). + - /var/run/docker.sock:/var/run/docker.sock + # The image runs unprivileged (uid 10001); grant the host docker group so + # the worker can drive the daemon. provision.sh fills DOCKER_GID in .env. + group_add: + - "${DOCKER_GID:?set in .env (getent group docker | cut -d: -f3)}" + environment: + RUST_LOG: "coven_github=info" + healthcheck: + test: ["CMD", "curl", "-fsS", "http://localhost:3000/healthz"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + + caddy: + image: caddy:2 + restart: unless-stopped + ports: + - "80:80" + - "443:443" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - caddy-data:/data + - caddy-config:/config + environment: + WEBHOOK_DOMAIN: ${WEBHOOK_DOMAIN:?set in .env, e.g. gh.opencoven.ai} + depends_on: + - coven-github + + litestream: + image: litestream/litestream:0.3 + restart: unless-stopped + command: ["replicate", "-config", "/etc/litestream.yml"] + volumes: + - ./litestream.yml:/etc/litestream.yml:ro + - ./data:/data + environment: + LITESTREAM_ACCESS_KEY_ID: ${R2_ACCESS_KEY_ID:?set in .env} + LITESTREAM_SECRET_ACCESS_KEY: ${R2_SECRET_ACCESS_KEY:?set in .env} + R2_BUCKET: ${R2_BUCKET:?set in .env} + R2_ACCOUNT_ID: ${R2_ACCOUNT_ID:?set in .env} + depends_on: + - coven-github + +volumes: + caddy-data: + caddy-config: diff --git a/deploy/hosted/litestream.yml b/deploy/hosted/litestream.yml new file mode 100644 index 0000000..5f79ac1 --- /dev/null +++ b/deploy/hosted/litestream.yml @@ -0,0 +1,14 @@ +# Litestream: continuous SQLite replication to Cloudflare R2 (S3-compatible). +# Credentials come from the environment (compose.yaml). Restore drill: +# litestream restore -config /etc/litestream.yml /data/coven-github.db +dbs: + - path: /data/coven-github.db + replicas: + - type: s3 + bucket: ${R2_BUCKET} + path: coven-github + endpoint: https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com + # R2 ignores regions but the S3 client requires one. + region: auto + retention: 720h # 30 days of point-in-time history + sync-interval: 10s diff --git a/deploy/hosted/provision.sh b/deploy/hosted/provision.sh new file mode 100755 index 0000000..e08820a --- /dev/null +++ b/deploy/hosted/provision.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Provision a fresh Ubuntu 24.04 host (e.g. Hetzner CPX41) to run hosted +# coven-github. Idempotent; run as root ON THE SERVER: +# +# curl -fsSL https://raw.githubusercontent.com/OpenCoven/coven-github/main/deploy/hosted/provision.sh | bash +# +# Then follow docs/hosted-deploy.md to place config, keys, and .env. +set -euo pipefail + +DEPLOY_DIR=/opt/coven-github +RAW=https://raw.githubusercontent.com/OpenCoven/coven-github/main/deploy/hosted + +echo "==> Installing Docker Engine + compose plugin" +if ! command -v docker >/dev/null 2>&1; then + curl -fsSL https://get.docker.com | sh +fi + +echo "==> Hardening: unattended upgrades + firewall (22/80/443 only)" +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y -qq unattended-upgrades ufw curl +ufw default deny incoming +ufw default allow outgoing +ufw allow 22/tcp +ufw allow 80/tcp +ufw allow 443/tcp +ufw --force enable + +echo "==> Laying out ${DEPLOY_DIR}" +mkdir -p "${DEPLOY_DIR}"/{config,keys,data,workspaces} +chmod 700 "${DEPLOY_DIR}/keys" +# The image runs as uid 10001 (user `coven`); it owns its state dirs. +chown 10001 "${DEPLOY_DIR}/data" "${DEPLOY_DIR}/workspaces" +cd "${DEPLOY_DIR}" +for f in compose.yaml Caddyfile litestream.yml .env.example; do + curl -fsSL "${RAW}/${f}" -o "${f}" +done +[ -f .env ] || { cp .env.example .env && chmod 600 .env; } + +echo "==> Recording the docker group id for the compose worker" +DOCKER_GID="$(getent group docker | cut -d: -f3)" +grep -q '^DOCKER_GID=' .env \ + && sed -i "s/^DOCKER_GID=.*/DOCKER_GID=${DOCKER_GID}/" .env \ + || echo "DOCKER_GID=${DOCKER_GID}" >> .env + +cat <<'NEXT' + +Provisioned. Next (docs/hosted-deploy.md): + 1. Fill /opt/coven-github/.env (domain, image tag, R2 credentials) + 2. Place /opt/coven-github/config/production.toml and keys/app.pem (0600) + 3. Point DNS A/AAAA for your webhook domain at this host + 4. docker compose run --rm coven-github doctor --config /config/production.toml + 5. docker compose up -d + 6. curl -fsS https:///healthz +NEXT diff --git a/docs/hosted-deploy.md b/docs/hosted-deploy.md new file mode 100644 index 0000000..dfef646 --- /dev/null +++ b/docs/hosted-deploy.md @@ -0,0 +1,117 @@ +# Hosted deployment runbook + +How the hosted OpenCoven GitHub adapter runs in production: one VM, Docker +Compose, Caddy TLS ingress, and continuous SQLite replication to Cloudflare +R2. This is the operational half of the [HOSTED.md](../HOSTED.md) beta gates; +the code half (durable queue, isolation, metering, entitlements) is shipped. + +**Architecture choice:** the store is single-writer SQLite and the container +worker backend drives the host Docker daemon, so the natural unit is one +well-sized VM (Hetzner CPX41-class: 8 vCPU / 16 GB). Tier task caps keep beta +load well inside that. Hosted Dedicated later = this same stack on a +customer-dedicated VM. + +Everything lives in [`deploy/hosted/`](../deploy/hosted/): + +| File | Role | +|---|---| +| `provision.sh` | One-shot Ubuntu 24.04 host setup (Docker, firewall, layout) | +| `compose.yaml` | Adapter + Caddy + Litestream services | +| `Caddyfile` | TLS ingress for the webhook domain | +| `litestream.yml` | SQLite → R2 continuous replication | +| `.env.example` | Secrets template (domain, image tag, R2 credentials) | + +## First deploy + +1. **VM**: create an Ubuntu 24.04 host (Hetzner CPX41 or similar). Point DNS + `A`/`AAAA` for the webhook domain (e.g. `gh.opencoven.ai`) at it. +2. **Provision** (as root on the VM): + ```sh + curl -fsSL https://raw.githubusercontent.com/OpenCoven/coven-github/main/deploy/hosted/provision.sh | bash + ``` + Installs Docker, enables unattended upgrades, opens only 22/80/443, and + lays out `/opt/coven-github`. +3. **Secrets** (never in git): + - Fill `/opt/coven-github/.env` — domain, image tag, R2 token (create an + R2 bucket + API token scoped to it first). + - Place the GitHub App PEM at `keys/app.pem`, mode `0600`. + - Write `config/production.toml` from [`config/example.toml`](../config/example.toml): + `storage.path = "/data/coven-github.db"`, `worker.backend = "container"`, + `api.mode = "token"` with tenant tokens, `[billing] require_plan = true` + at Marketplace go-live (see + [marketplace-listing.md](marketplace-listing.md)), and `[[installations]]` + entries for grandfathered beta tenants. +4. **Preflight + start**: + ```sh + cd /opt/coven-github + docker compose run --rm coven-github doctor --config /config/production.toml + docker compose up -d + curl -fsS https://gh.opencoven.ai/healthz + ``` +5. **Wire GitHub**: set the App's webhook URL to + `https://gh.opencoven.ai/webhook` and send a ping delivery — the adapter + answers `pong` and records the delivery. + +The image is published by [`.github/workflows/publish.yml`](../.github/workflows/publish.yml) +to `ghcr.io/opencoven/coven-github` (`latest` + SHA on main, semver on tags). + +## Monitoring + +- **Liveness/readiness**: `GET /healthz` — 200 when the store answers, 503 + otherwise. Point an external uptime monitor at it (the compose healthcheck + restarts the container; the external check catches host/DNS/TLS failure). +- **Queue and task health**: the tenant API is the dashboard — + `/api/github/tasks` (states), `/api/github/usage` (per-installation load), + `/api/github/audit` (ignored deliveries, attempt outcomes). A growing + `queued` count with idle workers, or repeated `attempt:failed` audit rows, + is the primary alert condition. +- **Logs**: `docker compose logs -f coven-github` (`RUST_LOG` in + compose.yaml). Secrets are redacted by the adapter. +- **Replication**: `docker compose exec litestream litestream dbs -config + /etc/litestream.yml` should list the db; alert if the R2 bucket's latest + WAL segment is older than a few minutes. + +## Backup and restore + +Litestream streams every WAL segment to R2 (10 s sync, 30-day retention). +**Run a restore drill before onboarding paying tenants**, and quarterly: + +```sh +docker compose stop coven-github +docker compose run --rm litestream restore -if-replica-exists \ + -config /etc/litestream.yml -o /data/restored.db /data/coven-github.db +# inspect, then swap in and restart +``` + +Losing this database loses delivery idempotency, task history, usage +metering, and **billing entitlements** — treat replication health as a +page-worthy alert. + +## Upgrades + +```sh +cd /opt/coven-github +docker compose pull coven-github +docker compose up -d coven-github +curl -fsS https://gh.opencoven.ai/healthz +``` + +Schema migrations run forward-only at boot. GitHub redelivers webhooks that +5xx during the restart window, and the delivery-id dedup makes redelivery +safe. Pin `COVEN_GITHUB_TAG` to a SHA or semver tag for controlled rollouts; +roll back by re-pinning the previous tag (schema rollbacks are not supported +— restore from R2 if a migration must be undone). + +## Incident basics + +- **Webhook 5xx / down**: GitHub retries; fix the host, deliveries catch up. + Check `docker compose ps`, `/healthz`, disk space (`df -h /opt`). +- **Stuck task**: find it via `/api/github/tasks`, inspect + `/api/github/audit`; the worker enforces timeouts and cleans workspaces — + a task that outlives its timeout is a bug, capture logs before restarting. +- **Runaway load from one tenant**: intake caps and claim-time concurrency + already gate per installation; drop the tenant's tier or set explicit + `[installations.limits]` and restart. +- **Compromise suspected**: rotate the webhook secret and App PEM in GitHub, + replace `keys/app.pem` and the secret in `production.toml`, restart, and + audit `/api/github/audit` for the exposure window.