From 1a1557d13dd70c8cee06337cc2dc0cfb93be6b44 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 02:44:18 -0500 Subject: [PATCH 01/20] Plan Tenant Wrapper subprojects --- .cass/playbook.yaml | 12 + .env.example | 44 +++ README.md | 154 ++++++--- configs/certs/.gitkeep | 0 configs/nginx/default.conf.template | 92 ++++++ configs/oauth2-proxy/credentials/.gitkeep | 0 .../oauth2-proxy/google-workspace.example.cfg | 20 ++ configs/oauth2-proxy/keycloak.example.cfg | 18 ++ configs/oauth2-proxy/oauth2-proxy.cfg | 33 ++ docker-compose.yml | 58 +++- docker/proxy/Dockerfile | 11 + docker/proxy/entrypoint.sh | 23 ++ docs/api.md | 304 +++++++----------- docs/architecture.md | 163 ++++------ docs/configuration.md | 95 ++++-- docs/deployment.md | 184 +++++------ docs/security.md | 135 ++++---- pyproject.toml | 1 + src/octw/api/app.py | 22 +- src/octw/api/app_logic.py | 128 ++++++++ src/octw/api/auth.py | 84 ++++- src/octw/api/routers/app_router.py | 142 ++++++++ src/octw/api/routers/auth_router.py | 11 +- src/octw/api/routers/internal_router.py | 15 +- src/octw/api/routers/provision_router.py | 97 ++---- src/octw/common/config.py | 19 ++ src/octw/db/engine.py | 39 ++- src/octw/db/tables.py | 5 + src/octw/edge/proxy.py | 176 ++++++++-- src/octw/models/tenant.py | 10 + src/octw/orchestrator/docker_orch.py | 258 +++++++++++---- src/octw/orchestrator/tenant_service.py | 73 ++++- src/octw/web/static/app.css | 122 +++++++ src/octw/web/static/app.js | 233 ++++++++++++++ src/octw/web/static/index.html | 40 +++ tests/unit/test_app_v1.py | 165 ++++++++++ uv.lock | 2 + 37 files changed, 2264 insertions(+), 724 deletions(-) create mode 100644 .cass/playbook.yaml create mode 100644 .env.example create mode 100644 configs/certs/.gitkeep create mode 100644 configs/nginx/default.conf.template create mode 100644 configs/oauth2-proxy/credentials/.gitkeep create mode 100644 configs/oauth2-proxy/google-workspace.example.cfg create mode 100644 configs/oauth2-proxy/keycloak.example.cfg create mode 100644 configs/oauth2-proxy/oauth2-proxy.cfg create mode 100644 docker/proxy/Dockerfile create mode 100644 docker/proxy/entrypoint.sh create mode 100644 src/octw/api/app_logic.py create mode 100644 src/octw/api/routers/app_router.py create mode 100644 src/octw/web/static/app.css create mode 100644 src/octw/web/static/app.js create mode 100644 src/octw/web/static/index.html create mode 100644 tests/unit/test_app_v1.py diff --git a/.cass/playbook.yaml b/.cass/playbook.yaml new file mode 100644 index 0000000..25ebb9a --- /dev/null +++ b/.cass/playbook.yaml @@ -0,0 +1,12 @@ +# Project-specific playbook rules +# These are merged with your global ~/.cass-memory/playbook.yaml +# Project rules take precedence over global rules +schema_version: 2 +name: repo-playbook +description: Project-specific rules for this repository +metadata: + createdAt: 2026-03-09T04:30:43.156Z + totalReflections: 0 + totalSessionsProcessed: 0 +deprecatedPatterns: [] +bullets: [] diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5f9ad99 --- /dev/null +++ b/.env.example @@ -0,0 +1,44 @@ +# Core control-plane settings +OCTW_DB_URL=postgresql+asyncpg://octw:octw@octw-db:5432/octw +OCTW_REDIS_URL=redis://octw-cache:6379/0 +OCTW_PUBLIC_BASE_URL=https://octw.example.com +OCTW_SERVER_NAME=octw.example.com +OCTW_EDGE_DOMAIN=octw.example.com +OCTW_API_INTERNAL_BASE_URL=http://octw-api:8000 +OCTW_TRUSTED_PROXY_ENABLED=true +OCTW_TRUSTED_PROXY_USER_HEADER=X-Forwarded-Email +OCTW_TRUSTED_PROXY_IPS=172.30.0.0/24 +OCTW_DEFAULT_PROVIDER=zai + +# Secrets +# Generate with: python3 -c "import secrets; print(secrets.token_urlsafe(32))" +OCTW_JWT_SECRET=replace-with-strong-random-token +# Generate with: python3 -c "import os; print(os.urandom(32).hex())" +OCTW_KEK=replace-with-64-hex-character-kek + +# Shared provider API keys (set at least one) +OCTW_ZAI_API_KEY= +OCTW_MOONSHOT_API_KEY= +OCTW_MINIMAX_API_KEY= + +# oauth2-proxy base settings +# Valid providers include: oidc, keycloak-oidc, google +OCTW_OIDC_PROVIDER=oidc +# Config file under configs/oauth2-proxy/ used by the oauth2-proxy container. +OCTW_OIDC_CONFIG_FILE=oauth2-proxy.cfg +# Required for generic OIDC and keycloak-oidc. Leave blank for google. +OCTW_OIDC_ISSUER_URL=https://issuer.example.com/realms/octw +OCTW_OIDC_CLIENT_ID=octw +OCTW_OIDC_CLIENT_SECRET=replace-with-oidc-client-secret +# Generate with: python3 -c "import base64, os; print(base64.urlsafe_b64encode(os.urandom(32)).decode().rstrip('='))" +OCTW_OIDC_COOKIE_SECRET=replace-with-base64-cookie-secret +# Restrict login to your workspace domain. Use * only for isolated test environments. +OCTW_OIDC_EMAIL_DOMAINS=example.com +# Allow oauth2-proxy redirects back to this host. Prefix subdomains with a dot. +OCTW_OIDC_WHITELIST_DOMAINS=.example.com + +# Provider-specific examples live in: +# configs/oauth2-proxy/keycloak.example.cfg +# configs/oauth2-proxy/google-workspace.example.cfg +# For Google Workspace group checks, place the Admin SDK service account JSON at: +# configs/oauth2-proxy/credentials/google-admin-sdk.json diff --git a/README.md b/README.md index c47ab57..150ef1f 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,128 @@ # OCTW — OpenClaw Tenant Wrapper -OCTW is a multi-tenant abstraction layer that provisions and operates multiple independent [OpenClaw](https://github.com/openclaw/openclaw) installations on a single host. Each tenant gets its own isolated OpenClaw gateway, network, filesystem, credentials, and webchat — using the upstream container image without forking. +OCTW provisions and operates multiple isolated [OpenClaw](https://github.com/openclaw/openclaw) installations on one host. This branch adds a browser-first app flow: reverse-proxy OIDC login, one-click tenant deployment, verified readiness, wake-on-resume, and a minimal OCTW-hosted chat UI. ## Quick Start ### Prerequisites -- Docker (with compose plugin) -- [uv](https://docs.astral.sh/uv/) (for CLI and development) +- Docker Engine with the Compose plugin +- [uv](https://docs.astral.sh/uv/) for local development and tests +- At least one supported model provider API key +- A public DNS name that points to the VM +- OIDC client credentials for your IdP -### 1. Clone and prepare the host +### 1. Prepare the host ```bash -git clone https://github.com/kumanday/OCTW.git && cd OCTW +git clone https://github.com/kumanday/OCTW.git +cd OCTW sudo mkdir -p /var/lib/octw /etc/octw -sudo chown $USER /var/lib/octw +sudo chown "$USER" /var/lib/octw sudo python3 -c "import os; open('/etc/octw/master.key','wb').write(os.urandom(32))" -sudo chown $USER /etc/octw/master.key +sudo chown "$USER" /etc/octw/master.key sudo chmod 600 /etc/octw/master.key ``` -### 2. Configure provider API keys +### 2. Create `.env` -Set one or more LLM provider keys. At least one is required: +```bash +cp .env.example .env +``` + +Fill in these values at minimum: + +- `OCTW_KEK` +- `OCTW_JWT_SECRET` +- `OCTW_PUBLIC_BASE_URL` +- `OCTW_SERVER_NAME` +- `OCTW_TRUSTED_PROXY_ENABLED=true` +- one or more provider API keys such as `OCTW_ZAI_API_KEY` +- `OCTW_OIDC_CLIENT_ID` +- `OCTW_OIDC_CLIENT_SECRET` +- `OCTW_OIDC_COOKIE_SECRET` + +Useful generators: ```bash -export OCTW_ZAI_API_KEY="your-zai-key" # Z.ai GLM Coding Plan -export OCTW_MOONSHOT_API_KEY="your-moonshot-key" # Moonshot AI Kimi Coding Plan -export OCTW_MINIMAX_API_KEY="your-minimax-key" # MiniMax Coding Plan +python3 -c "import os; print(os.urandom(32).hex())" +python3 -c "import secrets; print(secrets.token_urlsafe(32))" +python3 -c "import base64, os; print(base64.urlsafe_b64encode(os.urandom(32)).decode().rstrip('='))" +``` + +### 3. Pick an OIDC provider profile + +The stack now includes both public ingress and OIDC inside Docker: + +- `octw-proxy` terminates HTTPS and routes requests +- `octw-oidc` runs `oauth2-proxy` + +Base oauth2-proxy settings live in [configs/oauth2-proxy/oauth2-proxy.cfg](/Users/magos/dev/kumanday/OCTW/configs/oauth2-proxy/oauth2-proxy.cfg). + +Provider examples: + +- Keycloak: [configs/oauth2-proxy/keycloak.example.cfg](/Users/magos/dev/kumanday/OCTW/configs/oauth2-proxy/keycloak.example.cfg) +- Google Workspace: [configs/oauth2-proxy/google-workspace.example.cfg](/Users/magos/dev/kumanday/OCTW/configs/oauth2-proxy/google-workspace.example.cfg) + +Typical Keycloak settings in `.env`: + +```dotenv +OCTW_OIDC_PROVIDER=keycloak-oidc +OCTW_OIDC_ISSUER_URL=https://keycloak.example.com/realms/octw +OCTW_OIDC_EMAIL_DOMAINS=example.com +OCTW_OIDC_WHITELIST_DOMAINS=.example.com ``` -### 3. Launch the stack +Typical Google Workspace settings in `.env`: + +```dotenv +OCTW_OIDC_PROVIDER=google +OCTW_OIDC_ISSUER_URL= +OCTW_OIDC_EMAIL_DOMAINS=example.com +OCTW_OIDC_WHITELIST_DOMAINS=.example.com +``` + +If you want Google Group checks, place the Admin SDK service account JSON at `configs/oauth2-proxy/credentials/google-admin-sdk.json` and copy the relevant settings from the Google example config. + +### 4. Launch the stack ```bash -export OCTW_KEK=$(python3 -c "print(open('/etc/octw/master.key','rb').read().hex())") -export OCTW_JWT_SECRET=$(python3 -c "import secrets; print(secrets.token_urlsafe(32))") -docker compose up -d +docker compose up -d --build ``` -This starts four services: +This starts six services: | Service | Port | Purpose | |---|---|---| -| **octw-api** | 8000 | REST API and one-click provisioning | -| **octw-edge** | 8443 | Tenant proxy with wake-on-request | -| **octw-db** | 5432 | PostgreSQL (metadata, secrets, audit) | -| **octw-cache** | 6379 | Redis (sessions, locks) | +| `octw-proxy` | `80`, `443` | Public HTTPS ingress and routing | +| `octw-oidc` | internal | `oauth2-proxy` for OIDC login/session | +| `octw-api` | `127.0.0.1:8000` | API, browser app, provisioning workflow | +| `octw-edge` | `127.0.0.1:8443` | Auth-aware tenant HTTP and WebSocket proxy | +| `octw-db` | `5432` | PostgreSQL metadata store | +| `octw-cache` | `6379` | Redis sessions and locks | + +If `configs/certs/fullchain.pem` and `configs/certs/privkey.pem` do not exist, `octw-proxy` generates a self-signed certificate on first boot. Replace those files with a real certificate for production. + +### 5. Open the browser app + +Open: + +```text +https://octw.example.com/app +``` + +The browser flow is: -### 4. Provision a tenant +1. `octw-proxy` sends the request through `octw-oidc` +2. OCTW boots or refreshes the local `octw_session` cookie from the trusted forwarded email header +3. `POST /api/v1/app/deploy-or-resume` creates or resumes the user's single tenant +4. OCTW verifies the OpenClaw tenant before returning success +5. The browser connects to `/t/{slug}/ws` through `octw-edge` and lands in chat + +### 6. Operator / CLI provisioning still works ```bash -# Authenticate TOKEN=$(curl -s -X POST http://localhost:8000/api/v1/auth/login \ -H 'Content-Type: application/json' \ -d '{"email":"admin@example.com"}' | jq -r .dev_token) @@ -60,14 +131,14 @@ JWT=$(curl -s -X POST http://localhost:8000/api/v1/auth/verify \ -H 'Content-Type: application/json' \ -d "{\"token\":\"$TOKEN\"}" | jq -r .access_token) -# One-click provision curl -s -X POST http://localhost:8000/api/v1/provision \ -H "Authorization: Bearer $JWT" \ -H 'Content-Type: application/json' \ -d '{"slug":"acme", "name":"Acme Corp", "provider":"zai"}' | jq ``` -Response: +The provision response now includes readiness verification: + ```json { "tenant_id": "...", @@ -75,20 +146,13 @@ Response: "status": "running", "provider": "zai", "model": "zai-coding/glm-5", - "url": "https://octw.example.com/acme/" + "url": "https://octw.example.com/acme/", + "verification_status": "verified", + "verification_error": null } ``` -The tenant's OpenClaw webchat is accessible at `https://octw.example.com/acme/`. - -### 5. List available providers - -```bash -curl -s http://localhost:8000/api/v1/provision/providers \ - -H "Authorization: Bearer $JWT" | jq -``` - -## CLI +## Operator CLI ```bash uv sync @@ -97,19 +161,19 @@ uv run octw tenant status uv run octw tenant stop ``` -See [docs/cli.md](docs/cli.md) for the full command reference. +See [docs/cli.md](/Users/magos/dev/kumanday/OCTW/docs/cli.md). ## Documentation | Document | Contents | |---|---| -| [Architecture](docs/architecture.md) | Component overview, isolation model, data flow | -| [API Reference](docs/api.md) | All REST endpoints with request/response examples | -| [Providers](docs/providers.md) | Supported LLM providers, adding new ones | -| [Configuration](docs/configuration.md) | All environment variables and defaults | -| [Security](docs/security.md) | Threat model, secret strategy, container hardening | -| [CLI Reference](docs/cli.md) | Operator command-line tool | -| [Deployment](docs/deployment.md) | Production setup, Docker Compose, development mode | +| [docs/architecture.md](/Users/magos/dev/kumanday/OCTW/docs/architecture.md) | Control-plane and tenant data flow | +| [docs/api.md](/Users/magos/dev/kumanday/OCTW/docs/api.md) | Browser app, operator API, and internal endpoints | +| [docs/configuration.md](/Users/magos/dev/kumanday/OCTW/docs/configuration.md) | Environment variables and proxy config files | +| [docs/deployment.md](/Users/magos/dev/kumanday/OCTW/docs/deployment.md) | Compose deployment, OIDC setup, verification | +| [docs/security.md](/Users/magos/dev/kumanday/OCTW/docs/security.md) | Reverse-proxy auth, trusted-proxy boundaries, secrets | +| [docs/providers.md](/Users/magos/dev/kumanday/OCTW/docs/providers.md) | Supported model providers | +| [docs/cli.md](/Users/magos/dev/kumanday/OCTW/docs/cli.md) | Operator CLI reference | ## Running Tests @@ -119,4 +183,4 @@ uv run pytest tests/ -v ## License -See [LICENSE](LICENSE). +See [LICENSE](/Users/magos/dev/kumanday/OCTW/LICENSE). diff --git a/configs/certs/.gitkeep b/configs/certs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/configs/nginx/default.conf.template b/configs/nginx/default.conf.template new file mode 100644 index 0000000..6718bdf --- /dev/null +++ b/configs/nginx/default.conf.template @@ -0,0 +1,92 @@ +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +server { + listen 80; + server_name ${OCTW_SERVER_NAME}; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl http2; + server_name ${OCTW_SERVER_NAME}; + + ssl_certificate /etc/nginx/certs/fullchain.pem; + ssl_certificate_key /etc/nginx/certs/privkey.pem; + ssl_session_timeout 1d; + ssl_session_cache shared:SSL:10m; + ssl_protocols TLSv1.2 TLSv1.3; + + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Host $host; + + location /oauth2/ { + proxy_pass http://octw-oidc:4180; + proxy_set_header X-Auth-Request-Redirect $scheme://$host$request_uri; + proxy_set_header X-Forwarded-Uri $request_uri; + } + + location = /oauth2/auth { + internal; + proxy_pass http://octw-oidc:4180/oauth2/auth; + proxy_set_header X-Forwarded-Uri $request_uri; + proxy_set_header Content-Length ""; + proxy_pass_request_body off; + } + + error_page 401 = @oauth2_signin; + + location @oauth2_signin { + return 302 /oauth2/start?rd=$scheme://$host$request_uri; + } + + location = /health { + proxy_pass http://octw-api:8000/health; + } + + location = / { + return 302 /app; + } + + location /app { + auth_request /oauth2/auth; + auth_request_set $user_email $upstream_http_x_auth_request_email; + proxy_set_header X-Forwarded-Email $user_email; + proxy_pass http://octw-api:8000; + } + + location /api/v1/app/ { + auth_request /oauth2/auth; + auth_request_set $user_email $upstream_http_x_auth_request_email; + proxy_set_header X-Forwarded-Email $user_email; + proxy_pass http://octw-api:8000; + } + + location /t/ { + auth_request /oauth2/auth; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_pass http://octw-edge:8443; + } + + location /oauth2/sign_out { + proxy_pass http://octw-oidc:4180/oauth2/sign_out; + } + + location ~ ^/[a-z0-9][a-z0-9-]*/? { + auth_request /oauth2/auth; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_pass http://octw-edge:8443; + } + + location / { + return 404; + } +} diff --git a/configs/oauth2-proxy/credentials/.gitkeep b/configs/oauth2-proxy/credentials/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/configs/oauth2-proxy/google-workspace.example.cfg b/configs/oauth2-proxy/google-workspace.example.cfg new file mode 100644 index 0000000..d4bf430 --- /dev/null +++ b/configs/oauth2-proxy/google-workspace.example.cfg @@ -0,0 +1,20 @@ +# Google Workspace example for oauth2-proxy. +# Copy the relevant settings into oauth2-proxy.cfg or keep this file as a +# reference while setting OCTW_OIDC_PROVIDER=google. + +provider = "google" +email_domains = ["example.com"] +whitelist_domains = [".example.com"] + +# Optional: require membership in one or more Google Groups. +# This requires Admin SDK access. Mount the service account JSON at: +# configs/oauth2-proxy/credentials/google-admin-sdk.json +# inside the container as /var/run/oauth2-proxy/google-admin-sdk.json. +# google_admin_email = "admin@example.com" +# google_groups = ["octw-users@example.com"] +# google_service_account_json = "/var/run/oauth2-proxy/google-admin-sdk.json" + +# Optional: if you run on GCP with Workload Identity instead of a JSON key. +# google_use_application_default_credentials = true +# google_target_principal = "oauth2-proxy@example-project.iam.gserviceaccount.com" +# google_use_organization_id = true diff --git a/configs/oauth2-proxy/keycloak.example.cfg b/configs/oauth2-proxy/keycloak.example.cfg new file mode 100644 index 0000000..a36da15 --- /dev/null +++ b/configs/oauth2-proxy/keycloak.example.cfg @@ -0,0 +1,18 @@ +# Keycloak example for oauth2-proxy. +# Copy the relevant settings into oauth2-proxy.cfg or keep this file as a +# reference while setting OCTW_OIDC_PROVIDER=keycloak-oidc. + +provider = "keycloak-oidc" +oidc_issuer_url = "https://keycloak.example.com/realms/octw" +scope = "openid profile email roles groups" +email_domains = ["example.com"] +whitelist_domains = [".example.com"] + +# Optional: require realm or client roles presented by Keycloak. +# allowed_roles = ["octw-user"] + +# Optional: require membership in explicit Keycloak groups. +# keycloak_groups = ["/octw-users"] + +# Optional: trust a private CA when Keycloak uses an internal certificate. +# provider_ca_files = ["/var/run/oauth2-proxy/keycloak-ca.pem"] diff --git a/configs/oauth2-proxy/oauth2-proxy.cfg b/configs/oauth2-proxy/oauth2-proxy.cfg new file mode 100644 index 0000000..e1e51de --- /dev/null +++ b/configs/oauth2-proxy/oauth2-proxy.cfg @@ -0,0 +1,33 @@ +# Base oauth2-proxy configuration used by docker-compose.yml. +# Provider selection and secrets come from environment variables so the same +# file can be used for generic OIDC, Keycloak, or Google Workspace. + +http_address = "0.0.0.0:4180" +upstreams = ["static://202"] + +set_xauthrequest = true +reverse_proxy = true +proxy_websockets = true +skip_provider_button = true +scope = "openid profile email" + +cookie_name = "__Host-octw_oauth2" +cookie_secure = true +cookie_httponly = true +cookie_samesite = "lax" +cookie_path = "/" +cookie_refresh = "1h" +cookie_expire = "8h" + +pass_access_token = false +pass_authorization_header = false +set_authorization_header = false +skip_auth_strip_headers = true + +session_store_type = "redis" +redis_connection_url = "redis://octw-cache:6379/1" + +standard_logging = true +request_logging = true +auth_logging = true +silence_ping_logging = true diff --git a/docker-compose.yml b/docker-compose.yml index 61cf67d..d83548b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,7 @@ services: - octw_control octw-api: + container_name: octw-api build: context: . dockerfile: Dockerfile @@ -38,7 +39,7 @@ services: # The edge proxy runs as non-root (UID 1000). user: "0:0" ports: - - "8000:8000" + - "127.0.0.1:8000:8000" environment: OCTW_DB_URL: "postgresql+asyncpg://octw:octw@octw-db:5432/octw" OCTW_REDIS_URL: "redis://octw-cache:6379/0" @@ -47,6 +48,10 @@ services: OCTW_ZAI_API_KEY: "${OCTW_ZAI_API_KEY:-}" OCTW_MOONSHOT_API_KEY: "${OCTW_MOONSHOT_API_KEY:-}" OCTW_MINIMAX_API_KEY: "${OCTW_MINIMAX_API_KEY:-}" + OCTW_PUBLIC_BASE_URL: "${OCTW_PUBLIC_BASE_URL:-https://octw.example.com}" + OCTW_TRUSTED_PROXY_ENABLED: "${OCTW_TRUSTED_PROXY_ENABLED:-false}" + OCTW_TRUSTED_PROXY_USER_HEADER: "${OCTW_TRUSTED_PROXY_USER_HEADER:-X-Forwarded-Email}" + OCTW_TRUSTED_PROXY_IPS: "${OCTW_TRUSTED_PROXY_IPS:-172.30.0.0/24}" volumes: - /var/run/docker.sock:/var/run/docker.sock - /var/lib/octw:/var/lib/octw @@ -59,25 +64,74 @@ services: - octw_control octw-edge: + container_name: octw-edge build: context: . dockerfile: Dockerfile command: ["uvicorn", "octw.edge.proxy:edge_app", "--host", "0.0.0.0", "--port", "8443"] ports: - - "8443:8443" + - "127.0.0.1:8443:8443" environment: OCTW_DB_URL: "postgresql+asyncpg://octw:octw@octw-db:5432/octw" OCTW_REDIS_URL: "redis://octw-cache:6379/0" OCTW_JWT_SECRET: "${OCTW_JWT_SECRET:-change-me-in-production}" OCTW_EDGE_DOMAIN: "${OCTW_EDGE_DOMAIN:-octw.example.com}" + OCTW_API_INTERNAL_BASE_URL: "${OCTW_API_INTERNAL_BASE_URL:-http://octw-api:8000}" + OCTW_PUBLIC_BASE_URL: "${OCTW_PUBLIC_BASE_URL:-https://octw.example.com}" depends_on: - octw-api networks: - octw_control + octw-oidc: + image: quay.io/oauth2-proxy/oauth2-proxy:v7.14.3 + container_name: octw-oidc + command: + - --config=/etc/oauth2-proxy/${OCTW_OIDC_CONFIG_FILE:-oauth2-proxy.cfg} + environment: + OAUTH2_PROXY_PROVIDER: "${OCTW_OIDC_PROVIDER:-oidc}" + OAUTH2_PROXY_OIDC_ISSUER_URL: "${OCTW_OIDC_ISSUER_URL:-}" + OAUTH2_PROXY_REDIRECT_URL: "${OCTW_PUBLIC_BASE_URL:-https://octw.example.com}/oauth2/callback" + OAUTH2_PROXY_CLIENT_ID: "${OCTW_OIDC_CLIENT_ID}" + OAUTH2_PROXY_CLIENT_SECRET: "${OCTW_OIDC_CLIENT_SECRET}" + OAUTH2_PROXY_COOKIE_SECRET: "${OCTW_OIDC_COOKIE_SECRET}" + OAUTH2_PROXY_EMAIL_DOMAINS: "${OCTW_OIDC_EMAIL_DOMAINS:-*}" + OAUTH2_PROXY_WHITELIST_DOMAINS: "${OCTW_OIDC_WHITELIST_DOMAINS:-}" + depends_on: + octw-cache: + condition: service_healthy + volumes: + - ./configs/oauth2-proxy:/etc/oauth2-proxy:ro + - ./configs/oauth2-proxy/credentials:/var/run/oauth2-proxy:ro + networks: + - octw_control + + octw-proxy: + container_name: octw-proxy + build: + context: . + dockerfile: docker/proxy/Dockerfile + environment: + OCTW_SERVER_NAME: "${OCTW_SERVER_NAME:-octw.example.com}" + ports: + - "80:80" + - "443:443" + depends_on: + - octw-api + - octw-edge + - octw-oidc + volumes: + - ./configs/nginx:/etc/nginx/templates:ro + - ./configs/certs:/etc/nginx/certs + networks: + - octw_control + volumes: octw_pgdata: networks: octw_control: driver: bridge + ipam: + config: + - subnet: 172.30.0.0/24 diff --git a/docker/proxy/Dockerfile b/docker/proxy/Dockerfile new file mode 100644 index 0000000..455b4b7 --- /dev/null +++ b/docker/proxy/Dockerfile @@ -0,0 +1,11 @@ +FROM nginx:1.27-alpine + +RUN apk add --no-cache bash gettext openssl + +COPY docker/proxy/entrypoint.sh /entrypoint.sh +COPY configs/nginx/default.conf.template /etc/nginx/templates/default.conf.template + +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/docker/proxy/entrypoint.sh b/docker/proxy/entrypoint.sh new file mode 100644 index 0000000..9059142 --- /dev/null +++ b/docker/proxy/entrypoint.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu + +CERT_DIR=/etc/nginx/certs +CERT_FILE="$CERT_DIR/fullchain.pem" +KEY_FILE="$CERT_DIR/privkey.pem" +mkdir -p "$CERT_DIR" + +if [ ! -s "$CERT_FILE" ] || [ ! -s "$KEY_FILE" ]; then + SERVER_NAME="${OCTW_SERVER_NAME:-localhost}" + echo "Generating self-signed TLS certificate for ${SERVER_NAME}" >&2 + openssl req -x509 -nodes -newkey rsa:2048 \ + -keyout "$KEY_FILE" \ + -out "$CERT_FILE" \ + -days 365 \ + -subj "/CN=${SERVER_NAME}" +fi + +envsubst '${OCTW_SERVER_NAME}' \ + < /etc/nginx/templates/default.conf.template \ + > /etc/nginx/conf.d/default.conf + +exec "$@" diff --git a/docs/api.md b/docs/api.md index 47bbafe..e08106c 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,35 +1,107 @@ # API Reference -Base URL: `http://localhost:8000/api/v1` +Base operator API URL: `http://localhost:8000/api/v1` -All endpoints require a Bearer token except auth endpoints. Obtain a token via the login/verify flow. +There are now two auth modes: -## Authentication +- operator and CLI endpoints use OCTW bearer tokens +- browser app endpoints under `/api/v1/app/*` are expected to sit behind `octw-proxy` and OIDC + +## Browser App Routes + +### GET `/app` + +Serves the single-page OCTW browser shell. + +### GET `/app/chat` + +Serves the same shell and resumes the chat view. + +### GET `/api/v1/app/session` + +Bootstrap endpoint for the browser app. + +Behavior: + +- if `octw_session` exists, returns the current user and tenant state +- otherwise, if the request came through the trusted reverse proxy, creates the local user if needed, mints `octw_session`, and returns the same payload +- otherwise returns `401` + +Response: + +```json +{ + "user_id": "550e8400-e29b-41d4-a716-446655440000", + "email": "user@example.com", + "tenant": { + "tenant_id": "44262e00-b7fc-462a-bf2e-8f7b750f4cfe", + "slug": "u-0123456789abcdef0123", + "status": "running", + "verification_status": "verified", + "verification_error": null, + "chat_url": "https://octw.example.com/app/chat" + } +} +``` + +### POST `/api/v1/app/deploy-or-resume` + +Idempotent browser flow entrypoint. + +Behavior: + +- creates the signed-in user's single tenant if it does not exist yet +- otherwise wakes the existing tenant if it is paused or stopped +- re-runs verification if the tenant is not currently marked `verified` + +Response: + +```json +{ + "created": true, + "tenant": { + "tenant_id": "44262e00-b7fc-462a-bf2e-8f7b750f4cfe", + "slug": "u-0123456789abcdef0123", + "status": "running", + "verification_status": "verified", + "verification_error": null, + "chat_url": "https://octw.example.com/app/chat" + } +} +``` + +### WebSocket `/t/{slug}/ws` + +Browser chat connections go through `octw-edge` on this path. + +`octw-edge` validates the OCTW session, checks tenant membership, wakes the tenant if needed, and then proxies the WebSocket to the tenant gateway. + +## Authentication Endpoints ### POST `/auth/login` -Request a magic link token. +Request a development magic-link token. ```json { "email": "user@example.com" } ``` -Response (202): +Response: + ```json { "message": "Check your email for a login link", "dev_token": "..." } ``` -In development, `dev_token` is returned directly. In production, the token would be sent via email. - ### POST `/auth/verify` -Exchange a magic link token for a session JWT. +Exchange a dev token for an OCTW bearer token and `octw_session` cookie. ```json { "token": "" } ``` Response: + ```json { "access_token": "eyJ...", @@ -41,33 +113,20 @@ Response: ### POST `/auth/logout` -Response: 204 No Content - ---- +Placeholder endpoint. Browser logout is currently expected to happen through `/oauth2/sign_out` at the ingress layer. ## Provisioning ### GET `/provision/providers` -List available providers and whether they are configured on the server. - -Response: -```json -[ - { - "key": "zai", - "display_name": "Z.ai GLM Coding Plan", - "model": "zai-coding/glm-5", - "configured": true - } -] -``` +List supported providers and whether their server-side API key is configured. ### POST `/provision` -One-click tenant provisioning. Creates tenant, runs OpenClaw onboarding, configures webchat and model, starts container. +Operator one-click provisioning flow. Request: + ```json { "slug": "acme", @@ -77,232 +136,99 @@ Request: } ``` -- `slug` — DNS-safe identifier (3-50 chars, lowercase alphanumeric and hyphens) -- `name` — display name -- `provider` — one of `zai`, `moonshot`, `minimax` (default: `zai`) -- `secrets` — optional additional per-tenant secrets +Response: -Response (200): ```json { - "tenant_id": "550e8400-...", + "tenant_id": "550e8400-e29b-41d4-a716-446655440000", "slug": "acme", "status": "running", "provider": "zai", "model": "zai-coding/glm-5", - "url": "https://octw.example.com/acme/" + "url": "https://octw.example.com/acme/", + "verification_status": "verified", + "verification_error": null } ``` -Error responses: -- 400 — invalid provider or provider not configured on server -- 409 — slug already exists +A successful response implies the config exists, the gateway is healthy, and a server-side WebSocket connection to the tenant succeeded. ---- - -## Tenants +## Tenant Management ### POST `/tenants` -Create a tenant without provisioning OpenClaw (use `/provision` for the full flow). - -```json -{ - "slug": "acme", - "name": "Acme Corp", - "plan": "standard", - "isolation_mode": "standard", - "trusted_proxy_auth": true -} -``` - -Response (201): -```json -{ "tenantId": "...", "slug": "acme", "status": "stopped" } -``` +Create tenant metadata without running the provisioning workflow. ### GET `/tenants` -List tenants the authenticated user belongs to. +List tenants visible to the caller. ### GET `/tenants/{tenantId}` -Get tenant details. Requires membership. +Fetch tenant details. ### PATCH `/tenants/{tenantId}` -Update tenant (currently supports `name`). Requires `tenant_admin` role. +Update mutable tenant fields such as `name`. ### DELETE `/tenants/{tenantId}` -Delete tenant and all resources. Requires `tenant_admin` role. Response: 204. - ---- - -## Members - -### GET `/tenants/{tenantId}/members` - -List tenant members and their roles. - -### POST `/tenants/{tenantId}/members` - -Add a member. Requires `tenant_admin` role. - -```json -{ "email": "user@example.com", "role": "tenant_user" } -``` - -Roles: `tenant_admin`, `tenant_user`, `tenant_viewer`. - -### DELETE `/tenants/{tenantId}/members/{userId}` - -Remove a member. Requires `tenant_admin` role. Response: 204. - ---- +Delete the tenant and its resources. ## Secrets -Secrets are encrypted at rest (AES-256-GCM envelope encryption). The API never returns secret values. - ### GET `/tenants/{tenantId}/secrets` -List secret metadata (names, types, timestamps — never values). +List secret metadata. Values are never returned. ### PUT `/tenants/{tenantId}/secrets/{name}` -Create or update a secret. Requires `tenant_admin` role. - -```json -{ - "value": "sk-...", - "type": "env", - "target_env_var": "OPENAI_API_KEY" -} -``` +Create or replace a secret. ### DELETE `/tenants/{tenantId}/secrets/{name}` -Delete a secret. Requires `tenant_admin` role. Response: 204. +Delete a secret. ### POST `/tenants/{tenantId}/secrets/rotate` -Rotate a secret (provide new value). Requires `tenant_admin` role. - -```json -{ "name": "OPENAI_API_KEY", "value": "sk-new-..." } -``` - ---- +Rotate a secret value. ## Runtime Control ### GET `/tenants/{tenantId}/runtime` -Get tenant runtime state, container ID, last activity, and resource limits. - -```json -{ - "tenant_id": "...", - "state": "running", - "container_id": "abc123...", - "last_activity_at": "2026-02-28T...", - "limits": { "mem_limit": "1536m", "cpu_quota": 100000, "pids_limit": 512 } -} -``` - -States: `running`, `paused`, `stopped`, `error`, `not_found`. +Return runtime state and resource limits. ### POST `/tenants/{tenantId}/runtime/start` -Start the tenant container. Requires `tenant_user` or above. +Start the tenant container. ### POST `/tenants/{tenantId}/runtime/stop` -Stop and remove the tenant container. Requires `tenant_user` or above. +Stop and remove the tenant container. ### POST `/tenants/{tenantId}/runtime/pause` -Pause the container (memory retained, CPU released). Requires `tenant_user` or above. +Pause the tenant container. ### POST `/tenants/{tenantId}/runtime/wake` -Wake a paused or stopped tenant. Requires `tenant_user` or above. +Wake a paused or stopped tenant. ### GET `/tenants/{tenantId}/runtime/logs` -Get container logs. - -Query parameters: -- `since` — Unix timestamp -- `tail` — number of lines (default 200, max 5000) +Return container logs. ---- +## Internal API -## Internal Endpoints +Internal base URL: `http://octw-api:8000/internal/v1/tenants` -These are used by octw-edge and should be restricted to internal access (mTLS in production). +These endpoints are intended for `octw-edge` and internal automation. -Base URL: `http://localhost:8000/internal/v1/tenants` - -All endpoints accept either a UUID or slug as the `{identifier}`. - -| Method | Path | Description | -|---|---|---| -| POST | `/{identifier}/ensure-running` | Wake tenant if not running | -| POST | `/{identifier}/pause-if-idle` | Pause tenant | -| POST | `/{identifier}/stop-if-idle` | Stop tenant | -| GET | `/{identifier}/status` | Get container state and IP | - -### GET `/{identifier}/status` Response - -```json -{ "state": "running", "ip": "172.18.0.5" } -``` - ---- - -## Metrics - -### GET `/metrics` - -Prometheus-format metrics endpoint. No authentication required. - -```bash -curl http://localhost:8000/metrics -``` - -Available metrics: - -| Metric | Type | Description | +| Method | Path | Purpose | |---|---|---| -| `octw_tenant_container_starts_total` | Counter | Tenant container start events (label: `tenant_id`) | -| `octw_tenant_container_stops_total` | Counter | Tenant container stop events (label: `tenant_id`) | -| `octw_tenant_wake_events_total` | Counter | Wake-on-request events (label: `tenant_id`) | -| `octw_tenant_pause_events_total` | Counter | Tenant pause events (label: `tenant_id`) | -| `octw_active_tenants` | Gauge | Currently running tenant containers | -| `octw_proxy_request_duration_seconds` | Histogram | Edge proxy request duration (labels: `tenant_slug`, `method`, `status`) | -| `octw_secret_operations_total` | Counter | Secret lifecycle operations (labels: `tenant_id`, `operation`) | -| `octw_auth_failures_total` | Counter | Authentication failures (label: `reason`) | - -A Prometheus scrape config is provided at `configs/prometheus.yml`. - ---- - -## Health - -### GET `/health` - -Health check endpoint on octw-api. No authentication required. - -```json -{ "status": "ok" } -``` - -### GET `/health` (octw-edge, port 8443) - -Health check endpoint on octw-edge. - -```json -{ "status": "ok", "service": "octw-edge" } -``` +| `POST` | `/{identifier}/ensure-running` | Wake tenant if needed | +| `GET` | `/{identifier}/access/{user_id}` | Confirm membership before proxying | +| `POST` | `/{identifier}/pause-if-idle` | Pause tenant | +| `POST` | `/{identifier}/stop-if-idle` | Stop tenant | +| `GET` | `/{identifier}/status` | Return runtime state and container IP | diff --git a/docs/architecture.md b/docs/architecture.md index 133e87c..0e5a62a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,120 +2,91 @@ ## Overview -OCTW runs one OpenClaw gateway container per tenant. All tenants share a common control plane (API, database, cache, edge proxy) but are isolated at the container, network, and filesystem levels. - -``` - Internet / VPN / Tailnet - | - v - +-----------------+ - | octw-edge | - | //... | - | Auth + Wake-on | - +--------+--------+ - | - +-------------+--------------+ - | | - v v - +--------------------+ +--------------------+ - | tenant net: T1 | | tenant net: T2 | - | +---------------+ | | +---------------+ | - | | openclaw T1 | | | | openclaw T2 | | - | | port 18789 | | | | port 18789 | | - | +---------------+ | | +---------------+ | - +--------------------+ +--------------------+ - -Control plane (host internal): -+----------+ +---------+ +-------+ -| octw-api |---| octw-db |---| redis | -+----------+ +---------+ +-------+ - | - v -+----------------+ -| orchestrator | -| (Docker API) | -+----------------+ +OCTW runs one OpenClaw gateway container per tenant and keeps all tenant gateways private on Docker networks. A shared control plane handles browser auth, provisioning, verification, runtime control, and proxying. + +```mermaid +flowchart LR + browser["Browser"] --> proxy["octw-proxy\nNginx HTTPS ingress"] + proxy --> oidc["octw-oidc\noauth2-proxy"] + proxy --> api["octw-api\nFastAPI"] + proxy --> edge["octw-edge\nHTTP + WS proxy"] + api --> db["octw-db\nPostgreSQL"] + api --> cache["octw-cache\nRedis"] + api --> docker["Docker API"] + edge --> tenant1["Tenant gateway A"] + edge --> tenant2["Tenant gateway B"] ``` ## Components | Component | Technology | Role | |---|---|---| -| **octw-api** | FastAPI (Python) | REST API for provisioning, tenant CRUD, RBAC, secrets, runtime control | -| **octw-edge** | FastAPI + httpx | Reverse proxy: path-based tenant routing (`//`), JWT auth, wake-on-request | -| **octw-db** | PostgreSQL 16 | Metadata, memberships, audit logs, encrypted secret ciphertext, tenant DEKs | -| **octw-cache** | Redis 7 | Sessions, rate limiting, orchestration locks | -| **orchestrator** | Docker SDK for Python | Container lifecycle, network creation, resource limits, init jobs | -| **vault** | cryptography (Python) | AES-256-GCM envelope encryption for secrets at rest | -| **CLI** | Click | Operator tool for tenant and secret management | +| `octw-proxy` | Nginx | Public HTTPS entrypoint, `auth_request`, browser routing | +| `octw-oidc` | oauth2-proxy | OIDC login, secure proxy session cookie, forwarded identity headers | +| `octw-api` | FastAPI | Browser app shell, operator API, provisioning, tenant verification | +| `octw-edge` | FastAPI + httpx + websockets | Membership-aware tenant HTTP and WebSocket proxy | +| `octw-db` | PostgreSQL | Users, tenants, memberships, audit, encrypted secret metadata | +| `octw-cache` | Redis | Sessions, locks, coordination | +| tenant gateway | OpenClaw | Per-tenant runtime with trusted-proxy auth | -## Data Flow +## Browser Flow -### One-Click Provisioning +1. Browser requests `/app` +2. `octw-proxy` delegates auth to `octw-oidc` +3. `octw-api` mints or refreshes `octw_session` from the trusted forwarded email header +4. `/api/v1/app/deploy-or-resume` creates or resumes the user's single tenant +5. `octw-api` waits for tenant verification +6. Browser connects to `/t/{slug}/ws` +7. `octw-edge` validates membership, wakes the tenant if needed, and proxies the WebSocket to the tenant gateway -``` -Web App - │ - ├─► POST /api/v1/provision { slug, name, provider } - │ - ▼ -octw-api - ├─ 1. Create tenant row in DB - ├─ 2. Create filesystem dirs (/var/lib/octw/tenants//) - ├─ 3. Create isolated Docker network - ├─ 4. Run init container (openclaw onboard --non-interactive) - │ └─ Provider API key + model injected via env vars - ├─ 5. Patch openclaw.json (webchat, gateway bind, model config) - ├─ 6. Start tenant container with secrets injected - └─ 7. Return { tenant_id, slug, url, provider, model } -``` +## Provisioning Flow -### Request Routing +When OCTW provisions a tenant, it performs: -``` -User request: https://octw.example.com/acme/chat - │ - ▼ -octw-edge - ├─ Extract slug "acme" from path - ├─ Authenticate (JWT Bearer or session cookie) - ├─ Resolve slug → tenant_id → container IP - │ └─ If stopped/paused: call ensure-running, wait for container - ├─ Proxy to http://:18789/chat - └─ Return response -``` +1. create tenant rows and ownership membership +2. create tenant directories and a dedicated Docker network +3. connect `octw-edge` to that tenant network +4. run non-interactive OpenClaw onboarding +5. rewrite `openclaw.json` for trusted-proxy auth and explicit allowed origins +6. start the tenant container +7. verify the tenant by checking config presence, gateway health, and a WebSocket handshake -## Isolation Model +Only after step 7 does OCTW report the tenant as ready. -Each tenant container runs with: +## Request Routing -- **Dedicated Docker network** — per-tenant bridge network, `internal=true`. No inter-tenant connectivity. -- **Dedicated volumes** — separate state and workspace directories at `/var/lib/octw/tenants//`, permissions `0700`. -- **Non-root execution** — UID 1000:1000, all Linux capabilities dropped, `no-new-privileges`. -- **Resource limits** — configurable memory (default 1.5 GB), CPU quota, PID limit (512). -- **No host port publishing** — tenant containers are not reachable from the host network; only octw-edge connects. -- **Optional hardened runtime** — gVisor (`runsc`) per tenant via `isolation_mode: hardened`. +### Browser app -## Idle Hibernation +- `/app` and `/api/v1/app/*` terminate at `octw-api` +- `/oauth2/*` terminates at `octw-oidc` +- `/t/{slug}/ws` terminates at `octw-edge` -To conserve host resources when running many tenants: +### Legacy tenant path -| Trigger | Action | Effect | -|---|---|---| -| 30 min idle | Pause container | Memory retained, CPU released | -| 8 hours idle | Stop container | Memory and CPU fully reclaimed | -| First request after stop/pause | Wake-on-request | Edge proxy triggers ensure-running, waits, then proxies | +- `https://octw.example.com//...` still proxies through `octw-edge` +- this remains useful for compatibility and debugging +- the preferred v1 UX is `/app` and `/app/chat` + +## Isolation Model + +Each tenant still gets: + +- a dedicated Docker bridge network +- its own state and workspace directories +- no public host ports +- non-root execution with dropped capabilities +- independent runtime lifecycle and idle hibernation + +`octw-edge` is the only shared service attached to tenant networks. -Activity is defined as HTTP/WS traffic through octw-edge to the tenant. +## Persistent State -## Database Schema +Important persisted records now include: -Key tables: +- tenant owner membership +- provider selection +- verification status +- verification error message +- verification timestamp -- **tenants** — id, slug, name, plan, status, provider, resource_limits, container_id, network_id -- **users** — id, email -- **memberships** — tenant_id, user_id, role (tenant_admin / tenant_user / tenant_viewer) -- **secrets** — tenant_id, name, ciphertext, nonce, algorithm, key_version, target_env_var -- **tenant_deks** — tenant_id, encrypted_dek, dek_nonce, kek_version -- **audit_events** — tenant_id, user_id, action, detail, timestamp -- **image_pins** — image_ref, version, digest, is_default +That state lets OCTW resume a tenant later and distinguish a running container from a verified, usable deployment. diff --git a/docs/configuration.md b/docs/configuration.md index ef34e98..3ee2cdd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,66 +1,93 @@ # Configuration -All settings are controlled via environment variables with the `OCTW_` prefix. +OCTW reads settings from environment variables with the `OCTW_` prefix. Start with [`.env.example`](/Users/magos/dev/kumanday/OCTW/.env.example) and adjust only what you need. -## Core Settings +## Core Control Plane | Variable | Default | Description | |---|---|---| -| `OCTW_DB_URL` | `postgresql+asyncpg://octw:octw@localhost:5432/octw` | Database connection string | +| `OCTW_DB_URL` | `postgresql+asyncpg://octw:octw@localhost:5432/octw` | PostgreSQL connection string | | `OCTW_REDIS_URL` | `redis://localhost:6379/0` | Redis connection string | -| `OCTW_JWT_SECRET` | `CHANGE-ME-IN-PRODUCTION` | JWT signing secret (use a strong random value) | -| `OCTW_JWT_ALGORITHM` | `HS256` | JWT algorithm | -| `OCTW_JWT_EXPIRE_MINUTES` | `60` | JWT token expiry | -| `OCTW_LOG_LEVEL` | `INFO` | Log level | +| `OCTW_LOG_LEVEL` | `INFO` | API and edge log level | +| `OCTW_PUBLIC_BASE_URL` | `https://octw.example.com` | Public HTTPS origin used by the browser app and Control UI origin checks | +| `OCTW_EDGE_DOMAIN` | `octw.example.com` | Hostname used when building legacy tenant URLs | +| `OCTW_EDGE_LISTEN_HOST` | `0.0.0.0` | Bind host for the standalone edge server | +| `OCTW_EDGE_LISTEN_PORT` | `8443` | Bind port for the standalone edge server | +| `OCTW_API_INTERNAL_BASE_URL` | `http://octw-api:8000` | Internal API base URL used by `octw-edge` | +| `OCTW_EDGE_CONTAINER_NAME` | `octw-edge` | Docker container name trusted by tenant networks | -## Encryption +## Secrets and Session Signing | Variable | Default | Description | |---|---|---| -| `OCTW_KEK` | — | Master encryption key as hex string (32 bytes = 64 hex chars) | -| `OCTW_KEK_PATH` | `/etc/octw/master.key` | Path to 32-byte KEK file (used if `OCTW_KEK` is not set) | +| `OCTW_KEK` | none | Hex-encoded 32-byte master key for envelope encryption | +| `OCTW_KEK_PATH` | `/etc/octw/master.key` | Fallback path for the KEK if `OCTW_KEK` is unset | +| `OCTW_JWT_SECRET` | `CHANGE-ME-IN-PRODUCTION` | Signing key for API bearer tokens and the `octw_session` cookie | +| `OCTW_JWT_ALGORITHM` | `HS256` | JWT algorithm | +| `OCTW_JWT_EXPIRE_MINUTES` | `60` | Expiry for OCTW bearer tokens and `octw_session` | -The KEK encrypts per-tenant Data Encryption Keys (DEKs), which in turn encrypt individual secret values. See [security.md](security.md) for details. +## Shared Provider API Keys -## Provider API Keys +At least one provider key must be present for one-click provisioning. | Variable | Provider | Model | |---|---|---| -| `OCTW_ZAI_API_KEY` | Z.ai GLM Coding Plan | `zai-coding/glm-5` | -| `OCTW_MOONSHOT_API_KEY` | Moonshot AI Kimi Coding Plan | `kimi-coding/k2p5` | -| `OCTW_MINIMAX_API_KEY` | MiniMax Coding Plan | `minimax-coding/MiniMax-M2.5` | +| `OCTW_ZAI_API_KEY` | Z.ai | `zai-coding/glm-5` | +| `OCTW_MOONSHOT_API_KEY` | Moonshot AI | `kimi-coding/k2p5` | +| `OCTW_MINIMAX_API_KEY` | MiniMax | `minimax-coding/MiniMax-M2.5` | +| `OCTW_DEFAULT_PROVIDER` | `zai` | Provider key used by `/api/v1/app/deploy-or-resume` | -At least one provider key is required for provisioning. See [providers.md](providers.md) for details. - -## OpenClaw Image +## OpenClaw Runtime | Variable | Default | Description | |---|---|---| -| `OCTW_OPENCLAW_IMAGE` | `ghcr.io/openclaw/openclaw:latest` | OpenClaw container image | -| `OCTW_OPENCLAW_DIGEST` | — | Pin image by digest (recommended for production) | +| `OCTW_OPENCLAW_IMAGE` | `ghcr.io/openclaw/openclaw:latest` | OpenClaw image used for tenants | +| `OCTW_OPENCLAW_DIGEST` | none | Optional image digest pin | +| `OCTW_WEBCHAT_PORT` | `18790` | WebChat port enabled during onboarding | +| `OCTW_TENANT_BASE_DIR` | `/var/lib/octw/tenants` | Base directory for tenant state and workspace data | -## Networking +## Resource Limits and Idle Control | Variable | Default | Description | |---|---|---| -| `OCTW_EDGE_LISTEN_HOST` | `0.0.0.0` | Edge proxy listen address | -| `OCTW_EDGE_LISTEN_PORT` | `8443` | Edge proxy listen port | -| `OCTW_EDGE_DOMAIN` | `octw.example.com` | Domain used for building tenant URLs | -| `OCTW_WEBCHAT_PORT` | `18790` | Port for OpenClaw webchat inside tenant containers | +| `OCTW_DEFAULT_MEM_LIMIT` | `1536m` | Default container memory limit | +| `OCTW_DEFAULT_CPU_QUOTA` | `100000` | Default CPU quota in microseconds | +| `OCTW_DEFAULT_CPU_PERIOD` | `100000` | CPU period in microseconds | +| `OCTW_DEFAULT_PIDS_LIMIT` | `512` | Default PID limit per tenant | +| `OCTW_IDLE_PAUSE_SECONDS` | `1800` | Idle pause threshold | +| `OCTW_IDLE_STOP_SECONDS` | `28800` | Idle stop threshold | -## Tenant Defaults +## Browser Auth and Trusted Proxy + +These settings control the browser-first `/app` flow. | Variable | Default | Description | |---|---|---| -| `OCTW_TENANT_BASE_DIR` | `/var/lib/octw/tenants` | Base directory for tenant state and workspace volumes | -| `OCTW_DEFAULT_MEM_LIMIT` | `1536m` | Default memory limit per tenant container | -| `OCTW_DEFAULT_CPU_QUOTA` | `100000` | CPU quota in microseconds per period | -| `OCTW_DEFAULT_CPU_PERIOD` | `100000` | CPU period in microseconds | -| `OCTW_DEFAULT_PIDS_LIMIT` | `512` | Maximum PIDs per tenant container | +| `OCTW_TRUSTED_PROXY_ENABLED` | `false` | Allow the API to bootstrap browser sessions from a forwarded identity header | +| `OCTW_TRUSTED_PROXY_USER_HEADER` | `X-Forwarded-Email` | Header that carries the authenticated user email from Nginx/oauth2-proxy | +| `OCTW_TRUSTED_PROXY_IPS` | empty | Comma-separated list of IPs or CIDRs allowed to send that header | + +`OCTW_TRUSTED_PROXY_ENABLED` should only be enabled when the API is behind `octw-proxy` or another trusted reverse proxy that injects the configured header. + +## Containerized OIDC Ingress -## Hibernation +`docker-compose.yml` now runs `oauth2-proxy` inside the `octw-oidc` service. The service loads [configs/oauth2-proxy/oauth2-proxy.cfg](/Users/magos/dev/kumanday/OCTW/configs/oauth2-proxy/oauth2-proxy.cfg) and accepts provider-specific overrides from the environment. | Variable | Default | Description | |---|---|---| -| `OCTW_IDLE_PAUSE_SECONDS` | `1800` | Pause container after 30 minutes of inactivity | -| `OCTW_IDLE_STOP_SECONDS` | `28800` | Stop container after 8 hours of inactivity | +| `OCTW_SERVER_NAME` | `octw.example.com` | Nginx `server_name` and certificate CN fallback | +| `OCTW_OIDC_CONFIG_FILE` | `oauth2-proxy.cfg` | Config file under `configs/oauth2-proxy/` loaded by `octw-oidc` | +| `OCTW_OIDC_PROVIDER` | `oidc` | `oauth2-proxy` provider, typically `oidc`, `keycloak-oidc`, or `google` | +| `OCTW_OIDC_ISSUER_URL` | empty | Required for generic OIDC and Keycloak providers | +| `OCTW_OIDC_CLIENT_ID` | none | OIDC client ID | +| `OCTW_OIDC_CLIENT_SECRET` | none | OIDC client secret | +| `OCTW_OIDC_COOKIE_SECRET` | none | 16/24/32-byte `oauth2-proxy` cookie secret, raw or base64-encoded | +| `OCTW_OIDC_EMAIL_DOMAINS` | `*` | Allowed email domains passed to `oauth2-proxy` | +| `OCTW_OIDC_WHITELIST_DOMAINS` | empty | Redirect domains allowed by `oauth2-proxy` | + +Provider-specific references: + +- Keycloak: [configs/oauth2-proxy/keycloak.example.cfg](/Users/magos/dev/kumanday/OCTW/configs/oauth2-proxy/keycloak.example.cfg) +- Google Workspace: [configs/oauth2-proxy/google-workspace.example.cfg](/Users/magos/dev/kumanday/OCTW/configs/oauth2-proxy/google-workspace.example.cfg) + +For Google Group enforcement, mount an Admin SDK service-account JSON at `configs/oauth2-proxy/credentials/google-admin-sdk.json`. diff --git a/docs/deployment.md b/docs/deployment.md index fea9633..a70220c 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,145 +1,147 @@ # Deployment -## Docker Compose (Recommended) +## Compose Topology -### Prerequisites +The recommended deployment is the provided Docker Compose stack: -- Docker Engine with compose plugin -- At least one LLM provider API key +| Service | Exposure | Role | +|---|---|---| +| `octw-proxy` | public `80/443` | HTTPS termination, `auth_request`, browser routing | +| `octw-oidc` | internal | `oauth2-proxy` for OIDC login/session | +| `octw-api` | `127.0.0.1:8000` | REST API and browser app | +| `octw-edge` | `127.0.0.1:8443` | Tenant HTTP and WebSocket proxy | +| `octw-db` | internal plus `5432` | PostgreSQL metadata store | +| `octw-cache` | internal plus `6379` | Redis sessions and orchestration locks | + +The API and edge stay on localhost for operator access. End users only hit `octw-proxy` over HTTPS. -### Host Setup +## Host Preparation ```bash -# Create directories sudo mkdir -p /var/lib/octw /etc/octw -sudo chown $USER /var/lib/octw - -# Generate master encryption key +sudo chown "$USER" /var/lib/octw sudo python3 -c "import os; open('/etc/octw/master.key','wb').write(os.urandom(32))" -sudo chown $USER /etc/octw/master.key +sudo chown "$USER" /etc/octw/master.key sudo chmod 600 /etc/octw/master.key ``` -### Environment Variables +## Environment Setup ```bash -# Required -export OCTW_KEK=$(python3 -c "print(open('/etc/octw/master.key','rb').read().hex())") -export OCTW_JWT_SECRET=$(python3 -c "import secrets; print(secrets.token_urlsafe(32))") - -# Provider keys (at least one required) -export OCTW_ZAI_API_KEY="your-key" -export OCTW_MOONSHOT_API_KEY="your-key" -export OCTW_MINIMAX_API_KEY="your-key" +cp .env.example .env ``` -### Launch +Set at least: -```bash -docker compose up -d -``` +- `OCTW_KEK` +- `OCTW_JWT_SECRET` +- `OCTW_PUBLIC_BASE_URL` +- `OCTW_SERVER_NAME` +- `OCTW_TRUSTED_PROXY_ENABLED=true` +- `OCTW_OIDC_CLIENT_ID` +- `OCTW_OIDC_CLIENT_SECRET` +- `OCTW_OIDC_COOKIE_SECRET` +- one provider API key such as `OCTW_ZAI_API_KEY` -### Verify +## OIDC Provider Profiles -```bash -docker compose ps # all 4 services should be Up/Healthy -curl http://localhost:8000/health # {"status":"ok"} -curl http://localhost:8443/health # {"status":"ok","service":"octw-edge"} +### Keycloak + +In `.env`: + +```dotenv +OCTW_OIDC_PROVIDER=keycloak-oidc +OCTW_OIDC_ISSUER_URL=https://keycloak.example.com/realms/octw +OCTW_OIDC_EMAIL_DOMAINS=example.com +OCTW_OIDC_WHITELIST_DOMAINS=.example.com ``` -### Services +Use [configs/oauth2-proxy/keycloak.example.cfg](/Users/magos/dev/kumanday/OCTW/configs/oauth2-proxy/keycloak.example.cfg) as the reference for optional Keycloak group or role gates. -| Service | Port | Image | -|---|---|---| -| octw-api | 8000 | Built from Dockerfile | -| octw-edge | 8443 | Built from Dockerfile | -| octw-db | 5432 | postgres:16-alpine | -| octw-cache | 6379 | redis:7-alpine | +### Google Workspace -### Rebuild After Code Changes +In `.env`: -```bash -docker compose up -d --build +```dotenv +OCTW_OIDC_PROVIDER=google +OCTW_OIDC_ISSUER_URL= +OCTW_OIDC_EMAIL_DOMAINS=example.com +OCTW_OIDC_WHITELIST_DOMAINS=.example.com ``` -### View Logs +If you need Google Group enforcement: -```bash -docker compose logs -f octw-api -docker compose logs -f octw-edge -``` +1. Put the Admin SDK service account JSON at `configs/oauth2-proxy/credentials/google-admin-sdk.json` +2. Copy the relevant settings from [configs/oauth2-proxy/google-workspace.example.cfg](/Users/magos/dev/kumanday/OCTW/configs/oauth2-proxy/google-workspace.example.cfg) into your active oauth2-proxy config +3. Delegate domain-wide authority and grant the Admin SDK read scopes in Google Workspace -### Reset Everything +## Launch ```bash -docker compose down -v # removes containers and volumes -sudo rm -rf /var/lib/octw/tenants/* # removes tenant data +docker compose up -d --build ``` -## Development Setup - -For running services outside Docker Compose (e.g. for debugging with breakpoints): - -```bash -uv sync +`octw-proxy` will generate a self-signed certificate if `configs/certs/fullchain.pem` and `configs/certs/privkey.pem` are missing. -# Start only infrastructure -docker compose up -d octw-db octw-cache +## Verification -# Run API in one terminal -uv run octw server api +### Container health -# Run edge proxy in another terminal -uv run octw server edge +```bash +docker compose ps +curl http://localhost:8000/health +curl http://localhost:8443/health +curl -k https://localhost/health ``` -Both connect to the same Postgres and Redis instances from Docker Compose. - -## Production Considerations - -### TLS Termination +### Browser flow -The edge proxy (octw-edge) does not terminate TLS itself. In production, place a reverse proxy in front: +Open `https:///app`. -- **nginx** or **Caddy** with automatic ACME certificates -- Cloud load balancer with TLS termination -- Tailscale / WireGuard for private access +The first successful visit should: -### Database +1. redirect through your OIDC provider +2. return to OCTW with a valid `oauth2-proxy` session +3. mint an `octw_session` cookie from the trusted forwarded email header +4. create or resume the single user-owned tenant +5. land on `/app/chat` -The included Postgres is suitable for development. For production: +### Provision verification -- Use a managed Postgres instance (RDS, Cloud SQL, etc.) -- Enable SSL connections -- Set strong passwords -- Configure backups +A successful `POST /api/v1/provision` or `POST /api/v1/app/deploy-or-resume` now implies: -Update `OCTW_DB_URL` to point to your production database. +- `openclaw.json` exists after onboarding +- the tenant container started +- the gateway health check succeeded +- a server-side WebSocket handshake to the tenant gateway succeeded -### Secrets +If verification fails, OCTW stores the failure and returns an error instead of pretending the tenant is ready. -- Use a strong, unique `OCTW_JWT_SECRET` (at least 32 bytes) -- Protect `/etc/octw/master.key` with file permissions (`chmod 600`) -- Consider backing the KEK with cloud KMS for production -- Rotate the JWT secret periodically +## Logs -### Resource Limits +```bash +docker compose logs -f octw-proxy +docker compose logs -f octw-oidc +docker compose logs -f octw-api +docker compose logs -f octw-edge +``` -Default tenant limits (1.5 GB RAM, 1 vCPU, 512 PIDs) are suitable for development. Adjust per deployment: +## Rebuild After Code Changes ```bash -export OCTW_DEFAULT_MEM_LIMIT=2g -export OCTW_DEFAULT_PIDS_LIMIT=1024 +docker compose up -d --build ``` -### Monitoring +## Reset -Prometheus metrics are available at `http://localhost:8000/metrics`. Key metrics: +```bash +docker compose down -v +sudo rm -rf /var/lib/octw/tenants/* +``` -- `octw_tenant_container_starts_total` -- `octw_tenant_wake_events_total` -- `octw_active_tenants` -- `octw_proxy_request_duration_seconds` -- `octw_auth_failures_total` +## Production Notes -A Prometheus scrape config is provided at `configs/prometheus.yml`. +- Replace the generated certificate under `configs/certs/` with a real certificate before exposing the stack broadly. +- Keep `octw-api` and `octw-edge` bound to localhost. The public entrypoint should stay `octw-proxy` only. +- The default Postgres and Redis containers are adequate for development and small-scale testing. For production, move them to managed services and update `OCTW_DB_URL` and `OCTW_REDIS_URL`. +- Pin `OCTW_OPENCLAW_DIGEST` if you need deterministic tenant image rollouts. diff --git a/docs/security.md b/docs/security.md index 13f9f7d..61ec4fa 100644 --- a/docs/security.md +++ b/docs/security.md @@ -1,111 +1,84 @@ # Security -## Threat Model +## Trust Boundaries -OCTW assumes tenants are mutually untrusted. Tenants may be adversarial through prompt injection, may install compromised skills or MCP servers, and should never be able to access another tenant's data or network. +OCTW now has three distinct auth boundaries: -### Primary Threats +1. `octw-proxy` plus `octw-oidc` authenticate the browser user with OIDC +2. `octw-api` trusts the forwarded user header only from configured proxy IPs and converts it into the local `octw_session` cookie +3. each tenant OpenClaw gateway trusts only `octw-edge` via OpenClaw `trusted-proxy` mode -| Threat | Mitigation | -|---|---| -| Cross-tenant data access | Per-tenant volumes with `0700` permissions, separate Docker networks | -| Cross-tenant network movement | Dedicated bridge network per tenant (`internal=true`), no inter-tenant connectivity | -| Host compromise via container breakout | Non-root execution, all capabilities dropped, `no-new-privileges`, optional gVisor | -| Secret leakage to disk | AES-256-GCM encryption at rest, env var injection at runtime, SecretRef in OpenClaw config | -| Ingress auth bypass | JWT validation at edge proxy, forwarded header sanitization | +That split avoids device pairing, Tailscale login, and dangerous gateway auth bypasses for end users. -## Secret Strategy +## Public Ingress -### Envelope Encryption +Public browser traffic terminates at `octw-proxy` over HTTPS. -``` -Master KEK (file at /etc/octw/master.key or OCTW_KEK env var) - │ - └─► encrypts per-tenant DEK (stored in DB as ciphertext) - │ - └─► encrypts each secret value (AES-256-GCM, unique nonce per value) -``` +- `octw-proxy` uses Nginx `auth_request` against `octw-oidc` +- `octw-oidc` stores its own login session in a secure cookie +- Nginx forwards the authenticated email as `X-Forwarded-Email` to OCTW app endpoints +- `octw-api` accepts that header only when `OCTW_TRUSTED_PROXY_ENABLED=true` and the caller IP matches `OCTW_TRUSTED_PROXY_IPS` -- **KEK** — 32-byte AES key, stored outside the database (file or env var). Single point of trust. -- **DEK** — one random 32-byte key per tenant, encrypted by KEK and stored in `tenant_deks` table. -- **Secret values** — encrypted by the tenant's DEK using AES-256-GCM with a unique 12-byte nonce. +The browser never receives raw tenant credentials and never connects directly to tenant container IPs. -### Runtime Secret Delivery +## OCTW Browser Session -1. Secrets are decrypted **only** when starting or waking a tenant container. -2. Decrypted values are injected as **environment variables** into the container process. -3. Secrets are **never written** to the tenant's state volume or config files by OCTW. -4. OpenClaw config uses **SecretRef** (`{"$ref": "env:ZAI_API_KEY"}`) to reference env vars without embedding values. +After a successful OIDC-authenticated request, `octw-api` creates `octw_session`: -### Secret Categories +- signed with `OCTW_JWT_SECRET` +- `HttpOnly` +- `SameSite=Lax` +- `Secure` when `OCTW_PUBLIC_BASE_URL` is HTTPS -| Category | Example | Storage | -|---|---|---| -| Provider API keys (shared) | `ZAI_API_KEY` | Server config (`OCTW_ZAI_API_KEY`), injected at runtime | -| Per-tenant secrets | Channel tokens, custom keys | Encrypted in DB, injected at runtime | -| Platform secrets | JWT secret, DB password | Server environment only | -| Infrastructure secrets | KEK, TLS keys | File or env, never in DB | +`octw-edge` uses that cookie for both HTTP proxying and the `/t/{slug}/ws` WebSocket path. -## Container Hardening +## Tenant Gateway Auth -Every tenant container runs with: +When OCTW configures a tenant, it rewrites the OpenClaw gateway auth section to: -```yaml -user: "1000:1000" -cap_drop: ["ALL"] -security_opt: ["no-new-privileges:true"] -pids_limit: 512 -mem_limit: "1536m" -cpu_quota: 100000 -``` +- `gateway.auth.mode = "trusted-proxy"` +- `gateway.auth.trustedProxy.userHeader = "x-octw-user-email"` +- `gateway.trustedProxies = []` +- `gateway.controlUi.allowedOrigins = [OCTW_PUBLIC_BASE_URL]` -Additionally: -- No Docker socket mount -- No host port publishing -- No privileged mode -- Restart policy: `unless-stopped` -- Health check: `curl -sf http://localhost:18789/health` every 30s +OCTW also removes dangerous fallback controls such as insecure auth and device-auth disable flags. -### Isolation Modes +## Edge Proxy Controls -| Mode | Runtime | Description | -|---|---|---| -| `standard` | Default Docker (runc) | seccomp + AppArmor profiles | -| `hardened` | gVisor (runsc) | User-space kernel, stronger syscall isolation | +`octw-edge` now enforces all of the following before proxying to a tenant: -Set per tenant at creation: `"isolation_mode": "hardened"`. +- validates the OCTW bearer token or `octw_session` cookie +- checks tenant membership via the internal API +- wakes paused or stopped tenants on demand +- strips user-supplied forwarded identity headers and injects its own `x-octw-user-id` and `x-octw-user-email` +- proxies both HTTP and WebSocket traffic over private Docker networks only -## Network Isolation +## Secret Handling -- Each tenant gets a dedicated Docker bridge network (`octw_tenant_`). -- Networks are created with `internal=true` — no outbound internet access by default. -- Only the edge proxy (octw-edge) is connected to tenant networks. -- Tenant containers never join the default bridge network. -- No ports are published to the host. +Secrets still use envelope encryption: -## Authentication and Authorization +- KEK from `OCTW_KEK` or `OCTW_KEK_PATH` +- one DEK per tenant in the database +- AES-256-GCM for stored ciphertext +- runtime injection into container environment variables only -### Edge Proxy (octw-edge) +Provider API keys remain host-level shared secrets. Per-tenant secrets remain encrypted in the database and are never returned by the API. -- Authenticates every request via JWT Bearer token or `octw_session` cookie. -- Sanitizes forwarded headers (`X-Forwarded-For` is overwritten with client IP). -- Injects `x-octw-tenant-slug`, `x-octw-user-id`, `x-octw-user-email` headers. +## Container Isolation -### API (octw-api) +Each tenant still gets: -- JWT-based auth on all endpoints (except `/auth/login` and `/auth/verify`). -- RBAC: every tenant operation checks membership and role. -- Roles: `tenant_admin` > `tenant_user` > `tenant_viewer`. +- a dedicated Docker bridge network +- dedicated tenant directories under `/var/lib/octw/tenants//` +- no published host ports +- non-root execution and dropped Linux capabilities +- resource limits and idle pause/stop enforcement -## Audit Logging +`octw-edge` is the only control-plane container attached to tenant networks. -All sensitive actions emit audit events stored in the `audit_events` table: +## Operational Guidance -- Tenant create / delete -- Secret set / rotate / delete -- Container start / stop / pause / wake -- Member add / remove -- Route mapping changes -- Backup / restore - -Each event includes `tenant_id`, `user_id`, action type, and detail metadata. +- Keep `octw-api` and `octw-edge` on localhost-only bindings. +- Use a real TLS certificate in `configs/certs/` for production. +- Do not enable OpenClaw insecure auth fallbacks to make browser login easier. The trusted-proxy path replaces that need. +- If you use Google Group checks in `oauth2-proxy`, treat the Admin SDK service-account JSON under `configs/oauth2-proxy/credentials/` as a production secret. diff --git a/pyproject.toml b/pyproject.toml index 7c031c4..d50a70a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "pyjwt[crypto]>=2.10.0", "cryptography>=44.0.0", "httpx>=0.28.0", + "websockets>=16.0", "python-multipart>=0.0.18", "structlog>=24.4.0", "prometheus-client>=0.21.0", diff --git a/src/octw/api/app.py b/src/octw/api/app.py index f156bbb..60b759e 100644 --- a/src/octw/api/app.py +++ b/src/octw/api/app.py @@ -1,11 +1,15 @@ from __future__ import annotations from contextlib import asynccontextmanager +from pathlib import Path from fastapi import FastAPI +from fastapi.responses import FileResponse +from fastapi.staticfiles import StaticFiles from prometheus_client import make_asgi_app from octw.api.routers import ( + app_router, auth_router, internal_router, provision_router, @@ -13,14 +17,14 @@ secrets_router, tenants_router, ) -from octw.db.engine import engine -from octw.db.tables import Base +from octw.db.engine import engine, ensure_schema + +WEB_ROOT = Path(__file__).resolve().parent.parent / "web" / "static" @asynccontextmanager async def lifespan(app: FastAPI): - async with engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) + await ensure_schema(engine) yield await engine.dispose() @@ -32,12 +36,22 @@ async def lifespan(app: FastAPI): app.include_router(secrets_router.router, prefix="/api/v1") app.include_router(runtime_router.router, prefix="/api/v1") app.include_router(provision_router.router, prefix="/api/v1") +app.include_router(app_router.router) app.include_router(internal_router.router) +app.mount("/app/static", StaticFiles(directory=WEB_ROOT), name="app-static") + metrics_app = make_asgi_app() app.mount("/metrics", metrics_app) +@app.get("/app") +@app.get("/app/") +@app.get("/app/chat") +async def app_shell(): + return FileResponse(WEB_ROOT / "index.html") + + @app.get("/health") async def health(): return {"status": "ok"} diff --git a/src/octw/api/app_logic.py b/src/octw/api/app_logic.py new file mode 100644 index 0000000..ee92162 --- /dev/null +++ b/src/octw/api/app_logic.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import re +import uuid +from dataclasses import dataclass + +from sqlalchemy import update +from sqlalchemy.ext.asyncio import AsyncSession + +from octw.api.auth import TokenPayload +from octw.common.config import settings +from octw.common.logging import get_logger +from octw.db.tables import TenantRow +from octw.models.provider import get_provider +from octw.models.tenant import TenantCreate, TenantPlan +from octw.orchestrator.docker_orch import DockerOrchestrator +from octw.orchestrator.tenant_service import TenantService +from octw.vault.service import VaultService + +log = get_logger(__name__) + +_SLUG_SAFE = re.compile(r"[^a-z0-9]+") + + +@dataclass(slots=True) +class ProvisionedTenant: + tenant_id: str + slug: str + status: str + provider: str + model: str + url: str + verification_status: str + verification_error: str | None + + +def build_user_tenant_slug(user_id: uuid.UUID) -> str: + return f"u-{user_id.hex[:20]}" + + +def build_user_tenant_name(email: str) -> str: + local = email.split("@", 1)[0].strip().lower() or "user" + safe = _SLUG_SAFE.sub(" ", local).strip() + label = safe.title() if safe else "User" + return f"{label} Workspace" + + +def tenant_chat_url(slug: str) -> str: + return f"{settings.public_base_url.rstrip('/')}/app/chat" + + +async def provision_tenant( + *, + session: AsyncSession, + svc: TenantService, + orch: DockerOrchestrator, + vault_svc: VaultService, + user: TokenPayload, + slug: str, + name: str, + provider_key: str, + secrets: dict[str, str] | None = None, +) -> ProvisionedTenant: + provider_spec = get_provider(provider_key) + api_key = settings.get_provider_api_key(provider_spec.env_var) + if not api_key: + raise RuntimeError( + "Provider " + f"'{provider_spec.display_name}' is not configured. " + f"Set OCTW_{provider_spec.env_var}." + ) + + existing = await svc.get_tenant_by_slug(session, slug) + if existing: + raise RuntimeError(f"Tenant with slug '{slug}' already exists") + + tenant_req = TenantCreate( + slug=slug, + name=name, + plan=TenantPlan.STANDARD, + trusted_proxy_auth=True, + ) + tenant = await svc.create_tenant(session, tenant_req, creator_user_id=user.user_id) + await session.flush() + + await session.execute( + update(TenantRow) + .where(TenantRow.tenant_id == tenant.tenant_id) + .values(provider=provider_spec.key.value) + ) + await session.flush() + + if secrets: + for secret_name, secret_value in secrets.items(): + await vault_svc.store_secret( + session, tenant.tenant_id, secret_name, secret_value, + target_env_var=secret_name, + ) + await session.flush() + + env_secrets = await vault_svc.get_decrypted_secrets(session, tenant.tenant_id) + orch.run_init_job( + tenant, + provider_env_var=provider_spec.env_var, + provider_model=provider_spec.model_id, + env_secrets=env_secrets, + ) + + orch.configure_tenant(tenant.tenant_id, provider_spec=provider_spec) + await svc.start_tenant(session, tenant.tenant_id) + try: + verified = await svc.verify_tenant(session, tenant.tenant_id) + except Exception as exc: + await svc.mark_verification_failed(session, tenant.tenant_id, str(exc)) + log.error("tenant_verification_failed", tenant_id=str(tenant.tenant_id), error=str(exc)) + raise + + await session.commit() + return ProvisionedTenant( + tenant_id=str(verified.tenant_id), + slug=verified.slug, + status=verified.status.value, + provider=provider_spec.key.value, + model=provider_spec.model_id, + url=f"https://{settings.edge_domain.rstrip('/')}/{verified.slug}/", + verification_status=verified.verification_status.value, + verification_error=verified.verification_error, + ) diff --git a/src/octw/api/auth.py b/src/octw/api/auth.py index ffb7ac8..d4c0b62 100644 --- a/src/octw/api/auth.py +++ b/src/octw/api/auth.py @@ -1,10 +1,12 @@ from __future__ import annotations +import ipaddress import uuid from datetime import datetime, timedelta +from typing import Any import jwt -from fastapi import Depends, HTTPException +from fastapi import Depends, HTTPException, Request, Response from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -13,6 +15,7 @@ from octw.db.tables import UserRow security = HTTPBearer(auto_error=False) +SESSION_COOKIE_NAME = "octw_session" class TokenPayload: @@ -29,7 +32,7 @@ def create_access_token( expires_minutes: int | None = None, ) -> str: exp = datetime.utcnow() + timedelta(minutes=expires_minutes or settings.jwt_expire_minutes) - payload: dict = { + payload: dict[str, Any] = { "sub": str(user_id), "email": email, "exp": exp, @@ -48,12 +51,73 @@ def decode_token(token: str) -> TokenPayload: return TokenPayload(user_id=uuid.UUID(data["sub"]), email=data["email"], tid=tid) +def _extract_cookie_token(request: Request) -> str | None: + token = request.cookies.get(SESSION_COOKIE_NAME) + return token.strip() if token else None + + +def _client_host(request: Request) -> str: + if request.client and request.client.host: + return request.client.host + return "" + + +def _ip_in_networks(host: str, configured: list[str]) -> bool: + if not host: + return False + try: + addr = ipaddress.ip_address(host) + except ValueError: + return False + for raw in configured: + try: + network = ipaddress.ip_network(raw, strict=False) + except ValueError: + continue + if addr in network: + return True + return False + + +def get_trusted_proxy_email(request: Request) -> str | None: + if not settings.trusted_proxy_enabled: + return None + host = _client_host(request) + if settings.trusted_proxy_ips and not _ip_in_networks(host, settings.trusted_proxy_ips): + return None + header = settings.trusted_proxy_user_header + email = request.headers.get(header, "").strip() + return email or None + + async def get_current_user( + request: Request, credentials: HTTPAuthorizationCredentials | None = Depends(security), ) -> TokenPayload: - if not credentials: + token: str | None = None + if credentials: + token = credentials.credentials + if not token: + token = _extract_cookie_token(request) + if not token: + raise HTTPException(status_code=401, detail="Not authenticated") + return decode_token(token) + + +async def get_browser_user( + request: Request, + session: AsyncSession, +) -> tuple[TokenPayload, bool]: + token = _extract_cookie_token(request) + if token: + return decode_token(token), False + + forwarded_email = get_trusted_proxy_email(request) + if not forwarded_email: raise HTTPException(status_code=401, detail="Not authenticated") - return decode_token(credentials.credentials) + + user = await get_or_create_user(session, forwarded_email) + return TokenPayload(user.user_id, user.email), True async def get_or_create_user( @@ -68,3 +132,15 @@ async def get_or_create_user( session.add(row) await session.flush() return row + + +def set_session_cookie(response: Response, token: str) -> None: + response.set_cookie( + key=SESSION_COOKIE_NAME, + value=token, + httponly=True, + secure=settings.public_base_url.startswith("https://"), + samesite="lax", + path="/", + max_age=settings.jwt_expire_minutes * 60, + ) diff --git a/src/octw/api/routers/app_router.py b/src/octw/api/routers/app_router.py new file mode 100644 index 0000000..fad80c2 --- /dev/null +++ b/src/octw/api/routers/app_router.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException, Request, Response +from pydantic import BaseModel +from sqlalchemy.ext.asyncio import AsyncSession + +from octw.api.app_logic import ( + build_user_tenant_name, + build_user_tenant_slug, + provision_tenant, + tenant_chat_url, +) +from octw.api.auth import create_access_token, get_browser_user, set_session_cookie +from octw.api.deps import get_orchestrator, get_tenant_service, get_vault_service +from octw.common.config import settings +from octw.db.engine import get_session +from octw.models.tenant import VerificationStatus +from octw.orchestrator.docker_orch import DockerOrchestrator +from octw.orchestrator.tenant_service import TenantService +from octw.vault.service import VaultService + +router = APIRouter(prefix="/api/v1/app", tags=["app"]) + + +class AppTenantInfo(BaseModel): + tenant_id: str + slug: str + status: str + verification_status: str + verification_error: str | None = None + chat_url: str + + +class AppSessionResponse(BaseModel): + user_id: str + email: str + tenant: AppTenantInfo | None = None + + +class DeployOrResumeResponse(BaseModel): + created: bool + tenant: AppTenantInfo + + +async def _browser_auth( + request: Request, + response: Response, + session: AsyncSession, +): + user, issued = await get_browser_user(request, session) + if issued: + await session.commit() + set_session_cookie(response, create_access_token(user.user_id, user.email)) + return user + + +async def _tenant_info(session: AsyncSession, svc: TenantService, user_id): + tenant = await svc.get_owner_tenant(session, user_id) + if not tenant: + return None + return AppTenantInfo( + tenant_id=str(tenant.tenant_id), + slug=tenant.slug, + status=tenant.status.value, + verification_status=tenant.verification_status.value, + verification_error=tenant.verification_error, + chat_url=tenant_chat_url(tenant.slug), + ) + + +@router.get("/session", response_model=AppSessionResponse) +async def get_app_session( + request: Request, + response: Response, + session: AsyncSession = Depends(get_session), + svc: TenantService = Depends(get_tenant_service), +): + user = await _browser_auth(request, response, session) + tenant = await _tenant_info(session, svc, user.user_id) + return AppSessionResponse(user_id=str(user.user_id), email=user.email, tenant=tenant) + + +@router.post("/deploy-or-resume", response_model=DeployOrResumeResponse) +async def deploy_or_resume( + request: Request, + response: Response, + session: AsyncSession = Depends(get_session), + svc: TenantService = Depends(get_tenant_service), + orch: DockerOrchestrator = Depends(get_orchestrator), + vault_svc: VaultService = Depends(get_vault_service), +): + user = await _browser_auth(request, response, session) + tenant = await svc.get_owner_tenant(session, user.user_id) + created = False + + if tenant is None: + try: + provisioned = await provision_tenant( + session=session, + svc=svc, + orch=orch, + vault_svc=vault_svc, + user=user, + slug=build_user_tenant_slug(user.user_id), + name=build_user_tenant_name(user.email), + provider_key=settings.default_provider, + ) + except Exception as exc: + await session.rollback() + raise HTTPException(status_code=500, detail=str(exc)) from exc + created = True + tenant_info = AppTenantInfo( + tenant_id=provisioned.tenant_id, + slug=provisioned.slug, + status=provisioned.status, + verification_status=provisioned.verification_status, + verification_error=provisioned.verification_error, + chat_url=tenant_chat_url(provisioned.slug), + ) + return DeployOrResumeResponse(created=created, tenant=tenant_info) + + try: + await svc.wake_tenant(session, tenant.tenant_id) + if tenant.verification_status != VerificationStatus.VERIFIED: + await svc.verify_tenant(session, tenant.tenant_id) + await session.commit() + except Exception as exc: + await session.rollback() + raise HTTPException(status_code=500, detail=str(exc)) from exc + + refreshed = await svc.get_tenant(session, tenant.tenant_id) + if not refreshed: + raise HTTPException(status_code=404, detail="Tenant disappeared") + tenant_info = AppTenantInfo( + tenant_id=str(refreshed.tenant_id), + slug=refreshed.slug, + status=refreshed.status.value, + verification_status=refreshed.verification_status.value, + verification_error=refreshed.verification_error, + chat_url=tenant_chat_url(refreshed.slug), + ) + return DeployOrResumeResponse(created=created, tenant=tenant_info) diff --git a/src/octw/api/routers/auth_router.py b/src/octw/api/routers/auth_router.py index 1fd525b..8889af5 100644 --- a/src/octw/api/routers/auth_router.py +++ b/src/octw/api/routers/auth_router.py @@ -3,11 +3,11 @@ import secrets from datetime import datetime, timedelta -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Response from pydantic import BaseModel from sqlalchemy.ext.asyncio import AsyncSession -from octw.api.auth import create_access_token, get_or_create_user +from octw.api.auth import create_access_token, get_or_create_user, set_session_cookie from octw.db.engine import get_session router = APIRouter(prefix="/auth", tags=["auth"]) @@ -42,7 +42,11 @@ async def login(req: LoginRequest, session: AsyncSession = Depends(get_session)) @router.post("/verify") -async def verify(req: VerifyRequest, session: AsyncSession = Depends(get_session)): +async def verify( + req: VerifyRequest, + response: Response, + session: AsyncSession = Depends(get_session), +): pending = _pending_tokens.pop(req.token, None) if not pending or pending["expires"] < datetime.utcnow(): raise HTTPException(status_code=401, detail="Invalid or expired token") @@ -50,6 +54,7 @@ async def verify(req: VerifyRequest, session: AsyncSession = Depends(get_session user = await get_or_create_user(session, pending["email"]) await session.commit() access_token = create_access_token(user.user_id, user.email) + set_session_cookie(response, access_token) return SessionResponse( access_token=access_token, user_id=str(user.user_id), diff --git a/src/octw/api/routers/internal_router.py b/src/octw/api/routers/internal_router.py index cca8f2a..1d7017d 100644 --- a/src/octw/api/routers/internal_router.py +++ b/src/octw/api/routers/internal_router.py @@ -19,7 +19,6 @@ async def _resolve_tenant_id( session: AsyncSession, svc: TenantService, ) -> uuid.UUID: - """Resolve a tenant identifier that may be a UUID or a slug.""" try: return uuid.UUID(identifier) except ValueError: @@ -42,6 +41,20 @@ async def ensure_running( return {"status": "running", "containerId": container_id} +@router.get("/{identifier}/access/{user_id}") +async def check_access( + identifier: str, + user_id: uuid.UUID, + session: AsyncSession = Depends(get_session), + svc: TenantService = Depends(get_tenant_service), +): + tenant_id = await _resolve_tenant_id(identifier, session, svc) + role = await svc.check_membership(session, tenant_id, user_id) + if role is None: + raise HTTPException(status_code=403, detail="Forbidden") + return {"allowed": True, "role": role.value, "tenant_id": str(tenant_id)} + + @router.post("/{identifier}/pause-if-idle") async def pause_if_idle( identifier: str, diff --git a/src/octw/api/routers/provision_router.py b/src/octw/api/routers/provision_router.py index 1a5906a..1cede1b 100644 --- a/src/octw/api/routers/provision_router.py +++ b/src/octw/api/routers/provision_router.py @@ -3,17 +3,15 @@ from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, Field -from sqlalchemy import update from sqlalchemy.ext.asyncio import AsyncSession +from octw.api.app_logic import provision_tenant as provision_tenant_impl from octw.api.auth import TokenPayload, get_current_user from octw.api.deps import get_orchestrator, get_tenant_service, get_vault_service from octw.common.config import settings from octw.common.logging import get_logger from octw.db.engine import get_session -from octw.db.tables import TenantRow from octw.models.provider import PROVIDERS, get_provider -from octw.models.tenant import TenantCreate, TenantPlan from octw.orchestrator.docker_orch import DockerOrchestrator from octw.orchestrator.tenant_service import TenantService from octw.vault.service import VaultService @@ -40,6 +38,8 @@ class ProvisionResponse(BaseModel): provider: str model: str url: str + verification_status: str + verification_error: str | None = None @router.get("/providers") @@ -77,10 +77,11 @@ async def provision_tenant( """ # 0. Validate provider try: - provider_spec = get_provider(req.provider) + get_provider(req.provider) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) from e + provider_spec = get_provider(req.provider) api_key = settings.get_provider_api_key(provider_spec.env_var) if not api_key: raise HTTPException( @@ -91,82 +92,42 @@ async def provision_tenant( ), ) - existing = await svc.get_tenant_by_slug(session, req.slug) - if existing: - raise HTTPException( - status_code=409, - detail=f"Tenant with slug '{req.slug}' already exists", - ) - - # 1. Create tenant - tenant_req = TenantCreate( - slug=req.slug, - name=req.name, - plan=TenantPlan.STANDARD, - trusted_proxy_auth=True, - ) - tenant = await svc.create_tenant(session, tenant_req, creator_user_id=user.user_id) - await session.flush() - - # Store the provider choice on the tenant row - await session.execute( - update(TenantRow) - .where(TenantRow.tenant_id == tenant.tenant_id) - .values(provider=provider_spec.key.value) - ) - await session.flush() - - # 2. Store per-tenant secrets (if any beyond the shared provider key) - if req.secrets: - for secret_name, secret_value in req.secrets.items(): - await vault_svc.store_secret( - session, tenant.tenant_id, secret_name, secret_value, - target_env_var=secret_name, - ) - await session.flush() - - # 3. Run OpenClaw onboarding init job try: - env_secrets = await vault_svc.get_decrypted_secrets(session, tenant.tenant_id) - orch.run_init_job( - tenant, - provider_env_var=provider_spec.env_var, - provider_model=provider_spec.model_id, - env_secrets=env_secrets, + provisioned = await provision_tenant_impl( + session=session, + svc=svc, + orch=orch, + vault_svc=vault_svc, + user=user, + slug=req.slug, + name=req.name, + provider_key=req.provider, + secrets=req.secrets, ) except RuntimeError as e: log.error( "provision_init_failed", - tenant_id=str(tenant.tenant_id), + slug=req.slug, error=str(e), ) raise HTTPException(status_code=500, detail=f"Onboarding failed: {e}") from e - # 4. Configure gateway + model provider in openclaw.json - orch.configure_tenant(tenant.tenant_id, provider_spec=provider_spec) - - # 5. Start the tenant container - await svc.start_tenant(session, tenant.tenant_id) - await session.commit() - - # 6. Build access URL - base_url = settings.edge_domain.rstrip("/") - url = f"https://{base_url}/{req.slug}/" - log.info( "tenant_provisioned", - tenant_id=str(tenant.tenant_id), - slug=req.slug, - provider=provider_spec.key.value, - model=provider_spec.model_id, - url=url, + tenant_id=provisioned.tenant_id, + slug=provisioned.slug, + provider=provisioned.provider, + model=provisioned.model, + url=provisioned.url, ) return ProvisionResponse( - tenant_id=str(tenant.tenant_id), - slug=req.slug, - status="running", - provider=provider_spec.key.value, - model=provider_spec.model_id, - url=url, + tenant_id=provisioned.tenant_id, + slug=provisioned.slug, + status=provisioned.status, + provider=provisioned.provider, + model=provisioned.model, + url=provisioned.url, + verification_status=provisioned.verification_status, + verification_error=provisioned.verification_error, ) diff --git a/src/octw/common/config.py b/src/octw/common/config.py index 6315fe8..1adcf73 100644 --- a/src/octw/common/config.py +++ b/src/octw/common/config.py @@ -1,5 +1,6 @@ from __future__ import annotations +from pydantic import Field, field_validator from pydantic_settings import BaseSettings @@ -16,6 +17,9 @@ class OCTWSettings(BaseSettings): edge_listen_host: str = "0.0.0.0" edge_listen_port: int = 8443 edge_domain: str = "octw.example.com" + edge_container_name: str = "octw-edge" + api_internal_base_url: str = "http://octw-api:8000" + public_base_url: str = "https://octw.example.com" jwt_secret: str = "CHANGE-ME-IN-PRODUCTION" jwt_algorithm: str = "HS256" @@ -27,6 +31,7 @@ class OCTWSettings(BaseSettings): default_cpu_quota: int = 100000 # microseconds per period default_cpu_period: int = 100000 default_pids_limit: int = 512 + default_provider: str = "zai" idle_pause_seconds: int = 1800 # 30 min idle_stop_seconds: int = 28800 # 8 hours @@ -36,11 +41,25 @@ class OCTWSettings(BaseSettings): moonshot_api_key: str | None = None minimax_api_key: str | None = None + # Browser auth via reverse proxy. + trusted_proxy_enabled: bool = False + trusted_proxy_user_header: str = "X-Forwarded-Email" + trusted_proxy_ips: list[str] = Field(default_factory=list) + # WebChat is enabled by default during onboarding. webchat_port: int = 18790 log_level: str = "INFO" + @field_validator("trusted_proxy_ips", mode="before") + @classmethod + def _parse_proxy_ips(cls, value): + if value in (None, "", []): + return [] + if isinstance(value, str): + return [item.strip() for item in value.split(",") if item.strip()] + return value + def get_provider_api_key(self, env_var: str) -> str | None: mapping = { "ZAI_API_KEY": self.zai_api_key, diff --git a/src/octw/db/engine.py b/src/octw/db/engine.py index 8f5096d..44362e0 100644 --- a/src/octw/db/engine.py +++ b/src/octw/db/engine.py @@ -1,13 +1,50 @@ from __future__ import annotations -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy import text +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + AsyncSession, + async_sessionmaker, + create_async_engine, +) from octw.common.config import settings +from octw.db.tables import Base engine = create_async_engine(settings.db_url, echo=False, pool_size=10, max_overflow=20) async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) +async def ensure_schema(db_engine: AsyncEngine | None = None) -> None: + target = db_engine or engine + async with target.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + dialect = conn.dialect.name + if dialect == "postgresql": + await conn.execute( + text("ALTER TABLE tenants ADD COLUMN IF NOT EXISTS owner_user_id UUID") + ) + await conn.execute( + text( + "ALTER TABLE tenants " + "ADD COLUMN IF NOT EXISTS verification_status " + "VARCHAR(20) DEFAULT 'pending'" + ) + ) + await conn.execute( + text("ALTER TABLE tenants ADD COLUMN IF NOT EXISTS verification_error TEXT") + ) + await conn.execute( + text("ALTER TABLE tenants ADD COLUMN IF NOT EXISTS verified_at TIMESTAMP") + ) + await conn.execute( + text( + "UPDATE tenants SET verification_status = " + "COALESCE(verification_status, 'pending')" + ) + ) + + async def get_session() -> AsyncSession: # type: ignore[misc] async with async_session() as session: yield session # type: ignore[misc] diff --git a/src/octw/db/tables.py b/src/octw/db/tables.py index 9c15778..4b147a4 100644 --- a/src/octw/db/tables.py +++ b/src/octw/db/tables.py @@ -9,6 +9,7 @@ Index, LargeBinary, String, + Text, UniqueConstraint, func, ) @@ -28,8 +29,12 @@ class TenantRow(Base): ) slug: Mapped[str] = mapped_column(String(50), unique=True, nullable=False) name: Mapped[str] = mapped_column(String(200), nullable=False) + owner_user_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), nullable=True) plan: Mapped[str] = mapped_column(String(20), default="standard") status: Mapped[str] = mapped_column(String(20), default="provisioning") + verification_status: Mapped[str] = mapped_column(String(20), default="pending") + verification_error: Mapped[str | None] = mapped_column(Text, nullable=True) + verified_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) isolation_mode: Mapped[str] = mapped_column(String(20), default="standard") resource_limits: Mapped[dict | None] = mapped_column(JSON, nullable=True) trusted_proxy_auth: Mapped[bool] = mapped_column(default=True) diff --git a/src/octw/edge/proxy.py b/src/octw/edge/proxy.py index ed95391..aba133b 100644 --- a/src/octw/edge/proxy.py +++ b/src/octw/edge/proxy.py @@ -6,12 +6,17 @@ """ from __future__ import annotations +import asyncio import re +from collections.abc import AsyncIterator import httpx -from fastapi import FastAPI, HTTPException, Request, Response +import websockets +from fastapi import FastAPI, HTTPException, Request, Response, WebSocket, WebSocketDisconnect +from websockets.exceptions import ConnectionClosed from octw.api.auth import TokenPayload, decode_token +from octw.common.config import settings from octw.common.logging import get_logger from octw.orchestrator.docker_orch import OPENCLAW_GATEWAY_PORT @@ -20,10 +25,10 @@ edge_app = FastAPI(title="OCTW Edge Proxy") _http_client: httpx.AsyncClient | None = None -_INTERNAL_API = "http://127.0.0.1:8000" +_INTERNAL_API = settings.api_internal_base_url.rstrip("/") _SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9\-]*[a-z0-9]$") -_RESERVED_PATHS = {"api", "internal", "health", "metrics"} +_RESERVED_PATHS = {"api", "internal", "health", "metrics", "app", "t"} async def _get_client() -> httpx.AsyncClient: @@ -33,7 +38,7 @@ async def _get_client() -> httpx.AsyncClient: return _http_client -def _authenticate(request: Request) -> TokenPayload | None: +def _authenticate_request(request: Request) -> TokenPayload | None: auth = request.headers.get("authorization", "") if auth.startswith("Bearer "): try: @@ -49,17 +54,38 @@ def _authenticate(request: Request) -> TokenPayload | None: return None -async def _resolve_and_ensure(slug: str) -> str: - """Resolve tenant slug to container IP, waking if necessary.""" +def _authenticate_websocket(websocket: WebSocket) -> TokenPayload | None: + auth = websocket.headers.get("authorization", "") + if auth.startswith("Bearer "): + try: + return decode_token(auth[7:]) + except Exception: + pass + cookie_header = websocket.headers.get("cookie", "") + for chunk in cookie_header.split(";"): + name, _, value = chunk.strip().partition("=") + if name == "octw_session" and value: + try: + return decode_token(value) + except Exception: + pass + return None + + +async def _check_access(slug: str, user: TokenPayload) -> None: client = await _get_client() + resp = await client.get(f"{_INTERNAL_API}/internal/v1/tenants/{slug}/access/{user.user_id}") + if resp.status_code == 403: + raise HTTPException(status_code=403, detail="Forbidden") + if resp.status_code != 200: + raise HTTPException(status_code=404, detail=f"Tenant '{slug}' not found") - # Look up tenant by slug to get tenant_id - await client.get(f"{_INTERNAL_API}/api/v1/tenants", params={"slug": slug}) - # Check current status - status_resp = await client.get( - f"{_INTERNAL_API}/internal/v1/tenants/{slug}/status" - ) +async def _resolve_and_ensure(slug: str, user: TokenPayload) -> str: + client = await _get_client() + await _check_access(slug, user) + + status_resp = await client.get(f"{_INTERNAL_API}/internal/v1/tenants/{slug}/status") if status_resp.status_code != 200: raise HTTPException(status_code=404, detail=f"Tenant '{slug}' not found") @@ -71,14 +97,9 @@ async def _resolve_and_ensure(slug: str) -> str: return ip if state in ("paused", "stopped", "not_found"): - ensure = await client.post( - f"{_INTERNAL_API}/internal/v1/tenants/{slug}/ensure-running" - ) + ensure = await client.post(f"{_INTERNAL_API}/internal/v1/tenants/{slug}/ensure-running") if ensure.status_code == 200: - # Re-fetch IP after wake - status_resp = await client.get( - f"{_INTERNAL_API}/internal/v1/tenants/{slug}/status" - ) + status_resp = await client.get(f"{_INTERNAL_API}/internal/v1/tenants/{slug}/status") data = status_resp.json() ip = data.get("ip") if ip: @@ -87,6 +108,103 @@ async def _resolve_and_ensure(slug: str) -> str: raise HTTPException(status_code=503, detail="Tenant is not available") +def _proxy_headers_from_request(request: Request, slug: str, user: TokenPayload) -> dict[str, str]: + headers = dict(request.headers) + headers.pop("host", None) + headers.pop("origin", None) + headers.pop("x-octw-user-email", None) + headers.pop("x-forwarded-for", None) + headers["x-octw-tenant-slug"] = slug + headers["x-octw-user-id"] = str(user.user_id) + headers["x-octw-user-email"] = user.email + headers["x-forwarded-for"] = request.client.host if request.client else "unknown" + headers["x-forwarded-proto"] = "https" + headers["x-forwarded-host"] = request.headers.get("host", settings.edge_domain) + origin = request.headers.get("origin") + if origin: + headers["origin"] = origin + return headers + + +@edge_app.websocket("/t/{slug}/ws") +async def tenant_websocket(slug: str, websocket: WebSocket): + if slug in _RESERVED_PATHS or not _SLUG_RE.match(slug): + await websocket.close(code=4404) + return + + user = _authenticate_websocket(websocket) + if not user: + await websocket.close(code=4401) + return + + try: + ip = await _resolve_and_ensure(slug, user) + except HTTPException as exc: + await websocket.close(code=4403 if exc.status_code == 403 else 4404) + return + + await websocket.accept() + upstream_headers = { + "origin": settings.public_base_url.rstrip("/"), + "x-octw-user-id": str(user.user_id), + "x-octw-user-email": user.email, + "x-forwarded-for": websocket.client.host if websocket.client else "unknown", + "x-forwarded-proto": "https", + "x-forwarded-host": websocket.headers.get("host", settings.edge_domain), + } + upstream_url = f"ws://{ip}:{OPENCLAW_GATEWAY_PORT}" + + try: + async with websockets.connect( + upstream_url, additional_headers=upstream_headers + ) as upstream: + async def client_to_upstream() -> None: + while True: + message = await websocket.receive() + if message.get("type") == "websocket.disconnect": + break + if message.get("text") is not None: + await upstream.send(message["text"]) + elif message.get("bytes") is not None: + await upstream.send(message["bytes"]) + + async def upstream_to_client() -> None: + async for message in _iter_upstream_messages(upstream): + if isinstance(message, bytes): + await websocket.send_bytes(message) + else: + await websocket.send_text(message) + + done, pending = await asyncio.wait( + [ + asyncio.create_task(client_to_upstream()), + asyncio.create_task(upstream_to_client()), + ], + return_when=asyncio.FIRST_COMPLETED, + ) + for task in pending: + task.cancel() + for task in done: + exc = task.exception() + if exc and not isinstance(exc, WebSocketDisconnect): + raise exc + except ConnectionClosed: + pass + except WebSocketDisconnect: + pass + except Exception as exc: + log.warning("tenant_ws_proxy_failed", slug=slug, error=str(exc)) + await websocket.close(code=1011) + + +async def _iter_upstream_messages(upstream) -> AsyncIterator[str | bytes]: + while True: + try: + yield await upstream.recv() + except ConnectionClosed: + break + + @edge_app.api_route( "/{slug}/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"], @@ -95,24 +213,17 @@ async def proxy_path(slug: str, path: str, request: Request): if slug in _RESERVED_PATHS or not _SLUG_RE.match(slug): raise HTTPException(status_code=404, detail="Not found") - user = _authenticate(request) + user = _authenticate_request(request) if not user: raise HTTPException(status_code=401, detail="Authentication required") - ip = await _resolve_and_ensure(slug) + ip = await _resolve_and_ensure(slug, user) upstream_url = f"http://{ip}:{OPENCLAW_GATEWAY_PORT}/{path}" if request.url.query: upstream_url += f"?{request.url.query}" client = await _get_client() - headers = dict(request.headers) - headers.pop("host", None) - headers["x-octw-tenant-slug"] = slug - headers["x-octw-user-id"] = str(user.user_id) - headers["x-octw-user-email"] = user.email - headers.pop("x-forwarded-for", None) - headers["x-forwarded-for"] = request.client.host if request.client else "unknown" - headers["x-forwarded-proto"] = "https" + headers = _proxy_headers_from_request(request, slug, user) body = await request.body() resp = await client.request( @@ -122,10 +233,14 @@ async def proxy_path(slug: str, path: str, request: Request): content=body, ) + response_headers = { + key: value for key, value in resp.headers.items() + if key.lower() not in {"content-encoding", "transfer-encoding", "connection"} + } return Response( content=resp.content, status_code=resp.status_code, - headers=dict(resp.headers), + headers=response_headers, ) @@ -134,7 +249,6 @@ async def proxy_path(slug: str, path: str, request: Request): methods=["GET"], ) async def proxy_slug_root(slug: str, request: Request): - """Handle / without trailing slash.""" if slug in _RESERVED_PATHS or not _SLUG_RE.match(slug): raise HTTPException(status_code=404, detail="Not found") return await proxy_path(slug, "", request) diff --git a/src/octw/models/tenant.py b/src/octw/models/tenant.py index efc0cab..60d86f1 100644 --- a/src/octw/models/tenant.py +++ b/src/octw/models/tenant.py @@ -16,6 +16,12 @@ class TenantStatus(enum.StrEnum): DELETING = "deleting" +class VerificationStatus(enum.StrEnum): + PENDING = "pending" + VERIFIED = "verified" + FAILED = "failed" + + class ContainerState(enum.StrEnum): RUNNING = "running" PAUSED = "paused" @@ -56,6 +62,10 @@ class Tenant(BaseModel): name: str plan: TenantPlan = TenantPlan.STANDARD status: TenantStatus = TenantStatus.PROVISIONING + verification_status: VerificationStatus = VerificationStatus.PENDING + verification_error: str | None = None + verified_at: datetime | None = None + owner_user_id: uuid.UUID | None = None isolation_mode: IsolationMode = IsolationMode.STANDARD resource_limits: ResourceLimits = Field(default_factory=ResourceLimits) trusted_proxy_auth: bool = True diff --git a/src/octw/orchestrator/docker_orch.py b/src/octw/orchestrator/docker_orch.py index 4f6e11d..a31d054 100644 --- a/src/octw/orchestrator/docker_orch.py +++ b/src/octw/orchestrator/docker_orch.py @@ -2,6 +2,9 @@ import os import stat +import subprocess +import textwrap +import time import uuid from pathlib import Path @@ -21,6 +24,82 @@ CONTAINER_STATE_DIR = f"{CONTAINER_HOME}/.openclaw" CONTAINER_WORKSPACE_DIR = f"{CONTAINER_HOME}/.openclaw/workspace" +WS_PROBE_SCRIPT = textwrap.dedent( + """ + import asyncio + import json + import sys + import websockets + + async def main() -> int: + uri, origin, email = sys.argv[1:4] + async with websockets.connect( + uri, + additional_headers={ + "Origin": origin, + "x-octw-user-email": email, + "x-forwarded-for": "127.0.0.1", + "x-forwarded-proto": "https", + "x-forwarded-host": origin.replace("https://", "").replace("http://", ""), + }, + open_timeout=10, + close_timeout=5, + max_size=1_000_000, + ) as ws: + challenge = None + try: + first = json.loads(await asyncio.wait_for(ws.recv(), timeout=0.75)) + except asyncio.TimeoutError: + first = None + if first and first.get("type") == "event" and first.get("event") == "connect.challenge": + payload = first.get("payload") or {} + challenge = payload.get("nonce") + req = { + "type": "req", + "id": "octw-probe", + "method": "connect", + "params": { + "minProtocol": 3, + "maxProtocol": 3, + "client": { + "id": "control-ui", + "version": "octw-probe", + "platform": "linux", + "mode": "webchat", + }, + "role": "operator", + "scopes": ["operator.read", "operator.write", "operator.admin"], + "caps": [], + "commands": [], + "permissions": {}, + "locale": "en-US", + "userAgent": "octw-probe", + }, + } + if challenge: + req["params"]["challengeNonce"] = challenge + await ws.send(json.dumps(req)) + response = json.loads(await ws.recv()) + if response.get("type") != "res" or not response.get("ok"): + raise RuntimeError(json.dumps(response)) + payload = response.get("payload") or {} + if payload.get("type") != "hello-ok": + raise RuntimeError(json.dumps(response)) + await ws.send(json.dumps({ + "type": "req", + "id": "octw-status", + "method": "health", + "params": {}, + })) + health = json.loads(await ws.recv()) + if health.get("type") != "res" or not health.get("ok"): + raise RuntimeError(json.dumps(health)) + return 0 + + raise SystemExit(asyncio.run(main())) + """ +).strip() + class DockerOrchestrator: def __init__(self) -> None: @@ -42,6 +121,9 @@ def _image_ref(self, tenant: Tenant) -> str: return f"{ref}@{tenant.openclaw_digest}" return tenant.openclaw_image or settings.openclaw_image + def _edge_container(self): + return self._client.containers.get(settings.edge_container_name) + # --- Filesystem --- def create_tenant_dirs(self, tenant_id: uuid.UUID) -> dict[str, str]: @@ -61,17 +143,16 @@ def create_tenant_dirs(self, tenant_id: uuid.UUID) -> dict[str, str]: f"sudo chown $USER {settings.tenant_base_dir}" ) from None try: - os.chmod(d, stat.S_IRWXU) # 0700 + os.chmod(d, stat.S_IRWXU) if os.getuid() == 0: os.chown(d, 1000, 1000) except OSError: - pass # already exists with correct permissions + pass log.info("tenant_dirs_created", tenant_id=str(tenant_id), dirs=dirs) return dirs def remove_tenant_dirs(self, tenant_id: uuid.UUID) -> None: import shutil - import subprocess base = self._tenant_dir(tenant_id) if not base.exists(): @@ -79,12 +160,7 @@ def remove_tenant_dirs(self, tenant_id: uuid.UUID) -> None: try: shutil.rmtree(base) except PermissionError: - # Dirs may be owned by UID 1000; use sudo if available - subprocess.run( - ["sudo", "rm", "-rf", str(base)], - check=True, - capture_output=True, - ) + subprocess.run(["sudo", "rm", "-rf", str(base)], check=True, capture_output=True) log.info("tenant_dirs_removed", tenant_id=str(tenant_id)) # --- Network --- @@ -114,13 +190,34 @@ def remove_network(self, tenant_id: uuid.UUID) -> None: except NotFound: pass - def connect_edge_to_network(self, tenant_id: uuid.UUID, edge_container: str) -> None: + def connect_edge_to_network( + self, tenant_id: uuid.UUID, edge_container: str | None = None + ) -> str | None: name = self._network_name(tenant_id) + target = edge_container or settings.edge_container_name try: net = self._client.networks.get(name) - net.connect(edge_container) + net.reload() + connected = net.attrs.get("Containers", {}) or {} + edge = self._client.containers.get(target) + if edge.id not in connected: + net.connect(edge) + log.info("edge_connected", tenant_id=str(tenant_id), edge_container=edge.name) + return self.get_named_container_ip(target, name) except Exception as e: log.warning("edge_connect_failed", tenant_id=str(tenant_id), error=str(e)) + return None + + def get_named_container_ip(self, container_name: str, network_name: str) -> str | None: + try: + container = self._client.containers.get(container_name) + except NotFound: + return None + nets = container.attrs.get("NetworkSettings", {}).get("Networks", {}) + info = nets.get(network_name) + if info: + return info.get("IPAddress") + return None # --- Init / Onboarding --- @@ -132,7 +229,6 @@ def run_init_job( env_secrets: dict[str, str] | None = None, timeout: int = 120, ) -> str: - """Run a one-shot container to onboard OpenClaw for this tenant.""" init_name = f"octw_init_{tenant.tenant_id}" image = self._image_ref(tenant) dirs = self.create_tenant_dirs(tenant.tenant_id) @@ -186,10 +282,6 @@ def run_init_job( exit_code = result.get("StatusCode", -1) container.remove() - # The onboard command writes config/workspace then tries to start the - # gateway and connect via WebSocket. In a one-shot init container the - # gateway isn't running, so it exits 1 with a WS close error. This is - # expected -- check that the config file was actually created. config_created = "Updated" in output and "openclaw.json" in output if exit_code != 0 and not config_created: log.error( @@ -212,12 +304,15 @@ def run_init_job( log.info("init_job_completed", tenant_id=str(tenant.tenant_id)) return output + def tenant_config_exists(self, tenant_id: uuid.UUID) -> bool: + state_dir = Path(settings.tenant_base_dir) / str(tenant_id) / "state" + return (state_dir / "openclaw.json").exists() + def configure_tenant( self, tenant_id: uuid.UUID, provider_spec: object | None = None, ) -> None: - """Configure gateway and model provider in OpenClaw config.""" state_dir = Path(settings.tenant_base_dir) / str(tenant_id) / "state" config_path = state_dir / "openclaw.json" if not config_path.exists(): @@ -232,45 +327,42 @@ def configure_tenant( config = json.loads(config_path.read_text()) - # Gateway: bind to LAN so edge proxy can reach it config["gateway"] = config.get("gateway", {}) config["gateway"]["bind"] = "lan" config["gateway"]["port"] = OPENCLAW_GATEWAY_PORT - config["gateway"]["controlUi"] = config["gateway"].get("controlUi", {}) - config["gateway"]["controlUi"][ - "dangerouslyAllowHostHeaderOriginFallback" - ] = True + edge_ip = self.connect_edge_to_network(tenant_id) + config["gateway"]["trustedProxies"] = [ip for ip in [edge_ip] if ip] + config["gateway"]["auth"] = config["gateway"].get("auth", {}) + config["gateway"]["auth"]["mode"] = "trusted-proxy" + config["gateway"]["auth"]["trustedProxy"] = { + "userHeader": "x-octw-user-email", + "requiredHeaders": ["x-forwarded-proto", "x-forwarded-host"], + } + control_ui = config["gateway"].get("controlUi", {}) + control_ui["allowedOrigins"] = [settings.public_base_url.rstrip("/")] + control_ui.pop("dangerouslyAllowHostHeaderOriginFallback", None) + control_ui.pop("allowInsecureAuth", None) + control_ui.pop("dangerouslyDisableDeviceAuth", None) + config["gateway"]["controlUi"] = control_ui - # Model provider if spec: - # Set primary model in agents.defaults.model config["agents"] = config.get("agents", {}) config["agents"]["defaults"] = config["agents"].get("defaults", {}) - config["agents"]["defaults"]["model"] = { - "primary": spec.model_id, - } + config["agents"]["defaults"]["model"] = {"primary": spec.model_id} if spec.builtin: - # Built-in providers only need the env var set (done at - # container start) and the model selection above. - # Remove any stale custom providers entry for this provider. providers = config.get("models", {}).get("providers", {}) providers.pop(spec.provider_name, None) else: - # Custom provider: write full models.providers entry config["models"] = config.get("models", {}) config["models"]["mode"] = "merge" providers = config["models"].get("providers", {}) - provider_entry: dict = { - "apiKey": f"${{{spec.env_var}}}", - } + provider_entry: dict[str, object] = {"apiKey": f"${{{spec.env_var}}}"} if spec.base_url: provider_entry["baseUrl"] = spec.base_url if spec.api_type: provider_entry["api"] = spec.api_type - provider_entry["models"] = [ - {"id": spec.model_name, "name": spec.display_name}, - ] + provider_entry["models"] = [{"id": spec.model_name, "name": spec.display_name}] providers[spec.provider_name] = provider_entry config["models"]["providers"] = providers @@ -299,7 +391,6 @@ def start_container( "OPENCLAW_GATEWAY_PORT": str(OPENCLAW_GATEWAY_PORT), "OPENCLAW_CANVAS_PORT": str(OPENCLAW_CANVAS_PORT), } - # Inject the selected provider's API key from server config if provider_env_var: api_key = settings.get_provider_api_key(provider_env_var) if api_key: @@ -308,32 +399,23 @@ def start_container( env.update(env_secrets) mounts = [ - Mount( - target=CONTAINER_STATE_DIR, - source=dirs["state"], - type="bind", - ), - Mount( - target=CONTAINER_WORKSPACE_DIR, - source=dirs["workspace"], - type="bind", - ), + Mount(target=CONTAINER_STATE_DIR, source=dirs["state"], type="bind"), + Mount(target=CONTAINER_WORKSPACE_DIR, source=dirs["workspace"], type="bind"), ] - runtime = None - if tenant.isolation_mode == IsolationMode.HARDENED: - runtime = "runsc" - + runtime = "runsc" if tenant.isolation_mode == IsolationMode.HARDENED else None security_opt = ["no-new-privileges:true"] try: existing = self._client.containers.get(name) if existing.status == "paused": existing.unpause() + self.connect_edge_to_network(tenant.tenant_id) log.info("container_unpaused", tenant_id=str(tenant.tenant_id)) return existing.id if existing.status == "exited": existing.remove() elif existing.status == "running": + self.connect_edge_to_network(tenant.tenant_id) return existing.id else: existing.remove(force=True) @@ -342,6 +424,7 @@ def start_container( network_name = self._network_name(tenant.tenant_id) self.create_network(tenant.tenant_id) + self.connect_edge_to_network(tenant.tenant_id) container = self._client.containers.run( image=image, @@ -365,17 +448,13 @@ def start_container( restart_policy={"Name": "unless-stopped"}, healthcheck={ "test": ["CMD", "curl", "-sf", f"http://localhost:{OPENCLAW_GATEWAY_PORT}/health"], - "interval": 30_000_000_000, # 30s in ns + "interval": 30_000_000_000, "timeout": 5_000_000_000, "retries": 3, "start_period": 10_000_000_000, }, ) - log.info( - "container_started", - tenant_id=str(tenant.tenant_id), - container_id=container.id, - ) + log.info("container_started", tenant_id=str(tenant.tenant_id), container_id=container.id) return container.id def stop_container(self, tenant_id: uuid.UUID, timeout: int = 30) -> None: @@ -404,6 +483,7 @@ def unpause_container(self, tenant_id: uuid.UUID) -> None: container = self._client.containers.get(name) if container.status == "paused": container.unpause() + self.connect_edge_to_network(tenant_id) log.info("container_unpaused", tenant_id=str(tenant_id)) except NotFound: pass @@ -429,7 +509,7 @@ def get_container_logs( name = self._container_name(tenant_id) try: container = self._client.containers.get(name) - kwargs: dict = {"tail": tail, "timestamps": True} + kwargs: dict[str, object] = {"tail": tail, "timestamps": True} if since: kwargs["since"] = since return container.logs(**kwargs).decode(errors="replace") @@ -437,17 +517,55 @@ def get_container_logs( return "" def get_container_ip(self, tenant_id: uuid.UUID) -> str | None: - name = self._container_name(tenant_id) - network_name = self._network_name(tenant_id) - try: - container = self._client.containers.get(name) - nets = container.attrs.get("NetworkSettings", {}).get("Networks", {}) - net_info = nets.get(network_name) - if net_info: - return net_info.get("IPAddress") - except NotFound: - pass - return None + return self.get_named_container_ip( + self._container_name(tenant_id), self._network_name(tenant_id) + ) + + def wait_for_gateway_health(self, tenant_id: uuid.UUID, timeout: int = 45) -> None: + deadline = time.time() + timeout + ip: str | None = None + while time.time() < deadline: + ip = self.get_container_ip(tenant_id) + if not ip: + time.sleep(1) + continue + result = self._edge_container().exec_run([ + "curl", "-fsS", f"http://{ip}:{OPENCLAW_GATEWAY_PORT}/health" + ]) + if result.exit_code == 0: + return + time.sleep(1) + raise RuntimeError(f"Gateway health check failed for tenant {tenant_id} (last ip={ip})") + + def verify_gateway_ws(self, tenant_id: uuid.UUID, email: str, timeout: int = 45) -> None: + deadline = time.time() + timeout + ip: str | None = None + last_output = "" + while time.time() < deadline: + ip = self.get_container_ip(tenant_id) + if not ip: + time.sleep(1) + continue + result = self._edge_container().exec_run([ + "/app/.venv/bin/python", + "-c", + WS_PROBE_SCRIPT, + f"ws://{ip}:{OPENCLAW_GATEWAY_PORT}", + settings.public_base_url.rstrip("/"), + email, + ]) + if isinstance(result.output, (bytes, bytearray)): + output = result.output.decode(errors="replace") + else: + output = str(result.output) + last_output = output.strip() + if result.exit_code == 0: + return + time.sleep(1) + raise RuntimeError( + "Gateway websocket verification failed for tenant " + f"{tenant_id}: {last_output or 'unknown error'}" + ) def cleanup_tenant(self, tenant_id: uuid.UUID) -> None: self.stop_container(tenant_id) diff --git a/src/octw/orchestrator/tenant_service.py b/src/octw/orchestrator/tenant_service.py index d52ae25..c9726b7 100644 --- a/src/octw/orchestrator/tenant_service.py +++ b/src/octw/orchestrator/tenant_service.py @@ -19,6 +19,7 @@ TenantCreate, TenantRuntimeInfo, TenantStatus, + VerificationStatus, ) from octw.orchestrator.docker_orch import DockerOrchestrator from octw.vault.service import VaultService @@ -59,8 +60,10 @@ async def create_tenant( row = TenantRow( slug=req.slug, name=req.name, + owner_user_id=creator_user_id, plan=req.plan.value, status=TenantStatus.PROVISIONING.value, + verification_status=VerificationStatus.PENDING.value, isolation_mode=req.isolation_mode.value, resource_limits=limits.model_dump(), trusted_proxy_auth=req.trusted_proxy_auth, @@ -74,6 +77,7 @@ async def create_tenant( self._orch.create_tenant_dirs(tenant.tenant_id) network_id = self._orch.create_network(tenant.tenant_id) + self._orch.connect_edge_to_network(tenant.tenant_id) row.network_id = network_id row.status = TenantStatus.STOPPED.value await session.flush() @@ -92,6 +96,7 @@ async def create_tenant( tenant.status = TenantStatus.STOPPED tenant.network_id = network_id + tenant.owner_user_id = creator_user_id log.info("tenant_created", tenant_id=str(tenant.tenant_id), slug=req.slug) return tenant @@ -110,6 +115,15 @@ async def get_tenant_by_slug( row = result.scalar_one_or_none() return self._row_to_tenant(row) if row else None + async def get_owner_tenant( + self, session: AsyncSession, owner_user_id: uuid.UUID + ) -> Tenant | None: + result = await session.execute( + select(TenantRow).where(TenantRow.owner_user_id == owner_user_id) + ) + row = result.scalar_one_or_none() + return self._row_to_tenant(row) if row else None + async def list_tenants( self, session: AsyncSession, user_id: uuid.UUID | None = None ) -> list[Tenant]: @@ -155,11 +169,15 @@ async def start_tenant( secrets = await self._vault.get_decrypted_secrets(session, tenant_id) + provider_spec = None provider_env_var: str | None = None if tenant.provider: from octw.models.provider import get_provider - spec = get_provider(tenant.provider) - provider_env_var = spec.env_var + provider_spec = get_provider(tenant.provider) + provider_env_var = provider_spec.env_var + + if self._orch.tenant_config_exists(tenant_id): + self._orch.configure_tenant(tenant_id, provider_spec=provider_spec) container_id = self._orch.start_container( tenant, env_secrets=secrets, provider_env_var=provider_env_var, @@ -208,6 +226,7 @@ async def wake_tenant( ) -> str: state = self._orch.get_container_state(tenant_id) if state == ContainerState.RUNNING: + self._orch.connect_edge_to_network(tenant_id) await self._touch_activity(session, tenant_id) tenant = await self.get_tenant(session, tenant_id) return tenant.container_id or "" @@ -224,6 +243,50 @@ async def wake_tenant( return tenant.container_id or "" return await self.start_tenant(session, tenant_id) + async def verify_tenant(self, session: AsyncSession, tenant_id: uuid.UUID) -> Tenant: + tenant = await self.get_tenant(session, tenant_id) + if not tenant: + raise ValueError(f"Tenant {tenant_id} not found") + if not tenant.owner_user_id: + raise ValueError(f"Tenant {tenant_id} has no owner for verification") + + owner = await session.get(UserRow, tenant.owner_user_id) + if not owner: + raise ValueError(f"Tenant {tenant_id} owner not found") + if not self._orch.tenant_config_exists(tenant_id): + raise RuntimeError(f"Tenant {tenant.slug} is missing openclaw.json") + + self._orch.wait_for_gateway_health(tenant_id) + self._orch.verify_gateway_ws(tenant_id, owner.email) + + await session.execute( + update(TenantRow) + .where(TenantRow.tenant_id == tenant_id) + .values( + verification_status=VerificationStatus.VERIFIED.value, + verification_error=None, + verified_at=datetime.utcnow(), + ) + ) + await session.flush() + refreshed = await self.get_tenant(session, tenant_id) + if not refreshed: + raise ValueError(f"Tenant {tenant_id} disappeared after verification") + return refreshed + + async def mark_verification_failed( + self, session: AsyncSession, tenant_id: uuid.UUID, error: str + ) -> None: + await session.execute( + update(TenantRow) + .where(TenantRow.tenant_id == tenant_id) + .values( + verification_status=VerificationStatus.FAILED.value, + verification_error=error, + ) + ) + await session.flush() + async def get_runtime_info( self, session: AsyncSession, tenant_id: uuid.UUID ) -> TenantRuntimeInfo: @@ -254,8 +317,6 @@ async def _touch_activity( ) await session.flush() - # --- Membership --- - async def _ensure_membership( self, session: AsyncSession, @@ -367,8 +428,12 @@ def _row_to_tenant(row: TenantRow) -> Tenant: tenant_id=row.tenant_id, slug=row.slug, name=row.name, + owner_user_id=row.owner_user_id, plan=row.plan, status=row.status, + verification_status=row.verification_status, + verification_error=row.verification_error, + verified_at=row.verified_at, isolation_mode=row.isolation_mode, resource_limits=limits, trusted_proxy_auth=row.trusted_proxy_auth, diff --git a/src/octw/web/static/app.css b/src/octw/web/static/app.css new file mode 100644 index 0000000..702fcaa --- /dev/null +++ b/src/octw/web/static/app.css @@ -0,0 +1,122 @@ +:root { + color-scheme: light; + --bg: #f3efe6; + --panel: rgba(255, 252, 246, 0.92); + --line: rgba(38, 34, 28, 0.12); + --text: #1f1a15; + --muted: #6d6255; + --accent: #0b7a75; + --accent-2: #f08a24; +} +* { box-sizing: border-box; } +body { + margin: 0; + min-height: 100vh; + font-family: Georgia, "Iowan Old Style", serif; + background: radial-gradient(circle at top left, #fff7df, transparent 30%), linear-gradient(160deg, #efe4d0, #f5f1ea 50%, #e7efe8); + color: var(--text); +} +.shell { + max-width: 1180px; + margin: 0 auto; + padding: 32px 20px 48px; +} +.panel { + background: var(--panel); + border: 1px solid var(--line); + border-radius: 24px; + backdrop-filter: blur(16px); + box-shadow: 0 16px 40px rgba(31, 26, 21, 0.08); +} +.hero { + padding: 48px; + min-height: 280px; +} +.hero h1 { + font-size: clamp(2.4rem, 4vw, 4.5rem); + line-height: 0.95; + max-width: 10ch; + margin: 0 0 16px; +} +.eyebrow { + text-transform: uppercase; + letter-spacing: 0.18em; + color: var(--accent); + font-size: 0.76rem; + margin-bottom: 12px; +} +.lede, .muted, .status { + color: var(--muted); +} +.button { + border: 0; + border-radius: 999px; + padding: 12px 20px; + font: inherit; + cursor: pointer; + background: rgba(31, 26, 21, 0.07); + color: var(--text); +} +.button.primary { + background: linear-gradient(135deg, var(--accent), #05504c); + color: #fff; +} +.chat-shell { + display: grid; + grid-template-columns: 280px 1fr; + gap: 18px; +} +.sidebar, .chat-panel { + padding: 24px; +} +.messages { + height: min(65vh, 720px); + overflow: auto; + display: flex; + flex-direction: column; + gap: 12px; + margin-bottom: 18px; +} +.message { + padding: 14px 16px; + border-radius: 18px; + max-width: min(82%, 780px); + white-space: pre-wrap; +} +.message.user { + background: #f4d5b5; + margin-left: auto; +} +.message.assistant { + background: #fff; + border: 1px solid var(--line); +} +.message.system { + background: #edf5f4; + border: 1px dashed rgba(11, 122, 117, 0.28); + max-width: 100%; +} +.composer { + display: flex; + flex-direction: column; + gap: 12px; +} +textarea { + width: 100%; + resize: vertical; + border-radius: 18px; + border: 1px solid var(--line); + padding: 16px; + font: inherit; + background: rgba(255,255,255,0.76); +} +.composer-actions { + display: flex; + align-items: center; + justify-content: space-between; +} +@media (max-width: 900px) { + .hero { padding: 28px; } + .chat-shell { grid-template-columns: 1fr; } + .messages { height: 56vh; } +} diff --git a/src/octw/web/static/app.js b/src/octw/web/static/app.js new file mode 100644 index 0000000..c9201a0 --- /dev/null +++ b/src/octw/web/static/app.js @@ -0,0 +1,233 @@ +const state = { + session: null, + socket: null, + requestId: 0, + pending: new Map(), + sessionKey: "main", +}; + +const el = { + landing: document.getElementById("landing"), + summary: document.getElementById("summary"), + status: document.getElementById("status"), + deployButton: document.getElementById("deploy-button"), + chatShell: document.getElementById("chat-shell"), + messages: document.getElementById("messages"), + prompt: document.getElementById("prompt"), + composer: document.getElementById("composer"), + sendButton: document.getElementById("send-button"), + resumeButton: document.getElementById("resume-button"), + connection: document.getElementById("connection"), + tenantName: document.getElementById("tenant-name"), + tenantMeta: document.getElementById("tenant-meta"), +}; + +async function fetchJson(url, init) { + const response = await fetch(url, { + credentials: "include", + headers: { "Content-Type": "application/json", ...(init?.headers || {}) }, + ...init, + }); + if (!response.ok) { + const body = await response.text(); + throw new Error(body || `${response.status} ${response.statusText}`); + } + return response.json(); +} + +function setStatus(message) { + el.status.textContent = message; +} + +function addMessage(role, text) { + const node = document.createElement("div"); + node.className = `message ${role}`; + node.textContent = text; + el.messages.appendChild(node); + el.messages.scrollTop = el.messages.scrollHeight; +} + +function normalizeText(entry) { + if (!entry || typeof entry !== "object") return ""; + if (typeof entry.text === "string") return entry.text; + if (Array.isArray(entry.content)) { + return entry.content + .map((part) => (part && typeof part.text === "string" ? part.text : "")) + .filter(Boolean) + .join("\n\n"); + } + return ""; +} + +function renderHistory(messages) { + el.messages.innerHTML = ""; + for (const message of messages || []) { + addMessage(message.role === "user" ? "user" : "assistant", normalizeText(message)); + } +} + +function nextRequestId() { + state.requestId += 1; + return `octw-${state.requestId}`; +} + +function sendRpc(method, params = {}) { + if (!state.socket || state.socket.readyState !== WebSocket.OPEN) { + throw new Error("WebSocket not connected"); + } + const id = nextRequestId(); + const payload = { type: "req", id, method, params }; + state.socket.send(JSON.stringify(payload)); + return new Promise((resolve, reject) => { + state.pending.set(id, { resolve, reject }); + window.setTimeout(() => { + if (state.pending.has(id)) { + state.pending.delete(id); + reject(new Error(`Timed out waiting for ${method}`)); + } + }, 15000); + }); +} + +async function loadHistory() { + const response = await sendRpc("chat.history", { sessionKey: state.sessionKey, limit: 100 }); + renderHistory(response.messages || []); +} + +async function connectChat() { + if (!state.session?.tenant) return; + if (state.socket && (state.socket.readyState === WebSocket.OPEN || state.socket.readyState === WebSocket.CONNECTING)) { + return; + } + const url = `${window.location.protocol === "https:" ? "wss" : "ws"}://${window.location.host}/t/${state.session.tenant.slug}/ws`; + const socket = new WebSocket(url); + state.socket = socket; + el.connection.textContent = "Connecting"; + let connectSent = false; + + const sendConnect = () => { + if (connectSent || socket.readyState !== WebSocket.OPEN) return; + connectSent = true; + socket.send(JSON.stringify({ + type: "req", + id: nextRequestId(), + method: "connect", + params: { + minProtocol: 3, + maxProtocol: 3, + client: { id: "control-ui", version: "octw-web", platform: "web", mode: "webchat" }, + role: "operator", + scopes: ["operator.read", "operator.write", "operator.admin"], + caps: [], + commands: [], + permissions: {}, + locale: navigator.language, + userAgent: navigator.userAgent, + }, + })); + }; + + socket.addEventListener("open", () => { + el.connection.textContent = "Authorizing"; + window.setTimeout(sendConnect, 700); + }); + + socket.addEventListener("message", async (event) => { + const frame = JSON.parse(event.data); + if (frame.type === "event" && frame.event === "connect.challenge") { + sendConnect(); + return; + } + if (frame.type === "res") { + const pending = state.pending.get(frame.id); + if (pending) { + state.pending.delete(frame.id); + if (frame.ok) pending.resolve(frame.payload || {}); + else pending.reject(new Error(frame.error?.message || "Gateway error")); + } + if (frame.ok && frame.payload?.type === "hello-ok") { + el.connection.textContent = "Connected"; + await loadHistory(); + } + return; + } + if (frame.type === "event" && frame.event === "chat") { + await loadHistory(); + } + }); + + socket.addEventListener("close", () => { + el.connection.textContent = "Disconnected"; + state.socket = null; + }); +} + +async function bootstrap() { + try { + state.session = await fetchJson("/api/v1/app/session"); + } catch (error) { + setStatus(`Authentication failed: ${error.message}`); + return; + } + + if (!state.session.tenant) { + el.summary.textContent = `Signed in as ${state.session.email}. Create your dedicated OpenClaw workspace.`; + el.deployButton.hidden = false; + return; + } + + el.landing.hidden = true; + el.chatShell.hidden = false; + el.tenantName.textContent = state.session.tenant.slug; + el.tenantMeta.textContent = `${state.session.email} · ${state.session.tenant.status} · ${state.session.tenant.verification_status}`; + await connectChat(); +} + +el.deployButton.addEventListener("click", async () => { + el.deployButton.disabled = true; + setStatus("Provisioning your workspace. This can take a minute or two."); + try { + const response = await fetchJson("/api/v1/app/deploy-or-resume", { method: "POST", body: "{}" }); + state.session = { ...state.session, tenant: response.tenant }; + await bootstrap(); + setStatus(response.created ? "Workspace ready." : "Workspace resumed."); + } catch (error) { + setStatus(`Provisioning failed: ${error.message}`); + } finally { + el.deployButton.disabled = false; + } +}); + +el.resumeButton.addEventListener("click", async () => { + setStatus("Resuming workspace."); + try { + const response = await fetchJson("/api/v1/app/deploy-or-resume", { method: "POST", body: "{}" }); + state.session = { ...state.session, tenant: response.tenant }; + await connectChat(); + await loadHistory(); + setStatus("Workspace connected."); + } catch (error) { + setStatus(`Resume failed: ${error.message}`); + } +}); + +el.composer.addEventListener("submit", async (event) => { + event.preventDefault(); + const message = el.prompt.value.trim(); + if (!message) return; + el.prompt.value = ""; + addMessage("user", message); + addMessage("system", "Running..."); + try { + await sendRpc("chat.send", { + sessionKey: state.sessionKey, + message, + deliver: false, + idempotencyKey: nextRequestId(), + }); + } catch (error) { + addMessage("system", `Send failed: ${error.message}`); + } +}); + +bootstrap(); diff --git a/src/octw/web/static/index.html b/src/octw/web/static/index.html new file mode 100644 index 0000000..115db19 --- /dev/null +++ b/src/octw/web/static/index.html @@ -0,0 +1,40 @@ + + + + + + OCTW Chat + + + +
+
+

OpenClaw Tenant Wrapper

+

Deploy once. Resume straight into chat.

+

Checking your workspace.

+ +

+
+ + +
+ + + diff --git a/tests/unit/test_app_v1.py b/tests/unit/test_app_v1.py new file mode 100644 index 0000000..2277112 --- /dev/null +++ b/tests/unit/test_app_v1.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import uuid +from types import SimpleNamespace + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from starlette.requests import Request + +from octw.api import auth +from octw.api.app_logic import build_user_tenant_name, build_user_tenant_slug +from octw.api.auth import TokenPayload +from octw.api.routers import app_router +from octw.api.routers.app_router import router as app_api_router +from octw.common.config import settings +from octw.models.tenant import Tenant, TenantStatus, VerificationStatus +from octw.orchestrator.docker_orch import DockerOrchestrator + + +@pytest.mark.asyncio +async def test_get_browser_user_bootstraps_from_trusted_proxy(monkeypatch): + user_id = uuid.uuid4() + + async def fake_get_or_create_user(session, email: str): + return SimpleNamespace(user_id=user_id, email=email) + + monkeypatch.setattr(settings, "trusted_proxy_enabled", True) + monkeypatch.setattr(settings, "trusted_proxy_user_header", "X-Forwarded-Email") + monkeypatch.setattr(settings, "trusted_proxy_ips", ["127.0.0.1/32"]) + monkeypatch.setattr(auth, "get_or_create_user", fake_get_or_create_user) + + request = Request( + { + "type": "http", + "headers": [(b"x-forwarded-email", b"person@example.com")], + "client": ("127.0.0.1", 1234), + } + ) + + payload, issued = await auth.get_browser_user(request, session=object()) + assert issued is True + assert payload.user_id == user_id + assert payload.email == "person@example.com" + + +def test_build_user_tenant_slug_is_deterministic(): + user_id = uuid.UUID("11111111-2222-3333-4444-555555555555") + assert build_user_tenant_slug(user_id) == "u-11111111222233334444" + assert build_user_tenant_name("alice.smith@example.com") == "Alice Smith Workspace" + + +def test_app_session_sets_cookie(monkeypatch): + app = FastAPI() + app.include_router(app_api_router) + + class FakeTenantService: + async def get_owner_tenant(self, session, user_id): + return None + + class FakeSession: + async def commit(self): + return None + + async def fake_session_dep(): + yield FakeSession() + + async def fake_browser_user(request, session): + return TokenPayload(uuid.uuid4(), "oidc@example.com"), True + + monkeypatch.setattr(app_router, "get_browser_user", fake_browser_user) + monkeypatch.setattr(app_router, "create_access_token", lambda user_id, email: "session-token") + app.dependency_overrides[app_router.get_session] = fake_session_dep + app.dependency_overrides[app_router.get_tenant_service] = lambda: FakeTenantService() + + client = TestClient(app) + response = client.get("/api/v1/app/session") + + assert response.status_code == 200 + assert response.json()["email"] == "oidc@example.com" + assert response.cookies.get("octw_session") == "session-token" + + +def test_configure_tenant_sets_trusted_proxy(monkeypatch, tmp_path): + tenant_id = uuid.uuid4() + state_dir = tmp_path / str(tenant_id) / "state" + state_dir.mkdir(parents=True) + config_path = state_dir / "openclaw.json" + config_path.write_text( + '{"gateway": {"controlUi": {"dangerouslyAllowHostHeaderOriginFallback": true}}}' + ) + + monkeypatch.setattr(settings, "tenant_base_dir", str(tmp_path)) + monkeypatch.setattr(settings, "public_base_url", "https://chat.example.com") + + orch = DockerOrchestrator.__new__(DockerOrchestrator) + orch.connect_edge_to_network = lambda tenant_id: "172.18.0.9" + + orch.configure_tenant(tenant_id, provider_spec=None) + + data = config_path.read_text() + assert '"mode": "trusted-proxy"' in data + assert '"trustedProxies": [' in data + assert '"172.18.0.9"' in data + assert '"allowedOrigins": [' in data + assert 'dangerouslyAllowHostHeaderOriginFallback' not in data + + +def test_deploy_or_resume_reuses_existing_tenant(monkeypatch): + app = FastAPI() + app.include_router(app_api_router) + + tenant = Tenant( + tenant_id=uuid.uuid4(), + slug="u-tenant", + name="Workspace", + owner_user_id=uuid.uuid4(), + status=TenantStatus.STOPPED, + verification_status=VerificationStatus.VERIFIED, + ) + + class FakeTenantService: + def __init__(self): + self.wake_calls = 0 + + async def get_owner_tenant(self, session, user_id): + return tenant + + async def wake_tenant(self, session, tenant_id): + self.wake_calls += 1 + return "container-id" + + async def verify_tenant(self, session, tenant_id): + raise AssertionError("verified tenant should not be re-verified") + + async def get_tenant(self, session, tenant_id): + return tenant.model_copy(update={"status": TenantStatus.RUNNING}) + + fake_service = FakeTenantService() + + class FakeSession: + async def commit(self): + return None + + async def rollback(self): + return None + + async def fake_session_dep(): + yield FakeSession() + + async def fake_browser_user(request, session): + return TokenPayload(tenant.owner_user_id, "oidc@example.com"), False + + monkeypatch.setattr(app_router, "get_browser_user", fake_browser_user) + app.dependency_overrides[app_router.get_session] = fake_session_dep + app.dependency_overrides[app_router.get_tenant_service] = lambda: fake_service + app.dependency_overrides[app_router.get_orchestrator] = lambda: object() + app.dependency_overrides[app_router.get_vault_service] = lambda: object() + + client = TestClient(app) + response = client.post("/api/v1/app/deploy-or-resume") + + assert response.status_code == 200 + assert response.json()["created"] is False + assert fake_service.wake_calls == 1 diff --git a/uv.lock b/uv.lock index a14da8d..128948a 100644 --- a/uv.lock +++ b/uv.lock @@ -723,6 +723,7 @@ dependencies = [ { name = "sqlalchemy", extra = ["asyncio"] }, { name = "structlog" }, { name = "uvicorn", extra = ["standard"] }, + { name = "websockets" }, ] [package.dev-dependencies] @@ -753,6 +754,7 @@ requires-dist = [ { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0.36" }, { name = "structlog", specifier = ">=24.4.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" }, + { name = "websockets", specifier = ">=16.0" }, ] [package.metadata.requires-dev] From 5b2838db7bee185981a161c2bb7a1d84c1b6d7eb Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 03:35:59 -0500 Subject: [PATCH 02/20] doc: replace newlines with br tags for mermaid --- docs/architecture.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 0e5a62a..46bc0b0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -6,12 +6,12 @@ OCTW runs one OpenClaw gateway container per tenant and keeps all tenant gateway ```mermaid flowchart LR - browser["Browser"] --> proxy["octw-proxy\nNginx HTTPS ingress"] - proxy --> oidc["octw-oidc\noauth2-proxy"] - proxy --> api["octw-api\nFastAPI"] - proxy --> edge["octw-edge\nHTTP + WS proxy"] - api --> db["octw-db\nPostgreSQL"] - api --> cache["octw-cache\nRedis"] + browser["Browser"] --> proxy["octw-proxy
Nginx HTTPS ingress"] + proxy --> oidc["octw-oidc
oauth2-proxy"] + proxy --> api["octw-api
FastAPI"] + proxy --> edge["octw-edge
HTTP + WS proxy"] + api --> db["octw-db
PostgreSQL"] + api --> cache["octw-cache
Redis"] api --> docker["Docker API"] edge --> tenant1["Tenant gateway A"] edge --> tenant2["Tenant gateway B"] From f9038a4cb1759a98b104aca511f726d90e43e396 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 04:02:29 -0500 Subject: [PATCH 03/20] Implement OCTW one-click deploy --- docs/architecture-1.pdf | Bin 0 -> 22211 bytes docs/architecture-1.png | Bin 0 -> 36830 bytes docs/architecture.md-1.svg | 1 + src/octw/web/static/app.js | 15 +++++++++++++-- 4 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 docs/architecture-1.pdf create mode 100644 docs/architecture-1.png create mode 100644 docs/architecture.md-1.svg diff --git a/docs/architecture-1.pdf b/docs/architecture-1.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b016244b7d0f99c7d9d489330fae9645b88bc46b GIT binary patch literal 22211 zcmb4q1yEf}7A+(pxCD1D?(Ux8?(XjH4#6e31$TFMf&>o`+}(o{xOn(CB=cY1pZW7@ zs!ko*-MxFQ?$v$vKDREZoRA1DBOMD2>D1xD1`H#B0bpxj0mH*XFX&*b?_}!$pb#{3 zu(dXKwx)!kmvgW+ayB#u85J$f_31%v0oIJnjG&U|+UB-4Li$d|016>4CI%)p24)6M z26k2kR#s{T26E5=swZu0^xv!S@xd6|82!Y+{1+BOeE=g2y?``;UdGnJTHorgN-Td> z5`6cw(yyvgf&c~>dMSzLv+7T~v=o2^hFwu5 zp{)_9jg+yCsgoIiiIahqj}PGJ1R5Y~7`OD}4BKd&me`QB>!S6;ms*;>>dIXP?;Gmc znlZrXIA7j^#ajh-q3AiT5pbZk6~wYsoSX6B;3CxXswl2J8a9-s_7#cS`;iW01;Z~v za!`9Xc)Dr%Xj&kq>7%@}db(S>NPaSV0$E;#TBkSJR6OD1V^~~WkAJFn-qQ8e?YtLW zTzfL5fG~*Pp1gIwow_L6Y1(n4GaxqMF7toKE%{kvR51)bhMBh!`~A}0DX85PuvoQB z$KYl7+i8)wV|zjwq^T_&U36q{!!Ji$@CyEJi?XSEo&@Mj{22L_$-*H&l)TgiOMte{ zkJl~>RCwuwbzz(obsBkrnPCT}nz96t?|@Q0Gn%pmyagEiiSg|C7(FJ(BmR+g?bA;o zJ99yP6mqB~;FZ1{FV#OTZT0wlUJS!2=8C^WCakt8U#)l(s-OyIu8Y1O)m^*_P)N)$ z!1ZnmuGO!KD&?#Zh3GEqTkKTIR&$v%d!^&4tXplmcGp`8+#*5}6F?`AE$=NL3XLgY ztaduGpNjHML%bi3ju4P_-0>mI?ZWOaiOdPHi&?AP4DJij-WllYLM}1Qw8V^Y(70}G zgJ6}m!LG4A)0e%mKgiBW8pA-wlhYDN^K0SWl}2%?;5{1VqeIvXbLLN%bP~$}#1;CL z5A_mwk{#iA&d+#jy_gq{+@yHpTrX(Xpt+1XYu0Ao)7cWDy)i)gzBPng+w2vG;pr@D zoC1sOVzupwzG@RTOxKvd_Bbmt7Z#Z$bIfW*#B#0v+H!B@*_Ob-?mN-@gzPZv;!%Lu zvZ}Ds#_CQf{x#ju0_vCfyy!+S_KYnV?$vwtDubtRM9;>AQ(Ep+xx+3zX8XrbabgmB zC)<<`$Dk1Hm2gYZZ&njr9~zYn+)|_y=cAGHw3ixw_9Ndsl47?~_g*Gt^!=B5Ch4GR zH4`%(FA2NCu^9K*Hyf?a>@!S3+#|P8Vj_wy*Q$3?0=5vaZ&#nbfM@ zVuYoOFXWh&=2S-0#X@DVJ*L9FBM{q$XSz@%;}l`PI@RlEyuCi5NXBj|pRc>IQ#dgl zPVD}uh*AJierrlXifD4`mB7Hy+*A3%jjoNX8ae7iUiAk!T7&ro4Sy1IC;S$U9zjJq zk|>eOh{O3DZF=Sq=2;{yAoe9H(#)x9bYAIkNnohnOyt&C6Kt|Z_4X#Qvg z_RidvWopCBTi+foK3DRf%>KWXh6?aSp;6Z>BA!-+f)Udcg_ZGLieGBlcUN~WXkd6xCxy9Y48d( zNW`WA`foC-lr!~*H+5^yU{e&zE8ygPK6GK1tGDh_E;!$0DeeXKUoHSY=>WY6){!;dJCL9lz8`IgJ1}|((6Np1Yy?N@<2rfASZA)~_ zn5nW}p-4|Q(xCF80lYLS@L&L|Z~)`{lJ}N5tHn1D`O=ihfyv(7ccz}>VVG~ee$yx+ zbG$CTyi_4uok~^WDREOc&)*d9iw`G9lcmc1(-5t+JorX#K)$Q7o-*Zc) zt;VNX@hrI2o*Qiq^~Xn&Qv|Q|(v2c{m3Xe7CK4k-id8nbW z3NPlyeW8E;7I5vXyUF&gJqfOOs_?w}6;aGpg_lgPezwgigQth6dF`3e#!ydu5IL+Y zdL|8zy4#JeH)*4X#QK&YU}_ z^2rw{ACSyx<yIg6=SWaN+0ViOpBNpNMn0L|9+&|rKlIkBur z<_8`ucczO!4PFLD{B*n(hg#n_Ko_y4;?v|ldu?wGWJ|Uyz_v5E#R0fC?v6?`_JLsn zTfcnYz$=y4tnyWB#t`p(s33|h`zdC`YF~tXn7rk1z_yD{9nR28Y4D%J8HmlImi|v2 z{$B(|oHG`tW%|5ym4_13hl^@9po{WFXQdEtV@7rav8|;A8Z=SD(xA9bNUR z4n`sThfJ&=vtGmNw4VnC5|rOTY|iq*rQ*VxsWVr!`f7Jkr3FYh1XPFSTpf*)Py5K9 z3iY6VeWUG(Qlfi6GKoX!uSRa2nR>Vuf5!#w`_S%nt$Tcb`sseF=;=YQ7ubtP z@N{N=dtz!iy|kx`fj#3@jBx?HwZFx$SvI)p4FwYCI+w5Fq&|M-dosK|h|WsN;`_7^ zdxBTq+4(5KAE$Q@SZFr2zcX14GG7#~wn>vXFL6Kd1hn4=m;<`c~ASbuW)}OPqTHBo3WC ze01E?T^ww(b_!?}O<+TbR#t8>o((NyYWXB;6)P^_M8iMRdNnfTe2I~_Hm7t%v4Z1rQJQ4HzV3Ead)Io3X|?4o zT(1^A7aeNUCMHz+(*06Rxg~Y)>5&r=BZK`9{v^H7Lr*QK1|YOY)py0LqK_WJ?yIB>SoES;&maRRf%CAQFkOBS;7 z>sR@Wd+t6ZXvs9EQT}@p*COxQd13BSCHfBsMv_ETGziO!Hi&od12(5sygjOX_7uUK z@DX7xlFoR+zaC=gM-xj+K;l~60s(I7=crIdP^lU+7*N=-=E4o*PT?;-m2r=tR&XG1 zxwx*1)V^j#8F>t2kQwfaZf>7kAM5h8aPhojy%=-hhhU%~ZKTtKbO^smx_=bzEAr&~ zR<<2W%R*T8eI*}1Xm+s2yK_~(6~|KdnnJ+q=F5A>&lk6}JPZo#&1VhY8CnTi8?I~E zQ1c!RPyvC^8V-AJE>^Yuo33~izqvoQ5Z^b2^uvZ@u#a!~GY$A1WB*PFnEr~-mE7%&0rYbErl7D~!PwE( z*}>4*5dg|sKpBdmqOlV|^Eo-8S2A{U0?>kHVr0W=vvMm9z!06QlW z$e|#pwy}+qBj9H)0rI2>$~Fw0F$rL>KBav$DQ1>o8O}aYW&>)-vIrQy!}THLgp^dKK=sm3)P<}IQ}P6 z&m;N&kYWTd{ps-EM)yC``aN|&Y5hd-H(GzBi2sq+-^ck6a?j}gz{2!Lrv1;Jn0~4Z zB&Wa481tX*epdKz0{As!oWIF%FI^h}EBuTf@t7WSVWewO`@q@Gdz>EoYN^LJ9e)$=z}tN^w@9sZMue$jf~ZJw7V z)4z%5e_QT<4A3*WKd>aWm(?@1=l=hcA=5uC`u`xq zXHT3gTk|4Hh(?0=KtFI0b`_$Sr; zoz!pj{EZanpYHxiLcd7;epCD;^>5Por^Wv7GsXOj>Q5B^q?f;w`mLM4kz)Mw#l`$j z>iNa$c|HA_s(;hYKdtuv!0H*+AFNpZNict7#qwJ?e`m$`r@w!CS3I*~`TdRn&DAr2 zKLz+)_P+`67py-}fMT5gzL)-u72{vaf$@1q1u#Af2*CI};(%YfHh}3_5rF4auK@V9 zh5*dZssk`TPZQu5uRnqO({TK2ivAjqXY79t2;)EPd%t}=gMC)YFG7F7_$TQK*%~@q zgTg-0cVm4UQvijr4Xv``&u{2}-!cYG)H94fak2bs>-rs10e*!b|GNBoYXg40=K#On z3;(+O+DZYxHXp#RmxrX zaAaiQWCMl&|K*1O&@Uex41fL-FyrOvrPR0Z4??>yhN5dFcbp#U!xP2tKgo9geq{Vu^c_R<%J zMw3?>rC#%hf*C)xOYYK@d*ADEzSRY*He5xM6qvKvVt-+Qn+;2^Ht#-US+#Qgdff~% zVJ5?oUN;oX3asXPlar3u4#a)s!@}`fH)w{3y2{&RnrQLs`}gaxew{UCh9?i~g+ETL z{K~-?YRk$DD-XG@DBkXZVJ_KCOj81gs#$OkwD_+{QU!&%HE)JJoG&Hr${TRCM>t1lwK%kI4?` zd7%%7%weO!h=q-v6S~)H2lPF@RE#@F1XNzm;qJO>z54otm>2u&Q%_;d&YcMO`F5Ey zkG!AwDD#PhW8^J2jMKz@eOK{AtXE5HNOkri#F8c&Bo>E`KJSV z@$1QsNiOpYCnYYbC%Kaye&a$WMZBsgRsLC@O4&Nvdfrjr{^)z3s%@X$Js>2m^%!_1 znQHvZQSd$6wKaa)4e!48QQ7s(edFEucY-f%l&6JVlU+F7)O|9{N_B#-_cj#Y_`|&< z6kJmy1qWl=9{xJZ7rb}2Yw4j;Eh_U~ch}$I{a|zHdgJQUsi-NJCK-15} zm*jm`^#I(`RC?M=4qouWsD^afbnY6-@O6{^4^W5R{u9LAr329*oq#VW9N^DYY6b zW4{Mki|&NWu0`!0ZNJMaolIyx92fb(-xqJg0iqJwk&pKPHGQ9Z%U8G~eh>IIN!1|3 zj9_dwF+~F_!1>b2>KNy6kdJ;as_n&L0-aPm`=-u;v{d|Q2syioH$;?eXT*tN$C}3z zlvI4OLBSw1;SbJwO$T$sKR0E zW|i`D*k&V`wo9y|;dgHr!o7^Cy)GDTz+9m8L~nexS}9~-9;EoOz{u&m+=qL+0I{{! zci`)J++%ap&t=uWdlYXyAEe!kcD-2%CsQZu2YH^|oE2?pPi)i2wY)Uf30sx5o{qM`kc`{NK4iPnZATr+X%`gPdQ&0yNtpJifv1QDK1@92S7h~ z-~5Hd^PFmN5W@&6X)1Z>c=Af^o$9(m&FuZuCn1Q&&DkeUlKq>k{n$1R-kZLI^QY}Q zU!mw`#_wVG;txJezCBeQp&f0x`xv~R2~O##T4}IgbyGhC(Tn%@V2~SbM8wsc1r4dW zy<EZD4!4?(76h8tKzYR!~bu(GoKJyW2{(;zpbphubh9#c}ojegMM z&r+%W9(YhiKG3aLe~&ZhmJJK`I_BHS$fX@3)`B->RcR~z#!To*guC*B_nRuJR{Cw_ za!>H8U{}|33pH2rDw)GhKUKjtZa3C^Prj<$mG5rO7T34!Tm_zVRl{vd?#i<({biFU z-`$iaSBA^FR~+5Y4)~H)`&R1Q(N1hOl#AO^RE1aS+*ph6adqI2_|jB^R+!zDzw!pC zzHCi%OI+j4Q0+c3Z-;S`-BH%LM%B4(BmX`;=Z#*~z7pAf;Uv46*yc%8RethXr=|7K zedj2%i(1qbuj5HxuHHS7Hrv&ni^>K9hlIQ;%XQ`vLL~N9jNN|t@-vN z8{cQ3DCC;!>g*7Kw|$^8M3?#MtSlJcr>`(XclPQ`G8NxPex|n#jrYO7JR~dm^2|{Q zkzexijBbA;w61bq2ZQaKfQNRh2D?}#j@TiLJo{V3Z|<$K*ZWB))5onjhpnX}Tr)ZB zK(q1^#vEDhCU+D{TNJ@$VzU{IN#EoL_2Uqo6UTqF^>t zB5rA73xX{S^H^q_-gq*$Y1!hk3M_s)KH!(r zNY1UxiM&!#6?ksz{g{i%xbE1^-0X4d7TiMpDOTlJQVE89I;I@n1PW`}!GhS};G73O zCiVo14A02`ppcmj1!KbK8%RE2f)Q#*v5t%4r+)plO#KE?R~l z5T;4xkts~+!{PZ$2dkr$Foqz!%jK!HL!so44}DEr#qEs}bh2J0Ek^3>xJOK$p}%KDcYQXD-!n zDQKp3<$P76jXi z;ix{aitIhmeL+1;-?_m{JEwK#xFH3?K`r)RR8xKkXp8-Z%B>lh_-oHx>|mL>j97tl zS=yd%421>#$1JD@T_g)SDDq8OXr?C8$>oWj64V;xq9b~!?`5Ril5{2a{rk&$osJbk zh^$jDN`e%r36lfKNVek!Zy2E~K1&p*Xv$2S8w!l(gSF^~DFRJJ&s6w19K;2vO3c@_ zk+U?W6wwRMRdOzsF3UEz_#6*A15b3H-S>0U-*t^Qwc5@`6IeTl#`GKk#>%dmj?41&Ok%9 z?bZjet;>Z`|HTF77L90@F}2Yw5fNj2!g>#b7obr)7c;w}7!Un21sw~0T&5};8c0@d zqs!YAQo+HGTuW5O&dOG)qc%6DyOa)a^qkLYLM0(vUW}_=Y#Thl^)cFfn2E(2HH3Mk zD09eN4z0ml$Jv%Gul^bhFO8z#HatdoioI{B6ss<+N#m{A%$Hpo2{Y;Yo1~Vdtk_RR(hIwGL>{1%7^_xY_0Y5+qhp75LS zdHFyj1JO4kAH9PN0vD1)vq3lAq19D#{@q8C3AqW6c*YdQy1BT~XuyF4Zyx-?w4tI8$=i&xW&F?KgnPd?MW8UMU73*c?A)_5Je|SrJk5kIv?;RXl zY><_MO}t==85Db@_(6pA9L!z7{PXpE5oQGQg6sSByIZBTSd*O}L zJK}pxMSJvns40d-?+*tb$M{50Qs;FU1QVDU10Eqv4PW0w?X6n@F~0P09xehhG2Agm9+;Q68-@|A4ZrM@=cV+5NE(q3MMcc?o@7T#yC^l?5R$laQk z&0u&(U|ja#_ek3NMg$kdukOJxawEL`BC7pBCwoF}KS{eco&?F^7ff1TRtE&MhWWaP zw1!vrY7l^?T2aDB<7)nWpi}tBLY6&5QuU)+F!j6-%>dewY%}u6e3o4zMOA+y=;5_$ z&9xnErOD|AmsanOz(ad@Pef`vT2Bo2buJ*C4eE}s9t7w?AHoxIZ6NXpZ}~&oJhxid zX`O2g4E4g&&3g(}Lo`pErgg|8+GVQYrRvm2<$$(@!)6SA!>TQe8%|HWPoKvZESnLpt9ftGPK-V6_vPS~9l`Lq%c1uR zmTq95OsY2CS?N`6vb*_p_^)k(1F3vMAQ#GRc=3&@)(IbyI{aVxjO%oh$-8V30-5m9 zcTB7d=r<5^y^49+6(I>jQ}z4VafAMbDu;-}nk4dcKL)rv{z2fv=cIriMeWdu*pM1oKJ0K6l^bk(z=bty!yMeqX_5<^CNvi~4q=i6LNv4#p6;R0K zye()K4a%je?IETZ98*SBm(Qj^&MH1Dq@n(*3=w>scD8fsS5=C2qwu=3v?LrR6YiZ$ zb7?Vow<_})tMGgC;>Y!}6zFeF>y~mkL+zpgrLZ?51YU`{a>`+{+tNXwgjAL1IlXRu z{j1Dhbkd#U625yx_B?3eW8>EVY{lnHu0M>VKR$M+yc>@ABeyRvz%! zW%cG4mUuy@ej$gsp$e%gvHKx;X3b^geP;bN(za)gRXd~*_NKK`P+l3Yu&{gQ>%F|< zc(%wB!VMQ($~ER!_%stiG6w41Z!bD4N4$tas$gzfUyoeTcQ@G=gM1W9H-T63NgS}P zm;~4;zVb?=eYuu%XxC=-xW2@MV)eL@_BsScaf6r4sxSwKytfJZhApj!e9?+6iU9|g z4KLHSd#)3w6B_48{U&&&`YObB9_}TZU>7@Fz|_27uSvih)17@@?E9z4k2U5!2%rMJ zkOKyIl|ki=IrwHc9Iq2!h|Y{&_6?+vVelz)_9MSG#7=YULg-$l zTcIfy&PeHIxR*Xv3y7U5gcY~&H!cWhGKtq5dgEQ!nSQqHM`TipJ1EjqW6%aYOz_+G zH{|whyf4>Q2|9Zb$U*RA=Cu)~VuP}+wwL73X`Tgf>Oi-PllTy{a}w(W8?QrpAfgfEI03@X@J=RO{zdH7-%@vf}e5Sym6t1K%MutwpSY=8CE8FjJ!3SPR0tOnZ)jT_<+UZ|Z3RVOO4D zysG@NOt*M_5Y{eU1bL@-ZFsH&=Wm3f2YSud-*qkthz~ABFLhu|d*1hseLCwdLCYJ3~Tn~y)8E*UXM2T=y^=yZVA9cy~;DkO^l73=E@?sZY{ z7eViIkuaAFn4-wiVdrr<^pgX83CU5AG_WNLlI~L2%qUyEB zLc$!kTs-RXSIPxy-s)L<%|zO6K11dcBLn!OZ)@t+l)W3_DiUj%-bOYLss6nHd7r{=6@Hnyz5+={6yqv zlP9~9wE9-VhPduV%>fDod5#g&^_v;k=)2y3Q~Cmc@NL|*io0ZRP*gjs~T=I!>=Sd zod-lC3OGCJslp=a@G5mP#aH58Xxm#A_%0-TpEt52lg3%xv8u89ndnP^V;3$S z$?nE(wr?|S8`lam56_nHVEUsPJX|%rR!SOYXBlUiXIDEW%<3Ch(IVk(Nf(N3jnS~w zS&00|1vbNu>s~lr zKk8olv|uG@vR;@Nv$J4sdury?^`1N4Ivm@BC{f`VvtW*jB-bpof>iFTrtg$jaIU?(Viwim(Hg=SYEsb0s zpz$@fYl|5%nq`4fEUWe`T6m4&Kr3%vh;UxXL|#Q|%BcMwZiX1VM{=iDayGq0LqlT& zI?3g55(O5zwWao@4Hb5z#-@Flsnd_@?8I;l%Ip0JoALdLO>=5`P6`}GC%Lpr$#9cM zth`99DP$a(+|b~HYE;6+KrwxDThY@jx>Z?IrHPBxQG%EdapgOAEma0!wqt5*2~c^67#^%L^?ZZ(^7rV|!6Y~Fr4!z}xG4hziS zPc{j&+451mi^cGp`x6%&uIA{bilYu3(;^jW>x=p`2*sqr&Z)%f6pl*VMxuz^=1E$c-&NmXM0}qvS{cQO{u29l_A`<8!5syhh(4Q$2*_8(GXO*ixbu^q*SRC<7zz_G z)XM4`yKvoM9nDP2j9k8$Q(JjVhM5jSH!@3Y30e(fIdpa zhIXD$vScUi!*VmBb~W2$W#yRathp=#q6Oo`yJ)sP4twcqYwDW!aQeJxC{ z3_rC!i5r(~MSuFFm-jSX#Kz+;Hzb=?hVRkoYI#cCZ_TgW7CvGzhUZEpD;N@@)E9EI zna=VK(!Ae6Zog8Q?px{MWRj&*lE}V9WP+v8cY7`~ESjk=w@eVWnEvcH-fz!Bw9g>X z(qql9heh+vQ37rJ^{|jIfwE0h^Iwzq3?jzRa+Syn-)+}RVW1kz@CV`SvC_6RvdFjJr%K-jKfgp_mEvHdPRmoSzZD(i1P!YYdrRK6Y=xlh99F#M40u(7Ky;>?Zxsa3 zALjY;P9j4=`(1@XS&1=KP^w@U=vWLvR?Ze0%@Nw6*f3La+!H_67iDgihp_qF5wa!1 zDi%OfX7*J{d*o<^S;MRXh1I#s~BWm=>zqyvN9nyq1iW*e5d&*R2MD#A8sEn>7x?@>IQ!kbMT^lO_=L!>AT)C2!>?Rk#|YEX@qp zp!0F=yj|zlYTp&QR+$BlBg^ilqtFL$AK>6AfxJIwdLMDSF_LfraQr`E2=h_4up3Zvs#k1qUusPOGu+T2Hnl=UUP)u!0whHF29-OT@uTq~h{% z@qZqCJYp{p6%Y`SeX$_Pf-hXd=%S#-^@0PEKM%#+I6QcD$91#6Gnz4pcZwXw(pj}W zu1r~YsFol}NyDREUNrkA5A9PPS3s2mUK{i^G0^GU37ODwux}fiNIZk;eSTc1M#*e3 z^R)Z7ol+_YaZW5@Y%yka86}&5xw5)Iyh;J6Nnz_0|EH#6Yk64C9LN?1-rF&WnJ-o{ z6nUCV@S#E0DoGM*pq+!ZW@Fugu@uRtE3|ErR`!UsNjlGWbvHi5`!T2Xs+scfQO#JT zDv?UI*DrZrQ{R-J?`1ljNlbDEc8*Gh$!wfPsXaGI*}y7tJt}<8r{+gM0-9)*VsVYf zCy@~fh@B5#4n!QC^1Pk${B2l7*;APX&7!pGL-3*yceZn`I;VK!dFpr-66?m?8x`YF zrwl_BX{IR4Y4(KDB0X;4;`uM7l+60i>J{X#_w`H6>T7$n4=T5@57)y`Qmq{iiTae$ za^<5MV-pE0bDJRuvA5f`3E=YPt~%w~Z#*Jt#C3IKL@ztL8fF&E5xv{4Uj`%~aZ?l) z=*shf;t9x?=k8u#7a)YbVZSwjfU2lX(op*5glr+s_0C&AgSj`meqy0l2Z_Vk;hYFK z-P?((X1`m`JF%|j;!v3J`OOZ^a2Yi{w(_(VNyYTsp;LlP_91R|Cb>)+>DWinbb9@m zgMhb;v1)NC{PO2AENE$z(5UvKY#xZFMa`yu*Gi`ct7YxA{GUiZQTdv8E-x;|DS9lQ z$*7+ipC@yv#j*|rrHX2ng17$EzTf3av8xfnZNY z$XB*hS_XFSa_rEzrAoIc@-C~l`~eb(+++s%qYZ-j=RE%|tTV2VB{d_@A(kICk% z%Lt4E;}&i)c6Af!)o)8j50ctZpye+(tGI3?FUS|OsavI?GHBXVRFq7bSp$}BmvD7O z@=7SKhg*y+BHG#aG_TO(-!8|9i|gfT#*rt@GqAC;86N{bG`fiL=JCFtFj8Y_^_)MM zKl7I^1#_?C`)G##PP~|I`i%y5oGfeDv|MLRx{G_5YS~nHSp;`P!6rDS9Q&}i3Kw*` z@raQ?*=uJw4KA@#s#5B26dD>NIKZjn_#=$Ps#@yAJ|XNKl3;ZfF*@@z*85hk5%1b@ zZRu}AG?U^! z&yte9eQT#NOh`;jQi;S=SAX+WLXxxO?p5gVO)=gIJ;>M5`Gl%vE|xO;r1ZFy$3_f8d1`th+VR&orFb?=LO-eFeD z_a6a(2TkW&1Y>HJ@=!#oDX|u*dhSpjA+>DRm_;Uux6Xtsp7ZaUHO<1Z<=|9Q7`;*y z;UKqq$nx-gBuc6W%ozDSl`La$WW?@JO{-Cgw)LjDD2}S6J`cR=k+*y5Pzm>ge38#)bEt3@{Q7TZYx{-w|qiF!u%N0g&w zMGjpxNtlVF8KB2KQyY(jxm*%;oHu8}POUemI*fUV);O`vEB3{09A^n@NgSu#17nW{ zKFPIg>aT744iZB`5_;v}+GUoOM)O8#+DC+SDwQNm;8^(hR?^1)Oe&=fGst7^!{4l} zts@Hqg@y_G+0%?gDsdJyT3pP`<%t4AKf*o5M@LJqCHpv8wQd?KNc77BOyl-P_zvo} zWQ?!viE6gureS-YlwWF>@qk0`^$*wwRO`gCXvikZ|41Pz6q_a&Pmn73SStB0j4i@% zi|RneJa3x#I1VTtQjix1E&g7HIrwh1aaOp*RgqdYLdWtLU|k{Lr-B^5y2`4glF7jq zb?D$C#08kk1|!{W5FHkhDhtHPE!1Y=3zJ1PiDq9f$Y-vn5i)Nn>lUu*OLXek;M$IZ zmUy5m;8G(~R}gRO+CM6~FR8|vRy_SwV`sa+fEzH^kx%sND3$0+#mF8%h=W$b3t zoYvImQ^u!BUGz*Q2|~Fe0dZ*SDk#vwTNJ7u4n;;`5Uk+QDwL0WCKpM(C1Mq=%AsXp z*zc=>fW`7rciw1!jBEwXDX~qY1zT&hl3N=`o3~QC(z~*kj$6yCll?7@J2BtKPa``i zEM`Ce6}-h_@%4VaI(f6)Cy6434nE1EueWenvh#OK?4~RJy_`+!)$=T;;hr5j4f1ox zv1Asu7u#vmmu7<(+GGl!$LpcE^A|NLT>GG#bY#`rDqJ=5CRWI)#aE|_7%&PHxHT*A zY-F=?Uf{2iO(3%>Xlbw-*cl6Vg`@8IH`wTQ>1t6KlNcBav+`EoY#lbny+4iOJhXGY zuaA}eayMehoUDbP!FvM~&1`wBEt|7ZcV5eaL~41=Bs9Fnm8fdAR$FqrAYY5-VtW8X zrCxY*tF!~4fg=&&VHhD{;V4-3>dn8}^@vXmw!3r;T~7*UVE>{uKmNtt+2F(3dCLkC zl24~-=21%d1Q=}|^?|C=`|8@dU0`yG8&P5%(^oB%` zU4cL|Lh1K-MOYd7k4k{sv~&ng&v5olI)vRpkrmArr`5{VH>m?YUosWaChHn|7@M3=e<^XRmwgSGPUcd_EUp&a)qEh1>;^E*T z2kov)55;~L!NuLcrOnkhc~Kd%WNj{Sgm-lt45?GjBI?-f%H93BG$4iDXX^v8lr~q)939b ziw>!&OJ7>#V=ANfpt(;9?!mX`{iqbh@9^!5jO2HJI=;N$MinsSAw6z$Uk_y`@I6X# zzaAql&_#)fMR60Vk1fb29)8CWEBm_dI}DR$*3n>-Y^5ubXee_IF~+?UlntucY~92$ zz2n63ORT3g{k`SAef`rtgQcuKdZZfbtQdn%N@`^$3G4L$V32t*IYlr> zWkidWpc@2=p|at!k$4&Btdq>`B$7%jGo6?z+VM@fi&PSJ(Yknk2lfSQt3WPzd<-|C zKy)ch#Cs#y5SuC7^npCXJi{doVJ0H%V}hsc{isaTrz1RKWRrelso#i3`WQ>&UpmKHUNtyNJ`zENRnI=9nv(c z&IFoEJt+Nc(%B_#H&Rk|HJW-!#T=U%G81R1dbm_Wq2krlm3S-3=p^@*aclkocm~Xs zp^1TvXW$xHy-ebKk@wg74`HM5RuDoYHDX$yzits($t3K88@-K}E$<(Qra0Y>{9yhK z<>Bz~@(tGz`;utgV)KyA9TRy&L&Xd-oueh)acr#)#*3QBtffX_a?FxM*e|Y*?A8?} zc;2JJDaG8T)W|>xrVuqnV$k1vl~q`qcD_IGDU_FJxCyD39C7%xe_w96)@HIEY>BY- zK55K)+yt8PwV$_Mt{;w@GaXz8mngZH=d~O|3JMhc(Bma;^qgC@8$GEpdN-9onk<`1 znTzF^uYG|GH-I^bPcRv?mcEwwiAPsNy`D?L98z7P7_}CL_glinezBQgJ?A2tHoi96 zOzy*x<7c6k10n~kiHE5oR4z_wSoiFGlc;O~dk5HYA)|1^&k5z>F?0uosi*wFPxx**+wP8U7D<-I0U1dj0lAa@i^n3QL zn3O8SeagflD#@)ut}1w6g&ZUmgWkmrQH;cHD#RsjX<0V9{9A=!G|86iQ%T~ho%4_Z&lly9es*4LvC)Xn|Be~(jC&tUAQpa`- zWaq6P_7TaMQ^0uLPD6Etp=`zQ*|mo}Q8LPlitiY!H7-LBnk_~3$T6!*2tAUx_941r z_@gjt3E3fd-a@ZZ$s@FQTo2lpYuCBDMudI+P*yDMw%vo~fiT~DmwJIP_-_INvCp_} z88_>;bjFx6VPAuTxEG{?qj!b0gFZBbmJnlW6Wq~|%=&WSX~LTl}r z#nP41P6ckMQef~7k+EXPlINR_4tKB4jeTr=Dp_Tt8z}g zHXu-yY+;q;obwZH4`N94nCIUleDl?X`#3(i1$vojAUMrIsyV}}KCQ@(17>F{x`l)e%`_O>67?S?l6A?m;JG(G@!%oHAZE5e?7Nxhf-0 z=|)OOeFIM~vlMx2IR#TEOiB1KWWief(h}K*#FQDbL@DtM8fiYt-m5-WqvjX>ojn&Z z5Xplcvz=T?Bi6CT-3vp$3+ zzX}||4|S~R6ecuM@@p=7C3I4tn5fK|M(HDqOP?6xQP;BhaQfK!1mr+KLH zH%{)*J7G}>MJz+yOP)EZ4lpCV3q~keY&+2A6C?|P;adfZX3PcY1VcE!NG0^}hItb; zJ+kY_$4lb(eB|%cFjKs|S-$PemJW<+GrP4^Xcz}Y%-DbSI@IIx_Ie6|Ya8r3t&$~Z zKBwbarwA)plOd43F4QG2dXrDTa4lMEf><;#e2MEGs`U~BW^mhW^=XONaCd^%0)&?S=fY z8HlKLOH0sPy)dzpT~{v?Imtzkk|66PAYww>O(yC?XWNHkuiv)KZvL6FBuWd>_Oo}- z6C*35CQ4lcuCqg?Bd{#Y7Ai@&B!RW?bSqi-Ay7#2rsgbqTj;|0VUH)Xbhvrkd8twR zA^xR#@VLe4@OPRLvOU{njn3^4PHOHIx0{#mFUfWDvk2(m4>~#kA~wE1YUCHvS*~f) z-6;45L(_Wh5H)sY=_F;i*6b9DEYiwlP9{T>B>OBn6bo1$*}$za!4KIJbS(2`E(&== zcIWGx?6z^-r5c~5#>$n%)q-ZYl}dknSc(S*VcDvV>Y!ju^&{}GBnrVp0^1OL}vs+1^ z?Kb<7xu{WT^)gR9Y~ftZbS5Rw*X`i=+FqJ9(NMuWm|)ym*v~pTdS^}h+iG@l66Byn zgxaBV?h=$zni0x|$I|u(HJ)LJJSeXoq)kRc&22s;Fn*?maYlpzNSJpi$!`)i1A-L@ z=|<|Z3~-|{LOTgp8Fjun$O${LxQoTF=DSNtj~cxyPiS!b&`@`k|HV&kR{n=LH6otL zd~u>sM{MbEsl_>(tgcgj{kL1o$^{XLDcg)CMtg}3v?lGRn9F$Qd-B3HwK-3toy|D- z0Q^sQujXorCm}Bw+|HcazmRiSrbJ`LHw>l9yzuDglv>Lkp}wNMBh;Qe-b4vYzslJg zL$Nj5R%Y`axs_=y;^9rlyDQ-k=TdFjB*w*yNA!&HDBgv38|Fdg;jHWG-H6ab!w7h@ zp|HTI7JYNklLtE>fu2L99#|UahKle})xu#BuBfC8|B74L<Vwp>!yVG;c~)#hpy+K;J3y9J3VPb15?xWx_b5 z3iP-TEmQ?KWPJI~5ga=-es<{9$WOG@=oHaEYw6vijby&FcB_yj7EqO#8%o*ikYiAE3MtTsGj~CH=cP6$eJUOno z-u_RJKj5ckQq=NANh*X(IY$wPVPS&OSDZA`0MtAVN;4)z(sI4>#l*PqEwC-I>{4XoSKQTdxaXpR$TFQMGDi}l>ppCJ%mR7`G2^O_AV4)mDK?EBWTOk5shR7dGp@9dAn)zdri~h>P|ZUEM5Kj^{s7}_CEHcsdm>}*Eem?d@gOo zy$hi&^}Rbr|I^(XklVA{CpXHQw}#^9=51`;-*GPSv(%As#k)^m!P~sM{mmUk4RbwD z=h7V$%_-)ZKyz8Ge%aibTK|DNZfM-7HLWwq(1ZEcj=$3|=4#^Ygiz2Mth|4|W$zye z->0Tno5wV@H&$i8P`Q43{-pW0E^gj+bJff$CFiica(>I@!yTWK2i+6y)OAj@+@Cb> z?xczZy>s|x0hj~%7|CyQ`&~Y73G_AVBl}-<7C|3@EaMsvu+YMwYq2|!2Yfzs{_^+* zNj7C<`CWi<3A&<*I>j1v5o5D3ku1xiL{(j7xRJZ)H9=QJ_%oK~sz-0_hqNbwwpYx=dwJMk-PS32Cy36bTSH9m}Go z0+0k92G}5glH9U@4WKT9V&eELrrS5-q2GcL7ywnLIQ?#>Fu42n;KG2>V_|BpjcGC3 z3St4j3YNc|Ys+(%KpTZZ2!c0IYKZNneh5K7oR#nPc>MuisbI~5UT$u{-e&p=-99e^ z;rfSx&>k!)fffZUmH<+jGG+s33LhkiGS!LVV66(~5gChE!ic%6P*s#Pl{kRUin!xo zyxTmY>7s^N7zu@VO0Y37gi4AiYb=aNU{mQxHUL9Puqf+P(*U5;L`8;Bg9hjzDk@l| z%p8S1mox?bUkGvnn=7fnFH)6=sxc!O*dnYUQ73HQ2z(=y8vG;@GyXu_h;Z%dCyV?%yjLS-)rMhhtqo_oSTZ=$O&8sq_2>^A=NTPfc<9E1hlh zP|t7UkKTQEU-OFbm-dZb(stm$fr1-U9)tzTE|OP{lS z)xrH;r$VKt+CF|ZdGbVY&L_P|6S}*;4wrs4?Bdv3b9>Srd+|$WdoO2nE~STt6gK>h zT~B<}l+qHuwl;0b@T%TMYsEPC)-!7!+zVelPcDRiyotKXH zp;CV0Q{Tm#%T3!-pTAkY(>A*L{c@{4f06B6rKjfLlFXuthPZVb3#&p$z6g0t-HDmU z|1A_PJE}9M9j|@w=$D!8^p|8yu)1c(w_Q1pTf&|$t-$WHb^ zK}gT(uaSjLs8&!u04T+Mm^8FNSw5@S%Q`AT`ap(8V4y2~?nNec0!?V(5-1_SjV}@g zWUSc;yM*F?nIbcwG+Krv!s#JN2Z%XJPe!^9=Ojvopm!xKHd+QH0qR4vjIwhRBO^Mi z%h7s7Q^C_%86%2gWK`w+fQrZIDHL(ODMaS%!r>l7#@cp7+f`J?2}jGI+;BE9;`N|d zaPwgRW4ZZc5*vGjB^9bvj9tPA<`@~KP|0ItQ298SI*1KAh#p1!d^+X*)pR~C8XrF* zshqz=V$gZ?nhD~4BMO(3h$#GC!lR4xmr(XljIlu&Fdrks{2oDfRxYpLQOU)g!qYK! zKB)Myv66Jk?WLsa+*y?oiOlDK&k64`KGtv6?tRDj$&Ohg@!{K@E*!Pa>y7m+BMq1=HCMhNg3d(IUQ6V`LlpDAxDAx|qufsPh zAz>x(cFk5!1c6f6{a^(J5iC_U_)UvK>TM6&)!2RSH|J7?9Ip|bu2c3s4Jmc+?}$_k8iKxg;qO! zy-^`@9DEJsRq!MmMg?B4(^G{E;TQt4A6}n^Noct2343LOpQi}0^4I6w`^EP|1r0f< z|2?|va7TD~X$tX#Qs2SR@AUyuXv;wO6G#`|bmIE7eES$TSCDJuFfcTK{;AR{X3 zu6u(B>%cP(4)tmqT5j&*B24|kvfSL?v$HbpitFJh$cgM1*E+h3iwB<8Dc`z9qcyed zyw`bFr_ZXcsHB8?Ya+j>h>9nC#_Qi+vEFn$-ESHmrlO;h5Ea$$eo)G7+qu1aey;9$ zwN_oN!a-X3v2{N~LTR$fI-{=c2|s^L)p2CW?tB1&;>Gz3F7r_uM#eB)m+k4~#POTA zd;`!^%gXf0QzWFNnP+D|B{_&>wfTKKB_{HCz$M-N`?sc;v7Dx6Xx0+9Pft-J&#w-~ z*>E>E=0?vHK7+uwDk=f7u|hHwvN;3S1hl`9+ZQ}^az-H5oBb$m-Ezjq?>jxU4GvyR zc~Ol+VYdc zPaG)p|L*9JPU6`MG6$ZGpF7H)3j;(q&PB#v2M zScpoYps?TG)U>%VW~VoI>*R!6Q8CJC%;FX4_6^ii*~-Bi6oj}|%`GiCUmjUb*1i7m zNRx|&s0w5FDSN3p|{iHJIO@XgKZV`I^IYj3x=2P`ZY$ja*EDe=nX zlx1hLs*TxGv%04~{pu=w$xv<>s!og;qzl>ks|5U3{{=x7XL(J3P#V zKo=7W$&fStlOhnuqsYs~c9VjTaNf|YZhB*EKQ~wWhIh4wfJT zi{{xDR87sLgZ+0n1OIE{HnyccK0+|@nJl;$ws3O{i^1Q&mIntRE>cq1*o)Dh2^W@@ zKKh|CFy_f3XV(+MGH31Ji_7CS;=HLIQf>R}`gIEqPEMk`cbAu!=Zg6F8)VtoY__Hv z*A(8#$$bd?t&fNI)CV^}NY`*l5d9e!*Oj)Yw*P95i`((`L#3?Hzj8{6iHWGFt-ZZC zq@<*XS6k8voa_PuT*>~*zSxhN?|CN6Gz_kOcqBQGx>eEHHtObjD4lZX)W7RCJZ^!Vs# zATIvhsO6kY`)BWhJ7pTHo9_JQ$bI^xx4UPnQc0ST-h+qe>zm6~P&G5tS5s3PmSSLN z`1a7&c5iv@{_EGi0YMTbMw0ALRTMNeg|pi3-*aE%<~In8F5bBg$EBoX_r;4k1%{1- zHP3B{g9CdNm3wk>db~u^sc&Fo@$q%zB;U0y)YZ97VPIh2;8VmA6EpUCqpPc)b?|p& zglACrZq(!R^Xp*0dy8>+cxiHad4y#*YwXTewA|dz;WFg>bxU(|jgDK3drLC;`RiHIgPCte z2=H}nZA+g&f3DSd+19r7sc_Q_(^pPGVSBbk9s^^guTNJ;M;6ofM|bzb`)6RR-kO&Hw(+lpV88OXv!Axw_Erf_vzD*+xHzr#Y1zl>BIc14D~s* z)YVHeGPoZWOU9fD?N zX)V>LW?>U=!@#fG+Dx{#@D=8sil@R7Q6>v`e$C-dN=TsNIejVHEAsSd`|KFKE{C@!X7e(~~}UL6e8sO_ilamh8db8%B0Wt5fO`I91HF;PQG zd~_V6#VM7T&7<9x+7PI+mmwx?K3&M)Ov4SCg>QWP>EXs1vj)%WEP+?(adAxL#VX#o zs%5`wGBQp^pE>ee+!JW@syGZ@T@}Z}EB3`=vbABuA$Jf)cXm1sBV**X&%FabNKJjr z+%rC&o1ZVFpinn9!KC(1NlA%-WSN*`StO(H0hyzz@yhH&>ijEW7R{XlQ)kg%WITMD zYObhsOO@;CS>dSLM(-~N66qJ<(m6gB8K4puKuPG=f&}Vbr7%i}{5OLU= z6w2uP;5~v%uHJe*Lq9mE2q(kr#wFLu$&I=Gg!upDVMfSAg&aW{ar36IqegdY z>yWugpRqzh=wJLz%l(T9S^Ccwj>CAoyu1(JZ==1s-YhILm^Q}nGW8A55E_2KK=G-y zm7VRJT2V{Ms%>Q&*KLWOmHWsf>_7P$>SH+iT?t%t)YQ0S=qOXlYHCx}wpleb@ymiJ z*QP%`K+z7@)zg!Zl!TZe_x5c?S=qqgAPo;CYoCFkp;ndE%uKT%EiG+pYwP*>xsb?B z)H{&1dcDLvdRLH}>vDO1QtP-yE1hU$Y#h+@{b1+j7Xld6$jI^O!CGBiok-Zjt|VTi z&+e#nw-z}c;vv&A+{`au9tj9sbwF@1FnIm?H6f<2BrO-5f{Kdh&z}j2iA#r`u&_YP zm5`Cq`fB~YC%nn3S(}yL?f8wly1I&rikw^@>@4_}@$t^KwxgB)tll&cTU%St^W)ioDV zB_(z3+BIfoW;b?eSJyKbdr)BD_S=n(4N6K%cEj#@BNDu3iVq%9DJdz*$;k-`3CYQt zii#PKFm`k>QcR#~kU&6?*U%!rzkMH8*Vmg#RGjlRYU!VU}FFif|U3X%&ZVRTY zY;PmZJdXB&wsHWhiKL{Yv~)6u@h2}YFR<5;kdP{;qh`Ou^nwEOfoxfEaq+{$Lom;M z|Dp>b77}&jlJB$dBt0Y`*gHKi#li~O7_01PZ@-xS*wxjg*A|LQg6o`$V@ZM)Z(maJ zBuQuWH#axm#>VD&_H4L7?I?xUbZ>7@Ci%tR)>h$HuhNQ|Z+ep}~$gaO^ zP1HL6fmdhe`mYEoSO6ivSHXdSyJWm?El0k+5)o;f+XbszUf$-i?1SI~i5aA4s7^NH zIxKJ+SXo)0JsWIoUHl!z_-beIYJ2f%?)1XRN%hXoQ%Hj$>5Qb6YAGG1@agN*%*do=(rP#r`G`l(FkM85(i8KL z&t-mP#aLh8?qF4szd`fe70J6#kpwCH?q%+0{O4r_1?B1K*{P`w{U%v5agdQEUG-#hJN<0s7nhMCB93|T_%Q;#y@PS{ zudO5kv5LmU^5jXC$Ao)-riMG)D`_2_rq#iq9EEr|X6ovMQR2-Q1W6AYEvMcxMA0%b zf*r1JdAeVo)7-h!O^yGUit1odY6}Vs1hObg$^buqdKwrI;Oy)iLi()7HsTJdwY4nd zy7h(cbakKFW)>$;n)&n`Zsd{kWB;xDTRrx8=6l%L@y=Ym>(TD+u9>-cd0CnNoh%%B zddWJgrmThrX13kSC`LoXtBc{HsY~y%2G}b-d^zj)0 zGknO02{zIGGa#3of1crSj>TwMfAib}Uj!Z@VQ3&G8K3H}!2@-O2@|#a0Rh%bOkIJ2 zk!I$@m>2;A!^5&U91KxI)1XLME>>0x6O$MAtAa+$8a_R3Va|}n1qL$A&dP0W z(y?g1ct0#7CB?ve=|f&w#*vlY!^qeuE-DIcNiK(jiHQ#mZvGGN4;Fy|Vv>p3i!;`T z-ASfoN07=!e*XNWfG=%q>Qt-oEflJHx|FmsLqg~$C(dL7muQIW!a`0O-v-YsZ~~m> z8~4P;bD^U7TWM)(ZJnArbDo&kS)k_G-rg=J8wa_x%j&>oO^qrK-zDTjOicA+k(p5P zrKOR7n-qmgBYi&h@wmN++CPv}i`{>D&p$#+*z!hd^ z6*M$*@^~5f`7;X&qGDoXG&S)kI1Bpptwe^ZF= zAB5Q1?dK@Kfn$tpmlXBCkfsn9k8BM~d84Fcs#9Dyw=0$4z^rZ*z@qc>r%=A;#&?(Y zsVg4uUTwY7Q{I-%apyOZl0RDC9puQPef|1Fc}f{3q*l4v5?23zQ>2a->oq1#eCW^8 zxI9_PXlRIt`zz-Y{{H=rH`>dNn5*BdtwhAcG}P2TR=@L>)wv#RcE+)Pr9Z#3U|Ab0A7U zp#~{wO+MSzq`9HPi&teM8!?a+G20ryqTBF5GP6dvMt~9CAivS76{*nSFe(i zlA=C;_77b!Tv&Vub+V{LShI|x;-IZfNp46i_3O@gAn|Z%|TGml$+ZuA6MC?~ROz z6lrBi#(u%Sd-t7b?H$jSJ4{MNnvnNQlp6jSDc1AUKA^~{t6SeUCiV#r4jzwxrUuKl zdFcK2?OS=-xb+!pPA;wvsHCc@^oPHWj;gBIV_bz{HSuLwRruik{{34JeWHHNTTOXv zY|QG65n%*@36lIu%PTT`o2;@j@H!93$pyMst)xE=4k%pZP0GIN*mhHbEJ9vMsnYYx zgN7z!ZMcYlOk&E&!s5pTD=Q#hKDd#8@gKN#^zYS0MO{?edJ4E6K4s2ay#8q@*_jGi0#8$bs;w=7bgc3<_Bf`hEQz*bQ&ZDzz4O7A?(Xi< z+W7d7;o+kC^B1t<mL}5%@7_Ip;nXuUG}P4Q&%?|K=2Tdgy+>cd3hU$ z-kERcMZ#(XT+6+ra<7gt zw~lVtlM|t!T<3E?9f0vpPI`33KZ8?XNKMl=zqB-a^E$=nPY4n8oxQNb{TBk}qn%ow z6Zz>JV#>!?A4!)M7aucSaJG{%K5Kd`D<;+(rXGflUKNp&Qfu7b9qQ-~#hs~H-S-iv z_cbnD{gE^@G>?3Kw(w3VgDpyA^l3I;q6rif6?Odn{he4`Tq0;-GN5aN2m7#>@k+vWMQL|4!+)ncIc<)o}LuEfC0f`%+V_;Vo&I#Xr%Vq!|mkil`YwMW+ zg0pOeco-32)Cg*y@X*i($Vw9t=6_3$uEJc-H&jXT&;tD0RO)UXDI?HV*6tsCW}T?E zg^(zSZfqjmHmByW9`ND6g(XhUv^@7{yTI?TuuSEv*aDclv;QIs4vLD3l%ym=Q}Zcc z^)i3HHHMj|h)PLa%!LU^r||PZIrvrY-lK_7j@}KliCBF%apr|s##rG`*42ZY|baDOyq7AfgzCvWk;@Q&k-+WPwX%uLdwJ*dWw zO-%IdXA6sp=4QW^6vloV7#jLkYI{D`Fr&fejPJZYqDh|ehMAdxm^dmja_?-7J+51F&P zVaI{=`qeo%xa3SL$?YI-$xFMB<=v#sre@_>30HjESm}ONQSW)x{b%c?OzroWEZ)Ny z1iGSv+syRz`p8(=T^SV>0AFX4`4XVm$yw1gUamLLf-2Urlf6Ak?avHK5$>Y#*aR<}4 z@7ZPH_F(<_`FQfj_YV~g*Z-zvXFoS8r9s%8y>v|YBA=t+lZ~D_JNrY+LnZwStMy_h zH(&&r{V6j#O0Iq86Pt&cHy?xn&jI`K5{`lB_vv_S9==bfNuX6Cn< z|Ft@pmn9j8uzN!F88WGc%RGJ-2T{p=WbFo+z*wavObSf+4kR7XELZj?yIN@4T7;Od z-@GwGap2fS0s?lzS^X>;HBjGTu>bh*-f^xq1n>z!WC#Exy1Id+6tB?H(M2UBY8^I4 zaq_VQ$$713eSCa=b|>)$+_~x1jl9Zv3ScAKySorO!%g)j99^ zhu#W)UR_-cIXqwq&DxZPtieqHY6{nISx#2f)M(bbM1>WFq2XTtq6BMMWj!ZsWyRY2 z{y!`vNv|%ltdHCpuYP6sNShiH2j}W|C#|-Y7x@cb0A5I{&o3_ac6Sq#l3*UeLXMl` zkC~VzN(`{z7u&C3TG~D}PCJxzbaeU>Ubv8fQsZO5Px{4P4bx|*Ar`Od;Z8hLdd z$IV1olZ*@+25c9y>rkkn@~J{xf>tX4;~@ zA_Uwe7#--4MwbH^f7eSLk*V8<(cG+%FAqZ-In z%nKzKV5X%NwtIwf?g%_HYiCPao1c#lc9z)+S$*K`8m&9JS`zWp2HYw#0I)i*4m_5X zl|^{3ilpq_D`46X5_vj1;AT#7d;K8=tkHAV3fDcanQLH#AG7k5iYm&>Vfr7zz|g_@Eo^Qw1F8hm*uPyxjrN^&esPhO zl2X^)Tne_tN43H%E-Q;pKtKSw;fBcVtg3d}`I?`f9~XD;-*OG+D#EU&35QVfR7=Od z{-0U^T$EFMSY2$!v%`%9u(kb_ex!lC>4;|H^}KwcFQI)5IEB}Y-comxMD&y1p&?qZ zX9P@By~Ym1#o1d21C{-ncC3}1oqDnEEifK>0OGQwlRlVXU@P^20a$(WxHtvyO(vH0 zJ#seX9v3cvL}FlIz}&wyB5fqR(qxc9NolxPuMM9Jog|eLI7+>?JK68GoxI`Zsa=q1 zcXgSQgr#GnR>P$LDSP?yC0q?Ji|1K~& zISDu_7SWS`7eLxD_C4^1YeNNCn3$BXiZ&3_(WVl>l2EWYkQX53_q<$RUiNP+M0q4+ zWMrhG@)x`w><7wqfFQ~v{G9sw zq-7EmlnJC;ZSqE=P`iGU0UQN@=V4(uu%d#j$bqKwdz?Eh_fW{l$RG`v*t_XlMa4&* zCn%ul@OrsA*B0i7Lw>lu4Kru`%B1}5(W6HwnN!zLkV9leS_UekQsMsYE__761~QLC z9hMG!lJmmvyT}WUE_Nl5k&!`ui`et0grR2k%iE%3zzk0$^1#=CZHX< z^@WP$Eeiaq@aOxcX)s_Y5Wx}kU?5i<^VY3ftRGMwjSuE2LDrA7V&ro3AnAfx_tX9M zmX=#3G=lll)6;PIlDko#P)_9))S?ZnNg zsybfoO@opW>EyrGJMYmVY1_$q7~uEmTkf$ak0iy!+BYX^b>37KdkP8yT}2IKAKFCvYypv| z_0KOwUYRrYQlJPeP(-56uit`;Md7MO9OOnvV$wa=Eh}UF``53PW{Xxh;SGP^8su-?u5qDbf62`)sX8Wje+`>NH1>NB0c8Kl z$&z-2DEn|UgMlk4DCoR4Huv@QkcS5j=s0eVJiJ!CIeGB-@ncG!K)lqm_l(y_v_Tla zWuY!6B?Tgkp1wZd36|#Ph0veIb;5lh>5Ykr2_Y}~W|gX_q9XQR0r&@GM-TYxp!h2W zW*I@x$Veq1F#J;=50MRQ?DzK@644{lDHc7}T0oYzg>EFhP}!Sns{`N`gW$7L!Ksw8 za#?x#L72b;!>;(4goK0DK`g4Y6Hd-N$RVcJ)*>P!0YhXn4E+mS0rC=ISE-qqItB(| zxL^C{yuB^s64ZQs#-DOkg7Bry70SV>kzZC;R*ytH3IGUI?dYDNK&iDZUL%updM%E(t?5n9y7`$KbIZ+^T!sb zBPgBFQ_*lX+uNnyyovS41K{X2tm6t8q%4lfVSnu3_*6gPaoEj{3sbNC6X!?7;Xm<<~l`YF!SB^7Dg%1xyn;+TPw?U0xm@dW(j7 zI-@Z#pn!IB92Xz|-pCCM)E^)&ftaAGu5M)X98x3OwIMAVn-W|E92#}?L^y$qU24=% zp2R07cXo8pu@w2^#Ox%YZ|$z zxVYm4fM-8y9q$2b1#97l!#n4;yfz&8na~ArnECd2bP551B7m#}(J?SFD=R91g#flA z;5p|P96Q(1m*@ui=@ui}7XW~fk-0y676Otf09gUo2IwRYxfN zGe!lO%~Y`-6e7?7t>1s1eIQZjl2b z;NWl{P^5#yhHu(UY^G~k_yj4*&x^xSw7gQZw1HMbryz)I0oXn;2#ba~9iIS{BZ~PoPyAE2GtR)S$I5P{2adVu5)g9~e+eqN@HL6nC$J#8QlEX+z`RO)`@ zbAJBN%69MD+o55u8k>dS#6-$9x9oxfVgdpk!%7O!rGiwHfx+y+M1&yIuRDfGVi za(jkPv$~^msbDIOyKR->4jbEbAl-;2C*|bw|B=_nSA3{`6 zaQiGFC_s#!F^|uA_uJRm4iJ!XaAaMC>a>QTPG!XGBPn; zF2qRP=i01u+HtBJ9<8dX;u+FmO@~!9Hp+g59`cZBU~RD~iH)@&6!=zI3;Q;^s*pY`cwg*(z{&cjo0W3~)S@-T;P5Xja5f33I_J_AXhB5-A+g2hp zojpAqR8%0<`1SKALQ&DZlO%3?ThQE`&%@)IPtQnk34rRsA;j5vYjzW70#|iH=riv2 zl#jV-1q6tXp1(u~N##zAu7pGfxWb<;G;P5V+)mp^fNa+N7_VLgl5yz|`ANjbr>uma zI{{JaaM4wh%-Y&oNeLUMh-WuNj^)fms_?+&V-9_Sn1zQZE)IM0#LUUbiI}P64h~Mb zoUweCM_%H`kC|09nCQXG^uHkBYm~2Y+1lpq%-w|&pf-fz@#AY0)XaC7;+tPqW|4tM z1XJYXECChW8ZFrvS#`BD(7Hgz!Vtlp<6@sMu&|`{&xHgx96U&2dyS>Us-ohoso7u= zRByk=5G6iVJm*zYb7yUQr2qHC?83s_5-~mful}Cc!nCxoE0FYDju%sqP)m9_ot_FS zFmyG~nW>M@d7(_X9$7r5m;4-~G%(CHyF|Qw=m6>$mqa7=vs z{)sIH!sv8g~NgQ_=+y z1&77B)4hA#3{ja`hW?c;F|yltL!_;UC&1I)C7cJ0A8echG^Jh_49@WS(w`o)W7NK+HyZbu4ZBupwM5W+jmFg~)(^Iy z-@kj86GYIX8XkhnL1phX$tqfL-gSdW3(`+di&fq;(3!U zgHyBw)m(+`?KAxSM}GK5b92WX&#%nP2!QTlAcqHdn&V6&pYZBxWa%PP*=VP>4yPP^ z3%p&U&^NBeq#z`_&|dg`Fi|)zZ;&A1iQ(OAb+D>+ESZ;?DGPxu&H8AmX-$jTO-5Tp z3o{|vSD~|P(`|1?MqDhFj)u?D#012kppJ)zgObyHY<3C@^7W%IJ*t`=W{R>8-hQuy z?di$pL7Bo9hscoQWwJENHiWI+|p*2xqYo14XPHpX0Xz3Oi3 z{K&;(THYmemS#m82?c*MsAX5izCzF~j_57M!9@5sX|u*L<@-8K;=dg%$jgHqerknj z5&*&RaUVvLSFdgW@d5l}7}?m`9Kr9F*9)^>9maSDD@_2fQoCH<; z-WO@-OL7<;9W^n@wKnt)ZSW|V1~IN`bw38cW|{Iy>*-^{@8ICKQy0jm`9@n@H;F0GU?pk(J@L zs}vCDxwvq2Ii*ti=^_=yiXTAzh_z;TaI*1FgX7jas3?9%t@*vFxOsUWSR(TRZ9~H? z9$o4v@z={&9;kNW7p@m8BgK-WAwxSEvS~G2gEv3cBun!`r^v)RhBT3@KRC(KblrxmH{7r+{?pfEYr6*8$HYWQG`|V36(F7h z!oms*3gV#D10BYfFWWQC-BT%Y--jH`wZCW~eh|DU-0aI>1YOVT*C$XPZim-Gu>~cy zjkUG?Xj<)A7lpgx-BQRHOH0|GJ%jEr3PDj(YPOOskwa(*@Ijlr>7!htAAr=}1z?bC zu?S@PU2}J*a(5?5vJqrbud)Ifr}2EZ!v`aY4hMzmr(h*3B!tjL!eeI?Q&STP=~jbB z$|uBBqv!qm_Y**lM?<5?QnaU3n3&pI_3W8FQfXM#n+ZN*`0*_{xn-b6At)@&6M~|S zo11%W)dXOLhmifHrLj;_F3#EB!=ddp4pUc8a_z)^4Z>f*Qep6*&86faLO~CYi2=F8 zA~cF@ZE0y}NZCEtt}-??g+Gr1kwor!9+8v;UwWb3nE0_j{qhZnjX|^f;m6NVvPCJ$ zClwYGL2Z%fm^Iw4743x@XglHFXGfNswDP;FX`}k3WDD?F}mH=OaFhN4Xv?cH^dJ}dV)h|Ih$t)RHPyrI& zbsufG7{(#DVbN*`i78!-ygGi0Ly?~P^tM1tzZWs#d{=^DXci%qV(+9LYn#@FKFa{n z{t5_zAkWM%@GLGl^%YL5cTWh#{b4Id=>tkC5%l7Z@3uI+-N}LFy|_40%fU(#3@>01y5F9e&-D3T>_2Q`B_Jm!$0hK?g>(b* z7Dy^D&QI7cQxq?h2R|JM3JQKHDZ!6F{I!prZPI_BSI zWM~NC7pIH>_lp1UVRCr*H=&DJN5u7J?5FOr)HE3(A;cgtxY}J9BS*=@5lE)-9^CcW zx(yg-z4T+4g!Rd8(w9e#W0hA>OZA@Y4MUL2QBci#SOpNx)|BDZFO3^l_u!l)C6PUR zi!v4dcWCHiBfBykAsbSj7Ht$=|9Awa+(Y+EAdd#vog>$~qn2aV^Wr4LwlMuI%7t7D9(g!sj5? z+xykqw+^krTepASx@5AjczD)ws1Z}jA0dq)--eUj?>A`NrO7BoKzpvF6r+x#cQ5pjkNXJV`%Ox|yi9FWWAENIBy*b6dVD9{^=!p{rKJ@H zk~3b%n%T#Ebj^8t(2%=MffLtu{<6%K#m2-e3Z5S{gyGJ_+C18dEXEqj+`8p{yt5}3&8-} zLiy6H5DGW7X+Mn+5izu}i=5{gQ=LdK%4um3TtYH9YX zs`7}Ch`@9ERWIKb8%*Dem41=8Zx2B)gm%n?gDovB4GatTz*o(iZH;K!PNtTYF_DpK zWkwOz)!YyBvmZReB_kx^2bI3i>(}=7_HY^u4AAyPo|90cGVAPD7?Zdbzy%668rlAF9YshSw5vQfSrhr(j>DI6QJUaxCB-V{dZjgMyC8tIBPA4h z^X9mRq7sz%&wBxLj04;fvKnT{{f3&v z!pImBBTRd=aS8&u|i)ddH4Qd4Wk`$s^W>>3_UlXrXoQmNHd9sv~_ z+1{*5%l^gAr+Q3Hr4mo$Bq=DMQ{eHlXSdG{mF492mwPY%mP%>Q-t=nQbUxaYl9#9Q z{lV{$pI^N5Lz=3?Q^dvPcN{x28cuRd^g|^Ogzc}S8Fak8I9_a?oZk2bBDBE++f{q< z_ZGD_&Fk0P>7mODfyfW)xc8PIsi>$8(v8MQR$o6fkj?KzV#ov5*_ALU5mTs7BMGwn z%gbSNv!S1@!n0!Ec9_BeRCe;Uo@@1lwnhpCEv+_f4(R@TL%)8^DGYkA?}Az&)yd4H zec`?Wrz$!+4-*UPZIPyAMxTP>;N0w2XaF4c$@%u}`=37&%F5XIv>zmO&2T4&$JW+jFE}xy#LxEr5Ww;QyVl^` zuO=pT7oXM(7ld^|L9S08b7y8}qfme*(Rub~6#4qEv5{Z&k01Z21c}DMkKy6|@7l}* zjL}UAtnXh2NqZ8Jx;V}tI<|`5s+`>Y<6CCL<+R|f-1A7o)t0W_VB5~dG!hX+lj1$C8e05hPGfS$tUMkf7U zN>e~8$OU>sZichIznYzGAz)yT*Ed+;Wi~T5b_Q*rzP{+ISM`aBy3qGF<%#C!hqkz2 z5O$vnsRS@KE~2E`x=$KES#4X8lhgb2CsI@fqEo}}D(J&nA1O{jq<)Sjs;wpio%2B? zlbnD*05hsC{9cO>-+ofBt%6+$Qae+7 zKmhukhAw?JnZ)B#pwk*yO7bAw!6I%qo}VW+xQJLzrq84pfXR{sqbez7!9laDr44Ci z0t(?1q9`v9iRz!1dR|rJ<`#iNZEX`Fe^Eui^>|~nd2)lg5&&0|C|Pi<{LrTy2&iy@pj{^$V2GN5RMyF)({ zdTL+aY;dsW=-3#fSwGv_ruVP&+RV=aa0oU?M8qyHw+yBN1_MKNa!Gc*x|~ShGA3po zUtTf&-nAi0-VpKm^G|3K8W@OyCTa-GFjg`F&G*ntVapp2us#q04jV(#f8jw1K5>6b ze*HSjr|HkS4|N-MiY^?k=%^?VdP284XtWGg`l3LT8yzitO#EW6Cq3P=GN-0?3c6AOs{-yZ zOD+dOz9cO>1qIQcGauZ(|J}y$$NKyj1Ju6oSOAp_kY!K+YXHSbMtUpsxd8n_s6?{f z9_a@Q@|^MnZGLEIEgk}nG>D4PDS%Wu+A4*l=s>|zgcgWlPvAi6Y_sx$f(T3p z-rxuP)aLkh8U0J2ZBu9B?q)7vt8kM>&#OBu>g6U>fK3q)6kE@2VPcB&ZE(@kM@LCG zyJ~G8HD0oB&4M_sY7fv8EFf@05UG!R7XN5{&B<{EUqeCRC@v8MY^Qr8?hq4npJN`6 z-3coV#KoYCIX3z>G(0rajAP}u?|JFGGOgX_0wAstWDVehR(Ll4{HZhTS*khIF|o9i z5q0_$j(eY++&cyy9smy)$jO#G+PKpZC}@q_fnE%divlOHvlH?|pa$4)jNA1=xNw_h$D4EQzR;HG z&9#1*e&K!KKn=Q(a}ltc*jVL6?z!Kpy1*Z@_8hGaHbQqgw5fxX?&>0?F){JV_77Dz zgNu0SY4hbpR;KjeiVJ%tfL>{H77h-Fz?;L76?h(W$AcGq8a}=pRE&;Wnjl>VgbL8X z^}n?FgUT4>4W%+ljza@EqxnkEic%}R+i%?7669e;@h%+ePQ${A^CUd_VZ6FAh0pZ? z$+F+s=LdYNm&?8GEG(4(tg#zHAH^V#xqq#zgm|d)&g^69V5r04i5--bQ#>)_btmCr zmW1ljx2Bq)HPpbMC?lg%$GaS!GGe&&>ex(AN2i%|0GEQ}UGS3zR4$jY9!K%B2p3*N z@OZ_u?Y*IbvDQB<-(5`1FM^<7FNt>;S_L7qyUa^z94@MbYSq*fo`piT;&d%fOO}3p z>YlFt0R1%|T+^CnPlK@v0f4PowD>O$%}2YFK$O0!F&y!UGzd4hun_u34KET{U&cLa z4-4w}`{~oy^z@~q=xOpG?H<^eqyO3n5il4*bTrJ#uC}^C=7Sp;jNK$v0342!w1rO( z;8f7k#}@Yh--eEQUfPtAK`Qb5}xe>!s&7QPbtz8BY(M1mp49myrhg6 z7~p`T1e`TISHWVcArSX>r({fw5|JKpT47r?g{BEAh|2z=; z1ADVo2f6ss(Rc#7ONBSFRZcsYn7$>BTgA^zqmy4$Z$Tkac(zex zKD_46cOblOJXS~p=lh*OeXGDS5a3oKzb3`cQV>?y5DZ?v$_Ngy;CcF@^`tz)3ADE3-?QUxH3DvY6~_$<0Ia4^4W5xlDkUsTm`tC)z7fY?u)_R5 z8hi6_EWfsG_-c@$h)797l**Jjvs8wVqS08Cu}qogiX<|XF;kMEjFEXL5@ie}Q$m?? zQRdmZpWV;pGq1I@dbZaqRnk?CX}r?n3jFl=PGatu}Fu`t~Pt z4iD3%pNrl~zR8)Sf9gew#trK5I)*EnO_RZstHlDZ+im0TH7`3D&e!~v?VIF@`5QgF z?4B^(ZSCQ;WPYo6RGNJ1TE8U)MGf;AeG8_%ct-7bMa8?DHWq;10DLLtY6EfimCn)< z35n=^SDYRvC>b(Drox&gJ8j=k%ICYr-H$UUZ>;MH=Jx+STU?^T?YdkL#d0VUtU*S` z)qPG^Q#!x0y7M1);T6p0XmmSU#-DLUa@k^PV}{kbJl#rcf4}KT=~KK%)u=^dH($}? zJA3Zjtt3Z@rTL3{-mip`g*v>uPpzyQ-K64D|C4^1k9qnj$59(MclY)vp)4sW*Ij}i zL`n-jpP3gD70s9XMgH>Ti>IgIlkjq2OS}9}hK6!(+0y%^y~K0G84py#MH6`6fviol zS0z3$P@R11S4lvk_!2Lryk1D)c3HvJaVDz68K=Yisx}*mxaJ2a{26L};Y{aeX)Izl z(Q_*9HsgNiSk2oi9{5W#?%2_jsjDgH_32ygV6M0XNul&UzGiGCN7ZH-HOufJ(3VZ) zL%1VM%;;IW7;h1pvE+R;guRx6$XWOYkVr9ih)!=(OW+3{k4qQYnT9T^$f#4Eo32=E zy-55#OH3<@Nz(4qOS|B(0JEQ?RQ7JKb4LVl0p(VeYg(VL{;jldS2e;%z0XkH{=Y7{ zjfrsh@-|6?oe_y7x(N-Y(5|r@`sY{Hr~v6diDb6rzwTyF-yRC7`&DH$5i-ndekNru z=h9z(7h3NgjZ;`yIewVT#CH6Oe~9B|^BIFfRfb1PULLXFcI0PDp;K#qpg)_V{^xb9 zW60bX^qP*2j>2(QpR$TS9htq}c81gQi2fCwEi?f(Gd(CaATe0~`H`{fIO)CWg$oYr zYpWybq)gwc&3}%g;YvtIC|{fOAUY-n<@0h&G)afTm-@#&C{mN2KQ~V$(aYm@?GJii z#N`$^pu2^q1k}o(={9nBqN^1O?B#s(HZ#f%X6$&CF#`hwchmb?&N+~~9Gc^0p^Qz% zy|7`J%|5-ov#AeJ|MBA!(o_mhTUuJamAgquF)HK{Uwk+C_h{W_wy68}KOjq$TOJp| z4JYrRVN?(~I?F00qoD!w4W}fL4Wb)-sS{cBOYC!Va*FIncaS+qw^X#XZ$sHFzw&1j z{;5TSPq@hJ+kQ{Ir61ixNNAY`)k%*OaTDKLo}Ca87N()0A#Pc>NP_YYC#C{Fanq`a zxH|FOQ_eC5=g%{dnMh&YO`BWjaT5V%NYGhWdW&rDVIHftw6|}gqFR_Joy28An29U+ zUZ{MgeeU8#gF-8|xu=`1V!q|Kqlp>q@DNQY;{Ey96!&;6&uK&p`{Ei)cxs&u+Lh1p z&Rx2+j)pBbkElpfk#68(nclbur=v z@R}Jphb{z+GdB$vzCAQH_Apw##r)LClM72rkhrztC7&a%D-KVPp7483e4?WEXw8NX zL;gLI3LVDf6;NMvCPFzLg=ex@TU(w);|zLGiSH64%f4x=@P?Je;RK-C5SHUlE9}6N z%WYx4zLCRcGybEbOhpGg-Nu%opBFQAPDyW4Smxs8U4}*;yd3;<+VBjI(Rzt#ahdNT zCY8DA2D1~pF&$NQU*C9jWv<&AnJM-5?Zix@eRgz%JGY2P9=@BA!G~uXK}*0DbpE2V zhL|eE>>eyGDtiC^eao2*96N`a69k!Pfa78i6`o?AE;!5V6FhR{Ar_SR-)yf{HO#HH z6b(G^19lq5x3zq4i8 z8lU6mvBb~)!^1CFSxuk|gqSn?0X%v_fN5ZWK%JjQdO(Sbr&fKnBhf}jMU6wEUR^y1 zD-z;Y8;D;G{B^z@LY;^L=S^K*-2t_TsD-nY5vuDq` zyUXB_6E^!&D{RB!YVm=24p!hadZh6ja>h~v_my|TcO5O(W?d$7{Th7Pa z@OB;n7WnBCp+d*pNJpp9+HitbQ-b_DMBK!O?*7-zymt&`+#Aadz&5Ax7Nq~QY0A^G z%k}j3#yN&)UN}O(_C{PA{b$T3qGvc%yDL7Vgf7QIIk18mA?s1 z5-tdEcPX}-Q7&@7*m%wPM?kJf>1VoDe?3|^N#`doeYP@MecbTiaX`Smg7$tK{F|j$tSG&eL~IAVQA@Oc{Kv^; zuKoMrl(f3Kin}IaGb>hV0-$~?G!(&z2ZQ>pXU_-`Fc}dl0cW7a0uwp0_qLs#l;iki zGqaH(Hfb}{J55^=FEg)(P%G$tyEzB5A$a)6udm#exl41Nh=Ac6Hf_PS1;>astnlMS z#l?6uLVV z+GvyPMP=RQdh*S&k+cQNuPt;KdI2bEJF}S$1F66|U{{QuHC0trObB}gY{PzZY>1?B z;zT7T4fIzPm%L6~a;wuXQr8ndzgn0sxy_EfasE4iNhs$!^T2WcNYv&0O(c683{h5u^UOsJMG4+Tw(pM3>?t=LEXTCM`7cPh? z)&=Y;&doL5%jVKsGRxHQERcR!uq^6ES<;7%R4eRmQM32MeRj{$9`q+IX+`5u;u z*9~0${ID_`VUU%*w@c8DnsYDR{BXi*X2wm!N>7KMb$2{hw>Sn?`RLk_ZqZ@LxXpjO zy@Rdx#O2FT#l?&L)qcPPJ@GSSD*nHIYNx16y3J)=KJ#UV+uUY$X=bNCKhmvwU9)rx zxp{bWjUv}V($dZ<)-6uYR#M{=etw33=ld=h7!+9dF`pjoB6i!SXvu=!pc$_Nb9v+> zz^2+qtcB)nDTqw3($edD@Tk`Czfush^dT> z%)@{HM36OdVV?kW3Fz3vaGp_9Hj8^`E$LpN*#WiUu_|1 zI-d}D(0nNr8^HVFOvP&hLv|_jo#Os zvN@(FU%8WlVsU;lRjlDK6HRmnJVLq`boZM_Rd42?h7<$Mv5ZukEi5b$v+T9_m@NUV zVyish^CU9T)WpQ(I`JPG3<-xVu+v|WrJ&Ro70t-XGRf=2uziVrK z7KM)5)V{rQkyU&g-Qch0zoT-^yHRhkJ=olR?Yd%}KjZO@$3NfM^z8oqUTV(X(n)#u zV2_nF5ss>oB>f^E- z*ROXiEOVM8lcc!C$LJ4&26D?-mPo7wAJRW+l`WQ4*3|xDkKi6!G zo<2clhop+uhh9VDmSLadV;-;XCVg8hkT`Jr)DDurmp?_av%mp9S)V;lo*I8kgr5Pd4WVK!4N}4;^JRZ_dP2B)p6pDB5 z6Y}cPW2NAIweGWZp?P?AxON$>-d1D4#zu|6$3WwYwjE?~m|Aa@UxHw2Z-+j6(lo&X zo`)J#hj&tSFC`(9T3?&DQlttVewNYWTip*Ew*=Y9@8!H63xDDtHL7g7XXO)O*&z_P zyZ`bec`u^br^fC3lKHO)OGrsIG~P1GHb`M%(FEP(HZ$_2YTM$XAdF>N+uMTzx21as z2?&%G97TFf*AqdE-NV{JR9?*309W^jnzf*XG5RI4%kzL&_${9yXe6 z-*=Nd-mraJkCml#Yuj|*!a`c((@QR{XH0W{J`OV~FEfdXnX}%ns;9S^^Jk=CK~&7z zu-s6*?3!uQUa!@Xym7C<)So2oTJ+$-jwV0<339Ln6@-_^>bgb;H7F^_0l~rJT^B4H zW5~CMR^Gas<0kzX<+oB(8x-7J!zO|zS2#;@`K&AqS|}?UoP7WBM*R@aftpVrb5T%eg^V3Qk=@wtBF|aKR*)rE9vi{omADE3BQ!Z~7M=0jAoDzkc$Xx{F$K?eSi@xP&cUT$=fA}qL0)< zgEbu{Zqg8iu5b})i48zBysI`V+zAK>;Fge~BzbH45dVDoly&0M22ws3KF$Qv{WVb+*H+OD0~o>u-)#TmEYom(Bgt$R z?kmmK!J!5TB#FEckc1ed3s@b{iKGzMBv29s#^BoOyXquTD{@uugX1+@Ni71%v|6}% zd4EGv$;gQ68aM~)9B;_>CHU$#lQ@3i%7=ah86|1PkTxEr5zjJ_6zPhrpDBS*SAJFk7QCXo(f=s8~Gj7gBZXOJ+5Xl>fE z%^CLH7cVjsqlwwUQ3ITUq;Q&ePQiV_e#B*3Nw|zFA_fDOQmnL75SE~K6CHl`vOjAt z$y=1Tk_BXcFuJ^qFMA6RHvn6doeT`er71`Xsl-5%iLvmOB7T($i%&?qzJ0q)Je(BL zes9OOU%rS53y0wqdjCtt*+x3D4gb3iAdT1`kx~2KT7YCcYZoPE1u@?e@r;QcZ}nla z`|#h-(Q1KSeWO&o!6U>EXfp)yUbiA2EB%a|1lqru8T)^Km<@k;Zu33N!hcCV4@_b& z>*2Fg#Mh!xvm0CDs?;ZtxtgLm3l${E+XpW*Uj&AX7tfxB{R>fUlH#av@#X5_>3?ri z6*H0?j1~|rEv*|j4*dH71@YH?qVbo)u;f|Tsdxn~ z1}NA5ed1#Xga65V5$qK)$83$xo(+LE5!VAK`k<=tlw#Lm?~US@9NW1wBiNfQ>W`9I z3Zl~cP>T~q2g#$_k+Xw~Xnfw6KwJ==8q*Hi`M;@WtLc8ZY9_Al{MpCFHM&vkZ zFH#~Iv(BC#mmuAu#3k>c_}?w-YOk)ojTl8XBB!nux>n_bR>7{rm^mm^y@RUXzvmVd zoJ4~4@5V*(bv7CkVFWY^{F?4YPm!&^KTwPCf4_!xCW(!9OEih}KIoJ*iS&Raq1iCk zs0IZPk)sFtC|qDh54-9+x!G*6-aHJYu`N0=^Z#;+`%Jm-3*E-5FQ56)i-&jWcM}J)KoWF<)c%{Huq}v{a+Oow~?$riHqz1 zaWtrQ8XGbf$5prE$Jbt{{dI9^Zfw}HxX|#@X>_<*#MrnRlwx1y`rC#Ep4BB#3S9*Z0Y)a)7v~>F8o5CQ}W{ z)d_38)WAo}NFjK9dWs+KV47JDmd|QxaBuyyQn^}?;WlsZ{kx&iJ9OnX#RcGC2@${Q zbk<{%i$*RkEPmCGf`gxgRdMgE)+2E^y35J`CjcEzg5(f_VVJ|gNP_?{Mn^}(%9g7A z<||geBJ6a34!s1?}%6)cR)pqzG@839xa zFYR!mZWIu>OOF~JAock0yl+s@cjWiSk1qplBylEN8{Ve#d-cll>Mhx-Z$;MSYS_P4 z7X~v>8$q1Z|M|HWN}?&}c=i%Vh9&s<*}!7lk~OLex#knn+8PEE44#7@zIAnh(A`2o z?#{VZ92)wu+^tYh|6z7qfx}RP?GUKPeqY~gRAQ{Ml@SqEXIm;>zjkVPmwJW`aFa)Q z!AUCr2gfAsNB$szl6IT3cX25`>3Nc17e=BM+&!J0k0Gy^LGh2+49Qh~Qqo=e{ajp+ z;qk!2^0EEkv$Kbyce~9kAaewu-iJw1U?qFi#Kd;sB>nUOI=@nv+dq)&7GM7;r3f-w zf|@hW)N9w?y|UKd2Q;2;Jhs>B^rtUf+Ko>O0jx|`tUCIam$!xU`uotkx$(f)!FaGD zJ|4uL3N{UpQ64G3`PQ=6ZOZ5q}zv6`dae5%xm zJzlokdU1+YPA(7g_+{Fm!d#Zn5c zO#3|J!osG;JM$2$fQa2uXsPt_x}$J*J_?Q;`en}NYE52mo`QT`*3&(qy5DoYSA7V5 zYa;da^-(1$$;laMX)07gqT|P@K7MjNO$7&hO_c`~?vf}5x_v^bsyg17zKWByvglf` z^1Y3G2<4{#tB&?E(-i3 z2^170kTAzczln&gmQ_B~&z?Oyq~79!C#m`HAs$8?C>T-URNto^oL&UZVOrq|SDA(e z^`Yb9;9x{VL<|l4{A^$DpaO4xoJvS&RKh&Wi5&3ywcV_Vfk?u`llz~YExP_uLeOx? z<8Mt_@$FqG#Qp-Lq}6{QI>ai_$8&w3r`scBQ} zm7J?Yj&lo0Fz>Nr3xLTIyw(8JCG~>>V{~ey@VP#>7d{PGHnVH6yz2s!1z!7HV6KG6PJaJD$WBKs?AGdnxH zy^m-%kMc#hYiVUfc5EBEb4*$JJ1!|BD^H-SEiHETKM!!94Bji5;xDg=%jgM2*230| zZqTMEB4XBy8LTp@=;rvVfH{W3qeXa>loOH+qK=NTCUU`zDIh3ltwlvzp%h&ZB|zHW zzI}^vproLXl$Mt8SOIE|eOK)TKZ9aV&%wr6Nn%OzURr{3@L5t)S<&UQ(&=de(7g*> zGVibY6)A9_Aw~?kaN|4^{_8^**Vo_8e&3HHA&y?ND72D=zx!*u7k%}S#SWDjnVHgv z#t$Am=F!{-fNkM#?G>+ek3T0OIeZZ6gLn5k|9G2~;JI}0z^@p8)xIyU3d>!zb8H8j z;Q{}#sVU9lW@BsXMGXzK*j^gac5)H~1O$@ZC*)w0CKZc#!yWp8v9T-GW&9HJF6i)h z&Xrw&r%(!Zbosl#5a-O?ln)HvU%l&_u?@Vr<3Nxtr(;289!XZM}&N!>21TC0h(F58SFTx_d-X!m-XZ=$N-$(JW zEM3>+42q{pVUaEL%2M3c9}{9~ipC-PGDHLCuXohrG$S-(T0GxnF|dLrOv5}AxU>1G zYb=W`dzf*h7Tj>3eRZxrxE2wZ&T3`hFKnAmUb{;-2>W@|h=8{dn)37VV=^*sc4YDE zpSg(4*uWruv1oMohlQByR9RW9V8x%KVq|%XC(}T=0Cr3{?`t$jvYNb>_VT5ej7x7p z`yYMUW;%RI^)Kq+8%r~S<~J6O5y@yD+^k?0b+2kOSw~T^ z3JG{5zwZ5TVI4J9RX|PA0FaZRno>mb&OBNAv~*1^u^%r0t}lSY*UE}J)M-fWbTU&? zh9V6;&uMAFWByqjg_NMo0m_!cf$Z}8q+ckw*$;l8MMIHXLgn?3JG!nUY9n7CVTz)N zMm=d32baQ}oE(rYtvXvtiOhJu)VC&iwZZJ5v|Lau>7cDi0>1{@c^S+wCdr*b96d!%%*>+~bA44F! zyaEESB@;nJ(*Tn+qK9n@CaSp=B$ayzJBEga$aJ$SmL7s9BcvAgQYfxle-QL-ipKQ( zJiGYK-FWuK>em}dac4l$`6~9qPz;Epx#^uVM`UDHRu-oZO|%4E^`TA_scN@QGy~-X z?GqrC7oa?L66FJzsp&8d;sBWf`9WM$mG4$0>40V3iE*YZ{7I5J4Da=voTE65tLu1T z-XZjKp+dpumS=w}OV_U#v7qCE=Y%wcN{l+M*O2i3(6r3gByT3b)_6srrH+v~S`PPA zcy#ymt%FkFLn0DFlpUe5e`0{J|m+Gl?F=p?6bLZ!ghT&B3~ZOS5J@G4kVol_{XJfRbWNA3)3zZbYp@M{0FkM#9uctSz1`wOhGX_ z-uW2wtN`UPY5h*^ufz*khLsh7+_h8B9OmUE=++_pjUV#BiQq412wA^0RsJRh1_lWA z5NuG-syz4)_e@HLl_(04*up{yG6%;Ox_vU2u~%Jvw-F=3O^HSJ$B>cd3^En0)#MnM z4Mx_V(!JXRMu)Vf286q*NkCjY0ziRQ05kvV{QOw)TZA|x423^w-59C$?(27zu(h?p zj}L7s003x+TZ8uE^9ePz-(U&y?|Cy}oUn^k#fVw*@$uo!B@!~Fv8Tc}KEaz7amG9} zG&C3PrC2hod-k+K5kU)hu-z0#L&14w-aqkK^2wHv1dDqvM&}c%4CK&M%Q6t;} zk7hhr`a=O|g{7tKt@M(|4bC7fQ{Ym1v}YE*x5%vVV%qm28N!N1P$py=N3e_)29MUY zM}r|Kg|yR;-4pDig3iC6gPb=uGJ+`|ihotza#siN;_e=ajxkKFCCzX^; z5LfVkKlwmyjkTwPjwJ>klrL!^5ydw$094kPyE_KRGm<_6GfGqrz09`6)q7ZLRSgwG#whtjx1 zv-??D(4TrRGBV<^ga}d^-%im~Inp1SdE7H+r5k{8eQ6&KSCj1^o^$>>Rv<0dA4_2S!Js>S-$ARrjF}3X83_UoDW9R|)1x zuQ|C8??~f)@$FvRGy^(FA(cUXieV)wKs!Rpe**O{p??9g$3(n6wOO!-7cR{Hn|&mb z3S?GLW55d%@J=El)}Hsp5oDZq?i{zUID*MS+CeO}%)$PCTOvjPLE^>ny&6L zoW>;q`j;DhDxKx1etz$$iAMfyzl(M1-Ix%uZn2 z{F9FORRllNuESEHa7@M#K!DM`n*gekr)iHEc%tOptGU1YHhYOGiK_!5q1J z=?LlUE{yntchIRgPV}fsQ+V*4*<9`%s`h*A)G2E-{XL1*)m7mnjx9IEmw#th($UdD zf{_w;sv%y!S;qYshwLh$8ORgq>x^zLix2(#bK!4gyo^iTuOqZ9!tPT)o42!d8J-RC z8r5<)_s)|FI_^F zFL!Xr{z|7b@C612p7qtqqaGgBkVT*Hn@fH1!hST#2uMpSH@o;#jlrjsEwGnOWsaFwFUgn`kD}7ERX&A z|3-VV{~M~|@uBjq`tjqfh-r{W>Z5?If_(0cPvT4V$(7BM3oD9?JA;DCK7FF`S`SCs z%;ocQNN#;jw{LmA?L!b-LTIP}J(Ex^-1?Wug*TR#dO=hM0MIe=d0tU*&D{KBPjMR~ zqw&pK10V;f^z=|rAdxwYs?}Ax-teaB7RCd`8ET%F zw7J@`@K+nney6FZ>4LPhmL|ooLmhzrd@=Ef-=mUoE;S!aPAV$qXHUk0z6Hf@Ys+Bh zS?pci|Dop8!a{?|{h+a~yjnwT?dAEtb(7r7xNyb#T0JArT^X5_)y3g>^v-yD4}#S< zFc7I&G?C&w$-Tq({+8`*QY+ook@SiN6%9DrEJB4_%67jB#~~>jj`o%uY0hM~`BlH| zloigQ-8Q0INjhLANbggS8$+9h3^qZ^;Ru;;08Jz(G&IPNa0n;R?KKOJhkWJt3g$_7 zxAd1=YLDz}b{C>>0&lvX>qJ8XU%aeZaKP4%FJf1_c5+?^83+*tmrp~?8=I=l+4$P+ zV?C11U0O$k3c>kJwjKG&h?%{z`Bide^PTob-Fcuae6rkY$j0huy-DcfB+;4=-hLd#){Zz!%}bfttN^9o@^X zN|c4C(?~y_BbsgV4Z99lVx)>E90{Pf)gQ-=qwzS-} zAzlY4qow>sEvNuANtRnC7`m`V# z!O6UrcK9k@EAL3vSl)m5@Z7gtc`&9P?j{=XE0Uba|BZI2^K2sJhUmVr*a_CPF zNT>RWY>ffJ@q0z4-A~r%n=sD&8u3|2XHHX0u#6ktt{Q7l#&HGsK zqa!BPlle}QDdXdT*_CV0oUF0S!L8H2NbdknoW=30%E4>b@;aLnOerYHZEYe@mm~rF zN|X>hqWY+=#pi^CQls1F-g;{dztbIkQXzpEZUDJPc}@DGXWGP zCA9?g8xP<3B)a;8_K}ff=us2)CiNSt+%tj7r6Y)8{R>ABR@ zVS8EGO^lCmylO7O20A0Td@k#BiM#I)2APv*+ix;mpl3xY?A5H1Zxf0(B45&YjT|u>kw$Qxm>QNlis%Gw<38 zfU324W833)qy6sI)xS?mC^_6{WLO_wDVBZj+?13g0mXvlLP?lbsXX2KS;lQUt;VJr=6%7E>ZF#7sHaLz8u!4Wxa+uFI12p| zSFXw~pt`ID2NYF0vi3O0XLd(WacUGE7|Ne*dJCJC?0HMB|JgTj`-5fW%a;%DFw#J- zTJzyPqoPlh#ht_l)&0V3(AH&ZY}LN}?cB4RnCRMrvZ+hQ{QYyy&2iQ#6nI)VsHY8J zPlTUq{HI;#3aWyYM@^ zk6pe(-X`%tuL@F;miwUEy_U&^>|QI--84EF0>S~S4B-x7*Tl%IUJ(iQ>sH?%M$g9W z4X$OCan3c9d-e_>v~tSC5088nz*L3W<+oM+;GoE> z?=L2MRQ>%W!3Vr;Z9RB!r~zc&Ub&uUpVdTNU*4CViC zTbPx7B@f+M$s;2<WZ(_#}t3 zvGt_=@2|6ymFrWB)9NO!dhYH|D%YC_ zSQ&4l2OH8KKY`DG&Oe&?W^DG2OH6EWqW3&;=JFhLGlF^#j7@%^&<-h1bw^CZ@q@H% z7V{JOqa_Y%0|Wcf>jP`R6N-1EuR=bF<5i3p94MPm693FO`+; zMal|I7`-CiNcYj(I4j9{;qT0hU&DBL*PE6@#NMEa#tS`UcsQmNvS&;gRE70r0$g0v z$Yaye(z?^l8U3tM*%-Np$2+4uYK^Hm5g5Tr-K_qsD(?~yz5H%*4L73(r}J1SSd7S7dx=ZM~Jkvcp-cH`4c|L2*Rx|bIq zuR5WsYKvUhCnPCpovuaLDIGDpp^*~jbES3uBf<6^yjswbzFOq3){c$P!oGGN%&aK= zazWB!QF6$vywADz96X>aB*P#L-~(K|cvVs5)F~H`5x?rGf#+xGx4}2be7I?rn))+X z!ySyq$O>m>X;}?A5XG|ek1IYq%FmBRbQcIB0p;IZl`tSWBoyqRh9v~C4u+j1Jar~Uh-QI@t zUh+}ooE8|ej(b(VK;yrX$;;~oo2We2Tju84%Sk1!tEQaM`nCZk+F{#)wmO9mCOM1da{R8k@}oluOu4n(Y@gTt@s#nZ6@eqT*L2W<&^4mAEM z$7hGAsB1f!18&^B*?=hTUXdg-|6u(#wO&_C4kc%DQBj(&W&%#_rP_R|a;>EPee#-5 z^1=4rh*SZ~-w+`kGbpl|0Lk)Pxzv5}S0Nz#x$|Jxh$p)}{SG6I96~@~0{ z0An~CGNBorYGpAodjQ~*&$hfnLK+nnb@T!_-|#C%TEdayqT_N#K8U^<5fNG8;bi&s z9w@GE4tzdpBAbaZtE;b9R=Q8G2n?nlDWt#Jw{L+s;jzM|ADN?DOX9NzyOhB9@1xWr zcbGIzJ;vVSe-eTL3-eJ?TYuNosk!(!rornU9`VM;ESFN6oj z=_{F#IbsFi?2-z*1EKW7#vjTwm#K*+9nx8tmnleZw3pf?pck@o3JT$Ve0EUa+$6E-a_#Uxt)^zND;sC6Ls^_&%I(y<>i+4|Cv1Yz*%RG` zzs5V;YJ(HBapqK~)0veVF&ffauX4}TvAWPrkl|$Ts|yd#MS)tq6rNN3?kB+TN-o5BVw!uN4 zkeL}uh(S8Qb7;oI+g1cl47%aw4xOx8CzAO{w zsr2Pi+QeTFA5ASj1|;jbzQ(4e*8M`QL;9HL>zUB&GM?`1UtYn0oPVFZCr#!yb|~6N zKaA0U>E_a?qVm8KThDV}m|Yd8rP*K|-!~LKI`Xx=^fQ)g!&(Y*^M8F2(vro9IC|wV zW~ln>f1&8s>h9kY$EvHx_bunzV&d@=d9UMNq{@%9)sBOSS5i|O2HD1Yk#Dzb*Dn;e z64nL|c6Q6-d4i803;O!8H-G#%*4g&0-cqU3bJN2B3Ra1%oSfYQzr^m*hsC-d<{P(m za>@p1`lY+f=v&UYZ}(oTwsuI|G`d^t^VfNwOgtr-iG356i$bl_^4#pg!l5BAoZh&r z!yJC~@Pch^HJ^C}wS|^emd8jr)G*~vla8t%yP<5sp}QK`B#Of>8yKvC{YGKp@-^nl zwxb+ZyODT3icfy|Qdmq(QdG3a&527^wi?TMaoPiE%BlV5(98j~2D?lVJZGYS7yz^L z{B^n#6t0kQ4}?(*iyoozYd}mgy}SY-L5BsKp#1tT>q>s@ixWmr&e=@Uh(F~@nPgFL z^M$&4WWmB~HK67(&vq`Z697GHWDfV*+=%Px85R?pq_``A^s!-055kJsp?)3R;A8e? z>(DD1menBwY~7U=CEqtZNB-Rq)^g(hHZC9c>Bjwv&@tRa23xs2VT$8;l%^ZvSJ^~@ zY8qTls|VUT4lyc1JnlHL5)gf<0xc?HVzY1GGQxrq(hu#FKx%5qvg;qS+fqDi$vR<` z>!;z(iI4bhjbv2ZV`cDpM5h5zG-9*8yJg1&oLPOyzdzuo@#+Hval~UBaUZC0fTs)w#&mcDv~O zkh1a*wK+|m#;mU0dr@B>PS;wR&53SK&jFHuDM+&&?4h`u#B61-p@oG&gA+9XSHz%` zVSt1T1n}9{uklwts2cX=SQ}0c3?!lHRD^P6ZP|o^;*@sV8oD)bY}ND$X=AMH;qch~f)jZ@VBqLXPCeNN*ii_0o?*A%G|;3TI*d;@a z@A;#V2&gdSF1SuSFag?2z21K7z`6PGg9T5`24+pfN^iYHdmoym93r`Qi+m81U%x?4 zMdo^NXj|U+0`#ZIki=Z$5eAOBT|_~GX2qv3U+&2>INysNxP^#_iW@`0z&a_mM(F5v zuTD3|aVgWOM-Nzj3N^7?x3ecJjHigcRM1Ks08J4tc01xg!8=`BjS&sB;semi$| zzS-V#;o?PTiz_BpE0Ex>tVCK^=(Fmpugw0wBqwJjznULWTzbhVLhCuix2hj?G4CVt z--`E^CAC>S+Sp1&8&W>&?2$9QY(p$4JsPqmGo_mSaxU{Z%nnJPUhXa1_)j?u&GWjK z$hn5#$^5=XUQ!pDcS#CvFVfQ+`L1yokex6@1@~@HknrFCHfs2vt_qKFJYwQBT$O*y zxGIB*Is13Q<5iF{YA5jI7tf@0kbX%>-T+HP;VRpC^FT2ubY#4wxF>tnOlHpWcO8Zj z9qbi!;{X4Q8c;6$pLfHjM@?!?DtUQu)kKKQ#6g{%SO11~Cn3;)l$jJl1K$1~TRU|+ znzXr(XQw84TGN5u$>hM^Z)QqdDZ~O=Zv1b{|9mjajQ@ALi2rdH_*4JK0}QT{*wbp- UBdk&)@MDtdsWT_D6tCX>UuHJ;&j0`b literal 0 HcmV?d00001 diff --git a/docs/architecture.md-1.svg b/docs/architecture.md-1.svg new file mode 100644 index 0000000..a49d9fd --- /dev/null +++ b/docs/architecture.md-1.svg @@ -0,0 +1 @@ +

Browser

octw-proxy
Nginx HTTPS ingress

octw-oidc
oauth2-proxy

octw-api
FastAPI

octw-edge
HTTP + WS proxy

octw-db
PostgreSQL

octw-cache
Redis

Docker API

Tenant gateway A

Tenant gateway B

\ No newline at end of file diff --git a/src/octw/web/static/app.js b/src/octw/web/static/app.js index c9201a0..61bbb08 100644 --- a/src/octw/web/static/app.js +++ b/src/octw/web/static/app.js @@ -22,8 +22,19 @@ const el = { tenantMeta: document.getElementById("tenant-meta"), }; -async function fetchJson(url, init) { - const response = await fetch(url, { +function toSameOriginPath(path) { + if (typeof path !== "string" || !path.startsWith("/")) { + throw new Error("Expected a same-origin absolute path"); + } + const url = new URL(path, window.location.origin); + if (url.origin !== window.location.origin) { + throw new Error("Cross-origin requests are not allowed"); + } + return `${url.pathname}${url.search}`; +} + +async function fetchJson(path, init) { + const response = await fetch(toSameOriginPath(path), { credentials: "include", headers: { "Content-Type": "application/json", ...(init?.headers || {}) }, ...init, From 0c65803d9c7d42d0543a0291ff8c029a3ec716e6 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 04:17:17 -0500 Subject: [PATCH 04/20] Harden proxy config and docs --- README.md | 2 +- configs/nginx/default.conf.template | 8 ++++---- docker-compose.yml | 6 +++--- docker/proxy/Dockerfile | 4 +++- docker/proxy/entrypoint.sh | 16 ++++++++++++---- docs/deployment.md | 4 ++-- 6 files changed, 25 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 150ef1f..420f3ff 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ This starts six services: | `octw-db` | `5432` | PostgreSQL metadata store | | `octw-cache` | `6379` | Redis sessions and locks | -If `configs/certs/fullchain.pem` and `configs/certs/privkey.pem` do not exist, `octw-proxy` generates a self-signed certificate on first boot. Replace those files with a real certificate for production. +If `configs/certs/fullchain.pem` and `configs/certs/privkey.pem` do not exist, `octw-proxy` generates an ephemeral self-signed certificate inside the container for bootstrap only. Mount real cert files in `configs/certs/` for production. ### 5. Open the browser app diff --git a/configs/nginx/default.conf.template b/configs/nginx/default.conf.template index 6718bdf..9c885c3 100644 --- a/configs/nginx/default.conf.template +++ b/configs/nginx/default.conf.template @@ -4,17 +4,17 @@ map $http_upgrade $connection_upgrade { } server { - listen 80; + listen 8080; server_name ${OCTW_SERVER_NAME}; return 301 https://$host$request_uri; } server { - listen 443 ssl http2; + listen 8443 ssl http2; server_name ${OCTW_SERVER_NAME}; - ssl_certificate /etc/nginx/certs/fullchain.pem; - ssl_certificate_key /etc/nginx/certs/privkey.pem; + ssl_certificate ${OCTW_SSL_CERT_FILE}; + ssl_certificate_key ${OCTW_SSL_KEY_FILE}; ssl_session_timeout 1d; ssl_session_cache shared:SSL:10m; ssl_protocols TLSv1.2 TLSv1.3; diff --git a/docker-compose.yml b/docker-compose.yml index d83548b..c59364a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,15 +114,15 @@ services: environment: OCTW_SERVER_NAME: "${OCTW_SERVER_NAME:-octw.example.com}" ports: - - "80:80" - - "443:443" + - "80:8080" + - "443:8443" depends_on: - octw-api - octw-edge - octw-oidc volumes: - ./configs/nginx:/etc/nginx/templates:ro - - ./configs/certs:/etc/nginx/certs + - ./configs/certs:/etc/nginx/certs:ro networks: - octw_control diff --git a/docker/proxy/Dockerfile b/docker/proxy/Dockerfile index 455b4b7..6cdb1bd 100644 --- a/docker/proxy/Dockerfile +++ b/docker/proxy/Dockerfile @@ -1,4 +1,4 @@ -FROM nginx:1.27-alpine +FROM nginxinc/nginx-unprivileged:1.27-alpine RUN apk add --no-cache bash gettext openssl @@ -7,5 +7,7 @@ COPY configs/nginx/default.conf.template /etc/nginx/templates/default.conf.templ RUN chmod +x /entrypoint.sh +USER 101:101 + ENTRYPOINT ["/entrypoint.sh"] CMD ["nginx", "-g", "daemon off;"] diff --git a/docker/proxy/entrypoint.sh b/docker/proxy/entrypoint.sh index 9059142..ed29c8a 100644 --- a/docker/proxy/entrypoint.sh +++ b/docker/proxy/entrypoint.sh @@ -4,19 +4,27 @@ set -eu CERT_DIR=/etc/nginx/certs CERT_FILE="$CERT_DIR/fullchain.pem" KEY_FILE="$CERT_DIR/privkey.pem" -mkdir -p "$CERT_DIR" +GENERATED_CERT_DIR=/tmp/nginx-certs +GENERATED_CERT_FILE="$GENERATED_CERT_DIR/fullchain.pem" +GENERATED_KEY_FILE="$GENERATED_CERT_DIR/privkey.pem" if [ ! -s "$CERT_FILE" ] || [ ! -s "$KEY_FILE" ]; then SERVER_NAME="${OCTW_SERVER_NAME:-localhost}" echo "Generating self-signed TLS certificate for ${SERVER_NAME}" >&2 + mkdir -p "$GENERATED_CERT_DIR" openssl req -x509 -nodes -newkey rsa:2048 \ - -keyout "$KEY_FILE" \ - -out "$CERT_FILE" \ + -keyout "$GENERATED_KEY_FILE" \ + -out "$GENERATED_CERT_FILE" \ -days 365 \ -subj "/CN=${SERVER_NAME}" + export OCTW_SSL_CERT_FILE="$GENERATED_CERT_FILE" + export OCTW_SSL_KEY_FILE="$GENERATED_KEY_FILE" +else + export OCTW_SSL_CERT_FILE="$CERT_FILE" + export OCTW_SSL_KEY_FILE="$KEY_FILE" fi -envsubst '${OCTW_SERVER_NAME}' \ +envsubst '${OCTW_SERVER_NAME} ${OCTW_SSL_CERT_FILE} ${OCTW_SSL_KEY_FILE}' \ < /etc/nginx/templates/default.conf.template \ > /etc/nginx/conf.d/default.conf diff --git a/docs/deployment.md b/docs/deployment.md index a70220c..e92a91f 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -81,7 +81,7 @@ If you need Google Group enforcement: docker compose up -d --build ``` -`octw-proxy` will generate a self-signed certificate if `configs/certs/fullchain.pem` and `configs/certs/privkey.pem` are missing. +`octw-proxy` will generate an ephemeral self-signed certificate inside the container if `configs/certs/fullchain.pem` and `configs/certs/privkey.pem` are missing. ## Verification @@ -141,7 +141,7 @@ sudo rm -rf /var/lib/octw/tenants/* ## Production Notes -- Replace the generated certificate under `configs/certs/` with a real certificate before exposing the stack broadly. +- Mount a real certificate at `configs/certs/fullchain.pem` and `configs/certs/privkey.pem` before exposing the stack broadly. - Keep `octw-api` and `octw-edge` bound to localhost. The public entrypoint should stay `octw-proxy` only. - The default Postgres and Redis containers are adequate for development and small-scale testing. For production, move them to managed services and update `OCTW_DB_URL` and `OCTW_REDIS_URL`. - Pin `OCTW_OPENCLAW_DIGEST` if you need deterministic tenant image rollouts. From da4153bc5166d46b4bb69c96043e42e4a78374d5 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 04:40:00 -0500 Subject: [PATCH 05/20] Harden HTTP calls and Docker user --- docker/proxy/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/proxy/Dockerfile b/docker/proxy/Dockerfile index 6cdb1bd..641ed46 100644 --- a/docker/proxy/Dockerfile +++ b/docker/proxy/Dockerfile @@ -1,5 +1,7 @@ FROM nginxinc/nginx-unprivileged:1.27-alpine +USER root + RUN apk add --no-cache bash gettext openssl COPY docker/proxy/entrypoint.sh /entrypoint.sh From 44f4c5ecc3cb3e744ed5a084f3feaf00c0d559c4 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 04:47:27 -0500 Subject: [PATCH 06/20] Document Google Workspace secrets --- docker/proxy/entrypoint.sh | 4 ++-- src/octw/common/config.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/proxy/entrypoint.sh b/docker/proxy/entrypoint.sh index ed29c8a..7fbae15 100644 --- a/docker/proxy/entrypoint.sh +++ b/docker/proxy/entrypoint.sh @@ -8,9 +8,9 @@ GENERATED_CERT_DIR=/tmp/nginx-certs GENERATED_CERT_FILE="$GENERATED_CERT_DIR/fullchain.pem" GENERATED_KEY_FILE="$GENERATED_CERT_DIR/privkey.pem" -if [ ! -s "$CERT_FILE" ] || [ ! -s "$KEY_FILE" ]; then +if [ ! -r "$CERT_FILE" ] || [ ! -r "$KEY_FILE" ]; then SERVER_NAME="${OCTW_SERVER_NAME:-localhost}" - echo "Generating self-signed TLS certificate for ${SERVER_NAME}" >&2 + echo "Using ephemeral self-signed TLS certificate for ${SERVER_NAME}" >&2 mkdir -p "$GENERATED_CERT_DIR" openssl req -x509 -nodes -newkey rsa:2048 \ -keyout "$GENERATED_KEY_FILE" \ diff --git a/src/octw/common/config.py b/src/octw/common/config.py index 1adcf73..c71997c 100644 --- a/src/octw/common/config.py +++ b/src/octw/common/config.py @@ -1,7 +1,9 @@ from __future__ import annotations +from typing import Annotated + from pydantic import Field, field_validator -from pydantic_settings import BaseSettings +from pydantic_settings import BaseSettings, NoDecode class OCTWSettings(BaseSettings): @@ -44,7 +46,7 @@ class OCTWSettings(BaseSettings): # Browser auth via reverse proxy. trusted_proxy_enabled: bool = False trusted_proxy_user_header: str = "X-Forwarded-Email" - trusted_proxy_ips: list[str] = Field(default_factory=list) + trusted_proxy_ips: Annotated[list[str], NoDecode] = Field(default_factory=list) # WebChat is enabled by default during onboarding. webchat_port: int = 18790 From b51a69fe78aa853446009ec67d82cb6fb8f3a290 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 04:53:42 -0500 Subject: [PATCH 07/20] Add USER directive in proxy Dockerfi --- src/octw/edge/proxy.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/octw/edge/proxy.py b/src/octw/edge/proxy.py index aba133b..a9e9308 100644 --- a/src/octw/edge/proxy.py +++ b/src/octw/edge/proxy.py @@ -205,6 +205,11 @@ async def _iter_upstream_messages(upstream) -> AsyncIterator[str | bytes]: break +@edge_app.get("/health") +async def health(): + return {"status": "ok", "service": "octw-edge"} + + @edge_app.api_route( "/{slug}/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"], @@ -252,8 +257,3 @@ async def proxy_slug_root(slug: str, request: Request): if slug in _RESERVED_PATHS or not _SLUG_RE.match(slug): raise HTTPException(status_code=404, detail="Not found") return await proxy_path(slug, "", request) - - -@edge_app.get("/health") -async def health(): - return {"status": "ok", "service": "octw-edge"} From dd6fb0d4b40d144ba411c7f7ae49b3fcb3fbed5f Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 05:29:19 -0500 Subject: [PATCH 08/20] Monitor docker compose services --- configs/nginx/default.conf.template | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/configs/nginx/default.conf.template b/configs/nginx/default.conf.template index 9c885c3..aeec15a 100644 --- a/configs/nginx/default.conf.template +++ b/configs/nginx/default.conf.template @@ -6,12 +6,15 @@ map $http_upgrade $connection_upgrade { server { listen 8080; server_name ${OCTW_SERVER_NAME}; + port_in_redirect off; return 301 https://$host$request_uri; } server { - listen 8443 ssl http2; + listen 8443 ssl; + http2 on; server_name ${OCTW_SERVER_NAME}; + port_in_redirect off; ssl_certificate ${OCTW_SSL_CERT_FILE}; ssl_certificate_key ${OCTW_SSL_KEY_FILE}; @@ -25,6 +28,7 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto https; proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Port 443; location /oauth2/ { proxy_pass http://octw-oidc:4180; From 42a023bc633dce77175d433e6bcda3ff651afec2 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 05:34:25 -0500 Subject: [PATCH 09/20] Add /health endpoint to edge --- configs/nginx/default.conf.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/nginx/default.conf.template b/configs/nginx/default.conf.template index aeec15a..2bda136 100644 --- a/configs/nginx/default.conf.template +++ b/configs/nginx/default.conf.template @@ -32,8 +32,8 @@ server { location /oauth2/ { proxy_pass http://octw-oidc:4180; - proxy_set_header X-Auth-Request-Redirect $scheme://$host$request_uri; proxy_set_header X-Forwarded-Uri $request_uri; + proxy_set_header X-Scheme https; } location = /oauth2/auth { @@ -47,7 +47,7 @@ server { error_page 401 = @oauth2_signin; location @oauth2_signin { - return 302 /oauth2/start?rd=$scheme://$host$request_uri; + return 302 /oauth2/start?rd=$scheme://$host$escaped_request_uri; } location = /health { From 88bf4b165c3571593ab468fabeb7c65a6b6bab87 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 05:38:30 -0500 Subject: [PATCH 10/20] Investigate hanging site after cert --- configs/nginx/default.conf.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/nginx/default.conf.template b/configs/nginx/default.conf.template index 2bda136..5f0f31f 100644 --- a/configs/nginx/default.conf.template +++ b/configs/nginx/default.conf.template @@ -47,7 +47,7 @@ server { error_page 401 = @oauth2_signin; location @oauth2_signin { - return 302 /oauth2/start?rd=$scheme://$host$escaped_request_uri; + return 302 /oauth2/start?rd=$scheme://$http_host$request_uri; } location = /health { From b0f17f130833fc2129e3ab0edf7d07c36ad6617c Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 05:41:25 -0500 Subject: [PATCH 11/20] Troubleshoot octw-dev hanging page --- configs/nginx/default.conf.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/nginx/default.conf.template b/configs/nginx/default.conf.template index 5f0f31f..68c83ce 100644 --- a/configs/nginx/default.conf.template +++ b/configs/nginx/default.conf.template @@ -30,7 +30,7 @@ server { proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-Port 443; - location /oauth2/ { + location ^~ /oauth2/ { proxy_pass http://octw-oidc:4180; proxy_set_header X-Forwarded-Uri $request_uri; proxy_set_header X-Scheme https; From c155d686984ca55d48794b653a34749de26e3c2c Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 05:48:59 -0500 Subject: [PATCH 12/20] Fix oauth2 redirect loop --- configs/nginx/default.conf.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/nginx/default.conf.template b/configs/nginx/default.conf.template index 68c83ce..8750cb2 100644 --- a/configs/nginx/default.conf.template +++ b/configs/nginx/default.conf.template @@ -58,14 +58,14 @@ server { return 302 /app; } - location /app { + location ^~ /app { auth_request /oauth2/auth; auth_request_set $user_email $upstream_http_x_auth_request_email; proxy_set_header X-Forwarded-Email $user_email; proxy_pass http://octw-api:8000; } - location /api/v1/app/ { + location ^~ /api/v1/app/ { auth_request /oauth2/auth; auth_request_set $user_email $upstream_http_x_auth_request_email; proxy_set_header X-Forwarded-Email $user_email; From 8a27d58bbca3962e151aa77a077e819cbf06ebf1 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 05:58:44 -0500 Subject: [PATCH 13/20] Fix infinite oauth redirect loop --- configs/nginx/default.conf.template | 2 ++ src/octw/orchestrator/docker_orch.py | 34 ++++++++++++---------------- src/octw/web/static/app.css | 29 ++++++++++++++++++++++++ src/octw/web/static/app.js | 32 ++++++++++++++++++++------ src/octw/web/static/index.html | 4 ++++ 5 files changed, 74 insertions(+), 27 deletions(-) diff --git a/configs/nginx/default.conf.template b/configs/nginx/default.conf.template index 8750cb2..aeba890 100644 --- a/configs/nginx/default.conf.template +++ b/configs/nginx/default.conf.template @@ -69,6 +69,8 @@ server { auth_request /oauth2/auth; auth_request_set $user_email $upstream_http_x_auth_request_email; proxy_set_header X-Forwarded-Email $user_email; + proxy_read_timeout 300s; + proxy_send_timeout 300s; proxy_pass http://octw-api:8000; } diff --git a/src/octw/orchestrator/docker_orch.py b/src/octw/orchestrator/docker_orch.py index a31d054..67f6a19 100644 --- a/src/octw/orchestrator/docker_orch.py +++ b/src/octw/orchestrator/docker_orch.py @@ -8,10 +8,10 @@ import uuid from pathlib import Path -import docker from docker.errors import NotFound from docker.types import Mount +import docker from octw.common.config import settings from octw.common.logging import get_logger from octw.models.tenant import ContainerState, IsolationMode, Tenant @@ -46,14 +46,6 @@ async def main() -> int: close_timeout=5, max_size=1_000_000, ) as ws: - challenge = None - try: - first = json.loads(await asyncio.wait_for(ws.recv(), timeout=0.75)) - except asyncio.TimeoutError: - first = None - if first and first.get("type") == "event" and first.get("event") == "connect.challenge": - payload = first.get("payload") or {} - challenge = payload.get("nonce") req = { "type": "req", "id": "octw-probe", @@ -62,26 +54,28 @@ async def main() -> int: "minProtocol": 3, "maxProtocol": 3, "client": { - "id": "control-ui", - "version": "octw-probe", + "id": "openclaw-control-ui", + "version": "control-ui", "platform": "linux", "mode": "webchat", }, "role": "operator", - "scopes": ["operator.read", "operator.write", "operator.admin"], - "caps": [], - "commands": [], - "permissions": {}, + "scopes": ["operator.admin", "operator.approvals", "operator.pairing"], + "caps": ["tool-events"], "locale": "en-US", "userAgent": "octw-probe", }, } - if challenge: - req["params"]["challengeNonce"] = challenge await ws.send(json.dumps(req)) - response = json.loads(await ws.recv()) - if response.get("type") != "res" or not response.get("ok"): - raise RuntimeError(json.dumps(response)) + while True: + response = json.loads(await ws.recv()) + if response.get("type") == "event": + continue + if response.get("type") != "res" or response.get("id") != "octw-probe": + continue + if not response.get("ok"): + raise RuntimeError(json.dumps(response)) + break payload = response.get("payload") or {} if payload.get("type") != "hello-ok": raise RuntimeError(json.dumps(response)) diff --git a/src/octw/web/static/app.css b/src/octw/web/static/app.css index 702fcaa..21d19cb 100644 --- a/src/octw/web/static/app.css +++ b/src/octw/web/static/app.css @@ -61,6 +61,32 @@ body { background: linear-gradient(135deg, var(--accent), #05504c); color: #fff; } +.button:disabled { + cursor: wait; + opacity: 0.7; +} +.activity { + display: inline-flex; + align-items: center; + gap: 12px; + margin-top: 18px; + padding: 12px 16px; + border-radius: 999px; + background: rgba(11, 122, 117, 0.08); + color: var(--accent); + border: 1px solid rgba(11, 122, 117, 0.16); +} +.spinner { + width: 18px; + height: 18px; + border-radius: 50%; + border: 2px solid rgba(11, 122, 117, 0.2); + border-top-color: var(--accent); + animation: spin 0.9s linear infinite; +} +.activity[hidden] { + display: none; +} .chat-shell { display: grid; grid-template-columns: 280px 1fr; @@ -115,6 +141,9 @@ textarea { align-items: center; justify-content: space-between; } +@keyframes spin { + to { transform: rotate(360deg); } +} @media (max-width: 900px) { .hero { padding: 28px; } .chat-shell { grid-template-columns: 1fr; } diff --git a/src/octw/web/static/app.js b/src/octw/web/static/app.js index 61bbb08..50be735 100644 --- a/src/octw/web/static/app.js +++ b/src/octw/web/static/app.js @@ -11,6 +11,8 @@ const el = { summary: document.getElementById("summary"), status: document.getElementById("status"), deployButton: document.getElementById("deploy-button"), + activity: document.getElementById("activity"), + activityLabel: document.getElementById("activity-label"), chatShell: document.getElementById("chat-shell"), messages: document.getElementById("messages"), prompt: document.getElementById("prompt"), @@ -50,6 +52,14 @@ function setStatus(message) { el.status.textContent = message; } +function setActivity(message = "") { + const active = Boolean(message); + el.activity.hidden = !active; + el.activityLabel.textContent = message; + el.deployButton.disabled = active; + el.resumeButton.disabled = active; +} + function addMessage(role, text) { const node = document.createElement("div"); node.className = `message ${role}`; @@ -126,12 +136,15 @@ async function connectChat() { params: { minProtocol: 3, maxProtocol: 3, - client: { id: "control-ui", version: "octw-web", platform: "web", mode: "webchat" }, + client: { + id: "openclaw-control-ui", + version: "control-ui", + platform: navigator.platform || "web", + mode: "webchat", + }, role: "operator", - scopes: ["operator.read", "operator.write", "operator.admin"], - caps: [], - commands: [], - permissions: {}, + scopes: ["operator.admin", "operator.approvals", "operator.pairing"], + caps: ["tool-events"], locale: navigator.language, userAgent: navigator.userAgent, }, @@ -184,6 +197,7 @@ async function bootstrap() { if (!state.session.tenant) { el.summary.textContent = `Signed in as ${state.session.email}. Create your dedicated OpenClaw workspace.`; el.deployButton.hidden = false; + setActivity(""); return; } @@ -191,11 +205,12 @@ async function bootstrap() { el.chatShell.hidden = false; el.tenantName.textContent = state.session.tenant.slug; el.tenantMeta.textContent = `${state.session.email} · ${state.session.tenant.status} · ${state.session.tenant.verification_status}`; + setActivity(""); await connectChat(); } el.deployButton.addEventListener("click", async () => { - el.deployButton.disabled = true; + setActivity("Provisioning your workspace. This can take a minute or two."); setStatus("Provisioning your workspace. This can take a minute or two."); try { const response = await fetchJson("/api/v1/app/deploy-or-resume", { method: "POST", body: "{}" }); @@ -205,11 +220,12 @@ el.deployButton.addEventListener("click", async () => { } catch (error) { setStatus(`Provisioning failed: ${error.message}`); } finally { - el.deployButton.disabled = false; + setActivity(""); } }); el.resumeButton.addEventListener("click", async () => { + setActivity("Waking your workspace."); setStatus("Resuming workspace."); try { const response = await fetchJson("/api/v1/app/deploy-or-resume", { method: "POST", body: "{}" }); @@ -219,6 +235,8 @@ el.resumeButton.addEventListener("click", async () => { setStatus("Workspace connected."); } catch (error) { setStatus(`Resume failed: ${error.message}`); + } finally { + setActivity(""); } }); diff --git a/src/octw/web/static/index.html b/src/octw/web/static/index.html index 115db19..23cb196 100644 --- a/src/octw/web/static/index.html +++ b/src/octw/web/static/index.html @@ -13,6 +13,10 @@

Deploy once. Resume straight into chat.

Checking your workspace.

+

From 9f03bb30a97fbae24a251fc0757e60ab26ea345a Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 06:06:47 -0500 Subject: [PATCH 14/20] Fix slug regex overriding routes --- src/octw/api/auth.py | 5 ++++- tests/unit/test_app_v1.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/octw/api/auth.py b/src/octw/api/auth.py index d4c0b62..75f52cc 100644 --- a/src/octw/api/auth.py +++ b/src/octw/api/auth.py @@ -110,7 +110,10 @@ async def get_browser_user( ) -> tuple[TokenPayload, bool]: token = _extract_cookie_token(request) if token: - return decode_token(token), False + payload = decode_token(token) + row = await session.get(UserRow, payload.user_id) + if row: + return TokenPayload(row.user_id, row.email, payload.tid), False forwarded_email = get_trusted_proxy_email(request) if not forwarded_email: diff --git a/tests/unit/test_app_v1.py b/tests/unit/test_app_v1.py index 2277112..dfde31d 100644 --- a/tests/unit/test_app_v1.py +++ b/tests/unit/test_app_v1.py @@ -44,6 +44,42 @@ async def fake_get_or_create_user(session, email: str): assert payload.email == "person@example.com" +@pytest.mark.asyncio +async def test_get_browser_user_rehydrates_when_cookie_user_missing(monkeypatch): + stale_user_id = uuid.uuid4() + fresh_user_id = uuid.uuid4() + stale_token = auth.create_access_token(stale_user_id, "person@example.com") + + async def fake_get_or_create_user(session, email: str): + return SimpleNamespace(user_id=fresh_user_id, email=email) + + class FakeSession: + async def get(self, model, user_id): + assert user_id == stale_user_id + return None + + monkeypatch.setattr(settings, "trusted_proxy_enabled", True) + monkeypatch.setattr(settings, "trusted_proxy_user_header", "X-Forwarded-Email") + monkeypatch.setattr(settings, "trusted_proxy_ips", ["127.0.0.1/32"]) + monkeypatch.setattr(auth, "get_or_create_user", fake_get_or_create_user) + + request = Request( + { + "type": "http", + "headers": [ + (b"x-forwarded-email", b"person@example.com"), + (b"cookie", f"octw_session={stale_token}".encode()), + ], + "client": ("127.0.0.1", 1234), + } + ) + + payload, issued = await auth.get_browser_user(request, session=FakeSession()) + assert issued is True + assert payload.user_id == fresh_user_id + assert payload.email == "person@example.com" + + def test_build_user_tenant_slug_is_deterministic(): user_id = uuid.UUID("11111111-2222-3333-4444-555555555555") assert build_user_tenant_slug(user_id) == "u-11111111222233334444" From 27a6efbcf6c6c6830a84dafcbb44191d10e42267 Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Mon, 9 Mar 2026 06:14:40 -0500 Subject: [PATCH 15/20] Add spinner and fix provisioning --- src/octw/orchestrator/docker_orch.py | 15 ++- src/octw/web/static/app.css | 3 + src/octw/web/static/app.js | 176 +++++++++++++++++++++++---- src/octw/web/static/index.html | 8 +- 4 files changed, 174 insertions(+), 28 deletions(-) diff --git a/src/octw/orchestrator/docker_orch.py b/src/octw/orchestrator/docker_orch.py index 67f6a19..5a11f8c 100644 --- a/src/octw/orchestrator/docker_orch.py +++ b/src/octw/orchestrator/docker_orch.py @@ -85,9 +85,18 @@ async def main() -> int: "method": "health", "params": {}, })) - health = json.loads(await ws.recv()) - if health.get("type") != "res" or not health.get("ok"): - raise RuntimeError(json.dumps(health)) + while True: + health = json.loads(await ws.recv()) + if health.get("type") == "event" and health.get("event") == "health": + payload = health.get("payload") or {} + if not payload.get("ok"): + raise RuntimeError(json.dumps(health)) + break + if health.get("type") != "res" or health.get("id") != "octw-status": + continue + if not health.get("ok"): + raise RuntimeError(json.dumps(health)) + break return 0 raise SystemExit(asyncio.run(main())) diff --git a/src/octw/web/static/app.css b/src/octw/web/static/app.css index 21d19cb..af4cdc7 100644 --- a/src/octw/web/static/app.css +++ b/src/octw/web/static/app.css @@ -48,6 +48,9 @@ body { .lede, .muted, .status { color: var(--muted); } +.status[hidden] { + display: none; +} .button { border: 0; border-radius: 999px; diff --git a/src/octw/web/static/app.js b/src/octw/web/static/app.js index 50be735..4ab9bcd 100644 --- a/src/octw/web/static/app.js +++ b/src/octw/web/static/app.js @@ -4,9 +4,12 @@ const state = { requestId: 0, pending: new Map(), sessionKey: "main", + locale: "en", }; const el = { + heroEyebrow: document.getElementById("hero-eyebrow"), + heroTitle: document.getElementById("hero-title"), landing: document.getElementById("landing"), summary: document.getElementById("summary"), status: document.getElementById("status"), @@ -14,6 +17,7 @@ const el = { activity: document.getElementById("activity"), activityLabel: document.getElementById("activity-label"), chatShell: document.getElementById("chat-shell"), + workspaceEyebrow: document.getElementById("workspace-eyebrow"), messages: document.getElementById("messages"), prompt: document.getElementById("prompt"), composer: document.getElementById("composer"), @@ -24,6 +28,129 @@ const el = { tenantMeta: document.getElementById("tenant-meta"), }; +const STRINGS = { + en: { + appTitle: "OCTW Chat", + heroEyebrow: "OpenClaw Tenant Wrapper", + heroTitle: "Deploy once. Resume straight into chat.", + checkingWorkspace: "Checking your workspace.", + deployWorkspace: "Deploy Workspace", + working: "Working", + workspace: "Workspace", + chat: "Chat", + resumeWake: "Resume / Wake", + promptPlaceholder: "Ask OpenClaw to do something useful.", + disconnected: "Disconnected", + connecting: "Connecting", + authorizing: "Authorizing", + connected: "Connected", + send: "Send", + signedInSummary: "Signed in as {email}. Create your dedicated OpenClaw workspace.", + tenantMeta: "{email} · {status} · {verification}", + status_running: "running", + status_paused: "paused", + status_stopped: "stopped", + status_error: "error", + status_provisioning: "provisioning", + verification_verified: "verified", + verification_pending: "pending", + verification_failed: "failed", + authFailed: "Authentication failed: {message}", + provisioningActivity: "Provisioning your workspace. This can take a minute or two.", + workspaceReady: "Workspace ready.", + workspaceResumed: "Workspace resumed.", + provisioningFailed: "Provisioning failed: {message}", + wakingWorkspace: "Waking your workspace.", + workspaceConnected: "Workspace connected.", + resumeFailed: "Resume failed: {message}", + running: "Running...", + sendFailed: "Send failed: {message}", + timedOutWaiting: "Timed out waiting for {method}", + websocketNotConnected: "WebSocket not connected", + gatewayError: "Gateway error", + }, + es: { + appTitle: "Chat de OCTW", + heroEyebrow: "OpenClaw Tenant Wrapper", + heroTitle: "Despliega una vez. Retoma directo en el chat.", + checkingWorkspace: "Revisando tu espacio de trabajo.", + deployWorkspace: "Desplegar espacio", + working: "Procesando", + workspace: "Espacio de trabajo", + chat: "Chat", + resumeWake: "Reanudar / Activar", + promptPlaceholder: "Pídele a OpenClaw que haga algo útil.", + disconnected: "Desconectado", + connecting: "Conectando", + authorizing: "Autorizando", + connected: "Conectado", + send: "Enviar", + signedInSummary: "Sesión iniciada como {email}. Crea tu espacio dedicado de OpenClaw.", + tenantMeta: "{email} · {status} · {verification}", + status_running: "activo", + status_paused: "pausado", + status_stopped: "detenido", + status_error: "error", + status_provisioning: "provisionando", + verification_verified: "verificado", + verification_pending: "pendiente", + verification_failed: "fallido", + authFailed: "Error de autenticación: {message}", + provisioningActivity: "Provisionando tu espacio. Esto puede tardar uno o dos minutos.", + workspaceReady: "Espacio listo.", + workspaceResumed: "Espacio reanudado.", + provisioningFailed: "Falló el provisionamiento: {message}", + wakingWorkspace: "Activando tu espacio.", + workspaceConnected: "Espacio conectado.", + resumeFailed: "Falló la reanudación: {message}", + running: "Ejecutando...", + sendFailed: "Falló el envío: {message}", + timedOutWaiting: "Tiempo de espera agotado para {method}", + websocketNotConnected: "WebSocket no conectado", + gatewayError: "Error del gateway", + }, +}; + +function resolveLocale() { + const query = new URLSearchParams(window.location.search).get("lang"); + if (query) { + const normalized = query.toLowerCase(); + if (normalized.startsWith("es")) return "es"; + if (normalized.startsWith("en")) return "en"; + } + const raw = (navigator.language || "en").toLowerCase(); + if (raw.startsWith("es")) return "es"; + return "en"; +} + +function t(key, vars = {}) { + const strings = STRINGS[state.locale] || STRINGS.en; + const template = strings[key] || STRINGS.en[key] || key; + return template.replace(/\{(\w+)\}/g, (_, name) => `${vars[name] ?? ""}`); +} + +function applyLocale() { + state.locale = resolveLocale(); + document.documentElement.lang = state.locale; + document.title = t("appTitle"); + el.heroEyebrow.textContent = t("heroEyebrow"); + el.heroTitle.textContent = t("heroTitle"); + el.summary.textContent = t("checkingWorkspace"); + el.deployButton.textContent = t("deployWorkspace"); + el.activityLabel.textContent = t("working"); + el.workspaceEyebrow.textContent = t("workspace"); + el.tenantName.textContent = t("chat"); + el.resumeButton.textContent = t("resumeWake"); + el.prompt.placeholder = t("promptPlaceholder"); + el.connection.textContent = t("disconnected"); + el.sendButton.textContent = t("send"); +} + +function localizeTenantState(value, prefix) { + const key = `${prefix}_${String(value || "").toLowerCase()}`; + return STRINGS[state.locale][key] || STRINGS.en[key] || value; +} + function toSameOriginPath(path) { if (typeof path !== "string" || !path.startsWith("/")) { throw new Error("Expected a same-origin absolute path"); @@ -49,13 +176,15 @@ async function fetchJson(path, init) { } function setStatus(message) { + const visible = Boolean(message); + el.status.hidden = !visible; el.status.textContent = message; } function setActivity(message = "") { const active = Boolean(message); el.activity.hidden = !active; - el.activityLabel.textContent = message; + el.activityLabel.textContent = message || t("working"); el.deployButton.disabled = active; el.resumeButton.disabled = active; } @@ -94,7 +223,7 @@ function nextRequestId() { function sendRpc(method, params = {}) { if (!state.socket || state.socket.readyState !== WebSocket.OPEN) { - throw new Error("WebSocket not connected"); + throw new Error(t("websocketNotConnected")); } const id = nextRequestId(); const payload = { type: "req", id, method, params }; @@ -104,7 +233,7 @@ function sendRpc(method, params = {}) { window.setTimeout(() => { if (state.pending.has(id)) { state.pending.delete(id); - reject(new Error(`Timed out waiting for ${method}`)); + reject(new Error(t("timedOutWaiting", { method }))); } }, 15000); }); @@ -123,7 +252,7 @@ async function connectChat() { const url = `${window.location.protocol === "https:" ? "wss" : "ws"}://${window.location.host}/t/${state.session.tenant.slug}/ws`; const socket = new WebSocket(url); state.socket = socket; - el.connection.textContent = "Connecting"; + el.connection.textContent = t("connecting"); let connectSent = false; const sendConnect = () => { @@ -152,7 +281,7 @@ async function connectChat() { }; socket.addEventListener("open", () => { - el.connection.textContent = "Authorizing"; + el.connection.textContent = t("authorizing"); window.setTimeout(sendConnect, 700); }); @@ -167,10 +296,10 @@ async function connectChat() { if (pending) { state.pending.delete(frame.id); if (frame.ok) pending.resolve(frame.payload || {}); - else pending.reject(new Error(frame.error?.message || "Gateway error")); + else pending.reject(new Error(frame.error?.message || t("gatewayError"))); } if (frame.ok && frame.payload?.type === "hello-ok") { - el.connection.textContent = "Connected"; + el.connection.textContent = t("connected"); await loadHistory(); } return; @@ -181,7 +310,7 @@ async function connectChat() { }); socket.addEventListener("close", () => { - el.connection.textContent = "Disconnected"; + el.connection.textContent = t("disconnected"); state.socket = null; }); } @@ -190,12 +319,12 @@ async function bootstrap() { try { state.session = await fetchJson("/api/v1/app/session"); } catch (error) { - setStatus(`Authentication failed: ${error.message}`); + setStatus(t("authFailed", { message: error.message })); return; } if (!state.session.tenant) { - el.summary.textContent = `Signed in as ${state.session.email}. Create your dedicated OpenClaw workspace.`; + el.summary.textContent = t("signedInSummary", { email: state.session.email }); el.deployButton.hidden = false; setActivity(""); return; @@ -204,37 +333,41 @@ async function bootstrap() { el.landing.hidden = true; el.chatShell.hidden = false; el.tenantName.textContent = state.session.tenant.slug; - el.tenantMeta.textContent = `${state.session.email} · ${state.session.tenant.status} · ${state.session.tenant.verification_status}`; + el.tenantMeta.textContent = t("tenantMeta", { + email: state.session.email, + status: localizeTenantState(state.session.tenant.status, "status"), + verification: localizeTenantState(state.session.tenant.verification_status, "verification"), + }); setActivity(""); await connectChat(); } el.deployButton.addEventListener("click", async () => { - setActivity("Provisioning your workspace. This can take a minute or two."); - setStatus("Provisioning your workspace. This can take a minute or two."); + setActivity(t("provisioningActivity")); + setStatus(""); try { const response = await fetchJson("/api/v1/app/deploy-or-resume", { method: "POST", body: "{}" }); state.session = { ...state.session, tenant: response.tenant }; await bootstrap(); - setStatus(response.created ? "Workspace ready." : "Workspace resumed."); + setStatus(response.created ? t("workspaceReady") : t("workspaceResumed")); } catch (error) { - setStatus(`Provisioning failed: ${error.message}`); + setStatus(t("provisioningFailed", { message: error.message })); } finally { setActivity(""); } }); el.resumeButton.addEventListener("click", async () => { - setActivity("Waking your workspace."); - setStatus("Resuming workspace."); + setActivity(t("wakingWorkspace")); + setStatus(""); try { const response = await fetchJson("/api/v1/app/deploy-or-resume", { method: "POST", body: "{}" }); state.session = { ...state.session, tenant: response.tenant }; await connectChat(); await loadHistory(); - setStatus("Workspace connected."); + setStatus(t("workspaceConnected")); } catch (error) { - setStatus(`Resume failed: ${error.message}`); + setStatus(t("resumeFailed", { message: error.message })); } finally { setActivity(""); } @@ -246,7 +379,7 @@ el.composer.addEventListener("submit", async (event) => { if (!message) return; el.prompt.value = ""; addMessage("user", message); - addMessage("system", "Running..."); + addMessage("system", t("running")); try { await sendRpc("chat.send", { sessionKey: state.sessionKey, @@ -255,8 +388,9 @@ el.composer.addEventListener("submit", async (event) => { idempotencyKey: nextRequestId(), }); } catch (error) { - addMessage("system", `Send failed: ${error.message}`); + addMessage("system", t("sendFailed", { message: error.message })); } }); +applyLocale(); bootstrap(); diff --git a/src/octw/web/static/index.html b/src/octw/web/static/index.html index 23cb196..078cb20 100644 --- a/src/octw/web/static/index.html +++ b/src/octw/web/static/index.html @@ -9,20 +9,20 @@
-

OpenClaw Tenant Wrapper

-

Deploy once. Resume straight into chat.

+

OpenClaw Tenant Wrapper

+

Deploy once. Resume straight into chat.

Checking your workspace.

-

+