diff --git a/AGENTS.md b/AGENTS.md index 5ebe4cb..4aaaed4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,7 +68,29 @@ - `generate-compose.sh` merges templates into final compose files - `setup.sh` generates secrets using `generate-secrets.sh` - `.env.example` documents all configurable variables +- Documentation lives in `on-prem/docs/` +## Documentation Requirements + +**Always update documentation when making changes.** The docs folder (`on-prem/docs/`) contains user-facing documentation that must stay in sync with the codebase. + +### When to Update Each File + +| File | Update When... | +|------|----------------| +| `docs/configuration.md` | Adding/removing/changing environment variables, changing defaults, adding new `DC_*` variables | +| `docs/quickstart.md` | Changing setup flow, adding new features users need to configure, changing ports/volumes/services | +| `docs/support.md` | Changing support boundaries, adding new component categories | +| `docs/README.md` | Adding new documentation pages, changing known limitations | +| `.env.example` | Adding any new environment variable (always document generation commands for secrets) | +| `on-prem/README.md` | Changing scripts, profiles, or file structure | + +### Documentation Style + +- Use tables for configuration references (Variable | Type | Default | Description) +- Group settings into **Required**, **Frequently Used**, and **Other** sections +- Include example values and generation commands for secrets +- Document both localhost development and production deployment patterns ## Environment Variables Reference diff --git a/README.md b/README.md index 9fe9475..6422b30 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Docker configurations for running Currents services. ## On-Premises -For self-hosted Currents deployment, see the [On-Prem documentation](on-prem/README.md). +For self-hosted Currents deployment, see the [On-Prem documentation](on-prem/docs/). ```bash cd on-prem @@ -12,3 +12,6 @@ cd on-prem docker compose up -d ``` +## Development + +- [On-Prem Development Guide](on-prem/README.md) β€” Architecture, scripts, and release steps diff --git a/on-prem/.env.example b/on-prem/.env.example index 6a3d305..01eda50 100644 --- a/on-prem/.env.example +++ b/on-prem/.env.example @@ -1,7 +1,9 @@ # ============================================================================= # Required Configuration # ============================================================================= -# These values MUST be set before starting. Run `./scripts/setup.sh` to generate secrets. +# These values MUST be set before starting. +# Run `./scripts/setup.sh` to generate all secrets automatically, +# or use the generate-secrets.sh commands below to generate them manually. # Initial Setup ON_PREM_EMAIL=admin@localhost @@ -11,20 +13,35 @@ SMTP_HOST=localhost SMTP_USER=test SMTP_PASS=test -# Authentication (generated by setup.sh) +# Authentication +# Generate with: ./scripts/generate-secrets.sh token 64 JWT_SECRET= +# Generate with: ./scripts/generate-secrets.sh token 64 API_SECRET= -# MongoDB (password generated by setup.sh) +# Object Storage (additional settings) +# RUSTFS (optional, set only if using the included rustfs) +# Generate with: ./scripts/generate-secrets.sh token 32 +RUSTFS_SECRET_KEY= + +# MongoDB +# Generate with: ./scripts/generate-secrets.sh token 32 +# Or set to your existing password MONGODB_PASSWORD= -# ClickHouse (passwords generated by setup.sh) +# ClickHouse +# Default password only needed if using the included clickhouse +# Generate with: ./scripts/generate-secrets.sh token 32 CLICKHOUSE_DEFAULT_PASSWORD= +# Password for the currents-user for clickhouse +# Generate with: ./scripts/generate-secrets.sh token 32 +# Or provide your existing password CLICKHOUSE_CURRENTS_PASSWORD= -# GitLab Integration -# Run `./scripts/generate-secrets key gitlab-key.pem` -# Then `base64 < "gitlab-key.pem" | tr -d '\n'` to get the base64 secret +# GitLab Integration (base64-encoded RSA private key) +# Generate with: +# ./scripts/generate-secrets.sh key .gitlab-key.pem +# base64 < .gitlab-key.pem | tr -d '\n' GITLAB_STATE_SECRET= # ============================================================================= @@ -47,6 +64,16 @@ FILE_STORAGE_ACCESS_KEY_ID=${RUSTFS_ACCESS_KEY} FILE_STORAGE_SECRET_ACCESS_KEY=${RUSTFS_SECRET_KEY} # FILE_STORAGE_REGION=us-east-1 +# ClickHouse +CLICKHOUSE_URL=http://clickhouse:8123 +CLICKHOUSE_USERNAME=currents +# CLICKHOUSE_ACCESS_TOKEN= + +# MongoDB +MONGODB_USERNAME=currents-user +MONGODB_DATABASE=currents +MONGODB_URI=mongodb://${MONGODB_USERNAME}:${MONGODB_PASSWORD}@mongodb:27017/${MONGODB_DATABASE}?authSource=admin&replicaSet=rs0 + # Domain configuration for Traefik routing # TRAEFIK_DOMAIN=example.com # TRAEFIK_API_SUBDOMAIN=currents-app @@ -68,25 +95,16 @@ GITLAB_REDIRECT_URL=${APP_BASE_URL}/integrations/gitlab/callback # Authentication settings JWT_SECRET_EXPIRY=10m -# MongoDB -MONGODB_USERNAME=currents-user -MONGODB_DATABASE=currents -MONGODB_URI=mongodb://${MONGODB_USERNAME}:${MONGODB_PASSWORD}@mongodb:27017/${MONGODB_DATABASE}?authSource=admin&replicaSet=rs0 - # Redis (default points to redis service in compose network) REDIS_URI=redis://redis:6379 REDIS_URI_SLAVE=redis://redis:6379 -# ClickHouse -CLICKHOUSE_URL=http://clickhouse:8123 -CLICKHOUSE_USERNAME=currents -# CLICKHOUSE_ACCESS_TOKEN= - -# Object Storage (additional settings) +# Object Storage (addition configuration) +## RustFS (optional) RUSTFS_ACCESS_KEY=rustfs-access-key -RUSTFS_SECRET_KEY=rustfs-secret-key + FILE_STORAGE_INTERNAL_ENDPOINT="http://host.docker.internal:9000" -FILE_STORAGE_FORCE_PATH_STYLE=true +# FILE_STORAGE_FORCE_PATH_STYLE=true # Set automatically when using RustFS profile # Email (additional settings) AUTOMATED_REPORTS_EMAIL_BCC= @@ -96,7 +114,6 @@ AUTOMATED_REPORTS_EMAIL_BCC= # CORALOGIX_API_KEY= # CORALOGIX_APP_NAME="customer-name" - # ============================================================================= # Docker Compose Configuration # ============================================================================= @@ -146,4 +163,4 @@ AUTOMATED_REPORTS_EMAIL_BCC= # DC_TRAEFIK_CERTS_DIR=./data/traefik/certs # DC_TRAEFIK_CONFIG_DIR=./data/traefik/config # DC_TRAEFIK_DYNAMIC_CONFIG_PATH=/etc/traefik/dynamic.yml -# TRAEFIK_GENERATE_TEMP_CERTS=false \ No newline at end of file +# TRAEFIK_GENERATE_TEMP_CERTS=false diff --git a/on-prem/README.md b/on-prem/README.md index c5a1425..ef48a74 100644 --- a/on-prem/README.md +++ b/on-prem/README.md @@ -2,25 +2,35 @@ Docker Compose configuration for running Currents on-premises. -## Quick Start +## Documentation -```bash -# Run the interactive setup -./scripts/setup.sh -``` +πŸ“š **For setup and configuration instructions, see the [docs](./docs/) folder:** -This will: -1. Guide you through selecting a configuration profile -2. Generate the appropriate `docker-compose.yml` -3. Create a `.env` file with auto-generated secrets +- [Quickstart Guide](./docs/quickstart.md) β€” Get up and running +- [Configuration Reference](./docs/configuration.md) β€” All environment variables +- [Support Policy](./docs/support.md) β€” What's supported -Then start the services: +--- -```bash -docker compose up -d +## Development Guide + +This section is for developers working on the docker-compose configuration itself. + +### Architecture + +The configuration is built from modular templates that are merged together based on the selected profile. This allows users to include only the services they need. + +``` +templates/ +β”œβ”€β”€ compose.currents.yml # Core Currents application services (always included) +β”œβ”€β”€ compose.redis.yml # Redis cache +β”œβ”€β”€ compose.mongodb.yml # MongoDB database +β”œβ”€β”€ compose.clickhouse.yml # ClickHouse analytics +β”œβ”€β”€ compose.rustfs.yml # S3-compatible object storage +└── compose.traefik.yml # TLS termination proxy (optional profile) ``` -## Configuration Profiles +### Configuration Profiles | Profile | Services Included | Use Case | |---------|-------------------|----------| @@ -28,20 +38,16 @@ docker compose up -d | `database` | Redis, MongoDB, ClickHouse | External S3-compatible storage | | `cache` | Redis | External MongoDB, ClickHouse, and S3 | -## Scripts +### Scripts -### `scripts/setup.sh` +#### `scripts/setup.sh` Interactive setup wizard that: - Prompts for profile selection (or custom service selection) - Generates the docker-compose configuration - Creates `.env` from `.env.example` with auto-generated secrets -```bash -./scripts/setup.sh -``` - -### `scripts/generate-compose.sh` +#### `scripts/generate-compose.sh` Generates a docker-compose file for a specific profile. Used by `setup.sh` but can also be run directly. @@ -75,7 +81,7 @@ Generates a docker-compose file for a specific profile. Used by `setup.sh` but c | `clickhouse` | ClickHouse only | | `rustfs` | RustFS only | -### `scripts/generate-secrets.sh` +#### `scripts/generate-secrets.sh` Utility for generating secrets and keys. @@ -90,7 +96,7 @@ Utility for generating secrets and keys. ./scripts/generate-secrets.sh key mykey.pem ``` -## Files +### Files | File | Description | |------|-------------| @@ -101,32 +107,11 @@ Utility for generating secrets and keys. | `.env` | Your environment configuration (git-ignored) | | `.env.example` | Template for environment configuration | | `templates/` | Source templates for compose generation | +| `docs/` | User-facing documentation | -## Environment Configuration - -Copy `.env.example` to `.env` and configure as needed. The `setup.sh` script does this automatically and generates required secrets. - -### Application Variables - -- `JWT_SECRET` - Authentication token secret -- `RUSTFS_ACCESS_KEY` / `RUSTFS_SECRET_KEY` - RustFS credentials -- `GITLAB_STATE_SECRET` - GitLab integration key (base64-encoded PEM) - -### Compose-Only Variables +### Services -Variables prefixed with `DC_` are used only by docker-compose (not passed to containers): - -- `DC_IMAGE_REPOSITORY` - Docker image repository prefix -- `DC_IMAGE_TAG` - Docker image tag -- `DC_REDIS_VOLUME` - Redis data volume path -- `DC_MONGODB_VOLUME` - MongoDB data volume path -- `DC_CLICKHOUSE_VOLUME` - ClickHouse data volume path -- `DC_RUSTFS_VOLUME` - RustFS data volume path -- `DC_SCHEDULER_STARTUP_VOLUME` - Scheduler startup data volume path - -## Services - -### Currents Application +#### Currents Application - **director** (port 1234) - Test orchestration service - **api** (port 4000) - API and dashboard - **changestreams-worker** - MongoDB change stream processor @@ -134,15 +119,18 @@ Variables prefixed with `DC_` are used only by docker-compose (not passed to con - **scheduler** - Scheduled tasks and startup migrations - **webhooks** - Webhook delivery service -### Data Services (optional, based on profile) -- **redis** (port 6379, 8001) - Cache and pub/sub +#### Data Services (optional, based on profile) +- **redis** (port 6379) - Cache and pub/sub - **mongodb** (port 27017) - Primary database - **clickhouse** (port 8123, 9123) - Analytics database - **rustfs** (port 9000, 9001) - S3-compatible object storage -## Development +#### Optional Services +- **traefik** (ports 80, 443) - TLS termination proxy (enabled with `--profile tls`) -To regenerate the pre-committed docker-compose files after modifying templates: +### Regenerating Compose Files + +After modifying templates, regenerate the pre-committed docker-compose files: ```bash ./scripts/generate-compose.sh full @@ -152,3 +140,9 @@ To regenerate the pre-committed docker-compose files after modifying templates: A GitHub Action validates that committed compose files stay in sync with templates. +### Variable Naming Convention + +- **`DC_` prefix**: Docker-compose-only variables (not passed to containers) + - `DC_MONGODB_PORT`, `DC_CURRENTS_IMAGE_TAG`, `DC_REDIS_VOLUME` +- **No prefix**: App config variables that containers need + - `MONGODB_PASSWORD`, `APP_BASE_URL`, `CLICKHOUSE_CURRENTS_PASSWORD` diff --git a/on-prem/docker-compose.full.yml b/on-prem/docker-compose.full.yml index 0231939..688d3ab 100644 --- a/on-prem/docker-compose.full.yml +++ b/on-prem/docker-compose.full.yml @@ -11,7 +11,7 @@ name: currents_full # RustFS service configuration # S3-compatible object storage - usually excluded when using external S3/cloud storage services: - # Currents services depend on redis + # Configure Currents services to use path-style S3 URLs (required for RustFS) director: hostname: director image: ${DC_CURRENTS_IMAGE_REPOSITORY:-currents-}director:${DC_CURRENTS_IMAGE_TAG:-dev} @@ -25,6 +25,7 @@ services: environment: CURRENTS_ENV: onprem EMAIL_TRANSPORTER: smtp + FILE_STORAGE_FORCE_PATH_STYLE: "true" depends_on: clickhouse: condition: service_healthy @@ -50,6 +51,7 @@ services: INCLUDE_DASHBOARD: onprem CLICKHOUSE_PASSWORD: ${CLICKHOUSE_CURRENTS_PASSWORD} EMAIL_TRANSPORTER: smtp + FILE_STORAGE_FORCE_PATH_STYLE: "true" depends_on: clickhouse: condition: service_healthy @@ -71,6 +73,7 @@ services: CURRENTS_ENV: onprem CLICKHOUSE_PASSWORD: ${CLICKHOUSE_CURRENTS_PASSWORD} EMAIL_TRANSPORTER: smtp + FILE_STORAGE_FORCE_PATH_STYLE: "true" depends_on: clickhouse: condition: service_healthy @@ -92,6 +95,7 @@ services: CURRENTS_ENV: onprem CLICKHOUSE_PASSWORD: ${CLICKHOUSE_CURRENTS_PASSWORD} EMAIL_TRANSPORTER: smtp + FILE_STORAGE_FORCE_PATH_STYLE: "true" depends_on: clickhouse: condition: service_healthy @@ -114,6 +118,7 @@ services: CLICKHOUSE_PASSWORD: ${CLICKHOUSE_CURRENTS_PASSWORD} RUN_STARTUP_TASKS: "true" EMAIL_TRANSPORTER: smtp + FILE_STORAGE_FORCE_PATH_STYLE: "true" volumes: - ${DC_SCHEDULER_STARTUP_VOLUME:-./data/startup}:/app/packages/scheduler/dist/.startup depends_on: @@ -136,6 +141,7 @@ services: environment: CURRENTS_ENV: onprem EMAIL_TRANSPORTER: smtp + FILE_STORAGE_FORCE_PATH_STYLE: "true" depends_on: clickhouse: condition: service_healthy diff --git a/on-prem/docs/README.md b/on-prem/docs/README.md new file mode 100644 index 0000000..d112e53 --- /dev/null +++ b/on-prem/docs/README.md @@ -0,0 +1,30 @@ +# Currents Self-Hosted Documentation (Docker Compose) + +Currents on-premise installation using Docker Compose provides a containerized deployment for running Currents services on a single host or small-scale infrastructure. + +The Docker Compose configuration is modular, allowing you to choose which data services to run locally versus connecting to external managed services. + +## Resources + +- [πŸš€ Quickstart Guide](./quickstart.md) +- [Configuration Reference](./configuration.md) +- [Support Policy](./support.md) + +## Additional Resources + +- [Docker Documentation](https://docs.docker.com/) +- [Docker Compose Documentation](https://docs.docker.com/compose/) +- [Podman Documentation](https://docs.podman.io/) + +## Upstream Services + +- The self-hosted solution requires an existing Identity Provider for access provisioning. +- The recommended configuration for the stateful services (MongoDB, ClickHouse, Redis) may not be adequate for all production loads. +- Currents team doesn't provide support for the upstream services (MongoDB, ClickHouse, Redis), see [Support Policy](./support.md). + +## Known Limitations + +The following features are not fully available for self-hosted version. If you need them let us know in advance: + +- Code coverage collection and reporting +- Bitbucket and MS Teams integrations are still WIP diff --git a/on-prem/docs/configuration.md b/on-prem/docs/configuration.md new file mode 100644 index 0000000..2123c6d --- /dev/null +++ b/on-prem/docs/configuration.md @@ -0,0 +1,124 @@ +# Configuration Reference + +This document lists all configurable environment variables for the Currents Docker Compose deployment. + +Variables are configured in the `.env` file. Run `./scripts/setup.sh` to generate secrets automatically, or use the manual generation commands shown below. + +## Values + +### Required + +These values must be set before starting. Secrets can be generated with `./scripts/setup.sh` or manually. + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `ON_PREM_EMAIL` | string | `admin@localhost` | Email address for the root admin user | +| `SMTP_HOST` | string | `localhost` | SMTP server hostname | +| `SMTP_USER` | string | `test` | SMTP username | +| `SMTP_PASS` | string | `test` | SMTP password | +| `JWT_SECRET` | string | _(empty)_ | Authentication token secret. Generate with: `./scripts/generate-secrets.sh token 64` | +| `API_SECRET` | string | _(empty)_ | Internal API secret for service-to-service auth. Generate with: `./scripts/generate-secrets.sh token 64` | +| `RUSTFS_SECRET_KEY` | string | _(empty)_ | RustFS/S3 secret key (only if using included RustFS). Generate with: `./scripts/generate-secrets.sh token 32` | +| `MONGODB_PASSWORD` | string | _(empty)_ | MongoDB password. Generate with: `./scripts/generate-secrets.sh token 32` | +| `CLICKHOUSE_DEFAULT_PASSWORD` | string | _(empty)_ | ClickHouse default user password (only if using included ClickHouse). Generate with: `./scripts/generate-secrets.sh token 32` | +| `CLICKHOUSE_CURRENTS_PASSWORD` | string | _(empty)_ | ClickHouse currents user password. Generate with: `./scripts/generate-secrets.sh token 32` | +| `GITLAB_STATE_SECRET` | string | _(empty)_ | GitLab integration key (base64-encoded RSA private key) | + +### Frequently Used + +These have defaults but you'll likely want to customize them. + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `APP_BASE_URL` | string | `http://localhost:4000` | Base URL for the dashboard/API | +| `CURRENTS_RECORD_API_URL` | string | `http://localhost:1234` | Base URL for the recording endpoint (director) | +| `SMTP_PORT` | int | `587` | SMTP server port | +| `SMTP_SECURE` | bool | `false` | Whether SMTP uses TLS | +| `AUTOMATED_REPORTS_EMAIL_FROM` | string | `Currents Report ` | From address for automated report emails | +| `FILE_STORAGE_ENDPOINT` | string | `http://localhost:9000` | Object storage endpoint (external access) | +| `FILE_STORAGE_BUCKET` | string | `currents` | Object storage bucket name | +| `FILE_STORAGE_ACCESS_KEY_ID` | string | `${RUSTFS_ACCESS_KEY}` | Object storage access key ID | +| `FILE_STORAGE_SECRET_ACCESS_KEY` | string | `${RUSTFS_SECRET_KEY}` | Object storage secret access key | +| `FILE_STORAGE_REGION` | string | _(commented)_ | S3 region (required for AWS S3) | +| `CLICKHOUSE_URL` | string | `http://clickhouse:8123` | ClickHouse HTTP endpoint | +| `CLICKHOUSE_USERNAME` | string | `currents` | ClickHouse username | +| `MONGODB_USERNAME` | string | `currents-user` | MongoDB username | +| `MONGODB_DATABASE` | string | `currents` | MongoDB database name | +| `MONGODB_URI` | string | _(derived)_ | Full MongoDB connection string | +| `TRAEFIK_DOMAIN` | string | _(commented)_ | Base domain for Traefik TLS routing | +| `TRAEFIK_API_SUBDOMAIN` | string | `currents-app` | Subdomain for API/Dashboard | +| `TRAEFIK_DIRECTOR_SUBDOMAIN` | string | `currents-record` | Subdomain for Director | +| `TRAEFIK_STORAGE_SUBDOMAIN` | string | `currents-storage` | Subdomain for RustFS S3 API | +| `TRAEFIK_ENABLE_STORAGE` | bool | `false` | Enable storage routing (auto-set when rustfs included) | + +### Other Values + +Less commonly changed settings with sensible defaults. + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `JWT_SECRET_EXPIRY` | string | `10m` | How often to expire session tokens | +| `REDIS_URI` | string | `redis://redis:6379` | Redis connection URI | +| `REDIS_URI_SLAVE` | string | `redis://redis:6379` | Redis replica connection URI | +| `RUSTFS_ACCESS_KEY` | string | `rustfs-access-key` | RustFS/S3 access key | +| `FILE_STORAGE_INTERNAL_ENDPOINT` | string | `http://host.docker.internal:9000` | Object storage internal endpoint | +| `FILE_STORAGE_FORCE_PATH_STYLE` | bool | _(commented)_ | Use path-style S3 URLs (auto-set to `true` when using RustFS profile) | +| `AUTOMATED_REPORTS_EMAIL_BCC` | string | _(empty)_ | BCC address for automated reports | + +### Docker Compose Configuration + +These variables configure Docker Compose behavior only (not passed to containers). All are optional with sensible defaults. + +#### Image Configuration + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `DC_CURRENTS_IMAGE_REPOSITORY` | string | `currents-` | Image repository prefix for Currents services | +| `DC_CURRENTS_IMAGE_TAG` | string | `dev` | Image tag for Currents services | +| `DC_MONGODB_IMAGE` | string | `mongo:8.2.3` | MongoDB image | +| `DC_REDIS_IMAGE` | string | `redis/redis-stack-server:7.4.0-v8` | Redis image | +| `DC_CLICKHOUSE_IMAGE` | string | `clickhouse/clickhouse-server:25.8` | ClickHouse image | +| `DC_RUSTFS_IMAGE` | string | `rustfs/rustfs:1.0.0-alpha.79` | RustFS image | +| `DC_AWS_CLI_IMAGE` | string | `amazon/aws-cli:latest` | AWS CLI image (for bucket init) | + +#### Port Configuration + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `DC_DIRECTOR_PORT` | string | `1234` | Director API port (all interfaces) | +| `DC_API_PORT` | string | `4000` | Dashboard/API port (all interfaces) | +| `DC_MONGODB_PORT` | string | `127.0.0.1:27017` | MongoDB port (localhost only) | +| `DC_REDIS_PORT` | string | `127.0.0.1:6379` | Redis port (localhost only) | +| `DC_CLICKHOUSE_HTTP_PORT` | string | `127.0.0.1:8123` | ClickHouse HTTP port (localhost only) | +| `DC_CLICKHOUSE_TCP_PORT` | string | `127.0.0.1:9123` | ClickHouse TCP port (localhost only) | +| `DC_RUSTFS_S3_PORT` | string | `9000` | RustFS S3 API port (all interfaces) | +| `DC_RUSTFS_CONSOLE_PORT` | string | `9001` | RustFS Console port (all interfaces) | + +#### Volume Configuration + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `DC_REDIS_VOLUME` | string | `./data/redis` | Redis data volume path | +| `DC_MONGODB_VOLUME` | string | `./data/mongodb` | MongoDB data volume path | +| `DC_CLICKHOUSE_VOLUME` | string | `./data/clickhouse` | ClickHouse data volume path | +| `DC_RUSTFS_VOLUME` | string | `./data/rustfs` | RustFS data volume path | +| `DC_SCHEDULER_STARTUP_VOLUME` | string | `./data/startup` | Scheduler startup data volume | + +#### Traefik Configuration + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `DC_TRAEFIK_IMAGE` | string | `traefik:v3.3` | Traefik image | +| `DC_TRAEFIK_HTTP_PORT` | string | `80` | Traefik HTTP port | +| `DC_TRAEFIK_HTTPS_PORT` | string | `443` | Traefik HTTPS port | +| `DC_TRAEFIK_CERTS_DIR` | string | `./data/traefik/certs` | Certificate directory | +| `DC_TRAEFIK_CONFIG_DIR` | string | `./data/traefik/config` | Custom config directory | +| `TRAEFIK_GENERATE_TEMP_CERTS` | bool | `false` | Generate temporary self-signed certs | + +### Observability (Optional) + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `CORALOGIX_API_ENDPOINT` | string | _(empty)_ | Coralogix API endpoint | +| `CORALOGIX_API_KEY` | string | _(empty)_ | Coralogix API key | +| `CORALOGIX_APP_NAME` | string | `customer-name` | Application name for Coralogix | diff --git a/on-prem/docs/index.md b/on-prem/docs/index.md new file mode 100644 index 0000000..73af30c --- /dev/null +++ b/on-prem/docs/index.md @@ -0,0 +1,36 @@ +--- +title: Currents On-Prem Documentation +--- + +# Currents On-Prem + +Docker Compose configuration for self-hosted Currents deployment. + +## Getting Started + +- [Quickstart Guide](quickstart.md) β€” Get up and running with Docker Compose +- [Configuration Reference](configuration.md) β€” All environment variables and settings +- [Support Policy](support.md) β€” What's supported and maintenance responsibilities + +## Quick Setup + +```bash +git clone https://github.com/currents-dev/currents-dev-docker.git +cd currents-dev-docker/on-prem +./scripts/setup.sh +docker compose up -d +``` + +## Configuration Profiles + +| Profile | Services Included | Use Case | +|---------|-------------------|----------| +| `full` | Redis, MongoDB, ClickHouse, RustFS | Running everything locally | +| `database` | Redis, MongoDB, ClickHouse | Using external S3-compatible storage | +| `cache` | Redis | Using external MongoDB, ClickHouse, and S3 | + +## Additional Resources + +- [Docker Documentation](https://docs.docker.com/) +- [Docker Compose Documentation](https://docs.docker.com/compose/) +- [Podman Documentation](https://docs.podman.io/) diff --git a/on-prem/docs/quickstart.md b/on-prem/docs/quickstart.md new file mode 100644 index 0000000..c78e496 --- /dev/null +++ b/on-prem/docs/quickstart.md @@ -0,0 +1,373 @@ +# Docker Compose Quickstart + +This guide walks you through setting up Currents on-premises using Docker Compose. + +## Prerequisites + +- **Docker** 20.10+ with Docker Compose V2, or **Podman** 4.0+ with docker-compose +- At least 8GB RAM available for containers +- Git (for cloning the repository) + +## Step 1: Clone the Repository + +```bash +git clone https://github.com/currents-dev/currents-dev-docker.git +cd currents-dev-docker/on-prem +``` + +## Step 2: Create Environment File + +A `.env` file is required to run the services. You have two options: + +### Option A: Run Setup (Recommended) + +The interactive setup wizard will guide you through configuration: + +```bash +./scripts/setup.sh +``` + +This will: +1. Ask you to select a configuration profile +2. Generate the appropriate `docker-compose.yml` +3. Create a `.env` file with auto-generated secrets + +### Option B: Manual Setup + +If you prefer to configure manually: + +```bash +cp .env.example .env +``` + +Then edit `.env` to fill in the required secrets. See [Configuration Reference](./configuration.md) for generation commands. + +### Configuration Profiles + +| Profile | Services Included | Use Case | +|---------|-------------------|----------| +| `full` | Redis, MongoDB, ClickHouse, RustFS | Running everything locally | +| `database` | Redis, MongoDB, ClickHouse | Using external S3-compatible storage | +| `cache` | Redis | Using external MongoDB, ClickHouse, and S3 | + +## Step 3: Configure Environment + +Review and customize `.env` as needed. + +### Application URLs + +Configure the URLs where Currents will be accessible. For production, we recommend using subdomains: + +```bash +# Dashboard and API +APP_BASE_URL=https://currents-app.example.com + +# Recording endpoint (where test reporters send data) +CURRENTS_RECORD_API_URL=https://currents-record.example.com +``` + +For local development, use localhost with ports: + +```bash +APP_BASE_URL=http://localhost:4000 +CURRENTS_RECORD_API_URL=http://localhost:1234 +``` + +### Root User Account + +The `ON_PREM_EMAIL` is the email address used to create the initial root admin user: + +```bash +ON_PREM_EMAIL=admin@example.com +``` + +### Object Storage (Recommended: Bring Your Own) + +We recommend using your own S3-compatible object storage (AWS S3, Google Object Storage etc.) rather than the included RustFS service. Configure your storage provider: + +```bash +# Your S3-compatible endpoint +FILE_STORAGE_ENDPOINT=https://s3.us-east-1.amazonaws.com + +# Bucket name (must already exist) +FILE_STORAGE_BUCKET=currents-artifacts + +# Credentials +FILE_STORAGE_ACCESS_KEY_ID= +FILE_STORAGE_SECRET_ACCESS_KEY= + +# Region (required for AWS S3) +FILE_STORAGE_REGION=us-east-1 + +# Use path-style URLs (required for MinIO and most S3-compatible services, not needed for AWS S3) +# FILE_STORAGE_FORCE_PATH_STYLE=true +``` + +If using the included RustFS for testing, configure the `RUSTFS_*` variables instead. The RustFS profile automatically sets `FILE_STORAGE_FORCE_PATH_STYLE=true` for all services. + +> ⚠️ **Production Note:** RustFS is intended for local development and testing onlyβ€”it is **not recommended for production deployments**. The included Docker Compose configuration is designed for local development; production environments should use external, production-grade object storage backends such as AWS S3, Google Cloud Storage, or a managed MinIO cluster. + +### SMTP Configuration + +Email is required for notifications, invitations, and reports: + +```bash +# SMTP server +SMTP_HOST=smtp.example.com +SMTP_PORT=587 +SMTP_SECURE=false + +# SMTP credentials +SMTP_USER=your-smtp-username +SMTP_PASS=your-smtp-password + +# From address for automated emails +AUTOMATED_REPORTS_EMAIL_FROM=Currents Report +``` + +> **Note:** `SMTP_SECURE=false` uses STARTTLS (explicit TLS) which starts unencrypted then upgrades to TLSβ€”this is the standard for port 587 and recommended for most providers. Set `SMTP_SECURE=true` for implicit TLS connections (port 465), which establish TLS immediately without upgrading. + +Common SMTP configurations: + +| Provider | Host | Port | Secure | +|----------|------|------|--------| +| Amazon SES | `email-smtp.us-east-1.amazonaws.com` | 587 | false | +| SendGrid | `smtp.sendgrid.net` | 587 | false | +| Mailgun | `smtp.mailgun.org` | 587 | false | +| Gmail | `smtp.gmail.com` | 587 | false | + +See [Configuration Reference](./configuration.md) for all available options. + +## Step 4: Start Services + +```bash +docker compose up -d +``` + +Monitor startup progress: + +```bash +docker compose logs -f +``` + +## Step 5: Verify Installation + +Once all services are running, access: + +- **Dashboard**: http://localhost:4000 +- **Director API**: http://localhost:1234 + +Check service health: + +```bash +docker compose ps +``` + +All services should show as "healthy" or "running". + +## Production: TLS Termination + +For production deployments, we recommend setting up a reverse proxy with TLS termination in front of the Currents services. You can either: + +- **Bring your own** β€” Use your existing reverse proxy (nginx, HAProxy, AWS ALB, etc.) to handle TLS and route traffic to the Currents services +- **Use the included Traefik** β€” A pre-configured Traefik profile is included for convenience + +### Using Your Own Reverse Proxy + +Configure your reverse proxy to route: +- `https://currents-app.example.com` β†’ `http://localhost:4000` (API/Dashboard) +- `https://currents-record.example.com` β†’ `http://localhost:1234` (Director) + +Update your `.env` to match the external URLs: +```bash +APP_BASE_URL=https://currents-app.example.com +CURRENTS_RECORD_API_URL=https://currents-record.example.com +``` + +### Using the Included Traefik Profile + +1. Place your wildcard certificate files in `data/traefik/certs/`: + - `wildcard.crt` - Certificate file + - `wildcard.key` - Private key file + +2. Configure your domain in `.env`: + ```bash + TRAEFIK_DOMAIN=example.com + TRAEFIK_API_SUBDOMAIN=currents-app + TRAEFIK_DIRECTOR_SUBDOMAIN=currents-record + ``` + +3. Start with the TLS profile: + ```bash + docker compose --profile tls up -d + ``` + +## Common Operations + +### View Logs + +```bash +# All services +docker compose logs -f + +# Specific service +docker compose logs -f api +``` + +### Stop Services + +```bash +docker compose down +``` + +### Restart a Service + +```bash +docker compose restart api +``` + +### Update to Latest Version + +```bash +# Pull latest images +docker compose pull + +# Restart with new images +docker compose up -d +``` + +### Regenerate Secrets + +```bash +./scripts/setup.sh +# Select "Y" when asked to regenerate secrets +``` + +## Troubleshooting + +### Services Won't Start + +Check logs for errors: +```bash +docker compose logs --tail=50 +``` + +Verify `.env` file exists and has all required secrets populated. + +### Database Connection Issues + +Ensure MongoDB has initialized its replica set. Check logs: +```bash +docker compose logs mongodb +``` + +The replica set initialization runs automatically on first start. + +### Port Conflicts + +If ports are already in use, customize them in `.env`: +```bash +DC_API_PORT=4001 +DC_DIRECTOR_PORT=1235 +``` + +## Advanced: Port Binding Configuration + +By default, application ports (API, Director) bind to all interfaces while database ports bind to localhost only. You can customize this behavior using `DC_*_PORT` variables. + +### Binding to Specific Interfaces + +To restrict a service to localhost only (not accessible from other machines): + +```bash +# Bind API to localhost only +DC_API_PORT=127.0.0.1:4000 + +# Bind Director to localhost only +DC_DIRECTOR_PORT=127.0.0.1:1234 +``` + +To expose a database service to all interfaces (use with caution): + +```bash +# Expose MongoDB to all interfaces +DC_MONGODB_PORT=27017 + +# Expose Redis to all interfaces +DC_REDIS_PORT=6379 +``` + +### Port Configuration Reference + +| Variable | Default | Description | +|----------|---------|-------------| +| `DC_API_PORT` | `4000` | Dashboard/API (all interfaces) | +| `DC_DIRECTOR_PORT` | `1234` | Director API (all interfaces) | +| `DC_MONGODB_PORT` | `127.0.0.1:27017` | MongoDB (localhost only) | +| `DC_REDIS_PORT` | `127.0.0.1:6379` | Redis (localhost only) | +| `DC_CLICKHOUSE_HTTP_PORT` | `127.0.0.1:8123` | ClickHouse HTTP (localhost only) | +| `DC_CLICKHOUSE_TCP_PORT` | `127.0.0.1:9123` | ClickHouse TCP (localhost only) | +| `DC_RUSTFS_S3_PORT` | `9000` | RustFS S3 API (all interfaces) | +| `DC_RUSTFS_CONSOLE_PORT` | `9001` | RustFS Console (all interfaces) | + +> **Security Note:** Database ports default to localhost-only binding to prevent unintended external access. Only expose them to all interfaces if you have proper firewall rules in place. + +## Advanced: Volume Configuration + +By default, data is stored in local directories under `./data/`. You can customize volume paths or use named Docker volumes for more advanced storage configurations. + +### Using Custom Paths + +To store data in different directories: + +```bash +# Store MongoDB data on a separate disk +DC_MONGODB_VOLUME=/mnt/ssd/mongodb + +# Store ClickHouse data on high-performance storage +DC_CLICKHOUSE_VOLUME=/mnt/nvme/clickhouse +``` + +### Using Named Docker Volumes + +For advanced volume management (encryption, network storage, custom drivers), create Docker volumes first and reference them by name: + +```bash +# Create volumes with custom options +docker volume create --driver local \ + --opt type=none \ + --opt o=bind \ + --opt device=/mnt/encrypted/mongodb \ + mongodb-data + +docker volume create --driver local \ + --opt type=none \ + --opt o=bind \ + --opt device=/mnt/encrypted/clickhouse \ + clickhouse-data +``` + +Then reference them in `.env`: + +```bash +DC_MONGODB_VOLUME=mongodb-data +DC_CLICKHOUSE_VOLUME=clickhouse-data +``` + +### Volume Configuration Reference + +| Variable | Default | Description | +|----------|---------|-------------| +| `DC_REDIS_VOLUME` | `./data/redis` | Redis data storage | +| `DC_MONGODB_VOLUME` | `./data/mongodb` | MongoDB data storage | +| `DC_CLICKHOUSE_VOLUME` | `./data/clickhouse` | ClickHouse data storage | +| `DC_RUSTFS_VOLUME` | `./data/rustfs` | RustFS object storage | +| `DC_SCHEDULER_STARTUP_VOLUME` | `./data/startup` | Scheduler startup state | + +> **Tip:** Named Docker volumes are useful when you need encryption, network-attached storage, or custom volume drivers that aren't possible with bind mounts. + +## Next Steps + +- Review the [Configuration Reference](./configuration.md) for all available options +- Read the [Support Policy](./support.md) to understand support boundaries +- Set up monitoring and backups for production use diff --git a/on-prem/docs/support.md b/on-prem/docs/support.md new file mode 100644 index 0000000..24e35ff --- /dev/null +++ b/on-prem/docs/support.md @@ -0,0 +1,129 @@ +# Currents Support & Maintenance Policy + +## Introduction + +This document outlines Currents' support and maintenance policy for customers deploying Currents in self-managed (on-premises) environments using Docker Compose. It clearly defines our responsibilities, delineates the components we fully support, and sets expectations for areas where support is advisory or out of scope. + +This document may be shared with customers and incorporated into future support material and onboarding guides. + +--- + +## System Components & Support Scope + +Currents components fall into three categories: + +1. **Core (Fully Supported)** +2. **Advisory Support** +3. **Excluded (No Support)** + +--- + +### 1. Core Components (Fully Supported) + +Once Currents is deployed and the correct setup is confirmed, Currents is responsible for the reliable and performant operation of the core services. This includes: + +- **Currents services running within Docker Compose** +- **Docker Compose configuration** files, parameters, and definitions +- **Database configuration**: + - Index settings + - Query settings and performance +- **Redis**: + - Cache keys and content + - LUA scripts + - Associated application code + +- **Container configuration**: + - Service definitions + - Inter-service communication + +> πŸ›  For issues related to core components, please follow the "Troubleshooting" section. + +--- + +### 2. Advisory Support (Customer-Managed) + +Currents will provide **recommendations and minimal examples** for components in this category, but direct support is limited. These components are considered external to Currents' internal systems and fall under customer responsibility. + +- **Host Management**: + - Docker/Podman installation and configuration + - Host OS configuration and updates + - Resource allocation (CPU, memory, disk) + +- **Reverse Proxy / TLS**: + - Traefik configuration (optional profile included) + - Certificate management + - Load balancer setup + +- **Storage Infrastructure** (e.g., S3 buckets) + - Provisioning access and capacity + - Data encryption + +- **Database Infrastructure**: + - Resource allocation + - Clustering and scaling + - Backups and recovery + +- **Secrets management**: + - Secrets encryption and provisioning + - Creating, syncing, or rotating secrets as needed + +- **Network Configuration**: + - Connectivity between services + - Firewall and port configuration + +- **Instrumentation & Monitoring**: + - Currents can provide example metrics for performance and health + - Customers are responsible for setting up monitoring, alerting, and incident response + - Currents can offer ad-hoc consultation as needed + +- **Infrastructure Versions**: + - Minimum viable versions will be provided for required components + +--- + +### 3. Excluded Components (No Support) + +Currents does **not** provide support for the following: + +- Infrastructure component upgrades (e.g., DB engine upgrades, OS updates) +- Network configuration and external service management +- Provisioning access to the system +- Infrastructure availability and uptime + +--- + +## Maintenance Responsibilities + +### Service Upgrades + +- Currents will publish updated Docker images and docker-compose configurations +- A changelog and upgrade instructions/tools will be included +- Customers are responsible for applying upgrades at their discretion +- Currents is available for support in case of upgrade issues, within the defined support perimeter + +### Data Migration + +- If data migration is needed, Currents will supply instructions and tooling +- Execution and validation are the customer's responsibility + +### Infrastructure Maintenance + +- OS, network, and storage infrastructure are fully customer-managed + +--- + +## Troubleshooting + +To ensure efficient issue resolution, customer must provide certain technical information. + +- Detailed logs (use `docker compose logs `) +- A standalone example and reproduction steps +- Timely responses during the troubleshooting process +- Full breakdown of involved system setups: + - Versions (`docker --version`, `docker compose version`) + - Configuration files (`.env`, `docker-compose.yml`) +- It is **recommended** to use a shared external logging provider (e.g., Coralogix) for log access +- Feature requests are subject to our internal roadmap and will be prioritized based on availability, unless agreed otherwise in writing +- Support hours and SLA terms are governed by the customer's signed contract + +Failure to provide complete information or delayed responses may impact our ability to resolve issues effectively. diff --git a/on-prem/scripts/setup.sh b/on-prem/scripts/setup.sh index cf57fd6..96dbdd6 100755 --- a/on-prem/scripts/setup.sh +++ b/on-prem/scripts/setup.sh @@ -53,6 +53,12 @@ setup_env_file() { sed -i.bak "s|^JWT_SECRET=.*|JWT_SECRET=$JWT_SECRET|" "$ENV_FILE" fi + # Generate API_SECRET + API_SECRET=$("$SCRIPT_DIR/generate-secrets.sh" token 64) + if grep -q "^API_SECRET=" "$ENV_FILE"; then + sed -i.bak "s|^API_SECRET=.*|API_SECRET=$API_SECRET|" "$ENV_FILE" + fi + # Generate RUSTFS_SECRET_KEY RUSTFS_SECRET=$("$SCRIPT_DIR/generate-secrets.sh" token 32) if grep -q "^RUSTFS_SECRET_KEY=" "$ENV_FILE"; then diff --git a/on-prem/templates/compose.rustfs.yml b/on-prem/templates/compose.rustfs.yml index 16d7056..b778cc3 100644 --- a/on-prem/templates/compose.rustfs.yml +++ b/on-prem/templates/compose.rustfs.yml @@ -42,3 +42,28 @@ services: traefik: environment: TRAEFIK_ENABLE_STORAGE: "true" + + # Configure Currents services to use path-style S3 URLs (required for RustFS) + director: + environment: + FILE_STORAGE_FORCE_PATH_STYLE: "true" + + api: + environment: + FILE_STORAGE_FORCE_PATH_STYLE: "true" + + changestreams-worker: + environment: + FILE_STORAGE_FORCE_PATH_STYLE: "true" + + write-worker: + environment: + FILE_STORAGE_FORCE_PATH_STYLE: "true" + + scheduler: + environment: + FILE_STORAGE_FORCE_PATH_STYLE: "true" + + webhooks: + environment: + FILE_STORAGE_FORCE_PATH_STYLE: "true"