diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7cb1e3e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +node_modules +.wrangler +dist +.git +.github +.env +.env.* +.dev.vars +*.log +npm-debug.log* +Dockerfile +docker-compose*.yml +.dockerignore +.vscode +.idea +.DS_Store diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..afb6ad1 --- /dev/null +++ b/.env.example @@ -0,0 +1,20 @@ +# Copy to .dev.vars for local development (read by `wrangler dev`) and/or +# to .env for the production compose file (read by `wrangler deploy`). +# +# cp .env.example .dev.vars # for `docker compose ... up` (dev) +# cp .env.example .env # for `docker compose ... run` (prod deploy) + +# ----- Worker runtime secrets (read by `wrangler dev` from .dev.vars) ----- +# These are the agent-facing LLM/GitHub credentials the orchestrator injects +# into Dynamic Workers at runtime. In production, set them on Cloudflare +# with `wrangler secret put ANTHROPIC_API_KEY`, etc. +ANTHROPIC_API_KEY=sk-ant-... +OPENAI_API_KEY=sk-... +GITHUB_PAT=ghp_... + +# ----- Deploy-time credentials (read by `wrangler deploy` from env) ----- +# Required only for the production compose file. Generate an API token at +# https://dash.cloudflare.com/profile/api-tokens with the "Edit Cloudflare +# Workers" template. +CLOUDFLARE_API_TOKEN= +CLOUDFLARE_ACCOUNT_ID= diff --git a/.gitignore b/.gitignore index 9aff1e2..f214e82 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ node_modules/ dist/ .wrangler/ .dev.vars +.env +.env.local *.log diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..e69de29 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a81420c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,50 @@ +# syntax=docker/dockerfile:1.6 + +# ---------- base ---------- +# Shared base: pinned Node + system deps wrangler needs (git for git bindings, +# tini for PID 1 signal handling, ca-certificates for HTTPS to Cloudflare). +FROM node:20-alpine AS base +RUN apk add --no-cache git tini ca-certificates +WORKDIR /app +ENV CI=true \ + NPM_CONFIG_FUND=false \ + NPM_CONFIG_AUDIT=false + +# ---------- deps ---------- +# Install the full dependency tree once so downstream stages can reuse the +# layer. Copying only the manifests first keeps this cache hit on code-only +# changes. +FROM base AS deps +COPY package.json package-lock.json ./ +RUN --mount=type=cache,target=/root/.npm \ + npm ci + +# ---------- dev ---------- +# Development image: runs `wrangler dev` with hot reload. The source tree is +# bind-mounted in docker-compose.dev.yml, so we only need the toolchain baked +# into the image itself. +FROM deps AS dev +COPY . . +EXPOSE 8787 +ENTRYPOINT ["/sbin/tini", "--"] +# --ip 0.0.0.0 is required so the port is reachable from the host, not just +# the container's loopback. +CMD ["npx", "wrangler", "dev", "--ip", "0.0.0.0", "--port", "8787"] + +# ---------- typecheck ---------- +# Dedicated stage so CI can `docker build --target typecheck` as a gate +# without producing a runtime image. +FROM deps AS typecheck +COPY . . +RUN npm run typecheck + +# ---------- prod ---------- +# Production image: a reproducible deploy toolchain. Cloudflare Workers run +# on Cloudflare's edge — this image does NOT host the worker. It runs +# `wrangler deploy` (pushing to Cloudflare) and is the image used by CI and +# by docker-compose.prod.yml. Typecheck is a dependency so a failing build +# cannot be deployed. +FROM deps AS prod +COPY --from=typecheck /app /app +ENTRYPOINT ["/sbin/tini", "--"] +CMD ["npx", "wrangler", "deploy"] diff --git a/README.md b/README.md index 4120326..b6a5b43 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,19 @@ npm run dev curl http://localhost:8787/health ``` +### Local Development with Docker + +If you'd rather not install Node/wrangler on the host: + +```bash +cp .env.example .dev.vars # fill in your API keys +docker compose -f docker-compose.yml -f docker-compose.dev.yml up +# → http://localhost:8787 (hot reload enabled via source bind-mount) +``` + +See [docs/docker.md](docs/docker.md) for the full Compose guide, including +deploying to Cloudflare from a pinned toolchain image. + ### Production Deployment ```bash @@ -238,6 +251,11 @@ Full API documentation: [docs/api-reference.md](docs/api-reference.md) ├── wrangler.jsonc # Cloudflare Worker configuration ├── package.json # Dependencies and scripts ├── tsconfig.json # TypeScript (strict mode) +├── Dockerfile # Multi-stage: dev / typecheck / prod deploy +├── docker-compose.yml # Base Compose service definition +├── docker-compose.dev.yml # Dev override (wrangler dev + hot reload) +├── docker-compose.prod.yml # Prod override (wrangler deploy) +├── .env.example # Template for .dev.vars and .env ├── docs/ │ ├── architecture.md # System design and data flow │ ├── research-brief.md # Dynamic Workers platform research @@ -245,6 +263,7 @@ Full API documentation: [docs/api-reference.md](docs/api-reference.md) │ ├── agents.md # Agent types and tool API │ ├── security.md # Security model and egress control │ ├── deployment.md # Local dev, production, and CI/CD +│ ├── docker.md # Docker Compose usage and deploy workflow │ └── configuration.md # Wrangler config and tuning guide ├── src/ │ ├── index.ts # Orchestrator Worker (main entry) @@ -278,6 +297,7 @@ Full API documentation: [docs/api-reference.md](docs/api-reference.md) | [Agents](docs/agents.md) | Agent types, tool API interfaces, how to add new agents | | [Security](docs/security.md) | Sandboxing, egress control, credential separation, threat model | | [Deployment](docs/deployment.md) | Local dev, production deploy, CI/CD pipelines | +| [Docker](docs/docker.md) | Containerized dev server and pinned deploy toolchain via Docker Compose | | [Configuration](docs/configuration.md) | Wrangler config, secrets, tuning guide, multi-environment | --- @@ -307,6 +327,6 @@ This project is dual-licensed: The Community License grants full Apache 2.0 freedoms (including an explicit patent grant) with the Commons Clause restriction that prohibits selling the software or offering it as a paid service. All contributions are assigned to the copyright holder under the Contributor License Agreement in the LICENSE file. -For commercial licensing inquiries, contact David Brown via [GitHub](https://github.com/davidbrown). +For commercial licensing inquiries, contact David Brown via [GitHub](https://github.com/papismurf). See [LICENSE](LICENSE) for the full terms. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..b59f889 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,36 @@ +# Development override — runs `wrangler dev` with hot reload. +# +# Usage: +# docker compose -f docker-compose.yml -f docker-compose.dev.yml up +# +# Secrets come from .dev.vars (same file wrangler reads locally). Copy +# .env.example to .dev.vars and fill in your API keys before starting. + +services: + orchestrator: + build: + target: dev + command: ["npx", "wrangler", "dev", "--ip", "0.0.0.0", "--port", "8787"] + ports: + - "8787:8787" + environment: + # Wrangler writes to $HOME; keep it inside the volume so auth/config + # persist and don't collide with the host user's real $HOME. + - HOME=/app/.wrangler/home + volumes: + # Bind-mount the source tree so edits on the host trigger wrangler's + # file watcher inside the container. + - ./src:/app/src + - ./wrangler.jsonc:/app/wrangler.jsonc + - ./tsconfig.json:/app/tsconfig.json + - ./package.json:/app/package.json + - ./package-lock.json:/app/package-lock.json + # .dev.vars is wrangler's local secrets file. Mounted read-only so a + # misbehaving dep can't rewrite it. + - ./.dev.vars:/app/.dev.vars:ro + # Named volume for node_modules so host and container don't fight over + # platform-specific binaries (esbuild, etc.). + - node_modules:/app/node_modules + +volumes: + node_modules: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..04373d9 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,32 @@ +# Production override — runs `wrangler deploy` against Cloudflare. +# +# Important: Cloudflare Workers run on Cloudflare's edge. This compose file +# does NOT host the worker; it provides a reproducible, pinned toolchain for +# deploying. Use it from CI or from a workstation that doesn't have the +# correct Node/wrangler versions installed locally. +# +# Usage: +# # One-shot deploy (recommended): +# docker compose -f docker-compose.yml -f docker-compose.prod.yml run --rm orchestrator +# +# # Tail live production logs: +# docker compose -f docker-compose.yml -f docker-compose.prod.yml run --rm orchestrator npx wrangler tail +# +# Secrets are supplied via environment variables (e.g. an .env file or the +# CI provider's secret store). CLOUDFLARE_API_TOKEN is required; the +# per-provider API keys are consumed by `wrangler secret put` in a separate +# step — they are NOT baked into the Worker at deploy time. + +services: + orchestrator: + build: + target: prod + # `run --rm` + `restart: no` ensures deploys are one-shot, not a service + # that keeps restarting after a successful push. + restart: "no" + command: ["npx", "wrangler", "deploy"] + environment: + - CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN} + - CLOUDFLARE_ACCOUNT_ID=${CLOUDFLARE_ACCOUNT_ID:-} + # No source bind-mount: the image is the deploy artifact. Reproducibility + # comes from the built image, not from whatever happens to be on disk. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..13422a5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,23 @@ +# Base compose file — shared service definition consumed by both the dev and +# prod overrides. Invoke it via: +# docker compose -f docker-compose.yml -f docker-compose.dev.yml up +# docker compose -f docker-compose.yml -f docker-compose.prod.yml run --rm orchestrator +# +# On its own this file is not intended to `up` — the overrides pick the +# Dockerfile target and the command. + +services: + orchestrator: + image: agent-orchestrator:${TAG:-local} + build: + context: . + dockerfile: Dockerfile + # `tty` keeps wrangler's interactive output readable when attached. + tty: true + # Cache wrangler's local state (D1/KV/DO simulators, build artifacts) + # across container restarts so dev startup stays fast. + volumes: + - wrangler-state:/app/.wrangler + +volumes: + wrangler-state: diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 0000000..2230837 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,203 @@ +# Docker & Docker Compose + +This project ships with a multi-stage `Dockerfile` and two Compose overrides +— one for development, one for deploying to Cloudflare. Use Docker when you +want a reproducible toolchain (pinned Node + wrangler versions), don't want +to install Node on the host, or are wiring the deploy step into CI. + +## What the production image is (and isn't) + +Cloudflare Workers run on Cloudflare's global edge network. The production +image in this repo **does not host the Worker** — it is a pinned deploy +toolchain that runs `wrangler deploy` and pushes the Worker to Cloudflare. +Treat "production Docker" here like a CI builder, not a runtime server. + +If you want a long-running local HTTP endpoint, use the **development** +compose file — it runs `wrangler dev`, which serves the Worker on +`http://localhost:8787` backed by the local `workerd` runtime. + +--- + +## Files + +| File | Purpose | +|---|---| +| `Dockerfile` | Multi-stage build: `base` → `deps` → `dev` / `typecheck` / `prod` | +| `.dockerignore` | Keeps secrets, `node_modules`, and `.wrangler` out of the build context | +| `docker-compose.yml` | Shared service definition (image, build context, persistent wrangler state) | +| `docker-compose.dev.yml` | Dev override — `wrangler dev` + source bind-mount + port 8787 | +| `docker-compose.prod.yml` | Prod override — `wrangler deploy` as a one-shot run | +| `.env.example` | Template for `.dev.vars` (dev) and `.env` (deploy credentials) | + +--- + +## Development workflow + +### 1. Create `.dev.vars` + +`wrangler dev` reads Worker secrets from `.dev.vars` (not `.env`). Start +from the template: + +```bash +cp .env.example .dev.vars +# edit .dev.vars and fill in ANTHROPIC_API_KEY / OPENAI_API_KEY / GITHUB_PAT +``` + +`.dev.vars` is gitignored and mounted read-only into the container. + +### 2. Start the dev server + +```bash +docker compose -f docker-compose.yml -f docker-compose.dev.yml up +``` + +The Worker is now reachable at `http://localhost:8787`. Smoke-test it: + +```bash +curl http://localhost:8787/health +``` + +**Hot reload**: the `src/` directory is bind-mounted, so saving a file on +the host triggers wrangler's watcher inside the container. No rebuild +required. + +**node_modules**: stored in a named volume (`node_modules`) so host-built +platform binaries (esbuild, etc.) don't conflict with the Linux-built +versions inside the container. + +### 3. Useful dev commands + +```bash +# One-off commands inside the dev container +docker compose -f docker-compose.yml -f docker-compose.dev.yml \ + run --rm orchestrator npm run typecheck + +docker compose -f docker-compose.yml -f docker-compose.dev.yml \ + run --rm orchestrator npx wrangler kv namespace list + +# Stop and remove containers (keeps volumes) +docker compose -f docker-compose.yml -f docker-compose.dev.yml down + +# Full reset (drops node_modules + wrangler state volumes) +docker compose -f docker-compose.yml -f docker-compose.dev.yml down -v +``` + +--- + +## Production deploy workflow + +### 1. Create `.env` with deploy credentials + +```bash +cp .env.example .env +# fill in CLOUDFLARE_API_TOKEN (and CLOUDFLARE_ACCOUNT_ID if your token +# is scoped to multiple accounts) +``` + +Generate the API token at + using the **Edit +Cloudflare Workers** template. + +### 2. Set Worker secrets once (not through Docker) + +Worker runtime secrets (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GITHUB_PAT`) +are stored on Cloudflare, not baked into the image. Set them once per +environment: + +```bash +docker compose -f docker-compose.yml -f docker-compose.prod.yml \ + run --rm orchestrator npx wrangler secret put ANTHROPIC_API_KEY +# repeat for OPENAI_API_KEY, GITHUB_PAT +``` + +### 3. Deploy + +```bash +docker compose -f docker-compose.yml -f docker-compose.prod.yml \ + run --rm orchestrator +``` + +The `prod` stage depends on the `typecheck` stage, so a TypeScript error +fails the build *before* `wrangler deploy` runs. A broken tree can't be +pushed to production. + +### 4. Tail production logs + +```bash +docker compose -f docker-compose.yml -f docker-compose.prod.yml \ + run --rm orchestrator npx wrangler tail +``` + +--- + +## CI/CD integration + +The production compose file is designed to be called from CI. A minimal +GitHub Actions job: + +```yaml +deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Deploy + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + docker compose -f docker-compose.yml -f docker-compose.prod.yml \ + run --rm orchestrator +``` + +If your CI already caches Docker layers, the `deps` stage is the big win — +it only re-runs `npm ci` when `package-lock.json` changes. + +For a gate-only check (no deploy), target the typecheck stage directly: + +```bash +docker build --target typecheck -t orchestrator-check . +``` + +--- + +## Image layout + +``` +base alpine + node 20 + git + tini + ca-certificates + └── deps + node_modules (from npm ci, cached via BuildKit) + ├── dev + full source; runs `wrangler dev` + ├── typecheck + full source; runs `npm run typecheck` at build time + └── prod copies /app from typecheck; runs `wrangler deploy` +``` + +Why `tini`? Wrangler spawns child processes (workerd, esbuild). Without a +proper PID 1, `docker stop` can leave zombies and slow shutdown. + +Why Alpine? Small image, fast pulls in CI. If you hit a native-module +compatibility issue (rare with this dep tree), switch the base to +`node:20-bookworm-slim`. + +--- + +## Troubleshooting + +**`wrangler dev` exits immediately with "authentication required"** +You're hitting a command that needs a real Cloudflare login. `wrangler dev` +itself runs offline by default, but commands like `wrangler kv namespace +create` require auth. Either run `wrangler login` inside the container (a +browser URL will be printed) or set `CLOUDFLARE_API_TOKEN` in the +environment. + +**Port 8787 isn't reachable from the host** +Confirm wrangler is bound to `0.0.0.0`, not `127.0.0.1`. The dev compose +file passes `--ip 0.0.0.0` explicitly; if you override the command, keep +that flag. + +**File changes don't trigger a reload** +On Windows/macOS, Docker Desktop's file-event propagation can be flaky on +bind mounts. Restart the container, or switch to a polling watcher by +setting `CHOKIDAR_USEPOLLING=true` in the dev override's `environment`. + +**`npm ci` fails with EACCES on `node_modules`** +Delete the named volume so it rebuilds clean: +`docker compose -f docker-compose.yml -f docker-compose.dev.yml down -v`.