From 2bf57e2137ced084ef816d7ddd9c328250625ab9 Mon Sep 17 00:00:00 2001 From: Aditya Rana Date: Sun, 5 Apr 2026 19:03:06 +0530 Subject: [PATCH] infra: add Docker, docker-compose, and GitHub Actions CI/CD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - server/Dockerfile: multi-stage Go build (golang:1.22-alpine -> alpine:3.19) - web/Dockerfile: multi-stage Next.js build with standalone output - docker-compose.yml: 3-node Raft cluster + Next.js dashboard - .github/workflows/ci.yml: go test -race on every push/PR to main - .github/workflows/docker-publish.yml: push to Docker Hub on merge to main - Makefile (root): docker-up/down/logs/ps convenience targets - server/cmd/server/main.go: replace hardcoded Raft config with env vars (NODE_ID, HTTP_ADDR, TCP_ADDR, PEERS, DATA_DIR — all have safe defaults) - web/next.config.ts: enable standalone output for Docker runner stage --- .github/workflows/ci.yml | 64 ++++++++++++++ .github/workflows/docker-publish.yml | 86 ++++++++++++++++++ Makefile | 80 +++++++++++++++++ docker-compose.yml | 126 +++++++++++++++++++++++++++ server/.dockerignore | 22 +++++ server/Dockerfile | 42 +++++++++ server/cmd/server/main.go | 44 +++++++--- web/.dockerignore | 26 ++++++ web/Dockerfile | 55 ++++++++++++ web/next.config.ts | 2 +- 10 files changed, 535 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/docker-publish.yml create mode 100644 Makefile create mode 100644 docker-compose.yml create mode 100644 server/.dockerignore create mode 100644 server/Dockerfile create mode 100644 web/.dockerignore create mode 100644 web/Dockerfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2e896d2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,64 @@ +# ───────────────────────────────────────────────────────────────────────────── +# CI — Run tests, vet, and build on every push and pull request to main +# ───────────────────────────────────────────────────────────────────────────── +name: CI + +on: + push: + branches: ["main", "infra/docker-ci"] + paths: + - "server/**" + pull_request: + branches: ["main"] + paths: + - "server/**" + +jobs: + test: + name: Test & Vet + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: server/go.mod + cache-dependency-path: server/go.sum + + - name: Download modules + working-directory: server + run: go mod download + + - name: Vet + working-directory: server + run: go vet ./... + + - name: Test (with race detector) + working-directory: server + run: go test -race -count=1 -timeout 120s ./... + + build: + name: Build Binary + runs-on: ubuntu-latest + needs: test + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: server/go.mod + cache-dependency-path: server/go.sum + + - name: Build server + working-directory: server + run: CGO_ENABLED=0 go build -o kvstore ./cmd/server + + - name: Build CLI + working-directory: server + run: CGO_ENABLED=0 go build -o kvcli ./cmd/kvcli diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..1ffc88a --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,86 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Docker Publish — Build and push images to Docker Hub on merge to main +# +# Required GitHub Secrets: +# DOCKER_USERNAME — your Docker Hub username +# DOCKER_PASSWORD — your Docker Hub access token (NOT your password) +# +# Images produced: +# /kvstore-server:latest +# /kvstore-server: +# /kvstore-web:latest +# /kvstore-web: +# ───────────────────────────────────────────────────────────────────────────── +name: Docker Publish + +on: + push: + branches: ["main"] + +jobs: + publish: + name: Build & Push Docker Images + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + # ── Set up Docker Buildx (required for multi-arch + cache) ─────────── + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # ── Log in to Docker Hub ───────────────────────────────────────────── + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + # ── Extract metadata (tags + labels) for server image ──────────────── + - name: Docker metadata — server + id: meta-server + uses: docker/metadata-action@v5 + with: + images: ${{ secrets.DOCKER_USERNAME }}/kvstore-server + tags: | + type=raw,value=latest + type=sha,prefix=,format=short + + # ── Build and push the Go server image ─────────────────────────────── + - name: Build and push server image + uses: docker/build-push-action@v6 + with: + context: ./server + file: ./server/Dockerfile + push: true + tags: ${{ steps.meta-server.outputs.tags }} + labels: ${{ steps.meta-server.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # ── Extract metadata for web image ─────────────────────────────────── + - name: Docker metadata — web + id: meta-web + uses: docker/metadata-action@v5 + with: + images: ${{ secrets.DOCKER_USERNAME }}/kvstore-web + tags: | + type=raw,value=latest + type=sha,prefix=,format=short + + # ── Build and push the Next.js web image ───────────────────────────── + - name: Build and push web image + uses: docker/build-push-action@v6 + with: + context: ./web + file: ./web/Dockerfile + push: true + tags: ${{ steps.meta-web.outputs.tags }} + labels: ${{ steps.meta-web.outputs.labels }} + # Pass build-time API URL — override via repo variable if needed + build-args: | + NEXT_PUBLIC_API_URL=http://localhost:8080 + NEXT_PUBLIC_WS_URL=ws://localhost:8080 + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0df9a3a --- /dev/null +++ b/Makefile @@ -0,0 +1,80 @@ +# ====================================================================== +# KVStore — Root Makefile +# Operates on the full stack (server + web + Docker cluster). +# Run individual server targets from inside server/ using its own Makefile. +# ====================================================================== + +.DEFAULT_GOAL := help + +.PHONY: help +help: + @echo "" + @echo " KVStore — root targets" + @echo "" + @echo " Docker Cluster" + @echo " make docker-build build all images locally" + @echo " make docker-up build + start 3-node cluster and web dashboard" + @echo " make docker-down stop and remove containers (keeps volumes)" + @echo " make docker-clean stop, remove containers AND named volumes" + @echo " make docker-logs tail logs from all services" + @echo " make docker-ps show status of all services" + @echo "" + @echo " Individual services" + @echo " make docker-node1 tail logs for node1 only" + @echo " make docker-web tail logs for the web dashboard only" + @echo "" + @echo " Server (delegates to server/Makefile)" + @echo " make test run all Go tests with the race detector" + @echo " make build build server and CLI binaries" + @echo "" + +# ── Docker Cluster ────────────────────────────────────────────────────────── + +.PHONY: docker-build +docker-build: + docker compose build + +.PHONY: docker-up +docker-up: + docker compose up --build -d + @echo "" + @echo " Cluster is running:" + @echo " node1 HTTP API → http://localhost:8080" + @echo " node1 TCP → localhost:6379" + @echo " Dashboard → http://localhost:3000" + @echo "" + +.PHONY: docker-down +docker-down: + docker compose down + +.PHONY: docker-clean +docker-clean: + docker compose down -v + @echo " All containers and volumes removed." + +.PHONY: docker-logs +docker-logs: + docker compose logs -f + +.PHONY: docker-ps +docker-ps: + docker compose ps + +.PHONY: docker-node1 +docker-node1: + docker compose logs -f node1 + +.PHONY: docker-web +docker-web: + docker compose logs -f web + +# ── Server (delegate) ─────────────────────────────────────────────────────── + +.PHONY: test +test: + $(MAKE) -C server test + +.PHONY: build +build: + $(MAKE) -C server build diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..06d87f6 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,126 @@ +# ───────────────────────────────────────────────────────────────────────────── +# KVStore — 3-Node Raft Cluster + Next.js Dashboard +# +# Usage: +# docker compose up --build -d # start everything +# docker compose down -v # stop and remove volumes +# docker compose logs -f # tail all logs +# ───────────────────────────────────────────────────────────────────────────── + +services: + + # ── Raft Node 1 (bootstrap leader candidate) ──────────────────────────── + node1: + build: + context: ./server + dockerfile: Dockerfile + image: kvstore-server:local + container_name: kvstore-node1 + restart: unless-stopped + environment: + NODE_ID: node1 + HTTP_ADDR: :8080 + TCP_ADDR: :6379 + DATA_DIR: /app/data + # Peer list: all three nodes know about each other + PEERS: "node2=http://node2:8080,node3=http://node3:8080" + ports: + - "6379:6379" # TCP protocol (exposed to host for kvcli) + - "8080:8080" # HTTP API (exposed to host for curl / dashboard) + volumes: + - node1-data:/app/data + networks: + - kvstore-net + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8080/api/health || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 5s + + # ── Raft Node 2 ────────────────────────────────────────────────────────── + node2: + image: kvstore-server:local + container_name: kvstore-node2 + restart: unless-stopped + depends_on: + node1: + condition: service_started + environment: + NODE_ID: node2 + HTTP_ADDR: :8080 + TCP_ADDR: :6379 + DATA_DIR: /app/data + PEERS: "node1=http://node1:8080,node3=http://node3:8080" + # Internal ports only — node2/node3 are not exposed to the host + expose: + - "8080" + - "6379" + volumes: + - node2-data:/app/data + networks: + - kvstore-net + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8080/api/health || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 5s + + # ── Raft Node 3 ────────────────────────────────────────────────────────── + node3: + image: kvstore-server:local + container_name: kvstore-node3 + restart: unless-stopped + depends_on: + node1: + condition: service_started + environment: + NODE_ID: node3 + HTTP_ADDR: :8080 + TCP_ADDR: :6379 + DATA_DIR: /app/data + PEERS: "node1=http://node1:8080,node2=http://node2:8080" + expose: + - "8080" + - "6379" + volumes: + - node3-data:/app/data + networks: + - kvstore-net + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8080/api/health || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 5s + + # ── Next.js Dashboard ───────────────────────────────────────────────────── + web: + build: + context: ./web + dockerfile: Dockerfile + args: + # Dashboard points at node1's HTTP API + NEXT_PUBLIC_API_URL: http://localhost:8080 + NEXT_PUBLIC_WS_URL: ws://localhost:8080 + container_name: kvstore-web + restart: unless-stopped + depends_on: + node1: + condition: service_healthy + ports: + - "3000:3000" + networks: + - kvstore-net + +# ── Named volumes (data persists across container restarts) ──────────────── +volumes: + node1-data: + node2-data: + node3-data: + +# ── Internal bridge network ──────────────────────────────────────────────── +networks: + kvstore-net: + driver: bridge diff --git a/server/.dockerignore b/server/.dockerignore new file mode 100644 index 0000000..281c460 --- /dev/null +++ b/server/.dockerignore @@ -0,0 +1,22 @@ +# Build artifacts +kvstore +kvcli +*.exe +*.out +*.test +*.prof + +# Runtime data — never bake into image +data/ + +# Coverage reports +coverage.out +coverage.html + +# IDE / OS +.vscode/ +.idea/ +*.swp +*.swo +.DS_Store +Thumbs.db diff --git a/server/Dockerfile b/server/Dockerfile new file mode 100644 index 0000000..c1fa294 --- /dev/null +++ b/server/Dockerfile @@ -0,0 +1,42 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Stage 1: Build the Go binary +# ───────────────────────────────────────────────────────────────────────────── +FROM golang:1.22-alpine AS builder + +WORKDIR /app + +# Download dependencies first (layer cache) +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source and build a statically linked binary +COPY . . +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ + go build -ldflags="-s -w" -o kvstore ./cmd/server + +# ───────────────────────────────────────────────────────────────────────────── +# Stage 2: Minimal runtime image +# ───────────────────────────────────────────────────────────────────────────── +FROM alpine:3.19 + +# ca-certificates needed if the server ever makes outbound TLS calls +RUN apk add --no-cache ca-certificates + +WORKDIR /app + +COPY --from=builder /app/kvstore . + +# Data directory for AOF log + snapshot (mount a volume here in production) +RUN mkdir -p /app/data + +# TCP protocol port | HTTP API + Raft RPC port +EXPOSE 6379 8080 + +# Override via env vars: NODE_ID, HTTP_ADDR, TCP_ADDR, PEERS, DATA_DIR +ENV NODE_ID=node1 \ + HTTP_ADDR=:8080 \ + TCP_ADDR=:6379 \ + DATA_DIR=/app/data \ + PEERS="" + +ENTRYPOINT ["./kvstore"] diff --git a/server/cmd/server/main.go b/server/cmd/server/main.go index e5b226a..d44eef3 100644 --- a/server/cmd/server/main.go +++ b/server/cmd/server/main.go @@ -5,18 +5,40 @@ import ( "fmt" "os" "os/signal" + "strings" "syscall" "time" "github.com/ARCoder181105/kvstore/internal/api" aof "github.com/ARCoder181105/kvstore/internal/persistence" + "github.com/ARCoder181105/kvstore/internal/raft" "github.com/ARCoder181105/kvstore/internal/server" "github.com/ARCoder181105/kvstore/internal/store" - "github.com/ARCoder181105/kvstore/internal/raft" ) +func getEnv(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} + func main() { - if err := os.MkdirAll("./data", 0755); err != nil { + nodeID := getEnv("NODE_ID", "node1") + httpAddr := getEnv("HTTP_ADDR", ":8080") + tcpAddr := getEnv("TCP_ADDR", ":6379") + dataDir := getEnv("DATA_DIR", "./data") + peersStr := getEnv("PEERS", "node1=http://localhost:8080") + + peers := make(map[raft.NodeID]string) + for _, p := range strings.Split(peersStr, ",") { + parts := strings.Split(p, "=") + if len(parts) == 2 { + peers[raft.NodeID(parts[0])] = parts[1] + } + } + + if err := os.MkdirAll(dataDir, 0755); err != nil { fmt.Println("failed to create data directory:", err) os.Exit(1) } @@ -28,13 +50,13 @@ func main() { // with short remaining TTLs that were being loaded from disk. // 1. Load snapshot (absolute ExpiresAt — no time math required) - if err := aof.Load("./data/snapshot.db", s); err != nil { + if err := aof.Load(dataDir+"/snapshot.db", s); err != nil { fmt.Println("snapshot load error:", err) os.Exit(1) } // 2. Replay AOF on top of snapshot (also uses absolute ExpiresAt now) - if err := aof.Replay("./data/aof.log", s); err != nil { + if err := aof.Replay(dataDir+"/aof.log", s); err != nil { fmt.Println("AOF replay error:", err) os.Exit(1) } @@ -45,7 +67,7 @@ func main() { go s.StartEviction(ctx) // 4. Start AOF writer - aofWriter, err := aof.NewAOFWriter("./data/aof.log") + aofWriter, err := aof.NewAOFWriter(dataDir + "/aof.log") if err != nil { fmt.Println("failed to create AOF writer:", err) os.Exit(1) @@ -53,22 +75,22 @@ func main() { go aofWriter.Start(ctx) // 5. Start TCP server - srv := server.New(":6379", s, aofWriter) + srv := server.New(tcpAddr, s, aofWriter) if err := srv.Start(); err != nil { fmt.Println("failed to start server:", err) os.Exit(1) } - fmt.Println("kvstore listening on :6379") + fmt.Printf("kvstore listening on %s\n", tcpAddr) // 6. Initialize Raft - raftNode := raft.New("node1", map[raft.NodeID]string{"node1": "http://localhost:8080"}, s) + raftNode := raft.New(raft.NodeID(nodeID), peers, s) apiSrv := api.New(s, raftNode) - if err := apiSrv.Start(":8080"); err != nil { + if err := apiSrv.Start(httpAddr); err != nil { fmt.Println("failed to start HTTP server:", err) os.Exit(1) } - fmt.Println("HTTP API listening on :8080") + fmt.Printf("HTTP API listening on %s\n", httpAddr) quit := make(chan os.Signal, 1) signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) @@ -84,7 +106,7 @@ func main() { cancel() // stops eviction + AOF goroutines // Save final snapshot on clean shutdown - if err := aof.Save(s, "./data/snapshot.db"); err != nil { + if err := aof.Save(s, dataDir+"/snapshot.db"); err != nil { fmt.Println("snapshot save error:", err) } diff --git a/web/.dockerignore b/web/.dockerignore new file mode 100644 index 0000000..915bfb8 --- /dev/null +++ b/web/.dockerignore @@ -0,0 +1,26 @@ +# Dependencies +node_modules/ +.npm + +# Next.js build output +.next/ +out/ + +# Environment files (secrets) +.env +.env.local +.env.production + +# IDE / OS +.vscode/ +.idea/ +*.swp +.DS_Store +Thumbs.db + +# Git +.git/ +.gitignore + +# Docs +*.md diff --git a/web/Dockerfile b/web/Dockerfile new file mode 100644 index 0000000..2301f36 --- /dev/null +++ b/web/Dockerfile @@ -0,0 +1,55 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Stage 1: Install dependencies +# ───────────────────────────────────────────────────────────────────────────── +FROM node:20-alpine AS deps + +WORKDIR /app + +COPY package.json package-lock.json ./ +RUN npm ci + +# ───────────────────────────────────────────────────────────────────────────── +# Stage 2: Build the Next.js application +# ───────────────────────────────────────────────────────────────────────────── +FROM node:20-alpine AS builder + +WORKDIR /app + +COPY --from=deps /app/node_modules ./node_modules +COPY . . + +# Build-time env vars (injected by docker-compose or CI) +ARG NEXT_PUBLIC_API_URL=http://localhost:8080 +ARG NEXT_PUBLIC_WS_URL=ws://localhost:8080 + +ENV NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL \ + NEXT_PUBLIC_WS_URL=$NEXT_PUBLIC_WS_URL + +RUN npm run build + +# ───────────────────────────────────────────────────────────────────────────── +# Stage 3: Minimal production runtime +# ───────────────────────────────────────────────────────────────────────────── +FROM node:20-alpine AS runner + +WORKDIR /app + +ENV NODE_ENV=production + +# Create a non-root user for security +RUN addgroup --system --gid 1001 nodejs && \ + adduser --system --uid 1001 nextjs + +# Copy only what Next.js standalone needs +COPY --from=builder /app/public ./public +COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ +COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static + +USER nextjs + +EXPOSE 3000 + +ENV PORT=3000 \ + HOSTNAME=0.0.0.0 + +CMD ["node", "server.js"] diff --git a/web/next.config.ts b/web/next.config.ts index e9ffa30..68a6c64 100644 --- a/web/next.config.ts +++ b/web/next.config.ts @@ -1,7 +1,7 @@ import type { NextConfig } from "next"; const nextConfig: NextConfig = { - /* config options here */ + output: "standalone", }; export default nextConfig;