diff --git a/.gitignore b/.gitignore index e3be5b64..1bff456c 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,8 @@ __debug_bin *.pem *.crt vendor/ +*.duckdb +storage/* # Internal team references docs/SONARQUBE_SETUP_GUIDE.md @@ -53,3 +55,6 @@ internal/WAL/.tmp/* .code-review-graph/* .cursor/* test_results/ + +dlq/* +*.log diff --git a/Dockerfile b/Dockerfile index 7eed9af9..8fa67e89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,14 +2,14 @@ # JMDN (Jupiter MetaZK Decentralized Network) - Multi-stage Dockerfile # ============================================================================= # Build: docker build -t jmdn:latest . -# Run: docker run -d --name jmdn -p 8080:8080 -p 4001:4001 jmdn:latest -# Config: docker run -d -v /path/to/config.env:/etc/jmdn/config.env jmdn:latest +# Run: docker run -d --name jmdn -p 6090:6090 -p 6545:6545 jmdn:latest +# Config: docker run -d -v /path/to/jmdn.yaml:/etc/jmdn/jmdn.yaml jmdn:latest # ============================================================================= # ----------------------------------------------------------------------------- # Stage 1: Build # ----------------------------------------------------------------------------- -FROM golang:1.25-bookworm AS builder +FROM golang:1.25.3-bookworm AS builder # Install build dependencies (CGO_ENABLED=1 requires gcc) RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -51,11 +51,25 @@ RUN GIT_COMMIT=${GIT_COMMIT:-$(git rev-parse --short HEAD 2>/dev/null || echo "u # ----------------------------------------------------------------------------- FROM debian:bookworm-slim -# Install runtime dependencies +# Install runtime dependencies + Yggdrasil (required: network.yggdrasil: true in jmdn.yaml) RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ + gnupg \ libc6 \ + netcat-openbsd \ + wget \ + openssl \ + python3 \ + gawk \ + && mkdir -p /usr/local/apt-keys \ + && gpg --fetch-keys https://neilalexander.s3.dualstack.eu-west-2.amazonaws.com/deb/key.txt \ + && gpg --export 1C5162E133015D81A811239D1840CDAC6011C5EA \ + | tee /usr/local/apt-keys/yggdrasil-keyring.gpg > /dev/null \ + && echo 'deb [signed-by=/usr/local/apt-keys/yggdrasil-keyring.gpg] http://neilalexander.s3.dualstack.eu-west-2.amazonaws.com/deb/ debian yggdrasil' \ + > /etc/apt/sources.list.d/yggdrasil.list \ + && apt-get update && apt-get install -y --no-install-recommends \ + yggdrasil \ && rm -rf /var/lib/apt/lists/* # Install ImmuDB @@ -66,35 +80,56 @@ RUN ARCH=$([ "$TARGETARCH" = "arm64" ] && echo "arm64" || echo "amd64") && \ -o /usr/local/bin/immudb && \ chmod +x /usr/local/bin/immudb -# Create non-root user -RUN groupadd -r jmdn && useradd -r -g jmdn -d /home/jmdn -s /bin/bash -m jmdn - -# Create required directories -RUN mkdir -p /etc/jmdn /opt/jmdn/data /var/log/jmdn && \ - chown -R jmdn:jmdn /opt/jmdn /var/log/jmdn /etc/jmdn - -# Copy binary and default config from builder -COPY --from=builder /src/jmdn /usr/local/bin/jmdn -COPY --from=builder /src/jmdn_default.yaml /etc/jmdn/jmdn_default.yaml +# Create required directories (mirrors install_services.sh layout) +RUN mkdir -p \ + /etc/jmdn/certs \ + /opt/jmdn/data/data \ + /opt/jmdn/data/config \ + /opt/jmdn/data/DB \ + /var/log/jmdn + +# Copy binary and scripts from builder +COPY --from=builder /src/jmdn /usr/local/bin/jmdn +COPY --from=builder /src/Scripts/start_jmdn_wrapper.sh /usr/local/bin/start_jmdn_wrapper.sh +COPY --from=builder /src/Scripts/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY --from=builder /src/Scripts/bootstrap_sync.sh /usr/local/bin/bootstrap_sync.sh +RUN chmod +x /usr/local/bin/start_jmdn_wrapper.sh \ + /usr/local/bin/docker-entrypoint.sh \ + /usr/local/bin/bootstrap_sync.sh + +# Copy default config as jmdn.yaml (mirrors setup_config.sh: cp jmdn_default.yaml → jmdn.yaml) +COPY --from=builder /src/jmdn_default.yaml /etc/jmdn/jmdn.yaml + +# peer.json stored at two locations: +# /opt/jmdn/data/config/peer.json — runtime path (WORKDIR-relative, per config/constants.go) +# /etc/jmdn/peer.json — fallback used by entrypoint after bootstrap wipes the volume +COPY --from=builder /src/config/peer.json /opt/jmdn/data/config/peer.json COPY --from=builder /src/config/peer.json /etc/jmdn/peer.json -# Expose common ports -# 8080 - HTTP API / Explorer -# 4001 - P2P / libp2p -# 3323 - ImmuDB -# 50051 - gRPC -EXPOSE 8080 4001 3323 50051 - -# Health check -HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD curl -f http://localhost:8080/health || exit 1 - -# Data volume for ImmuDB persistence +# Expose ports per jmdn.yaml (localhost-bound ports excluded) +# 6090 - HTTP API / Explorer (ports.api) +# 16050 - Block generation (ports.blockgen) +# 16055 - Block propagation gRPC (ports.blockgrpc) +# 16052 - DID service (ports.did) +# 6545 - Facade / JSON-RPC (ports.facade) +# 6546 - WebSocket (ports.ws) +# ImmuDB (3322) is container-internal — not exposed +EXPOSE 6090 16050 16055 16052 6545 6546 + +# Health check against actual API port (ports.api: 6090) +# start-period extended to 60s to allow bootstrap sync on first run +HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ + CMD curl -f http://localhost:6090/health || exit 1 + +# Data volume — ImmuDB data, peer identity, certs, fastsync state VOLUME ["/opt/jmdn/data"] -USER jmdn -WORKDIR /home/jmdn +# Run as root — bootstrap_sync.sh needs root for chown after snapshot extract +# WORKDIR must match where jmdn resolves ./config/peer.json +# (hardcoded in config/constants.go: PeerFile = "./config/peer.json") +WORKDIR /opt/jmdn/data -# Default entrypoint - override config with -v /your/config.env:/etc/jmdn/config.env -ENTRYPOINT ["jmdn"] -CMD ["-config", "/etc/jmdn/config.env"] +# Startup order: ImmuDB → bootstrap sync → jmdn +# Override config: -v /your/jmdn.yaml:/etc/jmdn/jmdn.yaml +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +CMD ["-config", "/etc/jmdn/jmdn.yaml"] diff --git a/Scripts/bootstrap_sync.sh b/Scripts/bootstrap_sync.sh new file mode 100644 index 00000000..a0e3d07e --- /dev/null +++ b/Scripts/bootstrap_sync.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# bootstrap_sync.sh - JMDN Docker Bootstrap Sync +# +# Mirrors bootstrap.yml Ansible playbook for Docker deployments. +# Runs ONCE — guarded by /opt/jmdn/data/.bootstrapped sentinel on the volume. +# +# Flow (matches ansible task order): +# 1. List + download multipart files and checksums.md5 from GCS (public HTTP) +# 2. Normalise checksums (strip GCS paths → local basenames) and verify +# 3. Backup existing data → /opt/jmdn/backup/data_ +# 4. Clear immudb identity/state files +# 5. cat parts* | tar -xzf - into sandbox +# 6. Auto-discover real data root (parent dir of systemdb/) +# 7. Move to /opt/jmdn/data, fix permissions, clean up +# 8. Write sentinel +# +# Env overrides: +# GCS_BUCKET GCS bucket name (default: jmzk-releases) +# GCS_PREFIX Path prefix in bucket (default: jmdn_bootstrap_2306) +# PARTS_PREFIX Part filename prefix (default: data_backup_23062026.part) +# CHECKSUM_FILE Checksum filename (default: checksums.md5) +# +# To force a re-sync after wiping the volume: +# docker exec jmdn rm /opt/jmdn/data/.bootstrapped && docker restart jmdn + +set -euo pipefail + +# ── Config ─────────────────────────────────────────────── +GCS_BUCKET="${GCS_BUCKET:-jmzk-releases}" +GCS_PREFIX="${GCS_PREFIX:-jmdn_bootstrap_2306}" +PARTS_PREFIX="${PARTS_PREFIX:-data_backup_23062026.part}" +CHECKSUM_FILE="${CHECKSUM_FILE:-checksums.md5}" + +BASE_DIR="/opt/jmdn" +DATA_DIR="${BASE_DIR}/data" +WORK_DIR="${BASE_DIR}/bootstrap_tmp" +BACKUP_BASE="${BASE_DIR}/backup" +IMMUDB_STATE_DIR="${BASE_DIR}/.immudb_state" +SENTINEL="${DATA_DIR}/.bootstrapped" + +GCS_HTTP="https://storage.googleapis.com" +GCS_API="${GCS_HTTP}/storage/v1/b/${GCS_BUCKET}/o" + +log() { echo "[bootstrap] $*"; } +die() { echo "[bootstrap] ERROR: $*" >&2; exit 1; } + +# ── Guard ───────────────────────────────────────────────── +if [ -f "$SENTINEL" ]; then + log "Sentinel found — skipping bootstrap." + exit 0 +fi + +log "First run detected — starting bootstrap sync." + +# ── Ensure required tools ───────────────────────────────── +for tool in curl wget awk md5sum tar python3; do + command -v "$tool" >/dev/null 2>&1 || die "$tool is required but not found." +done + +# ── Prepare work directory ─────────────────────────────── +log "Preparing work directory: $WORK_DIR" +rm -rf "$WORK_DIR" +mkdir -p "$WORK_DIR" + +# ── List parts from GCS (public HTTP, no gsutil needed) ── +log "Listing parts from GCS: gs://${GCS_BUCKET}/${GCS_PREFIX}/${PARTS_PREFIX}*" +PARTS_JSON=$(curl -sf "${GCS_API}?prefix=${GCS_PREFIX}/${PARTS_PREFIX}" \ + || die "Failed to list objects from GCS. Check bucket name and network.") + +PART_NAMES=$(echo "$PARTS_JSON" | python3 -c " +import json, sys +data = json.load(sys.stdin) +items = data.get('items', []) +if not items: + raise SystemExit('No parts found matching prefix') +for item in sorted(items, key=lambda x: x['name']): + print(item['name']) +") || die "Failed to parse GCS listing." + +[ -z "$PART_NAMES" ] && die "No parts found for prefix ${GCS_PREFIX}/${PARTS_PREFIX}" + +log "Found parts:" +echo "$PART_NAMES" | while read -r p; do log " gs://${GCS_BUCKET}/$p"; done + +# ── Download parts ──────────────────────────────────────── +log "Downloading parts to $WORK_DIR..." +echo "$PART_NAMES" | while read -r part_path; do + fname=$(basename "$part_path") + log " $fname" + wget -q --show-progress -O "$WORK_DIR/$fname" "${GCS_HTTP}/${GCS_BUCKET}/${part_path}" +done + +# ── Download checksums ──────────────────────────────────── +log "Downloading ${CHECKSUM_FILE}..." +wget -q -O "$WORK_DIR/$CHECKSUM_FILE" \ + "${GCS_HTTP}/${GCS_BUCKET}/${GCS_PREFIX}/${CHECKSUM_FILE}" \ + || die "Failed to download ${CHECKSUM_FILE} from GCS." + +# ── Verify checksums ────────────────────────────────────── +# Mirrors ansible: +# awk '{n=split($2,a,"/"); print $1 " " a[n]}' checksums.md5 > checksums_local.md5 +# md5sum -c checksums_local.md5 +log "Normalising and verifying checksums..." +awk '{n=split($2,a,"/"); print $1 " " a[n]}' \ + "$WORK_DIR/$CHECKSUM_FILE" > "$WORK_DIR/checksums_local.md5" + +(cd "$WORK_DIR" && md5sum -c checksums_local.md5) \ + || die "Checksum verification failed — aborting to prevent corrupt data." +log "Checksums OK." + +# ── Backup existing data ────────────────────────────────── +if [ -d "$DATA_DIR" ] && [ "$(ls -A "$DATA_DIR" 2>/dev/null)" ]; then + TIMESTAMP=$(date -u '+%Y%m%dT%H%M%S') + BACKUP_DIR="${BACKUP_BASE}/data_${TIMESTAMP}" + log "Backing up existing data → $BACKUP_DIR" + mkdir -p "$BACKUP_BASE" + mv "$DATA_DIR" "$BACKUP_DIR" +fi + +# ── Clear immudb identity/state files ──────────────────── +log "Clearing immudb identity and state files..." +rm -f "${IMMUDB_STATE_DIR}"/.identity-* 2>/dev/null || true +rm -f "${IMMUDB_STATE_DIR}"/.state-* 2>/dev/null || true + +# ── Extract into sandbox ────────────────────────────────── +# Mirrors ansible: cat parts* | tar -xzf - -C sandbox +SANDBOX="${BASE_DIR}/data_tmp/sandbox" +log "Extracting parts into sandbox: $SANDBOX" +mkdir -p "$SANDBOX" + +# shellcheck disable=SC2086 +cat "$WORK_DIR"/${PARTS_PREFIX}* | tar -xzf - -C "$SANDBOX" \ + || die "Extraction failed." + +# ── Auto-discover real data root ────────────────────────── +# Mirrors ansible: find systemdb dir → its parent is the real data root +log "Discovering data root (searching for systemdb/)..." +REAL_DATA_DIR=$(find "$SANDBOX" -type d -name "systemdb" 2>/dev/null \ + | head -n 1 | xargs -I{} dirname {}) + +if [ -z "$REAL_DATA_DIR" ] || [ "$REAL_DATA_DIR" = "." ]; then + die "Could not find systemdb in extracted sandbox. Check snapshot integrity." +fi +log "Data root found: $REAL_DATA_DIR" + +# ── Move to final location ──────────────────────────────── +log "Moving data to $DATA_DIR..." +mv "$REAL_DATA_DIR" "$DATA_DIR" +rm -rf "${BASE_DIR}/data_tmp" + +# ── Fix permissions ─────────────────────────────────────── +log "Fixing permissions on $DATA_DIR..." +chown -R root:root "$DATA_DIR" + +# ── Clean up work directory ─────────────────────────────── +log "Cleaning up work directory..." +rm -rf "$WORK_DIR" + +# ── Write sentinel ──────────────────────────────────────── +touch "$SENTINEL" +log "Bootstrap complete. Sentinel written → $SENTINEL" diff --git a/Scripts/docker-entrypoint.sh b/Scripts/docker-entrypoint.sh new file mode 100644 index 00000000..fb1e78b2 --- /dev/null +++ b/Scripts/docker-entrypoint.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# docker-entrypoint.sh - Container startup orchestrator for JMDN +# +# Startup order (mirrors bootstrap.yml ansible play): +# 1. ImmuDB — start in background, wait for port 3322 +# 2. Bootstrap sync — first run only (download snapshot, verify, extract) +# 3. Restore paths — peer.json, certs, required dirs (wiped by bootstrap) +# 4. jmdn — exec only if all prior steps succeed + +set -euo pipefail + +IMMUDB_PORT="${IMMUDB_PORT:-3322}" +IMMUDB_DIR="${IMMUDB_DIR:-/opt/jmdn/data}" +IMMUDB_READY_TIMEOUT="${IMMUDB_READY_TIMEOUT:-30}" + +log() { echo "[entrypoint] $*"; } + +# ── Step 1: Start ImmuDB ───────────────────────────────── +log "Starting ImmuDB (dir: ${IMMUDB_DIR})..." +immudb --dir "${IMMUDB_DIR}" & +IMMUDB_PID=$! + +log "Waiting for ImmuDB on port ${IMMUDB_PORT}..." +elapsed=0 +until nc -z 127.0.0.1 "${IMMUDB_PORT}" 2>/dev/null; do + if [ "${elapsed}" -ge "${IMMUDB_READY_TIMEOUT}" ]; then + log "ERROR: ImmuDB did not start within ${IMMUDB_READY_TIMEOUT}s" + kill "${IMMUDB_PID}" 2>/dev/null || true + exit 1 + fi + sleep 1 + elapsed=$((elapsed + 1)) +done +log "ImmuDB ready (${elapsed}s)" + +_shutdown() { + log "Shutting down..." + kill "${IMMUDB_PID}" 2>/dev/null || true +} +trap _shutdown TERM INT + +# ── Step 2: Bootstrap sync (first run only) ────────────── +if ! /usr/local/bin/bootstrap_sync.sh; then + log "ERROR: Bootstrap failed — not starting jmdn." + kill "${IMMUDB_PID}" 2>/dev/null || true + exit 1 +fi + +# ── Step 3: Restore paths wiped by bootstrap ───────────── + +# peer.json — hardcoded relative path in config/constants.go: +# PeerFile = "./config/peer.json" resolved from WORKDIR (/opt/jmdn/data) +if [ ! -f /opt/jmdn/data/config/peer.json ]; then + log "peer.json missing — restoring from /etc/jmdn/peer.json" + mkdir -p /opt/jmdn/data/config + cp /etc/jmdn/peer.json /opt/jmdn/data/config/peer.json +fi + +# Required dirs (jmdn.yaml: security.cert_dir, cdc.dlq_path, thebe.kv_path) +mkdir -p \ + /opt/jmdn/data/certs + +# TLS certs — mirrors Scripts/setup_certs.sh +# Generated only if missing. For production mount real certs: +# -v /your/certs:/opt/jmdn/data/certs +if [ ! -f /opt/jmdn/data/certs/ca.crt ]; then + log "Generating self-signed TLS certs..." + CERT_DIR=/opt/jmdn/data/certs + + openssl req -x509 -newkey rsa:4096 -nodes -days 3650 \ + -keyout "$CERT_DIR/ca.key" -out "$CERT_DIR/ca.crt" \ + -subj "/C=US/O=JMDN/CN=JMDN Dev Root CA" 2>/dev/null + + for SVC in cli_admin block_ingest_grpc block_ingest_http did_service \ + explorer_api mempool_service admin_client explorer_client; do + openssl genrsa -out "$CERT_DIR/$SVC.key" 2048 2>/dev/null + openssl req -new -key "$CERT_DIR/$SVC.key" \ + -out "$CERT_DIR/$SVC.csr" \ + -subj "/C=US/O=JMDN/CN=$SVC" 2>/dev/null + openssl x509 -req -in "$CERT_DIR/$SVC.csr" \ + -CA "$CERT_DIR/ca.crt" -CAkey "$CERT_DIR/ca.key" -CAcreateserial \ + -out "$CERT_DIR/$SVC.crt" -days 365 -sha256 \ + -extfile <(printf "subjectAltName=DNS:localhost,IP:127.0.0.1") 2>/dev/null + rm -f "$CERT_DIR/$SVC.csr" + done + log "TLS certs generated." +fi + +# ── Step 4: Start JMDN ─────────────────────────────────── +log "Starting JMDN..." +exec /usr/local/bin/start_jmdn_wrapper.sh "$@"