Fix/dockerfile

.gitignore

@@ -44,6 +44,8 @@ __debug_bin
 *.pem
 *.crt
 vendor/
+*.duckdb
+storage/*
 
 # Internal team references
 docs/SONARQUBE_SETUP_GUIDE.md
@@ -53,3 +55,6 @@ internal/WAL/.tmp/*
 .code-review-graph/*
 .cursor/*
 test_results/
+
+dlq/*
+*.log

Dockerfile

@@ -2,14 +2,14 @@
 # JMDN (Jupiter MetaZK Decentralized Network) - Multi-stage Dockerfile
 # =============================================================================
 # Build:   docker build -t jmdn:latest .
-# Run:     docker run -d --name jmdn -p 8080:8080 -p 4001:4001 jmdn:latest
-# Config:  docker run -d -v /path/to/config.env:/etc/jmdn/config.env jmdn:latest
+# Run:     docker run -d --name jmdn -p 6090:6090 -p 6545:6545 jmdn:latest
+# Config:  docker run -d -v /path/to/jmdn.yaml:/etc/jmdn/jmdn.yaml jmdn:latest
 # =============================================================================
 
 # -----------------------------------------------------------------------------
 # Stage 1: Build
 # -----------------------------------------------------------------------------
-FROM golang:1.25-bookworm AS builder
+FROM golang:1.25.3-bookworm AS builder
 
 # Install build dependencies (CGO_ENABLED=1 requires gcc)
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -51,11 +51,25 @@ RUN GIT_COMMIT=${GIT_COMMIT:-$(git rev-parse --short HEAD 2>/dev/null || echo "u
 # -----------------------------------------------------------------------------
 FROM debian:bookworm-slim
 
-# Install runtime dependencies
+# Install runtime dependencies + Yggdrasil (required: network.yggdrasil: true in jmdn.yaml)
 RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
+    gnupg \
     libc6 \
+    netcat-openbsd \
+    wget \
+    openssl \
+    python3 \
+    gawk \
+    && mkdir -p /usr/local/apt-keys \
+    && gpg --fetch-keys https://neilalexander.s3.dualstack.eu-west-2.amazonaws.com/deb/key.txt \
+    && gpg --export 1C5162E133015D81A811239D1840CDAC6011C5EA \
+       | tee /usr/local/apt-keys/yggdrasil-keyring.gpg > /dev/null \
+    && echo 'deb [signed-by=/usr/local/apt-keys/yggdrasil-keyring.gpg] http://neilalexander.s3.dualstack.eu-west-2.amazonaws.com/deb/ debian yggdrasil' \
+       > /etc/apt/sources.list.d/yggdrasil.list \
+    && apt-get update && apt-get install -y --no-install-recommends \
+       yggdrasil \
     && rm -rf /var/lib/apt/lists/*
 
 # Install ImmuDB
@@ -66,35 +80,56 @@ RUN ARCH=$([ "$TARGETARCH" = "arm64" ] && echo "arm64" || echo "amd64") && \
     -o /usr/local/bin/immudb && \
     chmod +x /usr/local/bin/immudb
 
-# Create non-root user
-RUN groupadd -r jmdn && useradd -r -g jmdn -d /home/jmdn -s /bin/bash -m jmdn
-
-# Create required directories
-RUN mkdir -p /etc/jmdn /opt/jmdn/data /var/log/jmdn && \
-    chown -R jmdn:jmdn /opt/jmdn /var/log/jmdn /etc/jmdn
-
-# Copy binary and default config from builder
-COPY --from=builder /src/jmdn /usr/local/bin/jmdn
-COPY --from=builder /src/jmdn_default.yaml /etc/jmdn/jmdn_default.yaml
+# Create required directories (mirrors install_services.sh layout)
+RUN mkdir -p \
+    /etc/jmdn/certs \
+    /opt/jmdn/data/data \
+    /opt/jmdn/data/config \
+    /opt/jmdn/data/DB \
+    /var/log/jmdn
+
+# Copy binary and scripts from builder
+COPY --from=builder /src/jmdn                           /usr/local/bin/jmdn
+COPY --from=builder /src/Scripts/start_jmdn_wrapper.sh  /usr/local/bin/start_jmdn_wrapper.sh
+COPY --from=builder /src/Scripts/docker-entrypoint.sh   /usr/local/bin/docker-entrypoint.sh
+COPY --from=builder /src/Scripts/bootstrap_sync.sh      /usr/local/bin/bootstrap_sync.sh
+RUN chmod +x /usr/local/bin/start_jmdn_wrapper.sh \
+             /usr/local/bin/docker-entrypoint.sh \
+             /usr/local/bin/bootstrap_sync.sh
+
+# Copy default config as jmdn.yaml (mirrors setup_config.sh: cp jmdn_default.yaml → jmdn.yaml)
+COPY --from=builder /src/jmdn_default.yaml /etc/jmdn/jmdn.yaml
+
+# peer.json stored at two locations:
+#   /opt/jmdn/data/config/peer.json — runtime path (WORKDIR-relative, per config/constants.go)
+#   /etc/jmdn/peer.json             — fallback used by entrypoint after bootstrap wipes the volume
+COPY --from=builder /src/config/peer.json /opt/jmdn/data/config/peer.json
 COPY --from=builder /src/config/peer.json /etc/jmdn/peer.json
 
-# Expose common ports
-# 8080 - HTTP API / Explorer
-# 4001 - P2P / libp2p
-# 3323 - ImmuDB
-# 50051 - gRPC
-EXPOSE 8080 4001 3323 50051
-
-# Health check
-HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
-    CMD curl -f http://localhost:8080/health || exit 1
-
-# Data volume for ImmuDB persistence
+# Expose ports per jmdn.yaml (localhost-bound ports excluded)
+# 6090  - HTTP API / Explorer      (ports.api)
+# 16050 - Block generation         (ports.blockgen)
+# 16055 - Block propagation gRPC   (ports.blockgrpc)
+# 16052 - DID service              (ports.did)
+# 6545  - Facade / JSON-RPC        (ports.facade)
+# 6546  - WebSocket                (ports.ws)
+# ImmuDB (3322) is container-internal — not exposed
+EXPOSE 6090 16050 16055 16052 6545 6546
+
+# Health check against actual API port (ports.api: 6090)
+# start-period extended to 60s to allow bootstrap sync on first run
+HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
+    CMD curl -f http://localhost:6090/health || exit 1
+
+# Data volume — ImmuDB data, peer identity, certs, fastsync state
 VOLUME ["/opt/jmdn/data"]
 
-USER jmdn
-WORKDIR /home/jmdn
+# Run as root — bootstrap_sync.sh needs root for chown after snapshot extract
+# WORKDIR must match where jmdn resolves ./config/peer.json
+# (hardcoded in config/constants.go: PeerFile = "./config/peer.json")
+WORKDIR /opt/jmdn/data
 
-# Default entrypoint - override config with -v /your/config.env:/etc/jmdn/config.env
-ENTRYPOINT ["jmdn"]
-CMD ["-config", "/etc/jmdn/config.env"]
+# Startup order: ImmuDB → bootstrap sync → jmdn
+# Override config: -v /your/jmdn.yaml:/etc/jmdn/jmdn.yaml
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+CMD ["-config", "/etc/jmdn/jmdn.yaml"]

Scripts/bootstrap_sync.sh

@@ -0,0 +1,161 @@
+#!/usr/bin/env bash
+# bootstrap_sync.sh - JMDN Docker Bootstrap Sync
+#
+# Mirrors bootstrap.yml Ansible playbook for Docker deployments.
+# Runs ONCE — guarded by /opt/jmdn/data/.bootstrapped sentinel on the volume.
+#
+# Flow (matches ansible task order):
+#   1. List + download multipart files and checksums.md5 from GCS (public HTTP)
+#   2. Normalise checksums (strip GCS paths → local basenames) and verify
+#   3. Backup existing data → /opt/jmdn/backup/data_<timestamp>
+#   4. Clear immudb identity/state files
+#   5. cat parts* | tar -xzf - into sandbox
+#   6. Auto-discover real data root (parent dir of systemdb/)
+#   7. Move to /opt/jmdn/data, fix permissions, clean up
+#   8. Write sentinel
+#
+# Env overrides:
+#   GCS_BUCKET     GCS bucket name          (default: jmzk-releases)
+#   GCS_PREFIX     Path prefix in bucket    (default: jmdn_bootstrap_2306)
+#   PARTS_PREFIX   Part filename prefix     (default: data_backup_23062026.part)
+#   CHECKSUM_FILE  Checksum filename        (default: checksums.md5)
+#
+# To force a re-sync after wiping the volume:
+#   docker exec jmdn rm /opt/jmdn/data/.bootstrapped && docker restart jmdn
+
+set -euo pipefail
+
+# ── Config ───────────────────────────────────────────────
+GCS_BUCKET="${GCS_BUCKET:-jmzk-releases}"
+GCS_PREFIX="${GCS_PREFIX:-jmdn_bootstrap_2306}"
+PARTS_PREFIX="${PARTS_PREFIX:-data_backup_23062026.part}"
+CHECKSUM_FILE="${CHECKSUM_FILE:-checksums.md5}"
+
+BASE_DIR="/opt/jmdn"
+DATA_DIR="${BASE_DIR}/data"
+WORK_DIR="${BASE_DIR}/bootstrap_tmp"
+BACKUP_BASE="${BASE_DIR}/backup"
+IMMUDB_STATE_DIR="${BASE_DIR}/.immudb_state"
+SENTINEL="${DATA_DIR}/.bootstrapped"
+
+GCS_HTTP="https://storage.googleapis.com"
+GCS_API="${GCS_HTTP}/storage/v1/b/${GCS_BUCKET}/o"
+
+log() { echo "[bootstrap] $*"; }
+die() { echo "[bootstrap] ERROR: $*" >&2; exit 1; }
+
+# ── Guard ─────────────────────────────────────────────────
+if [ -f "$SENTINEL" ]; then
+    log "Sentinel found — skipping bootstrap."
+    exit 0
+fi
+
+log "First run detected — starting bootstrap sync."
+
+# ── Ensure required tools ─────────────────────────────────
+for tool in curl wget awk md5sum tar python3; do
+    command -v "$tool" >/dev/null 2>&1 || die "$tool is required but not found."
+done
+
+# ── Prepare work directory ───────────────────────────────
+log "Preparing work directory: $WORK_DIR"
+rm -rf "$WORK_DIR"
+mkdir -p "$WORK_DIR"
+
+# ── List parts from GCS (public HTTP, no gsutil needed) ──
+log "Listing parts from GCS: gs://${GCS_BUCKET}/${GCS_PREFIX}/${PARTS_PREFIX}*"
+PARTS_JSON=$(curl -sf "${GCS_API}?prefix=${GCS_PREFIX}/${PARTS_PREFIX}" \
+    || die "Failed to list objects from GCS. Check bucket name and network.")
+
+PART_NAMES=$(echo "$PARTS_JSON" | python3 -c "
+import json, sys
+data = json.load(sys.stdin)
+items = data.get('items', [])
+if not items:
+    raise SystemExit('No parts found matching prefix')
+for item in sorted(items, key=lambda x: x['name']):
+    print(item['name'])
+") || die "Failed to parse GCS listing."
+
+[ -z "$PART_NAMES" ] && die "No parts found for prefix ${GCS_PREFIX}/${PARTS_PREFIX}"
+
+log "Found parts:"
+echo "$PART_NAMES" | while read -r p; do log "  gs://${GCS_BUCKET}/$p"; done
+
+# ── Download parts ────────────────────────────────────────
+log "Downloading parts to $WORK_DIR..."
+echo "$PART_NAMES" | while read -r part_path; do
+    fname=$(basename "$part_path")
+    log "  $fname"
+    wget -q --show-progress -O "$WORK_DIR/$fname" "${GCS_HTTP}/${GCS_BUCKET}/${part_path}"
+done
+
+# ── Download checksums ────────────────────────────────────
+log "Downloading ${CHECKSUM_FILE}..."
+wget -q -O "$WORK_DIR/$CHECKSUM_FILE" \
+    "${GCS_HTTP}/${GCS_BUCKET}/${GCS_PREFIX}/${CHECKSUM_FILE}" \
+    || die "Failed to download ${CHECKSUM_FILE} from GCS."
+
+# ── Verify checksums ──────────────────────────────────────
+# Mirrors ansible:
+#   awk '{n=split($2,a,"/"); print $1 "  " a[n]}' checksums.md5 > checksums_local.md5
+#   md5sum -c checksums_local.md5
+log "Normalising and verifying checksums..."
+awk '{n=split($2,a,"/"); print $1 "  " a[n]}' \
+    "$WORK_DIR/$CHECKSUM_FILE" > "$WORK_DIR/checksums_local.md5"
+
+(cd "$WORK_DIR" && md5sum -c checksums_local.md5) \
+    || die "Checksum verification failed — aborting to prevent corrupt data."
+log "Checksums OK."
+
+# ── Backup existing data ──────────────────────────────────
+if [ -d "$DATA_DIR" ] && [ "$(ls -A "$DATA_DIR" 2>/dev/null)" ]; then
+    TIMESTAMP=$(date -u '+%Y%m%dT%H%M%S')
+    BACKUP_DIR="${BACKUP_BASE}/data_${TIMESTAMP}"
+    log "Backing up existing data → $BACKUP_DIR"
+    mkdir -p "$BACKUP_BASE"
+    mv "$DATA_DIR" "$BACKUP_DIR"
+fi
+
+# ── Clear immudb identity/state files ────────────────────
+log "Clearing immudb identity and state files..."
+rm -f "${IMMUDB_STATE_DIR}"/.identity-* 2>/dev/null || true
+rm -f "${IMMUDB_STATE_DIR}"/.state-*    2>/dev/null || true
+
+# ── Extract into sandbox ──────────────────────────────────
+# Mirrors ansible: cat parts* | tar -xzf - -C sandbox
+SANDBOX="${BASE_DIR}/data_tmp/sandbox"
+log "Extracting parts into sandbox: $SANDBOX"
+mkdir -p "$SANDBOX"
+
+# shellcheck disable=SC2086
+cat "$WORK_DIR"/${PARTS_PREFIX}* | tar -xzf - -C "$SANDBOX" \
+    || die "Extraction failed."
+
+# ── Auto-discover real data root ──────────────────────────
+# Mirrors ansible: find systemdb dir → its parent is the real data root
+log "Discovering data root (searching for systemdb/)..."
+REAL_DATA_DIR=$(find "$SANDBOX" -type d -name "systemdb" 2>/dev/null \
+    | head -n 1 | xargs -I{} dirname {})
+
+if [ -z "$REAL_DATA_DIR" ] || [ "$REAL_DATA_DIR" = "." ]; then
+    die "Could not find systemdb in extracted sandbox. Check snapshot integrity."
+fi
+log "Data root found: $REAL_DATA_DIR"
+
+# ── Move to final location ────────────────────────────────
+log "Moving data to $DATA_DIR..."
+mv "$REAL_DATA_DIR" "$DATA_DIR"
+rm -rf "${BASE_DIR}/data_tmp"
+
+# ── Fix permissions ───────────────────────────────────────
+log "Fixing permissions on $DATA_DIR..."
+chown -R root:root "$DATA_DIR"
+
+# ── Clean up work directory ───────────────────────────────
+log "Cleaning up work directory..."
+rm -rf "$WORK_DIR"
+
+# ── Write sentinel ────────────────────────────────────────
+touch "$SENTINEL"
+log "Bootstrap complete. Sentinel written → $SENTINEL"

Scripts/docker-entrypoint.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# docker-entrypoint.sh - Container startup orchestrator for JMDN
+#
+# Startup order (mirrors bootstrap.yml ansible play):
+#   1. ImmuDB          — start in background, wait for port 3322
+#   2. Bootstrap sync  — first run only (download snapshot, verify, extract)
+#   3. Restore paths   — peer.json, certs, required dirs (wiped by bootstrap)
+#   4. jmdn            — exec only if all prior steps succeed
+
+set -euo pipefail
+
+IMMUDB_PORT="${IMMUDB_PORT:-3322}"
+IMMUDB_DIR="${IMMUDB_DIR:-/opt/jmdn/data}"
+IMMUDB_READY_TIMEOUT="${IMMUDB_READY_TIMEOUT:-30}"
+
+log() { echo "[entrypoint] $*"; }
+
+# ── Step 1: Start ImmuDB ─────────────────────────────────
+log "Starting ImmuDB (dir: ${IMMUDB_DIR})..."
+immudb --dir "${IMMUDB_DIR}" &
+IMMUDB_PID=$!
+
+log "Waiting for ImmuDB on port ${IMMUDB_PORT}..."
+elapsed=0
+until nc -z 127.0.0.1 "${IMMUDB_PORT}" 2>/dev/null; do
+    if [ "${elapsed}" -ge "${IMMUDB_READY_TIMEOUT}" ]; then
+        log "ERROR: ImmuDB did not start within ${IMMUDB_READY_TIMEOUT}s"
+        kill "${IMMUDB_PID}" 2>/dev/null || true
+        exit 1
+    fi
+    sleep 1
+    elapsed=$((elapsed + 1))
+done
+log "ImmuDB ready (${elapsed}s)"
+
+_shutdown() {
+    log "Shutting down..."
+    kill "${IMMUDB_PID}" 2>/dev/null || true
+}
+trap _shutdown TERM INT
+
+# ── Step 2: Bootstrap sync (first run only) ──────────────
+if ! /usr/local/bin/bootstrap_sync.sh; then
+    log "ERROR: Bootstrap failed — not starting jmdn."
+    kill "${IMMUDB_PID}" 2>/dev/null || true
+    exit 1
+fi
+
+# ── Step 3: Restore paths wiped by bootstrap ─────────────
+
+# peer.json — hardcoded relative path in config/constants.go:
+#   PeerFile = "./config/peer.json" resolved from WORKDIR (/opt/jmdn/data)
+if [ ! -f /opt/jmdn/data/config/peer.json ]; then
+    log "peer.json missing — restoring from /etc/jmdn/peer.json"
+    mkdir -p /opt/jmdn/data/config
+    cp /etc/jmdn/peer.json /opt/jmdn/data/config/peer.json
+fi
+
+# Required dirs (jmdn.yaml: security.cert_dir, cdc.dlq_path, thebe.kv_path)
+mkdir -p \
+    /opt/jmdn/data/certs
+
+# TLS certs — mirrors Scripts/setup_certs.sh
+# Generated only if missing. For production mount real certs:
+#   -v /your/certs:/opt/jmdn/data/certs
+if [ ! -f /opt/jmdn/data/certs/ca.crt ]; then
+    log "Generating self-signed TLS certs..."
+    CERT_DIR=/opt/jmdn/data/certs
+
+    openssl req -x509 -newkey rsa:4096 -nodes -days 3650 \
+        -keyout "$CERT_DIR/ca.key" -out "$CERT_DIR/ca.crt" \
+        -subj "/C=US/O=JMDN/CN=JMDN Dev Root CA" 2>/dev/null
+
+    for SVC in cli_admin block_ingest_grpc block_ingest_http did_service \
+               explorer_api mempool_service admin_client explorer_client; do
+        openssl genrsa -out "$CERT_DIR/$SVC.key" 2048 2>/dev/null
+        openssl req -new -key "$CERT_DIR/$SVC.key" \
+            -out "$CERT_DIR/$SVC.csr" \
+            -subj "/C=US/O=JMDN/CN=$SVC" 2>/dev/null
+        openssl x509 -req -in "$CERT_DIR/$SVC.csr" \
+            -CA "$CERT_DIR/ca.crt" -CAkey "$CERT_DIR/ca.key" -CAcreateserial \
+            -out "$CERT_DIR/$SVC.crt" -days 365 -sha256 \
+            -extfile <(printf "subjectAltName=DNS:localhost,IP:127.0.0.1") 2>/dev/null
+        rm -f "$CERT_DIR/$SVC.csr"
+    done
+    log "TLS certs generated."
+fi
+
+# ── Step 4: Start JMDN ───────────────────────────────────
+log "Starting JMDN..."
+exec /usr/local/bin/start_jmdn_wrapper.sh "$@"