systemd node supervisor and makefile and small patches

beedb-gateway/pom.xml

@@ -15,6 +15,20 @@
   <artifactId>beedb-gateway</artifactId>
   <name>beedb-gateway</name>
 
+  <!-- beedb-server is plain Java and spring's bom should live here , it
+       should not inherit Spring's opinion about jackson, logback and the rest from parent -->
+  <dependencyManagement>
+    <dependencies>
+      <dependency>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>${spring.boot.version}</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
+
   <!-- No <version> elements below: the parent's <dependencyManagement> supplies them -->
   <dependencies>
     <dependency>

beedb-gateway/src/main/java/com/memcache/gateway/client/BeedbConfig.java

@@ -13,6 +13,7 @@
 import com.memcache.gateway.kv.WriteRateLimiter;
 import com.memcache.gateway.supervisor.NodeSupervisor;
 import com.memcache.gateway.supervisor.ProcessNodeSupervisor;
+import com.memcache.gateway.supervisor.SystemdNodeSupervisor;
 
 import ch.qos.logback.core.pattern.parser.Node;
 
@@ -76,6 +77,12 @@ public static class BeedbProperties {
 
         private int restartAfterSeconds = 15;
 
+        /** "process" on a laptop (nodes started by make), "systemd" on a server. */
+        private String supervisor = "process";
+
+        /** Only read when supervisor=systemd. %s is the node id. */
+        private String systemdUnitPattern = "beedb-node@%s.service";
+
 
         public int getKeyTtlSeconds() { return keyTtlSeconds; }
         public void setKeyTtlSeconds(int keyTtlSeconds) { this.keyTtlSeconds = keyTtlSeconds; }
@@ -97,6 +104,10 @@ public static class BeedbProperties {
         public void setSupervisorServerDir(String supervisorServerDir) { this.supervisorServerDir = supervisorServerDir; }
         public int getRestartAfterSeconds() { return restartAfterSeconds; }
         public void setRestartAfterSeconds(int restartAfterSeconds) { this.restartAfterSeconds = restartAfterSeconds; }
+        public String getSupervisor() { return supervisor; }
+        public void setSupervisor(String supervisor) { this.supervisor = supervisor; }
+        public String getSystemdUnitPattern() { return systemdUnitPattern; }
+        public void setSystemdUnitPattern(String systemdUnitPattern) { this.systemdUnitPattern = systemdUnitPattern; }
 
         public Map<String, String> getNodes() {
             return nodes;
@@ -150,6 +161,14 @@ public NodeSupervisor nodeSupervisor(){
         Set<String> nodeIds = beedbProperties.getNodes().keySet();
         String supervisorServerDir = beedbProperties.getSupervisorServerDir();
         int restartAfterSeconds = beedbProperties.getRestartAfterSeconds();
+        String supervisor = beedbProperties.getSupervisor();
+        if ("systemd".equalsIgnoreCase(supervisor)) {
+            return new SystemdNodeSupervisor(Set.copyOf(nodeIds),
+                beedbProperties.getSystemdUnitPattern(), restartAfterSeconds);
+        }
+        if (!"process".equalsIgnoreCase(supervisor)) {
+            throw new IllegalStateException("beedb.supervisor must be 'process' or 'systemd', got: " + supervisor);
+        }
         return new ProcessNodeSupervisor(Set.copyOf(nodeIds), Path.of(supervisorServerDir), restartAfterSeconds);
     }
 

beedb-gateway/src/main/java/com/memcache/gateway/supervisor/ProcessNodeSupervisor.java

@@ -1,6 +1,7 @@
 package com.memcache.gateway.supervisor;
 
 import java.io.IOException;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
@@ -37,7 +38,7 @@ public ProcessNodeSupervisor(Set<String> nodeIds, Path serverDirectory, int rest
     @Override
     public void kill(String nodeId) throws NodeCouldNotBeKilled {
         if(nodeId == null || !nodeIds.contains(nodeId)) throw new NodeCouldNotBeKilled("node "+nodeId+" is not in the set of nodes to kill");
-        String pattern = "Dnode\\.id="+nodeId+" .*com\\.memcache\\.Server";
+        String pattern = "Dnode\\.id="+nodeId+" -jar .*beedb-server.*\\.jar";
         try {
             Process process = new ProcessBuilder(List.of("pkill", "-f", pattern)).redirectErrorStream(true).start();
             if(!process.waitFor(PKILL_TIMEOUT_SECONDS, TimeUnit.SECONDS)){
@@ -68,7 +69,7 @@ public void kill(String nodeId) throws NodeCouldNotBeKilled {
     }
 
     private void chaosRestore(String nodeId) throws IOException{
-        String cp = Files.readString(serverDirectory.resolve("target/classpath.txt")).trim();
+        Path jar = serverJar();
         // stdout is discarded: logback's CONSOLE appender writes every log line there, and
         // logs/<node>.log already has them, rotated. stderr goes to a file instead it
         // stays empty unless the JVM dies of something only stderr sees, such as an
@@ -77,12 +78,27 @@ private void chaosRestore(String nodeId) throws IOException{
         Files.createDirectories(err.getParent());
         new ProcessBuilder(List.of(
         "setsid", "java", "-Dnode.id=" + nodeId,
-        "-cp", "target/classes:" + cp,
-        "com.memcache.Server", nodeId + ".properties"))
+        "-jar", jar.toString(), nodeId + ".properties"))
             .directory(serverDirectory.toFile())
             .redirectOutput(ProcessBuilder.Redirect.DISCARD)
             .redirectError(ProcessBuilder.Redirect.appendTo(err.toFile()))
             .start(); 
     }
+
+    /**
+     * the shaded jar in the server module's target/. found by glob rather than by a
+     * hard-coded version, so a version bump in the pom does not silently stop chaos
+     * from restoring nodes.
+     */
+    private Path serverJar() throws IOException {
+        Path target = serverDirectory.resolve("target");
+        try (DirectoryStream<Path> jars = Files.newDirectoryStream(target, "beedb-server-*.jar")) {
+            for (Path jar : jars) {
+                return jar;
+            }
+        }
+        throw new IOException("no beedb-server jar in " + target.toAbsolutePath()
+            + " -- run `make package` in the server module first");
+    }
 
 }

beedb-gateway/src/main/java/com/memcache/gateway/supervisor/SystemdNodeSupervisor.java

@@ -0,0 +1,97 @@
+package com.memcache.gateway.supervisor;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.memcache.gateway.chaos.chaosexception.NodeCouldNotBeKilled;
+
+/**
+ * when we have nodes on real server there we need systemd to stop and start them
+ * we can't user {@link ProcessNodeSupervisor} because pkill sends SIGTERM, the
+ * JVM exits 143, and the unit declares 143 a clean stop , so systemd would
+ * consider the node deliberately stopped, while this process later started a
+ * replacement outside systemd, in the wrong working directory, that systemd
+ * knows nothing about. 
+ *
+ * going through systemctl keeps one owner of the process. `stop` is an explicit
+ * request, so `Restart=on-failure` does not fire, the node stays down until we
+ * start it again, which is exactly what a chaos demo wants. 
+ */
+public class SystemdNodeSupervisor implements NodeSupervisor {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(SystemdNodeSupervisor.class);
+    private static final int SYSTEMCTL_TIMEOUT_SECONDS = 10;
+
+    private final Set<String> nodeIds;
+    private final String unitPattern;
+    private final int restartAfterSeconds;
+
+    private final ThreadFactory chaosRestoreThreadFactory = runnable -> {
+        Thread thread = new Thread(runnable, "chaos-restore");
+        thread.setDaemon(true);
+        return thread;
+    };
+    private final ScheduledExecutorService chaosRestorer =
+        Executors.newScheduledThreadPool(1, chaosRestoreThreadFactory);
+
+    public SystemdNodeSupervisor(Set<String> nodeIds, String unitPattern, int restartAfterSeconds) {
+        this.nodeIds = nodeIds;
+        this.unitPattern = unitPattern;
+        this.restartAfterSeconds = restartAfterSeconds;
+    }
+
+    @Override
+    public void kill(String nodeId) throws NodeCouldNotBeKilled {
+        // the node id becomes part of a command run as root, so it is checked
+        // against the configured set and never taken on trust. The sudoers rule
+        // names each unit in full as a second line of defence.
+        if (nodeId == null || !nodeIds.contains(nodeId)) {
+            throw new NodeCouldNotBeKilled("node " + nodeId + " is not in the set of nodes to kill");
+        }
+        systemctl("stop", nodeId);
+        LOGGER.info("chaos stopped {}, restoring in {}s", nodeId, restartAfterSeconds);
+
+        // the timer lives in this JVM: if the gateway restarts inside the
+        // window, the node stays stopped until someone starts it
+        chaosRestorer.schedule(() -> {
+            try {
+                systemctl("start", nodeId);
+                LOGGER.info("chaos restored {}", nodeId);
+            } catch (Exception e) {
+                LOGGER.error("failed to restore node {}", nodeId, e);
+            }
+        }, restartAfterSeconds, TimeUnit.SECONDS);
+    }
+
+    private void systemctl(String verb, String nodeId) throws NodeCouldNotBeKilled {
+        String unit = String.format(unitPattern, nodeId);
+        List<String> command = List.of("sudo", "-n", "systemctl", verb, unit);
+        try {
+            Process process = new ProcessBuilder(command).redirectErrorStream(true).start();
+            if (!process.waitFor(SYSTEMCTL_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
+                process.destroyForcibly();
+                throw new NodeCouldNotBeKilled("systemctl " + verb + " " + unit + " timed out");
+            }
+            int exitCode = process.exitValue();
+            if (exitCode != 0) {
+                // the sudoers rule is missing, or it does not
+                // cover this unit name. Both are silent until a kill is tried.
+                throw new NodeCouldNotBeKilled("systemctl " + verb + " " + unit
+                    + " exited " + exitCode + ": " + new String(process.getInputStream().readAllBytes()));
+            }
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new NodeCouldNotBeKilled("systemctl " + verb + " " + unit + " was interrupted", e);
+        } catch (IOException e) {
+            throw new NodeCouldNotBeKilled("systemctl " + verb + " " + unit + " failed: " + e.getMessage(), e);
+        }
+    }
+}

beedb-gateway/src/main/resources/application.properties

@@ -42,3 +42,9 @@ beedb.chaos-startup-grace-seconds=60
 beedb.supervisor-server-dir=../beedb-server
 
 beedb.restart-after-seconds=15
+
+#   process = pkill the JVM this gateway started (in case of laptop, `make up`)
+#   systemd = sudo systemctl stop/start the unit (in case of server)
+# The server value belongs in /etc/beedb/gateway.properties
+beedb.supervisor=process
+beedb.systemd-unit-pattern=beedb-node@%s.service

beedb-server/Makefile

@@ -0,0 +1,148 @@
+.PHONY: compile test classpath package run1 run2 run3 up down restart status tail logclean clean
+
+MVN      := mvn -q
+LOG_DIR  := logs
+CP_FILE  := target/classpath.txt
+MAIN     := com.memcache.Server
+# the same self-contained jar that gets deployed: running it here means local
+# and server start the node exactly the same way, dependencies included.
+JAR      := target/beedb-server-1.0-SNAPSHOT.jar
+
+# logback.xml reads -Dnode.id to name its rolling file appender, so each node
+# writes its own logs/<nodeId>.log with rotation and a totalSizeCap.
+# stdout is sent to /dev/null: logback owns the file now, and a shell redirect
+# writing to the same path would fight the rolling policy.
+JAVA_RUN = java -Dnode.id=$(1) -jar $(JAR) $(1).properties
+
+# wipe one node's logs so every start writes into an empty file: the live file,
+# the rotated .gz archives, and any .tmp left behind by an interrupted roll.
+CLEAN_LOGS = rm -f $(LOG_DIR)/$(1).log $(LOG_DIR)/$(1)-metrics.log \
+                   $(LOG_DIR)/$(1).*.log.gz $(LOG_DIR)/$(1)-metrics.*.log.gz \
+                   $(LOG_DIR)/$(1).*.tmp $(LOG_DIR)/$(1)-metrics.*.tmp
+
+# find a node's live JVMs by command line rather than by pid file. The pid file
+# is not authoritative: `up` overwrites it, so a node left running by an earlier
+# up/run* becomes untrackable and survives every restart still holding its
+# ports. Matching -Dnode.id=<node> finds those orphans too, and cannot be fooled
+# by a recycled pid the way a stale pid file can.
+NODE_PIDS = pgrep -f "Dnode\.id=$(1) -jar .*beedb-server.*\.jar"
+
+# both ports a node binds, read from its .properties so this can never drift
+# from the real config.
+NODE_PORTS = grep -hE '^(clientPort|raftPort)=' $(1).properties | cut -d= -f2 | tr -d ' \r'
+
+# ports currently in LISTEN state, one per line, IPv4/IPv6 prefix stripped.
+LISTENING = ss -ltnH 2>/dev/null | awk '{print $$4}' | sed 's/.*://'
+
+compile:
+	$(MVN) clean compile
+
+test:
+	$(MVN) test
+
+# resolve the dependency classpath once so `up`/`run*` don't pay Maven's
+# per-invocation bootstrap cost for every node.
+classpath: compile
+	$(MVN) dependency:build-classpath -Dmdep.outputFile=$(CP_FILE)
+
+# build the shaded jar. tests are skipped here on purpose: this is the inner
+# loop, `mvn verify` is where tests belong.
+package:
+	$(MVN) -DskipTests package
+
+run1: package
+	@mkdir -p $(LOG_DIR)
+	@$(call CLEAN_LOGS,node1)
+	$(call JAVA_RUN,node1)
+
+run2: package
+	@mkdir -p $(LOG_DIR)
+	@$(call CLEAN_LOGS,node2)
+	$(call JAVA_RUN,node2)
+
+run3: package
+	@mkdir -p $(LOG_DIR)
+	@$(call CLEAN_LOGS,node3)
+	$(call JAVA_RUN,node3)
+
+# start all 3 nodes as background processes. logs are written by logback
+# (logs/node<N>.log); only pids land here directly.
+up: package
+	@mkdir -p $(LOG_DIR)
+	@busy=""; \
+	listening=$$($(LISTENING)); \
+	for n in 1 2 3; do \
+		for p in $$($(call NODE_PORTS,node$$n)); do \
+			if echo "$$listening" | grep -qx "$$p"; then \
+				owner=$$(ss -lptnH "sport = :$$p" 2>/dev/null | grep -o 'pid=[0-9]*' | head -1); \
+				echo "node$$n: port $$p already bound ($$owner)"; \
+				busy="$$busy node$$n"; \
+			fi; \
+		done; \
+	done; \
+	if [ -n "$$busy" ]; then \
+		echo "refusing to start -- run 'make down' first"; \
+		exit 1; \
+	fi
+	@for n in 1 2 3; do \
+		$(call CLEAN_LOGS,node$$n); \
+		nohup java -Dnode.id=node$$n -jar $(JAR) node$$n.properties \
+			> /dev/null 2>&1 & echo $$! > $(LOG_DIR)/node$$n.pid; \
+	done
+	@sleep 1
+	@echo "Cluster up. node1=11211 node2=11212 node3=11213 (client ports)"
+	@$(MAKE) --no-print-directory status
+
+down:
+	@for n in 1 2 3; do \
+		rm -f $(LOG_DIR)/node$$n.pid; \
+		pids=$$($(call NODE_PIDS,node$$n) || true); \
+		if [ -z "$$pids" ]; then echo "node$$n: nothing running"; continue; fi; \
+		for pid in $$pids; do kill $$pid 2>/dev/null || true; done; \
+		i=0; \
+		while [ $$i -lt 20 ]; do \
+			alive=$$($(call NODE_PIDS,node$$n) || true); \
+			[ -z "$$alive" ] && break; \
+			sleep 0.5; i=$$((i+1)); \
+		done; \
+		for pid in $$($(call NODE_PIDS,node$$n) || true); do \
+			echo "node$$n: pid $$pid ignored SIGTERM, sending SIGKILL"; \
+			kill -9 $$pid 2>/dev/null || true; \
+		done; \
+		echo "stopped node$$n (pid $$(echo $$pids))"; \
+	done
+
+restart: down up
+
+status:
+	@for n in 1 2 3; do \
+		pids=$$($(call NODE_PIDS,node$$n) || true); \
+		if [ -z "$$pids" ]; then echo "node$$n: DOWN"; else \
+			echo "node$$n: RUNNING (pid $$(echo $$pids))"; \
+		fi; \
+	done
+
+# Wipe all three nodes' logs without starting anything.
+logclean:
+	@mkdir -p $(LOG_DIR)
+	@for n in 1 2 3; do $(call CLEAN_LOGS,node$$n); done
+	@echo "logs cleared for node1 node2 node3"
+
+tail:
+	tail -f $(LOG_DIR)/node1.log $(LOG_DIR)/node2.log $(LOG_DIR)/node3.log
+
+# dropping rotated archives too, not just the live logs.
+clean: down
+	$(MVN) clean
+	rm -rf $(LOG_DIR)
+
+# run the framing probe against a live cluster.
+#
+# the probe is a test class, so it still runs from target/test-classes with the
+# dependency classpath -- only the nodes moved to the jar.
+#
+#   make up && make probe
+probe:
+	$(MVN) test-compile
+	$(MVN) dependency:build-classpath -Dmdep.outputFile=$(CP_FILE)
+	java -cp "target/classes:target/test-classes:$$(cat $(CP_FILE))" com.memcache.FramingProbe