diff --git a/beedb-gateway/src/main/java/com/memcache/gateway/api/ChaosController.java b/beedb-gateway/src/main/java/com/memcache/gateway/api/ChaosController.java new file mode 100644 index 0000000..e977586 --- /dev/null +++ b/beedb-gateway/src/main/java/com/memcache/gateway/api/ChaosController.java @@ -0,0 +1,41 @@ +package com.memcache.gateway.api; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import com.memcache.gateway.chaos.ChaosKillState; +import com.memcache.gateway.chaos.ChaosService; +import com.memcache.gateway.cluster.ClusterSnapshot; +import com.memcache.gateway.cluster.ClusterStatusService; + +@RestController +@RequestMapping ("/api/chaos") +public class ChaosController { + + private final ClusterStatusService clusterStatusService; + private final ChaosService chaosService; + public ChaosController(ClusterStatusService clusterStatusService, ChaosService chaosService){ + this.clusterStatusService = clusterStatusService; + this.chaosService = chaosService; + } + + @PostMapping ("/kill") + public ResponseEntity kill(){ + ClusterSnapshot clusterSnapshot = clusterStatusService.getClusterSnapshot(); + ChaosKillState state = chaosService.requestKill(clusterSnapshot.leaderId(), clusterSnapshot); + if(state == ChaosKillState.SUCCESS){ + return ResponseEntity.status(HttpStatus.ACCEPTED).body(state.toString()); + } else if (state == ChaosKillState.UNDER_COOLDOWN) { + long secondsUntilKill = chaosService.secondsUntilEligible(); + return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS) + .header("Retry-After", String.valueOf(secondsUntilKill)) + .body(state.toString()); + } else { + return ResponseEntity.status(HttpStatus.CONFLICT).body(state.toString()); + } + + } +} diff --git a/beedb-gateway/src/main/java/com/memcache/gateway/api/ClusterController.java b/beedb-gateway/src/main/java/com/memcache/gateway/api/ClusterController.java index 2a8635b..770925d 100644 --- a/beedb-gateway/src/main/java/com/memcache/gateway/api/ClusterController.java +++ b/beedb-gateway/src/main/java/com/memcache/gateway/api/ClusterController.java @@ -1,6 +1,8 @@ package com.memcache.gateway.api; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; @@ -8,34 +10,34 @@ import org.springframework.web.bind.annotation.RestController; import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; -import com.memcache.gateway.cluster.ClusterSnapshot; import com.memcache.gateway.cluster.ClusterStatusService; +import com.memcache.gateway.cluster.ClusterView; @RestController @RequestMapping("/api") public class ClusterController { private final ClusterStatusService clusterStatusService; + private static final Logger LOGGER = LoggerFactory.getLogger(ClusterController.class); public ClusterController(ClusterStatusService clusterStatusService){ this.clusterStatusService = clusterStatusService; } @GetMapping ("/cluster") - public ResponseEntity getClusterStatus(){ - // System.out.println("getting cluster status"); - ClusterSnapshot clusterSnapshot = clusterStatusService.getClusterSnapshot(); - // System.out.println("cluster snapshot is " + clusterSnapshot); - if(clusterSnapshot == null){ - return ResponseEntity.notFound().build(); - } - return ResponseEntity.ok(clusterSnapshot); + public ResponseEntity getClusterStatus(){ + ClusterView clusterView = clusterStatusService.getClusterView(); + return ResponseEntity.ok(clusterView); } @GetMapping (value = "/events", produces = MediaType.TEXT_EVENT_STREAM_VALUE) public SseEmitter getClusterStatusEvents(){ - System.out.println("getting cluster status events"); - SseEmitter emitter = clusterStatusService.registerEmitter(); - return emitter; + try{ + SseEmitter emitter = clusterStatusService.registerEmitter(); + return emitter; + }catch(Exception e){ + LOGGER.error("failed to register emitter", e); + return null; + } } } diff --git a/beedb-gateway/src/main/java/com/memcache/gateway/api/KvController.java b/beedb-gateway/src/main/java/com/memcache/gateway/api/KvController.java index 4d1989a..9a3f2e5 100644 --- a/beedb-gateway/src/main/java/com/memcache/gateway/api/KvController.java +++ b/beedb-gateway/src/main/java/com/memcache/gateway/api/KvController.java @@ -2,7 +2,6 @@ import jakarta.servlet.http.HttpServletRequest; -import java.net.http.HttpHeaders; import java.nio.charset.StandardCharsets; import java.util.List; @@ -24,6 +23,7 @@ import com.memcache.gateway.client.BeedbClient; import com.memcache.gateway.client.BeedbConfig; +import com.memcache.gateway.demo.DemoWriter; import com.memcache.gateway.kv.KeyRegistry; import com.memcache.gateway.kv.WriteRateLimiter; @@ -73,6 +73,19 @@ private ResponseEntity rateLimited(HttpServletRequest request) { + " writes per hour. This is a shared demo."); } + /** + * 403 for keys the demo writer owns,checked before + * the rate limit, so a refused request does not also cost the visitor a write. + */ + private ResponseEntity reservedKey(String key) { + if (!key.startsWith(DemoWriter.KEY_PREFIX)) { + return null; + } + LOGGER.debug("rejected write to reserved demo key {}", key); + return ResponseEntity.status(HttpStatus.FORBIDDEN) + .body("key '" + key + "' is reserved for the demo writer"); + } + /** * Every key the gateway has written and who owns it. * @@ -101,18 +114,16 @@ public ResponseEntity set(@PathVariable("key") String key, // Size check FIRST, before the rate limit: an oversized body is rejected // outright, so it should not also cost the client one of their ten writes. // - // This is the one check that is load-bearing for safety rather than fairness. - // The server reads a command into a single 4096-byte buffer and indexes the - // value out of it; a larger value throws BufferUnderflowException, and the - // ClosedChannelException that follows has been seen killing the node. Until - // the server read path is fixed, this is what stops a stranger with curl - // taking the cluster down. + // The server accepts up to 8192 bytes per command, line and value together, so + // its real ceiling depends on how long the key is int valueBytes = value == null ? 0 : value.getBytes(StandardCharsets.UTF_8).length; if(valueBytes > maxValueBytes){ LOGGER.warn("rejected {}-byte value for key {} (max {})", valueBytes, key, maxValueBytes); return ResponseEntity.status(HttpStatus.PAYLOAD_TOO_LARGE) .body("value is " + valueBytes + " bytes; the limit is " + maxValueBytes); } + ResponseEntity reserved = reservedKey(key); + if(reserved != null) return reserved; ResponseEntity limited = rateLimited(request); if(limited != null) return limited; // 403 rather than 404: the key exists, you simply are not its owner. The UI @@ -136,6 +147,8 @@ public ResponseEntity set(@PathVariable("key") String key, public ResponseEntity delete(@PathVariable("key") String key, @RequestHeader(value = SESSION_HEADER, required = false) String sessionId, HttpServletRequest request){ + ResponseEntity reserved = reservedKey(key); + if(reserved != null) return reserved; ResponseEntity limited = rateLimited(request); if(limited != null) return limited; if(!keyRegistry.mayWrite(key, sessionId)){ diff --git a/beedb-gateway/src/main/java/com/memcache/gateway/chaos/ChaosKillState.java b/beedb-gateway/src/main/java/com/memcache/gateway/chaos/ChaosKillState.java index 493e085..a6c32fd 100644 --- a/beedb-gateway/src/main/java/com/memcache/gateway/chaos/ChaosKillState.java +++ b/beedb-gateway/src/main/java/com/memcache/gateway/chaos/ChaosKillState.java @@ -3,10 +3,8 @@ public enum ChaosKillState { SUCCESS, CLUSTER_DEGRADED, - KILLING_IN_PROGRESS, FAILED, UNDER_COOLDOWN, - NOT_CORRECT_NODE_ID, KILLING_TIME_ANNOUNCED, ALREADY_KILLED } diff --git a/beedb-gateway/src/main/java/com/memcache/gateway/chaos/ChaosService.java b/beedb-gateway/src/main/java/com/memcache/gateway/chaos/ChaosService.java index 7a12d4f..5f04367 100644 --- a/beedb-gateway/src/main/java/com/memcache/gateway/chaos/ChaosService.java +++ b/beedb-gateway/src/main/java/com/memcache/gateway/chaos/ChaosService.java @@ -1,9 +1,11 @@ package com.memcache.gateway.chaos; import java.time.Clock; +import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.Map; import java.util.concurrent.Executor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +47,6 @@ public ChaosState getState(){ public ChaosState tick(ClusterSnapshot clusterSnapshot){ ChaosState currentState = chaosState.get(); - LOGGER.info("tick chaoservice state is "+currentState+" current tiem is "+clock.instant()); boolean isHealthyCluster = healthyCluster(clusterSnapshot); Instant currentInstant = clock.instant(); if(isHealthyCluster && currentState.phase().equals(ChaosPhase.IDLE)){ @@ -76,13 +77,11 @@ else if(nodeStatus.raftRole() == null && !currentState.observedDown()){ } } } - LOGGER.info("chaoservice final state is "+chaosState.get()); return chaosState.get(); } public ChaosKillState requestKill(String nodeId, ClusterSnapshot clusterSnapshot){ - LOGGER.info("requestKill called by "+Thread.currentThread().getName()+" time is "+clock.instant()); ChaosState currentState = chaosState.get(); Instant nextEligibleAt = currentState.nextEligibleAt(); Instant currentInstant = clock.instant(); @@ -161,4 +160,27 @@ public boolean healthyCluster(ClusterSnapshot clusterSnapshot){ } return true; } + + public ChaosView getChaosView(){ + ChaosState chaosState = getState(); + Instant now = clock.instant(); + Instant killAt = chaosState.killAt(); + Long secondsUntilKill = null; + if(chaosState.phase().equals(ChaosPhase.ANNOUNCED) && killAt != null){ + long msUntilKill = Duration.between(now, killAt).toMillis(); + secondsUntilKill = (long) Math.max(0, Math.ceilDiv(msUntilKill, 1000L)); + + } + return new ChaosView( + chaosState.phase(), + chaosState.targetNodeId(), + chaosState.nextAutoAt(), + secondsUntilKill + ); + } + + public long secondsUntilEligible(){ + long msUntilKill = Duration.between(clock.instant(), getState().nextEligibleAt()).toMillis(); + return Math.max(0, Math.ceilDiv(msUntilKill, 1000L)); + } } diff --git a/beedb-gateway/src/main/java/com/memcache/gateway/chaos/ChaosView.java b/beedb-gateway/src/main/java/com/memcache/gateway/chaos/ChaosView.java new file mode 100644 index 0000000..2fa1a19 --- /dev/null +++ b/beedb-gateway/src/main/java/com/memcache/gateway/chaos/ChaosView.java @@ -0,0 +1,11 @@ +package com.memcache.gateway.chaos; + +import java.time.Instant; + +// this will be shared with the client(on frontend) +public record ChaosView( + ChaosPhase phase, + String targetNodeId, + Instant nextAutoAt, + Long secondsUntilKill +) {} diff --git a/beedb-gateway/src/main/java/com/memcache/gateway/client/BeedbConfig.java b/beedb-gateway/src/main/java/com/memcache/gateway/client/BeedbConfig.java index 780e43b..1a23815 100644 --- a/beedb-gateway/src/main/java/com/memcache/gateway/client/BeedbConfig.java +++ b/beedb-gateway/src/main/java/com/memcache/gateway/client/BeedbConfig.java @@ -1,10 +1,20 @@ package com.memcache.gateway.client; +import java.nio.file.Path; import java.time.Clock; import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import com.memcache.gateway.chaos.ChaosService; import com.memcache.gateway.kv.KeyRegistry; import com.memcache.gateway.kv.WriteRateLimiter; +import com.memcache.gateway.supervisor.NodeSupervisor; +import com.memcache.gateway.supervisor.ProcessNodeSupervisor; + +import ch.qos.logback.core.pattern.parser.Node; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -60,6 +70,13 @@ public static class BeedbProperties { /** if we dont receive kill command, automatically trigger leader kill after these many seconds */ private int cooldownSeconds = 30; + private int chaosStartupGraceSeconds = 60; + + private String supervisorServerDir = "../beedb-server"; + + private int restartAfterSeconds = 15; + + public int getKeyTtlSeconds() { return keyTtlSeconds; } public void setKeyTtlSeconds(int keyTtlSeconds) { this.keyTtlSeconds = keyTtlSeconds; } public int getMaxTrackedKeys() { return maxTrackedKeys; } @@ -74,6 +91,12 @@ public static class BeedbProperties { public void setAutoKillSeconds(int autoKillSeconds) { this.autoKillSeconds = autoKillSeconds; } public int getCooldownSeconds() { return cooldownSeconds; } public void setCooldownSeconds(int cooldownSeconds) { this.cooldownSeconds = cooldownSeconds; } + public int getChaosStartupGraceSeconds() { return chaosStartupGraceSeconds; } + public void setChaosStartupGraceSeconds(int chaosStartupGraceSeconds) { this.chaosStartupGraceSeconds = chaosStartupGraceSeconds; } + public String getSupervisorServerDir() { return supervisorServerDir; } + public void setSupervisorServerDir(String supervisorServerDir) { this.supervisorServerDir = supervisorServerDir; } + public int getRestartAfterSeconds() { return restartAfterSeconds; } + public void setRestartAfterSeconds(int restartAfterSeconds) { this.restartAfterSeconds = restartAfterSeconds; } public Map getNodes() { return nodes; @@ -121,4 +144,31 @@ public BeedbClient beedbClient() { public Clock clock(){ return Clock.systemUTC(); } + + @Bean + public NodeSupervisor nodeSupervisor(){ + Set nodeIds = beedbProperties.getNodes().keySet(); + String supervisorServerDir = beedbProperties.getSupervisorServerDir(); + int restartAfterSeconds = beedbProperties.getRestartAfterSeconds(); + return new ProcessNodeSupervisor(Set.copyOf(nodeIds), Path.of(supervisorServerDir), restartAfterSeconds); + } + + @Bean + public ChaosService chaosService(NodeSupervisor nodeSupervisor, Clock clock){ + int announceLeadSeconds = beedbProperties.getAnnounceLeadSeconds(); + int autoKillSeconds = beedbProperties.getAutoKillSeconds(); + int cooldownSeconds = beedbProperties.getCooldownSeconds(); + int startingSeconds = beedbProperties.getChaosStartupGraceSeconds(); + Executor killExecutor = Executors.newFixedThreadPool(1 , killThreadFactory()); + return new ChaosService(nodeSupervisor, announceLeadSeconds, autoKillSeconds, cooldownSeconds, clock, startingSeconds, killExecutor); + } + + private ThreadFactory killThreadFactory(){ + ThreadFactory threadFactory = runnable -> { + Thread thread = new Thread(runnable, "chaos-kill"); + thread.setDaemon(true); + return thread; + }; + return threadFactory; + } } diff --git a/beedb-gateway/src/main/java/com/memcache/gateway/cluster/ClusterStatusService.java b/beedb-gateway/src/main/java/com/memcache/gateway/cluster/ClusterStatusService.java index 0c69f5b..5da60f8 100644 --- a/beedb-gateway/src/main/java/com/memcache/gateway/cluster/ClusterStatusService.java +++ b/beedb-gateway/src/main/java/com/memcache/gateway/cluster/ClusterStatusService.java @@ -12,7 +12,9 @@ import org.springframework.stereotype.Service; import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; +import com.memcache.gateway.chaos.ChaosService; import com.memcache.gateway.client.BeedbClient; +import com.memcache.gateway.demo.DemoWriter; @Service public class ClusterStatusService { @@ -21,8 +23,14 @@ public class ClusterStatusService { private final BeedbClient beedbClient; private final List emitters = new CopyOnWriteArrayList<>(); private static final Logger LOGGER = LoggerFactory.getLogger(ClusterStatusService.class); - public ClusterStatusService(BeedbClient beedbClient){ + private final ChaosService chaosService; + volatile ClusterView clusterView; + private final DemoWriter demoWriter; + public ClusterStatusService(BeedbClient beedbClient, ChaosService chaosService, DemoWriter demoWriter){ this.beedbClient = beedbClient; + this.chaosService = chaosService; + this.demoWriter = demoWriter; + this.clusterView = new ClusterView(null, chaosService.getChaosView() , demoWriter.view()); } public ClusterSnapshot getClusterSnapshot(){ if(clusterSnapshot == null){ @@ -45,7 +53,7 @@ public SseEmitter registerEmitter(){ }); emitters.add(emitter); try { - emitter.send(clusterSnapshot); + emitter.send(clusterView); } catch (Exception e) { logFailure(e, "failed to send cluster status"); } @@ -56,7 +64,7 @@ public void sendClusterStatus(){ LOGGER.debug("sending cluster status"); emitters.forEach(emitter -> { try { - emitter.send(clusterSnapshot); + emitter.send(clusterView); LOGGER.debug("sent cluster status"); } catch (Exception e) { emitters.remove(emitter); @@ -94,6 +102,12 @@ public void updateClusterStatus(){ public void scheduledUpdateClusterStatus(){ updateClusterStatus(); // sending updated cluster status to all connected clients + try{ + chaosService.tick(clusterSnapshot); + }catch(Exception e){ + logFailure(e, "failed to tick chaos service"); + } + clusterView = new ClusterView(clusterSnapshot, chaosService.getChaosView(), demoWriter.view()); LOGGER.debug("cluster status sent to all connected clients"); sendClusterStatus(); } @@ -107,4 +121,9 @@ private void logFailure(Exception e, String message){ LOGGER.error("{} ", message , e); } } + + public ClusterView getClusterView(){ + return new ClusterView(clusterSnapshot, chaosService.getChaosView() ,demoWriter.view()); + } + } diff --git a/beedb-gateway/src/main/java/com/memcache/gateway/cluster/ClusterView.java b/beedb-gateway/src/main/java/com/memcache/gateway/cluster/ClusterView.java new file mode 100644 index 0000000..fbbf549 --- /dev/null +++ b/beedb-gateway/src/main/java/com/memcache/gateway/cluster/ClusterView.java @@ -0,0 +1,10 @@ +package com.memcache.gateway.cluster; + +import com.memcache.gateway.chaos.ChaosView; +import com.memcache.gateway.demo.DemoRecord; + +public record ClusterView ( + ClusterSnapshot clusterSnapshot, + ChaosView chaosView, + DemoRecord demoRecord +){} diff --git a/beedb-gateway/src/main/java/com/memcache/gateway/demo/DemoRecord.java b/beedb-gateway/src/main/java/com/memcache/gateway/demo/DemoRecord.java new file mode 100644 index 0000000..f4fca49 --- /dev/null +++ b/beedb-gateway/src/main/java/com/memcache/gateway/demo/DemoRecord.java @@ -0,0 +1,7 @@ +package com.memcache.gateway.demo; + +public record DemoRecord ( + long ok, long failed , long unconfirmed , String lastKey +){ + +} diff --git a/beedb-gateway/src/main/java/com/memcache/gateway/demo/DemoWriter.java b/beedb-gateway/src/main/java/com/memcache/gateway/demo/DemoWriter.java new file mode 100644 index 0000000..73890f9 --- /dev/null +++ b/beedb-gateway/src/main/java/com/memcache/gateway/demo/DemoWriter.java @@ -0,0 +1,84 @@ +package com.memcache.gateway.demo; + +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import com.memcache.gateway.client.BeedbClient; +import com.memcache.gateway.client.beedbexception.BeedbTimeoutException; +import com.memcache.gateway.client.beedbexception.NoLeaderException; +import com.memcache.gateway.client.beedbexception.NodeUnreachableException; +import com.memcache.gateway.kv.KeyRegistry; + +@Service +public class DemoWriter { + + // every demo key starts with this, and KvController refuses visitor writes to it. + // both sides use this one constant, so the guard can never check a different prefix. + public static final String KEY_PREFIX = "demo:"; + // owner recorded in the KeyRegistry. Anyone can send this as their session header, + // which is why KvController blocks the prefix instead of relying on ownership. + private static final String OWNER = "demo-session"; + private static final int RING_SIZE = 20; + + private final AtomicLong ok = new AtomicLong(); + private final AtomicLong failed = new AtomicLong(); + // a timeout is not a failure: the leader may have committed the write after we + // stopped waiting, so it is counted apart from the writes the cluster refused. + private final AtomicLong unconfirmed = new AtomicLong(); + // only this method touches it, and fixedDelay never runs it twice at once. + private long attempts = 0; + // written here, read by the status tick on another thread. + private volatile String lastKey = null; + private final BeedbClient beedbClient; + private final KeyRegistry keyRegistry; + private static final Logger LOGGER = LoggerFactory.getLogger(DemoWriter.class); + + public DemoWriter(BeedbClient beedbClient, KeyRegistry keyRegistry){ + this.beedbClient = beedbClient; + this.keyRegistry = keyRegistry; + } + + @Scheduled (fixedDelay = 1000) + public void write(){ + long n = attempts++; + String key = KEY_PREFIX + (n % RING_SIZE); + String value = "write #" + (n + 1); + lastKey = key; + // Expected failures log at DEBUG: BeedbClient already logs one WARN line per failed + // request, and a failover would otherwise add a stack trace every second. + try{ + String response = beedbClient.set(key, value); + if("STORED".equals(response)){ + // Only after the cluster confirmed it, same rule as KvController. + keyRegistry.recordWrite(key, OWNER); + ok.incrementAndGet(); + }else if(response != null && response.startsWith("SERVER_ERROR") && response.contains("timeout")){ + // The server's own propose timeout: the entry may still commit. + unconfirmed.incrementAndGet(); + LOGGER.debug("demo write {} unconfirmed: {}", key, response); + }else{ + failed.incrementAndGet(); + LOGGER.debug("demo write {} failed: {}", key, response); + } + }catch(NoLeaderException | NodeUnreachableException e){ + failed.incrementAndGet(); + LOGGER.debug("demo write {} failed: {}", key, e.getMessage()); + }catch(BeedbTimeoutException e){ + unconfirmed.incrementAndGet(); + LOGGER.debug("demo write {} unconfirmed: {}", key, e.getMessage()); + }catch(Exception e){ + // Not a failover but a bug. Spring would keep the task running either way; + // catching it keeps the attempt counted and the trace in the log. + failed.incrementAndGet(); + LOGGER.warn("demo write {} failed unexpectedly", key, e); + } + } + + public DemoRecord view(){ + return new DemoRecord(ok.get(), failed.get(), unconfirmed.get(), lastKey); + } +} diff --git a/beedb-gateway/src/main/java/com/memcache/gateway/supervisor/ProcessNodeSupervisor.java b/beedb-gateway/src/main/java/com/memcache/gateway/supervisor/ProcessNodeSupervisor.java index e9a34a4..2d97782 100644 --- a/beedb-gateway/src/main/java/com/memcache/gateway/supervisor/ProcessNodeSupervisor.java +++ b/beedb-gateway/src/main/java/com/memcache/gateway/supervisor/ProcessNodeSupervisor.java @@ -52,7 +52,7 @@ public void kill(String nodeId) throws NodeCouldNotBeKilled { try { chaosRestore(nodeId); } catch (Exception e) { - LOGGER.error("exceptino in chaos-restorer thread for node id "+nodeId+" "+e.getMessage(),e); + LOGGER.error("failed to restore node {}", nodeId, e); } }, restartAfterSeconds, TimeUnit.SECONDS); @@ -69,13 +69,19 @@ public void kill(String nodeId) throws NodeCouldNotBeKilled { private void chaosRestore(String nodeId) throws IOException{ String cp = Files.readString(serverDirectory.resolve("target/classpath.txt")).trim(); + // stdout is discarded: logback's CONSOLE appender writes every log line there, and + // logs/.log already has them, rotated. stderr goes to a file instead it + // stays empty unless the JVM dies of something only stderr sees, such as an + // uncaught Error + Path err = serverDirectory.resolve("logs").resolve(nodeId + ".err"); + Files.createDirectories(err.getParent()); new ProcessBuilder(List.of( - "java", "-Dnode.id=" + nodeId, + "setsid", "java", "-Dnode.id=" + nodeId, "-cp", "target/classes:" + cp, "com.memcache.Server", nodeId + ".properties")) .directory(serverDirectory.toFile()) .redirectOutput(ProcessBuilder.Redirect.DISCARD) - .redirectError(ProcessBuilder.Redirect.DISCARD) + .redirectError(ProcessBuilder.Redirect.appendTo(err.toFile())) .start(); } diff --git a/beedb-gateway/src/main/resources/application.properties b/beedb-gateway/src/main/resources/application.properties index cffb7c7..4113253 100644 --- a/beedb-gateway/src/main/resources/application.properties +++ b/beedb-gateway/src/main/resources/application.properties @@ -34,3 +34,11 @@ beedb.cooldown-seconds=30 # if we dont receive kill command, automatically trigger leader kill after these many seconds beedb.auto-kill-seconds=240 + +spring.task.scheduling.pool.size=3 + +beedb.chaos-startup-grace-seconds=60 + +beedb.supervisor-server-dir=../beedb-server + +beedb.restart-after-seconds=15 diff --git a/beedb-gateway/src/test/java/com/memcache/gateway/cluster/TickProbe.java b/beedb-gateway/src/test/java/com/memcache/gateway/cluster/TickProbe.java index ea0b404..b4b19d6 100644 --- a/beedb-gateway/src/test/java/com/memcache/gateway/cluster/TickProbe.java +++ b/beedb-gateway/src/test/java/com/memcache/gateway/cluster/TickProbe.java @@ -1,13 +1,21 @@ package com.memcache.gateway.cluster; +import java.time.Clock; import java.util.*; + +import com.memcache.gateway.chaos.ChaosService; +import com.memcache.gateway.chaos.FakeNodeSupervisor; import com.memcache.gateway.client.BeedbClient; +import com.memcache.gateway.demo.DemoWriter; public class TickProbe { public static void main(String[] a) throws Exception { Map nodes = new LinkedHashMap<>(); nodes.put("node1","localhost:11211"); nodes.put("node2","localhost:11212"); nodes.put("node3","localhost:11213"); - ClusterStatusService svc = new ClusterStatusService(new BeedbClient(nodes, 86400)); + ClusterStatusService svc = new ClusterStatusService( + new BeedbClient(nodes, 86400), + new ChaosService(new FakeNodeSupervisor(), 10, 240, 30, Clock.systemUTC(), 10, Runnable::run), + new DemoWriter(new BeedbClient(nodes, 86400), null)); for (int i = 1; i <= 3; i++) { try { svc.updateClusterStatus();