From dd495464901fcf12b6b931d0a2fdbd0cc0914753 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 14:15:38 +0100 Subject: [PATCH 1/6] fix(chaos): a member that does not lead injects no fault, and each fault carries its term and deadline J4b of the resilient-quorum plan. A paused leader that resumed after another member took over injected its plan's partition anyway (s09 of the failure study): leadership was checked when the plan started, not before the fault went in. Both providers now call Quorum.requireWrites() right after the fault's delay, before any write, and refuse with the leader's name. The NetworkPolicy, the ChaosEngine and the scaled StatefulSet or KafkaNodePool carry kates.io/term (the writer's Ratis term) and kates.io/deadline (now plus kates.engine.max-duration-ms), which the reconciler reads next. A scale-down writes them each time, since an earlier plan's snapshot stays on the object with an older term. --- .../com/bmscomp/kates/chaos/FaultMarks.java | 23 +++ .../kates/chaos/KubernetesChaosProvider.java | 51 ++++-- .../kates/chaos/LitmusChaosProvider.java | 12 ++ .../kates/chaos/NodePoolScaleDown.java | 17 +- .../java/com/bmscomp/kates/quorum/Quorum.java | 17 ++ .../chaos/KubernetesChaosProviderTest.java | 159 ++++++++++++++---- .../kates/chaos/LitmusChaosProviderTest.java | 41 +++++ .../com/bmscomp/kates/quorum/QuorumTest.java | 15 ++ 8 files changed, 285 insertions(+), 50 deletions(-) create mode 100644 kates/src/main/java/com/bmscomp/kates/chaos/FaultMarks.java diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/FaultMarks.java b/kates/src/main/java/com/bmscomp/kates/chaos/FaultMarks.java new file mode 100644 index 000000000..d100990ab --- /dev/null +++ b/kates/src/main/java/com/bmscomp/kates/chaos/FaultMarks.java @@ -0,0 +1,23 @@ +package com.bmscomp.kates.chaos; + +import java.time.Instant; +import java.util.Map; + +/** + * What every object a fault creates or changes carries, so that a later leader can tell a fault + * of an older term, or one past its deadline, from the faults it runs itself. + */ +public final class FaultMarks { + + /** The Ratis term of the leader that wrote the object; {@code 0} with no quorum. */ + public static final String TERM = "kates.io/term"; + + /** The instant after which any leader undoes the fault, whoever made it. */ + public static final String DEADLINE = "kates.io/deadline"; + + private FaultMarks() {} + + public static Map of(long term, Instant deadline) { + return Map.of(TERM, Long.toString(term), DEADLINE, deadline.toString()); + } +} diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java b/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java index b9c9996d9..4e57fcfb6 100644 --- a/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java +++ b/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java @@ -96,6 +96,22 @@ public IncompleteScaleDownException(String message) { @Inject com.bmscomp.kates.engine.KatesExecutor executor; + // Built without config, as the tests build it, a Quorum is alone and always leads. + @Inject + com.bmscomp.kates.quorum.Quorum quorum = new com.bmscomp.kates.quorum.Quorum(); + + @org.eclipse.microprofile.config.inject.ConfigProperty(name = "kates.engine.max-duration-ms", defaultValue = "7200000") + long maxDurationMs = 7_200_000; + + /** + * The marks a fault written now carries: this leader's term, and a deadline at the engine's run + * bound. shortcut: a fault the live leader leaves after a crashed plan thread lives up to that + * bound (2 h); upgrade to the plan's own bound if that is seen. + */ + Map marks() { + return FaultMarks.of(quorum.term(), Instant.now().plusMillis(maxDurationMs)); + } + @Override public String name() { return "kubernetes"; @@ -156,6 +172,9 @@ public CompletableFuture triggerFault(FaultSpec spec, LongConsumer start = Instant.now(); startNanos = System.nanoTime(); } + // Confirmed after the delay, right before any write: a member that stopped + // leading since the plan started writes nothing (J4 F1). + quorum.requireWrites(); // The fault goes in from here, once its delay is over. onInject.accept(startNanos); applyDisruption(spec, engineName); @@ -282,6 +301,7 @@ private void isolatePod(FaultSpec spec, String podName) { .withName(policyName) .withNamespace(spec.targetNamespace()) .addToLabels("managed-by", "kates") + .addToAnnotations(marks()) .endMetadata() .withNewSpec() .withNewPodSelector() @@ -452,7 +472,7 @@ private void executeScaleDown(FaultSpec spec) throws InterruptedException { } if (!targets.pools().isEmpty()) { - new NodePoolScaleDown(client, scalePollIntervalMs).run(spec, targets.pools()); + new NodePoolScaleDown(client, scalePollIntervalMs, marks()).run(spec, targets.pools()); } int scaled = 0; for (String name : targets.statefulSets()) { @@ -488,19 +508,22 @@ private boolean scaleDownStatefulSet(String namespace, String name) { // already-reduced value, so only stamp when absent. Map annotations = ss.getMetadata().getAnnotations(); boolean alreadySnapshotted = annotations != null && annotations.containsKey(ORIGINAL_REPLICAS_ANNOTATION); - if (!alreadySnapshotted) { - String scaledDownAt = String.valueOf(System.currentTimeMillis()); - client.apps() - .statefulSets() - .inNamespace(namespace) - .withName(name) - .edit(s -> new StatefulSetBuilder(s) - .editMetadata() - .addToAnnotations(ORIGINAL_REPLICAS_ANNOTATION, String.valueOf(current)) - .addToAnnotations(SCALED_DOWN_AT_ANNOTATION, scaledDownAt) - .endMetadata() - .build()); - } + // The marks are written every time: a snapshot an earlier plan left carries an older term, + // which the reconciler would take for a dead leader's fault and undo under this one. + Map marks = marks(); + String scaledDownAt = String.valueOf(System.currentTimeMillis()); + client.apps() + .statefulSets() + .inNamespace(namespace) + .withName(name) + .edit(s -> { + var metadata = new StatefulSetBuilder(s).editMetadata().addToAnnotations(marks); + if (!alreadySnapshotted) { + metadata.addToAnnotations(ORIGINAL_REPLICAS_ANNOTATION, String.valueOf(current)) + .addToAnnotations(SCALED_DOWN_AT_ANNOTATION, scaledDownAt); + } + return metadata.endMetadata().build(); + }); LOG.info("SCALE_DOWN: StatefulSet " + name + " from " + current + " → " + (current - 1)); client.apps().statefulSets().inNamespace(namespace).withName(name).scale(current - 1); return true; diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java b/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java index 2e3cd08d5..8928458d5 100644 --- a/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java +++ b/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java @@ -77,6 +77,13 @@ public String name() { return "litmus-crd"; } + // Built without config, as the tests build it, a Quorum is alone and always leads. + @Inject + com.bmscomp.kates.quorum.Quorum quorum = new com.bmscomp.kates.quorum.Quorum(); + + @org.eclipse.microprofile.config.inject.ConfigProperty(name = "kates.engine.max-duration-ms", defaultValue = "7200000") + long maxDurationMs = 7_200_000; + @Override public CompletableFuture triggerFault(FaultSpec spec) { return triggerFault(spec, injectedAt -> {}); @@ -117,6 +124,11 @@ private CompletableFuture inject(FaultSpec spec, LongConsumer onIn ChaosEngine engine = buildChaosEngine(spec, engineName, experimentName); String resultName = engineName + "-" + experimentName; + // Confirmed right before the engine exists: a member that stopped leading writes nothing. + quorum.requireWrites(); + engine.getMetadata() + .setAnnotations(FaultMarks.of(quorum.term(), Instant.now().plusMillis(maxDurationMs))); + // Once the pods are picked, before the engine that injects the fault exists. onInject.accept(startNanos); diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/NodePoolScaleDown.java b/kates/src/main/java/com/bmscomp/kates/chaos/NodePoolScaleDown.java index 63235129f..9adbdbdba 100644 --- a/kates/src/main/java/com/bmscomp/kates/chaos/NodePoolScaleDown.java +++ b/kates/src/main/java/com/bmscomp/kates/chaos/NodePoolScaleDown.java @@ -69,9 +69,12 @@ private enum Progress { REMOVED } - NodePoolScaleDown(KubernetesClient client, long pollIntervalMs) { + private final Map marks; + + NodePoolScaleDown(KubernetesClient client, long pollIntervalMs, Map marks) { this.client = client; this.pollIntervalMs = pollIntervalMs; + this.marks = marks; } /** @@ -176,15 +179,17 @@ private void lower(String namespace, Pool pool, GenericKubernetesResource before throw new IllegalStateException( "KafkaNodePool " + pool.name() + " changed size while SCALE_DOWN was lowering it"); } + Map a = new HashMap<>( + p.getMetadata().getAnnotations() != null + ? p.getMetadata().getAnnotations() + : Map.of()); + // Every time, as for a StatefulSet: an earlier plan's snapshot carries an older term. + a.putAll(marks); if (snapshot) { - Map a = new HashMap<>( - p.getMetadata().getAnnotations() != null - ? p.getMetadata().getAnnotations() - : Map.of()); a.put(KubernetesChaosProvider.ORIGINAL_REPLICAS_ANNOTATION, String.valueOf(replicas)); a.put(KubernetesChaosProvider.SCALED_DOWN_AT_ANNOTATION, scaledDownAt); - p.getMetadata().setAnnotations(a); } + p.getMetadata().setAnnotations(a); spec(p).put("replicas", replicas - 1); return p; }); diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java index cbeb30f83..6570a1b5b 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java @@ -179,6 +179,23 @@ QuorumMember member() { return member; } + /** The term this member is in, as it last heard; {@code 0} with no quorum or while it does not serve. */ + public long term() { + QuorumMember current = member; + return current == null || !current.serving() ? 0 : current.term(); + } + + /** + * Throws unless this member may write now, as {@link #acceptsWrites()} decides: called right + * before a fault goes into Kubernetes, so a member that stopped leading since its plan started + * injects nothing. + */ + public void requireWrites() { + if (!acceptsWrites()) { + throw new com.bmscomp.kates.engine.NotLeaderException(leaderId()); + } + } + /** The leader this member knows of, or null with no quorum or during an election. */ public String leaderId() { QuorumMember current = member; diff --git a/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java b/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java index b9f010e30..f09d9095c 100644 --- a/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java +++ b/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java @@ -343,36 +343,7 @@ void stressTakesTheContainersUserWhereOnlyTheContainerSetsIt() throws Exception @Test void testScaleDownSnapshotsOriginalReplicas() throws ExecutionException, InterruptedException, TimeoutException { - // A Kafka not run by Strimzi: pods of a StatefulSet. - StatefulSet ss = new StatefulSetBuilder() - .withNewMetadata() - .withName("kafka") - .withNamespace("default") - .addToLabels("app", "kafka") - .endMetadata() - .withNewSpec() - .withReplicas(3) - .endSpec() - .build(); - client.apps().statefulSets().inNamespace("default").resource(ss).create(); - for (int i = 0; i < 3; i++) { - client.pods() - .inNamespace("default") - .resource(new PodBuilder() - .withNewMetadata() - .withName("kafka-" + i) - .withNamespace("default") - .addToLabels("app", "kafka") - .addNewOwnerReference() - .withApiVersion("apps/v1") - .withKind("StatefulSet") - .withName("kafka") - .withUid("sts-uid") - .endOwnerReference() - .endMetadata() - .build()) - .create(); - } + createKafkaStatefulSet(); FaultSpec spec = FaultSpec.builder("scale-down") .targetNamespace("default") @@ -990,4 +961,132 @@ void testCleanupNetworkPolicies() { void testIsAvailable() { assertTrue(provider.isAvailable()); } + + /** A Kafka not run by Strimzi: three pods of the StatefulSet {@code kafka} in {@code default}. */ + private void createKafkaStatefulSet() { + StatefulSet ss = new StatefulSetBuilder() + .withNewMetadata() + .withName("kafka") + .withNamespace("default") + .addToLabels("app", "kafka") + .endMetadata() + .withNewSpec() + .withReplicas(3) + .endSpec() + .build(); + client.apps().statefulSets().inNamespace("default").resource(ss).create(); + for (int i = 0; i < 3; i++) { + client.pods() + .inNamespace("default") + .resource(new PodBuilder() + .withNewMetadata() + .withName("kafka-" + i) + .withNamespace("default") + .addToLabels("app", "kafka") + .addNewOwnerReference() + .withApiVersion("apps/v1") + .withKind("StatefulSet") + .withName("kafka") + .withUid("sts-uid") + .endOwnerReference() + .endMetadata() + .build()) + .create(); + } + + } + + private static com.bmscomp.kates.quorum.Quorum quorumAt(long term, boolean leads) { + com.bmscomp.kates.quorum.Quorum quorum = org.mockito.Mockito.mock(com.bmscomp.kates.quorum.Quorum.class); + org.mockito.Mockito.when(quorum.term()).thenReturn(term); + if (!leads) { + org.mockito.Mockito.doThrow(new com.bmscomp.kates.engine.NotLeaderException("m1")) + .when(quorum) + .requireWrites(); + } + return quorum; + } + + private static void assertMarked(java.util.Map annotations, String term) { + assertEquals(term, annotations.get(FaultMarks.TERM), "the term of the leader that wrote it"); + java.time.Instant deadline = java.time.Instant.parse(annotations.get(FaultMarks.DEADLINE)); + java.time.Instant expected = java.time.Instant.now().plusMillis(7_200_000); + assertTrue( + Math.abs(java.time.Duration.between(expected, deadline).toSeconds()) < 5, + "the engine's run bound from now: " + deadline); + } + + @Test + void aMemberThatDoesNotLeadInjectsNothing() throws Exception { + createZonedBrokers(); + createKafkaStatefulSet(); + provider.quorum = quorumAt(3, false); + requests(); + for (DisruptionType type : List.of( + DisruptionType.POD_KILL, + DisruptionType.NETWORK_PARTITION, + DisruptionType.SCALE_DOWN, + DisruptionType.ROLLING_RESTART, + DisruptionType.CPU_STRESS)) { + FaultSpec spec = type == DisruptionType.SCALE_DOWN + ? FaultSpec.builder("x").targetNamespace("default").targetLabel("app=kafka").disruptionType(type).build() + : FaultSpec.builder("x").targetPod("krafter-brokers-0").disruptionType(type).chaosDurationSec(1).build(); + + ChaosOutcome outcome = provider.triggerFault(spec).get(5, TimeUnit.SECONDS); + + assertFalse(outcome.isPass(), type + " went in"); + assertTrue(outcome.failureReason().contains("does not lead"), type + ": " + outcome.failureReason()); + } + List writes = requests().stream() + .filter(r -> !r.startsWith("GET ")) + .toList(); + assertEquals(List.of(), writes, "a member that does not lead writes nothing"); + } + + @Test + void aPartitionCarriesItsTermAndDeadline() throws Exception { + createZonedBrokers(); + provider.quorum = quorumAt(7, true); + FaultSpec spec = FaultSpec.builder("split") + .targetPod("krafter-brokers-0") + .disruptionType(DisruptionType.NETWORK_PARTITION) + .chaosDurationSec(30) + .build(); + + provider.triggerFault(spec); + var policy = await(() -> client.network() + .networkPolicies() + .inNamespace("kafka") + .withName("kates-netpol-krafter-brokers-0") + .get()); + + assertMarked(policy.getMetadata().getAnnotations(), "7"); + } + + @Test + void aScaleDownCarriesItsTermAndDeadline() throws Exception { + createKafkaStatefulSet(); + provider.quorum = quorumAt(7, true); + FaultSpec spec = FaultSpec.builder("scale-down") + .targetNamespace("default") + .targetLabel("app=kafka") + .disruptionType(DisruptionType.SCALE_DOWN) + .build(); + + assertTrue(provider.triggerFault(spec).get(5, TimeUnit.SECONDS).isPass()); + + StatefulSet after = client.apps().statefulSets().inNamespace("default").withName("kafka").get(); + assertMarked(after.getMetadata().getAnnotations(), "7"); + } + + private static T await(java.util.function.Supplier found) throws InterruptedException { + for (int i = 0; i < 50; i++) { + T value = found.get(); + if (value != null) { + return value; + } + TimeUnit.MILLISECONDS.sleep(100); + } + throw new AssertionError("not within 5 s"); + } } diff --git a/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java b/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java index a2e8b7b1c..2ba2b3e1d 100644 --- a/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java +++ b/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java @@ -171,6 +171,47 @@ void aFaultWhoseExperimentIsInstalledCreatesItsChaosEngine() throws Exception { assertTrue(outcome.isPass(), outcome.failureReason()); } + @Test + void anEngineCarriesItsTermAndDeadline() throws Exception { + install("disk-fill"); + provider.resultPollIntervalMs = 20; + com.bmscomp.kates.quorum.Quorum quorum = org.mockito.Mockito.mock(com.bmscomp.kates.quorum.Quorum.class); + when(quorum.term()).thenReturn(7L); + provider.quorum = quorum; + FaultSpec spec = FaultSpec.builder("fill") + .targetBrokerId(2) + .disruptionType(DisruptionType.DISK_FILL) + .build(); + + provider.triggerFault(spec); + ChaosEngine engine = awaitEngine(); + + assertEquals("7", engine.getMetadata().getAnnotations().get(FaultMarks.TERM)); + assertNotNull(java.time.Instant.parse(engine.getMetadata().getAnnotations().get(FaultMarks.DEADLINE))); + } + + @Test + void anEngineIsNotCreatedByAFollower() throws Exception { + install("disk-fill"); + com.bmscomp.kates.quorum.Quorum quorum = org.mockito.Mockito.mock(com.bmscomp.kates.quorum.Quorum.class); + org.mockito.Mockito.doThrow(new com.bmscomp.kates.engine.NotLeaderException("m1")) + .when(quorum) + .requireWrites(); + provider.quorum = quorum; + FaultSpec spec = FaultSpec.builder("fill") + .targetBrokerId(2) + .disruptionType(DisruptionType.DISK_FILL) + .build(); + + ChaosOutcome outcome = provider.triggerFault(spec).get(5, TimeUnit.SECONDS); + + assertFalse(outcome.isPass()); + assertTrue(outcome.failureReason().contains("does not lead"), outcome.failureReason()); + assertEquals( + List.of(), + client.resources(ChaosEngine.class).inNamespace("kafka").list().getItems()); + } + private static String env(ChaosEngine engine, String name) { return engine.getSpec().experiments.getFirst().spec.components.env.stream() .filter(e -> e.name.equals(name)) diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java index 9fc4a64b8..0afb1d9a3 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java @@ -71,6 +71,21 @@ void aLeaderWhoseRecoveryFailsTakesNoWriteUntilItSucceeds() throws Exception { assertTrue(takeovers.get(takeovers.size() - 1).startsWith(leader.memberId() + "@")); } + @Test + void requireWritesRefusesOnAFollower() throws Exception { + start(3, new AtomicBoolean(false)); + Quorum leader = soleWriter(members); + + leader.requireWrites(); + assertEquals(((Number) leader.health().get("term")).longValue(), leader.term()); + for (Quorum member : members) { + if (member != leader) { + org.junit.jupiter.api.Assertions.assertThrows( + com.bmscomp.kates.engine.NotLeaderException.class, member::requireWrites); + } + } + } + @Test void aClosedMemberIsRestartedBySchedule() throws Exception { start(1, new AtomicBoolean(false)); From 80fd798294cef2f1b84786c7cb1456e9beb0d886 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 14:22:51 +0100 Subject: [PATCH 2/6] fix(disruption): the leader undoes faults from an older term or past their deadline every 10 s DisruptionOrphanReconciler ran once, at start or takeover, judged by creation time and swallowed its errors. A paused leader's late partition, made after the takeover, was never removed, and nor was one a new leader could not reach the Kubernetes API to delete (s09b, s08b of the failure study). - A pass runs every kates.quorum.reconcile-interval (10 s) on the member that leads, and undoes a Kates fault past its kates.io/deadline or of an older kates.io/term. The age rule stays for the start or takeover pass only. - Kates ChaosEngines are reconciled too, and a node-drain engine's TARGET_NODE is uncordoned. - Scale-downs are restored in every namespace, falling back to the Kafka namespace where listing across namespaces is refused. - Reports are marked INTERRUPTED only in a pass whose deletes all succeeded. --- .../kates/chaos/ScaleDownSnapshots.java | 70 +++++- .../DisruptionOrphanReconciler.java | 164 ++++++++++++- .../DisruptionOrphanReconcilerTest.java | 225 ++++++++++++++++++ 3 files changed, 436 insertions(+), 23 deletions(-) diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/ScaleDownSnapshots.java b/kates/src/main/java/com/bmscomp/kates/chaos/ScaleDownSnapshots.java index fba1ee4b6..e2f8d817f 100644 --- a/kates/src/main/java/com/bmscomp/kates/chaos/ScaleDownSnapshots.java +++ b/kates/src/main/java/com/bmscomp/kates/chaos/ScaleDownSnapshots.java @@ -2,6 +2,7 @@ import java.net.HttpURLConnection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Predicate; @@ -43,19 +44,55 @@ public int total() { */ public static Restored restore(KubernetesClient client, String namespace, Predicate eligible) { return new Restored( - restoreStatefulSets(client, namespace, eligible), restoreNodePools(client, namespace, eligible)); + restoreStatefulSets(client, namespace, null, eligible), + restoreNodePools(client, namespace, null, eligible)); } - private static int restoreStatefulSets(KubernetesClient client, String namespace, Predicate eligible) { + /** + * As {@link #restore}, in every namespace: a playbook can scale down outside Kafka's. Where the + * backend may not list across namespaces (rbac.scope=namespace), {@code fallback} only. + */ + public static Restored restoreEverywhere( + KubernetesClient client, String fallback, Predicate eligible) { + return new Restored( + restoreStatefulSets(client, null, fallback, eligible), + restoreNodePools(client, null, fallback, eligible)); + } + + /** Lists in {@code namespace}, or in every namespace when it is null, else {@code fallback} on a 403. */ + private static List list( + String namespace, + String fallback, + java.util.function.Function> in, + java.util.function.Supplier> everywhere) { + if (namespace != null) { + return in.apply(namespace); + } + try { + return everywhere.get(); + } catch (KubernetesClientException e) { + if (e.getCode() != 403 || fallback == null) { + throw e; + } + return in.apply(fallback); + } + } + + private static int restoreStatefulSets( + KubernetesClient client, String namespace, String fallback, Predicate eligible) { int restored = 0; try { - for (StatefulSet ss : - client.apps().statefulSets().inNamespace(namespace).list().getItems()) { + for (StatefulSet ss : list( + namespace, + fallback, + ns -> client.apps().statefulSets().inNamespace(ns).list().getItems(), + () -> client.apps().statefulSets().inAnyNamespace().list().getItems())) { Integer original = snapshot(ss.getMetadata()); if (original == null || !eligible.test(ss.getMetadata())) { continue; } String name = ss.getMetadata().getName(); + String namespaceOf = ss.getMetadata().getNamespace(); try { int current = ss.getSpec().getReplicas() != null ? ss.getSpec().getReplicas() : original; @@ -63,14 +100,14 @@ private static int restoreStatefulSets(KubernetesClient client, String namespace LOG.warnf("Restoring StatefulSet %s from %d → %d", name, current, original); client.apps() .statefulSets() - .inNamespace(namespace) + .inNamespace(namespaceOf) .withName(name) .scale(original); restored++; } client.apps() .statefulSets() - .inNamespace(namespace) + .inNamespace(namespaceOf) .withName(name) .edit(s -> new StatefulSetBuilder(s) .editMetadata() @@ -88,24 +125,33 @@ private static int restoreStatefulSets(KubernetesClient client, String namespace return restored; } - private static int restoreNodePools(KubernetesClient client, String namespace, Predicate eligible) { + private static int restoreNodePools( + KubernetesClient client, String namespace, String fallback, Predicate eligible) { int restored = 0; try { - for (GenericKubernetesResource pool : client.genericKubernetesResources(NodePoolScaleDown.NODE_POOLS) - .inNamespace(namespace) - .list() - .getItems()) { + for (GenericKubernetesResource pool : list( + namespace, + fallback, + ns -> client.genericKubernetesResources(NodePoolScaleDown.NODE_POOLS) + .inNamespace(ns) + .list() + .getItems(), + () -> client.genericKubernetesResources(NodePoolScaleDown.NODE_POOLS) + .inAnyNamespace() + .list() + .getItems())) { Integer original = snapshot(pool.getMetadata()); if (original == null || !eligible.test(pool.getMetadata())) { continue; } String name = pool.getMetadata().getName(); + String namespaceOf = pool.getMetadata().getNamespace(); int current = NodePoolScaleDown.replicas(pool); try { // One patch: the replicas and the snapshot go together, so a // failure leaves the snapshot for the next attempt. client.genericKubernetesResources(NodePoolScaleDown.NODE_POOLS) - .inNamespace(namespace) + .inNamespace(namespaceOf) .withName(name) .edit(p -> { if (current < original) { diff --git a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java index c1df91fed..42a1e3e06 100644 --- a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java +++ b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java @@ -4,21 +4,27 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; +import java.util.Map; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.event.Observes; import jakarta.inject.Inject; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import io.fabric8.kubernetes.api.model.NodeBuilder; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicy; import io.fabric8.kubernetes.client.KubernetesClient; import io.quarkus.runtime.StartupEvent; +import io.quarkus.scheduler.Scheduled; import org.eclipse.microprofile.config.inject.ConfigProperty; import org.jboss.logging.Logger; +import com.bmscomp.kates.chaos.FaultMarks; import com.bmscomp.kates.chaos.KubernetesChaosProvider; import com.bmscomp.kates.chaos.ScaleDownSnapshots; +import com.bmscomp.kates.chaos.litmus.ChaosEngine; +import com.bmscomp.kates.chaos.litmus.ChaosEngineSpec; /** * Cleans up faults abandoned by a previous process. @@ -51,8 +57,15 @@ public class DisruptionOrphanReconciler { @Inject ObjectMapper objectMapper; + // Built without config, as the tests build it, a Quorum is alone and always leads. @Inject - com.bmscomp.kates.quorum.Quorum quorum; + com.bmscomp.kates.quorum.Quorum quorum = new com.bmscomp.kates.quorum.Quorum(); + + /** When this member took over, or this lone process started: its own faults are made after. */ + volatile Instant since; + + /** Whether an undo of the pass running now failed: its reports stay RUNNING until one succeeds. */ + private boolean undoFailed; @ConfigProperty(name = "kates.chaos.orphan-recovery.enabled", defaultValue = "true") boolean enabled; @@ -90,6 +103,7 @@ void onStart(@Observes StartupEvent event) { // // Recovery is cleanup of faults made before this process started. // Nothing about it needs to happen before the first request is served. + since = startedAt; Thread.ofVirtual().name("disruption-orphan-recovery").start(() -> reconcile(startedAt)); } @@ -100,20 +114,40 @@ void onStart(@Observes StartupEvent event) { */ void onTakeover(@Observes com.bmscomp.kates.quorum.LeadershipAcquired takeover) { if (enabled) { + since = takeover.at(); Thread.ofVirtual().name("disruption-orphan-recovery").start(() -> reconcile(takeover.at())); } } - private void reconcile(Instant startedAt) { + /** + * The leader keeps cleaning up, not only at takeover: a paused leader's late write carries an + * older term (J4 F1), and a Kubernetes API or database that was down at takeover is back later + * (F4, F8). Only faults of an older term or past their deadline are undone here, never by age. + */ + @Scheduled( + every = "${kates.quorum.reconcile-interval:10s}", + identity = "fault-reconcile", + concurrentExecution = Scheduled.ConcurrentExecution.SKIP) + void periodic() { + if (enabled && since != null && quorum.acceptsWrites()) { + reconcile(null); + } + } + + /** One pass: with {@code startedAt}, also every Kates fault made before it, as at start or takeover. */ + private synchronized void reconcile(Instant startedAt) { + undoFailed = false; try { int policies = reconcileNetworkPolicies(startedAt); + int engines = reconcileChaosEngines(startedAt); ScaleDownSnapshots.Restored scaled = reconcileScaleDowns(startedAt); - int reports = markInterruptedReports(startedAt); - if (policies + scaled.total() + reports > 0) { + // A report says INTERRUPTED only once its faults are out (s08b reported it with one in). + int reports = undoFailed ? 0 : markInterruptedReports(startedAt != null ? startedAt : since); + if (policies + engines + scaled.total() + reports > 0) { LOG.warnf( - "Orphan recovery: removed %d NetworkPolicy(ies), restored %d KafkaNodePool(s) and %d" - + " StatefulSet(s), marked %d report(s) INTERRUPTED", - policies, scaled.nodePools(), scaled.statefulSets(), reports); + "Orphan recovery: removed %d NetworkPolicy(ies) and %d ChaosEngine(s), restored %d" + + " KafkaNodePool(s) and %d StatefulSet(s), marked %d report(s) INTERRUPTED", + policies, engines, scaled.nodePools(), scaled.statefulSets(), reports); } } catch (Exception e) { // Never block startup on cleanup — the API must come up either way. @@ -121,13 +155,116 @@ private void reconcile(Instant startedAt) { } } + /** + * Whether a Kates fault's object is no live fault's: past its deadline, or written by a leader + * of an older term, or, in a pass with {@code ageCutoffMs}, made before it. Which is how a + * lone instance tells the previous process's faults from its own, and how objects from before + * the marks existed are judged. + */ + boolean isStray(ObjectMeta meta, Long ageCutoffMs, Instant now) { + Map marks = meta.getAnnotations() != null ? meta.getAnnotations() : Map.of(); + String deadline = marks.get(FaultMarks.DEADLINE); + if (deadline != null) { + try { + if (Instant.parse(deadline).isBefore(now)) { + return true; + } + } catch (java.time.format.DateTimeParseException ignored) { + // judged as if unmarked + } + } + String term = marks.get(FaultMarks.TERM); + if (term != null && quorum.memberId() != null) { + try { + if (Long.parseLong(term) < quorum.term()) { + return true; + } + } catch (NumberFormatException ignored) { + // judged as if unmarked + } + } + return ageCutoffMs != null && isOlderThan(meta.getCreationTimestamp(), ageCutoffMs); + } + + /** + * Deletes the Kates ChaosEngines no live fault owns, in every namespace, and uncordons the node + * a node-drain engine named: Kates does not cordon, Litmus does, and nothing else would undo it. + */ + int reconcileChaosEngines(Instant startedAt) { + Long cutoff = startedAt == null ? null : cutoff(startedAt); + List engines; + try { + engines = kubeClient.resources(ChaosEngine.class) + .inAnyNamespace() + .withLabel("managed-by", "kates") + .list() + .getItems(); + } catch (Exception e) { + // No Litmus, or no rights on its engines: nothing Kates made to undo there. + LOG.debugf("Could not list ChaosEngines: %s", e.getMessage()); + return 0; + } + Instant now = Instant.now(); + int removed = 0; + for (ChaosEngine engine : engines) { + if (!isStray(engine.getMetadata(), cutoff, now)) { + continue; + } + String namespace = engine.getMetadata().getNamespace(); + String name = engine.getMetadata().getName(); + try { + kubeClient.resources(ChaosEngine.class) + .inNamespace(namespace) + .withName(name) + .delete(); + String node = drainedNode(engine); + if (node != null) { + kubeClient.nodes() + .withName(node) + .edit(n -> new NodeBuilder(n) + .editOrNewSpec() + .withUnschedulable(false) + .endSpec() + .build()); + LOG.warnf("Orphan recovery: uncordoned node %s, drained by ChaosEngine %s/%s", node, namespace, name); + } + removed++; + } catch (Exception e) { + undoFailed = true; + LOG.warnf("Could not undo ChaosEngine %s/%s: %s", namespace, name, e.getMessage()); + } + } + return removed; + } + + /** The TARGET_NODE of a node-drain engine, or null. */ + static String drainedNode(ChaosEngine engine) { + if (engine.getSpec() == null || engine.getSpec().experiments == null) { + return null; + } + for (ChaosEngineSpec.Experiment experiment : engine.getSpec().experiments) { + if ("node-drain".equals(experiment.name) + && experiment.spec != null + && experiment.spec.components != null + && experiment.spec.components.env != null) { + for (ChaosEngineSpec.EnvVar env : experiment.spec.components.env) { + if ("TARGET_NODE".equals(env.name) && env.value != null && !env.value.isBlank()) { + return env.value; + } + } + } + } + return null; + } + /** * Deletes the Kates-managed NetworkPolicies (network partitions) left from * before this process started, in every namespace, as cleanup does: a * playbook can isolate pods outside the Kafka namespace. */ int reconcileNetworkPolicies(Instant startedAt) { - long cutoff = cutoff(startedAt); + Long cutoff = startedAt == null ? null : cutoff(startedAt); + Instant now = Instant.now(); List policies; try { policies = kubeClient @@ -146,7 +283,7 @@ int reconcileNetworkPolicies(Instant startedAt) { } int removed = 0; for (NetworkPolicy policy : policies) { - if (!isOlderThan(policy.getMetadata().getCreationTimestamp(), cutoff)) { + if (!isStray(policy.getMetadata(), cutoff, now)) { continue; } String namespace = policy.getMetadata().getNamespace(); @@ -163,6 +300,7 @@ int reconcileNetworkPolicies(Instant startedAt) { LOG.warnf("Orphan recovery: deleted abandoned NetworkPolicy %s/%s", namespace, name); removed++; } catch (Exception e) { + undoFailed = true; LOG.warnf("Could not delete orphaned NetworkPolicy %s/%s: %s", namespace, name, e.getMessage()); } } @@ -176,8 +314,12 @@ int reconcileNetworkPolicies(Instant startedAt) { * it, which is exactly what makes recovery possible here. */ ScaleDownSnapshots.Restored reconcileScaleDowns(Instant startedAt) { - long cutoff = cutoff(startedAt); - return ScaleDownSnapshots.restore(kubeClient, kafkaNamespace, meta -> isScaleDownOlderThan(meta, cutoff)); + Long cutoff = startedAt == null ? null : cutoff(startedAt); + Instant now = Instant.now(); + return ScaleDownSnapshots.restoreEverywhere( + kubeClient, + kafkaNamespace, + meta -> isStray(meta, null, now) || (cutoff != null && isScaleDownOlderThan(meta, cutoff))); } private long cutoff(Instant startedAt) { diff --git a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrphanReconcilerTest.java b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrphanReconcilerTest.java index 1afcba70f..069073901 100644 --- a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrphanReconcilerTest.java +++ b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrphanReconcilerTest.java @@ -113,4 +113,229 @@ private NetworkPolicy get(String namespace, String name) { .withName(name) .get(); } + + // ---- J4b: by term and deadline, every pass ---- + + private static final Instant NOW = Instant.parse("2026-11-15T12:00:00Z"); + + private com.bmscomp.kates.quorum.Quorum quorumAt(long term, boolean leads) { + com.bmscomp.kates.quorum.Quorum quorum = org.mockito.Mockito.mock(com.bmscomp.kates.quorum.Quorum.class); + org.mockito.Mockito.when(quorum.term()).thenReturn(term); + org.mockito.Mockito.when(quorum.memberId()).thenReturn("m0"); + org.mockito.Mockito.when(quorum.acceptsWrites()).thenReturn(leads); + return quorum; + } + + private static io.fabric8.kubernetes.api.model.ObjectMeta meta(String term, Instant deadline, Instant created) { + Map a = new java.util.HashMap<>(); + if (term != null) { + a.put(com.bmscomp.kates.chaos.FaultMarks.TERM, term); + } + if (deadline != null) { + a.put(com.bmscomp.kates.chaos.FaultMarks.DEADLINE, deadline.toString()); + } + return new io.fabric8.kubernetes.api.model.ObjectMetaBuilder() + .withAnnotations(a) + .withCreationTimestamp(created.toString()) + .build(); + } + + @org.junit.jupiter.params.ParameterizedTest + @org.junit.jupiter.params.provider.CsvSource( + nullValues = "-", + value = { + // term, deadline from NOW (s), created from cutoff (s), current term, in a quorum, age pass, stray + "3, -, 10, 4, true, false, true", + "4, -, 10, 4, true, false, false", + "4, -10, 10, 4, true, false, true", + "4, 3600, 10, 4, true, false, false", + "-, -, -10, 4, true, true, true", + "-, -, 10, 4, true, true, false", + "-, -, -10, 4, true, false, false", + "0, -, -10, 0, false, true, true", + "0, -, 10, 0, false, true, false", + "0, -, -10, 0, false, false, false", + }) + void strayRule( + String term, + Long deadlineIn, + long createdIn, + long current, + boolean inQuorum, + boolean agePass, + boolean stray) { + DisruptionOrphanReconciler reconciler = reconciler(); + reconciler.quorum = quorumAt(current, true); + if (!inQuorum) { + org.mockito.Mockito.when(reconciler.quorum.memberId()).thenReturn(null); + } + Instant cutoff = NOW.minusSeconds(60); + var meta = meta(term, deadlineIn == null ? null : NOW.plusSeconds(deadlineIn), cutoff.plusSeconds(createdIn)); + + assertEquals(stray, reconciler.isStray(meta, agePass ? cutoff.toEpochMilli() : null, NOW)); + } + + private NetworkPolicy marked(String namespace, String name, String term) { + return client.network() + .networkPolicies() + .inNamespace(namespace) + .resource(new NetworkPolicyBuilder() + .withNewMetadata() + .withName(name) + .withNamespace(namespace) + .withLabels(Map.of("managed-by", "kates")) + .withAnnotations(Map.of( + com.bmscomp.kates.chaos.FaultMarks.TERM, term, + com.bmscomp.kates.chaos.FaultMarks.DEADLINE, Instant.now().plusSeconds(3600).toString())) + .endMetadata() + .build()) + .create(); + } + + private DisruptionOrphanReconciler leaderAt(long term) { + DisruptionOrphanReconciler reconciler = reconciler(); + reconciler.enabled = true; + reconciler.quorum = quorumAt(term, true); + reconciler.repository = org.mockito.Mockito.mock(DisruptionReportRepository.class); + reconciler.since = Instant.now().minusSeconds(60); + return reconciler; + } + + @Test + void theCurrentTermsFaultIsKept() { + marked("kafka", "kates-netpol-a", "4"); + + leaderAt(4).periodic(); + + assertNotNull(get("kafka", "kates-netpol-a")); + } + + @Test + void anOlderTermsPartitionGoesOnTheNextPass() { + // A paused leader's late write: made after this member took over, with the old term. + marked("kafka", "kates-netpol-late", "3"); + + leaderAt(4).periodic(); + + assertNull(get("kafka", "kates-netpol-late")); + } + + @Test + void anOlderTermsEngineGoesAndItsNodeIsUncordoned() { + client.nodes() + .resource(new io.fabric8.kubernetes.api.model.NodeBuilder() + .withNewMetadata() + .withName("n1") + .endMetadata() + .withNewSpec() + .withUnschedulable(true) + .endSpec() + .build()) + .create(); + com.bmscomp.kates.chaos.litmus.ChaosEngine engine = new com.bmscomp.kates.chaos.litmus.ChaosEngine(); + engine.getMetadata().setName("kates-drain-1"); + engine.getMetadata().setNamespace("kafka"); + engine.getMetadata().setLabels(Map.of("managed-by", "kates")); + engine.getMetadata().setAnnotations(Map.of(com.bmscomp.kates.chaos.FaultMarks.TERM, "3")); + var spec = new com.bmscomp.kates.chaos.litmus.ChaosEngineSpec(); + var experiment = new com.bmscomp.kates.chaos.litmus.ChaosEngineSpec.Experiment(); + experiment.name = "node-drain"; + experiment.spec = new com.bmscomp.kates.chaos.litmus.ChaosEngineSpec.ExperimentSpec(); + experiment.spec.components = new com.bmscomp.kates.chaos.litmus.ChaosEngineSpec.Components(); + experiment.spec.components.env = + java.util.List.of(new com.bmscomp.kates.chaos.litmus.ChaosEngineSpec.EnvVar("TARGET_NODE", "n1")); + spec.experiments = java.util.List.of(experiment); + engine.setSpec(spec); + client.resources(com.bmscomp.kates.chaos.litmus.ChaosEngine.class) + .inNamespace("kafka") + .resource(engine) + .create(); + + leaderAt(4).periodic(); + + assertEquals( + java.util.List.of(), + client.resources(com.bmscomp.kates.chaos.litmus.ChaosEngine.class) + .inAnyNamespace() + .list() + .getItems()); + assertTrue(!Boolean.TRUE.equals(client.nodes().withName("n1").get().getSpec().getUnschedulable()), + "the drained node is schedulable again"); + } + + @Test + void aScaleDownOutsideTheKafkaNamespaceIsRestored() { + client.apps() + .statefulSets() + .inNamespace("other") + .resource(new io.fabric8.kubernetes.api.model.apps.StatefulSetBuilder() + .withNewMetadata() + .withName("victim") + .withNamespace("other") + .withAnnotations(Map.of( + com.bmscomp.kates.chaos.KubernetesChaosProvider.ORIGINAL_REPLICAS_ANNOTATION, "3", + com.bmscomp.kates.chaos.FaultMarks.TERM, "3")) + .endMetadata() + .withNewSpec() + .withReplicas(2) + .endSpec() + .build()) + .create(); + + leaderAt(4).periodic(); + + assertEquals( + 3, + client.apps() + .statefulSets() + .inNamespace("other") + .withName("victim") + .get() + .getSpec() + .getReplicas()); + } + + @Test + void aFailedUndoLeavesReportsRunning() { + marked("kafka", "kates-netpol-stuck", "3"); + server.expect() + .delete() + .withPath("/apis/networking.k8s.io/v1/namespaces/kafka/networkpolicies/kates-netpol-stuck") + .andReturn(403, "forbidden") + .once(); + DisruptionOrphanReconciler reconciler = leaderAt(4); + + reconciler.periodic(); + org.mockito.Mockito.verify(reconciler.repository, org.mockito.Mockito.never()) + .findByStatusCreatedBefore(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any()); + + reconciler.periodic(); + assertNull(get("kafka", "kates-netpol-stuck")); + org.mockito.Mockito.verify(reconciler.repository) + .findByStatusCreatedBefore(org.mockito.ArgumentMatchers.eq("RUNNING"), org.mockito.ArgumentMatchers.any()); + } + + @Test + void aFollowerReconcilesNothing() { + marked("kafka", "kates-netpol-old", "3"); + DisruptionOrphanReconciler reconciler = leaderAt(4); + org.mockito.Mockito.when(reconciler.quorum.acceptsWrites()).thenReturn(false); + int before = server.getRequestCount(); + + reconciler.periodic(); + + assertEquals(before, server.getRequestCount(), "a follower asks the API nothing"); + assertNotNull(get("kafka", "kates-netpol-old")); + } + + @Test + void aLoneInstanceKeepsItsOwnPartitionOnEveryPass() { + marked("kafka", "kates-netpol-mine", "0"); + DisruptionOrphanReconciler reconciler = leaderAt(0); + org.mockito.Mockito.when(reconciler.quorum.memberId()).thenReturn(null); + + reconciler.periodic(); + + assertNotNull(get("kafka", "kates-netpol-mine"), "the periodic pass of a lone instance judges by deadline only"); + } } From 96cb8ef1707fe657e176f84dd873121f76fbaa9d Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 14:31:48 +0100 Subject: [PATCH 3/6] fix(engine): takeover takes writes with the database down, and a deposed member stops its runs TestOrchestrator's takeover recovery threw when PostgreSQL was down, so a new leader took no write until the database came back, and whether fault cleanup ran depended on CDI observer order (F6 of the failure study). It now logs the failure, and the reconciler retries the recovery on each pass. A member that stops leading is told by Ratis (notifyLeaderChanged on its state machine) and fires LeadershipLost; the orchestrator settles its live runs, so their load stops and their permits come back. The new leader stores them FAILED. --- .../DisruptionOrphanReconciler.java | 11 ++++ .../kates/engine/TestOrchestrator.java | 32 ++++++++++-- .../bmscomp/kates/quorum/LeadershipLost.java | 9 ++++ .../java/com/bmscomp/kates/quorum/Quorum.java | 23 ++++++++- .../bmscomp/kates/quorum/QuorumMember.java | 50 +++++++++++++++++-- .../config/NativeReflectionRegistryTest.java | 2 + .../engine/TestOrchestratorQuorumTest.java | 31 ++++++++++++ .../kates/quorum/QuorumMemberTest.java | 18 +++++++ 8 files changed, 166 insertions(+), 10 deletions(-) create mode 100644 kates/src/main/java/com/bmscomp/kates/quorum/LeadershipLost.java diff --git a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java index 42a1e3e06..cd9c1678f 100644 --- a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java +++ b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java @@ -61,6 +61,9 @@ public class DisruptionOrphanReconciler { @Inject com.bmscomp.kates.quorum.Quorum quorum = new com.bmscomp.kates.quorum.Quorum(); + @Inject + com.bmscomp.kates.engine.TestOrchestrator orchestrator; + /** When this member took over, or this lone process started: its own faults are made after. */ volatile Instant since; @@ -137,6 +140,14 @@ void periodic() { /** One pass: with {@code startedAt}, also every Kates fault made before it, as at start or takeover. */ private synchronized void reconcile(Instant startedAt) { undoFailed = false; + if (orchestrator != null) { + try { + // Retried here every pass: a takeover with PostgreSQL down did not get to it (J4 F6). + orchestrator.recoverPreviousLeaders(); + } catch (RuntimeException e) { + LOG.debugf("The previous leader's runs are not recovered yet: %s", e.getMessage()); + } + } try { int policies = reconcileNetworkPolicies(startedAt); int engines = reconcileChaosEngines(startedAt); diff --git a/kates/src/main/java/com/bmscomp/kates/engine/TestOrchestrator.java b/kates/src/main/java/com/bmscomp/kates/engine/TestOrchestrator.java index df1d032c7..5e3b18ded 100644 --- a/kates/src/main/java/com/bmscomp/kates/engine/TestOrchestrator.java +++ b/kates/src/main/java/com/bmscomp/kates/engine/TestOrchestrator.java @@ -156,15 +156,39 @@ void recoverOrphans() { } /** - * Fails, before this member takes its first write as leader, the live runs any other member - * created: only a leader starts runs, so they are a previous leader's, and it is gone or has - * lost the lead. A failure here keeps the member from taking writes, and is tried again. + * At takeover, fails the live runs any other member created: only a leader starts runs, so they + * are a previous leader's, and it is gone or has lost the lead. A failure, such as PostgreSQL + * down, no longer keeps the member from leading (J4 F6): the reconciler tries again each pass. */ void recoverPreviousLeaders(@Observes LeadershipAcquired takeover) { + try { + recoverPreviousLeaders(); + } catch (RuntimeException e) { + LOG.warnf("Term %d: the previous leader's runs are not recovered yet, retried: %s", takeover.term(), e); + } + } + + /** Fails another member's live runs; with no quorum there is no other member, and nothing to do. */ + public void recoverPreviousLeaders() { + if (quorum.memberId() == null) { + return; + } recover( repository.findByStatusAndMember(ORPHAN_STATUSES, quorum.memberId(), false), "when the member running it stopped leading", - "takeover, term " + takeover.term()); + "takeover"); + } + + /** + * Stops this member's runs once it no longer leads: their load would otherwise go on hitting + * Kafka, counted by no leader's permits. Only settled here; the new leader stores them FAILED. + */ + void onLeadershipLost(@Observes com.bmscomp.kates.quorum.LeadershipLost lost) { + List runs = List.copyOf(activeHandles.keySet()); + if (!runs.isEmpty()) { + LOG.warnf("No longer leading term %d: stopping %d run(s)", lost.term(), runs.size()); + } + runs.forEach(this::settle); } private void recover(List orphans, String when, String by) { diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/LeadershipLost.java b/kates/src/main/java/com/bmscomp/kates/quorum/LeadershipLost.java new file mode 100644 index 000000000..6946a7c41 --- /dev/null +++ b/kates/src/main/java/com/bmscomp/kates/quorum/LeadershipLost.java @@ -0,0 +1,9 @@ +package com.bmscomp.kates.quorum; + +import java.time.Instant; + +/** + * This member led {@code term} and no longer does: another member leads, or none yet. It stops its + * own runs; undoing its faults is the next leader's. + */ +public record LeadershipLost(long term, Instant at) {} diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java index 6570a1b5b..7a86400b6 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java @@ -52,6 +52,9 @@ public class Quorum { @Inject Event takeover; + @Inject + Event lost; + private volatile QuorumMember member; private volatile long recoveredTerm = -1; private QuorumMember.Settings settings; @@ -67,7 +70,7 @@ void start(@Observes StartupEvent event) throws IOException { org.apache.ratis.util.ExitUtils.disableSystemExit(); settings = QuorumMember.Settings.parse( peers.get(), memberId(), dir.orElse(null), cert.orElse(null), key.orElse(null), ca.orElse(null)); - member = QuorumMember.start(settings); + member = QuorumMember.start(settings, this::lostLead); LOG.infof( "quorum member %s started, %d members", settings.memberId(), settings.peers().size()); @@ -161,7 +164,7 @@ private void revive() { LOG.debugf("closing the old member: %s", e); } try { - member = QuorumMember.start(settings); + member = QuorumMember.start(settings, this::lostLead); notServingSince = null; recoveredTerm = -1; LOG.infof("quorum member %s serves again", settings.memberId()); @@ -170,6 +173,22 @@ private void revive() { } } + /** Off Ratis' thread: the observers stop this member's runs, which takes a while. */ + private void lostLead(long term) { + Event event = lost; + if (event == null) { + return; + } + LOG.warnf("Member %s no longer leads term %d", memberId(), term); + Thread.ofVirtual().name("quorum-lost-lead").start(() -> { + try { + event.fire(new LeadershipLost(term, Instant.now())); + } catch (RuntimeException e) { + LOG.warnf("Stopping this member's work after it lost the lead failed: %s", e); + } + }); + } + /** Since when this member's Ratis server has not been running; empty while it serves, or with no quorum. */ public Optional notServingSince() { return Optional.ofNullable(notServingSince); diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java index 2b3faeeb2..d61391ffe 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java @@ -142,6 +142,11 @@ private QuorumMember(RaftServer server, RaftClient client, RaftPeerId selfId, St /** Starts the member. Its term and vote are kept in {@code dir}, which must outlive a restart. */ public static QuorumMember start(Settings settings) throws IOException { + return start(settings, lostTerm -> {}); + } + + /** As {@link #start(Settings)}, telling {@code onLost} each time this member stops leading. */ + public static QuorumMember start(Settings settings, java.util.function.LongConsumer onLost) throws IOException { RaftPeer self = settings.self(); String address = self.getAddress(); RaftProperties properties = new RaftProperties(); @@ -166,13 +171,21 @@ public static QuorumMember start(Settings settings) throws IOException { .setMutualTls(true) .setSslProvider(SslProvider.JDK) .build()); + java.util.concurrent.atomic.AtomicReference started = new java.util.concurrent.atomic.AtomicReference<>(); + Confirmations confirmations = new Confirmations( + self.getId(), + () -> { + QuorumMember member = started.get(); + return member == null ? 0 : member.term(); + }, + onLost); RaftGroup group = RaftGroup.valueOf(GROUP, settings.peers()); RaftServer server = RaftServer.newBuilder() .setServerId(self.getId()) .setGroup(group) .setProperties(properties) .setParameters(parameters) - .setStateMachine(new Confirmations()) + .setStateMachine(confirmations) .setOption(RaftStorage.StartupOption.RECOVER) .build(); try { @@ -191,7 +204,9 @@ public static QuorumMember start(Settings settings) throws IOException { .setParameters(parameters) .setRetryPolicy(RetryPolicies.noRetry()) .build(); - return new QuorumMember(server, client, self.getId(), settings.memberId()); + QuorumMember member = new QuorumMember(server, client, self.getId(), settings.memberId()); + started.set(member); + return member; } public String memberId() { @@ -270,11 +285,38 @@ public void close() throws IOException { } } - /** Replicates nothing yet; answers the reads that confirm leadership. */ - private static final class Confirmations extends BaseStateMachine { + /** + * Replicates nothing; answers the reads that confirm leadership, and tells {@code onLost} when + * this member stops leading, with the term it led. + */ + static final class Confirmations extends BaseStateMachine { + private final RaftPeerId self; + private final java.util.function.LongSupplier term; + private final java.util.function.LongConsumer onLost; + private boolean led; + private long ledTerm; + + Confirmations(RaftPeerId self, java.util.function.LongSupplier term, java.util.function.LongConsumer onLost) { + this.self = self; + this.term = term; + this.onLost = onLost; + } + @Override public CompletableFuture query(Message request) { return CompletableFuture.completedFuture(Message.EMPTY); } + + @Override + public synchronized void notifyLeaderChanged( + org.apache.ratis.protocol.RaftGroupMemberId member, RaftPeerId leader) { + if (self.equals(leader)) { + led = true; + ledTerm = term.getAsLong(); + } else if (led) { + led = false; + onLost.accept(ledTerm); + } + } } } diff --git a/kates/src/test/java/com/bmscomp/kates/config/NativeReflectionRegistryTest.java b/kates/src/test/java/com/bmscomp/kates/config/NativeReflectionRegistryTest.java index e90114741..c6913b162 100644 --- a/kates/src/test/java/com/bmscomp/kates/config/NativeReflectionRegistryTest.java +++ b/kates/src/test/java/com/bmscomp/kates/config/NativeReflectionRegistryTest.java @@ -72,6 +72,8 @@ class NativeReflectionRegistryTest { // serialised. "com.bmscomp.kates.chaos.K8sPodWatcher$WatchSession", "com.bmscomp.kates.chaos.ProbeRegistry", + // The annotation names and values a fault's objects carry; a constants holder. + "com.bmscomp.kates.chaos.FaultMarks", "com.bmscomp.kates.chaos.KafkaProbes", // Fault-targeting helpers; a FaultSpec carries its selector as a string. "com.bmscomp.kates.chaos.ParsedLabelSelector", diff --git a/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java b/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java index d2d3fbb9b..84d863740 100644 --- a/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java +++ b/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java @@ -72,6 +72,37 @@ void aTakeoverFailsAPreviousLeadersRuns() { verify(engine.repository, never()).findByStatusAndMember(any(), anyString(), eq(true)); } + @Test + @DisplayName("a takeover with the database down still lets the member lead, and the recovery is retried") + void aTakeoverWithTheDatabaseDownTakesWrites() { + String previous = engine.storeEnded(TaskStatus.RUNNING); + when(engine.quorum.memberId()).thenReturn("kates-1"); + when(engine.repository.findByStatusAndMember(any(), eq("kates-1"), eq(false))) + .thenThrow(new IllegalStateException("PostgreSQL is down")) + .thenReturn(List.of(engine.rows.get(previous))); + + org.junit.jupiter.api.Assertions.assertDoesNotThrow( + () -> orchestrator.recoverPreviousLeaders(new LeadershipAcquired(4, Instant.now()))); + assertEquals(TaskStatus.RUNNING, engine.rows.get(previous).getStatus(), "not recovered yet"); + + orchestrator.recoverPreviousLeaders(); + + assertFailedWith(previous, "when the member running it stopped leading"); + } + + @Test + @DisplayName("a member that stops leading stops its own runs and gives their permits back") + void aDeposedMemberStopsItsRuns() { + String first = engine.startRunning(); + String second = engine.startRunning(); + + orchestrator.onLeadershipLost(new com.bmscomp.kates.quorum.LeadershipLost(4, Instant.now())); + + assertTrue(engine.backend.stopped.containsAll(InMemoryEngine.taskIds(first)), "first run's tasks stopped"); + assertTrue(engine.backend.stopped.containsAll(InMemoryEngine.taskIds(second)), "second run's tasks stopped"); + engine.assertEverySlotFree(); + } + private void assertFailedWith(String id, String reason) { TestRun run = engine.rows.get(id); assertEquals(TaskStatus.FAILED, run.getStatus()); diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java index 7dd44f403..decd65ea8 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java @@ -142,6 +142,24 @@ void aMemberWhoseServerClosesRejoinsAsAnotherTerm() throws Exception { await(() -> leader.memberId().equals(again.leaderId()) ? again : null); } + @Test + void losingTheLeadIsToldOnceWithTheTermItHeld() { + List lost = new ArrayList<>(); + long[] term = {5}; + var self = org.apache.ratis.protocol.RaftPeerId.valueOf("m0"); + var other = org.apache.ratis.protocol.RaftPeerId.valueOf("m1"); + QuorumMember.Confirmations confirmations = new QuorumMember.Confirmations(self, () -> term[0], lost::add); + + confirmations.notifyLeaderChanged(null, other); + assertEquals(List.of(), lost, "never led: nothing lost"); + confirmations.notifyLeaderChanged(null, self); + term[0] = 6; + confirmations.notifyLeaderChanged(null, null); + confirmations.notifyLeaderChanged(null, other); + + assertEquals(List.of(5L), lost, "told once, with the term it led"); + } + @Test void aMemberWithAnotherCasCertificateCannotJoin() throws Exception { TestPki pki = new TestPki(tmp.resolve("pki")); From 53a1fb95be725effa6d3da32444e94dd651fcf44 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 14:42:12 +0100 Subject: [PATCH 4/6] docs(api): faults carry their term and deadline, and the leader reconciles every 10 s The /api/health quorum paragraph says what J4b changed: the check right before a fault goes in, kates.io/term and kates.io/deadline, the 10 s reconcile pass and what it removes, and a deposed member stopping its runs. The stale-report caveat says when a quorum's leader marks a report INTERRUPTED. 18 CLI refs re-pointed to the lines the backend edits moved. --- cli/cmd/mcp_caveats.go | 14 +++++++------- cli/cmd/mcp_tools_chaos.go | 13 +++++++------ cli/cmd/mcp_tools_runs_activity.go | 4 ++-- cli/cmd/mcp_tools_runs_list.go | 10 +++++----- cli/cmd/mcp_tools_scenario.go | 4 ++-- docs/book/11-api-reference.md | 2 +- 6 files changed, 24 insertions(+), 23 deletions(-) diff --git a/cli/cmd/mcp_caveats.go b/cli/cmd/mcp_caveats.go index 514d88239..fd3fc700a 100644 --- a/cli/cmd/mcp_caveats.go +++ b/cli/cmd/mcp_caveats.go @@ -76,7 +76,7 @@ var mcpCaveatsCore = []mcpCaveat{ Short: "A LOAD run is one producer and one consumer whatever numProducers says; only STRESS runs parallel producers.", Text: "A LOAD run is one producer and one consumer, whatever numProducers and numConsumers say, " + "so it cannot show how the cluster behaves under parallel clients. STRESS starts one producer per numProducers.", - Refs: []string{mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1796-1805", "case LOAD ->", "case STRESS ->")}, + Refs: []string{mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1820-1829", "case LOAD ->", "case STRESS ->")}, }, { ID: mcpCaveatReaperDeadline, @@ -99,7 +99,7 @@ var mcpCaveatsCore = []mcpCaveat{ "kates/src/main/resources/application.properties:336-350", mcpAnchoredRef(mcpJava+"engine/TestTimeoutReaper.java:49-51,67-76,170-173,201-214", "TestResult.TaskStatus.STOPPING", "kates.engine.reaper-grace-ms", "\"-submission\"", "\"Timeout: still \""), - mcpJava + "engine/TestOrchestrator.java:151-217,259-262,436,1251-1315,1381-1390,1445-1477", + mcpJava + "engine/TestOrchestrator.java:151-241,283-286,460,1275-1339,1405-1414,1469-1501", mcpJava + "persistence/EntityMapper.java:161-170", "kates/src/main/resources/db/migration/V24__run_planned_duration.sql:1-13", mcpJava + "domain/TestRun.java:520-523", @@ -123,10 +123,10 @@ var mcpCaveatsCore = []mcpCaveat{ "integrity-cg-integrity, without transactions.", Refs: []string{ mcpJava + "domain/TestSpec.java:14-32,111-113", - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:228,251-253,301", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:252,275-277,325", "applyTypeDefaults(type, request.getSpec())", ".withRequestedSpec(", "buildTasks(type, spec, run.getId())"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1171-1217", "TestSpec applyTypeDefaults(", "return merged;"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1775-1779,1870-1874", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1195-1241", "TestSpec applyTypeDefaults(", "return merged;"), + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1799-1803,1894-1898", "Only when asked", `"-integrity-0-group")`), mcpAnchoredRef(mcpJava+"engine/NativeKafkaBackend.java:792", `task.getConsumerGroup() + "-integrity"`), mcpAnchoredRef(mcpJava+"persistence/EntityMapper.java:56,85-86,146-148", @@ -177,7 +177,7 @@ var mcpCaveatsCore = []mcpCaveat{ Text: "A TUNE_* run executes one produce task with the spec's single configuration: it measures one setting " + "and compares none.", Refs: []string{ - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1881-1882", "case TUNE_REPLICATION,", "-tune-0"), + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1905-1906", "case TUNE_REPLICATION,", "-tune-0"), }, }, { @@ -241,7 +241,7 @@ var mcpCaveatsCore = []mcpCaveat{ Refs: []string{ mcpAnchoredRef("kates/src/main/resources/application.properties:274-275", "# Chaos coordination (noop | kubernetes | litmus-crd | hybrid)", "kates.chaos.provider=litmus-crd"), - mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:63,108", + mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:63,115", "int resultPollIntervalMs = 5_000;", "Instant start = Instant.now();"), }, }, diff --git a/cli/cmd/mcp_tools_chaos.go b/cli/cmd/mcp_tools_chaos.go index 12c3b08cc..b8d836790 100644 --- a/cli/cmd/mcp_tools_chaos.go +++ b/cli/cmd/mcp_tools_chaos.go @@ -147,7 +147,7 @@ var mcpCaveatsChaos = []mcpCaveat{ mcpAnchoredRef(mcpJava+"disruption/DisruptionSafetyGuard.java:298-301,874-943", "boolean canExecute = checkRbacPermissions(spec);", `"Insufficient RBAC permissions for "`, "boolean checkRbacPermissions(FaultSpec spec)", `"RBAC check failed, assuming permitted"`), - mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:42-60,87-89,270", + mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:42-60,94-96,282", "ON_KUBERNETES_API = EnumSet.of(", "DisruptionType.POD_DELETE,", "DisruptionType.NETWORK_PARTITION);", "return kubernetes.triggerFault(spec, onInject);", `engineSpec.chaosServiceAccount = "litmus-admin";`), mcpAnchoredRef("kates/src/main/resources/application.properties:274-275", @@ -275,12 +275,12 @@ var mcpCaveatsChaos = []mcpCaveat{ mcpAnchoredRef(mcpJava+"disruption/KafkaIntelligenceService.java:125-127,198-217,241-246,310-324", "public void markDisruptionStart(Instant at)", "private Duration computeTimeToFullIsr()", "this.baselineLag = timeline.getLast().totalLag();", "private Duration computeTimeToLagRecovery(long baseLag)"), - mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:87-99,108-127", + mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:94-106,115-139", "return kubernetes.triggerFault(spec, onInject);", "CompletableFuture.delayedExecutor(spec.delayBeforeSec(), TimeUnit.SECONDS, executor.get());", "long startNanos = System.nanoTime();", "buildChaosEngine(spec, engineName, experimentName)", "onInject.accept(startNanos);", ".resource(engine)"), - mcpAnchoredRef(mcpJava+"chaos/KubernetesChaosProvider.java:152-161", + mcpAnchoredRef(mcpJava+"chaos/KubernetesChaosProvider.java:168-180", "Thread.sleep(spec.delayBeforeSec() * 1000L);", "onInject.accept(startNanos);", "applyDisruption(spec, engineName);"), mcpAnchoredRef(mcpJava+"chaos/PodTargets.java:58,69-75", @@ -310,12 +310,13 @@ var mcpCaveatsChaos = []mcpCaveat{ "stopped mid-plan and has not started again: such a plan never finishes. When the backend starts, it " + "marks each report created before then that still says RUNNING as INTERRUPTED, in the report and in " + "the list of disruptions alike, unless kates.chaos.orphan-recovery.enabled is false. A backend in a " + - "quorum (kates.quorum.peers) marks nothing when it starts: the next member to lead marks each report " + - "created before it took the lead. Without a quorum this assumes one backend replica: a replica that " + + "quorum (kates.quorum.peers) marks nothing when it starts: the member that leads marks each report " + + "created before it took the lead, every 10 s until PostgreSQL answers, and only in a pass that removed " + + "every fault it found of an older term or past its deadline. Without a quorum this assumes one backend replica: a replica that " + "starts while another runs a plan marks that plan's report too, until the plan ends and stores its " + "outcome.", Refs: []string{ - mcpJava + "disruption/DisruptionOrphanReconciler.java:57-58,66-77,93-111,189-240", + mcpJava + "disruption/DisruptionOrphanReconciler.java:73-77,82-111,118-167,359-393", mcpJava + "disruption/DisruptionReportRepository.java:35-60,93-102", mcpJava + "disruption/DisruptionPersistence.java:26-37", mcpJava + "disruption/DisruptionLauncher.java:114-121,129-144", diff --git a/cli/cmd/mcp_tools_runs_activity.go b/cli/cmd/mcp_tools_runs_activity.go index 40addb788..a67257109 100644 --- a/cli/cmd/mcp_tools_runs_activity.go +++ b/cli/cmd/mcp_tools_runs_activity.go @@ -324,7 +324,7 @@ func mcpActivityTestsSince(ctx context.Context, call *mcpCall, since time.Time, // are injected for part of its run, or one whose backend process stopped // before it ended and has not started again to mark it INTERRUPTED // (DisruptionLauncher.java:95-123, DisruptionReportRepository.java:35-43, -// DisruptionOrphanReconciler.java:189-240). +// DisruptionOrphanReconciler.java:359-393). func mcpActivityDisruptionsSince(ctx context.Context, call *mcpCall, since time.Time, page mcpActivityPage) (mcpActivityDisruptions, error) { out := mcpActivityDisruptions{StoredAsRunning: []mcpActivityDisruptionRow{}, Since: []mcpActivityDisruptionRow{}} for p := 0; p < mcpActivityMaxPages; p++ { @@ -486,7 +486,7 @@ var mcpCaveatsRunsActivity = []mcpCaveat{ mcpJava + "disruption/DisruptionLauncher.java:114-144", mcpJava + "disruption/DisruptionPersistence.java:16-37", mcpJava + "disruption/DisruptionReportRepository.java:20-43", - mcpJava + "disruption/DisruptionOrphanReconciler.java:189-240", + mcpJava + "disruption/DisruptionOrphanReconciler.java:359-393", mcpJava + "disruption/DisruptionTemplateResource.java:70-72", mcpJava + "disruption/DisruptionScheduler.java:112-124", mcpJava + "disruption/DisruptionReportEntity.java:39-47", diff --git a/cli/cmd/mcp_tools_runs_list.go b/cli/cmd/mcp_tools_runs_list.go index a67742e2a..9f0e61209 100644 --- a/cli/cmd/mcp_tools_runs_list.go +++ b/cli/cmd/mcp_tools_runs_list.go @@ -1032,7 +1032,7 @@ var mcpCaveatsRunsList = []mcpCaveat{ mcpAnchoredRef(mcpJava+"report/ReportGenerator.java:220-249,283-294", "report.setSummary(MetricUtils.computeSummary(results));", "summaries.put(phase, MetricUtils.computeSummary(rows))"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:505-514,2235-2251", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:529-538,2259-2275", "TestResult.TaskStatus.PENDING : TestResult.TaskStatus.RUNNING", "private static TestRun withUnfinishedTasksFailed("), }, @@ -1071,7 +1071,7 @@ var mcpCaveatsRunsList = []mcpCaveat{ "except for a run cancelled before its tasks existed; a cancel through the REST API also leaves a CANCEL " + "audit row.", Refs: []string{ - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:2182-2264", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:2206-2288", `"Cancelled by user"`, `EventKind.FAILED, "cancelled"`, "return run.withResults(updatedResults);"), mcpAnchoredRef(mcpJava+"api/TestResource.java:572-617", `@Path("/{id}/cancel")`, `auditService.record("CANCEL"`, `"Test cancelled; it is stored as FAILED"`), @@ -1104,14 +1104,14 @@ var mcpCaveatsRunsList = []mcpCaveat{ "say which the API is, but a scenario run's tasks do: on an older API they all start within moments " + "of one another.", Refs: []string{ - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:422-424,481-492,494-510", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:446-448,505-516,518-534", "applyTypeDefaults(type, scenario.getBaseSpec())", ".withRequestedSpec(", "long phaseStartMs = System.currentTimeMillis();", "scenario.resolveSpecForPhase(phase, baseSpec)", "phaseStartMs = saturatedSum(phaseStartMs", "backend.submit(task)", "TestResult.TaskStatus.PENDING"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1583-1611,1624-1632,1692-1731,1741-1753", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1607-1635,1648-1656,1716-1755,1765-1777", "MAX_RAMP_STEPS = 100", "scenarioInapplicableFields(TestScenario scenario", "is a SPIKE phase", "has none (", "check no record CRCs", "has the name of an earlier phase"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1878-1947", "List buildPhaseTask(", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1902-1971", "List buildPhaseTask(", "int baseTarget = Math.max(1, spec.getThroughput() / steps);", "saturatedSum(startAtMs, s * stepMs)", `taskId + "-spike"`), mcpAnchoredRef(mcpJava+"domain/TestScenario.java:115-190", diff --git a/cli/cmd/mcp_tools_scenario.go b/cli/cmd/mcp_tools_scenario.go index 0bb74cc68..f385b8f45 100644 --- a/cli/cmd/mcp_tools_scenario.go +++ b/cli/cmd/mcp_tools_scenario.go @@ -1242,7 +1242,7 @@ var mcpCaveatsScenario = []mcpCaveat{ "plans/mcp-server.md:387-405", mcpJava + "api/TestResource.java:118-188", mcpJava + "domain/TestSpec.java:34-113", - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1503-1578", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1527-1602", "Map inapplicableFields(", "the trogdor backend cannot run a transactional producer"), mcpAnchoredRef("cli/cmd/apply.go:238-307", "for i, scenario := range sf.Scenarios {", "apiClient.CreateTest(a.ctx, scenarioToRequest(scenario))", @@ -1259,7 +1259,7 @@ var mcpCaveatsScenario = []mcpCaveat{ "listed as defaulted may differ on the cluster that runs the scenario.", Refs: []string{ mcpJava + "config/TestTypeDefaults.java:22-64,323-464", - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1171-1217", "TestSpec applyTypeDefaults(", "return merged;"), + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1195-1241", "TestSpec applyTypeDefaults(", "return merged;"), mcpAnchoredRef("kates/src/main/resources/application.properties:73-114", "# Per-type overrides: STRESS", "kates.tests.roundtrip.throughput=10000"), mcpAnchoredRef("charts/kates/values.yaml:690-758", "defaults:", "tests:", `throughput: "10000"`), diff --git a/docs/book/11-api-reference.md b/docs/book/11-api-reference.md index 6ab0a37b7..4f30e0e46 100644 --- a/docs/book/11-api-reference.md +++ b/docs/book/11-api-reference.md @@ -237,7 +237,7 @@ curl -s "$BASE/api/health" The response also contains a `tests` object with the resolved default configuration for every test type. When Kafka is unreachable, the endpoint still returns `200 OK`, but with `"status": "DEGRADED"` and `"kafka.status": "DOWN"`. -`quorum` says whether this Kates is a member of a quorum, which it is only when `kates.quorum.peers` is set. Without one it is `{"enabled": false}`, and the instance acts alone. A member also reports `state`, its Raft server's life cycle (`RUNNING` while it serves). Ratis closes a member's server after a pause longer than 60 s, such as a long garbage collection or a frozen node; the member then starts it again from its directory within a few seconds, and until it serves again only `state` and `memberId` are reported. If it still does not serve after `kates.quorum.liveness-after` (30 s), `/q/health/live` answers DOWN so the kubelet restarts the pod. A member reports its `memberId` and, while it serves, the `leaderId` it knows of (`null` during an election), the Raft `term`, `appliedIndex`, the last log entry it applied, and `leader`: true once it leads and has committed an entry of its term. A member whose log is empty, new or with its directory lost, is no voter until it has caught up to an entry: three members of which one is empty and one is down elect nobody. Members talk to each other over mutual TLS on their own port: `kates.quorum.tls.cert`, `kates.quorum.tls.key` (PKCS#8 PEM) and `kates.quorum.tls.ca` are required, and so is `kates.quorum.dir`, which keeps a member's term and vote and must outlive a restart. Only the leader takes writes: on any other member, every request but a read is answered `503` with `Retry-After`, and its message names the leader. `PUT /api/drain` is the exception, since a drain is set on each member. Before its first write in a term, a new leader marks `FAILED` the live runs every other member created, and removes the faults Kates made before it took the lead; a member that starts up recovers only its own runs. A leader that loses its majority stops taking writes once its lease runs out, within about a second, even before it steps down. A request that needs a database Kates cannot reach is answered `503` with `Retry-After: 5` within about 30 s, the JDBC socket timeout, instead of waiting on the connection. +`quorum` says whether this Kates is a member of a quorum, which it is only when `kates.quorum.peers` is set. Without one it is `{"enabled": false}`, and the instance acts alone. A member also reports `state`, its Raft server's life cycle (`RUNNING` while it serves). Ratis closes a member's server after a pause longer than 60 s, such as a long garbage collection or a frozen node; the member then starts it again from its directory within a few seconds, and until it serves again only `state` and `memberId` are reported. If it still does not serve after `kates.quorum.liveness-after` (30 s), `/q/health/live` answers DOWN so the kubelet restarts the pod. A member reports its `memberId` and, while it serves, the `leaderId` it knows of (`null` during an election), the Raft `term`, `appliedIndex`, the last log entry it applied, and `leader`: true once it leads and has committed an entry of its term. A member whose log is empty, new or with its directory lost, is no voter until it has caught up to an entry: three members of which one is empty and one is down elect nobody. Members talk to each other over mutual TLS on their own port: `kates.quorum.tls.cert`, `kates.quorum.tls.key` (PKCS#8 PEM) and `kates.quorum.tls.ca` are required, and so is `kates.quorum.dir`, which keeps a member's term and vote and must outlive a restart. Only the leader takes writes: on any other member, every request but a read is answered `503` with `Retry-After`, and its message names the leader. `PUT /api/drain` is the exception, since a drain is set on each member. A member checks that it still leads right before each fault goes into Kubernetes, so a member that stopped leading since its plan started injects nothing, and every object a fault writes carries `kates.io/term`, the writer's term, and `kates.io/deadline`, the write's time plus `kates.engine.max-duration-ms`. The leader reconciles every `kates.quorum.reconcile-interval` (10 s): it removes any Kates fault of an older term or past its deadline, NetworkPolicies, ChaosEngines (uncordoning the node a node-drain named) and scale-downs in every namespace alike, and marks `FAILED` the live runs every other member created, retrying while PostgreSQL is unreachable rather than waiting on it to lead. A member that stops leading stops its own runs at once; the new leader stores them `FAILED`. A member that starts up recovers only its own runs. A leader that loses its majority stops taking writes once its lease runs out, within about a second, even before it steps down. A request that needs a database Kates cannot reach is answered `503` with `Retry-After: 5` within about 30 s, the JDBC socket timeout, instead of waiting on the connection. #### GET /api/drain From 14d4202ff5bbff91c9cd4d6b3a0df5497262c42b Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 15:04:09 +0100 Subject: [PATCH 5/6] fix(backend): a fault carries the term it was confirmed in, and a plan stops when its term ends From the branch review: - requireWrites() returns the term it confirmed, and every object of that fault carries it: read again per object, a paused leader could stamp a fault with the new leader's term and keep it for 2 h. - A plan stops before its next step once the term it started in ends. A member that no longer leads skips the auto-rollback and gives back no replicas at release: the rollback deletes every Kates partition in the namespace, the next leader's too, which undoes this one's by term. - A member that leads again fails its own runs it settled when it lost the lead; a revived member that led says it lost the lead. - A refused NetworkPolicy or ChaosEngine list holds reports back, and the database step runs after the Kubernetes undos in each pass. - The node uncordon is removed: the shipped RBAC cannot patch nodes, and the cordon is Litmus'. --- cli/cmd/mcp_caveats.go | 12 +- cli/cmd/mcp_tools_chaos.go | 18 +-- cli/cmd/mcp_tools_runs_activity.go | 2 +- cli/cmd/mcp_tools_runs_list.go | 10 +- cli/cmd/mcp_tools_scenario.go | 4 +- docs/book/11-api-reference.md | 2 +- .../kates/chaos/KubernetesChaosProvider.java | 32 +++--- .../kates/chaos/LitmusChaosProvider.java | 5 +- .../DisruptionConcurrencyGuard.java | 3 +- .../disruption/DisruptionOrchestrator.java | 19 +++- .../DisruptionOrphanReconciler.java | 62 +++-------- .../kates/engine/TestOrchestrator.java | 8 ++ .../java/com/bmscomp/kates/quorum/Quorum.java | 14 ++- .../chaos/KubernetesChaosProviderTest.java | 26 ++++- .../kates/chaos/LitmusChaosProviderTest.java | 2 +- ...sruptionConcurrencyGuardScaleDownTest.java | 12 ++ .../DisruptionOrchestratorQuorumTest.java | 105 ++++++++++++++++++ .../DisruptionOrphanReconcilerTest.java | 54 +++++++-- .../engine/TestOrchestratorQuorumTest.java | 17 ++- .../com/bmscomp/kates/quorum/QuorumTest.java | 33 ++++++ 20 files changed, 335 insertions(+), 105 deletions(-) create mode 100644 kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrchestratorQuorumTest.java diff --git a/cli/cmd/mcp_caveats.go b/cli/cmd/mcp_caveats.go index fd3fc700a..16b5fc5eb 100644 --- a/cli/cmd/mcp_caveats.go +++ b/cli/cmd/mcp_caveats.go @@ -76,7 +76,7 @@ var mcpCaveatsCore = []mcpCaveat{ Short: "A LOAD run is one producer and one consumer whatever numProducers says; only STRESS runs parallel producers.", Text: "A LOAD run is one producer and one consumer, whatever numProducers and numConsumers say, " + "so it cannot show how the cluster behaves under parallel clients. STRESS starts one producer per numProducers.", - Refs: []string{mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1820-1829", "case LOAD ->", "case STRESS ->")}, + Refs: []string{mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1828-1837", "case LOAD ->", "case STRESS ->")}, }, { ID: mcpCaveatReaperDeadline, @@ -99,7 +99,7 @@ var mcpCaveatsCore = []mcpCaveat{ "kates/src/main/resources/application.properties:336-350", mcpAnchoredRef(mcpJava+"engine/TestTimeoutReaper.java:49-51,67-76,170-173,201-214", "TestResult.TaskStatus.STOPPING", "kates.engine.reaper-grace-ms", "\"-submission\"", "\"Timeout: still \""), - mcpJava + "engine/TestOrchestrator.java:151-241,283-286,460,1275-1339,1405-1414,1469-1501", + mcpJava + "engine/TestOrchestrator.java:151-249,291-294,468,1283-1347,1413-1422,1477-1509", mcpJava + "persistence/EntityMapper.java:161-170", "kates/src/main/resources/db/migration/V24__run_planned_duration.sql:1-13", mcpJava + "domain/TestRun.java:520-523", @@ -123,10 +123,10 @@ var mcpCaveatsCore = []mcpCaveat{ "integrity-cg-integrity, without transactions.", Refs: []string{ mcpJava + "domain/TestSpec.java:14-32,111-113", - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:252,275-277,325", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:260,283-285,333", "applyTypeDefaults(type, request.getSpec())", ".withRequestedSpec(", "buildTasks(type, spec, run.getId())"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1195-1241", "TestSpec applyTypeDefaults(", "return merged;"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1799-1803,1894-1898", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1203-1249", "TestSpec applyTypeDefaults(", "return merged;"), + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1807-1811,1902-1906", "Only when asked", `"-integrity-0-group")`), mcpAnchoredRef(mcpJava+"engine/NativeKafkaBackend.java:792", `task.getConsumerGroup() + "-integrity"`), mcpAnchoredRef(mcpJava+"persistence/EntityMapper.java:56,85-86,146-148", @@ -177,7 +177,7 @@ var mcpCaveatsCore = []mcpCaveat{ Text: "A TUNE_* run executes one produce task with the spec's single configuration: it measures one setting " + "and compares none.", Refs: []string{ - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1905-1906", "case TUNE_REPLICATION,", "-tune-0"), + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1913-1914", "case TUNE_REPLICATION,", "-tune-0"), }, }, { diff --git a/cli/cmd/mcp_tools_chaos.go b/cli/cmd/mcp_tools_chaos.go index b8d836790..9be93f32c 100644 --- a/cli/cmd/mcp_tools_chaos.go +++ b/cli/cmd/mcp_tools_chaos.go @@ -147,7 +147,7 @@ var mcpCaveatsChaos = []mcpCaveat{ mcpAnchoredRef(mcpJava+"disruption/DisruptionSafetyGuard.java:298-301,874-943", "boolean canExecute = checkRbacPermissions(spec);", `"Insufficient RBAC permissions for "`, "boolean checkRbacPermissions(FaultSpec spec)", `"RBAC check failed, assuming permitted"`), - mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:42-60,94-96,282", + mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:42-60,94-96,281", "ON_KUBERNETES_API = EnumSet.of(", "DisruptionType.POD_DELETE,", "DisruptionType.NETWORK_PARTITION);", "return kubernetes.triggerFault(spec, onInject);", `engineSpec.chaosServiceAccount = "litmus-admin";`), mcpAnchoredRef("kates/src/main/resources/application.properties:274-275", @@ -226,8 +226,8 @@ var mcpCaveatsChaos = []mcpCaveat{ "if (selected != null && selected.isAvailable())", "new NoOpChaosProvider()", "matches no chaos provider"), mcpJava + "chaos/NoOpChaosProvider.java:27-32", mcpJava + "chaos/ChaosOutcome.java:77-81", - mcpJava + "disruption/DisruptionOrchestrator.java:157-160,221", - mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:347-351", + mcpJava + "disruption/DisruptionOrchestrator.java:170-173,234", + mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:360-364", "boolean injected = disruptionStart != null;", "The fault never went in"), mcpJava + "disruption/DisruptionImpactScorer.java:72-77", }, @@ -242,7 +242,7 @@ var mcpCaveatsChaos = []mcpCaveat{ "scores as degradation. Without Prometheus both are 0. Judge latency and throughput from the summary " + "and the steps' impactDeltas instead.", Refs: []string{ - mcpJava + "disruption/DisruptionOrchestrator.java:184-189,200-201", + mcpJava + "disruption/DisruptionOrchestrator.java:197-202,213-214", mcpJava + "disruption/PrometheusMetricsCapture.java:152-171", mcpJava + "disruption/DisruptionImpactScorer.java:92-130", }, @@ -264,7 +264,7 @@ var mcpCaveatsChaos = []mcpCaveat{ "measured them for a step whose fault never went in as well; a report does not say which of the two counted " + "its times.", Refs: []string{ - mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:302,313-315,347-356,393-399,503-509", + mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:315,326-328,360-369,406-412,518-524", "new RecoveryClocks(session, isrTracker, lagTracker)", ".triggerFault(spec, clocks::start)", "boolean injected = disruptionStart != null;", "if (step.requireRecovery() && injected)", "Duration strimziRecovery = injected", "injected && isrTracker != null ? isrTracker.stop() : null", @@ -275,14 +275,14 @@ var mcpCaveatsChaos = []mcpCaveat{ mcpAnchoredRef(mcpJava+"disruption/KafkaIntelligenceService.java:125-127,198-217,241-246,310-324", "public void markDisruptionStart(Instant at)", "private Duration computeTimeToFullIsr()", "this.baselineLag = timeline.getLast().totalLag();", "private Duration computeTimeToLagRecovery(long baseLag)"), - mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:94-106,115-139", + mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:94-106,115-138", "return kubernetes.triggerFault(spec, onInject);", "CompletableFuture.delayedExecutor(spec.delayBeforeSec(), TimeUnit.SECONDS, executor.get());", "long startNanos = System.nanoTime();", "buildChaosEngine(spec, engineName, experimentName)", "onInject.accept(startNanos);", ".resource(engine)"), mcpAnchoredRef(mcpJava+"chaos/KubernetesChaosProvider.java:168-180", "Thread.sleep(spec.delayBeforeSec() * 1000L);", "onInject.accept(startNanos);", - "applyDisruption(spec, engineName);"), + "applyDisruption(spec, engineName, marks);"), mcpAnchoredRef(mcpJava+"chaos/PodTargets.java:58,69-75", "public static List resolve(KubernetesClient client, FaultSpec spec)", "throw new IllegalStateException("), @@ -299,7 +299,7 @@ var mcpCaveatsChaos = []mcpCaveat{ Refs: []string{ mcpJava + "disruption/KafkaIntelligenceService.java:129-154,165-168", mcpJava + "disruption/DisruptionImpactScorer.java:42-45,132-157", - mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:266-269,398", + mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:279-282,411", "intelligence.startIsrTracking(", "isrTracker.stop()"), }, }, @@ -316,7 +316,7 @@ var mcpCaveatsChaos = []mcpCaveat{ "starts while another runs a plan marks that plan's report too, until the plan ends and stores its " + "outcome.", Refs: []string{ - mcpJava + "disruption/DisruptionOrphanReconciler.java:73-77,82-111,118-167,359-393", + mcpJava + "disruption/DisruptionOrphanReconciler.java:72-76,81-110,117-167,333-367", mcpJava + "disruption/DisruptionReportRepository.java:35-60,93-102", mcpJava + "disruption/DisruptionPersistence.java:26-37", mcpJava + "disruption/DisruptionLauncher.java:114-121,129-144", diff --git a/cli/cmd/mcp_tools_runs_activity.go b/cli/cmd/mcp_tools_runs_activity.go index a67257109..3da8a1181 100644 --- a/cli/cmd/mcp_tools_runs_activity.go +++ b/cli/cmd/mcp_tools_runs_activity.go @@ -486,7 +486,7 @@ var mcpCaveatsRunsActivity = []mcpCaveat{ mcpJava + "disruption/DisruptionLauncher.java:114-144", mcpJava + "disruption/DisruptionPersistence.java:16-37", mcpJava + "disruption/DisruptionReportRepository.java:20-43", - mcpJava + "disruption/DisruptionOrphanReconciler.java:359-393", + mcpJava + "disruption/DisruptionOrphanReconciler.java:333-367", mcpJava + "disruption/DisruptionTemplateResource.java:70-72", mcpJava + "disruption/DisruptionScheduler.java:112-124", mcpJava + "disruption/DisruptionReportEntity.java:39-47", diff --git a/cli/cmd/mcp_tools_runs_list.go b/cli/cmd/mcp_tools_runs_list.go index 9f0e61209..c989ce290 100644 --- a/cli/cmd/mcp_tools_runs_list.go +++ b/cli/cmd/mcp_tools_runs_list.go @@ -1032,7 +1032,7 @@ var mcpCaveatsRunsList = []mcpCaveat{ mcpAnchoredRef(mcpJava+"report/ReportGenerator.java:220-249,283-294", "report.setSummary(MetricUtils.computeSummary(results));", "summaries.put(phase, MetricUtils.computeSummary(rows))"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:529-538,2259-2275", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:537-546,2267-2283", "TestResult.TaskStatus.PENDING : TestResult.TaskStatus.RUNNING", "private static TestRun withUnfinishedTasksFailed("), }, @@ -1071,7 +1071,7 @@ var mcpCaveatsRunsList = []mcpCaveat{ "except for a run cancelled before its tasks existed; a cancel through the REST API also leaves a CANCEL " + "audit row.", Refs: []string{ - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:2206-2288", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:2214-2296", `"Cancelled by user"`, `EventKind.FAILED, "cancelled"`, "return run.withResults(updatedResults);"), mcpAnchoredRef(mcpJava+"api/TestResource.java:572-617", `@Path("/{id}/cancel")`, `auditService.record("CANCEL"`, `"Test cancelled; it is stored as FAILED"`), @@ -1104,14 +1104,14 @@ var mcpCaveatsRunsList = []mcpCaveat{ "say which the API is, but a scenario run's tasks do: on an older API they all start within moments " + "of one another.", Refs: []string{ - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:446-448,505-516,518-534", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:454-456,513-524,526-542", "applyTypeDefaults(type, scenario.getBaseSpec())", ".withRequestedSpec(", "long phaseStartMs = System.currentTimeMillis();", "scenario.resolveSpecForPhase(phase, baseSpec)", "phaseStartMs = saturatedSum(phaseStartMs", "backend.submit(task)", "TestResult.TaskStatus.PENDING"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1607-1635,1648-1656,1716-1755,1765-1777", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1615-1643,1656-1664,1724-1763,1773-1785", "MAX_RAMP_STEPS = 100", "scenarioInapplicableFields(TestScenario scenario", "is a SPIKE phase", "has none (", "check no record CRCs", "has the name of an earlier phase"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1902-1971", "List buildPhaseTask(", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1910-1979", "List buildPhaseTask(", "int baseTarget = Math.max(1, spec.getThroughput() / steps);", "saturatedSum(startAtMs, s * stepMs)", `taskId + "-spike"`), mcpAnchoredRef(mcpJava+"domain/TestScenario.java:115-190", diff --git a/cli/cmd/mcp_tools_scenario.go b/cli/cmd/mcp_tools_scenario.go index f385b8f45..f2592f4a8 100644 --- a/cli/cmd/mcp_tools_scenario.go +++ b/cli/cmd/mcp_tools_scenario.go @@ -1242,7 +1242,7 @@ var mcpCaveatsScenario = []mcpCaveat{ "plans/mcp-server.md:387-405", mcpJava + "api/TestResource.java:118-188", mcpJava + "domain/TestSpec.java:34-113", - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1527-1602", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1535-1610", "Map inapplicableFields(", "the trogdor backend cannot run a transactional producer"), mcpAnchoredRef("cli/cmd/apply.go:238-307", "for i, scenario := range sf.Scenarios {", "apiClient.CreateTest(a.ctx, scenarioToRequest(scenario))", @@ -1259,7 +1259,7 @@ var mcpCaveatsScenario = []mcpCaveat{ "listed as defaulted may differ on the cluster that runs the scenario.", Refs: []string{ mcpJava + "config/TestTypeDefaults.java:22-64,323-464", - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1195-1241", "TestSpec applyTypeDefaults(", "return merged;"), + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1203-1249", "TestSpec applyTypeDefaults(", "return merged;"), mcpAnchoredRef("kates/src/main/resources/application.properties:73-114", "# Per-type overrides: STRESS", "kates.tests.roundtrip.throughput=10000"), mcpAnchoredRef("charts/kates/values.yaml:690-758", "defaults:", "tests:", `throughput: "10000"`), diff --git a/docs/book/11-api-reference.md b/docs/book/11-api-reference.md index 4f30e0e46..c25c122c4 100644 --- a/docs/book/11-api-reference.md +++ b/docs/book/11-api-reference.md @@ -237,7 +237,7 @@ curl -s "$BASE/api/health" The response also contains a `tests` object with the resolved default configuration for every test type. When Kafka is unreachable, the endpoint still returns `200 OK`, but with `"status": "DEGRADED"` and `"kafka.status": "DOWN"`. -`quorum` says whether this Kates is a member of a quorum, which it is only when `kates.quorum.peers` is set. Without one it is `{"enabled": false}`, and the instance acts alone. A member also reports `state`, its Raft server's life cycle (`RUNNING` while it serves). Ratis closes a member's server after a pause longer than 60 s, such as a long garbage collection or a frozen node; the member then starts it again from its directory within a few seconds, and until it serves again only `state` and `memberId` are reported. If it still does not serve after `kates.quorum.liveness-after` (30 s), `/q/health/live` answers DOWN so the kubelet restarts the pod. A member reports its `memberId` and, while it serves, the `leaderId` it knows of (`null` during an election), the Raft `term`, `appliedIndex`, the last log entry it applied, and `leader`: true once it leads and has committed an entry of its term. A member whose log is empty, new or with its directory lost, is no voter until it has caught up to an entry: three members of which one is empty and one is down elect nobody. Members talk to each other over mutual TLS on their own port: `kates.quorum.tls.cert`, `kates.quorum.tls.key` (PKCS#8 PEM) and `kates.quorum.tls.ca` are required, and so is `kates.quorum.dir`, which keeps a member's term and vote and must outlive a restart. Only the leader takes writes: on any other member, every request but a read is answered `503` with `Retry-After`, and its message names the leader. `PUT /api/drain` is the exception, since a drain is set on each member. A member checks that it still leads right before each fault goes into Kubernetes, so a member that stopped leading since its plan started injects nothing, and every object a fault writes carries `kates.io/term`, the writer's term, and `kates.io/deadline`, the write's time plus `kates.engine.max-duration-ms`. The leader reconciles every `kates.quorum.reconcile-interval` (10 s): it removes any Kates fault of an older term or past its deadline, NetworkPolicies, ChaosEngines (uncordoning the node a node-drain named) and scale-downs in every namespace alike, and marks `FAILED` the live runs every other member created, retrying while PostgreSQL is unreachable rather than waiting on it to lead. A member that stops leading stops its own runs at once; the new leader stores them `FAILED`. A member that starts up recovers only its own runs. A leader that loses its majority stops taking writes once its lease runs out, within about a second, even before it steps down. A request that needs a database Kates cannot reach is answered `503` with `Retry-After: 5` within about 30 s, the JDBC socket timeout, instead of waiting on the connection. +`quorum` says whether this Kates is a member of a quorum, which it is only when `kates.quorum.peers` is set. Without one it is `{"enabled": false}`, and the instance acts alone. A member also reports `state`, its Raft server's life cycle (`RUNNING` while it serves). Ratis closes a member's server after a pause longer than 60 s, such as a long garbage collection or a frozen node; the member then starts it again from its directory within a few seconds, and until it serves again only `state` and `memberId` are reported. If it still does not serve after `kates.quorum.liveness-after` (30 s), `/q/health/live` answers DOWN so the kubelet restarts the pod. A member reports its `memberId` and, while it serves, the `leaderId` it knows of (`null` during an election), the Raft `term`, `appliedIndex`, the last log entry it applied, and `leader`: true once it leads and has committed an entry of its term. A member whose log is empty, new or with its directory lost, is no voter until it has caught up to an entry: three members of which one is empty and one is down elect nobody. Members talk to each other over mutual TLS on their own port: `kates.quorum.tls.cert`, `kates.quorum.tls.key` (PKCS#8 PEM) and `kates.quorum.tls.ca` are required, and so is `kates.quorum.dir`, which keeps a member's term and vote and must outlive a restart. Only the leader takes writes: on any other member, every request but a read is answered `503` with `Retry-After`, and its message names the leader. `PUT /api/drain` is the exception, since a drain is set on each member. A member checks that it still leads right before each fault goes into Kubernetes, so a member that stopped leading since its plan started injects nothing, and every object a fault writes carries `kates.io/term`, the writer's term, and `kates.io/deadline`, the write's time plus `kates.engine.max-duration-ms`. The leader reconciles every `kates.quorum.reconcile-interval` (10 s): it removes any Kates fault of an older term or past its deadline, NetworkPolicies, ChaosEngines and scale-downs in every namespace alike, and marks `FAILED` the live runs every other member created, retrying while PostgreSQL is unreachable rather than waiting on it to lead. A member that stops leading stops its own runs at once; the new leader stores them `FAILED`. A member that starts up recovers only its own runs. A leader that loses its majority stops taking writes once its lease runs out, within about a second, even before it steps down. A request that needs a database Kates cannot reach is answered `503` with `Retry-After: 5` within about 30 s, the JDBC socket timeout, instead of waiting on the connection. #### GET /api/drain diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java b/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java index 4e57fcfb6..bbfafac72 100644 --- a/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java +++ b/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java @@ -108,8 +108,8 @@ public IncompleteScaleDownException(String message) { * bound. shortcut: a fault the live leader leaves after a crashed plan thread lives up to that * bound (2 h); upgrade to the plan's own bound if that is seen. */ - Map marks() { - return FaultMarks.of(quorum.term(), Instant.now().plusMillis(maxDurationMs)); + Map marks(long term) { + return FaultMarks.of(term, Instant.now().plusMillis(maxDurationMs)); } @Override @@ -132,13 +132,13 @@ public String name() { * error that said nothing about it, and a few faults per plan are no load * to shed from the API server. */ - private void applyDisruption(FaultSpec spec, String engineName) throws Exception { + private void applyDisruption(FaultSpec spec, String engineName, Map marks) throws Exception { switch (spec.disruptionType()) { case POD_KILL -> executePodKill(spec); case POD_DELETE -> executePodDelete(spec); - case NETWORK_PARTITION -> executeNetworkPartition(spec, engineName); + case NETWORK_PARTITION -> executeNetworkPartition(spec, engineName, marks); case ROLLING_RESTART -> executeRollingRestart(spec); - case SCALE_DOWN -> executeScaleDown(spec); + case SCALE_DOWN -> executeScaleDown(spec, marks); case LEADER_ELECTION -> executePodKill(spec); case CPU_STRESS -> executeCpuStress(spec, engineName); case IO_STRESS -> executeIoStress(spec, engineName); @@ -174,10 +174,10 @@ public CompletableFuture triggerFault(FaultSpec spec, LongConsumer } // Confirmed after the delay, right before any write: a member that stopped // leading since the plan started writes nothing (J4 F1). - quorum.requireWrites(); + Map marks = marks(quorum.requireWrites()); // The fault goes in from here, once its delay is over. onInject.accept(startNanos); - applyDisruption(spec, engineName); + applyDisruption(spec, engineName, marks); return ChaosOutcome.success( engineName, spec.experimentName(), start, Instant.now(), startNanos, null, null, null); @@ -238,11 +238,12 @@ private void executePodDelete(FaultSpec spec) { * failed or the wait was interrupted: a NetworkPolicy left behind cuts the * pod off until rollback, which a failed fault does not get. */ - private void executeNetworkPartition(FaultSpec spec, String engineName) throws InterruptedException { + private void executeNetworkPartition(FaultSpec spec, String engineName, Map marks) + throws InterruptedException { Exception failure = null; try { for (String podName : PodTargets.resolve(client, spec)) { - isolatePod(spec, podName); + isolatePod(spec, podName, marks); } // The chaos limits refuse a partition without a duration; one that // gets here anyway is a blip. @@ -281,7 +282,7 @@ private void removePartition(String namespace, String engineName, Exception fail } } - private void isolatePod(FaultSpec spec, String podName) { + private void isolatePod(FaultSpec spec, String podName, Map marks) { String policyName = "kates-netpol-" + podName; LOG.info("NETWORK_PARTITION: applying deny NetworkPolicy for " + podName); @@ -301,7 +302,7 @@ private void isolatePod(FaultSpec spec, String podName) { .withName(policyName) .withNamespace(spec.targetNamespace()) .addToLabels("managed-by", "kates") - .addToAnnotations(marks()) + .addToAnnotations(marks) .endMetadata() .withNewSpec() .withNewPodSelector() @@ -463,7 +464,7 @@ private void removeManualRollingUpdate(String namespace, String podName) { * not run by Strimzi) is scaled down by one, but never below one replica. * Each records its original replica count for rollback. */ - private void executeScaleDown(FaultSpec spec) throws InterruptedException { + private void executeScaleDown(FaultSpec spec, Map marks) throws InterruptedException { ScaleDownTargets.Targets targets = ScaleDownTargets.resolve(client, spec); targets.skipped().forEach(reason -> LOG.info("SCALE_DOWN: skipping, " + reason)); if (targets.isEmpty()) { @@ -472,11 +473,11 @@ private void executeScaleDown(FaultSpec spec) throws InterruptedException { } if (!targets.pools().isEmpty()) { - new NodePoolScaleDown(client, scalePollIntervalMs, marks()).run(spec, targets.pools()); + new NodePoolScaleDown(client, scalePollIntervalMs, marks).run(spec, targets.pools()); } int scaled = 0; for (String name : targets.statefulSets()) { - if (scaleDownStatefulSet(spec.targetNamespace(), name)) { + if (scaleDownStatefulSet(spec.targetNamespace(), name, marks)) { scaled++; } } @@ -486,7 +487,7 @@ private void executeScaleDown(FaultSpec spec) throws InterruptedException { } } - private boolean scaleDownStatefulSet(String namespace, String name) { + private boolean scaleDownStatefulSet(String namespace, String name, Map marks) { StatefulSet ss = client.apps() .statefulSets() .inNamespace(namespace) @@ -510,7 +511,6 @@ private boolean scaleDownStatefulSet(String namespace, String name) { boolean alreadySnapshotted = annotations != null && annotations.containsKey(ORIGINAL_REPLICAS_ANNOTATION); // The marks are written every time: a snapshot an earlier plan left carries an older term, // which the reconciler would take for a dead leader's fault and undo under this one. - Map marks = marks(); String scaledDownAt = String.valueOf(System.currentTimeMillis()); client.apps() .statefulSets() diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java b/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java index 8928458d5..7797e710e 100644 --- a/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java +++ b/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java @@ -125,9 +125,8 @@ private CompletableFuture inject(FaultSpec spec, LongConsumer onIn String resultName = engineName + "-" + experimentName; // Confirmed right before the engine exists: a member that stopped leading writes nothing. - quorum.requireWrites(); - engine.getMetadata() - .setAnnotations(FaultMarks.of(quorum.term(), Instant.now().plusMillis(maxDurationMs))); + long term = quorum.requireWrites(); + engine.getMetadata().setAnnotations(FaultMarks.of(term, Instant.now().plusMillis(maxDurationMs))); // Once the pods are picked, before the engine that injects the fault exists. onInject.accept(startNanos); diff --git a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuard.java b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuard.java index c7680ffde..d72a02e1d 100644 --- a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuard.java +++ b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuard.java @@ -85,7 +85,8 @@ public boolean tryAcquire(String target, String token) { */ public void release(String target, String token) { Long since = acquiredAt.remove(token); - if (since != null && token.equals(activeByTarget.get(target))) { + // A holder that no longer leads gives nothing back: the next leader restores by term. + if (since != null && token.equals(activeByTarget.get(target)) && quorum.acceptsWrites()) { restoreScaleDownsSince(since); } activeByTarget.remove(target, token); diff --git a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrchestrator.java b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrchestrator.java index d68e70540..cea7f8cf7 100644 --- a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrchestrator.java +++ b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrchestrator.java @@ -56,6 +56,10 @@ public class DisruptionOrchestrator { @Inject DisruptionConcurrencyGuard concurrencyGuard; + // Built without config, as the tests build it, a Quorum is alone and always leads. + @Inject + com.bmscomp.kates.quorum.Quorum quorum = new com.bmscomp.kates.quorum.Quorum(); + @ConfigProperty(name = "kates.chaos.kafka.namespace", defaultValue = "kafka") String kafkaNamespace; @@ -144,8 +148,17 @@ private DisruptionReport executePlan(String disruptionId, DisruptionPlan plan) { double totalThroughputDelta = 0; double maxP99Spike = 0; + // A plan belongs to the term it started in: a member that lost the lead, or leads again in a + // later term, injects none of its remaining steps (J4 review). The next leader undoes the rest. + long term = quorum.term(); try { for (DisruptionPlan.DisruptionStep step : plan.getSteps()) { + if (quorum.term() != term || !quorum.acceptsWrites()) { + LOG.warnf( + "Plan '%s' stopped before step '%s': this member no longer leads term %d", + plan.getName(), step.name(), term); + break; + } LOG.info("Executing disruption step: " + step.name()); eventBus.emit( disruptionId, @@ -367,7 +380,7 @@ private DisruptionReport.StepReport executeStep( recovered = safetyGuard.verifyClusterState(); } - if (!recovered && plan.isAutoRollback()) { + if (!recovered && plan.isAutoRollback() && quorum.acceptsWrites()) { LOG.warn(" Recovery timeout — triggering auto-rollback for " + step.name()); safetyGuard.rollback(spec, outcome.engineName()); eventBus.emit( @@ -423,7 +436,9 @@ private DisruptionReport.StepReport executeStep( } catch (Exception e) { LOG.error("Step '" + step.name() + "' failed", e); - if (plan.isAutoRollback()) { + // A member that no longer leads undoes nothing: the rollback deletes every Kates partition + // in the namespace, the next leader's too, and that leader undoes this one's by term. + if (plan.isAutoRollback() && quorum.acceptsWrites()) { try { LOG.warn(" Auto-rollback on exception for step " + step.name()); safetyGuard.rollback(step.faultSpec(), ""); diff --git a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java index cd9c1678f..a6748ba66 100644 --- a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java +++ b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java @@ -11,7 +11,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import io.fabric8.kubernetes.api.model.NodeBuilder; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicy; import io.fabric8.kubernetes.client.KubernetesClient; @@ -140,18 +139,19 @@ void periodic() { /** One pass: with {@code startedAt}, also every Kates fault made before it, as at start or takeover. */ private synchronized void reconcile(Instant startedAt) { undoFailed = false; - if (orchestrator != null) { - try { - // Retried here every pass: a takeover with PostgreSQL down did not get to it (J4 F6). - orchestrator.recoverPreviousLeaders(); - } catch (RuntimeException e) { - LOG.debugf("The previous leader's runs are not recovered yet: %s", e.getMessage()); - } - } try { int policies = reconcileNetworkPolicies(startedAt); int engines = reconcileChaosEngines(startedAt); ScaleDownSnapshots.Restored scaled = reconcileScaleDowns(startedAt); + if (orchestrator != null) { + try { + // After the faults: a database that hangs must not hold a partition (J4 review). + // Retried every pass, since a takeover with PostgreSQL down did not get to it. + orchestrator.recoverPreviousLeaders(); + } catch (RuntimeException e) { + LOG.debugf("The previous leader's runs are not recovered yet: %s", e.getMessage()); + } + } // A report says INTERRUPTED only once its faults are out (s08b reported it with one in). int reports = undoFailed ? 0 : markInterruptedReports(startedAt != null ? startedAt : since); if (policies + engines + scaled.total() + reports > 0) { @@ -198,8 +198,8 @@ boolean isStray(ObjectMeta meta, Long ageCutoffMs, Instant now) { } /** - * Deletes the Kates ChaosEngines no live fault owns, in every namespace, and uncordons the node - * a node-drain engine named: Kates does not cordon, Litmus does, and nothing else would undo it. + * Deletes the Kates ChaosEngines no live fault owns, in every namespace, as Litmus cleanup does + * in-process. A node-drain's cordon is Litmus' to revert: Kates may not patch nodes. */ int reconcileChaosEngines(Instant startedAt) { Long cutoff = startedAt == null ? null : cutoff(startedAt); @@ -210,8 +210,11 @@ int reconcileChaosEngines(Instant startedAt) { .withLabel("managed-by", "kates") .list() .getItems(); - } catch (Exception e) { - // No Litmus, or no rights on its engines: nothing Kates made to undo there. + } catch (io.fabric8.kubernetes.client.KubernetesClientException e) { + // 404: no Litmus, so no engine Kates made. Anything else: whether one is in is unknown. + if (e.getCode() != 404) { + undoFailed = true; + } LOG.debugf("Could not list ChaosEngines: %s", e.getMessage()); return 0; } @@ -228,17 +231,6 @@ int reconcileChaosEngines(Instant startedAt) { .inNamespace(namespace) .withName(name) .delete(); - String node = drainedNode(engine); - if (node != null) { - kubeClient.nodes() - .withName(node) - .edit(n -> new NodeBuilder(n) - .editOrNewSpec() - .withUnschedulable(false) - .endSpec() - .build()); - LOG.warnf("Orphan recovery: uncordoned node %s, drained by ChaosEngine %s/%s", node, namespace, name); - } removed++; } catch (Exception e) { undoFailed = true; @@ -248,26 +240,6 @@ int reconcileChaosEngines(Instant startedAt) { return removed; } - /** The TARGET_NODE of a node-drain engine, or null. */ - static String drainedNode(ChaosEngine engine) { - if (engine.getSpec() == null || engine.getSpec().experiments == null) { - return null; - } - for (ChaosEngineSpec.Experiment experiment : engine.getSpec().experiments) { - if ("node-drain".equals(experiment.name) - && experiment.spec != null - && experiment.spec.components != null - && experiment.spec.components.env != null) { - for (ChaosEngineSpec.EnvVar env : experiment.spec.components.env) { - if ("TARGET_NODE".equals(env.name) && env.value != null && !env.value.isBlank()) { - return env.value; - } - } - } - } - return null; - } - /** * Deletes the Kates-managed NetworkPolicies (network partitions) left from * before this process started, in every namespace, as cleanup does: a @@ -290,6 +262,8 @@ int reconcileNetworkPolicies(Instant startedAt) { // (dev, unit tests) is normal and must not dump a stack trace on // every boot. LOG.warnf("Could not reconcile orphaned NetworkPolicies: %s", e.getMessage()); + // Whether a partition is still in is unknown: no report says INTERRUPTED on that. + undoFailed = true; return 0; } int removed = 0; diff --git a/kates/src/main/java/com/bmscomp/kates/engine/TestOrchestrator.java b/kates/src/main/java/com/bmscomp/kates/engine/TestOrchestrator.java index 5e3b18ded..678279235 100644 --- a/kates/src/main/java/com/bmscomp/kates/engine/TestOrchestrator.java +++ b/kates/src/main/java/com/bmscomp/kates/engine/TestOrchestrator.java @@ -177,6 +177,14 @@ public void recoverPreviousLeaders() { repository.findByStatusAndMember(ORPHAN_STATUSES, quorum.memberId(), false), "when the member running it stopped leading", "takeover"); + // Its own too, once it no longer runs them: it settled them when it lost the lead, and + // leads again (J4 review). A run it still runs has its handles. + recover( + repository.findByStatusAndMember(ORPHAN_STATUSES, quorum.memberId(), true).stream() + .filter(run -> !activeHandles.containsKey(run.getId())) + .toList(), + "when the member running it stopped leading", + "takeover"); } /** diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java index 7a86400b6..b9b2bdc96 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java @@ -158,6 +158,10 @@ private void revive() { LOG.warnf( "quorum member %s: Ratis server %s, starting it again from its disk", current.memberId(), current.state()); + // Ratis closes a paused server without telling its state machine the lead changed. + if (recoveredTerm >= 0) { + lostLead(recoveredTerm); + } try { current.close(); } catch (IOException | RuntimeException e) { @@ -205,14 +209,16 @@ public long term() { } /** - * Throws unless this member may write now, as {@link #acceptsWrites()} decides: called right - * before a fault goes into Kubernetes, so a member that stopped leading since its plan started - * injects nothing. + * Throws unless this member may write now, as {@link #acceptsWrites()} decides, and returns the + * term it confirmed: called right before a fault goes into Kubernetes, so a member that stopped + * leading since its plan started injects nothing, and what it writes carries that term, not one + * read later. */ - public void requireWrites() { + public long requireWrites() { if (!acceptsWrites()) { throw new com.bmscomp.kates.engine.NotLeaderException(leaderId()); } + return term(); } /** The leader this member knows of, or null with no quorum or during an election. */ diff --git a/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java b/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java index f09d9095c..081866c8e 100644 --- a/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java +++ b/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java @@ -999,7 +999,9 @@ private void createKafkaStatefulSet() { private static com.bmscomp.kates.quorum.Quorum quorumAt(long term, boolean leads) { com.bmscomp.kates.quorum.Quorum quorum = org.mockito.Mockito.mock(com.bmscomp.kates.quorum.Quorum.class); org.mockito.Mockito.when(quorum.term()).thenReturn(term); - if (!leads) { + if (leads) { + org.mockito.Mockito.when(quorum.requireWrites()).thenReturn(term); + } else { org.mockito.Mockito.doThrow(new com.bmscomp.kates.engine.NotLeaderException("m1")) .when(quorum) .requireWrites(); @@ -1089,4 +1091,26 @@ private static T await(java.util.function.Supplier found) throws Interrup } throw new AssertionError("not within 5 s"); } + + @Test + void aFaultIsMarkedWithTheTermItWasConfirmedIn() throws Exception { + createZonedBrokers(); + // Confirmed in term 7; by the time the object is built the member has heard of term 8. + provider.quorum = quorumAt(7, true); + org.mockito.Mockito.when(provider.quorum.term()).thenReturn(8L); + FaultSpec spec = FaultSpec.builder("split") + .targetPod("krafter-brokers-0") + .disruptionType(DisruptionType.NETWORK_PARTITION) + .chaosDurationSec(30) + .build(); + + provider.triggerFault(spec); + var policy = await(() -> client.network() + .networkPolicies() + .inNamespace("kafka") + .withName("kates-netpol-krafter-brokers-0") + .get()); + + assertMarked(policy.getMetadata().getAnnotations(), "7"); + } } diff --git a/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java b/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java index 2ba2b3e1d..560f3c1ec 100644 --- a/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java +++ b/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java @@ -176,7 +176,7 @@ void anEngineCarriesItsTermAndDeadline() throws Exception { install("disk-fill"); provider.resultPollIntervalMs = 20; com.bmscomp.kates.quorum.Quorum quorum = org.mockito.Mockito.mock(com.bmscomp.kates.quorum.Quorum.class); - when(quorum.term()).thenReturn(7L); + when(quorum.requireWrites()).thenReturn(7L); provider.quorum = quorum; FaultSpec spec = FaultSpec.builder("fill") .targetBrokerId(2) diff --git a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuardScaleDownTest.java b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuardScaleDownTest.java index c090c94fd..12b3b8fdc 100644 --- a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuardScaleDownTest.java +++ b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuardScaleDownTest.java @@ -93,4 +93,16 @@ void aReleaseByAnotherTokenRestoresNothing() { assertEquals(2, get("brokers").getSpec().getReplicas()); } + + @Test + void aHolderThatNoLongerLeadsGivesNothingBack() { + String target = guard.currentTarget(); + guard.tryAcquire(target, "plan#1"); + scaledDown("brokers", System.currentTimeMillis()); + guard.quorum = org.mockito.Mockito.mock(com.bmscomp.kates.quorum.Quorum.class); + + guard.release(target, "plan#1"); + + assertEquals(2, get("brokers").getSpec().getReplicas(), "the next leader gives it back"); + } } diff --git a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrchestratorQuorumTest.java b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrchestratorQuorumTest.java new file mode 100644 index 000000000..444d72f42 --- /dev/null +++ b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrchestratorQuorumTest.java @@ -0,0 +1,105 @@ +package com.bmscomp.kates.disruption; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.Instant; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.jupiter.api.Test; + +import com.bmscomp.kates.chaos.ChaosCoordinator; +import com.bmscomp.kates.chaos.ChaosOutcome; +import com.bmscomp.kates.chaos.DisruptionType; +import com.bmscomp.kates.chaos.FaultSpec; +import com.bmscomp.kates.chaos.K8sPodWatcher; +import com.bmscomp.kates.chaos.StrimziStateTracker; +import com.bmscomp.kates.engine.KatesMetrics; +import com.bmscomp.kates.quorum.Quorum; + +/** A plan belongs to the term it started in, and a member that no longer leads undoes nothing. */ +class DisruptionOrchestratorQuorumTest { + + private final DisruptionOrchestrator orchestrator = new DisruptionOrchestrator(); + private final Quorum quorum = mock(Quorum.class); + private final AtomicLong term = new AtomicLong(4); + + DisruptionOrchestratorQuorumTest() { + orchestrator.chaosCoordinator = mock(ChaosCoordinator.class); + orchestrator.podWatcher = mock(K8sPodWatcher.class); + orchestrator.strimziTracker = mock(StrimziStateTracker.class); + orchestrator.intelligence = mock(KafkaIntelligenceService.class); + orchestrator.safetyGuard = mock(DisruptionSafetyGuard.class); + orchestrator.prometheusCapture = mock(PrometheusMetricsCapture.class); + orchestrator.slaGrader = mock(SlaGrader.class); + orchestrator.eventBus = mock(DisruptionEventBus.class); + orchestrator.katesMetrics = mock(KatesMetrics.class); + orchestrator.concurrencyGuard = new DisruptionConcurrencyGuard(); + orchestrator.quorum = quorum; + when(quorum.term()).thenAnswer(invocation -> term.get()); + when(quorum.acceptsWrites()).thenReturn(true); + when(orchestrator.safetyGuard.validatePlan(any())) + .thenReturn(DisruptionSafetyGuard.ValidationResult.ok(List.of())); + when(orchestrator.safetyGuard.verifyClusterState()).thenReturn(true); + } + + private static DisruptionPlan.DisruptionStep kill(String name, boolean requireRecovery) { + return new DisruptionPlan.DisruptionStep( + name, + FaultSpec.builder(name).disruptionType(DisruptionType.POD_KILL).build(), + 0, + 0, + requireRecovery); + } + + private static CompletableFuture passed() { + Instant now = Instant.now(); + return CompletableFuture.completedFuture( + ChaosOutcome.success("engine", "kill", now, now, System.nanoTime(), null, null, null)); + } + + @Test + void aPlanStopsOnceTheTermItStartedInEnds() { + when(orchestrator.podWatcher.startWatching(any(), any())).thenReturn(mock(K8sPodWatcher.WatchSession.class)); + when(orchestrator.chaosCoordinator.triggerFault(any(), any())).thenAnswer(invocation -> { + term.set(5); // the member lost the lead and won it back during the first step + return passed(); + }); + DisruptionPlan plan = new DisruptionPlan(); + plan.setName("two-kills"); + plan.getSteps().add(kill("first", false)); + plan.getSteps().add(kill("second", false)); + + orchestrator.execute("d-1", plan); + + verify(orchestrator.chaosCoordinator, times(1)).triggerFault(any(), any()); + } + + @Test + void aMemberThatNoLongerLeadsDoesNotRollBack() throws Exception { + K8sPodWatcher.WatchSession session = mock(K8sPodWatcher.WatchSession.class); + when(session.awaitRecovery(org.mockito.ArgumentMatchers.anyLong(), any())).thenReturn(false); + when(orchestrator.podWatcher.startWatching(any(), any())).thenReturn(session); + when(orchestrator.chaosCoordinator.triggerFault(any(), any())).thenAnswer(invocation -> { + invocation.getArgument(1).accept(System.nanoTime()); + // The member loses the lead while its fault is in. + when(quorum.acceptsWrites()).thenReturn(false); + return passed(); + }); + DisruptionPlan plan = new DisruptionPlan(); + plan.setName("unrecovered"); + plan.setAutoRollback(true); + plan.getSteps().add(kill("kill", true)); + + orchestrator.execute("d-2", plan); + + verify(orchestrator.safetyGuard, never()).rollback(any(), any()); + } +} diff --git a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrphanReconcilerTest.java b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrphanReconcilerTest.java index 069073901..8e7d1db4e 100644 --- a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrphanReconcilerTest.java +++ b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrphanReconcilerTest.java @@ -185,8 +185,10 @@ private NetworkPolicy marked(String namespace, String name, String term) { .withNamespace(namespace) .withLabels(Map.of("managed-by", "kates")) .withAnnotations(Map.of( - com.bmscomp.kates.chaos.FaultMarks.TERM, term, - com.bmscomp.kates.chaos.FaultMarks.DEADLINE, Instant.now().plusSeconds(3600).toString())) + com.bmscomp.kates.chaos.FaultMarks.TERM, + term, + com.bmscomp.kates.chaos.FaultMarks.DEADLINE, + Instant.now().plusSeconds(3600).toString())) .endMetadata() .build()) .create(); @@ -221,7 +223,7 @@ void anOlderTermsPartitionGoesOnTheNextPass() { } @Test - void anOlderTermsEngineGoesAndItsNodeIsUncordoned() { + void anOlderTermsEngineGoes() { client.nodes() .resource(new io.fabric8.kubernetes.api.model.NodeBuilder() .withNewMetadata() @@ -259,8 +261,6 @@ void anOlderTermsEngineGoesAndItsNodeIsUncordoned() { .inAnyNamespace() .list() .getItems()); - assertTrue(!Boolean.TRUE.equals(client.nodes().withName("n1").get().getSpec().getUnschedulable()), - "the drained node is schedulable again"); } @Test @@ -307,12 +307,14 @@ void aFailedUndoLeavesReportsRunning() { reconciler.periodic(); org.mockito.Mockito.verify(reconciler.repository, org.mockito.Mockito.never()) - .findByStatusCreatedBefore(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any()); + .findByStatusCreatedBefore( + org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any()); reconciler.periodic(); assertNull(get("kafka", "kates-netpol-stuck")); org.mockito.Mockito.verify(reconciler.repository) - .findByStatusCreatedBefore(org.mockito.ArgumentMatchers.eq("RUNNING"), org.mockito.ArgumentMatchers.any()); + .findByStatusCreatedBefore( + org.mockito.ArgumentMatchers.eq("RUNNING"), org.mockito.ArgumentMatchers.any()); } @Test @@ -336,6 +338,42 @@ void aLoneInstanceKeepsItsOwnPartitionOnEveryPass() { reconciler.periodic(); - assertNotNull(get("kafka", "kates-netpol-mine"), "the periodic pass of a lone instance judges by deadline only"); + assertNotNull( + get("kafka", "kates-netpol-mine"), "the periodic pass of a lone instance judges by deadline only"); + } + + @Test + void aFailedListLeavesReportsRunning() { + // The Kubernetes API refuses the list: whether a partition is still in is unknown. + server.expect() + .get() + .withPath("/apis/networking.k8s.io/v1/networkpolicies?labelSelector=managed-by%3Dkates") + .andReturn(403, "forbidden") + .once(); + DisruptionOrphanReconciler reconciler = leaderAt(4); + + reconciler.periodic(); + + org.mockito.Mockito.verify(reconciler.repository, org.mockito.Mockito.never()) + .findByStatusCreatedBefore( + org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any()); + } + + @Test + void aPassRemovesFaultsBeforeAskingTheDatabase() { + marked("kafka", "kates-netpol-late", "3"); + DisruptionOrphanReconciler reconciler = leaderAt(4); + reconciler.orchestrator = org.mockito.Mockito.mock(com.bmscomp.kates.engine.TestOrchestrator.class); + java.util.concurrent.atomic.AtomicBoolean goneFirst = new java.util.concurrent.atomic.AtomicBoolean(); + org.mockito.Mockito.doAnswer(invocation -> { + goneFirst.set(get("kafka", "kates-netpol-late") == null); + return null; + }) + .when(reconciler.orchestrator) + .recoverPreviousLeaders(); + + reconciler.periodic(); + + assertTrue(goneFirst.get(), "a database that hangs must not hold the partition"); } } diff --git a/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java b/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java index 84d863740..90879d75d 100644 --- a/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java +++ b/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java @@ -69,7 +69,6 @@ void aTakeoverFailsAPreviousLeadersRuns() { orchestrator.recoverPreviousLeaders(new LeadershipAcquired(4, Instant.now())); assertFailedWith(previous, "when the member running it stopped leading"); - verify(engine.repository, never()).findByStatusAndMember(any(), anyString(), eq(true)); } @Test @@ -103,6 +102,22 @@ void aDeposedMemberStopsItsRuns() { engine.assertEverySlotFree(); } + @Test + @DisplayName("a member that leads again fails its own runs it stopped when it lost the lead") + void aRegainedLeaderFailsItsOwnRunsItNoLongerRuns() { + String live = engine.startRunning(); + String stopped = engine.storeEnded(TaskStatus.RUNNING); + when(engine.quorum.memberId()).thenReturn("kates-1"); + when(engine.repository.findByStatusAndMember(any(), eq("kates-1"), eq(false))).thenReturn(List.of()); + when(engine.repository.findByStatusAndMember(any(), eq("kates-1"), eq(true))) + .thenReturn(List.of(engine.rows.get(live), engine.rows.get(stopped))); + + orchestrator.recoverPreviousLeaders(); + + assertFailedWith(stopped, "when the member running it stopped leading"); + assertEquals(TaskStatus.RUNNING, engine.rows.get(live).getStatus(), "a run this member still runs is left"); + } + private void assertFailedWith(String id, String reason) { TestRun run = engine.rows.get(id); assertEquals(TaskStatus.FAILED, run.getStatus()); diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java index 0afb1d9a3..89addcb89 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java @@ -168,11 +168,44 @@ private void start(int count, AtomicBoolean failing) throws Exception { quorum.key = Optional.of(files[1].toString()); quorum.ca = Optional.of(files[2].toString()); quorum.takeover = recorder(quorum, failing); + quorum.lost = lostRecorder(quorum); quorum.start(null); members.add(quorum); } } + /** Each LeadershipLost any member fired. */ + private final List losses = new CopyOnWriteArrayList<>(); + + @SuppressWarnings("unchecked") + private Event lostRecorder(Quorum quorum) { + Event event = mock(Event.class); + doAnswer(invocation -> { + losses.add(quorum.memberId() + "@" + invocation.getArgument(0).term()); + return null; + }) + .when(event) + .fire(any()); + return event; + } + + @Test + void aLeaderWhoseServerClosedSaysItLostTheLead() throws Exception { + start(1, new AtomicBoolean(false)); + Quorum quorum = members.get(0); + soleWriter(members); + long term = quorum.term(); + + quorum.member().closeServer(); + quorum.takeOver(); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (losses.isEmpty() && System.nanoTime() < deadline) { + TimeUnit.MILLISECONDS.sleep(50); + } + assertEquals(List.of("m0@" + term), losses, "its runs are stopped like any deposed leader's"); + } + @SuppressWarnings("unchecked") private Event recorder(Quorum quorum, AtomicBoolean failing) { Event event = mock(Event.class); From 4163600684d73130a743892782a3aa1834ae187d Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 15:14:29 +0100 Subject: [PATCH 6/6] style(backend): Spotless on the files this branch changed The earlier spotless:apply ran with paths that matched nothing. Six CLI caveat refs follow the lines the formatting moved. --- cli/cmd/mcp_caveats.go | 2 +- cli/cmd/mcp_tools_chaos.go | 8 +++---- cli/cmd/mcp_tools_runs_activity.go | 2 +- .../kates/chaos/KubernetesChaosProvider.java | 24 +++++++++---------- .../kates/chaos/LitmusChaosProvider.java | 7 ++++-- .../kates/chaos/ScaleDownSnapshots.java | 3 +-- .../DisruptionOrphanReconciler.java | 7 +++--- .../bmscomp/kates/quorum/QuorumMember.java | 3 ++- .../chaos/KubernetesChaosProviderTest.java | 24 +++++++++++++------ .../kates/chaos/LitmusChaosProviderTest.java | 3 ++- .../DisruptionOrchestratorQuorumTest.java | 4 ++-- .../engine/TestOrchestratorQuorumTest.java | 4 ++-- .../com/bmscomp/kates/quorum/QuorumTest.java | 3 ++- 13 files changed, 54 insertions(+), 40 deletions(-) diff --git a/cli/cmd/mcp_caveats.go b/cli/cmd/mcp_caveats.go index 16b5fc5eb..32ea899af 100644 --- a/cli/cmd/mcp_caveats.go +++ b/cli/cmd/mcp_caveats.go @@ -241,7 +241,7 @@ var mcpCaveatsCore = []mcpCaveat{ Refs: []string{ mcpAnchoredRef("kates/src/main/resources/application.properties:274-275", "# Chaos coordination (noop | kubernetes | litmus-crd | hybrid)", "kates.chaos.provider=litmus-crd"), - mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:63,115", + mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:63,117", "int resultPollIntervalMs = 5_000;", "Instant start = Instant.now();"), }, }, diff --git a/cli/cmd/mcp_tools_chaos.go b/cli/cmd/mcp_tools_chaos.go index 9be93f32c..c573c9672 100644 --- a/cli/cmd/mcp_tools_chaos.go +++ b/cli/cmd/mcp_tools_chaos.go @@ -147,7 +147,7 @@ var mcpCaveatsChaos = []mcpCaveat{ mcpAnchoredRef(mcpJava+"disruption/DisruptionSafetyGuard.java:298-301,874-943", "boolean canExecute = checkRbacPermissions(spec);", `"Insufficient RBAC permissions for "`, "boolean checkRbacPermissions(FaultSpec spec)", `"RBAC check failed, assuming permitted"`), - mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:42-60,94-96,281", + mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:42-60,96-98,284", "ON_KUBERNETES_API = EnumSet.of(", "DisruptionType.POD_DELETE,", "DisruptionType.NETWORK_PARTITION);", "return kubernetes.triggerFault(spec, onInject);", `engineSpec.chaosServiceAccount = "litmus-admin";`), mcpAnchoredRef("kates/src/main/resources/application.properties:274-275", @@ -275,12 +275,12 @@ var mcpCaveatsChaos = []mcpCaveat{ mcpAnchoredRef(mcpJava+"disruption/KafkaIntelligenceService.java:125-127,198-217,241-246,310-324", "public void markDisruptionStart(Instant at)", "private Duration computeTimeToFullIsr()", "this.baselineLag = timeline.getLast().totalLag();", "private Duration computeTimeToLagRecovery(long baseLag)"), - mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:94-106,115-138", + mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:96-108,117-141", "return kubernetes.triggerFault(spec, onInject);", "CompletableFuture.delayedExecutor(spec.delayBeforeSec(), TimeUnit.SECONDS, executor.get());", "long startNanos = System.nanoTime();", "buildChaosEngine(spec, engineName, experimentName)", "onInject.accept(startNanos);", ".resource(engine)"), - mcpAnchoredRef(mcpJava+"chaos/KubernetesChaosProvider.java:168-180", + mcpAnchoredRef(mcpJava+"chaos/KubernetesChaosProvider.java:170-182", "Thread.sleep(spec.delayBeforeSec() * 1000L);", "onInject.accept(startNanos);", "applyDisruption(spec, engineName, marks);"), mcpAnchoredRef(mcpJava+"chaos/PodTargets.java:58,69-75", @@ -316,7 +316,7 @@ var mcpCaveatsChaos = []mcpCaveat{ "starts while another runs a plan marks that plan's report too, until the plan ends and stores its " + "outcome.", Refs: []string{ - mcpJava + "disruption/DisruptionOrphanReconciler.java:72-76,81-110,117-167,333-367", + mcpJava + "disruption/DisruptionOrphanReconciler.java:71-75,80-109,116-166,334-368", mcpJava + "disruption/DisruptionReportRepository.java:35-60,93-102", mcpJava + "disruption/DisruptionPersistence.java:26-37", mcpJava + "disruption/DisruptionLauncher.java:114-121,129-144", diff --git a/cli/cmd/mcp_tools_runs_activity.go b/cli/cmd/mcp_tools_runs_activity.go index 3da8a1181..748ed5ac8 100644 --- a/cli/cmd/mcp_tools_runs_activity.go +++ b/cli/cmd/mcp_tools_runs_activity.go @@ -486,7 +486,7 @@ var mcpCaveatsRunsActivity = []mcpCaveat{ mcpJava + "disruption/DisruptionLauncher.java:114-144", mcpJava + "disruption/DisruptionPersistence.java:16-37", mcpJava + "disruption/DisruptionReportRepository.java:20-43", - mcpJava + "disruption/DisruptionOrphanReconciler.java:333-367", + mcpJava + "disruption/DisruptionOrphanReconciler.java:334-368", mcpJava + "disruption/DisruptionTemplateResource.java:70-72", mcpJava + "disruption/DisruptionScheduler.java:112-124", mcpJava + "disruption/DisruptionReportEntity.java:39-47", diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java b/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java index bbfafac72..27a9719d0 100644 --- a/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java +++ b/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java @@ -100,7 +100,9 @@ public IncompleteScaleDownException(String message) { @Inject com.bmscomp.kates.quorum.Quorum quorum = new com.bmscomp.kates.quorum.Quorum(); - @org.eclipse.microprofile.config.inject.ConfigProperty(name = "kates.engine.max-duration-ms", defaultValue = "7200000") + @org.eclipse.microprofile.config.inject.ConfigProperty( + name = "kates.engine.max-duration-ms", + defaultValue = "7200000") long maxDurationMs = 7_200_000; /** @@ -512,18 +514,14 @@ private boolean scaleDownStatefulSet(String namespace, String name, Map { - var metadata = new StatefulSetBuilder(s).editMetadata().addToAnnotations(marks); - if (!alreadySnapshotted) { - metadata.addToAnnotations(ORIGINAL_REPLICAS_ANNOTATION, String.valueOf(current)) - .addToAnnotations(SCALED_DOWN_AT_ANNOTATION, scaledDownAt); - } - return metadata.endMetadata().build(); - }); + client.apps().statefulSets().inNamespace(namespace).withName(name).edit(s -> { + var metadata = new StatefulSetBuilder(s).editMetadata().addToAnnotations(marks); + if (!alreadySnapshotted) { + metadata.addToAnnotations(ORIGINAL_REPLICAS_ANNOTATION, String.valueOf(current)) + .addToAnnotations(SCALED_DOWN_AT_ANNOTATION, scaledDownAt); + } + return metadata.endMetadata().build(); + }); LOG.info("SCALE_DOWN: StatefulSet " + name + " from " + current + " → " + (current - 1)); client.apps().statefulSets().inNamespace(namespace).withName(name).scale(current - 1); return true; diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java b/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java index 7797e710e..fe7281207 100644 --- a/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java +++ b/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java @@ -81,7 +81,9 @@ public String name() { @Inject com.bmscomp.kates.quorum.Quorum quorum = new com.bmscomp.kates.quorum.Quorum(); - @org.eclipse.microprofile.config.inject.ConfigProperty(name = "kates.engine.max-duration-ms", defaultValue = "7200000") + @org.eclipse.microprofile.config.inject.ConfigProperty( + name = "kates.engine.max-duration-ms", + defaultValue = "7200000") long maxDurationMs = 7_200_000; @Override @@ -126,7 +128,8 @@ private CompletableFuture inject(FaultSpec spec, LongConsumer onIn // Confirmed right before the engine exists: a member that stopped leading writes nothing. long term = quorum.requireWrites(); - engine.getMetadata().setAnnotations(FaultMarks.of(term, Instant.now().plusMillis(maxDurationMs))); + engine.getMetadata() + .setAnnotations(FaultMarks.of(term, Instant.now().plusMillis(maxDurationMs))); // Once the pods are picked, before the engine that injects the fault exists. onInject.accept(startNanos); diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/ScaleDownSnapshots.java b/kates/src/main/java/com/bmscomp/kates/chaos/ScaleDownSnapshots.java index e2f8d817f..1d3ca201f 100644 --- a/kates/src/main/java/com/bmscomp/kates/chaos/ScaleDownSnapshots.java +++ b/kates/src/main/java/com/bmscomp/kates/chaos/ScaleDownSnapshots.java @@ -52,8 +52,7 @@ public static Restored restore(KubernetesClient client, String namespace, Predic * As {@link #restore}, in every namespace: a playbook can scale down outside Kafka's. Where the * backend may not list across namespaces (rbac.scope=namespace), {@code fallback} only. */ - public static Restored restoreEverywhere( - KubernetesClient client, String fallback, Predicate eligible) { + public static Restored restoreEverywhere(KubernetesClient client, String fallback, Predicate eligible) { return new Restored( restoreStatefulSets(client, null, fallback, eligible), restoreNodePools(client, null, fallback, eligible)); diff --git a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java index a6748ba66..dff03e58e 100644 --- a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java +++ b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java @@ -23,7 +23,6 @@ import com.bmscomp.kates.chaos.KubernetesChaosProvider; import com.bmscomp.kates.chaos.ScaleDownSnapshots; import com.bmscomp.kates.chaos.litmus.ChaosEngine; -import com.bmscomp.kates.chaos.litmus.ChaosEngineSpec; /** * Cleans up faults abandoned by a previous process. @@ -205,7 +204,8 @@ int reconcileChaosEngines(Instant startedAt) { Long cutoff = startedAt == null ? null : cutoff(startedAt); List engines; try { - engines = kubeClient.resources(ChaosEngine.class) + engines = kubeClient + .resources(ChaosEngine.class) .inAnyNamespace() .withLabel("managed-by", "kates") .list() @@ -227,7 +227,8 @@ int reconcileChaosEngines(Instant startedAt) { String namespace = engine.getMetadata().getNamespace(); String name = engine.getMetadata().getName(); try { - kubeClient.resources(ChaosEngine.class) + kubeClient + .resources(ChaosEngine.class) .inNamespace(namespace) .withName(name) .delete(); diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java index d61391ffe..557abc4a4 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java @@ -171,7 +171,8 @@ public static QuorumMember start(Settings settings, java.util.function.LongConsu .setMutualTls(true) .setSslProvider(SslProvider.JDK) .build()); - java.util.concurrent.atomic.AtomicReference started = new java.util.concurrent.atomic.AtomicReference<>(); + java.util.concurrent.atomic.AtomicReference started = + new java.util.concurrent.atomic.AtomicReference<>(); Confirmations confirmations = new Confirmations( self.getId(), () -> { diff --git a/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java b/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java index 081866c8e..5e3a99bbe 100644 --- a/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java +++ b/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java @@ -993,7 +993,6 @@ private void createKafkaStatefulSet() { .build()) .create(); } - } private static com.bmscomp.kates.quorum.Quorum quorumAt(long term, boolean leads) { @@ -1031,17 +1030,24 @@ void aMemberThatDoesNotLeadInjectsNothing() throws Exception { DisruptionType.ROLLING_RESTART, DisruptionType.CPU_STRESS)) { FaultSpec spec = type == DisruptionType.SCALE_DOWN - ? FaultSpec.builder("x").targetNamespace("default").targetLabel("app=kafka").disruptionType(type).build() - : FaultSpec.builder("x").targetPod("krafter-brokers-0").disruptionType(type).chaosDurationSec(1).build(); + ? FaultSpec.builder("x") + .targetNamespace("default") + .targetLabel("app=kafka") + .disruptionType(type) + .build() + : FaultSpec.builder("x") + .targetPod("krafter-brokers-0") + .disruptionType(type) + .chaosDurationSec(1) + .build(); ChaosOutcome outcome = provider.triggerFault(spec).get(5, TimeUnit.SECONDS); assertFalse(outcome.isPass(), type + " went in"); assertTrue(outcome.failureReason().contains("does not lead"), type + ": " + outcome.failureReason()); } - List writes = requests().stream() - .filter(r -> !r.startsWith("GET ")) - .toList(); + List writes = + requests().stream().filter(r -> !r.startsWith("GET ")).toList(); assertEquals(List.of(), writes, "a member that does not lead writes nothing"); } @@ -1077,7 +1083,11 @@ void aScaleDownCarriesItsTermAndDeadline() throws Exception { assertTrue(provider.triggerFault(spec).get(5, TimeUnit.SECONDS).isPass()); - StatefulSet after = client.apps().statefulSets().inNamespace("default").withName("kafka").get(); + StatefulSet after = client.apps() + .statefulSets() + .inNamespace("default") + .withName("kafka") + .get(); assertMarked(after.getMetadata().getAnnotations(), "7"); } diff --git a/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java b/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java index 560f3c1ec..1aebab321 100644 --- a/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java +++ b/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java @@ -187,7 +187,8 @@ void anEngineCarriesItsTermAndDeadline() throws Exception { ChaosEngine engine = awaitEngine(); assertEquals("7", engine.getMetadata().getAnnotations().get(FaultMarks.TERM)); - assertNotNull(java.time.Instant.parse(engine.getMetadata().getAnnotations().get(FaultMarks.DEADLINE))); + assertNotNull( + java.time.Instant.parse(engine.getMetadata().getAnnotations().get(FaultMarks.DEADLINE))); } @Test diff --git a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrchestratorQuorumTest.java b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrchestratorQuorumTest.java index 444d72f42..8dc921655 100644 --- a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrchestratorQuorumTest.java +++ b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrchestratorQuorumTest.java @@ -1,6 +1,5 @@ package com.bmscomp.kates.disruption; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -85,7 +84,8 @@ void aPlanStopsOnceTheTermItStartedInEnds() { @Test void aMemberThatNoLongerLeadsDoesNotRollBack() throws Exception { K8sPodWatcher.WatchSession session = mock(K8sPodWatcher.WatchSession.class); - when(session.awaitRecovery(org.mockito.ArgumentMatchers.anyLong(), any())).thenReturn(false); + when(session.awaitRecovery(org.mockito.ArgumentMatchers.anyLong(), any())) + .thenReturn(false); when(orchestrator.podWatcher.startWatching(any(), any())).thenReturn(session); when(orchestrator.chaosCoordinator.triggerFault(any(), any())).thenAnswer(invocation -> { invocation.getArgument(1).accept(System.nanoTime()); diff --git a/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java b/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java index 90879d75d..4a84fcab1 100644 --- a/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java +++ b/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java @@ -4,7 +4,6 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -108,7 +107,8 @@ void aRegainedLeaderFailsItsOwnRunsItNoLongerRuns() { String live = engine.startRunning(); String stopped = engine.storeEnded(TaskStatus.RUNNING); when(engine.quorum.memberId()).thenReturn("kates-1"); - when(engine.repository.findByStatusAndMember(any(), eq("kates-1"), eq(false))).thenReturn(List.of()); + when(engine.repository.findByStatusAndMember(any(), eq("kates-1"), eq(false))) + .thenReturn(List.of()); when(engine.repository.findByStatusAndMember(any(), eq("kates-1"), eq(true))) .thenReturn(List.of(engine.rows.get(live), engine.rows.get(stopped))); diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java index 89addcb89..bd3694665 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java @@ -181,7 +181,8 @@ private void start(int count, AtomicBoolean failing) throws Exception { private Event lostRecorder(Quorum quorum) { Event event = mock(Event.class); doAnswer(invocation -> { - losses.add(quorum.memberId() + "@" + invocation.getArgument(0).term()); + losses.add(quorum.memberId() + "@" + + invocation.getArgument(0).term()); return null; }) .when(event)