From 40630a346c6aec93e71fbebcbe1d0d6c92b333af Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Mon, 16 Nov 2026 01:09:03 +0100 Subject: [PATCH 1/6] test(operator): print the quorum members' logs when the PostgreSQL handover check fails The resource is deleted before the workflow's Show the stack step, so a failed handover left nothing to read. main fails this check on every run since #675 merged on top of #674. --- operator/e2e-quorum.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/operator/e2e-quorum.sh b/operator/e2e-quorum.sh index 060cf4dc..94f8a108 100755 --- a/operator/e2e-quorum.sh +++ b/operator/e2e-quorum.sh @@ -266,7 +266,17 @@ for _ in $(seq 1 90); do second=; sleep 1 done handed=$(( $(date +%s) - start )) -[ -n "$second" ] && pass "$second leads $handed s after $cut lost PostgreSQL" || fail "$cut still leads 90 s after losing PostgreSQL" +if [ -n "$second" ]; then + pass "$second leads $handed s after $cut lost PostgreSQL" +else + fail "$cut still leads 90 s after losing PostgreSQL" + # The resource is deleted before CI's Show the stack: each member's last lines, here. + echo "--- status: $("${k[@]}" get kates "$name" -o jsonpath="{.status}")"; "${k[@]}" get pods -o wide + for pod in "$name-0" "$name-1" "$name-2"; do + echo "--- $pod" + "${k[@]}" logs "$pod" --since=150s | grep -vE 'VertxGrpcSender|"level":"DEBUG"' | cut -c1-800 | tail -60 + done +fi connect # An empty body: the leader answers 400, a member that does not lead 503, before validation. check "the new leader takes writes" 400 "$(curl -s -m 5 -o /dev/null -w '%{http_code}' -X POST "$api/api/tests" \ From 7711f4d62d3e259b6110da5bf885ceeb1bbe8de0 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Mon, 16 Nov 2026 01:36:18 +0100 Subject: [PATCH 2/6] fix(quorum): a failed hand-over is retried at the next check, with another member A leader cut off from PostgreSQL handed over to the next member in kates.quorum.peers, and a failed transfer counted as a hand-over: the next try waited 2 minutes. In the e2e that member was the one fenced a step before; its outbound connections had not recovered, its election got no votes, and the old leader won the next one. Main has failed the check on every Operator run since. A transfer that fails, or after which this member still leads, no longer starts the 2-minute rest, and the next check, 5 s on, asks the member after the one that failed. e2e-node-loss.sh: on two nodes, a follower sharing one gets a new volume, which the scheduler places on the third, instead of failing. The anti-affinity is preferred, and a local volume pins a member. --- .../kates/quorum/DatabaseHandover.java | 22 +++++++++++++-- .../java/com/bmscomp/kates/quorum/Quorum.java | 20 +++++++++---- .../kates/quorum/DatabaseHandoverTest.java | 21 ++++++++++++++ operator/e2e-node-loss.sh | 28 +++++++++++++++++-- 4 files changed, 81 insertions(+), 10 deletions(-) diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java b/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java index 8aeb5b2d..a93101ef 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java @@ -19,7 +19,8 @@ * plans/j4-resilient-quorum.md): every write it takes fails, while another member may reach the * database. After 15 s without it, once nothing is live or after 2 minutes, since handing over * fails a live run. At most once per 2 minutes, so members that all lack the database do not pass - * it back and forth. + * it back and forth. A transfer that failed is no hand-over: the next check, 5 s on, asks the + * member after the one that failed, which may not reach its peers yet, as after a partition. */ @ApplicationScoped public class DatabaseHandover { @@ -44,6 +45,7 @@ public class DatabaseHandover { private Instant unreachableSince; private Instant handedOverAt; + private int failedTransfers; @Scheduled( every = "5s", @@ -59,9 +61,19 @@ void check() { return; } try { - String to = quorum.handOver(TRANSFER_TIMEOUT); + String to = quorum.handOver(TRANSFER_TIMEOUT, failedTransfers); + if (quorum.isLeader()) { + // The target could not win its election, and this member won the next one. + transferFailed(); + LOG.warnf( + "PostgreSQL unreachable since %s: handed to %s, but this member leads again", + unreachableSince, to); + return; + } + failedTransfers = 0; LOG.warnf("PostgreSQL unreachable since %s: leadership handed to %s", unreachableSince, to); } catch (java.io.IOException | RuntimeException e) { + transferFailed(); LOG.warnf("PostgreSQL unreachable since %s, and handing leadership on failed: %s", unreachableSince, e); } } @@ -87,6 +99,12 @@ boolean decide(Instant now, boolean reachable, int live) { return false; } + /** A transfer that failed: still leading, so the next check may try another member. */ + void transferFailed() { + handedOverAt = null; + failedTransfers++; + } + private boolean reachable() { try (Connection connection = database.getConnection()) { return connection.isValid(2); diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java index f5690a8a..06fa37c1 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java @@ -222,24 +222,34 @@ public long requireWrites() { } /** - * Hands leadership to the member after this one in {@code kates.quorum.peers}, so members that - * hand over in turn reach each in order; returns that member's id. + * Hands leadership to the member {@code 1 + attempt} after this one in + * {@code kates.quorum.peers}, skipping this one, so members that hand over in turn reach each in + * order, and a retry after a failed transfer asks another member; returns that member's id. * * @throws IOException if the transfer did not happen within {@code timeout} - * @throws IllegalStateException with no quorum + * @throws IllegalStateException with no quorum, or no other member */ - public String handOver(java.time.Duration timeout) throws IOException { + public String handOver(java.time.Duration timeout, int attempt) throws IOException { QuorumMember current = member; if (current == null) { throw new IllegalStateException("no quorum to hand over in"); } List ids = settings.peers().stream().map(peer -> peer.getId().toString()).toList(); - String target = ids.get((ids.indexOf(current.memberId()) + 1) % ids.size()); + String target = target(ids, current.memberId(), attempt); current.handOver(target, timeout); return target; } + /** The member {@code 1 + attempt} after {@code self} in {@code ids}, never {@code self}. */ + static String target(List ids, String self, int attempt) { + if (ids.size() < 2) { + throw new IllegalStateException("no other member to hand over to"); + } + int others = ids.size() - 1; + return ids.get((ids.indexOf(self) + 1 + Math.floorMod(attempt, others)) % ids.size()); + } + /** The leader this member knows of, or null with no quorum or during an election. */ public String leaderId() { QuorumMember current = member; diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/DatabaseHandoverTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/DatabaseHandoverTest.java index 16596193..a6b386bd 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/DatabaseHandoverTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/DatabaseHandoverTest.java @@ -1,10 +1,12 @@ package com.bmscomp.kates.quorum; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.time.Duration; import java.time.Instant; +import java.util.List; import org.junit.jupiter.api.Test; @@ -55,4 +57,23 @@ void theDatabaseBackResetsTheClock() { assertFalse(handover.decide(at(20), false, 0)); assertTrue(handover.decide(at(35), false, 0)); } + + @Test + void aFailedTransferIsRetriedAtTheNextCheck() { + DatabaseHandover handover = new DatabaseHandover(); + handover.decide(at(0), false, 0); + assertTrue(handover.decide(at(15), false, 0)); + handover.transferFailed(); + assertTrue(handover.decide(at(20), false, 0), "still leading without the database"); + } + + @Test + void eachRetryAsksTheNextOtherMember() { + List ids = List.of("q-0", "q-1", "q-2"); + assertEquals("q-1", Quorum.target(ids, "q-0", 0)); + assertEquals("q-2", Quorum.target(ids, "q-0", 1)); + assertEquals("q-1", Quorum.target(ids, "q-0", 2)); + assertEquals("q-0", Quorum.target(ids, "q-2", 0)); + assertEquals("q-1", Quorum.target(ids, "q-2", 1)); + } } diff --git a/operator/e2e-node-loss.sh b/operator/e2e-node-loss.sh index 7c36bee8..cb70a55c 100755 --- a/operator/e2e-node-loss.sh +++ b/operator/e2e-node-loss.sh @@ -24,9 +24,31 @@ ready() { "${k[@]}" get kates "$name" -o jsonpath='{.status.conditions[?(@.type= echo "== three members, one a node" "${k[@]}" wait --for=condition=Ready "kates/$name" --timeout=5m >/dev/null || { fail "not Ready"; exit 1; } members=("$name-0" "$name-1" "$name-2") -nodes=$(for pod in "${members[@]}"; do node_of "$pod"; done | sort -u | wc -l | tr -d ' ') -# Two on the lost node would take the majority, and nothing would be replaced: by design. -[ "$nodes" = 3 ] && pass "spread over three nodes" || { fail "on $nodes nodes: give one a new volume first"; exit 1; } +spread() { for pod in "${members[@]}"; do node_of "$pod"; done | sort -u | wc -l | tr -d ' '; } +# Two on the lost node would take the majority, and nothing would be replaced: by design. The +# anti-affinity is preferred, and a local volume pins a member to its node: on two nodes, a +# follower sharing one gets a new volume, which the scheduler places on the third. +if [ "$(spread)" != 3 ]; then + shared= + for pod in "${members[@]}"; do + [ "$pod" = "$(leader)" ] && continue + for other in "${members[@]}"; do + [ "$other" != "$pod" ] && [ "$(node_of "$other")" = "$(node_of "$pod")" ] && shared=$pod + done + [ -n "$shared" ] && break + done + if [ -n "$shared" ]; then + "${k[@]}" delete pvc "quorum-$shared" --wait=false >/dev/null + "${k[@]}" delete pod "$shared" >/dev/null + for _ in $(seq 1 90); do + [ "$(spread)" = 3 ] && [ "$(ready)" = True ] && break + sleep 2 + done + fi +fi +nodes=$(spread) +[ "$nodes" = 3 ] && pass "spread over three nodes${shared:+, $shared on a new volume}" \ + || { fail "on $nodes nodes after giving ${shared:-no member} a new volume"; exit 1; } # A follower on a worker: the control plane's node holds the API server. lead=$(leader) From c43b46a928e574930ec827512fb5fd168cde9541 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Mon, 16 Nov 2026 02:01:16 +0100 Subject: [PATCH 3/6] ci(operator): untaint Kind's control plane for the node-loss test Kind keeps the control plane's NoSchedule taint when the cluster has workers, so the three members ran on two nodes and the node-loss test had no third to spread them over. --- .github/workflows/ci-operator.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-operator.yml b/.github/workflows/ci-operator.yml index 551e111d..49fca1a0 100644 --- a/.github/workflows/ci-operator.yml +++ b/.github/workflows/ci-operator.yml @@ -203,7 +203,14 @@ jobs: - name: Three members keep a leader, recover its run, roll the leader last, fence a cut-off one and hand over without PostgreSQL env: KATES_E2E_NODE_LOSS: ${{ github.event_name != 'pull_request' && github.event_name != 'push' && '1' || '' }} - run: operator/e2e-quorum.sh payments + run: | + # Kind taints its control plane when it has workers: the node-loss test needs a member + # on each of the three nodes, and still takes a worker's away. + if [ "${KATES_E2E_NODE_LOSS}" = 1 ] \ + && kubectl get node alpha -o jsonpath='{.spec.taints[*].key}' | grep -q node-role.kubernetes.io/control-plane; then + kubectl taint node alpha node-role.kubernetes.io/control-plane:NoSchedule- + fi + operator/e2e-quorum.sh payments - name: Show the stack if: failure() From 74a0c986a8bbdc7dcbf8dd92e43bde4b26f2e2a6 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Mon, 16 Nov 2026 02:23:06 +0100 Subject: [PATCH 4/6] test(operator): print the leader's and the replaced member's logs when it has not caught up --- operator/e2e-node-loss.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/operator/e2e-node-loss.sh b/operator/e2e-node-loss.sh index cb70a55c..cd44cdef 100755 --- a/operator/e2e-node-loss.sh +++ b/operator/e2e-node-loss.sh @@ -89,7 +89,15 @@ for _ in $(seq 1 30); do [ "${applied:--1}" -ge 0 ] && break sleep 2 done -[ "${applied:--1}" -ge 0 ] && pass "$victim caught up (applied index $applied)" || fail "$victim has not caught up: '$applied'" +if [ "${applied:--1}" -ge 0 ]; then + pass "$victim caught up (applied index $applied)" +else + fail "$victim has not caught up: '$applied'" + for pod in "$lead" "$victim"; do + echo "--- $pod" + "${k[@]}" logs "$pod" --since=300s | grep -E 'ratis|Quorum' | cut -c1-800 | tail -60 + done +fi echo "== $node comes back" docker start "$container" >/dev/null From 821e6662c21f4e04db09727ca54341f9cb787d09 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Mon, 16 Nov 2026 02:48:15 +0100 Subject: [PATCH 5/6] test(operator): the node-loss test takes a worker without the test database The test PostgreSQL could share the lost node: every member then lost it, and the leader handed over by design while the test checked that it led throughout. The database now prefers the control plane, which the test never takes, and the lost node is never the database's. --- operator/e2e-node-loss.sh | 9 ++++++--- operator/e2e-quorum.sh | 8 ++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/operator/e2e-node-loss.sh b/operator/e2e-node-loss.sh index cd44cdef..e35820e3 100755 --- a/operator/e2e-node-loss.sh +++ b/operator/e2e-node-loss.sh @@ -50,15 +50,18 @@ nodes=$(spread) [ "$nodes" = 3 ] && pass "spread over three nodes${shared:+, $shared on a new volume}" \ || { fail "on $nodes nodes after giving ${shared:-no member} a new volume"; exit 1; } -# A follower on a worker: the control plane's node holds the API server. +# A follower on a worker: the control plane's node holds the API server. Not on the database's +# node either: without PostgreSQL, the leader hands over by design. lead=$(leader) +db_node=$("${k[@]}" get pods -l "app=$name-db" -o jsonpath='{.items[0].spec.nodeName}' 2>/dev/null) victim= for pod in "${members[@]}"; do node=$(node_of "$pod") - [ "$pod" != "$lead" ] && [[ "$(kubectl get node "$node" -o jsonpath='{.metadata.labels}')" != *control-plane* ]] \ + [ "$pod" != "$lead" ] && [ "$node" != "$db_node" ] \ + && [[ "$(kubectl get node "$node" -o jsonpath='{.metadata.labels}')" != *control-plane* ]] \ && victim=$pod && break done -[ -n "$victim" ] || { fail "no follower on a worker node"; exit 1; } +[ -n "$victim" ] || { fail "no follower on a worker node without the database ($db_node)"; exit 1; } node=$(node_of "$victim") container=$(kubectl get node "$node" -o jsonpath='{.spec.providerID}' | sed 's|.*/||') before=$(claim_uid "$victim") diff --git a/operator/e2e-quorum.sh b/operator/e2e-quorum.sh index 94f8a108..884ef0dc 100755 --- a/operator/e2e-quorum.sh +++ b/operator/e2e-quorum.sh @@ -37,6 +37,14 @@ spec: template: metadata: {labels: {app: $name-db}} spec: + # On the control plane where it can: e2e-node-loss.sh takes a worker away, and the members + # would all lose PostgreSQL with it. + tolerations: [{key: node-role.kubernetes.io/control-plane, operator: Exists, effect: NoSchedule}] + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: {matchExpressions: [{key: node-role.kubernetes.io/control-plane, operator: Exists}]} containers: - name: postgres image: postgres:17-alpine From 72c91427497ecb5a8c6ff1026331a19072e69c07 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Mon, 16 Nov 2026 06:55:13 +0100 Subject: [PATCH 6/6] fix(quorum): a member rebuilt on an empty disk has the leader step down Ratis 3.3.1's leader keeps the index a member matched before it lost its disk: it answers the member's INCONSISTENCY nextIndex=0 with heartbeats from that index, and the member stays at applied index -1 until the leader changes. A follower whose log is still empty 15 s after it knows a leader now asks that leader to step down, at most once a minute; the next leader starts replication over and sends the whole log. The node-loss e2e checks the leader only while the member is gone, and gives the catch-up 90 s. Co-Authored-By: Claude Opus 5.5 --- .../bmscomp/kates/quorum/EmptyLogCatchUp.java | 74 +++++++++++++++++++ .../bmscomp/kates/quorum/QuorumMember.java | 29 +++++++- .../kates/quorum/EmptyLogCatchUpTest.java | 41 ++++++++++ .../kates/quorum/QuorumMemberTest.java | 45 +++++++++++ operator/e2e-node-loss.sh | 14 ++-- 5 files changed, 195 insertions(+), 8 deletions(-) create mode 100644 kates/src/main/java/com/bmscomp/kates/quorum/EmptyLogCatchUp.java create mode 100644 kates/src/test/java/com/bmscomp/kates/quorum/EmptyLogCatchUpTest.java diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/EmptyLogCatchUp.java b/kates/src/main/java/com/bmscomp/kates/quorum/EmptyLogCatchUp.java new file mode 100644 index 00000000..62f82d5c --- /dev/null +++ b/kates/src/main/java/com/bmscomp/kates/quorum/EmptyLogCatchUp.java @@ -0,0 +1,74 @@ +package com.bmscomp.kates.quorum; + +import java.time.Duration; +import java.time.Instant; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +import io.quarkus.scheduler.Scheduled; +import org.jboss.logging.Logger; + +/** + * A member that follows a leader with an empty log, as one rebuilt on a new volume does, asks that + * leader to step down: Ratis 3.3.1's leader trusts the index the member matched before it lost its + * disk and sends it heartbeats only, while the next leader sends it the whole log + * ({@link QuorumMember#askLeaderToStepDown}). After 15 s, since a member that just started gets + * the log within a heartbeat; at most once a minute, should the next leader not help either. + */ +@ApplicationScoped +public class EmptyLogCatchUp { + + private static final Logger LOG = Logger.getLogger(EmptyLogCatchUp.class); + static final Duration AFTER = Duration.ofSeconds(15); + static final Duration AGAIN_AFTER = Duration.ofMinutes(1); + private static final Duration STEP_DOWN_TIMEOUT = Duration.ofSeconds(10); + + @Inject + Quorum quorum; + + private Instant emptySince; + private Instant askedAt; + + @Scheduled( + every = "5s", + identity = "quorum-empty-log-catch-up", + concurrentExecution = Scheduled.ConcurrentExecution.SKIP) + void check() { + QuorumMember current = quorum.member(); + boolean emptyUnderLeader = current != null + && current.serving() + && !current.isLeader() + && current.leaderId() != null + && current.appliedIndex() < 0; + if (!decide(Instant.now(), emptyUnderLeader)) { + return; + } + String leader = current.leaderId(); + try { + current.askLeaderToStepDown(STEP_DOWN_TIMEOUT); + LOG.warnf( + "Empty log under %s since %s: it stepped down, so the next leader sends the log", + leader, emptySince); + } catch (java.io.IOException | RuntimeException e) { + LOG.warnf("Empty log under %s since %s, and asking it to step down failed: %s", leader, emptySince, e); + } + } + + /** Whether to ask the leader to step down now, given whether this member follows it with an empty log. */ + boolean decide(Instant now, boolean emptyUnderLeader) { + if (!emptyUnderLeader) { + emptySince = null; + return false; + } + if (emptySince == null) { + emptySince = now; + } + boolean due = Duration.between(emptySince, now).compareTo(AFTER) >= 0; + boolean rested = askedAt == null || Duration.between(askedAt, now).compareTo(AGAIN_AFTER) >= 0; + if (due && rested) { + askedAt = now; + return true; + } + return false; + } +} diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java index 14d1e355..221a8d7e 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java @@ -269,19 +269,42 @@ public boolean confirmLeadership() { * @throws IOException if the transfer did not happen within {@code timeout} */ public void handOver(String target, Duration timeout) throws IOException { + transfer(selfId, RaftPeerId.valueOf(target), timeout); + } + + /** + * Asks the leader this member knows of to step down, so that the next leader, whichever wins, + * starts each member's replication over from what that member reports. Ratis 3.3.1's leader + * goes on trusting the index a member had matched before it lost its disk, and sends it only + * heartbeats; a new leader sends it the whole log. + * + * @throws IOException if there is no leader, or it did not step down within {@code timeout} + */ + public void askLeaderToStepDown(Duration timeout) throws IOException { + RaftPeerId leader = info().getLeaderId(); + if (leader == null) { + throw new IOException("no leader to ask"); + } + transfer(leader, null, timeout); + } + + /** A transfer to {@code target}, or a step-down with none, asked of {@code via}, the leader. */ + private void transfer(RaftPeerId via, RaftPeerId target, Duration timeout) throws IOException { RaftProperties properties = new RaftProperties(); RaftClientConfigKeys.Rpc.setRequestTimeout( properties, TimeDuration.valueOf(timeout.toMillis() + 2000, TimeUnit.MILLISECONDS)); try (RaftClient admin = RaftClient.newBuilder() .setRaftGroup(group) - .setLeaderId(selfId) + .setLeaderId(via) .setProperties(properties) .setParameters(parameters) .setRetryPolicy(RetryPolicies.noRetry()) .build()) { - var reply = admin.admin().transferLeadership(RaftPeerId.valueOf(target), timeout.toMillis()); + var reply = admin.admin().transferLeadership(target, timeout.toMillis()); if (!reply.isSuccess()) { - throw new IOException("leadership not handed to " + target + ": " + reply.getException()); + throw new IOException( + (target == null ? via + " did not step down: " : "leadership not handed to " + target + ": ") + + reply.getException()); } } } diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/EmptyLogCatchUpTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/EmptyLogCatchUpTest.java new file mode 100644 index 00000000..eece3ed6 --- /dev/null +++ b/kates/src/test/java/com/bmscomp/kates/quorum/EmptyLogCatchUpTest.java @@ -0,0 +1,41 @@ +package com.bmscomp.kates.quorum; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Instant; + +import org.junit.jupiter.api.Test; + +class EmptyLogCatchUpTest { + + private final EmptyLogCatchUp catchUp = new EmptyLogCatchUp(); + + private static Instant at(long seconds) { + return Instant.EPOCH.plusSeconds(seconds); + } + + @Test + void asksAfterFifteenSecondsWithAnEmptyLog() { + assertFalse(catchUp.decide(at(0), true)); + assertFalse(catchUp.decide(at(10), true)); + assertTrue(catchUp.decide(at(15), true)); + } + + @Test + void asksAgainOnlyAMinuteLater() { + catchUp.decide(at(0), true); + assertTrue(catchUp.decide(at(15), true)); + assertFalse(catchUp.decide(at(20), true)); + assertFalse(catchUp.decide(at(70), true)); + assertTrue(catchUp.decide(at(75), true)); + } + + @Test + void aLogThatFillsStartsTheClockOver() { + catchUp.decide(at(0), true); + assertFalse(catchUp.decide(at(10), false)); + assertFalse(catchUp.decide(at(15), true)); + assertTrue(catchUp.decide(at(30), true)); + } +} diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java index a8bfbde3..2fcf0b36 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java @@ -259,6 +259,51 @@ void theLeaderHandsLeadershipToTheMemberItNames() throws Exception { assertFalse(leader.isLeader()); } + @Test + void aMemberRebuiltOnAnEmptyDiskCatchesUpOnceTheLeaderStepsDown() throws Exception { + TestPki pki = new TestPki(tmp.resolve("pki")); + List peers = peers(3); + Path[][] files = certificates(pki, 3); + for (int i = 0; i < 3; i++) { + started.add(QuorumMember.start(settings(peers, i, files[i]))); + } + QuorumMember first = await(() -> soleLeader(started)); + // A few terms, so the leader has matched more than the first entry. + QuorumMember other = + started.stream().filter(m -> m != first).findFirst().orElseThrow(); + first.handOver(other.memberId(), Duration.ofSeconds(10)); + await(() -> other.isLeader() ? other : null); + other.handOver(first.memberId(), Duration.ofSeconds(10)); + QuorumMember leader = await(() -> first.isLeader() ? first : null); + QuorumMember lost = started.stream() + .filter(m -> m != leader && m != other) + .findFirst() + .orElseThrow(); + await(() -> lost.appliedIndex() >= leader.appliedIndex() ? lost : null); + int index = Integer.parseInt(lost.memberId().substring(1)); + lost.close(); + started.remove(lost); + deleteTree(tmp.resolve("data-m" + index)); + + QuorumMember rebuilt = QuorumMember.start(settings(peers, index, files[index])); + started.add(rebuilt); + await(() -> leader.memberId().equals(rebuilt.leaderId()) ? rebuilt : null); + TimeUnit.SECONDS.sleep(3); + // Ratis' gap EmptyLogCatchUp works around: should it fail, Ratis catches the member up itself. + assertEquals(-1, rebuilt.appliedIndex(), "under the same leader, it gets heartbeats only"); + + rebuilt.askLeaderToStepDown(Duration.ofSeconds(10)); + await(() -> rebuilt.appliedIndex() >= 0 ? rebuilt : null); + } + + private static void deleteTree(Path root) throws IOException { + try (var paths = Files.walk(root)) { + for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) { + Files.delete(path); + } + } + } + private static void assertRefused( String message, List peers, String id, String dir, String cert, String key, String ca) { IllegalArgumentException refused = assertThrows( diff --git a/operator/e2e-node-loss.sh b/operator/e2e-node-loss.sh index e35820e3..c1a68be9 100755 --- a/operator/e2e-node-loss.sh +++ b/operator/e2e-node-loss.sh @@ -72,7 +72,11 @@ start=$(date +%s) docker stop "$container" >/dev/null kubectl delete node "$node" --wait=false >/dev/null moved= +changed= for _ in $(seq 1 240); do + # Sampled while the victim is not back: once back on an empty log, it has the leader step down. + current=$(leader) + [ -n "$current" ] && [ "$current" != "$lead" ] && changed=$current now=$(claim_uid "$victim") [ -n "$now" ] && [ "$now" != "$before" ] && [ -n "$(node_of "$victim")" ] && [ "$(node_of "$victim")" != "$node" ] \ && [ "$("${k[@]}" get pod "$victim" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')" = True ] \ @@ -81,19 +85,19 @@ for _ in $(seq 1 240); do done [ -n "$moved" ] && pass "$victim runs on $(node_of "$victim") with a new volume, $(( $(date +%s) - start )) s after the loss" \ || fail "$victim was not replaced within 8 minutes ($("${k[@]}" get pod "$victim" -o jsonpath='{.status.phase}'))" -check_lead=$(leader) -[ "$check_lead" = "$lead" ] && pass "$lead led throughout" || fail "the leader changed from $lead to '$check_lead'" +[ -z "$changed" ] && pass "$lead led while $victim was gone" || fail "the leader changed from $lead to '$changed'" for _ in $(seq 1 60); do [ "$(ready)" = True ] && break; sleep 2; done [ "$(ready)" = True ] && pass "Ready with three members" \ || fail "not Ready: $("${k[@]}" get kates "$name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}')" -# Ready before caught up: the pod's readiness does not wait for the log. -for _ in $(seq 1 30); do +# Ready before caught up: the pod's readiness does not wait for the log. Under the same leader, +# the member waits 15 s, then has it step down (EmptyLogCatchUp), and the next leader sends the log. +for _ in $(seq 1 45); do applied=$(kubectl get --raw "/api/v1/namespaces/$ns/services/$victim:8080/proxy/api/health" | jq '.quorum.appliedIndex') [ "${applied:--1}" -ge 0 ] && break sleep 2 done if [ "${applied:--1}" -ge 0 ]; then - pass "$victim caught up (applied index $applied)" + pass "$victim caught up (applied index $applied), $(leader) leads" else fail "$victim has not caught up: '$applied'" for pod in "$lead" "$victim"; do