From 55da1c40f014911c0141882e5ff1859fb8afcbb9 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Mon, 16 Nov 2026 00:03:45 +0100 Subject: [PATCH] fix(quorum): a handover that fails is passed to the next member and retried The first nightly run of the operator e2e (43767783062) found the leader cut off from PostgreSQL still leading 90 s later, where a Kind lab handed over in 27 s. The handover targets the next member only, and a failed transfer used the 2-minute rest meant for successful ones, so nothing was tried again within the stage's 90 s. - Quorum.handOver tries each other member in turn, from the next one. - A failed handover no longer starts the rest: the next 5-second check tries again. - The e2e stage prints the cut leader's handover log when it fails. --- .../kates/quorum/DatabaseHandover.java | 6 ++++++ .../java/com/bmscomp/kates/quorum/Quorum.java | 21 ++++++++++++++----- .../kates/quorum/DatabaseHandoverTest.java | 9 ++++++++ .../com/bmscomp/kates/quorum/QuorumTest.java | 13 ++++++++++++ operator/e2e-quorum.sh | 7 ++++++- 5 files changed, 50 insertions(+), 6 deletions(-) diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java b/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java index 8aeb5b2d..8e407396 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java @@ -62,10 +62,16 @@ void check() { String to = quorum.handOver(TRANSFER_TIMEOUT); LOG.warnf("PostgreSQL unreachable since %s: leadership handed to %s", unreachableSince, to); } catch (java.io.IOException | RuntimeException e) { + handOverFailed(); LOG.warnf("PostgreSQL unreachable since %s, and handing leadership on failed: %s", unreachableSince, e); } } + /** The rest stops handovers bouncing between members; one that failed is retried at the next check. */ + void handOverFailed() { + handedOverAt = null; + } + /** Whether to hand over now, given whether the database answered and how much is live. */ boolean decide(Instant now, boolean reachable, int live) { if (reachable) { diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java index f5690a8a..a3c3f7a1 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java @@ -223,9 +223,10 @@ public long requireWrites() { /** * Hands leadership to the member after this one in {@code kates.quorum.peers}, so members that - * hand over in turn reach each in order; returns that member's id. + * hand over in turn reach each in order, or, if that one cannot take it (a member that has + * just come back may not have caught up), to the next; returns the member that took it. * - * @throws IOException if the transfer did not happen within {@code timeout} + * @throws IOException if no member took it, each within {@code timeout} * @throws IllegalStateException with no quorum */ public String handOver(java.time.Duration timeout) throws IOException { @@ -235,9 +236,19 @@ public String handOver(java.time.Duration timeout) throws IOException { } List ids = settings.peers().stream().map(peer -> peer.getId().toString()).toList(); - String target = ids.get((ids.indexOf(current.memberId()) + 1) % ids.size()); - current.handOver(target, timeout); - return target; + int self = ids.indexOf(current.memberId()); + IOException last = null; + for (int i = 1; i < ids.size(); i++) { + String target = ids.get((self + i) % ids.size()); + try { + current.handOver(target, timeout); + return target; + } catch (IOException e) { + LOG.warnf("Leadership not handed to %s: %s", target, e.getMessage()); + last = e; + } + } + throw last != null ? last : new IOException("no other member to hand leadership to"); } /** The leader this member knows of, or null with no quorum or during an election. */ diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/DatabaseHandoverTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/DatabaseHandoverTest.java index 16596193..63b67c54 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/DatabaseHandoverTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/DatabaseHandoverTest.java @@ -47,6 +47,15 @@ void handsOverAtMostOnceIn2Minutes() { assertTrue(handover.decide(at(135), false, 0)); } + @Test + void aFailedHandoverIsRetriedWithoutTheRest() { + DatabaseHandover handover = new DatabaseHandover(); + handover.decide(at(0), false, 0); + assertTrue(handover.decide(at(15), false, 0)); + handover.handOverFailed(); + assertTrue(handover.decide(at(20), false, 0), "retried at the next check, not 2 minutes later"); + } + @Test void theDatabaseBackResetsTheClock() { DatabaseHandover handover = new DatabaseHandover(); diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java index bd369466..5cdd560e 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java @@ -190,6 +190,19 @@ private Event lostRecorder(Quorum quorum) { return event; } + @Test + void aHandoverPassesOverAMemberThatCannotTakeIt() throws Exception { + start(3, new AtomicBoolean(false)); + Quorum leader = soleWriter(members); + int self = members.indexOf(leader); + Quorum next = members.get((self + 1) % 3); + Quorum after = members.get((self + 2) % 3); + next.member().closeServer(); + + assertEquals(after.memberId(), leader.handOver(java.time.Duration.ofSeconds(3))); + assertEquals(after, soleWriter(members)); + } + @Test void aLeaderWhoseServerClosedSaysItLostTheLead() throws Exception { start(1, new AtomicBoolean(false)); diff --git a/operator/e2e-quorum.sh b/operator/e2e-quorum.sh index 060cf4dc..59357b28 100755 --- a/operator/e2e-quorum.sh +++ b/operator/e2e-quorum.sh @@ -266,7 +266,12 @@ for _ in $(seq 1 90); do second=; sleep 1 done handed=$(( $(date +%s) - start )) -[ -n "$second" ] && pass "$second leads $handed s after $cut lost PostgreSQL" || fail "$cut still leads 90 s after losing PostgreSQL" +if [ -n "$second" ]; then + pass "$second leads $handed s after $cut lost PostgreSQL" +else + fail "$cut still leads 90 s after losing PostgreSQL" + "${k[@]}" logs "$cut" --since=3m | grep -E 'PostgreSQL|[Ll]eadership|leads term' | tail -20 +fi connect # An empty body: the leader answers 400, a member that does not lead 503, before validation. check "the new leader takes writes" 400 "$(curl -s -m 5 -o /dev/null -w '%{http_code}' -X POST "$api/api/tests" \