Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,16 @@ void check() {
String to = quorum.handOver(TRANSFER_TIMEOUT);
LOG.warnf("PostgreSQL unreachable since %s: leadership handed to %s", unreachableSince, to);
} catch (java.io.IOException | RuntimeException e) {
handOverFailed();
LOG.warnf("PostgreSQL unreachable since %s, and handing leadership on failed: %s", unreachableSince, e);
}
}

/** The rest stops handovers bouncing between members; one that failed is retried at the next check. */
void handOverFailed() {
handedOverAt = null;
}

/** Whether to hand over now, given whether the database answered and how much is live. */
boolean decide(Instant now, boolean reachable, int live) {
if (reachable) {
Expand Down
21 changes: 16 additions & 5 deletions kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java
Original file line number Diff line number Diff line change
Expand Up @@ -223,9 +223,10 @@ public long requireWrites() {

/**
* Hands leadership to the member after this one in {@code kates.quorum.peers}, so members that
* hand over in turn reach each in order; returns that member's id.
* hand over in turn reach each in order, or, if that one cannot take it (a member that has
* just come back may not have caught up), to the next; returns the member that took it.
*
* @throws IOException if the transfer did not happen within {@code timeout}
* @throws IOException if no member took it, each within {@code timeout}
* @throws IllegalStateException with no quorum
*/
public String handOver(java.time.Duration timeout) throws IOException {
Expand All @@ -235,9 +236,19 @@ public String handOver(java.time.Duration timeout) throws IOException {
}
List<String> ids =
settings.peers().stream().map(peer -> peer.getId().toString()).toList();
String target = ids.get((ids.indexOf(current.memberId()) + 1) % ids.size());
current.handOver(target, timeout);
return target;
int self = ids.indexOf(current.memberId());
IOException last = null;
for (int i = 1; i < ids.size(); i++) {
String target = ids.get((self + i) % ids.size());
try {
current.handOver(target, timeout);
return target;
} catch (IOException e) {
LOG.warnf("Leadership not handed to %s: %s", target, e.getMessage());
last = e;
}
}
throw last != null ? last : new IOException("no other member to hand leadership to");
}

/** The leader this member knows of, or null with no quorum or during an election. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,15 @@ void handsOverAtMostOnceIn2Minutes() {
assertTrue(handover.decide(at(135), false, 0));
}

@Test
void aFailedHandoverIsRetriedWithoutTheRest() {
DatabaseHandover handover = new DatabaseHandover();
handover.decide(at(0), false, 0);
assertTrue(handover.decide(at(15), false, 0));
handover.handOverFailed();
assertTrue(handover.decide(at(20), false, 0), "retried at the next check, not 2 minutes later");
}

@Test
void theDatabaseBackResetsTheClock() {
DatabaseHandover handover = new DatabaseHandover();
Expand Down
13 changes: 13 additions & 0 deletions kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,19 @@ private Event<LeadershipLost> lostRecorder(Quorum quorum) {
return event;
}

@Test
void aHandoverPassesOverAMemberThatCannotTakeIt() throws Exception {
start(3, new AtomicBoolean(false));
Quorum leader = soleWriter(members);
int self = members.indexOf(leader);
Quorum next = members.get((self + 1) % 3);
Quorum after = members.get((self + 2) % 3);
next.member().closeServer();

assertEquals(after.memberId(), leader.handOver(java.time.Duration.ofSeconds(3)));
assertEquals(after, soleWriter(members));
}

@Test
void aLeaderWhoseServerClosedSaysItLostTheLead() throws Exception {
start(1, new AtomicBoolean(false));
Expand Down
7 changes: 6 additions & 1 deletion operator/e2e-quorum.sh
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,12 @@ for _ in $(seq 1 90); do
second=; sleep 1
done
handed=$(( $(date +%s) - start ))
[ -n "$second" ] && pass "$second leads $handed s after $cut lost PostgreSQL" || fail "$cut still leads 90 s after losing PostgreSQL"
if [ -n "$second" ]; then
pass "$second leads $handed s after $cut lost PostgreSQL"
else
fail "$cut still leads 90 s after losing PostgreSQL"
"${k[@]}" logs "$cut" --since=3m | grep -E 'PostgreSQL|[Ll]eadership|leads term' | tail -20
fi
connect
# An empty body: the leader answers 400, a member that does not lead 503, before validation.
check "the new leader takes writes" 400 "$(curl -s -m 5 -o /dev/null -w '%{http_code}' -X POST "$api/api/tests" \
Expand Down
Loading