From ca9d2233e8ea9612310cbba3d1c99176785de13f Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 21:53:34 +0100 Subject: [PATCH] fix(quorum): a leader cut off from PostgreSQL hands leadership on J4d and J4e of plans/j4-resilient-quorum.md. J4d: the leader checks PostgreSQL every 5 s. After 15 s without it, once nothing is live or after 2 minutes, it hands leadership to the next member in kates.quorum.peers with Ratis' transferLeadership, at most once per 2 minutes, so members that all lack the database do not pass it back and forth. Before, it kept leading and every write failed while a member that could reach the database stood by (F5). J4e: e2e-quorum.sh cuts the leader from PostgreSQL with iptables and checks that another member leads and takes writes. The operator workflow runs nightly, and then also e2e-node-loss.sh on CI's three-node Kind. Changes under kates/.../quorum/ start the e2e. --- .github/workflows/ci-operator.yml | 15 ++- .../kates/quorum/DatabaseHandover.java | 97 +++++++++++++++++++ .../java/com/bmscomp/kates/quorum/Quorum.java | 19 ++++ .../bmscomp/kates/quorum/QuorumMember.java | 40 +++++++- .../kates/quorum/DatabaseHandoverTest.java | 58 +++++++++++ .../kates/quorum/QuorumMemberTest.java | 21 ++++ operator/e2e-quorum.sh | 39 +++++++- 7 files changed, 281 insertions(+), 8 deletions(-) create mode 100644 kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java create mode 100644 kates/src/test/java/com/bmscomp/kates/quorum/DatabaseHandoverTest.java diff --git a/.github/workflows/ci-operator.yml b/.github/workflows/ci-operator.yml index 16cdb328..c7e40d4c 100644 --- a/.github/workflows/ci-operator.yml +++ b/.github/workflows/ci-operator.yml @@ -9,6 +9,10 @@ on: branches: [main, operator] pull_request: branches: [main] + # Nightly, the e2e also loses a node (operator/e2e-node-loss.sh). + schedule: + - cron: "17 3 * * *" + workflow_dispatch: permissions: contents: read @@ -40,6 +44,7 @@ jobs: - '.github/workflows/ci-operator.yml' e2e: - 'operator/**' + - 'kates/src/main/java/com/bmscomp/kates/quorum/**' - 'charts/kates-operator/**' - 'charts/kates/**' - 'charts/kafka-cluster/**' @@ -126,9 +131,9 @@ jobs: e2e: name: E2E on Kind needs: changes - if: needs.changes.outputs.e2e == 'true' + if: needs.changes.outputs.e2e == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ./.github/actions/load-versions @@ -199,7 +204,9 @@ jobs: - name: A Kates resource becomes a backend that runs a test run: operator/e2e-kates.sh payments - - name: Three members keep a leader, recover its run, roll the leader last and fence a cut-off one + - name: Three members keep a leader, recover its run, roll the leader last, fence a cut-off one and hand over without PostgreSQL + env: + KATES_E2E_NODE_LOSS: ${{ github.event_name != 'pull_request' && github.event_name != 'push' && '1' || '' }} run: operator/e2e-quorum.sh payments - name: Show the stack @@ -207,7 +214,7 @@ jobs: run: | kubectl get pods -A kubectl -n kates logs deploy/kates -c kates --tail=50 || true - kubectl -n kates-operator logs deploy/kates-operator --tail=50 || true + kubectl -n kates-operator logs -l app.kubernetes.io/name=kates-operator --prefix --tail=50 || true gate: name: Gate ยท Operator diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java b/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java new file mode 100644 index 00000000..8aeb5b2d --- /dev/null +++ b/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java @@ -0,0 +1,97 @@ +package com.bmscomp.kates.quorum; + +import java.sql.Connection; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +import io.agroal.api.AgroalDataSource; +import io.quarkus.scheduler.Scheduled; +import org.jboss.logging.Logger; + +import com.bmscomp.kates.disruption.DisruptionConcurrencyGuard; +import com.bmscomp.kates.engine.TestOrchestrator; + +/** + * A leader that cannot reach PostgreSQL hands leadership on (J4d, section 4.6 of + * plans/j4-resilient-quorum.md): every write it takes fails, while another member may reach the + * database. After 15 s without it, once nothing is live or after 2 minutes, since handing over + * fails a live run. At most once per 2 minutes, so members that all lack the database do not pass + * it back and forth. + */ +@ApplicationScoped +public class DatabaseHandover { + + private static final Logger LOG = Logger.getLogger(DatabaseHandover.class); + static final Duration AFTER = Duration.ofSeconds(15); + static final Duration LIVE_WAIT = Duration.ofMinutes(2); + static final Duration AGAIN_AFTER = Duration.ofMinutes(2); + private static final Duration TRANSFER_TIMEOUT = Duration.ofSeconds(10); + + @Inject + Quorum quorum; + + @Inject + AgroalDataSource database; + + @Inject + TestOrchestrator runs; + + @Inject + DisruptionConcurrencyGuard faults; + + private Instant unreachableSince; + private Instant handedOverAt; + + @Scheduled( + every = "5s", + identity = "quorum-database-handover", + concurrentExecution = Scheduled.ConcurrentExecution.SKIP) + void check() { + if (quorum.memberId() == null || !quorum.isLeader()) { + unreachableSince = null; + return; + } + Instant now = Instant.now(); + if (!decide(now, reachable(), runs.liveRuns() + faults.inFlight())) { + return; + } + try { + String to = quorum.handOver(TRANSFER_TIMEOUT); + LOG.warnf("PostgreSQL unreachable since %s: leadership handed to %s", unreachableSince, to); + } catch (java.io.IOException | RuntimeException e) { + LOG.warnf("PostgreSQL unreachable since %s, and handing leadership on failed: %s", unreachableSince, e); + } + } + + /** Whether to hand over now, given whether the database answered and how much is live. */ + boolean decide(Instant now, boolean reachable, int live) { + if (reachable) { + unreachableSince = null; + return false; + } + if (unreachableSince == null) { + unreachableSince = now; + LOG.warnf("Leading without PostgreSQL; handing over after %s", AFTER); + } + Duration without = Duration.between(unreachableSince, now); + boolean due = without.compareTo(AFTER) >= 0 && (live == 0 || without.compareTo(LIVE_WAIT) >= 0); + boolean rested = + handedOverAt == null || Duration.between(handedOverAt, now).compareTo(AGAIN_AFTER) >= 0; + if (due && rested) { + handedOverAt = now; + return true; + } + return false; + } + + private boolean reachable() { + try (Connection connection = database.getConnection()) { + return connection.isValid(2); + } catch (SQLException e) { + return false; + } + } +} diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java index b9b2bdc9..f5690a8a 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java @@ -221,6 +221,25 @@ public long requireWrites() { return term(); } + /** + * Hands leadership to the member after this one in {@code kates.quorum.peers}, so members that + * hand over in turn reach each in order; returns that member's id. + * + * @throws IOException if the transfer did not happen within {@code timeout} + * @throws IllegalStateException with no quorum + */ + public String handOver(java.time.Duration timeout) throws IOException { + QuorumMember current = member; + if (current == null) { + throw new IllegalStateException("no quorum to hand over in"); + } + List ids = + settings.peers().stream().map(peer -> peer.getId().toString()).toList(); + String target = ids.get((ids.indexOf(current.memberId()) + 1) % ids.size()); + current.handOver(target, timeout); + return target; + } + /** The leader this member knows of, or null with no quorum or during an election. */ public String leaderId() { QuorumMember current = member; diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java index 557abc4a..14d1e355 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java @@ -4,6 +4,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -132,12 +133,22 @@ private static Path file(String setting, String value) { private final RaftClient client; private final RaftPeerId selfId; + private final RaftGroup group; + private final Parameters parameters; - private QuorumMember(RaftServer server, RaftClient client, RaftPeerId selfId, String memberId) { + private QuorumMember( + RaftServer server, + RaftClient client, + RaftPeerId selfId, + String memberId, + RaftGroup group, + Parameters parameters) { this.server = server; this.client = client; this.selfId = selfId; this.memberId = memberId; + this.group = group; + this.parameters = parameters; } /** Starts the member. Its term and vote are kept in {@code dir}, which must outlive a restart. */ @@ -205,7 +216,7 @@ public static QuorumMember start(Settings settings, java.util.function.LongConsu .setParameters(parameters) .setRetryPolicy(RetryPolicies.noRetry()) .build(); - QuorumMember member = new QuorumMember(server, client, self.getId(), settings.memberId()); + QuorumMember member = new QuorumMember(server, client, self.getId(), settings.memberId(), group, parameters); started.set(member); return member; } @@ -250,6 +261,31 @@ public boolean confirmLeadership() { } } + /** + * Asks this member, as leader, to hand leadership to {@code target}: it waits for the target + * to catch up, then the target holds an election it wins. On its own client, since the + * transfer takes longer than the confirming read's timeout allows. + * + * @throws IOException if the transfer did not happen within {@code timeout} + */ + public void handOver(String target, Duration timeout) throws IOException { + RaftProperties properties = new RaftProperties(); + RaftClientConfigKeys.Rpc.setRequestTimeout( + properties, TimeDuration.valueOf(timeout.toMillis() + 2000, TimeUnit.MILLISECONDS)); + try (RaftClient admin = RaftClient.newBuilder() + .setRaftGroup(group) + .setLeaderId(selfId) + .setProperties(properties) + .setParameters(parameters) + .setRetryPolicy(RetryPolicies.noRetry()) + .build()) { + var reply = admin.admin().transferLeadership(RaftPeerId.valueOf(target), timeout.toMillis()); + if (!reply.isSuccess()) { + throw new IOException("leadership not handed to " + target + ": " + reply.getException()); + } + } + } + /** The leader this member knows of, or null during an election. */ public String leaderId() { RaftPeerId leader = info().getLeaderId(); diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/DatabaseHandoverTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/DatabaseHandoverTest.java new file mode 100644 index 00000000..16596193 --- /dev/null +++ b/kates/src/test/java/com/bmscomp/kates/quorum/DatabaseHandoverTest.java @@ -0,0 +1,58 @@ +package com.bmscomp.kates.quorum; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.time.Instant; + +import org.junit.jupiter.api.Test; + +class DatabaseHandoverTest { + + private static final Instant T0 = Instant.parse("2026-11-15T12:00:00Z"); + + private static Instant at(int seconds) { + return T0.plus(Duration.ofSeconds(seconds)); + } + + @Test + void handsOverAfter15sWithNothingLive() { + DatabaseHandover handover = new DatabaseHandover(); + assertFalse(handover.decide(at(0), false, 0)); + assertFalse(handover.decide(at(10), false, 0)); + assertTrue(handover.decide(at(15), false, 0)); + } + + @Test + void aLiveRunWaitsUpTo2Minutes() { + DatabaseHandover handover = new DatabaseHandover(); + assertFalse(handover.decide(at(0), false, 1)); + assertFalse(handover.decide(at(60), false, 1)); + assertTrue(handover.decide(at(61), false, 0), "the run ended: hand over"); + DatabaseHandover stuck = new DatabaseHandover(); + stuck.decide(at(0), false, 1); + assertFalse(stuck.decide(at(119), false, 1)); + assertTrue(stuck.decide(at(120), false, 1)); + } + + @Test + void handsOverAtMostOnceIn2Minutes() { + DatabaseHandover handover = new DatabaseHandover(); + handover.decide(at(0), false, 0); + assertTrue(handover.decide(at(15), false, 0)); + // Leading again, as every member lacks the database. + assertFalse(handover.decide(at(30), false, 0)); + assertFalse(handover.decide(at(134), false, 0)); + assertTrue(handover.decide(at(135), false, 0)); + } + + @Test + void theDatabaseBackResetsTheClock() { + DatabaseHandover handover = new DatabaseHandover(); + handover.decide(at(0), false, 0); + assertFalse(handover.decide(at(10), true, 0)); + assertFalse(handover.decide(at(20), false, 0)); + assertTrue(handover.decide(at(35), false, 0)); + } +} diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java index decd65ea..a8bfbde3 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java @@ -238,6 +238,27 @@ void aLeaderThatLosesItsMajorityStopsLeading() throws Exception { assertFalse(leader.isLeader(), "alone, it does not lead again"); } + @Test + void theLeaderHandsLeadershipToTheMemberItNames() throws Exception { + TestPki pki = new TestPki(tmp.resolve("pki")); + List peers = peers(3); + Path[][] files = certificates(pki, 3); + for (int i = 0; i < 3; i++) { + started.add(QuorumMember.start(settings(peers, i, files[i]))); + } + QuorumMember leader = await(() -> soleLeader(started)); + long term = leader.term(); + QuorumMember target = + started.stream().filter(m -> m != leader).findFirst().orElseThrow(); + + leader.handOver(target.memberId(), Duration.ofSeconds(10)); + + QuorumMember next = await(() -> soleLeader(started)); + assertEquals(target.memberId(), next.memberId()); + assertTrue(next.term() > term, "a later term"); + assertFalse(leader.isLeader()); + } + private static void assertRefused( String message, List peers, String id, String dir, String cert, String key, String ca) { IllegalArgumentException refused = assertThrows( diff --git a/operator/e2e-quorum.sh b/operator/e2e-quorum.sh index 9d46c577..060cf4dc 100755 --- a/operator/e2e-quorum.sh +++ b/operator/e2e-quorum.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # A Kates resource with three members: one leader takes the traffic, a new one is elected when it -# dies and recovers its run, a version change keeps a majority, the leader replaced last, and a -# leader cut off from its peers takes no write once another leads. +# dies and recovers its run, a version change keeps a majority, the leader replaced last, a +# leader cut off from its peers takes no write once another leads, and a leader cut off from +# PostgreSQL hands over. # # KUBECONFIG= operator/e2e-quorum.sh [namespace] # @@ -243,6 +244,40 @@ done | sort -u | awk '{print $1}' | uniq -d) kill "${forwards[@]}" 2>/dev/null rm -f "$claims" +echo "== the leader cut off from PostgreSQL hands over" +"${k[@]}" wait --for=condition=Ready "kates/$name" --timeout=5m >/dev/null +for _ in $(seq 1 30); do [ -n "$(leader)" ] && break; sleep 2; done +cut=$(leader) +cut_ip=$("${k[@]}" get pod "$cut" -o jsonpath='{.status.podIP}') +db_ip=$("${k[@]}" get pods -l "app=$name-db" -o jsonpath='{.items[0].status.podIP}') +# Its database alone: the members, Kafka and port 8080 stay reachable. +db_cut() { # db_cut -I|-D + for node in $nodes; do + docker exec "$node" iptables "$1" FORWARD -s "$cut_ip" -d "$db_ip" -p tcp --dport 5432 -j DROP + docker exec "$node" iptables "$1" FORWARD -s "$db_ip" -d "$cut_ip" -p tcp --sport 5432 -j DROP + done +} +db_cut -I +start=$(date +%s) +second= +for _ in $(seq 1 90); do + second=$(leader) + [ -n "$second" ] && [ "$second" != "$cut" ] && [ "$(labelled)" = "$second" ] && break + second=; sleep 1 +done +handed=$(( $(date +%s) - start )) +[ -n "$second" ] && pass "$second leads $handed s after $cut lost PostgreSQL" || fail "$cut still leads 90 s after losing PostgreSQL" +connect +# An empty body: the leader answers 400, a member that does not lead 503, before validation. +check "the new leader takes writes" 400 "$(curl -s -m 5 -o /dev/null -w '%{http_code}' -X POST "$api/api/tests" \ + -H 'Content-Type: application/json' -d '{}')" +db_cut -D + +# Nightly: CI's Kind has three nodes, and this takes one away for a few minutes. +if [ "${KATES_E2E_NODE_LOSS:-}" = 1 ]; then + operator/e2e-node-loss.sh "$ns" "$name" || failed=1 +fi + echo "== deleting the resource removes what it made" "${k[@]}" delete kates "$name" --timeout=120s >/dev/null "${k[@]}" wait --for=delete "statefulset/$name" "secret/kates-quorum-ca" "secret/kates-quorum-tls" "pdb/$name" \