Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions .github/workflows/ci-operator.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ on:
branches: [main, operator]
pull_request:
branches: [main]
# Nightly, the e2e also loses a node (operator/e2e-node-loss.sh).
schedule:
- cron: "17 3 * * *"
workflow_dispatch:

permissions:
contents: read
Expand Down Expand Up @@ -40,6 +44,7 @@ jobs:
- '.github/workflows/ci-operator.yml'
e2e:
- 'operator/**'
- 'kates/src/main/java/com/bmscomp/kates/quorum/**'
- 'charts/kates-operator/**'
- 'charts/kates/**'
- 'charts/kafka-cluster/**'
Expand Down Expand Up @@ -126,9 +131,9 @@ jobs:
e2e:
name: E2E on Kind
needs: changes
if: needs.changes.outputs.e2e == 'true'
if: needs.changes.outputs.e2e == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
timeout-minutes: 45
timeout-minutes: 60
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: ./.github/actions/load-versions
Expand Down Expand Up @@ -199,15 +204,17 @@ jobs:
- name: A Kates resource becomes a backend that runs a test
run: operator/e2e-kates.sh payments

- name: Three members keep a leader, recover its run, roll the leader last and fence a cut-off one
- name: Three members keep a leader, recover its run, roll the leader last, fence a cut-off one and hand over without PostgreSQL
env:
KATES_E2E_NODE_LOSS: ${{ github.event_name != 'pull_request' && github.event_name != 'push' && '1' || '' }}
run: operator/e2e-quorum.sh payments

- name: Show the stack
if: failure()
run: |
kubectl get pods -A
kubectl -n kates logs deploy/kates -c kates --tail=50 || true
kubectl -n kates-operator logs deploy/kates-operator --tail=50 || true
kubectl -n kates-operator logs -l app.kubernetes.io/name=kates-operator --prefix --tail=50 || true

gate:
name: Gate · Operator
Expand Down
97 changes: 97 additions & 0 deletions kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package com.bmscomp.kates.quorum;

import java.sql.Connection;
import java.sql.SQLException;
import java.time.Duration;
import java.time.Instant;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;

import io.agroal.api.AgroalDataSource;
import io.quarkus.scheduler.Scheduled;
import org.jboss.logging.Logger;

import com.bmscomp.kates.disruption.DisruptionConcurrencyGuard;
import com.bmscomp.kates.engine.TestOrchestrator;

/**
* A leader that cannot reach PostgreSQL hands leadership on (J4d, section 4.6 of
* plans/j4-resilient-quorum.md): every write it takes fails, while another member may reach the
* database. After 15 s without it, once nothing is live or after 2 minutes, since handing over
* fails a live run. At most once per 2 minutes, so members that all lack the database do not pass
* it back and forth.
*/
@ApplicationScoped
public class DatabaseHandover {

private static final Logger LOG = Logger.getLogger(DatabaseHandover.class);
static final Duration AFTER = Duration.ofSeconds(15);
static final Duration LIVE_WAIT = Duration.ofMinutes(2);
static final Duration AGAIN_AFTER = Duration.ofMinutes(2);
private static final Duration TRANSFER_TIMEOUT = Duration.ofSeconds(10);

@Inject
Quorum quorum;

@Inject
AgroalDataSource database;

@Inject
TestOrchestrator runs;

@Inject
DisruptionConcurrencyGuard faults;

private Instant unreachableSince;
private Instant handedOverAt;

@Scheduled(
every = "5s",
identity = "quorum-database-handover",
concurrentExecution = Scheduled.ConcurrentExecution.SKIP)
void check() {
if (quorum.memberId() == null || !quorum.isLeader()) {
unreachableSince = null;
return;
}
Instant now = Instant.now();
if (!decide(now, reachable(), runs.liveRuns() + faults.inFlight())) {
return;
}
try {
String to = quorum.handOver(TRANSFER_TIMEOUT);
LOG.warnf("PostgreSQL unreachable since %s: leadership handed to %s", unreachableSince, to);
} catch (java.io.IOException | RuntimeException e) {
LOG.warnf("PostgreSQL unreachable since %s, and handing leadership on failed: %s", unreachableSince, e);
}
}

/** Whether to hand over now, given whether the database answered and how much is live. */
boolean decide(Instant now, boolean reachable, int live) {
if (reachable) {
unreachableSince = null;
return false;
}
if (unreachableSince == null) {
unreachableSince = now;
LOG.warnf("Leading without PostgreSQL; handing over after %s", AFTER);
}
Duration without = Duration.between(unreachableSince, now);
boolean due = without.compareTo(AFTER) >= 0 && (live == 0 || without.compareTo(LIVE_WAIT) >= 0);
boolean rested =
handedOverAt == null || Duration.between(handedOverAt, now).compareTo(AGAIN_AFTER) >= 0;
if (due && rested) {
handedOverAt = now;
return true;
}
return false;
}

private boolean reachable() {
try (Connection connection = database.getConnection()) {
return connection.isValid(2);
} catch (SQLException e) {
return false;
}
}
}
19 changes: 19 additions & 0 deletions kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,25 @@ public long requireWrites() {
return term();
}

/**
* Hands leadership to the member after this one in {@code kates.quorum.peers}, so members that
* hand over in turn reach each in order; returns that member's id.
*
* @throws IOException if the transfer did not happen within {@code timeout}
* @throws IllegalStateException with no quorum
*/
public String handOver(java.time.Duration timeout) throws IOException {
QuorumMember current = member;
if (current == null) {
throw new IllegalStateException("no quorum to hand over in");
}
List<String> ids =
settings.peers().stream().map(peer -> peer.getId().toString()).toList();
String target = ids.get((ids.indexOf(current.memberId()) + 1) % ids.size());
current.handOver(target, timeout);
return target;
}

/** The leader this member knows of, or null with no quorum or during an election. */
public String leaderId() {
QuorumMember current = member;
Expand Down
40 changes: 38 additions & 2 deletions kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
Expand Down Expand Up @@ -132,12 +133,22 @@ private static Path file(String setting, String value) {

private final RaftClient client;
private final RaftPeerId selfId;
private final RaftGroup group;
private final Parameters parameters;

private QuorumMember(RaftServer server, RaftClient client, RaftPeerId selfId, String memberId) {
private QuorumMember(
RaftServer server,
RaftClient client,
RaftPeerId selfId,
String memberId,
RaftGroup group,
Parameters parameters) {
this.server = server;
this.client = client;
this.selfId = selfId;
this.memberId = memberId;
this.group = group;
this.parameters = parameters;
}

/** Starts the member. Its term and vote are kept in {@code dir}, which must outlive a restart. */
Expand Down Expand Up @@ -205,7 +216,7 @@ public static QuorumMember start(Settings settings, java.util.function.LongConsu
.setParameters(parameters)
.setRetryPolicy(RetryPolicies.noRetry())
.build();
QuorumMember member = new QuorumMember(server, client, self.getId(), settings.memberId());
QuorumMember member = new QuorumMember(server, client, self.getId(), settings.memberId(), group, parameters);
started.set(member);
return member;
}
Expand Down Expand Up @@ -250,6 +261,31 @@ public boolean confirmLeadership() {
}
}

/**
* Asks this member, as leader, to hand leadership to {@code target}: it waits for the target
* to catch up, then the target holds an election it wins. On its own client, since the
* transfer takes longer than the confirming read's timeout allows.
*
* @throws IOException if the transfer did not happen within {@code timeout}
*/
public void handOver(String target, Duration timeout) throws IOException {
RaftProperties properties = new RaftProperties();
RaftClientConfigKeys.Rpc.setRequestTimeout(
properties, TimeDuration.valueOf(timeout.toMillis() + 2000, TimeUnit.MILLISECONDS));
try (RaftClient admin = RaftClient.newBuilder()
.setRaftGroup(group)
.setLeaderId(selfId)
.setProperties(properties)
.setParameters(parameters)
.setRetryPolicy(RetryPolicies.noRetry())
.build()) {
var reply = admin.admin().transferLeadership(RaftPeerId.valueOf(target), timeout.toMillis());
if (!reply.isSuccess()) {
throw new IOException("leadership not handed to " + target + ": " + reply.getException());
}
}
}

/** The leader this member knows of, or null during an election. */
public String leaderId() {
RaftPeerId leader = info().getLeaderId();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package com.bmscomp.kates.quorum;

import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.time.Duration;
import java.time.Instant;

import org.junit.jupiter.api.Test;

class DatabaseHandoverTest {

private static final Instant T0 = Instant.parse("2026-11-15T12:00:00Z");

private static Instant at(int seconds) {
return T0.plus(Duration.ofSeconds(seconds));
}

@Test
void handsOverAfter15sWithNothingLive() {
DatabaseHandover handover = new DatabaseHandover();
assertFalse(handover.decide(at(0), false, 0));
assertFalse(handover.decide(at(10), false, 0));
assertTrue(handover.decide(at(15), false, 0));
}

@Test
void aLiveRunWaitsUpTo2Minutes() {
DatabaseHandover handover = new DatabaseHandover();
assertFalse(handover.decide(at(0), false, 1));
assertFalse(handover.decide(at(60), false, 1));
assertTrue(handover.decide(at(61), false, 0), "the run ended: hand over");
DatabaseHandover stuck = new DatabaseHandover();
stuck.decide(at(0), false, 1);
assertFalse(stuck.decide(at(119), false, 1));
assertTrue(stuck.decide(at(120), false, 1));
}

@Test
void handsOverAtMostOnceIn2Minutes() {
DatabaseHandover handover = new DatabaseHandover();
handover.decide(at(0), false, 0);
assertTrue(handover.decide(at(15), false, 0));
// Leading again, as every member lacks the database.
assertFalse(handover.decide(at(30), false, 0));
assertFalse(handover.decide(at(134), false, 0));
assertTrue(handover.decide(at(135), false, 0));
}

@Test
void theDatabaseBackResetsTheClock() {
DatabaseHandover handover = new DatabaseHandover();
handover.decide(at(0), false, 0);
assertFalse(handover.decide(at(10), true, 0));
assertFalse(handover.decide(at(20), false, 0));
assertTrue(handover.decide(at(35), false, 0));
}
}
21 changes: 21 additions & 0 deletions kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,27 @@ void aLeaderThatLosesItsMajorityStopsLeading() throws Exception {
assertFalse(leader.isLeader(), "alone, it does not lead again");
}

@Test
void theLeaderHandsLeadershipToTheMemberItNames() throws Exception {
TestPki pki = new TestPki(tmp.resolve("pki"));
List<String> peers = peers(3);
Path[][] files = certificates(pki, 3);
for (int i = 0; i < 3; i++) {
started.add(QuorumMember.start(settings(peers, i, files[i])));
}
QuorumMember leader = await(() -> soleLeader(started));
long term = leader.term();
QuorumMember target =
started.stream().filter(m -> m != leader).findFirst().orElseThrow();

leader.handOver(target.memberId(), Duration.ofSeconds(10));

QuorumMember next = await(() -> soleLeader(started));
assertEquals(target.memberId(), next.memberId());
assertTrue(next.term() > term, "a later term");
assertFalse(leader.isLeader());
}

private static void assertRefused(
String message, List<String> peers, String id, String dir, String cert, String key, String ca) {
IllegalArgumentException refused = assertThrows(
Expand Down
39 changes: 37 additions & 2 deletions operator/e2e-quorum.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
#!/usr/bin/env bash
# A Kates resource with three members: one leader takes the traffic, a new one is elected when it
# dies and recovers its run, a version change keeps a majority, the leader replaced last, and a
# leader cut off from its peers takes no write once another leads.
# dies and recovers its run, a version change keeps a majority, the leader replaced last, a
# leader cut off from its peers takes no write once another leads, and a leader cut off from
# PostgreSQL hands over.
#
# KUBECONFIG=<scratch> operator/e2e-quorum.sh [namespace]
#
Expand Down Expand Up @@ -243,6 +244,40 @@ done | sort -u | awk '{print $1}' | uniq -d)
kill "${forwards[@]}" 2>/dev/null
rm -f "$claims"

echo "== the leader cut off from PostgreSQL hands over"
"${k[@]}" wait --for=condition=Ready "kates/$name" --timeout=5m >/dev/null
for _ in $(seq 1 30); do [ -n "$(leader)" ] && break; sleep 2; done
cut=$(leader)
cut_ip=$("${k[@]}" get pod "$cut" -o jsonpath='{.status.podIP}')
db_ip=$("${k[@]}" get pods -l "app=$name-db" -o jsonpath='{.items[0].status.podIP}')
# Its database alone: the members, Kafka and port 8080 stay reachable.
db_cut() { # db_cut -I|-D
for node in $nodes; do
docker exec "$node" iptables "$1" FORWARD -s "$cut_ip" -d "$db_ip" -p tcp --dport 5432 -j DROP
docker exec "$node" iptables "$1" FORWARD -s "$db_ip" -d "$cut_ip" -p tcp --sport 5432 -j DROP
done
}
db_cut -I
start=$(date +%s)
second=
for _ in $(seq 1 90); do
second=$(leader)
[ -n "$second" ] && [ "$second" != "$cut" ] && [ "$(labelled)" = "$second" ] && break
second=; sleep 1
done
handed=$(( $(date +%s) - start ))
[ -n "$second" ] && pass "$second leads $handed s after $cut lost PostgreSQL" || fail "$cut still leads 90 s after losing PostgreSQL"
connect
# An empty body: the leader answers 400, a member that does not lead 503, before validation.
check "the new leader takes writes" 400 "$(curl -s -m 5 -o /dev/null -w '%{http_code}' -X POST "$api/api/tests" \
-H 'Content-Type: application/json' -d '{}')"
db_cut -D

# Nightly: CI's Kind has three nodes, and this takes one away for a few minutes.
if [ "${KATES_E2E_NODE_LOSS:-}" = 1 ]; then
operator/e2e-node-loss.sh "$ns" "$name" || failed=1
fi

echo "== deleting the resource removes what it made"
"${k[@]}" delete kates "$name" --timeout=120s >/dev/null
"${k[@]}" wait --for=delete "statefulset/$name" "secret/kates-quorum-ca" "secret/kates-quorum-tls" "pdb/$name" \
Expand Down
Loading