Skip to content
9 changes: 8 additions & 1 deletion .github/workflows/ci-operator.yml
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,14 @@ jobs:
- name: Three members keep a leader, recover its run, roll the leader last, fence a cut-off one and hand over without PostgreSQL
env:
KATES_E2E_NODE_LOSS: ${{ github.event_name != 'pull_request' && github.event_name != 'push' && '1' || '' }}
run: operator/e2e-quorum.sh payments
run: |
# Kind taints its control plane when it has workers: the node-loss test needs a member
# on each of the three nodes, and still takes a worker's away.
if [ "${KATES_E2E_NODE_LOSS}" = 1 ] \
&& kubectl get node alpha -o jsonpath='{.spec.taints[*].key}' | grep -q node-role.kubernetes.io/control-plane; then
kubectl taint node alpha node-role.kubernetes.io/control-plane:NoSchedule-
fi
operator/e2e-quorum.sh payments

- name: Show the stack
if: failure()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
* plans/j4-resilient-quorum.md): every write it takes fails, while another member may reach the
* database. After 15 s without it, once nothing is live or after 2 minutes, since handing over
* fails a live run. At most once per 2 minutes, so members that all lack the database do not pass
* it back and forth.
* it back and forth. A transfer after which this member leads again, its target having lost the
* election, failed too: the next check, 5 s on, tries again.
*/
@ApplicationScoped
public class DatabaseHandover {
Expand Down Expand Up @@ -60,6 +61,14 @@ void check() {
}
try {
String to = quorum.handOver(TRANSFER_TIMEOUT);
if (quorum.isLeader()) {
// The target could not win its election, and this member won the next one.
handOverFailed();
LOG.warnf(
"PostgreSQL unreachable since %s: handed to %s, but this member leads again",
unreachableSince, to);
return;
}
LOG.warnf("PostgreSQL unreachable since %s: leadership handed to %s", unreachableSince, to);
} catch (java.io.IOException | RuntimeException e) {
handOverFailed();
Expand Down
74 changes: 74 additions & 0 deletions kates/src/main/java/com/bmscomp/kates/quorum/EmptyLogCatchUp.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package com.bmscomp.kates.quorum;

import java.time.Duration;
import java.time.Instant;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;

import io.quarkus.scheduler.Scheduled;
import org.jboss.logging.Logger;

/**
* A member that follows a leader with an empty log, as one rebuilt on a new volume does, asks that
* leader to step down: Ratis 3.3.1's leader trusts the index the member matched before it lost its
* disk and sends it heartbeats only, while the next leader sends it the whole log
* ({@link QuorumMember#askLeaderToStepDown}). After 15 s, since a member that just started gets
* the log within a heartbeat; at most once a minute, should the next leader not help either.
*/
@ApplicationScoped
public class EmptyLogCatchUp {

private static final Logger LOG = Logger.getLogger(EmptyLogCatchUp.class);
static final Duration AFTER = Duration.ofSeconds(15);
static final Duration AGAIN_AFTER = Duration.ofMinutes(1);
private static final Duration STEP_DOWN_TIMEOUT = Duration.ofSeconds(10);

@Inject
Quorum quorum;

private Instant emptySince;
private Instant askedAt;

@Scheduled(
every = "5s",
identity = "quorum-empty-log-catch-up",
concurrentExecution = Scheduled.ConcurrentExecution.SKIP)
void check() {
QuorumMember current = quorum.member();
boolean emptyUnderLeader = current != null
&& current.serving()
&& !current.isLeader()
&& current.leaderId() != null
&& current.appliedIndex() < 0;
if (!decide(Instant.now(), emptyUnderLeader)) {
return;
}
String leader = current.leaderId();
try {
current.askLeaderToStepDown(STEP_DOWN_TIMEOUT);
LOG.warnf(
"Empty log under %s since %s: it stepped down, so the next leader sends the log",
leader, emptySince);
} catch (java.io.IOException | RuntimeException e) {
LOG.warnf("Empty log under %s since %s, and asking it to step down failed: %s", leader, emptySince, e);
}
}

/** Whether to ask the leader to step down now, given whether this member follows it with an empty log. */
boolean decide(Instant now, boolean emptyUnderLeader) {
if (!emptyUnderLeader) {
emptySince = null;
return false;
}
if (emptySince == null) {
emptySince = now;
}
boolean due = Duration.between(emptySince, now).compareTo(AFTER) >= 0;
boolean rested = askedAt == null || Duration.between(askedAt, now).compareTo(AGAIN_AFTER) >= 0;
if (due && rested) {
askedAt = now;
return true;
}
return false;
}
}
29 changes: 26 additions & 3 deletions kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java
Original file line number Diff line number Diff line change
Expand Up @@ -269,19 +269,42 @@ public boolean confirmLeadership() {
* @throws IOException if the transfer did not happen within {@code timeout}
*/
public void handOver(String target, Duration timeout) throws IOException {
transfer(selfId, RaftPeerId.valueOf(target), timeout);
}

/**
* Asks the leader this member knows of to step down, so that the next leader, whichever wins,
* starts each member's replication over from what that member reports. Ratis 3.3.1's leader
* goes on trusting the index a member had matched before it lost its disk, and sends it only
* heartbeats; a new leader sends it the whole log.
*
* @throws IOException if there is no leader, or it did not step down within {@code timeout}
*/
public void askLeaderToStepDown(Duration timeout) throws IOException {
RaftPeerId leader = info().getLeaderId();
if (leader == null) {
throw new IOException("no leader to ask");
}
transfer(leader, null, timeout);
}

/** A transfer to {@code target}, or a step-down with none, asked of {@code via}, the leader. */
private void transfer(RaftPeerId via, RaftPeerId target, Duration timeout) throws IOException {
RaftProperties properties = new RaftProperties();
RaftClientConfigKeys.Rpc.setRequestTimeout(
properties, TimeDuration.valueOf(timeout.toMillis() + 2000, TimeUnit.MILLISECONDS));
try (RaftClient admin = RaftClient.newBuilder()
.setRaftGroup(group)
.setLeaderId(selfId)
.setLeaderId(via)
.setProperties(properties)
.setParameters(parameters)
.setRetryPolicy(RetryPolicies.noRetry())
.build()) {
var reply = admin.admin().transferLeadership(RaftPeerId.valueOf(target), timeout.toMillis());
var reply = admin.admin().transferLeadership(target, timeout.toMillis());
if (!reply.isSuccess()) {
throw new IOException("leadership not handed to " + target + ": " + reply.getException());
throw new IOException(
(target == null ? via + " did not step down: " : "leadership not handed to " + target + ": ")
+ reply.getException());
}
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package com.bmscomp.kates.quorum;

import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.time.Instant;

import org.junit.jupiter.api.Test;

class EmptyLogCatchUpTest {

private final EmptyLogCatchUp catchUp = new EmptyLogCatchUp();

private static Instant at(long seconds) {
return Instant.EPOCH.plusSeconds(seconds);
}

@Test
void asksAfterFifteenSecondsWithAnEmptyLog() {
assertFalse(catchUp.decide(at(0), true));
assertFalse(catchUp.decide(at(10), true));
assertTrue(catchUp.decide(at(15), true));
}

@Test
void asksAgainOnlyAMinuteLater() {
catchUp.decide(at(0), true);
assertTrue(catchUp.decide(at(15), true));
assertFalse(catchUp.decide(at(20), true));
assertFalse(catchUp.decide(at(70), true));
assertTrue(catchUp.decide(at(75), true));
}

@Test
void aLogThatFillsStartsTheClockOver() {
catchUp.decide(at(0), true);
assertFalse(catchUp.decide(at(10), false));
assertFalse(catchUp.decide(at(15), true));
assertTrue(catchUp.decide(at(30), true));
}
}
45 changes: 45 additions & 0 deletions kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,51 @@ void theLeaderHandsLeadershipToTheMemberItNames() throws Exception {
assertFalse(leader.isLeader());
}

@Test
void aMemberRebuiltOnAnEmptyDiskCatchesUpOnceTheLeaderStepsDown() throws Exception {
TestPki pki = new TestPki(tmp.resolve("pki"));
List<String> peers = peers(3);
Path[][] files = certificates(pki, 3);
for (int i = 0; i < 3; i++) {
started.add(QuorumMember.start(settings(peers, i, files[i])));
}
QuorumMember first = await(() -> soleLeader(started));
// A few terms, so the leader has matched more than the first entry.
QuorumMember other =
started.stream().filter(m -> m != first).findFirst().orElseThrow();
first.handOver(other.memberId(), Duration.ofSeconds(10));
await(() -> other.isLeader() ? other : null);
other.handOver(first.memberId(), Duration.ofSeconds(10));
QuorumMember leader = await(() -> first.isLeader() ? first : null);
QuorumMember lost = started.stream()
.filter(m -> m != leader && m != other)
.findFirst()
.orElseThrow();
await(() -> lost.appliedIndex() >= leader.appliedIndex() ? lost : null);
int index = Integer.parseInt(lost.memberId().substring(1));
lost.close();
started.remove(lost);
deleteTree(tmp.resolve("data-m" + index));

QuorumMember rebuilt = QuorumMember.start(settings(peers, index, files[index]));
started.add(rebuilt);
await(() -> leader.memberId().equals(rebuilt.leaderId()) ? rebuilt : null);
TimeUnit.SECONDS.sleep(3);
// Ratis' gap EmptyLogCatchUp works around: should it fail, Ratis catches the member up itself.
assertEquals(-1, rebuilt.appliedIndex(), "under the same leader, it gets heartbeats only");

rebuilt.askLeaderToStepDown(Duration.ofSeconds(10));
await(() -> rebuilt.appliedIndex() >= 0 ? rebuilt : null);
}

private static void deleteTree(Path root) throws IOException {
try (var paths = Files.walk(root)) {
for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) {
Files.delete(path);
}
}
}

private static void assertRefused(
String message, List<String> peers, String id, String dir, String cert, String key, String ca) {
IllegalArgumentException refused = assertThrows(
Expand Down
59 changes: 48 additions & 11 deletions operator/e2e-node-loss.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,44 @@ ready() { "${k[@]}" get kates "$name" -o jsonpath='{.status.conditions[?(@.type=
echo "== three members, one a node"
"${k[@]}" wait --for=condition=Ready "kates/$name" --timeout=5m >/dev/null || { fail "not Ready"; exit 1; }
members=("$name-0" "$name-1" "$name-2")
nodes=$(for pod in "${members[@]}"; do node_of "$pod"; done | sort -u | wc -l | tr -d ' ')
# Two on the lost node would take the majority, and nothing would be replaced: by design.
[ "$nodes" = 3 ] && pass "spread over three nodes" || { fail "on $nodes nodes: give one a new volume first"; exit 1; }
spread() { for pod in "${members[@]}"; do node_of "$pod"; done | sort -u | wc -l | tr -d ' '; }
# Two on the lost node would take the majority, and nothing would be replaced: by design. The
# anti-affinity is preferred, and a local volume pins a member to its node: on two nodes, a
# follower sharing one gets a new volume, which the scheduler places on the third.
if [ "$(spread)" != 3 ]; then
shared=
for pod in "${members[@]}"; do
[ "$pod" = "$(leader)" ] && continue
for other in "${members[@]}"; do
[ "$other" != "$pod" ] && [ "$(node_of "$other")" = "$(node_of "$pod")" ] && shared=$pod
done
[ -n "$shared" ] && break
done
if [ -n "$shared" ]; then
"${k[@]}" delete pvc "quorum-$shared" --wait=false >/dev/null
"${k[@]}" delete pod "$shared" >/dev/null
for _ in $(seq 1 90); do
[ "$(spread)" = 3 ] && [ "$(ready)" = True ] && break
sleep 2
done
fi
fi
nodes=$(spread)
[ "$nodes" = 3 ] && pass "spread over three nodes${shared:+, $shared on a new volume}" \
|| { fail "on $nodes nodes after giving ${shared:-no member} a new volume"; exit 1; }

# A follower on a worker: the control plane's node holds the API server.
# A follower on a worker: the control plane's node holds the API server. Not on the database's
# node either: without PostgreSQL, the leader hands over by design.
lead=$(leader)
db_node=$("${k[@]}" get pods -l "app=$name-db" -o jsonpath='{.items[0].spec.nodeName}' 2>/dev/null)
victim=
for pod in "${members[@]}"; do
node=$(node_of "$pod")
[ "$pod" != "$lead" ] && [[ "$(kubectl get node "$node" -o jsonpath='{.metadata.labels}')" != *control-plane* ]] \
[ "$pod" != "$lead" ] && [ "$node" != "$db_node" ] \
&& [[ "$(kubectl get node "$node" -o jsonpath='{.metadata.labels}')" != *control-plane* ]] \
&& victim=$pod && break
done
[ -n "$victim" ] || { fail "no follower on a worker node"; exit 1; }
[ -n "$victim" ] || { fail "no follower on a worker node without the database ($db_node)"; exit 1; }
node=$(node_of "$victim")
container=$(kubectl get node "$node" -o jsonpath='{.spec.providerID}' | sed 's|.*/||')
before=$(claim_uid "$victim")
Expand All @@ -47,7 +72,11 @@ start=$(date +%s)
docker stop "$container" >/dev/null
kubectl delete node "$node" --wait=false >/dev/null
moved=
changed=
for _ in $(seq 1 240); do
# Sampled while the victim is not back: once back on an empty log, it has the leader step down.
current=$(leader)
[ -n "$current" ] && [ "$current" != "$lead" ] && changed=$current
now=$(claim_uid "$victim")
[ -n "$now" ] && [ "$now" != "$before" ] && [ -n "$(node_of "$victim")" ] && [ "$(node_of "$victim")" != "$node" ] \
&& [ "$("${k[@]}" get pod "$victim" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')" = True ] \
Expand All @@ -56,18 +85,26 @@ for _ in $(seq 1 240); do
done
[ -n "$moved" ] && pass "$victim runs on $(node_of "$victim") with a new volume, $(( $(date +%s) - start )) s after the loss" \
|| fail "$victim was not replaced within 8 minutes ($("${k[@]}" get pod "$victim" -o jsonpath='{.status.phase}'))"
check_lead=$(leader)
[ "$check_lead" = "$lead" ] && pass "$lead led throughout" || fail "the leader changed from $lead to '$check_lead'"
[ -z "$changed" ] && pass "$lead led while $victim was gone" || fail "the leader changed from $lead to '$changed'"
for _ in $(seq 1 60); do [ "$(ready)" = True ] && break; sleep 2; done
[ "$(ready)" = True ] && pass "Ready with three members" \
|| fail "not Ready: $("${k[@]}" get kates "$name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}')"
# Ready before caught up: the pod's readiness does not wait for the log.
for _ in $(seq 1 30); do
# Ready before caught up: the pod's readiness does not wait for the log. Under the same leader,
# the member waits 15 s, then has it step down (EmptyLogCatchUp), and the next leader sends the log.
for _ in $(seq 1 45); do
applied=$(kubectl get --raw "/api/v1/namespaces/$ns/services/$victim:8080/proxy/api/health" | jq '.quorum.appliedIndex')
[ "${applied:--1}" -ge 0 ] && break
sleep 2
done
[ "${applied:--1}" -ge 0 ] && pass "$victim caught up (applied index $applied)" || fail "$victim has not caught up: '$applied'"
if [ "${applied:--1}" -ge 0 ]; then
pass "$victim caught up (applied index $applied), $(leader) leads"
else
fail "$victim has not caught up: '$applied'"
for pod in "$lead" "$victim"; do
echo "--- $pod"
"${k[@]}" logs "$pod" --since=300s | grep -E 'ratis|Quorum' | cut -c1-800 | tail -60
done
fi

echo "== $node comes back"
docker start "$container" >/dev/null
Expand Down
15 changes: 14 additions & 1 deletion operator/e2e-quorum.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,14 @@ spec:
template:
metadata: {labels: {app: $name-db}}
spec:
# On the control plane where it can: e2e-node-loss.sh takes a worker away, and the members
# would all lose PostgreSQL with it.
tolerations: [{key: node-role.kubernetes.io/control-plane, operator: Exists, effect: NoSchedule}]
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference: {matchExpressions: [{key: node-role.kubernetes.io/control-plane, operator: Exists}]}
containers:
- name: postgres
image: postgres:17-alpine
Expand Down Expand Up @@ -270,7 +278,12 @@ if [ -n "$second" ]; then
pass "$second leads $handed s after $cut lost PostgreSQL"
else
fail "$cut still leads 90 s after losing PostgreSQL"
"${k[@]}" logs "$cut" --since=3m | grep -E 'PostgreSQL|[Ll]eadership|leads term' | tail -20
# The resource is deleted before CI's Show the stack: each member's last lines, here.
echo "--- status: $("${k[@]}" get kates "$name" -o jsonpath="{.status}")"; "${k[@]}" get pods -o wide
for pod in "$name-0" "$name-1" "$name-2"; do
echo "--- $pod"
"${k[@]}" logs "$pod" --since=150s | grep -vE 'VertxGrpcSender|"level":"DEBUG"' | cut -c1-800 | tail -60
done
fi
connect
# An empty body: the leader answers 400, a member that does not lead 503, before validation.
Expand Down
Loading