diff --git a/.github/workflows/ci-operator.yml b/.github/workflows/ci-operator.yml index 551e111d..49fca1a0 100644 --- a/.github/workflows/ci-operator.yml +++ b/.github/workflows/ci-operator.yml @@ -203,7 +203,14 @@ jobs: - name: Three members keep a leader, recover its run, roll the leader last, fence a cut-off one and hand over without PostgreSQL env: KATES_E2E_NODE_LOSS: ${{ github.event_name != 'pull_request' && github.event_name != 'push' && '1' || '' }} - run: operator/e2e-quorum.sh payments + run: | + # Kind taints its control plane when it has workers: the node-loss test needs a member + # on each of the three nodes, and still takes a worker's away. + if [ "${KATES_E2E_NODE_LOSS}" = 1 ] \ + && kubectl get node alpha -o jsonpath='{.spec.taints[*].key}' | grep -q node-role.kubernetes.io/control-plane; then + kubectl taint node alpha node-role.kubernetes.io/control-plane:NoSchedule- + fi + operator/e2e-quorum.sh payments - name: Show the stack if: failure() diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java b/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java index 8e407396..91699d74 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/DatabaseHandover.java @@ -19,7 +19,8 @@ * plans/j4-resilient-quorum.md): every write it takes fails, while another member may reach the * database. After 15 s without it, once nothing is live or after 2 minutes, since handing over * fails a live run. At most once per 2 minutes, so members that all lack the database do not pass - * it back and forth. + * it back and forth. A transfer after which this member leads again, its target having lost the + * election, failed too: the next check, 5 s on, tries again. */ @ApplicationScoped public class DatabaseHandover { @@ -60,6 +61,14 @@ void check() { } try { String to = quorum.handOver(TRANSFER_TIMEOUT); + if (quorum.isLeader()) { + // The target could not win its election, and this member won the next one. + handOverFailed(); + LOG.warnf( + "PostgreSQL unreachable since %s: handed to %s, but this member leads again", + unreachableSince, to); + return; + } LOG.warnf("PostgreSQL unreachable since %s: leadership handed to %s", unreachableSince, to); } catch (java.io.IOException | RuntimeException e) { handOverFailed(); diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/EmptyLogCatchUp.java b/kates/src/main/java/com/bmscomp/kates/quorum/EmptyLogCatchUp.java new file mode 100644 index 00000000..62f82d5c --- /dev/null +++ b/kates/src/main/java/com/bmscomp/kates/quorum/EmptyLogCatchUp.java @@ -0,0 +1,74 @@ +package com.bmscomp.kates.quorum; + +import java.time.Duration; +import java.time.Instant; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +import io.quarkus.scheduler.Scheduled; +import org.jboss.logging.Logger; + +/** + * A member that follows a leader with an empty log, as one rebuilt on a new volume does, asks that + * leader to step down: Ratis 3.3.1's leader trusts the index the member matched before it lost its + * disk and sends it heartbeats only, while the next leader sends it the whole log + * ({@link QuorumMember#askLeaderToStepDown}). After 15 s, since a member that just started gets + * the log within a heartbeat; at most once a minute, should the next leader not help either. + */ +@ApplicationScoped +public class EmptyLogCatchUp { + + private static final Logger LOG = Logger.getLogger(EmptyLogCatchUp.class); + static final Duration AFTER = Duration.ofSeconds(15); + static final Duration AGAIN_AFTER = Duration.ofMinutes(1); + private static final Duration STEP_DOWN_TIMEOUT = Duration.ofSeconds(10); + + @Inject + Quorum quorum; + + private Instant emptySince; + private Instant askedAt; + + @Scheduled( + every = "5s", + identity = "quorum-empty-log-catch-up", + concurrentExecution = Scheduled.ConcurrentExecution.SKIP) + void check() { + QuorumMember current = quorum.member(); + boolean emptyUnderLeader = current != null + && current.serving() + && !current.isLeader() + && current.leaderId() != null + && current.appliedIndex() < 0; + if (!decide(Instant.now(), emptyUnderLeader)) { + return; + } + String leader = current.leaderId(); + try { + current.askLeaderToStepDown(STEP_DOWN_TIMEOUT); + LOG.warnf( + "Empty log under %s since %s: it stepped down, so the next leader sends the log", + leader, emptySince); + } catch (java.io.IOException | RuntimeException e) { + LOG.warnf("Empty log under %s since %s, and asking it to step down failed: %s", leader, emptySince, e); + } + } + + /** Whether to ask the leader to step down now, given whether this member follows it with an empty log. */ + boolean decide(Instant now, boolean emptyUnderLeader) { + if (!emptyUnderLeader) { + emptySince = null; + return false; + } + if (emptySince == null) { + emptySince = now; + } + boolean due = Duration.between(emptySince, now).compareTo(AFTER) >= 0; + boolean rested = askedAt == null || Duration.between(askedAt, now).compareTo(AGAIN_AFTER) >= 0; + if (due && rested) { + askedAt = now; + return true; + } + return false; + } +} diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java index 14d1e355..221a8d7e 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java @@ -269,19 +269,42 @@ public boolean confirmLeadership() { * @throws IOException if the transfer did not happen within {@code timeout} */ public void handOver(String target, Duration timeout) throws IOException { + transfer(selfId, RaftPeerId.valueOf(target), timeout); + } + + /** + * Asks the leader this member knows of to step down, so that the next leader, whichever wins, + * starts each member's replication over from what that member reports. Ratis 3.3.1's leader + * goes on trusting the index a member had matched before it lost its disk, and sends it only + * heartbeats; a new leader sends it the whole log. + * + * @throws IOException if there is no leader, or it did not step down within {@code timeout} + */ + public void askLeaderToStepDown(Duration timeout) throws IOException { + RaftPeerId leader = info().getLeaderId(); + if (leader == null) { + throw new IOException("no leader to ask"); + } + transfer(leader, null, timeout); + } + + /** A transfer to {@code target}, or a step-down with none, asked of {@code via}, the leader. */ + private void transfer(RaftPeerId via, RaftPeerId target, Duration timeout) throws IOException { RaftProperties properties = new RaftProperties(); RaftClientConfigKeys.Rpc.setRequestTimeout( properties, TimeDuration.valueOf(timeout.toMillis() + 2000, TimeUnit.MILLISECONDS)); try (RaftClient admin = RaftClient.newBuilder() .setRaftGroup(group) - .setLeaderId(selfId) + .setLeaderId(via) .setProperties(properties) .setParameters(parameters) .setRetryPolicy(RetryPolicies.noRetry()) .build()) { - var reply = admin.admin().transferLeadership(RaftPeerId.valueOf(target), timeout.toMillis()); + var reply = admin.admin().transferLeadership(target, timeout.toMillis()); if (!reply.isSuccess()) { - throw new IOException("leadership not handed to " + target + ": " + reply.getException()); + throw new IOException( + (target == null ? via + " did not step down: " : "leadership not handed to " + target + ": ") + + reply.getException()); } } } diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/EmptyLogCatchUpTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/EmptyLogCatchUpTest.java new file mode 100644 index 00000000..eece3ed6 --- /dev/null +++ b/kates/src/test/java/com/bmscomp/kates/quorum/EmptyLogCatchUpTest.java @@ -0,0 +1,41 @@ +package com.bmscomp.kates.quorum; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Instant; + +import org.junit.jupiter.api.Test; + +class EmptyLogCatchUpTest { + + private final EmptyLogCatchUp catchUp = new EmptyLogCatchUp(); + + private static Instant at(long seconds) { + return Instant.EPOCH.plusSeconds(seconds); + } + + @Test + void asksAfterFifteenSecondsWithAnEmptyLog() { + assertFalse(catchUp.decide(at(0), true)); + assertFalse(catchUp.decide(at(10), true)); + assertTrue(catchUp.decide(at(15), true)); + } + + @Test + void asksAgainOnlyAMinuteLater() { + catchUp.decide(at(0), true); + assertTrue(catchUp.decide(at(15), true)); + assertFalse(catchUp.decide(at(20), true)); + assertFalse(catchUp.decide(at(70), true)); + assertTrue(catchUp.decide(at(75), true)); + } + + @Test + void aLogThatFillsStartsTheClockOver() { + catchUp.decide(at(0), true); + assertFalse(catchUp.decide(at(10), false)); + assertFalse(catchUp.decide(at(15), true)); + assertTrue(catchUp.decide(at(30), true)); + } +} diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java index a8bfbde3..2fcf0b36 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java @@ -259,6 +259,51 @@ void theLeaderHandsLeadershipToTheMemberItNames() throws Exception { assertFalse(leader.isLeader()); } + @Test + void aMemberRebuiltOnAnEmptyDiskCatchesUpOnceTheLeaderStepsDown() throws Exception { + TestPki pki = new TestPki(tmp.resolve("pki")); + List peers = peers(3); + Path[][] files = certificates(pki, 3); + for (int i = 0; i < 3; i++) { + started.add(QuorumMember.start(settings(peers, i, files[i]))); + } + QuorumMember first = await(() -> soleLeader(started)); + // A few terms, so the leader has matched more than the first entry. + QuorumMember other = + started.stream().filter(m -> m != first).findFirst().orElseThrow(); + first.handOver(other.memberId(), Duration.ofSeconds(10)); + await(() -> other.isLeader() ? other : null); + other.handOver(first.memberId(), Duration.ofSeconds(10)); + QuorumMember leader = await(() -> first.isLeader() ? first : null); + QuorumMember lost = started.stream() + .filter(m -> m != leader && m != other) + .findFirst() + .orElseThrow(); + await(() -> lost.appliedIndex() >= leader.appliedIndex() ? lost : null); + int index = Integer.parseInt(lost.memberId().substring(1)); + lost.close(); + started.remove(lost); + deleteTree(tmp.resolve("data-m" + index)); + + QuorumMember rebuilt = QuorumMember.start(settings(peers, index, files[index])); + started.add(rebuilt); + await(() -> leader.memberId().equals(rebuilt.leaderId()) ? rebuilt : null); + TimeUnit.SECONDS.sleep(3); + // Ratis' gap EmptyLogCatchUp works around: should it fail, Ratis catches the member up itself. + assertEquals(-1, rebuilt.appliedIndex(), "under the same leader, it gets heartbeats only"); + + rebuilt.askLeaderToStepDown(Duration.ofSeconds(10)); + await(() -> rebuilt.appliedIndex() >= 0 ? rebuilt : null); + } + + private static void deleteTree(Path root) throws IOException { + try (var paths = Files.walk(root)) { + for (Path path : paths.sorted(java.util.Comparator.reverseOrder()).toList()) { + Files.delete(path); + } + } + } + private static void assertRefused( String message, List peers, String id, String dir, String cert, String key, String ca) { IllegalArgumentException refused = assertThrows( diff --git a/operator/e2e-node-loss.sh b/operator/e2e-node-loss.sh index 7c36bee8..c1a68be9 100755 --- a/operator/e2e-node-loss.sh +++ b/operator/e2e-node-loss.sh @@ -24,19 +24,44 @@ ready() { "${k[@]}" get kates "$name" -o jsonpath='{.status.conditions[?(@.type= echo "== three members, one a node" "${k[@]}" wait --for=condition=Ready "kates/$name" --timeout=5m >/dev/null || { fail "not Ready"; exit 1; } members=("$name-0" "$name-1" "$name-2") -nodes=$(for pod in "${members[@]}"; do node_of "$pod"; done | sort -u | wc -l | tr -d ' ') -# Two on the lost node would take the majority, and nothing would be replaced: by design. -[ "$nodes" = 3 ] && pass "spread over three nodes" || { fail "on $nodes nodes: give one a new volume first"; exit 1; } +spread() { for pod in "${members[@]}"; do node_of "$pod"; done | sort -u | wc -l | tr -d ' '; } +# Two on the lost node would take the majority, and nothing would be replaced: by design. The +# anti-affinity is preferred, and a local volume pins a member to its node: on two nodes, a +# follower sharing one gets a new volume, which the scheduler places on the third. +if [ "$(spread)" != 3 ]; then + shared= + for pod in "${members[@]}"; do + [ "$pod" = "$(leader)" ] && continue + for other in "${members[@]}"; do + [ "$other" != "$pod" ] && [ "$(node_of "$other")" = "$(node_of "$pod")" ] && shared=$pod + done + [ -n "$shared" ] && break + done + if [ -n "$shared" ]; then + "${k[@]}" delete pvc "quorum-$shared" --wait=false >/dev/null + "${k[@]}" delete pod "$shared" >/dev/null + for _ in $(seq 1 90); do + [ "$(spread)" = 3 ] && [ "$(ready)" = True ] && break + sleep 2 + done + fi +fi +nodes=$(spread) +[ "$nodes" = 3 ] && pass "spread over three nodes${shared:+, $shared on a new volume}" \ + || { fail "on $nodes nodes after giving ${shared:-no member} a new volume"; exit 1; } -# A follower on a worker: the control plane's node holds the API server. +# A follower on a worker: the control plane's node holds the API server. Not on the database's +# node either: without PostgreSQL, the leader hands over by design. lead=$(leader) +db_node=$("${k[@]}" get pods -l "app=$name-db" -o jsonpath='{.items[0].spec.nodeName}' 2>/dev/null) victim= for pod in "${members[@]}"; do node=$(node_of "$pod") - [ "$pod" != "$lead" ] && [[ "$(kubectl get node "$node" -o jsonpath='{.metadata.labels}')" != *control-plane* ]] \ + [ "$pod" != "$lead" ] && [ "$node" != "$db_node" ] \ + && [[ "$(kubectl get node "$node" -o jsonpath='{.metadata.labels}')" != *control-plane* ]] \ && victim=$pod && break done -[ -n "$victim" ] || { fail "no follower on a worker node"; exit 1; } +[ -n "$victim" ] || { fail "no follower on a worker node without the database ($db_node)"; exit 1; } node=$(node_of "$victim") container=$(kubectl get node "$node" -o jsonpath='{.spec.providerID}' | sed 's|.*/||') before=$(claim_uid "$victim") @@ -47,7 +72,11 @@ start=$(date +%s) docker stop "$container" >/dev/null kubectl delete node "$node" --wait=false >/dev/null moved= +changed= for _ in $(seq 1 240); do + # Sampled while the victim is not back: once back on an empty log, it has the leader step down. + current=$(leader) + [ -n "$current" ] && [ "$current" != "$lead" ] && changed=$current now=$(claim_uid "$victim") [ -n "$now" ] && [ "$now" != "$before" ] && [ -n "$(node_of "$victim")" ] && [ "$(node_of "$victim")" != "$node" ] \ && [ "$("${k[@]}" get pod "$victim" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')" = True ] \ @@ -56,18 +85,26 @@ for _ in $(seq 1 240); do done [ -n "$moved" ] && pass "$victim runs on $(node_of "$victim") with a new volume, $(( $(date +%s) - start )) s after the loss" \ || fail "$victim was not replaced within 8 minutes ($("${k[@]}" get pod "$victim" -o jsonpath='{.status.phase}'))" -check_lead=$(leader) -[ "$check_lead" = "$lead" ] && pass "$lead led throughout" || fail "the leader changed from $lead to '$check_lead'" +[ -z "$changed" ] && pass "$lead led while $victim was gone" || fail "the leader changed from $lead to '$changed'" for _ in $(seq 1 60); do [ "$(ready)" = True ] && break; sleep 2; done [ "$(ready)" = True ] && pass "Ready with three members" \ || fail "not Ready: $("${k[@]}" get kates "$name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}')" -# Ready before caught up: the pod's readiness does not wait for the log. -for _ in $(seq 1 30); do +# Ready before caught up: the pod's readiness does not wait for the log. Under the same leader, +# the member waits 15 s, then has it step down (EmptyLogCatchUp), and the next leader sends the log. +for _ in $(seq 1 45); do applied=$(kubectl get --raw "/api/v1/namespaces/$ns/services/$victim:8080/proxy/api/health" | jq '.quorum.appliedIndex') [ "${applied:--1}" -ge 0 ] && break sleep 2 done -[ "${applied:--1}" -ge 0 ] && pass "$victim caught up (applied index $applied)" || fail "$victim has not caught up: '$applied'" +if [ "${applied:--1}" -ge 0 ]; then + pass "$victim caught up (applied index $applied), $(leader) leads" +else + fail "$victim has not caught up: '$applied'" + for pod in "$lead" "$victim"; do + echo "--- $pod" + "${k[@]}" logs "$pod" --since=300s | grep -E 'ratis|Quorum' | cut -c1-800 | tail -60 + done +fi echo "== $node comes back" docker start "$container" >/dev/null diff --git a/operator/e2e-quorum.sh b/operator/e2e-quorum.sh index 59357b28..884ef0dc 100755 --- a/operator/e2e-quorum.sh +++ b/operator/e2e-quorum.sh @@ -37,6 +37,14 @@ spec: template: metadata: {labels: {app: $name-db}} spec: + # On the control plane where it can: e2e-node-loss.sh takes a worker away, and the members + # would all lose PostgreSQL with it. + tolerations: [{key: node-role.kubernetes.io/control-plane, operator: Exists, effect: NoSchedule}] + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: {matchExpressions: [{key: node-role.kubernetes.io/control-plane, operator: Exists}]} containers: - name: postgres image: postgres:17-alpine @@ -270,7 +278,12 @@ if [ -n "$second" ]; then pass "$second leads $handed s after $cut lost PostgreSQL" else fail "$cut still leads 90 s after losing PostgreSQL" - "${k[@]}" logs "$cut" --since=3m | grep -E 'PostgreSQL|[Ll]eadership|leads term' | tail -20 + # The resource is deleted before CI's Show the stack: each member's last lines, here. + echo "--- status: $("${k[@]}" get kates "$name" -o jsonpath="{.status}")"; "${k[@]}" get pods -o wide + for pod in "$name-0" "$name-1" "$name-2"; do + echo "--- $pod" + "${k[@]}" logs "$pod" --since=150s | grep -vE 'VertxGrpcSender|"level":"DEBUG"' | cut -c1-800 | tail -60 + done fi connect # An empty body: the leader answers 400, a member that does not lead 503, before validation.