From 4c689088913eb5c491336960710492d3d63c6653 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 12:08:39 +0100 Subject: [PATCH 1/6] fix(backend): a quorum member whose Ratis server closed starts it again from its disk Ratis closes its own server after a JVM pause longer than raft.server.close.threshold (60 s). The Kates process lived on and kept answering /api/health with its old term and leader, so the quorum was a member short until the pod restarted. Quorum's 2 s schedule now starts a new member on the same disk when the server is no longer running, and health shows the server's state without the stale fields. Ratis exits the JVM when a server it starts cannot bind; that is now an ExitException the next tick retries. --- .../java/com/bmscomp/kates/quorum/Quorum.java | 57 +++++++++++++++++-- .../bmscomp/kates/quorum/QuorumMember.java | 16 ++++++ .../kates/quorum/QuorumMemberTest.java | 46 +++++++++++++++ .../com/bmscomp/kates/quorum/QuorumTest.java | 48 ++++++++++++++++ 4 files changed, 163 insertions(+), 4 deletions(-) diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java index 722d8bfc..5404a588 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java @@ -54,12 +54,18 @@ public class Quorum { private volatile QuorumMember member; private volatile long recoveredTerm = -1; + private QuorumMember.Settings settings; + private volatile boolean stopping; + private volatile Instant notServingSince; void start(@Observes StartupEvent event) throws IOException { if (peers.isEmpty()) { return; } - QuorumMember.Settings settings = QuorumMember.Settings.parse( + // Ratis exits the JVM when a server it starts cannot bind, which would kill live runs over a + // member this class starts again anyway: an ExitException instead, caught by revive(). + org.apache.ratis.util.ExitUtils.disableSystemExit(); + settings = QuorumMember.Settings.parse( peers.get(), memberId(), dir.orElse(null), cert.orElse(null), key.orElse(null), ca.orElse(null)); member = QuorumMember.start(settings); LOG.infof( @@ -68,6 +74,7 @@ void start(@Observes StartupEvent event) throws IOException { } void stop(@Observes ShutdownEvent event) { + stopping = true; QuorumMember current = member; if (current == null) { return; @@ -87,7 +94,7 @@ public String memberId() { /** Whether this instance leads, as it last heard: cheap, and late by up to a step-down. */ public boolean isLeader() { QuorumMember current = member; - return current == null || current.isLeader(); + return current == null || (current.serving() && current.isLeader()); } /** @@ -101,7 +108,7 @@ public boolean acceptsWrites() { if (current == null) { return peers.isEmpty(); } - if (!current.confirmLeadership()) { + if (!current.serving() || !current.confirmLeadership()) { return false; } long term = current.term(); @@ -127,14 +134,53 @@ public boolean acceptsWrites() { @Scheduled(every = "2s", identity = "quorum-takeover", concurrentExecution = Scheduled.ConcurrentExecution.SKIP) void takeOver() { if (member != null) { + revive(); acceptsWrites(); } } + /** + * Starts the member again from its disk if Ratis has closed its server, as it does after a + * pause longer than raft.server.close.threshold (60 s): the process lives on, and without this + * the member is lost to the quorum until the pod restarts, answering health with its old term. + */ + private void revive() { + QuorumMember current = member; + if (stopping || current.serving()) { + return; + } + if (notServingSince == null) { + notServingSince = Instant.now(); + } + LOG.warnf("quorum member %s: Ratis server %s, starting it again from its disk", current.memberId(), current.state()); + try { + current.close(); + } catch (IOException | RuntimeException e) { + LOG.debugf("closing the old member: %s", e); + } + try { + member = QuorumMember.start(settings); + notServingSince = null; + recoveredTerm = -1; + LOG.infof("quorum member %s serves again", settings.memberId()); + } catch (IOException | RuntimeException e) { + LOG.warnf("quorum member %s could not start again, retrying: %s", settings.memberId(), e); + } + } + + /** Since when this member's Ratis server has not been running; empty while it serves, or with no quorum. */ + public Optional notServingSince() { + return Optional.ofNullable(notServingSince); + } + + QuorumMember member() { + return member; + } + /** The leader this member knows of, or null with no quorum or during an election. */ public String leaderId() { QuorumMember current = member; - return current == null ? null : current.leaderId(); + return current == null || !current.serving() ? null : current.leaderId(); } /** What {@code /api/health} shows. */ @@ -143,7 +189,10 @@ public Map health() { Map health = new LinkedHashMap<>(); health.put("enabled", current != null); if (current != null) { + health.put("state", current.state().name()); health.put("memberId", current.memberId()); + } + if (current != null && current.serving()) { health.put("leader", current.isLeader()); health.put("leaderId", current.leaderId()); health.put("term", current.term()); diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java index bfc08e2a..6f35d8f8 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java @@ -30,6 +30,7 @@ import org.apache.ratis.server.RaftServerConfigKeys; import org.apache.ratis.server.storage.RaftStorage; import org.apache.ratis.statemachine.impl.BaseStateMachine; +import org.apache.ratis.util.LifeCycle; import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslProvider; import org.apache.ratis.util.TimeDuration; @@ -190,6 +191,21 @@ public String memberId() { return memberId; } + /** The Ratis server's life cycle: RUNNING while it serves. */ + public LifeCycle.State state() { + return server.getLifeCycleState(); + } + + /** False once Ratis has closed the server, as it does after a pause past its close threshold. */ + public boolean serving() { + return state().isRunning(); + } + + /** Closes the server only, as Ratis' JvmPauseMonitor does; for tests. */ + void closeServer() throws IOException { + server.close(); + } + /** True once this member leads and has committed an entry of its term, so it knows it leads. */ public boolean isLeader() { return info().isLeaderReady(); diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java index bff804eb..7dd44f40 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java @@ -96,6 +96,52 @@ void threeMembersElectOneLeaderAndAnotherWhenItGoes() throws Exception { assertTrue(second.term() > term, "a new leader has a higher term"); } + @Test + void aClosedServerSaysSo() throws Exception { + TestPki pki = new TestPki(tmp.resolve("pki")); + List peers = peers(1); + QuorumMember member = QuorumMember.start(settings(peers, 0, certificates(pki, 1)[0])); + started.add(member); + assertTrue(member.serving(), "a started member serves"); + + member.closeServer(); + + assertEquals(org.apache.ratis.util.LifeCycle.State.CLOSED, member.state()); + assertFalse(member.serving(), "a member whose server closed does not serve"); + } + + @Test + void aMemberWhoseServerClosesRejoinsAsAnotherTerm() throws Exception { + TestPki pki = new TestPki(tmp.resolve("pki")); + List peers = peers(3); + Path[][] files = certificates(pki, 3); + for (int i = 0; i < 3; i++) { + started.add(QuorumMember.start(settings(peers, i, files[i]))); + } + QuorumMember first = await(() -> soleLeader(started)); + long term = first.term(); + long applied = first.appliedIndex(); + for (QuorumMember member : started) { + await(() -> member.appliedIndex() >= applied ? member : null); + } + + // As Ratis does past raft.server.close.threshold, then a new server on the same disk. + first.closeServer(); + int index = Integer.parseInt(first.memberId().substring(1)); + QuorumMember again = QuorumMember.start(settings(peers, index, files[index])); + started.remove(first); + first.close(); + started.add(again); + + QuorumMember leader = await(() -> { + QuorumMember sole = soleLeader(started); + return sole != null && sole.term() > term ? sole : null; + }); + assertTrue(leader.term() > term, "a later term leads"); + assertTrue(again.serving(), "the restarted member serves"); + await(() -> leader.memberId().equals(again.leaderId()) ? again : null); + } + @Test void aMemberWithAnotherCasCertificateCannotJoin() throws Exception { TestPki pki = new TestPki(tmp.resolve("pki")); diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java index 7360d9d1..4e7c1de4 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java @@ -71,6 +71,54 @@ void aLeaderWhoseRecoveryFailsTakesNoWriteUntilItSucceeds() throws Exception { assertTrue(takeovers.get(takeovers.size() - 1).startsWith(leader.memberId() + "@")); } + @Test + void aClosedMemberIsRestartedBySchedule() throws Exception { + start(1, new AtomicBoolean(false)); + Quorum quorum = members.get(0); + soleWriter(members); + + quorum.member().closeServer(); + assertTrue(quorum.notServingSince().isEmpty(), "nothing noticed before the schedule runs"); + quorum.takeOver(); + + assertTrue(quorum.notServingSince().isEmpty(), "restarted: " + quorum.health()); + assertEquals("RUNNING", quorum.health().get("state")); + assertTrue(quorum.member().serving()); + soleWriter(members); + } + + @Test + void healthWhileClosedNamesTheState() throws Exception { + start(1, new AtomicBoolean(false)); + Quorum quorum = members.get(0); + soleWriter(members); + int port = Integer.parseInt(quorum.peers.orElseThrow().get(0).replaceAll(".*:", "")); + + quorum.member().closeServer(); + // The port held by someone else: the new server cannot bind, so the member stays down. + try (java.net.ServerSocket held = new java.net.ServerSocket(port)) { + quorum.takeOver(); + + assertEquals("CLOSED", quorum.health().get("state")); + assertFalse(quorum.health().containsKey("term"), "no stale term: " + quorum.health()); + assertTrue(quorum.notServingSince().isPresent()); + assertFalse(quorum.acceptsWrites(), "a member that does not serve takes no write"); + } + } + + @Test + void noRestartAfterShutdown() throws Exception { + start(1, new AtomicBoolean(false)); + Quorum quorum = members.get(0); + QuorumMember before = quorum.member(); + + quorum.stop(null); + quorum.takeOver(); + + assertTrue(before == quorum.member(), "no member started after shutdown"); + assertFalse(quorum.member().serving()); + } + private void start(int count, AtomicBoolean failing) throws Exception { TestPki pki = new TestPki(tmp.resolve("pki")); Path[][] certificates = QuorumMemberTest.certificates(pki, count); From 4f20e4171de584e26af225b43acc0a226b949254 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 12:19:23 +0100 Subject: [PATCH 2/6] fix(backend): liveness fails when a quorum member's Ratis server stays closed The member starts a closed server again itself within seconds; if that keeps failing for kates.quorum.liveness-after (30 s), liveness goes DOWN so the kubelet restarts the pod. Liveness reports quorumState and quorumNotServingSec. --- .../bmscomp/kates/api/KatesLivenessCheck.java | 24 +++++++++++-- .../kates/api/KatesLivenessCheckTest.java | 36 +++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/kates/src/main/java/com/bmscomp/kates/api/KatesLivenessCheck.java b/kates/src/main/java/com/bmscomp/kates/api/KatesLivenessCheck.java index ef157464..9ba0e9f3 100644 --- a/kates/src/main/java/com/bmscomp/kates/api/KatesLivenessCheck.java +++ b/kates/src/main/java/com/bmscomp/kates/api/KatesLivenessCheck.java @@ -2,6 +2,9 @@ import java.lang.management.ManagementFactory; import java.lang.management.ThreadMXBean; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; import jakarta.inject.Inject; import org.eclipse.microprofile.config.inject.ConfigProperty; @@ -19,6 +22,9 @@ * exactly the outage it was meant to detect, in a restart loop that got worse * the more load the pod carried. Genuine exhaustion still surfaces as an * OOMKill, which the kubelet handles on its own. + * + *

The second condition a restart fixes: a quorum member whose Ratis server stays closed + * after the member has tried to start it again for {@code kates.quorum.liveness-after}. */ @Liveness public class KatesLivenessCheck implements HealthCheck { @@ -29,6 +35,12 @@ public class KatesLivenessCheck implements HealthCheck { @ConfigProperty(name = "kates.health.min-free-heap-mb", defaultValue = "64") int minFreeHeapMb; + @Inject + com.bmscomp.kates.quorum.Quorum quorum; + + @ConfigProperty(name = "kates.quorum.liveness-after", defaultValue = "30s") + Duration livenessAfter; + @Override public HealthCheckResponse call() { Runtime rt = Runtime.getRuntime(); @@ -47,6 +59,12 @@ public HealthCheckResponse call() { } catch (Exception ignored) { } + Optional notServingSince = quorum.notServingSince(); + long notServingSec = notServingSince + .map(since -> Duration.between(since, Instant.now()).toSeconds()) + .orElse(0L); + boolean quorumDead = notServingSince.isPresent() && notServingSec >= livenessAfter.toSeconds(); + var builder = HealthCheckResponse.named("kates-liveness") .withData("freeHeapMb", freeHeapMb) .withData("minFreeHeapMb", minFreeHeapMb) @@ -55,9 +73,11 @@ public HealthCheckResponse call() { // on it. .withData("heapLow", heapLow) .withData("deadlockedThreads", deadlocked != null ? deadlocked.length : 0) - .withData("activeTests", activeTests + "/" + maxTests); + .withData("activeTests", activeTests + "/" + maxTests) + .withData("quorumState", notServingSince.isPresent() ? "NOT_SERVING" : "SERVING") + .withData("quorumNotServingSec", notServingSec); - if (noDeadlocks) { + if (noDeadlocks && !quorumDead) { return builder.up().build(); } return builder.down().build(); diff --git a/kates/src/test/java/com/bmscomp/kates/api/KatesLivenessCheckTest.java b/kates/src/test/java/com/bmscomp/kates/api/KatesLivenessCheckTest.java index a911a175..c2aacbfa 100644 --- a/kates/src/test/java/com/bmscomp/kates/api/KatesLivenessCheckTest.java +++ b/kates/src/test/java/com/bmscomp/kates/api/KatesLivenessCheckTest.java @@ -8,7 +8,12 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; + import com.bmscomp.kates.engine.TestOrchestrator; +import com.bmscomp.kates.quorum.Quorum; /** * Liveness decides whether the kubelet KILLS the pod, so it fails only on @@ -19,6 +24,7 @@ class KatesLivenessCheckTest { private KatesLivenessCheck check; private TestOrchestrator orchestrator; + private Quorum quorum; @BeforeEach void setUp() { @@ -27,6 +33,36 @@ void setUp() { check.orchestrator = orchestrator; when(orchestrator.activeTestCount()).thenReturn(2); when(orchestrator.maxConcurrentTests()).thenReturn(3); + quorum = mock(Quorum.class); + check.quorum = quorum; + when(quorum.notServingSince()).thenReturn(Optional.empty()); + check.livenessAfter = Duration.ofSeconds(30); + check.minFreeHeapMb = 1; + } + + @Test + @DisplayName("a quorum member whose Ratis server stays closed is restarted by the kubelet") + void downWhenTheQuorumServerStaysClosed() { + when(quorum.notServingSince()).thenReturn(Optional.of(Instant.now().minusSeconds(31))); + + HealthCheckResponse response = check.call(); + + assertEquals(HealthCheckResponse.Status.DOWN, response.getStatus()); + assertTrue(((Number) response.getData().orElseThrow().get("quorumNotServingSec")).longValue() >= 31); + } + + @Test + @DisplayName("alive while the member is starting its server again") + void upWhileTheServerIsBeingRestarted() { + when(quorum.notServingSince()).thenReturn(Optional.of(Instant.now().minusSeconds(5))); + + assertEquals(HealthCheckResponse.Status.UP, check.call().getStatus()); + } + + @Test + @DisplayName("alive with no quorum") + void upWithNoQuorum() { + assertEquals(HealthCheckResponse.Status.UP, check.call().getStatus()); } @Test From 870089efc021aa77e246d3fabfdd62f31c20e9ff Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 12:33:13 +0100 Subject: [PATCH 3/6] fix(backend): an unreachable database answers 503 within seconds instead of hanging With no JDBC socket timeout, a query on a dead connection waited on TCP for minutes and every write hung with it: a leader cut off from PostgreSQL answered nothing (s07 of the J4 failure study). PostgreSQL's socketTimeout 30 s, connectTimeout 5 s and TCP keepalive (prod, and the ITs), with Agroal's acquisition timeout 5 s and validation every 10 s. A failure whose cause is the database being unreachable (SQL state 08, JDBCConnectionException, SQLTransientConnectionException, Agroal's acquisition timeout) answers 503 with Retry-After 5, from GlobalExceptionMapper and from POST /api/tests. The audit filter writes no row for that answer, whose retries held it back another 30 s; it logs the attempt instead. DatabaseOutageIT pauses PostgreSQL and expects the 503 within 35 s. --- .../kates/api/GlobalExceptionMapper.java | 42 ++++++++++ .../com/bmscomp/kates/api/TestResource.java | 7 ++ .../kates/audit/AuditResponseFilter.java | 16 ++++ .../java/com/bmscomp/kates/quorum/Quorum.java | 4 +- .../bmscomp/kates/quorum/QuorumMember.java | 2 +- .../src/main/resources/application.properties | 7 ++ .../kates/api/GlobalExceptionMapperTest.java | 23 ++++++ .../kates/api/KatesLivenessCheckTest.java | 8 +- .../kates/audit/AuditResponseFilterTest.java | 82 +++++++++++++++++++ .../bmscomp/kates/it/DatabaseOutageIT.java | 64 +++++++++++++++ .../kates/it/IntegrationTestProfile.java | 6 +- .../kates/it/PostgresTestResource.java | 7 ++ 12 files changed, 261 insertions(+), 7 deletions(-) create mode 100644 kates/src/test/java/com/bmscomp/kates/audit/AuditResponseFilterTest.java create mode 100644 kates/src/test/java/com/bmscomp/kates/it/DatabaseOutageIT.java diff --git a/kates/src/main/java/com/bmscomp/kates/api/GlobalExceptionMapper.java b/kates/src/main/java/com/bmscomp/kates/api/GlobalExceptionMapper.java index dc08939d..eb0d2f0e 100644 --- a/kates/src/main/java/com/bmscomp/kates/api/GlobalExceptionMapper.java +++ b/kates/src/main/java/com/bmscomp/kates/api/GlobalExceptionMapper.java @@ -43,6 +43,11 @@ public Response toResponse(Exception exception) { return error(409, "Conflict", "The resource was modified concurrently — re-read it and retry"); } + if (isDatabaseUnavailable(exception)) { + LOG.warnf("Database unavailable: %s", root.toString()); + return databaseUnavailable(); + } + if (root instanceof TimeoutException) { LOG.warn("Request timed out", exception); return error(504, "Gateway Timeout", root.getMessage()); @@ -71,6 +76,43 @@ static boolean hasOptimisticLockCause(Throwable t) { return false; } + /** What a client is told when the database cannot be reached: retry in this many seconds. */ + public static final String DATABASE_RETRY_AFTER = "5"; + + public static final String DATABASE_UNAVAILABLE = "The Kates API cannot reach its database; retry once it is back"; + + /** + * Whether the cause chain says the database could not be reached, as opposed to a statement it + * refused: a connection-class SQL state (08), Hibernate's or JDBC's connection exception, or + * Agroal's acquisition timeout. Depth-capped against cycles. + */ + public static boolean isDatabaseUnavailable(Throwable t) { + for (int depth = 0; t != null && depth < 16; t = t.getCause(), depth++) { + if (t instanceof org.hibernate.exception.JDBCConnectionException + || t instanceof java.sql.SQLTransientConnectionException) { + return true; + } + if (t instanceof java.sql.SQLException sql) { + String state = sql.getSQLState(); + String message = sql.getMessage(); + if ((state != null && state.startsWith("08")) + || (message != null && message.contains("acquisition timeout"))) { + return true; + } + } + } + return false; + } + + /** 503 with Retry-After, for a database the Kates API cannot reach. */ + public static Response databaseUnavailable() { + return Response.status(503) + .type(MediaType.APPLICATION_JSON_TYPE) + .header("Retry-After", DATABASE_RETRY_AFTER) + .entity(ApiError.of(503, "Service Unavailable", DATABASE_UNAVAILABLE)) + .build(); + } + private static Response error(int status, String label, String message) { return Response.status(status) .type(MediaType.APPLICATION_JSON_TYPE) diff --git a/kates/src/main/java/com/bmscomp/kates/api/TestResource.java b/kates/src/main/java/com/bmscomp/kates/api/TestResource.java index 9c44d356..7b88dbbe 100644 --- a/kates/src/main/java/com/bmscomp/kates/api/TestResource.java +++ b/kates/src/main/java/com/bmscomp/kates/api/TestResource.java @@ -180,6 +180,9 @@ private Response start(CreateTestRequest request, String clientKey) { || failure instanceof com.bmscomp.kates.engine.DrainingException) { answer.header("Retry-After", RETRY_AFTER_SECONDS); } + if (GlobalExceptionMapper.isDatabaseUnavailable(failure)) { + answer.header("Retry-After", GlobalExceptionMapper.DATABASE_RETRY_AFTER); + } return answer.build(); } TestRun run = result.asSuccess().orElseThrow(); @@ -226,6 +229,10 @@ private static ApiError notStarted(CreateTestRequest request, Exception failure) if (failure instanceof com.bmscomp.kates.engine.DrainingException) { return ApiError.of(503, "Service Unavailable", failure.getMessage()); } + if (GlobalExceptionMapper.isDatabaseUnavailable(failure)) { + LOG.warnf("A test run did not start: its database is unreachable: %s", failure); + return ApiError.of(503, "Service Unavailable", GlobalExceptionMapper.DATABASE_UNAVAILABLE); + } if (failure instanceof com.bmscomp.kates.engine.InvalidTestSpecException invalid) { return ApiError.validationFailed(invalid.getMessage(), invalid.getFieldErrors()); } diff --git a/kates/src/main/java/com/bmscomp/kates/audit/AuditResponseFilter.java b/kates/src/main/java/com/bmscomp/kates/audit/AuditResponseFilter.java index 236c62c5..5a254ee4 100644 --- a/kates/src/main/java/com/bmscomp/kates/audit/AuditResponseFilter.java +++ b/kates/src/main/java/com/bmscomp/kates/audit/AuditResponseFilter.java @@ -24,6 +24,8 @@ @Provider public class AuditResponseFilter implements ContainerResponseFilter { + private static final org.jboss.logging.Logger LOG = org.jboss.logging.Logger.getLogger(AuditResponseFilter.class); + static final int TARGET_LENGTH = 128; @Context @@ -53,9 +55,23 @@ public void filter(ContainerRequestContext request, ContainerResponseContext res String what = id != null ? id : request.getUriInfo().getPath(); String target = what.length() <= TARGET_LENGTH ? what : what.substring(0, TARGET_LENGTH); String details = "HTTP " + response.getStatus(); + // The row cannot be written to a database the answer says is unreachable: its retries only + // held the 503 back by another 30 s (DatabaseOutageIT). The attempt goes to the log instead. + if (isDatabaseOutage(response)) { + LOG.warnf( + "Audit row not written, the database is unreachable: %s %s %s by %s", + audited.action(), audited.type(), target, actor.name()); + return; + } AuditService.blockingSafe(() -> auditService.record(audited.action(), audited.type(), target, details, actor)); } + private static boolean isDatabaseOutage(ContainerResponseContext response) { + return response.getStatus() == 503 + && response.getEntity() instanceof com.bmscomp.kates.api.ApiError error + && com.bmscomp.kates.api.GlobalExceptionMapper.DATABASE_UNAVAILABLE.equals(error.getMessage()); + } + /** The id an answer carries as an "id" entry, or null. */ static String idOf(Object entity) { return entity instanceof Map map && map.get("id") instanceof String id && !id.isBlank() ? id : null; diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java index 5404a588..cbeb30f8 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java @@ -152,7 +152,9 @@ private void revive() { if (notServingSince == null) { notServingSince = Instant.now(); } - LOG.warnf("quorum member %s: Ratis server %s, starting it again from its disk", current.memberId(), current.state()); + LOG.warnf( + "quorum member %s: Ratis server %s, starting it again from its disk", + current.memberId(), current.state()); try { current.close(); } catch (IOException | RuntimeException e) { diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java index 6f35d8f8..2fdc0c24 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java @@ -30,8 +30,8 @@ import org.apache.ratis.server.RaftServerConfigKeys; import org.apache.ratis.server.storage.RaftStorage; import org.apache.ratis.statemachine.impl.BaseStateMachine; -import org.apache.ratis.util.LifeCycle; import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslProvider; +import org.apache.ratis.util.LifeCycle; import org.apache.ratis.util.TimeDuration; /** diff --git a/kates/src/main/resources/application.properties b/kates/src/main/resources/application.properties index a6fe5747..21e45dda 100644 --- a/kates/src/main/resources/application.properties +++ b/kates/src/main/resources/application.properties @@ -315,6 +315,13 @@ quarkus.datasource.jdbc.min-size=2 quarkus.datasource.jdbc.idle-removal-interval=PT5M quarkus.datasource.jdbc.max-lifetime=PT30M quarkus.datasource.jdbc.telemetry=false +# A query on a dead connection otherwise waits on TCP for minutes, and every write hangs with it +# (s07 of plans/j4-resilient-quorum.md). PostgreSQL-only settings: %prod, so H2 in tests never sees them. +%prod.quarkus.datasource.jdbc.additional-jdbc-properties.socketTimeout=30 +%prod.quarkus.datasource.jdbc.additional-jdbc-properties.connectTimeout=5 +%prod.quarkus.datasource.jdbc.additional-jdbc-properties.tcpKeepAlive=true +quarkus.datasource.jdbc.acquisition-timeout=5S +quarkus.datasource.jdbc.background-validation-interval=10S # Hibernate ORM # Flyway owns the schema; Hibernate must never touch it. diff --git a/kates/src/test/java/com/bmscomp/kates/api/GlobalExceptionMapperTest.java b/kates/src/test/java/com/bmscomp/kates/api/GlobalExceptionMapperTest.java index 04f9c35c..e8dfa849 100644 --- a/kates/src/test/java/com/bmscomp/kates/api/GlobalExceptionMapperTest.java +++ b/kates/src/test/java/com/bmscomp/kates/api/GlobalExceptionMapperTest.java @@ -18,6 +18,29 @@ void setUp() { mapper = new GlobalExceptionMapper(); } + @Test + void aConnectionFailureIsAnOutage() { + Response response = mapper.toResponse(new RuntimeException(new java.sql.SQLException("x", "08006"))); + assertEquals(503, response.getStatus()); + assertEquals("5", response.getHeaderString("Retry-After")); + assertEquals( + "The Kates API cannot reach its database; retry once it is back", + ((ApiError) response.getEntity()).getMessage()); + } + + @Test + void anAcquisitionTimeoutIsAnOutage() { + Response response = + mapper.toResponse(new RuntimeException(new java.sql.SQLException("Sorry, acquisition timeout!"))); + assertEquals(503, response.getStatus()); + } + + @Test + void aConstraintViolationIsNotAnOutage() { + Response response = mapper.toResponse(new RuntimeException(new java.sql.SQLException("dup", "23505"))); + assertEquals(500, response.getStatus()); + } + @Test void illegalArgumentReturnsBadRequest() { Response response = mapper.toResponse(new IllegalArgumentException("bad param")); diff --git a/kates/src/test/java/com/bmscomp/kates/api/KatesLivenessCheckTest.java b/kates/src/test/java/com/bmscomp/kates/api/KatesLivenessCheckTest.java index c2aacbfa..332d963d 100644 --- a/kates/src/test/java/com/bmscomp/kates/api/KatesLivenessCheckTest.java +++ b/kates/src/test/java/com/bmscomp/kates/api/KatesLivenessCheckTest.java @@ -3,15 +3,15 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; + import org.eclipse.microprofile.health.HealthCheckResponse; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; - import com.bmscomp.kates.engine.TestOrchestrator; import com.bmscomp.kates.quorum.Quorum; diff --git a/kates/src/test/java/com/bmscomp/kates/audit/AuditResponseFilterTest.java b/kates/src/test/java/com/bmscomp/kates/audit/AuditResponseFilterTest.java new file mode 100644 index 00000000..b0857d85 --- /dev/null +++ b/kates/src/test/java/com/bmscomp/kates/audit/AuditResponseFilterTest.java @@ -0,0 +1,82 @@ +package com.bmscomp.kates.audit; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerResponseContext; +import jakarta.ws.rs.container.ResourceInfo; +import jakarta.ws.rs.core.UriInfo; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; + +import com.bmscomp.kates.api.ApiError; +import com.bmscomp.kates.api.GlobalExceptionMapper; +import com.bmscomp.kates.service.AuditService; + +class AuditResponseFilterTest { + + static class Resource { + @Audited(action = "CREATE", type = "test") + public void create() {} + } + + private final AuditResponseFilter filter = new AuditResponseFilter(); + private final ContainerRequestContext request = mock(ContainerRequestContext.class); + private final ContainerResponseContext response = mock(ContainerResponseContext.class); + private MockedStatic blocking; + + @AfterEach + void tearDown() { + blocking.close(); + } + + @BeforeEach + void setUp() throws NoSuchMethodException { + // Outside Quarkus there is no worker pool: the write runs on the calling thread. + blocking = mockStatic(AuditService.class); + blocking.when(() -> AuditService.blockingSafe(any())).thenAnswer(invocation -> { + invocation.getArgument(0).run(); + return null; + }); + filter.resourceInfo = mock(ResourceInfo.class); + when(filter.resourceInfo.getResourceMethod()).thenReturn(Resource.class.getMethod("create")); + filter.trail = mock(AuditTrail.class); + filter.actors = mock(Actors.class); + when(filter.actors.current()).thenReturn(Actor.SCHEDULER); + filter.auditService = mock(AuditService.class); + UriInfo uri = mock(UriInfo.class); + when(uri.getPath()).thenReturn("/api/tests"); + when(request.getUriInfo()).thenReturn(uri); + } + + @Test + void aDatabaseOutageAnswerIsNotHeldByItsAuditRow() { + when(response.getStatus()).thenReturn(503); + when(response.getEntity()) + .thenReturn(ApiError.of(503, "Service Unavailable", GlobalExceptionMapper.DATABASE_UNAVAILABLE)); + + filter.filter(request, response); + + verify(filter.auditService, never()).record(anyString(), anyString(), anyString(), anyString(), any()); + } + + @Test + void anotherRefusalIsStillAudited() { + when(response.getStatus()).thenReturn(503); + when(response.getEntity()).thenReturn(ApiError.of(503, "Service Unavailable", "draining")); + + filter.filter(request, response); + + verify(filter.auditService, times(1)).record(anyString(), anyString(), anyString(), anyString(), any()); + } +} diff --git a/kates/src/test/java/com/bmscomp/kates/it/DatabaseOutageIT.java b/kates/src/test/java/com/bmscomp/kates/it/DatabaseOutageIT.java new file mode 100644 index 00000000..aa3ccfcb --- /dev/null +++ b/kates/src/test/java/com/bmscomp/kates/it/DatabaseOutageIT.java @@ -0,0 +1,64 @@ +package com.bmscomp.kates.it; + +import static io.restassured.RestAssured.given; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; + +import io.quarkus.test.InjectMock; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; +import io.restassured.http.ContentType; +import io.restassured.response.Response; +import org.junit.jupiter.api.Test; + +import com.bmscomp.kates.domain.TestType; +import com.bmscomp.kates.service.TopicService; + +/** + * PostgreSQL unreachable while the Kates API runs: a write answers 503 with Retry-After within the + * JDBC socket timeout, instead of hanging on TCP (s07 of plans/j4-resilient-quorum.md). + */ +@QuarkusTest +@TestProfile(NoSchedulersTestProfile.class) +@QuarkusTestResource(value = PostgresTestResource.class, restrictToAnnotatedClass = true) +class DatabaseOutageIT { + + @InjectMock + TopicService topicService; + + @Test + void aCreateWithTheDatabasePausedAnswers503Fast() { + // A first request, so the pool holds connections that the pause then strands. + given().when().get("/api/tests").then().statusCode(200); + String id = PostgresTestResource.container().getContainerId(); + var docker = PostgresTestResource.container().getDockerClient(); + docker.pauseContainerCmd(id).exec(); + try { + long start = System.nanoTime(); + // Without the JDBC timeouts the request hangs on TCP for minutes: give up at 60 s. + Response response = given().config(io.restassured.RestAssured.config() + .httpClient(io.restassured.config.HttpClientConfig.httpClientConfig() + .setParam("http.socket.timeout", 60_000))) + .contentType(ContentType.JSON) + .body(ItSupport.createRequest(TestType.LOAD, ItSupport.singleBrokerSpec(null))) + .when() + .post("/api/tests"); + Duration took = Duration.ofNanos(System.nanoTime() - start); + + assertEquals(503, response.statusCode(), response.asString()); + assertNotNull(response.header("Retry-After")); + assertTrue(took.compareTo(Duration.ofSeconds(35)) < 0, "answered after " + took); + } finally { + docker.unpauseContainerCmd(id).exec(); + } + assertTrue( + ItSupport.waitUntil( + Duration.ofSeconds(20), + () -> given().when().get("/api/tests").statusCode() == 200), + "the Kates API answers again once its database is back"); + } +} diff --git a/kates/src/test/java/com/bmscomp/kates/it/IntegrationTestProfile.java b/kates/src/test/java/com/bmscomp/kates/it/IntegrationTestProfile.java index eb35bbb6..935d9522 100644 --- a/kates/src/test/java/com/bmscomp/kates/it/IntegrationTestProfile.java +++ b/kates/src/test/java/com/bmscomp/kates/it/IntegrationTestProfile.java @@ -19,6 +19,10 @@ public Map getConfigOverrides() { return Map.of( "quarkus.datasource.db-kind", "postgresql", "quarkus.hibernate-orm.schema-management.strategy", "none", - "quarkus.flyway.migrate-at-start", "true"); + "quarkus.flyway.migrate-at-start", "true", + // What %prod sets in application.properties: the ITs run against PostgreSQL as shipped. + "quarkus.datasource.jdbc.additional-jdbc-properties.socketTimeout", "30", + "quarkus.datasource.jdbc.additional-jdbc-properties.connectTimeout", "5", + "quarkus.datasource.jdbc.additional-jdbc-properties.tcpKeepAlive", "true"); } } diff --git a/kates/src/test/java/com/bmscomp/kates/it/PostgresTestResource.java b/kates/src/test/java/com/bmscomp/kates/it/PostgresTestResource.java index 3feae07b..e9a4c1c4 100644 --- a/kates/src/test/java/com/bmscomp/kates/it/PostgresTestResource.java +++ b/kates/src/test/java/com/bmscomp/kates/it/PostgresTestResource.java @@ -9,6 +9,12 @@ public class PostgresTestResource implements QuarkusTestResourceLifecycleManager { private PostgreSQLContainer postgres; + private static volatile PostgreSQLContainer current; + + /** The container of the class running now, for a test that pauses it. */ + static PostgreSQLContainer container() { + return current; + } @Override public Map start() { @@ -17,6 +23,7 @@ public Map start() { .withUsername("kates") .withPassword("kates"); postgres.start(); + current = postgres; return Map.of( "quarkus.datasource.jdbc.url", postgres.getJdbcUrl(), "quarkus.datasource.username", postgres.getUsername(), From a135afb8b5681b8bddf955ba348ad0f50c22ae57 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 12:42:45 +0100 Subject: [PATCH 4/6] docs(api): health names a quorum member's server state, and a database outage answers 503 The /api/health quorum object's state, what a member reports while its Ratis server is down, the liveness backstop, and the 503 with Retry-After for an unreachable database. Three CLI caveat refs re-pointed to the lines the backend edits moved. --- cli/cmd/mcp_caveats.go | 2 +- cli/cmd/mcp_tools_runs_activity.go | 2 +- cli/cmd/mcp_tools_runs_list.go | 2 +- docs/book/11-api-reference.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/cmd/mcp_caveats.go b/cli/cmd/mcp_caveats.go index 50c4377e..514d8823 100644 --- a/cli/cmd/mcp_caveats.go +++ b/cli/cmd/mcp_caveats.go @@ -217,7 +217,7 @@ var mcpCaveatsCore = []mcpCaveat{ "measure out of its grade and lists it, with the reason, under unevaluated (with nothing measured the grade " + "is \"-\"), so read a grade that has unevaluated checks as partial.", Refs: []string{ - mcpAnchoredRef("kates/src/main/resources/application.properties:329-334", + mcpAnchoredRef("kates/src/main/resources/application.properties:336-341", "# Prometheus server (for disruption metrics capture)", "kates.prometheus.url=http://monitoring-kube-prometheus-prometheus.monitoring.svc:9090"), mcpAnchoredRef("charts/kates/values.yaml:449-459", diff --git a/cli/cmd/mcp_tools_runs_activity.go b/cli/cmd/mcp_tools_runs_activity.go index 46d0b62a..8469b9b1 100644 --- a/cli/cmd/mcp_tools_runs_activity.go +++ b/cli/cmd/mcp_tools_runs_activity.go @@ -458,7 +458,7 @@ var mcpCaveatsRunsActivity = []mcpCaveat{ mcpAnchoredRef(mcpJava+"service/RunRetention.java:82", `auditService.record("DELETE", "test", id, auditDetails, actor);`), mcpAnchoredRef(mcpJava+"service/TestCleanupScheduler.java:97", "Actor.SCHEDULER"), - mcpAnchoredRef(mcpJava+"audit/AuditResponseFilter.java:42-62", + mcpAnchoredRef(mcpJava+"audit/AuditResponseFilter.java:44-78", "public void filter(ContainerRequestContext request", "if (actor == null)", "static String idOf("), mcpAnchoredRef(mcpJava+"security/GrpcApiKeyInterceptor.java:67-70,100,156-159", "static final Map AUDITED", "audit(action, method, Status.Code.PERMISSION_DENIED", diff --git a/cli/cmd/mcp_tools_runs_list.go b/cli/cmd/mcp_tools_runs_list.go index 2b57a298..a67742e2 100644 --- a/cli/cmd/mcp_tools_runs_list.go +++ b/cli/cmd/mcp_tools_runs_list.go @@ -1073,7 +1073,7 @@ var mcpCaveatsRunsList = []mcpCaveat{ Refs: []string{ mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:2182-2264", `"Cancelled by user"`, `EventKind.FAILED, "cancelled"`, "return run.withResults(updatedResults);"), - mcpAnchoredRef(mcpJava+"api/TestResource.java:565-610", + mcpAnchoredRef(mcpJava+"api/TestResource.java:572-617", `@Path("/{id}/cancel")`, `auditService.record("CANCEL"`, `"Test cancelled; it is stored as FAILED"`), mcpJava + "domain/TestResult.java:25-31", }, diff --git a/docs/book/11-api-reference.md b/docs/book/11-api-reference.md index 42d19f66..6ab0a37b 100644 --- a/docs/book/11-api-reference.md +++ b/docs/book/11-api-reference.md @@ -237,7 +237,7 @@ curl -s "$BASE/api/health" The response also contains a `tests` object with the resolved default configuration for every test type. When Kafka is unreachable, the endpoint still returns `200 OK`, but with `"status": "DEGRADED"` and `"kafka.status": "DOWN"`. -`quorum` says whether this Kates is a member of a quorum, which it is only when `kates.quorum.peers` is set. Without one it is `{"enabled": false}`, and the instance acts alone. A member also reports its `memberId`, the `leaderId` it knows of (`null` during an election), the Raft `term`, `appliedIndex`, the last log entry it applied, and `leader`: true once it leads and has committed an entry of its term. A member whose log is empty, new or with its directory lost, is no voter until it has caught up to an entry: three members of which one is empty and one is down elect nobody. Members talk to each other over mutual TLS on their own port: `kates.quorum.tls.cert`, `kates.quorum.tls.key` (PKCS#8 PEM) and `kates.quorum.tls.ca` are required, and so is `kates.quorum.dir`, which keeps a member's term and vote and must outlive a restart. Only the leader takes writes: on any other member, every request but a read is answered `503` with `Retry-After`, and its message names the leader. `PUT /api/drain` is the exception, since a drain is set on each member. Before its first write in a term, a new leader marks `FAILED` the live runs every other member created, and removes the faults Kates made before it took the lead; a member that starts up recovers only its own runs. A leader that loses its majority stops taking writes once its lease runs out, within about a second, even before it steps down. +`quorum` says whether this Kates is a member of a quorum, which it is only when `kates.quorum.peers` is set. Without one it is `{"enabled": false}`, and the instance acts alone. A member also reports `state`, its Raft server's life cycle (`RUNNING` while it serves). Ratis closes a member's server after a pause longer than 60 s, such as a long garbage collection or a frozen node; the member then starts it again from its directory within a few seconds, and until it serves again only `state` and `memberId` are reported. If it still does not serve after `kates.quorum.liveness-after` (30 s), `/q/health/live` answers DOWN so the kubelet restarts the pod. A member reports its `memberId` and, while it serves, the `leaderId` it knows of (`null` during an election), the Raft `term`, `appliedIndex`, the last log entry it applied, and `leader`: true once it leads and has committed an entry of its term. A member whose log is empty, new or with its directory lost, is no voter until it has caught up to an entry: three members of which one is empty and one is down elect nobody. Members talk to each other over mutual TLS on their own port: `kates.quorum.tls.cert`, `kates.quorum.tls.key` (PKCS#8 PEM) and `kates.quorum.tls.ca` are required, and so is `kates.quorum.dir`, which keeps a member's term and vote and must outlive a restart. Only the leader takes writes: on any other member, every request but a read is answered `503` with `Retry-After`, and its message names the leader. `PUT /api/drain` is the exception, since a drain is set on each member. Before its first write in a term, a new leader marks `FAILED` the live runs every other member created, and removes the faults Kates made before it took the lead; a member that starts up recovers only its own runs. A leader that loses its majority stops taking writes once its lease runs out, within about a second, even before it steps down. A request that needs a database Kates cannot reach is answered `503` with `Retry-After: 5` within about 30 s, the JDBC socket timeout, instead of waiting on the connection. #### GET /api/drain From b2256a2390a193420cddf686fb6e864845d580ea Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 13:04:27 +0100 Subject: [PATCH 5/6] fix(backend,chart): a failed restart leaves no locked server, and CNPG outages answer 503 in 30 s From the branch review: - QuorumMember.start closes the server when its start fails. Ratis starts the division, which locks the storage directory, before it binds, so a failed start left a running division and every later start found the directory locked. - SQL states 57P01 to 57P03 (the server shutting down or starting, what a CloudNativePG switchover leaves) count as the database being unreachable. - The chart's CNPG JDBC URL sets socketTimeout=30, the backend's own: a URL parameter overrides it, and 60 doubled the 503 bound on the HA path. Chart 0.19.5; ci.yml asserts it. - The MCP audit caveat says a database-outage 503 leaves a log line, not a row. --- .github/workflows/ci.yml | 2 ++ charts/kates/Chart.yaml | 7 ++++--- charts/kates/templates/_helpers.tpl | 5 +++-- cli/cmd/mcp_tools_runs_activity.go | 6 ++++-- docs/book/appendix-d-versions.md | 2 +- .../kates/api/GlobalExceptionMapper.java | 6 ++++-- .../bmscomp/kates/quorum/QuorumMember.java | 9 ++++++++- .../kates/api/GlobalExceptionMapperTest.java | 9 +++++++++ .../com/bmscomp/kates/quorum/QuorumTest.java | 20 +++++++++++++++++++ 9 files changed, 55 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfeae63e..97e2ff50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -234,6 +234,8 @@ jobs: [ "$(printf '%s\n' "$cluster" | yq .spec.bootstrap.initdb.database)" = kates ] if printf '%s\n' "$out" | grep -q '^kind: StatefulSet'; then exit 1; fi printf '%s\n' "$out" | grep -q 'jdbc:postgresql://kates-db-rw\.' + # The backend's own socket timeout, which the URL overrides: a 503 within about 30 s, not 60. + printf '%s\n' "$out" | grep -q 'jdbc:postgresql://kates-db-rw\..*socketTimeout=30&' printf '%s\n' "$out" | grep -A3 'name: QUARKUS_DATASOURCE_PASSWORD' | grep -q 'name: kates-db-app' printf '%s\n' "$out" | grep -q 'until nc -z kates-db-rw\.' printf '%s\n' "$out" | grep -q 'cnpg.io/cluster: kates-db' diff --git a/charts/kates/Chart.yaml b/charts/kates/Chart.yaml index bcce7241..7bc62855 100644 --- a/charts/kates/Chart.yaml +++ b/charts/kates/Chart.yaml @@ -2,9 +2,8 @@ apiVersion: v2 name: kates description: Kates — Kafka Advanced Testing & Engineering Suite type: application -# Patch: appVersion and the image tag move to 1.40.0, the tag the v1.40.0 -# release publishes. -version: 0.19.4 +# Patch: with cnpg.enabled, the JDBC URL's socket timeout is 30 s, as the backend's own. +version: 0.19.5 # The non-blocking health endpoints this chart's probes now point at come from # a BUILD-TIME property (quarkus.messaging.health.enabled=false), so they only # exist in the image built from this branch. Keep appVersion and @@ -28,6 +27,8 @@ annotations: artifacthub.io/category: integration-delivery artifacthub.io/license: Apache-2.0 artifacthub.io/changes: | + - kind: changed + description: "With cnpg.enabled, the JDBC URL's socketTimeout is 30 s instead of 60 s, the backend's own: a request against an unreachable database is answered 503 within about 30 s. Templates are otherwise unchanged." - kind: changed description: "appVersion 1.40.0 and the image tag move to 1.40.0: chaos runs that never outlive or escape their run (#632, #637-#640), numbers that are what was measured (#642-#645), a Kubernetes client that verifies the API server (#641) Trogdor tasks that reach the configured Kafka (#635) and pollers that stop at shutdown instead of logging errors (#646). Throughput summaries and scenario phases change behaviour; see the release notes." - kind: changed diff --git a/charts/kates/templates/_helpers.tpl b/charts/kates/templates/_helpers.tpl index 66571177..fb69ab94 100644 --- a/charts/kates/templates/_helpers.tpl +++ b/charts/kates/templates/_helpers.tpl @@ -65,8 +65,9 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- if .Values.postgresql.enabled -}} jdbc:postgresql://{{ include "kates.postgresql.fullname" . }}.{{ .Release.Namespace }}.svc:5432/{{ .Values.postgresql.auth.database }} {{- else if .Values.cnpg.enabled -}} -{{- /* Timeouts: a connection to a dead node otherwise hangs on TCP for minutes after a failover. */ -}} -jdbc:postgresql://{{ include "kates.dbHost" . }}:5432/{{ .Values.externalDatabase.database }}?socketTimeout=60&tcpKeepAlive=true&connectTimeout=5 +{{- /* Timeouts: a connection to a dead node otherwise hangs on TCP for minutes after a failover. + socketTimeout matches the backend's own (application.properties), which a URL parameter overrides. */ -}} +jdbc:postgresql://{{ include "kates.dbHost" . }}:5432/{{ .Values.externalDatabase.database }}?socketTimeout=30&tcpKeepAlive=true&connectTimeout=5 {{- else -}} jdbc:postgresql://{{ .Values.externalDatabase.host }}:{{ .Values.externalDatabase.port | default 5432 }}/{{ .Values.externalDatabase.database }} {{- end -}} diff --git a/cli/cmd/mcp_tools_runs_activity.go b/cli/cmd/mcp_tools_runs_activity.go index 8469b9b1..40addb78 100644 --- a/cli/cmd/mcp_tools_runs_activity.go +++ b/cli/cmd/mcp_tools_runs_activity.go @@ -451,7 +451,8 @@ var mcpCaveatsRunsActivity = []mcpCaveat{ "acted. Every REST and gRPC call that changes something leaves a row, a refused or failed one too, " + "with how it ended in details, and its target is the id of the run or disruption it started when it " + "started one. The Kates API's daily retention sweep leaves one as system:scheduler for each finished " + - "run it deletes. A call without a valid key, and a dry run, leave none. Rows written before the Kates " + + "run it deletes. A call without a valid key, and a dry run, leave none, and so does a REST call " + + "answered 503 because the Kates API could not reach its database, which it logs instead. Rows written before the Kates " + "API recorded actors name no one, and then only the REST test endpoints wrote rows at all. Runs and " + "disruption reports record no owner of their own: who started one shows only in its audit row.", Refs: []string{ @@ -459,7 +460,8 @@ var mcpCaveatsRunsActivity = []mcpCaveat{ `auditService.record("DELETE", "test", id, auditDetails, actor);`), mcpAnchoredRef(mcpJava+"service/TestCleanupScheduler.java:97", "Actor.SCHEDULER"), mcpAnchoredRef(mcpJava+"audit/AuditResponseFilter.java:44-78", - "public void filter(ContainerRequestContext request", "if (actor == null)", "static String idOf("), + "public void filter(ContainerRequestContext request", "if (actor == null)", "if (isDatabaseOutage(response))", + "static String idOf("), mcpAnchoredRef(mcpJava+"security/GrpcApiKeyInterceptor.java:67-70,100,156-159", "static final Map AUDITED", "audit(action, method, Status.Code.PERMISSION_DENIED", "private void audit("), diff --git a/docs/book/appendix-d-versions.md b/docs/book/appendix-d-versions.md index c743cd16..39eab5dd 100644 --- a/docs/book/appendix-d-versions.md +++ b/docs/book/appendix-d-versions.md @@ -23,7 +23,7 @@ Every version the platform pins, in one place. This table is generated from the | `kafka-ui` chart | 0.3.4 (app v1.5.0) | `charts/kafka-ui/Chart.yaml` | | `kates-chaos` chart | 2.6.0 (app 3.28.0) | `charts/kates-chaos/Chart.yaml` | | `kates-platform` chart | 0.9.4 (app 1.0.0) | `charts/kates-platform/Chart.yaml` | -| `kates` chart | 0.19.4 (app 1.40.0) | `charts/kates/Chart.yaml` | +| `kates` chart | 0.19.5 (app 1.40.0) | `charts/kates/Chart.yaml` | | `legacy-kafka` chart | 0.2.2 (app 3.9.1) | `charts/legacy-kafka/Chart.yaml` | | `mirror-maker2` chart | 0.12.3 (app 4.3.1) | `charts/mirror-maker2/Chart.yaml` | | `monitoring` chart | 1.9.0 (app 82.4.3) | `charts/monitoring/Chart.yaml` | diff --git a/kates/src/main/java/com/bmscomp/kates/api/GlobalExceptionMapper.java b/kates/src/main/java/com/bmscomp/kates/api/GlobalExceptionMapper.java index eb0d2f0e..340681fe 100644 --- a/kates/src/main/java/com/bmscomp/kates/api/GlobalExceptionMapper.java +++ b/kates/src/main/java/com/bmscomp/kates/api/GlobalExceptionMapper.java @@ -83,7 +83,7 @@ static boolean hasOptimisticLockCause(Throwable t) { /** * Whether the cause chain says the database could not be reached, as opposed to a statement it - * refused: a connection-class SQL state (08), Hibernate's or JDBC's connection exception, or + * refused: a connection-class SQL state (08) or a server shutting down or starting (57P01 to 57P03), Hibernate's or JDBC's connection exception, or * Agroal's acquisition timeout. Depth-capped against cycles. */ public static boolean isDatabaseUnavailable(Throwable t) { @@ -95,7 +95,9 @@ public static boolean isDatabaseUnavailable(Throwable t) { if (t instanceof java.sql.SQLException sql) { String state = sql.getSQLState(); String message = sql.getMessage(); - if ((state != null && state.startsWith("08")) + // 08: no connection. 57P01-57P03: the server shutting down or starting, as a + // CloudNativePG switchover leaves on the connections it kills. + if ((state != null && (state.startsWith("08") || state.matches("57P0[123]"))) || (message != null && message.contains("acquisition timeout"))) { return true; } diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java index 2fdc0c24..2b3faeeb 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java @@ -175,7 +175,14 @@ public static QuorumMember start(Settings settings) throws IOException { .setStateMachine(new Confirmations()) .setOption(RaftStorage.StartupOption.RECOVER) .build(); - server.start(); + try { + server.start(); + } catch (IOException | RuntimeException e) { + // Ratis starts the division, which locks the storage directory, before it binds: a start + // that fails must close what it started, or every later start finds the directory locked. + server.close(); + throw e; + } RaftProperties clientProperties = new RaftProperties(); RaftClientConfigKeys.Rpc.setRequestTimeout(clientProperties, CONFIRM_TIMEOUT); RaftClient client = RaftClient.newBuilder() diff --git a/kates/src/test/java/com/bmscomp/kates/api/GlobalExceptionMapperTest.java b/kates/src/test/java/com/bmscomp/kates/api/GlobalExceptionMapperTest.java index e8dfa849..a1555f4a 100644 --- a/kates/src/test/java/com/bmscomp/kates/api/GlobalExceptionMapperTest.java +++ b/kates/src/test/java/com/bmscomp/kates/api/GlobalExceptionMapperTest.java @@ -35,6 +35,15 @@ void anAcquisitionTimeoutIsAnOutage() { assertEquals(503, response.getStatus()); } + @Test + void aDatabaseShuttingDownOrStartingIsAnOutage() { + // What a CloudNativePG switchover leaves on the connections it kills. + for (String state : new String[] {"57P01", "57P02", "57P03"}) { + Response response = mapper.toResponse(new RuntimeException(new java.sql.SQLException("x", state))); + assertEquals(503, response.getStatus(), state); + } + } + @Test void aConstraintViolationIsNotAnOutage() { Response response = mapper.toResponse(new RuntimeException(new java.sql.SQLException("dup", "23505"))); diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java index 4e7c1de4..9fc4a64b 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java @@ -106,6 +106,26 @@ void healthWhileClosedNamesTheState() throws Exception { } } + @Test + void aRestartThatFailedSucceedsOnceThePortIsFree() throws Exception { + start(1, new AtomicBoolean(false)); + Quorum quorum = members.get(0); + soleWriter(members); + int port = Integer.parseInt(quorum.peers.orElseThrow().get(0).replaceAll(".*:", "")); + + quorum.member().closeServer(); + try (java.net.ServerSocket held = new java.net.ServerSocket(port)) { + quorum.takeOver(); + assertTrue(quorum.notServingSince().isPresent(), "the port is held: " + quorum.health()); + } + // A failed start must leave nothing behind, such as the storage directory's lock. + quorum.takeOver(); + + assertEquals("RUNNING", quorum.health().get("state"), quorum.health().toString()); + assertTrue(quorum.notServingSince().isEmpty()); + soleWriter(members); + } + @Test void noRestartAfterShutdown() throws Exception { start(1, new AtomicBoolean(false)); From 188cafe435ef2b5c90ccafc131b74f45fbd9fc86 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 13:11:12 +0100 Subject: [PATCH 6/6] chore(chart): kates-platform 0.9.5 packs kates 0.19.5 The chart-versions gate: an umbrella that packs a chart whose version moved carries a new version too. --- charts/kates-platform/Chart.yaml | 7 ++++--- docs/book/appendix-d-versions.md | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/charts/kates-platform/Chart.yaml b/charts/kates-platform/Chart.yaml index f3917d31..a2f517d2 100644 --- a/charts/kates-platform/Chart.yaml +++ b/charts/kates-platform/Chart.yaml @@ -3,9 +3,8 @@ name: kates-platform description: Umbrella chart for the full Kates platform — Kafka, Kates, and supporting infrastructure icon: https://raw.githubusercontent.com/bmscomp/kates/main/docs/assets/charts/kates-platform.svg type: application -# Patch: the umbrella packs the subcharts the v1.40.0 release pins: kates -# 0.19.4, kafka-cluster 1.1.3, connect-cluster 2.2.3, mirror-maker2 0.12.3. -version: 0.9.4 +# Patch: the umbrella packs kates 0.19.5 (the CNPG JDBC URL's 30 s socket timeout). +version: 0.9.5 appVersion: "1.0.0" keywords: - kafka @@ -42,6 +41,8 @@ dependencies: condition: connect-cluster.enabled annotations: artifacthub.io/changes: | + - kind: changed + description: "Packs kates 0.19.5: with cnpg.enabled, the JDBC URL's socketTimeout is 30 s, the backend's own. Templates are otherwise unchanged." - kind: changed description: "The umbrella moves with the v1.40.0 release: it packs kates 0.19.4, kafka-cluster 1.1.3, connect-cluster 2.2.3 and mirror-maker2 0.12.3 as they are at that tag." - kind: changed diff --git a/docs/book/appendix-d-versions.md b/docs/book/appendix-d-versions.md index 39eab5dd..140842e4 100644 --- a/docs/book/appendix-d-versions.md +++ b/docs/book/appendix-d-versions.md @@ -22,7 +22,7 @@ Every version the platform pins, in one place. This table is generated from the | `kafka-common` chart | 0.1.3 (library) | `charts/kafka-common/Chart.yaml` | | `kafka-ui` chart | 0.3.4 (app v1.5.0) | `charts/kafka-ui/Chart.yaml` | | `kates-chaos` chart | 2.6.0 (app 3.28.0) | `charts/kates-chaos/Chart.yaml` | -| `kates-platform` chart | 0.9.4 (app 1.0.0) | `charts/kates-platform/Chart.yaml` | +| `kates-platform` chart | 0.9.5 (app 1.0.0) | `charts/kates-platform/Chart.yaml` | | `kates` chart | 0.19.5 (app 1.40.0) | `charts/kates/Chart.yaml` | | `legacy-kafka` chart | 0.2.2 (app 3.9.1) | `charts/legacy-kafka/Chart.yaml` | | `mirror-maker2` chart | 0.12.3 (app 4.3.1) | `charts/mirror-maker2/Chart.yaml` |