Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,8 @@ jobs:
[ "$(printf '%s\n' "$cluster" | yq .spec.bootstrap.initdb.database)" = kates ]
if printf '%s\n' "$out" | grep -q '^kind: StatefulSet'; then exit 1; fi
printf '%s\n' "$out" | grep -q 'jdbc:postgresql://kates-db-rw\.'
# The backend's own socket timeout, which the URL overrides: a 503 within about 30 s, not 60.
printf '%s\n' "$out" | grep -q 'jdbc:postgresql://kates-db-rw\..*socketTimeout=30&'
printf '%s\n' "$out" | grep -A3 'name: QUARKUS_DATASOURCE_PASSWORD' | grep -q 'name: kates-db-app'
printf '%s\n' "$out" | grep -q 'until nc -z kates-db-rw\.'
printf '%s\n' "$out" | grep -q 'cnpg.io/cluster: kates-db'
Expand Down
7 changes: 4 additions & 3 deletions charts/kates-platform/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@ name: kates-platform
description: Umbrella chart for the full Kates platform — Kafka, Kates, and supporting infrastructure
icon: https://raw.githubusercontent.com/bmscomp/kates/main/docs/assets/charts/kates-platform.svg
type: application
# Patch: the umbrella packs the subcharts the v1.40.0 release pins: kates
# 0.19.4, kafka-cluster 1.1.3, connect-cluster 2.2.3, mirror-maker2 0.12.3.
version: 0.9.4
# Patch: the umbrella packs kates 0.19.5 (the CNPG JDBC URL's 30 s socket timeout).
version: 0.9.5
appVersion: "1.0.0"
keywords:
- kafka
Expand Down Expand Up @@ -42,6 +41,8 @@ dependencies:
condition: connect-cluster.enabled
annotations:
artifacthub.io/changes: |
- kind: changed
description: "Packs kates 0.19.5: with cnpg.enabled, the JDBC URL's socketTimeout is 30 s, the backend's own. Templates are otherwise unchanged."
- kind: changed
description: "The umbrella moves with the v1.40.0 release: it packs kates 0.19.4, kafka-cluster 1.1.3, connect-cluster 2.2.3 and mirror-maker2 0.12.3 as they are at that tag."
- kind: changed
Expand Down
7 changes: 4 additions & 3 deletions charts/kates/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@ apiVersion: v2
name: kates
description: Kates — Kafka Advanced Testing & Engineering Suite
type: application
# Patch: appVersion and the image tag move to 1.40.0, the tag the v1.40.0
# release publishes.
version: 0.19.4
# Patch: with cnpg.enabled, the JDBC URL's socket timeout is 30 s, as the backend's own.
version: 0.19.5
# The non-blocking health endpoints this chart's probes now point at come from
# a BUILD-TIME property (quarkus.messaging.health.enabled=false), so they only
# exist in the image built from this branch. Keep appVersion and
Expand All @@ -28,6 +27,8 @@ annotations:
artifacthub.io/category: integration-delivery
artifacthub.io/license: Apache-2.0
artifacthub.io/changes: |
- kind: changed
description: "With cnpg.enabled, the JDBC URL's socketTimeout is 30 s instead of 60 s, the backend's own: a request against an unreachable database is answered 503 within about 30 s. Templates are otherwise unchanged."
- kind: changed
description: "appVersion 1.40.0 and the image tag move to 1.40.0: chaos runs that never outlive or escape their run (#632, #637-#640), numbers that are what was measured (#642-#645), a Kubernetes client that verifies the API server (#641) Trogdor tasks that reach the configured Kafka (#635) and pollers that stop at shutdown instead of logging errors (#646). Throughput summaries and scenario phases change behaviour; see the release notes."
- kind: changed
Expand Down
5 changes: 3 additions & 2 deletions charts/kates/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,9 @@ app.kubernetes.io/instance: {{ .Release.Name }}
{{- if .Values.postgresql.enabled -}}
jdbc:postgresql://{{ include "kates.postgresql.fullname" . }}.{{ .Release.Namespace }}.svc:5432/{{ .Values.postgresql.auth.database }}
{{- else if .Values.cnpg.enabled -}}
{{- /* Timeouts: a connection to a dead node otherwise hangs on TCP for minutes after a failover. */ -}}
jdbc:postgresql://{{ include "kates.dbHost" . }}:5432/{{ .Values.externalDatabase.database }}?socketTimeout=60&tcpKeepAlive=true&connectTimeout=5
{{- /* Timeouts: a connection to a dead node otherwise hangs on TCP for minutes after a failover.
socketTimeout matches the backend's own (application.properties), which a URL parameter overrides. */ -}}
jdbc:postgresql://{{ include "kates.dbHost" . }}:5432/{{ .Values.externalDatabase.database }}?socketTimeout=30&tcpKeepAlive=true&connectTimeout=5
{{- else -}}
jdbc:postgresql://{{ .Values.externalDatabase.host }}:{{ .Values.externalDatabase.port | default 5432 }}/{{ .Values.externalDatabase.database }}
{{- end -}}
Expand Down
2 changes: 1 addition & 1 deletion cli/cmd/mcp_caveats.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ var mcpCaveatsCore = []mcpCaveat{
"measure out of its grade and lists it, with the reason, under unevaluated (with nothing measured the grade " +
"is \"-\"), so read a grade that has unevaluated checks as partial.",
Refs: []string{
mcpAnchoredRef("kates/src/main/resources/application.properties:329-334",
mcpAnchoredRef("kates/src/main/resources/application.properties:336-341",
"# Prometheus server (for disruption metrics capture)",
"kates.prometheus.url=http://monitoring-kube-prometheus-prometheus.monitoring.svc:9090"),
mcpAnchoredRef("charts/kates/values.yaml:449-459",
Expand Down
8 changes: 5 additions & 3 deletions cli/cmd/mcp_tools_runs_activity.go
Original file line number Diff line number Diff line change
Expand Up @@ -451,15 +451,17 @@ var mcpCaveatsRunsActivity = []mcpCaveat{
"acted. Every REST and gRPC call that changes something leaves a row, a refused or failed one too, " +
"with how it ended in details, and its target is the id of the run or disruption it started when it " +
"started one. The Kates API's daily retention sweep leaves one as system:scheduler for each finished " +
"run it deletes. A call without a valid key, and a dry run, leave none. Rows written before the Kates " +
"run it deletes. A call without a valid key, and a dry run, leave none, and so does a REST call " +
"answered 503 because the Kates API could not reach its database, which it logs instead. Rows written before the Kates " +
"API recorded actors name no one, and then only the REST test endpoints wrote rows at all. Runs and " +
"disruption reports record no owner of their own: who started one shows only in its audit row.",
Refs: []string{
mcpAnchoredRef(mcpJava+"service/RunRetention.java:82",
`auditService.record("DELETE", "test", id, auditDetails, actor);`),
mcpAnchoredRef(mcpJava+"service/TestCleanupScheduler.java:97", "Actor.SCHEDULER"),
mcpAnchoredRef(mcpJava+"audit/AuditResponseFilter.java:42-62",
"public void filter(ContainerRequestContext request", "if (actor == null)", "static String idOf("),
mcpAnchoredRef(mcpJava+"audit/AuditResponseFilter.java:44-78",
"public void filter(ContainerRequestContext request", "if (actor == null)", "if (isDatabaseOutage(response))",
"static String idOf("),
mcpAnchoredRef(mcpJava+"security/GrpcApiKeyInterceptor.java:67-70,100,156-159",
"static final Map<String, String> AUDITED", "audit(action, method, Status.Code.PERMISSION_DENIED",
"private void audit("),
Expand Down
2 changes: 1 addition & 1 deletion cli/cmd/mcp_tools_runs_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -1073,7 +1073,7 @@ var mcpCaveatsRunsList = []mcpCaveat{
Refs: []string{
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:2182-2264",
`"Cancelled by user"`, `EventKind.FAILED, "cancelled"`, "return run.withResults(updatedResults);"),
mcpAnchoredRef(mcpJava+"api/TestResource.java:565-610",
mcpAnchoredRef(mcpJava+"api/TestResource.java:572-617",
`@Path("/{id}/cancel")`, `auditService.record("CANCEL"`, `"Test cancelled; it is stored as FAILED"`),
mcpJava + "domain/TestResult.java:25-31",
},
Expand Down
2 changes: 1 addition & 1 deletion docs/book/11-api-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ curl -s "$BASE/api/health"

The response also contains a `tests` object with the resolved default configuration for every test type. When Kafka is unreachable, the endpoint still returns `200 OK`, but with `"status": "DEGRADED"` and `"kafka.status": "DOWN"`.

`quorum` says whether this Kates is a member of a quorum, which it is only when `kates.quorum.peers` is set. Without one it is `{"enabled": false}`, and the instance acts alone. A member also reports its `memberId`, the `leaderId` it knows of (`null` during an election), the Raft `term`, `appliedIndex`, the last log entry it applied, and `leader`: true once it leads and has committed an entry of its term. A member whose log is empty, new or with its directory lost, is no voter until it has caught up to an entry: three members of which one is empty and one is down elect nobody. Members talk to each other over mutual TLS on their own port: `kates.quorum.tls.cert`, `kates.quorum.tls.key` (PKCS#8 PEM) and `kates.quorum.tls.ca` are required, and so is `kates.quorum.dir`, which keeps a member's term and vote and must outlive a restart. Only the leader takes writes: on any other member, every request but a read is answered `503` with `Retry-After`, and its message names the leader. `PUT /api/drain` is the exception, since a drain is set on each member. Before its first write in a term, a new leader marks `FAILED` the live runs every other member created, and removes the faults Kates made before it took the lead; a member that starts up recovers only its own runs. A leader that loses its majority stops taking writes once its lease runs out, within about a second, even before it steps down.
`quorum` says whether this Kates is a member of a quorum, which it is only when `kates.quorum.peers` is set. Without one it is `{"enabled": false}`, and the instance acts alone. A member also reports `state`, its Raft server's life cycle (`RUNNING` while it serves). Ratis closes a member's server after a pause longer than 60 s, such as a long garbage collection or a frozen node; the member then starts it again from its directory within a few seconds, and until it serves again only `state` and `memberId` are reported. If it still does not serve after `kates.quorum.liveness-after` (30 s), `/q/health/live` answers DOWN so the kubelet restarts the pod. A member reports its `memberId` and, while it serves, the `leaderId` it knows of (`null` during an election), the Raft `term`, `appliedIndex`, the last log entry it applied, and `leader`: true once it leads and has committed an entry of its term. A member whose log is empty, new or with its directory lost, is no voter until it has caught up to an entry: three members of which one is empty and one is down elect nobody. Members talk to each other over mutual TLS on their own port: `kates.quorum.tls.cert`, `kates.quorum.tls.key` (PKCS#8 PEM) and `kates.quorum.tls.ca` are required, and so is `kates.quorum.dir`, which keeps a member's term and vote and must outlive a restart. Only the leader takes writes: on any other member, every request but a read is answered `503` with `Retry-After`, and its message names the leader. `PUT /api/drain` is the exception, since a drain is set on each member. Before its first write in a term, a new leader marks `FAILED` the live runs every other member created, and removes the faults Kates made before it took the lead; a member that starts up recovers only its own runs. A leader that loses its majority stops taking writes once its lease runs out, within about a second, even before it steps down. A request that needs a database Kates cannot reach is answered `503` with `Retry-After: 5` within about 30 s, the JDBC socket timeout, instead of waiting on the connection.

#### GET /api/drain

Expand Down
4 changes: 2 additions & 2 deletions docs/book/appendix-d-versions.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ Every version the platform pins, in one place. This table is generated from the
| `kafka-common` chart | 0.1.3 (library) | `charts/kafka-common/Chart.yaml` |
| `kafka-ui` chart | 0.3.4 (app v1.5.0) | `charts/kafka-ui/Chart.yaml` |
| `kates-chaos` chart | 2.6.0 (app 3.28.0) | `charts/kates-chaos/Chart.yaml` |
| `kates-platform` chart | 0.9.4 (app 1.0.0) | `charts/kates-platform/Chart.yaml` |
| `kates` chart | 0.19.4 (app 1.40.0) | `charts/kates/Chart.yaml` |
| `kates-platform` chart | 0.9.5 (app 1.0.0) | `charts/kates-platform/Chart.yaml` |
| `kates` chart | 0.19.5 (app 1.40.0) | `charts/kates/Chart.yaml` |
| `legacy-kafka` chart | 0.2.2 (app 3.9.1) | `charts/legacy-kafka/Chart.yaml` |
| `mirror-maker2` chart | 0.12.3 (app 4.3.1) | `charts/mirror-maker2/Chart.yaml` |
| `monitoring` chart | 1.9.0 (app 82.4.3) | `charts/monitoring/Chart.yaml` |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ public Response toResponse(Exception exception) {
return error(409, "Conflict", "The resource was modified concurrently — re-read it and retry");
}

if (isDatabaseUnavailable(exception)) {
LOG.warnf("Database unavailable: %s", root.toString());
return databaseUnavailable();
}

if (root instanceof TimeoutException) {
LOG.warn("Request timed out", exception);
return error(504, "Gateway Timeout", root.getMessage());
Expand Down Expand Up @@ -71,6 +76,45 @@ static boolean hasOptimisticLockCause(Throwable t) {
return false;
}

/** What a client is told when the database cannot be reached: retry in this many seconds. */
public static final String DATABASE_RETRY_AFTER = "5";

public static final String DATABASE_UNAVAILABLE = "The Kates API cannot reach its database; retry once it is back";

/**
* Whether the cause chain says the database could not be reached, as opposed to a statement it
* refused: a connection-class SQL state (08) or a server shutting down or starting (57P01 to 57P03), Hibernate's or JDBC's connection exception, or
* Agroal's acquisition timeout. Depth-capped against cycles.
*/
public static boolean isDatabaseUnavailable(Throwable t) {
for (int depth = 0; t != null && depth < 16; t = t.getCause(), depth++) {
if (t instanceof org.hibernate.exception.JDBCConnectionException
|| t instanceof java.sql.SQLTransientConnectionException) {
return true;
}
if (t instanceof java.sql.SQLException sql) {
String state = sql.getSQLState();
String message = sql.getMessage();
// 08: no connection. 57P01-57P03: the server shutting down or starting, as a
// CloudNativePG switchover leaves on the connections it kills.
if ((state != null && (state.startsWith("08") || state.matches("57P0[123]")))
|| (message != null && message.contains("acquisition timeout"))) {
return true;
}
}
}
return false;
}

/** 503 with Retry-After, for a database the Kates API cannot reach. */
public static Response databaseUnavailable() {
return Response.status(503)
.type(MediaType.APPLICATION_JSON_TYPE)
.header("Retry-After", DATABASE_RETRY_AFTER)
.entity(ApiError.of(503, "Service Unavailable", DATABASE_UNAVAILABLE))
.build();
}

private static Response error(int status, String label, String message) {
return Response.status(status)
.type(MediaType.APPLICATION_JSON_TYPE)
Expand Down
24 changes: 22 additions & 2 deletions kates/src/main/java/com/bmscomp/kates/api/KatesLivenessCheck.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
import jakarta.inject.Inject;

import org.eclipse.microprofile.config.inject.ConfigProperty;
Expand All @@ -19,6 +22,9 @@
* exactly the outage it was meant to detect, in a restart loop that got worse
* the more load the pod carried. Genuine exhaustion still surfaces as an
* OOMKill, which the kubelet handles on its own.
*
* <p>The second condition a restart fixes: a quorum member whose Ratis server stays closed
* after the member has tried to start it again for {@code kates.quorum.liveness-after}.
*/
@Liveness
public class KatesLivenessCheck implements HealthCheck {
Expand All @@ -29,6 +35,12 @@ public class KatesLivenessCheck implements HealthCheck {
@ConfigProperty(name = "kates.health.min-free-heap-mb", defaultValue = "64")
int minFreeHeapMb;

@Inject
com.bmscomp.kates.quorum.Quorum quorum;

@ConfigProperty(name = "kates.quorum.liveness-after", defaultValue = "30s")
Duration livenessAfter;

@Override
public HealthCheckResponse call() {
Runtime rt = Runtime.getRuntime();
Expand All @@ -47,6 +59,12 @@ public HealthCheckResponse call() {
} catch (Exception ignored) {
}

Optional<Instant> notServingSince = quorum.notServingSince();
long notServingSec = notServingSince
.map(since -> Duration.between(since, Instant.now()).toSeconds())
.orElse(0L);
boolean quorumDead = notServingSince.isPresent() && notServingSec >= livenessAfter.toSeconds();

var builder = HealthCheckResponse.named("kates-liveness")
.withData("freeHeapMb", freeHeapMb)
.withData("minFreeHeapMb", minFreeHeapMb)
Expand All @@ -55,9 +73,11 @@ public HealthCheckResponse call() {
// on it.
.withData("heapLow", heapLow)
.withData("deadlockedThreads", deadlocked != null ? deadlocked.length : 0)
.withData("activeTests", activeTests + "/" + maxTests);
.withData("activeTests", activeTests + "/" + maxTests)
.withData("quorumState", notServingSince.isPresent() ? "NOT_SERVING" : "SERVING")
.withData("quorumNotServingSec", notServingSec);

if (noDeadlocks) {
if (noDeadlocks && !quorumDead) {
return builder.up().build();
}
return builder.down().build();
Expand Down
7 changes: 7 additions & 0 deletions kates/src/main/java/com/bmscomp/kates/api/TestResource.java
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,9 @@ private Response start(CreateTestRequest request, String clientKey) {
|| failure instanceof com.bmscomp.kates.engine.DrainingException) {
answer.header("Retry-After", RETRY_AFTER_SECONDS);
}
if (GlobalExceptionMapper.isDatabaseUnavailable(failure)) {
answer.header("Retry-After", GlobalExceptionMapper.DATABASE_RETRY_AFTER);
}
return answer.build();
}
TestRun run = result.asSuccess().orElseThrow();
Expand Down Expand Up @@ -226,6 +229,10 @@ private static ApiError notStarted(CreateTestRequest request, Exception failure)
if (failure instanceof com.bmscomp.kates.engine.DrainingException) {
return ApiError.of(503, "Service Unavailable", failure.getMessage());
}
if (GlobalExceptionMapper.isDatabaseUnavailable(failure)) {
LOG.warnf("A test run did not start: its database is unreachable: %s", failure);
return ApiError.of(503, "Service Unavailable", GlobalExceptionMapper.DATABASE_UNAVAILABLE);
}
if (failure instanceof com.bmscomp.kates.engine.InvalidTestSpecException invalid) {
return ApiError.validationFailed(invalid.getMessage(), invalid.getFieldErrors());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
@Provider
public class AuditResponseFilter implements ContainerResponseFilter {

private static final org.jboss.logging.Logger LOG = org.jboss.logging.Logger.getLogger(AuditResponseFilter.class);

static final int TARGET_LENGTH = 128;

@Context
Expand Down Expand Up @@ -53,9 +55,23 @@ public void filter(ContainerRequestContext request, ContainerResponseContext res
String what = id != null ? id : request.getUriInfo().getPath();
String target = what.length() <= TARGET_LENGTH ? what : what.substring(0, TARGET_LENGTH);
String details = "HTTP " + response.getStatus();
// The row cannot be written to a database the answer says is unreachable: its retries only
// held the 503 back by another 30 s (DatabaseOutageIT). The attempt goes to the log instead.
if (isDatabaseOutage(response)) {
LOG.warnf(
"Audit row not written, the database is unreachable: %s %s %s by %s",
audited.action(), audited.type(), target, actor.name());
return;
}
AuditService.blockingSafe(() -> auditService.record(audited.action(), audited.type(), target, details, actor));
}

private static boolean isDatabaseOutage(ContainerResponseContext response) {
return response.getStatus() == 503
&& response.getEntity() instanceof com.bmscomp.kates.api.ApiError error
&& com.bmscomp.kates.api.GlobalExceptionMapper.DATABASE_UNAVAILABLE.equals(error.getMessage());
}

/** The id an answer carries as an "id" entry, or null. */
static String idOf(Object entity) {
return entity instanceof Map<?, ?> map && map.get("id") instanceof String id && !id.isBlank() ? id : null;
Expand Down
Loading
Loading