diff --git a/cli/cmd/mcp_caveats.go b/cli/cmd/mcp_caveats.go index 514d8823..32ea899a 100644 --- a/cli/cmd/mcp_caveats.go +++ b/cli/cmd/mcp_caveats.go @@ -76,7 +76,7 @@ var mcpCaveatsCore = []mcpCaveat{ Short: "A LOAD run is one producer and one consumer whatever numProducers says; only STRESS runs parallel producers.", Text: "A LOAD run is one producer and one consumer, whatever numProducers and numConsumers say, " + "so it cannot show how the cluster behaves under parallel clients. STRESS starts one producer per numProducers.", - Refs: []string{mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1796-1805", "case LOAD ->", "case STRESS ->")}, + Refs: []string{mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1828-1837", "case LOAD ->", "case STRESS ->")}, }, { ID: mcpCaveatReaperDeadline, @@ -99,7 +99,7 @@ var mcpCaveatsCore = []mcpCaveat{ "kates/src/main/resources/application.properties:336-350", mcpAnchoredRef(mcpJava+"engine/TestTimeoutReaper.java:49-51,67-76,170-173,201-214", "TestResult.TaskStatus.STOPPING", "kates.engine.reaper-grace-ms", "\"-submission\"", "\"Timeout: still \""), - mcpJava + "engine/TestOrchestrator.java:151-217,259-262,436,1251-1315,1381-1390,1445-1477", + mcpJava + "engine/TestOrchestrator.java:151-249,291-294,468,1283-1347,1413-1422,1477-1509", mcpJava + "persistence/EntityMapper.java:161-170", "kates/src/main/resources/db/migration/V24__run_planned_duration.sql:1-13", mcpJava + "domain/TestRun.java:520-523", @@ -123,10 +123,10 @@ var mcpCaveatsCore = []mcpCaveat{ "integrity-cg-integrity, without transactions.", Refs: []string{ mcpJava + "domain/TestSpec.java:14-32,111-113", - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:228,251-253,301", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:260,283-285,333", "applyTypeDefaults(type, request.getSpec())", ".withRequestedSpec(", "buildTasks(type, spec, run.getId())"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1171-1217", "TestSpec applyTypeDefaults(", "return merged;"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1775-1779,1870-1874", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1203-1249", "TestSpec applyTypeDefaults(", "return merged;"), + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1807-1811,1902-1906", "Only when asked", `"-integrity-0-group")`), mcpAnchoredRef(mcpJava+"engine/NativeKafkaBackend.java:792", `task.getConsumerGroup() + "-integrity"`), mcpAnchoredRef(mcpJava+"persistence/EntityMapper.java:56,85-86,146-148", @@ -177,7 +177,7 @@ var mcpCaveatsCore = []mcpCaveat{ Text: "A TUNE_* run executes one produce task with the spec's single configuration: it measures one setting " + "and compares none.", Refs: []string{ - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1881-1882", "case TUNE_REPLICATION,", "-tune-0"), + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1913-1914", "case TUNE_REPLICATION,", "-tune-0"), }, }, { @@ -241,7 +241,7 @@ var mcpCaveatsCore = []mcpCaveat{ Refs: []string{ mcpAnchoredRef("kates/src/main/resources/application.properties:274-275", "# Chaos coordination (noop | kubernetes | litmus-crd | hybrid)", "kates.chaos.provider=litmus-crd"), - mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:63,108", + mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:63,117", "int resultPollIntervalMs = 5_000;", "Instant start = Instant.now();"), }, }, diff --git a/cli/cmd/mcp_tools_chaos.go b/cli/cmd/mcp_tools_chaos.go index 12c3b08c..c573c967 100644 --- a/cli/cmd/mcp_tools_chaos.go +++ b/cli/cmd/mcp_tools_chaos.go @@ -147,7 +147,7 @@ var mcpCaveatsChaos = []mcpCaveat{ mcpAnchoredRef(mcpJava+"disruption/DisruptionSafetyGuard.java:298-301,874-943", "boolean canExecute = checkRbacPermissions(spec);", `"Insufficient RBAC permissions for "`, "boolean checkRbacPermissions(FaultSpec spec)", `"RBAC check failed, assuming permitted"`), - mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:42-60,87-89,270", + mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:42-60,96-98,284", "ON_KUBERNETES_API = EnumSet.of(", "DisruptionType.POD_DELETE,", "DisruptionType.NETWORK_PARTITION);", "return kubernetes.triggerFault(spec, onInject);", `engineSpec.chaosServiceAccount = "litmus-admin";`), mcpAnchoredRef("kates/src/main/resources/application.properties:274-275", @@ -226,8 +226,8 @@ var mcpCaveatsChaos = []mcpCaveat{ "if (selected != null && selected.isAvailable())", "new NoOpChaosProvider()", "matches no chaos provider"), mcpJava + "chaos/NoOpChaosProvider.java:27-32", mcpJava + "chaos/ChaosOutcome.java:77-81", - mcpJava + "disruption/DisruptionOrchestrator.java:157-160,221", - mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:347-351", + mcpJava + "disruption/DisruptionOrchestrator.java:170-173,234", + mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:360-364", "boolean injected = disruptionStart != null;", "The fault never went in"), mcpJava + "disruption/DisruptionImpactScorer.java:72-77", }, @@ -242,7 +242,7 @@ var mcpCaveatsChaos = []mcpCaveat{ "scores as degradation. Without Prometheus both are 0. Judge latency and throughput from the summary " + "and the steps' impactDeltas instead.", Refs: []string{ - mcpJava + "disruption/DisruptionOrchestrator.java:184-189,200-201", + mcpJava + "disruption/DisruptionOrchestrator.java:197-202,213-214", mcpJava + "disruption/PrometheusMetricsCapture.java:152-171", mcpJava + "disruption/DisruptionImpactScorer.java:92-130", }, @@ -264,7 +264,7 @@ var mcpCaveatsChaos = []mcpCaveat{ "measured them for a step whose fault never went in as well; a report does not say which of the two counted " + "its times.", Refs: []string{ - mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:302,313-315,347-356,393-399,503-509", + mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:315,326-328,360-369,406-412,518-524", "new RecoveryClocks(session, isrTracker, lagTracker)", ".triggerFault(spec, clocks::start)", "boolean injected = disruptionStart != null;", "if (step.requireRecovery() && injected)", "Duration strimziRecovery = injected", "injected && isrTracker != null ? isrTracker.stop() : null", @@ -275,14 +275,14 @@ var mcpCaveatsChaos = []mcpCaveat{ mcpAnchoredRef(mcpJava+"disruption/KafkaIntelligenceService.java:125-127,198-217,241-246,310-324", "public void markDisruptionStart(Instant at)", "private Duration computeTimeToFullIsr()", "this.baselineLag = timeline.getLast().totalLag();", "private Duration computeTimeToLagRecovery(long baseLag)"), - mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:87-99,108-127", + mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:96-108,117-141", "return kubernetes.triggerFault(spec, onInject);", "CompletableFuture.delayedExecutor(spec.delayBeforeSec(), TimeUnit.SECONDS, executor.get());", "long startNanos = System.nanoTime();", "buildChaosEngine(spec, engineName, experimentName)", "onInject.accept(startNanos);", ".resource(engine)"), - mcpAnchoredRef(mcpJava+"chaos/KubernetesChaosProvider.java:152-161", + mcpAnchoredRef(mcpJava+"chaos/KubernetesChaosProvider.java:170-182", "Thread.sleep(spec.delayBeforeSec() * 1000L);", "onInject.accept(startNanos);", - "applyDisruption(spec, engineName);"), + "applyDisruption(spec, engineName, marks);"), mcpAnchoredRef(mcpJava+"chaos/PodTargets.java:58,69-75", "public static List resolve(KubernetesClient client, FaultSpec spec)", "throw new IllegalStateException("), @@ -299,7 +299,7 @@ var mcpCaveatsChaos = []mcpCaveat{ Refs: []string{ mcpJava + "disruption/KafkaIntelligenceService.java:129-154,165-168", mcpJava + "disruption/DisruptionImpactScorer.java:42-45,132-157", - mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:266-269,398", + mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:279-282,411", "intelligence.startIsrTracking(", "isrTracker.stop()"), }, }, @@ -310,12 +310,13 @@ var mcpCaveatsChaos = []mcpCaveat{ "stopped mid-plan and has not started again: such a plan never finishes. When the backend starts, it " + "marks each report created before then that still says RUNNING as INTERRUPTED, in the report and in " + "the list of disruptions alike, unless kates.chaos.orphan-recovery.enabled is false. A backend in a " + - "quorum (kates.quorum.peers) marks nothing when it starts: the next member to lead marks each report " + - "created before it took the lead. Without a quorum this assumes one backend replica: a replica that " + + "quorum (kates.quorum.peers) marks nothing when it starts: the member that leads marks each report " + + "created before it took the lead, every 10 s until PostgreSQL answers, and only in a pass that removed " + + "every fault it found of an older term or past its deadline. Without a quorum this assumes one backend replica: a replica that " + "starts while another runs a plan marks that plan's report too, until the plan ends and stores its " + "outcome.", Refs: []string{ - mcpJava + "disruption/DisruptionOrphanReconciler.java:57-58,66-77,93-111,189-240", + mcpJava + "disruption/DisruptionOrphanReconciler.java:71-75,80-109,116-166,334-368", mcpJava + "disruption/DisruptionReportRepository.java:35-60,93-102", mcpJava + "disruption/DisruptionPersistence.java:26-37", mcpJava + "disruption/DisruptionLauncher.java:114-121,129-144", diff --git a/cli/cmd/mcp_tools_runs_activity.go b/cli/cmd/mcp_tools_runs_activity.go index 40addb78..748ed5ac 100644 --- a/cli/cmd/mcp_tools_runs_activity.go +++ b/cli/cmd/mcp_tools_runs_activity.go @@ -324,7 +324,7 @@ func mcpActivityTestsSince(ctx context.Context, call *mcpCall, since time.Time, // are injected for part of its run, or one whose backend process stopped // before it ended and has not started again to mark it INTERRUPTED // (DisruptionLauncher.java:95-123, DisruptionReportRepository.java:35-43, -// DisruptionOrphanReconciler.java:189-240). +// DisruptionOrphanReconciler.java:359-393). func mcpActivityDisruptionsSince(ctx context.Context, call *mcpCall, since time.Time, page mcpActivityPage) (mcpActivityDisruptions, error) { out := mcpActivityDisruptions{StoredAsRunning: []mcpActivityDisruptionRow{}, Since: []mcpActivityDisruptionRow{}} for p := 0; p < mcpActivityMaxPages; p++ { @@ -486,7 +486,7 @@ var mcpCaveatsRunsActivity = []mcpCaveat{ mcpJava + "disruption/DisruptionLauncher.java:114-144", mcpJava + "disruption/DisruptionPersistence.java:16-37", mcpJava + "disruption/DisruptionReportRepository.java:20-43", - mcpJava + "disruption/DisruptionOrphanReconciler.java:189-240", + mcpJava + "disruption/DisruptionOrphanReconciler.java:334-368", mcpJava + "disruption/DisruptionTemplateResource.java:70-72", mcpJava + "disruption/DisruptionScheduler.java:112-124", mcpJava + "disruption/DisruptionReportEntity.java:39-47", diff --git a/cli/cmd/mcp_tools_runs_list.go b/cli/cmd/mcp_tools_runs_list.go index a67742e2..c989ce29 100644 --- a/cli/cmd/mcp_tools_runs_list.go +++ b/cli/cmd/mcp_tools_runs_list.go @@ -1032,7 +1032,7 @@ var mcpCaveatsRunsList = []mcpCaveat{ mcpAnchoredRef(mcpJava+"report/ReportGenerator.java:220-249,283-294", "report.setSummary(MetricUtils.computeSummary(results));", "summaries.put(phase, MetricUtils.computeSummary(rows))"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:505-514,2235-2251", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:537-546,2267-2283", "TestResult.TaskStatus.PENDING : TestResult.TaskStatus.RUNNING", "private static TestRun withUnfinishedTasksFailed("), }, @@ -1071,7 +1071,7 @@ var mcpCaveatsRunsList = []mcpCaveat{ "except for a run cancelled before its tasks existed; a cancel through the REST API also leaves a CANCEL " + "audit row.", Refs: []string{ - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:2182-2264", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:2214-2296", `"Cancelled by user"`, `EventKind.FAILED, "cancelled"`, "return run.withResults(updatedResults);"), mcpAnchoredRef(mcpJava+"api/TestResource.java:572-617", `@Path("/{id}/cancel")`, `auditService.record("CANCEL"`, `"Test cancelled; it is stored as FAILED"`), @@ -1104,14 +1104,14 @@ var mcpCaveatsRunsList = []mcpCaveat{ "say which the API is, but a scenario run's tasks do: on an older API they all start within moments " + "of one another.", Refs: []string{ - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:422-424,481-492,494-510", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:454-456,513-524,526-542", "applyTypeDefaults(type, scenario.getBaseSpec())", ".withRequestedSpec(", "long phaseStartMs = System.currentTimeMillis();", "scenario.resolveSpecForPhase(phase, baseSpec)", "phaseStartMs = saturatedSum(phaseStartMs", "backend.submit(task)", "TestResult.TaskStatus.PENDING"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1583-1611,1624-1632,1692-1731,1741-1753", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1615-1643,1656-1664,1724-1763,1773-1785", "MAX_RAMP_STEPS = 100", "scenarioInapplicableFields(TestScenario scenario", "is a SPIKE phase", "has none (", "check no record CRCs", "has the name of an earlier phase"), - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1878-1947", "List buildPhaseTask(", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1910-1979", "List buildPhaseTask(", "int baseTarget = Math.max(1, spec.getThroughput() / steps);", "saturatedSum(startAtMs, s * stepMs)", `taskId + "-spike"`), mcpAnchoredRef(mcpJava+"domain/TestScenario.java:115-190", diff --git a/cli/cmd/mcp_tools_scenario.go b/cli/cmd/mcp_tools_scenario.go index 0bb74cc6..f2592f4a 100644 --- a/cli/cmd/mcp_tools_scenario.go +++ b/cli/cmd/mcp_tools_scenario.go @@ -1242,7 +1242,7 @@ var mcpCaveatsScenario = []mcpCaveat{ "plans/mcp-server.md:387-405", mcpJava + "api/TestResource.java:118-188", mcpJava + "domain/TestSpec.java:34-113", - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1503-1578", + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1535-1610", "Map inapplicableFields(", "the trogdor backend cannot run a transactional producer"), mcpAnchoredRef("cli/cmd/apply.go:238-307", "for i, scenario := range sf.Scenarios {", "apiClient.CreateTest(a.ctx, scenarioToRequest(scenario))", @@ -1259,7 +1259,7 @@ var mcpCaveatsScenario = []mcpCaveat{ "listed as defaulted may differ on the cluster that runs the scenario.", Refs: []string{ mcpJava + "config/TestTypeDefaults.java:22-64,323-464", - mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1171-1217", "TestSpec applyTypeDefaults(", "return merged;"), + mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1203-1249", "TestSpec applyTypeDefaults(", "return merged;"), mcpAnchoredRef("kates/src/main/resources/application.properties:73-114", "# Per-type overrides: STRESS", "kates.tests.roundtrip.throughput=10000"), mcpAnchoredRef("charts/kates/values.yaml:690-758", "defaults:", "tests:", `throughput: "10000"`), diff --git a/docs/book/11-api-reference.md b/docs/book/11-api-reference.md index 6ab0a37b..c25c122c 100644 --- a/docs/book/11-api-reference.md +++ b/docs/book/11-api-reference.md @@ -237,7 +237,7 @@ curl -s "$BASE/api/health" The response also contains a `tests` object with the resolved default configuration for every test type. When Kafka is unreachable, the endpoint still returns `200 OK`, but with `"status": "DEGRADED"` and `"kafka.status": "DOWN"`. -`quorum` says whether this Kates is a member of a quorum, which it is only when `kates.quorum.peers` is set. Without one it is `{"enabled": false}`, and the instance acts alone. A member also reports `state`, its Raft server's life cycle (`RUNNING` while it serves). Ratis closes a member's server after a pause longer than 60 s, such as a long garbage collection or a frozen node; the member then starts it again from its directory within a few seconds, and until it serves again only `state` and `memberId` are reported. If it still does not serve after `kates.quorum.liveness-after` (30 s), `/q/health/live` answers DOWN so the kubelet restarts the pod. A member reports its `memberId` and, while it serves, the `leaderId` it knows of (`null` during an election), the Raft `term`, `appliedIndex`, the last log entry it applied, and `leader`: true once it leads and has committed an entry of its term. A member whose log is empty, new or with its directory lost, is no voter until it has caught up to an entry: three members of which one is empty and one is down elect nobody. Members talk to each other over mutual TLS on their own port: `kates.quorum.tls.cert`, `kates.quorum.tls.key` (PKCS#8 PEM) and `kates.quorum.tls.ca` are required, and so is `kates.quorum.dir`, which keeps a member's term and vote and must outlive a restart. Only the leader takes writes: on any other member, every request but a read is answered `503` with `Retry-After`, and its message names the leader. `PUT /api/drain` is the exception, since a drain is set on each member. Before its first write in a term, a new leader marks `FAILED` the live runs every other member created, and removes the faults Kates made before it took the lead; a member that starts up recovers only its own runs. A leader that loses its majority stops taking writes once its lease runs out, within about a second, even before it steps down. A request that needs a database Kates cannot reach is answered `503` with `Retry-After: 5` within about 30 s, the JDBC socket timeout, instead of waiting on the connection. +`quorum` says whether this Kates is a member of a quorum, which it is only when `kates.quorum.peers` is set. Without one it is `{"enabled": false}`, and the instance acts alone. A member also reports `state`, its Raft server's life cycle (`RUNNING` while it serves). Ratis closes a member's server after a pause longer than 60 s, such as a long garbage collection or a frozen node; the member then starts it again from its directory within a few seconds, and until it serves again only `state` and `memberId` are reported. If it still does not serve after `kates.quorum.liveness-after` (30 s), `/q/health/live` answers DOWN so the kubelet restarts the pod. A member reports its `memberId` and, while it serves, the `leaderId` it knows of (`null` during an election), the Raft `term`, `appliedIndex`, the last log entry it applied, and `leader`: true once it leads and has committed an entry of its term. A member whose log is empty, new or with its directory lost, is no voter until it has caught up to an entry: three members of which one is empty and one is down elect nobody. Members talk to each other over mutual TLS on their own port: `kates.quorum.tls.cert`, `kates.quorum.tls.key` (PKCS#8 PEM) and `kates.quorum.tls.ca` are required, and so is `kates.quorum.dir`, which keeps a member's term and vote and must outlive a restart. Only the leader takes writes: on any other member, every request but a read is answered `503` with `Retry-After`, and its message names the leader. `PUT /api/drain` is the exception, since a drain is set on each member. A member checks that it still leads right before each fault goes into Kubernetes, so a member that stopped leading since its plan started injects nothing, and every object a fault writes carries `kates.io/term`, the writer's term, and `kates.io/deadline`, the write's time plus `kates.engine.max-duration-ms`. The leader reconciles every `kates.quorum.reconcile-interval` (10 s): it removes any Kates fault of an older term or past its deadline, NetworkPolicies, ChaosEngines and scale-downs in every namespace alike, and marks `FAILED` the live runs every other member created, retrying while PostgreSQL is unreachable rather than waiting on it to lead. A member that stops leading stops its own runs at once; the new leader stores them `FAILED`. A member that starts up recovers only its own runs. A leader that loses its majority stops taking writes once its lease runs out, within about a second, even before it steps down. A request that needs a database Kates cannot reach is answered `503` with `Retry-After: 5` within about 30 s, the JDBC socket timeout, instead of waiting on the connection. #### GET /api/drain diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/FaultMarks.java b/kates/src/main/java/com/bmscomp/kates/chaos/FaultMarks.java new file mode 100644 index 00000000..d100990a --- /dev/null +++ b/kates/src/main/java/com/bmscomp/kates/chaos/FaultMarks.java @@ -0,0 +1,23 @@ +package com.bmscomp.kates.chaos; + +import java.time.Instant; +import java.util.Map; + +/** + * What every object a fault creates or changes carries, so that a later leader can tell a fault + * of an older term, or one past its deadline, from the faults it runs itself. + */ +public final class FaultMarks { + + /** The Ratis term of the leader that wrote the object; {@code 0} with no quorum. */ + public static final String TERM = "kates.io/term"; + + /** The instant after which any leader undoes the fault, whoever made it. */ + public static final String DEADLINE = "kates.io/deadline"; + + private FaultMarks() {} + + public static Map of(long term, Instant deadline) { + return Map.of(TERM, Long.toString(term), DEADLINE, deadline.toString()); + } +} diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java b/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java index b9c9996d..27a9719d 100644 --- a/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java +++ b/kates/src/main/java/com/bmscomp/kates/chaos/KubernetesChaosProvider.java @@ -96,6 +96,24 @@ public IncompleteScaleDownException(String message) { @Inject com.bmscomp.kates.engine.KatesExecutor executor; + // Built without config, as the tests build it, a Quorum is alone and always leads. + @Inject + com.bmscomp.kates.quorum.Quorum quorum = new com.bmscomp.kates.quorum.Quorum(); + + @org.eclipse.microprofile.config.inject.ConfigProperty( + name = "kates.engine.max-duration-ms", + defaultValue = "7200000") + long maxDurationMs = 7_200_000; + + /** + * The marks a fault written now carries: this leader's term, and a deadline at the engine's run + * bound. shortcut: a fault the live leader leaves after a crashed plan thread lives up to that + * bound (2 h); upgrade to the plan's own bound if that is seen. + */ + Map marks(long term) { + return FaultMarks.of(term, Instant.now().plusMillis(maxDurationMs)); + } + @Override public String name() { return "kubernetes"; @@ -116,13 +134,13 @@ public String name() { * error that said nothing about it, and a few faults per plan are no load * to shed from the API server. */ - private void applyDisruption(FaultSpec spec, String engineName) throws Exception { + private void applyDisruption(FaultSpec spec, String engineName, Map marks) throws Exception { switch (spec.disruptionType()) { case POD_KILL -> executePodKill(spec); case POD_DELETE -> executePodDelete(spec); - case NETWORK_PARTITION -> executeNetworkPartition(spec, engineName); + case NETWORK_PARTITION -> executeNetworkPartition(spec, engineName, marks); case ROLLING_RESTART -> executeRollingRestart(spec); - case SCALE_DOWN -> executeScaleDown(spec); + case SCALE_DOWN -> executeScaleDown(spec, marks); case LEADER_ELECTION -> executePodKill(spec); case CPU_STRESS -> executeCpuStress(spec, engineName); case IO_STRESS -> executeIoStress(spec, engineName); @@ -156,9 +174,12 @@ public CompletableFuture triggerFault(FaultSpec spec, LongConsumer start = Instant.now(); startNanos = System.nanoTime(); } + // Confirmed after the delay, right before any write: a member that stopped + // leading since the plan started writes nothing (J4 F1). + Map marks = marks(quorum.requireWrites()); // The fault goes in from here, once its delay is over. onInject.accept(startNanos); - applyDisruption(spec, engineName); + applyDisruption(spec, engineName, marks); return ChaosOutcome.success( engineName, spec.experimentName(), start, Instant.now(), startNanos, null, null, null); @@ -219,11 +240,12 @@ private void executePodDelete(FaultSpec spec) { * failed or the wait was interrupted: a NetworkPolicy left behind cuts the * pod off until rollback, which a failed fault does not get. */ - private void executeNetworkPartition(FaultSpec spec, String engineName) throws InterruptedException { + private void executeNetworkPartition(FaultSpec spec, String engineName, Map marks) + throws InterruptedException { Exception failure = null; try { for (String podName : PodTargets.resolve(client, spec)) { - isolatePod(spec, podName); + isolatePod(spec, podName, marks); } // The chaos limits refuse a partition without a duration; one that // gets here anyway is a blip. @@ -262,7 +284,7 @@ private void removePartition(String namespace, String engineName, Exception fail } } - private void isolatePod(FaultSpec spec, String podName) { + private void isolatePod(FaultSpec spec, String podName, Map marks) { String policyName = "kates-netpol-" + podName; LOG.info("NETWORK_PARTITION: applying deny NetworkPolicy for " + podName); @@ -282,6 +304,7 @@ private void isolatePod(FaultSpec spec, String podName) { .withName(policyName) .withNamespace(spec.targetNamespace()) .addToLabels("managed-by", "kates") + .addToAnnotations(marks) .endMetadata() .withNewSpec() .withNewPodSelector() @@ -443,7 +466,7 @@ private void removeManualRollingUpdate(String namespace, String podName) { * not run by Strimzi) is scaled down by one, but never below one replica. * Each records its original replica count for rollback. */ - private void executeScaleDown(FaultSpec spec) throws InterruptedException { + private void executeScaleDown(FaultSpec spec, Map marks) throws InterruptedException { ScaleDownTargets.Targets targets = ScaleDownTargets.resolve(client, spec); targets.skipped().forEach(reason -> LOG.info("SCALE_DOWN: skipping, " + reason)); if (targets.isEmpty()) { @@ -452,11 +475,11 @@ private void executeScaleDown(FaultSpec spec) throws InterruptedException { } if (!targets.pools().isEmpty()) { - new NodePoolScaleDown(client, scalePollIntervalMs).run(spec, targets.pools()); + new NodePoolScaleDown(client, scalePollIntervalMs, marks).run(spec, targets.pools()); } int scaled = 0; for (String name : targets.statefulSets()) { - if (scaleDownStatefulSet(spec.targetNamespace(), name)) { + if (scaleDownStatefulSet(spec.targetNamespace(), name, marks)) { scaled++; } } @@ -466,7 +489,7 @@ private void executeScaleDown(FaultSpec spec) throws InterruptedException { } } - private boolean scaleDownStatefulSet(String namespace, String name) { + private boolean scaleDownStatefulSet(String namespace, String name, Map marks) { StatefulSet ss = client.apps() .statefulSets() .inNamespace(namespace) @@ -488,19 +511,17 @@ private boolean scaleDownStatefulSet(String namespace, String name) { // already-reduced value, so only stamp when absent. Map annotations = ss.getMetadata().getAnnotations(); boolean alreadySnapshotted = annotations != null && annotations.containsKey(ORIGINAL_REPLICAS_ANNOTATION); - if (!alreadySnapshotted) { - String scaledDownAt = String.valueOf(System.currentTimeMillis()); - client.apps() - .statefulSets() - .inNamespace(namespace) - .withName(name) - .edit(s -> new StatefulSetBuilder(s) - .editMetadata() - .addToAnnotations(ORIGINAL_REPLICAS_ANNOTATION, String.valueOf(current)) - .addToAnnotations(SCALED_DOWN_AT_ANNOTATION, scaledDownAt) - .endMetadata() - .build()); - } + // The marks are written every time: a snapshot an earlier plan left carries an older term, + // which the reconciler would take for a dead leader's fault and undo under this one. + String scaledDownAt = String.valueOf(System.currentTimeMillis()); + client.apps().statefulSets().inNamespace(namespace).withName(name).edit(s -> { + var metadata = new StatefulSetBuilder(s).editMetadata().addToAnnotations(marks); + if (!alreadySnapshotted) { + metadata.addToAnnotations(ORIGINAL_REPLICAS_ANNOTATION, String.valueOf(current)) + .addToAnnotations(SCALED_DOWN_AT_ANNOTATION, scaledDownAt); + } + return metadata.endMetadata().build(); + }); LOG.info("SCALE_DOWN: StatefulSet " + name + " from " + current + " → " + (current - 1)); client.apps().statefulSets().inNamespace(namespace).withName(name).scale(current - 1); return true; diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java b/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java index 2e3cd08d..fe728120 100644 --- a/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java +++ b/kates/src/main/java/com/bmscomp/kates/chaos/LitmusChaosProvider.java @@ -77,6 +77,15 @@ public String name() { return "litmus-crd"; } + // Built without config, as the tests build it, a Quorum is alone and always leads. + @Inject + com.bmscomp.kates.quorum.Quorum quorum = new com.bmscomp.kates.quorum.Quorum(); + + @org.eclipse.microprofile.config.inject.ConfigProperty( + name = "kates.engine.max-duration-ms", + defaultValue = "7200000") + long maxDurationMs = 7_200_000; + @Override public CompletableFuture triggerFault(FaultSpec spec) { return triggerFault(spec, injectedAt -> {}); @@ -117,6 +126,11 @@ private CompletableFuture inject(FaultSpec spec, LongConsumer onIn ChaosEngine engine = buildChaosEngine(spec, engineName, experimentName); String resultName = engineName + "-" + experimentName; + // Confirmed right before the engine exists: a member that stopped leading writes nothing. + long term = quorum.requireWrites(); + engine.getMetadata() + .setAnnotations(FaultMarks.of(term, Instant.now().plusMillis(maxDurationMs))); + // Once the pods are picked, before the engine that injects the fault exists. onInject.accept(startNanos); diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/NodePoolScaleDown.java b/kates/src/main/java/com/bmscomp/kates/chaos/NodePoolScaleDown.java index 63235129..9adbdbdb 100644 --- a/kates/src/main/java/com/bmscomp/kates/chaos/NodePoolScaleDown.java +++ b/kates/src/main/java/com/bmscomp/kates/chaos/NodePoolScaleDown.java @@ -69,9 +69,12 @@ private enum Progress { REMOVED } - NodePoolScaleDown(KubernetesClient client, long pollIntervalMs) { + private final Map marks; + + NodePoolScaleDown(KubernetesClient client, long pollIntervalMs, Map marks) { this.client = client; this.pollIntervalMs = pollIntervalMs; + this.marks = marks; } /** @@ -176,15 +179,17 @@ private void lower(String namespace, Pool pool, GenericKubernetesResource before throw new IllegalStateException( "KafkaNodePool " + pool.name() + " changed size while SCALE_DOWN was lowering it"); } + Map a = new HashMap<>( + p.getMetadata().getAnnotations() != null + ? p.getMetadata().getAnnotations() + : Map.of()); + // Every time, as for a StatefulSet: an earlier plan's snapshot carries an older term. + a.putAll(marks); if (snapshot) { - Map a = new HashMap<>( - p.getMetadata().getAnnotations() != null - ? p.getMetadata().getAnnotations() - : Map.of()); a.put(KubernetesChaosProvider.ORIGINAL_REPLICAS_ANNOTATION, String.valueOf(replicas)); a.put(KubernetesChaosProvider.SCALED_DOWN_AT_ANNOTATION, scaledDownAt); - p.getMetadata().setAnnotations(a); } + p.getMetadata().setAnnotations(a); spec(p).put("replicas", replicas - 1); return p; }); diff --git a/kates/src/main/java/com/bmscomp/kates/chaos/ScaleDownSnapshots.java b/kates/src/main/java/com/bmscomp/kates/chaos/ScaleDownSnapshots.java index fba1ee4b..1d3ca201 100644 --- a/kates/src/main/java/com/bmscomp/kates/chaos/ScaleDownSnapshots.java +++ b/kates/src/main/java/com/bmscomp/kates/chaos/ScaleDownSnapshots.java @@ -2,6 +2,7 @@ import java.net.HttpURLConnection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Predicate; @@ -43,19 +44,54 @@ public int total() { */ public static Restored restore(KubernetesClient client, String namespace, Predicate eligible) { return new Restored( - restoreStatefulSets(client, namespace, eligible), restoreNodePools(client, namespace, eligible)); + restoreStatefulSets(client, namespace, null, eligible), + restoreNodePools(client, namespace, null, eligible)); } - private static int restoreStatefulSets(KubernetesClient client, String namespace, Predicate eligible) { + /** + * As {@link #restore}, in every namespace: a playbook can scale down outside Kafka's. Where the + * backend may not list across namespaces (rbac.scope=namespace), {@code fallback} only. + */ + public static Restored restoreEverywhere(KubernetesClient client, String fallback, Predicate eligible) { + return new Restored( + restoreStatefulSets(client, null, fallback, eligible), + restoreNodePools(client, null, fallback, eligible)); + } + + /** Lists in {@code namespace}, or in every namespace when it is null, else {@code fallback} on a 403. */ + private static List list( + String namespace, + String fallback, + java.util.function.Function> in, + java.util.function.Supplier> everywhere) { + if (namespace != null) { + return in.apply(namespace); + } + try { + return everywhere.get(); + } catch (KubernetesClientException e) { + if (e.getCode() != 403 || fallback == null) { + throw e; + } + return in.apply(fallback); + } + } + + private static int restoreStatefulSets( + KubernetesClient client, String namespace, String fallback, Predicate eligible) { int restored = 0; try { - for (StatefulSet ss : - client.apps().statefulSets().inNamespace(namespace).list().getItems()) { + for (StatefulSet ss : list( + namespace, + fallback, + ns -> client.apps().statefulSets().inNamespace(ns).list().getItems(), + () -> client.apps().statefulSets().inAnyNamespace().list().getItems())) { Integer original = snapshot(ss.getMetadata()); if (original == null || !eligible.test(ss.getMetadata())) { continue; } String name = ss.getMetadata().getName(); + String namespaceOf = ss.getMetadata().getNamespace(); try { int current = ss.getSpec().getReplicas() != null ? ss.getSpec().getReplicas() : original; @@ -63,14 +99,14 @@ private static int restoreStatefulSets(KubernetesClient client, String namespace LOG.warnf("Restoring StatefulSet %s from %d → %d", name, current, original); client.apps() .statefulSets() - .inNamespace(namespace) + .inNamespace(namespaceOf) .withName(name) .scale(original); restored++; } client.apps() .statefulSets() - .inNamespace(namespace) + .inNamespace(namespaceOf) .withName(name) .edit(s -> new StatefulSetBuilder(s) .editMetadata() @@ -88,24 +124,33 @@ private static int restoreStatefulSets(KubernetesClient client, String namespace return restored; } - private static int restoreNodePools(KubernetesClient client, String namespace, Predicate eligible) { + private static int restoreNodePools( + KubernetesClient client, String namespace, String fallback, Predicate eligible) { int restored = 0; try { - for (GenericKubernetesResource pool : client.genericKubernetesResources(NodePoolScaleDown.NODE_POOLS) - .inNamespace(namespace) - .list() - .getItems()) { + for (GenericKubernetesResource pool : list( + namespace, + fallback, + ns -> client.genericKubernetesResources(NodePoolScaleDown.NODE_POOLS) + .inNamespace(ns) + .list() + .getItems(), + () -> client.genericKubernetesResources(NodePoolScaleDown.NODE_POOLS) + .inAnyNamespace() + .list() + .getItems())) { Integer original = snapshot(pool.getMetadata()); if (original == null || !eligible.test(pool.getMetadata())) { continue; } String name = pool.getMetadata().getName(); + String namespaceOf = pool.getMetadata().getNamespace(); int current = NodePoolScaleDown.replicas(pool); try { // One patch: the replicas and the snapshot go together, so a // failure leaves the snapshot for the next attempt. client.genericKubernetesResources(NodePoolScaleDown.NODE_POOLS) - .inNamespace(namespace) + .inNamespace(namespaceOf) .withName(name) .edit(p -> { if (current < original) { diff --git a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuard.java b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuard.java index c7680ffd..d72a02e1 100644 --- a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuard.java +++ b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuard.java @@ -85,7 +85,8 @@ public boolean tryAcquire(String target, String token) { */ public void release(String target, String token) { Long since = acquiredAt.remove(token); - if (since != null && token.equals(activeByTarget.get(target))) { + // A holder that no longer leads gives nothing back: the next leader restores by term. + if (since != null && token.equals(activeByTarget.get(target)) && quorum.acceptsWrites()) { restoreScaleDownsSince(since); } activeByTarget.remove(target, token); diff --git a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrchestrator.java b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrchestrator.java index d68e7054..cea7f8cf 100644 --- a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrchestrator.java +++ b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrchestrator.java @@ -56,6 +56,10 @@ public class DisruptionOrchestrator { @Inject DisruptionConcurrencyGuard concurrencyGuard; + // Built without config, as the tests build it, a Quorum is alone and always leads. + @Inject + com.bmscomp.kates.quorum.Quorum quorum = new com.bmscomp.kates.quorum.Quorum(); + @ConfigProperty(name = "kates.chaos.kafka.namespace", defaultValue = "kafka") String kafkaNamespace; @@ -144,8 +148,17 @@ private DisruptionReport executePlan(String disruptionId, DisruptionPlan plan) { double totalThroughputDelta = 0; double maxP99Spike = 0; + // A plan belongs to the term it started in: a member that lost the lead, or leads again in a + // later term, injects none of its remaining steps (J4 review). The next leader undoes the rest. + long term = quorum.term(); try { for (DisruptionPlan.DisruptionStep step : plan.getSteps()) { + if (quorum.term() != term || !quorum.acceptsWrites()) { + LOG.warnf( + "Plan '%s' stopped before step '%s': this member no longer leads term %d", + plan.getName(), step.name(), term); + break; + } LOG.info("Executing disruption step: " + step.name()); eventBus.emit( disruptionId, @@ -367,7 +380,7 @@ private DisruptionReport.StepReport executeStep( recovered = safetyGuard.verifyClusterState(); } - if (!recovered && plan.isAutoRollback()) { + if (!recovered && plan.isAutoRollback() && quorum.acceptsWrites()) { LOG.warn(" Recovery timeout — triggering auto-rollback for " + step.name()); safetyGuard.rollback(spec, outcome.engineName()); eventBus.emit( @@ -423,7 +436,9 @@ private DisruptionReport.StepReport executeStep( } catch (Exception e) { LOG.error("Step '" + step.name() + "' failed", e); - if (plan.isAutoRollback()) { + // A member that no longer leads undoes nothing: the rollback deletes every Kates partition + // in the namespace, the next leader's too, and that leader undoes this one's by term. + if (plan.isAutoRollback() && quorum.acceptsWrites()) { try { LOG.warn(" Auto-rollback on exception for step " + step.name()); safetyGuard.rollback(step.faultSpec(), ""); diff --git a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java index c1df91fe..dff03e58 100644 --- a/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java +++ b/kates/src/main/java/com/bmscomp/kates/disruption/DisruptionOrphanReconciler.java @@ -4,6 +4,7 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; +import java.util.Map; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.event.Observes; import jakarta.inject.Inject; @@ -14,11 +15,14 @@ import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicy; import io.fabric8.kubernetes.client.KubernetesClient; import io.quarkus.runtime.StartupEvent; +import io.quarkus.scheduler.Scheduled; import org.eclipse.microprofile.config.inject.ConfigProperty; import org.jboss.logging.Logger; +import com.bmscomp.kates.chaos.FaultMarks; import com.bmscomp.kates.chaos.KubernetesChaosProvider; import com.bmscomp.kates.chaos.ScaleDownSnapshots; +import com.bmscomp.kates.chaos.litmus.ChaosEngine; /** * Cleans up faults abandoned by a previous process. @@ -51,8 +55,18 @@ public class DisruptionOrphanReconciler { @Inject ObjectMapper objectMapper; + // Built without config, as the tests build it, a Quorum is alone and always leads. @Inject - com.bmscomp.kates.quorum.Quorum quorum; + com.bmscomp.kates.quorum.Quorum quorum = new com.bmscomp.kates.quorum.Quorum(); + + @Inject + com.bmscomp.kates.engine.TestOrchestrator orchestrator; + + /** When this member took over, or this lone process started: its own faults are made after. */ + volatile Instant since; + + /** Whether an undo of the pass running now failed: its reports stay RUNNING until one succeeds. */ + private boolean undoFailed; @ConfigProperty(name = "kates.chaos.orphan-recovery.enabled", defaultValue = "true") boolean enabled; @@ -90,6 +104,7 @@ void onStart(@Observes StartupEvent event) { // // Recovery is cleanup of faults made before this process started. // Nothing about it needs to happen before the first request is served. + since = startedAt; Thread.ofVirtual().name("disruption-orphan-recovery").start(() -> reconcile(startedAt)); } @@ -100,20 +115,49 @@ void onStart(@Observes StartupEvent event) { */ void onTakeover(@Observes com.bmscomp.kates.quorum.LeadershipAcquired takeover) { if (enabled) { + since = takeover.at(); Thread.ofVirtual().name("disruption-orphan-recovery").start(() -> reconcile(takeover.at())); } } - private void reconcile(Instant startedAt) { + /** + * The leader keeps cleaning up, not only at takeover: a paused leader's late write carries an + * older term (J4 F1), and a Kubernetes API or database that was down at takeover is back later + * (F4, F8). Only faults of an older term or past their deadline are undone here, never by age. + */ + @Scheduled( + every = "${kates.quorum.reconcile-interval:10s}", + identity = "fault-reconcile", + concurrentExecution = Scheduled.ConcurrentExecution.SKIP) + void periodic() { + if (enabled && since != null && quorum.acceptsWrites()) { + reconcile(null); + } + } + + /** One pass: with {@code startedAt}, also every Kates fault made before it, as at start or takeover. */ + private synchronized void reconcile(Instant startedAt) { + undoFailed = false; try { int policies = reconcileNetworkPolicies(startedAt); + int engines = reconcileChaosEngines(startedAt); ScaleDownSnapshots.Restored scaled = reconcileScaleDowns(startedAt); - int reports = markInterruptedReports(startedAt); - if (policies + scaled.total() + reports > 0) { + if (orchestrator != null) { + try { + // After the faults: a database that hangs must not hold a partition (J4 review). + // Retried every pass, since a takeover with PostgreSQL down did not get to it. + orchestrator.recoverPreviousLeaders(); + } catch (RuntimeException e) { + LOG.debugf("The previous leader's runs are not recovered yet: %s", e.getMessage()); + } + } + // A report says INTERRUPTED only once its faults are out (s08b reported it with one in). + int reports = undoFailed ? 0 : markInterruptedReports(startedAt != null ? startedAt : since); + if (policies + engines + scaled.total() + reports > 0) { LOG.warnf( - "Orphan recovery: removed %d NetworkPolicy(ies), restored %d KafkaNodePool(s) and %d" - + " StatefulSet(s), marked %d report(s) INTERRUPTED", - policies, scaled.nodePools(), scaled.statefulSets(), reports); + "Orphan recovery: removed %d NetworkPolicy(ies) and %d ChaosEngine(s), restored %d" + + " KafkaNodePool(s) and %d StatefulSet(s), marked %d report(s) INTERRUPTED", + policies, engines, scaled.nodePools(), scaled.statefulSets(), reports); } } catch (Exception e) { // Never block startup on cleanup — the API must come up either way. @@ -121,13 +165,90 @@ private void reconcile(Instant startedAt) { } } + /** + * Whether a Kates fault's object is no live fault's: past its deadline, or written by a leader + * of an older term, or, in a pass with {@code ageCutoffMs}, made before it. Which is how a + * lone instance tells the previous process's faults from its own, and how objects from before + * the marks existed are judged. + */ + boolean isStray(ObjectMeta meta, Long ageCutoffMs, Instant now) { + Map marks = meta.getAnnotations() != null ? meta.getAnnotations() : Map.of(); + String deadline = marks.get(FaultMarks.DEADLINE); + if (deadline != null) { + try { + if (Instant.parse(deadline).isBefore(now)) { + return true; + } + } catch (java.time.format.DateTimeParseException ignored) { + // judged as if unmarked + } + } + String term = marks.get(FaultMarks.TERM); + if (term != null && quorum.memberId() != null) { + try { + if (Long.parseLong(term) < quorum.term()) { + return true; + } + } catch (NumberFormatException ignored) { + // judged as if unmarked + } + } + return ageCutoffMs != null && isOlderThan(meta.getCreationTimestamp(), ageCutoffMs); + } + + /** + * Deletes the Kates ChaosEngines no live fault owns, in every namespace, as Litmus cleanup does + * in-process. A node-drain's cordon is Litmus' to revert: Kates may not patch nodes. + */ + int reconcileChaosEngines(Instant startedAt) { + Long cutoff = startedAt == null ? null : cutoff(startedAt); + List engines; + try { + engines = kubeClient + .resources(ChaosEngine.class) + .inAnyNamespace() + .withLabel("managed-by", "kates") + .list() + .getItems(); + } catch (io.fabric8.kubernetes.client.KubernetesClientException e) { + // 404: no Litmus, so no engine Kates made. Anything else: whether one is in is unknown. + if (e.getCode() != 404) { + undoFailed = true; + } + LOG.debugf("Could not list ChaosEngines: %s", e.getMessage()); + return 0; + } + Instant now = Instant.now(); + int removed = 0; + for (ChaosEngine engine : engines) { + if (!isStray(engine.getMetadata(), cutoff, now)) { + continue; + } + String namespace = engine.getMetadata().getNamespace(); + String name = engine.getMetadata().getName(); + try { + kubeClient + .resources(ChaosEngine.class) + .inNamespace(namespace) + .withName(name) + .delete(); + removed++; + } catch (Exception e) { + undoFailed = true; + LOG.warnf("Could not undo ChaosEngine %s/%s: %s", namespace, name, e.getMessage()); + } + } + return removed; + } + /** * Deletes the Kates-managed NetworkPolicies (network partitions) left from * before this process started, in every namespace, as cleanup does: a * playbook can isolate pods outside the Kafka namespace. */ int reconcileNetworkPolicies(Instant startedAt) { - long cutoff = cutoff(startedAt); + Long cutoff = startedAt == null ? null : cutoff(startedAt); + Instant now = Instant.now(); List policies; try { policies = kubeClient @@ -142,11 +263,13 @@ int reconcileNetworkPolicies(Instant startedAt) { // (dev, unit tests) is normal and must not dump a stack trace on // every boot. LOG.warnf("Could not reconcile orphaned NetworkPolicies: %s", e.getMessage()); + // Whether a partition is still in is unknown: no report says INTERRUPTED on that. + undoFailed = true; return 0; } int removed = 0; for (NetworkPolicy policy : policies) { - if (!isOlderThan(policy.getMetadata().getCreationTimestamp(), cutoff)) { + if (!isStray(policy.getMetadata(), cutoff, now)) { continue; } String namespace = policy.getMetadata().getNamespace(); @@ -163,6 +286,7 @@ int reconcileNetworkPolicies(Instant startedAt) { LOG.warnf("Orphan recovery: deleted abandoned NetworkPolicy %s/%s", namespace, name); removed++; } catch (Exception e) { + undoFailed = true; LOG.warnf("Could not delete orphaned NetworkPolicy %s/%s: %s", namespace, name, e.getMessage()); } } @@ -176,8 +300,12 @@ int reconcileNetworkPolicies(Instant startedAt) { * it, which is exactly what makes recovery possible here. */ ScaleDownSnapshots.Restored reconcileScaleDowns(Instant startedAt) { - long cutoff = cutoff(startedAt); - return ScaleDownSnapshots.restore(kubeClient, kafkaNamespace, meta -> isScaleDownOlderThan(meta, cutoff)); + Long cutoff = startedAt == null ? null : cutoff(startedAt); + Instant now = Instant.now(); + return ScaleDownSnapshots.restoreEverywhere( + kubeClient, + kafkaNamespace, + meta -> isStray(meta, null, now) || (cutoff != null && isScaleDownOlderThan(meta, cutoff))); } private long cutoff(Instant startedAt) { diff --git a/kates/src/main/java/com/bmscomp/kates/engine/TestOrchestrator.java b/kates/src/main/java/com/bmscomp/kates/engine/TestOrchestrator.java index df1d032c..67827923 100644 --- a/kates/src/main/java/com/bmscomp/kates/engine/TestOrchestrator.java +++ b/kates/src/main/java/com/bmscomp/kates/engine/TestOrchestrator.java @@ -156,15 +156,47 @@ void recoverOrphans() { } /** - * Fails, before this member takes its first write as leader, the live runs any other member - * created: only a leader starts runs, so they are a previous leader's, and it is gone or has - * lost the lead. A failure here keeps the member from taking writes, and is tried again. + * At takeover, fails the live runs any other member created: only a leader starts runs, so they + * are a previous leader's, and it is gone or has lost the lead. A failure, such as PostgreSQL + * down, no longer keeps the member from leading (J4 F6): the reconciler tries again each pass. */ void recoverPreviousLeaders(@Observes LeadershipAcquired takeover) { + try { + recoverPreviousLeaders(); + } catch (RuntimeException e) { + LOG.warnf("Term %d: the previous leader's runs are not recovered yet, retried: %s", takeover.term(), e); + } + } + + /** Fails another member's live runs; with no quorum there is no other member, and nothing to do. */ + public void recoverPreviousLeaders() { + if (quorum.memberId() == null) { + return; + } recover( repository.findByStatusAndMember(ORPHAN_STATUSES, quorum.memberId(), false), "when the member running it stopped leading", - "takeover, term " + takeover.term()); + "takeover"); + // Its own too, once it no longer runs them: it settled them when it lost the lead, and + // leads again (J4 review). A run it still runs has its handles. + recover( + repository.findByStatusAndMember(ORPHAN_STATUSES, quorum.memberId(), true).stream() + .filter(run -> !activeHandles.containsKey(run.getId())) + .toList(), + "when the member running it stopped leading", + "takeover"); + } + + /** + * Stops this member's runs once it no longer leads: their load would otherwise go on hitting + * Kafka, counted by no leader's permits. Only settled here; the new leader stores them FAILED. + */ + void onLeadershipLost(@Observes com.bmscomp.kates.quorum.LeadershipLost lost) { + List runs = List.copyOf(activeHandles.keySet()); + if (!runs.isEmpty()) { + LOG.warnf("No longer leading term %d: stopping %d run(s)", lost.term(), runs.size()); + } + runs.forEach(this::settle); } private void recover(List orphans, String when, String by) { diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/LeadershipLost.java b/kates/src/main/java/com/bmscomp/kates/quorum/LeadershipLost.java new file mode 100644 index 00000000..6946a7c4 --- /dev/null +++ b/kates/src/main/java/com/bmscomp/kates/quorum/LeadershipLost.java @@ -0,0 +1,9 @@ +package com.bmscomp.kates.quorum; + +import java.time.Instant; + +/** + * This member led {@code term} and no longer does: another member leads, or none yet. It stops its + * own runs; undoing its faults is the next leader's. + */ +public record LeadershipLost(long term, Instant at) {} diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java index cbeb30f8..b9b2bdc9 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/Quorum.java @@ -52,6 +52,9 @@ public class Quorum { @Inject Event takeover; + @Inject + Event lost; + private volatile QuorumMember member; private volatile long recoveredTerm = -1; private QuorumMember.Settings settings; @@ -67,7 +70,7 @@ void start(@Observes StartupEvent event) throws IOException { org.apache.ratis.util.ExitUtils.disableSystemExit(); settings = QuorumMember.Settings.parse( peers.get(), memberId(), dir.orElse(null), cert.orElse(null), key.orElse(null), ca.orElse(null)); - member = QuorumMember.start(settings); + member = QuorumMember.start(settings, this::lostLead); LOG.infof( "quorum member %s started, %d members", settings.memberId(), settings.peers().size()); @@ -155,13 +158,17 @@ private void revive() { LOG.warnf( "quorum member %s: Ratis server %s, starting it again from its disk", current.memberId(), current.state()); + // Ratis closes a paused server without telling its state machine the lead changed. + if (recoveredTerm >= 0) { + lostLead(recoveredTerm); + } try { current.close(); } catch (IOException | RuntimeException e) { LOG.debugf("closing the old member: %s", e); } try { - member = QuorumMember.start(settings); + member = QuorumMember.start(settings, this::lostLead); notServingSince = null; recoveredTerm = -1; LOG.infof("quorum member %s serves again", settings.memberId()); @@ -170,6 +177,22 @@ private void revive() { } } + /** Off Ratis' thread: the observers stop this member's runs, which takes a while. */ + private void lostLead(long term) { + Event event = lost; + if (event == null) { + return; + } + LOG.warnf("Member %s no longer leads term %d", memberId(), term); + Thread.ofVirtual().name("quorum-lost-lead").start(() -> { + try { + event.fire(new LeadershipLost(term, Instant.now())); + } catch (RuntimeException e) { + LOG.warnf("Stopping this member's work after it lost the lead failed: %s", e); + } + }); + } + /** Since when this member's Ratis server has not been running; empty while it serves, or with no quorum. */ public Optional notServingSince() { return Optional.ofNullable(notServingSince); @@ -179,6 +202,25 @@ QuorumMember member() { return member; } + /** The term this member is in, as it last heard; {@code 0} with no quorum or while it does not serve. */ + public long term() { + QuorumMember current = member; + return current == null || !current.serving() ? 0 : current.term(); + } + + /** + * Throws unless this member may write now, as {@link #acceptsWrites()} decides, and returns the + * term it confirmed: called right before a fault goes into Kubernetes, so a member that stopped + * leading since its plan started injects nothing, and what it writes carries that term, not one + * read later. + */ + public long requireWrites() { + if (!acceptsWrites()) { + throw new com.bmscomp.kates.engine.NotLeaderException(leaderId()); + } + return term(); + } + /** The leader this member knows of, or null with no quorum or during an election. */ public String leaderId() { QuorumMember current = member; diff --git a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java index 2b3faeeb..557abc4a 100644 --- a/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java +++ b/kates/src/main/java/com/bmscomp/kates/quorum/QuorumMember.java @@ -142,6 +142,11 @@ private QuorumMember(RaftServer server, RaftClient client, RaftPeerId selfId, St /** Starts the member. Its term and vote are kept in {@code dir}, which must outlive a restart. */ public static QuorumMember start(Settings settings) throws IOException { + return start(settings, lostTerm -> {}); + } + + /** As {@link #start(Settings)}, telling {@code onLost} each time this member stops leading. */ + public static QuorumMember start(Settings settings, java.util.function.LongConsumer onLost) throws IOException { RaftPeer self = settings.self(); String address = self.getAddress(); RaftProperties properties = new RaftProperties(); @@ -166,13 +171,22 @@ public static QuorumMember start(Settings settings) throws IOException { .setMutualTls(true) .setSslProvider(SslProvider.JDK) .build()); + java.util.concurrent.atomic.AtomicReference started = + new java.util.concurrent.atomic.AtomicReference<>(); + Confirmations confirmations = new Confirmations( + self.getId(), + () -> { + QuorumMember member = started.get(); + return member == null ? 0 : member.term(); + }, + onLost); RaftGroup group = RaftGroup.valueOf(GROUP, settings.peers()); RaftServer server = RaftServer.newBuilder() .setServerId(self.getId()) .setGroup(group) .setProperties(properties) .setParameters(parameters) - .setStateMachine(new Confirmations()) + .setStateMachine(confirmations) .setOption(RaftStorage.StartupOption.RECOVER) .build(); try { @@ -191,7 +205,9 @@ public static QuorumMember start(Settings settings) throws IOException { .setParameters(parameters) .setRetryPolicy(RetryPolicies.noRetry()) .build(); - return new QuorumMember(server, client, self.getId(), settings.memberId()); + QuorumMember member = new QuorumMember(server, client, self.getId(), settings.memberId()); + started.set(member); + return member; } public String memberId() { @@ -270,11 +286,38 @@ public void close() throws IOException { } } - /** Replicates nothing yet; answers the reads that confirm leadership. */ - private static final class Confirmations extends BaseStateMachine { + /** + * Replicates nothing; answers the reads that confirm leadership, and tells {@code onLost} when + * this member stops leading, with the term it led. + */ + static final class Confirmations extends BaseStateMachine { + private final RaftPeerId self; + private final java.util.function.LongSupplier term; + private final java.util.function.LongConsumer onLost; + private boolean led; + private long ledTerm; + + Confirmations(RaftPeerId self, java.util.function.LongSupplier term, java.util.function.LongConsumer onLost) { + this.self = self; + this.term = term; + this.onLost = onLost; + } + @Override public CompletableFuture query(Message request) { return CompletableFuture.completedFuture(Message.EMPTY); } + + @Override + public synchronized void notifyLeaderChanged( + org.apache.ratis.protocol.RaftGroupMemberId member, RaftPeerId leader) { + if (self.equals(leader)) { + led = true; + ledTerm = term.getAsLong(); + } else if (led) { + led = false; + onLost.accept(ledTerm); + } + } } } diff --git a/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java b/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java index b9f010e3..5e3a99bb 100644 --- a/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java +++ b/kates/src/test/java/com/bmscomp/kates/chaos/KubernetesChaosProviderTest.java @@ -343,36 +343,7 @@ void stressTakesTheContainersUserWhereOnlyTheContainerSetsIt() throws Exception @Test void testScaleDownSnapshotsOriginalReplicas() throws ExecutionException, InterruptedException, TimeoutException { - // A Kafka not run by Strimzi: pods of a StatefulSet. - StatefulSet ss = new StatefulSetBuilder() - .withNewMetadata() - .withName("kafka") - .withNamespace("default") - .addToLabels("app", "kafka") - .endMetadata() - .withNewSpec() - .withReplicas(3) - .endSpec() - .build(); - client.apps().statefulSets().inNamespace("default").resource(ss).create(); - for (int i = 0; i < 3; i++) { - client.pods() - .inNamespace("default") - .resource(new PodBuilder() - .withNewMetadata() - .withName("kafka-" + i) - .withNamespace("default") - .addToLabels("app", "kafka") - .addNewOwnerReference() - .withApiVersion("apps/v1") - .withKind("StatefulSet") - .withName("kafka") - .withUid("sts-uid") - .endOwnerReference() - .endMetadata() - .build()) - .create(); - } + createKafkaStatefulSet(); FaultSpec spec = FaultSpec.builder("scale-down") .targetNamespace("default") @@ -990,4 +961,166 @@ void testCleanupNetworkPolicies() { void testIsAvailable() { assertTrue(provider.isAvailable()); } + + /** A Kafka not run by Strimzi: three pods of the StatefulSet {@code kafka} in {@code default}. */ + private void createKafkaStatefulSet() { + StatefulSet ss = new StatefulSetBuilder() + .withNewMetadata() + .withName("kafka") + .withNamespace("default") + .addToLabels("app", "kafka") + .endMetadata() + .withNewSpec() + .withReplicas(3) + .endSpec() + .build(); + client.apps().statefulSets().inNamespace("default").resource(ss).create(); + for (int i = 0; i < 3; i++) { + client.pods() + .inNamespace("default") + .resource(new PodBuilder() + .withNewMetadata() + .withName("kafka-" + i) + .withNamespace("default") + .addToLabels("app", "kafka") + .addNewOwnerReference() + .withApiVersion("apps/v1") + .withKind("StatefulSet") + .withName("kafka") + .withUid("sts-uid") + .endOwnerReference() + .endMetadata() + .build()) + .create(); + } + } + + private static com.bmscomp.kates.quorum.Quorum quorumAt(long term, boolean leads) { + com.bmscomp.kates.quorum.Quorum quorum = org.mockito.Mockito.mock(com.bmscomp.kates.quorum.Quorum.class); + org.mockito.Mockito.when(quorum.term()).thenReturn(term); + if (leads) { + org.mockito.Mockito.when(quorum.requireWrites()).thenReturn(term); + } else { + org.mockito.Mockito.doThrow(new com.bmscomp.kates.engine.NotLeaderException("m1")) + .when(quorum) + .requireWrites(); + } + return quorum; + } + + private static void assertMarked(java.util.Map annotations, String term) { + assertEquals(term, annotations.get(FaultMarks.TERM), "the term of the leader that wrote it"); + java.time.Instant deadline = java.time.Instant.parse(annotations.get(FaultMarks.DEADLINE)); + java.time.Instant expected = java.time.Instant.now().plusMillis(7_200_000); + assertTrue( + Math.abs(java.time.Duration.between(expected, deadline).toSeconds()) < 5, + "the engine's run bound from now: " + deadline); + } + + @Test + void aMemberThatDoesNotLeadInjectsNothing() throws Exception { + createZonedBrokers(); + createKafkaStatefulSet(); + provider.quorum = quorumAt(3, false); + requests(); + for (DisruptionType type : List.of( + DisruptionType.POD_KILL, + DisruptionType.NETWORK_PARTITION, + DisruptionType.SCALE_DOWN, + DisruptionType.ROLLING_RESTART, + DisruptionType.CPU_STRESS)) { + FaultSpec spec = type == DisruptionType.SCALE_DOWN + ? FaultSpec.builder("x") + .targetNamespace("default") + .targetLabel("app=kafka") + .disruptionType(type) + .build() + : FaultSpec.builder("x") + .targetPod("krafter-brokers-0") + .disruptionType(type) + .chaosDurationSec(1) + .build(); + + ChaosOutcome outcome = provider.triggerFault(spec).get(5, TimeUnit.SECONDS); + + assertFalse(outcome.isPass(), type + " went in"); + assertTrue(outcome.failureReason().contains("does not lead"), type + ": " + outcome.failureReason()); + } + List writes = + requests().stream().filter(r -> !r.startsWith("GET ")).toList(); + assertEquals(List.of(), writes, "a member that does not lead writes nothing"); + } + + @Test + void aPartitionCarriesItsTermAndDeadline() throws Exception { + createZonedBrokers(); + provider.quorum = quorumAt(7, true); + FaultSpec spec = FaultSpec.builder("split") + .targetPod("krafter-brokers-0") + .disruptionType(DisruptionType.NETWORK_PARTITION) + .chaosDurationSec(30) + .build(); + + provider.triggerFault(spec); + var policy = await(() -> client.network() + .networkPolicies() + .inNamespace("kafka") + .withName("kates-netpol-krafter-brokers-0") + .get()); + + assertMarked(policy.getMetadata().getAnnotations(), "7"); + } + + @Test + void aScaleDownCarriesItsTermAndDeadline() throws Exception { + createKafkaStatefulSet(); + provider.quorum = quorumAt(7, true); + FaultSpec spec = FaultSpec.builder("scale-down") + .targetNamespace("default") + .targetLabel("app=kafka") + .disruptionType(DisruptionType.SCALE_DOWN) + .build(); + + assertTrue(provider.triggerFault(spec).get(5, TimeUnit.SECONDS).isPass()); + + StatefulSet after = client.apps() + .statefulSets() + .inNamespace("default") + .withName("kafka") + .get(); + assertMarked(after.getMetadata().getAnnotations(), "7"); + } + + private static T await(java.util.function.Supplier found) throws InterruptedException { + for (int i = 0; i < 50; i++) { + T value = found.get(); + if (value != null) { + return value; + } + TimeUnit.MILLISECONDS.sleep(100); + } + throw new AssertionError("not within 5 s"); + } + + @Test + void aFaultIsMarkedWithTheTermItWasConfirmedIn() throws Exception { + createZonedBrokers(); + // Confirmed in term 7; by the time the object is built the member has heard of term 8. + provider.quorum = quorumAt(7, true); + org.mockito.Mockito.when(provider.quorum.term()).thenReturn(8L); + FaultSpec spec = FaultSpec.builder("split") + .targetPod("krafter-brokers-0") + .disruptionType(DisruptionType.NETWORK_PARTITION) + .chaosDurationSec(30) + .build(); + + provider.triggerFault(spec); + var policy = await(() -> client.network() + .networkPolicies() + .inNamespace("kafka") + .withName("kates-netpol-krafter-brokers-0") + .get()); + + assertMarked(policy.getMetadata().getAnnotations(), "7"); + } } diff --git a/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java b/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java index a2e8b7b1..1aebab32 100644 --- a/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java +++ b/kates/src/test/java/com/bmscomp/kates/chaos/LitmusChaosProviderTest.java @@ -171,6 +171,48 @@ void aFaultWhoseExperimentIsInstalledCreatesItsChaosEngine() throws Exception { assertTrue(outcome.isPass(), outcome.failureReason()); } + @Test + void anEngineCarriesItsTermAndDeadline() throws Exception { + install("disk-fill"); + provider.resultPollIntervalMs = 20; + com.bmscomp.kates.quorum.Quorum quorum = org.mockito.Mockito.mock(com.bmscomp.kates.quorum.Quorum.class); + when(quorum.requireWrites()).thenReturn(7L); + provider.quorum = quorum; + FaultSpec spec = FaultSpec.builder("fill") + .targetBrokerId(2) + .disruptionType(DisruptionType.DISK_FILL) + .build(); + + provider.triggerFault(spec); + ChaosEngine engine = awaitEngine(); + + assertEquals("7", engine.getMetadata().getAnnotations().get(FaultMarks.TERM)); + assertNotNull( + java.time.Instant.parse(engine.getMetadata().getAnnotations().get(FaultMarks.DEADLINE))); + } + + @Test + void anEngineIsNotCreatedByAFollower() throws Exception { + install("disk-fill"); + com.bmscomp.kates.quorum.Quorum quorum = org.mockito.Mockito.mock(com.bmscomp.kates.quorum.Quorum.class); + org.mockito.Mockito.doThrow(new com.bmscomp.kates.engine.NotLeaderException("m1")) + .when(quorum) + .requireWrites(); + provider.quorum = quorum; + FaultSpec spec = FaultSpec.builder("fill") + .targetBrokerId(2) + .disruptionType(DisruptionType.DISK_FILL) + .build(); + + ChaosOutcome outcome = provider.triggerFault(spec).get(5, TimeUnit.SECONDS); + + assertFalse(outcome.isPass()); + assertTrue(outcome.failureReason().contains("does not lead"), outcome.failureReason()); + assertEquals( + List.of(), + client.resources(ChaosEngine.class).inNamespace("kafka").list().getItems()); + } + private static String env(ChaosEngine engine, String name) { return engine.getSpec().experiments.getFirst().spec.components.env.stream() .filter(e -> e.name.equals(name)) diff --git a/kates/src/test/java/com/bmscomp/kates/config/NativeReflectionRegistryTest.java b/kates/src/test/java/com/bmscomp/kates/config/NativeReflectionRegistryTest.java index e9011474..c6913b16 100644 --- a/kates/src/test/java/com/bmscomp/kates/config/NativeReflectionRegistryTest.java +++ b/kates/src/test/java/com/bmscomp/kates/config/NativeReflectionRegistryTest.java @@ -72,6 +72,8 @@ class NativeReflectionRegistryTest { // serialised. "com.bmscomp.kates.chaos.K8sPodWatcher$WatchSession", "com.bmscomp.kates.chaos.ProbeRegistry", + // The annotation names and values a fault's objects carry; a constants holder. + "com.bmscomp.kates.chaos.FaultMarks", "com.bmscomp.kates.chaos.KafkaProbes", // Fault-targeting helpers; a FaultSpec carries its selector as a string. "com.bmscomp.kates.chaos.ParsedLabelSelector", diff --git a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuardScaleDownTest.java b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuardScaleDownTest.java index c090c94f..12b3b8fd 100644 --- a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuardScaleDownTest.java +++ b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionConcurrencyGuardScaleDownTest.java @@ -93,4 +93,16 @@ void aReleaseByAnotherTokenRestoresNothing() { assertEquals(2, get("brokers").getSpec().getReplicas()); } + + @Test + void aHolderThatNoLongerLeadsGivesNothingBack() { + String target = guard.currentTarget(); + guard.tryAcquire(target, "plan#1"); + scaledDown("brokers", System.currentTimeMillis()); + guard.quorum = org.mockito.Mockito.mock(com.bmscomp.kates.quorum.Quorum.class); + + guard.release(target, "plan#1"); + + assertEquals(2, get("brokers").getSpec().getReplicas(), "the next leader gives it back"); + } } diff --git a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrchestratorQuorumTest.java b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrchestratorQuorumTest.java new file mode 100644 index 00000000..8dc92165 --- /dev/null +++ b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrchestratorQuorumTest.java @@ -0,0 +1,105 @@ +package com.bmscomp.kates.disruption; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.Instant; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.jupiter.api.Test; + +import com.bmscomp.kates.chaos.ChaosCoordinator; +import com.bmscomp.kates.chaos.ChaosOutcome; +import com.bmscomp.kates.chaos.DisruptionType; +import com.bmscomp.kates.chaos.FaultSpec; +import com.bmscomp.kates.chaos.K8sPodWatcher; +import com.bmscomp.kates.chaos.StrimziStateTracker; +import com.bmscomp.kates.engine.KatesMetrics; +import com.bmscomp.kates.quorum.Quorum; + +/** A plan belongs to the term it started in, and a member that no longer leads undoes nothing. */ +class DisruptionOrchestratorQuorumTest { + + private final DisruptionOrchestrator orchestrator = new DisruptionOrchestrator(); + private final Quorum quorum = mock(Quorum.class); + private final AtomicLong term = new AtomicLong(4); + + DisruptionOrchestratorQuorumTest() { + orchestrator.chaosCoordinator = mock(ChaosCoordinator.class); + orchestrator.podWatcher = mock(K8sPodWatcher.class); + orchestrator.strimziTracker = mock(StrimziStateTracker.class); + orchestrator.intelligence = mock(KafkaIntelligenceService.class); + orchestrator.safetyGuard = mock(DisruptionSafetyGuard.class); + orchestrator.prometheusCapture = mock(PrometheusMetricsCapture.class); + orchestrator.slaGrader = mock(SlaGrader.class); + orchestrator.eventBus = mock(DisruptionEventBus.class); + orchestrator.katesMetrics = mock(KatesMetrics.class); + orchestrator.concurrencyGuard = new DisruptionConcurrencyGuard(); + orchestrator.quorum = quorum; + when(quorum.term()).thenAnswer(invocation -> term.get()); + when(quorum.acceptsWrites()).thenReturn(true); + when(orchestrator.safetyGuard.validatePlan(any())) + .thenReturn(DisruptionSafetyGuard.ValidationResult.ok(List.of())); + when(orchestrator.safetyGuard.verifyClusterState()).thenReturn(true); + } + + private static DisruptionPlan.DisruptionStep kill(String name, boolean requireRecovery) { + return new DisruptionPlan.DisruptionStep( + name, + FaultSpec.builder(name).disruptionType(DisruptionType.POD_KILL).build(), + 0, + 0, + requireRecovery); + } + + private static CompletableFuture passed() { + Instant now = Instant.now(); + return CompletableFuture.completedFuture( + ChaosOutcome.success("engine", "kill", now, now, System.nanoTime(), null, null, null)); + } + + @Test + void aPlanStopsOnceTheTermItStartedInEnds() { + when(orchestrator.podWatcher.startWatching(any(), any())).thenReturn(mock(K8sPodWatcher.WatchSession.class)); + when(orchestrator.chaosCoordinator.triggerFault(any(), any())).thenAnswer(invocation -> { + term.set(5); // the member lost the lead and won it back during the first step + return passed(); + }); + DisruptionPlan plan = new DisruptionPlan(); + plan.setName("two-kills"); + plan.getSteps().add(kill("first", false)); + plan.getSteps().add(kill("second", false)); + + orchestrator.execute("d-1", plan); + + verify(orchestrator.chaosCoordinator, times(1)).triggerFault(any(), any()); + } + + @Test + void aMemberThatNoLongerLeadsDoesNotRollBack() throws Exception { + K8sPodWatcher.WatchSession session = mock(K8sPodWatcher.WatchSession.class); + when(session.awaitRecovery(org.mockito.ArgumentMatchers.anyLong(), any())) + .thenReturn(false); + when(orchestrator.podWatcher.startWatching(any(), any())).thenReturn(session); + when(orchestrator.chaosCoordinator.triggerFault(any(), any())).thenAnswer(invocation -> { + invocation.getArgument(1).accept(System.nanoTime()); + // The member loses the lead while its fault is in. + when(quorum.acceptsWrites()).thenReturn(false); + return passed(); + }); + DisruptionPlan plan = new DisruptionPlan(); + plan.setName("unrecovered"); + plan.setAutoRollback(true); + plan.getSteps().add(kill("kill", true)); + + orchestrator.execute("d-2", plan); + + verify(orchestrator.safetyGuard, never()).rollback(any(), any()); + } +} diff --git a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrphanReconcilerTest.java b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrphanReconcilerTest.java index 1afcba70..8e7d1db4 100644 --- a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrphanReconcilerTest.java +++ b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionOrphanReconcilerTest.java @@ -113,4 +113,267 @@ private NetworkPolicy get(String namespace, String name) { .withName(name) .get(); } + + // ---- J4b: by term and deadline, every pass ---- + + private static final Instant NOW = Instant.parse("2026-11-15T12:00:00Z"); + + private com.bmscomp.kates.quorum.Quorum quorumAt(long term, boolean leads) { + com.bmscomp.kates.quorum.Quorum quorum = org.mockito.Mockito.mock(com.bmscomp.kates.quorum.Quorum.class); + org.mockito.Mockito.when(quorum.term()).thenReturn(term); + org.mockito.Mockito.when(quorum.memberId()).thenReturn("m0"); + org.mockito.Mockito.when(quorum.acceptsWrites()).thenReturn(leads); + return quorum; + } + + private static io.fabric8.kubernetes.api.model.ObjectMeta meta(String term, Instant deadline, Instant created) { + Map a = new java.util.HashMap<>(); + if (term != null) { + a.put(com.bmscomp.kates.chaos.FaultMarks.TERM, term); + } + if (deadline != null) { + a.put(com.bmscomp.kates.chaos.FaultMarks.DEADLINE, deadline.toString()); + } + return new io.fabric8.kubernetes.api.model.ObjectMetaBuilder() + .withAnnotations(a) + .withCreationTimestamp(created.toString()) + .build(); + } + + @org.junit.jupiter.params.ParameterizedTest + @org.junit.jupiter.params.provider.CsvSource( + nullValues = "-", + value = { + // term, deadline from NOW (s), created from cutoff (s), current term, in a quorum, age pass, stray + "3, -, 10, 4, true, false, true", + "4, -, 10, 4, true, false, false", + "4, -10, 10, 4, true, false, true", + "4, 3600, 10, 4, true, false, false", + "-, -, -10, 4, true, true, true", + "-, -, 10, 4, true, true, false", + "-, -, -10, 4, true, false, false", + "0, -, -10, 0, false, true, true", + "0, -, 10, 0, false, true, false", + "0, -, -10, 0, false, false, false", + }) + void strayRule( + String term, + Long deadlineIn, + long createdIn, + long current, + boolean inQuorum, + boolean agePass, + boolean stray) { + DisruptionOrphanReconciler reconciler = reconciler(); + reconciler.quorum = quorumAt(current, true); + if (!inQuorum) { + org.mockito.Mockito.when(reconciler.quorum.memberId()).thenReturn(null); + } + Instant cutoff = NOW.minusSeconds(60); + var meta = meta(term, deadlineIn == null ? null : NOW.plusSeconds(deadlineIn), cutoff.plusSeconds(createdIn)); + + assertEquals(stray, reconciler.isStray(meta, agePass ? cutoff.toEpochMilli() : null, NOW)); + } + + private NetworkPolicy marked(String namespace, String name, String term) { + return client.network() + .networkPolicies() + .inNamespace(namespace) + .resource(new NetworkPolicyBuilder() + .withNewMetadata() + .withName(name) + .withNamespace(namespace) + .withLabels(Map.of("managed-by", "kates")) + .withAnnotations(Map.of( + com.bmscomp.kates.chaos.FaultMarks.TERM, + term, + com.bmscomp.kates.chaos.FaultMarks.DEADLINE, + Instant.now().plusSeconds(3600).toString())) + .endMetadata() + .build()) + .create(); + } + + private DisruptionOrphanReconciler leaderAt(long term) { + DisruptionOrphanReconciler reconciler = reconciler(); + reconciler.enabled = true; + reconciler.quorum = quorumAt(term, true); + reconciler.repository = org.mockito.Mockito.mock(DisruptionReportRepository.class); + reconciler.since = Instant.now().minusSeconds(60); + return reconciler; + } + + @Test + void theCurrentTermsFaultIsKept() { + marked("kafka", "kates-netpol-a", "4"); + + leaderAt(4).periodic(); + + assertNotNull(get("kafka", "kates-netpol-a")); + } + + @Test + void anOlderTermsPartitionGoesOnTheNextPass() { + // A paused leader's late write: made after this member took over, with the old term. + marked("kafka", "kates-netpol-late", "3"); + + leaderAt(4).periodic(); + + assertNull(get("kafka", "kates-netpol-late")); + } + + @Test + void anOlderTermsEngineGoes() { + client.nodes() + .resource(new io.fabric8.kubernetes.api.model.NodeBuilder() + .withNewMetadata() + .withName("n1") + .endMetadata() + .withNewSpec() + .withUnschedulable(true) + .endSpec() + .build()) + .create(); + com.bmscomp.kates.chaos.litmus.ChaosEngine engine = new com.bmscomp.kates.chaos.litmus.ChaosEngine(); + engine.getMetadata().setName("kates-drain-1"); + engine.getMetadata().setNamespace("kafka"); + engine.getMetadata().setLabels(Map.of("managed-by", "kates")); + engine.getMetadata().setAnnotations(Map.of(com.bmscomp.kates.chaos.FaultMarks.TERM, "3")); + var spec = new com.bmscomp.kates.chaos.litmus.ChaosEngineSpec(); + var experiment = new com.bmscomp.kates.chaos.litmus.ChaosEngineSpec.Experiment(); + experiment.name = "node-drain"; + experiment.spec = new com.bmscomp.kates.chaos.litmus.ChaosEngineSpec.ExperimentSpec(); + experiment.spec.components = new com.bmscomp.kates.chaos.litmus.ChaosEngineSpec.Components(); + experiment.spec.components.env = + java.util.List.of(new com.bmscomp.kates.chaos.litmus.ChaosEngineSpec.EnvVar("TARGET_NODE", "n1")); + spec.experiments = java.util.List.of(experiment); + engine.setSpec(spec); + client.resources(com.bmscomp.kates.chaos.litmus.ChaosEngine.class) + .inNamespace("kafka") + .resource(engine) + .create(); + + leaderAt(4).periodic(); + + assertEquals( + java.util.List.of(), + client.resources(com.bmscomp.kates.chaos.litmus.ChaosEngine.class) + .inAnyNamespace() + .list() + .getItems()); + } + + @Test + void aScaleDownOutsideTheKafkaNamespaceIsRestored() { + client.apps() + .statefulSets() + .inNamespace("other") + .resource(new io.fabric8.kubernetes.api.model.apps.StatefulSetBuilder() + .withNewMetadata() + .withName("victim") + .withNamespace("other") + .withAnnotations(Map.of( + com.bmscomp.kates.chaos.KubernetesChaosProvider.ORIGINAL_REPLICAS_ANNOTATION, "3", + com.bmscomp.kates.chaos.FaultMarks.TERM, "3")) + .endMetadata() + .withNewSpec() + .withReplicas(2) + .endSpec() + .build()) + .create(); + + leaderAt(4).periodic(); + + assertEquals( + 3, + client.apps() + .statefulSets() + .inNamespace("other") + .withName("victim") + .get() + .getSpec() + .getReplicas()); + } + + @Test + void aFailedUndoLeavesReportsRunning() { + marked("kafka", "kates-netpol-stuck", "3"); + server.expect() + .delete() + .withPath("/apis/networking.k8s.io/v1/namespaces/kafka/networkpolicies/kates-netpol-stuck") + .andReturn(403, "forbidden") + .once(); + DisruptionOrphanReconciler reconciler = leaderAt(4); + + reconciler.periodic(); + org.mockito.Mockito.verify(reconciler.repository, org.mockito.Mockito.never()) + .findByStatusCreatedBefore( + org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any()); + + reconciler.periodic(); + assertNull(get("kafka", "kates-netpol-stuck")); + org.mockito.Mockito.verify(reconciler.repository) + .findByStatusCreatedBefore( + org.mockito.ArgumentMatchers.eq("RUNNING"), org.mockito.ArgumentMatchers.any()); + } + + @Test + void aFollowerReconcilesNothing() { + marked("kafka", "kates-netpol-old", "3"); + DisruptionOrphanReconciler reconciler = leaderAt(4); + org.mockito.Mockito.when(reconciler.quorum.acceptsWrites()).thenReturn(false); + int before = server.getRequestCount(); + + reconciler.periodic(); + + assertEquals(before, server.getRequestCount(), "a follower asks the API nothing"); + assertNotNull(get("kafka", "kates-netpol-old")); + } + + @Test + void aLoneInstanceKeepsItsOwnPartitionOnEveryPass() { + marked("kafka", "kates-netpol-mine", "0"); + DisruptionOrphanReconciler reconciler = leaderAt(0); + org.mockito.Mockito.when(reconciler.quorum.memberId()).thenReturn(null); + + reconciler.periodic(); + + assertNotNull( + get("kafka", "kates-netpol-mine"), "the periodic pass of a lone instance judges by deadline only"); + } + + @Test + void aFailedListLeavesReportsRunning() { + // The Kubernetes API refuses the list: whether a partition is still in is unknown. + server.expect() + .get() + .withPath("/apis/networking.k8s.io/v1/networkpolicies?labelSelector=managed-by%3Dkates") + .andReturn(403, "forbidden") + .once(); + DisruptionOrphanReconciler reconciler = leaderAt(4); + + reconciler.periodic(); + + org.mockito.Mockito.verify(reconciler.repository, org.mockito.Mockito.never()) + .findByStatusCreatedBefore( + org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any()); + } + + @Test + void aPassRemovesFaultsBeforeAskingTheDatabase() { + marked("kafka", "kates-netpol-late", "3"); + DisruptionOrphanReconciler reconciler = leaderAt(4); + reconciler.orchestrator = org.mockito.Mockito.mock(com.bmscomp.kates.engine.TestOrchestrator.class); + java.util.concurrent.atomic.AtomicBoolean goneFirst = new java.util.concurrent.atomic.AtomicBoolean(); + org.mockito.Mockito.doAnswer(invocation -> { + goneFirst.set(get("kafka", "kates-netpol-late") == null); + return null; + }) + .when(reconciler.orchestrator) + .recoverPreviousLeaders(); + + reconciler.periodic(); + + assertTrue(goneFirst.get(), "a database that hangs must not hold the partition"); + } } diff --git a/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java b/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java index d2d3fbb9..4a84fcab 100644 --- a/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java +++ b/kates/src/test/java/com/bmscomp/kates/engine/TestOrchestratorQuorumTest.java @@ -4,7 +4,6 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -69,7 +68,54 @@ void aTakeoverFailsAPreviousLeadersRuns() { orchestrator.recoverPreviousLeaders(new LeadershipAcquired(4, Instant.now())); assertFailedWith(previous, "when the member running it stopped leading"); - verify(engine.repository, never()).findByStatusAndMember(any(), anyString(), eq(true)); + } + + @Test + @DisplayName("a takeover with the database down still lets the member lead, and the recovery is retried") + void aTakeoverWithTheDatabaseDownTakesWrites() { + String previous = engine.storeEnded(TaskStatus.RUNNING); + when(engine.quorum.memberId()).thenReturn("kates-1"); + when(engine.repository.findByStatusAndMember(any(), eq("kates-1"), eq(false))) + .thenThrow(new IllegalStateException("PostgreSQL is down")) + .thenReturn(List.of(engine.rows.get(previous))); + + org.junit.jupiter.api.Assertions.assertDoesNotThrow( + () -> orchestrator.recoverPreviousLeaders(new LeadershipAcquired(4, Instant.now()))); + assertEquals(TaskStatus.RUNNING, engine.rows.get(previous).getStatus(), "not recovered yet"); + + orchestrator.recoverPreviousLeaders(); + + assertFailedWith(previous, "when the member running it stopped leading"); + } + + @Test + @DisplayName("a member that stops leading stops its own runs and gives their permits back") + void aDeposedMemberStopsItsRuns() { + String first = engine.startRunning(); + String second = engine.startRunning(); + + orchestrator.onLeadershipLost(new com.bmscomp.kates.quorum.LeadershipLost(4, Instant.now())); + + assertTrue(engine.backend.stopped.containsAll(InMemoryEngine.taskIds(first)), "first run's tasks stopped"); + assertTrue(engine.backend.stopped.containsAll(InMemoryEngine.taskIds(second)), "second run's tasks stopped"); + engine.assertEverySlotFree(); + } + + @Test + @DisplayName("a member that leads again fails its own runs it stopped when it lost the lead") + void aRegainedLeaderFailsItsOwnRunsItNoLongerRuns() { + String live = engine.startRunning(); + String stopped = engine.storeEnded(TaskStatus.RUNNING); + when(engine.quorum.memberId()).thenReturn("kates-1"); + when(engine.repository.findByStatusAndMember(any(), eq("kates-1"), eq(false))) + .thenReturn(List.of()); + when(engine.repository.findByStatusAndMember(any(), eq("kates-1"), eq(true))) + .thenReturn(List.of(engine.rows.get(live), engine.rows.get(stopped))); + + orchestrator.recoverPreviousLeaders(); + + assertFailedWith(stopped, "when the member running it stopped leading"); + assertEquals(TaskStatus.RUNNING, engine.rows.get(live).getStatus(), "a run this member still runs is left"); } private void assertFailedWith(String id, String reason) { diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java index 7dd44f40..decd65ea 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumMemberTest.java @@ -142,6 +142,24 @@ void aMemberWhoseServerClosesRejoinsAsAnotherTerm() throws Exception { await(() -> leader.memberId().equals(again.leaderId()) ? again : null); } + @Test + void losingTheLeadIsToldOnceWithTheTermItHeld() { + List lost = new ArrayList<>(); + long[] term = {5}; + var self = org.apache.ratis.protocol.RaftPeerId.valueOf("m0"); + var other = org.apache.ratis.protocol.RaftPeerId.valueOf("m1"); + QuorumMember.Confirmations confirmations = new QuorumMember.Confirmations(self, () -> term[0], lost::add); + + confirmations.notifyLeaderChanged(null, other); + assertEquals(List.of(), lost, "never led: nothing lost"); + confirmations.notifyLeaderChanged(null, self); + term[0] = 6; + confirmations.notifyLeaderChanged(null, null); + confirmations.notifyLeaderChanged(null, other); + + assertEquals(List.of(5L), lost, "told once, with the term it led"); + } + @Test void aMemberWithAnotherCasCertificateCannotJoin() throws Exception { TestPki pki = new TestPki(tmp.resolve("pki")); diff --git a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java index 9fc4a64b..bd369466 100644 --- a/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java +++ b/kates/src/test/java/com/bmscomp/kates/quorum/QuorumTest.java @@ -71,6 +71,21 @@ void aLeaderWhoseRecoveryFailsTakesNoWriteUntilItSucceeds() throws Exception { assertTrue(takeovers.get(takeovers.size() - 1).startsWith(leader.memberId() + "@")); } + @Test + void requireWritesRefusesOnAFollower() throws Exception { + start(3, new AtomicBoolean(false)); + Quorum leader = soleWriter(members); + + leader.requireWrites(); + assertEquals(((Number) leader.health().get("term")).longValue(), leader.term()); + for (Quorum member : members) { + if (member != leader) { + org.junit.jupiter.api.Assertions.assertThrows( + com.bmscomp.kates.engine.NotLeaderException.class, member::requireWrites); + } + } + } + @Test void aClosedMemberIsRestartedBySchedule() throws Exception { start(1, new AtomicBoolean(false)); @@ -153,11 +168,45 @@ private void start(int count, AtomicBoolean failing) throws Exception { quorum.key = Optional.of(files[1].toString()); quorum.ca = Optional.of(files[2].toString()); quorum.takeover = recorder(quorum, failing); + quorum.lost = lostRecorder(quorum); quorum.start(null); members.add(quorum); } } + /** Each LeadershipLost any member fired. */ + private final List losses = new CopyOnWriteArrayList<>(); + + @SuppressWarnings("unchecked") + private Event lostRecorder(Quorum quorum) { + Event event = mock(Event.class); + doAnswer(invocation -> { + losses.add(quorum.memberId() + "@" + + invocation.getArgument(0).term()); + return null; + }) + .when(event) + .fire(any()); + return event; + } + + @Test + void aLeaderWhoseServerClosedSaysItLostTheLead() throws Exception { + start(1, new AtomicBoolean(false)); + Quorum quorum = members.get(0); + soleWriter(members); + long term = quorum.term(); + + quorum.member().closeServer(); + quorum.takeOver(); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (losses.isEmpty() && System.nanoTime() < deadline) { + TimeUnit.MILLISECONDS.sleep(50); + } + assertEquals(List.of("m0@" + term), losses, "its runs are stopped like any deposed leader's"); + } + @SuppressWarnings("unchecked") private Event recorder(Quorum quorum, AtomicBoolean failing) { Event event = mock(Event.class);