Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions cli/cmd/mcp_caveats.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ var mcpCaveatsCore = []mcpCaveat{
Short: "A LOAD run is one producer and one consumer whatever numProducers says; only STRESS runs parallel producers.",
Text: "A LOAD run is one producer and one consumer, whatever numProducers and numConsumers say, " +
"so it cannot show how the cluster behaves under parallel clients. STRESS starts one producer per numProducers.",
Refs: []string{mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1796-1805", "case LOAD ->", "case STRESS ->")},
Refs: []string{mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1828-1837", "case LOAD ->", "case STRESS ->")},
},
{
ID: mcpCaveatReaperDeadline,
Expand All @@ -99,7 +99,7 @@ var mcpCaveatsCore = []mcpCaveat{
"kates/src/main/resources/application.properties:336-350",
mcpAnchoredRef(mcpJava+"engine/TestTimeoutReaper.java:49-51,67-76,170-173,201-214",
"TestResult.TaskStatus.STOPPING", "kates.engine.reaper-grace-ms", "\"-submission\"", "\"Timeout: still \""),
mcpJava + "engine/TestOrchestrator.java:151-217,259-262,436,1251-1315,1381-1390,1445-1477",
mcpJava + "engine/TestOrchestrator.java:151-249,291-294,468,1283-1347,1413-1422,1477-1509",
mcpJava + "persistence/EntityMapper.java:161-170",
"kates/src/main/resources/db/migration/V24__run_planned_duration.sql:1-13",
mcpJava + "domain/TestRun.java:520-523",
Expand All @@ -123,10 +123,10 @@ var mcpCaveatsCore = []mcpCaveat{
"integrity-cg-integrity, without transactions.",
Refs: []string{
mcpJava + "domain/TestSpec.java:14-32,111-113",
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:228,251-253,301",
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:260,283-285,333",
"applyTypeDefaults(type, request.getSpec())", ".withRequestedSpec(", "buildTasks(type, spec, run.getId())"),
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1171-1217", "TestSpec applyTypeDefaults(", "return merged;"),
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1775-1779,1870-1874",
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1203-1249", "TestSpec applyTypeDefaults(", "return merged;"),
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1807-1811,1902-1906",
"Only when asked", `"-integrity-0-group")`),
mcpAnchoredRef(mcpJava+"engine/NativeKafkaBackend.java:792", `task.getConsumerGroup() + "-integrity"`),
mcpAnchoredRef(mcpJava+"persistence/EntityMapper.java:56,85-86,146-148",
Expand Down Expand Up @@ -177,7 +177,7 @@ var mcpCaveatsCore = []mcpCaveat{
Text: "A TUNE_* run executes one produce task with the spec's single configuration: it measures one setting " +
"and compares none.",
Refs: []string{
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1881-1882", "case TUNE_REPLICATION,", "-tune-0"),
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1913-1914", "case TUNE_REPLICATION,", "-tune-0"),
},
},
{
Expand Down Expand Up @@ -241,7 +241,7 @@ var mcpCaveatsCore = []mcpCaveat{
Refs: []string{
mcpAnchoredRef("kates/src/main/resources/application.properties:274-275",
"# Chaos coordination (noop | kubernetes | litmus-crd | hybrid)", "kates.chaos.provider=litmus-crd"),
mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:63,108",
mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:63,117",
"int resultPollIntervalMs = 5_000;", "Instant start = Instant.now();"),
},
},
Expand Down
25 changes: 13 additions & 12 deletions cli/cmd/mcp_tools_chaos.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ var mcpCaveatsChaos = []mcpCaveat{
mcpAnchoredRef(mcpJava+"disruption/DisruptionSafetyGuard.java:298-301,874-943",
"boolean canExecute = checkRbacPermissions(spec);", `"Insufficient RBAC permissions for "`,
"boolean checkRbacPermissions(FaultSpec spec)", `"RBAC check failed, assuming permitted"`),
mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:42-60,87-89,270",
mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:42-60,96-98,284",
"ON_KUBERNETES_API = EnumSet.of(", "DisruptionType.POD_DELETE,", "DisruptionType.NETWORK_PARTITION);",
"return kubernetes.triggerFault(spec, onInject);", `engineSpec.chaosServiceAccount = "litmus-admin";`),
mcpAnchoredRef("kates/src/main/resources/application.properties:274-275",
Expand Down Expand Up @@ -226,8 +226,8 @@ var mcpCaveatsChaos = []mcpCaveat{
"if (selected != null && selected.isAvailable())", "new NoOpChaosProvider()", "matches no chaos provider"),
mcpJava + "chaos/NoOpChaosProvider.java:27-32",
mcpJava + "chaos/ChaosOutcome.java:77-81",
mcpJava + "disruption/DisruptionOrchestrator.java:157-160,221",
mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:347-351",
mcpJava + "disruption/DisruptionOrchestrator.java:170-173,234",
mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:360-364",
"boolean injected = disruptionStart != null;", "The fault never went in"),
mcpJava + "disruption/DisruptionImpactScorer.java:72-77",
},
Expand All @@ -242,7 +242,7 @@ var mcpCaveatsChaos = []mcpCaveat{
"scores as degradation. Without Prometheus both are 0. Judge latency and throughput from the summary " +
"and the steps' impactDeltas instead.",
Refs: []string{
mcpJava + "disruption/DisruptionOrchestrator.java:184-189,200-201",
mcpJava + "disruption/DisruptionOrchestrator.java:197-202,213-214",
mcpJava + "disruption/PrometheusMetricsCapture.java:152-171",
mcpJava + "disruption/DisruptionImpactScorer.java:92-130",
},
Expand All @@ -264,7 +264,7 @@ var mcpCaveatsChaos = []mcpCaveat{
"measured them for a step whose fault never went in as well; a report does not say which of the two counted " +
"its times.",
Refs: []string{
mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:302,313-315,347-356,393-399,503-509",
mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:315,326-328,360-369,406-412,518-524",
"new RecoveryClocks(session, isrTracker, lagTracker)", ".triggerFault(spec, clocks::start)",
"boolean injected = disruptionStart != null;", "if (step.requireRecovery() && injected)",
"Duration strimziRecovery = injected", "injected && isrTracker != null ? isrTracker.stop() : null",
Expand All @@ -275,14 +275,14 @@ var mcpCaveatsChaos = []mcpCaveat{
mcpAnchoredRef(mcpJava+"disruption/KafkaIntelligenceService.java:125-127,198-217,241-246,310-324",
"public void markDisruptionStart(Instant at)", "private Duration computeTimeToFullIsr()",
"this.baselineLag = timeline.getLast().totalLag();", "private Duration computeTimeToLagRecovery(long baseLag)"),
mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:87-99,108-127",
mcpAnchoredRef(mcpJava+"chaos/LitmusChaosProvider.java:96-108,117-141",
"return kubernetes.triggerFault(spec, onInject);",
"CompletableFuture.delayedExecutor(spec.delayBeforeSec(), TimeUnit.SECONDS, executor.get());",
"long startNanos = System.nanoTime();", "buildChaosEngine(spec, engineName, experimentName)",
"onInject.accept(startNanos);", ".resource(engine)"),
mcpAnchoredRef(mcpJava+"chaos/KubernetesChaosProvider.java:152-161",
mcpAnchoredRef(mcpJava+"chaos/KubernetesChaosProvider.java:170-182",
"Thread.sleep(spec.delayBeforeSec() * 1000L);", "onInject.accept(startNanos);",
"applyDisruption(spec, engineName);"),
"applyDisruption(spec, engineName, marks);"),
mcpAnchoredRef(mcpJava+"chaos/PodTargets.java:58,69-75",
"public static List<String> resolve(KubernetesClient client, FaultSpec spec)",
"throw new IllegalStateException("),
Expand All @@ -299,7 +299,7 @@ var mcpCaveatsChaos = []mcpCaveat{
Refs: []string{
mcpJava + "disruption/KafkaIntelligenceService.java:129-154,165-168",
mcpJava + "disruption/DisruptionImpactScorer.java:42-45,132-157",
mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:266-269,398",
mcpAnchoredRef(mcpJava+"disruption/DisruptionOrchestrator.java:279-282,411",
"intelligence.startIsrTracking(", "isrTracker.stop()"),
},
},
Expand All @@ -310,12 +310,13 @@ var mcpCaveatsChaos = []mcpCaveat{
"stopped mid-plan and has not started again: such a plan never finishes. When the backend starts, it " +
"marks each report created before then that still says RUNNING as INTERRUPTED, in the report and in " +
"the list of disruptions alike, unless kates.chaos.orphan-recovery.enabled is false. A backend in a " +
"quorum (kates.quorum.peers) marks nothing when it starts: the next member to lead marks each report " +
"created before it took the lead. Without a quorum this assumes one backend replica: a replica that " +
"quorum (kates.quorum.peers) marks nothing when it starts: the member that leads marks each report " +
"created before it took the lead, every 10 s until PostgreSQL answers, and only in a pass that removed " +
"every fault it found of an older term or past its deadline. Without a quorum this assumes one backend replica: a replica that " +
"starts while another runs a plan marks that plan's report too, until the plan ends and stores its " +
"outcome.",
Refs: []string{
mcpJava + "disruption/DisruptionOrphanReconciler.java:57-58,66-77,93-111,189-240",
mcpJava + "disruption/DisruptionOrphanReconciler.java:71-75,80-109,116-166,334-368",
mcpJava + "disruption/DisruptionReportRepository.java:35-60,93-102",
mcpJava + "disruption/DisruptionPersistence.java:26-37",
mcpJava + "disruption/DisruptionLauncher.java:114-121,129-144",
Expand Down
4 changes: 2 additions & 2 deletions cli/cmd/mcp_tools_runs_activity.go
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,7 @@ func mcpActivityTestsSince(ctx context.Context, call *mcpCall, since time.Time,
// are injected for part of its run, or one whose backend process stopped
// before it ended and has not started again to mark it INTERRUPTED
// (DisruptionLauncher.java:95-123, DisruptionReportRepository.java:35-43,
// DisruptionOrphanReconciler.java:189-240).
// DisruptionOrphanReconciler.java:359-393).
func mcpActivityDisruptionsSince(ctx context.Context, call *mcpCall, since time.Time, page mcpActivityPage) (mcpActivityDisruptions, error) {
out := mcpActivityDisruptions{StoredAsRunning: []mcpActivityDisruptionRow{}, Since: []mcpActivityDisruptionRow{}}
for p := 0; p < mcpActivityMaxPages; p++ {
Expand Down Expand Up @@ -486,7 +486,7 @@ var mcpCaveatsRunsActivity = []mcpCaveat{
mcpJava + "disruption/DisruptionLauncher.java:114-144",
mcpJava + "disruption/DisruptionPersistence.java:16-37",
mcpJava + "disruption/DisruptionReportRepository.java:20-43",
mcpJava + "disruption/DisruptionOrphanReconciler.java:189-240",
mcpJava + "disruption/DisruptionOrphanReconciler.java:334-368",
mcpJava + "disruption/DisruptionTemplateResource.java:70-72",
mcpJava + "disruption/DisruptionScheduler.java:112-124",
mcpJava + "disruption/DisruptionReportEntity.java:39-47",
Expand Down
10 changes: 5 additions & 5 deletions cli/cmd/mcp_tools_runs_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -1032,7 +1032,7 @@ var mcpCaveatsRunsList = []mcpCaveat{
mcpAnchoredRef(mcpJava+"report/ReportGenerator.java:220-249,283-294",
"report.setSummary(MetricUtils.computeSummary(results));",
"summaries.put(phase, MetricUtils.computeSummary(rows))"),
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:505-514,2235-2251",
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:537-546,2267-2283",
"TestResult.TaskStatus.PENDING : TestResult.TaskStatus.RUNNING",
"private static TestRun withUnfinishedTasksFailed("),
},
Expand Down Expand Up @@ -1071,7 +1071,7 @@ var mcpCaveatsRunsList = []mcpCaveat{
"except for a run cancelled before its tasks existed; a cancel through the REST API also leaves a CANCEL " +
"audit row.",
Refs: []string{
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:2182-2264",
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:2214-2296",
`"Cancelled by user"`, `EventKind.FAILED, "cancelled"`, "return run.withResults(updatedResults);"),
mcpAnchoredRef(mcpJava+"api/TestResource.java:572-617",
`@Path("/{id}/cancel")`, `auditService.record("CANCEL"`, `"Test cancelled; it is stored as FAILED"`),
Expand Down Expand Up @@ -1104,14 +1104,14 @@ var mcpCaveatsRunsList = []mcpCaveat{
"say which the API is, but a scenario run's tasks do: on an older API they all start within moments " +
"of one another.",
Refs: []string{
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:422-424,481-492,494-510",
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:454-456,513-524,526-542",
"applyTypeDefaults(type, scenario.getBaseSpec())", ".withRequestedSpec(",
"long phaseStartMs = System.currentTimeMillis();", "scenario.resolveSpecForPhase(phase, baseSpec)",
"phaseStartMs = saturatedSum(phaseStartMs", "backend.submit(task)", "TestResult.TaskStatus.PENDING"),
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1583-1611,1624-1632,1692-1731,1741-1753",
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1615-1643,1656-1664,1724-1763,1773-1785",
"MAX_RAMP_STEPS = 100", "scenarioInapplicableFields(TestScenario scenario", "is a SPIKE phase",
"has none (", "check no record CRCs", "has the name of an earlier phase"),
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1878-1947", "List<BenchmarkTask> buildPhaseTask(",
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1910-1979", "List<BenchmarkTask> buildPhaseTask(",
"int baseTarget = Math.max(1, spec.getThroughput() / steps);", "saturatedSum(startAtMs, s * stepMs)",
`taskId + "-spike"`),
mcpAnchoredRef(mcpJava+"domain/TestScenario.java:115-190",
Expand Down
4 changes: 2 additions & 2 deletions cli/cmd/mcp_tools_scenario.go
Original file line number Diff line number Diff line change
Expand Up @@ -1242,7 +1242,7 @@ var mcpCaveatsScenario = []mcpCaveat{
"plans/mcp-server.md:387-405",
mcpJava + "api/TestResource.java:118-188",
mcpJava + "domain/TestSpec.java:34-113",
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1503-1578",
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1535-1610",
"Map<String, String> inapplicableFields(", "the trogdor backend cannot run a transactional producer"),
mcpAnchoredRef("cli/cmd/apply.go:238-307",
"for i, scenario := range sf.Scenarios {", "apiClient.CreateTest(a.ctx, scenarioToRequest(scenario))",
Expand All @@ -1259,7 +1259,7 @@ var mcpCaveatsScenario = []mcpCaveat{
"listed as defaulted may differ on the cluster that runs the scenario.",
Refs: []string{
mcpJava + "config/TestTypeDefaults.java:22-64,323-464",
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1171-1217", "TestSpec applyTypeDefaults(", "return merged;"),
mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1203-1249", "TestSpec applyTypeDefaults(", "return merged;"),
mcpAnchoredRef("kates/src/main/resources/application.properties:73-114",
"# Per-type overrides: STRESS", "kates.tests.roundtrip.throughput=10000"),
mcpAnchoredRef("charts/kates/values.yaml:690-758", "defaults:", "tests:", `throughput: "10000"`),
Expand Down
2 changes: 1 addition & 1 deletion docs/book/11-api-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ curl -s "$BASE/api/health"

The response also contains a `tests` object with the resolved default configuration for every test type. When Kafka is unreachable, the endpoint still returns `200 OK`, but with `"status": "DEGRADED"` and `"kafka.status": "DOWN"`.

`quorum` says whether this Kates is a member of a quorum, which it is only when `kates.quorum.peers` is set. Without one it is `{"enabled": false}`, and the instance acts alone. A member also reports `state`, its Raft server's life cycle (`RUNNING` while it serves). Ratis closes a member's server after a pause longer than 60 s, such as a long garbage collection or a frozen node; the member then starts it again from its directory within a few seconds, and until it serves again only `state` and `memberId` are reported. If it still does not serve after `kates.quorum.liveness-after` (30 s), `/q/health/live` answers DOWN so the kubelet restarts the pod. A member reports its `memberId` and, while it serves, the `leaderId` it knows of (`null` during an election), the Raft `term`, `appliedIndex`, the last log entry it applied, and `leader`: true once it leads and has committed an entry of its term. A member whose log is empty, new or with its directory lost, is no voter until it has caught up to an entry: three members of which one is empty and one is down elect nobody. Members talk to each other over mutual TLS on their own port: `kates.quorum.tls.cert`, `kates.quorum.tls.key` (PKCS#8 PEM) and `kates.quorum.tls.ca` are required, and so is `kates.quorum.dir`, which keeps a member's term and vote and must outlive a restart. Only the leader takes writes: on any other member, every request but a read is answered `503` with `Retry-After`, and its message names the leader. `PUT /api/drain` is the exception, since a drain is set on each member. Before its first write in a term, a new leader marks `FAILED` the live runs every other member created, and removes the faults Kates made before it took the lead; a member that starts up recovers only its own runs. A leader that loses its majority stops taking writes once its lease runs out, within about a second, even before it steps down. A request that needs a database Kates cannot reach is answered `503` with `Retry-After: 5` within about 30 s, the JDBC socket timeout, instead of waiting on the connection.
`quorum` says whether this Kates is a member of a quorum, which it is only when `kates.quorum.peers` is set. Without one it is `{"enabled": false}`, and the instance acts alone. A member also reports `state`, its Raft server's life cycle (`RUNNING` while it serves). Ratis closes a member's server after a pause longer than 60 s, such as a long garbage collection or a frozen node; the member then starts it again from its directory within a few seconds, and until it serves again only `state` and `memberId` are reported. If it still does not serve after `kates.quorum.liveness-after` (30 s), `/q/health/live` answers DOWN so the kubelet restarts the pod. A member reports its `memberId` and, while it serves, the `leaderId` it knows of (`null` during an election), the Raft `term`, `appliedIndex`, the last log entry it applied, and `leader`: true once it leads and has committed an entry of its term. A member whose log is empty, new or with its directory lost, is no voter until it has caught up to an entry: three members of which one is empty and one is down elect nobody. Members talk to each other over mutual TLS on their own port: `kates.quorum.tls.cert`, `kates.quorum.tls.key` (PKCS#8 PEM) and `kates.quorum.tls.ca` are required, and so is `kates.quorum.dir`, which keeps a member's term and vote and must outlive a restart. Only the leader takes writes: on any other member, every request but a read is answered `503` with `Retry-After`, and its message names the leader. `PUT /api/drain` is the exception, since a drain is set on each member. A member checks that it still leads right before each fault goes into Kubernetes, so a member that stopped leading since its plan started injects nothing, and every object a fault writes carries `kates.io/term`, the writer's term, and `kates.io/deadline`, the write's time plus `kates.engine.max-duration-ms`. The leader reconciles every `kates.quorum.reconcile-interval` (10 s): it removes any Kates fault of an older term or past its deadline, NetworkPolicies, ChaosEngines and scale-downs in every namespace alike, and marks `FAILED` the live runs every other member created, retrying while PostgreSQL is unreachable rather than waiting on it to lead. A member that stops leading stops its own runs at once; the new leader stores them `FAILED`. A member that starts up recovers only its own runs. A leader that loses its majority stops taking writes once its lease runs out, within about a second, even before it steps down. A request that needs a database Kates cannot reach is answered `503` with `Retry-After: 5` within about 30 s, the JDBC socket timeout, instead of waiting on the connection.

#### GET /api/drain

Expand Down
23 changes: 23 additions & 0 deletions kates/src/main/java/com/bmscomp/kates/chaos/FaultMarks.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package com.bmscomp.kates.chaos;

import java.time.Instant;
import java.util.Map;

/**
* What every object a fault creates or changes carries, so that a later leader can tell a fault
* of an older term, or one past its deadline, from the faults it runs itself.
*/
public final class FaultMarks {

/** The Ratis term of the leader that wrote the object; {@code 0} with no quorum. */
public static final String TERM = "kates.io/term";

/** The instant after which any leader undoes the fault, whoever made it. */
public static final String DEADLINE = "kates.io/deadline";

private FaultMarks() {}

public static Map<String, String> of(long term, Instant deadline) {
return Map.of(TERM, Long.toString(term), DEADLINE, deadline.toString());
}
}
Loading
Loading