From ddac746aa236191a81580576d447afe54c7049da Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Sun, 15 Nov 2026 22:19:59 +0100 Subject: [PATCH] feat(operator)!: remove KatesTest and KafkaCompliance O3b of plans/operator-kates-kind.md: 0.2.0 deprecated both run kinds, and 0.3.0 removes them. The operator now reconciles the Kates kind alone. Gone with them: both reconcilers and CRDs, their examples and e2e.sh; the kates-connection Secret and the connection.fallback mount; the Strimzi Kafka watch, kafka.namespaces, rbac.kafkaNamespaceRole and the kafka-reader Role; the kates_operator_tests and kates_operator_compliance_rules gauges; the client calls only they made (create, get, cancel, report, compliance, capabilities); the deprecation warning at start; the reconcile-at annotation, which only they read. BREAKING CHANGE: values setting kafka, connection or rbac.kafkaNamespaceRole are refused by the chart's schema. Delete every KatesTest and KafkaCompliance while 0.2.x runs, since 0.3.0 no longer clears a live KatesTest's finalizer, then upgrade, then delete the two CRDs. The chart README's "Upgrading to 0.3.0" gives the order. --- .github/workflows/ci-operator.yml | 14 +- operator/chart/README.md | 123 +-- .../chart/crds/kafkacompliances.kates.io.yaml | 194 ----- operator/chart/crds/katestests.kates.io.yaml | 565 -------------- operator/chart/templates/NOTES.txt | 17 +- operator/chart/templates/_helpers.tpl | 8 +- operator/chart/templates/aggregate-roles.yaml | 4 +- operator/chart/templates/crd-upgrade.yaml | 2 +- operator/chart/templates/deployment.yaml | 19 - operator/chart/templates/kafka-reader.yaml | 34 - operator/chart/templates/networkpolicy.yaml | 2 +- operator/chart/templates/rbac.yaml | 8 - operator/chart/values.schema.json | 181 +++-- operator/chart/values.yaml | 22 +- operator/crates/kates-client/src/lib.rs | 336 +------- .../crates/kates-crds/src/kafkacompliance.rs | 101 --- operator/crates/kates-crds/src/katestest.rs | 157 ---- operator/crates/kates-crds/src/lib.rs | 24 +- operator/crates/kates-crds/src/wire.rs | 100 --- .../kates-crds/tests/kafkacompliance.rs | 42 - operator/crates/kates-crds/tests/katestest.rs | 340 -------- .../crates/kates-operator/src/connection.rs | 186 +---- operator/crates/kates-operator/src/context.rs | 31 - operator/crates/kates-operator/src/crds.rs | 4 +- operator/crates/kates-operator/src/lib.rs | 2 +- operator/crates/kates-operator/src/metrics.rs | 106 +-- .../src/reconcile/compliance.rs | 621 --------------- .../kates-operator/src/reconcile/mod.rs | 19 - .../kates-operator/src/reconcile/test.rs | 732 ------------------ operator/crates/kates-operator/src/run.rs | 220 +----- operator/e2e-stack.sh | 11 +- operator/e2e.sh | 119 --- operator/examples/compliance.yaml | 13 - operator/examples/load-test.yaml | 14 - 34 files changed, 232 insertions(+), 4139 deletions(-) delete mode 100644 operator/chart/crds/kafkacompliances.kates.io.yaml delete mode 100644 operator/chart/crds/katestests.kates.io.yaml delete mode 100644 operator/chart/templates/kafka-reader.yaml delete mode 100644 operator/crates/kates-crds/src/kafkacompliance.rs delete mode 100644 operator/crates/kates-crds/src/katestest.rs delete mode 100644 operator/crates/kates-crds/src/wire.rs delete mode 100644 operator/crates/kates-crds/tests/kafkacompliance.rs delete mode 100644 operator/crates/kates-crds/tests/katestest.rs delete mode 100644 operator/crates/kates-operator/src/reconcile/compliance.rs delete mode 100644 operator/crates/kates-operator/src/reconcile/test.rs delete mode 100755 operator/e2e.sh delete mode 100644 operator/examples/compliance.yaml delete mode 100644 operator/examples/load-test.yaml diff --git a/.github/workflows/ci-operator.yml b/.github/workflows/ci-operator.yml index 16cdb328..d23ccbe5 100644 --- a/.github/workflows/ci-operator.yml +++ b/.github/workflows/ci-operator.yml @@ -34,7 +34,6 @@ jobs: rust: - 'operator/**' - 'charts/kates-operator/**' - - 'schemas/test-request.v1.schema.json' - 'rust-toolchain.toml' - 'versions.env' - '.github/workflows/ci-operator.yml' @@ -109,16 +108,16 @@ jobs: set -euo pipefail schemas="${RUNNER_TEMP}/schemas" mkdir -p "${schemas}" - yq -o=json '.spec.versions[0].schema.openAPIV3Schema' operator/chart/crds/katestests.kates.io.yaml \ + yq -o=json '.spec.versions[0].schema.openAPIV3Schema' operator/chart/crds/kates.kates.io.yaml \ | jq '.properties.spec |= walk(if type == "object" and (.properties | type) == "object" then . + {additionalProperties: false} else . end)' \ - > "${schemas}/katestest_v1alpha1.json" + > "${schemas}/kates_v1alpha1.json" validate() { - printf 'apiVersion: kates.io/v1alpha1\nkind: KatesTest\nmetadata: {name: ci}\nspec: %s\n' "$1" \ + printf 'apiVersion: kates.io/v1alpha1\nkind: Kates\nmetadata: {name: ci}\nspec: %s\n' "$1" \ | kubeconform -strict -summary -schema-location "${schemas}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json" - } - validate '{type: LOAD, spec: {numRecords: 1, acks: all}, sla: {maxP99LatencyMs: 100}}' - if validate '{type: LOAD, spec: {records: 1}}'; then + validate '{version: "1.40.0", database: {secretRef: {name: db}}, kafka: {bootstrapServers: "k:9092"}}' + if validate '{version: "1.40.0", database: {secretRef: {name: db}}, kafka: {bootstrapServers: "k:9092"}, replica: 3}'; then echo "::error::kubeconform accepted a mistyped key" exit 1 fi @@ -193,9 +192,6 @@ jobs: - name: Deploy Kafka, Kates and the operator run: operator/e2e-stack.sh docker.io/library/kates-operator:ci - - name: Run the e2e checks - run: operator/e2e.sh payments kafka krafter - - name: A Kates resource becomes a backend that runs a test run: operator/e2e-kates.sh payments diff --git a/operator/chart/README.md b/operator/chart/README.md index 51bd8934..a4c3e547 100644 --- a/operator/chart/README.md +++ b/operator/chart/README.md @@ -1,10 +1,8 @@ # kates-operator -Kates runs and compliance checks as Kubernetes resources. `KatesTest` is one run and `KafkaCompliance` a standing check. A team can commit `loadtest.yaml`, watch it with `kubectl get katestests`, and gate a pipeline on it. +A `Kates` resource installs one Kates backend, or with `replicas: 3` or `5` a quorum of them that keeps a leader when a member is lost. The operator upgrades it without losing a live run, and removes what it made when the resource is deleted. -The operator drives a Kates API through its REST API: one that is already running, or one it installs from a [`Kates`](#kates) resource. It needs Kates 1.38.0 or newer: older versions can't honour an idempotency key, so every `KatesTest` waits at `Synced=False`, reason `ApiTooOld`, and starts no run. - -`KatesTest` and `KafkaCompliance` are deprecated in 0.2.0 and go in 0.3.0. The API server prints a warning on every request for either kind, and the operator logs one at start while any of their resources is left. Move each to a `CronJob` that runs the `kates` CLI, as [Replacing the run kinds](#replacing-the-run-kinds) shows. Both still work in 0.2.0. The `Kates` kind stays. +0.3.0 removed `KatesTest` and `KafkaCompliance`, which 0.2.0 deprecated. Before upgrading from 0.2.x, read [Upgrading to 0.3.0](#upgrading-to-030). To run a test or a compliance check on a schedule, run the `kates` CLI from a `CronJob`, as [Replacing the run kinds](#replacing-the-run-kinds) shows. ## Install @@ -49,81 +47,7 @@ helm install kates-operator ./kates-operator -n team --skip-crds \ CRDs are applied by a pre-install and pre-upgrade hook that never downgrades: with several releases on a cluster, the newest wins. On a cluster that already has the CRDs, pass `--skip-crds`. A conflict on `customresourcedefinitions.apiextensions.k8s.io` during install means the flag was missed, and `--force-conflicts` is a downgrade, not the fix. -Don't let two releases watch the same namespace: each would reconcile the same resources. - -## Connect a namespace - -Each watched namespace holds a Secret named `kates-connection`, with the keys `url`, `apiKey` and optional `caBundle`: - -```bash -kubectl create secret generic kates-connection -n payments \ - --from-literal=url=http://kates.kates.svc:8080 --from-literal=apiKey= -``` - -The key needs `read` and `test:run`. Use a named key from the Kates keys file, so the audit log names the team. `connection.fallback: true` lends the Secret `connection.existingSecret` to every watched namespace that has no `kates-connection`. With `watchAnyNamespace`, that lends the key to everyone who can create a `KatesTest`. - -To rotate a key, add the new hash to the Kates keys file, swap the Secret, then remove the old hash. The operator reads the key on every pass, so it needs no restart. - -## KatesTest - -```yaml -apiVersion: kates.io/v1alpha1 -kind: KatesTest -metadata: - name: nightly-load -spec: - type: LOAD - spec: - numRecords: 100000 - acks: all - sla: - maxP99LatencyMs: 100 -``` - -`spec.spec` takes the wire names of `POST /api/tests`, and the CRD refuses a mistyped key. `spec.sla` is graded by the Kates API, the same way the report and the JUnit export grade it. Once the run has started, the whole `spec` is immutable, because the resource is the evidence of what the run was judged against. - -A CI gate is two commands, race-free on either outcome: - -```bash -kubectl wait --for=jsonpath='{.status.finishedAt}' katestest/nightly-load --timeout=30m -kubectl wait --for=condition=Ready katestest/nightly-load --timeout=0 -``` - -The first returns on every ending, and the second exits non-zero on a failed run. `kubectl describe` shows the failing key. The value-less `jsonpath` form needs kubectl 1.29 or newer. - -Deleting a `KatesTest` whose run is live cancels the run first. The run's record stays in Kates. A deleted and re-created `KatesTest` is a new run. - -### On a schedule - -There is no schedule kind. Run `kates test apply` from a `CronJob`, as [Replacing the run kinds](#replacing-the-run-kinds) shows. - -### GitOps - -To rerun a `KatesTest`, rename it: the new name is a new run. With pruning on, the old resource is deleted, and its run is cancelled first if it is still live. - -- A spec edit under the same name, after the run started, is refused at dry-run. Flux dry-runs every object first, so that one refusal fails the whole Kustomization. Keep `KatesTest`s in their own Kustomization or Application. -- Don't use Flux `spec.force: true`, or Argo CD `Replace=true,Force=true`. A refused edit then deletes and re-creates the resource, which starts a new run with no review. -- Never set `ttlSecondsAfterFinished` on a resource a GitOps tool holds: the tool re-creates it, and the run repeats every TTL. The TTL is for one-shot resources a pipeline creates under a unique name. - -## KafkaCompliance - -```yaml -apiVersion: kates.io/v1alpha1 -kind: KafkaCompliance -metadata: - name: krafter-rules -spec: - interval: 1h - reference: - target: {namespace: kafka, name: krafter} - brokers: - count: {min: 3} - zones: {min: 3} -``` - -`spec.reference` is a `KafkaReference` `.spec`, inline, and the API's strict reader checks it. The resource's name becomes the stored check's name, so it must be a DNS label. Intervals under a minute are raised to one minute, and intervals over a year lowered to one year. - -A check also runs when the spec changes, and when the Strimzi `Kafka` it names has finished reconciling a change, if that namespace is in `kafka.namespaces`. Any other target is checked on its interval alone, and `KafkaWatched=False` says why. `spec.suspend: true` stops new checks. +Don't let two releases watch the same namespace: each would reconcile the same `Kates` resources. ## Kates @@ -203,6 +127,20 @@ What it doesn't do yet: - It doesn't create PostgreSQL. A changed Secret is not noticed: delete the pod to pick it up. - Kafka over TLS with a truststore file isn't expressible yet. `spec.env` takes any other setting of the container, except the names `database`, `kafka` and `apiKeySecretRef` set. +## Upgrading to 0.3.0 + +0.2.x holds a finalizer on each live `KatesTest`, and 0.3.0 no longer clears it. So, in this order: + +1. **While 0.2.x still runs**, move each `KatesTest` and `KafkaCompliance` to a `CronJob` (below), then delete them. Deleting a live `KatesTest` cancels its run, so wait for it to finish first. +2. **Remove** `kafka`, `connection` and `rbac.kafkaNamespaceRole` from your values: the chart's schema refuses them. Upgrade with `helm upgrade`. +3. **Delete** the two CRDs, which Helm leaves in place: + +```bash +kubectl delete crd katestests.kates.io kafkacompliances.kates.io +``` + +A `KatesTest` left with its finalizer keeps its CRD in `Terminating`. Clear it by hand: `kubectl patch katestest -n --type=merge -p '{"metadata":{"finalizers":null}}'`. + ## Replacing the run kinds A `CronJob` runs the `kates` CLI against the API: `kates test apply --wait` for a `KatesTest`, `kates compliance check` for a `KafkaCompliance`. The Job fails when the run misses a gate or the check fails, so the `CronJob`'s history shows each miss. The scenario file takes the keys of `kates test apply`, not the wire names of `spec.spec`: see `kates test apply --help`. @@ -244,9 +182,9 @@ spec: - name: KATES_VERSION value: "1.40.0" - name: KATES_URL - valueFrom: {secretKeyRef: {name: kates-connection, key: url}} + valueFrom: {secretKeyRef: {name: kates-cli, key: url}} - name: KATES_API_KEY - valueFrom: {secretKeyRef: {name: kates-connection, key: apiKey}} + valueFrom: {secretKeyRef: {name: kates-cli, key: apiKey}} command: [/bin/sh, -ec] args: - | @@ -264,38 +202,27 @@ spec: configMap: {name: nightly-load} ``` -For a `KafkaCompliance`, put its `spec.reference` in the ConfigMap as a `KafkaReference` file, and run `./kates compliance check -f /scenario/reference.yaml` instead. The check runs on the schedule only, not when the Strimzi `Kafka` changes. A test needs a key with `read` and `test:run`, a check `read` alone, so the `kates-connection` key the operator uses serves both. +For a `KafkaCompliance`, put its `spec.reference` in the ConfigMap as a `KafkaReference` file, and run `./kates compliance check -f /scenario/reference.yaml` instead. The check runs on the schedule only, not when the Strimzi `Kafka` changes. The Secret `kates-cli` holds the API's `url` and an `apiKey`: a test needs a key with `read` and `test:run`, a check `read` alone. A `kates-connection` Secret left from 0.2.x has the same keys and serves as well. The pod downloads the CLI from GitHub and checks it against the release's `checksums.txt`. Where pods have no route to GitHub, serve the tarball and `checksums.txt` from a mirror and point `release` at it. `kates-tester` is unsigned. Where the Kates chart's `kyvernoPolicy.cosign` is on with Enforce, its pod is refused in that release's namespace: run the `CronJob` in another namespace, or use your own signed image with the CLI in it. -Once a namespace has its `CronJob`s, delete its `KatesTest`s and `KafkaCompliance`s. Deleting a live `KatesTest` cancels its run, so wait for it to finish first. - ## Operating it | Annotation | Effect | | --- | --- | -| `kates.io/pause-reconciliation: "true"` | No calls to the Kates API. A deleted resource is never paused | -| `kates.io/reconcile-at: ` | Forces a pass, recorded in `status.lastHandledReconcileAt` | - -Before `helm uninstall`, before dropping a namespace from `watchNamespaces`, and before deleting a namespace, delete its live `KatesTest`s and its `Kates` resources. A finished `KatesTest` holds no finalizer and blocks nothing. - -A live one left behind needs a hand. With `watchNamespaces` or the default mode, the RoleBinding that gives the operator its rights lives in the namespace, so `kubectl delete namespace` removes those rights along with everything else. The operator can then no longer clear its finalizer, and the namespace stays Terminating until you do: - -```bash -kubectl patch katestest -n --type=merge -p '{"metadata":{"finalizers":null}}' -``` +| `kates.io/pause-reconciliation: "true"` | The operator leaves the resource as it is. A deleted resource is never paused | -Kates ends the run at its own timeout. With `watchAnyNamespace` the ClusterRoleBinding survives: if the Secret goes first, the finalizer comes off with a `CancelSkipped` Event. +Before `helm uninstall`, before dropping a namespace from `watchNamespaces`, and before deleting a namespace, delete its `Kates` resources. With `watchNamespaces` or the default mode, the RoleBinding that gives the operator its rights lives in the namespace, so `kubectl delete namespace` removes those rights too, and the operator can no longer clear its finalizer. -A `Kates` left behind is cleared the same way, and its ClusterRoleBinding then stays: delete it too, or a ServiceAccount later created under that name in that namespace inherits its rights. +A `Kates` left behind is cleared by hand, and its ClusterRoleBinding then stays: delete it too, or a ServiceAccount later created under that name in that namespace inherits its rights. ```bash kubectl patch kates -n --type=merge -p '{"metadata":{"finalizers":null}}' kubectl delete clusterrolebinding kates-backend.. ``` -The NetworkPolicy allows egress on 8080, 443 and 6443. A Kates URL on any other port needs `networkPolicy.egressRules`, or the blocked call shows as `ApiUnavailable`. +The NetworkPolicy allows egress on 8080, 443 and 6443, the Kates pods' API and the Kubernetes API. -Metrics are on `:8080/metrics` under `kates_operator_`, and `monitoring.enabled` renders a PodMonitor. `/readyz` fails on an unreachable API server or a stale served CRD, and it reports the Lease and any refused Kafka namespace in its body without failing on them. +Metrics are on `:8080/metrics` under `kates_operator_`, and `monitoring.enabled` renders a PodMonitor. `/readyz` fails on an unreachable API server or a stale served CRD, and it reports whether the pod holds the Lease in its body without failing on it. One pod holds the Lease `kates-operator` in the release namespace. `replicas: 2` only shortens the gap after a crash, to between 5 and 15 seconds. diff --git a/operator/chart/crds/kafkacompliances.kates.io.yaml b/operator/chart/crds/kafkacompliances.kates.io.yaml deleted file mode 100644 index ce36d5c7..00000000 --- a/operator/chart/crds/kafkacompliances.kates.io.yaml +++ /dev/null @@ -1,194 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kafkacompliances.kates.io -spec: - group: kates.io - names: - kind: KafkaCompliance - plural: kafkacompliances - singular: kafkacompliance - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.verdict - name: Verdict - type: string - - jsonPath: .status.checkedAt - name: Checked - type: date - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - deprecated: true - deprecationWarning: "kates.io/v1alpha1 KafkaCompliance is deprecated and goes in kates-operator 0.3.0: run kates compliance check from a CronJob instead" - name: v1alpha1 - schema: - openAPIV3Schema: - description: "A standing compliance check of one Kafka. crd-version: 1" - properties: - spec: - description: A standing compliance check of one Kafka. - properties: - interval: - description: How often to check, such as 30m or 1h. - pattern: ^([0-9]+(s|m|h))+$ - type: string - reference: - description: A `KafkaReference` `.spec`, inline; its target names the Kafka. - properties: - target: - properties: - name: - type: string - namespace: - type: string - required: - - name - - namespace - type: object - x-kubernetes-preserve-unknown-fields: true - required: - - target - type: object - x-kubernetes-preserve-unknown-fields: true - suspend: - default: false - description: Stops new checks without touching what the last one found. - type: boolean - required: - - interval - - reference - type: object - status: - nullable: true - properties: - checkId: - description: "`/api/conformity/checks/{id}`; absent when the API kept no report." - nullable: true - type: string - checkedAt: - description: >- - Time is a wrapper around time.Time which supports correct marshaling to YAML - and JSON. Wrappers are provided for many of the factory methods that the time - package offers. - format: date-time - nullable: true - type: string - conditions: - items: - description: >- - Condition contains details for one aspect of the current state of this API - Resource. - properties: - lastTransitionTime: - description: >- - lastTransitionTime is the last time the condition transitioned from one status - to another. This should be when the underlying condition changed. If that is not - known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: >- - message is a human readable message indicating details about the transition. - This may be an empty string. - type: string - observedGeneration: - description: >- - observedGeneration represents the .metadata.generation that the condition was - set based upon. For instance, if .metadata.generation is currently 12, but the - .status.conditions[x].observedGeneration is 9, the condition is out of date with - respect to the current state of the instance. - format: int64 - type: integer - reason: - description: >- - reason contains a programmatic identifier indicating the reason for the - condition's last transition. Producers of specific condition types may define expected - values and meanings for this field, and whether the values are considered a - guaranteed API. The value should be a CamelCase string. This field may not be empty. - type: string - status: - description: status of the condition, one of True, False, Unknown. - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - counts: - nullable: true - properties: - fail: - format: int64 - type: integer - pass: - format: int64 - type: integer - unknown: - format: int64 - type: integer - required: - - fail - - pass - - unknown - type: object - failed: - description: At most 20; the report has the rest. - items: - properties: - detail: - nullable: true - type: string - rule: - type: string - required: - - rule - type: object - type: array - lastHandledReconcileAt: - description: The `kates.io/reconcile-at` value last acted on. - nullable: true - type: string - nextCheckAt: - description: >- - Time is a wrapper around time.Time which supports correct marshaling to YAML - and JSON. Wrappers are provided for many of the factory methods that the time - package offers. - format: date-time - nullable: true - type: string - observedGeneration: - format: int64 - nullable: true - type: integer - observedKafkaGeneration: - description: The Kafka's `status.observedGeneration` at the last answered check. - format: int64 - nullable: true - type: integer - verdict: - description: PASS, FAIL or UNKNOWN. - nullable: true - type: string - type: object - required: - - spec - title: KafkaCompliance - type: object - x-kubernetes-validations: - - message: "name must be a DNS label (at most 63 lower-case letters, digits and -): it names the stored compliance check" - rule: self.metadata.name.matches('^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$') - served: true - storage: true - subresources: - status: {} diff --git a/operator/chart/crds/katestests.kates.io.yaml b/operator/chart/crds/katestests.kates.io.yaml deleted file mode 100644 index f892ae44..00000000 --- a/operator/chart/crds/katestests.kates.io.yaml +++ /dev/null @@ -1,565 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: katestests.kates.io -spec: - group: kates.io - names: - kind: KatesTest - plural: katestests - singular: katestest - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .status.phase - name: Phase - type: string - - jsonPath: .status.runId - name: Run - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - deprecated: true - deprecationWarning: "kates.io/v1alpha1 KatesTest is deprecated and goes in kates-operator 0.3.0: run kates test apply from a CronJob instead" - name: v1alpha1 - schema: - openAPIV3Schema: - description: "One Kates run. crd-version: 1" - properties: - spec: - description: One Kates run, as `kubectl apply` takes it. - properties: - backend: - description: "Execution backend: native (the API's default) or trogdor." - nullable: true - type: string - scenario: - properties: - backend: - description: "Benchmark execution engine: native or trogdor." - type: string - baseSpec: - description: Base TestSpec inherited by all phases unless overridden in phases[].spec. - properties: - acks: - description: "Producer acknowledgment mode: \"all\", \"-1\", \"1\", or \"0\"." - nullable: true - x-kubernetes-int-or-string: true - batchSize: - description: Producer batch.size in bytes (1..16,777,216). - type: integer - compressionType: - description: "Producer compression codec: none, gzip, snappy, lz4, or zstd." - type: string - consumerGroup: - description: >- - Consumer group ID (1..255 printable ASCII chars). Allowed only on single-phase - LOAD, ENDURANCE, and INTEGRITY runs. - type: string - durationMs: - description: >- - Maximum test duration in milliseconds (0 for record-count bounded, or - 1,000..86,400,000). - type: integer - enableCrc: - description: Enable CRC32 payload validation (allowed only on single-phase INTEGRITY runs). - type: boolean - enableIdempotence: - description: "Enable producer idempotence (requires acks: \"all\")." - type: boolean - enableTransactions: - description: "Enable Kafka transactions (requires acks: \"all\", enableIdempotence: true, and backend: native)." - type: boolean - fetchMaxWaitMs: - description: >- - Consumer fetch.max.wait.ms (1..60,000). Allowed only on single-phase LOAD, - ENDURANCE, and INTEGRITY runs. - type: integer - fetchMinBytes: - description: >- - Consumer fetch.min.bytes (1..16,777,216). Allowed only on single-phase LOAD, - ENDURANCE, and INTEGRITY runs. - type: integer - lingerMs: - description: Producer linger.ms in milliseconds (0..60,000). - type: integer - minInsyncReplicas: - description: Topic min.insync.replicas (1..10). Must be <= replicationFactor. - type: integer - numConsumers: - description: Number of concurrent consumer tasks (1..64). - type: integer - numProducers: - description: Number of concurrent producer tasks (1..64). - type: integer - numRecords: - description: Number of records each producer task sends (1..100,000,000). - type: integer - partitions: - description: Number of topic partitions (1..1,000). - type: integer - recordSize: - description: Payload size of each record in bytes (1..10,485,760). - type: integer - replicationFactor: - description: Topic replication factor (1..10). Must be >= minInsyncReplicas. - type: integer - targetThroughput: - description: "Alias for throughput: target producer rate in records/sec, or -1 for unlimited." - type: integer - throughput: - description: Target producer throughput in records/sec per producer, or -1 for unlimited. - type: integer - topic: - description: Kafka topic name (1..249 ASCII alphanumeric, '.', '_', or '-' chars). - type: string - type: object - description: - description: Human-readable description of the multi-phase scenario. - type: string - labels: - additionalProperties: - type: string - description: Arbitrary key/value string labels attached to the scenario. - type: object - name: - description: Scenario name (1..128 characters). - type: string - phases: - description: Ordered sequence of execution phases (WARMUP, RAMP, STEADY, SPIKE, COOLDOWN). - items: - description: A single execution phase within a multi-phase TestScenario. - properties: - durationMs: - description: Phase duration in milliseconds (0 or 1,000..86,400,000). - type: integer - name: - description: Phase display name (1..100 characters). - type: string - phaseType: - description: "Alias for type: execution phase behavior (WARMUP, RAMP, STEADY, SPIKE, or COOLDOWN)." - type: string - rampSteps: - description: >- - Number of incremental throughput steps (2..100). Only used when phase type is - RAMP. - type: integer - spec: - description: Phase-specific TestSpec overrides merged onto scenario.baseSpec. - properties: - acks: - description: "Producer acknowledgment mode: \"all\", \"-1\", \"1\", or \"0\"." - nullable: true - x-kubernetes-int-or-string: true - batchSize: - description: Producer batch.size in bytes (1..16,777,216). - type: integer - compressionType: - description: "Producer compression codec: none, gzip, snappy, lz4, or zstd." - type: string - consumerGroup: - description: >- - Consumer group ID (1..255 printable ASCII chars). Allowed only on single-phase - LOAD, ENDURANCE, and INTEGRITY runs. - type: string - durationMs: - description: >- - Maximum test duration in milliseconds (0 for record-count bounded, or - 1,000..86,400,000). - type: integer - enableCrc: - description: Enable CRC32 payload validation (allowed only on single-phase INTEGRITY runs). - type: boolean - enableIdempotence: - description: "Enable producer idempotence (requires acks: \"all\")." - type: boolean - enableTransactions: - description: "Enable Kafka transactions (requires acks: \"all\", enableIdempotence: true, and backend: native)." - type: boolean - fetchMaxWaitMs: - description: >- - Consumer fetch.max.wait.ms (1..60,000). Allowed only on single-phase LOAD, - ENDURANCE, and INTEGRITY runs. - type: integer - fetchMinBytes: - description: >- - Consumer fetch.min.bytes (1..16,777,216). Allowed only on single-phase LOAD, - ENDURANCE, and INTEGRITY runs. - type: integer - lingerMs: - description: Producer linger.ms in milliseconds (0..60,000). - type: integer - minInsyncReplicas: - description: Topic min.insync.replicas (1..10). Must be <= replicationFactor. - type: integer - numConsumers: - description: Number of concurrent consumer tasks (1..64). - type: integer - numProducers: - description: Number of concurrent producer tasks (1..64). - type: integer - numRecords: - description: Number of records each producer task sends (1..100,000,000). - type: integer - partitions: - description: Number of topic partitions (1..1,000). - type: integer - recordSize: - description: Payload size of each record in bytes (1..10,485,760). - type: integer - replicationFactor: - description: Topic replication factor (1..10). Must be >= minInsyncReplicas. - type: integer - targetThroughput: - description: "Alias for throughput: target producer rate in records/sec, or -1 for unlimited." - type: integer - throughput: - description: Target producer throughput in records/sec per producer, or -1 for unlimited. - type: integer - topic: - description: Kafka topic name (1..249 ASCII alphanumeric, '.', '_', or '-' chars). - type: string - type: object - targetThroughput: - description: Target producer throughput for this phase in records/sec, or -1 for unlimited. - type: integer - type: object - type: array - sla: - description: SLA definition evaluated across the scenario run. - properties: - maxAvgLatencyMs: - description: Maximum allowed average latency in milliseconds (> 0). - type: number - maxDataLossPercent: - description: Maximum allowed data loss percentage (0.0..100.0). - type: number - maxErrorRate: - description: Maximum allowed error rate as a fraction between 0.0 and 1.0. - type: number - maxP999LatencyMs: - description: Maximum allowed p99.9 latency in milliseconds (> 0). - type: number - maxP99LatencyMs: - description: Maximum allowed p99 latency in milliseconds (> 0). - type: number - maxRpoMs: - description: Maximum allowed Recovery Point Objective (RPO) in milliseconds (> 0). - type: number - maxRtoMs: - description: Maximum allowed Recovery Time Objective (RTO) in milliseconds (> 0). - type: number - minRecordsProcessed: - description: Minimum total records processed (>= 1). - type: integer - minThroughputRecPerSec: - description: Minimum required throughput in records/second (> 0). - type: number - type: object - type: - description: Primary test type for the scenario. - type: string - type: object - sla: - description: >- - Service Level Agreement thresholds used by Kates SlaGrader to grade a test or - disruption run. - properties: - maxAvgLatencyMs: - description: Maximum allowed average latency in milliseconds (> 0). - type: number - maxDataLossPercent: - description: Maximum allowed data loss percentage (0.0..100.0). - type: number - maxErrorRate: - description: Maximum allowed error rate as a fraction between 0.0 and 1.0. - type: number - maxP999LatencyMs: - description: Maximum allowed p99.9 latency in milliseconds (> 0). - type: number - maxP99LatencyMs: - description: Maximum allowed p99 latency in milliseconds (> 0). - type: number - maxRpoMs: - description: Maximum allowed Recovery Point Objective (RPO) in milliseconds (> 0). - type: number - maxRtoMs: - description: Maximum allowed Recovery Time Objective (RTO) in milliseconds (> 0). - type: number - minRecordsProcessed: - description: Minimum total records processed (>= 1). - type: integer - minThroughputRecPerSec: - description: Minimum required throughput in records/second (> 0). - type: number - type: object - spec: - description: >- - Kates API wire-format TestSpec. Uses numRecords, numProducers, recordSize, - durationMs, and throughput. - properties: - acks: - description: "Producer acknowledgment mode: \"all\", \"-1\", \"1\", or \"0\"." - nullable: true - x-kubernetes-int-or-string: true - batchSize: - description: Producer batch.size in bytes (1..16,777,216). - type: integer - compressionType: - description: "Producer compression codec: none, gzip, snappy, lz4, or zstd." - type: string - consumerGroup: - description: >- - Consumer group ID (1..255 printable ASCII chars). Allowed only on single-phase - LOAD, ENDURANCE, and INTEGRITY runs. - type: string - durationMs: - description: >- - Maximum test duration in milliseconds (0 for record-count bounded, or - 1,000..86,400,000). - type: integer - enableCrc: - description: Enable CRC32 payload validation (allowed only on single-phase INTEGRITY runs). - type: boolean - enableIdempotence: - description: "Enable producer idempotence (requires acks: \"all\")." - type: boolean - enableTransactions: - description: "Enable Kafka transactions (requires acks: \"all\", enableIdempotence: true, and backend: native)." - type: boolean - fetchMaxWaitMs: - description: >- - Consumer fetch.max.wait.ms (1..60,000). Allowed only on single-phase LOAD, - ENDURANCE, and INTEGRITY runs. - type: integer - fetchMinBytes: - description: >- - Consumer fetch.min.bytes (1..16,777,216). Allowed only on single-phase LOAD, - ENDURANCE, and INTEGRITY runs. - type: integer - lingerMs: - description: Producer linger.ms in milliseconds (0..60,000). - type: integer - minInsyncReplicas: - description: Topic min.insync.replicas (1..10). Must be <= replicationFactor. - type: integer - numConsumers: - description: Number of concurrent consumer tasks (1..64). - type: integer - numProducers: - description: Number of concurrent producer tasks (1..64). - type: integer - numRecords: - description: Number of records each producer task sends (1..100,000,000). - type: integer - partitions: - description: Number of topic partitions (1..1,000). - type: integer - recordSize: - description: Payload size of each record in bytes (1..10,485,760). - type: integer - replicationFactor: - description: Topic replication factor (1..10). Must be >= minInsyncReplicas. - type: integer - targetThroughput: - description: "Alias for throughput: target producer rate in records/sec, or -1 for unlimited." - type: integer - throughput: - description: Target producer throughput in records/sec per producer, or -1 for unlimited. - type: integer - topic: - description: Kafka topic name (1..249 ASCII alphanumeric, '.', '_', or '-' chars). - type: string - type: object - ttlSecondsAfterFinished: - description: |- - Seconds after the end of the run to delete this resource. Never set it on a resource Flux - or Argo CD holds: the tool re-creates it, and the run repeats every TTL. - format: int32 - minimum: 0.0 - nullable: true - type: integer - type: - description: "`TestType`, as `POST /api/tests` takes it." - enum: - - LOAD - - STRESS - - SPIKE - - ENDURANCE - - VOLUME - - CAPACITY - - ROUND_TRIP - - INTEGRITY - - TUNE_REPLICATION - - TUNE_ACKS - - TUNE_BATCHING - - TUNE_COMPRESSION - - TUNE_PARTITIONS - - INTEGRATION_CDC - type: string - required: - - type - type: object - status: - nullable: true - properties: - conditions: - items: - description: >- - Condition contains details for one aspect of the current state of this API - Resource. - properties: - lastTransitionTime: - description: >- - lastTransitionTime is the last time the condition transitioned from one status - to another. This should be when the underlying condition changed. If that is not - known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: >- - message is a human readable message indicating details about the transition. - This may be an empty string. - type: string - observedGeneration: - description: >- - observedGeneration represents the .metadata.generation that the condition was - set based upon. For instance, if .metadata.generation is currently 12, but the - .status.conditions[x].observedGeneration is 9, the condition is out of date with - respect to the current state of the instance. - format: int64 - type: integer - reason: - description: >- - reason contains a programmatic identifier indicating the reason for the - condition's last transition. Producers of specific condition types may define expected - values and meanings for this field, and whether the values are considered a - guaranteed API. The value should be a CamelCase string. This field may not be empty. - type: string - status: - description: status of the condition, one of True, False, Unknown. - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - finishedAt: - description: >- - Time is a wrapper around time.Time which supports correct marshaling to YAML - and JSON. Wrappers are provided for many of the factory methods that the time - package offers. - format: date-time - nullable: true - type: string - lastHandledReconcileAt: - description: The `kates.io/reconcile-at` value last acted on. - nullable: true - type: string - observedGeneration: - format: int64 - nullable: true - type: integer - phase: - enum: - - Pending - - Running - - Succeeded - - Failed - - null - nullable: true - type: string - runId: - nullable: true - type: string - startedAt: - description: >- - Time is a wrapper around time.Time which supports correct marshaling to YAML - and JSON. Wrappers are provided for many of the factory methods that the time - package offers. - format: date-time - nullable: true - type: string - summary: - description: "`ReportSummary`'s fields verbatim, from `GET /api/tests/{id}/report`." - nullable: true - properties: - avgLatencyMs: - format: double - nullable: true - type: number - avgThroughputMBPerSec: - format: double - nullable: true - type: number - avgThroughputRecPerSec: - format: double - nullable: true - type: number - durationMs: - format: int64 - nullable: true - type: integer - errorRate: - format: double - nullable: true - type: number - maxLatencyMs: - format: double - nullable: true - type: number - p50LatencyMs: - format: double - nullable: true - type: number - p95LatencyMs: - format: double - nullable: true - type: number - p999LatencyMs: - format: double - nullable: true - type: number - p99LatencyMs: - format: double - nullable: true - type: number - peakThroughputRecPerSec: - format: double - nullable: true - type: number - totalErrors: - format: int64 - nullable: true - type: integer - totalRecords: - format: int64 - nullable: true - type: integer - type: object - type: object - required: - - spec - title: KatesTest - type: object - x-kubernetes-validations: - - message: exactly one of spec.spec and spec.scenario is required - rule: has(self.spec.spec) != has(self.spec.scenario) - - message: "spec.sla goes with spec.spec: a scenario carries its own sla" - rule: "!has(self.spec.sla) || has(self.spec.spec)" - - message: "spec is immutable once the run has started: create a new KatesTest" - rule: "!has(oldSelf.status) || !has(oldSelf.status.runId) || oldSelf.spec == self.spec" - served: true - storage: true - subresources: - status: {} diff --git a/operator/chart/templates/NOTES.txt b/operator/chart/templates/NOTES.txt index 90c6b341..27b1ca9a 100644 --- a/operator/chart/templates/NOTES.txt +++ b/operator/chart/templates/NOTES.txt @@ -1,14 +1,7 @@ -kates-operator reconciles KatesTest and KafkaCompliance in: {{ include "kates-operator.namespaces" . }} +kates-operator reconciles Kates resources in: {{ include "kates-operator.namespaces" . }} -Each of those namespaces needs a Secret named kates-connection (keys url, apiKey, optional caBundle): + kubectl get kates -A - kubectl create secret generic kates-connection -n \ - --from-literal=url=http://kates.kates.svc:8080 --from-literal=apiKey= - -Then: - - kubectl get katestests,kafkacompliances -A - -KatesTest and KafkaCompliance are deprecated and go in kates-operator 0.3.0. -Run kates test apply or kates compliance check from a CronJob instead: see -"Replacing the run kinds" in the chart README. +KatesTest and KafkaCompliance were removed in 0.3.0. To run tests or compliance +checks on a schedule, run kates test apply or kates compliance check from a +CronJob: see "Replacing the run kinds" in the chart README. diff --git a/operator/chart/templates/_helpers.tpl b/operator/chart/templates/_helpers.tpl index 3b0363d8..2dbbe1c1 100644 --- a/operator/chart/templates/_helpers.tpl +++ b/operator/chart/templates/_helpers.tpl @@ -39,12 +39,6 @@ imagePullSecrets: {{/* What the operator may do where it reconciles. */}} {{- define "kates-operator.rules" -}} -- apiGroups: ["kates.io"] - resources: ["katestests", "kafkacompliances"] - verbs: ["get", "list", "watch", "patch", "update", "delete"] -- apiGroups: ["kates.io"] - resources: ["katestests/status", "kafkacompliances/status", "katestests/finalizers", "kafkacompliances/finalizers"] - verbs: ["get", "patch", "update"] {{- /* patch and update: the finalizer that removes a pod's ClusterRoleBinding. */}} - apiGroups: ["kates.io"] resources: ["kates"] @@ -76,7 +70,7 @@ quorum it labels the leader's pod, which the Service selects. */}} - apiGroups: [""] resources: ["secrets"] verbs: ["get"] - resourceNames: ["kates-connection", "kates-api-key"] + resourceNames: ["kates-api-key"] {{- /* A quorum's CA and its members' certificates, by name, so one replicated Kates per namespace. create cannot be narrowed by name, and reads nothing. */}} - apiGroups: [""] diff --git a/operator/chart/templates/aggregate-roles.yaml b/operator/chart/templates/aggregate-roles.yaml index 7bdf3cba..6500b68b 100644 --- a/operator/chart/templates/aggregate-roles.yaml +++ b/operator/chart/templates/aggregate-roles.yaml @@ -8,7 +8,7 @@ metadata: rbac.authorization.k8s.io/aggregate-to-view: "true" rules: - apiGroups: ["kates.io"] - resources: ["katestests", "kafkacompliances", "kates"] + resources: ["kates"] verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -20,7 +20,7 @@ metadata: rbac.authorization.k8s.io/aggregate-to-edit: "true" rules: - apiGroups: ["kates.io"] - resources: ["katestests", "kafkacompliances", "kates"] + resources: ["kates"] verbs: ["*"] --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/operator/chart/templates/crd-upgrade.yaml b/operator/chart/templates/crd-upgrade.yaml index 6bdc16e7..4d05e7e3 100644 --- a/operator/chart/templates/crd-upgrade.yaml +++ b/operator/chart/templates/crd-upgrade.yaml @@ -24,7 +24,7 @@ metadata: rules: - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] - resourceNames: ["katestests.kates.io", "kafkacompliances.kates.io", "kates.kates.io"] + resourceNames: ["kates.kates.io"] verbs: ["get", "create", "patch"] - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] diff --git a/operator/chart/templates/deployment.yaml b/operator/chart/templates/deployment.yaml index 3e340cd1..e4d90444 100644 --- a/operator/chart/templates/deployment.yaml +++ b/operator/chart/templates/deployment.yaml @@ -58,8 +58,6 @@ spec: fieldPath: metadata.namespace - name: KATES_OPERATOR_NAMESPACES value: {{ include "kates-operator.namespaces" . | quote }} - - name: KATES_OPERATOR_KAFKA_NAMESPACES - value: {{ if .Values.watchAnyNamespace }}"*"{{ else }}{{ join "," .Values.kafka.namespaces | quote }}{{ end }} {{- if .Values.rbac.clusterRoles.create }} - name: KATES_OPERATOR_BACKEND_ROLE value: kates-backend-{{ .Release.Namespace }} @@ -84,20 +82,3 @@ spec: periodSeconds: 10 resources: {{- toYaml .Values.resources | nindent 12 }} - {{- if .Values.connection.fallback }} - volumeMounts: - - name: connection - mountPath: /etc/kates-operator/connection - readOnly: true - {{- end }} - {{- if .Values.connection.fallback }} - # Projected with no subPath, so the kubelet refreshes a rotated key; optional, so a - # missing Secret leaves the resources at NoConnection and the pod running. - volumes: - - name: connection - projected: - sources: - - secret: - name: {{ .Values.connection.existingSecret }} - optional: true - {{- end }} diff --git a/operator/chart/templates/kafka-reader.yaml b/operator/chart/templates/kafka-reader.yaml deleted file mode 100644 index de5cc3f9..00000000 --- a/operator/chart/templates/kafka-reader.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if and .Values.rbac.kafkaNamespaceRole (not .Values.watchAnyNamespace) }} -{{- $root := . }} -{{- $name := printf "kates-operator-kafka-reader-%s" .Release.Namespace | trunc 63 | trimSuffix "-" }} -{{- range .Values.kafka.namespaces }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ $name }} - namespace: {{ . }} - labels: - {{- include "kates-operator.labels" $root | nindent 4 }} -rules: - - apiGroups: ["kafka.strimzi.io"] - resources: ["kafkas"] - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ $name }} - namespace: {{ . }} - labels: - {{- include "kates-operator.labels" $root | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ $name }} -subjects: - - kind: ServiceAccount - name: kates-operator - namespace: {{ $root.Release.Namespace }} -{{- end }} -{{- end }} diff --git a/operator/chart/templates/networkpolicy.yaml b/operator/chart/templates/networkpolicy.yaml index 677d7c5b..030a209d 100644 --- a/operator/chart/templates/networkpolicy.yaml +++ b/operator/chart/templates/networkpolicy.yaml @@ -26,7 +26,7 @@ spec: protocol: UDP - port: 53 protocol: TCP - # Ports only, with no to: the template cannot know which namespace a kates-connection names. + # Ports only, with no to: the Kates resources it reaches may be in any watched namespace. - ports: - port: 8080 protocol: TCP diff --git a/operator/chart/templates/rbac.yaml b/operator/chart/templates/rbac.yaml index b50386af..403bb335 100644 --- a/operator/chart/templates/rbac.yaml +++ b/operator/chart/templates/rbac.yaml @@ -4,9 +4,6 @@ {{- if or .Values.watchNamespaces .Values.watchAnyNamespace }} {{- fail "rbac.clusterRoles.create=false installs in the release namespace alone: leave watchNamespaces and watchAnyNamespace unset" }} {{- end }} -{{- if .Values.kafka.namespaces }} -{{- fail "rbac.clusterRoles.create=false takes no kafka.namespaces: a Kafka namespace's Role is that namespace's admin's to apply, and compliance then runs on its interval" }} -{{- end }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -39,11 +36,6 @@ metadata: {{- include "kates-operator.labels" . | nindent 4 }} rules: {{- include "kates-operator.rules" . | nindent 2 }} - {{- if .Values.watchAnyNamespace }} - - apiGroups: ["kafka.strimzi.io"] - resources: ["kafkas"] - verbs: ["get", "list", "watch"] - {{- end }} {{- if .Values.watchAnyNamespace }} --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/operator/chart/values.schema.json b/operator/chart/values.schema.json index ff6a5950..34522c48 100644 --- a/operator/chart/values.schema.json +++ b/operator/chart/values.schema.json @@ -7,27 +7,49 @@ "global": { "type": "object", "properties": { - "imageRegistry": { "type": "string" }, - "imagePullSecrets": { "$ref": "#/$defs/pullSecrets" } + "imageRegistry": { + "type": "string" + }, + "imagePullSecrets": { + "$ref": "#/$defs/pullSecrets" + } } }, "image": { "type": "object", "additionalProperties": false, "properties": { - "repository": { "type": "string", "minLength": 1 }, - "tag": { "type": "string", "not": { "const": "latest" } }, - "digest": { "type": "string", "pattern": "^(sha256:[0-9a-f]{64})?$" }, - "pullPolicy": { "enum": ["Always", "IfNotPresent", "Never"] } + "repository": { + "type": "string", + "minLength": 1 + }, + "tag": { + "type": "string", + "not": { + "const": "latest" + } + }, + "digest": { + "type": "string", + "pattern": "^(sha256:[0-9a-f]{64})?$" + }, + "pullPolicy": { + "enum": [ + "Always", + "IfNotPresent", + "Never" + ] + } } }, - "imagePullSecrets": { "$ref": "#/$defs/pullSecrets" }, - "watchNamespaces": { "$ref": "#/$defs/names" }, - "watchAnyNamespace": { "type": "boolean" }, - "kafka": { - "type": "object", - "additionalProperties": false, - "properties": { "namespaces": { "$ref": "#/$defs/names" } } + "imagePullSecrets": { + "$ref": "#/$defs/pullSecrets" + }, + "watchNamespaces": { + "$ref": "#/$defs/names" + }, + "watchAnyNamespace": { + "type": "boolean" }, "rbac": { "type": "object", @@ -36,70 +58,141 @@ "clusterRoles": { "type": "object", "additionalProperties": false, - "properties": { "create": { "type": "boolean" } } + "properties": { + "create": { + "type": "boolean" + } + } }, "aggregateRoles": { "type": "object", "additionalProperties": false, - "properties": { "create": { "type": "boolean" } } + "properties": { + "create": { + "type": "boolean" + } + } }, - "kafkaNamespaceRole": { "type": "boolean" }, - "backendNamespaces": { "type": "array", "items": { "type": "string", "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" } }, - "backendDirectChaos": { "type": "boolean" } - } - }, - "connection": { - "type": "object", - "additionalProperties": false, - "properties": { - "fallback": { "type": "boolean" }, - "existingSecret": { "type": "string", "minLength": 1 } + "backendNamespaces": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" + } + }, + "backendDirectChaos": { + "type": "boolean" + } } }, "crdUpgrade": { "type": "object", "additionalProperties": false, "properties": { - "enabled": { "type": "boolean" }, - "resources": { "type": "object" } + "enabled": { + "type": "boolean" + }, + "resources": { + "type": "object" + } } }, - "replicas": { "type": "integer", "minimum": 1 }, - "resources": { "type": "object" }, - "logFormat": { "enum": ["text", "json"] }, + "replicas": { + "type": "integer", + "minimum": 1 + }, + "resources": { + "type": "object" + }, + "logFormat": { + "enum": [ + "text", + "json" + ] + }, "tracing": { "type": "object", "additionalProperties": false, - "properties": { "endpoint": { "type": "string" } } + "properties": { + "endpoint": { + "type": "string" + } + } }, "networkPolicy": { "type": "object", "additionalProperties": false, "properties": { - "enabled": { "type": "boolean" }, - "monitoringNamespace": { "type": "string" }, - "egressRules": { "type": "array", "items": { "type": "object" } } + "enabled": { + "type": "boolean" + }, + "monitoringNamespace": { + "type": "string" + }, + "egressRules": { + "type": "array", + "items": { + "type": "object" + } + } } }, "monitoring": { "type": "object", "additionalProperties": false, - "properties": { "enabled": { "type": "boolean" } } + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "podLabels": { + "type": "object", + "additionalProperties": { + "type": "string" + } }, - "podLabels": { "type": "object", "additionalProperties": { "type": "string" } }, - "podAnnotations": { "type": "object" }, - "nodeSelector": { "type": "object" }, - "tolerations": { "type": "array" }, - "affinity": { "type": "object" } + "podAnnotations": { + "type": "object" + }, + "nodeSelector": { + "type": "object" + }, + "tolerations": { + "type": "array" + }, + "affinity": { + "type": "object" + } }, "$defs": { - "names": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "names": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, "pullSecrets": { "type": "array", "items": { "anyOf": [ - { "type": "string", "minLength": 1 }, - { "type": "object", "required": ["name"], "properties": { "name": { "type": "string", "minLength": 1 } } } + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1 + } + } + } ] } } diff --git a/operator/chart/values.yaml b/operator/chart/values.yaml index 623e48b6..1249490d 100644 --- a/operator/chart/values.yaml +++ b/operator/chart/values.yaml @@ -1,6 +1,5 @@ -# The Kates operator. It drives a Kates API that is already running and never -# installs one. A resource reads its API and key from the Secret -# kates-connection in its own namespace (keys url, apiKey, optional caBundle). +# The Kates operator. A Kates resource installs one Kates backend, and with +# replicas 3 or 5 a quorum of them. global: # -- Registry prepended to the image, with a slash, for clusters that pull @@ -25,11 +24,6 @@ watchNamespaces: [] # -- Reconcile every namespace, with a ClusterRoleBinding. watchAnyNamespace: false -kafka: - # -- Namespaces whose Strimzi Kafkas trigger a compliance check when they - # change. Empty: checks run on their interval only. - namespaces: [] - rbac: clusterRoles: # -- false renders nothing cluster-scoped, for a namespace admin: apply the @@ -39,9 +33,6 @@ rbac: aggregateRoles: # -- Fold the kinds into the built-in view, edit and admin roles. create: true - # -- Render the Role that lets the operator read Kafkas in kafka.namespaces. - # false: someone with rights there applies it from helm template --show-only. - kafkaNamespaceRole: true # -- Namespaces whose Kates pods get the backend's cluster role, for faults, # topology and compliance reads. Empty: none, and faults are off. Name only # namespaces where you trust everyone who can run a pod with that role in @@ -54,15 +45,6 @@ rbac: # rbac.directChaos. Needs rbac.clusterRoles.create. backendDirectChaos: false -connection: - # -- Lend every watched namespace the Secret below when it has no - # kates-connection of its own. With watchAnyNamespace that lends this key to - # every user who can create a KatesTest. - fallback: false - # -- The fallback Secret (keys url, apiKey, optional caBundle). This chart - # never renders it, so no key sits in Helm values. - existingSecret: kates-operator-connection - crdUpgrade: # -- Apply the CRDs from a pre-install and pre-upgrade hook, never over a # newer version. Off with rbac.clusterRoles.create=false. diff --git a/operator/crates/kates-client/src/lib.rs b/operator/crates/kates-client/src/lib.rs index 4b26897c..0ddbd905 100644 --- a/operator/crates/kates-client/src/lib.rs +++ b/operator/crates/kates-client/src/lib.rs @@ -5,7 +5,7 @@ use std::collections::BTreeMap; use std::sync::Arc; use std::time::Duration; -use reqwest::header::{HeaderMap, LOCATION, RETRY_AFTER}; +use reqwest::header::{HeaderMap, RETRY_AFTER}; use reqwest::{Certificate, Method, Response, StatusCode, redirect}; use secrecy::{ExposeSecret, SecretString}; use serde::Deserialize; @@ -83,125 +83,6 @@ struct ApiError { field_errors: BTreeMap, } -/// `TestResult.TaskStatus`; a status this build does not know reads as still running. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum RunStatus { - Pending, - Running, - Done, - Failed, -} - -impl RunStatus { - fn parse(status: &str) -> Self { - match status { - "PENDING" => Self::Pending, - "DONE" => Self::Done, - "FAILED" => Self::Failed, - _ => Self::Running, - } - } - - #[must_use] - pub fn is_terminal(self) -> bool { - matches!(self, Self::Done | Self::Failed) - } -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct WireRun { - id: String, - status: String, - client_key: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Run { - pub id: String, - pub status: RunStatus, - pub client_key: Option, -} - -impl From for Run { - fn from(run: WireRun) -> Self { - Self { - status: RunStatus::parse(&run.status), - id: run.id, - client_key: run.client_key, - } - } -} - -#[derive(Debug, Clone, Deserialize)] -pub struct Violation { - pub metric: String, - pub threshold: f64, - pub actual: f64, - pub reason: Option, -} - -impl std::fmt::Display for Violation { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match &self.reason { - Some(reason) => write!(f, "{}: {reason}", self.metric), - None => write!( - f, - "{}: {} against {}", - self.metric, self.actual, self.threshold - ), - } - } -} - -#[derive(Debug, Clone, Deserialize)] -pub struct SlaVerdict { - pub passed: bool, - #[serde(default)] - pub violations: Vec, -} - -/// What the operator reads of `GET /api/tests/{id}/report`. -#[derive(Debug, Clone, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Report { - #[serde(default)] - pub summary: Value, - pub overall_sla_verdict: Option, -} - -#[derive(Debug, Clone, Default, Deserialize)] -pub struct ComplianceSummary { - pub pass: i64, - pub fail: i64, - pub warn: i64, - pub waived: i64, - pub unknown: i64, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct ComplianceResult { - pub rule: String, - pub status: String, - pub detail: Option, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct ComplianceReport { - pub verdict: String, - #[serde(default)] - pub summary: ComplianceSummary, - #[serde(default)] - pub results: Vec, -} - -/// An answered check. `check_id` is absent when the API ran it but could not store the report. -#[derive(Debug, Clone)] -pub struct ComplianceAnswer { - pub report: ComplianceReport, - pub check_id: Option, -} - /// The answer of `/api/drain`. `drained`: the switch is set and nothing is live, so the pod can /// be replaced without losing a run or leaving a fault in. #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] @@ -286,22 +167,6 @@ impl Client { self } - /// `GET /api/health`'s `capabilities`; an API older than the field lists none. - /// - /// # Errors - /// See [`Error`]. - pub async fn capabilities(&self, key: &SecretString) -> Result, Error> { - #[derive(Deserialize)] - struct Health { - #[serde(default)] - capabilities: Vec, - } - let health: Health = self - .json(Method::GET, "/api/health", "/api/health", key, None, None) - .await?; - Ok(health.capabilities) - } - /// The member's view of its quorum, from the public `GET /api/health`: no key is sent. A /// backend with no quorum, or one older than it, says `enabled: false`. /// @@ -346,121 +211,6 @@ impl Client { .await } - /// Starts a run, or finds the one `idempotency_key` already started. - /// - /// # Errors - /// See [`Error`]. - pub async fn create_test( - &self, - key: &SecretString, - idempotency_key: &str, - request: &Value, - ) -> Result { - let run: WireRun = self - .json( - Method::POST, - "/api/tests", - "/api/tests", - key, - Some(idempotency_key), - Some(request), - ) - .await?; - Ok(run.into()) - } - - /// # Errors - /// See [`Error`]. - pub async fn get_test(&self, key: &SecretString, id: &str) -> Result { - let path = format!("/api/tests/{id}"); - let run: WireRun = self - .json(Method::GET, "/api/tests/{id}", &path, key, None, None) - .await?; - Ok(run.into()) - } - - /// The run `client_key` created. An API older than the filter lists everything, so an item - /// counts only if it carries the key itself. - /// - /// # Errors - /// See [`Error`]. - pub async fn find_by_client_key( - &self, - key: &SecretString, - client_key: &str, - ) -> Result, Error> { - #[derive(Deserialize)] - struct Page { - #[serde(default)] - items: Vec, - } - let path = format!("/api/tests?clientKey={client_key}"); - let page: Page = self - .json(Method::GET, "/api/tests", &path, key, None, None) - .await?; - Ok(page - .items - .into_iter() - .find(|run| run.client_key.as_deref() == Some(client_key)) - .map(Run::from)) - } - - /// # Errors - /// `Conflict` when the run is not running. See [`Error`]. - pub async fn cancel_test(&self, key: &SecretString, id: &str) -> Result<(), Error> { - let path = format!("/api/tests/{id}/cancel"); - let _: Value = self - .json( - Method::POST, - "/api/tests/{id}/cancel", - &path, - key, - None, - None, - ) - .await?; - Ok(()) - } - - /// # Errors - /// See [`Error`]. - pub async fn report(&self, key: &SecretString, id: &str) -> Result { - let path = format!("/api/tests/{id}/report"); - self.json( - Method::GET, - "/api/tests/{id}/report", - &path, - key, - None, - None, - ) - .await - } - - /// Runs a compliance check. A stored report's id comes from the `Location` header. - /// - /// # Errors - /// See [`Error`]. - pub async fn check_compliance( - &self, - key: &SecretString, - reference: &Value, - ) -> Result { - const ROUTE: &str = "/api/conformity/check"; - let response = self - .call(Method::POST, ROUTE, ROUTE, key, None, Some(reference)) - .await?; - let check_id = response - .headers() - .get(LOCATION) - .and_then(|location| location.to_str().ok()) - .and_then(|location| location.rsplit('/').next()) - .filter(|id| !id.is_empty()) - .map(str::to_owned); - let report = decode(response).await?; - Ok(ComplianceAnswer { report, check_id }) - } - async fn json( &self, method: Method, @@ -619,7 +369,7 @@ fn classify(status: StatusCode, headers: &HeaderMap, body: &str, host: &str) -> #[cfg(test)] mod tests { use serde_json::json; - use wiremock::matchers::{header, method, path, query_param}; + use wiremock::matchers::{header, method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; use super::*; @@ -643,7 +393,7 @@ mod tests { .respond_with(response) .mount(&server) .await; - client(&server).get_test(&key(), "r").await.unwrap_err() + client(&server).drain(&key()).await.unwrap_err() } #[tokio::test] @@ -672,46 +422,6 @@ mod tests { ); } - #[tokio::test] - async fn create_sends_the_key_and_the_idempotency_key() { - let server = MockServer::start().await; - Mock::given(method("POST")) - .and(path("/prefix/api/tests")) - .and(header("authorization", "Bearer the-key")) - .and(header("idempotency-key", "uid-1")) - .respond_with(ResponseTemplate::new(202).set_body_json( - json!({"id": "r1", "status": "PENDING", "clientKey": "uid-1", "results": []}), - )) - .expect(1) - .mount(&server) - .await; - - let url = format!("{}/prefix/", server.uri()); - let run = Client::new(&url, None) - .unwrap() - .create_test(&key(), "uid-1", &json!({"type": "LOAD"})) - .await - .unwrap(); - assert_eq!(run.id, "r1"); - assert_eq!(run.status, RunStatus::Pending); - assert_eq!(run.client_key.as_deref(), Some("uid-1")); - } - - #[tokio::test] - async fn an_older_api_that_lists_everything_finds_nothing() { - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/tests")) - .and(query_param("clientKey", "uid-1")) - .respond_with(ResponseTemplate::new(200).set_body_json( - json!({"items": [{"id": "other", "status": "RUNNING"}], "total": 1}), - )) - .mount(&server) - .await; - let found = client(&server).find_by_client_key(&key(), "uid-1").await; - assert_eq!(found, Ok(None)); - } - #[tokio::test] async fn the_engine_full_answer_carries_retry_after() { let error = answering( @@ -800,44 +510,14 @@ mod tests { } #[tokio::test] - async fn the_check_id_comes_from_location_and_a_lost_report_has_none() { - let body = json!({"verdict": "FAIL", "summary": {"pass": 1, "fail": 1, "warn": 0, - "waived": 0, "unknown": 0}, "results": [{"rule": "kafka.replicas.min", - "status": "FAIL", "detail": "2 < 3"}]}); - let server = MockServer::start().await; - Mock::given(method("POST")) - .and(path("/api/conformity/check")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(&body) - .insert_header("Location", "/api/conformity/checks/c1"), - ) - .up_to_n_times(1) - .mount(&server) - .await; - Mock::given(method("POST")) - .and(path("/api/conformity/check")) - .respond_with(ResponseTemplate::new(200).set_body_json(&body)) - .mount(&server) - .await; - - let client = client(&server); - let stored = client.check_compliance(&key(), &json!({})).await.unwrap(); - assert_eq!(stored.check_id.as_deref(), Some("c1")); - assert_eq!(stored.report.summary.fail, 1); - let lost = client.check_compliance(&key(), &json!({})).await.unwrap(); - assert_eq!(lost.check_id, None); - } - - #[tokio::test] - async fn the_observer_gets_the_route_template_and_never_the_url() { + async fn the_observer_gets_the_route_template() { let seen = Arc::new(std::sync::Mutex::new(Vec::new())); let sink = seen.clone(); let server = MockServer::start().await; Mock::given(method("GET")) .respond_with( ResponseTemplate::new(200) - .set_body_json(json!({"id": "secret-id", "status": "DONE"})), + .set_body_json(json!({"draining": false, "drained": false})), ) .mount(&server) .await; @@ -845,16 +525,16 @@ mod tests { .observed_by(Arc::new(move |m, r, s| { sink.lock().unwrap().push((m, r, s)); })) - .get_test(&key(), "secret-id") + .drain(&key()) .await .unwrap(); - assert_eq!(*seen.lock().unwrap(), [("GET", "/api/tests/{id}", 200)]); + assert_eq!(*seen.lock().unwrap(), [("GET", "/api/drain", 200)]); } #[tokio::test] async fn a_network_error_says_why() { let client = Client::new("http://127.0.0.1:1", None).unwrap(); - let error = client.get_test(&key(), "r").await.unwrap_err(); + let error = client.drain(&key()).await.unwrap_err(); let Error::Unavailable { status, detail, .. } = error else { panic!("{error:?}"); }; diff --git a/operator/crates/kates-crds/src/kafkacompliance.rs b/operator/crates/kates-crds/src/kafkacompliance.rs deleted file mode 100644 index d278f83e..00000000 --- a/operator/crates/kates-crds/src/kafkacompliance.rs +++ /dev/null @@ -1,101 +0,0 @@ -use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time}; -use kube::{CustomResource, KubeSchema}; -use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -/// A standing compliance check of one Kafka. -#[derive(CustomResource, KubeSchema, Serialize, Deserialize, Clone, Debug)] -#[kube( - group = "kates.io", - version = "v1alpha1", - kind = "KafkaCompliance", - namespaced, - status = "KafkaComplianceStatus", - doc = "A standing compliance check of one Kafka. crd-version: 1", - deprecated = "kates.io/v1alpha1 KafkaCompliance is deprecated and goes in kates-operator 0.3.0: run kates compliance check from a CronJob instead", - cel, - validation = Rule::new("self.metadata.name.matches('^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$')") - .message("name must be a DNS label (at most 63 lower-case letters, digits and -): it names the stored compliance check"), - printcolumn = r#"{"name":"Verdict","type":"string","jsonPath":".status.verdict"}"#, - printcolumn = r#"{"name":"Checked","type":"date","jsonPath":".status.checkedAt"}"#, - printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"# -)] -#[serde(rename_all = "camelCase")] -pub struct KafkaComplianceSpec { - /// A `KafkaReference` `.spec`, inline; its target names the Kafka. - #[schemars(schema_with = "reference")] - pub reference: Value, - /// How often to check, such as 30m or 1h. - #[schemars(regex(pattern = r"^([0-9]+(s|m|h))+$"))] - pub interval: String, - /// Stops new checks without touching what the last one found. - #[serde(default)] - pub suspend: bool, -} - -// Pruning resumes at every declared property, so target keeps the flag too: without it the -// apiserver would drop clusterId, platform or a mistyped key before the strict reader sees them. -fn reference(_: &mut SchemaGenerator) -> Schema { - json_schema!({ - "type": "object", - "x-kubernetes-preserve-unknown-fields": true, - "required": ["target"], - "properties": { - "target": { - "type": "object", - "x-kubernetes-preserve-unknown-fields": true, - "required": ["namespace", "name"], - "properties": { - "namespace": {"type": "string"}, - "name": {"type": "string"}, - }, - }, - }, - }) -} - -#[derive(Serialize, Deserialize, JsonSchema, Clone, Debug, Default, PartialEq, Eq)] -pub struct ComplianceCounts { - pub pass: i64, - pub fail: i64, - pub unknown: i64, -} - -#[derive(Serialize, Deserialize, JsonSchema, Clone, Debug, PartialEq, Eq)] -pub struct FailedRule { - pub rule: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detail: Option, -} - -#[derive(KubeSchema, Serialize, Deserialize, Clone, Debug, Default)] -#[serde(rename_all = "camelCase")] -pub struct KafkaComplianceStatus { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub observed_generation: Option, - /// PASS, FAIL or UNKNOWN. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub verdict: Option, - /// `/api/conformity/checks/{id}`; absent when the API kept no report. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub check_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub checked_at: Option