From e2165c61ffb9caeb826410b0652e89128f649c73 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Mon, 16 Nov 2026 00:40:17 +0100 Subject: [PATCH 1/3] feat(cli): kates deploy installs Kates standalone or under the Kates operator --kates-mode picks how deploy installs the Kates API: - standalone (default, unchanged): the kates chart with its own PostgreSQL. - operator: the Kates operator (operator/chart, or the published kates-operator chart), a PostgreSQL for Kates (the kates-db release of charts/cdc-postgresql, open to the Kates pods alone, or an existing Secret named by --kates-db-secret), and a Kates resource named kates that the operator runs as one member or, with --kates-replicas 3 or 5, a quorum. The Service stays kates and the key stays in Secret kates-api-key, so kates ports, health and the context key work in either mode. --operator-ns places the operator (a scope flag, so clean and status take it too). The wizard asks the mode after the components, then the members, namespace and database. Passwords go to Helm through a 0600 values file and to kubectl on stdin, never on a command line. Deploy refuses to install one mode over the other, and refuses operator mode with --adminless. When the Kates resource does not get Ready it prints the operator's reason. The component table gains kates-operator, kates-db and the Kates CRD group, so the plan, clean and status cover them; clean deletes the Kates resource while the operator still runs. Release builds now carry the kates-operator chart version, deploy checks that chart against publish-operator.yml's signature, and publish-operator.yml attests the chart's build provenance. --- .github/workflows/publish-operator.yml | 14 +- .github/workflows/release.yml | 3 +- cli/cmd/charts_test.go | 2 +- cli/cmd/clean.go | 9 +- cli/cmd/deploy.go | 29 +- cli/cmd/deploy_chart_verify.go | 22 +- cli/cmd/deploy_charts.go | 5 +- cli/cmd/deploy_components.go | 17 +- cli/cmd/deploy_flags.go | 13 +- cli/cmd/deploy_kates_operator.go | 406 +++++++++++++++++++++++++ cli/cmd/deploy_kates_operator_test.go | 242 +++++++++++++++ cli/cmd/deploy_plan.go | 84 ++++- cli/cmd/deploy_status.go | 63 ++-- cli/cmd/deploy_status_test.go | 2 +- cli/cmd/image_mirror.go | 1 + cli/pkg/components/components.go | 21 +- cli/pkg/components/components_test.go | 13 +- docs/book/10-cli-reference.md | 4 + docs/book/12-deployment.md | 18 ++ 19 files changed, 901 insertions(+), 67 deletions(-) create mode 100644 cli/cmd/deploy_kates_operator.go create mode 100644 cli/cmd/deploy_kates_operator_test.go diff --git a/.github/workflows/publish-operator.yml b/.github/workflows/publish-operator.yml index 21046b7c6..30dec7e31 100644 --- a/.github/workflows/publish-operator.yml +++ b/.github/workflows/publish-operator.yml @@ -83,7 +83,8 @@ jobs: permissions: contents: read packages: write - id-token: write # cosign keyless signing + id-token: write # cosign keyless signing, build provenance + attestations: write # actions/attest-build-provenance env: VERSION: ${{ needs.check.outputs.version }} steps: @@ -115,6 +116,7 @@ jobs: --certificate-oidc-issuer "${ISSUER}" > /dev/null - name: Push and sign the chart + id: chart run: | set -euo pipefail # helm package is not reproducible: a re-run would replace a digest @@ -131,3 +133,13 @@ jobs: cosign sign --yes "${repo}@${digest}" cosign verify "${repo}@${digest}" --certificate-identity "${IDENTITY}" \ --certificate-oidc-issuer "${ISSUER}" > /dev/null + echo "digest=${digest}" >> "${GITHUB_OUTPUT}" + + # kates deploy --kates-mode operator refuses a published chart without it, + # as it refuses every other chart without one (cli/cmd/deploy_chart_verify.go). + - name: Attest the chart's build provenance + if: steps.chart.outputs.digest != '' + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ghcr.io/bmscomp/charts/kates-operator + subject-digest: ${{ steps.chart.outputs.digest }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6669821fa..ee25e10a2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,7 +71,8 @@ jobs: # reads them: the versions this release publishes, which `kates # deploy` pulls (cli/cmd/deploy_charts.go). CHART_VERSIONS="" - for chart in $(git -C .. ls-files ':(glob)charts/*/Chart.yaml'); do + # operator/chart is the kates-operator chart, which deploy --kates-mode operator pulls. + for chart in $(git -C .. ls-files ':(glob)charts/*/Chart.yaml' operator/chart/Chart.yaml); do name=$(sed -nE 's/^name:[[:space:]]*"?([^"[:space:]]+)"?.*$/\1/p' "../${chart}") version=$(sed -nE 's/^version:[[:space:]]*"?([^"[:space:]]+)"?.*$/\1/p' "../${chart}") CHART_VERSIONS="${CHART_VERSIONS:+${CHART_VERSIONS},}${name}=${version}" diff --git a/cli/cmd/charts_test.go b/cli/cmd/charts_test.go index 50af7a7f9..25074c87a 100644 --- a/cli/cmd/charts_test.go +++ b/cli/cmd/charts_test.go @@ -57,7 +57,7 @@ func TestAPlainHTTPMirrorIsVerifiedOverHTTP(t *testing.T) { } // allChartVersions names every chart deploy installs, as a release build does. -const allChartVersions = "kates-platform=0.8.7,cdc-postgresql=0.1.0,legacy-kafka=0.2.1,apicurio-registry=0.4.1,connect-cluster=2.1.14,kafka-cluster=1.0.18,kafka-ui=0.3.2,kates=0.14.1,kates-chaos=2.3.4,mirror-maker2=0.11.12,kates-monitoring=1.7.1,strimzi-operator=0.4.5" +const allChartVersions = "kates-platform=0.8.7,cdc-postgresql=0.1.0,legacy-kafka=0.2.1,apicurio-registry=0.4.1,connect-cluster=2.1.14,kafka-cluster=1.0.18,kafka-ui=0.3.2,kates=0.14.1,kates-chaos=2.3.4,kates-operator=0.3.0,mirror-maker2=0.11.12,kates-monitoring=1.7.1,strimzi-operator=0.4.5" func TestAChartBundleIsPulledThenReadWithNoNetwork(t *testing.T) { withChartBuild(t, allChartVersions) diff --git a/cli/cmd/clean.go b/cli/cmd/clean.go index 0bf75157c..2cc1576e1 100644 --- a/cli/cmd/clean.go +++ b/cli/cmd/clean.go @@ -41,6 +41,7 @@ var ( cleanConnectNS string cleanDbNS string cleanMM2NS string + cleanOperatorNS string cleanAppNS string cleanChaosNS string cleanMonitoringNS string @@ -58,7 +59,7 @@ func init() { Topology: &cleanTopology, Namespace: &cleanNamespace, KafkaNS: &cleanKafkaNS, ConnectNS: &cleanConnectNS, DbNS: &cleanDbNS, AppNS: &cleanAppNS, ChaosNS: &cleanChaosNS, MonitoringNS: &cleanMonitoringNS, - KafkaUINS: &cleanKafkaUINS, MM2NS: &cleanMM2NS, + KafkaUINS: &cleanKafkaUINS, MM2NS: &cleanMM2NS, OperatorNS: &cleanOperatorNS, }), scopeTopologyDefault("", "Topology to clean: 'isolated' or 'single'. If empty, cleans both.")) rootCmd.AddCommand(cleanCmd) @@ -69,12 +70,14 @@ func init() { func cleanPlacements() []components.Placement { var placements []components.Placement if cleanTopology == "single" || cleanTopology == "" { - placements = append(placements, components.SinglePlacement(cleanNamespace, cleanNamespace, deployKafkaName)) + single := components.SinglePlacement(cleanNamespace, cleanNamespace, deployKafkaName) + single.Operator = cleanOperatorNS + placements = append(placements, single) } if cleanTopology == "isolated" || cleanTopology == "" { placements = append(placements, components.Placement{ Kafka: cleanKafkaNS, Connect: cleanConnectNS, Database: cleanDbNS, App: cleanAppNS, - Chaos: cleanChaosNS, Monitoring: cleanMonitoringNS, KafkaUI: cleanKafkaUINS, MM2: cleanMM2NS, + Chaos: cleanChaosNS, Monitoring: cleanMonitoringNS, KafkaUI: cleanKafkaUINS, MM2: cleanMM2NS, Operator: cleanOperatorNS, KafkaName: deployKafkaName, }) } diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go index a9ada13fb..d0c18b981 100644 --- a/cli/cmd/deploy.go +++ b/cli/cmd/deploy.go @@ -105,10 +105,16 @@ var ( deployWithKates bool deployWithMirrorMaker2 bool deployMM2NS string - deployNamespaceLabels []string - deployNoWriteProbes bool - deployYes bool - deployUpgrade []string + deployOperatorNS string + // Standalone (the kates chart, with its PostgreSQL) or operator (the Kates + // operator, a Kates resource and a PostgreSQL of its own): deploy_kates_operator.go. + deployKatesMode string + deployKatesReplicas int + deployKatesDBSecret string + deployNamespaceLabels []string + deployNoWriteProbes bool + deployYes bool + deployUpgrade []string // Versions and scope (multi-version plan ยง3.1โ€“3.3). deployOperatorScope string @@ -133,6 +139,9 @@ func init() { deployCmd.Flags().BoolVar(&deployWithStrimzi, "with-strimzi", true, "Deploy Strimzi Operator") deployCmd.Flags().BoolVar(&deployWithKafkaConnect, "with-kafka-connect", false, "Deploy Kafka Connect with PostgreSQL CDC (Debezium)") deployCmd.Flags().BoolVar(&deployWithKates, "with-kates", true, "Deploy the Kates backend and its PostgreSQL database") + deployCmd.Flags().StringVar(&deployKatesMode, "kates-mode", katesModeStandalone, "How Kates is installed: 'standalone' (the kates chart, with its own PostgreSQL) or 'operator' (the Kates operator, a Kates resource and a PostgreSQL for it)") + deployCmd.Flags().IntVar(&deployKatesReplicas, "kates-replicas", 1, "Kates members with --kates-mode operator: 1, or 3 or 5 for a quorum that keeps a leader when members are lost. Fixed once the Kates resource exists") + deployCmd.Flags().StringVar(&deployKatesDBSecret, "kates-db-secret", "", "With --kates-mode operator: an existing Secret in the Kates namespace with keys url (jdbc:postgresql://...), username and password, instead of installing PostgreSQL") deployCmd.Flags().BoolVar(&deployWithKafkaUI, "with-kafka-ui", true, "Deploy Kafka UI for browser-based cluster monitoring") deployCmd.Flags().BoolVar(&deployWithMirrorMaker2, "with-mirror-maker2", false, "Deploy MirrorMaker 2 as a loopback mirror of the primary (replication lab; kates migrate for real sources)") deployCmd.Flags().StringArrayVar(&deployNamespaceLabels, "namespace-label", nil, "Label to apply to the namespaces deploy ensures (key=value, repeatable; overrides KATES_NAMESPACE_LABELS, which overrides the labels file)") @@ -452,6 +461,18 @@ func configureDeploySelection(cmd *cobra.Command, wizard bool) error { deployDbNS = deployNamespace } } + if err := validateKatesMode(); err != nil { + return err + } + if err := checkKatesModeSwitch(context.Background(), resolveNamespaces().app); err != nil { + return err + } + // The charts were pulled before the wizard, which may have chosen operator mode. + if katesOperatorMode() && chartsPublished { + if err := resolveChartRoot(context.Background(), deployCheckout, false, deployChartDirs()...); err != nil { + return err + } + } if err := checkUpgradeInDeploy(deployUpgrade, buildSharedEntries(resolveNamespaces())); err != nil { return err } diff --git a/cli/cmd/deploy_chart_verify.go b/cli/cmd/deploy_chart_verify.go index a831cf5da..f16634fe3 100644 --- a/cli/cmd/deploy_chart_verify.go +++ b/cli/cmd/deploy_chart_verify.go @@ -23,9 +23,19 @@ import ( // identity names the workflow rather than this build's tag. const ( chartSignerIdentity = `^https://github\.com/bmscomp/kates/\.github/workflows/publish-charts\.yml@refs/(heads/main|tags/v.+)$` - chartSignerIssuer = "https://token.actions.githubusercontent.com" + // The operator's chart is released apart, by publish-operator.yml on its own tags. + operatorChartSignerIdentity = `^https://github\.com/bmscomp/kates/\.github/workflows/publish-operator\.yml@refs/tags/operator-v.+$` + chartSignerIssuer = "https://token.actions.githubusercontent.com" ) +// chartSigner is the workflow identity that signs the chart at ref. +func chartSigner(ref string) string { + if strings.HasSuffix(ref, "/kates-operator") { + return operatorChartSignerIdentity + } + return chartSignerIdentity +} + // cosignPathFn finds cosign; tests replace it. var cosignPathFn = func() (string, error) { return exec.LookPath("cosign") } @@ -108,7 +118,7 @@ func verifyChart(ctx context.Context, ref, digest, dest string) (provenance bool return false, err } } - return verifyProvenance(ctx, cosign, digest) + return verifyProvenance(ctx, cosign, digest, chartSigner(ref)) } // verifySignature is cosign verify of ref@digest. @@ -117,7 +127,7 @@ func verifySignature(ctx context.Context, cosign, ref, digest string) error { vctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() args := []string{"verify", image, - "--certificate-identity-regexp", chartSignerIdentity, + "--certificate-identity-regexp", chartSigner(ref), "--certificate-oidc-issuer", chartSignerIssuer} if chartRegistryPlainHTTP() { args = append(args, "--allow-http-registry") @@ -191,9 +201,9 @@ func (e *chartProvenanceError) Error() string { } // verifyProvenance checks that one of GitHub's attestations for digest is a -// SLSA provenance signed by publish-charts.yml whose subject is digest. It +// SLSA provenance signed by signer's workflow whose subject is digest. It // returns false, with no error, when the attestations could not be read. -func verifyProvenance(ctx context.Context, cosign, digest string) (bool, error) { +func verifyProvenance(ctx context.Context, cosign, digest, signer string) (bool, error) { bundles, err := attestationsFn(ctx, digest) if errors.Is(err, errProvenanceUnavailable) { provenanceUnchecked = err @@ -218,7 +228,7 @@ func verifyProvenance(ctx context.Context, cosign, digest string) (bool, error) } vctx, cancel := context.WithTimeout(ctx, 2*time.Minute) out, err := runExecCombinedFn(vctx, cosign, "verify-blob-attestation", "--bundle", path, - "--certificate-identity-regexp", chartSignerIdentity, + "--certificate-identity-regexp", signer, "--certificate-oidc-issuer", chartSignerIssuer, "--type", "slsaprovenance1", "--digest", strings.TrimPrefix(digest, "sha256:"), "--digestAlg", "sha256") diff --git a/cli/cmd/deploy_charts.go b/cli/cmd/deploy_charts.go index 51edd2c65..68218462c 100644 --- a/cli/cmd/deploy_charts.go +++ b/cli/cmd/deploy_charts.go @@ -67,6 +67,7 @@ var deployChartNames = map[string]string{ "kafka-ui": "kafka-ui", "kates": "kates", "kates-chaos": "kates-chaos", + "kates-operator": "kates-operator", "kates-platform": "kates-platform", "mirror-maker2": "mirror-maker2", "monitoring": "kates-monitoring", @@ -363,7 +364,9 @@ func chartSource() string { // kates-platform umbrella, which deploy never installs but a bundle carries, // verified, for a helm install of it without network. func deployChartDirs() []string { - return slices.DeleteFunc(publishedChartDirs(), func(dir string) bool { return dir == "legacy-kafka" || dir == "kates-platform" }) + return slices.DeleteFunc(publishedChartDirs(), func(dir string) bool { + return dir == "legacy-kafka" || dir == "kates-platform" || (dir == "kates-operator" && !katesOperatorMode()) + }) } // publishedChartDirs lists every chart the CLI pulls, deploy's and migrate's. diff --git a/cli/cmd/deploy_components.go b/cli/cmd/deploy_components.go index 5e0faa6e3..8ace552e0 100644 --- a/cli/cmd/deploy_components.go +++ b/cli/cmd/deploy_components.go @@ -255,6 +255,9 @@ func selectedDeployNamespaces(dc *deployContext) []string { if deployWithKates { add(dc.ns.app) } + if componentSelected("kates-operator") { + add(dc.ns.operator) + } if deployWithMonitoring { add(dc.ns.jaeger) } @@ -631,6 +634,8 @@ func deployComponents(dc *deployContext) []component { dc.kafkaConnectComponent(), dc.kafkaConnectorComponent(), dc.apicurioComponent(), + dc.katesOperatorComponent(), + dc.katesDBComponent(), dc.katesComponent(), dc.kafkaUIComponent(), dc.mirrorMaker2Component(), @@ -1684,9 +1689,13 @@ func (dc *deployContext) katesComponent() component { kafkaNS := dc.ns.kafka appNS := dc.ns.app chaosNS := dc.ns.chaos + deps := []string{"kafka-users", "postgres", "monitoring"} + if katesOperatorMode() { + deps = append(deps, "kates-operator", "kates-db") + } return component{ id: "kates", name: "Kates Backend", namespace: appNS, - deps: []string{"kafka-users", "postgres", "monitoring"}, timeout: 8 * time.Minute, + deps: deps, timeout: 10 * time.Minute, enabled: func() bool { return deployWithKates }, // The KafkaUser was already waited on (kafka-users) โ€” just read the Secret. secrets: []secretCopy{{name: "kates-backend"}}, @@ -1698,6 +1707,9 @@ func (dc *deployContext) katesComponent() component { return fmt.Sprintf("\n๐Ÿ“ฆ %s Kates Backend (Namespace: %s)...", verb, appNS) }, install: func(ctx context.Context) error { + if katesOperatorMode() { + return dc.installKatesResource(ctx) + } // Auto-cleanup stale ClusterRole ownership from previous topology switches cleanupStaleClusterResource(ctx, "clusterrole", "kates", appNS) cleanupStaleClusterResource(ctx, "clusterrolebinding", "kates", appNS) @@ -1759,6 +1771,9 @@ func (dc *deployContext) katesComponent() component { return runKatesHelmUpgrade(ctx, appNS, katesArgs...) }, wait: func(ctx context.Context) error { + if katesOperatorMode() { + return waitKatesResource(ctx, appNS) + } return dc.waitPods(ctx, appNS, "app.kubernetes.io/instance=kates", 8*time.Minute, "") }, } diff --git a/cli/cmd/deploy_flags.go b/cli/cmd/deploy_flags.go index 14cf4450d..761cb00e5 100644 --- a/cli/cmd/deploy_flags.go +++ b/cli/cmd/deploy_flags.go @@ -1,6 +1,10 @@ package cmd -import "github.com/spf13/cobra" +import ( + "github.com/spf13/cobra" + + "github.com/bmscomp/kates/cli/pkg/components" +) // The topology and namespace flags place the stack. deploy, deploy status, // clean and kyverno apply all take them, and each used to declare its own @@ -12,7 +16,7 @@ import "github.com/spf13/cobra" // deployScopeVars are the variables the scope flags bind. deploy, deploy // status and kyverno apply share deploy's package globals; clean has its own. type deployScopeVars struct { - Topology, Namespace, KafkaNS, ConnectNS, DbNS, AppNS, ChaosNS, MonitoringNS, KafkaUINS, MM2NS *string + Topology, Namespace, KafkaNS, ConnectNS, DbNS, AppNS, ChaosNS, MonitoringNS, KafkaUINS, MM2NS, OperatorNS *string } // deployGlobalScope binds the flags to deploy's package-level variables. @@ -21,7 +25,7 @@ func deployGlobalScope() deployScopeVars { Topology: &deployTopology, Namespace: &deployNamespace, KafkaNS: &deployKafkaNS, ConnectNS: &deployConnectNS, DbNS: &deployDbNS, AppNS: &deployAppNS, ChaosNS: &deployChaosNS, MonitoringNS: &deployMonitoringNS, - KafkaUINS: &deployKafkaUINS, MM2NS: &deployMM2NS, + KafkaUINS: &deployKafkaUINS, MM2NS: &deployMM2NS, OperatorNS: &deployOperatorNS, } } @@ -46,7 +50,7 @@ func scopeTopologyDefault(value, usage string) deployScopeOption { // deployScopeFlagNames is the set addDeployScopeFlags registers, for the test // that checks every command has it. -var deployScopeFlagNames = []string{"topology", "namespace", "kafka-ns", "connect-ns", "db-ns", "app-ns", "chaos-ns", "monitoring-ns", "ui-ns", "mm2-ns", "kafka-name"} +var deployScopeFlagNames = []string{"topology", "namespace", "kafka-ns", "connect-ns", "db-ns", "app-ns", "chaos-ns", "monitoring-ns", "ui-ns", "mm2-ns", "operator-ns", "kafka-name"} // addDeployScopeFlags registers the topology and namespace flags on cmd. func addDeployScopeFlags(cmd *cobra.Command, opts ...deployScopeOption) { @@ -70,6 +74,7 @@ func addDeployScopeFlags(cmd *cobra.Command, opts ...deployScopeOption) { f.StringVar(v.MonitoringNS, "monitoring-ns", "monitoring", "Namespace for monitoring components (Jaeger) when topology is 'isolated'") f.StringVar(v.KafkaUINS, "ui-ns", "kafka", "Namespace for Kafka UI when topology is 'isolated'") f.StringVar(v.MM2NS, "mm2-ns", "kafka", "Namespace for MirrorMaker 2 when topology is 'isolated'") + f.StringVar(v.OperatorNS, "operator-ns", components.DefaultOperatorNamespace, "Namespace for the Kates operator (--kates-mode operator), under either topology") // The Kafka release is named by --kafka-name everywhere; clean already // bound deploy's variable, so it stays shared. f.StringVar(&deployKafkaName, "kafka-name", "krafter", "Name of the primary Kafka cluster") diff --git a/cli/cmd/deploy_kates_operator.go b/cli/cmd/deploy_kates_operator.go new file mode 100644 index 000000000..1497a15c1 --- /dev/null +++ b/cli/cmd/deploy_kates_operator.go @@ -0,0 +1,406 @@ +package cmd + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "math/big" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +// --kates-mode: how deploy installs Kates. Standalone is the kates chart with +// its own PostgreSQL. Operator installs the Kates operator, a PostgreSQL for +// Kates (the kates-db release of charts/cdc-postgresql, unless +// --kates-db-secret names one) and a Kates resource the operator turns into +// one backend, or a quorum of three or five. +const ( + katesModeStandalone = "standalone" + katesModeOperator = "operator" + + // The Kates resource's name: its Service is then kates, as the chart's + // is, so ports, the API key and the dashboards find it unchanged. + katesResourceName = "kates" + katesDBRelease = "kates-db" + // The Secret the Kates resource reads its database from, with --kates-db-secret unset. + katesDBConnection = "kates-db-connection" +) + +func katesOperatorMode() bool { return deployKatesMode == katesModeOperator } + +// validateKatesMode refuses flag combinations the chosen mode cannot honour, +// before anything reaches the cluster. +func validateKatesMode() error { + switch deployKatesMode { + case katesModeStandalone, katesModeOperator: + default: + return fmt.Errorf("--kates-mode %q: use 'standalone' or 'operator'", deployKatesMode) + } + if !slices.Contains([]int{1, 3, 5}, deployKatesReplicas) { + return fmt.Errorf("--kates-replicas %d: use 1, or 3 or 5 for a quorum", deployKatesReplicas) + } + if katesOperatorMode() { + if deployAdminless { + return errors.New("--kates-mode operator installs the Kates operator and its CRD, which needs cluster admin: drop --adminless, or keep --kates-mode standalone") + } + return nil + } + if deployKatesReplicas != 1 { + return errors.New("--kates-replicas needs --kates-mode operator: the kates chart runs one backend") + } + if deployKatesDBSecret != "" { + return errors.New("--kates-db-secret needs --kates-mode operator: the kates chart brings its own PostgreSQL") + } + return nil +} + +// checkKatesModeSwitch refuses to install Kates one way over an install made +// the other way: both would claim the Service kates. +func checkKatesModeSwitch(ctx context.Context, appNS string) error { + if !deployWithKates { + return nil + } + if katesOperatorMode() && isHelmReleaseDeployedFn(ctx, "kates", appNS) { + return fmt.Errorf("kates is installed standalone in %s (Helm release kates): uninstall it first (kates clean, or helm uninstall kates -n %s), or keep --kates-mode standalone", appNS, appNS) + } + if !katesOperatorMode() && katesResourceExists(ctx, appNS) { + return fmt.Errorf("kates is installed by the Kates operator in %s (Kates resource %s): use --kates-mode operator, or delete it first (kubectl delete kates %s -n %s)", appNS, katesResourceName, katesResourceName, appNS) + } + return nil +} + +func katesResourceExists(ctx context.Context, namespace string) bool { + out, err := runExecOutputFn(ctx, "kubectl", "get", "kates.kates.io", katesResourceName, "-n", namespace, + "--ignore-not-found", "-o", "name") + return err == nil && len(bytes.TrimSpace(out)) > 0 +} + +// operatorChartPath is the operator chart: operator/chart in a checkout, +// charts/kates-operator among the published packages. +func operatorChartPath() string { + if chartsPublished { + return chartRootPath("charts/kates-operator") + } + return chartRootPath("operator/chart") +} + +// katesOperatorComponent is the Kates operator, watching the Kates namespace +// and binding the backend's cluster role to the Kates pods there. +func (dc *deployContext) katesOperatorComponent() component { + operatorNS := dc.ns.operator + appNS := dc.ns.app + return component{ + id: "kates-operator", name: "Kates Operator", namespace: operatorNS, timeout: 5 * time.Minute, + enabled: func() bool { return componentSelected("kates-operator") }, + banner: func(context.Context) string { + return fmt.Sprintf("\n๐Ÿค– Deploying the Kates Operator (Namespace: %s, watching %s)...", operatorNS, appNS) + }, + install: func(ctx context.Context) error { + // Deploy makes each namespace itself: Helm's --create-namespace is dropped. + if err := ensureConfiguredDeployNamespace(ctx, operatorNS); err != nil { + return err + } + args := []string{"upgrade", "--install", "kates-operator", operatorChartPath(), + "-n", operatorNS, "--create-namespace", + "--set-json", "watchNamespaces=" + jsonList(appNS), + "--set-json", "rbac.backendNamespaces=" + jsonList(appNS), + "--wait", "--timeout", "5m"} + if mirror := imageRegistry(); mirror != "" { + args = append(args, "--set-string", "global.imageRegistry="+mirror) + } + args = append(args, dc.policyArgs("kates-operator")...) + return runHelmFn(ctx, args...) + }, + wait: func(ctx context.Context) error { + return dc.waitPods(ctx, operatorNS, "app.kubernetes.io/name=kates-operator", 5*time.Minute, "") + }, + } +} + +// katesDBComponent is the PostgreSQL the Kates resource stores in: this +// repository's cdc-postgresql chart as release kates-db, open to the Kates +// pods alone, and the Secret kates-db-connection that names it. +func (dc *deployContext) katesDBComponent() component { + appNS := dc.ns.app + return component{ + id: "kates-db", name: "PostgreSQL (Kates)", namespace: appNS, timeout: 5 * time.Minute, + enabled: func() bool { return componentSelected("kates-db") }, + banner: func(context.Context) string { + return fmt.Sprintf("\n๐Ÿ˜ Deploying PostgreSQL for Kates (Namespace: %s)...", appNS) + }, + install: func(ctx context.Context) error { + if err := ensureConfiguredDeployNamespace(ctx, appNS); err != nil { + return err + } + // Kept across runs: a new password would lock Kates out of its data. + password, err := secretValueOrNew(ctx, appNS, katesDBRelease, "password") + if err != nil { + return err + } + admin, err := secretValueOrNew(ctx, appNS, katesDBRelease, "postgres-password") + if err != nil { + return err + } + // In a 0600 file, not --set: --verbose prints every Helm command. + values, err := writePrivateValues(dc.valuesFile, "kates-db-values.yaml", map[string]any{ + "auth": map[string]any{ + "username": "kates", "database": "kates", + "password": password, "postgresPassword": admin, + }, + "networkPolicy": map[string]any{ + "allowPods": []map[string]string{{"app.kubernetes.io/instance": katesResourceName}}, + }, + }) + if err != nil { + return err + } + defer os.Remove(values) + args := append([]string{"upgrade", "--install", katesDBRelease, chartRootPath("charts/cdc-postgresql"), + "-n", appNS, "--create-namespace", "-f", values, "--timeout", "5m"}, + dc.policyArgs("cdc-postgresql")...) + if err := runHelmFn(ctx, args...); err != nil { + return err + } + return applySecret(ctx, appNS, katesDBConnection, map[string]string{ + "url": fmt.Sprintf("jdbc:postgresql://%s.%s.svc:5432/kates", katesDBRelease, appNS), + "username": "kates", + "password": password, + }) + }, + wait: func(ctx context.Context) error { + return dc.waitPods(ctx, appNS, "app.kubernetes.io/instance="+katesDBRelease, 5*time.Minute, "") + }, + } +} + +// installKatesResource is the Kates backend under the operator: its API key, +// then the Kates resource, which the operator turns into a StatefulSet and a +// Service named kates. +func (dc *deployContext) installKatesResource(ctx context.Context) error { + appNS := dc.ns.app + if err := ensureConfiguredDeployNamespace(ctx, appNS); err != nil { + return err + } + database := katesDBConnection + if deployKatesDBSecret != "" { + database = deployKatesDBSecret + if out, err := runExecOutputFn(ctx, "kubectl", "get", "secret", database, "-n", appNS, + "--ignore-not-found", "-o", "name"); err != nil || len(bytes.TrimSpace(out)) == 0 { + return fmt.Errorf("--kates-db-secret %s: no such Secret in %s; it needs keys url (jdbc:postgresql://...), username and password", database, appNS) + } + } + // The key Secret the kates chart makes, under the name and key the CLI and + // the operator's drain read; kept across runs, as the chart keeps it. + key, err := secretValueOrNew(ctx, appNS, apiKeySecretName, "api-key") + if err != nil { + return err + } + if err := applySecret(ctx, appNS, apiKeySecretName, map[string]string{"api-key": key}); err != nil { + return err + } + version, image, err := dc.katesImage(ctx) + if err != nil { + return err + } + manifest, err := katesResourceManifest(katesResourceSpec{ + namespace: appNS, version: version, image: image, replicas: deployKatesReplicas, + database: database, + bootstrap: fmt.Sprintf("%s-kafka-bootstrap.%s.svc:9092", dc.primary.Name, dc.primary.Namespace), + kafkaNS: dc.ns.kafka, cluster: dc.primary.Name, oneBroker: oneBroker(), + }) + if err != nil { + return err + } + dl.Printf(" - Kates resource %s/%s: %d member(s), image %s:%s\n", appNS, katesResourceName, deployKatesReplicas, image, version) + return runExecStdinFn(ctx, "kubectl", []string{"apply", "--server-side", "--field-manager", "kates-deploy", "-f", "-"}, manifest) +} + +// waitKatesResource waits for the operator to report the backend Ready: every +// member up and, with a quorum, a leader. +func waitKatesResource(ctx context.Context, appNS string) error { + err := runExecFn(ctx, "kubectl", "wait", "--for=condition=Ready", "kates.kates.io/"+katesResourceName, + "-n", appNS, "--timeout=10m") + if err == nil { + return nil + } + // The operator says why in the condition; a bare timeout says nothing. + out, readErr := runExecOutputFn(ctx, "kubectl", "get", "kates.kates.io", katesResourceName, "-n", appNS, + "-o", `jsonpath={.status.conditions[?(@.type=="Ready")].reason}: {.status.conditions[?(@.type=="Ready")].message}`) + if why := strings.TrimSpace(string(out)); readErr == nil && why != "" && why != ":" { + return fmt.Errorf("the Kates resource %s/%s is not Ready (%s): %w", appNS, katesResourceName, why, err) + } + return err +} + +// katesImage is the image the backend runs, as the chart path picks it: a +// local build on kind when one is there, else the published native image. +func (dc *deployContext) katesImage(ctx context.Context) (version, image string, err error) { + if dc.isKind { + args, err := kindBackendImageArgs(ctx, dc.report.Context) + if err != nil { + return "", "", err + } + for _, a := range args { + if v, ok := strings.CutPrefix(a, "image.tag="); ok { + version = v + } + if v, ok := strings.CutPrefix(a, "image.repository="); ok { + image = v + } + } + } + if version == "" { + if version, err = valuesImageTag(chartRootPath(publishedNativeValues)); err != nil { + return "", "", err + } + } + if image == "" { + image = "ghcr.io/bmscomp/kates" + if mirror := imageRegistry(); mirror != "" { + image = mirror + "/bmscomp/kates" + } + } + return version, image, nil +} + +// valuesImageTag reads image.tag from a values file. +func valuesImageTag(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + var values struct { + Image struct { + Tag string `yaml:"tag"` + } `yaml:"image"` + } + if err := yaml.Unmarshal(data, &values); err != nil { + return "", fmt.Errorf("%s: %w", path, err) + } + if values.Image.Tag == "" { + return "", fmt.Errorf("%s sets no image.tag", path) + } + return values.Image.Tag, nil +} + +type katesResourceSpec struct { + namespace, version, image, database, bootstrap, kafkaNS, cluster string + replicas int + oneBroker bool +} + +// katesResourceManifest is the Kates resource, with the Kafka settings the +// kates chart's values give the backend. +func katesResourceManifest(s katesResourceSpec) (string, error) { + env := []map[string]string{ + // What the chart's ConfigMap sets from kafka.clusterName and the Kafka namespace. + {"name": "KATES_CHAOS_KAFKA_NAMESPACE", "value": s.kafkaNS}, + {"name": "KATES_TOPOLOGY_KAFKA_NAMESPACE", "value": s.kafkaNS}, + {"name": "KATES_CHAOS_KAFKA_CLUSTER", "value": s.cluster}, + {"name": "KATES_TOPOLOGY_KAFKA_CLUSTER", "value": s.cluster}, + {"name": "KATES_CDC_TOPIC_NAMESPACE", "value": s.kafkaNS}, + } + if s.oneBroker { + for _, name := range []string{"KATES_DEFAULTS_REPLICATION_FACTOR", "KATES_DEFAULTS_MIN_INSYNC_REPLICAS", "KATES_OUTBOX_TOPIC_REPLICATION_FACTOR"} { + env = append(env, map[string]string{"name": name, "value": "1"}) + } + } + resource := map[string]any{ + "apiVersion": "kates.io/v1alpha1", + "kind": "Kates", + "metadata": map[string]any{"name": katesResourceName, "namespace": s.namespace}, + "spec": map[string]any{ + "version": s.version, + "image": s.image, + "replicas": s.replicas, + "database": map[string]any{"secretRef": map[string]any{"name": s.database}}, + "kafka": map[string]any{ + "bootstrapServers": s.bootstrap, + "securityProtocol": "SASL_PLAINTEXT", + "sasl": map[string]any{ + "mechanism": "SCRAM-SHA-512", + "username": "kates-backend", + "passwordSecretRef": map[string]any{"name": "kates-backend", "key": "password"}, + }, + }, + "apiKeySecretRef": map[string]any{"name": apiKeySecretName, "key": "api-key"}, + "env": env, + }, + } + out, err := yaml.Marshal(resource) + return string(out), err +} + +// secretValueOrNew is key of an existing Secret, or a new random value. +func secretValueOrNew(ctx context.Context, namespace, name, key string) (string, error) { + out, err := runExecOutputFn(ctx, "kubectl", "get", "secret", name, "-n", namespace, + "--ignore-not-found", "-o", "jsonpath={.data."+jsonpathKey(key)+"}") + if err == nil && len(bytes.TrimSpace(out)) > 0 { + raw, decodeErr := base64.StdEncoding.DecodeString(string(bytes.TrimSpace(out))) + if decodeErr != nil { + return "", fmt.Errorf("secret %s/%s key %s is not base64", namespace, name, key) + } + return string(raw), nil + } + return randomAlnum(32) +} + +const alnum = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +func randomAlnum(n int) (string, error) { + var b strings.Builder + for range n { + i, err := rand.Int(rand.Reader, big.NewInt(int64(len(alnum)))) + if err != nil { + return "", err + } + b.WriteByte(alnum[i.Int64()]) + } + return b.String(), nil +} + +// applySecret writes Secret name in namespace, through stdin so no value is +// on a command line. +func applySecret(ctx context.Context, namespace, name string, data map[string]string) error { + secret := map[string]any{ + "apiVersion": "v1", "kind": "Secret", "type": "Opaque", + "metadata": map[string]any{"name": name, "namespace": namespace}, + "stringData": data, + } + out, err := yaml.Marshal(secret) + if err != nil { + return err + } + if err := runExecStdinFn(ctx, "kubectl", []string{"apply", "-f", "-"}, string(out)); err != nil { + return fmt.Errorf("secret %s could not be written in %s: %w", name, namespace, err) + } + return nil +} + +// writePrivateValues writes values beside the deploy's values file, readable +// by this user alone. +func writePrivateValues(beside, name string, values map[string]any) (string, error) { + out, err := yaml.Marshal(values) + if err != nil { + return "", err + } + path := filepath.Join(filepath.Dir(beside), name) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return "", err + } + return path, os.WriteFile(path, out, 0o600) +} + +func jsonList(items ...string) string { + out, _ := json.Marshal(items) + return string(out) +} diff --git a/cli/cmd/deploy_kates_operator_test.go b/cli/cmd/deploy_kates_operator_test.go new file mode 100644 index 000000000..05d02ac5c --- /dev/null +++ b/cli/cmd/deploy_kates_operator_test.go @@ -0,0 +1,242 @@ +package cmd + +import ( + "context" + "encoding/base64" + "errors" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// setKatesMode sets the operator-mode flags for one test. +func setKatesMode(t *testing.T, mode string, replicas int, dbSecret string, adminless bool) { + t.Helper() + m, r, s, a, k := deployKatesMode, deployKatesReplicas, deployKatesDBSecret, deployAdminless, deployWithKates + t.Cleanup(func() { + deployKatesMode, deployKatesReplicas, deployKatesDBSecret, deployAdminless, deployWithKates = m, r, s, a, k + }) + deployKatesMode, deployKatesReplicas, deployKatesDBSecret, deployAdminless, deployWithKates = mode, replicas, dbSecret, adminless, true +} + +func TestKatesModeFlagsAreCheckedBeforeAnyInstall(t *testing.T) { + for _, tc := range []struct { + name, mode, secret string + replicas int + adminless bool + wantErr string + }{ + {name: "standalone default", mode: "standalone", replicas: 1}, + {name: "operator, one member", mode: "operator", replicas: 1}, + {name: "operator quorum and own database", mode: "operator", replicas: 5, secret: "db"}, + {name: "unknown mode", mode: "helm", replicas: 1, wantErr: "--kates-mode"}, + {name: "even quorum", mode: "operator", replicas: 2, wantErr: "--kates-replicas 2"}, + {name: "replicas without the operator", mode: "standalone", replicas: 3, wantErr: "needs --kates-mode operator"}, + {name: "database without the operator", mode: "standalone", replicas: 1, secret: "db", wantErr: "--kates-db-secret needs"}, + {name: "operator admin-less", mode: "operator", replicas: 1, adminless: true, wantErr: "cluster admin"}, + } { + t.Run(tc.name, func(t *testing.T) { + setKatesMode(t, tc.mode, tc.replicas, tc.secret, tc.adminless) + err := validateKatesMode() + if tc.wantErr == "" && err != nil { + t.Fatalf("refused: %v", err) + } + if tc.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tc.wantErr)) { + t.Fatalf("error = %v, want one naming %q", err, tc.wantErr) + } + }) + } +} + +func TestOperatorModeSelectsTheOperatorAndADatabaseUnlessOneIsNamed(t *testing.T) { + setKatesMode(t, "standalone", 1, "", false) + if componentSelected("kates-operator") || componentSelected("kates-db") || slices.Contains(deployChartDirs(), "kates-operator") { + t.Fatal("standalone selects the operator, its database or its chart") + } + setKatesMode(t, "operator", 3, "", false) + if !componentSelected("kates-operator") || !componentSelected("kates-db") || !slices.Contains(deployChartDirs(), "kates-operator") { + t.Fatal("operator mode lacks the operator, the database or the operator chart") + } + setKatesMode(t, "operator", 3, "mine", false) + if componentSelected("kates-db") { + t.Fatal("--kates-db-secret still installs PostgreSQL") + } +} + +func TestTheKatesResourceCarriesTheChartsKafkaSettings(t *testing.T) { + manifest, err := katesResourceManifest(katesResourceSpec{ + namespace: "kates", version: "1.41.0-native", image: "ghcr.io/bmscomp/kates", replicas: 3, + database: katesDBConnection, bootstrap: "krafter-kafka-bootstrap.kafka.svc:9092", + kafkaNS: "kafka", cluster: "krafter", oneBroker: true, + }) + if err != nil { + t.Fatal(err) + } + var got struct { + Kind string `yaml:"kind"` + Metadata struct{ Name, Namespace string } + Spec struct { + Version string `yaml:"version"` + Replicas int `yaml:"replicas"` + Database struct { + SecretRef struct{ Name string } `yaml:"secretRef"` + } `yaml:"database"` + Kafka struct { + BootstrapServers string `yaml:"bootstrapServers"` + Sasl struct { + Username string `yaml:"username"` + PasswordSecretRef struct{ Name, Key string } `yaml:"passwordSecretRef"` + } `yaml:"sasl"` + } `yaml:"kafka"` + APIKeySecretRef struct{ Name, Key string } `yaml:"apiKeySecretRef"` + Env []struct{ Name, Value string } + } `yaml:"spec"` + } + if err := yaml.Unmarshal([]byte(manifest), &got); err != nil { + t.Fatalf("%v\n%s", err, manifest) + } + if got.Kind != "Kates" || got.Metadata.Name != "kates" || got.Spec.Replicas != 3 || got.Spec.Version != "1.41.0-native" { + t.Fatalf("resource = %+v", got) + } + if got.Spec.Database.SecretRef.Name != katesDBConnection || got.Spec.Kafka.Sasl.PasswordSecretRef.Name != "kates-backend" || + got.Spec.APIKeySecretRef.Name != "kates-api-key" || got.Spec.APIKeySecretRef.Key != "api-key" { + t.Fatalf("secret references = %+v", got.Spec) + } + env := map[string]string{} + for _, e := range got.Spec.Env { + env[e.Name] = e.Value + } + if env["KATES_TOPOLOGY_KAFKA_CLUSTER"] != "krafter" || env["KATES_CHAOS_KAFKA_NAMESPACE"] != "kafka" || env["KATES_OUTBOX_TOPIC_REPLICATION_FACTOR"] != "1" { + t.Fatalf("env = %v", env) + } +} + +// operatorModeCluster stubs the cluster calls operator mode makes: Secret +// reads answer from secrets, applies and helm calls are recorded. +func operatorModeCluster(t *testing.T, secrets map[string]string) (applied, helm *[]string) { + t.Helper() + origOut, origStdin, origHelm, origDeployed := runExecOutputFn, runExecStdinFn, runHelmFn, isHelmReleaseDeployedFn + t.Cleanup(func() { + runExecOutputFn, runExecStdinFn, runHelmFn, isHelmReleaseDeployedFn = origOut, origStdin, origHelm, origDeployed + }) + applied, helm = &[]string{}, &[]string{} + runExecOutputFn = func(_ context.Context, _ string, args ...string) ([]byte, error) { + joined := strings.Join(args, " ") + for key, value := range secrets { + if strings.Contains(joined, key) { + return []byte(value), nil + } + } + return nil, nil + } + runExecStdinFn = func(_ context.Context, _ string, _ []string, stdin string) error { + *applied = append(*applied, stdin) + return nil + } + runHelmFn = func(_ context.Context, args ...string) error { + *helm = append(*helm, strings.Join(args, " ")) + return nil + } + isHelmReleaseDeployedFn = func(context.Context, string, string) bool { return false } + return applied, helm +} + +func TestKatesDBKeepsItsPasswordOffTheCommandLineAndAcrossRuns(t *testing.T) { + setKatesMode(t, "operator", 1, "", false) + kept := base64.StdEncoding.EncodeToString([]byte("kept-password")) + applied, helm := operatorModeCluster(t, map[string]string{ + "secret kates-db -n kates --ignore-not-found -o jsonpath={.data.password}": kept, + "get namespace": `{"metadata":{"name":"kates"}}`, + }) + dc := &deployContext{ctx: context.Background(), ns: resolvedNamespaces{app: "kates"}, + valuesFile: filepath.Join(t.TempDir(), "values-detected.yaml")} + if err := dc.katesDBComponent().install(context.Background()); err != nil { + t.Fatal(err) + } + if len(*helm) != 1 || !strings.Contains((*helm)[0], "upgrade --install kates-db") || strings.Contains((*helm)[0], "kept-password") { + t.Fatalf("helm = %v, want one kates-db upgrade with no password in it", *helm) + } + values := (*helm)[0][strings.Index((*helm)[0], "-f ")+3:] + values = strings.Fields(values)[0] + if _, err := os.Stat(values); !os.IsNotExist(err) { + t.Errorf("the values file %s was left behind", values) + } + connection := (*applied)[len(*applied)-1] + for _, want := range []string{"name: kates-db-connection", "jdbc:postgresql://kates-db.kates.svc:5432/kates", "password: kept-password"} { + if !strings.Contains(connection, want) { + t.Fatalf("connection Secret lacks %q:\n%s", want, connection) + } + } +} + +func TestTheKatesResourceGetsAKeyAndNamesAnExistingDatabase(t *testing.T) { + setKatesMode(t, "operator", 3, "own-db", false) + origRoot := chartRoot + t.Cleanup(func() { chartRoot = origRoot }) + chartRoot = "../.." + key := base64.StdEncoding.EncodeToString([]byte("existing-key")) + applied, _ := operatorModeCluster(t, map[string]string{ + "secret own-db -n kates": "secret/own-db", + "kates-api-key -n kates": key, + "get namespace": `{"metadata":{"name":"kates"}}`, + }) + dc := &deployContext{ctx: context.Background(), ns: resolvedNamespaces{app: "kates", kafka: "kafka"}, + primary: primaryCluster{Name: "krafter", Namespace: "kafka"}} + if err := dc.installKatesResource(context.Background()); err != nil { + t.Fatal(err) + } + all := strings.Join(*applied, "\n---\n") + for _, want := range []string{"api-key: existing-key", "kind: Kates", "name: own-db", "replicas: 3"} { + if !strings.Contains(all, want) { + t.Fatalf("applied lacks %q:\n%s", want, all) + } + } +} + +func TestAMissingDatabaseSecretIsNamed(t *testing.T) { + setKatesMode(t, "operator", 1, "nope", false) + operatorModeCluster(t, map[string]string{"get namespace": `{"metadata":{"name":"kates"}}`}) + dc := &deployContext{ctx: context.Background(), ns: resolvedNamespaces{app: "kates"}} + err := dc.installKatesResource(context.Background()) + if err == nil || !strings.Contains(err.Error(), "--kates-db-secret nope") { + t.Fatalf("error = %v", err) + } +} + +func TestOneModeIsNotInstalledOverTheOther(t *testing.T) { + setKatesMode(t, "operator", 1, "", false) + operatorModeCluster(t, nil) + isHelmReleaseDeployedFn = func(_ context.Context, release, _ string) bool { return release == "kates" } + if err := checkKatesModeSwitch(context.Background(), "kates"); err == nil || !strings.Contains(err.Error(), "installed standalone") { + t.Fatalf("operator over a standalone install: %v", err) + } + setKatesMode(t, "standalone", 1, "", false) + operatorModeCluster(t, map[string]string{"kates.kates.io kates": "kates.kates.io/kates"}) + if err := checkKatesModeSwitch(context.Background(), "kates"); err == nil || !strings.Contains(err.Error(), "by the Kates operator") { + t.Fatalf("standalone over an operator install: %v", err) + } +} + +func TestTheOperatorChartIsCheckedAgainstItsOwnSigner(t *testing.T) { + if chartSigner("oci://ghcr.io/bmscomp/charts/kates-operator") != operatorChartSignerIdentity { + t.Fatal("the operator chart is checked against publish-charts.yml") + } + if chartSigner("oci://ghcr.io/bmscomp/charts/kates") != chartSignerIdentity { + t.Fatal("the kates chart is not checked against publish-charts.yml") + } +} + +func TestANotReadyKatesResourceSaysWhy(t *testing.T) { + operatorModeCluster(t, map[string]string{"jsonpath={.status.conditions": "NoLeader: the quorum has no leader"}) + origExec := runExecFn + t.Cleanup(func() { runExecFn = origExec }) + runExecFn = func(context.Context, string, ...string) error { return errors.New("timed out") } + err := waitKatesResource(context.Background(), "kates") + if err == nil || !strings.Contains(err.Error(), "NoLeader: the quorum has no leader") { + t.Fatalf("error = %v", err) + } +} diff --git a/cli/cmd/deploy_plan.go b/cli/cmd/deploy_plan.go index 3102feb4f..42fc28b12 100644 --- a/cli/cmd/deploy_plan.go +++ b/cli/cmd/deploy_plan.go @@ -32,6 +32,13 @@ func runInteractiveForms() error { applyPickedComponents(picked) + // โ”€โ”€ Form: how Kates is installed โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + if deployWithKates && !deployAdminless { + if err := katesModeFormFn(); err != nil { + return err + } + } + // โ”€โ”€ Form: Versions and scope โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ // The Kafka options are a function of the operator chosen above them: // pick 1.0.1 and the list is its window, pick 1.1.0 and it is 1.1.0's. @@ -53,6 +60,45 @@ func runInteractiveForms() error { return runDeployReviewForm() } +// katesModeFormFn asks how Kates is installed; tests replace it. +var katesModeFormFn = runKatesModeForm + +// runKatesModeForm asks standalone or operator and, for operator, the +// members, the operator's namespace and an existing database. Two forms, so +// the second is built once the mode is known. +func runKatesModeForm() error { + err := huh.NewForm(huh.NewGroup( + huh.NewSelect[string](). + Title("How should Kates be installed?"). + Options( + huh.NewOption("Standalone: the kates chart, with its own PostgreSQL", katesModeStandalone), + huh.NewOption("Operator: the Kates operator, a Kates resource and a PostgreSQL for it", katesModeOperator), + ). + Value(&deployKatesMode), + )).WithTheme(ThemeKates()).WithAccessible(accessibleUI()).Run() + if err != nil || !katesOperatorMode() { + return err + } + return huh.NewForm(huh.NewGroup( + huh.NewSelect[int](). + Title("Kates members"). + Description("3 or 5 run a quorum that keeps a leader with one or two members lost. Fixed once installed"). + Options( + huh.NewOption("1", 1), + huh.NewOption("3 (quorum)", 3), + huh.NewOption("5 (quorum)", 5), + ). + Value(&deployKatesReplicas), + huh.NewInput(). + Title("Operator Namespace"). + Value(&deployOperatorNS), + huh.NewInput(). + Title("Existing database Secret (optional)"). + Description("A Secret in the Kates namespace with keys url, username and password. Empty: install PostgreSQL"). + Value(&deployKatesDBSecret), + )).WithTheme(ThemeKates()).WithAccessible(accessibleUI()).Run() +} + // runComponentForm is Form 1: the topology, the schema registry, HA and // the components, which it stores in picked. func runComponentForm(picked *[]string) error { @@ -339,29 +385,33 @@ type resolvedNamespaces struct { jaeger string kafkaUI string mm2 string + // operator is the Kates operator's namespace, under either topology. + operator string } // resolveNamespaces resolves namespace values based on the selected topology. func resolveNamespaces() resolvedNamespaces { if deployTopology == "single" { return resolvedNamespaces{ - kafka: deployNamespace, - connect: deployNamespace, - app: deployNamespace, - chaos: deployNamespace, - jaeger: deployNamespace, - kafkaUI: deployNamespace, - mm2: deployNamespace, + kafka: deployNamespace, + connect: deployNamespace, + app: deployNamespace, + chaos: deployNamespace, + jaeger: deployNamespace, + kafkaUI: deployNamespace, + mm2: deployNamespace, + operator: deployOperatorNS, } } return resolvedNamespaces{ - kafka: deployKafkaNS, - connect: deployConnectNS, - app: deployAppNS, - chaos: deployChaosNS, - jaeger: deployMonitoringNS, - kafkaUI: deployKafkaUINS, - mm2: deployMM2NS, + kafka: deployKafkaNS, + connect: deployConnectNS, + app: deployAppNS, + chaos: deployChaosNS, + jaeger: deployMonitoringNS, + kafkaUI: deployKafkaUINS, + mm2: deployMM2NS, + operator: deployOperatorNS, } } @@ -372,7 +422,7 @@ func deployPlacement(ns resolvedNamespaces) components.Placement { return components.Placement{ Kafka: ns.kafka, Connect: ns.connect, Database: deployDbNS, App: ns.app, Chaos: ns.chaos, Monitoring: ns.jaeger, KafkaUI: ns.kafkaUI, MM2: ns.mm2, - KafkaName: deployKafkaName, + Operator: ns.operator, KafkaName: deployKafkaName, } } @@ -400,6 +450,10 @@ func componentSelected(id string) bool { return deployWithChaos case "kates": return deployWithKates + case "kates-operator": + return deployWithKates && katesOperatorMode() + case "kates-db": + return deployWithKates && katesOperatorMode() && deployKatesDBSecret == "" case "kafka", "kafka-users": return true } diff --git a/cli/cmd/deploy_status.go b/cli/cmd/deploy_status.go index 21cad76f6..1a5d6df8e 100644 --- a/cli/cmd/deploy_status.go +++ b/cli/cmd/deploy_status.go @@ -150,6 +150,8 @@ type componentSpec struct { Namespace string Kind string Resource string + // Managed: no Helm release of its own, the Kates operator runs it. + Managed bool } // statusComponents is what `kates deploy status` inspects, in display order. @@ -157,21 +159,23 @@ type componentSpec struct { // check handles โ€” the check used to name the Strimzi kinds it knew, and a // component whose kind was not on that list reported a blank "Unknown" // forever without anything failing. -func statusComponents(kafkaNS, connectNS, appNS, chaosNS, jaegerNS, dbNS, kafkaUINS, mm2NS string) []componentSpec { +func statusComponents(kafkaNS, connectNS, appNS, chaosNS, jaegerNS, dbNS, kafkaUINS, mm2NS, operatorNS string) []componentSpec { return []componentSpec{ - {"A", "๐ŸฆŠ", "Strimzi Operator", "strimzi-operator", "strimzi-operator", "pod", "-l name=strimzi-cluster-operator"}, - {"A", "๐Ÿ”", "Cert-Manager", "cert-manager", "cert-manager", "pod", "-l app.kubernetes.io/instance=cert-manager"}, - {"A", "๐Ÿšฆ", "Kyverno", "kyverno", "kyverno", "pod", "-l app.kubernetes.io/instance=kyverno"}, - {"B", "๐Ÿ“จ", "Kafka (" + deployKafkaName + ")", deployKafkaName, kafkaNS, "kafkas.kafka.strimzi.io", deployKafkaName}, - {"B", "๐Ÿ˜", "PostgreSQL (CDC)", "postgresql", dbNS, "statefulset", "postgresql"}, - {"B", "๐Ÿ”—", "Kafka Connect", "connect-cluster", connectNS, "kafkaconnects.kafka.strimzi.io", "connect-cluster"}, - {"B", "๐Ÿ“Š", "Monitoring Stack", "monitoring", jaegerNS, "pod", "-l release=monitoring"}, - {"C", "๐Ÿ“‹", "Apicurio Registry", "apicurio", kafkaNS, "pod", "-l app.kubernetes.io/instance=apicurio"}, + {"A", "๐ŸฆŠ", "Strimzi Operator", "strimzi-operator", "strimzi-operator", "pod", "-l name=strimzi-cluster-operator", false}, + {"A", "๐Ÿ”", "Cert-Manager", "cert-manager", "cert-manager", "pod", "-l app.kubernetes.io/instance=cert-manager", false}, + {"A", "๐Ÿšฆ", "Kyverno", "kyverno", "kyverno", "pod", "-l app.kubernetes.io/instance=kyverno", false}, + {"A", "๐Ÿค–", "Kates Operator", "kates-operator", operatorNS, "pod", "-l app.kubernetes.io/name=kates-operator", false}, + {"B", "๐Ÿ“จ", "Kafka (" + deployKafkaName + ")", deployKafkaName, kafkaNS, "kafkas.kafka.strimzi.io", deployKafkaName, false}, + {"B", "๐Ÿ˜", "PostgreSQL (CDC)", "postgresql", dbNS, "statefulset", "postgresql", false}, + {"B", "๐Ÿ”—", "Kafka Connect", "connect-cluster", connectNS, "kafkaconnects.kafka.strimzi.io", "connect-cluster", false}, + {"B", "๐Ÿ“Š", "Monitoring Stack", "monitoring", jaegerNS, "pod", "-l release=monitoring", false}, + {"C", "๐Ÿ“‹", "Apicurio Registry", "apicurio", kafkaNS, "pod", "-l app.kubernetes.io/instance=apicurio", false}, + {"C", "๐Ÿ˜", "PostgreSQL (Kates)", "kates-db", appNS, "statefulset", "kates-db", false}, // By name: the instance label also matches the chart's embedded PostgreSQL pod. - {"C", "๐Ÿ“ฆ", "Kates Backend", "kates", appNS, "deployment", "kates"}, - {"C", "๐Ÿ’ป", "Kafka UI", "kafka-ui", kafkaUINS, "pod", "-l app.kubernetes.io/name=kafka-ui"}, - {"C", "๐Ÿ”", "MirrorMaker 2", "mm2", mm2NS, "kafkamirrormaker2s.kafka.strimzi.io", "mm2-mirror-maker2"}, - {"C", "๐Ÿงช", "Litmus Chaos", "chaos", chaosNS, "pod", "-l app.kubernetes.io/instance=chaos"}, + {"C", "๐Ÿ“ฆ", "Kates Backend", "kates", appNS, "deployment", "kates", false}, + {"C", "๐Ÿ’ป", "Kafka UI", "kafka-ui", kafkaUINS, "pod", "-l app.kubernetes.io/name=kafka-ui", false}, + {"C", "๐Ÿ”", "MirrorMaker 2", "mm2", mm2NS, "kafkamirrormaker2s.kafka.strimzi.io", "mm2-mirror-maker2", false}, + {"C", "๐Ÿงช", "Litmus Chaos", "chaos", chaosNS, "pod", "-l app.kubernetes.io/instance=chaos", false}, } } @@ -194,7 +198,15 @@ func runDeployStatus(cmd *cobra.Command, args []string) error { helmIndex := loadHelmReleaseIndex(ctx) - components := statusComponents(ns.kafka, ns.connect, ns.app, ns.chaos, ns.jaeger, dbNS, ns.kafkaUI, ns.mm2) + components := statusComponents(ns.kafka, ns.connect, ns.app, ns.chaos, ns.jaeger, dbNS, ns.kafkaUI, ns.mm2, ns.operator) + // Under the operator the backend is a Kates resource's StatefulSet. + if katesResourceExists(ctx, ns.app) { + for i := range components { + if components[i].Release == "kates" { + components[i].Kind, components[i].Managed = "statefulset", true + } + } + } // Route based on flags if deployStatusOutput == "json" { @@ -239,8 +251,8 @@ func runDeployStatus(cmd *cobra.Command, args []string) error { Release: c.Release, Namespace: c.Namespace, } - st.HelmStatus, st.Namespace = getHelmStatusWithNamespace(ctx, helmIndex, c.Release, c.Namespace) - if st.HelmStatus == "deployed" { + st.HelmStatus, st.Namespace = componentHelmStatus(ctx, helmIndex, c) + if st.HelmStatus == "deployed" || st.HelmStatus == operatorManaged { st.Health, st.Details = getHealthStatus(ctx, c.Kind, c.Resource, st.Namespace) } else { st.Health = "N/A" @@ -307,6 +319,17 @@ func lookupHelmRelease(index map[string][]helm.Release, release, namespace strin return "elsewhere (" + strings.Join(elsewhere, ", ") + ")", namespace, true } +// operatorManaged is the Helm status column of a component the Kates operator runs. +const operatorManaged = "operator" + +// componentHelmStatus is a row's Helm status, or operatorManaged for one the operator runs. +func componentHelmStatus(ctx context.Context, index map[string][]helm.Release, c componentSpec) (status, namespace string) { + if c.Managed { + return operatorManaged, c.Namespace + } + return getHelmStatusWithNamespace(ctx, index, c.Release, c.Namespace) +} + func getHelmStatusWithNamespace(ctx context.Context, index map[string][]helm.Release, release, namespace string) (status, resolvedNamespace string) { if st, ns, ok := lookupHelmRelease(index, release, namespace); ok { return st, ns @@ -598,8 +621,8 @@ func runJSONStatus(ctx context.Context, components []componentSpec) ([]Component Release: comp.Release, Namespace: comp.Namespace, } - st.HelmStatus, st.Namespace = getHelmStatusWithNamespace(ctx, helmIndex, comp.Release, comp.Namespace) - if st.HelmStatus == "deployed" { + st.HelmStatus, st.Namespace = componentHelmStatus(ctx, helmIndex, comp) + if st.HelmStatus == "deployed" || st.HelmStatus == operatorManaged { st.Health, st.Details, st.RawDetails = getHealthStatusV2(ctx, comp.Kind, comp.Resource, st.Namespace) } else { st.Health = "N/A" @@ -751,8 +774,8 @@ func runInteractiveStatus(ctx context.Context, components []componentSpec) ([]Co Release: comp.Release, Namespace: comp.Namespace, } - st.HelmStatus, st.Namespace = getHelmStatusWithNamespace(ctx, helmIndex, comp.Release, comp.Namespace) - if st.HelmStatus == "deployed" { + st.HelmStatus, st.Namespace = componentHelmStatus(ctx, helmIndex, comp) + if st.HelmStatus == "deployed" || st.HelmStatus == operatorManaged { st.Health, st.Details, st.RawDetails = getHealthStatusV2(ctx, comp.Kind, comp.Resource, st.Namespace) } else { st.Health = "N/A" diff --git a/cli/cmd/deploy_status_test.go b/cli/cmd/deploy_status_test.go index 928b321b0..69c1821e0 100644 --- a/cli/cmd/deploy_status_test.go +++ b/cli/cmd/deploy_status_test.go @@ -268,7 +268,7 @@ func TestParsePodListHealth(t *testing.T) { // "Unknown" in every status output. Nothing failed, because nothing checked // that the table and the health check agreed. func TestEveryStatusComponentCanBeChecked(t *testing.T) { - for _, c := range statusComponents("kafka", "connect", "kates", "litmus", "monitoring", "database", "kafka", "kafka") { + for _, c := range statusComponents("kafka", "connect", "kates", "litmus", "monitoring", "database", "kafka", "kafka", "kates-operator") { if !healthCheckKinds(c.Kind) { t.Errorf("%s is listed with kind %q, which no health check handles โ€” it would report a blank Unknown", c.Name, c.Kind) } diff --git a/cli/cmd/image_mirror.go b/cli/cmd/image_mirror.go index 1e844f8eb..f93f3c91c 100644 --- a/cli/cmd/image_mirror.go +++ b/cli/cmd/image_mirror.go @@ -55,6 +55,7 @@ var mirrorSets = map[string][]mirrorSet{ "connect-cluster": {{key: "global.imageRegistry"}}, "mirror-maker2": {{key: "global.imageRegistry"}}, "cdc-postgresql": {{key: "global.imageRegistry"}}, + "kates-operator": {{key: "global.imageRegistry"}}, // kube-prometheus-stack's global.imageRegistry replaces each image's // registry: the second name. "monitoring": {{key: "global.imageRegistry"}}, diff --git a/cli/pkg/components/components.go b/cli/pkg/components/components.go index d75292fd9..2afbb1d8b 100644 --- a/cli/pkg/components/components.go +++ b/cli/pkg/components/components.go @@ -36,6 +36,7 @@ const ( NSMonitoring NamespaceKey = "monitoring" NSKafkaUI NamespaceKey = "ui" NSMM2 NamespaceKey = "mm2" + NSOperator NamespaceKey = "operator" ) // Placement is where the user put each part of the stack: the --*-ns flags @@ -43,7 +44,9 @@ const ( // KafkaName is the primary Kafka cluster's name, which is also its release. type Placement struct { Kafka, Connect, Database, App, Chaos, Monitoring, KafkaUI, MM2 string - KafkaName string + // Operator is the Kates operator's namespace, the same in both topologies. + Operator string + KafkaName string } // DefaultPlacement is the isolated topology's defaults, the ones the deploy @@ -52,13 +55,16 @@ func DefaultPlacement() Placement { return Placement{ Kafka: "kafka", Connect: "connect", Database: "database", App: "kates", Chaos: "litmus", Monitoring: "monitoring", KafkaUI: "kafka", MM2: "kafka", - KafkaName: "krafter", + Operator: DefaultOperatorNamespace, KafkaName: "krafter", } } // SingleNamespace is the single topology's default namespace. const SingleNamespace = "kates-stack" +// DefaultOperatorNamespace is where the Kates operator goes unless --operator-ns says otherwise. +const DefaultOperatorNamespace = "kates-operator" + // SinglePlacement puts every namespaced component in namespace. The database // is placed separately because deploy keeps it on --db-ns under both // topologies (an existing database is never moved), while clean folds it in. @@ -66,7 +72,7 @@ func SinglePlacement(namespace, database, kafkaName string) Placement { return Placement{ Kafka: namespace, Connect: namespace, Database: database, App: namespace, Chaos: namespace, Monitoring: namespace, KafkaUI: namespace, MM2: namespace, - KafkaName: kafkaName, + Operator: DefaultOperatorNamespace, KafkaName: kafkaName, } } @@ -135,6 +141,8 @@ func (c Component) Namespace(p Placement) string { return p.KafkaUI case NSMM2: return p.MM2 + case NSOperator: + return p.Operator } return "" } @@ -151,6 +159,7 @@ const ( CRDKyverno = "Kyverno" CRDLitmus = "Litmus" CRDPrometheus = "Prometheus Operator" + CRDKates = "Kates" ) // table is the stack, in install order. @@ -161,6 +170,8 @@ var table = []Component{ FixedNamespace: "cert-manager", Operator: true, CRDGroup: CRDCertManager, Selector: "app.kubernetes.io/instance=cert-manager"}, {ID: "kyverno", Name: "Kyverno", Icon: "๐Ÿšฆ", Group: GroupOperators, Release: "kyverno", Chart: "kyverno/kyverno", FixedNamespace: "kyverno", Operator: true, CRDGroup: CRDKyverno, Selector: "app.kubernetes.io/instance=kyverno"}, + {ID: "kates-operator", Name: "Kates Operator", Icon: "๐Ÿค–", Group: GroupOperators, Release: "kates-operator", Chart: "operator/chart", + NamespaceKey: NSOperator, Operator: true, CRDGroup: CRDKates, Selector: "app.kubernetes.io/name=kates-operator"}, {ID: "kafka", Name: "Kafka Cluster", Icon: "๐Ÿ“จ", Group: GroupCore, Release: KafkaRelease, Chart: "charts/kafka-cluster", NamespaceKey: NSKafka, Selector: "strimzi.io/cluster=%s"}, @@ -175,6 +186,8 @@ var table = []Component{ {ID: "apicurio", Name: "Apicurio Registry", Icon: "๐Ÿ“‹", Group: GroupApps, Release: "apicurio", Chart: "charts/apicurio-registry", NamespaceKey: NSKafka, Selector: "app.kubernetes.io/instance=apicurio"}, + {ID: "kates-db", Name: "PostgreSQL (Kates)", Icon: "๐Ÿ˜", Group: GroupApps, Release: "kates-db", Chart: "charts/cdc-postgresql", + NamespaceKey: NSApp, Selector: "app.kubernetes.io/instance=kates-db"}, {ID: "kates", Name: "Kates Backend", Icon: "๐Ÿ“ฆ", Group: GroupApps, Release: "kates", Chart: "charts/kates", NamespaceKey: NSApp, Selector: "app.kubernetes.io/instance=kates"}, {ID: "kafka-ui", Name: "Kafka UI", Icon: "๐Ÿ’ป", Group: GroupApps, Release: "kafka-ui", Chart: "charts/kafka-ui", @@ -334,6 +347,8 @@ var ( }, {Name: CRDLitmus, CRDs: litmusCRDs, UsedBy: litmusCRDs, Namespaced: litmusCRDs}, {Name: CRDPrometheus, CRDs: prometheusCRDs, UsedBy: prometheusCRDs}, + // No Owner: the operator's namespace is --operator-ns, not fixed. + {Name: CRDKates, CRDs: []string{"kates.kates.io"}, UsedBy: []string{"kates.kates.io"}, Namespaced: []string{"kates.kates.io"}}, } ) diff --git a/cli/pkg/components/components_test.go b/cli/pkg/components/components_test.go index e16b1deb8..eee1db036 100644 --- a/cli/pkg/components/components_test.go +++ b/cli/pkg/components/components_test.go @@ -25,7 +25,7 @@ func TestTableIsConsistent(t *testing.T) { if c.Installs() != (c.Chart != "") || c.Installs() != (c.Icon != "") { t.Errorf("%s: a release needs a chart and an icon, a step neither (release %q chart %q icon %q)", c.ID, c.Release, c.Chart, c.Icon) } - if c.Operator && c.FixedNamespace == "" { + if c.Operator && c.FixedNamespace == "" && c.NamespaceKey != NSOperator { t.Errorf("%s: an operator has a namespace of its own", c.ID) } if c.CRDGroup != "" { @@ -78,13 +78,14 @@ func TestReleasesAndNamespaces(t *testing.T) { "strimzi-operator/strimzi-operator", "cert-manager/cert-manager", "kyverno/kyverno", "krafter/kafka", "monitoring/monitoring", "postgresql/database", "connect-cluster/connect", "apicurio/kafka", "kates/kates", "kafka-ui/kafka", "mm2/kafka", "chaos/litmus", + "kates-operator/kates-operator", "kates-db/kates", } { if !slices.Contains(names, want) { t.Errorf("Releases lacks %s: %v", want, names) } } - if len(names) != 12 { - t.Errorf("Releases = %v, want exactly the 12 releases", names) + if len(names) != 14 { + t.Errorf("Releases = %v, want exactly the 14 releases", names) } for _, n := range names { if strings.HasPrefix(n, "jaeger/") { @@ -93,7 +94,7 @@ func TestReleasesAndNamespaces(t *testing.T) { } ns := Namespaces(p) - want := []string{"kafka", "monitoring", "database", "connect", "kates", "litmus", "strimzi-operator", "cert-manager", "kyverno"} + want := []string{"kafka", "monitoring", "database", "connect", "kates", "litmus", "strimzi-operator", "cert-manager", "kyverno", "kates-operator"} if !slices.Equal(ns, want) { t.Errorf("Namespaces = %v, want %v", ns, want) } @@ -101,8 +102,8 @@ func TestReleasesAndNamespaces(t *testing.T) { func TestCRDGroups(t *testing.T) { all := CRDs() - if len(all) != 39 { - t.Errorf("CRDs() lists %d, want 39", len(all)) + if len(all) != 40 { + t.Errorf("CRDs() lists %d, want 40", len(all)) } for _, g := range Groups() { for _, crd := range g.UsedBy { diff --git a/docs/book/10-cli-reference.md b/docs/book/10-cli-reference.md index dbf9c4598..11c3be85d 100644 --- a/docs/book/10-cli-reference.md +++ b/docs/book/10-cli-reference.md @@ -1554,6 +1554,10 @@ Its flags choose the shape of the stack and how much it asks: | `--checkout` | Install the charts of the Kates checkout in this directory instead of the published ones; `KATES_CHECKOUT` does the same for every command | | `--chart-bundle` | Install from the bundle that [`kates charts pull --to`](#charts-pull) wrote in this directory, with no network; `KATES_CHART_BUNDLE` does the same for every command | | `--adminless` | Install without cluster admin, into namespaces that exist, on a Strimzi operator a cluster admin installed; see [Deploying Without Cluster Admin](12-deployment.md#deploying-without-cluster-admin) | +| `--kates-mode` | `standalone` (default): the `kates` chart, with its own PostgreSQL. `operator`: the Kates operator, a PostgreSQL for Kates, and a `Kates` resource the operator runs; see [Standalone or Under the Operator](12-deployment.md#standalone-or-under-the-operator) | +| `--kates-replicas` | With `--kates-mode operator`: `1` (default), or `3` or `5` for a quorum that keeps a leader with one or two members lost. Fixed once the `Kates` resource exists | +| `--kates-db-secret` | With `--kates-mode operator`: an existing Secret in the Kates namespace, with keys `url` (`jdbc:postgresql://โ€ฆ`), `username` and `password`, instead of installing PostgreSQL | +| `--operator-ns` | Namespace of the Kates operator, under either topology (default: `kates-operator`) | `--operator-scope namespace` lets older Kafka lines run beside the primary under an operator of their own. The loopback MirrorMaker 2 of `--with-mirror-maker2` is the chart's shipped shape, enough to exercise connectors, offset syncs, checkpoints and the `kates-mm2` ACLs without a second cluster. `--adminless` installs Kafka, Kafka Connect with its PostgreSQL, MirrorMaker 2, Apicurio, Kafka UI and Kates, placed by `--topology` and the `--*-ns` flags as in a full deploy; `--topology single` takes `--namespace`, by default the kube context's. diff --git a/docs/book/12-deployment.md b/docs/book/12-deployment.md index 0f6e388b0..70a055054 100644 --- a/docs/book/12-deployment.md +++ b/docs/book/12-deployment.md @@ -131,6 +131,24 @@ kubectl scale deployment/kates --replicas=1 The old PersistentVolumeClaim stays until you delete it. +### Standalone or Under the Operator + +`kates deploy` installs the Kates API one of two ways, chosen with `--kates-mode`, or in the wizard right after the components: + +| | `standalone` (default) | `operator` | +|---|---|---| +| What runs the API | The `kates` chart: a Deployment | The Kates operator (`operator/chart`), from a `Kates` resource: a StatefulSet | +| PostgreSQL | The chart's own | The `kates-db` release of `charts/cdc-postgresql`, open to the Kates pods alone, or your own with `--kates-db-secret` | +| Members | One | One, or three or five with `--kates-replicas`: a quorum whose leader takes the writes | +| Upgrades | A rollout | The operator rolls followers first and the leader last, and under `mode: Drain` waits for live runs | +| Needs cluster admin | No (`--adminless`) | Yes: the operator brings a CRD and a ClusterRole | + +```bash +kates deploy --kates-mode operator --kates-replicas 3 +``` + +Either way the Service is `kates` in the Kates namespace, and the API key is in Secret `kates-api-key`, so `kates ports`, `kates health` and the rest work the same. Deploy refuses to install one mode over the other: uninstall first (`kates clean`), or keep the mode you have. `kates clean` removes the `Kates` resource while the operator still runs, then the operator. The operator doesn't yet support the `INTEGRATION_CDC` test. + For Kafka broker storage, always use persistent volumes โ€” even in development. Kafka's log retention depends on data being durable, and losing broker data mid-test invalidates results. ## Resource Sizing From 36685bdba5cd193f414ef83bc6f431476bb31ba9 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Mon, 16 Nov 2026 00:47:02 +0100 Subject: [PATCH 2/3] docs(book): regenerate the index for the deployment chapter's new section --- docs/book/appendix-index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/book/appendix-index.md b/docs/book/appendix-index.md index 6aca047c9..b8cefe2ef 100644 --- a/docs/book/appendix-index.md +++ b/docs/book/appendix-index.md @@ -419,7 +419,7 @@ Every term the book teaches, with the sections that teach it: the ones that use ## Q [Quorum]{#idx-quorum} -: [The Cluster Under Test](03-cluster.md): [Kafka You Need for This Book](03-cluster.md#kafka-you-need-for-this-book), [Replication Configuration](03-cluster.md#replication-configuration); [Observability & Monitoring](09-observability.md): [The Dashboard Set](09-observability.md#the-dashboard-set); [Deploying the Strimzi Operator](deploying-strimzi-operator.md): [Adopting an Operator Installed from the Upstream Chart](deploying-strimzi-operator.md#migrating-from-the-ad-hoc-install); [Kafka Deployment Engineering](15-kafka-deployment.md): [Cluster Architecture](15-kafka-deployment.md#cluster-architecture); [Compliance Checks](compliance.md): [Writing Constraints](compliance.md#writing-constraints), [Placing Pods Across Zones and Nodes](compliance.md#placing-pods-across-zones-and-nodes); [REST API Reference](11-api-reference.md): [Endpoints](11-api-reference.md#endpoints) +: [The Cluster Under Test](03-cluster.md): [Kafka You Need for This Book](03-cluster.md#kafka-you-need-for-this-book), [Replication Configuration](03-cluster.md#replication-configuration); [Observability & Monitoring](09-observability.md): [The Dashboard Set](09-observability.md#the-dashboard-set); [Kafka Deployment Engineering](15-kafka-deployment.md): [Cluster Architecture](15-kafka-deployment.md#cluster-architecture); [Compliance Checks](compliance.md): [Writing Constraints](compliance.md#writing-constraints), [Placing Pods Across Zones and Nodes](compliance.md#placing-pods-across-zones-and-nodes); [CLI Reference](10-cli-reference.md): [Commands](10-cli-reference.md#commands); [REST API Reference](11-api-reference.md): [Endpoints](11-api-reference.md#endpoints) [Quota]{#idx-quota} : [Kafka Deployment Engineering](15-kafka-deployment.md): [Broker Settings and Why](15-kafka-deployment.md#broker-settings-and-why); [Multi-Tenancy](19-multi-tenancy.md): [How Quotas Work](19-multi-tenancy.md#how-quotas-work), [Monitoring a Tenant](19-multi-tenancy.md#monitoring-a-tenant) From f64b59ee7c4ca37f3435b31b257e073e05999718 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Mon, 16 Nov 2026 01:13:06 +0100 Subject: [PATCH 3/3] test(disruption): the scheduled-run test ticks twice inside one minute aScheduledRunStoresItsReportOnce evaluated an every-minute schedule at now + 61 s and 10 s later. With now past second 49, the second tick fell in the next minute, whose due time rightly ran too: "execute wanted 1 time, was 2", about one run in six. Both ticks now sit in the minute after now's. --- .../kates/disruption/DisruptionReportPersistenceTest.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionReportPersistenceTest.java b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionReportPersistenceTest.java index 8b52b3140..35c07c331 100644 --- a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionReportPersistenceTest.java +++ b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionReportPersistenceTest.java @@ -240,8 +240,12 @@ void aScheduledRunStoresItsReportOnce() { .extract() .path("id"); try { - // Two members' ticks after the next minute: the due time runs once. - java.time.Instant late = java.time.Instant.now().plusSeconds(61); + // Two members' ticks after the next minute: the due time runs once. Both inside the + // same minute: from now + 61 s, the second tick crossed into the minute after once + // now was past second 49, and that next due time rightly ran too. + java.time.Instant late = java.time.Instant.now() + .truncatedTo(java.time.temporal.ChronoUnit.MINUTES) + .plusSeconds(61); scheduler.evaluate(late); scheduler.evaluate(late.plusSeconds(10)); verify(orchestrator, times(1)).execute(any(), any());