diff --git a/.github/workflows/publish-operator.yml b/.github/workflows/publish-operator.yml index 21046b7c6..30dec7e31 100644 --- a/.github/workflows/publish-operator.yml +++ b/.github/workflows/publish-operator.yml @@ -83,7 +83,8 @@ jobs: permissions: contents: read packages: write - id-token: write # cosign keyless signing + id-token: write # cosign keyless signing, build provenance + attestations: write # actions/attest-build-provenance env: VERSION: ${{ needs.check.outputs.version }} steps: @@ -115,6 +116,7 @@ jobs: --certificate-oidc-issuer "${ISSUER}" > /dev/null - name: Push and sign the chart + id: chart run: | set -euo pipefail # helm package is not reproducible: a re-run would replace a digest @@ -131,3 +133,13 @@ jobs: cosign sign --yes "${repo}@${digest}" cosign verify "${repo}@${digest}" --certificate-identity "${IDENTITY}" \ --certificate-oidc-issuer "${ISSUER}" > /dev/null + echo "digest=${digest}" >> "${GITHUB_OUTPUT}" + + # kates deploy --kates-mode operator refuses a published chart without it, + # as it refuses every other chart without one (cli/cmd/deploy_chart_verify.go). + - name: Attest the chart's build provenance + if: steps.chart.outputs.digest != '' + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ghcr.io/bmscomp/charts/kates-operator + subject-digest: ${{ steps.chart.outputs.digest }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6669821fa..ee25e10a2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,7 +71,8 @@ jobs: # reads them: the versions this release publishes, which `kates # deploy` pulls (cli/cmd/deploy_charts.go). CHART_VERSIONS="" - for chart in $(git -C .. ls-files ':(glob)charts/*/Chart.yaml'); do + # operator/chart is the kates-operator chart, which deploy --kates-mode operator pulls. + for chart in $(git -C .. ls-files ':(glob)charts/*/Chart.yaml' operator/chart/Chart.yaml); do name=$(sed -nE 's/^name:[[:space:]]*"?([^"[:space:]]+)"?.*$/\1/p' "../${chart}") version=$(sed -nE 's/^version:[[:space:]]*"?([^"[:space:]]+)"?.*$/\1/p' "../${chart}") CHART_VERSIONS="${CHART_VERSIONS:+${CHART_VERSIONS},}${name}=${version}" diff --git a/cli/cmd/charts_test.go b/cli/cmd/charts_test.go index 50af7a7f9..25074c87a 100644 --- a/cli/cmd/charts_test.go +++ b/cli/cmd/charts_test.go @@ -57,7 +57,7 @@ func TestAPlainHTTPMirrorIsVerifiedOverHTTP(t *testing.T) { } // allChartVersions names every chart deploy installs, as a release build does. -const allChartVersions = "kates-platform=0.8.7,cdc-postgresql=0.1.0,legacy-kafka=0.2.1,apicurio-registry=0.4.1,connect-cluster=2.1.14,kafka-cluster=1.0.18,kafka-ui=0.3.2,kates=0.14.1,kates-chaos=2.3.4,mirror-maker2=0.11.12,kates-monitoring=1.7.1,strimzi-operator=0.4.5" +const allChartVersions = "kates-platform=0.8.7,cdc-postgresql=0.1.0,legacy-kafka=0.2.1,apicurio-registry=0.4.1,connect-cluster=2.1.14,kafka-cluster=1.0.18,kafka-ui=0.3.2,kates=0.14.1,kates-chaos=2.3.4,kates-operator=0.3.0,mirror-maker2=0.11.12,kates-monitoring=1.7.1,strimzi-operator=0.4.5" func TestAChartBundleIsPulledThenReadWithNoNetwork(t *testing.T) { withChartBuild(t, allChartVersions) diff --git a/cli/cmd/clean.go b/cli/cmd/clean.go index 0bf75157c..2cc1576e1 100644 --- a/cli/cmd/clean.go +++ b/cli/cmd/clean.go @@ -41,6 +41,7 @@ var ( cleanConnectNS string cleanDbNS string cleanMM2NS string + cleanOperatorNS string cleanAppNS string cleanChaosNS string cleanMonitoringNS string @@ -58,7 +59,7 @@ func init() { Topology: &cleanTopology, Namespace: &cleanNamespace, KafkaNS: &cleanKafkaNS, ConnectNS: &cleanConnectNS, DbNS: &cleanDbNS, AppNS: &cleanAppNS, ChaosNS: &cleanChaosNS, MonitoringNS: &cleanMonitoringNS, - KafkaUINS: &cleanKafkaUINS, MM2NS: &cleanMM2NS, + KafkaUINS: &cleanKafkaUINS, MM2NS: &cleanMM2NS, OperatorNS: &cleanOperatorNS, }), scopeTopologyDefault("", "Topology to clean: 'isolated' or 'single'. If empty, cleans both.")) rootCmd.AddCommand(cleanCmd) @@ -69,12 +70,14 @@ func init() { func cleanPlacements() []components.Placement { var placements []components.Placement if cleanTopology == "single" || cleanTopology == "" { - placements = append(placements, components.SinglePlacement(cleanNamespace, cleanNamespace, deployKafkaName)) + single := components.SinglePlacement(cleanNamespace, cleanNamespace, deployKafkaName) + single.Operator = cleanOperatorNS + placements = append(placements, single) } if cleanTopology == "isolated" || cleanTopology == "" { placements = append(placements, components.Placement{ Kafka: cleanKafkaNS, Connect: cleanConnectNS, Database: cleanDbNS, App: cleanAppNS, - Chaos: cleanChaosNS, Monitoring: cleanMonitoringNS, KafkaUI: cleanKafkaUINS, MM2: cleanMM2NS, + Chaos: cleanChaosNS, Monitoring: cleanMonitoringNS, KafkaUI: cleanKafkaUINS, MM2: cleanMM2NS, Operator: cleanOperatorNS, KafkaName: deployKafkaName, }) } diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go index a9ada13fb..d0c18b981 100644 --- a/cli/cmd/deploy.go +++ b/cli/cmd/deploy.go @@ -105,10 +105,16 @@ var ( deployWithKates bool deployWithMirrorMaker2 bool deployMM2NS string - deployNamespaceLabels []string - deployNoWriteProbes bool - deployYes bool - deployUpgrade []string + deployOperatorNS string + // Standalone (the kates chart, with its PostgreSQL) or operator (the Kates + // operator, a Kates resource and a PostgreSQL of its own): deploy_kates_operator.go. + deployKatesMode string + deployKatesReplicas int + deployKatesDBSecret string + deployNamespaceLabels []string + deployNoWriteProbes bool + deployYes bool + deployUpgrade []string // Versions and scope (multi-version plan ยง3.1โ€“3.3). deployOperatorScope string @@ -133,6 +139,9 @@ func init() { deployCmd.Flags().BoolVar(&deployWithStrimzi, "with-strimzi", true, "Deploy Strimzi Operator") deployCmd.Flags().BoolVar(&deployWithKafkaConnect, "with-kafka-connect", false, "Deploy Kafka Connect with PostgreSQL CDC (Debezium)") deployCmd.Flags().BoolVar(&deployWithKates, "with-kates", true, "Deploy the Kates backend and its PostgreSQL database") + deployCmd.Flags().StringVar(&deployKatesMode, "kates-mode", katesModeStandalone, "How Kates is installed: 'standalone' (the kates chart, with its own PostgreSQL) or 'operator' (the Kates operator, a Kates resource and a PostgreSQL for it)") + deployCmd.Flags().IntVar(&deployKatesReplicas, "kates-replicas", 1, "Kates members with --kates-mode operator: 1, or 3 or 5 for a quorum that keeps a leader when members are lost. Fixed once the Kates resource exists") + deployCmd.Flags().StringVar(&deployKatesDBSecret, "kates-db-secret", "", "With --kates-mode operator: an existing Secret in the Kates namespace with keys url (jdbc:postgresql://...), username and password, instead of installing PostgreSQL") deployCmd.Flags().BoolVar(&deployWithKafkaUI, "with-kafka-ui", true, "Deploy Kafka UI for browser-based cluster monitoring") deployCmd.Flags().BoolVar(&deployWithMirrorMaker2, "with-mirror-maker2", false, "Deploy MirrorMaker 2 as a loopback mirror of the primary (replication lab; kates migrate for real sources)") deployCmd.Flags().StringArrayVar(&deployNamespaceLabels, "namespace-label", nil, "Label to apply to the namespaces deploy ensures (key=value, repeatable; overrides KATES_NAMESPACE_LABELS, which overrides the labels file)") @@ -452,6 +461,18 @@ func configureDeploySelection(cmd *cobra.Command, wizard bool) error { deployDbNS = deployNamespace } } + if err := validateKatesMode(); err != nil { + return err + } + if err := checkKatesModeSwitch(context.Background(), resolveNamespaces().app); err != nil { + return err + } + // The charts were pulled before the wizard, which may have chosen operator mode. + if katesOperatorMode() && chartsPublished { + if err := resolveChartRoot(context.Background(), deployCheckout, false, deployChartDirs()...); err != nil { + return err + } + } if err := checkUpgradeInDeploy(deployUpgrade, buildSharedEntries(resolveNamespaces())); err != nil { return err } diff --git a/cli/cmd/deploy_chart_verify.go b/cli/cmd/deploy_chart_verify.go index a831cf5da..f16634fe3 100644 --- a/cli/cmd/deploy_chart_verify.go +++ b/cli/cmd/deploy_chart_verify.go @@ -23,9 +23,19 @@ import ( // identity names the workflow rather than this build's tag. const ( chartSignerIdentity = `^https://github\.com/bmscomp/kates/\.github/workflows/publish-charts\.yml@refs/(heads/main|tags/v.+)$` - chartSignerIssuer = "https://token.actions.githubusercontent.com" + // The operator's chart is released apart, by publish-operator.yml on its own tags. + operatorChartSignerIdentity = `^https://github\.com/bmscomp/kates/\.github/workflows/publish-operator\.yml@refs/tags/operator-v.+$` + chartSignerIssuer = "https://token.actions.githubusercontent.com" ) +// chartSigner is the workflow identity that signs the chart at ref. +func chartSigner(ref string) string { + if strings.HasSuffix(ref, "/kates-operator") { + return operatorChartSignerIdentity + } + return chartSignerIdentity +} + // cosignPathFn finds cosign; tests replace it. var cosignPathFn = func() (string, error) { return exec.LookPath("cosign") } @@ -108,7 +118,7 @@ func verifyChart(ctx context.Context, ref, digest, dest string) (provenance bool return false, err } } - return verifyProvenance(ctx, cosign, digest) + return verifyProvenance(ctx, cosign, digest, chartSigner(ref)) } // verifySignature is cosign verify of ref@digest. @@ -117,7 +127,7 @@ func verifySignature(ctx context.Context, cosign, ref, digest string) error { vctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() args := []string{"verify", image, - "--certificate-identity-regexp", chartSignerIdentity, + "--certificate-identity-regexp", chartSigner(ref), "--certificate-oidc-issuer", chartSignerIssuer} if chartRegistryPlainHTTP() { args = append(args, "--allow-http-registry") @@ -191,9 +201,9 @@ func (e *chartProvenanceError) Error() string { } // verifyProvenance checks that one of GitHub's attestations for digest is a -// SLSA provenance signed by publish-charts.yml whose subject is digest. It +// SLSA provenance signed by signer's workflow whose subject is digest. It // returns false, with no error, when the attestations could not be read. -func verifyProvenance(ctx context.Context, cosign, digest string) (bool, error) { +func verifyProvenance(ctx context.Context, cosign, digest, signer string) (bool, error) { bundles, err := attestationsFn(ctx, digest) if errors.Is(err, errProvenanceUnavailable) { provenanceUnchecked = err @@ -218,7 +228,7 @@ func verifyProvenance(ctx context.Context, cosign, digest string) (bool, error) } vctx, cancel := context.WithTimeout(ctx, 2*time.Minute) out, err := runExecCombinedFn(vctx, cosign, "verify-blob-attestation", "--bundle", path, - "--certificate-identity-regexp", chartSignerIdentity, + "--certificate-identity-regexp", signer, "--certificate-oidc-issuer", chartSignerIssuer, "--type", "slsaprovenance1", "--digest", strings.TrimPrefix(digest, "sha256:"), "--digestAlg", "sha256") diff --git a/cli/cmd/deploy_charts.go b/cli/cmd/deploy_charts.go index 51edd2c65..68218462c 100644 --- a/cli/cmd/deploy_charts.go +++ b/cli/cmd/deploy_charts.go @@ -67,6 +67,7 @@ var deployChartNames = map[string]string{ "kafka-ui": "kafka-ui", "kates": "kates", "kates-chaos": "kates-chaos", + "kates-operator": "kates-operator", "kates-platform": "kates-platform", "mirror-maker2": "mirror-maker2", "monitoring": "kates-monitoring", @@ -363,7 +364,9 @@ func chartSource() string { // kates-platform umbrella, which deploy never installs but a bundle carries, // verified, for a helm install of it without network. func deployChartDirs() []string { - return slices.DeleteFunc(publishedChartDirs(), func(dir string) bool { return dir == "legacy-kafka" || dir == "kates-platform" }) + return slices.DeleteFunc(publishedChartDirs(), func(dir string) bool { + return dir == "legacy-kafka" || dir == "kates-platform" || (dir == "kates-operator" && !katesOperatorMode()) + }) } // publishedChartDirs lists every chart the CLI pulls, deploy's and migrate's. diff --git a/cli/cmd/deploy_components.go b/cli/cmd/deploy_components.go index 5e0faa6e3..8ace552e0 100644 --- a/cli/cmd/deploy_components.go +++ b/cli/cmd/deploy_components.go @@ -255,6 +255,9 @@ func selectedDeployNamespaces(dc *deployContext) []string { if deployWithKates { add(dc.ns.app) } + if componentSelected("kates-operator") { + add(dc.ns.operator) + } if deployWithMonitoring { add(dc.ns.jaeger) } @@ -631,6 +634,8 @@ func deployComponents(dc *deployContext) []component { dc.kafkaConnectComponent(), dc.kafkaConnectorComponent(), dc.apicurioComponent(), + dc.katesOperatorComponent(), + dc.katesDBComponent(), dc.katesComponent(), dc.kafkaUIComponent(), dc.mirrorMaker2Component(), @@ -1684,9 +1689,13 @@ func (dc *deployContext) katesComponent() component { kafkaNS := dc.ns.kafka appNS := dc.ns.app chaosNS := dc.ns.chaos + deps := []string{"kafka-users", "postgres", "monitoring"} + if katesOperatorMode() { + deps = append(deps, "kates-operator", "kates-db") + } return component{ id: "kates", name: "Kates Backend", namespace: appNS, - deps: []string{"kafka-users", "postgres", "monitoring"}, timeout: 8 * time.Minute, + deps: deps, timeout: 10 * time.Minute, enabled: func() bool { return deployWithKates }, // The KafkaUser was already waited on (kafka-users) โ€” just read the Secret. secrets: []secretCopy{{name: "kates-backend"}}, @@ -1698,6 +1707,9 @@ func (dc *deployContext) katesComponent() component { return fmt.Sprintf("\n๐Ÿ“ฆ %s Kates Backend (Namespace: %s)...", verb, appNS) }, install: func(ctx context.Context) error { + if katesOperatorMode() { + return dc.installKatesResource(ctx) + } // Auto-cleanup stale ClusterRole ownership from previous topology switches cleanupStaleClusterResource(ctx, "clusterrole", "kates", appNS) cleanupStaleClusterResource(ctx, "clusterrolebinding", "kates", appNS) @@ -1759,6 +1771,9 @@ func (dc *deployContext) katesComponent() component { return runKatesHelmUpgrade(ctx, appNS, katesArgs...) }, wait: func(ctx context.Context) error { + if katesOperatorMode() { + return waitKatesResource(ctx, appNS) + } return dc.waitPods(ctx, appNS, "app.kubernetes.io/instance=kates", 8*time.Minute, "") }, } diff --git a/cli/cmd/deploy_flags.go b/cli/cmd/deploy_flags.go index 14cf4450d..761cb00e5 100644 --- a/cli/cmd/deploy_flags.go +++ b/cli/cmd/deploy_flags.go @@ -1,6 +1,10 @@ package cmd -import "github.com/spf13/cobra" +import ( + "github.com/spf13/cobra" + + "github.com/bmscomp/kates/cli/pkg/components" +) // The topology and namespace flags place the stack. deploy, deploy status, // clean and kyverno apply all take them, and each used to declare its own @@ -12,7 +16,7 @@ import "github.com/spf13/cobra" // deployScopeVars are the variables the scope flags bind. deploy, deploy // status and kyverno apply share deploy's package globals; clean has its own. type deployScopeVars struct { - Topology, Namespace, KafkaNS, ConnectNS, DbNS, AppNS, ChaosNS, MonitoringNS, KafkaUINS, MM2NS *string + Topology, Namespace, KafkaNS, ConnectNS, DbNS, AppNS, ChaosNS, MonitoringNS, KafkaUINS, MM2NS, OperatorNS *string } // deployGlobalScope binds the flags to deploy's package-level variables. @@ -21,7 +25,7 @@ func deployGlobalScope() deployScopeVars { Topology: &deployTopology, Namespace: &deployNamespace, KafkaNS: &deployKafkaNS, ConnectNS: &deployConnectNS, DbNS: &deployDbNS, AppNS: &deployAppNS, ChaosNS: &deployChaosNS, MonitoringNS: &deployMonitoringNS, - KafkaUINS: &deployKafkaUINS, MM2NS: &deployMM2NS, + KafkaUINS: &deployKafkaUINS, MM2NS: &deployMM2NS, OperatorNS: &deployOperatorNS, } } @@ -46,7 +50,7 @@ func scopeTopologyDefault(value, usage string) deployScopeOption { // deployScopeFlagNames is the set addDeployScopeFlags registers, for the test // that checks every command has it. -var deployScopeFlagNames = []string{"topology", "namespace", "kafka-ns", "connect-ns", "db-ns", "app-ns", "chaos-ns", "monitoring-ns", "ui-ns", "mm2-ns", "kafka-name"} +var deployScopeFlagNames = []string{"topology", "namespace", "kafka-ns", "connect-ns", "db-ns", "app-ns", "chaos-ns", "monitoring-ns", "ui-ns", "mm2-ns", "operator-ns", "kafka-name"} // addDeployScopeFlags registers the topology and namespace flags on cmd. func addDeployScopeFlags(cmd *cobra.Command, opts ...deployScopeOption) { @@ -70,6 +74,7 @@ func addDeployScopeFlags(cmd *cobra.Command, opts ...deployScopeOption) { f.StringVar(v.MonitoringNS, "monitoring-ns", "monitoring", "Namespace for monitoring components (Jaeger) when topology is 'isolated'") f.StringVar(v.KafkaUINS, "ui-ns", "kafka", "Namespace for Kafka UI when topology is 'isolated'") f.StringVar(v.MM2NS, "mm2-ns", "kafka", "Namespace for MirrorMaker 2 when topology is 'isolated'") + f.StringVar(v.OperatorNS, "operator-ns", components.DefaultOperatorNamespace, "Namespace for the Kates operator (--kates-mode operator), under either topology") // The Kafka release is named by --kafka-name everywhere; clean already // bound deploy's variable, so it stays shared. f.StringVar(&deployKafkaName, "kafka-name", "krafter", "Name of the primary Kafka cluster") diff --git a/cli/cmd/deploy_kates_operator.go b/cli/cmd/deploy_kates_operator.go new file mode 100644 index 000000000..1497a15c1 --- /dev/null +++ b/cli/cmd/deploy_kates_operator.go @@ -0,0 +1,406 @@ +package cmd + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "math/big" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +// --kates-mode: how deploy installs Kates. Standalone is the kates chart with +// its own PostgreSQL. Operator installs the Kates operator, a PostgreSQL for +// Kates (the kates-db release of charts/cdc-postgresql, unless +// --kates-db-secret names one) and a Kates resource the operator turns into +// one backend, or a quorum of three or five. +const ( + katesModeStandalone = "standalone" + katesModeOperator = "operator" + + // The Kates resource's name: its Service is then kates, as the chart's + // is, so ports, the API key and the dashboards find it unchanged. + katesResourceName = "kates" + katesDBRelease = "kates-db" + // The Secret the Kates resource reads its database from, with --kates-db-secret unset. + katesDBConnection = "kates-db-connection" +) + +func katesOperatorMode() bool { return deployKatesMode == katesModeOperator } + +// validateKatesMode refuses flag combinations the chosen mode cannot honour, +// before anything reaches the cluster. +func validateKatesMode() error { + switch deployKatesMode { + case katesModeStandalone, katesModeOperator: + default: + return fmt.Errorf("--kates-mode %q: use 'standalone' or 'operator'", deployKatesMode) + } + if !slices.Contains([]int{1, 3, 5}, deployKatesReplicas) { + return fmt.Errorf("--kates-replicas %d: use 1, or 3 or 5 for a quorum", deployKatesReplicas) + } + if katesOperatorMode() { + if deployAdminless { + return errors.New("--kates-mode operator installs the Kates operator and its CRD, which needs cluster admin: drop --adminless, or keep --kates-mode standalone") + } + return nil + } + if deployKatesReplicas != 1 { + return errors.New("--kates-replicas needs --kates-mode operator: the kates chart runs one backend") + } + if deployKatesDBSecret != "" { + return errors.New("--kates-db-secret needs --kates-mode operator: the kates chart brings its own PostgreSQL") + } + return nil +} + +// checkKatesModeSwitch refuses to install Kates one way over an install made +// the other way: both would claim the Service kates. +func checkKatesModeSwitch(ctx context.Context, appNS string) error { + if !deployWithKates { + return nil + } + if katesOperatorMode() && isHelmReleaseDeployedFn(ctx, "kates", appNS) { + return fmt.Errorf("kates is installed standalone in %s (Helm release kates): uninstall it first (kates clean, or helm uninstall kates -n %s), or keep --kates-mode standalone", appNS, appNS) + } + if !katesOperatorMode() && katesResourceExists(ctx, appNS) { + return fmt.Errorf("kates is installed by the Kates operator in %s (Kates resource %s): use --kates-mode operator, or delete it first (kubectl delete kates %s -n %s)", appNS, katesResourceName, katesResourceName, appNS) + } + return nil +} + +func katesResourceExists(ctx context.Context, namespace string) bool { + out, err := runExecOutputFn(ctx, "kubectl", "get", "kates.kates.io", katesResourceName, "-n", namespace, + "--ignore-not-found", "-o", "name") + return err == nil && len(bytes.TrimSpace(out)) > 0 +} + +// operatorChartPath is the operator chart: operator/chart in a checkout, +// charts/kates-operator among the published packages. +func operatorChartPath() string { + if chartsPublished { + return chartRootPath("charts/kates-operator") + } + return chartRootPath("operator/chart") +} + +// katesOperatorComponent is the Kates operator, watching the Kates namespace +// and binding the backend's cluster role to the Kates pods there. +func (dc *deployContext) katesOperatorComponent() component { + operatorNS := dc.ns.operator + appNS := dc.ns.app + return component{ + id: "kates-operator", name: "Kates Operator", namespace: operatorNS, timeout: 5 * time.Minute, + enabled: func() bool { return componentSelected("kates-operator") }, + banner: func(context.Context) string { + return fmt.Sprintf("\n๐Ÿค– Deploying the Kates Operator (Namespace: %s, watching %s)...", operatorNS, appNS) + }, + install: func(ctx context.Context) error { + // Deploy makes each namespace itself: Helm's --create-namespace is dropped. + if err := ensureConfiguredDeployNamespace(ctx, operatorNS); err != nil { + return err + } + args := []string{"upgrade", "--install", "kates-operator", operatorChartPath(), + "-n", operatorNS, "--create-namespace", + "--set-json", "watchNamespaces=" + jsonList(appNS), + "--set-json", "rbac.backendNamespaces=" + jsonList(appNS), + "--wait", "--timeout", "5m"} + if mirror := imageRegistry(); mirror != "" { + args = append(args, "--set-string", "global.imageRegistry="+mirror) + } + args = append(args, dc.policyArgs("kates-operator")...) + return runHelmFn(ctx, args...) + }, + wait: func(ctx context.Context) error { + return dc.waitPods(ctx, operatorNS, "app.kubernetes.io/name=kates-operator", 5*time.Minute, "") + }, + } +} + +// katesDBComponent is the PostgreSQL the Kates resource stores in: this +// repository's cdc-postgresql chart as release kates-db, open to the Kates +// pods alone, and the Secret kates-db-connection that names it. +func (dc *deployContext) katesDBComponent() component { + appNS := dc.ns.app + return component{ + id: "kates-db", name: "PostgreSQL (Kates)", namespace: appNS, timeout: 5 * time.Minute, + enabled: func() bool { return componentSelected("kates-db") }, + banner: func(context.Context) string { + return fmt.Sprintf("\n๐Ÿ˜ Deploying PostgreSQL for Kates (Namespace: %s)...", appNS) + }, + install: func(ctx context.Context) error { + if err := ensureConfiguredDeployNamespace(ctx, appNS); err != nil { + return err + } + // Kept across runs: a new password would lock Kates out of its data. + password, err := secretValueOrNew(ctx, appNS, katesDBRelease, "password") + if err != nil { + return err + } + admin, err := secretValueOrNew(ctx, appNS, katesDBRelease, "postgres-password") + if err != nil { + return err + } + // In a 0600 file, not --set: --verbose prints every Helm command. + values, err := writePrivateValues(dc.valuesFile, "kates-db-values.yaml", map[string]any{ + "auth": map[string]any{ + "username": "kates", "database": "kates", + "password": password, "postgresPassword": admin, + }, + "networkPolicy": map[string]any{ + "allowPods": []map[string]string{{"app.kubernetes.io/instance": katesResourceName}}, + }, + }) + if err != nil { + return err + } + defer os.Remove(values) + args := append([]string{"upgrade", "--install", katesDBRelease, chartRootPath("charts/cdc-postgresql"), + "-n", appNS, "--create-namespace", "-f", values, "--timeout", "5m"}, + dc.policyArgs("cdc-postgresql")...) + if err := runHelmFn(ctx, args...); err != nil { + return err + } + return applySecret(ctx, appNS, katesDBConnection, map[string]string{ + "url": fmt.Sprintf("jdbc:postgresql://%s.%s.svc:5432/kates", katesDBRelease, appNS), + "username": "kates", + "password": password, + }) + }, + wait: func(ctx context.Context) error { + return dc.waitPods(ctx, appNS, "app.kubernetes.io/instance="+katesDBRelease, 5*time.Minute, "") + }, + } +} + +// installKatesResource is the Kates backend under the operator: its API key, +// then the Kates resource, which the operator turns into a StatefulSet and a +// Service named kates. +func (dc *deployContext) installKatesResource(ctx context.Context) error { + appNS := dc.ns.app + if err := ensureConfiguredDeployNamespace(ctx, appNS); err != nil { + return err + } + database := katesDBConnection + if deployKatesDBSecret != "" { + database = deployKatesDBSecret + if out, err := runExecOutputFn(ctx, "kubectl", "get", "secret", database, "-n", appNS, + "--ignore-not-found", "-o", "name"); err != nil || len(bytes.TrimSpace(out)) == 0 { + return fmt.Errorf("--kates-db-secret %s: no such Secret in %s; it needs keys url (jdbc:postgresql://...), username and password", database, appNS) + } + } + // The key Secret the kates chart makes, under the name and key the CLI and + // the operator's drain read; kept across runs, as the chart keeps it. + key, err := secretValueOrNew(ctx, appNS, apiKeySecretName, "api-key") + if err != nil { + return err + } + if err := applySecret(ctx, appNS, apiKeySecretName, map[string]string{"api-key": key}); err != nil { + return err + } + version, image, err := dc.katesImage(ctx) + if err != nil { + return err + } + manifest, err := katesResourceManifest(katesResourceSpec{ + namespace: appNS, version: version, image: image, replicas: deployKatesReplicas, + database: database, + bootstrap: fmt.Sprintf("%s-kafka-bootstrap.%s.svc:9092", dc.primary.Name, dc.primary.Namespace), + kafkaNS: dc.ns.kafka, cluster: dc.primary.Name, oneBroker: oneBroker(), + }) + if err != nil { + return err + } + dl.Printf(" - Kates resource %s/%s: %d member(s), image %s:%s\n", appNS, katesResourceName, deployKatesReplicas, image, version) + return runExecStdinFn(ctx, "kubectl", []string{"apply", "--server-side", "--field-manager", "kates-deploy", "-f", "-"}, manifest) +} + +// waitKatesResource waits for the operator to report the backend Ready: every +// member up and, with a quorum, a leader. +func waitKatesResource(ctx context.Context, appNS string) error { + err := runExecFn(ctx, "kubectl", "wait", "--for=condition=Ready", "kates.kates.io/"+katesResourceName, + "-n", appNS, "--timeout=10m") + if err == nil { + return nil + } + // The operator says why in the condition; a bare timeout says nothing. + out, readErr := runExecOutputFn(ctx, "kubectl", "get", "kates.kates.io", katesResourceName, "-n", appNS, + "-o", `jsonpath={.status.conditions[?(@.type=="Ready")].reason}: {.status.conditions[?(@.type=="Ready")].message}`) + if why := strings.TrimSpace(string(out)); readErr == nil && why != "" && why != ":" { + return fmt.Errorf("the Kates resource %s/%s is not Ready (%s): %w", appNS, katesResourceName, why, err) + } + return err +} + +// katesImage is the image the backend runs, as the chart path picks it: a +// local build on kind when one is there, else the published native image. +func (dc *deployContext) katesImage(ctx context.Context) (version, image string, err error) { + if dc.isKind { + args, err := kindBackendImageArgs(ctx, dc.report.Context) + if err != nil { + return "", "", err + } + for _, a := range args { + if v, ok := strings.CutPrefix(a, "image.tag="); ok { + version = v + } + if v, ok := strings.CutPrefix(a, "image.repository="); ok { + image = v + } + } + } + if version == "" { + if version, err = valuesImageTag(chartRootPath(publishedNativeValues)); err != nil { + return "", "", err + } + } + if image == "" { + image = "ghcr.io/bmscomp/kates" + if mirror := imageRegistry(); mirror != "" { + image = mirror + "/bmscomp/kates" + } + } + return version, image, nil +} + +// valuesImageTag reads image.tag from a values file. +func valuesImageTag(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + var values struct { + Image struct { + Tag string `yaml:"tag"` + } `yaml:"image"` + } + if err := yaml.Unmarshal(data, &values); err != nil { + return "", fmt.Errorf("%s: %w", path, err) + } + if values.Image.Tag == "" { + return "", fmt.Errorf("%s sets no image.tag", path) + } + return values.Image.Tag, nil +} + +type katesResourceSpec struct { + namespace, version, image, database, bootstrap, kafkaNS, cluster string + replicas int + oneBroker bool +} + +// katesResourceManifest is the Kates resource, with the Kafka settings the +// kates chart's values give the backend. +func katesResourceManifest(s katesResourceSpec) (string, error) { + env := []map[string]string{ + // What the chart's ConfigMap sets from kafka.clusterName and the Kafka namespace. + {"name": "KATES_CHAOS_KAFKA_NAMESPACE", "value": s.kafkaNS}, + {"name": "KATES_TOPOLOGY_KAFKA_NAMESPACE", "value": s.kafkaNS}, + {"name": "KATES_CHAOS_KAFKA_CLUSTER", "value": s.cluster}, + {"name": "KATES_TOPOLOGY_KAFKA_CLUSTER", "value": s.cluster}, + {"name": "KATES_CDC_TOPIC_NAMESPACE", "value": s.kafkaNS}, + } + if s.oneBroker { + for _, name := range []string{"KATES_DEFAULTS_REPLICATION_FACTOR", "KATES_DEFAULTS_MIN_INSYNC_REPLICAS", "KATES_OUTBOX_TOPIC_REPLICATION_FACTOR"} { + env = append(env, map[string]string{"name": name, "value": "1"}) + } + } + resource := map[string]any{ + "apiVersion": "kates.io/v1alpha1", + "kind": "Kates", + "metadata": map[string]any{"name": katesResourceName, "namespace": s.namespace}, + "spec": map[string]any{ + "version": s.version, + "image": s.image, + "replicas": s.replicas, + "database": map[string]any{"secretRef": map[string]any{"name": s.database}}, + "kafka": map[string]any{ + "bootstrapServers": s.bootstrap, + "securityProtocol": "SASL_PLAINTEXT", + "sasl": map[string]any{ + "mechanism": "SCRAM-SHA-512", + "username": "kates-backend", + "passwordSecretRef": map[string]any{"name": "kates-backend", "key": "password"}, + }, + }, + "apiKeySecretRef": map[string]any{"name": apiKeySecretName, "key": "api-key"}, + "env": env, + }, + } + out, err := yaml.Marshal(resource) + return string(out), err +} + +// secretValueOrNew is key of an existing Secret, or a new random value. +func secretValueOrNew(ctx context.Context, namespace, name, key string) (string, error) { + out, err := runExecOutputFn(ctx, "kubectl", "get", "secret", name, "-n", namespace, + "--ignore-not-found", "-o", "jsonpath={.data."+jsonpathKey(key)+"}") + if err == nil && len(bytes.TrimSpace(out)) > 0 { + raw, decodeErr := base64.StdEncoding.DecodeString(string(bytes.TrimSpace(out))) + if decodeErr != nil { + return "", fmt.Errorf("secret %s/%s key %s is not base64", namespace, name, key) + } + return string(raw), nil + } + return randomAlnum(32) +} + +const alnum = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +func randomAlnum(n int) (string, error) { + var b strings.Builder + for range n { + i, err := rand.Int(rand.Reader, big.NewInt(int64(len(alnum)))) + if err != nil { + return "", err + } + b.WriteByte(alnum[i.Int64()]) + } + return b.String(), nil +} + +// applySecret writes Secret name in namespace, through stdin so no value is +// on a command line. +func applySecret(ctx context.Context, namespace, name string, data map[string]string) error { + secret := map[string]any{ + "apiVersion": "v1", "kind": "Secret", "type": "Opaque", + "metadata": map[string]any{"name": name, "namespace": namespace}, + "stringData": data, + } + out, err := yaml.Marshal(secret) + if err != nil { + return err + } + if err := runExecStdinFn(ctx, "kubectl", []string{"apply", "-f", "-"}, string(out)); err != nil { + return fmt.Errorf("secret %s could not be written in %s: %w", name, namespace, err) + } + return nil +} + +// writePrivateValues writes values beside the deploy's values file, readable +// by this user alone. +func writePrivateValues(beside, name string, values map[string]any) (string, error) { + out, err := yaml.Marshal(values) + if err != nil { + return "", err + } + path := filepath.Join(filepath.Dir(beside), name) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return "", err + } + return path, os.WriteFile(path, out, 0o600) +} + +func jsonList(items ...string) string { + out, _ := json.Marshal(items) + return string(out) +} diff --git a/cli/cmd/deploy_kates_operator_test.go b/cli/cmd/deploy_kates_operator_test.go new file mode 100644 index 000000000..05d02ac5c --- /dev/null +++ b/cli/cmd/deploy_kates_operator_test.go @@ -0,0 +1,242 @@ +package cmd + +import ( + "context" + "encoding/base64" + "errors" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// setKatesMode sets the operator-mode flags for one test. +func setKatesMode(t *testing.T, mode string, replicas int, dbSecret string, adminless bool) { + t.Helper() + m, r, s, a, k := deployKatesMode, deployKatesReplicas, deployKatesDBSecret, deployAdminless, deployWithKates + t.Cleanup(func() { + deployKatesMode, deployKatesReplicas, deployKatesDBSecret, deployAdminless, deployWithKates = m, r, s, a, k + }) + deployKatesMode, deployKatesReplicas, deployKatesDBSecret, deployAdminless, deployWithKates = mode, replicas, dbSecret, adminless, true +} + +func TestKatesModeFlagsAreCheckedBeforeAnyInstall(t *testing.T) { + for _, tc := range []struct { + name, mode, secret string + replicas int + adminless bool + wantErr string + }{ + {name: "standalone default", mode: "standalone", replicas: 1}, + {name: "operator, one member", mode: "operator", replicas: 1}, + {name: "operator quorum and own database", mode: "operator", replicas: 5, secret: "db"}, + {name: "unknown mode", mode: "helm", replicas: 1, wantErr: "--kates-mode"}, + {name: "even quorum", mode: "operator", replicas: 2, wantErr: "--kates-replicas 2"}, + {name: "replicas without the operator", mode: "standalone", replicas: 3, wantErr: "needs --kates-mode operator"}, + {name: "database without the operator", mode: "standalone", replicas: 1, secret: "db", wantErr: "--kates-db-secret needs"}, + {name: "operator admin-less", mode: "operator", replicas: 1, adminless: true, wantErr: "cluster admin"}, + } { + t.Run(tc.name, func(t *testing.T) { + setKatesMode(t, tc.mode, tc.replicas, tc.secret, tc.adminless) + err := validateKatesMode() + if tc.wantErr == "" && err != nil { + t.Fatalf("refused: %v", err) + } + if tc.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tc.wantErr)) { + t.Fatalf("error = %v, want one naming %q", err, tc.wantErr) + } + }) + } +} + +func TestOperatorModeSelectsTheOperatorAndADatabaseUnlessOneIsNamed(t *testing.T) { + setKatesMode(t, "standalone", 1, "", false) + if componentSelected("kates-operator") || componentSelected("kates-db") || slices.Contains(deployChartDirs(), "kates-operator") { + t.Fatal("standalone selects the operator, its database or its chart") + } + setKatesMode(t, "operator", 3, "", false) + if !componentSelected("kates-operator") || !componentSelected("kates-db") || !slices.Contains(deployChartDirs(), "kates-operator") { + t.Fatal("operator mode lacks the operator, the database or the operator chart") + } + setKatesMode(t, "operator", 3, "mine", false) + if componentSelected("kates-db") { + t.Fatal("--kates-db-secret still installs PostgreSQL") + } +} + +func TestTheKatesResourceCarriesTheChartsKafkaSettings(t *testing.T) { + manifest, err := katesResourceManifest(katesResourceSpec{ + namespace: "kates", version: "1.41.0-native", image: "ghcr.io/bmscomp/kates", replicas: 3, + database: katesDBConnection, bootstrap: "krafter-kafka-bootstrap.kafka.svc:9092", + kafkaNS: "kafka", cluster: "krafter", oneBroker: true, + }) + if err != nil { + t.Fatal(err) + } + var got struct { + Kind string `yaml:"kind"` + Metadata struct{ Name, Namespace string } + Spec struct { + Version string `yaml:"version"` + Replicas int `yaml:"replicas"` + Database struct { + SecretRef struct{ Name string } `yaml:"secretRef"` + } `yaml:"database"` + Kafka struct { + BootstrapServers string `yaml:"bootstrapServers"` + Sasl struct { + Username string `yaml:"username"` + PasswordSecretRef struct{ Name, Key string } `yaml:"passwordSecretRef"` + } `yaml:"sasl"` + } `yaml:"kafka"` + APIKeySecretRef struct{ Name, Key string } `yaml:"apiKeySecretRef"` + Env []struct{ Name, Value string } + } `yaml:"spec"` + } + if err := yaml.Unmarshal([]byte(manifest), &got); err != nil { + t.Fatalf("%v\n%s", err, manifest) + } + if got.Kind != "Kates" || got.Metadata.Name != "kates" || got.Spec.Replicas != 3 || got.Spec.Version != "1.41.0-native" { + t.Fatalf("resource = %+v", got) + } + if got.Spec.Database.SecretRef.Name != katesDBConnection || got.Spec.Kafka.Sasl.PasswordSecretRef.Name != "kates-backend" || + got.Spec.APIKeySecretRef.Name != "kates-api-key" || got.Spec.APIKeySecretRef.Key != "api-key" { + t.Fatalf("secret references = %+v", got.Spec) + } + env := map[string]string{} + for _, e := range got.Spec.Env { + env[e.Name] = e.Value + } + if env["KATES_TOPOLOGY_KAFKA_CLUSTER"] != "krafter" || env["KATES_CHAOS_KAFKA_NAMESPACE"] != "kafka" || env["KATES_OUTBOX_TOPIC_REPLICATION_FACTOR"] != "1" { + t.Fatalf("env = %v", env) + } +} + +// operatorModeCluster stubs the cluster calls operator mode makes: Secret +// reads answer from secrets, applies and helm calls are recorded. +func operatorModeCluster(t *testing.T, secrets map[string]string) (applied, helm *[]string) { + t.Helper() + origOut, origStdin, origHelm, origDeployed := runExecOutputFn, runExecStdinFn, runHelmFn, isHelmReleaseDeployedFn + t.Cleanup(func() { + runExecOutputFn, runExecStdinFn, runHelmFn, isHelmReleaseDeployedFn = origOut, origStdin, origHelm, origDeployed + }) + applied, helm = &[]string{}, &[]string{} + runExecOutputFn = func(_ context.Context, _ string, args ...string) ([]byte, error) { + joined := strings.Join(args, " ") + for key, value := range secrets { + if strings.Contains(joined, key) { + return []byte(value), nil + } + } + return nil, nil + } + runExecStdinFn = func(_ context.Context, _ string, _ []string, stdin string) error { + *applied = append(*applied, stdin) + return nil + } + runHelmFn = func(_ context.Context, args ...string) error { + *helm = append(*helm, strings.Join(args, " ")) + return nil + } + isHelmReleaseDeployedFn = func(context.Context, string, string) bool { return false } + return applied, helm +} + +func TestKatesDBKeepsItsPasswordOffTheCommandLineAndAcrossRuns(t *testing.T) { + setKatesMode(t, "operator", 1, "", false) + kept := base64.StdEncoding.EncodeToString([]byte("kept-password")) + applied, helm := operatorModeCluster(t, map[string]string{ + "secret kates-db -n kates --ignore-not-found -o jsonpath={.data.password}": kept, + "get namespace": `{"metadata":{"name":"kates"}}`, + }) + dc := &deployContext{ctx: context.Background(), ns: resolvedNamespaces{app: "kates"}, + valuesFile: filepath.Join(t.TempDir(), "values-detected.yaml")} + if err := dc.katesDBComponent().install(context.Background()); err != nil { + t.Fatal(err) + } + if len(*helm) != 1 || !strings.Contains((*helm)[0], "upgrade --install kates-db") || strings.Contains((*helm)[0], "kept-password") { + t.Fatalf("helm = %v, want one kates-db upgrade with no password in it", *helm) + } + values := (*helm)[0][strings.Index((*helm)[0], "-f ")+3:] + values = strings.Fields(values)[0] + if _, err := os.Stat(values); !os.IsNotExist(err) { + t.Errorf("the values file %s was left behind", values) + } + connection := (*applied)[len(*applied)-1] + for _, want := range []string{"name: kates-db-connection", "jdbc:postgresql://kates-db.kates.svc:5432/kates", "password: kept-password"} { + if !strings.Contains(connection, want) { + t.Fatalf("connection Secret lacks %q:\n%s", want, connection) + } + } +} + +func TestTheKatesResourceGetsAKeyAndNamesAnExistingDatabase(t *testing.T) { + setKatesMode(t, "operator", 3, "own-db", false) + origRoot := chartRoot + t.Cleanup(func() { chartRoot = origRoot }) + chartRoot = "../.." + key := base64.StdEncoding.EncodeToString([]byte("existing-key")) + applied, _ := operatorModeCluster(t, map[string]string{ + "secret own-db -n kates": "secret/own-db", + "kates-api-key -n kates": key, + "get namespace": `{"metadata":{"name":"kates"}}`, + }) + dc := &deployContext{ctx: context.Background(), ns: resolvedNamespaces{app: "kates", kafka: "kafka"}, + primary: primaryCluster{Name: "krafter", Namespace: "kafka"}} + if err := dc.installKatesResource(context.Background()); err != nil { + t.Fatal(err) + } + all := strings.Join(*applied, "\n---\n") + for _, want := range []string{"api-key: existing-key", "kind: Kates", "name: own-db", "replicas: 3"} { + if !strings.Contains(all, want) { + t.Fatalf("applied lacks %q:\n%s", want, all) + } + } +} + +func TestAMissingDatabaseSecretIsNamed(t *testing.T) { + setKatesMode(t, "operator", 1, "nope", false) + operatorModeCluster(t, map[string]string{"get namespace": `{"metadata":{"name":"kates"}}`}) + dc := &deployContext{ctx: context.Background(), ns: resolvedNamespaces{app: "kates"}} + err := dc.installKatesResource(context.Background()) + if err == nil || !strings.Contains(err.Error(), "--kates-db-secret nope") { + t.Fatalf("error = %v", err) + } +} + +func TestOneModeIsNotInstalledOverTheOther(t *testing.T) { + setKatesMode(t, "operator", 1, "", false) + operatorModeCluster(t, nil) + isHelmReleaseDeployedFn = func(_ context.Context, release, _ string) bool { return release == "kates" } + if err := checkKatesModeSwitch(context.Background(), "kates"); err == nil || !strings.Contains(err.Error(), "installed standalone") { + t.Fatalf("operator over a standalone install: %v", err) + } + setKatesMode(t, "standalone", 1, "", false) + operatorModeCluster(t, map[string]string{"kates.kates.io kates": "kates.kates.io/kates"}) + if err := checkKatesModeSwitch(context.Background(), "kates"); err == nil || !strings.Contains(err.Error(), "by the Kates operator") { + t.Fatalf("standalone over an operator install: %v", err) + } +} + +func TestTheOperatorChartIsCheckedAgainstItsOwnSigner(t *testing.T) { + if chartSigner("oci://ghcr.io/bmscomp/charts/kates-operator") != operatorChartSignerIdentity { + t.Fatal("the operator chart is checked against publish-charts.yml") + } + if chartSigner("oci://ghcr.io/bmscomp/charts/kates") != chartSignerIdentity { + t.Fatal("the kates chart is not checked against publish-charts.yml") + } +} + +func TestANotReadyKatesResourceSaysWhy(t *testing.T) { + operatorModeCluster(t, map[string]string{"jsonpath={.status.conditions": "NoLeader: the quorum has no leader"}) + origExec := runExecFn + t.Cleanup(func() { runExecFn = origExec }) + runExecFn = func(context.Context, string, ...string) error { return errors.New("timed out") } + err := waitKatesResource(context.Background(), "kates") + if err == nil || !strings.Contains(err.Error(), "NoLeader: the quorum has no leader") { + t.Fatalf("error = %v", err) + } +} diff --git a/cli/cmd/deploy_plan.go b/cli/cmd/deploy_plan.go index 3102feb4f..42fc28b12 100644 --- a/cli/cmd/deploy_plan.go +++ b/cli/cmd/deploy_plan.go @@ -32,6 +32,13 @@ func runInteractiveForms() error { applyPickedComponents(picked) + // โ”€โ”€ Form: how Kates is installed โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + if deployWithKates && !deployAdminless { + if err := katesModeFormFn(); err != nil { + return err + } + } + // โ”€โ”€ Form: Versions and scope โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ // The Kafka options are a function of the operator chosen above them: // pick 1.0.1 and the list is its window, pick 1.1.0 and it is 1.1.0's. @@ -53,6 +60,45 @@ func runInteractiveForms() error { return runDeployReviewForm() } +// katesModeFormFn asks how Kates is installed; tests replace it. +var katesModeFormFn = runKatesModeForm + +// runKatesModeForm asks standalone or operator and, for operator, the +// members, the operator's namespace and an existing database. Two forms, so +// the second is built once the mode is known. +func runKatesModeForm() error { + err := huh.NewForm(huh.NewGroup( + huh.NewSelect[string](). + Title("How should Kates be installed?"). + Options( + huh.NewOption("Standalone: the kates chart, with its own PostgreSQL", katesModeStandalone), + huh.NewOption("Operator: the Kates operator, a Kates resource and a PostgreSQL for it", katesModeOperator), + ). + Value(&deployKatesMode), + )).WithTheme(ThemeKates()).WithAccessible(accessibleUI()).Run() + if err != nil || !katesOperatorMode() { + return err + } + return huh.NewForm(huh.NewGroup( + huh.NewSelect[int](). + Title("Kates members"). + Description("3 or 5 run a quorum that keeps a leader with one or two members lost. Fixed once installed"). + Options( + huh.NewOption("1", 1), + huh.NewOption("3 (quorum)", 3), + huh.NewOption("5 (quorum)", 5), + ). + Value(&deployKatesReplicas), + huh.NewInput(). + Title("Operator Namespace"). + Value(&deployOperatorNS), + huh.NewInput(). + Title("Existing database Secret (optional)"). + Description("A Secret in the Kates namespace with keys url, username and password. Empty: install PostgreSQL"). + Value(&deployKatesDBSecret), + )).WithTheme(ThemeKates()).WithAccessible(accessibleUI()).Run() +} + // runComponentForm is Form 1: the topology, the schema registry, HA and // the components, which it stores in picked. func runComponentForm(picked *[]string) error { @@ -339,29 +385,33 @@ type resolvedNamespaces struct { jaeger string kafkaUI string mm2 string + // operator is the Kates operator's namespace, under either topology. + operator string } // resolveNamespaces resolves namespace values based on the selected topology. func resolveNamespaces() resolvedNamespaces { if deployTopology == "single" { return resolvedNamespaces{ - kafka: deployNamespace, - connect: deployNamespace, - app: deployNamespace, - chaos: deployNamespace, - jaeger: deployNamespace, - kafkaUI: deployNamespace, - mm2: deployNamespace, + kafka: deployNamespace, + connect: deployNamespace, + app: deployNamespace, + chaos: deployNamespace, + jaeger: deployNamespace, + kafkaUI: deployNamespace, + mm2: deployNamespace, + operator: deployOperatorNS, } } return resolvedNamespaces{ - kafka: deployKafkaNS, - connect: deployConnectNS, - app: deployAppNS, - chaos: deployChaosNS, - jaeger: deployMonitoringNS, - kafkaUI: deployKafkaUINS, - mm2: deployMM2NS, + kafka: deployKafkaNS, + connect: deployConnectNS, + app: deployAppNS, + chaos: deployChaosNS, + jaeger: deployMonitoringNS, + kafkaUI: deployKafkaUINS, + mm2: deployMM2NS, + operator: deployOperatorNS, } } @@ -372,7 +422,7 @@ func deployPlacement(ns resolvedNamespaces) components.Placement { return components.Placement{ Kafka: ns.kafka, Connect: ns.connect, Database: deployDbNS, App: ns.app, Chaos: ns.chaos, Monitoring: ns.jaeger, KafkaUI: ns.kafkaUI, MM2: ns.mm2, - KafkaName: deployKafkaName, + Operator: ns.operator, KafkaName: deployKafkaName, } } @@ -400,6 +450,10 @@ func componentSelected(id string) bool { return deployWithChaos case "kates": return deployWithKates + case "kates-operator": + return deployWithKates && katesOperatorMode() + case "kates-db": + return deployWithKates && katesOperatorMode() && deployKatesDBSecret == "" case "kafka", "kafka-users": return true } diff --git a/cli/cmd/deploy_status.go b/cli/cmd/deploy_status.go index 21cad76f6..1a5d6df8e 100644 --- a/cli/cmd/deploy_status.go +++ b/cli/cmd/deploy_status.go @@ -150,6 +150,8 @@ type componentSpec struct { Namespace string Kind string Resource string + // Managed: no Helm release of its own, the Kates operator runs it. + Managed bool } // statusComponents is what `kates deploy status` inspects, in display order. @@ -157,21 +159,23 @@ type componentSpec struct { // check handles โ€” the check used to name the Strimzi kinds it knew, and a // component whose kind was not on that list reported a blank "Unknown" // forever without anything failing. -func statusComponents(kafkaNS, connectNS, appNS, chaosNS, jaegerNS, dbNS, kafkaUINS, mm2NS string) []componentSpec { +func statusComponents(kafkaNS, connectNS, appNS, chaosNS, jaegerNS, dbNS, kafkaUINS, mm2NS, operatorNS string) []componentSpec { return []componentSpec{ - {"A", "๐ŸฆŠ", "Strimzi Operator", "strimzi-operator", "strimzi-operator", "pod", "-l name=strimzi-cluster-operator"}, - {"A", "๐Ÿ”", "Cert-Manager", "cert-manager", "cert-manager", "pod", "-l app.kubernetes.io/instance=cert-manager"}, - {"A", "๐Ÿšฆ", "Kyverno", "kyverno", "kyverno", "pod", "-l app.kubernetes.io/instance=kyverno"}, - {"B", "๐Ÿ“จ", "Kafka (" + deployKafkaName + ")", deployKafkaName, kafkaNS, "kafkas.kafka.strimzi.io", deployKafkaName}, - {"B", "๐Ÿ˜", "PostgreSQL (CDC)", "postgresql", dbNS, "statefulset", "postgresql"}, - {"B", "๐Ÿ”—", "Kafka Connect", "connect-cluster", connectNS, "kafkaconnects.kafka.strimzi.io", "connect-cluster"}, - {"B", "๐Ÿ“Š", "Monitoring Stack", "monitoring", jaegerNS, "pod", "-l release=monitoring"}, - {"C", "๐Ÿ“‹", "Apicurio Registry", "apicurio", kafkaNS, "pod", "-l app.kubernetes.io/instance=apicurio"}, + {"A", "๐ŸฆŠ", "Strimzi Operator", "strimzi-operator", "strimzi-operator", "pod", "-l name=strimzi-cluster-operator", false}, + {"A", "๐Ÿ”", "Cert-Manager", "cert-manager", "cert-manager", "pod", "-l app.kubernetes.io/instance=cert-manager", false}, + {"A", "๐Ÿšฆ", "Kyverno", "kyverno", "kyverno", "pod", "-l app.kubernetes.io/instance=kyverno", false}, + {"A", "๐Ÿค–", "Kates Operator", "kates-operator", operatorNS, "pod", "-l app.kubernetes.io/name=kates-operator", false}, + {"B", "๐Ÿ“จ", "Kafka (" + deployKafkaName + ")", deployKafkaName, kafkaNS, "kafkas.kafka.strimzi.io", deployKafkaName, false}, + {"B", "๐Ÿ˜", "PostgreSQL (CDC)", "postgresql", dbNS, "statefulset", "postgresql", false}, + {"B", "๐Ÿ”—", "Kafka Connect", "connect-cluster", connectNS, "kafkaconnects.kafka.strimzi.io", "connect-cluster", false}, + {"B", "๐Ÿ“Š", "Monitoring Stack", "monitoring", jaegerNS, "pod", "-l release=monitoring", false}, + {"C", "๐Ÿ“‹", "Apicurio Registry", "apicurio", kafkaNS, "pod", "-l app.kubernetes.io/instance=apicurio", false}, + {"C", "๐Ÿ˜", "PostgreSQL (Kates)", "kates-db", appNS, "statefulset", "kates-db", false}, // By name: the instance label also matches the chart's embedded PostgreSQL pod. - {"C", "๐Ÿ“ฆ", "Kates Backend", "kates", appNS, "deployment", "kates"}, - {"C", "๐Ÿ’ป", "Kafka UI", "kafka-ui", kafkaUINS, "pod", "-l app.kubernetes.io/name=kafka-ui"}, - {"C", "๐Ÿ”", "MirrorMaker 2", "mm2", mm2NS, "kafkamirrormaker2s.kafka.strimzi.io", "mm2-mirror-maker2"}, - {"C", "๐Ÿงช", "Litmus Chaos", "chaos", chaosNS, "pod", "-l app.kubernetes.io/instance=chaos"}, + {"C", "๐Ÿ“ฆ", "Kates Backend", "kates", appNS, "deployment", "kates", false}, + {"C", "๐Ÿ’ป", "Kafka UI", "kafka-ui", kafkaUINS, "pod", "-l app.kubernetes.io/name=kafka-ui", false}, + {"C", "๐Ÿ”", "MirrorMaker 2", "mm2", mm2NS, "kafkamirrormaker2s.kafka.strimzi.io", "mm2-mirror-maker2", false}, + {"C", "๐Ÿงช", "Litmus Chaos", "chaos", chaosNS, "pod", "-l app.kubernetes.io/instance=chaos", false}, } } @@ -194,7 +198,15 @@ func runDeployStatus(cmd *cobra.Command, args []string) error { helmIndex := loadHelmReleaseIndex(ctx) - components := statusComponents(ns.kafka, ns.connect, ns.app, ns.chaos, ns.jaeger, dbNS, ns.kafkaUI, ns.mm2) + components := statusComponents(ns.kafka, ns.connect, ns.app, ns.chaos, ns.jaeger, dbNS, ns.kafkaUI, ns.mm2, ns.operator) + // Under the operator the backend is a Kates resource's StatefulSet. + if katesResourceExists(ctx, ns.app) { + for i := range components { + if components[i].Release == "kates" { + components[i].Kind, components[i].Managed = "statefulset", true + } + } + } // Route based on flags if deployStatusOutput == "json" { @@ -239,8 +251,8 @@ func runDeployStatus(cmd *cobra.Command, args []string) error { Release: c.Release, Namespace: c.Namespace, } - st.HelmStatus, st.Namespace = getHelmStatusWithNamespace(ctx, helmIndex, c.Release, c.Namespace) - if st.HelmStatus == "deployed" { + st.HelmStatus, st.Namespace = componentHelmStatus(ctx, helmIndex, c) + if st.HelmStatus == "deployed" || st.HelmStatus == operatorManaged { st.Health, st.Details = getHealthStatus(ctx, c.Kind, c.Resource, st.Namespace) } else { st.Health = "N/A" @@ -307,6 +319,17 @@ func lookupHelmRelease(index map[string][]helm.Release, release, namespace strin return "elsewhere (" + strings.Join(elsewhere, ", ") + ")", namespace, true } +// operatorManaged is the Helm status column of a component the Kates operator runs. +const operatorManaged = "operator" + +// componentHelmStatus is a row's Helm status, or operatorManaged for one the operator runs. +func componentHelmStatus(ctx context.Context, index map[string][]helm.Release, c componentSpec) (status, namespace string) { + if c.Managed { + return operatorManaged, c.Namespace + } + return getHelmStatusWithNamespace(ctx, index, c.Release, c.Namespace) +} + func getHelmStatusWithNamespace(ctx context.Context, index map[string][]helm.Release, release, namespace string) (status, resolvedNamespace string) { if st, ns, ok := lookupHelmRelease(index, release, namespace); ok { return st, ns @@ -598,8 +621,8 @@ func runJSONStatus(ctx context.Context, components []componentSpec) ([]Component Release: comp.Release, Namespace: comp.Namespace, } - st.HelmStatus, st.Namespace = getHelmStatusWithNamespace(ctx, helmIndex, comp.Release, comp.Namespace) - if st.HelmStatus == "deployed" { + st.HelmStatus, st.Namespace = componentHelmStatus(ctx, helmIndex, comp) + if st.HelmStatus == "deployed" || st.HelmStatus == operatorManaged { st.Health, st.Details, st.RawDetails = getHealthStatusV2(ctx, comp.Kind, comp.Resource, st.Namespace) } else { st.Health = "N/A" @@ -751,8 +774,8 @@ func runInteractiveStatus(ctx context.Context, components []componentSpec) ([]Co Release: comp.Release, Namespace: comp.Namespace, } - st.HelmStatus, st.Namespace = getHelmStatusWithNamespace(ctx, helmIndex, comp.Release, comp.Namespace) - if st.HelmStatus == "deployed" { + st.HelmStatus, st.Namespace = componentHelmStatus(ctx, helmIndex, comp) + if st.HelmStatus == "deployed" || st.HelmStatus == operatorManaged { st.Health, st.Details, st.RawDetails = getHealthStatusV2(ctx, comp.Kind, comp.Resource, st.Namespace) } else { st.Health = "N/A" diff --git a/cli/cmd/deploy_status_test.go b/cli/cmd/deploy_status_test.go index 928b321b0..69c1821e0 100644 --- a/cli/cmd/deploy_status_test.go +++ b/cli/cmd/deploy_status_test.go @@ -268,7 +268,7 @@ func TestParsePodListHealth(t *testing.T) { // "Unknown" in every status output. Nothing failed, because nothing checked // that the table and the health check agreed. func TestEveryStatusComponentCanBeChecked(t *testing.T) { - for _, c := range statusComponents("kafka", "connect", "kates", "litmus", "monitoring", "database", "kafka", "kafka") { + for _, c := range statusComponents("kafka", "connect", "kates", "litmus", "monitoring", "database", "kafka", "kafka", "kates-operator") { if !healthCheckKinds(c.Kind) { t.Errorf("%s is listed with kind %q, which no health check handles โ€” it would report a blank Unknown", c.Name, c.Kind) } diff --git a/cli/cmd/image_mirror.go b/cli/cmd/image_mirror.go index 1e844f8eb..f93f3c91c 100644 --- a/cli/cmd/image_mirror.go +++ b/cli/cmd/image_mirror.go @@ -55,6 +55,7 @@ var mirrorSets = map[string][]mirrorSet{ "connect-cluster": {{key: "global.imageRegistry"}}, "mirror-maker2": {{key: "global.imageRegistry"}}, "cdc-postgresql": {{key: "global.imageRegistry"}}, + "kates-operator": {{key: "global.imageRegistry"}}, // kube-prometheus-stack's global.imageRegistry replaces each image's // registry: the second name. "monitoring": {{key: "global.imageRegistry"}}, diff --git a/cli/pkg/components/components.go b/cli/pkg/components/components.go index d75292fd9..2afbb1d8b 100644 --- a/cli/pkg/components/components.go +++ b/cli/pkg/components/components.go @@ -36,6 +36,7 @@ const ( NSMonitoring NamespaceKey = "monitoring" NSKafkaUI NamespaceKey = "ui" NSMM2 NamespaceKey = "mm2" + NSOperator NamespaceKey = "operator" ) // Placement is where the user put each part of the stack: the --*-ns flags @@ -43,7 +44,9 @@ const ( // KafkaName is the primary Kafka cluster's name, which is also its release. type Placement struct { Kafka, Connect, Database, App, Chaos, Monitoring, KafkaUI, MM2 string - KafkaName string + // Operator is the Kates operator's namespace, the same in both topologies. + Operator string + KafkaName string } // DefaultPlacement is the isolated topology's defaults, the ones the deploy @@ -52,13 +55,16 @@ func DefaultPlacement() Placement { return Placement{ Kafka: "kafka", Connect: "connect", Database: "database", App: "kates", Chaos: "litmus", Monitoring: "monitoring", KafkaUI: "kafka", MM2: "kafka", - KafkaName: "krafter", + Operator: DefaultOperatorNamespace, KafkaName: "krafter", } } // SingleNamespace is the single topology's default namespace. const SingleNamespace = "kates-stack" +// DefaultOperatorNamespace is where the Kates operator goes unless --operator-ns says otherwise. +const DefaultOperatorNamespace = "kates-operator" + // SinglePlacement puts every namespaced component in namespace. The database // is placed separately because deploy keeps it on --db-ns under both // topologies (an existing database is never moved), while clean folds it in. @@ -66,7 +72,7 @@ func SinglePlacement(namespace, database, kafkaName string) Placement { return Placement{ Kafka: namespace, Connect: namespace, Database: database, App: namespace, Chaos: namespace, Monitoring: namespace, KafkaUI: namespace, MM2: namespace, - KafkaName: kafkaName, + Operator: DefaultOperatorNamespace, KafkaName: kafkaName, } } @@ -135,6 +141,8 @@ func (c Component) Namespace(p Placement) string { return p.KafkaUI case NSMM2: return p.MM2 + case NSOperator: + return p.Operator } return "" } @@ -151,6 +159,7 @@ const ( CRDKyverno = "Kyverno" CRDLitmus = "Litmus" CRDPrometheus = "Prometheus Operator" + CRDKates = "Kates" ) // table is the stack, in install order. @@ -161,6 +170,8 @@ var table = []Component{ FixedNamespace: "cert-manager", Operator: true, CRDGroup: CRDCertManager, Selector: "app.kubernetes.io/instance=cert-manager"}, {ID: "kyverno", Name: "Kyverno", Icon: "๐Ÿšฆ", Group: GroupOperators, Release: "kyverno", Chart: "kyverno/kyverno", FixedNamespace: "kyverno", Operator: true, CRDGroup: CRDKyverno, Selector: "app.kubernetes.io/instance=kyverno"}, + {ID: "kates-operator", Name: "Kates Operator", Icon: "๐Ÿค–", Group: GroupOperators, Release: "kates-operator", Chart: "operator/chart", + NamespaceKey: NSOperator, Operator: true, CRDGroup: CRDKates, Selector: "app.kubernetes.io/name=kates-operator"}, {ID: "kafka", Name: "Kafka Cluster", Icon: "๐Ÿ“จ", Group: GroupCore, Release: KafkaRelease, Chart: "charts/kafka-cluster", NamespaceKey: NSKafka, Selector: "strimzi.io/cluster=%s"}, @@ -175,6 +186,8 @@ var table = []Component{ {ID: "apicurio", Name: "Apicurio Registry", Icon: "๐Ÿ“‹", Group: GroupApps, Release: "apicurio", Chart: "charts/apicurio-registry", NamespaceKey: NSKafka, Selector: "app.kubernetes.io/instance=apicurio"}, + {ID: "kates-db", Name: "PostgreSQL (Kates)", Icon: "๐Ÿ˜", Group: GroupApps, Release: "kates-db", Chart: "charts/cdc-postgresql", + NamespaceKey: NSApp, Selector: "app.kubernetes.io/instance=kates-db"}, {ID: "kates", Name: "Kates Backend", Icon: "๐Ÿ“ฆ", Group: GroupApps, Release: "kates", Chart: "charts/kates", NamespaceKey: NSApp, Selector: "app.kubernetes.io/instance=kates"}, {ID: "kafka-ui", Name: "Kafka UI", Icon: "๐Ÿ’ป", Group: GroupApps, Release: "kafka-ui", Chart: "charts/kafka-ui", @@ -334,6 +347,8 @@ var ( }, {Name: CRDLitmus, CRDs: litmusCRDs, UsedBy: litmusCRDs, Namespaced: litmusCRDs}, {Name: CRDPrometheus, CRDs: prometheusCRDs, UsedBy: prometheusCRDs}, + // No Owner: the operator's namespace is --operator-ns, not fixed. + {Name: CRDKates, CRDs: []string{"kates.kates.io"}, UsedBy: []string{"kates.kates.io"}, Namespaced: []string{"kates.kates.io"}}, } ) diff --git a/cli/pkg/components/components_test.go b/cli/pkg/components/components_test.go index e16b1deb8..eee1db036 100644 --- a/cli/pkg/components/components_test.go +++ b/cli/pkg/components/components_test.go @@ -25,7 +25,7 @@ func TestTableIsConsistent(t *testing.T) { if c.Installs() != (c.Chart != "") || c.Installs() != (c.Icon != "") { t.Errorf("%s: a release needs a chart and an icon, a step neither (release %q chart %q icon %q)", c.ID, c.Release, c.Chart, c.Icon) } - if c.Operator && c.FixedNamespace == "" { + if c.Operator && c.FixedNamespace == "" && c.NamespaceKey != NSOperator { t.Errorf("%s: an operator has a namespace of its own", c.ID) } if c.CRDGroup != "" { @@ -78,13 +78,14 @@ func TestReleasesAndNamespaces(t *testing.T) { "strimzi-operator/strimzi-operator", "cert-manager/cert-manager", "kyverno/kyverno", "krafter/kafka", "monitoring/monitoring", "postgresql/database", "connect-cluster/connect", "apicurio/kafka", "kates/kates", "kafka-ui/kafka", "mm2/kafka", "chaos/litmus", + "kates-operator/kates-operator", "kates-db/kates", } { if !slices.Contains(names, want) { t.Errorf("Releases lacks %s: %v", want, names) } } - if len(names) != 12 { - t.Errorf("Releases = %v, want exactly the 12 releases", names) + if len(names) != 14 { + t.Errorf("Releases = %v, want exactly the 14 releases", names) } for _, n := range names { if strings.HasPrefix(n, "jaeger/") { @@ -93,7 +94,7 @@ func TestReleasesAndNamespaces(t *testing.T) { } ns := Namespaces(p) - want := []string{"kafka", "monitoring", "database", "connect", "kates", "litmus", "strimzi-operator", "cert-manager", "kyverno"} + want := []string{"kafka", "monitoring", "database", "connect", "kates", "litmus", "strimzi-operator", "cert-manager", "kyverno", "kates-operator"} if !slices.Equal(ns, want) { t.Errorf("Namespaces = %v, want %v", ns, want) } @@ -101,8 +102,8 @@ func TestReleasesAndNamespaces(t *testing.T) { func TestCRDGroups(t *testing.T) { all := CRDs() - if len(all) != 39 { - t.Errorf("CRDs() lists %d, want 39", len(all)) + if len(all) != 40 { + t.Errorf("CRDs() lists %d, want 40", len(all)) } for _, g := range Groups() { for _, crd := range g.UsedBy { diff --git a/docs/book/10-cli-reference.md b/docs/book/10-cli-reference.md index dbf9c4598..11c3be85d 100644 --- a/docs/book/10-cli-reference.md +++ b/docs/book/10-cli-reference.md @@ -1554,6 +1554,10 @@ Its flags choose the shape of the stack and how much it asks: | `--checkout` | Install the charts of the Kates checkout in this directory instead of the published ones; `KATES_CHECKOUT` does the same for every command | | `--chart-bundle` | Install from the bundle that [`kates charts pull --to`](#charts-pull) wrote in this directory, with no network; `KATES_CHART_BUNDLE` does the same for every command | | `--adminless` | Install without cluster admin, into namespaces that exist, on a Strimzi operator a cluster admin installed; see [Deploying Without Cluster Admin](12-deployment.md#deploying-without-cluster-admin) | +| `--kates-mode` | `standalone` (default): the `kates` chart, with its own PostgreSQL. `operator`: the Kates operator, a PostgreSQL for Kates, and a `Kates` resource the operator runs; see [Standalone or Under the Operator](12-deployment.md#standalone-or-under-the-operator) | +| `--kates-replicas` | With `--kates-mode operator`: `1` (default), or `3` or `5` for a quorum that keeps a leader with one or two members lost. Fixed once the `Kates` resource exists | +| `--kates-db-secret` | With `--kates-mode operator`: an existing Secret in the Kates namespace, with keys `url` (`jdbc:postgresql://โ€ฆ`), `username` and `password`, instead of installing PostgreSQL | +| `--operator-ns` | Namespace of the Kates operator, under either topology (default: `kates-operator`) | `--operator-scope namespace` lets older Kafka lines run beside the primary under an operator of their own. The loopback MirrorMaker 2 of `--with-mirror-maker2` is the chart's shipped shape, enough to exercise connectors, offset syncs, checkpoints and the `kates-mm2` ACLs without a second cluster. `--adminless` installs Kafka, Kafka Connect with its PostgreSQL, MirrorMaker 2, Apicurio, Kafka UI and Kates, placed by `--topology` and the `--*-ns` flags as in a full deploy; `--topology single` takes `--namespace`, by default the kube context's. diff --git a/docs/book/12-deployment.md b/docs/book/12-deployment.md index 0f6e388b0..70a055054 100644 --- a/docs/book/12-deployment.md +++ b/docs/book/12-deployment.md @@ -131,6 +131,24 @@ kubectl scale deployment/kates --replicas=1 The old PersistentVolumeClaim stays until you delete it. +### Standalone or Under the Operator + +`kates deploy` installs the Kates API one of two ways, chosen with `--kates-mode`, or in the wizard right after the components: + +| | `standalone` (default) | `operator` | +|---|---|---| +| What runs the API | The `kates` chart: a Deployment | The Kates operator (`operator/chart`), from a `Kates` resource: a StatefulSet | +| PostgreSQL | The chart's own | The `kates-db` release of `charts/cdc-postgresql`, open to the Kates pods alone, or your own with `--kates-db-secret` | +| Members | One | One, or three or five with `--kates-replicas`: a quorum whose leader takes the writes | +| Upgrades | A rollout | The operator rolls followers first and the leader last, and under `mode: Drain` waits for live runs | +| Needs cluster admin | No (`--adminless`) | Yes: the operator brings a CRD and a ClusterRole | + +```bash +kates deploy --kates-mode operator --kates-replicas 3 +``` + +Either way the Service is `kates` in the Kates namespace, and the API key is in Secret `kates-api-key`, so `kates ports`, `kates health` and the rest work the same. Deploy refuses to install one mode over the other: uninstall first (`kates clean`), or keep the mode you have. `kates clean` removes the `Kates` resource while the operator still runs, then the operator. The operator doesn't yet support the `INTEGRATION_CDC` test. + For Kafka broker storage, always use persistent volumes โ€” even in development. Kafka's log retention depends on data being durable, and losing broker data mid-test invalidates results. ## Resource Sizing diff --git a/docs/book/appendix-index.md b/docs/book/appendix-index.md index 6aca047c9..b8cefe2ef 100644 --- a/docs/book/appendix-index.md +++ b/docs/book/appendix-index.md @@ -419,7 +419,7 @@ Every term the book teaches, with the sections that teach it: the ones that use ## Q [Quorum]{#idx-quorum} -: [The Cluster Under Test](03-cluster.md): [Kafka You Need for This Book](03-cluster.md#kafka-you-need-for-this-book), [Replication Configuration](03-cluster.md#replication-configuration); [Observability & Monitoring](09-observability.md): [The Dashboard Set](09-observability.md#the-dashboard-set); [Deploying the Strimzi Operator](deploying-strimzi-operator.md): [Adopting an Operator Installed from the Upstream Chart](deploying-strimzi-operator.md#migrating-from-the-ad-hoc-install); [Kafka Deployment Engineering](15-kafka-deployment.md): [Cluster Architecture](15-kafka-deployment.md#cluster-architecture); [Compliance Checks](compliance.md): [Writing Constraints](compliance.md#writing-constraints), [Placing Pods Across Zones and Nodes](compliance.md#placing-pods-across-zones-and-nodes); [REST API Reference](11-api-reference.md): [Endpoints](11-api-reference.md#endpoints) +: [The Cluster Under Test](03-cluster.md): [Kafka You Need for This Book](03-cluster.md#kafka-you-need-for-this-book), [Replication Configuration](03-cluster.md#replication-configuration); [Observability & Monitoring](09-observability.md): [The Dashboard Set](09-observability.md#the-dashboard-set); [Kafka Deployment Engineering](15-kafka-deployment.md): [Cluster Architecture](15-kafka-deployment.md#cluster-architecture); [Compliance Checks](compliance.md): [Writing Constraints](compliance.md#writing-constraints), [Placing Pods Across Zones and Nodes](compliance.md#placing-pods-across-zones-and-nodes); [CLI Reference](10-cli-reference.md): [Commands](10-cli-reference.md#commands); [REST API Reference](11-api-reference.md): [Endpoints](11-api-reference.md#endpoints) [Quota]{#idx-quota} : [Kafka Deployment Engineering](15-kafka-deployment.md): [Broker Settings and Why](15-kafka-deployment.md#broker-settings-and-why); [Multi-Tenancy](19-multi-tenancy.md): [How Quotas Work](19-multi-tenancy.md#how-quotas-work), [Monitoring a Tenant](19-multi-tenancy.md#monitoring-a-tenant) diff --git a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionReportPersistenceTest.java b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionReportPersistenceTest.java index 8b52b3140..35c07c331 100644 --- a/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionReportPersistenceTest.java +++ b/kates/src/test/java/com/bmscomp/kates/disruption/DisruptionReportPersistenceTest.java @@ -240,8 +240,12 @@ void aScheduledRunStoresItsReportOnce() { .extract() .path("id"); try { - // Two members' ticks after the next minute: the due time runs once. - java.time.Instant late = java.time.Instant.now().plusSeconds(61); + // Two members' ticks after the next minute: the due time runs once. Both inside the + // same minute: from now + 61 s, the second tick crossed into the minute after once + // now was past second 49, and that next due time rightly ran too. + java.time.Instant late = java.time.Instant.now() + .truncatedTo(java.time.temporal.ChronoUnit.MINUTES) + .plusSeconds(61); scheduler.evaluate(late); scheduler.evaluate(late.plusSeconds(10)); verify(orchestrator, times(1)).execute(any(), any());