Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion .github/workflows/publish-operator.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,8 @@ jobs:
permissions:
contents: read
packages: write
id-token: write # cosign keyless signing
id-token: write # cosign keyless signing, build provenance
attestations: write # actions/attest-build-provenance
env:
VERSION: ${{ needs.check.outputs.version }}
steps:
Expand Down Expand Up @@ -115,6 +116,7 @@ jobs:
--certificate-oidc-issuer "${ISSUER}" > /dev/null

- name: Push and sign the chart
id: chart
run: |
set -euo pipefail
# helm package is not reproducible: a re-run would replace a digest
Expand All @@ -131,3 +133,13 @@ jobs:
cosign sign --yes "${repo}@${digest}"
cosign verify "${repo}@${digest}" --certificate-identity "${IDENTITY}" \
--certificate-oidc-issuer "${ISSUER}" > /dev/null
echo "digest=${digest}" >> "${GITHUB_OUTPUT}"

# kates deploy --kates-mode operator refuses a published chart without it,
# as it refuses every other chart without one (cli/cmd/deploy_chart_verify.go).
- name: Attest the chart's build provenance
if: steps.chart.outputs.digest != ''
uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
with:
subject-name: ghcr.io/bmscomp/charts/kates-operator
subject-digest: ${{ steps.chart.outputs.digest }}
3 changes: 2 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,8 @@ jobs:
# reads them: the versions this release publishes, which `kates
# deploy` pulls (cli/cmd/deploy_charts.go).
CHART_VERSIONS=""
for chart in $(git -C .. ls-files ':(glob)charts/*/Chart.yaml'); do
# operator/chart is the kates-operator chart, which deploy --kates-mode operator pulls.
for chart in $(git -C .. ls-files ':(glob)charts/*/Chart.yaml' operator/chart/Chart.yaml); do
name=$(sed -nE 's/^name:[[:space:]]*"?([^"[:space:]]+)"?.*$/\1/p' "../${chart}")
version=$(sed -nE 's/^version:[[:space:]]*"?([^"[:space:]]+)"?.*$/\1/p' "../${chart}")
CHART_VERSIONS="${CHART_VERSIONS:+${CHART_VERSIONS},}${name}=${version}"
Expand Down
2 changes: 1 addition & 1 deletion cli/cmd/charts_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func TestAPlainHTTPMirrorIsVerifiedOverHTTP(t *testing.T) {
}

// allChartVersions names every chart deploy installs, as a release build does.
const allChartVersions = "kates-platform=0.8.7,cdc-postgresql=0.1.0,legacy-kafka=0.2.1,apicurio-registry=0.4.1,connect-cluster=2.1.14,kafka-cluster=1.0.18,kafka-ui=0.3.2,kates=0.14.1,kates-chaos=2.3.4,mirror-maker2=0.11.12,kates-monitoring=1.7.1,strimzi-operator=0.4.5"
const allChartVersions = "kates-platform=0.8.7,cdc-postgresql=0.1.0,legacy-kafka=0.2.1,apicurio-registry=0.4.1,connect-cluster=2.1.14,kafka-cluster=1.0.18,kafka-ui=0.3.2,kates=0.14.1,kates-chaos=2.3.4,kates-operator=0.3.0,mirror-maker2=0.11.12,kates-monitoring=1.7.1,strimzi-operator=0.4.5"

func TestAChartBundleIsPulledThenReadWithNoNetwork(t *testing.T) {
withChartBuild(t, allChartVersions)
Expand Down
9 changes: 6 additions & 3 deletions cli/cmd/clean.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ var (
cleanConnectNS string
cleanDbNS string
cleanMM2NS string
cleanOperatorNS string
cleanAppNS string
cleanChaosNS string
cleanMonitoringNS string
Expand All @@ -58,7 +59,7 @@ func init() {
Topology: &cleanTopology, Namespace: &cleanNamespace,
KafkaNS: &cleanKafkaNS, ConnectNS: &cleanConnectNS, DbNS: &cleanDbNS,
AppNS: &cleanAppNS, ChaosNS: &cleanChaosNS, MonitoringNS: &cleanMonitoringNS,
KafkaUINS: &cleanKafkaUINS, MM2NS: &cleanMM2NS,
KafkaUINS: &cleanKafkaUINS, MM2NS: &cleanMM2NS, OperatorNS: &cleanOperatorNS,
}),
scopeTopologyDefault("", "Topology to clean: 'isolated' or 'single'. If empty, cleans both."))
rootCmd.AddCommand(cleanCmd)
Expand All @@ -69,12 +70,14 @@ func init() {
func cleanPlacements() []components.Placement {
var placements []components.Placement
if cleanTopology == "single" || cleanTopology == "" {
placements = append(placements, components.SinglePlacement(cleanNamespace, cleanNamespace, deployKafkaName))
single := components.SinglePlacement(cleanNamespace, cleanNamespace, deployKafkaName)
single.Operator = cleanOperatorNS
placements = append(placements, single)
}
if cleanTopology == "isolated" || cleanTopology == "" {
placements = append(placements, components.Placement{
Kafka: cleanKafkaNS, Connect: cleanConnectNS, Database: cleanDbNS, App: cleanAppNS,
Chaos: cleanChaosNS, Monitoring: cleanMonitoringNS, KafkaUI: cleanKafkaUINS, MM2: cleanMM2NS,
Chaos: cleanChaosNS, Monitoring: cleanMonitoringNS, KafkaUI: cleanKafkaUINS, MM2: cleanMM2NS, Operator: cleanOperatorNS,
KafkaName: deployKafkaName,
})
}
Expand Down
29 changes: 25 additions & 4 deletions cli/cmd/deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,10 +105,16 @@ var (
deployWithKates bool
deployWithMirrorMaker2 bool
deployMM2NS string
deployNamespaceLabels []string
deployNoWriteProbes bool
deployYes bool
deployUpgrade []string
deployOperatorNS string
// Standalone (the kates chart, with its PostgreSQL) or operator (the Kates
// operator, a Kates resource and a PostgreSQL of its own): deploy_kates_operator.go.
deployKatesMode string
deployKatesReplicas int
deployKatesDBSecret string
deployNamespaceLabels []string
deployNoWriteProbes bool
deployYes bool
deployUpgrade []string

// Versions and scope (multi-version plan §3.1–3.3).
deployOperatorScope string
Expand All @@ -133,6 +139,9 @@ func init() {
deployCmd.Flags().BoolVar(&deployWithStrimzi, "with-strimzi", true, "Deploy Strimzi Operator")
deployCmd.Flags().BoolVar(&deployWithKafkaConnect, "with-kafka-connect", false, "Deploy Kafka Connect with PostgreSQL CDC (Debezium)")
deployCmd.Flags().BoolVar(&deployWithKates, "with-kates", true, "Deploy the Kates backend and its PostgreSQL database")
deployCmd.Flags().StringVar(&deployKatesMode, "kates-mode", katesModeStandalone, "How Kates is installed: 'standalone' (the kates chart, with its own PostgreSQL) or 'operator' (the Kates operator, a Kates resource and a PostgreSQL for it)")
deployCmd.Flags().IntVar(&deployKatesReplicas, "kates-replicas", 1, "Kates members with --kates-mode operator: 1, or 3 or 5 for a quorum that keeps a leader when members are lost. Fixed once the Kates resource exists")
deployCmd.Flags().StringVar(&deployKatesDBSecret, "kates-db-secret", "", "With --kates-mode operator: an existing Secret in the Kates namespace with keys url (jdbc:postgresql://...), username and password, instead of installing PostgreSQL")
deployCmd.Flags().BoolVar(&deployWithKafkaUI, "with-kafka-ui", true, "Deploy Kafka UI for browser-based cluster monitoring")
deployCmd.Flags().BoolVar(&deployWithMirrorMaker2, "with-mirror-maker2", false, "Deploy MirrorMaker 2 as a loopback mirror of the primary (replication lab; kates migrate for real sources)")
deployCmd.Flags().StringArrayVar(&deployNamespaceLabels, "namespace-label", nil, "Label to apply to the namespaces deploy ensures (key=value, repeatable; overrides KATES_NAMESPACE_LABELS, which overrides the labels file)")
Expand Down Expand Up @@ -452,6 +461,18 @@ func configureDeploySelection(cmd *cobra.Command, wizard bool) error {
deployDbNS = deployNamespace
}
}
if err := validateKatesMode(); err != nil {
return err
}
if err := checkKatesModeSwitch(context.Background(), resolveNamespaces().app); err != nil {
return err
}
// The charts were pulled before the wizard, which may have chosen operator mode.
if katesOperatorMode() && chartsPublished {
if err := resolveChartRoot(context.Background(), deployCheckout, false, deployChartDirs()...); err != nil {
return err
}
}
if err := checkUpgradeInDeploy(deployUpgrade, buildSharedEntries(resolveNamespaces())); err != nil {
return err
}
Expand Down
22 changes: 16 additions & 6 deletions cli/cmd/deploy_chart_verify.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,19 @@ import (
// identity names the workflow rather than this build's tag.
const (
chartSignerIdentity = `^https://github\.com/bmscomp/kates/\.github/workflows/publish-charts\.yml@refs/(heads/main|tags/v.+)$`
chartSignerIssuer = "https://token.actions.githubusercontent.com"
// The operator's chart is released apart, by publish-operator.yml on its own tags.
operatorChartSignerIdentity = `^https://github\.com/bmscomp/kates/\.github/workflows/publish-operator\.yml@refs/tags/operator-v.+$`
chartSignerIssuer = "https://token.actions.githubusercontent.com"
)

// chartSigner is the workflow identity that signs the chart at ref.
func chartSigner(ref string) string {
if strings.HasSuffix(ref, "/kates-operator") {
return operatorChartSignerIdentity
}
return chartSignerIdentity
}

// cosignPathFn finds cosign; tests replace it.
var cosignPathFn = func() (string, error) { return exec.LookPath("cosign") }

Expand Down Expand Up @@ -108,7 +118,7 @@ func verifyChart(ctx context.Context, ref, digest, dest string) (provenance bool
return false, err
}
}
return verifyProvenance(ctx, cosign, digest)
return verifyProvenance(ctx, cosign, digest, chartSigner(ref))
}

// verifySignature is cosign verify of ref@digest.
Expand All @@ -117,7 +127,7 @@ func verifySignature(ctx context.Context, cosign, ref, digest string) error {
vctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
args := []string{"verify", image,
"--certificate-identity-regexp", chartSignerIdentity,
"--certificate-identity-regexp", chartSigner(ref),
"--certificate-oidc-issuer", chartSignerIssuer}
if chartRegistryPlainHTTP() {
args = append(args, "--allow-http-registry")
Expand Down Expand Up @@ -191,9 +201,9 @@ func (e *chartProvenanceError) Error() string {
}

// verifyProvenance checks that one of GitHub's attestations for digest is a
// SLSA provenance signed by publish-charts.yml whose subject is digest. It
// SLSA provenance signed by signer's workflow whose subject is digest. It
// returns false, with no error, when the attestations could not be read.
func verifyProvenance(ctx context.Context, cosign, digest string) (bool, error) {
func verifyProvenance(ctx context.Context, cosign, digest, signer string) (bool, error) {
bundles, err := attestationsFn(ctx, digest)
if errors.Is(err, errProvenanceUnavailable) {
provenanceUnchecked = err
Expand All @@ -218,7 +228,7 @@ func verifyProvenance(ctx context.Context, cosign, digest string) (bool, error)
}
vctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
out, err := runExecCombinedFn(vctx, cosign, "verify-blob-attestation", "--bundle", path,
"--certificate-identity-regexp", chartSignerIdentity,
"--certificate-identity-regexp", signer,
"--certificate-oidc-issuer", chartSignerIssuer,
"--type", "slsaprovenance1",
"--digest", strings.TrimPrefix(digest, "sha256:"), "--digestAlg", "sha256")
Expand Down
5 changes: 4 additions & 1 deletion cli/cmd/deploy_charts.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ var deployChartNames = map[string]string{
"kafka-ui": "kafka-ui",
"kates": "kates",
"kates-chaos": "kates-chaos",
"kates-operator": "kates-operator",
"kates-platform": "kates-platform",
"mirror-maker2": "mirror-maker2",
"monitoring": "kates-monitoring",
Expand Down Expand Up @@ -363,7 +364,9 @@ func chartSource() string {
// kates-platform umbrella, which deploy never installs but a bundle carries,
// verified, for a helm install of it without network.
func deployChartDirs() []string {
return slices.DeleteFunc(publishedChartDirs(), func(dir string) bool { return dir == "legacy-kafka" || dir == "kates-platform" })
return slices.DeleteFunc(publishedChartDirs(), func(dir string) bool {
return dir == "legacy-kafka" || dir == "kates-platform" || (dir == "kates-operator" && !katesOperatorMode())
})
}

// publishedChartDirs lists every chart the CLI pulls, deploy's and migrate's.
Expand Down
17 changes: 16 additions & 1 deletion cli/cmd/deploy_components.go
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,9 @@ func selectedDeployNamespaces(dc *deployContext) []string {
if deployWithKates {
add(dc.ns.app)
}
if componentSelected("kates-operator") {
add(dc.ns.operator)
}
if deployWithMonitoring {
add(dc.ns.jaeger)
}
Expand Down Expand Up @@ -631,6 +634,8 @@ func deployComponents(dc *deployContext) []component {
dc.kafkaConnectComponent(),
dc.kafkaConnectorComponent(),
dc.apicurioComponent(),
dc.katesOperatorComponent(),
dc.katesDBComponent(),
dc.katesComponent(),
dc.kafkaUIComponent(),
dc.mirrorMaker2Component(),
Expand Down Expand Up @@ -1684,9 +1689,13 @@ func (dc *deployContext) katesComponent() component {
kafkaNS := dc.ns.kafka
appNS := dc.ns.app
chaosNS := dc.ns.chaos
deps := []string{"kafka-users", "postgres", "monitoring"}
if katesOperatorMode() {
deps = append(deps, "kates-operator", "kates-db")
}
return component{
id: "kates", name: "Kates Backend", namespace: appNS,
deps: []string{"kafka-users", "postgres", "monitoring"}, timeout: 8 * time.Minute,
deps: deps, timeout: 10 * time.Minute,
enabled: func() bool { return deployWithKates },
// The KafkaUser was already waited on (kafka-users) — just read the Secret.
secrets: []secretCopy{{name: "kates-backend"}},
Expand All @@ -1698,6 +1707,9 @@ func (dc *deployContext) katesComponent() component {
return fmt.Sprintf("\n📦 %s Kates Backend (Namespace: %s)...", verb, appNS)
},
install: func(ctx context.Context) error {
if katesOperatorMode() {
return dc.installKatesResource(ctx)
}
// Auto-cleanup stale ClusterRole ownership from previous topology switches
cleanupStaleClusterResource(ctx, "clusterrole", "kates", appNS)
cleanupStaleClusterResource(ctx, "clusterrolebinding", "kates", appNS)
Expand Down Expand Up @@ -1759,6 +1771,9 @@ func (dc *deployContext) katesComponent() component {
return runKatesHelmUpgrade(ctx, appNS, katesArgs...)
},
wait: func(ctx context.Context) error {
if katesOperatorMode() {
return waitKatesResource(ctx, appNS)
}
return dc.waitPods(ctx, appNS, "app.kubernetes.io/instance=kates", 8*time.Minute, "")
},
}
Expand Down
13 changes: 9 additions & 4 deletions cli/cmd/deploy_flags.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
package cmd

import "github.com/spf13/cobra"
import (
"github.com/spf13/cobra"

"github.com/bmscomp/kates/cli/pkg/components"
)

// The topology and namespace flags place the stack. deploy, deploy status,
// clean and kyverno apply all take them, and each used to declare its own
Expand All @@ -12,7 +16,7 @@ import "github.com/spf13/cobra"
// deployScopeVars are the variables the scope flags bind. deploy, deploy
// status and kyverno apply share deploy's package globals; clean has its own.
type deployScopeVars struct {
Topology, Namespace, KafkaNS, ConnectNS, DbNS, AppNS, ChaosNS, MonitoringNS, KafkaUINS, MM2NS *string
Topology, Namespace, KafkaNS, ConnectNS, DbNS, AppNS, ChaosNS, MonitoringNS, KafkaUINS, MM2NS, OperatorNS *string
}

// deployGlobalScope binds the flags to deploy's package-level variables.
Expand All @@ -21,7 +25,7 @@ func deployGlobalScope() deployScopeVars {
Topology: &deployTopology, Namespace: &deployNamespace,
KafkaNS: &deployKafkaNS, ConnectNS: &deployConnectNS, DbNS: &deployDbNS,
AppNS: &deployAppNS, ChaosNS: &deployChaosNS, MonitoringNS: &deployMonitoringNS,
KafkaUINS: &deployKafkaUINS, MM2NS: &deployMM2NS,
KafkaUINS: &deployKafkaUINS, MM2NS: &deployMM2NS, OperatorNS: &deployOperatorNS,
}
}

Expand All @@ -46,7 +50,7 @@ func scopeTopologyDefault(value, usage string) deployScopeOption {

// deployScopeFlagNames is the set addDeployScopeFlags registers, for the test
// that checks every command has it.
var deployScopeFlagNames = []string{"topology", "namespace", "kafka-ns", "connect-ns", "db-ns", "app-ns", "chaos-ns", "monitoring-ns", "ui-ns", "mm2-ns", "kafka-name"}
var deployScopeFlagNames = []string{"topology", "namespace", "kafka-ns", "connect-ns", "db-ns", "app-ns", "chaos-ns", "monitoring-ns", "ui-ns", "mm2-ns", "operator-ns", "kafka-name"}

// addDeployScopeFlags registers the topology and namespace flags on cmd.
func addDeployScopeFlags(cmd *cobra.Command, opts ...deployScopeOption) {
Expand All @@ -70,6 +74,7 @@ func addDeployScopeFlags(cmd *cobra.Command, opts ...deployScopeOption) {
f.StringVar(v.MonitoringNS, "monitoring-ns", "monitoring", "Namespace for monitoring components (Jaeger) when topology is 'isolated'")
f.StringVar(v.KafkaUINS, "ui-ns", "kafka", "Namespace for Kafka UI when topology is 'isolated'")
f.StringVar(v.MM2NS, "mm2-ns", "kafka", "Namespace for MirrorMaker 2 when topology is 'isolated'")
f.StringVar(v.OperatorNS, "operator-ns", components.DefaultOperatorNamespace, "Namespace for the Kates operator (--kates-mode operator), under either topology")
// The Kafka release is named by --kafka-name everywhere; clean already
// bound deploy's variable, so it stays shared.
f.StringVar(&deployKafkaName, "kafka-name", "krafter", "Name of the primary Kafka cluster")
Expand Down
Loading
Loading