From 7dbf6fd7920f8780a574ead608552b7f3541f3ce Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Mon, 16 Nov 2026 22:29:10 +0100 Subject: [PATCH 1/2] fix(scripts): wait for KafkaUser Secrets to settle before they are copied MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Strimzi's User Operator reads a user's Secret from its informer cache. The reconcile its own status update triggers can run before the cache holds the Secret the first reconcile created: it issues a new password, sets it in Kafka and overwrites the Secret (create, 409, update). A copy taken at the first Ready, as e2e-stack.sh and integration.yml do, can hold the password that was just replaced, and every Kafka call then fails authentication. That failed CI · Operator twice on main at f566a19a, the tree #684's PR run passed with: both runs copied within a second of Ready. After the Ready wait, deploy-kafka-generic.sh now waits until the user Secrets' resource versions hold for 10 s (60 s at most). --- scripts/deploy-kafka-generic.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/deploy-kafka-generic.sh b/scripts/deploy-kafka-generic.sh index 2c7d24591..a647d9c42 100755 --- a/scripts/deploy-kafka-generic.sh +++ b/scripts/deploy-kafka-generic.sh @@ -241,6 +241,19 @@ kubectl wait kafka/krafter --for=condition=Ready --timeout=600s -n "${NAMESPACE} info "Waiting for user secrets..." kubectl wait kafkauser --all --for=condition=Ready --timeout=60s -n "${NAMESPACE}" 2>/dev/null || true +# Ready is not final: the User Operator's next reconcile reads the Secret from a cache that can +# still miss it, issues a new password and overwrites the Secret. A copy taken now can be stale. +user_secrets() { + kubectl get secret -n "${NAMESPACE}" -l strimzi.io/kind=KafkaUser \ + -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.resourceVersion} {end}' 2>/dev/null +} +seen=$(user_secrets) +for _ in 1 2 3 4 5 6; do + sleep 10 + now=$(user_secrets) + [ -n "$now" ] && [ "$now" = "$seen" ] && break + seen=$now +done # ── Step 6: Verify ──────────────────────────────────────────────────────────── echo "" From d7735378dc688019405dbf2e2ddbe31fb0485e25 Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Mon, 16 Nov 2026 22:51:46 +0100 Subject: [PATCH 2/2] fix(cli): kates deploy copies KafkaUser Secrets only once they hold The kafka-users wait now ends with the same settle as deploy-kafka-generic.sh: the KafkaUser Secrets' resource versions must hold for 10 s (a minute at most). Kates, Connect, Kafka UI and MirrorMaker 2 copy their Secrets after it, so none copies a password the User Operator replaces a moment later. --- cli/cmd/deploy_components.go | 2 ++ cli/cmd/silent_wait.go | 26 ++++++++++++++++++++++++ cli/cmd/silent_wait_test.go | 39 ++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/cli/cmd/deploy_components.go b/cli/cmd/deploy_components.go index 8ace552e0..4ee84ee41 100644 --- a/cli/cmd/deploy_components.go +++ b/cli/cmd/deploy_components.go @@ -1303,6 +1303,8 @@ func (dc *deployContext) kafkaUsersComponent() component { if err := waitKafkaUsersReadySilent(ctx, kafkaNS, 8*time.Minute); err != nil { dl.Printf(" %s KafkaUsers not all ready after 8m — downstream deploys will retry secret lookup\n", output.WarningStyle.Render("⚠")) } + dl.Println(" - Waiting for the user Secrets to settle...") + waitUserSecretsSettled(ctx, kafkaNS) return nil }, } diff --git a/cli/cmd/silent_wait.go b/cli/cmd/silent_wait.go index 60a63adc2..ad63122ce 100644 --- a/cli/cmd/silent_wait.go +++ b/cli/cmd/silent_wait.go @@ -345,6 +345,32 @@ func waitKafkaUsersReadySilent(ctx context.Context, namespace string, timeout ti return waitCustomResourceReadySilent(ctx, "kafkauser", namespace, "strimzi.io/cluster="+deployKafkaName, timeout) } +// waitUserSecretsSettled returns once the KafkaUser Secrets in namespace have +// kept their resource versions for 10 s, after six tries at most. Ready is +// not final: the User Operator's next reconcile reads the Secret from a cache +// that can still miss it, issues a new password and overwrites the Secret, so +// a copy taken at Ready can hold a password Kafka no longer accepts. +func waitUserSecretsSettled(ctx context.Context, namespace string) { + versions := func() string { + out, _ := runExecOutputFn(ctx, "kubectl", "get", "secret", "-n", namespace, "-l", "strimzi.io/kind=KafkaUser", + "-o", "jsonpath={range .items[*]}{.metadata.name}={.metadata.resourceVersion} {end}") + return strings.TrimSpace(string(out)) + } + seen := versions() + for range 6 { + select { + case <-ctx.Done(): + return + case <-deployClock.After(10 * time.Second): + } + now := versions() + if now != "" && now == seen { + return + } + seen = now + } +} + func waitKafkaConnectReadySilent(ctx context.Context, namespace, name string, timeout time.Duration) error { return waitNamedCustomResourceReadySilent(ctx, "kafkaconnect", namespace, name, timeout) } diff --git a/cli/cmd/silent_wait_test.go b/cli/cmd/silent_wait_test.go index 1fc01acfe..6306c0882 100644 --- a/cli/cmd/silent_wait_test.go +++ b/cli/cmd/silent_wait_test.go @@ -245,3 +245,42 @@ func TestWaitKafkaReady_NamesWhyPodsWait(t *testing.T) { t.Errorf("err = %v, want the pods' reason", err) } } + +// userSecretReads answers the settle wait's reads in turn, repeating the last. +func userSecretReads(t *testing.T, answers ...string) *int { + t.Helper() + origOut, prevClock := runExecOutputFn, deployClock + t.Cleanup(func() { runExecOutputFn, deployClock = origOut, prevClock }) + deployClock = newFakeClock() + reads := 0 + runExecOutputFn = func(_ context.Context, _ string, args ...string) ([]byte, error) { + if !strings.Contains(strings.Join(args, " "), "get secret -n kafka -l strimzi.io/kind=KafkaUser") { + t.Fatalf("unexpected read %v", args) + } + reads++ + return []byte(answers[min(reads, len(answers))-1]), nil + } + return &reads +} + +// A Secret the User Operator rewrites right after Ready is copied only once it holds. +func TestUserSecretsAreCopiedOnlyOnceTheyHold(t *testing.T) { + reads := userSecretReads(t, "kates-backend=1 ", "kates-backend=2 ", "kates-backend=2 ") + start := deployClock.Now() + waitUserSecretsSettled(context.Background(), "kafka") + if *reads != 3 || deployClock.Now().Sub(start) != 20*time.Second { + t.Fatalf("returned after %d reads and %v, want 3 reads and 20s", *reads, deployClock.Now().Sub(start)) + } +} + +// Secrets that never hold, or none at all, do not stall the deploy past a minute. +func TestUserSecretsThatNeverHoldStopTheWaitAfterAMinute(t *testing.T) { + for _, answers := range [][]string{{"a=1", "a=2", "a=3", "a=4", "a=5", "a=6", "a=7"}, {""}} { + reads := userSecretReads(t, answers...) + start := deployClock.Now() + waitUserSecretsSettled(context.Background(), "kafka") + if *reads != 7 || deployClock.Now().Sub(start) != time.Minute { + t.Fatalf("%v: returned after %d reads and %v, want 7 reads and 1m", answers, *reads, deployClock.Now().Sub(start)) + } + } +}