Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cli/cmd/deploy_components.go
Original file line number Diff line number Diff line change
Expand Up @@ -1303,6 +1303,8 @@ func (dc *deployContext) kafkaUsersComponent() component {
if err := waitKafkaUsersReadySilent(ctx, kafkaNS, 8*time.Minute); err != nil {
dl.Printf(" %s KafkaUsers not all ready after 8m — downstream deploys will retry secret lookup\n", output.WarningStyle.Render("⚠"))
}
dl.Println(" - Waiting for the user Secrets to settle...")
waitUserSecretsSettled(ctx, kafkaNS)
return nil
},
}
Expand Down
26 changes: 26 additions & 0 deletions cli/cmd/silent_wait.go
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,32 @@ func waitKafkaUsersReadySilent(ctx context.Context, namespace string, timeout ti
return waitCustomResourceReadySilent(ctx, "kafkauser", namespace, "strimzi.io/cluster="+deployKafkaName, timeout)
}

// waitUserSecretsSettled returns once the KafkaUser Secrets in namespace have
// kept their resource versions for 10 s, after six tries at most. Ready is
// not final: the User Operator's next reconcile reads the Secret from a cache
// that can still miss it, issues a new password and overwrites the Secret, so
// a copy taken at Ready can hold a password Kafka no longer accepts.
func waitUserSecretsSettled(ctx context.Context, namespace string) {
versions := func() string {
out, _ := runExecOutputFn(ctx, "kubectl", "get", "secret", "-n", namespace, "-l", "strimzi.io/kind=KafkaUser",
"-o", "jsonpath={range .items[*]}{.metadata.name}={.metadata.resourceVersion} {end}")
return strings.TrimSpace(string(out))
}
seen := versions()
for range 6 {
select {
case <-ctx.Done():
return
case <-deployClock.After(10 * time.Second):
}
now := versions()
if now != "" && now == seen {
return
}
seen = now
}
}

func waitKafkaConnectReadySilent(ctx context.Context, namespace, name string, timeout time.Duration) error {
return waitNamedCustomResourceReadySilent(ctx, "kafkaconnect", namespace, name, timeout)
}
Expand Down
39 changes: 39 additions & 0 deletions cli/cmd/silent_wait_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -245,3 +245,42 @@ func TestWaitKafkaReady_NamesWhyPodsWait(t *testing.T) {
t.Errorf("err = %v, want the pods' reason", err)
}
}

// userSecretReads answers the settle wait's reads in turn, repeating the last.
func userSecretReads(t *testing.T, answers ...string) *int {
t.Helper()
origOut, prevClock := runExecOutputFn, deployClock
t.Cleanup(func() { runExecOutputFn, deployClock = origOut, prevClock })
deployClock = newFakeClock()
reads := 0
runExecOutputFn = func(_ context.Context, _ string, args ...string) ([]byte, error) {
if !strings.Contains(strings.Join(args, " "), "get secret -n kafka -l strimzi.io/kind=KafkaUser") {
t.Fatalf("unexpected read %v", args)
}
reads++
return []byte(answers[min(reads, len(answers))-1]), nil
}
return &reads
}

// A Secret the User Operator rewrites right after Ready is copied only once it holds.
func TestUserSecretsAreCopiedOnlyOnceTheyHold(t *testing.T) {
reads := userSecretReads(t, "kates-backend=1 ", "kates-backend=2 ", "kates-backend=2 ")
start := deployClock.Now()
waitUserSecretsSettled(context.Background(), "kafka")
if *reads != 3 || deployClock.Now().Sub(start) != 20*time.Second {
t.Fatalf("returned after %d reads and %v, want 3 reads and 20s", *reads, deployClock.Now().Sub(start))
}
}

// Secrets that never hold, or none at all, do not stall the deploy past a minute.
func TestUserSecretsThatNeverHoldStopTheWaitAfterAMinute(t *testing.T) {
for _, answers := range [][]string{{"a=1", "a=2", "a=3", "a=4", "a=5", "a=6", "a=7"}, {""}} {
reads := userSecretReads(t, answers...)
start := deployClock.Now()
waitUserSecretsSettled(context.Background(), "kafka")
if *reads != 7 || deployClock.Now().Sub(start) != time.Minute {
t.Fatalf("%v: returned after %d reads and %v, want 7 reads and 1m", answers, *reads, deployClock.Now().Sub(start))
}
}
}
13 changes: 13 additions & 0 deletions scripts/deploy-kafka-generic.sh
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,19 @@ kubectl wait kafka/krafter --for=condition=Ready --timeout=600s -n "${NAMESPACE}

info "Waiting for user secrets..."
kubectl wait kafkauser --all --for=condition=Ready --timeout=60s -n "${NAMESPACE}" 2>/dev/null || true
# Ready is not final: the User Operator's next reconcile reads the Secret from a cache that can
# still miss it, issues a new password and overwrites the Secret. A copy taken now can be stale.
user_secrets() {
kubectl get secret -n "${NAMESPACE}" -l strimzi.io/kind=KafkaUser \
-o jsonpath='{range .items[*]}{.metadata.name}={.metadata.resourceVersion} {end}' 2>/dev/null
}
seen=$(user_secrets)
for _ in 1 2 3 4 5 6; do
sleep 10
now=$(user_secrets)
[ -n "$now" ] && [ "$now" = "$seen" ] && break
seen=$now
done

# ── Step 6: Verify ────────────────────────────────────────────────────────────
echo ""
Expand Down
Loading