[CRDB-53973] e2e/operator: advanced e2e tests for bare-metal infrastructure(kind).

.github/workflows/integration-tests-advanced.yaml

@@ -0,0 +1,139 @@
+name: advanced-features-tests
+
+on:
+  schedule:
+    - cron: "15 6 * * 1"  # Run at 6:15 AM on every Monday UTC to avoid peak times
+  # Allow manual trigger on any branch
+  workflow_dispatch:
+
+jobs:
+  integration-tests-advanced-features:
+    runs-on: ubuntu-latest-4-core
+    strategy:
+      fail-fast: false
+      matrix:
+        provider: [k3d, kind]
+    timeout-minutes: 120
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.23
+      - name: Authenticate to Google Cloud
+        uses: 'google-github-actions/auth@v2'
+        id: auth
+        with:
+          token_format: access_token
+          project_id: 'cockroach-helm-testing'
+          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+      - name: Set up gcloud CLI
+        uses: google-github-actions/setup-gcloud@v2
+      - name: Install gke-gcloud-auth-plugin
+        run: |
+          gcloud components install gke-gcloud-auth-plugin
+      - name: Run tests (advanced features)
+        env:
+          PROVIDER: ${{ matrix.provider }}
+          TEST_ADVANCED_FEATURES: true
+          USE_GKE_GCLOUD_AUTH_PLUGIN: True
+        run: |
+          set -euo pipefail
+          make test/nightly-e2e/advanced | tee test_output.log
+      - name: Run multi-region tests (advanced features)
+        env:
+          PROVIDER: ${{ matrix.provider }}
+          TEST_ADVANCED_FEATURES: true
+          USE_GKE_GCLOUD_AUTH_PLUGIN: True
+        run: |
+          set -euo pipefail
+          make test/nightly-e2e/advanced/multi-region | tee -a test_output.log
+      - name: Archive test results
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: advanced-features-test-results-${{ matrix.provider }}
+          path: |
+            test_output.log
+            test_output.json
+      - name: Slack notification (advanced features)
+        if: ${{ always() && github.ref_name == 'master' }}
+        run: |
+          echo "Preparing Slack message for advanced features tests..."
+
+          if [ ! -f test_output.json ]; then
+            echo "Generating test_output.json from log..."
+            grep -E "=== RUN|--- PASS:|--- FAIL:" test_output.log | awk '
+              /=== RUN/ {
+                test_name = $3
+                tests[test_name] = "running"
+              }
+              /--- PASS:/ {
+                test_name = $3
+                if (test_name in tests) {
+                  print "{\"Action\":\"pass\",\"Test\":\"" test_name "\"}"
+                }
+              }
+              /--- FAIL:/ {
+                test_name = $3
+                if (test_name in tests) {
+                  print "{\"Action\":\"fail\",\"Test\":\"" test_name "\"}"
+                }
+              }' > test_output.json
+          fi
+
+          PASSED_TESTS=$(jq -r 'select(.Action=="pass") | .Test' test_output.json | sed 's|.*/||' | grep '^Test' | grep -v '^TestOperatorIn' | sort | uniq | head -n 20 | paste -sd ", " -)
+          FAILED_TESTS=$(jq -r 'select(.Action=="fail") | .Test' test_output.json | sed 's|.*/||' | grep '^Test' | grep -v '^TestOperatorIn' | sort | uniq | head -n 20 | paste -sd ", " -)
+          PASSED_COUNT=$(jq -r 'select(.Action=="pass") | .Test' test_output.json | sed 's|.*/||' | grep '^Test' | grep -v '^TestOperatorIn' | sort | uniq | wc -l)
+          FAILED_COUNT=$(jq -r 'select(.Action=="fail") | .Test' test_output.json | sed 's|.*/||' | grep '^Test' | grep -v '^TestOperatorIn' | sort | uniq | wc -l)
+
+          if [ "$FAILED_COUNT" -gt 0 ]; then
+            STATUS="❌ *Advanced features integration tests failed!*"
+            COLOR="#ff0000"
+          else
+            STATUS="✅ *Advanced features integration tests passed!*"
+            COLOR="#36a64f"
+          fi
+
+          REPO_NAME="helm-charts"
+
+          PAYLOAD=$(jq -n \
+            --arg text "$STATUS" \
+            --arg color "$COLOR" \
+            --arg repo "$REPO_NAME" \
+            --arg branch "${{ github.ref_name }}" \
+            --arg workflow "${{ github.workflow }}" \
+            --arg provider "${{ matrix.provider }}" \
+            --arg url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
+            --arg passed "$PASSED_COUNT" \
+            --arg failed "$FAILED_COUNT" \
+            --arg passed_tests "$PASSED_TESTS" \
+            --arg failed_tests "$FAILED_TESTS" \
+            '{
+              text: $text,
+              attachments: [
+                {
+                  color: $color,
+                  fields: [
+                    {"title": "Repository", "value": $repo, "short": true},
+                    {"title": "Branch", "value": $branch, "short": true},
+                    {"title": "Provider", "value": $provider, "short": true},
+                    {"title": "✅ Passed", "value": $passed, "short": true},
+                    {"title": "❌ Failed", "value": $failed, "short": true},
+                    {"title": "❌ Failed Tests", "value": (if $failed_tests == "" then "None" else $failed_tests end), "short": false},
+                    {"title": "✅ Passed Tests", "value": (if $passed_tests == "" then "None" else $passed_tests end), "short": false},
+                    {"title": "Run URL", "value": $url, "short": false}
+                  ]
+                }
+              ]
+            }')
+
+          curl -X POST -H 'Content-type: application/json' \
+            --data "$PAYLOAD" \
+            "${{ secrets.SLACK_WEBHOOK_URL }}"

Makefile

@@ -137,6 +137,34 @@ test/single-cluster/up: bin/k3d
 test/multi-cluster/down: bin/k3d
 	 ./tests/k3d/dev-multi-cluster.sh down
 
+test/nightly-e2e/advanced: bin/cockroach bin/kubectl bin/helm build/self-signer bin/k3d bin/kind
+	@PATH="$(PWD)/bin:${PATH}" TEST_ADVANCED_FEATURES=true go test -timeout 90m -v -test.run TestOperatorInSingleRegion ./tests/e2e/operator/singleRegion/... || (echo "Advanced features tests failed with exit code $$?" && exit 1)
+
+test/nightly-e2e/advanced/multi-region: bin/cockroach bin/kubectl bin/helm build/self-signer bin/k3d bin/kind
+	@PATH="$(PWD)/bin:${PATH}" TEST_ADVANCED_FEATURES=true go test -timeout 90m -v -test.run TestOperatorInMultiRegion ./tests/e2e/operator/multiRegion/... || (echo "Advanced features multi-region tests failed with exit code $$?" && exit 1)
+
+# Run advanced e2e tests using kind clusters on a GCE worker.
+# Creates kind clusters, runs all advanced feature tests, and leaves clusters running (no cleanup).
+# Use this for the first run or whenever you want a fresh cluster.
+test/nightly-e2e/advanced/kind: bin/cockroach bin/kubectl bin/helm build/self-signer bin/kind
+	@PATH="$(PWD)/bin:${PATH}" PROVIDER=kind SKIP_CLEANUP=true TEST_ADVANCED_FEATURES=true go test -timeout 90m -v -test.run TestOperatorInSingleRegion ./tests/e2e/operator/singleRegion/... || (echo "Advanced features tests (kind) failed with exit code $$?" && exit 1)
+
+# Re-run advanced e2e tests on existing kind clusters (skips cluster creation).
+# Use this after test/nightly-e2e/advanced/kind when clusters are already up and you want to
+# iterate on failures without recreating infrastructure.
+test/nightly-e2e/advanced/kind/reuse: bin/cockroach bin/kubectl bin/helm build/self-signer bin/kind
+	@PATH="$(PWD)/bin:${PATH}" PROVIDER=kind REUSE_INFRA=true SKIP_CLEANUP=true TEST_ADVANCED_FEATURES=true go test -timeout 90m -v -test.run TestOperatorInSingleRegion ./tests/e2e/operator/singleRegion/... || (echo "Advanced features tests (kind reuse) failed with exit code $$?" && exit 1)
+
+# Run multi-region advanced e2e tests using kind clusters on a GCE worker.
+# Creates kind clusters, runs all multi-region advanced feature tests, and leaves clusters running.
+test/nightly-e2e/advanced/kind/multi-region: bin/cockroach bin/kubectl bin/helm build/self-signer bin/kind
+	@PATH="$(PWD)/bin:${PATH}" PROVIDER=kind SKIP_CLEANUP=true TEST_ADVANCED_FEATURES=true go test -timeout 90m -v -test.run TestOperatorInMultiRegion ./tests/e2e/operator/multiRegion/... || (echo "Advanced features tests (kind multi-region) failed with exit code $$?" && exit 1)
+
+# Re-run multi-region advanced e2e tests on existing kind clusters (skips cluster creation).
+test/nightly-e2e/advanced/kind/multi-region/reuse: bin/cockroach bin/kubectl bin/helm build/self-signer bin/kind
+	@PATH="$(PWD)/bin:${PATH}" PROVIDER=kind REUSE_INFRA=true SKIP_CLEANUP=true TEST_ADVANCED_FEATURES=true go test -timeout 90m -v -test.run TestOperatorInMultiRegion ./tests/e2e/operator/multiRegion/... || (echo "Advanced features tests (kind multi-region reuse) failed with exit code $$?" && exit 1)
+
+
 test/lint: bin/helm ## lint the helm chart
 	@build/lint.sh && \
 	bin/helm lint cockroachdb && \

tests/e2e/migrate/helm_chart_to_cockroach_enterprise_operator_test.go

@@ -143,7 +143,7 @@ func (h *HelmChartToOperator) TestDefaultMigration(t *testing.T) {
 		k8s.RunKubectl(t, kubectlOptions, "delete", "priorityclass", "crdb-critical")
 	}()
 
-	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
+	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, "us-east-1")
 	defer func() {
 		t.Log("Uninstall the cockroachdb enterprise operator")
 		operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)
@@ -261,7 +261,7 @@ func (h *HelmChartToOperator) TestCertManagerMigration(t *testing.T) {
 		k8s.RunKubectl(t, kubectlOptions, "delete", "priorityclass", "crdb-critical")
 	}()
 
-	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
+	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, "us-east-1")
 	defer func() {
 		t.Log("Uninstall the cockroachdb enterprise operator")
 		operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)
@@ -348,7 +348,7 @@ func (h *HelmChartToOperator) TestPCRPrimaryMigration(t *testing.T) {
 		k8s.RunKubectl(t, kubectlOptions, "delete", "priorityclass", "crdb-critical")
 	}()
 
-	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
+	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, "us-east-1")
 	defer func() {
 		t.Log("Uninstall the cockroachdb enterprise operator")
 		operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)

tests/e2e/migrate/public_operator_to_cockroach_enterprise_operator_test.go

@@ -156,7 +156,7 @@ func (o *PublicOperatorToCockroachEnterpriseOperator) TestDefaultMigration(t *te
 	k8s.KubectlApply(t, kubectlOptions, filepath.Join(manifestsDirPath, "rbac.yaml"))
 
 	t.Log("Install the cockroachdb enterprise operator")
-	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
+	operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, "us-east-1")
 	defer func() {
 		t.Log("Uninstall the cockroachdb enterprise operator")
 		operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)

tests/e2e/operator/infra/local.go

@@ -41,9 +41,22 @@ type LocalRegion struct {
 //   - CoreDNS instances forward requests for other cluster domains; endpoints can be
 //     ClusterIP/pod IPs (with Calico) or LB IPs (with MetalLB).
 func (r *LocalRegion) SetUpInfra(t *testing.T) {
-	// If using existing infra return clients.
+	// If using existing infra, skip cluster creation and CNI setup but still
+	// populate clients so ValidateCRDB and other helpers can use them.
 	if r.ReusingInfra {
 		t.Logf("[%s] Reusing existing infrastructure", r.ProviderType)
+		if r.Clients == nil {
+			r.Clients = make(map[string]client.Client)
+		}
+		for _, cluster := range r.Clusters {
+			cfg, err := config.GetConfigWithContext(cluster)
+			require.NoError(t, err)
+			k8sClient, err := client.New(cfg, client.Options{})
+			require.NoError(t, err)
+			_ = apiextv1.AddToScheme(k8sClient.Scheme())
+			calico.RegisterCalicoGVK(k8sClient.Scheme())
+			r.Clients[cluster] = k8sClient
+		}
 		return
 	}