diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 618e7b656..c101dbff3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,6 +22,8 @@ jobs: pull-requests: read outputs: code: ${{ steps.filter.outputs.code }} + clickhouse-latest: ${{ steps.clickhouse.outputs.latest }} + clickhouse-supported: ${{ steps.clickhouse.outputs.supported }} steps: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -38,6 +40,11 @@ jobs: - '!docs/**' - '!examples/**' - '!LICENSE' + - name: Read supported ClickHouse versions + id: clickhouse + run: | + jq -r '[.[][] | split(".") | map(tonumber)] | group_by(.[:2]) | map(max) | sort | reverse | map(map(tostring) | join(".")) + | "latest=\(.[0])", "supported=\(join(","))"' test/supported/versions.json >> "$GITHUB_OUTPUT" lint: runs-on: ubuntu-latest @@ -277,23 +284,23 @@ jobs: include: - name: minimal-k8s-all-deploy-methods k8s_image: v1.28.15 - clickhouse_version: "26.7.5.10" + clickhouse_version: "${{ needs.changes.outputs.clickhouse-latest }}" deploy_target: test-compat-e2e - name: maximal-k8s-all-deploy-methods k8s_image: v1.36.1 - clickhouse_version: "26.7.5.10" + clickhouse_version: "${{ needs.changes.outputs.clickhouse-latest }}" deploy_target: test-compat-e2e - name: olm-deploy-method k8s_image: v1.28.15 - clickhouse_version: "26.7.5.10" + clickhouse_version: "${{ needs.changes.outputs.clickhouse-latest }}" deploy_target: test-compat-e2e-olm - name: supported-clickhouse-compatibility k8s_image: v1.30.13 - clickhouse_version: "26.7.5.10-distroless,26.6.3.62,26.5.7.64,26.3.22.7,25.8.32.4" + clickhouse_version: "${{ needs.changes.outputs.clickhouse-supported }},${{ needs.changes.outputs.clickhouse-latest }}-distroless" deploy_target: test-compat-e2e-manifest - name: operator-upgrade k8s_image: v1.30.13 - clickhouse_version: "26.7.5.10" + clickhouse_version: "${{ needs.changes.outputs.clickhouse-latest }}" deploy_target: test-compat-e2e-upgrade steps: - name: Checkout code diff --git a/.github/workflows/update-clickhouse-versions.yaml b/.github/workflows/update-clickhouse-versions.yaml new file mode 100644 index 000000000..34bb934c5 --- /dev/null +++ b/.github/workflows/update-clickhouse-versions.yaml @@ -0,0 +1,68 @@ +name: Update ClickHouse versions +on: + schedule: + - cron: '0 6 * * 1' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + update: + runs-on: ubuntu-latest + timeout-minutes: 15 + if: github.repository == 'ClickHouse/clickhouse-operator' && github.ref == 'refs/heads/main' + permissions: + contents: write + pull-requests: write + env: + BRANCH: ci/update-clickhouse-versions + TITLE: "ci: bump supported ClickHouse versions" + BOT_EMAIL: "41898282+github-actions[bot]@users.noreply.github.com" + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: 'go.mod' + + - name: Update supported ClickHouse versions + run: go run ./tools/update-clickhouse-versions + + - name: Open or update pull request + env: + GH_TOKEN: ${{ github.token }} + run: | + if git diff --quiet; then + echo "Supported ClickHouse versions are up to date" + exit 0 + fi + + pr=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[].number') + if [ -n "$pr" ]; then + git fetch --depth=1 origin "$BRANCH" + if git diff --quiet FETCH_HEAD -- test/supported/versions.json; then + echo "Update #$pr is up to date" + exit 0 + fi + + authors=$(gh pr view "$pr" --json commits --jq '[.commits[].authors[].email] | unique | join(",")') + if [ "$authors" != "$BOT_EMAIL" ]; then + echo "Update #$pr has manual commits, leaving it as is" + exit 0 + fi + fi + + git config user.name "github-actions[bot]" + git config user.email "$BOT_EMAIL" + git switch -C "$BRANCH" + git commit -am "$TITLE" + git push --force origin "$BRANCH" + + if [ -z "$pr" ]; then + gh pr create --head "$BRANCH" --title "$TITLE" --body "Updates \`test/supported/versions.json\` to the currently supported ClickHouse releases." + fi diff --git a/AGENTS.md b/AGENTS.md index 3c2082c05..276a2f5ac 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,6 +44,7 @@ go test ./internal/controller/keeper/ -v --ginkgo.v --ginkgo.focus="spec name pa - **Functional tests** (`controller_test.go`): Use `testutil.SetupEnvironment()` which starts `envtest` (real API server + `etcd`). Use for full reconciliation flow testing. - **Unit tests** (`sync_test.go`, `commands_test.go`): Use `fake.NewClientBuilder()` for faster, focused tests on individual methods. - **E2E tests** (`test/e2e/`): Real Kind cluster. Label tests with `Label("clickhouse")` or `Label("keeper")` for filtered runs. +- **ClickHouse versions**: Tests that run real ClickHouse images take their versions from the `test/supported` package instead of literal image tags. The weekly `update-clickhouse-versions` workflow bumps `test/supported/versions.json`. ### Code Generation (run after modifying api/v1alpha1/ types) ```bash diff --git a/internal/controller/clickhouse/commands_test.go b/internal/controller/clickhouse/commands_test.go index 5475facf5..e2e1f23b1 100644 --- a/internal/controller/clickhouse/commands_test.go +++ b/internal/controller/clickhouse/commands_test.go @@ -24,6 +24,7 @@ import ( v1 "github.com/ClickHouse/clickhouse-operator/api/v1alpha1" "github.com/ClickHouse/clickhouse-operator/internal/controller/keeper" "github.com/ClickHouse/clickhouse-operator/internal/controllerutil" + "github.com/ClickHouse/clickhouse-operator/test/supported" ) const ( @@ -31,11 +32,14 @@ const ( keeperHostname = "test-keeper" clickhouseHostnameFormat = "test-clickhouse-0-%d-0" testPassword = "test-password" - keeperImage = "clickhouse/clickhouse-keeper:26.7.5.10" - clickhouseImage = "clickhouse/clickhouse-server:26.7.5.10" testConfigRevision = "test-revision-v1" ) +var ( + keeperImage = "clickhouse/clickhouse-keeper:" + supported.Latest.Version() + clickhouseImage = "clickhouse/clickhouse-server:" + supported.Latest.Version() +) + func generateKeeperConfig() *strings.Reader { return strings.NewReader(fmt.Sprintf(`listen_host: 0.0.0.0 keeper_server: diff --git a/internal/upgrade/fetcher.go b/internal/upgrade/fetcher.go index d84299702..68f2d4776 100644 --- a/internal/upgrade/fetcher.go +++ b/internal/upgrade/fetcher.go @@ -111,6 +111,16 @@ type ReleaseData struct { Supported map[ClickHouseRelease]bool } +// NewReleaseData preprocesses fetched releases, keeping the latest version of every release. +func NewReleaseData(allReleases map[string][]ClickHouseVersion) *ReleaseData { + latestReleases := filterLatestVersions(allReleases) + + return &ReleaseData{ + Releases: latestReleases, + Supported: buildSupportedMap(latestReleases), + } +} + // ReleaseUpdater periodically fetches and updates ClickHouse release data, // maintaining a cache of the latest preprocessed information. type ReleaseUpdater struct { @@ -173,12 +183,7 @@ func (u *ReleaseUpdater) updateReleases(ctx context.Context) bool { return false } - latestReleases := filterLatestVersions(allReleases) - supported := buildSupportedMap(latestReleases) - u.cached.Store(&ReleaseData{ - Releases: latestReleases, - Supported: supported, - }) + u.cached.Store(NewReleaseData(allReleases)) return true } diff --git a/test/deploy/deploy_test.go b/test/deploy/deploy_test.go index 33fb95014..5030ee68b 100644 --- a/test/deploy/deploy_test.go +++ b/test/deploy/deploy_test.go @@ -26,6 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" v1 "github.com/ClickHouse/clickhouse-operator/api/v1alpha1" + "github.com/ClickHouse/clickhouse-operator/test/supported" "github.com/ClickHouse/clickhouse-operator/test/testutil" ) @@ -34,8 +35,7 @@ const ( testTag = "test" testImage = testRepo + ":" + testTag - defaultVersion = "latest" - reportDir = "report" + reportDir = "report" ) var ( @@ -52,7 +52,7 @@ var ( func TestDeploy(t *testing.T) { RegisterFailHandler(Fail) - versions := []string{defaultVersion} + versions := []string{supported.Latest.Version()} if vers := os.Getenv("CLICKHOUSE_VERSION"); vers != "" { versions = strings.Split(vers, ",") } @@ -333,9 +333,10 @@ var _ = Describe("Operator upgrade", Ordered, ContinueOnFailure, Label("upgrade" deploymentName = namespace + "-controller-manager" keeperName = "keeper" chName = "ch" - version = testutil.UpdateVersion ) + version := testutil.UpdateVersion + helmArgs := []string{"-n", namespace, "--set", "controller.watchNamespaces={" + namespace + "}", "--set", "crd.enable=true", "--set", "crd.keep=false", @@ -376,13 +377,13 @@ var _ = Describe("Operator upgrade", Ordered, ContinueOnFailure, Label("upgrade" keeperCR := testutil.NewKeeperCluster(namespace, keeperName). WithReplicas(3). WithStorage(*storage). - WithTag(version). + WithTag(version.Version()). Cluster() chCR := testutil.NewClickHouseCluster(namespace, chName). WithReplicas(2). WithStorage(*storage). WithKeeper(keeperName). - WithTag(version). + WithTag(version.Version()). Cluster() By("deploying keeper and clickhouse on the released operator") diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 661155eb1..108743d04 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -21,28 +21,20 @@ import ( ctrltestutil "github.com/ClickHouse/clickhouse-operator/internal/controller/testutil" "github.com/ClickHouse/clickhouse-operator/internal/controllerutil" "github.com/ClickHouse/clickhouse-operator/internal/upgrade" + "github.com/ClickHouse/clickhouse-operator/test/supported" "github.com/ClickHouse/clickhouse-operator/test/testutil" ) const ( pollingInterval = testutil.PollInterval - BaseVersion = testutil.BaseVersion - UpdateVersion = testutil.UpdateVersion - managerStopTimeout = 30 * time.Second ) -var releases = map[string][]upgrade.ClickHouseVersion{ - upgrade.ChannelStable: { - {Major: 26, Minor: 7, Patch: 5, Build: 10}, - {Major: 26, Minor: 6, Patch: 3, Build: 62}, - }, - upgrade.ChannelLTS: { - {Major: 26, Minor: 3, Patch: 22, Build: 7}, - {Major: 25, Minor: 8, Patch: 32, Build: 4}, - }, -} +var ( + BaseVersion = testutil.BaseVersion.Version() + UpdateVersion = testutil.UpdateVersion.Version() +) var ( sharding testutil.ShardingConfig @@ -137,7 +129,7 @@ var _ = BeforeSuite(func(ctx context.Context) { RestConfig: config, Logger: zapLogger, Dialer: podDialer, - Fetcher: &upgrade.StaticFetcher{Releases: releases}, + Fetcher: &upgrade.StaticFetcher{Releases: supported.Releases}, }, settings) Expect(err).NotTo(HaveOccurred()) diff --git a/test/supported/supported.go b/test/supported/supported.go new file mode 100644 index 000000000..2692624cf --- /dev/null +++ b/test/supported/supported.go @@ -0,0 +1,60 @@ +package supported + +import ( + _ "embed" + "encoding/json" + "fmt" + "slices" + + "github.com/ClickHouse/clickhouse-operator/internal/upgrade" +) + +//go:embed versions.json +var clickhouseJSON []byte + +var ( + // Releases contains the supported ClickHouse versions grouped by channel. + Releases map[string][]upgrade.ClickHouseVersion + // Base ClickHouse version used in tests. + Base upgrade.ClickHouseVersion + // Latest is the newest supported ClickHouse version, guaranteed greater than Base. + Latest upgrade.ClickHouseVersion +) + +func init() { + Releases = parseVersions(clickhouseJSON) + + latestLTS := slices.MaxFunc(Releases[upgrade.ChannelLTS], upgrade.CompareVersions) + latestStable := slices.MaxFunc(Releases[upgrade.ChannelStable], upgrade.CompareVersions) + + if upgrade.CompareVersions(latestLTS, latestStable) > 0 { + Base = latestStable + Latest = latestLTS + } else { + Base = latestLTS + Latest = latestStable + } +} + +func parseVersions(data []byte) map[string][]upgrade.ClickHouseVersion { + var raw map[string][]string + if err := json.Unmarshal(data, &raw); err != nil { + panic(fmt.Sprintf("parse supported ClickHouse versions: %v", err)) + } + + releases := make(map[string][]upgrade.ClickHouseVersion, len(raw)) + for channel, versions := range raw { + for _, value := range versions { + version, err := upgrade.ParseBareVersion(value) + if err != nil { + panic(fmt.Sprintf("parse supported ClickHouse version: %v", err)) + } + + releases[channel] = append(releases[channel], version) + } + + slices.SortFunc(releases[channel], func(a, b upgrade.ClickHouseVersion) int { return b.Compare(a) }) + } + + return releases +} diff --git a/test/supported/versions.json b/test/supported/versions.json new file mode 100644 index 000000000..497c62002 --- /dev/null +++ b/test/supported/versions.json @@ -0,0 +1,10 @@ +{ + "lts": [ + "26.8.14.3", + "26.3.37.3" + ], + "stable": [ + "26.9.7.9", + "26.7.18.2" + ] +} diff --git a/test/testutil/fixtures.go b/test/testutil/fixtures.go index fe353e348..87822785a 100644 --- a/test/testutil/fixtures.go +++ b/test/testutil/fixtures.go @@ -20,7 +20,7 @@ func NewClickHouseCluster(namespace, name string) *ClickHouseBuilder { Spec: v1.ClickHouseClusterSpec{ Replicas: new(int32(1)), ContainerTemplate: v1.ContainerTemplateSpec{ - Image: v1.ContainerImage{Tag: BaseVersion}, + Image: v1.ContainerImage{Tag: BaseVersion.Version()}, }, }, }} @@ -97,7 +97,7 @@ func NewKeeperCluster(namespace, name string) *KeeperBuilder { Spec: v1.KeeperClusterSpec{ Replicas: new(int32(1)), ContainerTemplate: v1.ContainerTemplateSpec{ - Image: v1.ContainerImage{Tag: BaseVersion}, + Image: v1.ContainerImage{Tag: BaseVersion.Version()}, }, }, }} diff --git a/test/testutil/utils.go b/test/testutil/utils.go index 93f505ef0..a06d3286a 100644 --- a/test/testutil/utils.go +++ b/test/testutil/utils.go @@ -20,6 +20,8 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/ClickHouse/clickhouse-operator/test/supported" ) const ( @@ -30,9 +32,14 @@ const ( networkPolicyControllerURLTmpl = "https://raw.githubusercontent.com" + "/kubernetes-sigs/kube-network-policies/%s/install.yaml" - logTailLines = 10 - BaseVersion = "26.3.22.7" - UpdateVersion = "26.7.5.10" + logTailLines = 10 +) + +var ( + // BaseVersion is the ClickHouse version test clusters start from. + BaseVersion = supported.Base + // UpdateVersion is the ClickHouse version test clusters upgrade to. + UpdateVersion = supported.Latest ) // CurrentSpecHash returns a stable hash for the currently running Ginkgo spec. diff --git a/tools/update-clickhouse-versions/main.go b/tools/update-clickhouse-versions/main.go new file mode 100644 index 000000000..3cbb5b50f --- /dev/null +++ b/tools/update-clickhouse-versions/main.go @@ -0,0 +1,142 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log" + "net/http" + "os" + "slices" + "time" + + "github.com/ClickHouse/clickhouse-operator/internal/upgrade" +) + +const ( + versionsURL = "https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/utils/list-versions/version_date.tsv" + tagURLFormat = "https://hub.docker.com/v2/namespaces/clickhouse/repositories/%s/tags/%s" + requestTimeout = 30 * time.Second + filePermission = 0o600 +) + +var ( + logger = log.New(os.Stderr, "", 0) + + repositories = []string{"clickhouse-server", "clickhouse-keeper"} + tagSuffixes = []string{"", "-distroless"} +) + +func main() { + if err := run(); err != nil { + logger.Fatal(err) + } +} + +func run() error { + var output string + + flag.StringVar(&output, "output", "test/supported/versions.json", "path to write the supported versions") + flag.Parse() + + ctx := context.Background() + client := &http.Client{Timeout: requestTimeout} + fetcher := &upgrade.URLFetcher{RequestTimeout: requestTimeout, HTTPClient: client, VersionsURL: versionsURL} + + releases, err := fetcher.FetchReleases(ctx) + if err != nil { + return fmt.Errorf("fetch releases: %w", err) + } + + pinned, err := pinSupported(ctx, client, releases) + if err != nil { + return err + } + + data, err := json.MarshalIndent(pinned, "", " ") + if err != nil { + return fmt.Errorf("marshal versions: %w", err) + } + + if err := os.WriteFile(output, append(data, '\n'), filePermission); err != nil { + return fmt.Errorf("write versions: %w", err) + } + + return nil +} + +// pinSupported returns the newest published version of every supported release by channel, newest first. +func pinSupported( + ctx context.Context, client *http.Client, releases map[string][]upgrade.ClickHouseVersion, +) (map[string][]string, error) { + supported := upgrade.NewReleaseData(releases).Supported + pinned := map[string][]string{} + found := map[upgrade.ClickHouseRelease]bool{} + + for channel, versions := range releases { + slices.SortFunc(versions, func(a, b upgrade.ClickHouseVersion) int { + return upgrade.CompareVersions(b, a) + }) + + for _, version := range versions { + if !supported[version.Release()] { + continue + } + + if !found[version.Release()] { + ok, err := imagesPublished(ctx, client, version.Version()) + if err != nil { + return nil, err + } + + if !ok { + logger.Printf("Skipping %s: images are not published", version.Version()) + continue + } + + found[version.Release()] = true + pinned[channel] = append(pinned[channel], version.Version()) + } + } + } + + return pinned, nil +} + +func imagesPublished(ctx context.Context, client *http.Client, version string) (bool, error) { + for _, repository := range repositories { + for _, suffix := range tagSuffixes { + if ok, err := tagExists(ctx, client, repository, version+suffix); err != nil || !ok { + return false, err + } + } + } + + return true, nil +} + +func tagExists(ctx context.Context, client *http.Client, repository, tag string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf(tagURLFormat, repository, tag), nil) + if err != nil { + return false, fmt.Errorf("create tag request: %w", err) + } + + resp, err := client.Do(req) + if err != nil { + return false, fmt.Errorf("request tag %s:%s: %w", repository, tag, err) + } + + defer func() { + _ = resp.Body.Close() + }() + + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusNotFound: + return false, nil + default: + return false, fmt.Errorf("request tag %s:%s: unexpected status code %d", repository, tag, resp.StatusCode) + } +}