diff --git a/.claude/skills/ci-results/SKILL.md b/.claude/skills/ci-results/SKILL.md new file mode 100644 index 000000000..92e4fc294 --- /dev/null +++ b/.claude/skills/ci-results/SKILL.md @@ -0,0 +1,107 @@ +--- +name: ci-results +description: Fetch and interpret Praktika CI structured JSON results for this repo (clickhouse-operator). Use whenever the user asks to investigate, address, or fix CI failures, or refers to a PR / CI report without specifying what broke. Always fetch the result JSON before reading code or proposing fixes. +--- + +# Praktika CI results (clickhouse-operator) + +Whenever the user asks to investigate, address, or fix CI failures — or refers to a +PR without specifying what's broken — **always fetch the CI result JSON first** +before reading code or proposing fixes. The `info` field of failing nodes is the +primary signal for what went wrong. + +This repo's Praktika config: + +| Field | Value | +|---|---| +| Artifact/report bucket | `clickhouseoperator-artifacts-eu-north-1` | +| HTTP endpoint | `clickhouseoperator-artifacts-eu-north-1.s3.amazonaws.com` | +| AWS region / profile | `eu-north-1` / `Box` | +| PR workflow name | `PR` → normalized `pr` | +| Main workflow name | `Main` → normalized `main` | +| Main branch | `main` | + +> The directory/filename segment comes from the workflow's **`name=` field**, not the +> `.py` filename. E.g. `ci/workflows/main_ci.py` declares `name="Main"`, so the segment +> is `main` (not `main_ci`); `pull_request.py` declares `name="PR"` → `pr`. If unsure, +> grep `name=` in `ci/workflows/*.py` and normalize per the rules below. + +Normalization (`Utils.normalize_string`): lowercase, then replace each of +`space ( ) { } ' [ ] , / - : " &` with `_`, collapse repeated `_`, strip trailing `_`. + +(Source: `ci/settings/settings.py`, `ci/workflows/`, `praktika/utils.py`.) + +## Step 1 — Determine PR/branch, sha, and workflow name + +**From a bare PR number** (or the current branch): +```bash +gh pr view {PR} --json number,headRefOid,title,url --jq '{number,sha:.headRefOid,title,url}' +# omit {PR} to use the current branch's PR +``` +Use workflow `pr` for PR runs. + +**From a Praktika report URL** (`...praktika.html?PR=357&sha=abc123&name_0=PR`): +- Extract `PR`, `sha`, `name_0` from the query params. +- Normalize `name_0`: lowercase, spaces → underscores (`"PR"` → `"pr"`). + +## Step 2 — Build the JSON URL + +The result object lives under the **normalized workflow name** directory, and the +filename embeds that same normalized name. + +**PR run:** +``` +https://clickhouseoperator-artifacts-eu-north-1.s3.amazonaws.com/PRs/{PR}/{sha}/pr/result_pr.json +``` + +**Main/branch run** (push to a ref — no PR number): +``` +https://clickhouseoperator-artifacts-eu-north-1.s3.amazonaws.com/REFs/{branch}/{sha}/main/result_main.json +``` + +Latest run for a PR/branch (sha unknown): swap the sha segment for `latest`, e.g. +`.../PRs/{PR}/latest/pr/result_pr.json`. + +Fetch with WebFetch (or `curl -s`). The bucket returns **HTTP 403, not 404**, for a +missing key — so a 403 almost always means the path is wrong (bad sha, branch, or +workflow name), not an auth problem. + +## Step 3 — Walk the result tree + +The JSON is a serialized `praktika.Result`: + +``` +{ + "name": str, + "status": str, # OK | FAIL | ERROR | SKIPPED | UNKNOWN | XFAIL | XPASS | PENDING | RUNNING | DROPPED + "start_time": float?, + "duration": float?, + "info": str, # <-- primary failure signal + "results": [...], # nested Result objects, same shape, recursive + "files": [...], + "links": [...], + "ext": { + "labels": [{"name": str, "link": str?, "hint": str?}, ...], + "warnings": [...], + "errors": [...], + "report_url": str?, + ... + } +} +``` + +Recurse `results` to find every node whose `status` is `FAIL` or `ERROR`, and read +its `info`. Each sub-job may also have its own `result_.json` +artifact and log `files`/`links` worth fetching for detail. + +Quick triage with `jq` after downloading: +```bash +curl -s "{json_url}" -o /tmp/result.json + +# all FAIL/ERROR nodes (includes parent aggregates like "Lint: Failures 1/6") +jq -r '.. | objects | select(.status=="FAIL" or .status=="ERROR") | "\(.name): \(.status)\n\(.info)\n"' /tmp/result.json + +# failed LEAVES only (no children) with info + links + files — the actionable list +jq -r '[.. | objects | select((.status=="FAIL" or .status=="ERROR") and ((.results|length)==0))][] | "### \(.name) [\(.status)]\n\(.info // "")\nlinks: \((.links // [])|join(", "))\nfiles: \((.files // [])|join(", "))\n"' /tmp/result.json +``` +Leaves carry the per-job `job.log` link; fetch it for full output when `info` is truncated. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index 18411811e..000000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,591 +0,0 @@ -name: Operator CI -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - workflow_dispatch: -env: - kind-version: 'v0.32.0' -permissions: - contents: read -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - changes: - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - pull-requests: read - outputs: - code: ${{ steps.filter.outputs.code }} - clickhouse-latest: ${{ steps.clickhouse.outputs.latest }} - clickhouse-supported: ${{ steps.clickhouse.outputs.supported }} - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - name: Detect non-docs changes - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4 - id: filter - with: - predicate-quantifier: 'every' - filters: | - code: - - '**' - - '!**/*.md' - - '!**/*.mdx' - - '!docs/**' - - '!examples/**' - - '!LICENSE' - - name: Read supported ClickHouse versions - id: clickhouse - run: | - jq -r '[.[][] | split(".") | map(tonumber)] | group_by(.[:2]) | map(max) | sort | reverse | map(map(tostring) | join(".")) - | "latest=\(.[0])", "supported=\(join(","))"' test/supported/versions.json >> "$GITHUB_OUTPUT" - - lint: - runs-on: ubuntu-latest - timeout-minutes: 15 - needs: [ changes ] - if: needs.changes.outputs.code == 'true' - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version-file: 'go.mod' - cache: false - - - name: Go cache - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: go-lint-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: | - go-lint- - - - name: Go Tidy - run: go mod tidy && git diff --exit-code - - - name: Check controller-gen generated - run: make generate && git diff --exit-code - - - name: Check manifests generated - run: make manifests && git diff --exit-code - - - name: Lint with golangci-lint - uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9 - with: - version: v2.13.1 - - - name: Lint with codespell - uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # v2.2 - with: - config: ci/.codespellrc - - - name: actionlint - uses: raven-actions/actionlint@3d39aea434753780c3b3d4a1a31c854b4dbf49d7 # v2 - with: - flags: '-config-file ci/actionlint.yaml' - - check-crd-compat: - # Only run on pull requests, and skip when the PR is explicitly labeled as - # an intentional CRD-breaking change. The job is not part of ci-success-check, - # so a failure here does not block the aggregate gate. - needs: [ changes ] - if: | - needs.changes.outputs.code == 'true' && - github.event_name == 'pull_request' && - !contains(github.event.pull_request.labels.*.name, 'crd-breaking-change') - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version-file: 'go.mod' - - - name: Check CRD backward compatibility - run: make check-crd-compat CRD_BASE_REF=${{ github.event.pull_request.base.sha }} - - bundle: - runs-on: ubuntu-latest - timeout-minutes: 20 - needs: [ changes ] - if: needs.changes.outputs.code == 'true' - env: - VERSION: 0.0.1 - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version-file: 'go.mod' - - - name: Generate bundle - run: make bundle - - - name: Create k8s Kind Cluster - uses: helm/kind-action@06c1ae10762d3b9c1644e7fe69596ae519e015a2 # v1.15.0 - with: - cluster_name: 'test-bundle' - version: ${{ env.kind-version }} - - - name: Export kubeconfig - run: kind export kubeconfig --name test-bundle - - - name: Create bundle image - run: make bundle-build - - - name: Load image into Kind cluster - run: kind load docker-image ghcr.io/clickhouse/clickhouse-operator-bundle:v0.0.1 --name test-bundle - - - name: Run scorecard tests - run: make scorecard - - build_and_test: - runs-on: ubuntu-latest - timeout-minutes: 25 - permissions: - contents: read - checks: write - needs: [ changes ] - if: needs.changes.outputs.code == 'true' - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version-file: 'go.mod' - cache: false - - - name: Go cache - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: go-unit-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: | - go-unit- - - - name: Go Mod - run: go mod download - - - name: Go Mod Verify - run: go mod verify - - - name: Build - run: go build -v cmd/main.go - - - name: Run tests - run: | - go list ./... | grep -v /e2e - make test-ci - - - name: Test Report - uses: dorny/test-reporter@d1b5dff021aa968b288b02285a8bf47471c5928a # v3 - if: ${{ !cancelled() }} - with: - name: Unit tests - badge-title: Unit tests - path: "**/report/*.xml" - reporter: java-junit - - fuzz_specs: - runs-on: ubuntu-latest - timeout-minutes: 20 - needs: [ changes ] - if: needs.changes.outputs.code == 'true' - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version-file: 'go.mod' - cache: false - - - name: Go cache - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: go-fuzz-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: | - go-fuzz- - - - name: Go Mod - run: go mod download - - - name: Run fuzz tests - run: make fuzz - - helm-test: - runs-on: ubuntu-latest - timeout-minutes: 15 - needs: [ changes ] - if: needs.changes.outputs.code == 'true' - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version-file: 'go.mod' - - - name: Install Helm - run: curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - - - name: Verify Helm installation - run: helm version - - - name: Check helmchart generated - run: make generate-helmchart-ci && git diff --exit-code dist/chart/ dist/chart-cluster/ - - - name: Build Helm Chart Dependencies - run: make build-helmchart-dependencies - - - name: Lint Helm Chart - run: helm lint ./dist/chart - - - name: Lint Cluster Helm Chart - run: make lint-cluster-chart - - compat-e2e-test: - runs-on: [ubuntu-latest] - timeout-minutes: 60 - needs: [ changes ] - if: needs.changes.outputs.code == 'true' - strategy: - fail-fast: false - matrix: - include: - - name: minimal-k8s-all-deploy-methods - k8s_image: v1.28.15 - clickhouse_version: "${{ needs.changes.outputs.clickhouse-latest }}" - deploy_target: test-compat-e2e - - name: maximal-k8s-all-deploy-methods - k8s_image: v1.36.1 - clickhouse_version: "${{ needs.changes.outputs.clickhouse-latest }}" - deploy_target: test-compat-e2e - - name: olm-deploy-method - k8s_image: v1.28.15 - clickhouse_version: "${{ needs.changes.outputs.clickhouse-latest }}" - deploy_target: test-compat-e2e-olm - - name: supported-clickhouse-compatibility - k8s_image: v1.30.13 - clickhouse_version: "${{ needs.changes.outputs.clickhouse-supported }},${{ needs.changes.outputs.clickhouse-latest }}-distroless" - deploy_target: test-compat-e2e-manifest - - name: operator-upgrade - k8s_image: v1.30.13 - clickhouse_version: "${{ needs.changes.outputs.clickhouse-latest }}" - deploy_target: test-compat-e2e-upgrade - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-tags: 'true' - - - name: Free disk space - uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # v2.0.0 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version-file: 'go.mod' - cache: false - - - name: Go cache - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: go-compat-e2e-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: | - go-compat-e2e- - - - name: Create k8s Kind Cluster - uses: helm/kind-action@06c1ae10762d3b9c1644e7fe69596ae519e015a2 # v1.15.0 - with: - cluster_name: kind - version: ${{ env.kind-version }} - config: ci/kind-cluster.config - node_image: "kindest/node:${{ matrix.k8s_image }}" - - - name: Go Mod - run: go mod download - - - name: Install tools - run: make operator-sdk opm - - - name: Pre-pull ClickHouse images into Kind - run: | - pull() { - for _ in 1 2 3; do - if docker pull "$1"; then return 0; fi - sleep 15 - done - return 1 - } - for version in $(echo "${{ matrix.clickhouse_version }}" | tr ',' ' '); do - for image in clickhouse/clickhouse-keeper clickhouse/clickhouse-server; do - pull "docker.io/${image}:${version}" - kind load docker-image "docker.io/${image}:${version}" - done - done - - - name: Run compatibility e2e tests - run: make ${{ matrix.deploy_target }} - env: - CLICKHOUSE_VERSION: ${{ matrix.clickhouse_version }} - - - name: Upload test report - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: ${{ !cancelled() }} - with: - name: e2e-report-${{ matrix.name }} - path: "**/report/*" - if-no-files-found: error - overwrite: true - - e2e-shard-plan: - needs: [ changes ] - if: needs.changes.outputs.code == 'true' - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - actions: read - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version-file: 'go.mod' - - - name: Go Mod - run: go mod download - - - name: Download previous e2e timings - run: bash ci/download-e2e-last-run.sh .ci/e2e-timings - env: - GH_TOKEN: ${{ github.token }} - - - name: Plan e2e shards - run: | - go run ./tools/e2e-shard-planner \ - -report-path .ci/e2e-timings/latest \ - -shards 4 \ - -output .ci/e2e-shard-plan/e2e-shard-plan.json - - - name: Upload e2e shard plan - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: e2e-shard-plan - path: .ci/e2e-shard-plan/e2e-shard-plan.json - if-no-files-found: error - - e2e-test: - needs: [ lint, changes, e2e-shard-plan ] - if: needs.changes.outputs.code == 'true' - strategy: - fail-fast: false - matrix: - shard: [ 1, 2, 3, 4 ] - runs-on: [self-hosted, amd-medium] - timeout-minutes: 45 - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Download e2e shard plan - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: e2e-shard-plan - path: .ci/e2e-shard-plan - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version-file: 'go.mod' - - - name: Go Mod - run: go mod download - - - name: Create k8s Kind Cluster - uses: helm/kind-action@06c1ae10762d3b9c1644e7fe69596ae519e015a2 # v1.15.0 - with: - cluster_name: kind - version: ${{ env.kind-version }} - config: ci/kind-cluster.config - node_image: "kindest/node:v1.34.3" # Can't use newer node image as self-hosted runners use cgroups v1 - - - name: Run e2e tests - env: - E2E_SHARD_INDEX: ${{ matrix.shard }} - E2E_SHARD_TOTAL: "4" - E2E_SHARD_PLAN: ${{ github.workspace }}/.ci/e2e-shard-plan/e2e-shard-plan.json - run: make test-e2e - - - name: Collect failure diagnostics - if: ${{ failure() && !cancelled() }} - run: | - mkdir -p failure-diagnostics - kind export logs --name kind failure-diagnostics/kind || true - kubectl get pods,jobs -A -o wide > failure-diagnostics/pods-jobs.txt 2>&1 || true - kubectl get events -A --sort-by=.metadata.creationTimestamp > failure-diagnostics/events.txt 2>&1 || true - - - name: Upload failure diagnostics - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: ${{ failure() && !cancelled() }} - with: - name: e2e-kind-logs-shard-${{ matrix.shard }} - path: failure-diagnostics - if-no-files-found: warn - retention-days: 7 - - - name: Upload test report - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - if: ${{ !cancelled() }} - with: - name: e2e-report-shard-${{ matrix.shard }} - path: "**/report/*" - if-no-files-found: error - overwrite: true - - e2e-report: - needs: [ changes, compat-e2e-test, e2e-test ] - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - checks: write - if: ${{ !cancelled() && needs.changes.outputs.code == 'true' }} - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Download all e2e test reports - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - pattern: e2e-report-* - - - name: Merge sharded e2e reports - run: python3 ci/merge-junit-reports.py - - - name: Test Report - uses: dorny/test-reporter@d1b5dff021aa968b288b02285a8bf47471c5928a # v3 - with: - name: E2E tests - badge-title: E2E tests - reporter: java-junit - path: "**/*-report.xml" - - openshift-compat: - needs: [ changes, compat-e2e-test, e2e-test ] - if: | - needs.changes.outputs.code == 'true' && - ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository ) - runs-on: [self-hosted, openshift, operator] - continue-on-error: true - timeout-minutes: 60 - concurrency: - group: openshift-compatibility - cancel-in-progress: false - permissions: - contents: read - packages: write - checks: write - env: - KUBECONFIG: /home/runner/okd-install/auth/kubeconfig - VERSION: 0.0.99-test - PLATFORMS: linux/amd64 - IMG: ghcr.io/clickhouse/clickhouse-operator:test - BUNDLE_IMG: ghcr.io/clickhouse/clickhouse-operator-bundle:test - CATALOG_IMG: ghcr.io/clickhouse/clickhouse-operator-catalog:test - SINGLE_BUNDLE_IMAGE: ghcr.io/clickhouse/clickhouse-operator-bundle:test - OPENSHIFT_CATALOG_IMAGE: ghcr.io/clickhouse/clickhouse-operator-catalog:test - OPENSHIFT_CHANNEL: stable-v0 - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - persist-credentials: false - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version-file: 'go.mod' - - - name: Log in to ghcr.io - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push test operator + bundle + catalog - run: make docker-buildx bundle bundle-buildx catalog-buildx - - - name: Clean up cluster state from previous runs - run: sudo /usr/local/bin/okd-cleanup.sh - - - name: Install OLM tooling - run: make operator-sdk opm - - - name: Run OLM compatibility e2e - run: make test-compat-e2e-olm-openshift - - - name: Test Report - uses: dorny/test-reporter@d1b5dff021aa968b288b02285a8bf47471c5928a # v3 - with: - name: E2E tests - badge-title: E2E tests - reporter: java-junit - path: "**/report/*" - - - name: Wipe workspace - if: always() - run: rm -rf "$GITHUB_WORKSPACE/.git" "$GITHUB_WORKSPACE/_work" || true - - ci-success-check: - name: All CI checks passed - runs-on: ubuntu-latest - timeout-minutes: 5 - needs: [ changes, lint, bundle, build_and_test, fuzz_specs, helm-test, compat-e2e-test, e2e-shard-plan, e2e-test, check-crd-compat ] - if: always() - steps: - - name: Determine CI status - id: status - run: | - if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" || "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then - echo "One or more CI jobs failed or were cancelled" - exit 1 - else - echo "All required jobs passed successfully" - fi diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 500ccdf1f..777edf3d7 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -2,8 +2,8 @@ name: CodeQL on: push: branches: [ main ] - pull_request: - branches: [ main ] + # pull_request: + # branches: [ main ] schedule: - cron: '30 4 * * 1' permissions: diff --git a/.github/workflows/dependabot-regenerate.yaml b/.github/workflows/dependabot-regenerate.yaml deleted file mode 100644 index 550c69aad..000000000 --- a/.github/workflows/dependabot-regenerate.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: Dependabot Regenerate -on: - pull_request: - paths: - - 'go.mod' - - 'go.sum' -permissions: - contents: read - -jobs: - regenerate: - runs-on: ubuntu-latest - timeout-minutes: 20 - if: github.actor == 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository - permissions: - contents: write - actions: write - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.event.pull_request.head.ref }} - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version-file: 'go.mod' - cache: false - - - name: Go cache - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: go-lint-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: | - go-lint- - - - name: Regenerate after dependency bump - run: | - go mod tidy - make generate manifests generate-helmchart-ci docs-generate-api-ref - - - name: Commit and push regenerated files - id: push - run: | - if [ -z "$(git status --porcelain)" ]; then - echo "Nothing to regenerate" - exit 0 - fi - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - git commit -m "chores: regenerate after dependency bump" - git push - echo "pushed=true" >> "$GITHUB_OUTPUT" - - - name: Re-run CI on the regenerated head - if: steps.push.outputs.pushed == 'true' - env: - GH_TOKEN: ${{ github.token }} - HEAD_REF: ${{ github.event.pull_request.head.ref }} - run: gh workflow run ci.yaml --ref "$HEAD_REF" diff --git a/.github/workflows/docs-lint.yaml b/.github/workflows/docs-lint.yaml deleted file mode 100644 index 4da385717..000000000 --- a/.github/workflows/docs-lint.yaml +++ /dev/null @@ -1,87 +0,0 @@ -name: Documentation Lint - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] -permissions: - contents: read -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - vale: - name: vale-linter - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - checks: write - pull-requests: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: errata-ai/vale-action@518a9136acc6e6668ce7c00d367051e0941e87ff # 3.0.0 - with: - fail_on_error: 'true' - filter_mode: nofilter - - doc-links: - name: Doc links - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - pull-requests: write - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4 - id: changes - with: - filters: | - docs: - - 'docs/**' - - if: steps.changes.outputs.docs == 'true' - uses: UmbrellaDocs/action-linkspector@568ec8d29fa92b31fd9ea5381e155c51e922af83 # v1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - reporter: github-pr-review - filter_mode: nofilter - fail_level: any - config_file: .linkspector.yml - - api-reference-generated: - name: API Reference Generated - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version-file: 'go.mod' - - - name: Generate API Reference - run: make docs-generate-api-ref - - - name: Check API Reference generated - run: git diff --exit-code docs/ - - ci-success-check: - name: All Docs CI checks passed - runs-on: ubuntu-latest - timeout-minutes: 5 - needs: [ vale, doc-links, api-reference-generated ] - if: always() - steps: - - name: Determine CI status - id: status - run: | - if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" || "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then - echo "One or more CI jobs failed or were cancelled" - exit 1 - else - echo "All required jobs passed successfully" - fi diff --git a/.github/workflows/openshift-compat.yaml b/.github/workflows/openshift-compat.yaml new file mode 100644 index 000000000..d0132ad46 --- /dev/null +++ b/.github/workflows/openshift-compat.yaml @@ -0,0 +1,103 @@ +name: OpenShift Compatibility +on: + pull_request: + branches: [ main ] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + changes: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + code: ${{ steps.filter.outputs.code }} + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Detect non-docs changes + uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4 + id: filter + with: + predicate-quantifier: 'every' + filters: | + code: + - '**' + - '!**/*.md' + - '!**/*.mdx' + - '!docs/**' + - '!examples/**' + - '!LICENSE' + + openshift-compat: + needs: [ changes ] + if: | + needs.changes.outputs.code == 'true' && + ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository ) + runs-on: [self-hosted, openshift, operator] + continue-on-error: true + timeout-minutes: 60 + concurrency: + group: openshift-compatibility + cancel-in-progress: false + permissions: + contents: read + packages: write + checks: write + env: + KUBECONFIG: /home/runner/okd-install/auth/kubeconfig + VERSION: 0.0.99-test + PLATFORMS: linux/amd64 + IMG: ghcr.io/clickhouse/clickhouse-operator:test + BUNDLE_IMG: ghcr.io/clickhouse/clickhouse-operator-bundle:test + CATALOG_IMG: ghcr.io/clickhouse/clickhouse-operator-catalog:test + SINGLE_BUNDLE_IMAGE: ghcr.io/clickhouse/clickhouse-operator-bundle:test + OPENSHIFT_CATALOG_IMAGE: ghcr.io/clickhouse/clickhouse-operator-catalog:test + OPENSHIFT_CHANNEL: stable-v0 + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: 'go.mod' + + - name: Log in to ghcr.io + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push test operator + bundle + catalog + run: make docker-buildx bundle bundle-buildx catalog-buildx + + - name: Clean up cluster state from previous runs + run: sudo /usr/local/bin/okd-cleanup.sh + + - name: Install OLM tooling + run: make operator-sdk opm + + - name: Run OLM compatibility e2e + run: make test-compat-e2e-olm-openshift + + - name: Test Report + uses: dorny/test-reporter@d1b5dff021aa968b288b02285a8bf47471c5928a # v3 + with: + name: E2E tests + badge-title: E2E tests + reporter: java-junit + path: "**/report/*" + + - name: Wipe workspace + if: always() + run: rm -rf "$GITHUB_WORKSPACE/.git" "$GITHUB_WORKSPACE/_work" || true diff --git a/.gitignore b/.gitignore index bfedd3f7d..702cd7f35 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,7 @@ go.work *.swp *.swo *~ + +# ci runtime +__pycache__/ +ci/tmp/ diff --git a/Makefile b/Makefile index 3c97ed4a1..cef26b5cf 100644 --- a/Makefile +++ b/Makefile @@ -464,6 +464,12 @@ $(ACTIONLINT): $(LOCALBIN) $(call go-install-tool,$(ACTIONLINT),github.com/rhysd/actionlint/cmd/actionlint,$(ACTIONLINT_VERSION)) CRD_BASE_REF ?= origin/main +# Optional pre-staged baseline directory. When set, each baseline is read from +# $(CRD_BASELINE_DIR)/ instead of `git show $(CRD_BASE_REF):...`. +# CI uses this because the S3 repo snapshot has no git history/remote to read the +# base manifests from (see ci/jobs/check_crd_compat.py). Empty for local runs, +# which fall back to git. +CRD_BASELINE_DIR ?= # Violations from embedded Kubernetes core types (securityContext bools, resource maps, etc.) # that are structural to the k8s API, not operator design choices. These are automatically # suppressed by ratcheting for existing fields, but appear when adding new fields that embed core types. @@ -482,12 +488,20 @@ check-crd-size: ## Fail if any CRD manifest exceeds $(CRD_MAX_BYTES) bytes. done; \ exit $$FAILED .PHONY: check-crd-compat -check-crd-compat: crd-schema-checker check-crd-size ## Check CRD backward compatibility against $(CRD_BASE_REF). +check-crd-compat: crd-schema-checker check-crd-size ## Check CRD backward compatibility against $(CRD_BASE_REF) (or $(CRD_BASELINE_DIR) if set). @FAILED=0; \ for crd in config/crd/bases/*.yaml; do \ - echo "Checking $$crd against $(CRD_BASE_REF)..."; \ + echo "Checking $$crd against baseline..."; \ BASELINE=$$(mktemp); \ - if ! git show $(CRD_BASE_REF):$$crd > "$$BASELINE" 2>/dev/null; then \ + if [ -n "$(CRD_BASELINE_DIR)" ]; then \ + SRC="$(CRD_BASELINE_DIR)/$$(basename $$crd)"; \ + if [ ! -s "$$SRC" ]; then \ + echo " No baseline at $$SRC — skipping (new CRD)"; \ + rm -f "$$BASELINE"; \ + continue; \ + fi; \ + cp "$$SRC" "$$BASELINE"; \ + elif ! git show $(CRD_BASE_REF):$$crd > "$$BASELINE" 2>/dev/null; then \ echo " No baseline found at $(CRD_BASE_REF):$$crd — skipping (new CRD)"; \ rm -f "$$BASELINE"; \ continue; \ diff --git a/ci/.codespellrc b/ci/.codespellrc index eef37b2a1..110fb18e7 100644 --- a/ci/.codespellrc +++ b/ci/.codespellrc @@ -1,6 +1,6 @@ [codespell] # Comma-separated list of files to skip -skip = *.svg,*.sum,*.tgz,go.sum,dist/chart/charts,bin,.git +skip = *.svg,*.sum,*.tgz,go.sum,dist/chart/charts,bin,.git,ci/tmp,*.log,golangci-report.json # File containing words to ignore (one per line) ignore-words = ci/codespell-ignore-words.txt diff --git a/ci/infrastructure/projects.py b/ci/infrastructure/projects.py new file mode 100644 index 000000000..826133838 --- /dev/null +++ b/ci/infrastructure/projects.py @@ -0,0 +1,357 @@ +import base64 + +from praktika.infrastructure import ( + CloudInfrastructure, + Components, + ImageBuilder, + Storage, + VPC, +) +from praktika.settings import Settings + + +# S3 prefixes the runner instance roles are scoped to. Both the artifact +# and the report bucket are granted (they are often the same bucket, in +# which case the duplicate collapses away). +_PROJECT_S3_PREFIXES = list( + dict.fromkeys( + [ + f"{Settings.S3_ARTIFACT_BUCKET}/*", + f"{Settings.S3_REPORT_BUCKET}/*", + ] + ) +) + + +# until published in pip +_PRAKTIKA_PACKAGE_BASE_URL = "https://praktika-artifacts-eu-north-1.s3.amazonaws.com/packages" +_PRAKTIKA_WHL = f"{_PRAKTIKA_PACKAGE_BASE_URL}/praktika-0.1.15-py3-none-any.whl" +_PRAKTIKA_CONTROLLER_WHL = f"{_PRAKTIKA_PACKAGE_BASE_URL}/praktika_controller-0.1.10-py3-none-any.whl" + + +# Rarely-changing tooling baked into the runner image for the documentation-lint +# jobs (vale_linter, doc_links). Fast-moving Go tooling (Go, helm, kubebuilder, +# controller-gen, ...) is NOT baked — it is installed per job and cached on S3 by +# the go-env pre-hook (ci/jobs/go_env.py), so version bumps need no AMI rebuild. +# This component runs on both the arm64 and x86_64 builders, so every download is +# arch-aware. +_NODE_MAJOR = "20" + + +def _doc_lint_tools_component(): + """Build-phase Image Builder component installing Vale and Node + linkspector + for the documentation-lint jobs.""" + commands = [ + # Resolve the Debian CPU arch once (arm64 / amd64) for arch-specific URLs. + "arch=$(dpkg --print-architecture)", + # --- Vale (latest release, matching vale-action's default) --- + # amd64 release assets use the `64-bit` arch token; arm64 uses `arm64`. + 'if [ "$arch" = "amd64" ]; then vale_arch="64-bit"; else vale_arch="arm64"; fi', + "vale_ver=$(curl -fsSL https://api.github.com/repos/errata-ai/vale/releases/latest | jq -r .tag_name | sed 's/^v//')", + 'curl -fsSL "https://github.com/errata-ai/vale/releases/download/v${vale_ver}/vale_${vale_ver}_Linux_${vale_arch}.tar.gz" -o /tmp/vale.tgz', + "tar -xzf /tmp/vale.tgz -C /usr/local/bin vale && rm -f /tmp/vale.tgz", + "vale --version", + # --- Node + linkspector (docs link checker) --- + f"curl -fsSL https://deb.nodesource.com/setup_{_NODE_MAJOR}.x | bash -", + "DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends nodejs", + "npm install -g @umbrelladocs/linkspector", + "linkspector --version", + ] + return { + "name": "docs-lint-tools", + "platform": "Linux", + "phase": "build", + "description": "Install docs-lint toolchain: Vale, Node + linkspector", + "commands": commands, + } + + +def _docker_kind_component(): + """Build-phase component that disables Docker's containerd image store so + `kind load docker-image` works on the Docker+Kind jobs (compat-e2e, e2e). + + praktika's base setup installs docker-ce and writes /etc/docker/daemon.json; + this component runs afterwards (custom components are appended last) and merges + `features.containerd-snapshotter = false` into it, preserving praktika's keys. + + Why: `kind load docker-image` exports each image with `docker save` and + re-imports it via `ctr ... --all-platforms --digests`. With the containerd + image store enabled, even a single-arch `docker pull` keeps a multi-platform + image index, so the import fails on the other platforms' / attestation blobs + ("content digest ... not found"). The classic image store emits a flat + single-platform archive, matching GitHub-hosted runners where `kind load` + works. + + The image-builder component YAML escaper only escapes double quotes, so the + JSON-heavy script is base64-wrapped (same technique as praktika's docker + registry-mirror component). + """ + script = """#!/usr/bin/env bash +set -euo pipefail +f=/etc/docker/daemon.json +[ -f "$f" ] || echo '{}' > "$f" +tmp=$(mktemp) +jq '.features."containerd-snapshotter" = false' "$f" > "$tmp" +mv "$tmp" "$f" +jq -e '.features."containerd-snapshotter" == false' "$f" +""" + b64 = base64.b64encode(script.encode()).decode() + return { + "name": "docker-kind-image-store", + "platform": "Linux", + "phase": "build", + "description": "Disable Docker containerd image store so kind load works", + "commands": [f"printf '%s' '{b64}' | base64 -d | bash"], + } + + +def _image_builders(): + # Bump whenever the recipe/components change so Image Builder creates a new + # recipe + component versions and rebuilds the AMI. + image_recipe_version = "1.0.9" + prebuilt_venvs = [ + # The `infrastructure` extra pulls Praktika's runtime deps + # (boto3/PyJWT/cryptography/requests) automatically; pytest and the + # Bedrock AI SDK are optional extras, so list them explicitly. The + # orchestrator's AI advisor (AI_PROVIDER="bedrock-anthropic") imports + # `anthropic[bedrock]` lazily at decide() time; baking it into this shared + # venv keeps it present on every AMI (harmless on job runners). + ImageBuilder.PrebuiltVenv( + name=Settings.PRAKTIKA_BASE_VENV, + packages=[ + "pytest>=7.0.0", + "pytest-reportlog>=0.4.0", + "anthropic[bedrock]", + f"praktika[infrastructure] @ {_PRAKTIKA_WHL}", + ], + description="CI runtime venv (+ Bedrock AI SDK for the orchestrator)", + ), + ] + custom_components = [ + # Build-phase: install the docs-lint toolchain (Vale + Node/linkspector). + # Go tooling is provisioned per job via the go-env pre-hook, not baked. + _doc_lint_tools_component(), + # Build-phase: disable Docker's containerd image store (kind load fix). + _docker_kind_component(), + # Test-phase: validate the image after build. + Components.create_image_test_component( + name="project-image-test", + commands=[ + "test -d /opt/praktika/work", + "test -w /opt/praktika/work", + "vale --version", + "linkspector --version", + ], + ), + ] + # The Ubuntu 24.04 parent AMI ships an 8 GB gp3 root volume on /dev/sda1; + # double it to 16 GB so the baked Go/pip caches + Docker layers have room. + block_device_mappings = [ + { + "deviceName": "/dev/sda1", + "ebs": { + "volumeSize": 16, + "volumeType": "gp3", + "deleteOnTermination": True, + }, + } + ] + builders = [ + Components.create_ubuntu_image_builder_config( + name="ci-arm64-image", + version=image_recipe_version, + controller_package=_PRAKTIKA_CONTROLLER_WHL, + prebuilt_venvs=prebuilt_venvs, + components=custom_components, + instance_types=["c7g.2xlarge"], + ), + Components.create_ubuntu_image_builder_config( + name="ci-x86_64-image", + version=image_recipe_version, + controller_package=_PRAKTIKA_CONTROLLER_WHL, + prebuilt_venvs=prebuilt_venvs, + components=custom_components, + instance_types=["c7i.2xlarge"], + ), + ] + for builder in builders: + builder.block_device_mappings = block_device_mappings + return builders + + +_GH_TOKEN_MINTER = Components.GitHubTokenMinter( + permissions={ + "actions": "write", + "checks": "write", + "contents": "write", + "issues": "write", + "metadata": "read", + "pages": "write", + "pull_requests": "write", + "statuses": "write", + }, + repositories=[Settings.PROJECT_NAME], +) +_IMAGE_BUILDERS = _image_builders() +_IMAGE_BUILDERS_BY_NAME = {builder.name: builder for builder in _IMAGE_BUILDERS} + + +# The Code Review job (`praktika review`) calls an OpenAI model on Bedrock via +# the Converse API, which requires bedrock:InvokeModel. Only the dedicated +# code-review runner pool below carries this grant (scoped to Bedrock +# foundation-model / inference-profile resources) — general job runners stay +# Bedrock-less, so an arbitrary job cannot reach the model API. +_CODE_REVIEW_BEDROCK_IAM_STATEMENT = { + "Sid": "BedrockRuntimeInference", + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": [ + "arn:aws:bedrock:*::foundation-model/*", + "arn:aws:bedrock:*:*:inference-profile/*", + ], +} + + +# The orchestrator's AI advisor (AI_PROVIDER="bedrock-anthropic") reaches Claude +# through Bedrock Runtime via `anthropic[bedrock]`, so the orchestrator instance +# role must be allowed to invoke the model. Scoped to Bedrock foundation-model / +# inference-profile resources (covers the global.anthropic.* inference profiles +# the advisor targets). +_ORCHESTRATOR_BEDROCK_IAM_STATEMENT = { + "Sid": "BedrockRuntimeInference", + "Effect": "Allow", + "Action": ["bedrock:InvokeModel"], + "Resource": [ + "arn:aws:bedrock:*::foundation-model/*", + "arn:aws:bedrock:*:*:inference-profile/*", + ], +} + + +PROJECTS = [ + CloudInfrastructure.Config( + name=Settings.PROJECT_NAME, + min_praktika_version="0.1.14", + vpcs=[ + VPC.Config( + subnets=[ + VPC.Subnet(availability_zone="eu-north-1a"), + ], + ) + ], + storages=[ + Storage.Config( + name="artifacts-eu-north-1", + retention_days=30, + public=True, + ), + ], + report_pages=[Components.report_page_config], + image_builders=_IMAGE_BUILDERS, + github_token_minters=[_GH_TOKEN_MINTER], + orchestrator_pool=Components.OrchestratorPool( + instance_type="t4g.xlarge", + scaling=Components.OrchestratorPool.Scaling.Auto, + size=0, + max_size=50, + volume_size_gb=100, + capacity_reserve=4, + image_builder=_IMAGE_BUILDERS_BY_NAME["ci-arm64-image"], + ext={ + "allowed_push_branches": ['main'], + "allowed_pr_base_branches": ['main'], + "allowed_users": [], + # The orchestrator's AI advisor calls Bedrock, so its role needs + # a scoped bedrock:InvokeModel grant. + "iam_statements": [_ORCHESTRATOR_BEDROCK_IAM_STATEMENT], + }, + ), + runner_pools=[ + Components.RunnerPool( + name="arm-2cpu-4g", + instance_type="c7g.large", + scaling=Components.RunnerPool.Scaling.Auto, + size=0, + capacity_reserve=4, + max_size=50, + volume_size_gb=100, + image_builder=_IMAGE_BUILDERS_BY_NAME["ci-arm64-image"], + allowed_ssm_parameters=[], + allowed_secrets=[], + allowed_s3_prefixes=_PROJECT_S3_PREFIXES, + allow_all_ssm_parameters=False, + allow_all_secrets=False, + allow_all_s3_prefixes=False, + allow_ssm_debug=False, + ), + Components.RunnerPool( + name="amd-2cpu-4g", + instance_type="c7a.large", + scaling=Components.RunnerPool.Scaling.Auto, + size=0, + max_size=50, + volume_size_gb=100, + image_builder=_IMAGE_BUILDERS_BY_NAME["ci-x86_64-image"], + allowed_ssm_parameters=[], + allowed_secrets=[], + allowed_s3_prefixes=_PROJECT_S3_PREFIXES, + allow_all_ssm_parameters=False, + allow_all_secrets=False, + allow_all_s3_prefixes=False, + allow_ssm_debug=False, + ), + Components.RunnerPool( + name="arm-8cpu-16g", + instance_type="c7g.2xlarge", + scaling=Components.RunnerPool.Scaling.Auto, + size=0, + max_size=50, + volume_size_gb=100, + image_builder=_IMAGE_BUILDERS_BY_NAME["ci-arm64-image"], + allowed_ssm_parameters=[], + allowed_secrets=[], + allowed_s3_prefixes=_PROJECT_S3_PREFIXES, + allow_all_ssm_parameters=False, + allow_all_secrets=False, + allow_all_s3_prefixes=False, + allow_ssm_debug=False, + ), + Components.RunnerPool( + name="amd-8cpu-16g", + instance_type="c7a.2xlarge", + scaling=Components.RunnerPool.Scaling.Auto, + size=0, + max_size=50, + volume_size_gb=100, + image_builder=_IMAGE_BUILDERS_BY_NAME["ci-x86_64-image"], + allowed_ssm_parameters=[], + allowed_secrets=[], + allowed_s3_prefixes=_PROJECT_S3_PREFIXES, + allow_all_ssm_parameters=False, + allow_all_secrets=False, + allow_all_s3_prefixes=False, + allow_ssm_debug=False, + ), + # Dedicated pool for the AI Code Review job. Identical to arm-2cpu-4g, + # plus a scoped bedrock:InvokeModel grant via ext["iam_statements"] + # so only this pool's role can call the Bedrock model API. + Components.RunnerPool( + name="arm-2cpu-4g-bedrock", + instance_type="c7g.large", + scaling=Components.RunnerPool.Scaling.Auto, + size=0, + max_size=50, + volume_size_gb=100, + image_builder=_IMAGE_BUILDERS_BY_NAME["ci-arm64-image"], + allowed_ssm_parameters=[], + allowed_secrets=[], + allowed_s3_prefixes=_PROJECT_S3_PREFIXES, + allow_all_ssm_parameters=False, + allow_all_secrets=False, + allow_all_s3_prefixes=False, + allow_ssm_debug=False, + ext={"iam_statements": [_CODE_REVIEW_BEDROCK_IAM_STATEMENT]}, + ), + ], + ) +] diff --git a/ci/jobs/bundle.py b/ci/jobs/bundle.py new file mode 100644 index 000000000..ee093fc4f --- /dev/null +++ b/ci/jobs/bundle.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +""" +ci.yaml :: bundle, migrated to praktika. + +Builds the OLM bundle and runs operator-sdk scorecard against a throwaway Kind +cluster. Reproduces, in-script, what the GitHub Actions job did via marketplace +actions (shared Docker/Kind plumbing in ci/jobs/kind_env.py): + + - `make bundle` -> generate + validate the bundle manifests + - `helm/kind-action` -> plain Kind cluster named "test-bundle" + - `kind export kubeconfig` -> point kubectl/scorecard at that cluster + - `make bundle-build` -> docker build the bundle image + - `kind load docker-image` -> load it into the cluster + - `make scorecard` -> operator-sdk scorecard bundle --wait-time=2m + +Env: + VERSION bundle version (default 0.0.1, matching ci.yaml), used to derive the + bundle image tag and FULL_VERSION the Makefile builds. + +operator-sdk is installed by the Makefile targets themselves (the `operator-sdk` +prereq), so there is no separate install step. kustomize/controller-gen come from +the go-env bundle. Docker must be provided by the runner image. + +Verdict is the scorecard exit code (no JUnit report), surfaced via the job exit +code (enable_exit_code_result). +""" +import os +import sys + +sys.path.insert(0, os.getcwd()) # ensure repo root is importable for ci.* +from ci.jobs import kind_env # noqa: E402 + +CLUSTER_NAME = "test-bundle" + + +def main() -> int: + version = os.environ.get("VERSION") or "0.0.1" + # Mirrors the Makefile's BUNDLE_IMG for a fixed VERSION (FULL_VERSION=v$VERSION + # when VERSION is set); same ref ci.yaml kind-loads. + bundle_img = f"ghcr.io/clickhouse/clickhouse-operator-bundle:v{version}" + + kind_env.require_docker() + # Before the cluster is created / the bundle image is built: may restart Docker. + kind_env.disable_containerd_image_store() + kind_env.install_kind() + kind_env.install_kubectl() + + kind_env.sh("make bundle") + # Plain default cluster (no ci/kind-cluster.config, Kind's default node image), + # matching the bundle job's kind-action usage. + kind_env.create_cluster(name=CLUSTER_NAME, config=None) + kind_env.sh(f"kind export kubeconfig --name {CLUSTER_NAME}") + kind_env.sh("make bundle-build") + kind_env.sh(f"kind load docker-image {bundle_img} --name {CLUSTER_NAME}") + kind_env.sh("make scorecard") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/jobs/check_crd_compat.py b/ci/jobs/check_crd_compat.py new file mode 100644 index 000000000..f77b0b267 --- /dev/null +++ b/ci/jobs/check_crd_compat.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +""" +ci.yaml :: check-crd-compat, migrated to praktika. + +The crd-breaking-change label gate lives in the workflow filter hook +(ci/jobs/filter_job_hook.py), not here. + +What the job does: it obtains each CRD's baseline (the manifest as it currently +exists on the base branch) and runs `make check-crd-compat`, which compares each +baseline against the PR's current CRD with crd-schema-checker and fails on +backward-incompatible schema changes. The verdict is returned as a praktika +Result (OK / FAIL / ERROR) carrying the baseline ref/source, staged counts, and +— on failure — the checker's violations. + +The baseline commit is the PR's merge-base from Info().base_git_history() (entry +0, newest first) — a real base-branch commit the controller resolves even for +jobs on a history-free S3 snapshot, which keeps a 404 unambiguous. Falls back to +the base branch name when unavailable (local runs). + +Two ways to read the manifests at that ref, picked automatically: + +1. git history (preferred): a clone with `origin` shallow-fetches the ref and + reads `git show FETCH_HEAD:...`, no per-file network call. + +2. GitHub contents API (fallback): on an S3 snapshot there is no `origin`, so the + fetch fails; we pull the base manifests via `gh` into a temp dir and pass it as + CRD_BASELINE_DIR. A 404 means the path is new on the base and is skipped. +""" +import glob +import os +import subprocess +import sys +import tempfile + +from praktika.info import Info +from praktika.result import Result +from praktika.utils import Utils + +CRD_GLOB = "config/crd/bases/*.yaml" + + +def _resolve_base_ref(info): + """Baseline ref: the PR merge-base SHA from Info().base_git_history() (entry 0, + newest first), falling back to the base branch name when unavailable (local + runs). Returns (ref, human_label).""" + history = info.base_git_history() + if history: + sha = history[0] + return sha, f"merge-base {sha[:12]}" + base = info.base_branch or "main" + return base, f"base branch {base}" + + +def _try_fetch_git_base(ref): + """Shallow-fetch `ref` so `git show FETCH_HEAD:` resolves base CRDs. + Returns True on success (clone with `origin`), False otherwise (snapshot has + no remote).""" + cmd = ["git", "fetch", "--no-tags", "--depth=1", "origin", ref] + print(f"+ {' '.join(cmd)}", flush=True) + return subprocess.run(cmd).returncode == 0 + + +def _stage_baselines(repo, ref, dest): + """Download each CRD's version at `ref` into `dest` via gh api. + + `--method GET` is required: `gh api` switches to POST once any `-f` field is + present, and the contents endpoint 404s on POST — which we'd misread as "new + CRD" for every file and pass vacuously. + + Returns (staged, skipped): a genuine 404 (file absent at `ref`) is left + unstaged so the Makefile treats it as new; raises on any other gh failure.""" + staged = skipped = 0 + for crd in sorted(glob.glob(CRD_GLOB)): + cmd = [ + "gh", "api", + "--method", "GET", + "-H", "Accept: application/vnd.github.raw", + f"repos/{repo}/contents/{crd}", + "-f", f"ref={ref}", + ] + print(f"+ {' '.join(cmd)}", flush=True) + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode == 0: + with open(os.path.join(dest, os.path.basename(crd)), "w") as f: + f.write(proc.stdout) + print(f" staged baseline for {crd}", flush=True) + staged += 1 + elif "404" in proc.stderr or "Not Found" in proc.stderr: + print(f" {crd} absent at {ref} — new CRD, will skip", flush=True) + skipped += 1 + else: + sys.stderr.write(proc.stderr) + raise RuntimeError(f"gh api failed for {crd}: {proc.stderr.strip()}") + return staged, skipped + + +def _run_make(make_args): + """Run `make check-crd-compat `, echoing output. Returns (rc, output).""" + cmd = ["make", "check-crd-compat"] + make_args + print(f"+ {' '.join(cmd)}", flush=True) + proc = subprocess.run(cmd, capture_output=True, text=True) + output = (proc.stdout or "") + (proc.stderr or "") + print(output, end="", flush=True) + return proc.returncode, output + + +def _check(info_lines): + """Run the compatibility check. Returns (rc, output); appends to info_lines.""" + info = Info() + ref, label = _resolve_base_ref(info) + info_lines.append(f"baseline ref: {label}") + + if _try_fetch_git_base(ref): + info_lines.append("baseline source: git history (FETCH_HEAD)") + return _run_make(["CRD_BASE_REF=FETCH_HEAD"]) + + info_lines.append( + "baseline source: GitHub API (snapshot checkout, no local git base)" + ) + repo = info.repo_name + if not repo: + raise RuntimeError("Could not resolve repository (Info().repo_name is empty)") + with tempfile.TemporaryDirectory(prefix="crd-baseline-") as dest: + staged, skipped = _stage_baselines(repo, ref, dest) + info_lines.append(f"baselines: {staged} staged, {skipped} new/absent (skipped)") + return _run_make([f"CRD_BASELINE_DIR={dest}"]) + + +def main(): + sw = Utils.Stopwatch() + info_lines = [] + try: + rc, output = _check(info_lines) + except Exception as e: # unexpected: resolution/gh/make plumbing failure + Result.create_from( + status=Result.Status.ERROR, + stopwatch=sw, + info="\n".join(info_lines + [f"ERROR: {e}"]), + ).complete_job() + return + + if rc == 0: + info_lines.append("No backward-incompatible CRD changes.") + status = Result.Status.OK + else: + status = Result.Status.FAIL + info_lines += ["", "Backward-incompatible CRD changes detected:", output.strip()] + + Result.create_from( + status=status, + stopwatch=sw, + info="\n".join(info_lines), + ).complete_job() + + +if __name__ == "__main__": + main() diff --git a/ci/jobs/ci_links.py b/ci/jobs/ci_links.py new file mode 100644 index 000000000..7987636c6 --- /dev/null +++ b/ci/jobs/ci_links.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +""" +Workflow pre-hook: post the CI report link into the PR description. + +Wired via `Workflow.Config(pre_hooks=["python3 ./ci/jobs/ci_links.py"])`. It runs +once at the start of a PR run (before the jobs) and appends a machine-owned block +with a link to this run's report, so a reviewer can jump from the PR straight to +the live CI report without hunting for the check run. + +Modeled on ClickHouse/ClickHouse ci/jobs/scripts/workflow_hooks/ci_links.py, minus +the upstream-sync bits that don't apply to this single-repo project. +""" +import tempfile +import traceback + +from praktika.gh import GH +from praktika.info import Info + +BLOCK_START = "" +BLOCK_END = "" + + +def has_block(body): + return BLOCK_START in body and BLOCK_END in body + + +def append_block(body, block): + if body.strip(): + return body.rstrip() + "\n\n" + block + "\n" + return block + "\n" + + +def main(): + info = Info() + if info.pr_number <= 0: + print("NOTE: Not a PR run - skip PR description update") + return + + if has_block(info.pr_body or ""): + print("NOTE: CI links already present - skip PR description update") + return + + workflow_line = ( + f"Workflow [[{info.workflow_name}]({info.get_report_url(latest=True)})]" + ) + # A horizontal rule visually separates the machine-owned block from the + # user-authored description. It lives inside the block markers so it is + # dropped together with the block. + block = f"{BLOCK_START}\n\n---\n{workflow_line}\n{BLOCK_END}" + + title, body, _labels, _is_draft = GH.get_pr_title_body_labels() + if not title: + print("WARNING: Failed to fetch PR data - skip PR description update") + return + + new_body = append_block(body or "", block) + + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", encoding="utf-8") as f: + f.write(new_body) + f.flush() + if not GH.update_pr_body(body_file=f.name): + print("WARNING: Failed to update PR description") + return + + info.env.PR_BODY = new_body + info.env.dump() + print("PR description updated with CI links") + + +if __name__ == "__main__": + try: + main() + except Exception: + traceback.print_exc() diff --git a/ci/jobs/compat_e2e.py b/ci/jobs/compat_e2e.py new file mode 100644 index 000000000..3c187ed08 --- /dev/null +++ b/ci/jobs/compat_e2e.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +""" +ci.yaml :: compat-e2e-test, migrated to praktika. + +Driver for every compat-e2e matrix variant. The 5-way matrix is fanned out in +ci/workflows/job_configs.py (_COMPAT_E2E_MATRIX + Job.parametrize); each variant +invokes this script with a different K8S_IMAGE / CLICKHOUSE_VERSION / +DEPLOY_TARGET. It reproduces, in-script, what the GitHub Actions job did via +marketplace actions: + + - `helm/kind-action` -> install the pinned Kind CLI + `kind create cluster` + with the per-variant node image and ci/kind-cluster.config + - pre-pull step -> `docker pull` (with retry) + `kind load docker-image` + for each ClickHouse version + - `make ` -> the actual compatibility test run + +It is driven by three env vars so the same script serves every matrix variant +once parametrized: + + K8S_IMAGE Kind node image tag, e.g. "v1.36.1" + CLICKHOUSE_VERSION comma-separated ClickHouse versions to pre-pull / test + DEPLOY_TARGET Makefile target, e.g. "test-compat-e2e" + FETCH_TAGS if set, resolve the release to upgrade from (the upgrade + variant deploys the latest release). Legacy clones fetch + tags so the test's own `git tag --list` works; snapshot + checkouts (no `origin` remote / history) resolve the highest + vX.Y.Z tag via the GitHub API and export + UPGRADE_FROM_VERSION, which deploy_test.go honors instead. + +Go/helm/kubebuilder come from the go-env pre-hook (on PATH via /usr/local/bin); +Docker must be provided by the runner image. The Docker/Kind plumbing is shared +with the e2e job in ci/jobs/kind_env.py. + +Reporting: Ginkgo writes a JUnit report per suite (`--ginkgo.junit-report`); we +parse them into a praktika Result with one sub-result per test case +(ci/jobs/junit_result.py) and `complete_job()` it, so the report page shows a +per-spec table — the replacement for the dropped dorny/test-reporter. The job's +verdict comes from the parsed cases reconciled with the `make` exit code, so +`enable_exit_code_result` is only a last-resort fallback here. + +Report-artifact upload (GHA `upload-artifact e2e-report-*`) is intentionally NOT +carried over: its only consumer (`e2e-report`) is not migrated yet. +""" +import os +import re +import subprocess +import sys + +sys.path.insert(0, os.getcwd()) # ensure repo root is importable for ci.* +from ci.jobs import kind_env # noqa: E402 +from ci.jobs.junit_result import build_job_result # noqa: E402 + +from praktika.info import Info # noqa: E402 +from praktika.result import Result # noqa: E402 +from praktika.utils import Utils # noqa: E402 + + +def _latest_release_tag(repo): + """Highest vX.Y.Z tag in `repo` via the GitHub API (mirrors deploy_test.go's + `git tag --list "v[0-9]*.[0-9]*.[0-9]*" --sort=-v:refname`).""" + out = subprocess.run( + ["gh", "api", "--paginate", f"repos/{repo}/tags", "--jq", ".[].name"], + capture_output=True, text=True, check=True, + ).stdout + versions = [] + for name in out.split(): + m = re.fullmatch(r"v(\d+)\.(\d+)\.(\d+)", name) + if m: + versions.append((tuple(int(x) for x in m.groups()), name)) + if not versions: + raise RuntimeError(f"no vX.Y.Z release tags found in {repo}") + return max(versions)[1] + + +def _resolve_upgrade_from(): + """Make the release-to-upgrade-from discoverable by the upgrade test. + + Honors an explicit UPGRADE_FROM_VERSION. Otherwise tries to fetch tags into + local git (legacy clone — the test then reads them itself); if there is no + `origin` remote (snapshot checkout), resolves the latest tag via the GitHub + API and exports UPGRADE_FROM_VERSION, which deploy_test.go honors instead. + """ + if os.environ.get("UPGRADE_FROM_VERSION"): + return + if kind_env.sh("git fetch --tags --force origin", check=False).returncode == 0: + return + print( + "NOTE: no git remote for tags — resolving latest release via GitHub API", + flush=True, + ) + tag = _latest_release_tag(Info().repo_name) + os.environ["UPGRADE_FROM_VERSION"] = tag + print(f" UPGRADE_FROM_VERSION={tag}", flush=True) + + +def main(): + sw = Utils.Stopwatch() + try: + node_image = os.environ["K8S_IMAGE"] + clickhouse_version = os.environ["CLICKHOUSE_VERSION"] + deploy_target = os.environ["DEPLOY_TARGET"] + + if os.environ.get("FETCH_TAGS"): + _resolve_upgrade_from() + kind_env.require_docker() + # Must run before the cluster is created / any image is pulled: it may + # restart the Docker daemon, which would wipe cluster containers and + # pulled images. + kind_env.disable_containerd_image_store() + kind_env.install_kind() + kind_env.install_kubectl() + # operator-sdk + opm, matching ci.yaml's unconditional + # `make operator-sdk opm` (needed by the OLM deploy path; cheap no-ops + # for the others). + kind_env.sh("make operator-sdk opm") + kind_env.create_cluster(node_image) + kind_env.prepull(clickhouse_version) + + # Run the suite without raising on failure: Ginkgo still writes the JUnit + # report on test failures, and we want to parse it either way. The job + # verdict comes from the parsed cases reconciled with this exit code. + env = {**os.environ, "CLICKHOUSE_VERSION": clickhouse_version} + print(f"+ make {deploy_target}", flush=True) + exit_code = subprocess.run( + f"make {deploy_target}", shell=True, env=env + ).returncode + except Exception as e: # setup/provisioning failure before the suite runs + Result.create_from( + status=Result.Status.ERROR, + stopwatch=sw, + info=f"compat-e2e setup failed: {e}", + ).complete_job() + return + + # Ginkgo writes report/junit-report.xml under each suite's package dir. + result = build_job_result( + report_glob="**/report/junit-report.xml", + run_exit_code=exit_code, + stopwatch=sw, + ) + result.complete_job() # dumps the Result and exits with the matching code + + +if __name__ == "__main__": + main() diff --git a/ci/jobs/dependabot_regenerate.py b/ci/jobs/dependabot_regenerate.py new file mode 100644 index 000000000..24d8447f1 --- /dev/null +++ b/ci/jobs/dependabot_regenerate.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +""" +.github/workflows/dependabot-regenerate.yaml, migrated to praktika. + +What it does: when Dependabot opens a PR that bumps go.mod/go.sum, the committed +*generated* artifacts (CRDs/RBAC, deepcopy, helm chart, API-reference docs) go +stale. This job re-derives them and pushes the result back onto the PR branch, so +the bot PR is self-consistent before the rest of the suite judges it. + +Applicability is decided *outside* this script, in the workflow filter hook +(ci/jobs/filter_job_hook.py): it runs only for a same-repo `dependabot[bot]` PR +that actually touched go.mod/go.sum. So here we assume we should regenerate. + +Why this can't be a `git diff --exit-code` check like Lint: Lint only *detects* +staleness and fails; this job *fixes* it and pushes, which is the whole point of +the original dependabot-regenerate workflow — a human never has to touch a bot PR. + +Branch handling: praktika checks out the PR's ephemeral `refs/pull//merge` +commit (detached, no remote). Committing there would commit the merge, not the +head. So we fetch the real head branch, check it out, regenerate on top of it, and +push that — exactly what the original workflow's `checkout ref: head.ref` did. + +Push credentials: the original pushed with GITHUB_TOKEN, whose pushes GitHub +deliberately does NOT re-trigger workflows on — hence its explicit trailing +`gh workflow run`. We push with a GitHub App installation token minted here +(GHAuth), and App-token pushes DO emit a `synchronize` event, so the fresh PR run +starts on its own; no manual re-dispatch needed. The token minter must grant +`contents: write`. +""" +import base64 +import subprocess +import sys + +from praktika.info import Info +from praktika.gh_auth import GHAuth + +# Mirrors dependabot-regenerate.yaml's commit identity. +_GIT_USER_NAME = "github-actions[bot]" +_GIT_USER_EMAIL = "41898282+github-actions[bot]@users.noreply.github.com" + +# dependabot-regenerate.yaml :: "Regenerate after dependency bump". KUBEBUILDER +# points generate-helmchart-ci at the binary the go-env pre-hook provisioned, +# matching the Helm Test job, instead of re-downloading it. +_REGEN_CMDS = [ + ["go", "mod", "tidy"], + [ + "make", + "generate", + "manifests", + "generate-helmchart-ci", + "docs-generate-api-ref", + "KUBEBUILDER=/usr/local/bin/kubebuilder", + ], +] + + +def _run(cmd, **kw): + print(f"+ {' '.join(cmd)}", flush=True) + return subprocess.run(cmd, check=True, **kw) + + +def _dirty() -> bool: + out = subprocess.run( + ["git", "status", "--porcelain"], check=True, capture_output=True, text=True + ).stdout.strip() + return bool(out) + + +def main() -> int: + info = Info() + repo = info.repo_name # owner/repo + head = info.git_branch # dependabot's head branch (same-repo, hook-guarded) + if not head: + print("No head branch in context; nothing to regenerate") + return 0 + + # contents:write to push the regenerated commit back onto the PR branch. + token = GHAuth.get_installation_token(required_permissions={"contents": "write"}) + + # Authenticate with an HTTP header, NOT a token-in-URL remote: _run echoes + # every argument and git prints the remote URL in error messages, so a + # token-bearing URL would leak this live contents:write credential into the + # public Praktika log. The remote URL stays token-free; the Authorization + # header (set below without echoing its value) carries the credential, and + # fetch/push reference the remote by name so they never surface the token. + remote = f"https://github.com/{repo}.git" + auth = base64.b64encode(f"x-access-token:{token}".encode()).decode() + auth_header_key = "http.https://github.com/.extraheader" + + # The ephemeral merge checkout has no remote; add one and lay the real head + # branch on top of the working tree before regenerating. + subprocess.run(["git", "remote", "remove", "origin"], check=False) + _run(["git", "remote", "add", "origin", remote]) + print(f"+ git config --local {auth_header_key} 'AUTHORIZATION: basic '", flush=True) + subprocess.run( + ["git", "config", "--local", auth_header_key, f"AUTHORIZATION: basic {auth}"], + check=True, + ) + _run(["git", "fetch", "--no-tags", "--depth=1", "origin", head]) + _run(["git", "checkout", "-B", head, "FETCH_HEAD"]) + + for cmd in _REGEN_CMDS: + _run(cmd) + + if not _dirty(): + print("Nothing to regenerate") + return 0 + + _run(["git", "config", "user.name", _GIT_USER_NAME]) + _run(["git", "config", "user.email", _GIT_USER_EMAIL]) + _run(["git", "add", "-A"]) + _run(["git", "commit", "-m", "chores: regenerate after dependency bump"]) + # App-token push → GitHub emits `synchronize` → fresh PR run starts itself. + _run(["git", "push", "origin", f"HEAD:refs/heads/{head}"]) + print(f"Pushed regenerated files to {head}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/jobs/e2e.py b/ci/jobs/e2e.py new file mode 100644 index 000000000..79c57010e --- /dev/null +++ b/ci/jobs/e2e.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +""" +ci.yaml :: e2e-test, migrated to praktika. + +Driver for one e2e shard. The 4-way shard matrix is fanned out in +ci/workflows/job_configs.py (Job.parametrize); each shard invokes this script +with a different E2E_SHARD ("index/total", e.g. "2/4"). It reproduces, in-script, +what the GitHub Actions job did via marketplace actions: + + - `helm/kind-action` -> install the pinned Kind CLI + `kind create cluster` + (shared Docker/Kind plumbing in ci/jobs/kind_env.py) + - failure diagnostics -> `kind export logs` + `kubectl get pods/events` + - `make test-e2e` -> the sharded e2e run + +Sharding is round-robin: the Go harness (test/testutil/sharding.go) hashes each +spec name to a shard, so no pre-computed plan is needed. (The legacy plan-based +"smart" mode still works via E2E_SHARD_INDEX/TOTAL/PLAN and is what ci.yaml +drives; this job uses the simpler E2E_SHARD form.) + +Env: + E2E_SHARD "index/total" for the Go harness, e.g. "2/4" (passed to `make`) + K8S_IMAGE Kind node image tag (default v1.34.3, matching ci.yaml) + +Reporting: Ginkgo writes report/junit-report.xml; we parse it into a praktika +Result with one sub-result per spec (ci/jobs/junit_result.py) and complete_job() +it. On failure, kind/kubectl diagnostics are collected and attached to the +Result. Unlike compat-e2e, there is no image pre-pull and no operator-sdk/opm, +matching the GitHub e2e-test job. +""" +import os +import subprocess +import sys + +sys.path.insert(0, os.getcwd()) # ensure repo root is importable for ci.* +from ci.jobs import kind_env # noqa: E402 +from ci.jobs.junit_result import build_job_result # noqa: E402 + +from praktika.result import Result # noqa: E402 +from praktika.utils import Utils # noqa: E402 + +# ci.yaml pins the e2e Kind node image (its comment cited cgroups v1 on the old +# self-hosted runners; praktika runners are cgroups v2, where this image is fine). +DEFAULT_K8S_IMAGE = "v1.34.3" +DIAG_DIR = "failure-diagnostics" +DIAG_ARCHIVE = "failure-diagnostics.tgz" + + +def _collect_diagnostics(): + """Mirror ci.yaml's failure-diagnostics step; return the archive path (or None). + Best-effort: every command is allowed to fail.""" + kind_env.sh(f"mkdir -p {DIAG_DIR}", check=False) + kind_env.sh(f"kind export logs --name {kind_env.DEFAULT_CLUSTER} {DIAG_DIR}/kind", check=False) + kind_env.sh(f"kubectl get pods,jobs -A -o wide > {DIAG_DIR}/pods-jobs.txt 2>&1", check=False) + kind_env.sh( + f"kubectl get events -A --sort-by=.metadata.creationTimestamp " + f"> {DIAG_DIR}/events.txt 2>&1", + check=False, + ) + proc = kind_env.sh(f"tar czf {DIAG_ARCHIVE} {DIAG_DIR}", check=False) + return DIAG_ARCHIVE if proc.returncode == 0 and os.path.isfile(DIAG_ARCHIVE) else None + + +def main(): + sw = Utils.Stopwatch() + try: + node_image = os.environ.get("K8S_IMAGE") or DEFAULT_K8S_IMAGE + + kind_env.require_docker() + # Must run before the cluster is created: it may restart the Docker + # daemon, which would wipe cluster containers. + kind_env.disable_containerd_image_store() + kind_env.install_kind() + kind_env.install_kubectl() + kind_env.create_cluster(node_image) + + # Run the suite without raising on failure: Ginkgo still writes the JUnit + # report on test failures, and we want to parse it either way. E2E_SHARD + # is already in the environment (set by the job command) and inherited by + # `make`. + print("+ make test-e2e", flush=True) + exit_code = subprocess.run("make test-e2e", shell=True).returncode + except Exception as e: # setup/provisioning failure before the suite runs + Result.create_from( + status=Result.Status.ERROR, + stopwatch=sw, + info=f"e2e setup failed: {e}", + ).complete_job() + return + + extra_files = [] + if exit_code != 0: + archive = _collect_diagnostics() + if archive: + extra_files.append(archive) + # Ginkgo's JSON report (if present) carries per-spec timings for future + # duration-weighted sharding; attach it for download. + ginkgo_json = "test/e2e/report/ginkgo-report.json" + if os.path.isfile(ginkgo_json): + extra_files.append(ginkgo_json) + + result = build_job_result( + report_glob="**/report/junit-report.xml", + run_exit_code=exit_code, + stopwatch=sw, + extra_files=extra_files, + ) + result.complete_job() # dumps the Result and exits with the matching code + + +if __name__ == "__main__": + main() diff --git a/ci/jobs/filter_job_hook.py b/ci/jobs/filter_job_hook.py new file mode 100644 index 000000000..18fccb58d --- /dev/null +++ b/ci/jobs/filter_job_hook.py @@ -0,0 +1,42 @@ +""" +Workflow filter hooks for the praktika workflows. + +A hook has the signature `should_skip_job(job_name) -> (skip: bool, reason: str)` +and is wired via `Workflow.Config(workflow_filter_hooks=[should_skip_job])`. The +orchestrator calls it once per job while building the run (see +praktika.native_jobs), i.e. where `praktika.info.Info` is populated — so label +gating needs no `gh` call from inside the job. +""" +from praktika.info import Info + +_info = None + + +def should_skip_job(job_name): + global _info + if _info is None: + _info = Info() + + # ci.yaml :: check-crd-compat only ran when the PR was NOT labeled as an + # intentional CRD-breaking change. Reproduce that gate here. + if ( + job_name == "Check CRD Compatibility" + and "crd-breaking-change" in _info.pr_labels + ): + return True, "PR labeled 'crd-breaking-change'" + + # dependabot-regenerate.yaml's `if:` guard, moved here so the job is listed + # in every PR run but only *runs* for a Dependabot dependency bump. It pushes + # to the PR head branch, so it is restricted to same-repo PRs (a fork head is + # not ours to push to) — exactly the original `head.repo == repository` check. + if job_name == "Dependabot Regenerate": + actor = _info.user_name or "" + if actor != "dependabot[bot]": + return True, f"PR actor '{actor}' is not dependabot[bot]" + if _info.fork_name and _info.fork_name != _info.repo_name: + return True, "PR is from a fork" + changed = _info.get_changed_files() or [] + if not any(f in ("go.mod", "go.sum") for f in changed): + return True, "no go.mod/go.sum changes" + + return False, "" diff --git a/ci/jobs/go_env.py b/ci/jobs/go_env.py new file mode 100644 index 000000000..4073e03b7 --- /dev/null +++ b/ci/jobs/go_env.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +""" +Pre-hook that provisions the Go toolchain + CI tools for the Operator CI jobs, +backed by an S3 cache so version bumps need no AMI rebuild. + +What it provisions (into default, env-free locations so the job command needs no +extra env): the Go SDK (/usr/local/go), helm + kubebuilder + the go-installed +CLIs (controller-gen, kustomize, setup-envtest, golangci-lint, actionlint, +crd-schema-checker, crd-ref-docs) on PATH, a warm Go module + build cache +(/root/go, /root/.cache/go-build), the codespell pip wheel cache, and the +envtest K8s assets. + +Versions are parsed from go.mod / Makefile at runtime (helm is pinned here), so +the cache key tracks the repo's own pins — bump a version in the Makefile and the +next run rebuilds + repopulates the cache automatically. + +The whole tree is cached as one bundle (S3PathCache). The key is the version set, +NOT go.sum: dependency bumps are served incrementally from the warm module cache +rather than forcing a full rebuild on every dependabot PR. +""" +import glob +import os +import re +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, os.getcwd()) # ensure repo root is importable for ci.* +from ci.jobs.s3_cache import S3PathCache +from ci.settings.settings import AWS_REGION, CACHE_S3_PATH +from praktika.settings import Settings as PraktikaSettings + +# Basename pattern for the per-arch go-env bundle distributed as a praktika +# artifact (see JobConfigs prep jobs). The consumer's `install` mode extracts +# whichever one its `requires` placed in the input dir. +ARTIFACT_GLOB = "go-env*.tar.zst" + +HELM_VERSION = "v3.19.0" # helm has no Makefile pin; bump here +ENVTEST_ASSETS_DIR = "/opt/ci-go/envtest" +GOBIN = "/usr/local/bin" +# HOME-independent cache locations. The consumer job runs under a different $HOME +# than this script's build (which uses /root), so caches under /root/go would be +# ignored and modules re-downloaded. We put them at fixed /opt paths and point the +# job's `go` at them with `go env -w` (writes a config file, so it survives the +# pre-hook -> command boundary regardless of $HOME). +GO_MODCACHE = "/opt/ci-go/gomodcache" +GO_BUILDCACHE = "/opt/ci-go/gocache" +GOPATH_DIR = "/opt/ci-go/gopath" + +# go-installed CLIs: (binary, go package, Makefile version variable) +GO_TOOLS = [ + ("controller-gen", "sigs.k8s.io/controller-tools/cmd/controller-gen", "CONTROLLER_TOOLS_VERSION"), + ("kustomize", "sigs.k8s.io/kustomize/kustomize/v5", "KUSTOMIZE_VERSION"), + ("setup-envtest", "sigs.k8s.io/controller-runtime/tools/setup-envtest", "ENVTEST_VERSION"), + ("golangci-lint", "github.com/golangci/golangci-lint/v2/cmd/golangci-lint", "GOLANGCI_LINT_VERSION"), + ("actionlint", "github.com/rhysd/actionlint/cmd/actionlint", "ACTIONLINT_VERSION"), + ("crd-schema-checker", "github.com/openshift/crd-schema-checker/cmd/crd-schema-checker", "CRD_SCHEMA_CHECKER_VERSION"), + ("crd-ref-docs", "github.com/elastic/crd-ref-docs", "CRD_REF_DOCS_VERSION"), +] + +# Everything the bundle captures (absolute; missing paths are skipped on save). +BUNDLE_PATHS = [ + "/usr/local/go", + GO_MODCACHE, + GO_BUILDCACHE, + ENVTEST_ASSETS_DIR, +] + [f"{GOBIN}/{b}" for b in ("go", "gofmt", "helm", "kubebuilder")] + [ + f"{GOBIN}/{b}" for (b, _, _) in GO_TOOLS +] + +_GO_ENV = { + **os.environ, + "HOME": "/root", + "GOPATH": GOPATH_DIR, + "GOMODCACHE": GO_MODCACHE, + "GOCACHE": GO_BUILDCACHE, + "GOBIN": GOBIN, + "PATH": f"/usr/local/go/bin:{GOBIN}:" + os.environ.get("PATH", ""), + "DEBIAN_FRONTEND": "noninteractive", +} + + +# Ambient env + the bundle's bin dirs on PATH, but WITHOUT forcing HOME or the +# GO*CACHE vars — so `go env -w` in the consumer writes to the job user's own +# config (read by the job command) and isn't shadowed by an OS-env override. +_PATH_ENV = { + **os.environ, + "PATH": f"/usr/local/go/bin:{GOBIN}:" + os.environ.get("PATH", ""), +} + + +def _sh(cmd, env=None): + print(f"+ {cmd}", flush=True) + subprocess.run(cmd, shell=True, check=True, env=env or _GO_ENV) + + +def _makefile_version(var): + m = re.search(rf"^{var}\s*[:?]?=\s*(\S+)", Path("Makefile").read_text(), re.M) + if not m: + raise RuntimeError(f"could not find {var} in Makefile") + return m.group(1) + + +def _go_version(): + m = re.search(r"^go\s+(\d+\.\d+(?:\.\d+)?)", Path("go.mod").read_text(), re.M) + if not m: + raise RuntimeError("could not find go version in go.mod") + return m.group(1) + + +def _arch(): + return subprocess.check_output(["dpkg", "--print-architecture"], text=True).strip() + + +def _versions(): + v = { + "go": _go_version(), + "helm": HELM_VERSION, + "kubebuilder": _makefile_version("KUBEBUILDER_VERSION"), + "envtest_k8s": _makefile_version("ENVTEST_K8S_VERSION"), + "codespell": _makefile_version("CODESPELL_VERSION"), + } + for _, _, var in GO_TOOLS: + v[var] = _makefile_version(var) + return v + + +def _install(arch, versions): + print("Installing Go toolchain + CI tools (cache miss)...", flush=True) + # Go SDK + _sh(f'curl -fsSL "https://go.dev/dl/go{versions["go"]}.linux-{arch}.tar.gz" -o /tmp/go.tgz') + _sh("rm -rf /usr/local/go && tar -C /usr/local -xzf /tmp/go.tgz && rm -f /tmp/go.tgz") + _sh(f"ln -sf /usr/local/go/bin/go {GOBIN}/go && ln -sf /usr/local/go/bin/gofmt {GOBIN}/gofmt") + _sh("go version") + # helm + _sh(f'curl -fsSL "https://get.helm.sh/helm-{versions["helm"]}-linux-{arch}.tar.gz" -o /tmp/helm.tgz') + _sh(f"tar -xzf /tmp/helm.tgz -C /tmp && install -m 0755 /tmp/linux-{arch}/helm {GOBIN}/helm && rm -rf /tmp/helm.tgz /tmp/linux-{arch}") + # kubebuilder + _sh(f'curl -fsSL "https://github.com/kubernetes-sigs/kubebuilder/releases/download/{versions["kubebuilder"]}/kubebuilder_linux_{arch}" -o {GOBIN}/kubebuilder && chmod +x {GOBIN}/kubebuilder') + # go-installed CLIs (also warms the module + build cache) + for _, pkg, var in GO_TOOLS: + _sh(f"go install {pkg}@{versions[var]}") + # warm the repo's module cache + _sh("go mod download") + # envtest K8s assets + _sh(f"mkdir -p {ENVTEST_ASSETS_DIR} && setup-envtest use {versions['envtest_k8s']} --bin-dir {ENVTEST_ASSETS_DIR} -p path") + # codespell wheel into the pip cache (Makefile reinstalls it with --target) + _sh(f"python3 -m pip install --break-system-packages codespell=={versions['codespell']}") + + +NAMESPACE = "go-env" +# Bump when the bundle's on-disk LAYOUT changes (paths in BUNDLE_PATHS / cache +# locations) so a layout change invalidates the key and the prep job rebuilds +# instead of restoring a stale-layout bundle under the same version key. +BUNDLE_VERSION = "2" + + +def _key_and_cache(): + arch = _arch() + versions = _versions() + key = S3PathCache.key_from( + [f"bundle={BUNDLE_VERSION}", arch] + + [f"{k}={v}" for k, v in sorted(versions.items())] + ) + print(f"go-env cache key: {key} (arch={arch}, bundle=v{BUNDLE_VERSION}, versions={versions})") + cache = None + try: + bucket, _, prefix = CACHE_S3_PATH.partition("/") + cache = S3PathCache(bucket=bucket, prefix=prefix, region=AWS_REGION) + except Exception as e: + print(f"WARNING: S3 cache unavailable ({e})") + return arch, versions, key, cache + + +def ensure(): + """Workflow pre-hook: populate the S3 bundle once (in the Config job) if it is + missing, so the per-job pre-hooks only download instead of each rebuilding on a + cache miss. Best-effort — on any failure the jobs still self-provision.""" + arch, versions, key, cache = _key_and_cache() + if not cache: + print("No S3 cache; jobs will self-provision") + return 0 + try: + if cache.exists(key, namespace=NAMESPACE): + print("go-env cache already present; nothing to prepare") + return 0 + print("go-env cache miss; building bundle once for the run") + _install(arch, versions) + cache.save(key, BUNDLE_PATHS, namespace=NAMESPACE) + except Exception as e: + print(f"WARNING: could not prepare go-env cache ({e}); jobs will self-provision") + return 0 + + +def setup(): + """Per-job pre-hook: restore the bundle; on a miss (e.g. the ensure step did + not run or a different arch) build it and save it so the run self-heals.""" + arch, versions, key, cache = _key_and_cache() + restored = False + if cache: + try: + restored = cache.restore(key, namespace=NAMESPACE) + except Exception as e: + print(f"WARNING: cache restore failed ({e}); installing fresh") + if not restored: + _install(arch, versions) + if cache: + try: + cache.save(key, BUNDLE_PATHS, namespace=NAMESPACE) + except Exception as e: + print(f"WARNING: cache save failed ({e}); continuing") + _sh("go version && helm version && kubebuilder version && controller-gen --version") + return 0 + + +def _make_tarball(outpath): + rel = [os.path.relpath(p, "/") for p in BUNDLE_PATHS if Path(p).exists()] + Path(outpath).parent.mkdir(parents=True, exist_ok=True) + _sh(f"tar -C / -cf - {' '.join(rel)} | zstd -c -T0 > {outpath}") + + +def prepare(outpath): + """Prep-job entry point: produce the go-env bundle tarball at ``outpath`` (to + be published as a praktika artifact). Restores it from S3 if cached, otherwise + builds the toolchain, tars it, and uploads it to S3. One prep job per arch.""" + arch, versions, key, cache = _key_and_cache() + if cache: + try: + if cache.download(key, NAMESPACE, outpath): + print(f"go-env bundle restored from cache -> {outpath}") + return 0 + except Exception as e: + print(f"WARNING: cache download failed ({e}); building") + _install(arch, versions) + _make_tarball(outpath) + if cache: + try: + cache.upload(key, NAMESPACE, outpath) + except Exception as e: + print(f"WARNING: cache upload failed ({e}); continuing") + return 0 + + +def install(): + """Consumer-job pre-hook: extract the go-env bundle that `requires` placed in + the input dir. Falls back to self-provisioning (restore-or-build) if the + artifact is missing, so the job still works off the happy path.""" + tarballs = sorted(glob.glob(os.path.join(PraktikaSettings.INPUT_DIR, ARTIFACT_GLOB))) + if not tarballs: + print("No go-env artifact in input dir; self-provisioning") + return setup() + tb = tarballs[0] + print(f"Installing go-env from artifact {tb}") + # Run with the job's ambient env (its real $HOME), so `go env -w` writes to the + # config the job command will read, and the bundled GO*CACHE aren't shadowed by + # an OS-env override. + _sh(f"zstd -d {tb} --stdout | tar -xf - -C /", env=_PATH_ENV) + _sh( + f"go env -w GOMODCACHE={GO_MODCACHE} GOCACHE={GO_BUILDCACHE} GOPATH={GOPATH_DIR}", + env=_PATH_ENV, + ) + _sh( + "go version && helm version && kubebuilder version && controller-gen --version", + env=_PATH_ENV, + ) + _sh("go env GOMODCACHE GOCACHE", env=_PATH_ENV) + return 0 + + +def main(): + mode = sys.argv[1] if len(sys.argv) > 1 else "setup" + if mode == "prepare": + return prepare(sys.argv[2]) + if mode == "install": + return install() + if mode == "ensure": + return ensure() + return setup() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/jobs/junit_result.py b/ci/jobs/junit_result.py new file mode 100644 index 000000000..767b3dc23 --- /dev/null +++ b/ci/jobs/junit_result.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""JUnit XML -> praktika Result translator (for Ginkgo / `go test` output). + +praktika ships gtest and pytest translators (Result.from_gtest_run / +from_pytest_run) but no JUnit one. Our Go suites run under Ginkgo, which emits +JUnit XML via `--ginkgo.junit-report`. This builds a praktika Result whose +sub-results are the individual test cases, so the report page renders a per-spec +table (failures first) instead of a single pass/fail synthesized from the job +exit code — the replacement for the dropped dorny/test-reporter step. + +Reusable across every JUnit-producing job (compat-e2e, e2e, unit tests). + +JUnit shape consumed (Ginkgo's output): + + + + + ...body... # or / + + + +""" +import glob +import xml.etree.ElementTree as ET + +from praktika.result import Result + +# JUnit child tag -> praktika status. A carries at most one of these; +# its absence means the case passed. +_TAG_STATUS = ( + ("failure", Result.Status.FAIL), + ("error", Result.Status.ERROR), + ("skipped", Result.Status.SKIPPED), +) + + +def _case_status(testcase): + for tag, status in _TAG_STATUS: + if testcase.find(tag) is not None: + return status + return Result.Status.OK + + +def _case_info(testcase): + """Failure/error/skip message + body, for Result.info.""" + for tag, _ in _TAG_STATUS: + el = testcase.find(tag) + if el is not None: + msg = (el.get("message") or "").strip() + body = (el.text or "").strip() + return "\n".join(p for p in (msg, body) if p) + return "" + + +def _case_duration(testcase): + try: + return float(testcase.get("time") or 0) + except ValueError: + return None + + +def _parse_file(path): + """Parse one JUnit file into a list of per-testcase Results.""" + results = [] + root = ET.parse(path).getroot() + # root may be (wrapping many) or a lone ; iter() + # yields the element itself too, so both shapes are covered. + for suite in root.iter("testsuite"): + suite_name = suite.get("name", "") + for tc in suite.findall("testcase"): + # Ginkgo puts the spec text in name and the suite/package in + # classname; join them for a readable, stable row label. + prefix = tc.get("classname", "") or suite_name + name = tc.get("name", "") + full = " :: ".join(p for p in (prefix, name) if p) or name or "(unnamed)" + results.append( + Result( + name=full, + status=_case_status(tc), + duration=_case_duration(tc), + info=_case_info(tc), + ) + ) + return results + + +def results_from_junit(report_glob): + """Parse every JUnit file matching `report_glob` (recursive) into a flat list + of per-testcase Results. Returns (results, matched_paths, error_str).""" + paths = sorted(glob.glob(report_glob, recursive=True)) + if not paths: + return [], [], f"no JUnit report matched [{report_glob}]" + results = [] + for p in paths: + try: + results.extend(_parse_file(p)) + except Exception as e: # malformed XML / truncated file + return results, paths, f"failed to parse [{p}]: {e}" + return results, paths, "" + + +def build_job_result(report_glob, run_exit_code, stopwatch, name="", extra_files=None): + """Assemble a job-level Result from JUnit reports, reconciled with the test + process exit code. + + - No report / no cases (process killed before writing, or glob miss) -> ERROR. + - Cases parsed -> status derived from the cases (any FAIL -> FAIL, etc.). + - All cases green but the process exited non-zero (panic or teardown failure + after the specs finished) -> force FAIL; a green suite must not mask it. + (Mirrors Result.from_gtest_run's binary-failed reconciliation.) + + `name` defaults to the praktika JOB_NAME (Result.create_from fills it in), + which must match so the runner's `Result.from_fs(job.name)` finds this file. + """ + results, paths, err = results_from_junit(report_glob) + base_files = list(extra_files or []) + if not results: + return Result.create_from( + name=name, + status=Result.Status.ERROR, + stopwatch=stopwatch, + info=err or "JUnit report contained no test cases", + files=base_files + paths, + ) + result = Result.create_from( + name=name, + results=results, + stopwatch=stopwatch, + files=base_files + paths, + info=err, # non-fatal note if some (but not all) files failed to parse + ) + if run_exit_code != 0 and result.is_ok(): + result.set_status(Result.Status.FAIL) + result.set_info( + f"test process exited with code [{run_exit_code}] despite all parsed cases passing" + ) + return result diff --git a/ci/jobs/kind_env.py b/ci/jobs/kind_env.py new file mode 100644 index 000000000..f892c494f --- /dev/null +++ b/ci/jobs/kind_env.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Shared Docker + Kind provisioning for the praktika Docker/Kind jobs +(compat-e2e, e2e). + +Reproduces, in plain shell, what the GitHub Actions jobs did via marketplace +actions (helm/kind-action + the pre-pull step) so the praktika jobs are +self-contained. Kind + kubectl are installed here (not baked into the go-env +bundle) because only the Docker/Kind jobs need them; Docker itself must be +provided by the runner image. +""" +import json +import os +import subprocess +import sys +import time + +# Pinned to match ci.yaml's `env.kind-version`. +KIND_VERSION = "v0.32.0" +# kubectl is not strictly required by the Go test harness (it uses client-go), +# but several deploy/diagnostic helpers shell out to it; install a recent build. +KUBECTL_VERSION = "v1.31.1" +INSTALL_DIR = "/usr/local/bin" +KIND_CONFIG = "ci/kind-cluster.config" +DEFAULT_CLUSTER = "kind" +CLICKHOUSE_IMAGES = ( + "clickhouse/clickhouse-keeper", + "clickhouse/clickhouse-server", +) + + +def sh(cmd, check=True, **kwargs): + print(f"+ {cmd}", flush=True) + return subprocess.run(cmd, shell=True, check=check, **kwargs) + + +def arch(): + # kind/kubectl release assets use the Go arch tokens amd64 / arm64, which is + # exactly what `dpkg --print-architecture` returns on Ubuntu. + return subprocess.check_output(["dpkg", "--print-architecture"], text=True).strip() + + +def require_docker(): + try: + sh("docker version") + except subprocess.CalledProcessError: + print( + "ERROR: Docker daemon not available on this runner. Kind-in-Docker " + "needs a working Docker daemon provided by the runner image.", + file=sys.stderr, + ) + raise + + +def _docker_driver(): + return subprocess.check_output( + ["docker", "info", "-f", "{{.Driver}}"], text=True + ).strip() + + +def _restart_docker(): + for cmd in ("systemctl restart docker", "service docker restart"): + try: + sh(cmd) + break + except subprocess.CalledProcessError: + continue + else: + raise RuntimeError("could not restart the Docker daemon") + # Wait for the daemon to come back before anything else touches it. + for _ in range(30): + if subprocess.run("docker version", shell=True).returncode == 0: + return + time.sleep(2) + raise RuntimeError("Docker daemon did not come back after restart") + + +def disable_containerd_image_store(): + """Transitional fallback for `kind load docker-image`. + + The authoritative fix lives in the runner AMI (ci/infrastructure/projects.py, + `_docker_kind_component`, recipe >= 1.0.6): it bakes + `features.containerd-snapshotter = false` into /etc/docker/daemon.json. Until + that AMI is built and the pools are rolled, runners still boot with the + containerd image store, where `kind load` exports via `docker save` and + re-imports with `ctr ... --all-platforms --digests` — which fails on the other + platforms' / attestation blobs ("content digest ... not found") because a + single-arch pull keeps a multi-platform index. + + So on such a runner (Driver=overlayfs) we disable the snapshotter and restart + Docker here. Once the baked AMI is in use the driver is the classic store and + this is a no-op (early return, no restart), so it can be deleted after the roll. + """ + if _docker_driver() != "overlayfs": + return # classic image store already in use (e.g. baked AMI); nothing to do + print("Docker containerd image store detected; disabling for kind load", flush=True) + path = "/etc/docker/daemon.json" + try: + with open(path) as f: + cfg = json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + cfg = {} + cfg.setdefault("features", {})["containerd-snapshotter"] = False + os.makedirs("/etc/docker", exist_ok=True) + with open(path, "w") as f: + json.dump(cfg, f) + _restart_docker() + + +def install_kind(a=None): + a = a or arch() + url = f"https://github.com/kubernetes-sigs/kind/releases/download/{KIND_VERSION}/kind-linux-{a}" + sh(f'curl -fsSL "{url}" -o {INSTALL_DIR}/kind && chmod +x {INSTALL_DIR}/kind') + sh("kind version") + + +def install_kubectl(a=None): + a = a or arch() + url = f"https://dl.k8s.io/release/{KUBECTL_VERSION}/bin/linux/{a}/kubectl" + sh(f'curl -fsSL "{url}" -o {INSTALL_DIR}/kubectl && chmod +x {INSTALL_DIR}/kubectl') + sh("kubectl version --client") + + +def create_cluster(node_image=None, name=DEFAULT_CLUSTER, config=KIND_CONFIG): + """Create a Kind cluster. node_image=None uses Kind's default image for the + installed version; config=None creates a plain single-node cluster (no + ci/kind-cluster.config topology).""" + # Tear down any leftover cluster of the same name first: autoscaled runners + # are reused, so a prior job's cluster can still be present, and `kind create` + # refuses to clobber it ("node(s) already exist"). `kind delete` is idempotent. + sh(f"kind delete cluster --name {name}", check=False) + # `kind delete` can leave the node containers behind when they are only + # stopped (e.g. after disable_containerd_image_store() restarted the Docker + # daemon), which still trips the "already exist" check. Force-remove any + # container carrying this cluster's kind label as a backstop. + sh( + f"ids=$(docker ps -aq --filter label=io.x-k8s.kind.cluster={name}); " + f'[ -n "$ids" ] && docker rm -f $ids || true', + check=False, + ) + cmd = f"kind create cluster --name {name} --wait 120s" + if node_image: + cmd += f" --image kindest/node:{node_image}" + if config: + cmd += f" --config {config}" + sh(cmd) + + +def prepull(versions, cluster=DEFAULT_CLUSTER): + """Pull each ClickHouse image (comma-separated versions) and load it into the + Kind cluster, so the test pods don't pull from docker.io in-cluster.""" + for version in (v.strip() for v in versions.split(",") if v.strip()): + for image in CLICKHOUSE_IMAGES: + ref = f"docker.io/{image}:{version}" + # Retry: the public registry occasionally rate-limits / flakes. + sh( + f"for i in 1 2 3; do docker pull {ref} && break || sleep 15; done; " + f"docker image inspect {ref} >/dev/null" + ) + sh(f"kind load docker-image {ref} --name {cluster}") diff --git a/ci/jobs/lint.py b/ci/jobs/lint.py new file mode 100644 index 000000000..ba70c3810 --- /dev/null +++ b/ci/jobs/lint.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +""" +ci.yaml :: lint, migrated to praktika. + +Reproduces the GitHub `lint` job as a driver that emits a structured praktika +Result instead of the bare pass/fail praktika would otherwise synthesize from the +shell exit code. The job is two concerns, each broken into sub-results so the +report page shows exactly what failed: + + 1. "regeneration is committed" gates — `go mod tidy`, `make generate`, + `make manifests` must not change any tracked file. Each is one sub-result; + on drift it carries the offending `git diff`. Tracked changes are reverted + between gates so one gate's drift never pollutes the next gate's diff. + + 2. `make lint`'s three linters, run individually so each is its own sub-result: + - golangci-lint — JSON output parsed into one sub-result per issue + (file:line [linter] -> message), so the report renders an issue table + instead of a single pass/fail (the replacement for reading raw logs). + - codespell + - actionlint + +The top-level Result aggregates (any sub-result FAIL -> job FAIL). The linter +binaries are installed into ./bin by the Makefile targets, served from the warm +Go module + pip caches provisioned by the go-env pre-hook (ci/jobs/go_env.py). +""" +import json +import os +import subprocess +import sys + +sys.path.insert(0, os.getcwd()) # ensure repo root is importable for ci.* + +from praktika.result import Result # noqa: E402 +from praktika.utils import Utils # noqa: E402 + +# Makefile installs the linters into ./bin ($(LOCALBIN)); we invoke them directly +# (instead of `make lint`) to capture each linter's output separately. +GOLANGCI = "bin/golangci-lint" +CODESPELL = "bin/codespell" +ACTIONLINT = "bin/actionlint" + + +def _run(cmd, env=None): + """Run a shell command, echoing it and its combined output. Returns (rc, output).""" + print(f"+ {cmd}", flush=True) + proc = subprocess.run(cmd, shell=True, text=True, capture_output=True, env=env) + output = (proc.stdout or "") + (proc.stderr or "") + if output: + print(output, end="", flush=True) + return proc.returncode, output + + +def _git_restore(): + """Revert tracked-file modifications (best-effort), so each regeneration gate + starts from the committed tree. Untracked files are left alone.""" + _run("git checkout -- .") + + +def _ensure_tools(): + """Install the lint tool binaries into ./bin via the Makefile. Cheap thanks to + the warm Go module + pip caches from the go-env pre-hook. Raises on failure.""" + rc, out = _run("make golangci-lint codespell actionlint") + if rc != 0: + raise RuntimeError(f"failed to install lint tools (exit {rc}):\n{out.strip()}") + + +def _diff_gate(name, cmd): + """Run a regeneration command, then fail if it dirtied any tracked file — + mirroring ci.yaml's ` && git diff --exit-code`.""" + sw = Utils.Stopwatch() + rc, out = _run(cmd) + if rc != 0: + _git_restore() + return Result.create_from( + name=name, status=Result.Status.FAIL, stopwatch=sw, + info=f"`{cmd}` failed (exit {rc}):\n{out.strip()}", + ) + diff = _run("git diff")[1] + _git_restore() + if diff.strip(): + return Result.create_from( + name=name, status=Result.Status.FAIL, stopwatch=sw, + info=f"`{cmd}` is not committed — it changed tracked files:\n{diff.strip()}", + ) + return Result.create_from(name=name, status=Result.Status.OK, stopwatch=sw) + + +def _golangci_result(): + """Run golangci-lint with JSON output; one leaf result whose info lists every + reported issue (lint findings aren't independent test cases, so a flat list + reads better than a nested table).""" + sw = Utils.Stopwatch() + # json -> file (parsed here), text -> stdout (human-readable in the job log). + # They must not share a stream: a single stdout would interleave the JSON + # object with the text summary and break json.load. + # Written under ci/tmp (git-ignored scratch) and skipped by codespell via the + # golangci-report.json entry in ci/.codespellrc, so the later codespell pass + # does not scan golangci's linter names and flag them as typos. + report = "ci/tmp/golangci-report.json" + os.makedirs("ci/tmp", exist_ok=True) + rc, out = _run(f"{GOLANGCI} run --output.text.path=stdout --output.json.path={report}") + + try: + with open(report) as f: + issues = (json.load(f) or {}).get("Issues") or [] + except (OSError, json.JSONDecodeError) as e: + # No/invalid report -> golangci-lint itself failed (bad config, panic, OOM). + return Result.create_from( + name="golangci-lint", status=Result.Status.ERROR, stopwatch=sw, + info=f"could not read golangci-lint report (exit {rc}): {e}\n{out.strip()[:2000]}", + ) + + if not issues: + if rc == 0: + return Result.create_from(name="golangci-lint", status=Result.Status.OK, stopwatch=sw) + # Non-zero exit with no issues parsed -> a run error, not a lint finding. + return Result.create_from( + name="golangci-lint", status=Result.Status.ERROR, stopwatch=sw, + info=f"golangci-lint exited {rc} with no issues reported", + ) + + lines = [f"{len(issues)} issue(s):", ""] + for it in issues: + pos = it.get("Pos") or {} + loc = f"{pos.get('Filename', '?')}:{pos.get('Line', 0)}:{pos.get('Column', 0)}" + linter = it.get("FromLinter", "") + lines.append(f"{loc}: {(it.get('Text') or '').strip()} ({linter})") + return Result.create_from( + name="golangci-lint", status=Result.Status.FAIL, stopwatch=sw, + info="\n".join(lines), files=[report], + ) + + +def _cmd_result(name, cmd): + """A leaf sub-result: OK when `cmd` exits 0, else FAIL carrying its output.""" + sw = Utils.Stopwatch() + rc, out = _run(cmd) + if rc == 0: + return Result.create_from(name=name, status=Result.Status.OK, stopwatch=sw) + return Result.create_from( + name=name, status=Result.Status.FAIL, stopwatch=sw, info=out.strip(), + ) + + +def main(): + sw = Utils.Stopwatch() + try: + _ensure_tools() + except Exception as e: + Result.create_from( + status=Result.Status.ERROR, stopwatch=sw, info=str(e), + ).complete_job() + return + + results = [ + _diff_gate("go mod tidy", "go mod tidy"), + _diff_gate("make generate", "make generate"), + _diff_gate("make manifests", "make manifests"), + _golangci_result(), + _cmd_result("codespell", f"{CODESPELL} --config ci/.codespellrc"), + _cmd_result("actionlint", f"{ACTIONLINT} -config-file ci/actionlint.yaml"), + ] + # name/status omitted: create_from defaults name to JOB_NAME ("Lint") so the + # runner's Result.from_fs(job.name) finds this file, and derives the status by + # aggregating the sub-results (any FAIL/ERROR -> job fails). + Result.create_from(results=results, stopwatch=sw).complete_job() + + +if __name__ == "__main__": + main() diff --git a/ci/jobs/s3_cache.py b/ci/jobs/s3_cache.py new file mode 100644 index 000000000..e2372f103 --- /dev/null +++ b/ci/jobs/s3_cache.py @@ -0,0 +1,232 @@ +""" +Generic S3-backed filesystem cache (content-addressed tar + zstd bundles). + +Intentionally free of any project specifics so it can later move into praktika +as-is. A bundle is a set of paths archived together and keyed by a caller-chosen +string; `restore` extracts it, `save` creates it **atomically write-once**. Large +bundles use multipart transfers. + +Write-once is enforced server-side with an S3 conditional write (If-None-Match: +"*") on the finalizing call — PutObject for small bundles, CompleteMultipartUpload +for large ones. The first writer wins; a loser's conditional fails with +PreconditionFailed (treated as success — the entry is already populated), so +concurrent first-run builders can't clobber or interleave a half-written object. +This also makes entries immutable: an existing key is never overwritten. + +Example (restore-or-build-and-save): + + cache = S3PathCache(bucket="my-bucket", prefix="ci_cache", region="eu-north-1") + key = S3PathCache.key_from(["go1.27.0", "arm64", helm_ver]) + if not cache.restore(key, namespace="go-env"): + ...install the toolchain into the paths below... + cache.save(key, ["/usr/local/go", "/root/go"], namespace="go-env") +""" +import hashlib +import os +import subprocess +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import List + +import boto3 +from boto3.s3.transfer import TransferConfig +from botocore.exceptions import ClientError + +_MULTIPART_THRESHOLD = 16 * 1024 * 1024 +_MULTIPART_CHUNKSIZE = 32 * 1024 * 1024 +_MAX_CONCURRENCY = 8 + +# Multipart, parallel download. (Upload is driven manually in _upload_write_once +# so the finalizing call can carry the If-None-Match conditional, which the +# high-level upload_file/ExtraArgs path does not support.) +_TRANSFER = TransferConfig( + multipart_threshold=_MULTIPART_THRESHOLD, + multipart_chunksize=_MULTIPART_CHUNKSIZE, + max_concurrency=_MAX_CONCURRENCY, + use_threads=True, +) + + +def _is_precondition_failed(e: ClientError) -> bool: + code = e.response.get("Error", {}).get("Code", "") + status = e.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + return code in ("PreconditionFailed", "ConditionalRequestConflict") or status == 412 + + +class S3PathCache: + def __init__(self, bucket: str, prefix: str, region: str, tmp_dir: str = "./ci/tmp"): + self.bucket = bucket + self.prefix = prefix.strip("/") + self.tmp_dir = tmp_dir + self._client = boto3.client("s3", region_name=region) + + @staticmethod + def key_from(parts: List[str]) -> str: + """Short, stable cache key from the given parts.""" + joined = "\n".join(str(p) for p in parts) + return hashlib.sha256(joined.encode()).hexdigest()[:16] + + def _s3_key(self, key: str, namespace: str) -> str: + return f"{self.prefix}/{namespace}/{key}.tar.zst" + + def _uri(self, s3_key: str) -> str: + return f"s3://{self.bucket}/{s3_key}" + + def exists(self, key: str, namespace: str) -> bool: + s3_key = self._s3_key(key, namespace) + try: + self._client.head_object(Bucket=self.bucket, Key=s3_key) + return True + except ClientError as e: + if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey", "NotFound"): + return False + raise + + def restore(self, key: str, namespace: str, base: str = "/") -> bool: + """Download + extract the bundle under ``base``. True on hit, False on miss + or a failed/corrupt extraction.""" + s3_key = self._s3_key(key, namespace) + if not self.exists(key, namespace): + print(f"S3 cache miss: {self._uri(s3_key)}") + return False + Path(self.tmp_dir).mkdir(parents=True, exist_ok=True) + local = os.path.join(self.tmp_dir, f"cache_{key}.tar.zst") + print(f"S3 cache hit, downloading: {self._uri(s3_key)}") + self._client.download_file(self.bucket, s3_key, local, Config=_TRANSFER) + try: + subprocess.run( + f"zstd -d {local} --stdout | tar -xf - -C {base}", + shell=True, check=True, + ) + except subprocess.CalledProcessError as e: + print(f"WARNING: cache extraction failed: {e}") + return False + finally: + _rm(local) + return True + + def save(self, key: str, paths: List[str], namespace: str, base: str = "/") -> bool: + """Archive ``paths`` (relative to ``base``) and upload atomically + write-once. Existing entries are never overwritten; on a lost first-run + race the conditional upload fails cleanly (another writer populated it).""" + s3_key = self._s3_key(key, namespace) + # Fast path: skip building the tarball when already cached. The upload is + # still conditional, so this is an optimization, not the correctness gate. + if self.exists(key, namespace): + print(f"S3 cache already present, skip save: {self._uri(s3_key)}") + return True + + rel = [] + for p in paths: + if Path(p).exists(): + rel.append(os.path.relpath(p, base)) + else: + print(f"WARNING: skipping missing path [{p}]") + if not rel: + print("WARNING: nothing to cache") + return False + + Path(self.tmp_dir).mkdir(parents=True, exist_ok=True) + local = os.path.join(self.tmp_dir, f"cache_{key}.tar.zst") + subprocess.run( + f"tar -C {base} -cf - {' '.join(rel)} | zstd -c -T0 > {local}", + shell=True, check=True, + ) + try: + won = self._upload_write_once(local, s3_key) + print( + f"S3 cache {'saved' if won else 'already populated by another run'}: " + f"{self._uri(s3_key)}" + ) + finally: + _rm(local) + return True + + def download(self, key: str, namespace: str, dest: str) -> bool: + """Download the raw bundle object to ``dest`` (no extraction). Returns True + on a hit, False on a miss. Use when another layer (e.g. a praktika + artifact) distributes the tarball itself.""" + s3_key = self._s3_key(key, namespace) + if not self.exists(key, namespace): + print(f"S3 cache miss: {self._uri(s3_key)}") + return False + Path(dest).parent.mkdir(parents=True, exist_ok=True) + print(f"S3 cache hit, downloading: {self._uri(s3_key)}") + self._client.download_file(self.bucket, s3_key, dest, Config=_TRANSFER) + return True + + def upload(self, key: str, namespace: str, src: str) -> bool: + """Upload an existing bundle file ``src`` write-once (no tarring). No-op if + the entry already exists.""" + s3_key = self._s3_key(key, namespace) + if self.exists(key, namespace): + print(f"S3 cache already present, skip upload: {self._uri(s3_key)}") + return True + won = self._upload_write_once(src, s3_key) + print( + f"S3 cache {'saved' if won else 'already populated by another run'}: " + f"{self._uri(s3_key)}" + ) + return True + + def _upload_write_once(self, local: str, s3_key: str) -> bool: + """Upload with a server-side If-None-Match:* conditional on the finalizing + call. Returns True if this writer created the object, False if it lost the + race (object already exists). Raises on any other error.""" + size = os.path.getsize(local) + try: + if size < _MULTIPART_THRESHOLD: + with open(local, "rb") as f: + self._client.put_object( + Bucket=self.bucket, Key=s3_key, Body=f, IfNoneMatch="*" + ) + else: + self._multipart_upload_write_once(local, s3_key, size) + return True + except ClientError as e: + if _is_precondition_failed(e): + return False + raise + + def _multipart_upload_write_once(self, local: str, s3_key: str, size: int) -> None: + """Manual multipart upload whose CompleteMultipartUpload carries + If-None-Match:* (the high-level transfer API cannot). Aborts the upload on + any failure so no orphaned parts are left behind.""" + mpu = self._client.create_multipart_upload(Bucket=self.bucket, Key=s3_key) + upload_id = mpu["UploadId"] + try: + n_parts = (size + _MULTIPART_CHUNKSIZE - 1) // _MULTIPART_CHUNKSIZE + + def _upload_part(idx): + with open(local, "rb") as f: + f.seek(idx * _MULTIPART_CHUNKSIZE) + data = f.read(_MULTIPART_CHUNKSIZE) + resp = self._client.upload_part( + Bucket=self.bucket, Key=s3_key, UploadId=upload_id, + PartNumber=idx + 1, Body=data, + ) + return {"PartNumber": idx + 1, "ETag": resp["ETag"]} + + with ThreadPoolExecutor(max_workers=_MAX_CONCURRENCY) as ex: + parts = list(ex.map(_upload_part, range(n_parts))) + parts.sort(key=lambda p: p["PartNumber"]) + + self._client.complete_multipart_upload( + Bucket=self.bucket, Key=s3_key, UploadId=upload_id, + MultipartUpload={"Parts": parts}, IfNoneMatch="*", + ) + except Exception: + try: + self._client.abort_multipart_upload( + Bucket=self.bucket, Key=s3_key, UploadId=upload_id + ) + except ClientError: + pass + raise + + +def _rm(path: str) -> None: + try: + os.remove(path) + except OSError: + pass diff --git a/ci/prompts/code_review.md b/ci/prompts/code_review.md new file mode 100644 index 000000000..57089c99e --- /dev/null +++ b/ci/prompts/code_review.md @@ -0,0 +1,48 @@ +# clickhouse-operator code-review guidance + +Project-specific guidance for the `praktika review` job. This is appended to the +fixed review protocol; keep it focused on what a generic reviewer would not know +about this repository. + +## What this repo is +A Kubernetes operator for managing ClickHouse database clusters and ClickHouse +Keeper clusters, built with Kubebuilder and controller-runtime. It manages two +CRDs — `ClickHouseCluster` and `KeeperCluster` — both `v1alpha1` in the +`clickhouse.com` API group. The code is Go; reconcilers live under +`internal/controller/`, the API types under `api/v1alpha1/`. + +## What to prioritise +- **Reconciliation correctness**: reconcilers must be idempotent and converge. + Flag non-deterministic desired-state construction, missing requeue on + transient errors, swallowed errors, and status updates that don't reflect + actual cluster state. Watch for unbounded or hot-looping requeues. +- **StatefulSet / PVC / ordinal handling**: ClickHouse shards+replicas and + Keeper members are stateful. Flag changes that could reorder, renumber, or + orphan pods/PVCs, or that mutate immutable StatefulSet fields (forcing a + recreate that drops data). Scale-down must not silently delete persistent data. +- **Keeper quorum & safety**: adding/removing Keeper members and leadership + changes must preserve raft quorum. Flag graceful-shutdown / leadership-yield + ordering bugs and anything that could lose quorum during a rollout. +- **API / CRD compatibility**: `api/v1alpha1` is a published API. Flag + breaking changes to existing fields, defaults that change behaviour on + existing clusters, and validation that would reject previously valid specs. + Generated artifacts (deepcopy, CRDs, Helm chart, API-ref docs) must stay in + sync with the types — call out when a type change isn't regenerated. +- **Concurrency**: controller-runtime caches are shared; never mutate objects + returned from the client cache without a DeepCopy. Flag data races on + shared maps/fields in the manager. +- **Security/RBAC**: flag over-broad RBAC in generated roles, secrets logged or + put into env/args, and privilege-escalating pod security settings. + +## What to skip +- Formatting, import ordering, and lint nits — `make lint-fix` / other jobs + cover these. +- Speculative refactors unrelated to the diff. +- Generated files' contents line-by-line (`zz_generated.*`, CRD bases, chart + templates, `docs/reference/api-reference.mdx`) — instead verify they are + consistent with the source types, not whether their formatting is pretty. + +## Style +- Prefer a small number of high-signal findings over many low-value ones. +- When you flag something, say concretely how it fails (inputs -> wrong outcome), + e.g. "scaling replicas 3->2 here deletes the PVC for replica-2, losing its data". diff --git a/ci/settings/settings.py b/ci/settings/settings.py new file mode 100644 index 000000000..a0ad37fd2 --- /dev/null +++ b/ci/settings/settings.py @@ -0,0 +1,35 @@ +class RunnerLabels: + # Labels are arch-Ncpu-Mg: N vCPUs, M GiB of memory. + ARM_2CPU_4G = "arm-2cpu-4g" + AMD_2CPU_4G = "amd-2cpu-4g" + ARM_8CPU_16G = "arm-8cpu-16g" + AMD_8CPU_16G = "amd-8cpu-16g" + # arm-2cpu-4g plus a scoped bedrock:InvokeModel grant for the AI Code Review job. + ARM_2CPU_4G_BEDROCK = "arm-2cpu-4g-bedrock" + + +PROJECT_NAME = "clickhouse-operator" +PROJECT_SLUG = "clickhouseoperator" +MAIN_BRANCH = "main" + +CI_CONFIG_RUNS_ON = [RunnerLabels.ARM_2CPU_4G] + +AWS_REGION = "eu-north-1" +AWS_PROFILE = "Box" + +S3_ARTIFACT_BUCKET = f"{PROJECT_SLUG}-artifacts-{AWS_REGION}" +S3_REPORT_BUCKET = S3_ARTIFACT_BUCKET +CACHE_S3_PATH = f"{S3_ARTIFACT_BUCKET}/ci_cache" +ENABLE_S3_REPO_SNAPSHOT = True +ENABLE_PR_EPHEMERAL_MERGE_COMMIT = True + +S3_BUCKET_TO_HTTP_ENDPOINT = { + S3_REPORT_BUCKET: f"{S3_REPORT_BUCKET}.s3.amazonaws.com", +} + +VALIDATE_FILE_PATHS = False + +GH_AUTH_LAMBDA_NAME = f"{PROJECT_SLUG}-gh-token" +GH_AUTH_LAMBDA_REGION = AWS_REGION +PRAKTIKA_BASE_VENV = "praktika-runtime-0.0.1" + diff --git a/ci/workflows/job_configs.py b/ci/workflows/job_configs.py new file mode 100644 index 000000000..2a258709e --- /dev/null +++ b/ci/workflows/job_configs.py @@ -0,0 +1,481 @@ +""" +Central registry of reusable Job.Config definitions. + +Workflows import this class and reference the jobs they need, e.g.: + + from ci.workflows.job_configs import JobConfigs + ... + jobs=[JobConfigs.vale_linter, JobConfigs.doc_links] + +Keeping every Job.Config in one place avoids duplicating job definitions across +workflow files. praktika's parser reads job attributes into a separate +per-workflow config and never mutates these instances, so the same Job.Config +can be shared by multiple workflows safely. + +This module intentionally exposes no WORKFLOWS, so praktika's workflow scan +skips it. +""" +import json +from pathlib import Path + +from praktika import Artifact, Job +from ci.settings.settings import RunnerLabels + + +def _clickhouse_versions(): + """Latest + supported ClickHouse versions from the repo's single source of + truth (test/supported/versions.json), mirroring ci.yaml's jq: keep the + highest patch per major.minor, sorted descending. Returns (latest, supported) + where `supported` is a comma-joined list (latest first).""" + path = Path(__file__).resolve().parents[2] / "test" / "supported" / "versions.json" + data = json.loads(path.read_text()) + best = {} # (major, minor) -> (numeric-parts tuple, version string) + for version in (v for group in data.values() for v in group): + parts = tuple(int(x) for x in version.split(".")) + mm = parts[:2] + if mm not in best or parts > best[mm][0]: + best[mm] = (parts, version) + ordered = [v for _, v in sorted(best.values(), reverse=True)] + return ordered[0], ",".join(ordered) + + +_CH_LATEST, _CH_SUPPORTED = _clickhouse_versions() + +# Consumer pre-hook: extract the go-env bundle delivered as a required artifact +# (falls back to self-provisioning from S3/build). The bundle holds the Go SDK + +# helm/kubebuilder/controller-gen/kustomize/golangci-lint/actionlint/ +# crd-schema-checker/crd-ref-docs + warm Go/pip caches + envtest assets, so no Go +# tooling is baked into the AMI. See ci/jobs/go_env.py. +_GO_ENV_INSTALL = "python3 ci/jobs/go_env.py install" + +# Per-arch go-env bundle, built once per run by the Prepare Go Env jobs and +# consumed (via `requires`) by every Go job of that arch. +GO_ENV_ARM_ARTIFACT = Artifact.Config( + name="go-env-arm", type=Artifact.Type.S3, path="ci/tmp/go-env-arm.tar.zst" +) +GO_ENV_AMD_ARTIFACT = Artifact.Config( + name="go-env-amd", type=Artifact.Type.S3, path="ci/tmp/go-env-amd.tar.zst" +) + + +# ci.yaml :: compat-e2e-test matrix (5 variants). Each entry maps to the three +# env vars ci/jobs/compat_e2e.py consumes; the first field is the GitHub matrix +# `name`, kept as the praktika job-name suffix via Job.ParamSet(parameter=...). +# fetch_tags mirrors ci.yaml's checkout `fetch-tags: true` — only the upgrade +# variant needs release tags (it deploys the latest release, then upgrades). +# ClickHouse versions come from test/supported/versions.json (single source of +# truth, auto-updated) — see _clickhouse_versions(), matching ci.yaml #355. +_COMPAT_E2E_MATRIX = [ + # (name, k8s node image, clickhouse version(s), make target, fetch_tags) + ("minimal-k8s-all-deploy-methods", "v1.28.15", _CH_LATEST, "test-compat-e2e", False), + ("maximal-k8s-all-deploy-methods", "v1.36.1", _CH_LATEST, "test-compat-e2e", False), + ("olm-deploy-method", "v1.28.15", _CH_LATEST, "test-compat-e2e-olm", False), + ( + "supported-clickhouse-compatibility", + "v1.30.13", + f"{_CH_SUPPORTED},{_CH_LATEST}-distroless", + "test-compat-e2e-manifest", + False, + ), + ("operator-upgrade", "v1.30.13", _CH_LATEST, "test-compat-e2e-upgrade", True), +] + + +def _compat_e2e_command(k8s_image, clickhouse_version, deploy_target, fetch_tags): + cmd = ( + f"K8S_IMAGE={k8s_image} " + f"CLICKHOUSE_VERSION={clickhouse_version} " + f"DEPLOY_TARGET={deploy_target} " + ) + if fetch_tags: + cmd += "FETCH_TAGS=1 " + return cmd + "python3 ci/jobs/compat_e2e.py" + + +# ci.yaml :: e2e-test — number of shards. Each shard runs the full e2e suite +# filtered to its slice; sharding is round-robin by spec-name hash in the Go +# harness, so no e2e-shard-plan job / cross-run timings artifact is needed +# (ci.yaml's plan-based mode still exists for GitHub Actions — see sharding.go). +_E2E_SHARD_TOTAL = 4 +# e2e Kind node image, matching ci.yaml's pin. +_E2E_K8S_IMAGE = "v1.34.3" + + +def _e2e_command(shard_index): + return ( + f"E2E_SHARD={shard_index}/{_E2E_SHARD_TOTAL} " + f"K8S_IMAGE={_E2E_K8S_IMAGE} " + "python3 ci/jobs/e2e.py" + ) + + +class JobConfigs: + # Prep jobs rebuild the bundle only when the tool version sources or the + # provisioning scripts change (praktika job cache + the go-env S3 cache both + # key off this). Each builds natively on its arch and publishes the tarball. + _GO_ENV_DIGEST = Job.CacheDigestConfig( + include_paths=[ + "./go.mod", + "./go.sum", + "./Makefile", + "./ci/jobs/go_env.py", + "./ci/jobs/s3_cache.py", + ], + ) + + prepare_go_env_arm = Job.Config( + name="Prepare Go Env (arm)", + runs_on=[RunnerLabels.ARM_2CPU_4G], + command=f"python3 ci/jobs/go_env.py prepare {GO_ENV_ARM_ARTIFACT.path}", + provides=[GO_ENV_ARM_ARTIFACT.name], + timeout=30 * 60, + digest_config=_GO_ENV_DIGEST, + ) + prepare_go_env_amd = Job.Config( + name="Prepare Go Env (amd)", + runs_on=[RunnerLabels.AMD_2CPU_4G], + command=f"python3 ci/jobs/go_env.py prepare {GO_ENV_AMD_ARTIFACT.path}", + provides=[GO_ENV_AMD_ARTIFACT.name], + timeout=30 * 60, + digest_config=_GO_ENV_DIGEST, + ) + # dependabot-regenerate.yaml, migrated. Re-derives the generated artifacts + # after a Dependabot go.mod/go.sum bump and pushes them onto the PR branch. + # Applicability (actor is dependabot[bot], same-repo, go.mod/go.sum touched) + # is enforced by the workflow filter hook (ci/jobs/filter_job_hook.py), so + # this job is listed in every PR run but only executes for a bot bump. + # Needs the full Go toolchain (controller-gen/kubebuilder/crd-ref-docs) from + # the go-env pre-hook, hence `requires` the arm bundle like the other Go jobs. + # enable_gh_auth so it can mint a contents:write token to push; allow_failure + # so a regenerate/push hiccup never blocks merge — Lint still independently + # fails on any stale generated file, so correctness stays guarded. The digest + # keys on go.mod/go.sum so an unrelated PR that somehow reaches here (it + # won't, the hook gates it) is change-filtered out too. + dependabot_regenerate = Job.Config( + name="Dependabot Regenerate", + runs_on=[RunnerLabels.ARM_2CPU_4G], + command="python3 ci/jobs/dependabot_regenerate.py", + timeout=20 * 60, + allow_failure=True, + enable_gh_auth=True, + pre_hooks=[_GO_ENV_INSTALL], + requires=[GO_ENV_ARM_ARTIFACT.name], + digest_config=Job.CacheDigestConfig( + include_paths=["./go.mod", "./go.sum"], + ), + ) + + # --- Documentation lint (migrated from .github/workflows/docs-lint.yaml) --- + # The lint toolchain (Vale, Node + linkspector, Go, pre-warmed crd-ref-docs) + # is baked into the runner image by ci/infrastructure/projects.py + # (_doc_lint_tools_component), so these jobs just run the Makefile targets. + + # docs-lint.yaml :: vale (vale-linter) + vale_linter = Job.Config( + name="Vale Linter", + runs_on=[RunnerLabels.ARM_2CPU_4G], + command="make docs-lint-vale", + timeout=10 * 60, + digest_config=Job.CacheDigestConfig( + include_paths=["./docs", "./*.md", "./.vale.ini"], + ), + ) + + # docs-lint.yaml :: doc-links (Doc links) + doc_links = Job.Config( + name="Doc Links", + runs_on=[RunnerLabels.ARM_2CPU_4G], + command="make docs-link-check", + timeout=10 * 60, + digest_config=Job.CacheDigestConfig( + include_paths=["./docs", "./.linkspector.yml"], + ), + ) + + # docs-lint.yaml :: api-reference-generated (API Reference Generated) + # Needs Go + crd-ref-docs, provisioned by the go-env pre-hook. + api_reference_generated = Job.Config( + name="API Reference Generated", + runs_on=[RunnerLabels.ARM_2CPU_4G], + command="make docs-generate-api-ref && git diff --exit-code docs/", + timeout=15 * 60, + pre_hooks=[_GO_ENV_INSTALL], + requires=[GO_ENV_ARM_ARTIFACT.name], + digest_config=Job.CacheDigestConfig( + include_paths=[ + "./api/v1alpha1", + "./docs/templates", + "./docs/reference", + "./Makefile", + "./go.mod", + ], + ), + ) + + # --- Operator CI (migrated job-by-job from .github/workflows/ci.yaml) --- + # Go source paths that gate the Go jobs below — the praktika equivalent of + # ci.yaml's `changes` non-docs paths-filter. Shared (read-only) across jobs. + _GO_CODE_DIGEST = Job.CacheDigestConfig( + include_paths=[ + "./api", + "./cmd", + "./internal", + "./test", + "./tools", + "./config", + "./hack", + "./go.mod", + "./go.sum", + "./Makefile", + "./.golangci.yml", + ], + ) + + # ci.yaml :: build_and_test. Go is baked into the runner image; controller-gen + # and setup-envtest self-install via `go-install-tool`, and envtest downloads + # the kubebuilder assets (K8s 1.36.2) — all in-process, no Docker/cluster. + # Runs on a medium runner because `go test -race` across the suite is heavier + # than the small pool's 4 GB. The dorny/test-reporter step is dropped; the + # job's exit code drives pass/fail (enable_exit_code_result). + build_and_test = Job.Config( + name="Build and Unit Tests", + runs_on=[RunnerLabels.ARM_8CPU_16G], + command=( + # The go-env pre-hook caches the envtest K8s assets at + # /opt/ci-go/envtest. Seed ./bin/k8s from there so `make test-ci`'s + # `setup-envtest use` is an offline hit; absent (e.g. local) it just + # downloads as usual. Works both ways. + "if [ -d /opt/ci-go/envtest/k8s ]; then mkdir -p bin && cp -rn /opt/ci-go/envtest/k8s bin/; fi && " + "go build -v cmd/main.go && make test-ci" + ), + timeout=25 * 60, + pre_hooks=[_GO_ENV_INSTALL], + requires=[GO_ENV_ARM_ARTIFACT.name], + digest_config=_GO_CODE_DIGEST, + ) + + # ci.yaml :: fuzz_specs. Go-only (two 60s fuzz runs). + fuzz_specs = Job.Config( + name="Fuzz Specs", + runs_on=[RunnerLabels.ARM_2CPU_4G], + command="make fuzz", + timeout=20 * 60, + pre_hooks=[_GO_ENV_INSTALL], + requires=[GO_ENV_ARM_ARTIFACT.name], + digest_config=_GO_CODE_DIGEST, + ) + + # ci.yaml :: lint. golangci-lint/codespell/actionlint are installed by the + # Makefile into ./bin using the warm Go/pip caches from the go-env pre-hook. + # The driver (ci/jobs/lint.py) runs the regeneration gates + the three linters + # as separate sub-results (golangci-lint issues become a per-issue table). + # Runs on a medium runner because golangci-lint over the whole module is + # memory-hungry. + lint = Job.Config( + name="Lint", + runs_on=[RunnerLabels.ARM_8CPU_16G], + command="python3 ci/jobs/lint.py", + timeout=15 * 60, + pre_hooks=[_GO_ENV_INSTALL], + requires=[GO_ENV_ARM_ARTIFACT.name], + digest_config=_GO_CODE_DIGEST, + ) + + # ci.yaml :: helm-test. helm + kubebuilder come from the go-env pre-hook (on + # PATH); KUBEBUILDER points the Makefile at the provisioned binary instead of + # re-downloading it. + helm_test = Job.Config( + name="Helm Test", + runs_on=[RunnerLabels.ARM_2CPU_4G], + command=( + "make generate-helmchart-ci KUBEBUILDER=/usr/local/bin/kubebuilder && " + "git diff --exit-code dist/chart/ dist/chart-cluster/ && " + "make build-helmchart-dependencies && " + "helm lint ./dist/chart && " + "make lint-cluster-chart" + ), + timeout=15 * 60, + pre_hooks=[_GO_ENV_INSTALL], + requires=[GO_ENV_ARM_ARTIFACT.name], + digest_config=Job.CacheDigestConfig( + include_paths=[ + "./api", + "./config", + "./dist/chart", + "./dist/chart-cluster", + "./tools/gen-cluster-chart", + "./go.mod", + "./go.sum", + "./Makefile", + ], + ), + ) + + # ci.yaml :: check-crd-compat. PR-only, advisory (allow_failure). The + # crd-breaking-change label gate is a workflow filter hook + # (ci/jobs/filter_job_hook.py); the helper fetches the base branch (praktika + # checks out an ephemeral merge commit with no base history) and runs the + # check. crd-schema-checker self-installs via go-install-tool (cache + # pre-warmed). + check_crd_compat = Job.Config( + name="Check CRD Compatibility", + runs_on=[RunnerLabels.ARM_2CPU_4G], + command="python3 ci/jobs/check_crd_compat.py", + timeout=15 * 60, + allow_failure=True, + # On ENABLE_S3_REPO_SNAPSHOT runs there is no `origin`, so the checker + # falls back to `gh api` for the base manifests — which needs auth, else + # it exits 4 and the job ERRORs instead of actually checking CRDs. + enable_gh_auth=True, + pre_hooks=[_GO_ENV_INSTALL], + requires=[GO_ENV_ARM_ARTIFACT.name], + digest_config=Job.CacheDigestConfig( + include_paths=[ + "./api", + "./config/crd", + "./ci/jobs/check_crd_compat.py", + "./go.mod", + "./go.sum", + "./Makefile", + ], + ), + ) + + # ci.yaml :: compat-e2e-test (SPIKE — one matrix variant of five). + # Proof-of-concept for the Docker+Kind path on praktika: the single + # `maximal-k8s-all-deploy-methods` variant (newest Kind node image, one + # ci.yaml :: compat-e2e-test — the full 5-variant matrix, fanned out with + # Job.parametrize (see _COMPAT_E2E_MATRIX / _compat_e2e_command above). Each + # variant is one Job.Config differing only in the K8S_IMAGE/CLICKHOUSE_VERSION/ + # DEPLOY_TARGET it passes to ci/jobs/compat_e2e.py. Runs on amd-8cpu-16g + # (c7a.4xlarge) to match the existing self-hosted e2e placement and to pull + # amd64 ClickHouse images, as the GitHub job does. Needs a working Docker + # daemon on the runner; compat_e2e.py installs Kind + kubectl, creates the + # cluster, pre-pulls images, runs the suite, and renders per-spec results. + # Go/helm come from the go-env artifact (amd). The base `command` is a + # placeholder — parametrize overrides it per variant. This attribute is a LIST + # of Job.Config; spread it into a workflow's jobs with `*JobConfigs...`. + compat_e2e_test = Job.Config( + name="Compat E2E", + runs_on=[RunnerLabels.AMD_8CPU_16G], + command=_compat_e2e_command(*_COMPAT_E2E_MATRIX[0][1:]), # overridden per variant + timeout=60 * 60, + # The operator-upgrade variant resolves the latest release tag via `gh api` + # (FETCH_TAGS=1) from a snapshot checkout with no `origin` remote; without + # pre-authenticated gh that call exits 4. parametrize/ParamSet can't set + # enable_gh_auth per variant, so it's set on the base for the whole matrix. + enable_gh_auth=True, + pre_hooks=[_GO_ENV_INSTALL], + requires=[GO_ENV_AMD_ARTIFACT.name], + digest_config=Job.CacheDigestConfig( + include_paths=[ + "./api", + "./cmd", + "./internal", + "./config", + "./test", + "./tools", + "./ci/kind-cluster.config", + "./ci/jobs/compat_e2e.py", + "./ci/jobs/junit_result.py", + "./go.mod", + "./go.sum", + "./Makefile", + ], + ), + ).parametrize( + *[ + Job.ParamSet( + parameter=name, + command=_compat_e2e_command(img, ver, tgt, fetch), + ) + for (name, img, ver, tgt, fetch) in _COMPAT_E2E_MATRIX + ] + ) + + # ci.yaml :: e2e-test — the full e2e suite, split into _E2E_SHARD_TOTAL shards + # fanned out with Job.parametrize. Each shard passes E2E_SHARD="n/total" to + # ci/jobs/e2e.py; the Go harness assigns specs round-robin by hash, so there + # is no e2e-shard-plan job. Same Docker+Kind runner (amd-8cpu-16g) and go-env + # (amd) as compat-e2e; e2e.py creates the cluster, runs `make test-e2e`, + # collects kind/kubectl diagnostics on failure, and renders per-spec results. + # Base `command` is a placeholder — parametrize overrides it per shard. This + # attribute is a LIST of Job.Config; spread it with `*JobConfigs...`. + e2e_test = Job.Config( + name="E2E Tests", + runs_on=[RunnerLabels.AMD_8CPU_16G], + command=_e2e_command(1), # overridden per shard + timeout=45 * 60, + pre_hooks=[_GO_ENV_INSTALL], + requires=[GO_ENV_AMD_ARTIFACT.name], + digest_config=Job.CacheDigestConfig( + include_paths=[ + "./api", + "./cmd", + "./internal", + "./config", + "./test", + "./ci/kind-cluster.config", + "./ci/jobs/e2e.py", + "./ci/jobs/kind_env.py", + "./ci/jobs/junit_result.py", + "./go.mod", + "./go.sum", + "./Makefile", + ], + ), + ).parametrize( + *[ + Job.ParamSet( + parameter=f"{shard}/{_E2E_SHARD_TOTAL}", + command=_e2e_command(shard), + ) + for shard in range(1, _E2E_SHARD_TOTAL + 1) + ] + ) + + # ci.yaml :: bundle. Builds the OLM bundle image and runs operator-sdk + # scorecard against a throwaway Kind cluster — ci/jobs/bundle.py. Same + # Docker+Kind runner (amd-8cpu-16g) and go-env (amd) as compat/e2e; the bundle + # image is manifest-only (arch-independent). operator-sdk self-installs via the + # Makefile targets. Verdict is the scorecard exit code. + bundle = Job.Config( + name="Bundle", + runs_on=[RunnerLabels.AMD_8CPU_16G], + command="VERSION=0.0.1 python3 ci/jobs/bundle.py", + timeout=30 * 60, + pre_hooks=[_GO_ENV_INSTALL], + requires=[GO_ENV_AMD_ARTIFACT.name], + digest_config=Job.CacheDigestConfig( + include_paths=[ + "./api", + "./cmd", + "./internal", + "./config", + "./bundle.Dockerfile", + "./ci/jobs/bundle.py", + "./ci/jobs/kind_env.py", + "./go.mod", + "./go.sum", + "./Makefile", + ], + ), + ) + + # --- AI code review --- + # `praktika review` consults an OpenAI model on Bedrock and posts a summary + # plus inline findings, managing its own review threads. It runs on the + # dedicated arm-2cpu-4g-bedrock pool (the only one granted bedrock:InvokeModel). + # allow_failure so a review hiccup never blocks merge; enable_gh_auth so the + # job can post comments and resolve threads. + code_review = Job.Config( + name="Code Review", + runs_on=[RunnerLabels.ARM_2CPU_4G_BEDROCK], + command=( + "python3 -I -m praktika review --provider bedrock-openai " + "--model global.openai.gpt-5.6-sol --reasoning-effort high " + "--prompt ./ci/prompts/code_review.md --fail-for-draft-pr" + ), + allow_failure=True, + enable_gh_auth=True, + ) diff --git a/ci/workflows/main_ci.py b/ci/workflows/main_ci.py new file mode 100644 index 000000000..6fc6a6444 --- /dev/null +++ b/ci/workflows/main_ci.py @@ -0,0 +1,38 @@ +from praktika import Workflow +from ci.workflows.job_configs import ( + JobConfigs, + GO_ENV_ARM_ARTIFACT, + GO_ENV_AMD_ARTIFACT, +) + + +WORKFLOWS = [ + Workflow.Config( + name="Main", + event=Workflow.Event.PUSH, + branches=["main", "ci/migrate-docs-lint-to-praktika"], + # Both per-arch go-env bundles: the arm Go/docs jobs and the amd + # Docker/Kind jobs (bundle, compat-e2e, e2e) each require their arch's. + artifacts=[GO_ENV_ARM_ARTIFACT, GO_ENV_AMD_ARTIFACT], + jobs=[ + # Build the per-arch Go toolchain bundles once; the Go jobs below + # `require` the matching one and extract it in their pre-hook. + JobConfigs.prepare_go_env_arm, + JobConfigs.prepare_go_env_amd, + JobConfigs.vale_linter, + JobConfigs.doc_links, + JobConfigs.api_reference_generated, + JobConfigs.build_and_test, + JobConfigs.fuzz_specs, + JobConfigs.lint, + JobConfigs.helm_test, + # Docker/Kind jobs (amd-8cpu-16g) — the last pieces moved off ci.yaml. + JobConfigs.bundle, + *JobConfigs.compat_e2e_test, + *JobConfigs.e2e_test, + ], + enable_cache=True, + enable_report=True, + enable_exit_code_result=True, + ) +] diff --git a/ci/workflows/pull_request.py b/ci/workflows/pull_request.py new file mode 100644 index 000000000..65a380426 --- /dev/null +++ b/ci/workflows/pull_request.py @@ -0,0 +1,57 @@ +from praktika import Workflow +from ci.workflows.job_configs import ( + JobConfigs, + GO_ENV_ARM_ARTIFACT, + GO_ENV_AMD_ARTIFACT, +) +from ci.jobs.filter_job_hook import should_skip_job + + +WORKFLOWS = [ + Workflow.Config( + name="PR", + event=Workflow.Event.PULL_REQUEST, + base_branches=["main"], + ai_orchestrator=Workflow.OrchestratorAI.Config( + enabled=True, + provider="bedrock-anthropic", + model="global.anthropic.claude-sonnet-5", + exclude_jobs=["Code Review"], + ), + enable_job_filtering_by_changes=True, + workflow_filter_hooks=[should_skip_job], + artifacts=[GO_ENV_ARM_ARTIFACT, GO_ENV_AMD_ARTIFACT], + jobs=[ + # Build the per-arch Go toolchain bundles once; the Go jobs below + # `require` the matching one and extract it in their pre-hook. + JobConfigs.prepare_go_env_arm, + JobConfigs.prepare_go_env_amd, + # Runs first (gated to Dependabot go.mod/go.sum bumps by the filter + # hook): regenerates derived files and pushes them to the PR branch + # before the rest of the suite evaluates the bot PR. + JobConfigs.dependabot_regenerate, + JobConfigs.vale_linter, + JobConfigs.doc_links, + JobConfigs.api_reference_generated, + JobConfigs.build_and_test, + JobConfigs.fuzz_specs, + JobConfigs.lint, + JobConfigs.helm_test, + JobConfigs.check_crd_compat, + JobConfigs.bundle, + *JobConfigs.compat_e2e_test, + *JobConfigs.e2e_test, + JobConfigs.code_review, + ], + pre_hooks=[ + # Post this run's report link into the PR description before the jobs + # start, so reviewers can jump from the PR straight to the live report. + "python3 ./ci/jobs/ci_links.py", + ], + enable_cache=True, + enable_report=True, + enable_gh_summary_comment=True, + enable_exit_code_result=True, + enable_merge_ready_status=True, + ) +] diff --git a/test/testutil/sharding.go b/test/testutil/sharding.go index 6b3fcc2a9..09a06e04b 100644 --- a/test/testutil/sharding.go +++ b/test/testutil/sharding.go @@ -5,27 +5,69 @@ import ( "encoding/json" "errors" "fmt" + "hash/fnv" "os" "path/filepath" + "strconv" + "strings" "github.com/sethvargo/go-envconfig" ) -// ShardingConfig assigns specs to CI shards from a pre-computed plan. +// ShardingConfig assigns specs to CI shards. Two modes are supported: +// +// - Round-robin (new): set E2E_SHARD to "index/total" (e.g. "2/4"). Specs are +// spread by a deterministic hash of the spec name, so every shard derives the +// same assignment independently and each spec runs on exactly one shard — no +// pre-computed plan. Takes precedence when E2E_SHARD is set. +// +// - Plan-based / "smart" (legacy): set E2E_SHARD_INDEX + E2E_SHARD_TOTAL + +// E2E_SHARD_PLAN (a JSON spec->shard map from the e2e-shard-planner). Used +// when E2E_SHARD is empty; this is what .github/workflows/ci.yaml drives. +// +// Duration-weighted sharding can later feed the plan-based mode (or replace the +// round-robin hash) using per-spec timings collected by main CI. type ShardingConfig struct { + // Round-robin mode: "index/total". Takes precedence when non-empty. + Shard string `env:"E2E_SHARD"` + + // Plan-based (legacy) mode. Index int `env:"E2E_SHARD_INDEX, default=0"` Total int `env:"E2E_SHARD_TOTAL, default=0"` PlanPath string `env:"E2E_SHARD_PLAN"` + roundRobin bool shardAssignments map[string]int } -// Load reads the sharding configuration from the environment and the plan file. +// Load reads the sharding configuration from the environment (and, in plan-based +// mode, the plan file). func (c *ShardingConfig) Load() error { if err := envconfig.Process(context.Background(), c); err != nil { return fmt.Errorf("load sharding config from env: %w", err) } + // Round-robin mode takes precedence over the legacy plan env vars. + if c.Shard != "" { + idx, total, ok := strings.Cut(c.Shard, "/") + index, indexErr := strconv.Atoi(idx) + totalN, totalErr := strconv.Atoi(total) + + if !ok || indexErr != nil || totalErr != nil { + return fmt.Errorf("invalid E2E_SHARD %q, want \"index/total\" e.g. \"2/4\"", c.Shard) + } + + c.Index, c.Total = index, totalN + if c.Total < 1 || c.Index < 1 || c.Index > c.Total { + return fmt.Errorf("invalid shard %d/%d", c.Index, c.Total) + } + + c.roundRobin = true + + return nil + } + + // Legacy plan-based mode. if c.Total <= 1 { return nil } @@ -56,6 +98,17 @@ func (c *ShardingConfig) Enabled(spec string) (bool, error) { return true, nil } + if c.roundRobin { + h := fnv.New32a() + _, _ = h.Write([]byte(spec)) + // Mask to 31 bits so the hash fits a signed int on every platform, then + // map it onto [1, Total]. c.Total is validated >= 1 in Load. + hash := int(h.Sum32() & 0x7fffffff) + shard := hash%c.Total + 1 + + return shard == c.Index, nil + } + shard, ok := c.shardAssignments[spec] if !ok { return false, fmt.Errorf("test %q is not assigned in sharding plan", spec)