From f38070b204cf0af664752db045ce92d3b047a2ed Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Wed, 12 Aug 2026 16:46:36 +0200
Subject: [PATCH 01/33] Fix nginx in CI

---
 helm-chart/values.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml
index 9709fea..bae49d6 100644
--- a/helm-chart/values.yaml
+++ b/helm-chart/values.yaml
@@ -113,11 +113,13 @@ ui:
               location /nginx_status {
                  stub_status;
                  allow 127.0.0.1;
+                 allow ::1;
                  deny all;
               }
               location ~ ^/(status|ping)$ {
                   include fastcgi_params;
                   allow 127.0.0.1;
+                  allow ::1;
                   fastcgi_param REQUEST_METHOD $request_method;
                   fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
                   fastcgi_pass 127.0.0.1:9000;

From b461d72ceb717feac36bba6b29cab1e465f7f7f3 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Wed, 12 Aug 2026 19:02:44 +0200
Subject: [PATCH 02/33] Fix CI

---
 helm-chart/templates/migration-job.yaml | 30 +++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/helm-chart/templates/migration-job.yaml b/helm-chart/templates/migration-job.yaml
index ea07f8f..0d89e96 100644
--- a/helm-chart/templates/migration-job.yaml
+++ b/helm-chart/templates/migration-job.yaml
@@ -18,6 +18,36 @@ spec:
       serviceAccountName: ui-admin-sa-{{ .Release.Namespace }}
       securityContext:
         fsGroup: {{ .Values.ui.userId }}
+      initContainers:
+      - name: wait-{{ .Chart.Name }}-ui-mysql
+        image: {{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag | default .Chart.AppVersion}}
+        imagePullPolicy: {{ .Values.ui.image.imagePullPolicy }}
+        command:
+          - sh
+          - -c
+          - |
+            for i in $(seq 1 {{ .Values.ui.migrationWait.maxAttempts }}); do
+              if php -r 'exit(@fsockopen(getenv("DB_HOST"), (int) getenv("DB_PORT"), $errno, $errstr, 2) ? 0 : 1);'; then
+                echo "MySQL is reachable"
+                exit 0
+              fi
+
+              echo "Waiting for MySQL to become reachable ($i/{{ .Values.ui.migrationWait.maxAttempts }})"
+              sleep {{ .Values.ui.migrationWait.sleepSeconds }}
+            done
+
+            echo "MySQL did not become reachable in time"
+            exit 1
+        securityContext:
+          runAsGroup: {{ .Values.ui.userId }}
+          runAsUser: {{ .Values.ui.userId }}
+        env:
+          - name: DB_HOST
+            value: "{{ include "manticore-streams.fullname" . }}-mysql-service"
+          - name: DB_PORT
+            value: "{{ .Values.mysql.service.port }}"
+        resources:
+          {{- toYaml .Values.ui.initResources | nindent 10 }}
       containers:
       - name: {{ .Chart.Name }}-ui-migrate
         image: {{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag | default .Chart.AppVersion}}

From 982b2d1b3350637f96b1637caf5c4880880c4b09 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Wed, 12 Aug 2026 19:27:12 +0200
Subject: [PATCH 03/33] Fix CI

---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2bc45a4..05e3a6f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -267,18 +267,18 @@ jobs:
           set -eux
           kubectl cp dev-environment/kafka/test_data.tar.gz "$KAFKA_NAMESPACE/$KAFKA_POD:/tmp/test_data.tar.gz"
           kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- tar -xzf /tmp/test_data.tar.gz -C /tmp
-          start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- sh -c '/opt/bitnami/kafka/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list localhost:9092 --topic my-results' | awk -F: 'END { print $3 }')
+          start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
           echo "START_OFFSET=${start_offset:-0}" >> "$GITHUB_ENV"
       - name: Publish Kafka test fixture
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- sh -c '/opt/bitnami/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic my-docs < /tmp/test_data.json'
+        run: kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- sh -c '/opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server localhost:9092 --topic my-docs < /tmp/test_data.json'
       - name: Verify Kafka processing result
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |
           set -eux
           end=$(( $(date +%s) + TIMEOUT_SECONDS ))
           while [ "$(date +%s)" -lt "$end" ]; do
-            current_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- sh -c '/opt/bitnami/kafka/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list localhost:9092 --topic my-results' | awk -F: 'END { print $3 }')
+            current_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
             current_offset=${current_offset:-0}
             if [ "$current_offset" -eq $((START_OFFSET + EXPECTED_RECORDS)) ]; then
               printf 'Processed exactly %s records\n' "$EXPECTED_RECORDS"

From efbe13a35e8120a64c088f47ace418fdb693babe Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Wed, 12 Aug 2026 20:19:23 +0200
Subject: [PATCH 04/33] Fix CI

---
 .github/workflows/ci.yml | 57 +++++++++++++++++++++++++++++++---------
 1 file changed, 44 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 05e3a6f..e9684b5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -144,12 +144,13 @@ jobs:
       KAFKA_NAMESPACE: kafka
       RELEASE_NAME: manticore-streams
       KAFKA_RELEASE_NAME: my-kafka
+      KAFKA_FIXTURE_PRODUCER_POD: kafka-fixture-producer
       TIMEOUT_SECONDS: 600
       EXPECTED_RECORDS: 9418
       KAFKA_POD: my-kafka-controller-0
       UI_DEPLOYMENT: manticore-streams-manticoresearch-ui
       UI_CONTAINER: manticoresearch-ui
-      KIND_CLUSTER_NAME: chart-testing
+      K3S_VERSION: v1.32.5+k3s1
     steps:
       - uses: actions/checkout@v4
       - name: Skip unrelated pull request
@@ -165,13 +166,14 @@ jobs:
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
       - uses: azure/setup-kubectl@v4
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-      - uses: helm/kind-action@v1.12.0
+      - name: Install K3s
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        with:
-          version: v0.27.0
-          node_image: kindest/node:v1.32.5
-          config: dev-environment/k8s_tests/kind-config.yaml
-          cluster_name: chart-testing
+        run: |
+          set -eux
+          curl -fsSL https://get.k3s.io | INSTALL_K3S_VERSION="$K3S_VERSION" sh -s - server --disable=traefik --disable=servicelb
+          sudo chmod 644 /etc/rancher/k3s/k3s.yaml
+          echo 'KUBECONFIG=/etc/rancher/k3s/k3s.yaml' >> "$GITHUB_ENV"
+          sudo k3s kubectl wait --for=condition=Ready node --all --timeout="${TIMEOUT_SECONDS}s"
       - name: Materialize development chart version
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |
@@ -191,7 +193,7 @@ jobs:
           pattern: ci-image-*
           path: ci-images
           merge-multiple: true
-      - name: Load validated images into kind
+      - name: Import validated images into K3s
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |
           set -eux
@@ -204,8 +206,11 @@ jobs:
             docker image inspect "streams/${component}:ci" >/dev/null
             docker tag "streams/${component}:ci" "ghcr.io/manticoresoftware/streams/${component}:${tag}"
           done
-          mapfile -t images < <(printf '%s\n' "${components[@]}" | sed "s#^#ghcr.io/manticoresoftware/streams/#; s#\$#:${tag}#")
-          kind load docker-image --name "$KIND_CLUSTER_NAME" "${images[@]}"
+          for component in "${components[@]}"; do
+            image="ghcr.io/manticoresoftware/streams/${component}:${tag}"
+            docker image save "$image" | sudo k3s ctr -n k8s.io images import -
+            sudo k3s ctr -n k8s.io images list -q | grep -Fx "$image"
+          done
       - name: Install Kafka and chart
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |
@@ -265,13 +270,39 @@ jobs:
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |
           set -eux
-          kubectl cp dev-environment/kafka/test_data.tar.gz "$KAFKA_NAMESPACE/$KAFKA_POD:/tmp/test_data.tar.gz"
-          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- tar -xzf /tmp/test_data.tar.gz -C /tmp
           start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
           echo "START_OFFSET=${start_offset:-0}" >> "$GITHUB_ENV"
+      - name: Start Kafka fixture producer
+        if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
+        run: |
+          set -eux
+          kubectl -n "$KAFKA_NAMESPACE" apply -f - <<EOF
+          apiVersion: v1
+          kind: Pod
+          metadata:
+            name: ${KAFKA_FIXTURE_PRODUCER_POD}
+            labels:
+              app.kubernetes.io/name: kafka-fixture-producer
+          spec:
+            restartPolicy: Never
+            containers:
+              - name: producer
+                image: docker.io/bitnamilegacy/kafka:4.0.0-debian-12-r10
+                command: ["sh", "-c", "sleep infinity"]
+                resources:
+                  requests:
+                    cpu: 250m
+                    memory: 512Mi
+                  limits:
+                    cpu: "1"
+                    memory: 1Gi
+          EOF
+          kubectl -n "$KAFKA_NAMESPACE" wait --for=condition=Ready "pod/${KAFKA_FIXTURE_PRODUCER_POD}" --timeout="${TIMEOUT_SECONDS}s"
+          kubectl -n "$KAFKA_NAMESPACE" cp dev-environment/kafka/test_data.tar.gz "${KAFKA_FIXTURE_PRODUCER_POD}:/tmp/test_data.tar.gz"
+          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- tar -xzf /tmp/test_data.tar.gz -C /tmp
       - name: Publish Kafka test fixture
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- sh -c '/opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server localhost:9092 --topic my-docs < /tmp/test_data.json'
+        run: kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- sh -c '/opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server my-kafka.kafka.svc.cluster.local:9092 --topic my-docs < /tmp/test_data.json'
       - name: Verify Kafka processing result
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |

From 4adec1e62637313f54383ff9f0f9d28d38f3b5d6 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Wed, 12 Aug 2026 20:48:11 +0200
Subject: [PATCH 05/33] Fix CI

---
 .github/workflows/ci.yml                      | 5 +++++
 .gitignore                                    | 1 +
 docs/README.md                                | 1 +
 docs/ci/github-actions-migration-decisions.md | 2 +-
 4 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e9684b5..16fdd21 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -173,6 +173,11 @@ jobs:
           curl -fsSL https://get.k3s.io | INSTALL_K3S_VERSION="$K3S_VERSION" sh -s - server --disable=traefik --disable=servicelb
           sudo chmod 644 /etc/rancher/k3s/k3s.yaml
           echo 'KUBECONFIG=/etc/rancher/k3s/k3s.yaml' >> "$GITHUB_ENV"
+          end=$(( $(date +%s) + TIMEOUT_SECONDS ))
+          until sudo k3s kubectl get nodes --no-headers 2>/dev/null | grep -q .; do
+            [ "$(date +%s)" -lt "$end" ] || { echo 'K3s did not register a node in time' >&2; sudo journalctl -u k3s --no-pager; exit 1; }
+            sleep 2
+          done
           sudo k3s kubectl wait --for=condition=Ready node --all --timeout="${TIMEOUT_SECONDS}s"
       - name: Materialize development chart version
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
diff --git a/.gitignore b/.gitignore
index 2148361..1240f14 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,4 @@
 /docker/worker/KafkaHandler.jar
 /docker/ui/source/
 /docker/ui-nginx/source/
+/artifacts/
diff --git a/docs/README.md b/docs/README.md
index c1a59b0..697faa8 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -2,6 +2,7 @@
 * [⚡ How it works](HowItWorks.md)
 * [✔ ️Installation](Installation.md)
     * [Helm chart variables](HelmVariables.md)
+* [Local k3s CI stand](ci/local-k3s.md)
 * Ready to work
     * [Roles](ReadyToWork/Roles.md)
     * [Admin Section](ReadyToWork/AdminSection.md)
diff --git a/docs/ci/github-actions-migration-decisions.md b/docs/ci/github-actions-migration-decisions.md
index 71f4ceb..672539d 100644
--- a/docs/ci/github-actions-migration-decisions.md
+++ b/docs/ci/github-actions-migration-decisions.md
@@ -6,6 +6,6 @@ GitHub Actions covers builds and tests only. It does not receive a kubeconfig, c
 
 `helm-chart/Chart.yaml` keeps the development template `7.4.6-$Format:%h$`. GitHub source archives expand it to Git's abbreviated commit ID, so clients downloading a source archive receive a unique chart and app version such as `7.4.6-25d56244`. Actions materializes the same abbreviated SHA only in its disposable checkout before linting, E2E, and publishing image tags. It never commits that generated version; this preserves straightforward Helm development-to-development upgrades without version-only CI commits. After every successful `master` run, the release job waits for all six image publications, packages the materialized chart as `manticoresearch-<version>.tgz`, and creates or updates the matching GitHub Release. The release tag, chart version, app version, and image tags are the same `<base>-<short-sha>` value.
 
-Kubernetes E2E runs in an ephemeral kind `v1.32.5` cluster on a GitHub-hosted runner. This is the closest available official kind node image to the requested `v1.32.6`, which is not published. It builds all six images locally, loads them into kind, installs a pinned Kafka chart, and verifies the 9,418-record output delta. Kafka chart version `32.0.1` is pinned in `install-kafka.sh`; its first GitHub Actions execution must be recorded before making E2E a protected required check.
+Kubernetes E2E runs in an ephemeral K3s `v1.32.5+k3s1` cluster on a GitHub-hosted runner. It builds all six images locally, imports them into K3s containerd, installs the pinned Kafka chart, and verifies the 9,418-record output delta. Kafka chart version `32.4.3` is pinned in `install-kafka.sh`. `docs/ci/local-k3s.md` documents the persistent local k3d/K3s equivalent used to reproduce the same deployment and test path.
 
 Before enabling protected-branch enforcement, a repository administrator must confirm GHCR package creation/retention policy and configure branch protection for `master`. Required checks are `worker-tests`, Helm validation, image smoke checks, and `k8s-e2e` after five consecutive relevant successful E2E runs.

From 467b5fad2b766728e238a0686a7c525b5091f1ae Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Wed, 12 Aug 2026 21:05:34 +0200
Subject: [PATCH 06/33] Fix CI

---
 .github/workflows/ci.yml | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 16fdd21..4967d5e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -239,7 +239,23 @@ jobs:
             --set-string "rulesChecker.image.tag=${tag}"
           )
           helm template manticore-streams helm-chart --kube-version v1.32.5 "${ci_overrides[@]}" > artifacts-rendered-helm.yaml
-          helm upgrade --install manticore-streams helm-chart --namespace manticore-streams --create-namespace --wait --timeout 600s "${ci_overrides[@]}"
+          mkdir -p artifacts
+          helm upgrade --install manticore-streams helm-chart --namespace manticore-streams --create-namespace --wait --timeout 600s "${ci_overrides[@]}" > artifacts/helm-upgrade.log 2>&1 &
+          helm_pid=$!
+          while kill -0 "$helm_pid" 2>/dev/null; do
+            echo '--- waiting for Helm deployment ---'
+            kubectl -n "$APP_NAMESPACE" get pods,jobs -o wide || true
+            kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -20 || true
+            tail -20 artifacts/helm-upgrade.log || true
+            sleep 10
+          done
+          if wait "$helm_pid"; then
+            cat artifacts/helm-upgrade.log
+          else
+            status=$?
+            cat artifacts/helm-upgrade.log
+            exit "$status"
+          fi
       - name: Wait for application workloads
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |

From 9237f0d04785bddd8a7859674388ac5a78043f45 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Wed, 12 Aug 2026 21:28:47 +0200
Subject: [PATCH 07/33] Fix CI

---
 .github/workflows/ci.yml | 36 ++++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4967d5e..83813d6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -327,19 +327,47 @@ jobs:
       - name: Verify Kafka processing result
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |
-          set -eux
+          set -eu
           end=$(( $(date +%s) + TIMEOUT_SECONDS ))
+          pipeline=''
+          while [ "$(date +%s)" -lt "$end" ]; do
+            pipeline=$(kubectl -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker -o jsonpath='{.items[0].metadata.name}')
+            [ -n "$pipeline" ] && break
+            echo 'Waiting for the cluster test to create a pipeline StatefulSet'
+            sleep 5
+          done
+          [ -n "$pipeline" ] || { echo "Cluster test did not create a pipeline StatefulSet within ${TIMEOUT_SECONDS}s" >&2; exit 1; }
+
+          echo "Pipeline StatefulSet: $pipeline"
+          kubectl -n "$APP_NAMESPACE" rollout status "statefulset/$pipeline" --timeout="${TIMEOUT_SECONDS}s"
+          kubectl -n "$APP_NAMESPACE" get statefulset "$pipeline"
+          kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${pipeline#${RELEASE_NAME}-}" -o wide
+
+          expected_offset=$((START_OFFSET + EXPECTED_RECORDS))
+          next_report=0
           while [ "$(date +%s)" -lt "$end" ]; do
             current_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
             current_offset=${current_offset:-0}
-            if [ "$current_offset" -eq $((START_OFFSET + EXPECTED_RECORDS)) ]; then
+            if [ "$current_offset" -eq "$expected_offset" ]; then
               printf 'Processed exactly %s records\n' "$EXPECTED_RECORDS"
               exit 0
             fi
-            printf 'Waiting for output records: start=%s current=%s expected=%s\n' "$START_OFFSET" "$current_offset" "$EXPECTED_RECORDS"
+            now=$(date +%s)
+            if [ "$now" -ge "$next_report" ]; then
+              printf 'Kafka output: start=%s current=%s expected=%s\n' "$START_OFFSET" "$current_offset" "$expected_offset"
+              kubectl -n "$APP_NAMESPACE" get statefulset "$pipeline"
+              kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${pipeline#${RELEASE_NAME}-}" -o wide
+              next_report=$((now + 30))
+            fi
             sleep 10
           done
-          echo "output offset did not reach $((START_OFFSET + EXPECTED_RECORDS)) within ${TIMEOUT_SECONDS}s" >&2
+          echo "Pipeline $pipeline did not produce offset $expected_offset within ${TIMEOUT_SECONDS}s" >&2
+          kubectl -n "$APP_NAMESPACE" describe "statefulset/$pipeline" || true
+          kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -30 || true
+          for pod in $(kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${pipeline#${RELEASE_NAME}-}" -o name); do
+            kubectl -n "$APP_NAMESPACE" describe "$pod" || true
+            kubectl -n "$APP_NAMESPACE" logs "$pod" --all-containers --prefix || true
+          done
           exit 1
       - name: Collect diagnostics
         if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true')

From c8caea8d6c91dabba2ed7dd231c8fcea5575b9dc Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Wed, 12 Aug 2026 22:18:21 +0200
Subject: [PATCH 08/33] Fix CI

---
 .github/workflows/ci.yml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 83813d6..7124e6b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -364,9 +364,17 @@ jobs:
           echo "Pipeline $pipeline did not produce offset $expected_offset within ${TIMEOUT_SECONDS}s" >&2
           kubectl -n "$APP_NAMESPACE" describe "statefulset/$pipeline" || true
           kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -30 || true
+          kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${pipeline#${RELEASE_NAME}-}" -o wide || true
           for pod in $(kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${pipeline#${RELEASE_NAME}-}" -o name); do
             kubectl -n "$APP_NAMESPACE" describe "$pod" || true
-            kubectl -n "$APP_NAMESPACE" logs "$pod" --all-containers --prefix || true
+            for container in $(kubectl -n "$APP_NAMESPACE" get "$pod" -o jsonpath='{.spec.containers[*].name}'); do
+              case "$container" in
+                *-manticore|*-worker)
+                  echo "--- Pipeline container logs: $pod ($container) ---"
+                  kubectl -n "$APP_NAMESPACE" logs "$pod" -c "$container" --prefix || true
+                  ;;
+              esac
+            done
           done
           exit 1
       - name: Collect diagnostics

From c9ecc288b76910549717579169373529bfcc14aa Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Thu, 13 Aug 2026 09:16:48 +0200
Subject: [PATCH 09/33] Fix CI

---
 .github/workflows/ci.yml | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7124e6b..c6e10ad 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -242,11 +242,29 @@ jobs:
           mkdir -p artifacts
           helm upgrade --install manticore-streams helm-chart --namespace manticore-streams --create-namespace --wait --timeout 600s "${ci_overrides[@]}" > artifacts/helm-upgrade.log 2>&1 &
           helm_pid=$!
+          next_report=0
           while kill -0 "$helm_pid" 2>/dev/null; do
-            echo '--- waiting for Helm deployment ---'
-            kubectl -n "$APP_NAMESPACE" get pods,jobs -o wide || true
-            kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -20 || true
-            tail -20 artifacts/helm-upgrade.log || true
+            failed_jobs=$(kubectl -n "$APP_NAMESPACE" get jobs --no-headers -o custom-columns=NAME:.metadata.name,FAILED:.status.failed 2>/dev/null | awk '$2 != "" && $2 != "0" { print $1 }' || true)
+            if [ -n "$failed_jobs" ]; then
+              echo "Helm deployment has failed Job(s): $failed_jobs" >&2
+              for job in $failed_jobs; do
+                echo "--- Failed Job: $job ---" >&2
+                kubectl -n "$APP_NAMESPACE" describe "job/$job" || true
+                kubectl -n "$APP_NAMESPACE" logs "job/$job" --all-containers --prefix || true
+              done
+              kill "$helm_pid" 2>/dev/null || true
+              wait "$helm_pid" 2>/dev/null || true
+              cat artifacts/helm-upgrade.log
+              exit 1
+            fi
+            now=$(date +%s)
+            if [ "$now" -ge "$next_report" ]; then
+              echo '--- waiting for Helm deployment ---'
+              kubectl -n "$APP_NAMESPACE" get pods,jobs -o wide || true
+              kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -20 || true
+              tail -20 artifacts/helm-upgrade.log || true
+              next_report=$((now + 30))
+            fi
             sleep 10
           done
           if wait "$helm_pid"; then

From 3f9f9f8511bf8fd06d71e4212322ed77fce42c79 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Thu, 13 Aug 2026 09:27:17 +0200
Subject: [PATCH 10/33] Fix CI

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c6e10ad..47fcd39 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -244,7 +244,7 @@ jobs:
           helm_pid=$!
           next_report=0
           while kill -0 "$helm_pid" 2>/dev/null; do
-            failed_jobs=$(kubectl -n "$APP_NAMESPACE" get jobs --no-headers -o custom-columns=NAME:.metadata.name,FAILED:.status.failed 2>/dev/null | awk '$2 != "" && $2 != "0" { print $1 }' || true)
+            failed_jobs=$(kubectl -n "$APP_NAMESPACE" get jobs -o jsonpath='{range .items[?(@.status.conditions)]}{.metadata.name}{"\t"}{range .status.conditions[?(@.type=="Failed")]}{.status}{end}{"\n"}{end}' 2>/dev/null | awk '$2 == "True" { print $1 }' || true)
             if [ -n "$failed_jobs" ]; then
               echo "Helm deployment has failed Job(s): $failed_jobs" >&2
               for job in $failed_jobs; do

From 8565383f364f8e98a2dad6b35aff4a884b4c7e48 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Thu, 13 Aug 2026 09:46:28 +0200
Subject: [PATCH 11/33] Fix CI

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 47fcd39..f0475c1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -349,7 +349,7 @@ jobs:
           end=$(( $(date +%s) + TIMEOUT_SECONDS ))
           pipeline=''
           while [ "$(date +%s)" -lt "$end" ]; do
-            pipeline=$(kubectl -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker -o jsonpath='{.items[0].metadata.name}')
+            pipeline=$(kubectl -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker -o jsonpath='{range .items[*]}{.metadata.name}{end}')
             [ -n "$pipeline" ] && break
             echo 'Waiting for the cluster test to create a pipeline StatefulSet'
             sleep 5

From 3fdf4fbc41c5f6a26f77b6cf5a08a7001ced6595 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Thu, 13 Aug 2026 11:33:27 +0200
Subject: [PATCH 12/33] Fix CI

---
 .github/workflows/ci.yml | 44 ++++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f0475c1..457bc1b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -305,6 +305,24 @@ jobs:
       - name: Run Laravel cluster integration tests
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'php -d short_open_tag=off ./vendor/phpunit/phpunit --do-not-cache-result --testsuite Cluster --stderr'
+      - name: Wait for generated pipeline
+        if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
+        run: |
+          set -eu
+          end=$(( $(date +%s) + TIMEOUT_SECONDS ))
+          pipeline=''
+          while [ "$(date +%s)" -lt "$end" ]; do
+            pipeline=$(kubectl -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker -o jsonpath='{range .items[*]}{.metadata.name}{end}')
+            [ -n "$pipeline" ] && break
+            echo 'Waiting for the cluster test to create a pipeline StatefulSet'
+            sleep 5
+          done
+          [ -n "$pipeline" ] || { echo "Cluster test did not create a pipeline StatefulSet within ${TIMEOUT_SECONDS}s" >&2; kubectl -n "$APP_NAMESPACE" get pods,deployments,statefulsets,jobs -o wide || true; exit 1; }
+          echo "Pipeline StatefulSet: $pipeline"
+          kubectl -n "$APP_NAMESPACE" rollout status "statefulset/$pipeline" --timeout="${TIMEOUT_SECONDS}s"
+          kubectl -n "$APP_NAMESPACE" get statefulset "$pipeline"
+          kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${pipeline#${RELEASE_NAME}-}" -o wide
+          echo "PIPELINE=$pipeline" >> "$GITHUB_ENV"
       - name: Load Kafka test fixture
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |
@@ -347,20 +365,6 @@ jobs:
         run: |
           set -eu
           end=$(( $(date +%s) + TIMEOUT_SECONDS ))
-          pipeline=''
-          while [ "$(date +%s)" -lt "$end" ]; do
-            pipeline=$(kubectl -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker -o jsonpath='{range .items[*]}{.metadata.name}{end}')
-            [ -n "$pipeline" ] && break
-            echo 'Waiting for the cluster test to create a pipeline StatefulSet'
-            sleep 5
-          done
-          [ -n "$pipeline" ] || { echo "Cluster test did not create a pipeline StatefulSet within ${TIMEOUT_SECONDS}s" >&2; exit 1; }
-
-          echo "Pipeline StatefulSet: $pipeline"
-          kubectl -n "$APP_NAMESPACE" rollout status "statefulset/$pipeline" --timeout="${TIMEOUT_SECONDS}s"
-          kubectl -n "$APP_NAMESPACE" get statefulset "$pipeline"
-          kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${pipeline#${RELEASE_NAME}-}" -o wide
-
           expected_offset=$((START_OFFSET + EXPECTED_RECORDS))
           next_report=0
           while [ "$(date +%s)" -lt "$end" ]; do
@@ -373,17 +377,17 @@ jobs:
             now=$(date +%s)
             if [ "$now" -ge "$next_report" ]; then
               printf 'Kafka output: start=%s current=%s expected=%s\n' "$START_OFFSET" "$current_offset" "$expected_offset"
-              kubectl -n "$APP_NAMESPACE" get statefulset "$pipeline"
-              kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${pipeline#${RELEASE_NAME}-}" -o wide
+              kubectl -n "$APP_NAMESPACE" get statefulset "$PIPELINE"
+              kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${PIPELINE#${RELEASE_NAME}-}" -o wide
               next_report=$((now + 30))
             fi
             sleep 10
           done
-          echo "Pipeline $pipeline did not produce offset $expected_offset within ${TIMEOUT_SECONDS}s" >&2
-          kubectl -n "$APP_NAMESPACE" describe "statefulset/$pipeline" || true
+          echo "Pipeline $PIPELINE did not produce offset $expected_offset within ${TIMEOUT_SECONDS}s" >&2
+          kubectl -n "$APP_NAMESPACE" describe "statefulset/$PIPELINE" || true
           kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -30 || true
-          kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${pipeline#${RELEASE_NAME}-}" -o wide || true
-          for pod in $(kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${pipeline#${RELEASE_NAME}-}" -o name); do
+          kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${PIPELINE#${RELEASE_NAME}-}" -o wide || true
+          for pod in $(kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${PIPELINE#${RELEASE_NAME}-}" -o name); do
             kubectl -n "$APP_NAMESPACE" describe "$pod" || true
             for container in $(kubectl -n "$APP_NAMESPACE" get "$pod" -o jsonpath='{.spec.containers[*].name}'); do
               case "$container" in

From 84144827f1e9a0e89090c61036cf6cfc7e94d7ec Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Thu, 13 Aug 2026 11:58:12 +0200
Subject: [PATCH 13/33] Fix CI

---
 .github/workflows/ci.yml                      | 14 ++++++++++++-
 .../k8s_tests/collect-diagnostics.sh          | 20 +++++++++++--------
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 457bc1b..12c9690 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -317,7 +317,19 @@ jobs:
             echo 'Waiting for the cluster test to create a pipeline StatefulSet'
             sleep 5
           done
-          [ -n "$pipeline" ] || { echo "Cluster test did not create a pipeline StatefulSet within ${TIMEOUT_SECONDS}s" >&2; kubectl -n "$APP_NAMESPACE" get pods,deployments,statefulsets,jobs -o wide || true; exit 1; }
+          if [ -z "$pipeline" ]; then
+            echo "Cluster test did not create a pipeline StatefulSet within ${TIMEOUT_SECONDS}s" >&2
+            echo '--- Application resources after Cluster PHPUnit ---' >&2
+            kubectl -n "$APP_NAMESPACE" get all,configmaps,pvc -o wide || true
+            echo '--- UI service account permissions ---' >&2
+            kubectl auth can-i create statefulsets.apps --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true
+            kubectl auth can-i create services --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true
+            kubectl auth can-i create persistentvolumeclaims --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true
+            echo '--- UI container logs ---' >&2
+            kubectl -n "$APP_NAMESPACE" logs "$UI_POD" -c "$UI_CONTAINER" --tail=300 --prefix || true
+            kubectl -n "$APP_NAMESPACE" logs "$UI_POD" -c manticoresearch-nginx --tail=100 --prefix || true
+            exit 1
+          fi
           echo "Pipeline StatefulSet: $pipeline"
           kubectl -n "$APP_NAMESPACE" rollout status "statefulset/$pipeline" --timeout="${TIMEOUT_SECONDS}s"
           kubectl -n "$APP_NAMESPACE" get statefulset "$pipeline"
diff --git a/dev-environment/k8s_tests/collect-diagnostics.sh b/dev-environment/k8s_tests/collect-diagnostics.sh
index 6588251..e1f4c70 100644
--- a/dev-environment/k8s_tests/collect-diagnostics.sh
+++ b/dev-environment/k8s_tests/collect-diagnostics.sh
@@ -5,17 +5,21 @@ ARTIFACT_DIR=${1:-artifacts/k8s}
 APP_NAMESPACE=${APP_NAMESPACE:-manticore-streams}
 KAFKA_NAMESPACE=${KAFKA_NAMESPACE:-kafka}
 RELEASE_NAME=${RELEASE_NAME:-manticore-streams}
+KUBECTL_TIMEOUT=${KUBECTL_TIMEOUT:-30s}
 mkdir -p "$ARTIFACT_DIR"
 
 for namespace in "$APP_NAMESPACE" "$KAFKA_NAMESPACE"; do
-  kubectl get all,cm,pvc -n "$namespace" -o wide >"$ARTIFACT_DIR/${namespace}-resources.txt" 2>&1 || true
-  kubectl get events -n "$namespace" --sort-by=.lastTimestamp >"$ARTIFACT_DIR/${namespace}-events.txt" 2>&1 || true
-  for pod in $(kubectl get pods -n "$namespace" -o name 2>/dev/null || true); do
+  kubectl --request-timeout="$KUBECTL_TIMEOUT" get all,cm,pvc -n "$namespace" -o wide >"$ARTIFACT_DIR/${namespace}-resources.txt" 2>&1 || true
+  kubectl --request-timeout="$KUBECTL_TIMEOUT" get events -n "$namespace" --sort-by=.lastTimestamp >"$ARTIFACT_DIR/${namespace}-events.txt" 2>&1 || true
+  for pod in $(kubectl --request-timeout="$KUBECTL_TIMEOUT" get pods -n "$namespace" -o name 2>/dev/null || true); do
     safe_name=$(printf '%s' "$pod" | tr '/' '_')
-    kubectl describe -n "$namespace" "$pod" >"$ARTIFACT_DIR/${namespace}-${safe_name}-describe.txt" 2>&1 || true
-    kubectl logs -n "$namespace" "$pod" --all-containers --prefix >"$ARTIFACT_DIR/${namespace}-${safe_name}.log" 2>&1 || true
+    (
+      kubectl --request-timeout="$KUBECTL_TIMEOUT" describe -n "$namespace" "$pod" >"$ARTIFACT_DIR/${namespace}-${safe_name}-describe.txt" 2>&1 || true
+      timeout 45s kubectl --request-timeout="$KUBECTL_TIMEOUT" logs -n "$namespace" "$pod" --all-containers --prefix >"$ARTIFACT_DIR/${namespace}-${safe_name}.log" 2>&1 || true
+    ) &
   done
+  wait
 done
-helm status "$RELEASE_NAME" -n "$APP_NAMESPACE" >"$ARTIFACT_DIR/helm-status.txt" 2>&1 || true
-helm get values "$RELEASE_NAME" -n "$APP_NAMESPACE" --all >"$ARTIFACT_DIR/helm-values.yaml" 2>&1 || true
-kubectl get nodes -o wide >"$ARTIFACT_DIR/nodes.txt" 2>&1 || true
+timeout 45s helm status "$RELEASE_NAME" -n "$APP_NAMESPACE" >"$ARTIFACT_DIR/helm-status.txt" 2>&1 || true
+timeout 45s helm get values "$RELEASE_NAME" -n "$APP_NAMESPACE" --all >"$ARTIFACT_DIR/helm-values.yaml" 2>&1 || true
+kubectl --request-timeout="$KUBECTL_TIMEOUT" get nodes -o wide >"$ARTIFACT_DIR/nodes.txt" 2>&1 || true

From 99e522e2d9402e30e60e1174d2b23d381808a58f Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Thu, 13 Aug 2026 12:34:26 +0200
Subject: [PATCH 14/33] Fix CI

---
 .github/workflows/ci.yml | 42 +++++++++++++++++++++++++++-------------
 1 file changed, 29 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 12c9690..7503b0e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -146,6 +146,7 @@ jobs:
       KAFKA_RELEASE_NAME: my-kafka
       KAFKA_FIXTURE_PRODUCER_POD: kafka-fixture-producer
       TIMEOUT_SECONDS: 600
+      PIPELINE_CREATION_TIMEOUT_SECONDS: 60
       EXPECTED_RECORDS: 9418
       KAFKA_POD: my-kafka-controller-0
       UI_DEPLOYMENT: manticore-streams-manticoresearch-ui
@@ -301,36 +302,51 @@ jobs:
         run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan migrate:fresh --seed
       - name: Run Laravel Unit and Feature tests
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'php -d short_open_tag=off ./vendor/phpunit/phpunit --do-not-cache-result --testsuite Dev --stderr'
+        run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Dev --stderr'
       - name: Run Laravel cluster integration tests
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'php -d short_open_tag=off ./vendor/phpunit/phpunit --do-not-cache-result --testsuite Cluster --stderr'
+        run: |
+          set -eu
+          kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c '
+            test -x ./vendor/bin/phpunit
+            php -d short_open_tag=off ./vendor/bin/phpunit --testsuite Cluster --list-tests --stderr | tee /tmp/cluster-test-list.txt
+            grep -Fqx " - Tests\\Cluster\\ClusterTest::assignUser" /tmp/cluster-test-list.txt
+            php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Cluster --stderr
+          '
       - name: Wait for generated pipeline
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |
           set -eu
-          end=$(( $(date +%s) + TIMEOUT_SECONDS ))
+          started=$(date +%s)
+          end=$(( $(date +%s) + PIPELINE_CREATION_TIMEOUT_SECONDS ))
+          next_report=$started
           pipeline=''
           while [ "$(date +%s)" -lt "$end" ]; do
             pipeline=$(kubectl -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker -o jsonpath='{range .items[*]}{.metadata.name}{end}')
             [ -n "$pipeline" ] && break
-            echo 'Waiting for the cluster test to create a pipeline StatefulSet'
+            now=$(date +%s)
+            if [ "$now" -ge "$next_report" ]; then
+              printf 'Pipeline creation: waiting (%ss elapsed, %ss remaining)\n' "$((now - started))" "$((end - now))"
+              kubectl -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker
+              next_report=$((now + 30))
+            fi
             sleep 5
           done
           if [ -z "$pipeline" ]; then
-            echo "Cluster test did not create a pipeline StatefulSet within ${TIMEOUT_SECONDS}s" >&2
-            echo '--- Application resources after Cluster PHPUnit ---' >&2
-            kubectl -n "$APP_NAMESPACE" get all,configmaps,pvc -o wide || true
-            echo '--- UI service account permissions ---' >&2
+            echo "Pipeline creation: no StatefulSet after ${PIPELINE_CREATION_TIMEOUT_SECONDS}s" >&2
+            echo '--- Pipeline failure summary ---' >&2
+            kubectl -n "$APP_NAMESPACE" get pods,statefulsets,jobs -o wide || true
+            kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -20 || true
+            echo '--- UI service account permissions (statefulsets/services/PVCs) ---' >&2
             kubectl auth can-i create statefulsets.apps --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true
             kubectl auth can-i create services --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true
             kubectl auth can-i create persistentvolumeclaims --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true
-            echo '--- UI container logs ---' >&2
-            kubectl -n "$APP_NAMESPACE" logs "$UI_POD" -c "$UI_CONTAINER" --tail=300 --prefix || true
-            kubectl -n "$APP_NAMESPACE" logs "$UI_POD" -c manticoresearch-nginx --tail=100 --prefix || true
+            echo '--- UI PHP logs (last 100 lines) ---' >&2
+            kubectl -n "$APP_NAMESPACE" logs "$UI_POD" -c "$UI_CONTAINER" --tail=100 --prefix || true
+            echo 'Detailed pod logs and full resources are in the k8s-e2e-diagnostics artifact.' >&2
             exit 1
           fi
-          echo "Pipeline StatefulSet: $pipeline"
+          printf 'Pipeline creation: found %s after %ss\n' "$pipeline" "$(( $(date +%s) - started ))"
           kubectl -n "$APP_NAMESPACE" rollout status "statefulset/$pipeline" --timeout="${TIMEOUT_SECONDS}s"
           kubectl -n "$APP_NAMESPACE" get statefulset "$pipeline"
           kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${pipeline#${RELEASE_NAME}-}" -o wide
@@ -371,7 +387,7 @@ jobs:
           kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- tar -xzf /tmp/test_data.tar.gz -C /tmp
       - name: Publish Kafka test fixture
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- sh -c '/opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server my-kafka.kafka.svc.cluster.local:9092 --topic my-docs < /tmp/test_data.json'
+        run: kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- sh -c 'timeout 120s /opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server my-kafka.kafka.svc.cluster.local:9092 --topic my-docs < /tmp/test_data.json'
       - name: Verify Kafka processing result
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |

From abbd011615f3562a3835d8d4ded47a39361cb727 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Thu, 13 Aug 2026 12:54:55 +0200
Subject: [PATCH 15/33] Fix CI

---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7503b0e..ae9ad31 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -294,15 +294,15 @@ jobs:
           set -eux
           kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-docs --partitions=1 --replication-factor=1
           kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-results --partitions=1 --replication-factor=1
-      - name: Configure Laravel cluster test environment
-        if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'sed -i "s/production/cluster_testing/g" .env && sed -i "s/testing/cluster_testing/g" phpunit.xml'
       - name: Migrate and seed Laravel test database
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan migrate:fresh --seed
       - name: Run Laravel Unit and Feature tests
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Dev --stderr'
+      - name: Configure Laravel cluster test environment
+        if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
+        run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'sed -i "s/production/cluster_testing/g" .env && sed -i "s/testing/cluster_testing/g" phpunit.xml'
       - name: Run Laravel cluster integration tests
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |

From d021c32ade14bd0cefe48812548ef4213781bdcd Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Thu, 13 Aug 2026 13:04:52 +0200
Subject: [PATCH 16/33] Fix CI

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ae9ad31..eaf85a7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -296,7 +296,7 @@ jobs:
           kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-results --partitions=1 --replication-factor=1
       - name: Migrate and seed Laravel test database
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan migrate:fresh --seed
+        run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan migrate:fresh --seed --force
       - name: Run Laravel Unit and Feature tests
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Dev --stderr'

From 8e3b3320f094b5fd354d0abaf51eb310d5313ef1 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Thu, 13 Aug 2026 15:54:37 +0200
Subject: [PATCH 17/33] Fix CI

---
 .github/workflows/ci.yml                    | 11 ++-
 ui/config/columnar.php                      |  2 +-
 ui/config/manticore.php                     |  2 +-
 ui/tests/Unit/ServiceEndpointConfigTest.php | 83 +++++++++++++++++++++
 4 files changed, 95 insertions(+), 3 deletions(-)
 create mode 100644 ui/tests/Unit/ServiceEndpointConfigTest.php

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index eaf85a7..1ef0408 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -294,12 +294,21 @@ jobs:
           set -eux
           kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-docs --partitions=1 --replication-factor=1
           kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-results --partitions=1 --replication-factor=1
+      - name: Install Laravel Dev test Manticore
+        if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
+        run: |
+          set -eux
+          tag=$(awk '/^appVersion:/ { print $2; exit }' helm-chart/Chart.yaml)
+          APP_NAMESPACE="$APP_NAMESPACE" \
+          DEV_TEST_MANTICORE_IMAGE="ghcr.io/manticoresoftware/streams/manticore:${tag}" \
+          TIMEOUT_SECONDS="$TIMEOUT_SECONDS" \
+          sh dev-environment/k8s_tests/install-dev-test-manticore.sh
       - name: Migrate and seed Laravel test database
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan migrate:fresh --seed --force
       - name: Run Laravel Unit and Feature tests
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Dev --stderr'
+        run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- env MANTICORE_HOST=manticore-dev-tests MANTICORE_PORT=9306 COLUMNAR_HOST="${RELEASE_NAME}-manticoresearch-columnar-service.${APP_NAMESPACE}.svc.cluster.local" COLUMNAR_PORT=9306 php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Dev --stderr
       - name: Configure Laravel cluster test environment
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'sed -i "s/production/cluster_testing/g" .env && sed -i "s/testing/cluster_testing/g" phpunit.xml'
diff --git a/ui/config/columnar.php b/ui/config/columnar.php
index ff6acb5..411cb8b 100644
--- a/ui/config/columnar.php
+++ b/ui/config/columnar.php
@@ -1,7 +1,7 @@
 <?php
 
 return [
-    'host' => (in_array(config('app.env'), ['production', 'cluster_testing']) ) ?
+    'host' => getenv('COLUMNAR_HOST') ?
         getenv('COLUMNAR_HOST').':'.getenv('COLUMNAR_PORT')
         : 'columnar:9306',
     'index' => 'metrics'
diff --git a/ui/config/manticore.php b/ui/config/manticore.php
index b207dcd..2fcd542 100644
--- a/ui/config/manticore.php
+++ b/ui/config/manticore.php
@@ -1,7 +1,7 @@
 <?php
 
 return [
-    'host' => (in_array(config('app.env'), ['production', 'cluster_testing']) ) ?
+    'host' => getenv('MANTICORE_HOST') ?
         getenv('MANTICORE_HOST').':'.getenv('MANTICORE_PORT')
         : 'manticore:9306',
     'index' => 'pq'
diff --git a/ui/tests/Unit/ServiceEndpointConfigTest.php b/ui/tests/Unit/ServiceEndpointConfigTest.php
new file mode 100644
index 0000000..caefbb0
--- /dev/null
+++ b/ui/tests/Unit/ServiceEndpointConfigTest.php
@@ -0,0 +1,83 @@
+<?php
+
+namespace Tests\Unit;
+
+use Tests\TestCase;
+
+/**
+ * @group application
+ */
+class ServiceEndpointConfigTest extends TestCase
+{
+    public function testManticoreConfigFallsBackToComposeEndpointWithoutExplicitHost(): void
+    {
+        $host = getenv('MANTICORE_HOST');
+        $port = getenv('MANTICORE_PORT');
+
+        try {
+            putenv('MANTICORE_HOST');
+            putenv('MANTICORE_PORT');
+
+            $config = require base_path('config/manticore.php');
+
+            self::assertSame('manticore:9306', $config['host']);
+        } finally {
+            putenv('MANTICORE_HOST'.($host === false ? '' : '='.$host));
+            putenv('MANTICORE_PORT'.($port === false ? '' : '='.$port));
+        }
+    }
+
+    public function testManticoreConfigPrefersExplicitEndpointInTesting(): void
+    {
+        $host = getenv('MANTICORE_HOST');
+        $port = getenv('MANTICORE_PORT');
+
+        try {
+            putenv('MANTICORE_HOST=manticore-dev-tests');
+            putenv('MANTICORE_PORT=9306');
+
+            $config = require base_path('config/manticore.php');
+
+            self::assertSame('manticore-dev-tests:9306', $config['host']);
+        } finally {
+            putenv('MANTICORE_HOST'.($host === false ? '' : '='.$host));
+            putenv('MANTICORE_PORT'.($port === false ? '' : '='.$port));
+        }
+    }
+
+    public function testColumnarConfigPrefersExplicitEndpointInTesting(): void
+    {
+        $host = getenv('COLUMNAR_HOST');
+        $port = getenv('COLUMNAR_PORT');
+
+        try {
+            putenv('COLUMNAR_HOST=columnar-dev-tests');
+            putenv('COLUMNAR_PORT=9306');
+
+            $config = require base_path('config/columnar.php');
+
+            self::assertSame('columnar-dev-tests:9306', $config['host']);
+        } finally {
+            putenv('COLUMNAR_HOST'.($host === false ? '' : '='.$host));
+            putenv('COLUMNAR_PORT'.($port === false ? '' : '='.$port));
+        }
+    }
+
+    public function testColumnarConfigFallsBackToComposeEndpointWithoutExplicitHost(): void
+    {
+        $host = getenv('COLUMNAR_HOST');
+        $port = getenv('COLUMNAR_PORT');
+
+        try {
+            putenv('COLUMNAR_HOST');
+            putenv('COLUMNAR_PORT');
+
+            $config = require base_path('config/columnar.php');
+
+            self::assertSame('columnar:9306', $config['host']);
+        } finally {
+            putenv('COLUMNAR_HOST'.($host === false ? '' : '='.$host));
+            putenv('COLUMNAR_PORT'.($port === false ? '' : '='.$port));
+        }
+    }
+}

From 0f537ffcefd3cd9810468d12b119eac291b151c2 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Thu, 13 Aug 2026 16:45:05 +0200
Subject: [PATCH 18/33] Fix CI

---
 .github/workflows/ci.yml                      | 62 ++++++++++++++-----
 docs/ci/github-actions-migration-decisions.md |  4 +-
 2 files changed, 50 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1ef0408..837f5c9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -135,8 +135,54 @@ jobs:
               - 'dev-environment/k8s_tests/**'
               - '.github/workflows/ci.yml'
 
+  laravel-tests:
+    needs: [image-build, ui-image-build, changes]
+    if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download Laravel test images
+        uses: actions/download-artifact@v4
+        with:
+          pattern: ci-image-*
+          path: ci-images
+          merge-multiple: true
+      - name: Load Laravel test images
+        run: |
+          set -eux
+          for component in manticore ui; do
+            gzip -dc "ci-images/${component}.tar.gz" | docker load
+            docker image inspect "streams/${component}:ci" >/dev/null
+          done
+      - name: Start Laravel test dependencies
+        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml up -d --wait mysql manticore columnar
+      - name: Wait for Laravel database connection
+        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml run --rm --no-deps ui php -r 'for ($attempt = 0; $attempt < 60; $attempt++) { try { new PDO("mysql:host=mysql;dbname=mysql", "root", "mysql"); exit(0); } catch (Throwable $exception) { sleep(1); } } fwrite(STDERR, "MySQL did not accept a Laravel connection within 60 seconds\n"); exit(1);'
+      - name: Initialize Laravel test Manticore schema
+        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml exec -T manticore php /etc/manticoresearch/alter.php --table=pq --batch=500
+      - name: Initialize Laravel test metrics storage
+        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml exec -T columnar mysql -h 127.0.0.1 -P 9306 -e 'CREATE TABLE IF NOT EXISTS metrics (scrapTime timestamp, metric_name text, value float, tag string)'
+      - name: Migrate and seed Laravel test database
+        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml run --rm --no-deps ui php artisan migrate:fresh --seed --force
+      - name: Run Laravel Unit and Feature tests
+        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml run --rm --no-deps ui php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Dev --stderr
+      - name: Collect Laravel test dependency logs
+        if: always()
+        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml logs --no-color > laravel-test-dependencies.log || true
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: laravel-test-dependencies
+          path: laravel-test-dependencies.log
+          retention-days: 7
+          if-no-files-found: warn
+      - name: Remove Laravel test dependencies
+        if: always()
+        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml down --volumes --remove-orphans
+
   k8s-e2e:
-    needs: [worker-tests, image-build, ui-image-build, worker-image-build, helm, changes]
+    needs: [worker-tests, image-build, ui-image-build, worker-image-build, helm, changes, laravel-tests]
     runs-on: ubuntu-24.04
     timeout-minutes: 75
     env:
@@ -294,21 +340,9 @@ jobs:
           set -eux
           kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-docs --partitions=1 --replication-factor=1
           kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-results --partitions=1 --replication-factor=1
-      - name: Install Laravel Dev test Manticore
-        if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: |
-          set -eux
-          tag=$(awk '/^appVersion:/ { print $2; exit }' helm-chart/Chart.yaml)
-          APP_NAMESPACE="$APP_NAMESPACE" \
-          DEV_TEST_MANTICORE_IMAGE="ghcr.io/manticoresoftware/streams/manticore:${tag}" \
-          TIMEOUT_SECONDS="$TIMEOUT_SECONDS" \
-          sh dev-environment/k8s_tests/install-dev-test-manticore.sh
-      - name: Migrate and seed Laravel test database
+      - name: Migrate and seed Laravel cluster test database
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan migrate:fresh --seed --force
-      - name: Run Laravel Unit and Feature tests
-        if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- env MANTICORE_HOST=manticore-dev-tests MANTICORE_PORT=9306 COLUMNAR_HOST="${RELEASE_NAME}-manticoresearch-columnar-service.${APP_NAMESPACE}.svc.cluster.local" COLUMNAR_PORT=9306 php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Dev --stderr
       - name: Configure Laravel cluster test environment
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'sed -i "s/production/cluster_testing/g" .env && sed -i "s/testing/cluster_testing/g" phpunit.xml'
diff --git a/docs/ci/github-actions-migration-decisions.md b/docs/ci/github-actions-migration-decisions.md
index 672539d..80013ec 100644
--- a/docs/ci/github-actions-migration-decisions.md
+++ b/docs/ci/github-actions-migration-decisions.md
@@ -2,10 +2,10 @@
 
 The canonical repository is `manticoresoftware/streams` with `master` as its protected default branch. The only supported CI registry is `ghcr.io/manticoresoftware/streams`.
 
-GitHub Actions covers builds and tests only. It does not receive a kubeconfig, cloud credential, deployment environment permission, or documentation-deployment credential. The single `.github/workflows/ci.yml` workflow runs worker tests, image smoke builds, Helm validation, and change detection in parallel. Kubernetes E2E waits for those gates; publishing waits for successful E2E and runs only for a `master` push. Pull requests use `contents: read` and never publish images. Publishing uses the workflow-provided `GITHUB_TOKEN` with `packages: write`. Published GHCR images are private; runtime clusters provide their own pull secret through the chart's `imagePullSecrets` value. CI uses locally loaded kind images and disables this value.
+GitHub Actions covers builds and tests only. It does not receive a kubeconfig, cloud credential, deployment environment permission, or documentation-deployment credential. The single `.github/workflows/ci.yml` workflow runs worker tests, image smoke builds, Helm validation, change detection, and a Docker-based Laravel Dev suite. The Laravel job loads the CI-built UI and Manticore images, starts disposable MySQL, Manticore, and Columnar services, initializes the test schemas, migrates and seeds the Laravel database, and runs the `Dev` PHPUnit suite. Kubernetes E2E waits for those gates and retains only Helm deployment, in-cluster migration, the `Cluster` PHPUnit suite, generated pipeline readiness, and the Kafka fixture/output assertion. Publishing waits for successful E2E and runs only for a `master` push. Pull requests use `contents: read` and never publish images. Publishing uses the workflow-provided `GITHUB_TOKEN` with `packages: write`. Published GHCR images are private; runtime clusters provide their own pull secret through the chart's `imagePullSecrets` value. CI uses locally loaded kind images and disables this value.
 
 `helm-chart/Chart.yaml` keeps the development template `7.4.6-$Format:%h$`. GitHub source archives expand it to Git's abbreviated commit ID, so clients downloading a source archive receive a unique chart and app version such as `7.4.6-25d56244`. Actions materializes the same abbreviated SHA only in its disposable checkout before linting, E2E, and publishing image tags. It never commits that generated version; this preserves straightforward Helm development-to-development upgrades without version-only CI commits. After every successful `master` run, the release job waits for all six image publications, packages the materialized chart as `manticoresearch-<version>.tgz`, and creates or updates the matching GitHub Release. The release tag, chart version, app version, and image tags are the same `<base>-<short-sha>` value.
 
 Kubernetes E2E runs in an ephemeral K3s `v1.32.5+k3s1` cluster on a GitHub-hosted runner. It builds all six images locally, imports them into K3s containerd, installs the pinned Kafka chart, and verifies the 9,418-record output delta. Kafka chart version `32.4.3` is pinned in `install-kafka.sh`. `docs/ci/local-k3s.md` documents the persistent local k3d/K3s equivalent used to reproduce the same deployment and test path.
 
-Before enabling protected-branch enforcement, a repository administrator must confirm GHCR package creation/retention policy and configure branch protection for `master`. Required checks are `worker-tests`, Helm validation, image smoke checks, and `k8s-e2e` after five consecutive relevant successful E2E runs.
+Before enabling protected-branch enforcement, a repository administrator must confirm GHCR package creation/retention policy and configure branch protection for `master`. Required checks are `worker-tests`, `laravel-tests`, Helm validation, image smoke checks, and `k8s-e2e` after five consecutive relevant successful E2E runs.

From becd6d4fccfa1ac7320ce3b0ddbe0f6ed0e5c5f4 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Thu, 13 Aug 2026 17:06:33 +0200
Subject: [PATCH 19/33] Fix CI

---
 .github/workflows/ci.yml                      |  1 +
 .../docker-compose.laravel-tests.yml          | 66 +++++++++++++++++++
 2 files changed, 67 insertions(+)
 create mode 100644 dev-environment/docker-compose.laravel-tests.yml

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 837f5c9..0c9f47c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -132,6 +132,7 @@ jobs:
               - 'ui/**'
               - 'sources/**'
               - 'helm-chart/**'
+              - 'dev-environment/docker-compose.laravel-tests.yml'
               - 'dev-environment/k8s_tests/**'
               - '.github/workflows/ci.yml'
 
diff --git a/dev-environment/docker-compose.laravel-tests.yml b/dev-environment/docker-compose.laravel-tests.yml
new file mode 100644
index 0000000..67cf323
--- /dev/null
+++ b/dev-environment/docker-compose.laravel-tests.yml
@@ -0,0 +1,66 @@
+name: manticore-streams-laravel-tests
+
+services:
+  mysql:
+    image: mysql:5.7.22
+    platform: linux/amd64
+    environment:
+      MYSQL_DATABASE: mysql
+      MYSQL_ROOT_PASSWORD: mysql
+    healthcheck:
+      test: ["CMD-SHELL", "mysqladmin ping -h localhost -uroot -pmysql --silent"]
+      interval: 5s
+      timeout: 3s
+      retries: 60
+
+  manticore:
+    image: streams/manticore:ci
+    platform: linux/amd64
+    environment:
+      MANTICORE_RULES: json=json|url=url|text=source|text=text|text=title|text=statustext|url=merged|url=merged2
+      MANTICORE_PORT: "9306"
+      PIPELINE: m
+      INSTANCE_LABEL: laravel-tests
+      STAGE: dev
+      BLACKLIST_MODE: "1"
+    healthcheck:
+      test: ["CMD-SHELL", "mysql -h 127.0.0.1 -P 9306 -e 'SELECT * FROM m_cluster:pq LIMIT 0'"]
+      interval: 5s
+      timeout: 3s
+      retries: 60
+
+  columnar:
+    image: manticoresearch/manticore:25.0.0
+    platform: linux/amd64
+    environment:
+      EXTRA: "1"
+      searchd_mysql_version_string: "5.5.21"
+      searchd_max_packet_size: "128M"
+      searchd_pid_file: /tmp/searchd.pid
+    healthcheck:
+      test: ["CMD-SHELL", "mysql -h 127.0.0.1 -P 9306 -e 'SELECT 1'"]
+      interval: 5s
+      timeout: 3s
+      retries: 60
+
+  ui:
+    image: streams/ui:ci
+    platform: linux/amd64
+    environment:
+      DB_HOST: mysql
+      DB_DATABASE: mysql
+      DB_USERNAME: root
+      DB_PASSWORD: mysql
+      MANTICORE_HOST: manticore
+      MANTICORE_PORT: "9306"
+      COLUMNAR_HOST: columnar
+      COLUMNAR_PORT: "9306"
+    extra_hosts:
+      - "dev.manticoresearch.com=127.0.0.1"
+    depends_on:
+      mysql:
+        condition: service_healthy
+      manticore:
+        condition: service_healthy
+      columnar:
+        condition: service_healthy

From 6791106a0b4a506b691978e3a5195b00d6e837bc Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Thu, 13 Aug 2026 18:28:53 +0200
Subject: [PATCH 20/33] Fix CI

---
 .github/workflows/ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0c9f47c..e380634 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -347,6 +347,9 @@ jobs:
       - name: Configure Laravel cluster test environment
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'sed -i "s/production/cluster_testing/g" .env && sed -i "s/testing/cluster_testing/g" phpunit.xml'
+      - name: Seed Laravel cluster test user
+        if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
+        run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan db:seed --class=TestDataSeeder --force
       - name: Run Laravel cluster integration tests
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |

From 1938135ceb862789c60ce068b4af56f91a278ed0 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Fri, 14 Aug 2026 11:49:36 +0200
Subject: [PATCH 21/33] Fix CI

---
 .github/workflows/ci.yml | 35 +----------------------------------
 1 file changed, 1 insertion(+), 34 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e380634..b1d9e83 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -288,40 +288,7 @@ jobs:
           )
           helm template manticore-streams helm-chart --kube-version v1.32.5 "${ci_overrides[@]}" > artifacts-rendered-helm.yaml
           mkdir -p artifacts
-          helm upgrade --install manticore-streams helm-chart --namespace manticore-streams --create-namespace --wait --timeout 600s "${ci_overrides[@]}" > artifacts/helm-upgrade.log 2>&1 &
-          helm_pid=$!
-          next_report=0
-          while kill -0 "$helm_pid" 2>/dev/null; do
-            failed_jobs=$(kubectl -n "$APP_NAMESPACE" get jobs -o jsonpath='{range .items[?(@.status.conditions)]}{.metadata.name}{"\t"}{range .status.conditions[?(@.type=="Failed")]}{.status}{end}{"\n"}{end}' 2>/dev/null | awk '$2 == "True" { print $1 }' || true)
-            if [ -n "$failed_jobs" ]; then
-              echo "Helm deployment has failed Job(s): $failed_jobs" >&2
-              for job in $failed_jobs; do
-                echo "--- Failed Job: $job ---" >&2
-                kubectl -n "$APP_NAMESPACE" describe "job/$job" || true
-                kubectl -n "$APP_NAMESPACE" logs "job/$job" --all-containers --prefix || true
-              done
-              kill "$helm_pid" 2>/dev/null || true
-              wait "$helm_pid" 2>/dev/null || true
-              cat artifacts/helm-upgrade.log
-              exit 1
-            fi
-            now=$(date +%s)
-            if [ "$now" -ge "$next_report" ]; then
-              echo '--- waiting for Helm deployment ---'
-              kubectl -n "$APP_NAMESPACE" get pods,jobs -o wide || true
-              kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -20 || true
-              tail -20 artifacts/helm-upgrade.log || true
-              next_report=$((now + 30))
-            fi
-            sleep 10
-          done
-          if wait "$helm_pid"; then
-            cat artifacts/helm-upgrade.log
-          else
-            status=$?
-            cat artifacts/helm-upgrade.log
-            exit "$status"
-          fi
+          helm upgrade --install manticore-streams helm-chart --namespace manticore-streams --create-namespace --wait --timeout 600s "${ci_overrides[@]}" 2>&1 | tee artifacts/helm-upgrade.log
       - name: Wait for application workloads
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |

From 638104b264bbf0775804df6ed936d3c1ce37e2d0 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Fri, 14 Aug 2026 12:16:14 +0200
Subject: [PATCH 22/33] Fix CI

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b1d9e83..1565a25 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -316,7 +316,7 @@ jobs:
         run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'sed -i "s/production/cluster_testing/g" .env && sed -i "s/testing/cluster_testing/g" phpunit.xml'
       - name: Seed Laravel cluster test user
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan db:seed --class=TestDataSeeder --force
+        run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan db:seed --class='\TestDataSeeder' --force
       - name: Run Laravel cluster integration tests
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |

From cea462c157356af2be5dc2631e31381c8a7ebfc5 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Fri, 14 Aug 2026 12:18:15 +0200
Subject: [PATCH 23/33] Fix CI

---
 .gitignore                    | 2 ++
 docker/ui-nginx/.dockerignore | 2 ++
 docker/ui/.dockerignore       | 2 ++
 3 files changed, 6 insertions(+)
 create mode 100644 docker/ui-nginx/.dockerignore
 create mode 100644 docker/ui/.dockerignore

diff --git a/.gitignore b/.gitignore
index 1240f14..af4bb21 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,3 +9,5 @@
 /docker/ui/source/
 /docker/ui-nginx/source/
 /artifacts/
+**/.DS_Store
+**/._*
diff --git a/docker/ui-nginx/.dockerignore b/docker/ui-nginx/.dockerignore
new file mode 100644
index 0000000..ce54105
--- /dev/null
+++ b/docker/ui-nginx/.dockerignore
@@ -0,0 +1,2 @@
+**/.DS_Store
+**/._*
diff --git a/docker/ui/.dockerignore b/docker/ui/.dockerignore
new file mode 100644
index 0000000..ce54105
--- /dev/null
+++ b/docker/ui/.dockerignore
@@ -0,0 +1,2 @@
+**/.DS_Store
+**/._*

From b0181c1282b787dff8e95aa1e40a39446e5b6799 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Fri, 14 Aug 2026 12:51:34 +0200
Subject: [PATCH 24/33] Fix CI

---
 .github/workflows/ci.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1565a25..3f9a532 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -363,7 +363,7 @@ jobs:
           printf 'Pipeline creation: found %s after %ss\n' "$pipeline" "$(( $(date +%s) - started ))"
           kubectl -n "$APP_NAMESPACE" rollout status "statefulset/$pipeline" --timeout="${TIMEOUT_SECONDS}s"
           kubectl -n "$APP_NAMESPACE" get statefulset "$pipeline"
-          kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${pipeline#${RELEASE_NAME}-}" -o wide
+          kubectl -n "$APP_NAMESPACE" get pods -l "name=${pipeline#${RELEASE_NAME}-}" -o wide
           echo "PIPELINE=$pipeline" >> "$GITHUB_ENV"
       - name: Load Kafka test fixture
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
@@ -420,7 +420,7 @@ jobs:
             if [ "$now" -ge "$next_report" ]; then
               printf 'Kafka output: start=%s current=%s expected=%s\n' "$START_OFFSET" "$current_offset" "$expected_offset"
               kubectl -n "$APP_NAMESPACE" get statefulset "$PIPELINE"
-              kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${PIPELINE#${RELEASE_NAME}-}" -o wide
+              kubectl -n "$APP_NAMESPACE" get pods -l "name=${PIPELINE#${RELEASE_NAME}-}" -o wide
               next_report=$((now + 30))
             fi
             sleep 10
@@ -428,14 +428,16 @@ jobs:
           echo "Pipeline $PIPELINE did not produce offset $expected_offset within ${TIMEOUT_SECONDS}s" >&2
           kubectl -n "$APP_NAMESPACE" describe "statefulset/$PIPELINE" || true
           kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -30 || true
-          kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${PIPELINE#${RELEASE_NAME}-}" -o wide || true
-          for pod in $(kubectl -n "$APP_NAMESPACE" get pods -l "app.kubernetes.io/pipeline=${PIPELINE#${RELEASE_NAME}-}" -o name); do
+          kubectl -n "$APP_NAMESPACE" get pods -l "name=${PIPELINE#${RELEASE_NAME}-}" -o wide || true
+          for pod in $(kubectl -n "$APP_NAMESPACE" get pods -l "name=${PIPELINE#${RELEASE_NAME}-}" -o name); do
             kubectl -n "$APP_NAMESPACE" describe "$pod" || true
             for container in $(kubectl -n "$APP_NAMESPACE" get "$pod" -o jsonpath='{.spec.containers[*].name}'); do
               case "$container" in
                 *-manticore|*-worker)
                   echo "--- Pipeline container logs: $pod ($container) ---"
                   kubectl -n "$APP_NAMESPACE" logs "$pod" -c "$container" --prefix || true
+                  echo "--- Previous pipeline container logs: $pod ($container) ---"
+                  kubectl -n "$APP_NAMESPACE" logs "$pod" -c "$container" --previous --prefix || true
                   ;;
               esac
             done

From 1ec1124e2decff131e1499c54ade1ddffdcfef68 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Fri, 14 Aug 2026 13:49:40 +0200
Subject: [PATCH 25/33] Fix CI

---
 .github/workflows/ci.yml | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3f9a532..d87b5e3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -369,7 +369,9 @@ jobs:
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |
           set -eux
+          input_start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }')
           start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
+          echo "INPUT_START_OFFSET=${input_start_offset:-0}" >> "$GITHUB_ENV"
           echo "START_OFFSET=${start_offset:-0}" >> "$GITHUB_ENV"
       - name: Start Kafka fixture producer
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
@@ -399,9 +401,17 @@ jobs:
           kubectl -n "$KAFKA_NAMESPACE" wait --for=condition=Ready "pod/${KAFKA_FIXTURE_PRODUCER_POD}" --timeout="${TIMEOUT_SECONDS}s"
           kubectl -n "$KAFKA_NAMESPACE" cp dev-environment/kafka/test_data.tar.gz "${KAFKA_FIXTURE_PRODUCER_POD}:/tmp/test_data.tar.gz"
           kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- tar -xzf /tmp/test_data.tar.gz -C /tmp
+          fixture_records=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- wc -l < /tmp/test_data.json)
+          echo "FIXTURE_RECORDS=${fixture_records}" >> "$GITHUB_ENV"
       - name: Publish Kafka test fixture
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- sh -c 'timeout 120s /opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server my-kafka.kafka.svc.cluster.local:9092 --topic my-docs < /tmp/test_data.json'
+        run: |
+          set -eu
+          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- sh -c 'timeout 120s /opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server my-kafka.kafka.svc.cluster.local:9092 --topic my-docs < /tmp/test_data.json'
+          input_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }')
+          expected_input_offset=$((INPUT_START_OFFSET + FIXTURE_RECORDS))
+          printf 'Kafka input: start=%s current=%s expected=%s\n' "$INPUT_START_OFFSET" "${input_offset:-0}" "$expected_input_offset"
+          [ "${input_offset:-0}" -eq "$expected_input_offset" ]
       - name: Verify Kafka processing result
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |
@@ -426,6 +436,11 @@ jobs:
             sleep 10
           done
           echo "Pipeline $PIPELINE did not produce offset $expected_offset within ${TIMEOUT_SECONDS}s" >&2
+          echo '--- Kafka topic offsets ---'
+          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs || true
+          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results || true
+          echo '--- Kafka consumer group ms_test_stream ---'
+          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ms_test_stream || true
           kubectl -n "$APP_NAMESPACE" describe "statefulset/$PIPELINE" || true
           kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -30 || true
           kubectl -n "$APP_NAMESPACE" get pods -l "name=${PIPELINE#${RELEASE_NAME}-}" -o wide || true

From e14fd8e675a08cdf51a58adae403f25d15c24cd3 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Fri, 14 Aug 2026 13:59:49 +0200
Subject: [PATCH 26/33] Fix CI

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d87b5e3..4a44837 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -401,7 +401,7 @@ jobs:
           kubectl -n "$KAFKA_NAMESPACE" wait --for=condition=Ready "pod/${KAFKA_FIXTURE_PRODUCER_POD}" --timeout="${TIMEOUT_SECONDS}s"
           kubectl -n "$KAFKA_NAMESPACE" cp dev-environment/kafka/test_data.tar.gz "${KAFKA_FIXTURE_PRODUCER_POD}:/tmp/test_data.tar.gz"
           kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- tar -xzf /tmp/test_data.tar.gz -C /tmp
-          fixture_records=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- wc -l < /tmp/test_data.json)
+          fixture_records=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- sh -c 'wc -l < /tmp/test_data.json')
           echo "FIXTURE_RECORDS=${fixture_records}" >> "$GITHUB_ENV"
       - name: Publish Kafka test fixture
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'

From 2a1e87f38c77e00f67bbece1391178e4fe0dea1e Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Fri, 14 Aug 2026 14:29:45 +0200
Subject: [PATCH 27/33] Fix CI

---
 .github/workflows/ci.yml                   | 40 +++++++++++++++++-----
 dev-environment/k8s_tests/install-kafka.sh | 12 +++++++
 2 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4a44837..e605b63 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -306,8 +306,8 @@ jobs:
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |
           set -eux
-          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-docs --partitions=1 --replication-factor=1
-          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-results --partitions=1 --replication-factor=1
+          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-docs --partitions=1 --replication-factor=1
+          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-results --partitions=1 --replication-factor=1
       - name: Migrate and seed Laravel cluster test database
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan migrate:fresh --seed --force
@@ -365,12 +365,34 @@ jobs:
           kubectl -n "$APP_NAMESPACE" get statefulset "$pipeline"
           kubectl -n "$APP_NAMESPACE" get pods -l "name=${pipeline#${RELEASE_NAME}-}" -o wide
           echo "PIPELINE=$pipeline" >> "$GITHUB_ENV"
+      - name: Wait for Kafka consumer assignment
+        if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
+        run: |
+          set -eu
+          end=$(( $(date +%s) + TIMEOUT_SECONDS ))
+          next_report=0
+          group_members=''
+          while [ "$(date +%s)" -lt "$end" ]; do
+            group_members=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ms_test_stream --members --verbose 2>&1 || true)
+            if printf '%s\n' "$group_members" | grep -Fq 'my-docs(0)'; then
+              printf 'Kafka consumer group ms_test_stream is assigned my-docs(0)\n'
+              exit 0
+            fi
+            now=$(date +%s)
+            if [ "$now" -ge "$next_report" ]; then
+              printf 'Kafka consumer assignment: waiting for ms_test_stream to receive my-docs(0)\n%s\n' "$group_members"
+              next_report=$((now + 30))
+            fi
+            sleep 5
+          done
+          printf 'Kafka consumer group ms_test_stream was not assigned my-docs(0) within %ss\n%s\n' "$TIMEOUT_SECONDS" "$group_members" >&2
+          exit 1
       - name: Load Kafka test fixture
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: |
           set -eux
-          input_start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }')
-          start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
+          input_start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }')
+          start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
           echo "INPUT_START_OFFSET=${input_start_offset:-0}" >> "$GITHUB_ENV"
           echo "START_OFFSET=${start_offset:-0}" >> "$GITHUB_ENV"
       - name: Start Kafka fixture producer
@@ -408,7 +430,7 @@ jobs:
         run: |
           set -eu
           kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- sh -c 'timeout 120s /opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server my-kafka.kafka.svc.cluster.local:9092 --topic my-docs < /tmp/test_data.json'
-          input_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }')
+          input_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }')
           expected_input_offset=$((INPUT_START_OFFSET + FIXTURE_RECORDS))
           printf 'Kafka input: start=%s current=%s expected=%s\n' "$INPUT_START_OFFSET" "${input_offset:-0}" "$expected_input_offset"
           [ "${input_offset:-0}" -eq "$expected_input_offset" ]
@@ -420,7 +442,7 @@ jobs:
           expected_offset=$((START_OFFSET + EXPECTED_RECORDS))
           next_report=0
           while [ "$(date +%s)" -lt "$end" ]; do
-            current_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
+            current_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
             current_offset=${current_offset:-0}
             if [ "$current_offset" -eq "$expected_offset" ]; then
               printf 'Processed exactly %s records\n' "$EXPECTED_RECORDS"
@@ -437,10 +459,10 @@ jobs:
           done
           echo "Pipeline $PIPELINE did not produce offset $expected_offset within ${TIMEOUT_SECONDS}s" >&2
           echo '--- Kafka topic offsets ---'
-          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs || true
-          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results || true
+          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs || true
+          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results || true
           echo '--- Kafka consumer group ms_test_stream ---'
-          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ms_test_stream || true
+          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ms_test_stream || true
           kubectl -n "$APP_NAMESPACE" describe "statefulset/$PIPELINE" || true
           kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -30 || true
           kubectl -n "$APP_NAMESPACE" get pods -l "name=${PIPELINE#${RELEASE_NAME}-}" -o wide || true
diff --git a/dev-environment/k8s_tests/install-kafka.sh b/dev-environment/k8s_tests/install-kafka.sh
index 41a1948..01bbb3d 100644
--- a/dev-environment/k8s_tests/install-kafka.sh
+++ b/dev-environment/k8s_tests/install-kafka.sh
@@ -38,7 +38,19 @@ helm upgrade --install "$KAFKA_RELEASE_NAME" bitnami/kafka \
   --set listeners.client.protocol=PLAINTEXT \
   --set listeners.controller.protocol=PLAINTEXT \
   --set listeners.interbroker.protocol=PLAINTEXT \
+  --set config.offsets\\.topic\\.replication\\.factor=1 \
+  --set config.transaction\\.state\\.log\\.replication\\.factor=1 \
+  --set config.transaction\\.state\\.log\\.min\\.isr=1 \
   --set provisioning.enabled=false \
   --wait --timeout "${TIMEOUT_SECONDS}s"
 
 kubectl -n "$KAFKA_NAMESPACE" rollout status statefulset/"${KAFKA_RELEASE_NAME}"-controller --timeout="${TIMEOUT_SECONDS}s"
+kubectl -n "$KAFKA_NAMESPACE" exec "${KAFKA_RELEASE_NAME}"-controller-0 -c kafka -- \
+  /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists \
+  --topic ci-kafka-coordinator-readiness --partitions=1 --replication-factor=1
+kubectl -n "$KAFKA_NAMESPACE" exec "${KAFKA_RELEASE_NAME}"-controller-0 -c kafka -- sh -c \
+  "printf 'ready\\n' | /opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server localhost:9092 --topic ci-kafka-coordinator-readiness"
+timeout 60s kubectl -n "$KAFKA_NAMESPACE" exec "${KAFKA_RELEASE_NAME}"-controller-0 -c kafka -- \
+  /opt/bitnami/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 \
+  --topic ci-kafka-coordinator-readiness --group ci-kafka-coordinator-readiness --from-beginning \
+  --max-messages 1 --timeout-ms 30000 >/dev/null

From 2d82388bc88affd15a70216f76ccc3457b76d647 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Fri, 14 Aug 2026 16:14:51 +0200
Subject: [PATCH 28/33] Fix CI

---
 dev-environment/k8s_tests/collect-diagnostics.sh | 1 +
 dev-environment/k8s_tests/install-kafka.sh       | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/dev-environment/k8s_tests/collect-diagnostics.sh b/dev-environment/k8s_tests/collect-diagnostics.sh
index e1f4c70..e7742f1 100644
--- a/dev-environment/k8s_tests/collect-diagnostics.sh
+++ b/dev-environment/k8s_tests/collect-diagnostics.sh
@@ -16,6 +16,7 @@ for namespace in "$APP_NAMESPACE" "$KAFKA_NAMESPACE"; do
     (
       kubectl --request-timeout="$KUBECTL_TIMEOUT" describe -n "$namespace" "$pod" >"$ARTIFACT_DIR/${namespace}-${safe_name}-describe.txt" 2>&1 || true
       timeout 45s kubectl --request-timeout="$KUBECTL_TIMEOUT" logs -n "$namespace" "$pod" --all-containers --prefix >"$ARTIFACT_DIR/${namespace}-${safe_name}.log" 2>&1 || true
+      timeout 45s kubectl --request-timeout="$KUBECTL_TIMEOUT" logs -n "$namespace" "$pod" --all-containers --previous --prefix >"$ARTIFACT_DIR/${namespace}-${safe_name}-previous.log" 2>&1 || true
     ) &
   done
   wait
diff --git a/dev-environment/k8s_tests/install-kafka.sh b/dev-environment/k8s_tests/install-kafka.sh
index 01bbb3d..40f50b0 100644
--- a/dev-environment/k8s_tests/install-kafka.sh
+++ b/dev-environment/k8s_tests/install-kafka.sh
@@ -38,9 +38,9 @@ helm upgrade --install "$KAFKA_RELEASE_NAME" bitnami/kafka \
   --set listeners.client.protocol=PLAINTEXT \
   --set listeners.controller.protocol=PLAINTEXT \
   --set listeners.interbroker.protocol=PLAINTEXT \
-  --set config.offsets\\.topic\\.replication\\.factor=1 \
-  --set config.transaction\\.state\\.log\\.replication\\.factor=1 \
-  --set config.transaction\\.state\\.log\\.min\\.isr=1 \
+  --set overrideConfiguration.offsets\\.topic\\.replication\\.factor=1 \
+  --set overrideConfiguration.transaction\\.state\\.log\\.replication\\.factor=1 \
+  --set overrideConfiguration.transaction\\.state\\.log\\.min\\.isr=1 \
   --set provisioning.enabled=false \
   --wait --timeout "${TIMEOUT_SECONDS}s"
 

From 55b08c3e9f507266631f1533ce4e5cfcc4fc6dbe Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Fri, 14 Aug 2026 16:37:03 +0200
Subject: [PATCH 29/33] Fix CI

---
 .github/workflows/ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e605b63..c5924d6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -374,18 +374,18 @@ jobs:
           group_members=''
           while [ "$(date +%s)" -lt "$end" ]; do
             group_members=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ms_test_stream --members --verbose 2>&1 || true)
-            if printf '%s\n' "$group_members" | grep -Fq 'my-docs(0)'; then
-              printf 'Kafka consumer group ms_test_stream is assigned my-docs(0)\n'
+            if printf '%s\n' "$group_members" | grep -Eq 'my-docs(:0|\(0\))'; then
+              printf 'Kafka consumer group ms_test_stream is assigned my-docs partition 0\n'
               exit 0
             fi
             now=$(date +%s)
             if [ "$now" -ge "$next_report" ]; then
-              printf 'Kafka consumer assignment: waiting for ms_test_stream to receive my-docs(0)\n%s\n' "$group_members"
+              printf 'Kafka consumer assignment: waiting for ms_test_stream to receive my-docs partition 0\n%s\n' "$group_members"
               next_report=$((now + 30))
             fi
             sleep 5
           done
-          printf 'Kafka consumer group ms_test_stream was not assigned my-docs(0) within %ss\n%s\n' "$TIMEOUT_SECONDS" "$group_members" >&2
+          printf 'Kafka consumer group ms_test_stream was not assigned my-docs partition 0 within %ss\n%s\n' "$TIMEOUT_SECONDS" "$group_members" >&2
           exit 1
       - name: Load Kafka test fixture
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'

From 1467ce38c2c50b8c4f22d42c8f9580989cba55b8 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Sat, 15 Aug 2026 10:38:06 +0200
Subject: [PATCH 30/33] CI refactor

---
 .github/scripts/README.md                     |   8 +
 .../k8s/capture-kafka-start-offsets.sh        |   7 +
 .../scripts/k8s}/collect-diagnostics.sh       |   0
 .../scripts/k8s/create-kafka-test-topics.sh   |   5 +
 .github/scripts/k8s/import-ci-images.sh       |  17 ++
 .github/scripts/k8s/install-k3s.sh            |  12 +
 .../scripts/k8s}/install-kafka.sh             |   0
 .github/scripts/k8s/install-streams-chart.sh  |  24 ++
 .github/scripts/k8s/publish-kafka-fixture.sh  |   8 +
 .../scripts/k8s/run-laravel-cluster-tests.sh  |   9 +
 .../k8s/start-kafka-fixture-producer.sh       |  29 ++
 .../scripts/k8s/verify-kafka-processing.sh    |  56 ++++
 .../k8s/wait-for-application-workloads.sh     |  12 +
 .../k8s/wait-for-generated-pipeline.sh        |  38 +++
 .../k8s/wait-for-kafka-consumer-assignment.sh |  21 ++
 .github/scripts/materialize-chart-version.sh  |  15 +
 .github/workflows/ci.yml                      | 288 ++----------------
 17 files changed, 292 insertions(+), 257 deletions(-)
 create mode 100644 .github/scripts/README.md
 create mode 100644 .github/scripts/k8s/capture-kafka-start-offsets.sh
 rename {dev-environment/k8s_tests => .github/scripts/k8s}/collect-diagnostics.sh (100%)
 create mode 100644 .github/scripts/k8s/create-kafka-test-topics.sh
 create mode 100644 .github/scripts/k8s/import-ci-images.sh
 create mode 100644 .github/scripts/k8s/install-k3s.sh
 rename {dev-environment/k8s_tests => .github/scripts/k8s}/install-kafka.sh (100%)
 create mode 100644 .github/scripts/k8s/install-streams-chart.sh
 create mode 100644 .github/scripts/k8s/publish-kafka-fixture.sh
 create mode 100644 .github/scripts/k8s/run-laravel-cluster-tests.sh
 create mode 100644 .github/scripts/k8s/start-kafka-fixture-producer.sh
 create mode 100644 .github/scripts/k8s/verify-kafka-processing.sh
 create mode 100644 .github/scripts/k8s/wait-for-application-workloads.sh
 create mode 100644 .github/scripts/k8s/wait-for-generated-pipeline.sh
 create mode 100644 .github/scripts/k8s/wait-for-kafka-consumer-assignment.sh
 create mode 100644 .github/scripts/materialize-chart-version.sh

diff --git a/.github/scripts/README.md b/.github/scripts/README.md
new file mode 100644
index 0000000..4c3e1de
--- /dev/null
+++ b/.github/scripts/README.md
@@ -0,0 +1,8 @@
+# CI scripts
+
+This directory contains helpers owned by GitHub Actions workflows.
+
+- `k8s/` contains K3s E2E orchestration and diagnostics invoked by `.github/workflows/ci.yml`.
+- `materialize-chart-version.sh` materializes the source-archive chart version for CI packaging and releases.
+
+The developer-run local K3d/K3s harness may reuse these CI helpers; its entry points remain in `dev-environment/`.
diff --git a/.github/scripts/k8s/capture-kafka-start-offsets.sh b/.github/scripts/k8s/capture-kafka-start-offsets.sh
new file mode 100644
index 0000000..0b5373e
--- /dev/null
+++ b/.github/scripts/k8s/capture-kafka-start-offsets.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+input_start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }')
+start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
+echo "INPUT_START_OFFSET=${input_start_offset:-0}" >> "$GITHUB_ENV"
+echo "START_OFFSET=${start_offset:-0}" >> "$GITHUB_ENV"
diff --git a/dev-environment/k8s_tests/collect-diagnostics.sh b/.github/scripts/k8s/collect-diagnostics.sh
similarity index 100%
rename from dev-environment/k8s_tests/collect-diagnostics.sh
rename to .github/scripts/k8s/collect-diagnostics.sh
diff --git a/.github/scripts/k8s/create-kafka-test-topics.sh b/.github/scripts/k8s/create-kafka-test-topics.sh
new file mode 100644
index 0000000..ffe6583
--- /dev/null
+++ b/.github/scripts/k8s/create-kafka-test-topics.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-docs --partitions=1 --replication-factor=1
+kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-results --partitions=1 --replication-factor=1
diff --git a/.github/scripts/k8s/import-ci-images.sh b/.github/scripts/k8s/import-ci-images.sh
new file mode 100644
index 0000000..ec728d7
--- /dev/null
+++ b/.github/scripts/k8s/import-ci-images.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+tag=$(awk '/^appVersion:/ { print $2; exit }' helm-chart/Chart.yaml)
+for image_archive in ci-images/*.tar.gz; do
+  gzip -dc "$image_archive" | docker load
+done
+components=(manticore scaler rules_checker worker ui ui-nginx)
+for component in "${components[@]}"; do
+  docker image inspect "streams/${component}:ci" >/dev/null
+  docker tag "streams/${component}:ci" "ghcr.io/manticoresoftware/streams/${component}:${tag}"
+done
+for component in "${components[@]}"; do
+  image="ghcr.io/manticoresoftware/streams/${component}:${tag}"
+  docker image save "$image" | sudo k3s ctr -n k8s.io images import -
+  sudo k3s ctr -n k8s.io images list -q | grep -Fx "$image"
+done
diff --git a/.github/scripts/k8s/install-k3s.sh b/.github/scripts/k8s/install-k3s.sh
new file mode 100644
index 0000000..6b5aaaa
--- /dev/null
+++ b/.github/scripts/k8s/install-k3s.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+curl -fsSL https://get.k3s.io | INSTALL_K3S_VERSION="$K3S_VERSION" sh -s - server --disable=traefik --disable=servicelb
+sudo chmod 644 /etc/rancher/k3s/k3s.yaml
+echo 'KUBECONFIG=/etc/rancher/k3s/k3s.yaml' >> "$GITHUB_ENV"
+end=$(( $(date +%s) + TIMEOUT_SECONDS ))
+until sudo k3s kubectl get nodes --no-headers 2>/dev/null | grep -q .; do
+  [ "$(date +%s)" -lt "$end" ] || { echo 'K3s did not register a node in time' >&2; sudo journalctl -u k3s --no-pager; exit 1; }
+  sleep 2
+done
+sudo k3s kubectl wait --for=condition=Ready node --all --timeout="${TIMEOUT_SECONDS}s"
diff --git a/dev-environment/k8s_tests/install-kafka.sh b/.github/scripts/k8s/install-kafka.sh
similarity index 100%
rename from dev-environment/k8s_tests/install-kafka.sh
rename to .github/scripts/k8s/install-kafka.sh
diff --git a/.github/scripts/k8s/install-streams-chart.sh b/.github/scripts/k8s/install-streams-chart.sh
new file mode 100644
index 0000000..0d6b228
--- /dev/null
+++ b/.github/scripts/k8s/install-streams-chart.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export TIMEOUT_SECONDS=600
+bash .github/scripts/k8s/install-kafka.sh
+tag=$(awk '/^appVersion:/ { print $2; exit }' helm-chart/Chart.yaml)
+ci_overrides=(
+  --set ingress.enabled=false
+  --set podSecurityPolicy.enabled=false
+  --set-json 'imagePullSecrets=[]'
+  --set 'worker.resources.requests.cpu=250m'
+  --set 'worker.resources.requests.memory=512Mi'
+  --set 'worker.resources.limits.cpu=1'
+  --set 'worker.resources.limits.memory=1Gi'
+  --set-string "ui.image.tag=${tag}"
+  --set-string "ui.nginx.image.tag=${tag}"
+  --set-string "scaler.php.image.tag=${tag}"
+  --set-string "worker.image.tag=${tag}"
+  --set-string "manticore.image.tag=${tag}"
+  --set-string "rulesChecker.image.tag=${tag}"
+)
+helm template "$RELEASE_NAME" helm-chart --kube-version v1.32.5 "${ci_overrides[@]}" > artifacts-rendered-helm.yaml
+mkdir -p artifacts
+helm upgrade --install "$RELEASE_NAME" helm-chart --namespace "$APP_NAMESPACE" --create-namespace --wait --timeout 600s "${ci_overrides[@]}" 2>&1 | tee artifacts/helm-upgrade.log
diff --git a/.github/scripts/k8s/publish-kafka-fixture.sh b/.github/scripts/k8s/publish-kafka-fixture.sh
new file mode 100644
index 0000000..d227706
--- /dev/null
+++ b/.github/scripts/k8s/publish-kafka-fixture.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- sh -c 'timeout 120s /opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server my-kafka.kafka.svc.cluster.local:9092 --topic my-docs < /tmp/test_data.json'
+input_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }')
+expected_input_offset=$((INPUT_START_OFFSET + FIXTURE_RECORDS))
+printf 'Kafka input: start=%s current=%s expected=%s\n' "$INPUT_START_OFFSET" "${input_offset:-0}" "$expected_input_offset"
+[ "${input_offset:-0}" -eq "$expected_input_offset" ]
diff --git a/.github/scripts/k8s/run-laravel-cluster-tests.sh b/.github/scripts/k8s/run-laravel-cluster-tests.sh
new file mode 100644
index 0000000..aead368
--- /dev/null
+++ b/.github/scripts/k8s/run-laravel-cluster-tests.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c '
+  test -x ./vendor/bin/phpunit
+  php -d short_open_tag=off ./vendor/bin/phpunit --testsuite Cluster --list-tests --stderr | tee /tmp/cluster-test-list.txt
+  grep -Fqx " - Tests\\Cluster\\ClusterTest::assignUser" /tmp/cluster-test-list.txt
+  php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Cluster --stderr
+'
diff --git a/.github/scripts/k8s/start-kafka-fixture-producer.sh b/.github/scripts/k8s/start-kafka-fixture-producer.sh
new file mode 100644
index 0000000..216a437
--- /dev/null
+++ b/.github/scripts/k8s/start-kafka-fixture-producer.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+kubectl -n "$KAFKA_NAMESPACE" apply -f - <<EOF
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ${KAFKA_FIXTURE_PRODUCER_POD}
+  labels:
+    app.kubernetes.io/name: kafka-fixture-producer
+spec:
+  restartPolicy: Never
+  containers:
+    - name: producer
+      image: docker.io/bitnamilegacy/kafka:4.0.0-debian-12-r10
+      command: ["sh", "-c", "sleep infinity"]
+      resources:
+        requests:
+          cpu: 250m
+          memory: 512Mi
+        limits:
+          cpu: "1"
+          memory: 1Gi
+EOF
+kubectl -n "$KAFKA_NAMESPACE" wait --for=condition=Ready "pod/${KAFKA_FIXTURE_PRODUCER_POD}" --timeout="${TIMEOUT_SECONDS}s"
+kubectl -n "$KAFKA_NAMESPACE" cp dev-environment/kafka/test_data.tar.gz "${KAFKA_FIXTURE_PRODUCER_POD}:/tmp/test_data.tar.gz"
+kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- tar -xzf /tmp/test_data.tar.gz -C /tmp
+fixture_records=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- sh -c 'wc -l < /tmp/test_data.json')
+echo "FIXTURE_RECORDS=${fixture_records}" >> "$GITHUB_ENV"
diff --git a/.github/scripts/k8s/verify-kafka-processing.sh b/.github/scripts/k8s/verify-kafka-processing.sh
new file mode 100644
index 0000000..8341813
--- /dev/null
+++ b/.github/scripts/k8s/verify-kafka-processing.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+end=$(( $(date +%s) + TIMEOUT_SECONDS ))
+expected_offset=$((START_OFFSET + EXPECTED_RECORDS))
+next_report=0
+pipeline_name=${PIPELINE#"${RELEASE_NAME}"-}
+while [ "$(date +%s)" -lt "$end" ]; do
+  current_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
+  current_offset=${current_offset:-0}
+  if [ "$current_offset" -eq "$expected_offset" ]; then
+    printf 'Processed exactly %s records\n' "$EXPECTED_RECORDS"
+    exit 0
+  fi
+  now=$(date +%s)
+  if [ "$now" -ge "$next_report" ]; then
+    printf 'Kafka output: start=%s current=%s expected=%s\n' "$START_OFFSET" "$current_offset" "$expected_offset"
+    kubectl -n "$APP_NAMESPACE" get statefulset "$PIPELINE"
+    kubectl -n "$APP_NAMESPACE" get pods -l "name=${pipeline_name}" -o wide
+    for pod in $(kubectl -n "$APP_NAMESPACE" get pods -l "name=${pipeline_name}" -o name); do
+      for container in $(kubectl -n "$APP_NAMESPACE" get "$pod" -o jsonpath='{.spec.containers[*].name}'); do
+        case "$container" in
+          *-worker)
+            echo "--- Pipeline worker logs: $pod ($container), last 100 lines ---"
+            kubectl -n "$APP_NAMESPACE" logs "$pod" -c "$container" --tail=100 --prefix || true
+            ;;
+        esac
+      done
+    done
+    next_report=$((now + 30))
+  fi
+  sleep 10
+done
+echo "Pipeline $PIPELINE did not produce offset $expected_offset within ${TIMEOUT_SECONDS}s" >&2
+echo '--- Kafka topic offsets ---'
+kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs || true
+kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results || true
+echo '--- Kafka consumer group ms_test_stream ---'
+kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ms_test_stream || true
+kubectl -n "$APP_NAMESPACE" describe "statefulset/$PIPELINE" || true
+kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -30 || true
+kubectl -n "$APP_NAMESPACE" get pods -l "name=${pipeline_name}" -o wide || true
+for pod in $(kubectl -n "$APP_NAMESPACE" get pods -l "name=${pipeline_name}" -o name); do
+  kubectl -n "$APP_NAMESPACE" describe "$pod" || true
+  for container in $(kubectl -n "$APP_NAMESPACE" get "$pod" -o jsonpath='{.spec.containers[*].name}'); do
+    case "$container" in
+      *-manticore|*-worker)
+        echo "--- Pipeline container logs: $pod ($container) ---"
+        kubectl -n "$APP_NAMESPACE" logs "$pod" -c "$container" --prefix || true
+        echo "--- Previous pipeline container logs: $pod ($container) ---"
+        kubectl -n "$APP_NAMESPACE" logs "$pod" -c "$container" --previous --prefix || true
+        ;;
+    esac
+  done
+done
+exit 1
diff --git a/.github/scripts/k8s/wait-for-application-workloads.sh b/.github/scripts/k8s/wait-for-application-workloads.sh
new file mode 100644
index 0000000..d0c9ec3
--- /dev/null
+++ b/.github/scripts/k8s/wait-for-application-workloads.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+kubectl get namespace "$APP_NAMESPACE"
+kubectl -n "$KAFKA_NAMESPACE" get pod "$KAFKA_POD"
+kubectl -n "$APP_NAMESPACE" rollout status deployment/"$UI_DEPLOYMENT" --timeout="${TIMEOUT_SECONDS}s"
+kubectl -n "$APP_NAMESPACE" rollout status deployment/"${RELEASE_NAME}-manticoresearch-scaler" --timeout="${TIMEOUT_SECONDS}s"
+kubectl -n "$APP_NAMESPACE" rollout status statefulset/"${RELEASE_NAME}-manticoresearch-columnar" --timeout="${TIMEOUT_SECONDS}s"
+kubectl -n "$APP_NAMESPACE" rollout status statefulset/"${RELEASE_NAME}-manticoresearch-ui-mysql" --timeout="${TIMEOUT_SECONDS}s"
+ui_pod=$(kubectl -n "$APP_NAMESPACE" get pods -l name=manticoresearch-ui -o jsonpath='{.items[0].metadata.name}')
+test -n "$ui_pod"
+echo "UI_POD=$ui_pod" >> "$GITHUB_ENV"
diff --git a/.github/scripts/k8s/wait-for-generated-pipeline.sh b/.github/scripts/k8s/wait-for-generated-pipeline.sh
new file mode 100644
index 0000000..67cb5c9
--- /dev/null
+++ b/.github/scripts/k8s/wait-for-generated-pipeline.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+started=$(date +%s)
+end=$(( $(date +%s) + PIPELINE_CREATION_TIMEOUT_SECONDS ))
+next_report=$started
+pipeline=''
+while [ "$(date +%s)" -lt "$end" ]; do
+  pipeline=$(kubectl -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker -o jsonpath='{range .items[*]}{.metadata.name}{end}')
+  [ -n "$pipeline" ] && break
+  now=$(date +%s)
+  if [ "$now" -ge "$next_report" ]; then
+    printf 'Pipeline creation: waiting (%ss elapsed, %ss remaining)\n' "$((now - started))" "$((end - now))"
+    kubectl -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker
+    next_report=$((now + 30))
+  fi
+  sleep 5
+done
+if [ -z "$pipeline" ]; then
+  echo "Pipeline creation: no StatefulSet after ${PIPELINE_CREATION_TIMEOUT_SECONDS}s" >&2
+  echo '--- Pipeline failure summary ---' >&2
+  kubectl -n "$APP_NAMESPACE" get pods,statefulsets,jobs -o wide || true
+  kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -20 || true
+  echo '--- UI service account permissions (statefulsets/services/PVCs) ---' >&2
+  kubectl auth can-i create statefulsets.apps --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true
+  kubectl auth can-i create services --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true
+  kubectl auth can-i create persistentvolumeclaims --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true
+  echo '--- UI PHP logs (last 100 lines) ---' >&2
+  kubectl -n "$APP_NAMESPACE" logs "$UI_POD" -c "$UI_CONTAINER" --tail=100 --prefix || true
+  echo 'Detailed pod logs and full resources are in the k8s-e2e-diagnostics artifact.' >&2
+  exit 1
+fi
+printf 'Pipeline creation: found %s after %ss\n' "$pipeline" "$(( $(date +%s) - started ))"
+kubectl -n "$APP_NAMESPACE" rollout status "statefulset/$pipeline" --timeout="${TIMEOUT_SECONDS}s"
+kubectl -n "$APP_NAMESPACE" get statefulset "$pipeline"
+pipeline_name=${pipeline#"${RELEASE_NAME}"-}
+kubectl -n "$APP_NAMESPACE" get pods -l "name=${pipeline_name}" -o wide
+echo "PIPELINE=$pipeline" >> "$GITHUB_ENV"
diff --git a/.github/scripts/k8s/wait-for-kafka-consumer-assignment.sh b/.github/scripts/k8s/wait-for-kafka-consumer-assignment.sh
new file mode 100644
index 0000000..5d2a451
--- /dev/null
+++ b/.github/scripts/k8s/wait-for-kafka-consumer-assignment.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+end=$(( $(date +%s) + TIMEOUT_SECONDS ))
+next_report=0
+group_members=''
+while [ "$(date +%s)" -lt "$end" ]; do
+  group_members=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ms_test_stream --members --verbose 2>&1 || true)
+  if printf '%s\n' "$group_members" | grep -Eq 'my-docs(:0|\(0\))'; then
+    printf 'Kafka consumer group ms_test_stream is assigned my-docs partition 0\n'
+    exit 0
+  fi
+  now=$(date +%s)
+  if [ "$now" -ge "$next_report" ]; then
+    printf 'Kafka consumer assignment: waiting for ms_test_stream to receive my-docs partition 0\n%s\n' "$group_members"
+    next_report=$((now + 30))
+  fi
+  sleep 5
+done
+printf 'Kafka consumer group ms_test_stream was not assigned my-docs partition 0 within %ss\n%s\n' "$TIMEOUT_SECONDS" "$group_members" >&2
+exit 1
diff --git a/.github/scripts/materialize-chart-version.sh b/.github/scripts/materialize-chart-version.sh
new file mode 100644
index 0000000..a43c52f
--- /dev/null
+++ b/.github/scripts/materialize-chart-version.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+chart=${1:-helm-chart/Chart.yaml}
+short_sha=${SHORT_SHA:-$(git rev-parse --short HEAD)}
+token="\$Format:%h\$"
+
+source=$(<"$chart")
+stripped=${source//"$token"/}
+replacements=$(( (${#source} - ${#stripped}) / ${#token} ))
+[ "$replacements" -eq 2 ] || {
+  echo "expected two development-version tokens in $chart, found $replacements" >&2
+  exit 1
+}
+printf '%s\n' "${source//"$token"/$short_sha}" > "$chart"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c5924d6..4e4ba89 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -42,7 +42,10 @@ jobs:
       - uses: docker/setup-buildx-action@v3
       - run: docker build -t streams/${{ matrix.component }}:ci -f docker/${{ matrix.component }}/Dockerfile docker/${{ matrix.component }}
       - name: Archive validated image
-        run: docker save streams/${{ matrix.component }}:ci | gzip > ${{ matrix.component }}.tar.gz
+        run: |
+          set -euo pipefail
+          docker save streams/${{ matrix.component }}:ci | gzip > ${{ matrix.component }}.tar.gz
+          gzip -t ${{ matrix.component }}.tar.gz
       - uses: actions/upload-artifact@v4
         with:
           name: ci-image-${{ matrix.component }}
@@ -64,7 +67,10 @@ jobs:
       - run: cp -a ui/. docker/${{ matrix.component }}/source/
       - run: docker build -t streams/${{ matrix.component }}:ci -f docker/${{ matrix.component }}/Dockerfile docker/${{ matrix.component }}
       - name: Archive validated image
-        run: docker save streams/${{ matrix.component }}:ci | gzip > ${{ matrix.component }}.tar.gz
+        run: |
+          set -euo pipefail
+          docker save streams/${{ matrix.component }}:ci | gzip > ${{ matrix.component }}.tar.gz
+          gzip -t ${{ matrix.component }}.tar.gz
       - uses: actions/upload-artifact@v4
         with:
           name: ci-image-${{ matrix.component }}
@@ -88,7 +94,10 @@ jobs:
       - run: cp sources/src/target/KafkaPublisher-1.0-SNAPSHOT-jar-with-dependencies.jar docker/worker/KafkaHandler.jar
       - run: docker build -t streams/worker:ci docker/worker
       - name: Archive validated image
-        run: docker save streams/worker:ci | gzip > worker.tar.gz
+        run: |
+          set -euo pipefail
+          docker save streams/worker:ci | gzip > worker.tar.gz
+          gzip -t worker.tar.gz
       - uses: actions/upload-artifact@v4
         with:
           name: ci-image-worker
@@ -103,16 +112,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: azure/setup-helm@v4
       - name: Materialize development chart version
-        run: |
-          set -eux
-          short_sha=$(git rev-parse --short HEAD)
-          chart=helm-chart/Chart.yaml
-          token="\$Format:%h\$"
-          source=$(<"$chart")
-          stripped=${source//"$token"/}
-          replacements=$(( (${#source} - ${#stripped}) / ${#token} ))
-          [ "$replacements" -eq 2 ] || { echo "expected two development-version tokens, found $replacements"; exit 1; }
-          printf '%s\n' "${source//"$token"/$short_sha}" > "$chart"
+        run: bash dev-environment/ci/materialize-chart-version.sh
       - run: helm lint helm-chart
       - run: helm template manticore-streams helm-chart --kube-version v1.32.5 > rendered-default.yaml
       - run: docker run --rm -v "$PWD:/work" -w /work ghcr.io/yannh/kubeconform:v0.6.7 -strict -summary -kubernetes-version 1.32.5 rendered-default.yaml
@@ -151,7 +151,7 @@ jobs:
           merge-multiple: true
       - name: Load Laravel test images
         run: |
-          set -eux
+          set -euxo pipefail
           for component in manticore ui; do
             gzip -dc "ci-images/${component}.tar.gz" | docker load
             docker image inspect "streams/${component}:ci" >/dev/null
@@ -216,29 +216,10 @@ jobs:
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
       - name: Install K3s
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: |
-          set -eux
-          curl -fsSL https://get.k3s.io | INSTALL_K3S_VERSION="$K3S_VERSION" sh -s - server --disable=traefik --disable=servicelb
-          sudo chmod 644 /etc/rancher/k3s/k3s.yaml
-          echo 'KUBECONFIG=/etc/rancher/k3s/k3s.yaml' >> "$GITHUB_ENV"
-          end=$(( $(date +%s) + TIMEOUT_SECONDS ))
-          until sudo k3s kubectl get nodes --no-headers 2>/dev/null | grep -q .; do
-            [ "$(date +%s)" -lt "$end" ] || { echo 'K3s did not register a node in time' >&2; sudo journalctl -u k3s --no-pager; exit 1; }
-            sleep 2
-          done
-          sudo k3s kubectl wait --for=condition=Ready node --all --timeout="${TIMEOUT_SECONDS}s"
+        run: bash .github/scripts/k8s/install-k3s.sh
       - name: Materialize development chart version
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: |
-          set -eux
-          short_sha=$(git rev-parse --short HEAD)
-          chart=helm-chart/Chart.yaml
-          token="\$Format:%h\$"
-          source=$(<"$chart")
-          stripped=${source//"$token"/}
-          replacements=$(( (${#source} - ${#stripped}) / ${#token} ))
-          [ "$replacements" -eq 2 ] || { echo "expected two development-version tokens, found $replacements"; exit 1; }
-          printf '%s\n' "${source//"$token"/$short_sha}" > "$chart"
+        run: bash .github/scripts/materialize-chart-version.sh
       - name: Download validated images
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         uses: actions/download-artifact@v4
@@ -248,66 +229,16 @@ jobs:
           merge-multiple: true
       - name: Import validated images into K3s
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: |
-          set -eux
-          tag=$(awk '/^appVersion:/ { print $2; exit }' helm-chart/Chart.yaml)
-          for image_archive in ci-images/*.tar.gz; do
-            gzip -dc "$image_archive" | docker load
-          done
-          components=(manticore scaler rules_checker worker ui ui-nginx)
-          for component in "${components[@]}"; do
-            docker image inspect "streams/${component}:ci" >/dev/null
-            docker tag "streams/${component}:ci" "ghcr.io/manticoresoftware/streams/${component}:${tag}"
-          done
-          for component in "${components[@]}"; do
-            image="ghcr.io/manticoresoftware/streams/${component}:${tag}"
-            docker image save "$image" | sudo k3s ctr -n k8s.io images import -
-            sudo k3s ctr -n k8s.io images list -q | grep -Fx "$image"
-          done
+        run: bash .github/scripts/k8s/import-ci-images.sh
       - name: Install Kafka and chart
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: |
-          set -eux
-          export TIMEOUT_SECONDS=600
-          sh dev-environment/k8s_tests/install-kafka.sh
-          tag=$(awk '/^appVersion:/ { print $2; exit }' helm-chart/Chart.yaml)
-          ci_overrides=(
-            --set ingress.enabled=false
-            --set podSecurityPolicy.enabled=false
-            --set-json 'imagePullSecrets=[]'
-            --set 'worker.resources.requests.cpu=250m'
-            --set 'worker.resources.requests.memory=512Mi'
-            --set 'worker.resources.limits.cpu=1'
-            --set 'worker.resources.limits.memory=1Gi'
-            --set-string "ui.image.tag=${tag}"
-            --set-string "ui.nginx.image.tag=${tag}"
-            --set-string "scaler.php.image.tag=${tag}"
-            --set-string "worker.image.tag=${tag}"
-            --set-string "manticore.image.tag=${tag}"
-            --set-string "rulesChecker.image.tag=${tag}"
-          )
-          helm template manticore-streams helm-chart --kube-version v1.32.5 "${ci_overrides[@]}" > artifacts-rendered-helm.yaml
-          mkdir -p artifacts
-          helm upgrade --install manticore-streams helm-chart --namespace manticore-streams --create-namespace --wait --timeout 600s "${ci_overrides[@]}" 2>&1 | tee artifacts/helm-upgrade.log
+        run: bash .github/scripts/k8s/install-streams-chart.sh
       - name: Wait for application workloads
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: |
-          set -eux
-          kubectl get namespace "$APP_NAMESPACE"
-          kubectl -n "$KAFKA_NAMESPACE" get pod "$KAFKA_POD"
-          kubectl -n "$APP_NAMESPACE" rollout status deployment/"$UI_DEPLOYMENT" --timeout="${TIMEOUT_SECONDS}s"
-          kubectl -n "$APP_NAMESPACE" rollout status deployment/"${RELEASE_NAME}-manticoresearch-scaler" --timeout="${TIMEOUT_SECONDS}s"
-          kubectl -n "$APP_NAMESPACE" rollout status statefulset/"${RELEASE_NAME}-manticoresearch-columnar" --timeout="${TIMEOUT_SECONDS}s"
-          kubectl -n "$APP_NAMESPACE" rollout status statefulset/"${RELEASE_NAME}-manticoresearch-ui-mysql" --timeout="${TIMEOUT_SECONDS}s"
-          ui_pod=$(kubectl -n "$APP_NAMESPACE" get pods -l name=manticoresearch-ui -o jsonpath='{.items[0].metadata.name}')
-          test -n "$ui_pod"
-          echo "UI_POD=$ui_pod" >> "$GITHUB_ENV"
+        run: bash .github/scripts/k8s/wait-for-application-workloads.sh
       - name: Create Kafka test topics
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: |
-          set -eux
-          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-docs --partitions=1 --replication-factor=1
-          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-results --partitions=1 --replication-factor=1
+        run: bash .github/scripts/k8s/create-kafka-test-topics.sh
       - name: Migrate and seed Laravel cluster test database
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan migrate:fresh --seed --force
@@ -319,170 +250,28 @@ jobs:
         run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan db:seed --class='\TestDataSeeder' --force
       - name: Run Laravel cluster integration tests
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: |
-          set -eu
-          kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c '
-            test -x ./vendor/bin/phpunit
-            php -d short_open_tag=off ./vendor/bin/phpunit --testsuite Cluster --list-tests --stderr | tee /tmp/cluster-test-list.txt
-            grep -Fqx " - Tests\\Cluster\\ClusterTest::assignUser" /tmp/cluster-test-list.txt
-            php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Cluster --stderr
-          '
+        run: bash .github/scripts/k8s/run-laravel-cluster-tests.sh
       - name: Wait for generated pipeline
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: |
-          set -eu
-          started=$(date +%s)
-          end=$(( $(date +%s) + PIPELINE_CREATION_TIMEOUT_SECONDS ))
-          next_report=$started
-          pipeline=''
-          while [ "$(date +%s)" -lt "$end" ]; do
-            pipeline=$(kubectl -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker -o jsonpath='{range .items[*]}{.metadata.name}{end}')
-            [ -n "$pipeline" ] && break
-            now=$(date +%s)
-            if [ "$now" -ge "$next_report" ]; then
-              printf 'Pipeline creation: waiting (%ss elapsed, %ss remaining)\n' "$((now - started))" "$((end - now))"
-              kubectl -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker
-              next_report=$((now + 30))
-            fi
-            sleep 5
-          done
-          if [ -z "$pipeline" ]; then
-            echo "Pipeline creation: no StatefulSet after ${PIPELINE_CREATION_TIMEOUT_SECONDS}s" >&2
-            echo '--- Pipeline failure summary ---' >&2
-            kubectl -n "$APP_NAMESPACE" get pods,statefulsets,jobs -o wide || true
-            kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -20 || true
-            echo '--- UI service account permissions (statefulsets/services/PVCs) ---' >&2
-            kubectl auth can-i create statefulsets.apps --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true
-            kubectl auth can-i create services --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true
-            kubectl auth can-i create persistentvolumeclaims --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true
-            echo '--- UI PHP logs (last 100 lines) ---' >&2
-            kubectl -n "$APP_NAMESPACE" logs "$UI_POD" -c "$UI_CONTAINER" --tail=100 --prefix || true
-            echo 'Detailed pod logs and full resources are in the k8s-e2e-diagnostics artifact.' >&2
-            exit 1
-          fi
-          printf 'Pipeline creation: found %s after %ss\n' "$pipeline" "$(( $(date +%s) - started ))"
-          kubectl -n "$APP_NAMESPACE" rollout status "statefulset/$pipeline" --timeout="${TIMEOUT_SECONDS}s"
-          kubectl -n "$APP_NAMESPACE" get statefulset "$pipeline"
-          kubectl -n "$APP_NAMESPACE" get pods -l "name=${pipeline#${RELEASE_NAME}-}" -o wide
-          echo "PIPELINE=$pipeline" >> "$GITHUB_ENV"
+        run: bash .github/scripts/k8s/wait-for-generated-pipeline.sh
       - name: Wait for Kafka consumer assignment
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: |
-          set -eu
-          end=$(( $(date +%s) + TIMEOUT_SECONDS ))
-          next_report=0
-          group_members=''
-          while [ "$(date +%s)" -lt "$end" ]; do
-            group_members=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ms_test_stream --members --verbose 2>&1 || true)
-            if printf '%s\n' "$group_members" | grep -Eq 'my-docs(:0|\(0\))'; then
-              printf 'Kafka consumer group ms_test_stream is assigned my-docs partition 0\n'
-              exit 0
-            fi
-            now=$(date +%s)
-            if [ "$now" -ge "$next_report" ]; then
-              printf 'Kafka consumer assignment: waiting for ms_test_stream to receive my-docs partition 0\n%s\n' "$group_members"
-              next_report=$((now + 30))
-            fi
-            sleep 5
-          done
-          printf 'Kafka consumer group ms_test_stream was not assigned my-docs partition 0 within %ss\n%s\n' "$TIMEOUT_SECONDS" "$group_members" >&2
-          exit 1
+        run: bash .github/scripts/k8s/wait-for-kafka-consumer-assignment.sh
       - name: Load Kafka test fixture
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: |
-          set -eux
-          input_start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }')
-          start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
-          echo "INPUT_START_OFFSET=${input_start_offset:-0}" >> "$GITHUB_ENV"
-          echo "START_OFFSET=${start_offset:-0}" >> "$GITHUB_ENV"
+        run: bash .github/scripts/k8s/capture-kafka-start-offsets.sh
       - name: Start Kafka fixture producer
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: |
-          set -eux
-          kubectl -n "$KAFKA_NAMESPACE" apply -f - <<EOF
-          apiVersion: v1
-          kind: Pod
-          metadata:
-            name: ${KAFKA_FIXTURE_PRODUCER_POD}
-            labels:
-              app.kubernetes.io/name: kafka-fixture-producer
-          spec:
-            restartPolicy: Never
-            containers:
-              - name: producer
-                image: docker.io/bitnamilegacy/kafka:4.0.0-debian-12-r10
-                command: ["sh", "-c", "sleep infinity"]
-                resources:
-                  requests:
-                    cpu: 250m
-                    memory: 512Mi
-                  limits:
-                    cpu: "1"
-                    memory: 1Gi
-          EOF
-          kubectl -n "$KAFKA_NAMESPACE" wait --for=condition=Ready "pod/${KAFKA_FIXTURE_PRODUCER_POD}" --timeout="${TIMEOUT_SECONDS}s"
-          kubectl -n "$KAFKA_NAMESPACE" cp dev-environment/kafka/test_data.tar.gz "${KAFKA_FIXTURE_PRODUCER_POD}:/tmp/test_data.tar.gz"
-          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- tar -xzf /tmp/test_data.tar.gz -C /tmp
-          fixture_records=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- sh -c 'wc -l < /tmp/test_data.json')
-          echo "FIXTURE_RECORDS=${fixture_records}" >> "$GITHUB_ENV"
+        run: bash .github/scripts/k8s/start-kafka-fixture-producer.sh
       - name: Publish Kafka test fixture
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: |
-          set -eu
-          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- sh -c 'timeout 120s /opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server my-kafka.kafka.svc.cluster.local:9092 --topic my-docs < /tmp/test_data.json'
-          input_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }')
-          expected_input_offset=$((INPUT_START_OFFSET + FIXTURE_RECORDS))
-          printf 'Kafka input: start=%s current=%s expected=%s\n' "$INPUT_START_OFFSET" "${input_offset:-0}" "$expected_input_offset"
-          [ "${input_offset:-0}" -eq "$expected_input_offset" ]
+        run: bash .github/scripts/k8s/publish-kafka-fixture.sh
       - name: Verify Kafka processing result
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        run: |
-          set -eu
-          end=$(( $(date +%s) + TIMEOUT_SECONDS ))
-          expected_offset=$((START_OFFSET + EXPECTED_RECORDS))
-          next_report=0
-          while [ "$(date +%s)" -lt "$end" ]; do
-            current_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
-            current_offset=${current_offset:-0}
-            if [ "$current_offset" -eq "$expected_offset" ]; then
-              printf 'Processed exactly %s records\n' "$EXPECTED_RECORDS"
-              exit 0
-            fi
-            now=$(date +%s)
-            if [ "$now" -ge "$next_report" ]; then
-              printf 'Kafka output: start=%s current=%s expected=%s\n' "$START_OFFSET" "$current_offset" "$expected_offset"
-              kubectl -n "$APP_NAMESPACE" get statefulset "$PIPELINE"
-              kubectl -n "$APP_NAMESPACE" get pods -l "name=${PIPELINE#${RELEASE_NAME}-}" -o wide
-              next_report=$((now + 30))
-            fi
-            sleep 10
-          done
-          echo "Pipeline $PIPELINE did not produce offset $expected_offset within ${TIMEOUT_SECONDS}s" >&2
-          echo '--- Kafka topic offsets ---'
-          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs || true
-          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results || true
-          echo '--- Kafka consumer group ms_test_stream ---'
-          kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ms_test_stream || true
-          kubectl -n "$APP_NAMESPACE" describe "statefulset/$PIPELINE" || true
-          kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -30 || true
-          kubectl -n "$APP_NAMESPACE" get pods -l "name=${PIPELINE#${RELEASE_NAME}-}" -o wide || true
-          for pod in $(kubectl -n "$APP_NAMESPACE" get pods -l "name=${PIPELINE#${RELEASE_NAME}-}" -o name); do
-            kubectl -n "$APP_NAMESPACE" describe "$pod" || true
-            for container in $(kubectl -n "$APP_NAMESPACE" get "$pod" -o jsonpath='{.spec.containers[*].name}'); do
-              case "$container" in
-                *-manticore|*-worker)
-                  echo "--- Pipeline container logs: $pod ($container) ---"
-                  kubectl -n "$APP_NAMESPACE" logs "$pod" -c "$container" --prefix || true
-                  echo "--- Previous pipeline container logs: $pod ($container) ---"
-                  kubectl -n "$APP_NAMESPACE" logs "$pod" -c "$container" --previous --prefix || true
-                  ;;
-              esac
-            done
-          done
-          exit 1
+        run: bash .github/scripts/k8s/verify-kafka-processing.sh
       - name: Collect diagnostics
         if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true')
-        run: sh dev-environment/k8s_tests/collect-diagnostics.sh artifacts/k8s
+        run: bash .github/scripts/k8s/collect-diagnostics.sh artifacts/k8s
       - uses: actions/upload-artifact@v4
         if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true')
         with:
@@ -509,15 +298,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Materialize image version
         run: |
-          set -eux
-          short_sha=$(git rev-parse --short HEAD)
-          chart=helm-chart/Chart.yaml
-          token="\$Format:%h\$"
-          source=$(<"$chart")
-          stripped=${source//"$token"/}
-          replacements=$(( (${#source} - ${#stripped}) / ${#token} ))
-          [ "$replacements" -eq 2 ] || { echo "expected two development-version tokens, found $replacements"; exit 1; }
-          printf '%s\n' "${source//"$token"/$short_sha}" > "$chart"
+          bash .github/scripts/materialize-chart-version.sh
           version=$(awk '/^appVersion:/ { print $2; exit }' helm-chart/Chart.yaml)
           case "$version" in *"\$Format:"*) echo 'Chart version was not materialized'; exit 1;; esac
           echo "IMAGE_TAG=${version}" >> "$GITHUB_ENV"
@@ -529,7 +310,7 @@ jobs:
           merge-multiple: true
       - name: Push tested images to GHCR
         run: |
-          set -eux
+          set -euxo pipefail
           for image_archive in ci-images/*.tar.gz; do
             gzip -dc "$image_archive" | docker load
           done
@@ -555,14 +336,7 @@ jobs:
         id: package
         run: |
           set -eux
-          short_sha=$(git rev-parse --short HEAD)
-          chart=helm-chart/Chart.yaml
-          token="\$Format:%h\$"
-          source=$(<"$chart")
-          stripped=${source//"$token"/}
-          replacements=$(( (${#source} - ${#stripped}) / ${#token} ))
-          [ "$replacements" -eq 2 ] || { echo "expected two development-version tokens, found $replacements"; exit 1; }
-          printf '%s\n' "${source//"$token"/$short_sha}" > "$chart"
+          bash .github/scripts/materialize-chart-version.sh
           version=$(awk '/^version:/ { print $2; exit }' helm-chart/Chart.yaml)
           app_version=$(awk '/^appVersion:/ { print $2; exit }' helm-chart/Chart.yaml)
           test "$version" = "$app_version"

From 3d3e2375f45b8b910674f45f6b6a73a553e74041 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Sat, 15 Aug 2026 10:44:44 +0200
Subject: [PATCH 31/33] CI refactor

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4e4ba89..08421ef 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -112,7 +112,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: azure/setup-helm@v4
       - name: Materialize development chart version
-        run: bash dev-environment/ci/materialize-chart-version.sh
+        run: bash .github/scripts/materialize-chart-version.sh
       - run: helm lint helm-chart
       - run: helm template manticore-streams helm-chart --kube-version v1.32.5 > rendered-default.yaml
       - run: docker run --rm -v "$PWD:/work" -w /work ghcr.io/yannh/kubeconform:v0.6.7 -strict -summary -kubernetes-version 1.32.5 rendered-default.yaml

From dae8b9fd95c3530782f51e554d1c66941a16b48a Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Mon, 17 Aug 2026 13:55:45 +0200
Subject: [PATCH 32/33] Add dev server for issues investigation

---
 .gitignore                                  |   1 +
 dev-environment/k8s_tests/.env.example      |  14 +
 dev-environment/k8s_tests/README.md         |  60 +++
 dev-environment/k8s_tests/dev-server-k3d.sh | 547 ++++++++++++++++++++
 docs/ci/dev-server-k3d.md                   |  57 ++
 5 files changed, 679 insertions(+)
 create mode 100644 dev-environment/k8s_tests/.env.example
 create mode 100644 dev-environment/k8s_tests/README.md
 create mode 100755 dev-environment/k8s_tests/dev-server-k3d.sh
 create mode 100644 docs/ci/dev-server-k3d.md

diff --git a/.gitignore b/.gitignore
index af4bb21..75531a8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,5 +9,6 @@
 /docker/ui/source/
 /docker/ui-nginx/source/
 /artifacts/
+/dev-environment/k8s_tests/.env
 **/.DS_Store
 **/._*
diff --git a/dev-environment/k8s_tests/.env.example b/dev-environment/k8s_tests/.env.example
new file mode 100644
index 0000000..6a49a5d
--- /dev/null
+++ b/dev-environment/k8s_tests/.env.example
@@ -0,0 +1,14 @@
+# Development-server K3d SSH target. Copy this file to .env and fill in the
+# server details. Authentication uses your SSH agent or ~/.ssh/config by default.
+DEV_CLUSTER_IP=
+DEV_CLUSTER_USER=
+
+# Optional explicit private-key path. Leave empty to use SSH agent/configuration.
+DEV_CLUSTER_SSH_KEY=
+
+# K3d cluster to create, reuse, and stop on the development server.
+CLUSTER_NAME=manticore-streams-dev
+
+# Scratch checkout on the remote server. The remote command synchronizes the
+# current working tree here before building and deploying it.
+DEV_CLUSTER_WORKDIR=/tmp/manticore-streams-k3s
diff --git a/dev-environment/k8s_tests/README.md b/dev-environment/k8s_tests/README.md
new file mode 100644
index 0000000..1c3dbbc
--- /dev/null
+++ b/dev-environment/k8s_tests/README.md
@@ -0,0 +1,60 @@
+# Development-server K3d validation
+
+Run the CI-equivalent Kubernetes deployment and Kafka E2E check in K3d on the amd64 development server over SSH.
+
+Run the commands below from this `dev-environment/k8s_tests` directory.
+
+## Configure
+
+```sh
+cp .env.example .env
+```
+
+Set `DEV_CLUSTER_IP`, `DEV_CLUSTER_USER`, and `CLUSTER_NAME` in `.env`. SSH uses the agent or `~/.ssh/config`; set `DEV_CLUSTER_SSH_KEY` only when a specific key file is needed.
+
+The remote user must have Docker, Java 21, Maven, Helm, `kubectl`, and K3d. The runner adds `$HOME/.local/bin` to the remote `PATH`, which is where the development-server K3d, Helm, and kubectl tools were installed. It uses an isolated empty Docker configuration at `$HOME/.local/share/manticore-streams/docker-anonymous` so stale private registry credentials cannot block public base-image pulls. No sudo is required.
+
+## Development-server commands
+
+```sh
+./dev-server-k3d.sh up
+./dev-server-k3d.sh build
+./dev-server-k3d.sh deploy
+./dev-server-k3d.sh test
+```
+
+Every command uses the SSH target from `.env`; this runner never operates a local cluster. Commands other than `down` first synchronize the current checkout to `DEV_CLUSTER_WORKDIR`.
+
+`up` first checks the remote K3d state. When the configured cluster is already running, it returns immediately without synchronizing the checkout, building images, or deploying workloads.
+
+| Command | Purpose |
+| --- | --- |
+| `up` | Create the configured K3d cluster if needed, or start it when stopped, and wait for its node to become Ready. It does not build or deploy. |
+| `build` | Build the six CI images on the development server without changing the cluster. |
+| `deploy` | Import previously built images and install or upgrade Kafka and Streams. Run `build` first after code changes. |
+| `test` | Run the CI Cluster PHPUnit suite, create its pipeline, publish the Kafka fixture, and verify the exact output count. |
+| `status` | Show cluster nodes and the Kafka and Streams workloads. |
+| `kubectl <args>` | Run `kubectl` against the remote K3d context without manually handling its kubeconfig. |
+| `down` | Delete the configured K3d cluster and all Kubernetes data in it. |
+
+## Debug pods
+
+After `deploy` or while a failing `test` leaves the cluster up, use the runner as a remote `kubectl` proxy. It uses the remote `k3d-$CLUSTER_NAME` context and allocates a TTY for an interactive shell:
+
+```sh
+./dev-server-k3d.sh kubectl get pods -A -o wide
+./dev-server-k3d.sh kubectl -n manticore-streams get events --sort-by=.lastTimestamp
+./dev-server-k3d.sh kubectl -n manticore-streams logs deployment/manticore-streams-manticoresearch-ui --all-containers --tail=200
+./dev-server-k3d.sh kubectl -n manticore-streams exec -it <pod-name> -c manticoresearch-ui -- sh
+./dev-server-k3d.sh kubectl -n kafka logs my-kafka-controller-0 -c kafka --tail=200
+```
+
+To delete the configured K3d cluster and its workloads, persistent volumes, and Kubernetes data, run:
+
+```sh
+./dev-server-k3d.sh down
+```
+
+This runs `k3d cluster delete "$CLUSTER_NAME"` on the remote server and prints a ten-second heartbeat while deletion is in progress. It then removes and verifies any residual cluster Docker resources. It does not require sudo. It retains unrelated Docker images and local diagnostic artifacts, but the K3d cluster and all data stored in it are removed. Recreate it with `./dev-server-k3d.sh up`, then run `build` and `deploy`.
+
+Use `./dev-server-k3d.sh --help` to display the command synopsis.
diff --git a/dev-environment/k8s_tests/dev-server-k3d.sh b/dev-environment/k8s_tests/dev-server-k3d.sh
new file mode 100755
index 0000000..5d2f304
--- /dev/null
+++ b/dev-environment/k8s_tests/dev-server-k3d.sh
@@ -0,0 +1,547 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# A remote-only K3d reproduction of the CI E2E environment on the amd64
+# development server. This machine only synchronizes the checkout and invokes SSH.
+
+ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ENV_FILE=${DEV_CLUSTER_ENV_FILE:-"$SCRIPT_DIR/.env"}
+
+if [ -f "$ENV_FILE" ]; then
+  set -a
+  # shellcheck disable=SC1090
+  . "$ENV_FILE"
+  set +a
+fi
+
+CLUSTER_MODE=${CLUSTER_MODE:-k3d}
+CLUSTER_NAME=${CLUSTER_NAME:-manticore-streams-dev}
+K3S_VERSION=${K3S_VERSION:-v1.32.5-k3s1}
+APP_NAMESPACE=${APP_NAMESPACE:-manticore-streams}
+KAFKA_NAMESPACE=${KAFKA_NAMESPACE:-kafka}
+RELEASE_NAME=${RELEASE_NAME:-manticore-streams}
+KAFKA_RELEASE_NAME=${KAFKA_RELEASE_NAME:-my-kafka}
+TIMEOUT_SECONDS=${TIMEOUT_SECONDS:-600}
+PIPELINE_CREATION_TIMEOUT_SECONDS=${PIPELINE_CREATION_TIMEOUT_SECONDS:-60}
+EXPECTED_RECORDS=${EXPECTED_RECORDS:-9418}
+KAFKA_POD=${KAFKA_POD:-my-kafka-controller-0}
+KAFKA_FIXTURE_PRODUCER_POD=${KAFKA_FIXTURE_PRODUCER_POD:-kafka-fixture-producer}
+UI_DEPLOYMENT=${UI_DEPLOYMENT:-manticore-streams-manticoresearch-ui}
+UI_CONTAINER=${UI_CONTAINER:-manticoresearch-ui}
+PLATFORM=${PLATFORM:-linux/amd64}
+DEV_CLUSTER_WORKDIR=${DEV_CLUSTER_WORKDIR:-/tmp/manticore-streams-k3s}
+
+components=(manticore scaler rules_checker worker ui ui-nginx)
+
+die() {
+  printf 'error: %s\n' "$*" >&2
+  exit 1
+}
+
+require_commands() {
+  local command commands=(docker helm kubectl k3d mvn python3)
+  [ "$CLUSTER_MODE" = k3d ] || die "unsupported CLUSTER_MODE: $CLUSTER_MODE"
+  for command in "${commands[@]}"; do
+    command -v "$command" >/dev/null 2>&1 || die "required command is not installed: $command"
+  done
+  docker info >/dev/null 2>&1 || die 'Docker is not running'
+}
+
+java_home() {
+  if [ -n "${JAVA_HOME:-}" ] && [ -x "$JAVA_HOME/bin/java" ]; then
+    printf '%s\n' "$JAVA_HOME"
+    return
+  fi
+
+  if [ "$(uname -s)" = Darwin ] && [ -x /usr/libexec/java_home ]; then
+    /usr/libexec/java_home -v 21 2>/dev/null || die 'Java 21 is required; install it or set JAVA_HOME'
+    return
+  fi
+
+  die 'JAVA_HOME must point to a Java 21 installation'
+}
+
+kubectl_dev() {
+  kubectl --context "k3d-${CLUSTER_NAME}" "$@"
+}
+
+helm_dev() {
+  helm --kube-context "k3d-${CLUSTER_NAME}" "$@"
+}
+
+image_tag() {
+  if [ -n "${IMAGE_TAG:-}" ]; then
+    printf '%s\n' "$IMAGE_TAG"
+    return
+  fi
+  git -C "$ROOT" rev-parse --short HEAD
+}
+
+version() {
+  local tag
+  tag=$(image_tag)
+  python3 - "$ROOT/helm-chart/Chart.yaml" "$tag" <<'PY'
+import pathlib
+import sys
+
+chart = pathlib.Path(sys.argv[1])
+short_sha = sys.argv[2]
+source = chart.read_text()
+token = '$Format:%h$'
+if source.count(token) != 2:
+    raise SystemExit(f'expected two development-version tokens, found {source.count(token)}')
+for line in source.splitlines():
+    if line.startswith('appVersion: '):
+        base = line.removeprefix('appVersion: ').removesuffix(token)
+        print(f'{base}{short_sha}')
+        break
+else:
+    raise SystemExit('Chart.yaml has no appVersion')
+PY
+}
+
+materialize_chart() {
+  local destination=$1 tag=$2
+  cp -a "$ROOT/helm-chart" "$destination"
+  python3 - "$destination/Chart.yaml" "$tag" <<'PY'
+import pathlib
+import sys
+
+chart = pathlib.Path(sys.argv[1])
+short_sha = sys.argv[2]
+source = chart.read_text()
+token = '$Format:%h$'
+if source.count(token) != 2:
+    raise SystemExit(f'expected two development-version tokens, found {source.count(token)}')
+chart.write_text(source.replace(token, short_sha))
+PY
+}
+
+create_cluster() {
+  if k3d cluster list -o json | python3 -c '
+import json
+import sys
+name = sys.argv[1]
+print("present" if any(cluster["name"] == name for cluster in json.load(sys.stdin)) else "")
+' "$CLUSTER_NAME" | grep -qx present; then
+    k3d cluster start "$CLUSTER_NAME"
+    return
+  fi
+
+  k3d cluster create "$CLUSTER_NAME" \
+    --image "rancher/k3s:${K3S_VERSION}" \
+    --servers 1 \
+    --agents 0 \
+    --k3s-arg '--disable=traefik@server:*' \
+    --k3s-arg '--disable=servicelb@server:*' \
+    --wait
+}
+
+require_cluster() {
+  if ! k3d cluster list -o json | python3 -c '
+import json
+import sys
+name = sys.argv[1]
+raise SystemExit(0 if any(cluster["name"] == name for cluster in json.load(sys.stdin)) else 1)
+' "$CLUSTER_NAME"; then
+    die "k3d cluster does not exist: $CLUSTER_NAME; run up first"
+  fi
+}
+
+wait_for_node() {
+  kubectl_dev wait --for=condition=Ready node --all --timeout="${TIMEOUT_SECONDS}s"
+}
+
+build_images() (
+  local tag staging build_java_home
+  tag=$(version)
+  build_java_home=$(java_home)
+  staging=$(mktemp -d)
+  trap 'rm -rf "$staging"' EXIT
+
+  (
+    cd "$ROOT/sources/src"
+    JAVA_HOME="$build_java_home" mvn clean compile assembly:single
+  )
+
+  cp -a "$ROOT/docker/worker" "$staging/worker"
+  cp "$ROOT/sources/src/target/KafkaPublisher-1.0-SNAPSHOT-jar-with-dependencies.jar" "$staging/worker/KafkaHandler.jar"
+  cp -a "$ROOT/docker/ui" "$staging/ui"
+  cp -a "$ROOT/ui/." "$staging/ui/source/"
+  cp -a "$ROOT/docker/ui-nginx" "$staging/ui-nginx"
+  cp -a "$ROOT/ui/." "$staging/ui-nginx/source/"
+
+  for component in manticore scaler rules_checker; do
+    docker buildx build --platform "$PLATFORM" --load \
+      -t "streams/${component}:ci" \
+      -f "$ROOT/docker/${component}/Dockerfile" "$ROOT/docker/${component}"
+  done
+  for component in worker ui ui-nginx; do
+    docker buildx build --platform "$PLATFORM" --load \
+      -t "streams/${component}:ci" \
+      -f "$staging/${component}/Dockerfile" "$staging/${component}"
+  done
+
+  for component in "${components[@]}"; do
+    docker image inspect "streams/${component}:ci" >/dev/null
+    docker tag "streams/${component}:ci" "ghcr.io/manticoresoftware/streams/${component}:${tag}"
+  done
+)
+
+import_images() {
+  local tag component
+  tag=$(version)
+  for component in "${components[@]}"; do
+    local_image="ghcr.io/manticoresoftware/streams/${component}:${tag}"
+    docker image inspect "$local_image" >/dev/null || die "image was not built: $local_image"
+    k3d image import --cluster "$CLUSTER_NAME" "$local_image"
+  done
+}
+
+chart_overrides() {
+  local tag=$1
+  printf '%s\n' \
+    '--set' 'ingress.enabled=false' \
+    '--set' 'podSecurityPolicy.enabled=false' \
+    '--set-json' 'imagePullSecrets=[]' \
+    '--set' 'worker.resources.requests.cpu=250m' \
+    '--set' 'worker.resources.requests.memory=512Mi' \
+    '--set' 'worker.resources.limits.cpu=1' \
+    '--set' 'worker.resources.limits.memory=1Gi' \
+    '--set-string' "ui.image.tag=${tag}" \
+    '--set-string' "ui.nginx.image.tag=${tag}" \
+    '--set-string' "scaler.php.image.tag=${tag}" \
+    '--set-string' "worker.image.tag=${tag}" \
+    '--set-string' "manticore.image.tag=${tag}" \
+    '--set-string' "rulesChecker.image.tag=${tag}"
+}
+
+deploy() {
+  local tag chart
+  tag=$(version)
+  chart=$(mktemp -d)
+  materialize_chart "$chart/chart" "$tag"
+  overrides=()
+  while IFS= read -r override; do
+    overrides+=("$override")
+  done < <(chart_overrides "$tag")
+
+  export KAFKA_NAMESPACE KAFKA_RELEASE_NAME TIMEOUT_SECONDS
+  (
+    cd "$ROOT"
+    bash .github/scripts/k8s/install-kafka.sh
+  )
+  helm_dev upgrade --install "$RELEASE_NAME" "$chart/chart" \
+    --namespace "$APP_NAMESPACE" --create-namespace --wait --timeout "${TIMEOUT_SECONDS}s" \
+    "${overrides[@]}"
+  rm -rf "$chart"
+}
+
+wait_for_workloads() {
+  local ui_pod
+  kubectl_dev get namespace "$APP_NAMESPACE"
+  kubectl_dev -n "$KAFKA_NAMESPACE" get pod "$KAFKA_POD"
+  kubectl_dev -n "$APP_NAMESPACE" rollout status "deployment/${UI_DEPLOYMENT}" --timeout="${TIMEOUT_SECONDS}s"
+  kubectl_dev -n "$APP_NAMESPACE" rollout status "deployment/${RELEASE_NAME}-manticoresearch-scaler" --timeout="${TIMEOUT_SECONDS}s"
+  kubectl_dev -n "$APP_NAMESPACE" rollout status "statefulset/${RELEASE_NAME}-manticoresearch-columnar" --timeout="${TIMEOUT_SECONDS}s"
+  kubectl_dev -n "$APP_NAMESPACE" rollout status "statefulset/${RELEASE_NAME}-manticoresearch-ui-mysql" --timeout="${TIMEOUT_SECONDS}s"
+  ui_pod=$(kubectl_dev -n "$APP_NAMESPACE" get pods -l name=manticoresearch-ui -o jsonpath='{.items[0].metadata.name}')
+  test -n "$ui_pod" || die 'UI pod was not found'
+  printf '%s\n' "$ui_pod"
+}
+
+restore_ui_test_config() {
+  local ui_pod=$1
+  kubectl_dev -n "$APP_NAMESPACE" exec "$ui_pod" -c "$UI_CONTAINER" -- sh -c '
+    sed -i "s/^APP_ENV=cluster_testing$/APP_ENV=production/" .env
+    sed -i "/name=\"APP_ENV\"/ s/value=\"[^\"]*\"/value=\"testing\"/" phpunit.xml
+  '
+}
+
+configure_ui_cluster_test_config() {
+  local ui_pod=$1
+  kubectl_dev -n "$APP_NAMESPACE" exec "$ui_pod" -c "$UI_CONTAINER" -- sh -c '
+    sed -i "s/^APP_ENV=production$/APP_ENV=cluster_testing/" .env
+    sed -i "/name=\"APP_ENV\"/ s/value=\"[^\"]*\"/value=\"cluster_testing\"/" phpunit.xml
+  '
+}
+
+run_e2e() (
+  local ui_pod pipeline input_start_offset fixture_records input_offset expected_input_offset start_offset current_offset end started next_report now group_members
+  wait_for_workloads >/dev/null
+  ui_pod=$(kubectl_dev -n "$APP_NAMESPACE" get pods -l name=manticoresearch-ui -o jsonpath='{.items[0].metadata.name}')
+  test -n "$ui_pod" || die 'UI pod was not found'
+  restore_ui_test_config "$ui_pod"
+  trap 'restore_ui_test_config "$ui_pod" || true' EXIT
+  kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-docs --partitions=1 --replication-factor=1
+  kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-results --partitions=1 --replication-factor=1
+  kubectl_dev -n "$APP_NAMESPACE" exec "$ui_pod" -c "$UI_CONTAINER" -- php artisan migrate:fresh --seed --force
+  configure_ui_cluster_test_config "$ui_pod"
+  kubectl_dev -n "$APP_NAMESPACE" exec "$ui_pod" -c "$UI_CONTAINER" -- php artisan db:seed --class='\TestDataSeeder' --force
+  kubectl_dev -n "$APP_NAMESPACE" exec "$ui_pod" -c "$UI_CONTAINER" -- sh -c '
+    test -x ./vendor/bin/phpunit
+    php -d short_open_tag=off ./vendor/bin/phpunit --testsuite Cluster --list-tests --stderr | tee /tmp/cluster-test-list.txt
+    grep -Fqx " - Tests\\Cluster\\ClusterTest::assignUser" /tmp/cluster-test-list.txt
+    php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Cluster --stderr
+  '
+
+  started=$(date +%s)
+  end=$(( started + PIPELINE_CREATION_TIMEOUT_SECONDS ))
+  next_report=$started
+  pipeline=''
+  while [ "$(date +%s)" -lt "$end" ]; do
+    pipeline=$(kubectl_dev -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker -o jsonpath='{range .items[*]}{.metadata.name}{end}')
+    [ -n "$pipeline" ] && break
+    now=$(date +%s)
+    if [ "$now" -ge "$next_report" ]; then
+      printf 'Pipeline creation: waiting (%ss elapsed, %ss remaining)\n' "$((now - started))" "$((end - now))"
+      kubectl_dev -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker
+      next_report=$((now + 30))
+    fi
+    sleep 5
+  done
+  [ -n "$pipeline" ] || die "Cluster test did not create a pipeline StatefulSet within ${PIPELINE_CREATION_TIMEOUT_SECONDS}s"
+  kubectl_dev -n "$APP_NAMESPACE" rollout status "statefulset/$pipeline" --timeout="${TIMEOUT_SECONDS}s"
+
+  end=$(( $(date +%s) + TIMEOUT_SECONDS ))
+  next_report=0
+  group_members=''
+  while [ "$(date +%s)" -lt "$end" ]; do
+    group_members=$(kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ms_test_stream --members --verbose 2>&1 || true)
+    if printf '%s\n' "$group_members" | grep -Eq 'my-docs(:0|\(0\))'; then
+      printf 'Kafka consumer group ms_test_stream is assigned my-docs partition 0\n'
+      break
+    fi
+    now=$(date +%s)
+    if [ "$now" -ge "$next_report" ]; then
+      printf 'Kafka consumer assignment: waiting for ms_test_stream to receive my-docs partition 0\n%s\n' "$group_members"
+      next_report=$((now + 30))
+    fi
+    sleep 5
+  done
+  printf '%s\n' "$group_members" | grep -Eq 'my-docs(:0|\(0\))' || die "Kafka consumer group ms_test_stream was not assigned my-docs partition 0 within ${TIMEOUT_SECONDS}s"
+
+  input_start_offset=$(kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }')
+  start_offset=$(kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
+  kubectl_dev -n "$KAFKA_NAMESPACE" apply -f - <<EOF
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ${KAFKA_FIXTURE_PRODUCER_POD}
+spec:
+  restartPolicy: Never
+  containers:
+    - name: producer
+      image: docker.io/bitnamilegacy/kafka:4.0.0-debian-12-r10
+      command: ["sh", "-c", "sleep infinity"]
+      resources:
+        requests: {cpu: 250m, memory: 512Mi}
+        limits: {cpu: "1", memory: 1Gi}
+EOF
+  kubectl_dev -n "$KAFKA_NAMESPACE" wait --for=condition=Ready "pod/${KAFKA_FIXTURE_PRODUCER_POD}" --timeout="${TIMEOUT_SECONDS}s"
+  kubectl_dev -n "$KAFKA_NAMESPACE" cp "$ROOT/dev-environment/kafka/test_data.tar.gz" "${KAFKA_FIXTURE_PRODUCER_POD}:/tmp/test_data.tar.gz"
+  kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- tar -xzf /tmp/test_data.tar.gz -C /tmp
+  fixture_records=$(kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- sh -c 'wc -l < /tmp/test_data.json')
+  kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- sh -c 'timeout 120s /opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server my-kafka.kafka.svc.cluster.local:9092 --topic my-docs < /tmp/test_data.json'
+  input_offset=$(kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }')
+  expected_input_offset=$(( ${input_start_offset:-0} + fixture_records ))
+  printf 'Kafka input: start=%s current=%s expected=%s\n' "${input_start_offset:-0}" "${input_offset:-0}" "$expected_input_offset"
+  [ "${input_offset:-0}" -eq "$expected_input_offset" ] || die 'Kafka fixture was not fully published'
+
+  end=$(( $(date +%s) + TIMEOUT_SECONDS ))
+  while [ "$(date +%s)" -lt "$end" ]; do
+    current_offset=$(kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }')
+    current_offset=${current_offset:-0}
+    if [ "$current_offset" -eq $(( ${start_offset:-0} + EXPECTED_RECORDS )) ]; then
+      printf 'Processed exactly %s records\n' "$EXPECTED_RECORDS"
+      return
+    fi
+    printf 'Waiting for output records: start=%s current=%s expected=%s\n' "${start_offset:-0}" "$current_offset" "$EXPECTED_RECORDS"
+    sleep 10
+  done
+  kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs || true
+  kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results || true
+  kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ms_test_stream || true
+  die "output offset did not reach $(( ${start_offset:-0} + EXPECTED_RECORDS )) within ${TIMEOUT_SECONDS}s"
+)
+
+run_remote_kubectl() {
+  local target remote_command quoted_arguments
+  local -a ssh_options=(ssh -o BatchMode=yes)
+
+  shift
+  [ "$#" -gt 0 ] || die 'usage: dev-server-k3d.sh kubectl <kubectl arguments>'
+  [ -n "${DEV_CLUSTER_IP:-}" ] || die "DEV_CLUSTER_IP is required in $ENV_FILE"
+  [ -n "${DEV_CLUSTER_USER:-}" ] || die "DEV_CLUSTER_USER is required in $ENV_FILE"
+  command -v ssh >/dev/null 2>&1 || die 'required command is not installed: ssh'
+  if [ -n "${DEV_CLUSTER_SSH_KEY:-}" ]; then
+    [ -f "$DEV_CLUSTER_SSH_KEY" ] || die "DEV_CLUSTER_SSH_KEY does not exist: $DEV_CLUSTER_SSH_KEY"
+    ssh_options+=(-i "$DEV_CLUSTER_SSH_KEY")
+  fi
+  if [ -t 0 ] && [ -t 1 ]; then
+    ssh_options+=(-tt)
+  fi
+
+  target="${DEV_CLUSTER_USER}@${DEV_CLUSTER_IP}"
+  printf -v quoted_arguments ' %q' "$@"
+  # shellcheck disable=SC2016 # $HOME and $PATH must expand on the remote host.
+  printf -v remote_command 'export PATH="$HOME/.local/bin:$PATH"; kubectl --context %q%s' "k3d-${CLUSTER_NAME}" "$quoted_arguments"
+  "${ssh_options[@]}" "$target" "$remote_command"
+}
+
+run_remote() {
+  local command=${1:-up} target ssh_transport remote_command remote_image_tag cluster_check status=0
+  local -a ssh_options=(ssh -o BatchMode=yes)
+
+  case "$command" in
+    up|build|deploy|test|status|down) ;;
+    *) die "remote command must be one of: up, build, deploy, test, status, down" ;;
+  esac
+  [ -n "${DEV_CLUSTER_IP:-}" ] || die "DEV_CLUSTER_IP is required in $ENV_FILE"
+  [ -n "${DEV_CLUSTER_USER:-}" ] || die "DEV_CLUSTER_USER is required in $ENV_FILE"
+  command -v ssh >/dev/null 2>&1 || die 'required command is not installed: ssh'
+  if [ -n "${DEV_CLUSTER_SSH_KEY:-}" ]; then
+    [ -f "$DEV_CLUSTER_SSH_KEY" ] || die "DEV_CLUSTER_SSH_KEY does not exist: $DEV_CLUSTER_SSH_KEY"
+    ssh_options+=(-i "$DEV_CLUSTER_SSH_KEY")
+  fi
+
+  target="${DEV_CLUSTER_USER}@${DEV_CLUSTER_IP}"
+  if [ "$command" = up ]; then
+    # shellcheck disable=SC2016 # $HOME and $PATH must expand on the remote host.
+    printf -v cluster_check 'export PATH="$HOME/.local/bin:$PATH"; k3d cluster list -o json | python3 -c %q %q' \
+      'import json, sys
+name = sys.argv[1]
+clusters = json.load(sys.stdin)
+for cluster in clusters:
+    if cluster["name"] == name:
+        servers = [node for node in cluster["nodes"] if node["role"] == "server"]
+        raise SystemExit(0 if servers and all(node["State"]["Running"] for node in servers) else 1)
+raise SystemExit(1)' "$CLUSTER_NAME"
+    if "${ssh_options[@]}" "$target" "$cluster_check"; then
+      printf 'Remote k3d cluster %s is already running; nothing to do.\n' "$CLUSTER_NAME"
+      return
+    fi
+  fi
+  if [ "$command" = down ]; then
+    local delete_pid elapsed=0 cleanup_command verify_command
+    printf 'Deleting remote K3d cluster %s and all of its data...\n' "$CLUSTER_NAME"
+    "${ssh_options[@]}" "$target" "export PATH=\"\$HOME/.local/bin:\$PATH\"; k3d cluster delete $(printf '%q' "$CLUSTER_NAME")" &
+    delete_pid=$!
+    while kill -0 "$delete_pid" 2>/dev/null; do
+      sleep 10
+      elapsed=$((elapsed + 10))
+      if kill -0 "$delete_pid" 2>/dev/null; then
+        printf 'Still deleting remote cluster (%ss elapsed)...\n' "$elapsed"
+      fi
+    done
+    if ! wait "$delete_pid"; then
+      die "remote K3d cluster deletion failed after ${elapsed}s; run status to inspect the remaining resources"
+    fi
+    # shellcheck disable=SC2016 # $cluster must expand on the remote host.
+    printf -v cleanup_command 'set -e; cluster=%q; containers=$(docker ps -aq --filter "label=k3d.cluster=$cluster"); if [ -n "$containers" ]; then printf "Removing remaining cluster container(s): %%s\\n" "$containers"; docker rm -f $containers; fi; if docker network inspect "k3d-$cluster" >/dev/null 2>&1; then printf "Removing remaining cluster network k3d-%%s\\n" "$cluster"; docker network rm "k3d-$cluster"; fi; if docker volume inspect "k3d-$cluster-images" >/dev/null 2>&1; then printf "Removing remaining cluster image volume k3d-%%s-images\\n" "$cluster"; docker volume rm "k3d-$cluster-images"; fi' "$CLUSTER_NAME"
+    printf 'Checking for remaining remote Docker resources...\n'
+    if ! "${ssh_options[@]}" "$target" "$cleanup_command"; then
+      die 'remote K3d deletion left Docker resources that could not be removed'
+    fi
+    # shellcheck disable=SC2016 # $HOME and $PATH must expand on the remote host.
+    if "${ssh_options[@]}" "$target" "export PATH=\"\$HOME/.local/bin:\$PATH\"; k3d cluster list -o json | python3 -c 'import json, sys; raise SystemExit(0 if any(cluster[\"name\"] == sys.argv[1] for cluster in json.load(sys.stdin)) else 1)' $(printf '%q' "$CLUSTER_NAME")"; then
+      die 'remote K3d reported success, but the cluster still exists'
+    fi
+    # shellcheck disable=SC2016 # $cluster must expand on the remote host.
+    printf -v verify_command 'cluster=%q; test -z "$(docker ps -aq --filter "label=k3d.cluster=$cluster")" && ! docker network inspect "k3d-$cluster" >/dev/null 2>&1 && ! docker volume inspect "k3d-$cluster-images" >/dev/null 2>&1' "$CLUSTER_NAME"
+    if ! "${ssh_options[@]}" "$target" "$verify_command"; then
+      die 'remote K3d cluster metadata was removed, but Docker resources remain'
+    fi
+    printf 'Remote K3d cluster %s was deleted.\n' "$CLUSTER_NAME"
+    return
+  fi
+
+  command -v rsync >/dev/null 2>&1 || die 'required command is not installed: rsync'
+  "${ssh_options[@]}" "$target" "export PATH=\"\$HOME/.local/bin:\$PATH\"; mkdir -p $(printf '%q' "$DEV_CLUSTER_WORKDIR")"
+  printf -v ssh_transport '%q ' "${ssh_options[@]}"
+  printf 'Synchronizing checkout to %s...\n' "$target"
+  rsync -az --progress --delete \
+    --exclude .git --exclude artifacts --exclude dev-environment/k8s_tests/.env \
+    -e "$ssh_transport" "$ROOT/" "${target}:${DEV_CLUSTER_WORKDIR}/"
+  printf 'Checkout synchronized.\n'
+
+  remote_image_tag=$(git -C "$ROOT" rev-parse --short HEAD) || die 'remote mode requires a Git checkout on the local machine'
+  # shellcheck disable=SC2016 # $HOME and $PATH must expand on the remote host.
+  printf -v remote_command 'export PATH="$HOME/.local/bin:$PATH" DOCKER_CONFIG="$HOME/.local/share/manticore-streams/docker-anonymous"; mkdir -p "$DOCKER_CONFIG"; if [ -z "${JAVA_HOME:-}" ] && [ -x "$HOME/.local/jdks/temurin-21/bin/java" ]; then export JAVA_HOME="$HOME/.local/jdks/temurin-21"; fi; cd %q && REMOTE_EXECUTION=1 CLUSTER_MODE=k3d CLUSTER_NAME=%q IMAGE_TAG=%q bash dev-environment/k8s_tests/dev-server-k3d.sh %q' \
+    "$DEV_CLUSTER_WORKDIR" "$CLUSTER_NAME" "$remote_image_tag" "$command"
+  if "${ssh_options[@]}" "$target" "$remote_command"; then
+    :
+  else
+    status=$?
+  fi
+
+  return "$status"
+}
+
+usage() {
+  cat <<'EOF'
+Usage: dev-environment/k8s_tests/dev-server-k3d.sh <command>
+
+Commands:
+  up           Create or start the remote k3d cluster and wait for its node to become Ready.
+  build        Build the six CI images on the development server.
+  deploy       Import existing images and install/upgrade Kafka and Streams on the development server.
+  test         Run the CI Kubernetes application and Kafka fixture checks on the development server.
+  status       Show remote nodes and application/Kafka workloads.
+  kubectl      Run kubectl arguments against the remote K3d cluster.
+  down         Delete the configured remote k3d cluster and all of its data.
+
+Environment overrides: CLUSTER_NAME, K3S_VERSION, PLATFORM, TIMEOUT_SECONDS.
+Remote configuration: dev-environment/k8s_tests/.env (see .env.example).
+EOF
+}
+
+remote_main() {
+  local command=${1:-}
+  case "$command" in
+    up)
+      require_commands
+      create_cluster
+      wait_for_node
+      ;;
+    build) require_commands; build_images ;;
+    deploy)
+      require_commands
+      require_cluster
+      wait_for_node
+      import_images
+      deploy
+      wait_for_workloads >/dev/null
+      ;;
+    test)
+      require_commands
+      run_e2e
+      ;;
+    status)
+      require_commands
+      kubectl_dev get nodes -o wide
+      kubectl_dev -n "$KAFKA_NAMESPACE" get all,cm,pvc -o wide
+      kubectl_dev -n "$APP_NAMESPACE" get all,cm,pvc -o wide
+      ;;
+    -h|--help|help|'') usage ;;
+    *) die "unknown command: $command" ;;
+  esac
+}
+
+main() {
+  local command=${1:-}
+  if [ "${REMOTE_EXECUTION:-0}" = 1 ]; then
+    remote_main "$@"
+    return
+  fi
+
+  case "$command" in
+    up|build|deploy|test|status|down) run_remote "$command" ;;
+    kubectl) run_remote_kubectl "$@" ;;
+    -h|--help|help|'') usage ;;
+    *) die "unknown command: $command" ;;
+  esac
+}
+
+main "$@"
diff --git a/docs/ci/dev-server-k3d.md b/docs/ci/dev-server-k3d.md
new file mode 100644
index 0000000..9f1b7eb
--- /dev/null
+++ b/docs/ci/dev-server-k3d.md
@@ -0,0 +1,57 @@
+# Development-server K3d CI validation
+
+`dev-environment/k8s_tests/dev-server-k3d.sh` reproduces the Kubernetes E2E path in `.github/workflows/ci.yml` on the amd64 development server over SSH. It is a remote-only runner: it does not create, build, deploy, or delete a local Mac K3d cluster.
+
+## Prerequisites
+
+Create the ignored target configuration:
+
+```bash
+cp dev-environment/k8s_tests/.env.example dev-environment/k8s_tests/.env
+```
+
+Set the server IP address, SSH user, and K3d cluster name. SSH uses the agent or `~/.ssh/config`; set `DEV_CLUSTER_SSH_KEY` only when an explicit private-key path is needed. `DEV_CLUSTER_WORKDIR` defaults to a disposable remote checkout at `/tmp/manticore-streams-k3s`.
+
+The remote user needs Docker, Java 21, Maven, Helm, `kubectl`, and K3d. The runner adds `$HOME/.local/bin` to the remote `PATH`, uses an isolated empty Docker configuration at `$HOME/.local/share/manticore-streams/docker-anonymous` so stale credentials cannot block public base-image pulls, and synchronizes the current local working tree (excluding `.git`, artifacts, and the local `.env`). No sudo is required.
+
+## Workflow
+
+Run the commands from `dev-environment/k8s_tests`:
+
+```bash
+./dev-server-k3d.sh up
+./dev-server-k3d.sh build
+./dev-server-k3d.sh deploy
+./dev-server-k3d.sh test
+```
+
+`up` creates the configured K3d cluster if necessary, or starts it when stopped, then waits for its node to become Ready. It does not build images or deploy workloads.
+
+When the configured cluster is already running, `up` exits immediately without synchronizing the checkout or changing the cluster.
+
+`build` compiles the Java worker and builds the six `linux/amd64` CI images on the development server. `deploy` imports those previously built images, installs or upgrades the pinned Kafka chart and Streams Helm chart, and waits for workloads. `deploy` requires a cluster created by `up` and images created by `build`.
+
+`test` runs the Kubernetes-only Cluster PHPUnit suite, including its `cluster_testing` seed, pipeline creation, Kafka fixture publishing, and the exact 9,418-record output assertion from CI.
+
+## Debugging pods
+
+Use the runner as a remote `kubectl` proxy after `deploy`, or while a failing `test` leaves the cluster running. It uses the remote `k3d-$CLUSTER_NAME` context; interactive `exec` sessions receive a TTY:
+
+```bash
+./dev-server-k3d.sh kubectl get pods -A -o wide
+./dev-server-k3d.sh kubectl -n manticore-streams get events --sort-by=.lastTimestamp
+./dev-server-k3d.sh kubectl -n manticore-streams logs deployment/manticore-streams-manticoresearch-ui --all-containers --tail=200
+./dev-server-k3d.sh kubectl -n manticore-streams exec -it <pod-name> -c manticoresearch-ui -- sh
+./dev-server-k3d.sh kubectl -n kafka logs my-kafka-controller-0 -c kafka --tail=200
+```
+
+## Investigation and reset
+
+```bash
+./dev-server-k3d.sh status
+./dev-server-k3d.sh down
+```
+
+`status` shows the remote cluster nodes and Kafka/Streams workloads. Use the `kubectl` proxy for events, pod descriptions, logs, and interactive shells.
+
+`down` runs `k3d cluster delete "$CLUSTER_NAME"` as the remote user and prints a ten-second heartbeat while it waits. It removes and verifies any remaining container, network, or K3d image volume. It deletes workloads, persistent volumes, and Kubernetes data in that K3d cluster, while retaining unrelated Docker images. Recreate the stand with `up`, then run `build` and `deploy`.

From f89fdb457d0051b2fafa21801f68c7e093c03145 Mon Sep 17 00:00:00 2001
From: djklim87 <klim@manticoresearch.com>
Date: Mon, 17 Aug 2026 14:38:12 +0200
Subject: [PATCH 33/33] Security improvements

---
 .../docker-compose/laravel-tests.yml          |  4 +-
 .github/scripts/k8s/collect-diagnostics.sh    |  3 -
 .github/scripts/k8s/install-k3s.sh            | 45 +++++++++-
 .github/scripts/k8s/install-kafka.sh          | 14 +--
 .../k8s/start-kafka-fixture-producer.sh       |  2 +-
 .github/workflows/ci.yml                      | 89 ++++++++++---------
 docker/manticore/Dockerfile                   |  2 +-
 docker/rules_checker/Dockerfile               |  2 +-
 docker/scaler/Dockerfile                      |  2 +-
 docker/ui-nginx/Dockerfile                    |  2 +-
 docker/ui/Dockerfile                          |  2 +-
 docker/worker/Dockerfile                      |  2 +-
 docs/ci/github-actions-migration-decisions.md | 11 ---
 13 files changed, 104 insertions(+), 76 deletions(-)
 rename dev-environment/docker-compose.laravel-tests.yml => .github/docker-compose/laravel-tests.yml (88%)
 delete mode 100644 docs/ci/github-actions-migration-decisions.md

diff --git a/dev-environment/docker-compose.laravel-tests.yml b/.github/docker-compose/laravel-tests.yml
similarity index 88%
rename from dev-environment/docker-compose.laravel-tests.yml
rename to .github/docker-compose/laravel-tests.yml
index 67cf323..0a53f61 100644
--- a/dev-environment/docker-compose.laravel-tests.yml
+++ b/.github/docker-compose/laravel-tests.yml
@@ -2,7 +2,7 @@ name: manticore-streams-laravel-tests
 
 services:
   mysql:
-    image: mysql:5.7.22
+    image: mysql:5.7.22@sha256:aaba540cdd9313645d892f4f20573e8b42b30e5be71c054b7befed2f7da5f85b
     platform: linux/amd64
     environment:
       MYSQL_DATABASE: mysql
@@ -30,7 +30,7 @@ services:
       retries: 60
 
   columnar:
-    image: manticoresearch/manticore:25.0.0
+    image: manticoresearch/manticore:25.0.0@sha256:4b1595624dbe4a9f917c34e65a06455793fecee15d3b74c4f03c0eb28de7e1c1
     platform: linux/amd64
     environment:
       EXTRA: "1"
diff --git a/.github/scripts/k8s/collect-diagnostics.sh b/.github/scripts/k8s/collect-diagnostics.sh
index e7742f1..e62ff5c 100644
--- a/.github/scripts/k8s/collect-diagnostics.sh
+++ b/.github/scripts/k8s/collect-diagnostics.sh
@@ -15,12 +15,9 @@ for namespace in "$APP_NAMESPACE" "$KAFKA_NAMESPACE"; do
     safe_name=$(printf '%s' "$pod" | tr '/' '_')
     (
       kubectl --request-timeout="$KUBECTL_TIMEOUT" describe -n "$namespace" "$pod" >"$ARTIFACT_DIR/${namespace}-${safe_name}-describe.txt" 2>&1 || true
-      timeout 45s kubectl --request-timeout="$KUBECTL_TIMEOUT" logs -n "$namespace" "$pod" --all-containers --prefix >"$ARTIFACT_DIR/${namespace}-${safe_name}.log" 2>&1 || true
-      timeout 45s kubectl --request-timeout="$KUBECTL_TIMEOUT" logs -n "$namespace" "$pod" --all-containers --previous --prefix >"$ARTIFACT_DIR/${namespace}-${safe_name}-previous.log" 2>&1 || true
     ) &
   done
   wait
 done
 timeout 45s helm status "$RELEASE_NAME" -n "$APP_NAMESPACE" >"$ARTIFACT_DIR/helm-status.txt" 2>&1 || true
-timeout 45s helm get values "$RELEASE_NAME" -n "$APP_NAMESPACE" --all >"$ARTIFACT_DIR/helm-values.yaml" 2>&1 || true
 kubectl --request-timeout="$KUBECTL_TIMEOUT" get nodes -o wide >"$ARTIFACT_DIR/nodes.txt" 2>&1 || true
diff --git a/.github/scripts/k8s/install-k3s.sh b/.github/scripts/k8s/install-k3s.sh
index 6b5aaaa..c61d164 100644
--- a/.github/scripts/k8s/install-k3s.sh
+++ b/.github/scripts/k8s/install-k3s.sh
@@ -1,9 +1,46 @@
 #!/usr/bin/env bash
-set -euxo pipefail
+set -euo pipefail
 
-curl -fsSL https://get.k3s.io | INSTALL_K3S_VERSION="$K3S_VERSION" sh -s - server --disable=traefik --disable=servicelb
-sudo chmod 644 /etc/rancher/k3s/k3s.yaml
-echo 'KUBECONFIG=/etc/rancher/k3s/k3s.yaml' >> "$GITHUB_ENV"
+: "${K3S_VERSION:?K3S_VERSION must be set}"
+: "${K3S_BINARY_SHA256:?K3S_BINARY_SHA256 must be set}"
+
+workdir=$(mktemp -d)
+trap 'rm -rf "$workdir"' EXIT
+k3s_binary="$workdir/k3s"
+curl --fail --location --proto '=https' --tlsv1.2 --retry 3 \
+  --output "$k3s_binary" \
+  "https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}/k3s"
+printf '%s  %s\n' "$K3S_BINARY_SHA256" "$k3s_binary" | sha256sum --check --status
+
+sudo install -m 0755 "$k3s_binary" /usr/local/bin/k3s
+sudo tee /etc/systemd/system/k3s.service >/dev/null <<'EOF'
+[Unit]
+Description=Lightweight Kubernetes
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=notify
+KillMode=process
+Delegate=yes
+LimitNOFILE=1048576
+LimitNPROC=infinity
+LimitCORE=infinity
+TasksMax=infinity
+TimeoutStartSec=0
+Restart=always
+RestartSec=5s
+ExecStart=/usr/local/bin/k3s server --disable=traefik --disable=servicelb
+
+[Install]
+WantedBy=multi-user.target
+EOF
+sudo systemctl daemon-reload
+sudo systemctl enable --now k3s
+
+kubeconfig="${RUNNER_TEMP:-/tmp}/k3s.yaml"
+sudo install -m 0600 -o "$(id -u)" -g "$(id -g)" /etc/rancher/k3s/k3s.yaml "$kubeconfig"
+echo "KUBECONFIG=$kubeconfig" >> "$GITHUB_ENV"
 end=$(( $(date +%s) + TIMEOUT_SECONDS ))
 until sudo k3s kubectl get nodes --no-headers 2>/dev/null | grep -q .; do
   [ "$(date +%s)" -lt "$end" ] || { echo 'K3s did not register a node in time' >&2; sudo journalctl -u k3s --no-pager; exit 1; }
diff --git a/.github/scripts/k8s/install-kafka.sh b/.github/scripts/k8s/install-kafka.sh
index 40f50b0..b7346dd 100644
--- a/.github/scripts/k8s/install-kafka.sh
+++ b/.github/scripts/k8s/install-kafka.sh
@@ -4,6 +4,7 @@ set -eu
 KAFKA_NAMESPACE=${KAFKA_NAMESPACE:-kafka}
 KAFKA_RELEASE_NAME=${KAFKA_RELEASE_NAME:-my-kafka}
 KAFKA_CHART_VERSION=${KAFKA_CHART_VERSION:-32.4.3}
+KAFKA_CHART_SHA256=${KAFKA_CHART_SHA256:-235ae1c09c837fbb1e670ddb83c816352ee91c05a5336e31bbd51f243a7a3687}
 TIMEOUT_SECONDS=${TIMEOUT_SECONDS:-600}
 
 collect_diagnostics() {
@@ -15,7 +16,6 @@ collect_diagnostics() {
   kubectl -n "$KAFKA_NAMESPACE" get all,cm,pvc -o wide >&2
   kubectl -n "$KAFKA_NAMESPACE" get events --sort-by=.lastTimestamp >&2
   helm status "$KAFKA_RELEASE_NAME" -n "$KAFKA_NAMESPACE" >&2
-  helm get values "$KAFKA_RELEASE_NAME" -n "$KAFKA_NAMESPACE" --all >&2
   for pod in $(kubectl -n "$KAFKA_NAMESPACE" get pods -o name 2>/dev/null); do
     kubectl -n "$KAFKA_NAMESPACE" describe "$pod" >&2
     kubectl -n "$KAFKA_NAMESPACE" logs "$pod" --all-containers --prefix >&2
@@ -26,13 +26,17 @@ collect_diagnostics() {
 
 trap collect_diagnostics EXIT
 
-helm repo add bitnami https://charts.bitnami.com/bitnami
-helm repo update bitnami
-helm upgrade --install "$KAFKA_RELEASE_NAME" bitnami/kafka \
+chart_archive=$(mktemp)
+curl --fail --location --proto '=https' --tlsv1.2 --retry 3 \
+  --output "$chart_archive" \
+  "https://charts.bitnami.com/bitnami/kafka-${KAFKA_CHART_VERSION}.tgz"
+printf '%s  %s\n' "$KAFKA_CHART_SHA256" "$chart_archive" | sha256sum --check --status
+helm upgrade --install "$KAFKA_RELEASE_NAME" "$chart_archive" \
   --namespace "$KAFKA_NAMESPACE" --create-namespace \
   --version "$KAFKA_CHART_VERSION" \
-  --set global.security.allowInsecureImages=true \
   --set image.repository=bitnamilegacy/kafka \
+  --set image.tag=4.0.0-debian-12-r10 \
+  --set image.digest=sha256:aa0b2aee8c5610dd1d18d48b4f1df0dbe3267b5d4c338d36c9af9cbf0529c0b0 \
   --set controller.replicaCount=1 \
   --set broker.replicaCount=0 \
   --set listeners.client.protocol=PLAINTEXT \
diff --git a/.github/scripts/k8s/start-kafka-fixture-producer.sh b/.github/scripts/k8s/start-kafka-fixture-producer.sh
index 216a437..27d617c 100644
--- a/.github/scripts/k8s/start-kafka-fixture-producer.sh
+++ b/.github/scripts/k8s/start-kafka-fixture-producer.sh
@@ -12,7 +12,7 @@ spec:
   restartPolicy: Never
   containers:
     - name: producer
-      image: docker.io/bitnamilegacy/kafka:4.0.0-debian-12-r10
+      image: docker.io/bitnamilegacy/kafka:4.0.0-debian-12-r10@sha256:aa0b2aee8c5610dd1d18d48b4f1df0dbe3267b5d4c338d36c9af9cbf0529c0b0
       command: ["sh", "-c", "sleep infinity"]
       resources:
         requests:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 08421ef..931b262 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,8 +20,8 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-java@v5
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0
+      - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.0.0
         with:
           distribution: temurin
           java-version: '21'
@@ -38,15 +38,15 @@ jobs:
       matrix:
         component: [manticore, scaler, rules_checker]
     steps:
-      - uses: actions/checkout@v4
-      - uses: docker/setup-buildx-action@v3
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0
+      - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v3.10.0
       - run: docker build -t streams/${{ matrix.component }}:ci -f docker/${{ matrix.component }}/Dockerfile docker/${{ matrix.component }}
       - name: Archive validated image
         run: |
           set -euo pipefail
           docker save streams/${{ matrix.component }}:ci | gzip > ${{ matrix.component }}.tar.gz
           gzip -t ${{ matrix.component }}.tar.gz
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4.6.2
         with:
           name: ci-image-${{ matrix.component }}
           path: ${{ matrix.component }}.tar.gz
@@ -62,8 +62,8 @@ jobs:
       matrix:
         component: [ui, ui-nginx]
     steps:
-      - uses: actions/checkout@v4
-      - uses: docker/setup-buildx-action@v3
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0
+      - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v3.10.0
       - run: cp -a ui/. docker/${{ matrix.component }}/source/
       - run: docker build -t streams/${{ matrix.component }}:ci -f docker/${{ matrix.component }}/Dockerfile docker/${{ matrix.component }}
       - name: Archive validated image
@@ -71,7 +71,7 @@ jobs:
           set -euo pipefail
           docker save streams/${{ matrix.component }}:ci | gzip > ${{ matrix.component }}.tar.gz
           gzip -t ${{ matrix.component }}.tar.gz
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4.6.2
         with:
           name: ci-image-${{ matrix.component }}
           path: ${{ matrix.component }}.tar.gz
@@ -83,8 +83,8 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 45
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-java@v5
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0
+      - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.0.0
         with:
           distribution: temurin
           java-version: '21'
@@ -98,7 +98,7 @@ jobs:
           set -euo pipefail
           docker save streams/worker:ci | gzip > worker.tar.gz
           gzip -t worker.tar.gz
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4.6.2
         with:
           name: ci-image-worker
           path: worker.tar.gz
@@ -109,21 +109,21 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 15
     steps:
-      - uses: actions/checkout@v4
-      - uses: azure/setup-helm@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0
+      - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v4.3.1
       - name: Materialize development chart version
         run: bash .github/scripts/materialize-chart-version.sh
       - run: helm lint helm-chart
       - run: helm template manticore-streams helm-chart --kube-version v1.32.5 > rendered-default.yaml
-      - run: docker run --rm -v "$PWD:/work" -w /work ghcr.io/yannh/kubeconform:v0.6.7 -strict -summary -kubernetes-version 1.32.5 rendered-default.yaml
+      - run: docker run --rm -v "$PWD:/work" -w /work ghcr.io/yannh/kubeconform:v0.6.7@sha256:0925177fb05b44ce18574076141b5c3d83235e1904d3f952182ac99ddc45762c -strict -summary -kubernetes-version 1.32.5 rendered-default.yaml
 
   changes:
     runs-on: ubuntu-24.04
     outputs:
       relevant: ${{ steps.filter.outputs.relevant }}
     steps:
-      - uses: actions/checkout@v4
-      - uses: dorny/paths-filter@v3
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0
+      - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v3.0.2
         id: filter
         with:
           filters: |
@@ -132,7 +132,7 @@ jobs:
               - 'ui/**'
               - 'sources/**'
               - 'helm-chart/**'
-              - 'dev-environment/docker-compose.laravel-tests.yml'
+              - '.github/docker-compose/laravel-tests.yml'
               - 'dev-environment/k8s_tests/**'
               - '.github/workflows/ci.yml'
 
@@ -142,9 +142,9 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0
       - name: Download Laravel test images
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v4.3.0
         with:
           pattern: ci-image-*
           path: ci-images
@@ -157,30 +157,30 @@ jobs:
             docker image inspect "streams/${component}:ci" >/dev/null
           done
       - name: Start Laravel test dependencies
-        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml up -d --wait mysql manticore columnar
+        run: docker compose -f .github/docker-compose/laravel-tests.yml up -d --wait mysql manticore columnar
       - name: Wait for Laravel database connection
-        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml run --rm --no-deps ui php -r 'for ($attempt = 0; $attempt < 60; $attempt++) { try { new PDO("mysql:host=mysql;dbname=mysql", "root", "mysql"); exit(0); } catch (Throwable $exception) { sleep(1); } } fwrite(STDERR, "MySQL did not accept a Laravel connection within 60 seconds\n"); exit(1);'
+        run: docker compose -f .github/docker-compose/laravel-tests.yml run --rm --no-deps ui php -r 'for ($attempt = 0; $attempt < 60; $attempt++) { try { new PDO("mysql:host=mysql;dbname=mysql", "root", "mysql"); exit(0); } catch (Throwable $exception) { sleep(1); } } fwrite(STDERR, "MySQL did not accept a Laravel connection within 60 seconds\n"); exit(1);'
       - name: Initialize Laravel test Manticore schema
-        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml exec -T manticore php /etc/manticoresearch/alter.php --table=pq --batch=500
+        run: docker compose -f .github/docker-compose/laravel-tests.yml exec -T manticore php /etc/manticoresearch/alter.php --table=pq --batch=500
       - name: Initialize Laravel test metrics storage
-        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml exec -T columnar mysql -h 127.0.0.1 -P 9306 -e 'CREATE TABLE IF NOT EXISTS metrics (scrapTime timestamp, metric_name text, value float, tag string)'
+        run: docker compose -f .github/docker-compose/laravel-tests.yml exec -T columnar mysql -h 127.0.0.1 -P 9306 -e 'CREATE TABLE IF NOT EXISTS metrics (scrapTime timestamp, metric_name text, value float, tag string)'
       - name: Migrate and seed Laravel test database
-        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml run --rm --no-deps ui php artisan migrate:fresh --seed --force
+        run: docker compose -f .github/docker-compose/laravel-tests.yml run --rm --no-deps ui php artisan migrate:fresh --seed --force
       - name: Run Laravel Unit and Feature tests
-        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml run --rm --no-deps ui php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Dev --stderr
+        run: docker compose -f .github/docker-compose/laravel-tests.yml run --rm --no-deps ui php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Dev --stderr
       - name: Collect Laravel test dependency logs
         if: always()
-        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml logs --no-color > laravel-test-dependencies.log || true
-      - uses: actions/upload-artifact@v4
-        if: always()
+        run: docker compose -f .github/docker-compose/laravel-tests.yml logs --no-color > laravel-test-dependencies.log || true
+      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4.6.2
+        if: failure()
         with:
           name: laravel-test-dependencies
           path: laravel-test-dependencies.log
-          retention-days: 7
+          retention-days: 3
           if-no-files-found: warn
       - name: Remove Laravel test dependencies
         if: always()
-        run: docker compose -f dev-environment/docker-compose.laravel-tests.yml down --volumes --remove-orphans
+        run: docker compose -f .github/docker-compose/laravel-tests.yml down --volumes --remove-orphans
 
   k8s-e2e:
     needs: [worker-tests, image-build, ui-image-build, worker-image-build, helm, changes, laravel-tests]
@@ -199,20 +199,21 @@ jobs:
       UI_DEPLOYMENT: manticore-streams-manticoresearch-ui
       UI_CONTAINER: manticoresearch-ui
       K3S_VERSION: v1.32.5+k3s1
+      K3S_BINARY_SHA256: 9624ef0534b1be432b4ce8dd94dcd7241ed02451d970a4d38943baccf5c60dce
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0
       - name: Skip unrelated pull request
         if: github.event_name == 'pull_request' && needs.changes.outputs.relevant != 'true'
         run: echo 'No Kubernetes-relevant files changed.'
-      - uses: actions/setup-java@v5
+      - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.0.0
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
         with:
           distribution: temurin
           java-version: '21'
           cache: maven
-      - uses: azure/setup-helm@v4
+      - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v4.3.1
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-      - uses: azure/setup-kubectl@v4
+      - uses: azure/setup-kubectl@829323503d1be3d00ca8346e5391ca0b07a9ab0d # v4.3.0
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
       - name: Install K3s
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
@@ -222,7 +223,7 @@ jobs:
         run: bash .github/scripts/materialize-chart-version.sh
       - name: Download validated images
         if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true'
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v4.3.0
         with:
           pattern: ci-image-*
           path: ci-images
@@ -272,14 +273,14 @@ jobs:
       - name: Collect diagnostics
         if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true')
         run: bash .github/scripts/k8s/collect-diagnostics.sh artifacts/k8s
-      - uses: actions/upload-artifact@v4
-        if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true')
+      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4.6.2
+        if: failure() && (github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true')
         with:
           name: k8s-e2e-diagnostics
           path: |
             artifacts/
             artifacts-rendered-helm.yaml
-          retention-days: 14
+          retention-days: 3
 
   publish:
     if: github.event_name == 'push' && github.ref == 'refs/heads/master'
@@ -290,8 +291,8 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v4
-      - uses: docker/login-action@v3
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0
+      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v3.4.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -303,7 +304,7 @@ jobs:
           case "$version" in *"\$Format:"*) echo 'Chart version was not materialized'; exit 1;; esac
           echo "IMAGE_TAG=${version}" >> "$GITHUB_ENV"
       - name: Download tested images
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v4.3.0
         with:
           pattern: ci-image-*
           path: ci-images
@@ -330,8 +331,8 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 15
     steps:
-      - uses: actions/checkout@v4
-      - uses: azure/setup-helm@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0
+      - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v4.3.1
       - name: Package release chart
         id: package
         run: |
@@ -346,7 +347,7 @@ jobs:
           echo "version=${version}" >> "$GITHUB_OUTPUT"
           echo "asset=release-artifacts/manticoresearch-${version}.tgz" >> "$GITHUB_OUTPUT"
       - name: Create GitHub Release
-        uses: softprops/action-gh-release@v2
+        uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v2.3.2
         with:
           tag_name: ${{ steps.package.outputs.version }}
           name: ${{ steps.package.outputs.version }}
diff --git a/docker/manticore/Dockerfile b/docker/manticore/Dockerfile
index 203983e..43ba80b 100644
--- a/docker/manticore/Dockerfile
+++ b/docker/manticore/Dockerfile
@@ -1,4 +1,4 @@
-FROM manticoresearch/manticore:7.4.6
+FROM manticoresearch/manticore:7.4.6@sha256:9fb96a2bfeec4b81b84aad7d55db396c6389e135cc3be32d856c3f91e01a56e9
 
 RUN apt-get update && echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections \
     && apt-get -y install wget software-properties-common supervisor logrotate git zip unzip wget gnupg curl tree cmake \
diff --git a/docker/rules_checker/Dockerfile b/docker/rules_checker/Dockerfile
index eb98dcf..4cba554 100644
--- a/docker/rules_checker/Dockerfile
+++ b/docker/rules_checker/Dockerfile
@@ -1,4 +1,4 @@
-FROM php:8.3-fpm
+FROM php:8.3-fpm@sha256:b7b2846437277f3d0a6f43161e4e91f82f3649c3e773d36509d53ca6519e2446
 
 # Install Nginx, kcat, mysql-client, curl, supervisor, procps, and oniguruma
 RUN apt-get update && apt-get install -y \
diff --git a/docker/scaler/Dockerfile b/docker/scaler/Dockerfile
index a5f20c4..00386e9 100644
--- a/docker/scaler/Dockerfile
+++ b/docker/scaler/Dockerfile
@@ -1,4 +1,4 @@
-FROM php:8.4.3-fpm-alpine3.20
+FROM php:8.4.3-fpm-alpine3.20@sha256:5f6c116c7f4ac2cd8e23c2f94bcff487351e0591d1813e4c6d38bff2c0563c5a
 
 # Copy from docker/scaler/ context
 COPY index.php curl.php scaler.php response.php init.sh /scaler_source_code/
diff --git a/docker/ui-nginx/Dockerfile b/docker/ui-nginx/Dockerfile
index bcc1f02..a1ff45c 100644
--- a/docker/ui-nginx/Dockerfile
+++ b/docker/ui-nginx/Dockerfile
@@ -1,4 +1,4 @@
-FROM nginx:1.24-alpine
+FROM nginx:1.24-alpine@sha256:77e5d4a6ad906c5d3793764085706577fa705b1dc6f244ea0241c4b5e2155385
 
 RUN apk add --no-cache curl \
     && mkdir -p /var/www/html/public \
diff --git a/docker/ui/Dockerfile b/docker/ui/Dockerfile
index 21d6cbc..f8cb6da 100644
--- a/docker/ui/Dockerfile
+++ b/docker/ui/Dockerfile
@@ -1,4 +1,4 @@
-FROM php:8.3-fpm
+FROM php:8.3-fpm@sha256:b7b2846437277f3d0a6f43161e4e91f82f3649c3e773d36509d53ca6519e2446
 
 RUN apt update && apt install -y \
     librdkafka-dev \
diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile
index af8ad85..c01322a 100644
--- a/docker/worker/Dockerfile
+++ b/docker/worker/Dockerfile
@@ -1,4 +1,4 @@
-FROM eclipse-temurin:21-jre-noble
+FROM eclipse-temurin:21-jre-noble@sha256:ca397720325ceefe39ce397f186759fc87d9efafb2dc4ce53315980844c2f4f2
 
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
diff --git a/docs/ci/github-actions-migration-decisions.md b/docs/ci/github-actions-migration-decisions.md
deleted file mode 100644
index 80013ec..0000000
--- a/docs/ci/github-actions-migration-decisions.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# GitHub Actions CI decisions
-
-The canonical repository is `manticoresoftware/streams` with `master` as its protected default branch. The only supported CI registry is `ghcr.io/manticoresoftware/streams`.
-
-GitHub Actions covers builds and tests only. It does not receive a kubeconfig, cloud credential, deployment environment permission, or documentation-deployment credential. The single `.github/workflows/ci.yml` workflow runs worker tests, image smoke builds, Helm validation, change detection, and a Docker-based Laravel Dev suite. The Laravel job loads the CI-built UI and Manticore images, starts disposable MySQL, Manticore, and Columnar services, initializes the test schemas, migrates and seeds the Laravel database, and runs the `Dev` PHPUnit suite. Kubernetes E2E waits for those gates and retains only Helm deployment, in-cluster migration, the `Cluster` PHPUnit suite, generated pipeline readiness, and the Kafka fixture/output assertion. Publishing waits for successful E2E and runs only for a `master` push. Pull requests use `contents: read` and never publish images. Publishing uses the workflow-provided `GITHUB_TOKEN` with `packages: write`. Published GHCR images are private; runtime clusters provide their own pull secret through the chart's `imagePullSecrets` value. CI uses locally loaded kind images and disables this value.
-
-`helm-chart/Chart.yaml` keeps the development template `7.4.6-$Format:%h$`. GitHub source archives expand it to Git's abbreviated commit ID, so clients downloading a source archive receive a unique chart and app version such as `7.4.6-25d56244`. Actions materializes the same abbreviated SHA only in its disposable checkout before linting, E2E, and publishing image tags. It never commits that generated version; this preserves straightforward Helm development-to-development upgrades without version-only CI commits. After every successful `master` run, the release job waits for all six image publications, packages the materialized chart as `manticoresearch-<version>.tgz`, and creates or updates the matching GitHub Release. The release tag, chart version, app version, and image tags are the same `<base>-<short-sha>` value.
-
-Kubernetes E2E runs in an ephemeral K3s `v1.32.5+k3s1` cluster on a GitHub-hosted runner. It builds all six images locally, imports them into K3s containerd, installs the pinned Kafka chart, and verifies the 9,418-record output delta. Kafka chart version `32.4.3` is pinned in `install-kafka.sh`. `docs/ci/local-k3s.md` documents the persistent local k3d/K3s equivalent used to reproduce the same deployment and test path.
-
-Before enabling protected-branch enforcement, a repository administrator must confirm GHCR package creation/retention policy and configure branch protection for `master`. Required checks are `worker-tests`, `laravel-tests`, Helm validation, image smoke checks, and `k8s-e2e` after five consecutive relevant successful E2E runs.