diff --git a/.github/docker-compose/laravel-tests.yml b/.github/docker-compose/laravel-tests.yml new file mode 100644 index 0000000..0a53f61 --- /dev/null +++ b/.github/docker-compose/laravel-tests.yml @@ -0,0 +1,66 @@ +name: manticore-streams-laravel-tests + +services: + mysql: + image: mysql:5.7.22@sha256:aaba540cdd9313645d892f4f20573e8b42b30e5be71c054b7befed2f7da5f85b + platform: linux/amd64 + environment: + MYSQL_DATABASE: mysql + MYSQL_ROOT_PASSWORD: mysql + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h localhost -uroot -pmysql --silent"] + interval: 5s + timeout: 3s + retries: 60 + + manticore: + image: streams/manticore:ci + platform: linux/amd64 + environment: + MANTICORE_RULES: json=json|url=url|text=source|text=text|text=title|text=statustext|url=merged|url=merged2 + MANTICORE_PORT: "9306" + PIPELINE: m + INSTANCE_LABEL: laravel-tests + STAGE: dev + BLACKLIST_MODE: "1" + healthcheck: + test: ["CMD-SHELL", "mysql -h 127.0.0.1 -P 9306 -e 'SELECT * FROM m_cluster:pq LIMIT 0'"] + interval: 5s + timeout: 3s + retries: 60 + + columnar: + image: manticoresearch/manticore:25.0.0@sha256:4b1595624dbe4a9f917c34e65a06455793fecee15d3b74c4f03c0eb28de7e1c1 + platform: linux/amd64 + environment: + EXTRA: "1" + searchd_mysql_version_string: "5.5.21" + searchd_max_packet_size: "128M" + searchd_pid_file: /tmp/searchd.pid + healthcheck: + test: ["CMD-SHELL", "mysql -h 127.0.0.1 -P 9306 -e 'SELECT 1'"] + interval: 5s + timeout: 3s + retries: 60 + + ui: + image: streams/ui:ci + platform: linux/amd64 + environment: + DB_HOST: mysql + DB_DATABASE: mysql + DB_USERNAME: root + DB_PASSWORD: mysql + MANTICORE_HOST: manticore + MANTICORE_PORT: "9306" + COLUMNAR_HOST: columnar + COLUMNAR_PORT: "9306" + extra_hosts: + - "dev.manticoresearch.com=127.0.0.1" + depends_on: + mysql: + condition: service_healthy + manticore: + condition: service_healthy + columnar: + condition: service_healthy diff --git a/.github/scripts/README.md b/.github/scripts/README.md new file mode 100644 index 0000000..4c3e1de --- /dev/null +++ b/.github/scripts/README.md @@ -0,0 +1,8 @@ +# CI scripts + +This directory contains helpers owned by GitHub Actions workflows. + +- `k8s/` contains K3s E2E orchestration and diagnostics invoked by `.github/workflows/ci.yml`. +- `materialize-chart-version.sh` materializes the source-archive chart version for CI packaging and releases. + +The developer-run local K3d/K3s harness may reuse these CI helpers; its entry points remain in `dev-environment/`. diff --git a/.github/scripts/k8s/capture-kafka-start-offsets.sh b/.github/scripts/k8s/capture-kafka-start-offsets.sh new file mode 100644 index 0000000..0b5373e --- /dev/null +++ b/.github/scripts/k8s/capture-kafka-start-offsets.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euxo pipefail + +input_start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }') +start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }') +echo "INPUT_START_OFFSET=${input_start_offset:-0}" >> "$GITHUB_ENV" +echo "START_OFFSET=${start_offset:-0}" >> "$GITHUB_ENV" diff --git a/.github/scripts/k8s/collect-diagnostics.sh b/.github/scripts/k8s/collect-diagnostics.sh new file mode 100644 index 0000000..e62ff5c --- /dev/null +++ b/.github/scripts/k8s/collect-diagnostics.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env sh +set -u + +ARTIFACT_DIR=${1:-artifacts/k8s} +APP_NAMESPACE=${APP_NAMESPACE:-manticore-streams} +KAFKA_NAMESPACE=${KAFKA_NAMESPACE:-kafka} +RELEASE_NAME=${RELEASE_NAME:-manticore-streams} +KUBECTL_TIMEOUT=${KUBECTL_TIMEOUT:-30s} +mkdir -p "$ARTIFACT_DIR" + +for namespace in "$APP_NAMESPACE" "$KAFKA_NAMESPACE"; do + kubectl --request-timeout="$KUBECTL_TIMEOUT" get all,cm,pvc -n "$namespace" -o wide >"$ARTIFACT_DIR/${namespace}-resources.txt" 2>&1 || true + kubectl --request-timeout="$KUBECTL_TIMEOUT" get events -n "$namespace" --sort-by=.lastTimestamp >"$ARTIFACT_DIR/${namespace}-events.txt" 2>&1 || true + for pod in $(kubectl --request-timeout="$KUBECTL_TIMEOUT" get pods -n "$namespace" -o name 2>/dev/null || true); do + safe_name=$(printf '%s' "$pod" | tr '/' '_') + ( + kubectl --request-timeout="$KUBECTL_TIMEOUT" describe -n "$namespace" "$pod" >"$ARTIFACT_DIR/${namespace}-${safe_name}-describe.txt" 2>&1 || true + ) & + done + wait +done +timeout 45s helm status "$RELEASE_NAME" -n "$APP_NAMESPACE" >"$ARTIFACT_DIR/helm-status.txt" 2>&1 || true +kubectl --request-timeout="$KUBECTL_TIMEOUT" get nodes -o wide >"$ARTIFACT_DIR/nodes.txt" 2>&1 || true diff --git a/.github/scripts/k8s/create-kafka-test-topics.sh b/.github/scripts/k8s/create-kafka-test-topics.sh new file mode 100644 index 0000000..ffe6583 --- /dev/null +++ b/.github/scripts/k8s/create-kafka-test-topics.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euxo pipefail + +kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-docs --partitions=1 --replication-factor=1 +kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-results --partitions=1 --replication-factor=1 diff --git a/.github/scripts/k8s/import-ci-images.sh b/.github/scripts/k8s/import-ci-images.sh new file mode 100644 index 0000000..ec728d7 --- /dev/null +++ b/.github/scripts/k8s/import-ci-images.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euxo pipefail + +tag=$(awk '/^appVersion:/ { print $2; exit }' helm-chart/Chart.yaml) +for image_archive in ci-images/*.tar.gz; do + gzip -dc "$image_archive" | docker load +done +components=(manticore scaler rules_checker worker ui ui-nginx) +for component in "${components[@]}"; do + docker image inspect "streams/${component}:ci" >/dev/null + docker tag "streams/${component}:ci" "ghcr.io/manticoresoftware/streams/${component}:${tag}" +done +for component in "${components[@]}"; do + image="ghcr.io/manticoresoftware/streams/${component}:${tag}" + docker image save "$image" | sudo k3s ctr -n k8s.io images import - + sudo k3s ctr -n k8s.io images list -q | grep -Fx "$image" +done diff --git a/.github/scripts/k8s/install-k3s.sh b/.github/scripts/k8s/install-k3s.sh new file mode 100644 index 0000000..c61d164 --- /dev/null +++ b/.github/scripts/k8s/install-k3s.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${K3S_VERSION:?K3S_VERSION must be set}" +: "${K3S_BINARY_SHA256:?K3S_BINARY_SHA256 must be set}" + +workdir=$(mktemp -d) +trap 'rm -rf "$workdir"' EXIT +k3s_binary="$workdir/k3s" +curl --fail --location --proto '=https' --tlsv1.2 --retry 3 \ + --output "$k3s_binary" \ + "https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}/k3s" +printf '%s %s\n' "$K3S_BINARY_SHA256" "$k3s_binary" | sha256sum --check --status + +sudo install -m 0755 "$k3s_binary" /usr/local/bin/k3s +sudo tee /etc/systemd/system/k3s.service >/dev/null <<'EOF' +[Unit] +Description=Lightweight Kubernetes +After=network-online.target +Wants=network-online.target + +[Service] +Type=notify +KillMode=process +Delegate=yes +LimitNOFILE=1048576 +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +TimeoutStartSec=0 +Restart=always +RestartSec=5s +ExecStart=/usr/local/bin/k3s server --disable=traefik --disable=servicelb + +[Install] +WantedBy=multi-user.target +EOF +sudo systemctl daemon-reload +sudo systemctl enable --now k3s + +kubeconfig="${RUNNER_TEMP:-/tmp}/k3s.yaml" +sudo install -m 0600 -o "$(id -u)" -g "$(id -g)" /etc/rancher/k3s/k3s.yaml "$kubeconfig" +echo "KUBECONFIG=$kubeconfig" >> "$GITHUB_ENV" +end=$(( $(date +%s) + TIMEOUT_SECONDS )) +until sudo k3s kubectl get nodes --no-headers 2>/dev/null | grep -q .; do + [ "$(date +%s)" -lt "$end" ] || { echo 'K3s did not register a node in time' >&2; sudo journalctl -u k3s --no-pager; exit 1; } + sleep 2 +done +sudo k3s kubectl wait --for=condition=Ready node --all --timeout="${TIMEOUT_SECONDS}s" diff --git a/.github/scripts/k8s/install-kafka.sh b/.github/scripts/k8s/install-kafka.sh new file mode 100644 index 0000000..b7346dd --- /dev/null +++ b/.github/scripts/k8s/install-kafka.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env sh +set -eu + +KAFKA_NAMESPACE=${KAFKA_NAMESPACE:-kafka} +KAFKA_RELEASE_NAME=${KAFKA_RELEASE_NAME:-my-kafka} +KAFKA_CHART_VERSION=${KAFKA_CHART_VERSION:-32.4.3} +KAFKA_CHART_SHA256=${KAFKA_CHART_SHA256:-235ae1c09c837fbb1e670ddb83c816352ee91c05a5336e31bbd51f243a7a3687} +TIMEOUT_SECONDS=${TIMEOUT_SECONDS:-600} + +collect_diagnostics() { + status=$? + [ "$status" -eq 0 ] && return + + set +e + echo "Kafka installation failed; collecting diagnostics" >&2 + kubectl -n "$KAFKA_NAMESPACE" get all,cm,pvc -o wide >&2 + kubectl -n "$KAFKA_NAMESPACE" get events --sort-by=.lastTimestamp >&2 + helm status "$KAFKA_RELEASE_NAME" -n "$KAFKA_NAMESPACE" >&2 + for pod in $(kubectl -n "$KAFKA_NAMESPACE" get pods -o name 2>/dev/null); do + kubectl -n "$KAFKA_NAMESPACE" describe "$pod" >&2 + kubectl -n "$KAFKA_NAMESPACE" logs "$pod" --all-containers --prefix >&2 + kubectl -n "$KAFKA_NAMESPACE" logs "$pod" --all-containers --prefix --previous >&2 + done + exit "$status" +} + +trap collect_diagnostics EXIT + +chart_archive=$(mktemp) +curl --fail --location --proto '=https' --tlsv1.2 --retry 3 \ + --output "$chart_archive" \ + "https://charts.bitnami.com/bitnami/kafka-${KAFKA_CHART_VERSION}.tgz" +printf '%s %s\n' "$KAFKA_CHART_SHA256" "$chart_archive" | sha256sum --check --status +helm upgrade --install "$KAFKA_RELEASE_NAME" "$chart_archive" \ + --namespace "$KAFKA_NAMESPACE" --create-namespace \ + --version "$KAFKA_CHART_VERSION" \ + --set image.repository=bitnamilegacy/kafka \ + --set image.tag=4.0.0-debian-12-r10 \ + --set image.digest=sha256:aa0b2aee8c5610dd1d18d48b4f1df0dbe3267b5d4c338d36c9af9cbf0529c0b0 \ + --set controller.replicaCount=1 \ + --set broker.replicaCount=0 \ + --set listeners.client.protocol=PLAINTEXT \ + --set listeners.controller.protocol=PLAINTEXT \ + --set listeners.interbroker.protocol=PLAINTEXT \ + --set overrideConfiguration.offsets\\.topic\\.replication\\.factor=1 \ + --set overrideConfiguration.transaction\\.state\\.log\\.replication\\.factor=1 \ + --set overrideConfiguration.transaction\\.state\\.log\\.min\\.isr=1 \ + --set provisioning.enabled=false \ + --wait --timeout "${TIMEOUT_SECONDS}s" + +kubectl -n "$KAFKA_NAMESPACE" rollout status statefulset/"${KAFKA_RELEASE_NAME}"-controller --timeout="${TIMEOUT_SECONDS}s" +kubectl -n "$KAFKA_NAMESPACE" exec "${KAFKA_RELEASE_NAME}"-controller-0 -c kafka -- \ + /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists \ + --topic ci-kafka-coordinator-readiness --partitions=1 --replication-factor=1 +kubectl -n "$KAFKA_NAMESPACE" exec "${KAFKA_RELEASE_NAME}"-controller-0 -c kafka -- sh -c \ + "printf 'ready\\n' | /opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server localhost:9092 --topic ci-kafka-coordinator-readiness" +timeout 60s kubectl -n "$KAFKA_NAMESPACE" exec "${KAFKA_RELEASE_NAME}"-controller-0 -c kafka -- \ + /opt/bitnami/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 \ + --topic ci-kafka-coordinator-readiness --group ci-kafka-coordinator-readiness --from-beginning \ + --max-messages 1 --timeout-ms 30000 >/dev/null diff --git a/.github/scripts/k8s/install-streams-chart.sh b/.github/scripts/k8s/install-streams-chart.sh new file mode 100644 index 0000000..0d6b228 --- /dev/null +++ b/.github/scripts/k8s/install-streams-chart.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +export TIMEOUT_SECONDS=600 +bash .github/scripts/k8s/install-kafka.sh +tag=$(awk '/^appVersion:/ { print $2; exit }' helm-chart/Chart.yaml) +ci_overrides=( + --set ingress.enabled=false + --set podSecurityPolicy.enabled=false + --set-json 'imagePullSecrets=[]' + --set 'worker.resources.requests.cpu=250m' + --set 'worker.resources.requests.memory=512Mi' + --set 'worker.resources.limits.cpu=1' + --set 'worker.resources.limits.memory=1Gi' + --set-string "ui.image.tag=${tag}" + --set-string "ui.nginx.image.tag=${tag}" + --set-string "scaler.php.image.tag=${tag}" + --set-string "worker.image.tag=${tag}" + --set-string "manticore.image.tag=${tag}" + --set-string "rulesChecker.image.tag=${tag}" +) +helm template "$RELEASE_NAME" helm-chart --kube-version v1.32.5 "${ci_overrides[@]}" > artifacts-rendered-helm.yaml +mkdir -p artifacts +helm upgrade --install "$RELEASE_NAME" helm-chart --namespace "$APP_NAMESPACE" --create-namespace --wait --timeout 600s "${ci_overrides[@]}" 2>&1 | tee artifacts/helm-upgrade.log diff --git a/.github/scripts/k8s/publish-kafka-fixture.sh b/.github/scripts/k8s/publish-kafka-fixture.sh new file mode 100644 index 0000000..d227706 --- /dev/null +++ b/.github/scripts/k8s/publish-kafka-fixture.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_FIXTURE_PRODUCER_POD" -- sh -c 'timeout 120s /opt/bitnami/kafka/bin/kafka-console-producer.sh --bootstrap-server my-kafka.kafka.svc.cluster.local:9092 --topic my-docs < /tmp/test_data.json' +input_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }') +expected_input_offset=$((INPUT_START_OFFSET + FIXTURE_RECORDS)) +printf 'Kafka input: start=%s current=%s expected=%s\n' "$INPUT_START_OFFSET" "${input_offset:-0}" "$expected_input_offset" +[ "${input_offset:-0}" -eq "$expected_input_offset" ] diff --git a/.github/scripts/k8s/run-laravel-cluster-tests.sh b/.github/scripts/k8s/run-laravel-cluster-tests.sh new file mode 100644 index 0000000..aead368 --- /dev/null +++ b/.github/scripts/k8s/run-laravel-cluster-tests.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c ' + test -x ./vendor/bin/phpunit + php -d short_open_tag=off ./vendor/bin/phpunit --testsuite Cluster --list-tests --stderr | tee /tmp/cluster-test-list.txt + grep -Fqx " - Tests\\Cluster\\ClusterTest::assignUser" /tmp/cluster-test-list.txt + php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Cluster --stderr +' diff --git a/.github/scripts/k8s/start-kafka-fixture-producer.sh b/.github/scripts/k8s/start-kafka-fixture-producer.sh new file mode 100644 index 0000000..27d617c --- /dev/null +++ b/.github/scripts/k8s/start-kafka-fixture-producer.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euxo pipefail + +kubectl -n "$KAFKA_NAMESPACE" apply -f - <> "$GITHUB_ENV" diff --git a/.github/scripts/k8s/verify-kafka-processing.sh b/.github/scripts/k8s/verify-kafka-processing.sh new file mode 100644 index 0000000..8341813 --- /dev/null +++ b/.github/scripts/k8s/verify-kafka-processing.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +end=$(( $(date +%s) + TIMEOUT_SECONDS )) +expected_offset=$((START_OFFSET + EXPECTED_RECORDS)) +next_report=0 +pipeline_name=${PIPELINE#"${RELEASE_NAME}"-} +while [ "$(date +%s)" -lt "$end" ]; do + current_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }') + current_offset=${current_offset:-0} + if [ "$current_offset" -eq "$expected_offset" ]; then + printf 'Processed exactly %s records\n' "$EXPECTED_RECORDS" + exit 0 + fi + now=$(date +%s) + if [ "$now" -ge "$next_report" ]; then + printf 'Kafka output: start=%s current=%s expected=%s\n' "$START_OFFSET" "$current_offset" "$expected_offset" + kubectl -n "$APP_NAMESPACE" get statefulset "$PIPELINE" + kubectl -n "$APP_NAMESPACE" get pods -l "name=${pipeline_name}" -o wide + for pod in $(kubectl -n "$APP_NAMESPACE" get pods -l "name=${pipeline_name}" -o name); do + for container in $(kubectl -n "$APP_NAMESPACE" get "$pod" -o jsonpath='{.spec.containers[*].name}'); do + case "$container" in + *-worker) + echo "--- Pipeline worker logs: $pod ($container), last 100 lines ---" + kubectl -n "$APP_NAMESPACE" logs "$pod" -c "$container" --tail=100 --prefix || true + ;; + esac + done + done + next_report=$((now + 30)) + fi + sleep 10 +done +echo "Pipeline $PIPELINE did not produce offset $expected_offset within ${TIMEOUT_SECONDS}s" >&2 +echo '--- Kafka topic offsets ---' +kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs || true +kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results || true +echo '--- Kafka consumer group ms_test_stream ---' +kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ms_test_stream || true +kubectl -n "$APP_NAMESPACE" describe "statefulset/$PIPELINE" || true +kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -30 || true +kubectl -n "$APP_NAMESPACE" get pods -l "name=${pipeline_name}" -o wide || true +for pod in $(kubectl -n "$APP_NAMESPACE" get pods -l "name=${pipeline_name}" -o name); do + kubectl -n "$APP_NAMESPACE" describe "$pod" || true + for container in $(kubectl -n "$APP_NAMESPACE" get "$pod" -o jsonpath='{.spec.containers[*].name}'); do + case "$container" in + *-manticore|*-worker) + echo "--- Pipeline container logs: $pod ($container) ---" + kubectl -n "$APP_NAMESPACE" logs "$pod" -c "$container" --prefix || true + echo "--- Previous pipeline container logs: $pod ($container) ---" + kubectl -n "$APP_NAMESPACE" logs "$pod" -c "$container" --previous --prefix || true + ;; + esac + done +done +exit 1 diff --git a/.github/scripts/k8s/wait-for-application-workloads.sh b/.github/scripts/k8s/wait-for-application-workloads.sh new file mode 100644 index 0000000..d0c9ec3 --- /dev/null +++ b/.github/scripts/k8s/wait-for-application-workloads.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euxo pipefail + +kubectl get namespace "$APP_NAMESPACE" +kubectl -n "$KAFKA_NAMESPACE" get pod "$KAFKA_POD" +kubectl -n "$APP_NAMESPACE" rollout status deployment/"$UI_DEPLOYMENT" --timeout="${TIMEOUT_SECONDS}s" +kubectl -n "$APP_NAMESPACE" rollout status deployment/"${RELEASE_NAME}-manticoresearch-scaler" --timeout="${TIMEOUT_SECONDS}s" +kubectl -n "$APP_NAMESPACE" rollout status statefulset/"${RELEASE_NAME}-manticoresearch-columnar" --timeout="${TIMEOUT_SECONDS}s" +kubectl -n "$APP_NAMESPACE" rollout status statefulset/"${RELEASE_NAME}-manticoresearch-ui-mysql" --timeout="${TIMEOUT_SECONDS}s" +ui_pod=$(kubectl -n "$APP_NAMESPACE" get pods -l name=manticoresearch-ui -o jsonpath='{.items[0].metadata.name}') +test -n "$ui_pod" +echo "UI_POD=$ui_pod" >> "$GITHUB_ENV" diff --git a/.github/scripts/k8s/wait-for-generated-pipeline.sh b/.github/scripts/k8s/wait-for-generated-pipeline.sh new file mode 100644 index 0000000..67cb5c9 --- /dev/null +++ b/.github/scripts/k8s/wait-for-generated-pipeline.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +started=$(date +%s) +end=$(( $(date +%s) + PIPELINE_CREATION_TIMEOUT_SECONDS )) +next_report=$started +pipeline='' +while [ "$(date +%s)" -lt "$end" ]; do + pipeline=$(kubectl -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker -o jsonpath='{range .items[*]}{.metadata.name}{end}') + [ -n "$pipeline" ] && break + now=$(date +%s) + if [ "$now" -ge "$next_report" ]; then + printf 'Pipeline creation: waiting (%ss elapsed, %ss remaining)\n' "$((now - started))" "$((end - now))" + kubectl -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker + next_report=$((now + 30)) + fi + sleep 5 +done +if [ -z "$pipeline" ]; then + echo "Pipeline creation: no StatefulSet after ${PIPELINE_CREATION_TIMEOUT_SECONDS}s" >&2 + echo '--- Pipeline failure summary ---' >&2 + kubectl -n "$APP_NAMESPACE" get pods,statefulsets,jobs -o wide || true + kubectl -n "$APP_NAMESPACE" get events --sort-by=.lastTimestamp | tail -20 || true + echo '--- UI service account permissions (statefulsets/services/PVCs) ---' >&2 + kubectl auth can-i create statefulsets.apps --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true + kubectl auth can-i create services --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true + kubectl auth can-i create persistentvolumeclaims --as="system:serviceaccount:${APP_NAMESPACE}:ui-admin-sa-${APP_NAMESPACE}" -n "$APP_NAMESPACE" || true + echo '--- UI PHP logs (last 100 lines) ---' >&2 + kubectl -n "$APP_NAMESPACE" logs "$UI_POD" -c "$UI_CONTAINER" --tail=100 --prefix || true + echo 'Detailed pod logs and full resources are in the k8s-e2e-diagnostics artifact.' >&2 + exit 1 +fi +printf 'Pipeline creation: found %s after %ss\n' "$pipeline" "$(( $(date +%s) - started ))" +kubectl -n "$APP_NAMESPACE" rollout status "statefulset/$pipeline" --timeout="${TIMEOUT_SECONDS}s" +kubectl -n "$APP_NAMESPACE" get statefulset "$pipeline" +pipeline_name=${pipeline#"${RELEASE_NAME}"-} +kubectl -n "$APP_NAMESPACE" get pods -l "name=${pipeline_name}" -o wide +echo "PIPELINE=$pipeline" >> "$GITHUB_ENV" diff --git a/.github/scripts/k8s/wait-for-kafka-consumer-assignment.sh b/.github/scripts/k8s/wait-for-kafka-consumer-assignment.sh new file mode 100644 index 0000000..5d2a451 --- /dev/null +++ b/.github/scripts/k8s/wait-for-kafka-consumer-assignment.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +end=$(( $(date +%s) + TIMEOUT_SECONDS )) +next_report=0 +group_members='' +while [ "$(date +%s)" -lt "$end" ]; do + group_members=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ms_test_stream --members --verbose 2>&1 || true) + if printf '%s\n' "$group_members" | grep -Eq 'my-docs(:0|\(0\))'; then + printf 'Kafka consumer group ms_test_stream is assigned my-docs partition 0\n' + exit 0 + fi + now=$(date +%s) + if [ "$now" -ge "$next_report" ]; then + printf 'Kafka consumer assignment: waiting for ms_test_stream to receive my-docs partition 0\n%s\n' "$group_members" + next_report=$((now + 30)) + fi + sleep 5 +done +printf 'Kafka consumer group ms_test_stream was not assigned my-docs partition 0 within %ss\n%s\n' "$TIMEOUT_SECONDS" "$group_members" >&2 +exit 1 diff --git a/.github/scripts/materialize-chart-version.sh b/.github/scripts/materialize-chart-version.sh new file mode 100644 index 0000000..a43c52f --- /dev/null +++ b/.github/scripts/materialize-chart-version.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +chart=${1:-helm-chart/Chart.yaml} +short_sha=${SHORT_SHA:-$(git rev-parse --short HEAD)} +token="\$Format:%h\$" + +source=$(<"$chart") +stripped=${source//"$token"/} +replacements=$(( (${#source} - ${#stripped}) / ${#token} )) +[ "$replacements" -eq 2 ] || { + echo "expected two development-version tokens in $chart, found $replacements" >&2 + exit 1 +} +printf '%s\n' "${source//"$token"/$short_sha}" > "$chart" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bc45a4..931b262 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,8 +20,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 20 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0 + - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.0.0 with: distribution: temurin java-version: '21' @@ -38,12 +38,15 @@ jobs: matrix: component: [manticore, scaler, rules_checker] steps: - - uses: actions/checkout@v4 - - uses: docker/setup-buildx-action@v3 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0 + - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v3.10.0 - run: docker build -t streams/${{ matrix.component }}:ci -f docker/${{ matrix.component }}/Dockerfile docker/${{ matrix.component }} - name: Archive validated image - run: docker save streams/${{ matrix.component }}:ci | gzip > ${{ matrix.component }}.tar.gz - - uses: actions/upload-artifact@v4 + run: | + set -euo pipefail + docker save streams/${{ matrix.component }}:ci | gzip > ${{ matrix.component }}.tar.gz + gzip -t ${{ matrix.component }}.tar.gz + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4.6.2 with: name: ci-image-${{ matrix.component }} path: ${{ matrix.component }}.tar.gz @@ -59,13 +62,16 @@ jobs: matrix: component: [ui, ui-nginx] steps: - - uses: actions/checkout@v4 - - uses: docker/setup-buildx-action@v3 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0 + - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v3.10.0 - run: cp -a ui/. docker/${{ matrix.component }}/source/ - run: docker build -t streams/${{ matrix.component }}:ci -f docker/${{ matrix.component }}/Dockerfile docker/${{ matrix.component }} - name: Archive validated image - run: docker save streams/${{ matrix.component }}:ci | gzip > ${{ matrix.component }}.tar.gz - - uses: actions/upload-artifact@v4 + run: | + set -euo pipefail + docker save streams/${{ matrix.component }}:ci | gzip > ${{ matrix.component }}.tar.gz + gzip -t ${{ matrix.component }}.tar.gz + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4.6.2 with: name: ci-image-${{ matrix.component }} path: ${{ matrix.component }}.tar.gz @@ -77,8 +83,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 45 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0 + - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.0.0 with: distribution: temurin java-version: '21' @@ -88,8 +94,11 @@ jobs: - run: cp sources/src/target/KafkaPublisher-1.0-SNAPSHOT-jar-with-dependencies.jar docker/worker/KafkaHandler.jar - run: docker build -t streams/worker:ci docker/worker - name: Archive validated image - run: docker save streams/worker:ci | gzip > worker.tar.gz - - uses: actions/upload-artifact@v4 + run: | + set -euo pipefail + docker save streams/worker:ci | gzip > worker.tar.gz + gzip -t worker.tar.gz + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4.6.2 with: name: ci-image-worker path: worker.tar.gz @@ -100,30 +109,21 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: azure/setup-helm@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0 + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v4.3.1 - name: Materialize development chart version - run: | - set -eux - short_sha=$(git rev-parse --short HEAD) - chart=helm-chart/Chart.yaml - token="\$Format:%h\$" - source=$(<"$chart") - stripped=${source//"$token"/} - replacements=$(( (${#source} - ${#stripped}) / ${#token} )) - [ "$replacements" -eq 2 ] || { echo "expected two development-version tokens, found $replacements"; exit 1; } - printf '%s\n' "${source//"$token"/$short_sha}" > "$chart" + run: bash .github/scripts/materialize-chart-version.sh - run: helm lint helm-chart - run: helm template manticore-streams helm-chart --kube-version v1.32.5 > rendered-default.yaml - - run: docker run --rm -v "$PWD:/work" -w /work ghcr.io/yannh/kubeconform:v0.6.7 -strict -summary -kubernetes-version 1.32.5 rendered-default.yaml + - run: docker run --rm -v "$PWD:/work" -w /work ghcr.io/yannh/kubeconform:v0.6.7@sha256:0925177fb05b44ce18574076141b5c3d83235e1904d3f952182ac99ddc45762c -strict -summary -kubernetes-version 1.32.5 rendered-default.yaml changes: runs-on: ubuntu-24.04 outputs: relevant: ${{ steps.filter.outputs.relevant }} steps: - - uses: actions/checkout@v4 - - uses: dorny/paths-filter@v3 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0 + - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v3.0.2 id: filter with: filters: | @@ -132,11 +132,58 @@ jobs: - 'ui/**' - 'sources/**' - 'helm-chart/**' + - '.github/docker-compose/laravel-tests.yml' - 'dev-environment/k8s_tests/**' - '.github/workflows/ci.yml' + laravel-tests: + needs: [image-build, ui-image-build, changes] + if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0 + - name: Download Laravel test images + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v4.3.0 + with: + pattern: ci-image-* + path: ci-images + merge-multiple: true + - name: Load Laravel test images + run: | + set -euxo pipefail + for component in manticore ui; do + gzip -dc "ci-images/${component}.tar.gz" | docker load + docker image inspect "streams/${component}:ci" >/dev/null + done + - name: Start Laravel test dependencies + run: docker compose -f .github/docker-compose/laravel-tests.yml up -d --wait mysql manticore columnar + - name: Wait for Laravel database connection + run: docker compose -f .github/docker-compose/laravel-tests.yml run --rm --no-deps ui php -r 'for ($attempt = 0; $attempt < 60; $attempt++) { try { new PDO("mysql:host=mysql;dbname=mysql", "root", "mysql"); exit(0); } catch (Throwable $exception) { sleep(1); } } fwrite(STDERR, "MySQL did not accept a Laravel connection within 60 seconds\n"); exit(1);' + - name: Initialize Laravel test Manticore schema + run: docker compose -f .github/docker-compose/laravel-tests.yml exec -T manticore php /etc/manticoresearch/alter.php --table=pq --batch=500 + - name: Initialize Laravel test metrics storage + run: docker compose -f .github/docker-compose/laravel-tests.yml exec -T columnar mysql -h 127.0.0.1 -P 9306 -e 'CREATE TABLE IF NOT EXISTS metrics (scrapTime timestamp, metric_name text, value float, tag string)' + - name: Migrate and seed Laravel test database + run: docker compose -f .github/docker-compose/laravel-tests.yml run --rm --no-deps ui php artisan migrate:fresh --seed --force + - name: Run Laravel Unit and Feature tests + run: docker compose -f .github/docker-compose/laravel-tests.yml run --rm --no-deps ui php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Dev --stderr + - name: Collect Laravel test dependency logs + if: always() + run: docker compose -f .github/docker-compose/laravel-tests.yml logs --no-color > laravel-test-dependencies.log || true + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4.6.2 + if: failure() + with: + name: laravel-test-dependencies + path: laravel-test-dependencies.log + retention-days: 3 + if-no-files-found: warn + - name: Remove Laravel test dependencies + if: always() + run: docker compose -f .github/docker-compose/laravel-tests.yml down --volumes --remove-orphans + k8s-e2e: - needs: [worker-tests, image-build, ui-image-build, worker-image-build, helm, changes] + needs: [worker-tests, image-build, ui-image-build, worker-image-build, helm, changes, laravel-tests] runs-on: ubuntu-24.04 timeout-minutes: 75 env: @@ -144,162 +191,96 @@ jobs: KAFKA_NAMESPACE: kafka RELEASE_NAME: manticore-streams KAFKA_RELEASE_NAME: my-kafka + KAFKA_FIXTURE_PRODUCER_POD: kafka-fixture-producer TIMEOUT_SECONDS: 600 + PIPELINE_CREATION_TIMEOUT_SECONDS: 60 EXPECTED_RECORDS: 9418 KAFKA_POD: my-kafka-controller-0 UI_DEPLOYMENT: manticore-streams-manticoresearch-ui UI_CONTAINER: manticoresearch-ui - KIND_CLUSTER_NAME: chart-testing + K3S_VERSION: v1.32.5+k3s1 + K3S_BINARY_SHA256: 9624ef0534b1be432b4ce8dd94dcd7241ed02451d970a4d38943baccf5c60dce steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0 - name: Skip unrelated pull request if: github.event_name == 'pull_request' && needs.changes.outputs.relevant != 'true' run: echo 'No Kubernetes-relevant files changed.' - - uses: actions/setup-java@v5 + - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.0.0 if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' with: distribution: temurin java-version: '21' cache: maven - - uses: azure/setup-helm@v4 + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v4.3.1 if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - - uses: azure/setup-kubectl@v4 + - uses: azure/setup-kubectl@829323503d1be3d00ca8346e5391ca0b07a9ab0d # v4.3.0 if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - - uses: helm/kind-action@v1.12.0 + - name: Install K3s if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - with: - version: v0.27.0 - node_image: kindest/node:v1.32.5 - config: dev-environment/k8s_tests/kind-config.yaml - cluster_name: chart-testing + run: bash .github/scripts/k8s/install-k3s.sh - name: Materialize development chart version if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - run: | - set -eux - short_sha=$(git rev-parse --short HEAD) - chart=helm-chart/Chart.yaml - token="\$Format:%h\$" - source=$(<"$chart") - stripped=${source//"$token"/} - replacements=$(( (${#source} - ${#stripped}) / ${#token} )) - [ "$replacements" -eq 2 ] || { echo "expected two development-version tokens, found $replacements"; exit 1; } - printf '%s\n' "${source//"$token"/$short_sha}" > "$chart" + run: bash .github/scripts/materialize-chart-version.sh - name: Download validated images if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v4.3.0 with: pattern: ci-image-* path: ci-images merge-multiple: true - - name: Load validated images into kind + - name: Import validated images into K3s if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - run: | - set -eux - tag=$(awk '/^appVersion:/ { print $2; exit }' helm-chart/Chart.yaml) - for image_archive in ci-images/*.tar.gz; do - gzip -dc "$image_archive" | docker load - done - components=(manticore scaler rules_checker worker ui ui-nginx) - for component in "${components[@]}"; do - docker image inspect "streams/${component}:ci" >/dev/null - docker tag "streams/${component}:ci" "ghcr.io/manticoresoftware/streams/${component}:${tag}" - done - mapfile -t images < <(printf '%s\n' "${components[@]}" | sed "s#^#ghcr.io/manticoresoftware/streams/#; s#\$#:${tag}#") - kind load docker-image --name "$KIND_CLUSTER_NAME" "${images[@]}" + run: bash .github/scripts/k8s/import-ci-images.sh - name: Install Kafka and chart if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - run: | - set -eux - export TIMEOUT_SECONDS=600 - sh dev-environment/k8s_tests/install-kafka.sh - tag=$(awk '/^appVersion:/ { print $2; exit }' helm-chart/Chart.yaml) - ci_overrides=( - --set ingress.enabled=false - --set podSecurityPolicy.enabled=false - --set-json 'imagePullSecrets=[]' - --set 'worker.resources.requests.cpu=250m' - --set 'worker.resources.requests.memory=512Mi' - --set 'worker.resources.limits.cpu=1' - --set 'worker.resources.limits.memory=1Gi' - --set-string "ui.image.tag=${tag}" - --set-string "ui.nginx.image.tag=${tag}" - --set-string "scaler.php.image.tag=${tag}" - --set-string "worker.image.tag=${tag}" - --set-string "manticore.image.tag=${tag}" - --set-string "rulesChecker.image.tag=${tag}" - ) - helm template manticore-streams helm-chart --kube-version v1.32.5 "${ci_overrides[@]}" > artifacts-rendered-helm.yaml - helm upgrade --install manticore-streams helm-chart --namespace manticore-streams --create-namespace --wait --timeout 600s "${ci_overrides[@]}" + run: bash .github/scripts/k8s/install-streams-chart.sh - name: Wait for application workloads if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - run: | - set -eux - kubectl get namespace "$APP_NAMESPACE" - kubectl -n "$KAFKA_NAMESPACE" get pod "$KAFKA_POD" - kubectl -n "$APP_NAMESPACE" rollout status deployment/"$UI_DEPLOYMENT" --timeout="${TIMEOUT_SECONDS}s" - kubectl -n "$APP_NAMESPACE" rollout status deployment/"${RELEASE_NAME}-manticoresearch-scaler" --timeout="${TIMEOUT_SECONDS}s" - kubectl -n "$APP_NAMESPACE" rollout status statefulset/"${RELEASE_NAME}-manticoresearch-columnar" --timeout="${TIMEOUT_SECONDS}s" - kubectl -n "$APP_NAMESPACE" rollout status statefulset/"${RELEASE_NAME}-manticoresearch-ui-mysql" --timeout="${TIMEOUT_SECONDS}s" - ui_pod=$(kubectl -n "$APP_NAMESPACE" get pods -l name=manticoresearch-ui -o jsonpath='{.items[0].metadata.name}') - test -n "$ui_pod" - echo "UI_POD=$ui_pod" >> "$GITHUB_ENV" + run: bash .github/scripts/k8s/wait-for-application-workloads.sh - name: Create Kafka test topics if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - run: | - set -eux - kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-docs --partitions=1 --replication-factor=1 - kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-results --partitions=1 --replication-factor=1 + run: bash .github/scripts/k8s/create-kafka-test-topics.sh + - name: Migrate and seed Laravel cluster test database + if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' + run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan migrate:fresh --seed --force - name: Configure Laravel cluster test environment if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'sed -i "s/production/cluster_testing/g" .env && sed -i "s/testing/cluster_testing/g" phpunit.xml' - - name: Migrate and seed Laravel test database + - name: Seed Laravel cluster test user if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan migrate:fresh --seed - - name: Run Laravel Unit and Feature tests - if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'php -d short_open_tag=off ./vendor/phpunit/phpunit --do-not-cache-result --testsuite Dev --stderr' + run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- php artisan db:seed --class='\TestDataSeeder' --force - name: Run Laravel cluster integration tests if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - run: kubectl -n "$APP_NAMESPACE" exec "$UI_POD" -c "$UI_CONTAINER" -- sh -c 'php -d short_open_tag=off ./vendor/phpunit/phpunit --do-not-cache-result --testsuite Cluster --stderr' + run: bash .github/scripts/k8s/run-laravel-cluster-tests.sh + - name: Wait for generated pipeline + if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' + run: bash .github/scripts/k8s/wait-for-generated-pipeline.sh + - name: Wait for Kafka consumer assignment + if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' + run: bash .github/scripts/k8s/wait-for-kafka-consumer-assignment.sh - name: Load Kafka test fixture if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - run: | - set -eux - kubectl cp dev-environment/kafka/test_data.tar.gz "$KAFKA_NAMESPACE/$KAFKA_POD:/tmp/test_data.tar.gz" - kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- tar -xzf /tmp/test_data.tar.gz -C /tmp - start_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- sh -c '/opt/bitnami/kafka/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list localhost:9092 --topic my-results' | awk -F: 'END { print $3 }') - echo "START_OFFSET=${start_offset:-0}" >> "$GITHUB_ENV" + run: bash .github/scripts/k8s/capture-kafka-start-offsets.sh + - name: Start Kafka fixture producer + if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' + run: bash .github/scripts/k8s/start-kafka-fixture-producer.sh - name: Publish Kafka test fixture if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - run: kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- sh -c '/opt/bitnami/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic my-docs < /tmp/test_data.json' + run: bash .github/scripts/k8s/publish-kafka-fixture.sh - name: Verify Kafka processing result if: github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true' - run: | - set -eux - end=$(( $(date +%s) + TIMEOUT_SECONDS )) - while [ "$(date +%s)" -lt "$end" ]; do - current_offset=$(kubectl -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -- sh -c '/opt/bitnami/kafka/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list localhost:9092 --topic my-results' | awk -F: 'END { print $3 }') - current_offset=${current_offset:-0} - if [ "$current_offset" -eq $((START_OFFSET + EXPECTED_RECORDS)) ]; then - printf 'Processed exactly %s records\n' "$EXPECTED_RECORDS" - exit 0 - fi - printf 'Waiting for output records: start=%s current=%s expected=%s\n' "$START_OFFSET" "$current_offset" "$EXPECTED_RECORDS" - sleep 10 - done - echo "output offset did not reach $((START_OFFSET + EXPECTED_RECORDS)) within ${TIMEOUT_SECONDS}s" >&2 - exit 1 + run: bash .github/scripts/k8s/verify-kafka-processing.sh - name: Collect diagnostics if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true') - run: sh dev-environment/k8s_tests/collect-diagnostics.sh artifacts/k8s - - uses: actions/upload-artifact@v4 - if: always() && (github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true') + run: bash .github/scripts/k8s/collect-diagnostics.sh artifacts/k8s + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v4.6.2 + if: failure() && (github.event_name != 'pull_request' || needs.changes.outputs.relevant == 'true') with: name: k8s-e2e-diagnostics path: | artifacts/ artifacts-rendered-helm.yaml - retention-days: 14 + retention-days: 3 publish: if: github.event_name == 'push' && github.ref == 'refs/heads/master' @@ -310,35 +291,27 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - - uses: actions/checkout@v4 - - uses: docker/login-action@v3 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v3.4.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Materialize image version run: | - set -eux - short_sha=$(git rev-parse --short HEAD) - chart=helm-chart/Chart.yaml - token="\$Format:%h\$" - source=$(<"$chart") - stripped=${source//"$token"/} - replacements=$(( (${#source} - ${#stripped}) / ${#token} )) - [ "$replacements" -eq 2 ] || { echo "expected two development-version tokens, found $replacements"; exit 1; } - printf '%s\n' "${source//"$token"/$short_sha}" > "$chart" + bash .github/scripts/materialize-chart-version.sh version=$(awk '/^appVersion:/ { print $2; exit }' helm-chart/Chart.yaml) case "$version" in *"\$Format:"*) echo 'Chart version was not materialized'; exit 1;; esac echo "IMAGE_TAG=${version}" >> "$GITHUB_ENV" - name: Download tested images - uses: actions/download-artifact@v4 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v4.3.0 with: pattern: ci-image-* path: ci-images merge-multiple: true - name: Push tested images to GHCR run: | - set -eux + set -euxo pipefail for image_archive in ci-images/*.tar.gz; do gzip -dc "$image_archive" | docker load done @@ -358,20 +331,13 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: azure/setup-helm@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.2.0 + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v4.3.1 - name: Package release chart id: package run: | set -eux - short_sha=$(git rev-parse --short HEAD) - chart=helm-chart/Chart.yaml - token="\$Format:%h\$" - source=$(<"$chart") - stripped=${source//"$token"/} - replacements=$(( (${#source} - ${#stripped}) / ${#token} )) - [ "$replacements" -eq 2 ] || { echo "expected two development-version tokens, found $replacements"; exit 1; } - printf '%s\n' "${source//"$token"/$short_sha}" > "$chart" + bash .github/scripts/materialize-chart-version.sh version=$(awk '/^version:/ { print $2; exit }' helm-chart/Chart.yaml) app_version=$(awk '/^appVersion:/ { print $2; exit }' helm-chart/Chart.yaml) test "$version" = "$app_version" @@ -381,7 +347,7 @@ jobs: echo "version=${version}" >> "$GITHUB_OUTPUT" echo "asset=release-artifacts/manticoresearch-${version}.tgz" >> "$GITHUB_OUTPUT" - name: Create GitHub Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v2.3.2 with: tag_name: ${{ steps.package.outputs.version }} name: ${{ steps.package.outputs.version }} diff --git a/.gitignore b/.gitignore index 2148361..75531a8 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ /docker/worker/KafkaHandler.jar /docker/ui/source/ /docker/ui-nginx/source/ +/artifacts/ +/dev-environment/k8s_tests/.env +**/.DS_Store +**/._* diff --git a/dev-environment/k8s_tests/.env.example b/dev-environment/k8s_tests/.env.example new file mode 100644 index 0000000..6a49a5d --- /dev/null +++ b/dev-environment/k8s_tests/.env.example @@ -0,0 +1,14 @@ +# Development-server K3d SSH target. Copy this file to .env and fill in the +# server details. Authentication uses your SSH agent or ~/.ssh/config by default. +DEV_CLUSTER_IP= +DEV_CLUSTER_USER= + +# Optional explicit private-key path. Leave empty to use SSH agent/configuration. +DEV_CLUSTER_SSH_KEY= + +# K3d cluster to create, reuse, and stop on the development server. +CLUSTER_NAME=manticore-streams-dev + +# Scratch checkout on the remote server. The remote command synchronizes the +# current working tree here before building and deploying it. +DEV_CLUSTER_WORKDIR=/tmp/manticore-streams-k3s diff --git a/dev-environment/k8s_tests/README.md b/dev-environment/k8s_tests/README.md new file mode 100644 index 0000000..1c3dbbc --- /dev/null +++ b/dev-environment/k8s_tests/README.md @@ -0,0 +1,60 @@ +# Development-server K3d validation + +Run the CI-equivalent Kubernetes deployment and Kafka E2E check in K3d on the amd64 development server over SSH. + +Run the commands below from this `dev-environment/k8s_tests` directory. + +## Configure + +```sh +cp .env.example .env +``` + +Set `DEV_CLUSTER_IP`, `DEV_CLUSTER_USER`, and `CLUSTER_NAME` in `.env`. SSH uses the agent or `~/.ssh/config`; set `DEV_CLUSTER_SSH_KEY` only when a specific key file is needed. + +The remote user must have Docker, Java 21, Maven, Helm, `kubectl`, and K3d. The runner adds `$HOME/.local/bin` to the remote `PATH`, which is where the development-server K3d, Helm, and kubectl tools were installed. It uses an isolated empty Docker configuration at `$HOME/.local/share/manticore-streams/docker-anonymous` so stale private registry credentials cannot block public base-image pulls. No sudo is required. + +## Development-server commands + +```sh +./dev-server-k3d.sh up +./dev-server-k3d.sh build +./dev-server-k3d.sh deploy +./dev-server-k3d.sh test +``` + +Every command uses the SSH target from `.env`; this runner never operates a local cluster. Commands other than `down` first synchronize the current checkout to `DEV_CLUSTER_WORKDIR`. + +`up` first checks the remote K3d state. When the configured cluster is already running, it returns immediately without synchronizing the checkout, building images, or deploying workloads. + +| Command | Purpose | +| --- | --- | +| `up` | Create the configured K3d cluster if needed, or start it when stopped, and wait for its node to become Ready. It does not build or deploy. | +| `build` | Build the six CI images on the development server without changing the cluster. | +| `deploy` | Import previously built images and install or upgrade Kafka and Streams. Run `build` first after code changes. | +| `test` | Run the CI Cluster PHPUnit suite, create its pipeline, publish the Kafka fixture, and verify the exact output count. | +| `status` | Show cluster nodes and the Kafka and Streams workloads. | +| `kubectl ` | Run `kubectl` against the remote K3d context without manually handling its kubeconfig. | +| `down` | Delete the configured K3d cluster and all Kubernetes data in it. | + +## Debug pods + +After `deploy` or while a failing `test` leaves the cluster up, use the runner as a remote `kubectl` proxy. It uses the remote `k3d-$CLUSTER_NAME` context and allocates a TTY for an interactive shell: + +```sh +./dev-server-k3d.sh kubectl get pods -A -o wide +./dev-server-k3d.sh kubectl -n manticore-streams get events --sort-by=.lastTimestamp +./dev-server-k3d.sh kubectl -n manticore-streams logs deployment/manticore-streams-manticoresearch-ui --all-containers --tail=200 +./dev-server-k3d.sh kubectl -n manticore-streams exec -it -c manticoresearch-ui -- sh +./dev-server-k3d.sh kubectl -n kafka logs my-kafka-controller-0 -c kafka --tail=200 +``` + +To delete the configured K3d cluster and its workloads, persistent volumes, and Kubernetes data, run: + +```sh +./dev-server-k3d.sh down +``` + +This runs `k3d cluster delete "$CLUSTER_NAME"` on the remote server and prints a ten-second heartbeat while deletion is in progress. It then removes and verifies any residual cluster Docker resources. It does not require sudo. It retains unrelated Docker images and local diagnostic artifacts, but the K3d cluster and all data stored in it are removed. Recreate it with `./dev-server-k3d.sh up`, then run `build` and `deploy`. + +Use `./dev-server-k3d.sh --help` to display the command synopsis. diff --git a/dev-environment/k8s_tests/collect-diagnostics.sh b/dev-environment/k8s_tests/collect-diagnostics.sh deleted file mode 100644 index 6588251..0000000 --- a/dev-environment/k8s_tests/collect-diagnostics.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env sh -set -u - -ARTIFACT_DIR=${1:-artifacts/k8s} -APP_NAMESPACE=${APP_NAMESPACE:-manticore-streams} -KAFKA_NAMESPACE=${KAFKA_NAMESPACE:-kafka} -RELEASE_NAME=${RELEASE_NAME:-manticore-streams} -mkdir -p "$ARTIFACT_DIR" - -for namespace in "$APP_NAMESPACE" "$KAFKA_NAMESPACE"; do - kubectl get all,cm,pvc -n "$namespace" -o wide >"$ARTIFACT_DIR/${namespace}-resources.txt" 2>&1 || true - kubectl get events -n "$namespace" --sort-by=.lastTimestamp >"$ARTIFACT_DIR/${namespace}-events.txt" 2>&1 || true - for pod in $(kubectl get pods -n "$namespace" -o name 2>/dev/null || true); do - safe_name=$(printf '%s' "$pod" | tr '/' '_') - kubectl describe -n "$namespace" "$pod" >"$ARTIFACT_DIR/${namespace}-${safe_name}-describe.txt" 2>&1 || true - kubectl logs -n "$namespace" "$pod" --all-containers --prefix >"$ARTIFACT_DIR/${namespace}-${safe_name}.log" 2>&1 || true - done -done -helm status "$RELEASE_NAME" -n "$APP_NAMESPACE" >"$ARTIFACT_DIR/helm-status.txt" 2>&1 || true -helm get values "$RELEASE_NAME" -n "$APP_NAMESPACE" --all >"$ARTIFACT_DIR/helm-values.yaml" 2>&1 || true -kubectl get nodes -o wide >"$ARTIFACT_DIR/nodes.txt" 2>&1 || true diff --git a/dev-environment/k8s_tests/dev-server-k3d.sh b/dev-environment/k8s_tests/dev-server-k3d.sh new file mode 100755 index 0000000..5d2f304 --- /dev/null +++ b/dev-environment/k8s_tests/dev-server-k3d.sh @@ -0,0 +1,547 @@ +#!/usr/bin/env bash +set -euo pipefail + +# A remote-only K3d reproduction of the CI E2E environment on the amd64 +# development server. This machine only synchronizes the checkout and invokes SSH. + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +ENV_FILE=${DEV_CLUSTER_ENV_FILE:-"$SCRIPT_DIR/.env"} + +if [ -f "$ENV_FILE" ]; then + set -a + # shellcheck disable=SC1090 + . "$ENV_FILE" + set +a +fi + +CLUSTER_MODE=${CLUSTER_MODE:-k3d} +CLUSTER_NAME=${CLUSTER_NAME:-manticore-streams-dev} +K3S_VERSION=${K3S_VERSION:-v1.32.5-k3s1} +APP_NAMESPACE=${APP_NAMESPACE:-manticore-streams} +KAFKA_NAMESPACE=${KAFKA_NAMESPACE:-kafka} +RELEASE_NAME=${RELEASE_NAME:-manticore-streams} +KAFKA_RELEASE_NAME=${KAFKA_RELEASE_NAME:-my-kafka} +TIMEOUT_SECONDS=${TIMEOUT_SECONDS:-600} +PIPELINE_CREATION_TIMEOUT_SECONDS=${PIPELINE_CREATION_TIMEOUT_SECONDS:-60} +EXPECTED_RECORDS=${EXPECTED_RECORDS:-9418} +KAFKA_POD=${KAFKA_POD:-my-kafka-controller-0} +KAFKA_FIXTURE_PRODUCER_POD=${KAFKA_FIXTURE_PRODUCER_POD:-kafka-fixture-producer} +UI_DEPLOYMENT=${UI_DEPLOYMENT:-manticore-streams-manticoresearch-ui} +UI_CONTAINER=${UI_CONTAINER:-manticoresearch-ui} +PLATFORM=${PLATFORM:-linux/amd64} +DEV_CLUSTER_WORKDIR=${DEV_CLUSTER_WORKDIR:-/tmp/manticore-streams-k3s} + +components=(manticore scaler rules_checker worker ui ui-nginx) + +die() { + printf 'error: %s\n' "$*" >&2 + exit 1 +} + +require_commands() { + local command commands=(docker helm kubectl k3d mvn python3) + [ "$CLUSTER_MODE" = k3d ] || die "unsupported CLUSTER_MODE: $CLUSTER_MODE" + for command in "${commands[@]}"; do + command -v "$command" >/dev/null 2>&1 || die "required command is not installed: $command" + done + docker info >/dev/null 2>&1 || die 'Docker is not running' +} + +java_home() { + if [ -n "${JAVA_HOME:-}" ] && [ -x "$JAVA_HOME/bin/java" ]; then + printf '%s\n' "$JAVA_HOME" + return + fi + + if [ "$(uname -s)" = Darwin ] && [ -x /usr/libexec/java_home ]; then + /usr/libexec/java_home -v 21 2>/dev/null || die 'Java 21 is required; install it or set JAVA_HOME' + return + fi + + die 'JAVA_HOME must point to a Java 21 installation' +} + +kubectl_dev() { + kubectl --context "k3d-${CLUSTER_NAME}" "$@" +} + +helm_dev() { + helm --kube-context "k3d-${CLUSTER_NAME}" "$@" +} + +image_tag() { + if [ -n "${IMAGE_TAG:-}" ]; then + printf '%s\n' "$IMAGE_TAG" + return + fi + git -C "$ROOT" rev-parse --short HEAD +} + +version() { + local tag + tag=$(image_tag) + python3 - "$ROOT/helm-chart/Chart.yaml" "$tag" <<'PY' +import pathlib +import sys + +chart = pathlib.Path(sys.argv[1]) +short_sha = sys.argv[2] +source = chart.read_text() +token = '$Format:%h$' +if source.count(token) != 2: + raise SystemExit(f'expected two development-version tokens, found {source.count(token)}') +for line in source.splitlines(): + if line.startswith('appVersion: '): + base = line.removeprefix('appVersion: ').removesuffix(token) + print(f'{base}{short_sha}') + break +else: + raise SystemExit('Chart.yaml has no appVersion') +PY +} + +materialize_chart() { + local destination=$1 tag=$2 + cp -a "$ROOT/helm-chart" "$destination" + python3 - "$destination/Chart.yaml" "$tag" <<'PY' +import pathlib +import sys + +chart = pathlib.Path(sys.argv[1]) +short_sha = sys.argv[2] +source = chart.read_text() +token = '$Format:%h$' +if source.count(token) != 2: + raise SystemExit(f'expected two development-version tokens, found {source.count(token)}') +chart.write_text(source.replace(token, short_sha)) +PY +} + +create_cluster() { + if k3d cluster list -o json | python3 -c ' +import json +import sys +name = sys.argv[1] +print("present" if any(cluster["name"] == name for cluster in json.load(sys.stdin)) else "") +' "$CLUSTER_NAME" | grep -qx present; then + k3d cluster start "$CLUSTER_NAME" + return + fi + + k3d cluster create "$CLUSTER_NAME" \ + --image "rancher/k3s:${K3S_VERSION}" \ + --servers 1 \ + --agents 0 \ + --k3s-arg '--disable=traefik@server:*' \ + --k3s-arg '--disable=servicelb@server:*' \ + --wait +} + +require_cluster() { + if ! k3d cluster list -o json | python3 -c ' +import json +import sys +name = sys.argv[1] +raise SystemExit(0 if any(cluster["name"] == name for cluster in json.load(sys.stdin)) else 1) +' "$CLUSTER_NAME"; then + die "k3d cluster does not exist: $CLUSTER_NAME; run up first" + fi +} + +wait_for_node() { + kubectl_dev wait --for=condition=Ready node --all --timeout="${TIMEOUT_SECONDS}s" +} + +build_images() ( + local tag staging build_java_home + tag=$(version) + build_java_home=$(java_home) + staging=$(mktemp -d) + trap 'rm -rf "$staging"' EXIT + + ( + cd "$ROOT/sources/src" + JAVA_HOME="$build_java_home" mvn clean compile assembly:single + ) + + cp -a "$ROOT/docker/worker" "$staging/worker" + cp "$ROOT/sources/src/target/KafkaPublisher-1.0-SNAPSHOT-jar-with-dependencies.jar" "$staging/worker/KafkaHandler.jar" + cp -a "$ROOT/docker/ui" "$staging/ui" + cp -a "$ROOT/ui/." "$staging/ui/source/" + cp -a "$ROOT/docker/ui-nginx" "$staging/ui-nginx" + cp -a "$ROOT/ui/." "$staging/ui-nginx/source/" + + for component in manticore scaler rules_checker; do + docker buildx build --platform "$PLATFORM" --load \ + -t "streams/${component}:ci" \ + -f "$ROOT/docker/${component}/Dockerfile" "$ROOT/docker/${component}" + done + for component in worker ui ui-nginx; do + docker buildx build --platform "$PLATFORM" --load \ + -t "streams/${component}:ci" \ + -f "$staging/${component}/Dockerfile" "$staging/${component}" + done + + for component in "${components[@]}"; do + docker image inspect "streams/${component}:ci" >/dev/null + docker tag "streams/${component}:ci" "ghcr.io/manticoresoftware/streams/${component}:${tag}" + done +) + +import_images() { + local tag component + tag=$(version) + for component in "${components[@]}"; do + local_image="ghcr.io/manticoresoftware/streams/${component}:${tag}" + docker image inspect "$local_image" >/dev/null || die "image was not built: $local_image" + k3d image import --cluster "$CLUSTER_NAME" "$local_image" + done +} + +chart_overrides() { + local tag=$1 + printf '%s\n' \ + '--set' 'ingress.enabled=false' \ + '--set' 'podSecurityPolicy.enabled=false' \ + '--set-json' 'imagePullSecrets=[]' \ + '--set' 'worker.resources.requests.cpu=250m' \ + '--set' 'worker.resources.requests.memory=512Mi' \ + '--set' 'worker.resources.limits.cpu=1' \ + '--set' 'worker.resources.limits.memory=1Gi' \ + '--set-string' "ui.image.tag=${tag}" \ + '--set-string' "ui.nginx.image.tag=${tag}" \ + '--set-string' "scaler.php.image.tag=${tag}" \ + '--set-string' "worker.image.tag=${tag}" \ + '--set-string' "manticore.image.tag=${tag}" \ + '--set-string' "rulesChecker.image.tag=${tag}" +} + +deploy() { + local tag chart + tag=$(version) + chart=$(mktemp -d) + materialize_chart "$chart/chart" "$tag" + overrides=() + while IFS= read -r override; do + overrides+=("$override") + done < <(chart_overrides "$tag") + + export KAFKA_NAMESPACE KAFKA_RELEASE_NAME TIMEOUT_SECONDS + ( + cd "$ROOT" + bash .github/scripts/k8s/install-kafka.sh + ) + helm_dev upgrade --install "$RELEASE_NAME" "$chart/chart" \ + --namespace "$APP_NAMESPACE" --create-namespace --wait --timeout "${TIMEOUT_SECONDS}s" \ + "${overrides[@]}" + rm -rf "$chart" +} + +wait_for_workloads() { + local ui_pod + kubectl_dev get namespace "$APP_NAMESPACE" + kubectl_dev -n "$KAFKA_NAMESPACE" get pod "$KAFKA_POD" + kubectl_dev -n "$APP_NAMESPACE" rollout status "deployment/${UI_DEPLOYMENT}" --timeout="${TIMEOUT_SECONDS}s" + kubectl_dev -n "$APP_NAMESPACE" rollout status "deployment/${RELEASE_NAME}-manticoresearch-scaler" --timeout="${TIMEOUT_SECONDS}s" + kubectl_dev -n "$APP_NAMESPACE" rollout status "statefulset/${RELEASE_NAME}-manticoresearch-columnar" --timeout="${TIMEOUT_SECONDS}s" + kubectl_dev -n "$APP_NAMESPACE" rollout status "statefulset/${RELEASE_NAME}-manticoresearch-ui-mysql" --timeout="${TIMEOUT_SECONDS}s" + ui_pod=$(kubectl_dev -n "$APP_NAMESPACE" get pods -l name=manticoresearch-ui -o jsonpath='{.items[0].metadata.name}') + test -n "$ui_pod" || die 'UI pod was not found' + printf '%s\n' "$ui_pod" +} + +restore_ui_test_config() { + local ui_pod=$1 + kubectl_dev -n "$APP_NAMESPACE" exec "$ui_pod" -c "$UI_CONTAINER" -- sh -c ' + sed -i "s/^APP_ENV=cluster_testing$/APP_ENV=production/" .env + sed -i "/name=\"APP_ENV\"/ s/value=\"[^\"]*\"/value=\"testing\"/" phpunit.xml + ' +} + +configure_ui_cluster_test_config() { + local ui_pod=$1 + kubectl_dev -n "$APP_NAMESPACE" exec "$ui_pod" -c "$UI_CONTAINER" -- sh -c ' + sed -i "s/^APP_ENV=production$/APP_ENV=cluster_testing/" .env + sed -i "/name=\"APP_ENV\"/ s/value=\"[^\"]*\"/value=\"cluster_testing\"/" phpunit.xml + ' +} + +run_e2e() ( + local ui_pod pipeline input_start_offset fixture_records input_offset expected_input_offset start_offset current_offset end started next_report now group_members + wait_for_workloads >/dev/null + ui_pod=$(kubectl_dev -n "$APP_NAMESPACE" get pods -l name=manticoresearch-ui -o jsonpath='{.items[0].metadata.name}') + test -n "$ui_pod" || die 'UI pod was not found' + restore_ui_test_config "$ui_pod" + trap 'restore_ui_test_config "$ui_pod" || true' EXIT + kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-docs --partitions=1 --replication-factor=1 + kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --if-not-exists --topic my-results --partitions=1 --replication-factor=1 + kubectl_dev -n "$APP_NAMESPACE" exec "$ui_pod" -c "$UI_CONTAINER" -- php artisan migrate:fresh --seed --force + configure_ui_cluster_test_config "$ui_pod" + kubectl_dev -n "$APP_NAMESPACE" exec "$ui_pod" -c "$UI_CONTAINER" -- php artisan db:seed --class='\TestDataSeeder' --force + kubectl_dev -n "$APP_NAMESPACE" exec "$ui_pod" -c "$UI_CONTAINER" -- sh -c ' + test -x ./vendor/bin/phpunit + php -d short_open_tag=off ./vendor/bin/phpunit --testsuite Cluster --list-tests --stderr | tee /tmp/cluster-test-list.txt + grep -Fqx " - Tests\\Cluster\\ClusterTest::assignUser" /tmp/cluster-test-list.txt + php -d short_open_tag=off ./vendor/bin/phpunit --do-not-cache-result --testsuite Cluster --stderr + ' + + started=$(date +%s) + end=$(( started + PIPELINE_CREATION_TIMEOUT_SECONDS )) + next_report=$started + pipeline='' + while [ "$(date +%s)" -lt "$end" ]; do + pipeline=$(kubectl_dev -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker -o jsonpath='{range .items[*]}{.metadata.name}{end}') + [ -n "$pipeline" ] && break + now=$(date +%s) + if [ "$now" -ge "$next_report" ]; then + printf 'Pipeline creation: waiting (%ss elapsed, %ss remaining)\n' "$((now - started))" "$((end - now))" + kubectl_dev -n "$APP_NAMESPACE" get statefulsets -l app.kubernetes.io/component=worker + next_report=$((now + 30)) + fi + sleep 5 + done + [ -n "$pipeline" ] || die "Cluster test did not create a pipeline StatefulSet within ${PIPELINE_CREATION_TIMEOUT_SECONDS}s" + kubectl_dev -n "$APP_NAMESPACE" rollout status "statefulset/$pipeline" --timeout="${TIMEOUT_SECONDS}s" + + end=$(( $(date +%s) + TIMEOUT_SECONDS )) + next_report=0 + group_members='' + while [ "$(date +%s)" -lt "$end" ]; do + group_members=$(kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --describe --group ms_test_stream --members --verbose 2>&1 || true) + if printf '%s\n' "$group_members" | grep -Eq 'my-docs(:0|\(0\))'; then + printf 'Kafka consumer group ms_test_stream is assigned my-docs partition 0\n' + break + fi + now=$(date +%s) + if [ "$now" -ge "$next_report" ]; then + printf 'Kafka consumer assignment: waiting for ms_test_stream to receive my-docs partition 0\n%s\n' "$group_members" + next_report=$((now + 30)) + fi + sleep 5 + done + printf '%s\n' "$group_members" | grep -Eq 'my-docs(:0|\(0\))' || die "Kafka consumer group ms_test_stream was not assigned my-docs partition 0 within ${TIMEOUT_SECONDS}s" + + input_start_offset=$(kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-docs | awk -F: 'END { print $3 }') + start_offset=$(kubectl_dev -n "$KAFKA_NAMESPACE" exec "$KAFKA_POD" -c kafka -- /opt/bitnami/kafka/bin/kafka-get-offsets.sh --bootstrap-server localhost:9092 --topic my-results | awk -F: 'END { print $3 }') + kubectl_dev -n "$KAFKA_NAMESPACE" apply -f - <' + [ -n "${DEV_CLUSTER_IP:-}" ] || die "DEV_CLUSTER_IP is required in $ENV_FILE" + [ -n "${DEV_CLUSTER_USER:-}" ] || die "DEV_CLUSTER_USER is required in $ENV_FILE" + command -v ssh >/dev/null 2>&1 || die 'required command is not installed: ssh' + if [ -n "${DEV_CLUSTER_SSH_KEY:-}" ]; then + [ -f "$DEV_CLUSTER_SSH_KEY" ] || die "DEV_CLUSTER_SSH_KEY does not exist: $DEV_CLUSTER_SSH_KEY" + ssh_options+=(-i "$DEV_CLUSTER_SSH_KEY") + fi + if [ -t 0 ] && [ -t 1 ]; then + ssh_options+=(-tt) + fi + + target="${DEV_CLUSTER_USER}@${DEV_CLUSTER_IP}" + printf -v quoted_arguments ' %q' "$@" + # shellcheck disable=SC2016 # $HOME and $PATH must expand on the remote host. + printf -v remote_command 'export PATH="$HOME/.local/bin:$PATH"; kubectl --context %q%s' "k3d-${CLUSTER_NAME}" "$quoted_arguments" + "${ssh_options[@]}" "$target" "$remote_command" +} + +run_remote() { + local command=${1:-up} target ssh_transport remote_command remote_image_tag cluster_check status=0 + local -a ssh_options=(ssh -o BatchMode=yes) + + case "$command" in + up|build|deploy|test|status|down) ;; + *) die "remote command must be one of: up, build, deploy, test, status, down" ;; + esac + [ -n "${DEV_CLUSTER_IP:-}" ] || die "DEV_CLUSTER_IP is required in $ENV_FILE" + [ -n "${DEV_CLUSTER_USER:-}" ] || die "DEV_CLUSTER_USER is required in $ENV_FILE" + command -v ssh >/dev/null 2>&1 || die 'required command is not installed: ssh' + if [ -n "${DEV_CLUSTER_SSH_KEY:-}" ]; then + [ -f "$DEV_CLUSTER_SSH_KEY" ] || die "DEV_CLUSTER_SSH_KEY does not exist: $DEV_CLUSTER_SSH_KEY" + ssh_options+=(-i "$DEV_CLUSTER_SSH_KEY") + fi + + target="${DEV_CLUSTER_USER}@${DEV_CLUSTER_IP}" + if [ "$command" = up ]; then + # shellcheck disable=SC2016 # $HOME and $PATH must expand on the remote host. + printf -v cluster_check 'export PATH="$HOME/.local/bin:$PATH"; k3d cluster list -o json | python3 -c %q %q' \ + 'import json, sys +name = sys.argv[1] +clusters = json.load(sys.stdin) +for cluster in clusters: + if cluster["name"] == name: + servers = [node for node in cluster["nodes"] if node["role"] == "server"] + raise SystemExit(0 if servers and all(node["State"]["Running"] for node in servers) else 1) +raise SystemExit(1)' "$CLUSTER_NAME" + if "${ssh_options[@]}" "$target" "$cluster_check"; then + printf 'Remote k3d cluster %s is already running; nothing to do.\n' "$CLUSTER_NAME" + return + fi + fi + if [ "$command" = down ]; then + local delete_pid elapsed=0 cleanup_command verify_command + printf 'Deleting remote K3d cluster %s and all of its data...\n' "$CLUSTER_NAME" + "${ssh_options[@]}" "$target" "export PATH=\"\$HOME/.local/bin:\$PATH\"; k3d cluster delete $(printf '%q' "$CLUSTER_NAME")" & + delete_pid=$! + while kill -0 "$delete_pid" 2>/dev/null; do + sleep 10 + elapsed=$((elapsed + 10)) + if kill -0 "$delete_pid" 2>/dev/null; then + printf 'Still deleting remote cluster (%ss elapsed)...\n' "$elapsed" + fi + done + if ! wait "$delete_pid"; then + die "remote K3d cluster deletion failed after ${elapsed}s; run status to inspect the remaining resources" + fi + # shellcheck disable=SC2016 # $cluster must expand on the remote host. + printf -v cleanup_command 'set -e; cluster=%q; containers=$(docker ps -aq --filter "label=k3d.cluster=$cluster"); if [ -n "$containers" ]; then printf "Removing remaining cluster container(s): %%s\\n" "$containers"; docker rm -f $containers; fi; if docker network inspect "k3d-$cluster" >/dev/null 2>&1; then printf "Removing remaining cluster network k3d-%%s\\n" "$cluster"; docker network rm "k3d-$cluster"; fi; if docker volume inspect "k3d-$cluster-images" >/dev/null 2>&1; then printf "Removing remaining cluster image volume k3d-%%s-images\\n" "$cluster"; docker volume rm "k3d-$cluster-images"; fi' "$CLUSTER_NAME" + printf 'Checking for remaining remote Docker resources...\n' + if ! "${ssh_options[@]}" "$target" "$cleanup_command"; then + die 'remote K3d deletion left Docker resources that could not be removed' + fi + # shellcheck disable=SC2016 # $HOME and $PATH must expand on the remote host. + if "${ssh_options[@]}" "$target" "export PATH=\"\$HOME/.local/bin:\$PATH\"; k3d cluster list -o json | python3 -c 'import json, sys; raise SystemExit(0 if any(cluster[\"name\"] == sys.argv[1] for cluster in json.load(sys.stdin)) else 1)' $(printf '%q' "$CLUSTER_NAME")"; then + die 'remote K3d reported success, but the cluster still exists' + fi + # shellcheck disable=SC2016 # $cluster must expand on the remote host. + printf -v verify_command 'cluster=%q; test -z "$(docker ps -aq --filter "label=k3d.cluster=$cluster")" && ! docker network inspect "k3d-$cluster" >/dev/null 2>&1 && ! docker volume inspect "k3d-$cluster-images" >/dev/null 2>&1' "$CLUSTER_NAME" + if ! "${ssh_options[@]}" "$target" "$verify_command"; then + die 'remote K3d cluster metadata was removed, but Docker resources remain' + fi + printf 'Remote K3d cluster %s was deleted.\n' "$CLUSTER_NAME" + return + fi + + command -v rsync >/dev/null 2>&1 || die 'required command is not installed: rsync' + "${ssh_options[@]}" "$target" "export PATH=\"\$HOME/.local/bin:\$PATH\"; mkdir -p $(printf '%q' "$DEV_CLUSTER_WORKDIR")" + printf -v ssh_transport '%q ' "${ssh_options[@]}" + printf 'Synchronizing checkout to %s...\n' "$target" + rsync -az --progress --delete \ + --exclude .git --exclude artifacts --exclude dev-environment/k8s_tests/.env \ + -e "$ssh_transport" "$ROOT/" "${target}:${DEV_CLUSTER_WORKDIR}/" + printf 'Checkout synchronized.\n' + + remote_image_tag=$(git -C "$ROOT" rev-parse --short HEAD) || die 'remote mode requires a Git checkout on the local machine' + # shellcheck disable=SC2016 # $HOME and $PATH must expand on the remote host. + printf -v remote_command 'export PATH="$HOME/.local/bin:$PATH" DOCKER_CONFIG="$HOME/.local/share/manticore-streams/docker-anonymous"; mkdir -p "$DOCKER_CONFIG"; if [ -z "${JAVA_HOME:-}" ] && [ -x "$HOME/.local/jdks/temurin-21/bin/java" ]; then export JAVA_HOME="$HOME/.local/jdks/temurin-21"; fi; cd %q && REMOTE_EXECUTION=1 CLUSTER_MODE=k3d CLUSTER_NAME=%q IMAGE_TAG=%q bash dev-environment/k8s_tests/dev-server-k3d.sh %q' \ + "$DEV_CLUSTER_WORKDIR" "$CLUSTER_NAME" "$remote_image_tag" "$command" + if "${ssh_options[@]}" "$target" "$remote_command"; then + : + else + status=$? + fi + + return "$status" +} + +usage() { + cat <<'EOF' +Usage: dev-environment/k8s_tests/dev-server-k3d.sh + +Commands: + up Create or start the remote k3d cluster and wait for its node to become Ready. + build Build the six CI images on the development server. + deploy Import existing images and install/upgrade Kafka and Streams on the development server. + test Run the CI Kubernetes application and Kafka fixture checks on the development server. + status Show remote nodes and application/Kafka workloads. + kubectl Run kubectl arguments against the remote K3d cluster. + down Delete the configured remote k3d cluster and all of its data. + +Environment overrides: CLUSTER_NAME, K3S_VERSION, PLATFORM, TIMEOUT_SECONDS. +Remote configuration: dev-environment/k8s_tests/.env (see .env.example). +EOF +} + +remote_main() { + local command=${1:-} + case "$command" in + up) + require_commands + create_cluster + wait_for_node + ;; + build) require_commands; build_images ;; + deploy) + require_commands + require_cluster + wait_for_node + import_images + deploy + wait_for_workloads >/dev/null + ;; + test) + require_commands + run_e2e + ;; + status) + require_commands + kubectl_dev get nodes -o wide + kubectl_dev -n "$KAFKA_NAMESPACE" get all,cm,pvc -o wide + kubectl_dev -n "$APP_NAMESPACE" get all,cm,pvc -o wide + ;; + -h|--help|help|'') usage ;; + *) die "unknown command: $command" ;; + esac +} + +main() { + local command=${1:-} + if [ "${REMOTE_EXECUTION:-0}" = 1 ]; then + remote_main "$@" + return + fi + + case "$command" in + up|build|deploy|test|status|down) run_remote "$command" ;; + kubectl) run_remote_kubectl "$@" ;; + -h|--help|help|'') usage ;; + *) die "unknown command: $command" ;; + esac +} + +main "$@" diff --git a/dev-environment/k8s_tests/install-kafka.sh b/dev-environment/k8s_tests/install-kafka.sh deleted file mode 100644 index 41a1948..0000000 --- a/dev-environment/k8s_tests/install-kafka.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env sh -set -eu - -KAFKA_NAMESPACE=${KAFKA_NAMESPACE:-kafka} -KAFKA_RELEASE_NAME=${KAFKA_RELEASE_NAME:-my-kafka} -KAFKA_CHART_VERSION=${KAFKA_CHART_VERSION:-32.4.3} -TIMEOUT_SECONDS=${TIMEOUT_SECONDS:-600} - -collect_diagnostics() { - status=$? - [ "$status" -eq 0 ] && return - - set +e - echo "Kafka installation failed; collecting diagnostics" >&2 - kubectl -n "$KAFKA_NAMESPACE" get all,cm,pvc -o wide >&2 - kubectl -n "$KAFKA_NAMESPACE" get events --sort-by=.lastTimestamp >&2 - helm status "$KAFKA_RELEASE_NAME" -n "$KAFKA_NAMESPACE" >&2 - helm get values "$KAFKA_RELEASE_NAME" -n "$KAFKA_NAMESPACE" --all >&2 - for pod in $(kubectl -n "$KAFKA_NAMESPACE" get pods -o name 2>/dev/null); do - kubectl -n "$KAFKA_NAMESPACE" describe "$pod" >&2 - kubectl -n "$KAFKA_NAMESPACE" logs "$pod" --all-containers --prefix >&2 - kubectl -n "$KAFKA_NAMESPACE" logs "$pod" --all-containers --prefix --previous >&2 - done - exit "$status" -} - -trap collect_diagnostics EXIT - -helm repo add bitnami https://charts.bitnami.com/bitnami -helm repo update bitnami -helm upgrade --install "$KAFKA_RELEASE_NAME" bitnami/kafka \ - --namespace "$KAFKA_NAMESPACE" --create-namespace \ - --version "$KAFKA_CHART_VERSION" \ - --set global.security.allowInsecureImages=true \ - --set image.repository=bitnamilegacy/kafka \ - --set controller.replicaCount=1 \ - --set broker.replicaCount=0 \ - --set listeners.client.protocol=PLAINTEXT \ - --set listeners.controller.protocol=PLAINTEXT \ - --set listeners.interbroker.protocol=PLAINTEXT \ - --set provisioning.enabled=false \ - --wait --timeout "${TIMEOUT_SECONDS}s" - -kubectl -n "$KAFKA_NAMESPACE" rollout status statefulset/"${KAFKA_RELEASE_NAME}"-controller --timeout="${TIMEOUT_SECONDS}s" diff --git a/docker/manticore/Dockerfile b/docker/manticore/Dockerfile index 203983e..43ba80b 100644 --- a/docker/manticore/Dockerfile +++ b/docker/manticore/Dockerfile @@ -1,4 +1,4 @@ -FROM manticoresearch/manticore:7.4.6 +FROM manticoresearch/manticore:7.4.6@sha256:9fb96a2bfeec4b81b84aad7d55db396c6389e135cc3be32d856c3f91e01a56e9 RUN apt-get update && echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections \ && apt-get -y install wget software-properties-common supervisor logrotate git zip unzip wget gnupg curl tree cmake \ diff --git a/docker/rules_checker/Dockerfile b/docker/rules_checker/Dockerfile index eb98dcf..4cba554 100644 --- a/docker/rules_checker/Dockerfile +++ b/docker/rules_checker/Dockerfile @@ -1,4 +1,4 @@ -FROM php:8.3-fpm +FROM php:8.3-fpm@sha256:b7b2846437277f3d0a6f43161e4e91f82f3649c3e773d36509d53ca6519e2446 # Install Nginx, kcat, mysql-client, curl, supervisor, procps, and oniguruma RUN apt-get update && apt-get install -y \ diff --git a/docker/scaler/Dockerfile b/docker/scaler/Dockerfile index a5f20c4..00386e9 100644 --- a/docker/scaler/Dockerfile +++ b/docker/scaler/Dockerfile @@ -1,4 +1,4 @@ -FROM php:8.4.3-fpm-alpine3.20 +FROM php:8.4.3-fpm-alpine3.20@sha256:5f6c116c7f4ac2cd8e23c2f94bcff487351e0591d1813e4c6d38bff2c0563c5a # Copy from docker/scaler/ context COPY index.php curl.php scaler.php response.php init.sh /scaler_source_code/ diff --git a/docker/ui-nginx/.dockerignore b/docker/ui-nginx/.dockerignore new file mode 100644 index 0000000..ce54105 --- /dev/null +++ b/docker/ui-nginx/.dockerignore @@ -0,0 +1,2 @@ +**/.DS_Store +**/._* diff --git a/docker/ui-nginx/Dockerfile b/docker/ui-nginx/Dockerfile index bcc1f02..a1ff45c 100644 --- a/docker/ui-nginx/Dockerfile +++ b/docker/ui-nginx/Dockerfile @@ -1,4 +1,4 @@ -FROM nginx:1.24-alpine +FROM nginx:1.24-alpine@sha256:77e5d4a6ad906c5d3793764085706577fa705b1dc6f244ea0241c4b5e2155385 RUN apk add --no-cache curl \ && mkdir -p /var/www/html/public \ diff --git a/docker/ui/.dockerignore b/docker/ui/.dockerignore new file mode 100644 index 0000000..ce54105 --- /dev/null +++ b/docker/ui/.dockerignore @@ -0,0 +1,2 @@ +**/.DS_Store +**/._* diff --git a/docker/ui/Dockerfile b/docker/ui/Dockerfile index 21d6cbc..f8cb6da 100644 --- a/docker/ui/Dockerfile +++ b/docker/ui/Dockerfile @@ -1,4 +1,4 @@ -FROM php:8.3-fpm +FROM php:8.3-fpm@sha256:b7b2846437277f3d0a6f43161e4e91f82f3649c3e773d36509d53ca6519e2446 RUN apt update && apt install -y \ librdkafka-dev \ diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index af8ad85..c01322a 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -1,4 +1,4 @@ -FROM eclipse-temurin:21-jre-noble +FROM eclipse-temurin:21-jre-noble@sha256:ca397720325ceefe39ce397f186759fc87d9efafb2dc4ce53315980844c2f4f2 RUN apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/docs/README.md b/docs/README.md index c1a59b0..697faa8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,6 +2,7 @@ * [⚡ How it works](HowItWorks.md) * [✔ ️Installation](Installation.md) * [Helm chart variables](HelmVariables.md) +* [Local k3s CI stand](ci/local-k3s.md) * Ready to work * [Roles](ReadyToWork/Roles.md) * [Admin Section](ReadyToWork/AdminSection.md) diff --git a/docs/ci/dev-server-k3d.md b/docs/ci/dev-server-k3d.md new file mode 100644 index 0000000..9f1b7eb --- /dev/null +++ b/docs/ci/dev-server-k3d.md @@ -0,0 +1,57 @@ +# Development-server K3d CI validation + +`dev-environment/k8s_tests/dev-server-k3d.sh` reproduces the Kubernetes E2E path in `.github/workflows/ci.yml` on the amd64 development server over SSH. It is a remote-only runner: it does not create, build, deploy, or delete a local Mac K3d cluster. + +## Prerequisites + +Create the ignored target configuration: + +```bash +cp dev-environment/k8s_tests/.env.example dev-environment/k8s_tests/.env +``` + +Set the server IP address, SSH user, and K3d cluster name. SSH uses the agent or `~/.ssh/config`; set `DEV_CLUSTER_SSH_KEY` only when an explicit private-key path is needed. `DEV_CLUSTER_WORKDIR` defaults to a disposable remote checkout at `/tmp/manticore-streams-k3s`. + +The remote user needs Docker, Java 21, Maven, Helm, `kubectl`, and K3d. The runner adds `$HOME/.local/bin` to the remote `PATH`, uses an isolated empty Docker configuration at `$HOME/.local/share/manticore-streams/docker-anonymous` so stale credentials cannot block public base-image pulls, and synchronizes the current local working tree (excluding `.git`, artifacts, and the local `.env`). No sudo is required. + +## Workflow + +Run the commands from `dev-environment/k8s_tests`: + +```bash +./dev-server-k3d.sh up +./dev-server-k3d.sh build +./dev-server-k3d.sh deploy +./dev-server-k3d.sh test +``` + +`up` creates the configured K3d cluster if necessary, or starts it when stopped, then waits for its node to become Ready. It does not build images or deploy workloads. + +When the configured cluster is already running, `up` exits immediately without synchronizing the checkout or changing the cluster. + +`build` compiles the Java worker and builds the six `linux/amd64` CI images on the development server. `deploy` imports those previously built images, installs or upgrades the pinned Kafka chart and Streams Helm chart, and waits for workloads. `deploy` requires a cluster created by `up` and images created by `build`. + +`test` runs the Kubernetes-only Cluster PHPUnit suite, including its `cluster_testing` seed, pipeline creation, Kafka fixture publishing, and the exact 9,418-record output assertion from CI. + +## Debugging pods + +Use the runner as a remote `kubectl` proxy after `deploy`, or while a failing `test` leaves the cluster running. It uses the remote `k3d-$CLUSTER_NAME` context; interactive `exec` sessions receive a TTY: + +```bash +./dev-server-k3d.sh kubectl get pods -A -o wide +./dev-server-k3d.sh kubectl -n manticore-streams get events --sort-by=.lastTimestamp +./dev-server-k3d.sh kubectl -n manticore-streams logs deployment/manticore-streams-manticoresearch-ui --all-containers --tail=200 +./dev-server-k3d.sh kubectl -n manticore-streams exec -it -c manticoresearch-ui -- sh +./dev-server-k3d.sh kubectl -n kafka logs my-kafka-controller-0 -c kafka --tail=200 +``` + +## Investigation and reset + +```bash +./dev-server-k3d.sh status +./dev-server-k3d.sh down +``` + +`status` shows the remote cluster nodes and Kafka/Streams workloads. Use the `kubectl` proxy for events, pod descriptions, logs, and interactive shells. + +`down` runs `k3d cluster delete "$CLUSTER_NAME"` as the remote user and prints a ten-second heartbeat while it waits. It removes and verifies any remaining container, network, or K3d image volume. It deletes workloads, persistent volumes, and Kubernetes data in that K3d cluster, while retaining unrelated Docker images. Recreate the stand with `up`, then run `build` and `deploy`. diff --git a/docs/ci/github-actions-migration-decisions.md b/docs/ci/github-actions-migration-decisions.md deleted file mode 100644 index 71f4ceb..0000000 --- a/docs/ci/github-actions-migration-decisions.md +++ /dev/null @@ -1,11 +0,0 @@ -# GitHub Actions CI decisions - -The canonical repository is `manticoresoftware/streams` with `master` as its protected default branch. The only supported CI registry is `ghcr.io/manticoresoftware/streams`. - -GitHub Actions covers builds and tests only. It does not receive a kubeconfig, cloud credential, deployment environment permission, or documentation-deployment credential. The single `.github/workflows/ci.yml` workflow runs worker tests, image smoke builds, Helm validation, and change detection in parallel. Kubernetes E2E waits for those gates; publishing waits for successful E2E and runs only for a `master` push. Pull requests use `contents: read` and never publish images. Publishing uses the workflow-provided `GITHUB_TOKEN` with `packages: write`. Published GHCR images are private; runtime clusters provide their own pull secret through the chart's `imagePullSecrets` value. CI uses locally loaded kind images and disables this value. - -`helm-chart/Chart.yaml` keeps the development template `7.4.6-$Format:%h$`. GitHub source archives expand it to Git's abbreviated commit ID, so clients downloading a source archive receive a unique chart and app version such as `7.4.6-25d56244`. Actions materializes the same abbreviated SHA only in its disposable checkout before linting, E2E, and publishing image tags. It never commits that generated version; this preserves straightforward Helm development-to-development upgrades without version-only CI commits. After every successful `master` run, the release job waits for all six image publications, packages the materialized chart as `manticoresearch-.tgz`, and creates or updates the matching GitHub Release. The release tag, chart version, app version, and image tags are the same `-` value. - -Kubernetes E2E runs in an ephemeral kind `v1.32.5` cluster on a GitHub-hosted runner. This is the closest available official kind node image to the requested `v1.32.6`, which is not published. It builds all six images locally, loads them into kind, installs a pinned Kafka chart, and verifies the 9,418-record output delta. Kafka chart version `32.0.1` is pinned in `install-kafka.sh`; its first GitHub Actions execution must be recorded before making E2E a protected required check. - -Before enabling protected-branch enforcement, a repository administrator must confirm GHCR package creation/retention policy and configure branch protection for `master`. Required checks are `worker-tests`, Helm validation, image smoke checks, and `k8s-e2e` after five consecutive relevant successful E2E runs. diff --git a/helm-chart/templates/migration-job.yaml b/helm-chart/templates/migration-job.yaml index ea07f8f..0d89e96 100644 --- a/helm-chart/templates/migration-job.yaml +++ b/helm-chart/templates/migration-job.yaml @@ -18,6 +18,36 @@ spec: serviceAccountName: ui-admin-sa-{{ .Release.Namespace }} securityContext: fsGroup: {{ .Values.ui.userId }} + initContainers: + - name: wait-{{ .Chart.Name }}-ui-mysql + image: {{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag | default .Chart.AppVersion}} + imagePullPolicy: {{ .Values.ui.image.imagePullPolicy }} + command: + - sh + - -c + - | + for i in $(seq 1 {{ .Values.ui.migrationWait.maxAttempts }}); do + if php -r 'exit(@fsockopen(getenv("DB_HOST"), (int) getenv("DB_PORT"), $errno, $errstr, 2) ? 0 : 1);'; then + echo "MySQL is reachable" + exit 0 + fi + + echo "Waiting for MySQL to become reachable ($i/{{ .Values.ui.migrationWait.maxAttempts }})" + sleep {{ .Values.ui.migrationWait.sleepSeconds }} + done + + echo "MySQL did not become reachable in time" + exit 1 + securityContext: + runAsGroup: {{ .Values.ui.userId }} + runAsUser: {{ .Values.ui.userId }} + env: + - name: DB_HOST + value: "{{ include "manticore-streams.fullname" . }}-mysql-service" + - name: DB_PORT + value: "{{ .Values.mysql.service.port }}" + resources: + {{- toYaml .Values.ui.initResources | nindent 10 }} containers: - name: {{ .Chart.Name }}-ui-migrate image: {{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag | default .Chart.AppVersion}} diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 9709fea..bae49d6 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -113,11 +113,13 @@ ui: location /nginx_status { stub_status; allow 127.0.0.1; + allow ::1; deny all; } location ~ ^/(status|ping)$ { include fastcgi_params; allow 127.0.0.1; + allow ::1; fastcgi_param REQUEST_METHOD $request_method; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; fastcgi_pass 127.0.0.1:9000; diff --git a/ui/config/columnar.php b/ui/config/columnar.php index ff6acb5..411cb8b 100644 --- a/ui/config/columnar.php +++ b/ui/config/columnar.php @@ -1,7 +1,7 @@ (in_array(config('app.env'), ['production', 'cluster_testing']) ) ? + 'host' => getenv('COLUMNAR_HOST') ? getenv('COLUMNAR_HOST').':'.getenv('COLUMNAR_PORT') : 'columnar:9306', 'index' => 'metrics' diff --git a/ui/config/manticore.php b/ui/config/manticore.php index b207dcd..2fcd542 100644 --- a/ui/config/manticore.php +++ b/ui/config/manticore.php @@ -1,7 +1,7 @@ (in_array(config('app.env'), ['production', 'cluster_testing']) ) ? + 'host' => getenv('MANTICORE_HOST') ? getenv('MANTICORE_HOST').':'.getenv('MANTICORE_PORT') : 'manticore:9306', 'index' => 'pq' diff --git a/ui/tests/Unit/ServiceEndpointConfigTest.php b/ui/tests/Unit/ServiceEndpointConfigTest.php new file mode 100644 index 0000000..caefbb0 --- /dev/null +++ b/ui/tests/Unit/ServiceEndpointConfigTest.php @@ -0,0 +1,83 @@ +