From 58d64c630fe17e17fb9fe0b87eff8008d622db58 Mon Sep 17 00:00:00 2001 From: Ian Maxon Date: Thu, 8 Oct 2026 22:26:41 +0000 Subject: [PATCH 1/3] fix(k8s): give the webserver the object store endpoint Creating a per-user warehouse registers it in Lakekeeper with the webserver's own S3 endpoint and credentials. The chart never set them on the webserver, so it sent the localhost default, Lakekeeper's storage validation failed against it, and POST /api/warehouse answered 502 on every Kubernetes deployment. Co-Authored-By: Claude Opus 5.5 --- .../base/webserver/webserver-deployment.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/bin/k8s/templates/base/webserver/webserver-deployment.yaml b/bin/k8s/templates/base/webserver/webserver-deployment.yaml index 99a58870957..570be002cc4 100644 --- a/bin/k8s/templates/base/webserver/webserver-deployment.yaml +++ b/bin/k8s/templates/base/webserver/webserver-deployment.yaml @@ -74,6 +74,24 @@ spec: value: {{ .Values.lakekeeperInit.warehouse.region | quote }} - name: STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET value: {{ .Values.lakekeeperInit.warehouse.s3Bucket | quote }} + # Object store a per-user warehouse is registered on in Lakekeeper. Unset, it + # defaults to localhost, which Lakekeeper cannot reach. + - name: STORAGE_S3_ENDPOINT + value: {{ include "texera.s3.endpoint" . }} + {{- if .Values.storage.s3.endpoint }} + - name: STORAGE_S3_REGION + value: {{ .Values.storage.s3.region | quote }} + {{- end }} + - name: STORAGE_S3_AUTH_USERNAME + valueFrom: + secretKeyRef: + name: {{ include "texera.s3.secretName" . }} + key: {{ include "texera.s3.accessKeyIdKey" . }} + - name: STORAGE_S3_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "texera.s3.secretName" . }} + key: {{ include "texera.s3.secretAccessKeyKey" . }} {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" From 6b6f8ba0b3a7b52aff60b4c36575228a985e4e2f Mon Sep 17 00:00:00 2001 From: Ian Maxon Date: Thu, 8 Oct 2026 22:26:47 +0000 Subject: [PATCH 2/3] fix(k8s): hold traffic until each service passes its healthcheck The webserver, file-service, workflow-compiling-service and the computing-unit manager had no readiness probe, so Kubernetes marked them ready as soon as the container started and the gateway routed requests to a JVM that was not listening yet, answering 503. Probe /api/healthcheck as config-service and access-control-service already do. Co-Authored-By: Claude Opus 5.5 --- .../base/file-service/file-service-deployment.yaml | 8 +++++++- .../templates/base/webserver/webserver-deployment.yaml | 8 +++++++- .../workflow-compiling-service-deployment.yaml | 8 +++++++- .../workflow-computing-unit-manager-deployment.yaml | 8 +++++++- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/bin/k8s/templates/base/file-service/file-service-deployment.yaml b/bin/k8s/templates/base/file-service/file-service-deployment.yaml index b60eddf11b5..9a21c56686f 100644 --- a/bin/k8s/templates/base/file-service/file-service-deployment.yaml +++ b/bin/k8s/templates/base/file-service/file-service-deployment.yaml @@ -82,4 +82,10 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + readinessProbe: + httpGet: + path: /api/healthcheck + port: {{ .Values.fileService.service.port }} + initialDelaySeconds: 5 + periodSeconds: 5 diff --git a/bin/k8s/templates/base/webserver/webserver-deployment.yaml b/bin/k8s/templates/base/webserver/webserver-deployment.yaml index 570be002cc4..2669b396d7b 100644 --- a/bin/k8s/templates/base/webserver/webserver-deployment.yaml +++ b/bin/k8s/templates/base/webserver/webserver-deployment.yaml @@ -95,4 +95,10 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + readinessProbe: + httpGet: + path: /api/healthcheck + port: {{ .Values.webserver.service.port }} + initialDelaySeconds: 5 + periodSeconds: 5 diff --git a/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml index a538ce4a86f..234a31f0b52 100644 --- a/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml +++ b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml @@ -69,4 +69,10 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + readinessProbe: + httpGet: + path: /api/healthcheck + port: {{ .Values.workflowCompilingService.service.port }} + initialDelaySeconds: 5 + periodSeconds: 5 diff --git a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml index 1f6ef3f99b3..e96556ada30 100644 --- a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml +++ b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml @@ -151,4 +151,10 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + readinessProbe: + httpGet: + path: /api/healthcheck + port: {{ .Values.workflowComputingUnitManager.service.port }} + initialDelaySeconds: 5 + periodSeconds: 5 From 4de277974e77aa194e6d921afa1e3d1fd959e579 Mon Sep 17 00:00:00 2001 From: Ian Maxon Date: Thu, 8 Oct 2026 22:26:47 +0000 Subject: [PATCH 3/3] ci: deploy the Helm chart to k3d and run a workflow through it Nothing exercised the chart as a whole, so defects that only appear once the services talk to each other in a cluster shipped unnoticed; the two preceding fixes were found this way. The test deploys published images rather than building them, which keeps it to a few minutes; it runs after the nightly image build, on chart changes, and on demand. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/k8s-integration-test.yml | 111 ++++ .../tests/integration/k8s_integration_test.py | 585 ++++++++++++++++++ bin/k8s/tests/integration/requirements.txt | 19 + .../integration/run_k3d_integration_test.sh | 144 +++++ bin/k8s/tests/integration/values-ci.yaml | 45 ++ 5 files changed, 904 insertions(+) create mode 100644 .github/workflows/k8s-integration-test.yml create mode 100644 bin/k8s/tests/integration/k8s_integration_test.py create mode 100644 bin/k8s/tests/integration/requirements.txt create mode 100755 bin/k8s/tests/integration/run_k3d_integration_test.sh create mode 100644 bin/k8s/tests/integration/values-ci.yaml diff --git a/.github/workflows/k8s-integration-test.yml b/.github/workflows/k8s-integration-test.yml new file mode 100644 index 00000000000..288f044f815 --- /dev/null +++ b/.github/workflows/k8s-integration-test.yml @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Deploys the Helm chart into a k3d cluster and drives one user flow through the +# gateway: upload a dataset, create a workflow, run it on a Kubernetes computing +# unit, and check its result. It deploys published images rather than building +# them, so it tests the chart in this checkout against those images. +name: K8s integration test + +on: + workflow_dispatch: + inputs: + image_registry: + description: 'Registry prefix of the Texera images (e.g., ghcr.io/apache, docker.io/apache)' + required: false + default: 'ghcr.io/apache' + type: string + image_tag: + description: 'Tag of the Texera images' + required: false + default: 'latest' + type: string + # Once the nightly build has published fresh ghcr.io/apache/*:latest images. + workflow_run: + workflows: ['Build and push images'] + types: [completed] + pull_request: + paths: + - 'bin/k8s/**' + - '.github/workflows/k8s-integration-test.yml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + K3D_VERSION: v5.9.0 + HELM_VERSION: v4.3.0 + KUBECTL_VERSION: v1.35.5 + +jobs: + k3d: + runs-on: ubuntu-latest + # A manually dispatched image build pushes some other tag, so only the nightly one + # leaves `latest` worth testing. + if: >- + github.event_name != 'workflow_run' || + (github.event.workflow_run.event == 'schedule' && + github.event.workflow_run.conclusion == 'success') + timeout-minutes: 60 + env: + IMAGE_REGISTRY: ${{ github.event.inputs.image_registry || 'ghcr.io/apache' }} + IMAGE_TAG: ${{ github.event.inputs.image_tag || 'latest' }} + DIAGNOSTICS_DIR: ${{ github.workspace }}/k8s-integration-diagnostics + steps: + - name: Checkout Texera + uses: actions/checkout@v7 + + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/share/boost /usr/local/lib/android + df -h / + + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: '3.12' + + - name: Install test dependencies + run: python -m pip install -r bin/k8s/tests/integration/requirements.txt + + - name: Install k3d, kubectl and helm + run: | + bin_dir="$RUNNER_TEMP/k8s-tools" + mkdir -p "$bin_dir" + curl -fsSL --retry 5 -o "$bin_dir/k3d" \ + "https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64" + curl -fsSL --retry 5 -o "$bin_dir/kubectl" \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + curl -fsSL --retry 5 "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" \ + | tar -xz -C "$bin_dir" --strip-components=1 linux-amd64/helm + chmod +x "$bin_dir/k3d" "$bin_dir/kubectl" "$bin_dir/helm" + echo "$bin_dir" >> "$GITHUB_PATH" + + - name: Deploy to k3d and run the integration test + run: bin/k8s/tests/integration/run_k3d_integration_test.sh + + - name: Upload diagnostics + if: failure() + uses: actions/upload-artifact@v7 + with: + name: k8s-integration-diagnostics + path: ${{ env.DIAGNOSTICS_DIR }} + retention-days: 7 diff --git a/bin/k8s/tests/integration/k8s_integration_test.py b/bin/k8s/tests/integration/k8s_integration_test.py new file mode 100644 index 00000000000..994423f3f44 --- /dev/null +++ b/bin/k8s/tests/integration/k8s_integration_test.py @@ -0,0 +1,585 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +End-to-end check of a Texera deployment, driven the way the browser drives it: +every call goes through the Envoy gateway. + + login -> (warehouse) -> dataset + upload + version -> workflow create/read + -> computing unit -> websocket execute -> paginated result -> verify + +Configured by environment variables: + TEXERA_URL gateway base URL (default http://localhost:8080) + TEXERA_USERNAME (default texera) + TEXERA_PASSWORD (default texera) + TEXERA_STEP_TIMEOUT_S upper bound for each waiting step (default 900) +""" + +import base64 +import json +import os +import sys +import time +import uuid +from urllib.parse import quote, urlencode + +import requests +import websocket + +BASE_URL = os.environ.get("TEXERA_URL", "http://localhost:8080").rstrip("/") +USERNAME = os.environ.get("TEXERA_USERNAME", "texera") +PASSWORD = os.environ.get("TEXERA_PASSWORD", "texera") +STEP_TIMEOUT_S = int(os.environ.get("TEXERA_STEP_TIMEOUT_S", "900")) + +RUN_ID = uuid.uuid4().hex[:8] +DATASET_NAME = f"ci-people-{RUN_ID}" +CSV_NAME = "people.csv" +CSV_CONTENT = ( + "name,age,city\n" + "alice,34,Irvine\n" + "bob,27,Seattle\n" + "carol,45,Austin\n" + "dave,31,Boston\n" + "erin,19,Denver\n" +) +# Filter keeps age > 30; the Python UDF appends name_upper. +EXPECTED_ROWS = [ + {"name": "alice", "age": 34, "city": "Irvine", "name_upper": "ALICE"}, + {"name": "carol", "age": 45, "city": "Austin", "name_upper": "CAROL"}, + {"name": "dave", "age": 31, "city": "Boston", "name_upper": "DAVE"}, +] + +UDF_CODE = """from pytexera import * + +class ProcessTupleOperator(UDFOperatorV2): + @overrides + def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: + tuple_["name_upper"] = tuple_["name"].upper() + yield tuple_ +""" + +SCAN_ID = f"CSVFileScan-operator-{uuid.uuid4()}" +FILTER_ID = f"Filter-operator-{uuid.uuid4()}" +UDF_ID = f"PythonUDFV2-operator-{uuid.uuid4()}" + + +class IntegrationTestFailure(Exception): + pass + + +def step(title): + print(f"\n=== {title}", flush=True) + + +def info(message): + print(f" {message}", flush=True) + + +def check(condition, message): + if not condition: + raise IntegrationTestFailure(message) + + +def jwt_claims(token): + payload = token.split(".")[1] + payload += "=" * (-len(payload) % 4) + return json.loads(base64.urlsafe_b64decode(payload)) + + +def find_key(obj, key): + """Depth-first search for `key`, so the test does not pin response nesting.""" + if isinstance(obj, dict): + if key in obj: + return obj[key] + for value in obj.values(): + found = find_key(value, key) + if found is not None: + return found + elif isinstance(obj, list): + for value in obj: + found = find_key(value, key) + if found is not None: + return found + return None + + +class Client: + def __init__(self): + self.session = requests.Session() + self.token = None + + def call(self, method, path, expected=(200,), **kwargs): + headers = kwargs.pop("headers", {}) + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + response = self.session.request( + method, f"{BASE_URL}{path}", headers=headers, timeout=60, **kwargs + ) + if response.status_code not in expected: + raise IntegrationTestFailure( + f"{method} {path} -> HTTP {response.status_code}: {response.text[:2000]}" + ) + return response + + def json(self, method, path, **kwargs): + return self.call(method, path, **kwargs).json() + + +def wait_until(description, probe, interval_s=5): + deadline = time.monotonic() + STEP_TIMEOUT_S + last = None + while time.monotonic() < deadline: + try: + result = probe() + if result: + return result + last = "not ready" + except (requests.RequestException, IntegrationTestFailure) as e: + last = e + time.sleep(interval_s) + raise IntegrationTestFailure(f"timed out waiting for {description}: {last}") + + +def login(client): + step("Log in") + + def attempt(): + response = client.session.post( + f"{BASE_URL}/api/auth/login", + json={"username": USERNAME, "password": PASSWORD}, + timeout=30, + ) + if response.status_code != 200: + raise IntegrationTestFailure( + f"HTTP {response.status_code}: {response.text[:500]}" + ) + return response.json()["accessToken"] + + client.token = wait_until("the login endpoint behind the gateway", attempt) + claims = jwt_claims(client.token) + info(f"logged in as uid={claims.get('userId')} email={claims.get('email')}") + return claims + + +def prepare_warehouse(client): + step("Resolve the execution warehouse") + response = client.call("GET", "/api/warehouse/status", expected=(200, 404)) + if response.status_code == 404 or not response.json().get("enabled"): + info("per-user warehouses are off; executions use the shared default") + return None + created = client.json("POST", "/api/warehouse", json={"name": f"ci-{RUN_ID}"}) + info( + f"created warehouse whid={created['whid']} ({created['lakekeeperWarehouseName']})" + ) + return created["whid"] + + +def upload_dataset(client, owner_email): + step(f"Create dataset '{DATASET_NAME}' and upload {CSV_NAME}") + created = client.json( + "POST", + "/api/dataset/create", + json={ + "datasetName": DATASET_NAME, + "datasetDescription": "k3d integration test", + "isDatasetPublic": False, + }, + ) + did = find_key(created, "did") + check(did is not None, f"dataset create returned no did: {created}") + info(f"dataset did={did}") + + payload = CSV_CONTENT.encode() + common = { + "ownerEmail": owner_email, + "datasetName": DATASET_NAME, + "filePath": CSV_NAME, + } + init = client.json( + "POST", + "/api/dataset/multipart-upload?" + + urlencode( + { + **common, + "type": "init", + "fileSizeBytes": len(payload), + "partSizeBytes": len(payload), + } + ), + ) + check("missingParts" in init, f"multipart init returned no missingParts: {init}") + client.call( + "POST", + "/api/dataset/multipart-upload/part?" + urlencode({**common, "partNumber": 1}), + data=payload, + headers={"Content-Type": "application/octet-stream"}, + ) + client.call( + "POST", + "/api/dataset/multipart-upload?" + urlencode({**common, "type": "finish"}), + ) + + version = client.json( + "POST", + f"/api/dataset/{did}/version/create", + data="", + headers={"Content-Type": "text/plain"}, + ) + version_name = find_key(version, "name") + check(version_name, f"version create returned no version name: {version}") + info(f"created version '{version_name}'") + + listing = client.json("GET", "/api/dataset/list") + check( + any(find_key(entry, "name") == DATASET_NAME for entry in listing), + f"dataset '{DATASET_NAME}' missing from /api/dataset/list", + ) + return did, version_name + + +def port(port_id, display_name="", dependencies=None): + descriptor = { + "portID": port_id, + "displayName": display_name, + "allowMultiInputs": False, + "isDynamicPort": False, + } + if dependencies is not None: + descriptor["dependencies"] = dependencies + return descriptor + + +def operator(operator_id, operator_type, properties, inputs, view_result=False): + return { + "operatorID": operator_id, + "operatorType": operator_type, + "operatorVersion": "N/A", + "operatorProperties": properties, + "inputPorts": [port(p, dependencies=[]) for p in inputs], + "outputPorts": [port("output-0")], + "showAdvanced": False, + "isDisabled": False, + "customDisplayName": operator_type, + "dynamicInputPorts": False, + "dynamicOutputPorts": False, + "viewResult": view_result, + } + + +def link(source_id, target_id): + return { + "linkID": f"link-{uuid.uuid4()}", + "source": {"operatorID": source_id, "portID": "output-0"}, + "target": {"operatorID": target_id, "portID": "input-0"}, + } + + +def workflow_content(file_path): + operators = [ + operator( + SCAN_ID, + "CSVFileScan", + { + "fileEncoding": "UTF_8", + "customDelimiter": ",", + "hasHeader": True, + "fileName": file_path, + }, + inputs=[], + ), + operator( + FILTER_ID, + "Filter", + {"predicates": [{"attribute": "age", "condition": ">", "value": "30"}]}, + inputs=["input-0"], + ), + operator( + UDF_ID, + "PythonUDFV2", + { + "code": UDF_CODE, + "workers": 1, + "retainInputColumns": True, + "defaultEnv": True, + "outputColumns": [ + {"attributeName": "name_upper", "attributeType": "string"} + ], + }, + inputs=["input-0"], + view_result=True, + ), + ] + return { + "operators": operators, + "operatorPositions": { + op["operatorID"]: {"x": 200 * i, "y": 100} for i, op in enumerate(operators) + }, + "links": [link(SCAN_ID, FILTER_ID), link(FILTER_ID, UDF_ID)], + "commentBoxes": [], + "settings": {"dataTransferBatchSize": 400}, + } + + +def create_workflow(client, file_path): + step("Create and read back the workflow") + name = f"ci-workflow-{RUN_ID}" + created = client.json( + "POST", + "/api/workflow/create", + json={"name": name, "content": json.dumps(workflow_content(file_path))}, + ) + wid = find_key(created, "wid") + check(wid is not None, f"workflow create returned no wid: {created}") + info(f"workflow wid={wid}") + + stored = client.json("GET", f"/api/workflow/{wid}") + content = stored["content"] + content = json.loads(content) if isinstance(content, str) else content + stored_ids = sorted(op["operatorID"] for op in content["operators"]) + check( + stored_ids == sorted([SCAN_ID, FILTER_ID, UDF_ID]), + f"stored workflow operators differ: {stored_ids}", + ) + return wid, content + + +def logical_plan(content): + """Mirrors ExecuteWorkflowService.getLogicalPlanRequest in the frontend.""" + operators = {op["operatorID"]: op for op in content["operators"]} + + def ordinal(ports, port_id): + return next(i for i, p in enumerate(ports) if p["portID"] == port_id) + + return { + "operators": [ + { + **op["operatorProperties"], + "operatorID": op["operatorID"], + "operatorType": op["operatorType"], + "inputPorts": op["inputPorts"], + "outputPorts": op["outputPorts"], + } + for op in content["operators"] + ], + "links": [ + { + "fromOpId": l["source"]["operatorID"], + "fromPortId": { + "id": ordinal( + operators[l["source"]["operatorID"]]["outputPorts"], + l["source"]["portID"], + ), + "internal": False, + }, + "toOpId": l["target"]["operatorID"], + "toPortId": { + "id": ordinal( + operators[l["target"]["operatorID"]]["inputPorts"], + l["target"]["portID"], + ), + "internal": False, + }, + } + for l in content["links"] + ], + "opsToViewResult": [ + op["operatorID"] for op in content["operators"] if op.get("viewResult") + ], + "opsToReuseResult": [], + } + + +def create_computing_unit(client): + step("Create a Kubernetes computing unit") + limits = client.json("GET", "/api/computing-unit/limits") + info(f"limit options: {limits}") + memory = limits["memoryLimitOptions"][0] + created = client.json( + "POST", + "/api/computing-unit/create", + json={ + "name": f"ci-cu-{RUN_ID}", + "unitType": "kubernetes", + "cpuLimit": limits["cpuLimitOptions"][0], + "memoryLimit": memory, + "gpuLimit": limits["gpuLimitOptions"][0], + "jvmMemorySize": "1G", + "shmSize": "64Mi", + }, + ) + cuid = created["computingUnit"]["cuid"] + info(f"computing unit cuid={cuid}, status={created['status']}") + + def running(): + unit = client.json("GET", f"/api/computing-unit/{cuid}") + info(f"cuid={cuid} status={unit['status']} reason={unit.get('statusReason')}") + return unit["status"] == "Running" + + wait_until(f"computing unit {cuid} to run", running, interval_s=10) + return cuid + + +def websocket_url(wid, uid, cuid, token): + ws_base = BASE_URL.replace("https://", "wss://").replace("http://", "ws://") + query = urlencode( + {"wid": wid, "uid": uid, "cuid": cuid, "access-token": token}, quote_via=quote + ) + return f"{ws_base}/wsapi/workflow-websocket?{query}" + + +def open_websocket(url): + # The gateway answers 5xx while the unit's websocket endpoint is still coming up. + def attempt(): + try: + return websocket.create_connection(url, timeout=30) + except (websocket.WebSocketException, OSError) as e: + raise IntegrationTestFailure(f"websocket connect failed: {e}") from e + + return wait_until("the computing unit's websocket", attempt) + + +def receive(ws, deadline): + while time.monotonic() < deadline: + try: + frame = ws.recv() + except websocket.WebSocketTimeoutException: + ws.send(json.dumps({"type": "HeartBeatRequest"})) + continue + if not frame: + continue + return json.loads(frame) + raise IntegrationTestFailure("timed out waiting for a websocket event") + + +def execute_workflow(ws, content, cuid, warehouse_id): + step("Execute the workflow over the websocket") + request = { + "type": "WorkflowExecuteRequest", + "executionName": f"ci-execution-{RUN_ID}", + "engineVersion": "ci", + "logicalPlan": logical_plan(content), + "replayFromExecution": None, + "workflowSettings": content["settings"], + "emailNotificationEnabled": False, + "computingUnitId": cuid, + } + if warehouse_id is not None: + request["warehouseId"] = warehouse_id + ws.send(json.dumps(request)) + + deadline = time.monotonic() + STEP_TIMEOUT_S + seen_running = False + while True: + event = receive(ws, deadline) + kind = event.get("type") + if kind == "WorkflowStateEvent": + state = event["state"] + info(f"state -> {state}") + seen_running |= state == "Running" + if state == "Completed": + check( + seen_running, "execution completed without ever reporting Running" + ) + return + if state in ("Failed", "Killed", "Terminated"): + raise IntegrationTestFailure(f"execution ended in state {state}") + elif kind == "WorkflowErrorEvent": + raise IntegrationTestFailure( + f"execution error: {json.dumps(event['fatalErrors'])[:4000]}" + ) + elif kind == "ConsoleUpdateEvent": + for message in event.get("messages", []): + info( + f"console[{event.get('operatorId')}]: {message.get('title', '')}".rstrip() + ) + + +def fetch_result(ws, operator_id): + step("Read the result page") + request_id = str(uuid.uuid4()) + ws.send( + json.dumps( + { + "type": "ResultPaginationRequest", + "requestID": request_id, + "operatorID": operator_id, + "pageIndex": 1, + "pageSize": 100, + } + ) + ) + deadline = time.monotonic() + 120 + while True: + event = receive(ws, deadline) + if ( + event.get("type") == "PaginatedResultEvent" + and event.get("requestID") == request_id + ): + return event["table"] + if event.get("type") == "WorkflowErrorEvent": + raise IntegrationTestFailure( + f"pagination error: {json.dumps(event)[:4000]}" + ) + + +def verify_rows(rows): + info(f"received {len(rows)} row(s): {json.dumps(rows)}") + actual = sorted( + ({key: row.get(key) for key in EXPECTED_ROWS[0]} for row in rows), + key=lambda row: row["name"], + ) + normalized = [{key: str(value) for key, value in row.items()} for row in actual] + expected = [ + {key: str(value) for key, value in row.items()} for row in EXPECTED_ROWS + ] + check( + normalized == expected, + f"result mismatch:\n expected {expected}\n actual {normalized}", + ) + info("result matches") + + +def terminate_computing_unit(client, cuid): + step(f"Terminate computing unit {cuid}") + client.call("DELETE", f"/api/computing-unit/{cuid}/terminate", expected=(200, 204)) + + +def main(): + started = time.monotonic() + client = Client() + claims = login(client) + warehouse_id = prepare_warehouse(client) + _, version_name = upload_dataset(client, claims["email"]) + file_path = f"/dataset/{claims['email']}/{DATASET_NAME}/{version_name}/{CSV_NAME}" + wid, content = create_workflow(client, file_path) + cuid = create_computing_unit(client) + ws = open_websocket(websocket_url(wid, claims["userId"], cuid, client.token)) + try: + execute_workflow(ws, content, cuid, warehouse_id) + verify_rows(fetch_result(ws, UDF_ID)) + finally: + ws.close() + terminate_computing_unit(client, cuid) + print(f"\nPASS in {time.monotonic() - started:.0f}s", flush=True) + + +if __name__ == "__main__": + try: + main() + except IntegrationTestFailure as failure: + print(f"\nFAIL: {failure}", file=sys.stderr, flush=True) + sys.exit(1) diff --git a/bin/k8s/tests/integration/requirements.txt b/bin/k8s/tests/integration/requirements.txt new file mode 100644 index 00000000000..3446497da02 --- /dev/null +++ b/bin/k8s/tests/integration/requirements.txt @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +requests==2.34.2 +websocket-client==1.8.0 diff --git a/bin/k8s/tests/integration/run_k3d_integration_test.sh b/bin/k8s/tests/integration/run_k3d_integration_test.sh new file mode 100755 index 00000000000..ab7344fa79f --- /dev/null +++ b/bin/k8s/tests/integration/run_k3d_integration_test.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Deploys the Helm chart in bin/k8s into a throwaway k3d cluster and runs +# k8s_integration_test.py against it through the Envoy gateway. +# +# Needs docker, k3d, kubectl, helm and python3 (with requirements.txt installed). +# Run from anywhere: +# bin/k8s/tests/integration/run_k3d_integration_test.sh +# +# Environment: +# IMAGE_REGISTRY registry prefix of the Texera images (default ghcr.io/apache) +# IMAGE_TAG tag of the Texera images (default latest) +# CLUSTER_NAME k3d cluster name (default texera-it) +# GATEWAY_PORT host port mapped to the gateway (default 8080) +# K3S_IMAGE node image (default rancher/k3s:v1.35.5-k3s1) +# K3D_CREATE_ARGS extra `k3d cluster create` arguments, e.g. --host-alias for a local resolver +# KEEP_CLUSTER "true" leaves the cluster up afterwards +# DIAGNOSTICS_DIR where pod state and logs go on failure (default ./k8s-integration-diagnostics) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHART_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +IMAGE_REGISTRY="${IMAGE_REGISTRY:-ghcr.io/apache}" +IMAGE_TAG="${IMAGE_TAG:-latest}" +CLUSTER_NAME="${CLUSTER_NAME:-texera-it}" +GATEWAY_PORT="${GATEWAY_PORT:-8080}" +K3S_IMAGE="${K3S_IMAGE:-rancher/k3s:v1.35.5-k3s1}" +read -r -a K3D_CREATE_ARGS <<< "${K3D_CREATE_ARGS:-}" +KEEP_CLUSTER="${KEEP_CLUSTER:-false}" +DIAGNOSTICS_DIR="${DIAGNOSTICS_DIR:-$PWD/k8s-integration-diagnostics}" + +# The gateway routes name their backends `texera-*`, so the release name is fixed. +RELEASE=texera +NAMESPACE=texera +CU_NAMESPACE=texera-workflow-computing-unit-pool +KUBE_CONTEXT="k3d-$CLUSTER_NAME" + +SECONDS=0 +log() { printf '\n[%4ds] %s\n' "$SECONDS" "$*"; } +kc() { kubectl --context "$KUBE_CONTEXT" "$@"; } + +collect_diagnostics() { + mkdir -p "$DIAGNOSTICS_DIR" + log "Collecting diagnostics into $DIAGNOSTICS_DIR" + kc get all,pvc,jobs,httproutes,gateways -A -o wide > "$DIAGNOSTICS_DIR/resources.txt" 2>&1 || true + kc get events -A --sort-by=.lastTimestamp > "$DIAGNOSTICS_DIR/events.txt" 2>&1 || true + kc describe pods -n "$NAMESPACE" > "$DIAGNOSTICS_DIR/describe-$NAMESPACE.txt" 2>&1 || true + kc describe pods -n "$CU_NAMESPACE" > "$DIAGNOSTICS_DIR/describe-$CU_NAMESPACE.txt" 2>&1 || true + local ns pod + for ns in "$NAMESPACE" "$CU_NAMESPACE"; do + for pod in $(kc get pods -n "$ns" -o name 2>/dev/null); do + kc logs -n "$ns" "$pod" --all-containers --prefix --tail=2000 \ + > "$DIAGNOSTICS_DIR/logs-$ns-${pod#pod/}.txt" 2>&1 || true + done + done + kc get pods -A -o wide || true +} + +cleanup() { + local status=$? + if [ "$status" -ne 0 ]; then + collect_diagnostics + fi + if [ "$KEEP_CLUSTER" != "true" ]; then + log "Deleting k3d cluster $CLUSTER_NAME" + k3d cluster delete "$CLUSTER_NAME" >/dev/null 2>&1 || true + fi + log "Finished with exit status $status" + exit "$status" +} +trap cleanup EXIT + +log "Creating k3d cluster $CLUSTER_NAME (gateway on localhost:$GATEWAY_PORT)" +k3d cluster delete "$CLUSTER_NAME" >/dev/null 2>&1 || true +k3d cluster create "$CLUSTER_NAME" \ + --image "$K3S_IMAGE" \ + --agents 0 \ + --k3s-arg "--disable=traefik@server:0" \ + --port "$GATEWAY_PORT:80@loadbalancer" \ + --wait --timeout 300s \ + "${K3D_CREATE_ARGS[@]}" +kc wait --for=condition=Ready node --all --timeout=300s + +log "Building chart dependencies" +helm dependency update "$CHART_DIR" + +log "Installing $RELEASE with $IMAGE_REGISTRY/*:$IMAGE_TAG" +helm --kube-context "$KUBE_CONTEXT" install "$RELEASE" "$CHART_DIR" \ + --namespace "$NAMESPACE" --create-namespace \ + --values "$SCRIPT_DIR/values-ci.yaml" \ + --set texera.imageRegistry="$IMAGE_REGISTRY" \ + --set texera.imageTag="$IMAGE_TAG" \ + --timeout 20m + +log "Waiting for the deployments to roll out" +DEPLOYMENTS=( + envoy-gateway + "$RELEASE-lakefs" + "$RELEASE-lakekeeper" + "$RELEASE-rustfs" + "$RELEASE-webserver" + "$RELEASE-file-service" + "$RELEASE-config-service" + "$RELEASE-access-control-service" + "$RELEASE-workflow-compiling-service" + "$RELEASE-workflow-computing-unit-manager" +) +for deployment in "${DEPLOYMENTS[@]}"; do + kc rollout status "deployment/$deployment" -n "$NAMESPACE" --timeout=20m +done +kc wait --for=condition=complete "job/$RELEASE-lakekeeper-init" -n "$NAMESPACE" --timeout=10m + +# Envoy Gateway creates the proxy deployment for the Gateway resource itself, so it only +# exists once the controller has reconciled it. +log "Waiting for the Envoy proxy" +for _ in $(seq 1 60); do + proxy=$(kc get deployment -n "$NAMESPACE" \ + -l gateway.envoyproxy.io/owning-gateway-name="$RELEASE-gateway" -o name 2>/dev/null || true) + [ -n "$proxy" ] && break + sleep 5 +done +[ -n "$proxy" ] || { echo "Envoy proxy deployment never appeared" >&2; exit 1; } +kc rollout status "$proxy" -n "$NAMESPACE" --timeout=10m +log "Deployment ready" + +log "Running the integration test" +TEXERA_URL="http://localhost:$GATEWAY_PORT" python3 "$SCRIPT_DIR/k8s_integration_test.py" diff --git a/bin/k8s/tests/integration/values-ci.yaml b/bin/k8s/tests/integration/values-ci.yaml new file mode 100644 index 00000000000..e95c5938157 --- /dev/null +++ b/bin/k8s/tests/integration/values-ci.yaml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Overrides for the k3d integration test (run_k3d_integration_test.sh). They +# shrink the deployment to what dataset upload and workflow execution need, so +# it fits a 4-vCPU / 16 GiB GitHub-hosted runner next to one computing unit. + +texeraImages: + pullPolicy: IfNotPresent + +postgresql: + primary: + resources: + requests: + cpu: "250m" + memory: "512Mi" + +# The test uploads its own dataset; the loader image is not part of the nightly build. +exampleDataLoader: + enabled: false + +# LLM features need provider keys a CI run does not have, and the litellm image is +# one of the largest in the chart. +agentService: + enabled: false +litellm: + enabled: false + +# k3s already serves the metrics API. +metrics-server: + enabled: false