diff --git a/.github/workflows/k8s-integration-test.yml b/.github/workflows/k8s-integration-test.yml new file mode 100644 index 00000000000..288f044f815 --- /dev/null +++ b/.github/workflows/k8s-integration-test.yml @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Deploys the Helm chart into a k3d cluster and drives one user flow through the +# gateway: upload a dataset, create a workflow, run it on a Kubernetes computing +# unit, and check its result. It deploys published images rather than building +# them, so it tests the chart in this checkout against those images. +name: K8s integration test + +on: + workflow_dispatch: + inputs: + image_registry: + description: 'Registry prefix of the Texera images (e.g., ghcr.io/apache, docker.io/apache)' + required: false + default: 'ghcr.io/apache' + type: string + image_tag: + description: 'Tag of the Texera images' + required: false + default: 'latest' + type: string + # Once the nightly build has published fresh ghcr.io/apache/*:latest images. + workflow_run: + workflows: ['Build and push images'] + types: [completed] + pull_request: + paths: + - 'bin/k8s/**' + - '.github/workflows/k8s-integration-test.yml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + K3D_VERSION: v5.9.0 + HELM_VERSION: v4.3.0 + KUBECTL_VERSION: v1.35.5 + +jobs: + k3d: + runs-on: ubuntu-latest + # A manually dispatched image build pushes some other tag, so only the nightly one + # leaves `latest` worth testing. + if: >- + github.event_name != 'workflow_run' || + (github.event.workflow_run.event == 'schedule' && + github.event.workflow_run.conclusion == 'success') + timeout-minutes: 60 + env: + IMAGE_REGISTRY: ${{ github.event.inputs.image_registry || 'ghcr.io/apache' }} + IMAGE_TAG: ${{ github.event.inputs.image_tag || 'latest' }} + DIAGNOSTICS_DIR: ${{ github.workspace }}/k8s-integration-diagnostics + steps: + - name: Checkout Texera + uses: actions/checkout@v7 + + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/share/boost /usr/local/lib/android + df -h / + + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: '3.12' + + - name: Install test dependencies + run: python -m pip install -r bin/k8s/tests/integration/requirements.txt + + - name: Install k3d, kubectl and helm + run: | + bin_dir="$RUNNER_TEMP/k8s-tools" + mkdir -p "$bin_dir" + curl -fsSL --retry 5 -o "$bin_dir/k3d" \ + "https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64" + curl -fsSL --retry 5 -o "$bin_dir/kubectl" \ + "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + curl -fsSL --retry 5 "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" \ + | tar -xz -C "$bin_dir" --strip-components=1 linux-amd64/helm + chmod +x "$bin_dir/k3d" "$bin_dir/kubectl" "$bin_dir/helm" + echo "$bin_dir" >> "$GITHUB_PATH" + + - name: Deploy to k3d and run the integration test + run: bin/k8s/tests/integration/run_k3d_integration_test.sh + + - name: Upload diagnostics + if: failure() + uses: actions/upload-artifact@v7 + with: + name: k8s-integration-diagnostics + path: ${{ env.DIAGNOSTICS_DIR }} + retention-days: 7 diff --git a/bin/k8s/templates/base/file-service/file-service-deployment.yaml b/bin/k8s/templates/base/file-service/file-service-deployment.yaml index e70f2d32135..f1246ea5f20 100644 --- a/bin/k8s/templates/base/file-service/file-service-deployment.yaml +++ b/bin/k8s/templates/base/file-service/file-service-deployment.yaml @@ -83,4 +83,10 @@ spec: - name: {{ .name }} value: "{{ .value }}" {{- end }} - {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file + {{- include "texera.observability.env" . | nindent 12 }} + readinessProbe: + httpGet: + path: /api/healthcheck + port: {{ .Values.fileService.service.port }} + initialDelaySeconds: 5 + periodSeconds: 5 diff --git a/bin/k8s/templates/base/webserver/webserver-deployment.yaml b/bin/k8s/templates/base/webserver/webserver-deployment.yaml index f71c2211903..d7cee50a311 100644 --- a/bin/k8s/templates/base/webserver/webserver-deployment.yaml +++ b/bin/k8s/templates/base/webserver/webserver-deployment.yaml @@ -74,8 +74,32 @@ spec: value: {{ .Values.lakekeeperInit.warehouse.region | quote }} - name: STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET value: {{ .Values.lakekeeperInit.warehouse.s3Bucket | quote }} + # Object store a per-user warehouse is registered on in Lakekeeper. Unset, it + # defaults to localhost, which Lakekeeper cannot reach. + - name: STORAGE_S3_ENDPOINT + value: {{ include "texera.s3.endpoint" . }} + {{- if .Values.storage.s3.endpoint }} + - name: STORAGE_S3_REGION + value: {{ .Values.storage.s3.region | quote }} + {{- end }} + - name: STORAGE_S3_AUTH_USERNAME + valueFrom: + secretKeyRef: + name: {{ include "texera.s3.secretName" . }} + key: {{ include "texera.s3.accessKeyIdKey" . }} + - name: STORAGE_S3_AUTH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "texera.s3.secretName" . }} + key: {{ include "texera.s3.secretAccessKeyKey" . }} {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" {{- end }} - {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file + {{- include "texera.observability.env" . | nindent 12 }} + readinessProbe: + httpGet: + path: /api/healthcheck + port: {{ .Values.webserver.service.port }} + initialDelaySeconds: 5 + periodSeconds: 5 diff --git a/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml index 0bc09731390..d9ad40088e0 100644 --- a/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml +++ b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml @@ -70,4 +70,10 @@ spec: - name: {{ .name }} value: "{{ .value }}" {{- end }} - {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file + {{- include "texera.observability.env" . | nindent 12 }} + readinessProbe: + httpGet: + path: /api/healthcheck + port: {{ .Values.workflowCompilingService.service.port }} + initialDelaySeconds: 5 + periodSeconds: 5 diff --git a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml index 18d6242c6a7..d41ed1bd633 100644 --- a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml +++ b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml @@ -152,4 +152,10 @@ spec: - name: {{ .name }} value: "{{ .value }}" {{- end }} - {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file + {{- include "texera.observability.env" . | nindent 12 }} + readinessProbe: + httpGet: + path: /api/healthcheck + port: {{ .Values.workflowComputingUnitManager.service.port }} + initialDelaySeconds: 5 + periodSeconds: 5 diff --git a/bin/k8s/tests/integration/k8s_integration_test.py b/bin/k8s/tests/integration/k8s_integration_test.py new file mode 100644 index 00000000000..994423f3f44 --- /dev/null +++ b/bin/k8s/tests/integration/k8s_integration_test.py @@ -0,0 +1,585 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +End-to-end check of a Texera deployment, driven the way the browser drives it: +every call goes through the Envoy gateway. + + login -> (warehouse) -> dataset + upload + version -> workflow create/read + -> computing unit -> websocket execute -> paginated result -> verify + +Configured by environment variables: + TEXERA_URL gateway base URL (default http://localhost:8080) + TEXERA_USERNAME (default texera) + TEXERA_PASSWORD (default texera) + TEXERA_STEP_TIMEOUT_S upper bound for each waiting step (default 900) +""" + +import base64 +import json +import os +import sys +import time +import uuid +from urllib.parse import quote, urlencode + +import requests +import websocket + +BASE_URL = os.environ.get("TEXERA_URL", "http://localhost:8080").rstrip("/") +USERNAME = os.environ.get("TEXERA_USERNAME", "texera") +PASSWORD = os.environ.get("TEXERA_PASSWORD", "texera") +STEP_TIMEOUT_S = int(os.environ.get("TEXERA_STEP_TIMEOUT_S", "900")) + +RUN_ID = uuid.uuid4().hex[:8] +DATASET_NAME = f"ci-people-{RUN_ID}" +CSV_NAME = "people.csv" +CSV_CONTENT = ( + "name,age,city\n" + "alice,34,Irvine\n" + "bob,27,Seattle\n" + "carol,45,Austin\n" + "dave,31,Boston\n" + "erin,19,Denver\n" +) +# Filter keeps age > 30; the Python UDF appends name_upper. +EXPECTED_ROWS = [ + {"name": "alice", "age": 34, "city": "Irvine", "name_upper": "ALICE"}, + {"name": "carol", "age": 45, "city": "Austin", "name_upper": "CAROL"}, + {"name": "dave", "age": 31, "city": "Boston", "name_upper": "DAVE"}, +] + +UDF_CODE = """from pytexera import * + +class ProcessTupleOperator(UDFOperatorV2): + @overrides + def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: + tuple_["name_upper"] = tuple_["name"].upper() + yield tuple_ +""" + +SCAN_ID = f"CSVFileScan-operator-{uuid.uuid4()}" +FILTER_ID = f"Filter-operator-{uuid.uuid4()}" +UDF_ID = f"PythonUDFV2-operator-{uuid.uuid4()}" + + +class IntegrationTestFailure(Exception): + pass + + +def step(title): + print(f"\n=== {title}", flush=True) + + +def info(message): + print(f" {message}", flush=True) + + +def check(condition, message): + if not condition: + raise IntegrationTestFailure(message) + + +def jwt_claims(token): + payload = token.split(".")[1] + payload += "=" * (-len(payload) % 4) + return json.loads(base64.urlsafe_b64decode(payload)) + + +def find_key(obj, key): + """Depth-first search for `key`, so the test does not pin response nesting.""" + if isinstance(obj, dict): + if key in obj: + return obj[key] + for value in obj.values(): + found = find_key(value, key) + if found is not None: + return found + elif isinstance(obj, list): + for value in obj: + found = find_key(value, key) + if found is not None: + return found + return None + + +class Client: + def __init__(self): + self.session = requests.Session() + self.token = None + + def call(self, method, path, expected=(200,), **kwargs): + headers = kwargs.pop("headers", {}) + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + response = self.session.request( + method, f"{BASE_URL}{path}", headers=headers, timeout=60, **kwargs + ) + if response.status_code not in expected: + raise IntegrationTestFailure( + f"{method} {path} -> HTTP {response.status_code}: {response.text[:2000]}" + ) + return response + + def json(self, method, path, **kwargs): + return self.call(method, path, **kwargs).json() + + +def wait_until(description, probe, interval_s=5): + deadline = time.monotonic() + STEP_TIMEOUT_S + last = None + while time.monotonic() < deadline: + try: + result = probe() + if result: + return result + last = "not ready" + except (requests.RequestException, IntegrationTestFailure) as e: + last = e + time.sleep(interval_s) + raise IntegrationTestFailure(f"timed out waiting for {description}: {last}") + + +def login(client): + step("Log in") + + def attempt(): + response = client.session.post( + f"{BASE_URL}/api/auth/login", + json={"username": USERNAME, "password": PASSWORD}, + timeout=30, + ) + if response.status_code != 200: + raise IntegrationTestFailure( + f"HTTP {response.status_code}: {response.text[:500]}" + ) + return response.json()["accessToken"] + + client.token = wait_until("the login endpoint behind the gateway", attempt) + claims = jwt_claims(client.token) + info(f"logged in as uid={claims.get('userId')} email={claims.get('email')}") + return claims + + +def prepare_warehouse(client): + step("Resolve the execution warehouse") + response = client.call("GET", "/api/warehouse/status", expected=(200, 404)) + if response.status_code == 404 or not response.json().get("enabled"): + info("per-user warehouses are off; executions use the shared default") + return None + created = client.json("POST", "/api/warehouse", json={"name": f"ci-{RUN_ID}"}) + info( + f"created warehouse whid={created['whid']} ({created['lakekeeperWarehouseName']})" + ) + return created["whid"] + + +def upload_dataset(client, owner_email): + step(f"Create dataset '{DATASET_NAME}' and upload {CSV_NAME}") + created = client.json( + "POST", + "/api/dataset/create", + json={ + "datasetName": DATASET_NAME, + "datasetDescription": "k3d integration test", + "isDatasetPublic": False, + }, + ) + did = find_key(created, "did") + check(did is not None, f"dataset create returned no did: {created}") + info(f"dataset did={did}") + + payload = CSV_CONTENT.encode() + common = { + "ownerEmail": owner_email, + "datasetName": DATASET_NAME, + "filePath": CSV_NAME, + } + init = client.json( + "POST", + "/api/dataset/multipart-upload?" + + urlencode( + { + **common, + "type": "init", + "fileSizeBytes": len(payload), + "partSizeBytes": len(payload), + } + ), + ) + check("missingParts" in init, f"multipart init returned no missingParts: {init}") + client.call( + "POST", + "/api/dataset/multipart-upload/part?" + urlencode({**common, "partNumber": 1}), + data=payload, + headers={"Content-Type": "application/octet-stream"}, + ) + client.call( + "POST", + "/api/dataset/multipart-upload?" + urlencode({**common, "type": "finish"}), + ) + + version = client.json( + "POST", + f"/api/dataset/{did}/version/create", + data="", + headers={"Content-Type": "text/plain"}, + ) + version_name = find_key(version, "name") + check(version_name, f"version create returned no version name: {version}") + info(f"created version '{version_name}'") + + listing = client.json("GET", "/api/dataset/list") + check( + any(find_key(entry, "name") == DATASET_NAME for entry in listing), + f"dataset '{DATASET_NAME}' missing from /api/dataset/list", + ) + return did, version_name + + +def port(port_id, display_name="", dependencies=None): + descriptor = { + "portID": port_id, + "displayName": display_name, + "allowMultiInputs": False, + "isDynamicPort": False, + } + if dependencies is not None: + descriptor["dependencies"] = dependencies + return descriptor + + +def operator(operator_id, operator_type, properties, inputs, view_result=False): + return { + "operatorID": operator_id, + "operatorType": operator_type, + "operatorVersion": "N/A", + "operatorProperties": properties, + "inputPorts": [port(p, dependencies=[]) for p in inputs], + "outputPorts": [port("output-0")], + "showAdvanced": False, + "isDisabled": False, + "customDisplayName": operator_type, + "dynamicInputPorts": False, + "dynamicOutputPorts": False, + "viewResult": view_result, + } + + +def link(source_id, target_id): + return { + "linkID": f"link-{uuid.uuid4()}", + "source": {"operatorID": source_id, "portID": "output-0"}, + "target": {"operatorID": target_id, "portID": "input-0"}, + } + + +def workflow_content(file_path): + operators = [ + operator( + SCAN_ID, + "CSVFileScan", + { + "fileEncoding": "UTF_8", + "customDelimiter": ",", + "hasHeader": True, + "fileName": file_path, + }, + inputs=[], + ), + operator( + FILTER_ID, + "Filter", + {"predicates": [{"attribute": "age", "condition": ">", "value": "30"}]}, + inputs=["input-0"], + ), + operator( + UDF_ID, + "PythonUDFV2", + { + "code": UDF_CODE, + "workers": 1, + "retainInputColumns": True, + "defaultEnv": True, + "outputColumns": [ + {"attributeName": "name_upper", "attributeType": "string"} + ], + }, + inputs=["input-0"], + view_result=True, + ), + ] + return { + "operators": operators, + "operatorPositions": { + op["operatorID"]: {"x": 200 * i, "y": 100} for i, op in enumerate(operators) + }, + "links": [link(SCAN_ID, FILTER_ID), link(FILTER_ID, UDF_ID)], + "commentBoxes": [], + "settings": {"dataTransferBatchSize": 400}, + } + + +def create_workflow(client, file_path): + step("Create and read back the workflow") + name = f"ci-workflow-{RUN_ID}" + created = client.json( + "POST", + "/api/workflow/create", + json={"name": name, "content": json.dumps(workflow_content(file_path))}, + ) + wid = find_key(created, "wid") + check(wid is not None, f"workflow create returned no wid: {created}") + info(f"workflow wid={wid}") + + stored = client.json("GET", f"/api/workflow/{wid}") + content = stored["content"] + content = json.loads(content) if isinstance(content, str) else content + stored_ids = sorted(op["operatorID"] for op in content["operators"]) + check( + stored_ids == sorted([SCAN_ID, FILTER_ID, UDF_ID]), + f"stored workflow operators differ: {stored_ids}", + ) + return wid, content + + +def logical_plan(content): + """Mirrors ExecuteWorkflowService.getLogicalPlanRequest in the frontend.""" + operators = {op["operatorID"]: op for op in content["operators"]} + + def ordinal(ports, port_id): + return next(i for i, p in enumerate(ports) if p["portID"] == port_id) + + return { + "operators": [ + { + **op["operatorProperties"], + "operatorID": op["operatorID"], + "operatorType": op["operatorType"], + "inputPorts": op["inputPorts"], + "outputPorts": op["outputPorts"], + } + for op in content["operators"] + ], + "links": [ + { + "fromOpId": l["source"]["operatorID"], + "fromPortId": { + "id": ordinal( + operators[l["source"]["operatorID"]]["outputPorts"], + l["source"]["portID"], + ), + "internal": False, + }, + "toOpId": l["target"]["operatorID"], + "toPortId": { + "id": ordinal( + operators[l["target"]["operatorID"]]["inputPorts"], + l["target"]["portID"], + ), + "internal": False, + }, + } + for l in content["links"] + ], + "opsToViewResult": [ + op["operatorID"] for op in content["operators"] if op.get("viewResult") + ], + "opsToReuseResult": [], + } + + +def create_computing_unit(client): + step("Create a Kubernetes computing unit") + limits = client.json("GET", "/api/computing-unit/limits") + info(f"limit options: {limits}") + memory = limits["memoryLimitOptions"][0] + created = client.json( + "POST", + "/api/computing-unit/create", + json={ + "name": f"ci-cu-{RUN_ID}", + "unitType": "kubernetes", + "cpuLimit": limits["cpuLimitOptions"][0], + "memoryLimit": memory, + "gpuLimit": limits["gpuLimitOptions"][0], + "jvmMemorySize": "1G", + "shmSize": "64Mi", + }, + ) + cuid = created["computingUnit"]["cuid"] + info(f"computing unit cuid={cuid}, status={created['status']}") + + def running(): + unit = client.json("GET", f"/api/computing-unit/{cuid}") + info(f"cuid={cuid} status={unit['status']} reason={unit.get('statusReason')}") + return unit["status"] == "Running" + + wait_until(f"computing unit {cuid} to run", running, interval_s=10) + return cuid + + +def websocket_url(wid, uid, cuid, token): + ws_base = BASE_URL.replace("https://", "wss://").replace("http://", "ws://") + query = urlencode( + {"wid": wid, "uid": uid, "cuid": cuid, "access-token": token}, quote_via=quote + ) + return f"{ws_base}/wsapi/workflow-websocket?{query}" + + +def open_websocket(url): + # The gateway answers 5xx while the unit's websocket endpoint is still coming up. + def attempt(): + try: + return websocket.create_connection(url, timeout=30) + except (websocket.WebSocketException, OSError) as e: + raise IntegrationTestFailure(f"websocket connect failed: {e}") from e + + return wait_until("the computing unit's websocket", attempt) + + +def receive(ws, deadline): + while time.monotonic() < deadline: + try: + frame = ws.recv() + except websocket.WebSocketTimeoutException: + ws.send(json.dumps({"type": "HeartBeatRequest"})) + continue + if not frame: + continue + return json.loads(frame) + raise IntegrationTestFailure("timed out waiting for a websocket event") + + +def execute_workflow(ws, content, cuid, warehouse_id): + step("Execute the workflow over the websocket") + request = { + "type": "WorkflowExecuteRequest", + "executionName": f"ci-execution-{RUN_ID}", + "engineVersion": "ci", + "logicalPlan": logical_plan(content), + "replayFromExecution": None, + "workflowSettings": content["settings"], + "emailNotificationEnabled": False, + "computingUnitId": cuid, + } + if warehouse_id is not None: + request["warehouseId"] = warehouse_id + ws.send(json.dumps(request)) + + deadline = time.monotonic() + STEP_TIMEOUT_S + seen_running = False + while True: + event = receive(ws, deadline) + kind = event.get("type") + if kind == "WorkflowStateEvent": + state = event["state"] + info(f"state -> {state}") + seen_running |= state == "Running" + if state == "Completed": + check( + seen_running, "execution completed without ever reporting Running" + ) + return + if state in ("Failed", "Killed", "Terminated"): + raise IntegrationTestFailure(f"execution ended in state {state}") + elif kind == "WorkflowErrorEvent": + raise IntegrationTestFailure( + f"execution error: {json.dumps(event['fatalErrors'])[:4000]}" + ) + elif kind == "ConsoleUpdateEvent": + for message in event.get("messages", []): + info( + f"console[{event.get('operatorId')}]: {message.get('title', '')}".rstrip() + ) + + +def fetch_result(ws, operator_id): + step("Read the result page") + request_id = str(uuid.uuid4()) + ws.send( + json.dumps( + { + "type": "ResultPaginationRequest", + "requestID": request_id, + "operatorID": operator_id, + "pageIndex": 1, + "pageSize": 100, + } + ) + ) + deadline = time.monotonic() + 120 + while True: + event = receive(ws, deadline) + if ( + event.get("type") == "PaginatedResultEvent" + and event.get("requestID") == request_id + ): + return event["table"] + if event.get("type") == "WorkflowErrorEvent": + raise IntegrationTestFailure( + f"pagination error: {json.dumps(event)[:4000]}" + ) + + +def verify_rows(rows): + info(f"received {len(rows)} row(s): {json.dumps(rows)}") + actual = sorted( + ({key: row.get(key) for key in EXPECTED_ROWS[0]} for row in rows), + key=lambda row: row["name"], + ) + normalized = [{key: str(value) for key, value in row.items()} for row in actual] + expected = [ + {key: str(value) for key, value in row.items()} for row in EXPECTED_ROWS + ] + check( + normalized == expected, + f"result mismatch:\n expected {expected}\n actual {normalized}", + ) + info("result matches") + + +def terminate_computing_unit(client, cuid): + step(f"Terminate computing unit {cuid}") + client.call("DELETE", f"/api/computing-unit/{cuid}/terminate", expected=(200, 204)) + + +def main(): + started = time.monotonic() + client = Client() + claims = login(client) + warehouse_id = prepare_warehouse(client) + _, version_name = upload_dataset(client, claims["email"]) + file_path = f"/dataset/{claims['email']}/{DATASET_NAME}/{version_name}/{CSV_NAME}" + wid, content = create_workflow(client, file_path) + cuid = create_computing_unit(client) + ws = open_websocket(websocket_url(wid, claims["userId"], cuid, client.token)) + try: + execute_workflow(ws, content, cuid, warehouse_id) + verify_rows(fetch_result(ws, UDF_ID)) + finally: + ws.close() + terminate_computing_unit(client, cuid) + print(f"\nPASS in {time.monotonic() - started:.0f}s", flush=True) + + +if __name__ == "__main__": + try: + main() + except IntegrationTestFailure as failure: + print(f"\nFAIL: {failure}", file=sys.stderr, flush=True) + sys.exit(1) diff --git a/bin/k8s/tests/integration/requirements.txt b/bin/k8s/tests/integration/requirements.txt new file mode 100644 index 00000000000..3446497da02 --- /dev/null +++ b/bin/k8s/tests/integration/requirements.txt @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +requests==2.34.2 +websocket-client==1.8.0 diff --git a/bin/k8s/tests/integration/run_k3d_integration_test.sh b/bin/k8s/tests/integration/run_k3d_integration_test.sh new file mode 100755 index 00000000000..ab7344fa79f --- /dev/null +++ b/bin/k8s/tests/integration/run_k3d_integration_test.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Deploys the Helm chart in bin/k8s into a throwaway k3d cluster and runs +# k8s_integration_test.py against it through the Envoy gateway. +# +# Needs docker, k3d, kubectl, helm and python3 (with requirements.txt installed). +# Run from anywhere: +# bin/k8s/tests/integration/run_k3d_integration_test.sh +# +# Environment: +# IMAGE_REGISTRY registry prefix of the Texera images (default ghcr.io/apache) +# IMAGE_TAG tag of the Texera images (default latest) +# CLUSTER_NAME k3d cluster name (default texera-it) +# GATEWAY_PORT host port mapped to the gateway (default 8080) +# K3S_IMAGE node image (default rancher/k3s:v1.35.5-k3s1) +# K3D_CREATE_ARGS extra `k3d cluster create` arguments, e.g. --host-alias for a local resolver +# KEEP_CLUSTER "true" leaves the cluster up afterwards +# DIAGNOSTICS_DIR where pod state and logs go on failure (default ./k8s-integration-diagnostics) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHART_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +IMAGE_REGISTRY="${IMAGE_REGISTRY:-ghcr.io/apache}" +IMAGE_TAG="${IMAGE_TAG:-latest}" +CLUSTER_NAME="${CLUSTER_NAME:-texera-it}" +GATEWAY_PORT="${GATEWAY_PORT:-8080}" +K3S_IMAGE="${K3S_IMAGE:-rancher/k3s:v1.35.5-k3s1}" +read -r -a K3D_CREATE_ARGS <<< "${K3D_CREATE_ARGS:-}" +KEEP_CLUSTER="${KEEP_CLUSTER:-false}" +DIAGNOSTICS_DIR="${DIAGNOSTICS_DIR:-$PWD/k8s-integration-diagnostics}" + +# The gateway routes name their backends `texera-*`, so the release name is fixed. +RELEASE=texera +NAMESPACE=texera +CU_NAMESPACE=texera-workflow-computing-unit-pool +KUBE_CONTEXT="k3d-$CLUSTER_NAME" + +SECONDS=0 +log() { printf '\n[%4ds] %s\n' "$SECONDS" "$*"; } +kc() { kubectl --context "$KUBE_CONTEXT" "$@"; } + +collect_diagnostics() { + mkdir -p "$DIAGNOSTICS_DIR" + log "Collecting diagnostics into $DIAGNOSTICS_DIR" + kc get all,pvc,jobs,httproutes,gateways -A -o wide > "$DIAGNOSTICS_DIR/resources.txt" 2>&1 || true + kc get events -A --sort-by=.lastTimestamp > "$DIAGNOSTICS_DIR/events.txt" 2>&1 || true + kc describe pods -n "$NAMESPACE" > "$DIAGNOSTICS_DIR/describe-$NAMESPACE.txt" 2>&1 || true + kc describe pods -n "$CU_NAMESPACE" > "$DIAGNOSTICS_DIR/describe-$CU_NAMESPACE.txt" 2>&1 || true + local ns pod + for ns in "$NAMESPACE" "$CU_NAMESPACE"; do + for pod in $(kc get pods -n "$ns" -o name 2>/dev/null); do + kc logs -n "$ns" "$pod" --all-containers --prefix --tail=2000 \ + > "$DIAGNOSTICS_DIR/logs-$ns-${pod#pod/}.txt" 2>&1 || true + done + done + kc get pods -A -o wide || true +} + +cleanup() { + local status=$? + if [ "$status" -ne 0 ]; then + collect_diagnostics + fi + if [ "$KEEP_CLUSTER" != "true" ]; then + log "Deleting k3d cluster $CLUSTER_NAME" + k3d cluster delete "$CLUSTER_NAME" >/dev/null 2>&1 || true + fi + log "Finished with exit status $status" + exit "$status" +} +trap cleanup EXIT + +log "Creating k3d cluster $CLUSTER_NAME (gateway on localhost:$GATEWAY_PORT)" +k3d cluster delete "$CLUSTER_NAME" >/dev/null 2>&1 || true +k3d cluster create "$CLUSTER_NAME" \ + --image "$K3S_IMAGE" \ + --agents 0 \ + --k3s-arg "--disable=traefik@server:0" \ + --port "$GATEWAY_PORT:80@loadbalancer" \ + --wait --timeout 300s \ + "${K3D_CREATE_ARGS[@]}" +kc wait --for=condition=Ready node --all --timeout=300s + +log "Building chart dependencies" +helm dependency update "$CHART_DIR" + +log "Installing $RELEASE with $IMAGE_REGISTRY/*:$IMAGE_TAG" +helm --kube-context "$KUBE_CONTEXT" install "$RELEASE" "$CHART_DIR" \ + --namespace "$NAMESPACE" --create-namespace \ + --values "$SCRIPT_DIR/values-ci.yaml" \ + --set texera.imageRegistry="$IMAGE_REGISTRY" \ + --set texera.imageTag="$IMAGE_TAG" \ + --timeout 20m + +log "Waiting for the deployments to roll out" +DEPLOYMENTS=( + envoy-gateway + "$RELEASE-lakefs" + "$RELEASE-lakekeeper" + "$RELEASE-rustfs" + "$RELEASE-webserver" + "$RELEASE-file-service" + "$RELEASE-config-service" + "$RELEASE-access-control-service" + "$RELEASE-workflow-compiling-service" + "$RELEASE-workflow-computing-unit-manager" +) +for deployment in "${DEPLOYMENTS[@]}"; do + kc rollout status "deployment/$deployment" -n "$NAMESPACE" --timeout=20m +done +kc wait --for=condition=complete "job/$RELEASE-lakekeeper-init" -n "$NAMESPACE" --timeout=10m + +# Envoy Gateway creates the proxy deployment for the Gateway resource itself, so it only +# exists once the controller has reconciled it. +log "Waiting for the Envoy proxy" +for _ in $(seq 1 60); do + proxy=$(kc get deployment -n "$NAMESPACE" \ + -l gateway.envoyproxy.io/owning-gateway-name="$RELEASE-gateway" -o name 2>/dev/null || true) + [ -n "$proxy" ] && break + sleep 5 +done +[ -n "$proxy" ] || { echo "Envoy proxy deployment never appeared" >&2; exit 1; } +kc rollout status "$proxy" -n "$NAMESPACE" --timeout=10m +log "Deployment ready" + +log "Running the integration test" +TEXERA_URL="http://localhost:$GATEWAY_PORT" python3 "$SCRIPT_DIR/k8s_integration_test.py" diff --git a/bin/k8s/tests/integration/values-ci.yaml b/bin/k8s/tests/integration/values-ci.yaml new file mode 100644 index 00000000000..e95c5938157 --- /dev/null +++ b/bin/k8s/tests/integration/values-ci.yaml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Overrides for the k3d integration test (run_k3d_integration_test.sh). They +# shrink the deployment to what dataset upload and workflow execution need, so +# it fits a 4-vCPU / 16 GiB GitHub-hosted runner next to one computing unit. + +texeraImages: + pullPolicy: IfNotPresent + +postgresql: + primary: + resources: + requests: + cpu: "250m" + memory: "512Mi" + +# The test uploads its own dataset; the loader image is not part of the nightly build. +exampleDataLoader: + enabled: false + +# LLM features need provider keys a CI run does not have, and the litellm image is +# one of the largest in the chart. +agentService: + enabled: false +litellm: + enabled: false + +# k3s already serves the metrics API. +metrics-server: + enabled: false