diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index 830dbbe5..0dfa62f8 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -32,29 +32,21 @@ jobs: env: PYTEST_ADDOPTS: "--durations=0 --durations-min=5" run: | - docker compose up \ - --build \ - --abort-on-container-exit \ - --exit-code-from integration-tests + docker compose -f docker-compose.lakehouse.yml up -d --build + docker compose -f docker-compose.lakehouse.yml exec -T cdm-data-loaders pytest -m "requires_ceph" -v - name: Capture compose diagnostics on failure if: failure() run: | echo "=== docker compose ps ===" - docker compose ps -a || true + docker compose -f docker-compose.lakehouse.yml ps -a || true echo "=== ceph logs ===" - docker logs cdm-data-loaders-ceph-1 || true + docker compose -f docker-compose.lakehouse.yml logs ceph || true - echo "=== ceph integration test logs ===" - docker logs cdm-data-loaders-integration-tests-1 || true - - echo "=== ceph inspect state ===" - docker inspect cdm-data-loaders-ceph-1 --format '{{json .State}}' || true - - echo "=== ceph health state ===" - docker inspect cdm-data-loaders-ceph-1 --format '{{json .State.Health}}' || true + echo "=== cdm-data-loaders logs ===" + docker compose -f docker-compose.lakehouse.yml logs cdm-data-loaders || true - name: Tear down if: always() - run: docker compose down --volumes + run: docker compose -f docker-compose.lakehouse.yml down --volumes diff --git a/Dockerfile.lakehouse b/Dockerfile.lakehouse new file mode 100644 index 00000000..6b939a56 --- /dev/null +++ b/Dockerfile.lakehouse @@ -0,0 +1,46 @@ +ARG SPARK_IMAGE_TAG=main +FROM ghcr.io/kbasedatalakehouse/kube_spark_manager_image:${SPARK_IMAGE_TAG} + +ARG QSV_VERSION="28.0.0" +ARG XML_FILE_SPLITTER_VERSION="v0.1.4" + +USER root + +# Install qsv and xml_file_splitter binaries required by cdm-data-loaders readers +RUN ARCH=$(uname -m) && \ + curl -fsSL -o /tmp/xml_file_splitter.tar.gz "https://github.com/ialarmedalien/xml_file_splitter/releases/download/${XML_FILE_SPLITTER_VERSION}/xml_file_splitter-${ARCH}-unknown-linux-gnu.tar.gz" && \ + tar -xf /tmp/xml_file_splitter.tar.gz -C /tmp && \ + mv /tmp/xml_file_splitter-${ARCH}-unknown-linux-gnu/xml_file_splitter /usr/local/bin/ && \ + curl -fsSL -o /tmp/qsv.zip "https://github.com/dathere/qsv/releases/download/${QSV_VERSION}/qsv-${QSV_VERSION}-${ARCH}-unknown-linux-gnu.zip" && \ + unzip -q /tmp/qsv.zip -d /tmp/qsv && \ + mv /tmp/qsv/qsv /usr/local/bin/ && \ + chmod +x /usr/local/bin/xml_file_splitter /usr/local/bin/qsv && \ + rm -rf /tmp/* + +# Install uv for fast, locked dependency resolution +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ + +WORKDIR /app + +# Install cdm-data-loaders dependencies into the base image's conda environment +# (--inexact preserves pre-installed packages like berdl_notebook_utils and pyspark). +ENV UV_PROJECT_ENVIRONMENT=/opt/conda +ENV UV_COMPILE_BYTECODE=1 +ENV UV_LINK_MODE=copy + +COPY pyproject.toml uv.lock README.md ./ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --locked --inexact --no-install-project --all-groups + +COPY src ./src +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --python /opt/conda/bin/python --no-deps -e . + +ENV PATH="/opt/conda/bin:$PATH" +ENV PYTHONUNBUFFERED=1 + +# Override kube_spark_manager_image's Spark healthcheck and entrypoint so this +# container idles for interactive testing (pytest, CLI, Spark Connect client). +HEALTHCHECK NONE +ENTRYPOINT ["tini", "--"] +CMD ["sleep", "infinity"] diff --git a/docker-compose.lakehouse.yml b/docker-compose.lakehouse.yml new file mode 100644 index 00000000..fbbbc05e --- /dev/null +++ b/docker-compose.lakehouse.yml @@ -0,0 +1,286 @@ +# Lakehouse test stack for cdm-data-loaders: +# - Ceph RGW (S3 + IAM/STS) +# - Apache Polaris (stock, relational-jdbc on Postgres) +# - KBase Auth2 (test mode, on Mongo) for Spark Connect auth interceptor +# - Spark Master + Worker + Spark Connect (kube_spark_manager_image) +# - cdm-data-loaders interactive dev/test container (kube_spark_manager_image + repo) + +networks: + default: + +volumes: + polaris-credential: + postgres-data: + mongo-data: + ceph-data: + +services: + ceph: + image: ghcr.io/kbasetest/ceph-rgw-test-image:0.2.1 + networks: + default: + aliases: ["s3"] + environment: + RGW_PORT: "9090" + RGW_ACCESS_KEY: s3user + RGW_SECRET_KEY: s3password + RGW_USERS: "sql:sql:sqlpassword" + RGW_STS_KEY: cdmteststs123456 + ports: + - "9090:9090" + - "9091:8443" + volumes: + - ceph-data:/var/lib/ceph + healthcheck: + test: ["CMD-SHELL", "ceph osd stat 2>/dev/null | grep -qE '[1-9][0-9]* up' && curl -s http://127.0.0.1:9090/ >/dev/null 2>&1"] + interval: 10s + timeout: 10s + retries: 10 + start_period: 30s + + s3-create-bucket-ceph: + image: amazon/aws-cli:2.31.14 + depends_on: + ceph: + condition: service_healthy + environment: + S3_URL: http://s3:9090 + S3_USER: s3user + S3_PWD: s3password + S3_SQL_BUCKET: sql + S3_TEST_BUCKET: test-bucket + S3_SQL_RW_USER: sql + POLARIS_VENDING_STS_ROLE_NAME: cdm-polaris-vending + entrypoint: ["/bin/bash", "/s3_policies/ceph_create_bucket_entrypoint.sh"] + volumes: + - ./docker/lakehouse:/s3_policies:ro + + postgres: + image: postgres:16.3 + ports: + - "5432:5432" + environment: + POSTGRES_USER: polaris + POSTGRES_PASSWORD: polarispassword + POSTGRES_DB: polaris + healthcheck: + test: ["CMD-SHELL", "pg_isready -U polaris -d polaris"] + interval: 2s + timeout: 5s + retries: 20 + volumes: + - postgres-data:/var/lib/postgresql/data + + polaris-bootstrap: + image: apache/polaris-admin-tool:1.4.1 + depends_on: + postgres: + condition: service_healthy + environment: + POLARIS_PERSISTENCE_TYPE: relational-jdbc + QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/polaris + QUARKUS_DATASOURCE_USERNAME: polaris + QUARKUS_DATASOURCE_PASSWORD: polarispassword + POLARIS_REALM: POLARIS + POLARIS_BOOTSTRAP_CREDENTIAL: POLARIS,root,s3cr3t + entrypoint: ["/bin/sh", "/polaris/polaris_bootstrap.sh"] + volumes: + - ./docker/lakehouse:/polaris:ro + + polaris: + image: apache/polaris:1.4.1 + ports: + - "8181:8181" + - "8182:8182" + depends_on: + polaris-bootstrap: + condition: service_completed_successfully + environment: + POLARIS_PERSISTENCE_TYPE: relational-jdbc + QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/polaris + QUARKUS_DATASOURCE_USERNAME: polaris + QUARKUS_DATASOURCE_PASSWORD: polarispassword + POLARIS_REALM_CONTEXT_REALMS: POLARIS + AWS_REGION: us-east-1 + AWS_ACCESS_KEY_ID: sql + AWS_SECRET_ACCESS_KEY: sqlpassword + AWS_ENDPOINT_URL_S3: http://s3:9090 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8182/q/health"] + interval: 2s + timeout: 10s + retries: 60 + + polaris-setup: + image: curlimages/curl:8.11.1 + user: root + depends_on: + polaris: + condition: service_healthy + s3-create-bucket-ceph: + condition: service_completed_successfully + environment: + POLARIS_HOST: http://polaris:8181 + POLARIS_REALM: POLARIS + POLARIS_CLIENT_ID: root + POLARIS_CLIENT_SECRET: s3cr3t + POLARIS_SERVICE_PRINCIPAL: cdm_loader + POLARIS_SERVICE_PRINCIPAL_ROLE: service_admin + POLARIS_SERVICE_CREDENTIAL_FILE: /shared-polaris/polaris_credential.txt + POLARIS_CATALOG_NAME: cdm + S3_ENDPOINT: http://s3:9090 + CATALOG_BASE_LOCATION: s3://sql/warehouse + entrypoint: ["/bin/sh", "/polaris/polaris_setup.sh"] + volumes: + - ./docker/lakehouse:/polaris:ro + - polaris-credential:/shared-polaris + + mongo: + image: mongo:7.0 + ports: + - "27019:27019" + command: ["mongod", "--port", "27019"] + environment: + MONGO_INITDB_ROOT_USERNAME: cdm + MONGO_INITDB_ROOT_PASSWORD: cdmpassword + healthcheck: + test: ["CMD", "mongosh", "--quiet", "--port", "27019", "--eval", "db.adminCommand('ping')"] + interval: 2s + timeout: 5s + retries: 20 + volumes: + - mongo-data:/data/db + + mongo-init-auth2-user: + image: mongo:7.0 + depends_on: + mongo: + condition: service_healthy + environment: + MONGO_HOST: "mongo:27019" + MONGO_ROOT_USER: cdm + MONGO_ROOT_PWD: cdmpassword + AUTH2_MONGO_DB: auth2 + AUTH2_MONGO_USER: auth2 + AUTH2_MONGO_PWD: auth2password + entrypoint: ["/bin/sh", "/mongo-init/init_auth2_user.sh"] + volumes: + - ./docker/lakehouse:/mongo-init:ro + + auth2: + image: ghcr.io/kbase/auth2:0.8.1 + platform: linux/amd64 + depends_on: + mongo-init-auth2-user: + condition: service_completed_successfully + ports: + - "8180:8080" + environment: + mongo_host: "mongo:27019" + mongo_db: "auth2" + mongo_user: "auth2" + mongo_pwd: "auth2password" + test_mode_enabled: "true" + identity_providers: "" + command: + - "-template" + - "/kb/deployment/conf/.templates/deployment.cfg.templ:/kb/deployment/conf/deployment.cfg" + - "/kb/deployment/bin/start_auth2.sh" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/"] + interval: 2s + timeout: 5s + retries: 60 + + spark-master: + image: ghcr.io/kbasedatalakehouse/kube_spark_manager_image:${SPARK_IMAGE_TAG:-main} + platform: linux/amd64 + ports: + - "8090:8090" + - "7077:7077" + environment: + SPARK_MODE: master + SPARK_MASTER_HOST: spark-master + SPARK_MASTER_PORT: "7077" + SPARK_MASTER_WEBUI_PORT: "8090" + + spark-worker-1: + image: ghcr.io/kbasedatalakehouse/kube_spark_manager_image:${SPARK_IMAGE_TAG:-main} + platform: linux/amd64 + depends_on: + spark-master: + condition: service_healthy + ports: + - "8081:8081" + environment: + SPARK_MODE: worker + SPARK_MASTER_URL: spark://spark-master:7077 + SPARK_WORKER_CORES: "4" + SPARK_WORKER_MEMORY: 2G + SPARK_WORKER_PORT: "8083" + SPARK_WORKER_WEBUI_PORT: "8081" + + spark-connect: + image: ghcr.io/kbasedatalakehouse/kube_spark_manager_image:${SPARK_IMAGE_TAG:-main} + platform: linux/amd64 + depends_on: + spark-master: + condition: service_healthy + polaris-setup: + condition: service_completed_successfully + auth2: + condition: service_healthy + ports: + - "15002:15002" + environment: + SPARK_MODE: connect + USER: testuser + BERDL_POD_IP: spark-connect + SPARK_MASTER_URL: spark://spark-master:7077 + KBASE_AUTH_URL: http://auth2:8080/testmode/ + S3_ENDPOINT_URL: http://s3:9090 + S3_ACCESS_KEY: sql + S3_SECRET_KEY: sqlpassword + POLARIS_CATALOG_URI: http://polaris:8181/api/catalog + POLARIS_CREDENTIAL: root:s3cr3t + POLARIS_PERSONAL_CATALOG: cdm + BERDL_SPARK_DYNAMIC_ALLOCATION: "true" + SPARK_MAX_EXECUTORS: "2" + SPARK_WORKER_CORES: "4" + SPARK_WORKER_MEMORY: 2G + SPARK_MASTER_CORES: "4" + SPARK_MASTER_MEMORY: 2G + BERDL_SPARK_MODE: shared + BERDL_CONNECT_SESSION_TIMEOUT: 30m + + cdm-data-loaders: + build: + context: . + dockerfile: Dockerfile.lakehouse + args: + SPARK_IMAGE_TAG: ${SPARK_IMAGE_TAG:-main} + platform: linux/amd64 + depends_on: + s3-create-bucket-ceph: + condition: service_completed_successfully + polaris-setup: + condition: service_completed_successfully + spark-connect: + condition: service_healthy + environment: + AWS_ENDPOINT_URL: http://s3:9090 + AWS_ACCESS_KEY_ID: s3user + AWS_SECRET_ACCESS_KEY: s3password + AWS_DEFAULT_REGION: us-east-1 + S3_ENDPOINT_URL: http://s3:9090 + POLARIS_HOST: http://polaris:8181 + POLARIS_CATALOG_URI: http://polaris:8181/api/catalog + POLARIS_REALM: POLARIS + POLARIS_CATALOG: cdm + POLARIS_CREDENTIAL_FILE: /shared-polaris/polaris_credential.txt + SPARK_CONNECT_URL: sc://spark-connect:15002 + KBASE_AUTH_URL: http://auth2:8080/testmode/ + PYTEST_ADDOPTS: ${PYTEST_ADDOPTS:-} + volumes: + - .:/app + - polaris-credential:/shared-polaris:ro diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index d3e226c9..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,39 +0,0 @@ -services: - ceph: - image: ghcr.io/kbasetest/ceph-rgw-test-image:0.1.5 - environment: - RGW_PORT: "8080" - RGW_ACCESS_KEY: test_access_key - RGW_SECRET_KEY: test_access_secret - ports: - - "9000:8080" - - "9001:8443" - tmpfs: - - /var/lib/ceph - restart: on-failure - healthcheck: - # Two-part check: - # 1. ceph osd stat: verifies at least one OSD has joined the cluster. - # 2. curl: confirms the RGW HTTP listener is accepting connections. - # NOTE: ceph health is NOT used here — this image permanently sits at HEALTH_WARN due to - # AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED and MON_MSGR2_NOT_ENABLED, so it never passes. - test: [ "CMD-SHELL", "ceph osd stat 2>/dev/null | grep -qE '[1-9][0-9]* up' && curl -s http://127.0.0.1:8080/ >/dev/null 2>&1" ] - interval: 10s - timeout: 10s - retries: 10 - start_period: 30s - - integration-tests: - image: cdm-data-loaders-integration-tests:latest - build: - context: . - depends_on: - ceph: - condition: service_healthy - environment: - AWS_ENDPOINT_URL: http://ceph:8080 - AWS_ACCESS_KEY_ID: test_access_key - AWS_SECRET_ACCESS_KEY: test_access_secret - PYTEST_ADDOPTS: ${PYTEST_ADDOPTS:-} - entrypoint: [ "/app/scripts/entrypoint.sh", "ceph_integration_test" ] - command: [] diff --git a/docker/lakehouse/ceph-polaris-vending-role-policy.json b/docker/lakehouse/ceph-polaris-vending-role-policy.json new file mode 100644 index 00000000..32f2bde6 --- /dev/null +++ b/docker/lakehouse/ceph-polaris-vending-role-policy.json @@ -0,0 +1,20 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:s3:::sql", + "arn:aws:s3:::sql/*" + ] + } + ] +} diff --git a/docker/lakehouse/ceph_create_bucket_entrypoint.sh b/docker/lakehouse/ceph_create_bucket_entrypoint.sh new file mode 100755 index 00000000..d9abbe91 --- /dev/null +++ b/docker/lakehouse/ceph_create_bucket_entrypoint.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e + +printf "\n*** starting ceph bucket & policy setup ***\n" + +export AWS_ACCESS_KEY_ID="$S3_USER" +export AWS_SECRET_ACCESS_KEY="$S3_PWD" +export AWS_DEFAULT_REGION=us-east-1 + +AWS="aws --endpoint-url $S3_URL" + +make_bucket() { + if $AWS s3api head-bucket --bucket "$1" 2>/dev/null; then + echo "bucket $1 already exists" + else + $AWS s3api create-bucket --bucket "$1" && echo "Bucket $1 created" + fi +} + +make_bucket "$S3_SQL_BUCKET" +make_bucket "$S3_TEST_BUCKET" +make_bucket "cdm-spark-job-logs" +$AWS s3api put-object --bucket "cdm-spark-job-logs" --key "spark-job-logs/testuser/.keep" >/dev/null + +$AWS iam put-user-policy --user-name "$S3_SQL_RW_USER" --policy-name cdm-s3-access \ + --policy-document file:///s3_policies/sql-read-write-policy.json +echo "sql read-write user policy set" + +cat > /tmp/polaris-vending-trust-policy.json </dev/null 2>&1; then + echo "$POLARIS_VENDING_STS_ROLE_NAME role already exists" +else + $AWS iam create-role --role-name "$POLARIS_VENDING_STS_ROLE_NAME" \ + --max-session-duration 43200 \ + --assume-role-policy-document file:///tmp/polaris-vending-trust-policy.json + echo "$POLARIS_VENDING_STS_ROLE_NAME role created" +fi + +$AWS iam put-role-policy --role-name "$POLARIS_VENDING_STS_ROLE_NAME" \ + --policy-name "${POLARIS_VENDING_STS_ROLE_NAME}-access" \ + --policy-document file:///s3_policies/ceph-polaris-vending-role-policy.json +echo "$POLARIS_VENDING_STS_ROLE_NAME role policy set" diff --git a/docker/lakehouse/init_auth2_user.sh b/docker/lakehouse/init_auth2_user.sh new file mode 100755 index 00000000..dc87716d --- /dev/null +++ b/docker/lakehouse/init_auth2_user.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -e + +mongosh "mongodb://${MONGO_ROOT_USER}:${MONGO_ROOT_PWD}@${MONGO_HOST}/admin" --quiet --eval ' + const dbName = process.env.AUTH2_MONGO_DB; + const user = process.env.AUTH2_MONGO_USER; + const pwd = process.env.AUTH2_MONGO_PWD; + try { + db.getSiblingDB(dbName).createUser({ + user: user, + pwd: pwd, + roles: [{ role: "readWrite", db: dbName }], + }); + print("auth2 mongo user created"); + } catch (e) { + if (/already exists/i.test(e.message)) { + print("auth2 mongo user already exists"); + } else { + throw e; + } + } +' diff --git a/docker/lakehouse/polaris_bootstrap.sh b/docker/lakehouse/polaris_bootstrap.sh new file mode 100755 index 00000000..05187e03 --- /dev/null +++ b/docker/lakehouse/polaris_bootstrap.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +POLARIS_REALM="${POLARIS_REALM:-POLARIS}" +POLARIS_BOOTSTRAP_CREDENTIAL="${POLARIS_BOOTSTRAP_CREDENTIAL:-POLARIS,root,s3cr3t}" + +java -jar /deployments/polaris-admin-tool.jar bootstrap \ + "--realm=${POLARIS_REALM}" \ + "--credential=${POLARIS_BOOTSTRAP_CREDENTIAL}" + +status=$? +if [ "${status}" -eq 0 ] || [ "${status}" -eq 3 ]; then + exit 0 +fi +exit "${status}" diff --git a/docker/lakehouse/polaris_setup.sh b/docker/lakehouse/polaris_setup.sh new file mode 100755 index 00000000..3d44743a --- /dev/null +++ b/docker/lakehouse/polaris_setup.sh @@ -0,0 +1,180 @@ +#!/bin/sh +set -e + +POLARIS_HOST="${POLARIS_HOST:?}" +POLARIS_REALM="${POLARIS_REALM:?}" +POLARIS_CLIENT_ID="${POLARIS_CLIENT_ID:?}" +POLARIS_CLIENT_SECRET="${POLARIS_CLIENT_SECRET:?}" +POLARIS_SERVICE_PRINCIPAL="${POLARIS_SERVICE_PRINCIPAL:?}" +POLARIS_SERVICE_PRINCIPAL_ROLE="${POLARIS_SERVICE_PRINCIPAL_ROLE:?}" +POLARIS_SERVICE_CREDENTIAL_FILE="${POLARIS_SERVICE_CREDENTIAL_FILE:?}" +POLARIS_CATALOG_NAME="${POLARIS_CATALOG_NAME:-cdm}" +S3_ENDPOINT="${S3_ENDPOINT:-http://s3:9090}" +CATALOG_BASE_LOCATION="${CATALOG_BASE_LOCATION:-s3://sql/warehouse}" + +MGMT="${POLARIS_HOST}/api/management/v1" +REALM_HEADER="Polaris-Realm: ${POLARIS_REALM}" +CONTENT_TYPE_HEADER="Content-Type: application/json" + +echo "Getting OAuth token from Polaris..." +TOKEN_RESPONSE=$(curl -sf -X POST "${POLARIS_HOST}/api/catalog/v1/oauth/tokens" \ + -u "${POLARIS_CLIENT_ID}:${POLARIS_CLIENT_SECRET}" \ + -H "${REALM_HEADER}" \ + -d "grant_type=client_credentials&scope=PRINCIPAL_ROLE:ALL") + +TOKEN="${TOKEN_RESPONSE#*\"access_token\":\"}" +TOKEN="${TOKEN%%\"*}" + +if [ -z "${TOKEN}" ] || [ "${TOKEN}" = "${TOKEN_RESPONSE}" ]; then + echo "ERROR: Failed to get OAuth token. Response: ${TOKEN_RESPONSE}" + exit 1 +fi + +AUTH_HEADER="Authorization: Bearer ${TOKEN}" + +api() { + method="$1" + path="$2" + data="$3" + if [ -n "${data}" ]; then + curl -s -o /dev/null -w "%{http_code}" -X "${method}" "${MGMT}${path}" \ + -H "${AUTH_HEADER}" \ + -H "${CONTENT_TYPE_HEADER}" \ + -H "${REALM_HEADER}" \ + -d "${data}" + else + curl -s -o /dev/null -w "%{http_code}" -X "${method}" "${MGMT}${path}" \ + -H "${AUTH_HEADER}" \ + -H "${CONTENT_TYPE_HEADER}" \ + -H "${REALM_HEADER}" + fi +} + +api_response() { + method="$1" + path="$2" + data="$3" + tmpfile="$(mktemp)" + RESPONSE_STATUS=$(curl -s -o "${tmpfile}" -w "%{http_code}" -X "${method}" "${MGMT}${path}" \ + -H "${AUTH_HEADER}" \ + -H "${CONTENT_TYPE_HEADER}" \ + -H "${REALM_HEADER}" \ + -d "${data}") + RESPONSE_BODY="$(cat "${tmpfile}")" + rm -f "${tmpfile}" +} + +check_status() { + status="$1" + message="$2" + allow_existing_grant_status="${3:-false}" + case "${status}" in + 200|201|204) echo " -> ${message}" ;; + 409) echo " -> already exists" ;; + 500) + if [ "${allow_existing_grant_status}" = "true" ]; then + echo " -> already configured" + else + echo "ERROR: unexpected status ${status}" + exit 1 + fi + ;; + *) echo "ERROR: unexpected status ${status}"; exit 1 ;; + esac +} + +json_field() { + printf "%s" "$1" | sed -n "s/.*\"$2\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p" +} + +credential_is_valid() { + credential_file="$1" + [ -f "${credential_file}" ] || return 1 + client_id="$(cut -d: -f1 "${credential_file}")" + client_secret="$(cut -d: -f2- "${credential_file}")" + status=$(curl -s -o /dev/null -w "%{http_code}" -X POST "${POLARIS_HOST}/api/catalog/v1/oauth/tokens" \ + -u "${client_id}:${client_secret}" \ + -H "${REALM_HEADER}" \ + -d "grant_type=client_credentials&scope=PRINCIPAL_ROLE:ALL") || status="000" + [ "${status}" = "200" ] +} + +echo "Setting up Polaris principal ${POLARIS_SERVICE_PRINCIPAL}..." +if credential_is_valid "${POLARIS_SERVICE_CREDENTIAL_FILE}"; then + echo " -> existing credentials at ${POLARIS_SERVICE_CREDENTIAL_FILE} still valid" +else + status=$(api POST "/principals" "{ + \"principal\": { + \"name\": \"${POLARIS_SERVICE_PRINCIPAL}\", + \"type\": \"USER\", + \"properties\": {} + } + }") + check_status "${status}" "principal created" + + status=$(api PUT "/principals/${POLARIS_SERVICE_PRINCIPAL}/principal-roles" "{ + \"principalRole\": { + \"name\": \"${POLARIS_SERVICE_PRINCIPAL_ROLE}\" + } + }") + check_status "${status}" "principal role assigned" true + + api_response POST "/principals/${POLARIS_SERVICE_PRINCIPAL}/reset" "{}" + check_status "${RESPONSE_STATUS}" "principal credentials reset" + + client_id="$(json_field "${RESPONSE_BODY}" "clientId")" + client_secret="$(json_field "${RESPONSE_BODY}" "clientSecret")" + if [ -z "${client_id}" ] || [ -z "${client_secret}" ]; then + echo "ERROR: Could not parse principal credentials. Response: ${RESPONSE_BODY}" + exit 1 + fi + + mkdir -p "$(dirname "${POLARIS_SERVICE_CREDENTIAL_FILE}")" + printf "%s:%s" "${client_id}" "${client_secret}" > "${POLARIS_SERVICE_CREDENTIAL_FILE}" + chmod 600 "${POLARIS_SERVICE_CREDENTIAL_FILE}" + echo " -> principal credentials written to ${POLARIS_SERVICE_CREDENTIAL_FILE}" +fi + +echo "Ensuring default Polaris catalog '${POLARIS_CATALOG_NAME}'..." +status=$(api POST "/catalogs" "{ + \"catalog\": { + \"name\": \"${POLARIS_CATALOG_NAME}\", + \"type\": \"INTERNAL\", + \"properties\": { + \"default-base-location\": \"${CATALOG_BASE_LOCATION}\" + }, + \"storageConfigInfo\": { + \"storageType\": \"S3\", + \"allowedLocations\": [\"${CATALOG_BASE_LOCATION}\"], + \"endpoint\": \"${S3_ENDPOINT}\", + \"endpointInternal\": \"${S3_ENDPOINT}\", + \"pathStyleAccess\": true, + \"region\": \"us-east-1\", + \"stsUnavailable\": true + } + } +}") +check_status "${status}" "catalog '${POLARIS_CATALOG_NAME}' created" + +echo "Granting catalog admin role on '${POLARIS_CATALOG_NAME}' to '${POLARIS_SERVICE_PRINCIPAL_ROLE}'..." +status=$(api POST "/catalogs/${POLARIS_CATALOG_NAME}/catalog-roles" "{ + \"catalogRole\": { + \"name\": \"cdm_admin\" + } +}") +check_status "${status}" "catalog role created" + +status=$(api PUT "/catalogs/${POLARIS_CATALOG_NAME}/catalog-roles/cdm_admin/grants" "{ + \"type\": \"catalog\", + \"privilege\": \"CATALOG_MANAGE_CONTENT\" +}") +check_status "${status}" "CATALOG_MANAGE_CONTENT granted" true + +status=$(api PUT "/principal-roles/${POLARIS_SERVICE_PRINCIPAL_ROLE}/catalog-roles/${POLARIS_CATALOG_NAME}" "{ + \"catalogRole\": { + \"name\": \"cdm_admin\" + } +}") +check_status "${status}" "catalog role assigned to ${POLARIS_SERVICE_PRINCIPAL_ROLE}" true + +echo "Polaris lakehouse setup complete." diff --git a/docker/lakehouse/sql-read-write-policy.json b/docker/lakehouse/sql-read-write-policy.json new file mode 100644 index 00000000..670f63e3 --- /dev/null +++ b/docker/lakehouse/sql-read-write-policy.json @@ -0,0 +1,21 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:ListBucket", + "s3:PutObject", + "s3:DeleteObject", + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:s3:::sql", + "arn:aws:s3:::sql/*", + "arn:aws:s3:::cdm-spark-job-logs", + "arn:aws:s3:::cdm-spark-job-logs/*" + ] + } + ] +} diff --git a/docs/development_and_testing.md b/docs/development_and_testing.md index b4067906..5038aeb6 100644 --- a/docs/development_and_testing.md +++ b/docs/development_and_testing.md @@ -5,7 +5,7 @@ This guide covers local development setup, linting and code generation checks, S - **Project dependencies, lint rules, and `pytest` markers**: [`pyproject.toml`](https://github.com/kbase/cdm-data-loaders/blob/develop/pyproject.toml) - **Container entrypoint (`test` and `ceph_integration_test` modes)**: [`scripts/entrypoint.sh`](https://github.com/kbase/cdm-data-loaders/blob/develop/scripts/entrypoint.sh) - **Spark container test runner**: [`scripts/dev/run_tests.sh`](https://github.com/kbase/cdm-data-loaders/blob/develop/scripts/dev/run_tests.sh) -- **Ceph RGW integration Compose configuration**: [`docker-compose.yml`](https://github.com/kbase/cdm-data-loaders/blob/develop/docker-compose.yml) +- **Lakehouse integration Compose configuration**: [`docker-compose.lakehouse.yml`](https://github.com/kbase/cdm-data-loaders/blob/develop/docker-compose.lakehouse.yml) --- @@ -134,17 +134,15 @@ Tests marked with `@pytest.mark.requires_ceph` exercise real S3 transfers, bucke ### Docker Compose Workflow (Recommended) -[`docker-compose.yml`](https://github.com/kbase/cdm-data-loaders/blob/develop/docker-compose.yml) starts the Ceph RGW container, waits for its OSD and HTTP listener healthcheck, and runs the `ceph_integration_test` entrypoint mode (`pytest -m "requires_ceph" -v`): +[`docker-compose.lakehouse.yml`](https://github.com/kbase/cdm-data-loaders/blob/develop/docker-compose.lakehouse.yml) starts the lakehouse stack (Ceph RGW, Polaris, Auth2, Spark Master/Worker/Connect, and the `cdm-data-loaders` container), waits for healthchecks, and lets you run `pytest -m "requires_ceph" -v` inside `cdm-data-loaders`: ```sh -# build the image, start Ceph RGW, and run all requires_ceph integration tests -docker compose up \ - --build \ - --abort-on-container-exit \ - --exit-code-from integration-tests +# build and start the lakehouse stack, then run all requires_ceph integration tests +docker compose -f docker-compose.lakehouse.yml up -d --build +docker compose -f docker-compose.lakehouse.yml exec -T cdm-data-loaders pytest -m "requires_ceph" -v # stop containers and remove temporary volumes when finished -docker compose down --volumes +docker compose -f docker-compose.lakehouse.yml down --volumes ``` ### Running Against a Local Ceph Container from the Host diff --git a/docs/ncbi_ftp_e2e_walkthrough.md b/docs/ncbi_ftp_e2e_walkthrough.md index 0b5e5a7b..5d524810 100644 --- a/docs/ncbi_ftp_e2e_walkthrough.md +++ b/docs/ncbi_ftp_e2e_walkthrough.md @@ -26,13 +26,13 @@ Step-by-step instructions for running a full production transfer in the Lakehous ### Path Formats Used -| Format | Example | Description | -| --- | --- | --- | -| **s3:// URI** | `s3://cdm-lake/staging/run1/` | Full URI with scheme, bucket, and key | -| **bucket name** | `cdm-lake` | Bucket name without scheme | +| Format | Example | Description | +| ----------------- | ----------------------------------------------- | ------------------------------------------- | +| **s3:// URI** | `s3://cdm-lake/staging/run1/` | Full URI with scheme, bucket, and key | +| **bucket name** | `cdm-lake` | Bucket name without scheme | | **S3 key prefix** | `tenant-general-warehouse/kbase/datasets/ncbi/` | Path within a bucket (no scheme, no bucket) | -| **S3 object key** | `staging/transfer_manifest.txt` | Single object key within a bucket | -| **local path** | `output/removed_manifest.txt` | Filesystem path on the host | +| **S3 object key** | `staging/transfer_manifest.txt` | Single object key within a bucket | +| **local path** | `output/removed_manifest.txt` | Filesystem path on the host | ### Lakehouse Object (Final Location) @@ -89,8 +89,6 @@ export AWS_SECRET_ACCESS_KEY=test_access_secret export AWS_DEFAULT_REGION=us-east-1 ``` -(A matching service is configured in [`docker-compose.yml`](https://github.com/kbase/cdm-data-loaders/blob/develop/docker-compose.yml) for CI integration workflows.) - Create test buckets from the command line using the AWS CLI: ```sh @@ -121,17 +119,17 @@ Open [`notebooks/ncbi_ftp_manifest.ipynb`](https://github.com/kbase/cdm-data-loa ### Constants to Change (Cell 3) -| Constant | Walkthrough Value | Format | Purpose | -| --- | --- | --- | --- | -| `DATABASE` | `"refseq"` | string | Keep as-is | -| `PREFIX_FROM` | `"900"` | string | High-numbered prefix with few assemblies for fast diffing | -| `PREFIX_TO` | `"900"` | string | Single prefix bucket | -| `LIMIT` | `10` | int | Cap to 10 assemblies | -| `PREVIOUS_SUMMARY_URI` | `None` | s3:// URI | First run — treat every assembly as new | -| `SNAPSHOT_UPLOAD_URI` | `None` | s3:// URI | Skip S3 upload for local testing | -| `LAKEHOUSE_BUCKET` | `"cdm-lake"` (or `None`) | bucket name | Prune assemblies already present in the Lakehouse | -| `STORE_KEY_PREFIX` | `"tenant-general-warehouse/kbase/datasets/ncbi/"` | S3 key prefix | Default Lakehouse path prefix | -| `OUTPUT_DIR` | `Path("output")` | local path | Local manifest output directory | +| Constant | Walkthrough Value | Format | Purpose | +| ---------------------- | ------------------------------------------------- | ------------- | --------------------------------------------------------- | +| `DATABASE` | `"refseq"` | string | Keep as-is | +| `PREFIX_FROM` | `"900"` | string | High-numbered prefix with few assemblies for fast diffing | +| `PREFIX_TO` | `"900"` | string | Single prefix bucket | +| `LIMIT` | `10` | int | Cap to 10 assemblies | +| `PREVIOUS_SUMMARY_URI` | `None` | s3:// URI | First run — treat every assembly as new | +| `SNAPSHOT_UPLOAD_URI` | `None` | s3:// URI | Skip S3 upload for local testing | +| `LAKEHOUSE_BUCKET` | `"cdm-lake"` (or `None`) | bucket name | Prune assemblies already present in the Lakehouse | +| `STORE_KEY_PREFIX` | `"tenant-general-warehouse/kbase/datasets/ncbi/"` | S3 key prefix | Default Lakehouse path prefix | +| `OUTPUT_DIR` | `Path("output")` | local path | Local manifest output directory | ### Initialize the S3 Client for CEPH @@ -373,12 +371,12 @@ rm -rf staging/ output/ ## Troubleshooting -| Symptom | Cause | Fix | -| --- | --- | --- | -| `connect_ftp() timeout` | NCBI FTP may be slow or rate-limited | Retry and reduce `--threads` to 1 | -| Phase 3 shows 0 promoted | Staging prefix or bucket mismatch | Verify `--staging-path` matches the S3 upload path from Step 3d | -| Phase 3 S3 auth error | Missing CEPH credentials | Export `AWS_ENDPOINT_URL`, `AWS_ACCESS_KEY_ID`, and `AWS_SECRET_ACCESS_KEY` before running | -| Container cannot reach FTP | Docker network isolation | Pass `--network host` or verify DNS resolution inside the container | +| Symptom | Cause | Fix | +| -------------------------- | ------------------------------------ | ------------------------------------------------------------------------------------------ | +| `connect_ftp() timeout` | NCBI FTP may be slow or rate-limited | Retry and reduce `--threads` to 1 | +| Phase 3 shows 0 promoted | Staging prefix or bucket mismatch | Verify `--staging-path` matches the S3 upload path from Step 3d | +| Phase 3 S3 auth error | Missing CEPH credentials | Export `AWS_ENDPOINT_URL`, `AWS_ACCESS_KEY_ID`, and `AWS_SECRET_ACCESS_KEY` before running | +| Container cannot reach FTP | Docker network isolation | Pass `--network host` or verify DNS resolution inside the container | --- @@ -386,15 +384,15 @@ rm -rf staging/ output/ Phase 2 downloads files matching `FILE_FILTERS` in [`src/cdm_data_loaders/datasets/ncbi_ftp/assembly.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/datasets/ncbi_ftp/assembly.py) (plus `md5checksums.txt` for integrity verification): -| Suffix | Content | -| --- | --- | -| `_genomic.fna.gz` | Genome nucleotide sequences | -| `_genomic.gff.gz` | Genome annotations (GFF3) | -| `_protein.faa.gz` | Protein sequences | -| `_gene_ontology.gaf.gz` | GO annotations | -| `_assembly_report.txt` | Assembly metadata | -| `_assembly_stats.txt` | Assembly statistics | -| `_assembly_regions.txt` | Assembly regions | -| `_ani_contam_ranges.tsv` | ANI contamination ranges | -| `_gene_expression_counts.txt.gz` | Gene expression counts | +| Suffix | Content | +| ------------------------------------------- | ---------------------------- | +| `_genomic.fna.gz` | Genome nucleotide sequences | +| `_genomic.gff.gz` | Genome annotations (GFF3) | +| `_protein.faa.gz` | Protein sequences | +| `_gene_ontology.gaf.gz` | GO annotations | +| `_assembly_report.txt` | Assembly metadata | +| `_assembly_stats.txt` | Assembly statistics | +| `_assembly_regions.txt` | Assembly regions | +| `_ani_contam_ranges.tsv` | ANI contamination ranges | +| `_gene_expression_counts.txt.gz` | Gene expression counts | | `_normalized_gene_expression_counts.txt.gz` | Normalized expression counts | diff --git a/pyproject.toml b/pyproject.toml index cb029d92..5c4afd52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -266,9 +266,9 @@ AWS_DEFAULT_REGION = "us-east-1" # default to use containerized S3 test store AWS_ENDPOINT_URL = { value = "http://localhost:9000", skip_if_set = true } AWS_ENDPOINT_URL_S3 = { unset = true } -S3_ENDPOINT_URL = "http://localhost:9000" -AWS_ACCESS_KEY_ID = "test_access_key" -AWS_SECRET_ACCESS_KEY = "test_access_secret" +S3_ENDPOINT_URL = { value = "http://localhost:9000", skip_if_set = true } +AWS_ACCESS_KEY_ID = { value = "test_access_key", skip_if_set = true } +AWS_SECRET_ACCESS_KEY = { value = "test_access_secret", skip_if_set = true } # required for KBase Lakehouse env USER = "fake_user" KBASE_AUTH_TOKEN = "test-token-123" diff --git a/src/cdm_data_loaders/core/transforms/introspection.py b/src/cdm_data_loaders/core/transforms/introspection.py index f07834c7..8d38a41b 100644 --- a/src/cdm_data_loaders/core/transforms/introspection.py +++ b/src/cdm_data_loaders/core/transforms/introspection.py @@ -36,15 +36,17 @@ def _safe_type_hints(fn: Callable[..., Any]) -> dict[str, Any]: :return: mapping of parameter names to resolved annotations :rtype: dict[str, Any] """ - target = _unwrap_partial_target(fn) try: + target = inspect.unwrap(_unwrap_partial_target(fn)) + if ( + not inspect.isfunction(target) + and not inspect.ismethod(target) + and not inspect.isclass(target) + and callable(target) + ): + target = type(target).__call__ return get_type_hints(target) except (TypeError, NameError, AttributeError, ValueError): - if not inspect.isfunction(target) and not inspect.ismethod(target) and callable(target): - try: - return get_type_hints(type(target).__call__) - except (TypeError, NameError, AttributeError, ValueError): - return {} return {} diff --git a/tests/cdm_data_loaders/core/test_transforms.py b/tests/cdm_data_loaders/core/test_transforms.py index 77828a1d..58042277 100644 --- a/tests/cdm_data_loaders/core/test_transforms.py +++ b/tests/cdm_data_loaders/core/test_transforms.py @@ -1401,6 +1401,7 @@ def base_fn(table: pa.Table, *, _flag: bool = False) -> dict[str, Any]: [ pytest.param("nested_partial", id="nested-partial"), pytest.param("callable_object", id="callable-object"), + pytest.param("callable_object_with_attrs", id="callable-object-with-attrs"), pytest.param("unresolvable_annotation", id="unresolvable-annotation"), ], ) @@ -1422,6 +1423,16 @@ def __call__(self, table: pa.Table, ctx: TransformContext | None = None) -> dict return {} assert _safe_type_hints(_CallableObj())["table"] is pa.Table + elif case == "callable_object_with_attrs": + + class _CallableWithAttrs: + label: str = "x" + + def __call__(self, table: pa.Table, ctx: TransformContext | None = None) -> dict[str, Any]: + del table, ctx + return {} + + assert _safe_type_hints(_CallableWithAttrs())["table"] is pa.Table else: def bad_hint_fn(_x: object) -> dict[str, Any]: