Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 7 additions & 15 deletions .github/workflows/integration_tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,29 +32,21 @@ jobs:
env:
PYTEST_ADDOPTS: "--durations=0 --durations-min=5"
run: |
docker compose up \
--build \
--abort-on-container-exit \
--exit-code-from integration-tests
docker compose -f docker-compose.lakehouse.yml up -d --build
docker compose -f docker-compose.lakehouse.yml exec -T cdm-data-loaders pytest -m "requires_ceph" -v

- name: Capture compose diagnostics on failure
if: failure()
run: |
echo "=== docker compose ps ==="
docker compose ps -a || true
docker compose -f docker-compose.lakehouse.yml ps -a || true

echo "=== ceph logs ==="
docker logs cdm-data-loaders-ceph-1 || true
docker compose -f docker-compose.lakehouse.yml logs ceph || true

echo "=== ceph integration test logs ==="
docker logs cdm-data-loaders-integration-tests-1 || true

echo "=== ceph inspect state ==="
docker inspect cdm-data-loaders-ceph-1 --format '{{json .State}}' || true

echo "=== ceph health state ==="
docker inspect cdm-data-loaders-ceph-1 --format '{{json .State.Health}}' || true
echo "=== cdm-data-loaders logs ==="
docker compose -f docker-compose.lakehouse.yml logs cdm-data-loaders || true

- name: Tear down
if: always()
run: docker compose down --volumes
run: docker compose -f docker-compose.lakehouse.yml down --volumes
46 changes: 46 additions & 0 deletions Dockerfile.lakehouse
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
ARG SPARK_IMAGE_TAG=main
FROM ghcr.io/kbasedatalakehouse/kube_spark_manager_image:${SPARK_IMAGE_TAG}

ARG QSV_VERSION="28.0.0"
ARG XML_FILE_SPLITTER_VERSION="v0.1.4"

USER root

# Install qsv and xml_file_splitter binaries required by cdm-data-loaders readers
RUN ARCH=$(uname -m) && \
curl -fsSL -o /tmp/xml_file_splitter.tar.gz "https://github.com/ialarmedalien/xml_file_splitter/releases/download/${XML_FILE_SPLITTER_VERSION}/xml_file_splitter-${ARCH}-unknown-linux-gnu.tar.gz" && \
tar -xf /tmp/xml_file_splitter.tar.gz -C /tmp && \
mv /tmp/xml_file_splitter-${ARCH}-unknown-linux-gnu/xml_file_splitter /usr/local/bin/ && \
curl -fsSL -o /tmp/qsv.zip "https://github.com/dathere/qsv/releases/download/${QSV_VERSION}/qsv-${QSV_VERSION}-${ARCH}-unknown-linux-gnu.zip" && \
unzip -q /tmp/qsv.zip -d /tmp/qsv && \
mv /tmp/qsv/qsv /usr/local/bin/ && \
chmod +x /usr/local/bin/xml_file_splitter /usr/local/bin/qsv && \
rm -rf /tmp/*

# Install uv for fast, locked dependency resolution
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/

WORKDIR /app

# Install cdm-data-loaders dependencies into the base image's conda environment
# (--inexact preserves pre-installed packages like berdl_notebook_utils and pyspark).
ENV UV_PROJECT_ENVIRONMENT=/opt/conda
ENV UV_COMPILE_BYTECODE=1
ENV UV_LINK_MODE=copy

COPY pyproject.toml uv.lock README.md ./
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --locked --inexact --no-install-project --all-groups

COPY src ./src
RUN --mount=type=cache,target=/root/.cache/uv \
uv pip install --python /opt/conda/bin/python --no-deps -e .

ENV PATH="/opt/conda/bin:$PATH"
ENV PYTHONUNBUFFERED=1

# Override kube_spark_manager_image's Spark healthcheck and entrypoint so this
# container idles for interactive testing (pytest, CLI, Spark Connect client).
HEALTHCHECK NONE
ENTRYPOINT ["tini", "--"]
CMD ["sleep", "infinity"]
286 changes: 286 additions & 0 deletions docker-compose.lakehouse.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,286 @@
# Lakehouse test stack for cdm-data-loaders:
# - Ceph RGW (S3 + IAM/STS)
# - Apache Polaris (stock, relational-jdbc on Postgres)
# - KBase Auth2 (test mode, on Mongo) for Spark Connect auth interceptor
# - Spark Master + Worker + Spark Connect (kube_spark_manager_image)
# - cdm-data-loaders interactive dev/test container (kube_spark_manager_image + repo)

networks:
default:

volumes:
polaris-credential:
postgres-data:
mongo-data:
ceph-data:

services:
ceph:
image: ghcr.io/kbasetest/ceph-rgw-test-image:0.2.1
networks:
default:
aliases: ["s3"]
environment:
RGW_PORT: "9090"
RGW_ACCESS_KEY: s3user
RGW_SECRET_KEY: s3password
RGW_USERS: "sql:sql:sqlpassword"
RGW_STS_KEY: cdmteststs123456
ports:
- "9090:9090"
- "9091:8443"
volumes:
- ceph-data:/var/lib/ceph
healthcheck:
test: ["CMD-SHELL", "ceph osd stat 2>/dev/null | grep -qE '[1-9][0-9]* up' && curl -s http://127.0.0.1:9090/ >/dev/null 2>&1"]
interval: 10s
timeout: 10s
retries: 10
start_period: 30s

s3-create-bucket-ceph:
image: amazon/aws-cli:2.31.14
depends_on:
ceph:
condition: service_healthy
environment:
S3_URL: http://s3:9090
S3_USER: s3user
S3_PWD: s3password
S3_SQL_BUCKET: sql
S3_TEST_BUCKET: test-bucket
S3_SQL_RW_USER: sql
POLARIS_VENDING_STS_ROLE_NAME: cdm-polaris-vending
entrypoint: ["/bin/bash", "/s3_policies/ceph_create_bucket_entrypoint.sh"]
volumes:
- ./docker/lakehouse:/s3_policies:ro

postgres:
image: postgres:16.3
ports:
- "5432:5432"
environment:
POSTGRES_USER: polaris
POSTGRES_PASSWORD: polarispassword
POSTGRES_DB: polaris
healthcheck:
test: ["CMD-SHELL", "pg_isready -U polaris -d polaris"]
interval: 2s
timeout: 5s
retries: 20
volumes:
- postgres-data:/var/lib/postgresql/data

polaris-bootstrap:
image: apache/polaris-admin-tool:1.4.1
depends_on:
postgres:
condition: service_healthy
environment:
POLARIS_PERSISTENCE_TYPE: relational-jdbc
QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/polaris
QUARKUS_DATASOURCE_USERNAME: polaris
QUARKUS_DATASOURCE_PASSWORD: polarispassword
POLARIS_REALM: POLARIS
POLARIS_BOOTSTRAP_CREDENTIAL: POLARIS,root,s3cr3t
entrypoint: ["/bin/sh", "/polaris/polaris_bootstrap.sh"]
volumes:
- ./docker/lakehouse:/polaris:ro

polaris:
image: apache/polaris:1.4.1
ports:
- "8181:8181"
- "8182:8182"
depends_on:
polaris-bootstrap:
condition: service_completed_successfully
environment:
POLARIS_PERSISTENCE_TYPE: relational-jdbc
QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/polaris
QUARKUS_DATASOURCE_USERNAME: polaris
QUARKUS_DATASOURCE_PASSWORD: polarispassword
POLARIS_REALM_CONTEXT_REALMS: POLARIS
AWS_REGION: us-east-1
AWS_ACCESS_KEY_ID: sql
AWS_SECRET_ACCESS_KEY: sqlpassword
AWS_ENDPOINT_URL_S3: http://s3:9090
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8182/q/health"]
interval: 2s
timeout: 10s
retries: 60

polaris-setup:
image: curlimages/curl:8.11.1
user: root
depends_on:
polaris:
condition: service_healthy
s3-create-bucket-ceph:
condition: service_completed_successfully
environment:
POLARIS_HOST: http://polaris:8181
POLARIS_REALM: POLARIS
POLARIS_CLIENT_ID: root
POLARIS_CLIENT_SECRET: s3cr3t
POLARIS_SERVICE_PRINCIPAL: cdm_loader
POLARIS_SERVICE_PRINCIPAL_ROLE: service_admin
POLARIS_SERVICE_CREDENTIAL_FILE: /shared-polaris/polaris_credential.txt
POLARIS_CATALOG_NAME: cdm
S3_ENDPOINT: http://s3:9090
CATALOG_BASE_LOCATION: s3://sql/warehouse
entrypoint: ["/bin/sh", "/polaris/polaris_setup.sh"]
volumes:
- ./docker/lakehouse:/polaris:ro
- polaris-credential:/shared-polaris

mongo:
image: mongo:7.0
ports:
- "27019:27019"
command: ["mongod", "--port", "27019"]
environment:
MONGO_INITDB_ROOT_USERNAME: cdm
MONGO_INITDB_ROOT_PASSWORD: cdmpassword
healthcheck:
test: ["CMD", "mongosh", "--quiet", "--port", "27019", "--eval", "db.adminCommand('ping')"]
interval: 2s
timeout: 5s
retries: 20
volumes:
- mongo-data:/data/db

mongo-init-auth2-user:
image: mongo:7.0
depends_on:
mongo:
condition: service_healthy
environment:
MONGO_HOST: "mongo:27019"
MONGO_ROOT_USER: cdm
MONGO_ROOT_PWD: cdmpassword
AUTH2_MONGO_DB: auth2
AUTH2_MONGO_USER: auth2
AUTH2_MONGO_PWD: auth2password
entrypoint: ["/bin/sh", "/mongo-init/init_auth2_user.sh"]
volumes:
- ./docker/lakehouse:/mongo-init:ro

auth2:
image: ghcr.io/kbase/auth2:0.8.1
platform: linux/amd64
depends_on:
mongo-init-auth2-user:
condition: service_completed_successfully
ports:
- "8180:8080"
environment:
mongo_host: "mongo:27019"
mongo_db: "auth2"
mongo_user: "auth2"
mongo_pwd: "auth2password"
test_mode_enabled: "true"
identity_providers: ""
command:
- "-template"
- "/kb/deployment/conf/.templates/deployment.cfg.templ:/kb/deployment/conf/deployment.cfg"
- "/kb/deployment/bin/start_auth2.sh"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8080/"]
interval: 2s
timeout: 5s
retries: 60

spark-master:
image: ghcr.io/kbasedatalakehouse/kube_spark_manager_image:${SPARK_IMAGE_TAG:-main}
platform: linux/amd64
ports:
- "8090:8090"
- "7077:7077"
environment:
SPARK_MODE: master
SPARK_MASTER_HOST: spark-master
SPARK_MASTER_PORT: "7077"
SPARK_MASTER_WEBUI_PORT: "8090"

spark-worker-1:
image: ghcr.io/kbasedatalakehouse/kube_spark_manager_image:${SPARK_IMAGE_TAG:-main}
platform: linux/amd64
depends_on:
spark-master:
condition: service_healthy
ports:
- "8081:8081"
environment:
SPARK_MODE: worker
SPARK_MASTER_URL: spark://spark-master:7077
SPARK_WORKER_CORES: "4"
SPARK_WORKER_MEMORY: 2G
SPARK_WORKER_PORT: "8083"
SPARK_WORKER_WEBUI_PORT: "8081"

spark-connect:
image: ghcr.io/kbasedatalakehouse/kube_spark_manager_image:${SPARK_IMAGE_TAG:-main}
platform: linux/amd64
depends_on:
spark-master:
condition: service_healthy
polaris-setup:
condition: service_completed_successfully
auth2:
condition: service_healthy
ports:
- "15002:15002"
environment:
SPARK_MODE: connect
USER: testuser
BERDL_POD_IP: spark-connect
SPARK_MASTER_URL: spark://spark-master:7077
KBASE_AUTH_URL: http://auth2:8080/testmode/
S3_ENDPOINT_URL: http://s3:9090
S3_ACCESS_KEY: sql
S3_SECRET_KEY: sqlpassword
POLARIS_CATALOG_URI: http://polaris:8181/api/catalog
POLARIS_CREDENTIAL: root:s3cr3t
POLARIS_PERSONAL_CATALOG: cdm
BERDL_SPARK_DYNAMIC_ALLOCATION: "true"
SPARK_MAX_EXECUTORS: "2"
SPARK_WORKER_CORES: "4"
SPARK_WORKER_MEMORY: 2G
SPARK_MASTER_CORES: "4"
SPARK_MASTER_MEMORY: 2G
BERDL_SPARK_MODE: shared
BERDL_CONNECT_SESSION_TIMEOUT: 30m

cdm-data-loaders:
build:
context: .
dockerfile: Dockerfile.lakehouse
args:
SPARK_IMAGE_TAG: ${SPARK_IMAGE_TAG:-main}
platform: linux/amd64
depends_on:
s3-create-bucket-ceph:
condition: service_completed_successfully
polaris-setup:
condition: service_completed_successfully
spark-connect:
condition: service_healthy
environment:
AWS_ENDPOINT_URL: http://s3:9090
AWS_ACCESS_KEY_ID: s3user
AWS_SECRET_ACCESS_KEY: s3password
AWS_DEFAULT_REGION: us-east-1
S3_ENDPOINT_URL: http://s3:9090
POLARIS_HOST: http://polaris:8181
POLARIS_CATALOG_URI: http://polaris:8181/api/catalog
POLARIS_REALM: POLARIS
POLARIS_CATALOG: cdm
POLARIS_CREDENTIAL_FILE: /shared-polaris/polaris_credential.txt
SPARK_CONNECT_URL: sc://spark-connect:15002
KBASE_AUTH_URL: http://auth2:8080/testmode/
PYTEST_ADDOPTS: ${PYTEST_ADDOPTS:-}
volumes:
- .:/app
- polaris-credential:/shared-polaris:ro
Loading
Loading