Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 135 additions & 0 deletions .github/workflows/python-integration.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# Runs the Python client against a scheduler and executor built from the
# working tree.
#
# The "Python Release Build" workflow cannot catch Rust regressions: its
# tests call `setup_test_cluster()`, which starts the cluster IN-PROCESS from
# the crates.io release that `python/Cargo.toml` pins, so the code under
# review is never loaded. This workflow keeps the client on that same pinned
# release (it has to: `pyballista` re-exports datafusion-python types, so it
# can only move when datafusion-python does) and instead swaps the *cluster*
# for one built from the branch. That is the combination users actually run
# between releases, and it is the one that broke in #2367.

name: Python Client Integration

permissions:
contents: read

concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true

on:
push:
branches: [ "main" ]
paths:
- "ballista/**"
- "python/**"
- "Cargo.toml"
- "Cargo.lock"
- "rust-toolchain.toml"
- ".github/workflows/python-integration.yml"
# docs-only changes cannot affect this job; exclusions are applied last
- "!**/*.md"
pull_request:
paths:
- "ballista/**"
- "python/**"
- "Cargo.toml"
- "Cargo.lock"
- "rust-toolchain.toml"
- ".github/workflows/python-integration.yml"
# docs-only changes cannot affect this job; exclusions are applied last
- "!**/*.md"
workflow_dispatch:

jobs:
python-against-branch-cluster:
name: Python client vs branch-built cluster
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7.0.1

- uses: actions/setup-python@v7
with:
python-version: "3.12"

- name: Setup Rust toolchain
run: |
rustup toolchain install stable
rustup default stable
- uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 #2.9.2

- name: Install Protoc
uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
with:
version: "27.4"
repo-token: ${{ secrets.GITHUB_TOKEN }}

- uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
enable-cache: true

# The cluster comes from this branch. The client does not.
- name: Build scheduler and executor from the working tree
# Debug build on purpose: the test data is a handful of rows, so
# runtime speed is irrelevant and a release build would roughly
# double an already long job.
run: cargo build --locked -p ballista-scheduler -p ballista-executor

- name: Build Python client
working-directory: python
run: uv sync --dev --no-install-package ballista

- name: Start cluster
working-directory: python
run: |
# Started from python/ so the relative paths the tests register
# (testdata/test.parquet and friends) resolve on the executor too.
mkdir -p ../target/ci-logs
RUST_LOG=info ../target/debug/ballista-scheduler \
> ../target/ci-logs/scheduler.log 2>&1 &
RUST_LOG=info ../target/debug/ballista-executor \
> ../target/ci-logs/executor.log 2>&1 &

for _ in $(seq 1 60); do
if grep -q "Executor registration succeed" ../target/ci-logs/executor.log 2>/dev/null; then
echo "cluster is up"
exit 0
fi
sleep 1
done

echo "cluster did not come up within 60s"
echo "--- scheduler ---"; cat ../target/ci-logs/scheduler.log
echo "--- executor ---"; cat ../target/ci-logs/executor.log
exit 1

- name: Run Python tests against the cluster
working-directory: python
env:
BALLISTA_TEST_SCHEDULER: "localhost:50050"
run: uv run pytest

- name: Cluster logs
if: always()
run: |
echo "--- scheduler ---"; cat target/ci-logs/scheduler.log || true
echo "--- executor ---"; cat target/ci-logs/executor.log || true
32 changes: 31 additions & 1 deletion python/python/ballista/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.

import os

try:
import importlib.metadata as importlib_metadata
except ImportError:
Expand All @@ -23,7 +25,7 @@
from ._internal_ballista import (
BallistaScheduler,
BallistaExecutor,
setup_test_cluster,
setup_test_cluster as _setup_in_process_test_cluster,
)
from .extension import (
BallistaSessionContext,
Expand All @@ -33,8 +35,36 @@

__version__ = importlib_metadata.version(__name__)

#: Environment variable naming an already-running scheduler to test against,
#: as ``host:port`` (the port defaults to 50050 if omitted).
TEST_SCHEDULER_ENV_VAR = "BALLISTA_TEST_SCHEDULER"


def setup_test_cluster():
"""Return the ``(host, port)`` of a scheduler to run tests against.

By default this starts an in-process standalone scheduler and executor,
which is convenient but only ever exercises the Rust code the bindings
were compiled against.

If ``BALLISTA_TEST_SCHEDULER`` is set, no cluster is started and its
value is used instead. That lets CI point the Python client at a
scheduler and executor built from the working tree, so changes to the
Rust crates are actually covered by the Python tests.
"""
external = os.environ.get(TEST_SCHEDULER_ENV_VAR)
if not external:
return _setup_in_process_test_cluster()

host, sep, port = external.rpartition(":")
if not sep:
return (external, 50050)
return (host, int(port))


__all__ = [
"setup_test_cluster",
"TEST_SCHEDULER_ENV_VAR",
"BallistaScheduler",
"BallistaExecutor",
"BallistaSessionContext",
Expand Down
27 changes: 27 additions & 0 deletions python/python/tests/test_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,33 @@ def test_register_parquet(ctx):
assert len(batches[0]) == 8


def test_multi_stage_query(ctx):
"""A query with a shuffle boundary, so it runs as more than one stage.

Single-stage queries still round-trip the final stage's shuffle output
through Arrow Flight, but only a repartitioning query also exercises the
intermediate shuffle write/read path between executors. Regressions in
shuffle file naming or partition bookkeeping show up here.
"""
ctx.register_parquet("test", "testdata/test.parquet")
df = ctx.sql(
"SELECT bool_col, count(*) AS n, sum(int_col) AS total "
"FROM test GROUP BY bool_col ORDER BY bool_col"
)
batches = df.collect()

rows = [
(
batch.column(0)[i].as_py(),
batch.column(1)[i].as_py(),
batch.column(2)[i].as_py(),
)
for batch in batches
for i in range(len(batch))
]
assert sorted(rows) == [(False, 4, 4), (True, 4, 0)]


def test_read_dataframe_api(ctx):
df = (
ctx.read_csv("testdata/test.csv", has_header=True)
Expand Down
Loading