From 590353d9495399ca0ae710303311327c5f53e6aa Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 25 Aug 2026 12:55:28 -0600 Subject: [PATCH] ci: run Python client tests against a cluster built from the branch The existing "Python Release Build" job cannot catch Rust regressions. Its tests call setup_test_cluster(), which starts the scheduler and executor in-process from the crates.io release that python/Cargo.toml pins, so the code under review is never loaded. It also only triggers on python/** changes. Add a workflow that keeps the client on that pinned release and swaps the cluster for one built from the branch, which is the combination users actually run between releases. Trigger it on ballista/** too. Add BALLISTA_TEST_SCHEDULER to setup_test_cluster() so the existing tests can be pointed at an already-running scheduler, and add a test whose query crosses a shuffle boundary so intermediate shuffle write and read are covered rather than just the final stage fetch. --- .github/workflows/python-integration.yml | 135 +++++++++++++++++++++++ python/python/ballista/__init__.py | 32 +++++- python/python/tests/test_context.py | 27 +++++ 3 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/python-integration.yml diff --git a/.github/workflows/python-integration.yml b/.github/workflows/python-integration.yml new file mode 100644 index 0000000000..03717e56b3 --- /dev/null +++ b/.github/workflows/python-integration.yml @@ -0,0 +1,135 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Runs the Python client against a scheduler and executor built from the +# working tree. +# +# The "Python Release Build" workflow cannot catch Rust regressions: its +# tests call `setup_test_cluster()`, which starts the cluster IN-PROCESS from +# the crates.io release that `python/Cargo.toml` pins, so the code under +# review is never loaded. This workflow keeps the client on that same pinned +# release (it has to: `pyballista` re-exports datafusion-python types, so it +# can only move when datafusion-python does) and instead swaps the *cluster* +# for one built from the branch. That is the combination users actually run +# between releases, and it is the one that broke in #2367. + +name: Python Client Integration + +permissions: + contents: read + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +on: + push: + branches: [ "main" ] + paths: + - "ballista/**" + - "python/**" + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - ".github/workflows/python-integration.yml" + # docs-only changes cannot affect this job; exclusions are applied last + - "!**/*.md" + pull_request: + paths: + - "ballista/**" + - "python/**" + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - ".github/workflows/python-integration.yml" + # docs-only changes cannot affect this job; exclusions are applied last + - "!**/*.md" + workflow_dispatch: + +jobs: + python-against-branch-cluster: + name: Python client vs branch-built cluster + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.1 + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Setup Rust toolchain + run: | + rustup toolchain install stable + rustup default stable + - uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 #2.9.2 + + - name: Install Protoc + uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 + with: + version: "27.4" + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + enable-cache: true + + # The cluster comes from this branch. The client does not. + - name: Build scheduler and executor from the working tree + # Debug build on purpose: the test data is a handful of rows, so + # runtime speed is irrelevant and a release build would roughly + # double an already long job. + run: cargo build --locked -p ballista-scheduler -p ballista-executor + + - name: Build Python client + working-directory: python + run: uv sync --dev --no-install-package ballista + + - name: Start cluster + working-directory: python + run: | + # Started from python/ so the relative paths the tests register + # (testdata/test.parquet and friends) resolve on the executor too. + mkdir -p ../target/ci-logs + RUST_LOG=info ../target/debug/ballista-scheduler \ + > ../target/ci-logs/scheduler.log 2>&1 & + RUST_LOG=info ../target/debug/ballista-executor \ + > ../target/ci-logs/executor.log 2>&1 & + + for _ in $(seq 1 60); do + if grep -q "Executor registration succeed" ../target/ci-logs/executor.log 2>/dev/null; then + echo "cluster is up" + exit 0 + fi + sleep 1 + done + + echo "cluster did not come up within 60s" + echo "--- scheduler ---"; cat ../target/ci-logs/scheduler.log + echo "--- executor ---"; cat ../target/ci-logs/executor.log + exit 1 + + - name: Run Python tests against the cluster + working-directory: python + env: + BALLISTA_TEST_SCHEDULER: "localhost:50050" + run: uv run pytest + + - name: Cluster logs + if: always() + run: | + echo "--- scheduler ---"; cat target/ci-logs/scheduler.log || true + echo "--- executor ---"; cat target/ci-logs/executor.log || true diff --git a/python/python/ballista/__init__.py b/python/python/ballista/__init__.py index 5a5ef7bb02..a947800b90 100644 --- a/python/python/ballista/__init__.py +++ b/python/python/ballista/__init__.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +import os + try: import importlib.metadata as importlib_metadata except ImportError: @@ -23,7 +25,7 @@ from ._internal_ballista import ( BallistaScheduler, BallistaExecutor, - setup_test_cluster, + setup_test_cluster as _setup_in_process_test_cluster, ) from .extension import ( BallistaSessionContext, @@ -33,8 +35,36 @@ __version__ = importlib_metadata.version(__name__) +#: Environment variable naming an already-running scheduler to test against, +#: as ``host:port`` (the port defaults to 50050 if omitted). +TEST_SCHEDULER_ENV_VAR = "BALLISTA_TEST_SCHEDULER" + + +def setup_test_cluster(): + """Return the ``(host, port)`` of a scheduler to run tests against. + + By default this starts an in-process standalone scheduler and executor, + which is convenient but only ever exercises the Rust code the bindings + were compiled against. + + If ``BALLISTA_TEST_SCHEDULER`` is set, no cluster is started and its + value is used instead. That lets CI point the Python client at a + scheduler and executor built from the working tree, so changes to the + Rust crates are actually covered by the Python tests. + """ + external = os.environ.get(TEST_SCHEDULER_ENV_VAR) + if not external: + return _setup_in_process_test_cluster() + + host, sep, port = external.rpartition(":") + if not sep: + return (external, 50050) + return (host, int(port)) + + __all__ = [ "setup_test_cluster", + "TEST_SCHEDULER_ENV_VAR", "BallistaScheduler", "BallistaExecutor", "BallistaSessionContext", diff --git a/python/python/tests/test_context.py b/python/python/tests/test_context.py index 6e51803784..e6ec36e810 100644 --- a/python/python/tests/test_context.py +++ b/python/python/tests/test_context.py @@ -64,6 +64,33 @@ def test_register_parquet(ctx): assert len(batches[0]) == 8 +def test_multi_stage_query(ctx): + """A query with a shuffle boundary, so it runs as more than one stage. + + Single-stage queries still round-trip the final stage's shuffle output + through Arrow Flight, but only a repartitioning query also exercises the + intermediate shuffle write/read path between executors. Regressions in + shuffle file naming or partition bookkeeping show up here. + """ + ctx.register_parquet("test", "testdata/test.parquet") + df = ctx.sql( + "SELECT bool_col, count(*) AS n, sum(int_col) AS total " + "FROM test GROUP BY bool_col ORDER BY bool_col" + ) + batches = df.collect() + + rows = [ + ( + batch.column(0)[i].as_py(), + batch.column(1)[i].as_py(), + batch.column(2)[i].as_py(), + ) + for batch in batches + for i in range(len(batch)) + ] + assert sorted(rows) == [(False, 4, 4), (True, 4, 0)] + + def test_read_dataframe_api(ctx): df = ( ctx.read_csv("testdata/test.csv", has_header=True)