Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .env
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,6 @@ VCPKG="61d7eaeb094a6acbc23d9c2e54de9208175fda47" # 2026.10.24 (not release, u
# ci/docker/python-*-windows-*.dockerfile or the vcpkg config.
# This is a workaround for our CI problem that "archery docker build" doesn't
# use pulled built images in dev/tasks/python-wheels/github.windows.yml.
PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2026-10-28
PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2026-11-13
PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2026-11-28

6 changes: 6 additions & 0 deletions .github/workflows/dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,12 @@ jobs:
- name: Run pre-commit
run: |
pre-commit run --all-files --color=always --show-diff-on-failure
- name: Run pre-commit for type stubs
env:
# The previous step already ran the ruff hooks.
SKIP: ruff-check,ruff-format
run: |
pre-commit run --hook-stage manual python-stubs --all-files --color=always --show-diff-on-failure
release:
name: Source Release and Merge Script on ${{ matrix.runs-on }}
Expand Down
64 changes: 64 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,70 @@ repos:
(
?^python/pyarrow/vendored/|
)
# Linting, formatting and type checking of the type stubs. The type checkers
# don't run on commit, only on request (the Lint CI job runs them too):
# pre-commit run --hook-stage manual python-stubs --all-files
# The type checkers run with uvx, in the environment of
# python/requirements-stubs.txt. They don't need PyArrow installed: the
# modules that aren't annotated yet, e.g. pyarrow.compute, are treated as Any
# (see python/pyproject.toml).
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.17.2
hooks:
- id: ruff-check
alias: python-stubs
name: Python Stubs Lint (ruff)
args:
- "--fix"
files: >-
^python/pyarrow-stubs/
types:
- pyi
- id: ruff-format
alias: python-stubs
name: Python Stubs Format (ruff)
files: >-
^python/pyarrow-stubs/
types:
- pyi
- repo: local
hooks:
- id: mypy
alias: python-stubs
name: Python Stubs Type Check (mypy)
stages:
- manual
language: python
entry: bash -c "cd python && exec uvx --with-requirements requirements-stubs.txt mypy"
pass_filenames: false
files: >-
^python/(pyarrow-stubs(-tests)?/|(pyproject\.toml|requirements-stubs\.txt)$)
additional_dependencies: &uv
- uv==0.13.7
- id: pyright
alias: python-stubs
name: Python Stubs Type Check (pyright)
stages:
- manual
language: python
entry: uvx --with-requirements python/requirements-stubs.txt pyright -p python/pyproject.toml
pass_filenames: false
files: >-
^python/(pyarrow-stubs(-tests)?/|(pyproject\.toml|requirements-stubs\.txt)$)
additional_dependencies: *uv
- id: ty
alias: python-stubs
name: Python Stubs Type Check (ty)
stages:
- manual
language: python
# Unset the VIRTUAL_ENV of the hook, so that ty uses the environment
# of uvx.
entry: env -u VIRTUAL_ENV -u CONDA_PREFIX uvx --with-requirements python/requirements-stubs.txt ty check --project python
pass_filenames: false
files: >-
^python/(pyarrow-stubs(-tests)?/|(pyproject\.toml|requirements-stubs\.txt)$)
additional_dependencies: *uv
- repo: https://github.com/MarcoGorelli/cython-lint
rev: v0.21.1
hooks:
Expand Down
1 change: 0 additions & 1 deletion ci/conda_env_python.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ cython>=3.1
cloudpickle
fsspec
hypothesis
libcst>=1.8.6
numpy>=2.0
pytest
pytest-faulthandler
Expand Down
16 changes: 11 additions & 5 deletions ci/scripts/python_test_type_annotations.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,20 @@ if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
. "${ARROW_PYTHON_VENV}/bin/activate"
fi

# Install library stubs. Note some libraries contain their own type hints so they need to be installed.
pip install fsspec pandas-stubs scipy-stubs types-cffi types-psutil types-requests types-python-dateutil
cd "${pyarrow_dir}"

# Install type checkers
pip install mypy pyright ty
# Install type checkers and library stubs. Note some libraries contain their
# own type hints so they need to be installed.
pip install -r requirements-stubs.txt

# Run type checkers
cd "${pyarrow_dir}"
mypy
pyright --stats
ty check --verbose --output-format concise

# The type annotation tests are type checked above; also run them to verify
# the expected annotated types against the runtime types.
pytest -r s pyarrow-stubs-tests

# Check that the stubs match the signatures of the installed PyArrow.
python scripts/run_stubtest.py
15 changes: 6 additions & 9 deletions ci/scripts/python_wheel_macos_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,9 @@ else
exit 1
fi

wheel_build_binary_flag="--only-binary=:all:"
if python -c "import sys; sys.exit(0 if sys.version_info[:2] == (3, 15) else 1)"; then
# TODO: 3.15 workaround due to pyyaml (via libcst). Remove once pyyaml ships 3.15 wheels.
wheel_build_binary_flag=""
fi

pip install \
--force-reinstall \
${wheel_build_binary_flag} \
--only-binary=:all: \
--upgrade \
-r "${source_dir}/python/requirements-wheel-build.txt"
pip install "delocate>=0.10.3"
Expand Down Expand Up @@ -154,8 +148,7 @@ popd

echo "=== (${PYTHON_VERSION}) Building wheel ==="
export PYARROW_BUNDLE_ARROW_CPP=ON
# TODO(GH-32609): Re-enable when pyarrow-stubs are shipped in wheels again.
# export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
export PYARROW_WITH_ACERO="${ARROW_ACERO}"
export PYARROW_WITH_AZURE="${ARROW_AZURE}"
export PYARROW_WITH_DATASET="${ARROW_DATASET}"
Expand Down Expand Up @@ -193,3 +186,7 @@ fi
# Move the verified wheels
mkdir -p "${source_dir}/python/repaired_wheels"
mv "${source_dir}"/python/dist/*.whl "${source_dir}"/python/repaired_wheels/

echo "=== (${PYTHON_VERSION}) Check the type stubs against the wheel ==="
python "${source_dir}/python/scripts/run_stubtest.py" \
--wheel "${source_dir}"/python/repaired_wheels/pyarrow-*.whl
14 changes: 6 additions & 8 deletions ci/scripts/python_wheel_validate_contents.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ def _count_docstrings(source):
return count


# TODO(GH-48970): Check stubs ARE present once annotations are complete
def validate_wheel(path):
p = Path(path)
wheels = list(p.glob('*.whl'))
Expand All @@ -55,12 +54,14 @@ def validate_wheel(path):
info.filename.split("/")[-1] == filename for info in wheel_zip.filelist
), f"{filename} is missing from the wheel."

# TODO(GH-48970): Check that py.typed IS present once the stubs are
# complete.
assert not any(
info.filename == "pyarrow/py.typed" for info in wheel_zip.filelist
), "pyarrow/py.typed is present in the wheel."

source_root = Path(__file__).resolve().parents[2]
stubs_dir = source_root / "python" / "pyarrow-stubs" / "pyarrow"
stubs_dir = source_root / "python" / "pyarrow-stubs"
assert stubs_dir.exists(), f"Stub source directory not found: {stubs_dir}"

expected_stub_files = {
Expand All @@ -74,22 +75,19 @@ def validate_wheel(path):
if info.filename.startswith("pyarrow/") and info.filename.endswith(".pyi")
}

assert not (wheel_stub_files == expected_stub_files), (
"Wheel .pyi files do not differ from python/pyarrow-stubs/pyarrow.\n"
assert wheel_stub_files == expected_stub_files, (
"Wheel .pyi files differ from python/pyarrow-stubs.\n"
f"Missing in wheel: {sorted(expected_stub_files - wheel_stub_files)}\n"
f"Unexpected in wheel: {sorted(wheel_stub_files - expected_stub_files)}"
)
assert not wheel_stub_files, (
f"Wheel contains unexpected .pyi files: {sorted(wheel_stub_files)}"
)

wheel_docstring_count = sum(
_count_docstrings(wheel_zip.read(wsf).decode("utf-8"))
for wsf in wheel_stub_files
)

print(f"Found {wheel_docstring_count} docstring(s) in wheel stubs.")
assert wheel_docstring_count == 0, "Docstrings found in wheel stub files."
assert wheel_docstring_count, "No docstrings found in wheel stub files."

print(f"The wheel: {wheels[0]} seems valid.")

Expand Down
8 changes: 6 additions & 2 deletions ci/scripts/python_wheel_windows_build.bat
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,7 @@ popd

echo "=== (%PYTHON%) Building wheel ==="
set PYARROW_BUNDLE_ARROW_CPP=ON
rem TODO(GH-32609): Re-enable when pyarrow-stubs are shipped in wheels again.
rem set PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
set PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
set PYARROW_WITH_ACERO=%ARROW_ACERO%
set PYARROW_WITH_AZURE=%ARROW_AZURE%
set PYARROW_WITH_DATASET=%ARROW_DATASET%
Expand Down Expand Up @@ -155,4 +154,9 @@ echo "Wheel name: %WHEEL_NAME%"
--ignore-existing --with-mangle ^
-w repaired_wheels %WHEEL_NAME% || exit /B 1

@REM Check the type stubs against the wheel
for %%w in (repaired_wheels\pyarrow-*.whl) do (
%PYTHON_CMD% scripts\run_stubtest.py --wheel %%w || exit /B 1
)

popd
6 changes: 4 additions & 2 deletions ci/scripts/python_wheel_xlinux_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,7 @@ check_arrow_visibility

echo "=== (${PYTHON_VERSION}) Building wheel ==="
export PYARROW_BUNDLE_ARROW_CPP=ON
# TODO(GH-32609): Re-enable when pyarrow-stubs are shipped in wheels again.
# export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
export PYARROW_REQUIRE_STUB_DOCSTRINGS=ON
export PYARROW_WITH_ACERO="${ARROW_ACERO}"
export PYARROW_WITH_AZURE="${ARROW_AZURE}"
export PYARROW_WITH_DATASET="${ARROW_DATASET}"
Expand Down Expand Up @@ -224,4 +223,7 @@ rm -rf dist/temp-fix-wheel

echo "=== (${PYTHON_VERSION}) Tag the wheel with ${LINUX_WHEEL_KIND}${LINUX_WHEEL_VERSION} ==="
auditwheel repair dist/pyarrow-*.whl -w repaired_wheels

echo "=== (${PYTHON_VERSION}) Check the type stubs against the wheel ==="
python scripts/run_stubtest.py --wheel repaired_wheels/pyarrow-*.whl
popd
5 changes: 4 additions & 1 deletion cpp/cmake_modules/UseCython.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -145,8 +145,11 @@ function(compile_pyx
${no_docstrings_arg}
${cython_debug_arg}
${CYTHON_FLAGS}
# Necessary for autodoc of function arguments
# Necessary for autodoc of function arguments. The clinic format
# stores the signature in __text_signature__ rather than in
# __doc__, which keeps it out of the docstrings of the type stubs.
--directive embedsignature=True
--directive embedsignature.format=clinic
# Necessary for Cython code coverage
--working
${CMAKE_CURRENT_SOURCE_DIR}
Expand Down
Loading
Loading