diff --git a/docs/setuppy_tutorial.md b/docs/setuppy_tutorial.md index a36267cc..dcff8eab 100644 --- a/docs/setuppy_tutorial.md +++ b/docs/setuppy_tutorial.md @@ -121,6 +121,11 @@ python -c 'import hello_world; print(hello_world.sum_as_string(5, 7))' # => 12 to manually add an extra `build.rs` file, see [PyO3/setuptools-rust#351](https://github.com/PyO3/setuptools-rust/pull/351) for more information about the workaround. +- If your Rust extension generates files as part of its `build.rs` build script + that you want to be present in your Python wheel, you can use the `generated_files` + argument of `RustExtension` to define which files should be copied across, and into + which locations in the Python package. + - Since the adoption of {pep}`517`, running `python setup.py ...` directly as a CLI tool is [considered deprecated](https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html). Nevertheless, `setup.py` can be safely used as a configuration file diff --git a/examples/generated-files/Cargo.toml b/examples/generated-files/Cargo.toml new file mode 100644 index 00000000..341cfcce --- /dev/null +++ b/examples/generated-files/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "generated-files" +version = "0.1.0" +edition = "2021" +build = "build.rs" + +[dependencies] +pyo3 = "0.27" + +[lib] +name = "_lib" # private module to be nested into Python package +crate-type = ["cdylib"] +path = "rust/lib.rs" diff --git a/examples/generated-files/MANIFEST.in b/examples/generated-files/MANIFEST.in new file mode 100644 index 00000000..46e32918 --- /dev/null +++ b/examples/generated-files/MANIFEST.in @@ -0,0 +1,10 @@ +include build.rs +include Cargo.toml +include Cargo.lock +graft rust +graft tests + +# Data files generated by Rust build scripts generally do not need to be listed in the manifest +# bceause they are not generated as part of the `sdist`; the Rust extension is also not bulit in an +# sdist, so neither are the generated files. You should ensure that all source files needed to +# generate the build files _are_ included in the manifest, however. diff --git a/examples/generated-files/build.rs b/examples/generated-files/build.rs new file mode 100644 index 00000000..05e2a781 --- /dev/null +++ b/examples/generated-files/build.rs @@ -0,0 +1,16 @@ +use std::fs; +use std::io::{self, Write}; +use std::path::Path; + +fn main() -> io::Result<()> { + println!("cargo::rerun-if-changed=build.rs"); + let out_dir_raw = std::env::var("OUT_DIR").unwrap(); + let out_dir = Path::new(&out_dir_raw); + fs::File::create(out_dir.join("my_file.txt"))?.write_all(b"Generated by a build script.\n")?; + + let sub_dir = out_dir.join("dir"); + fs::create_dir_all(&sub_dir)?; + fs::File::create(sub_dir.join("a.txt"))?.write_all(b"This is file A.\n")?; + fs::File::create(sub_dir.join("b.txt"))?.write_all(b"This is file B.\n")?; + Ok(()) +} diff --git a/examples/generated-files/noxfile.py b/examples/generated-files/noxfile.py new file mode 100644 index 00000000..e996d236 --- /dev/null +++ b/examples/generated-files/noxfile.py @@ -0,0 +1,37 @@ +from pathlib import Path + +import nox + +SETUPTOOLS_RUST = Path(__file__).parents[2] + + +@nox.session() +def test(session: nox.Session): + session.install(str(SETUPTOOLS_RUST), "setuptools", "pytest") + session.install("--no-build-isolation", ".") + session.run("pytest", "tests", *session.posargs) + + +@nox.session() +def test_inplace(session: nox.Session): + session.install(str(SETUPTOOLS_RUST), "setuptools", "pytest") + session.install("--no-build-isolation", "--editable", ".") + try: + session.run("pytest", "tests", *session.posargs) + finally: + # Clear out any data files that _did_ exist + session.run( + "python", + "-c", + """ +import os +import shutil +from pathlib import Path +import generated_files + +try: + os.remove(Path(generated_files.__file__).parent / "my_file.txt") +except FileNotFoundError: + pass +shutil.rmtree(Path(generated_files.__file__).parent / "_data", ignore_errors=True)""", + ) diff --git a/examples/generated-files/pyproject.toml b/examples/generated-files/pyproject.toml new file mode 100644 index 00000000..c6b29c55 --- /dev/null +++ b/examples/generated-files/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools", "setuptools-rust"] +build-backend = "setuptools.build_meta" + +[project] +name = "generated-files" +version = "1.0" + +[tool.setuptools.packages] +# Pure Python packages/modules +find = { where = ["python"] } + +[[tool.setuptools-rust.ext-modules]] +target = "generated_files._lib" +[tool.setuptools-rust.ext-modules.generated-files] +# Keys correspond to files/directories in the Rust extension's build directory `OUT_DIR`. +# Values are Python packages that the corresponding file or directory should be placed inside. +"my_file.txt" = "generated_files" +"dir" = "generated_files._data" diff --git a/examples/generated-files/python/generated_files/__init__.py b/examples/generated-files/python/generated_files/__init__.py new file mode 100644 index 00000000..3ae4f105 --- /dev/null +++ b/examples/generated-files/python/generated_files/__init__.py @@ -0,0 +1,11 @@ +__all__ = ["library_ok", "data_files_content"] + +from pathlib import Path +from ._lib import library_ok + + +def data_files_content() -> dict[Path, str]: + us = Path(__file__).parent + paths = [us / "my_file.txt"] + paths.extend((us / "_data" / "dir").glob("*.txt")) + return {path.relative_to(us): path.read_text() for path in paths} diff --git a/examples/generated-files/rust/lib.rs b/examples/generated-files/rust/lib.rs new file mode 100644 index 00000000..581c482b --- /dev/null +++ b/examples/generated-files/rust/lib.rs @@ -0,0 +1,9 @@ +#[pyo3::pymodule] +mod _lib { + use pyo3::prelude::*; + + #[pyfunction] + fn library_ok() -> bool { + true + } +} diff --git a/examples/generated-files/tests/test_lib.py b/examples/generated-files/tests/test_lib.py new file mode 100644 index 00000000..04a25248 --- /dev/null +++ b/examples/generated-files/tests/test_lib.py @@ -0,0 +1,15 @@ +from pathlib import Path +import generated_files + + +def test_rust_extension(): + assert generated_files.library_ok() + + +def test_data_file_content(): + expected = { + Path("my_file.txt"): "Generated by a build script.\n", + Path("_data") / "dir" / "a.txt": "This is file A.\n", + Path("_data") / "dir" / "b.txt": "This is file B.\n", + } + assert generated_files.data_files_content() == expected diff --git a/setuptools_rust/build.py b/setuptools_rust/build.py index d8d2dabe..6d07b674 100644 --- a/setuptools_rust/build.py +++ b/setuptools_rust/build.py @@ -24,6 +24,7 @@ from setuptools import Distribution from setuptools.command.build_ext import build_ext as CommandBuildExt from setuptools.command.build_ext import get_abi3_suffix +from setuptools.command.build_py import build_py as setuptools_build_py from setuptools.command.install_scripts import install_scripts as CommandInstallScripts from ._utils import check_subprocess_output, format_called_process_error, Env @@ -124,10 +125,12 @@ def run_for_extension(self, ext: RustExtension) -> None: assert self.plat_name is not None if self.target is _Platform.CARGO_DEFAULT: self.target = _override_cargo_default_target(self.plat_name, ext.env) - dylib_paths = self.build_extension(ext) - self.install_extension(ext, dylib_paths) + dylib_paths, artifact_dirs = self.build_extension(ext) + self.install_extension(ext, dylib_paths, artifact_dirs) - def build_extension(self, ext: RustExtension) -> List["_BuiltModule"]: + def build_extension( + self, ext: RustExtension + ) -> Tuple[List["_BuiltModule"], List[Path]]: env = _prepare_build_environment(ext.env, ext) if not os.path.exists(ext.path): @@ -201,13 +204,21 @@ def build_extension(self, ext: RustExtension) -> List["_BuiltModule"]: targets: List[Optional[str]] = [None] elif self.target is _Platform.UNIVERSAL2: targets = list(_UNIVERSAL2_TARGETS) + if ext.generated_files: + raise PlatformError( + "generated files are not supported for universal2 wheels" + ) else: targets = [self.target] - cargo_messages = "" + cargo_messages: Dict[str, List[str]] = {} for target in targets: target_command = command.copy() - if target is not None: + if target is None: + # Normalize the entries in `cargo_messages` to always be in terms of the + # actual target triple. + target = get_rust_host(ext.env) + else: target_command += ["--target", target] if rustc_args: target_command += ["--"] @@ -221,12 +232,12 @@ def build_extension(self, ext: RustExtension) -> List["_BuiltModule"]: # If quiet, capture all output and only show it in the exception # If not quiet, forward all cargo output to stderr stderr = subprocess.PIPE if quiet else None - cargo_messages += check_subprocess_output( + cargo_messages[target] = check_subprocess_output( target_command, env=env, stderr=stderr, text=True, - ) + ).splitlines() except subprocess.CalledProcessError as e: # Don't include stdout in the formatted error as it is a huge dump # of cargo json lines which aren't helpful for the end user. @@ -246,7 +257,7 @@ def build_extension(self, ext: RustExtension) -> List["_BuiltModule"]: if ext._uses_exec_binding(): # Find artifact from cargo messages artifacts = _find_cargo_artifacts( - cargo_messages.splitlines(), + [line for messages in cargo_messages.values() for line in messages], package_id=package_id, kinds={"bin"}, ) @@ -276,7 +287,7 @@ def build_extension(self, ext: RustExtension) -> List["_BuiltModule"]: else: # Find artifact from cargo messages artifacts = _find_cargo_artifacts( - cargo_messages.splitlines(), + [line for messages in cargo_messages.values() for line in messages], package_id=package_id, kinds={"cdylib", "dylib"}, ) @@ -300,10 +311,28 @@ def build_extension(self, ext: RustExtension) -> List["_BuiltModule"]: # guaranteed to be just one element after checks above dylib_paths.append(_BuiltModule(ext.name, artifact_path)) - return dylib_paths + + if not ext.generated_files: + return dylib_paths, [] + + out_dirs = [ + out_dir + for target, messages in cargo_messages.items() + if (out_dir := _find_cargo_out_dir(messages, package_id)) is not None + ] + if not out_dirs: + raise ExecError( + f"extension {ext.name} requests data files, but no corresponding" + " build-script out directories could be found" + ) + + return dylib_paths, out_dirs def install_extension( - self, ext: RustExtension, dylib_paths: List["_BuiltModule"] + self, + ext: RustExtension, + dylib_paths: List["_BuiltModule"], + build_artifact_dirs: List[Path], ) -> None: debug_build = self._is_debug_build(ext) @@ -400,6 +429,39 @@ def install_extension( mode |= (mode & 0o444) >> 2 # copy R bits to X os.chmod(ext_path, mode) + if not ext.generated_files: + return + + # We'll delegate the finding of the package directories to Setuptools, so we + # can be sure we're handling editable installs and other complex situations + # correctly. + build_py = cast(setuptools_build_py, self.get_finalized_command("build_py")) + + def get_package_dir(package: str) -> Path: + if self.inplace: + # If `inplace`, we have to ask `build_py` (like `build_ext` would). + return Path(build_py.get_package_dir(package)) + # ... If not, `build_ext` knows where to put the package. + return Path(build_ext.build_lib) / Path(*package.split(".")) + + for source, package in ext.generated_files.items(): + dest = get_package_dir(package) + dest.mkdir(mode=0o755, parents=True, exist_ok=True) + for artifact_dir in build_artifact_dirs: + source_full = artifact_dir / source + dest_full = dest / source_full.name + if source_full.is_file(): + logger.info( + "Copying data file from %s to %s", source_full, dest_full + ) + shutil.copy2(source_full, dest_full) + elif source_full.is_dir(): + logger.info( + "Copying data directory from %s to %s", source_full, dest_full + ) + shutil.copytree(source_full, dest_full, dirs_exist_ok=True) + # This tacitly makes "no match" a silent non-error. + def get_dylib_ext_path(self, ext: RustExtension, target_fname: str) -> str: assert self.plat_name is not None build_ext = cast(CommandBuildExt, self.get_finalized_command("build_ext")) @@ -835,6 +897,20 @@ def _find_cargo_artifacts( return artifacts +def _find_cargo_out_dir(cargo_messages: List[str], package_id: str) -> Optional[Path]: + # Chances are that the line we're looking for will be the third laste line in the + # messages. The last is the completion report, the penultimate is generally the + # build of the final artifact. + for messsage in reversed(cargo_messages): + if "build-script-executed" not in messsage or package_id not in messsage: + continue + parsed = json.loads(messsage) + if parsed.get("package_id") == package_id: + out_dir = parsed.get("out_dir") + return None if out_dir is None else Path(out_dir) + return None + + def _replace_cross_target_dir(path: str, ext: RustExtension, *, quiet: bool) -> str: """Replaces target director from `cross` docker build with the correct local path. diff --git a/setuptools_rust/extension.py b/setuptools_rust/extension.py index 8c68e33e..c2ede380 100644 --- a/setuptools_rust/extension.py +++ b/setuptools_rust/extension.py @@ -115,6 +115,15 @@ class RustExtension: env: Environment variables to use when calling cargo or rustc (``env=`` in ``subprocess.Popen``). setuptools-rust may add additional variables or modify ``PATH``. + generated_files: Mapping of paths inside the extension's compilation ``OUT_DIR`` + to Python (sub)packages that they should be copied into. Each source path + can be either a file or a directory. The source will be copied + (recursively, in the case of a directory) inside the mapped location. This + option is incompatible with multiple targets. + + If this is populated, the built extension must have a build script that + populates its ``OUT_DIR``. Only the output of the build script of the extension itself + will be searched for data files. """ def __init__( @@ -135,6 +144,7 @@ def __init__( optional: bool = False, py_limited_api: Literal["auto", True, False] = "auto", env: Optional[Dict[str, str]] = None, + generated_files: Optional[Dict[str, str]] = None, ): if isinstance(target, dict): name = "; ".join("%s=%s" % (key, val) for key, val in target.items()) @@ -158,6 +168,12 @@ def __init__( self.optional = optional self.py_limited_api = py_limited_api self.env = Env(env) + self.generated_files = generated_files or {} + + if self.generated_files and len(self.target) > 1: + raise ValueError( + "using 'generated_files' with multiple targets is not supported" + ) if native: warnings.warn(