Add installation of data-files build artifacts into Python package

docs/setuppy_tutorial.md

@@ -121,6 +121,11 @@ python -c 'import hello_world; print(hello_world.sum_as_string(5, 7))'  # => 12
   to manually add an extra `build.rs` file, see [PyO3/setuptools-rust#351](https://github.com/PyO3/setuptools-rust/pull/351)
   for more information about the workaround.
 
+- If your Rust extension generates files as part of its `build.rs` build script
+  that you want to be present in your Python wheel, you can use the `generated_files`
+  argument of `RustExtension` to define which files should be copied across, and into
+  which locations in the Python package.
+
 - Since the adoption of {pep}`517`, running `python setup.py ...` directly as a CLI tool is
   [considered deprecated](https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html).
   Nevertheless, `setup.py` can be safely used as a configuration file

examples/generated-files/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "generated-files"
+version = "0.1.0"
+edition = "2021"
+build = "build.rs"
+
+[dependencies]
+pyo3 = "0.27"
+
+[lib]
+name = "_lib"  # private module to be nested into Python package
+crate-type = ["cdylib"]
+path = "rust/lib.rs"

examples/generated-files/MANIFEST.in

@@ -0,0 +1,10 @@
+include build.rs
+include Cargo.toml
+include Cargo.lock
+graft rust
+graft tests
+
+# Data files generated by Rust build scripts generally do not need to be listed in the manifest
+# bceause they are not generated as part of the `sdist`; the Rust extension is also not bulit in an
+# sdist, so neither are the generated files.  You should ensure that all source files needed to
+# generate the build files _are_ included in the manifest, however.

examples/generated-files/build.rs

@@ -0,0 +1,16 @@
+use std::fs;
+use std::io::{self, Write};
+use std::path::Path;
+
+fn main() -> io::Result<()> {
+    println!("cargo::rerun-if-changed=build.rs");
+    let out_dir_raw = std::env::var("OUT_DIR").unwrap();
+    let out_dir = Path::new(&out_dir_raw);
+    fs::File::create(out_dir.join("my_file.txt"))?.write_all(b"Generated by a build script.\n")?;
+
+    let sub_dir = out_dir.join("dir");
+    fs::create_dir_all(&sub_dir)?;
+    fs::File::create(sub_dir.join("a.txt"))?.write_all(b"This is file A.\n")?;
+    fs::File::create(sub_dir.join("b.txt"))?.write_all(b"This is file B.\n")?;
+    Ok(())
+}

examples/generated-files/noxfile.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+
+import nox
+
+SETUPTOOLS_RUST = Path(__file__).parents[2]
+
+
+@nox.session()
+def test(session: nox.Session):
+    session.install(str(SETUPTOOLS_RUST), "setuptools", "pytest")
+    session.install("--no-build-isolation", ".")
+    session.run("pytest", "tests", *session.posargs)
+
+
+@nox.session()
+def test_inplace(session: nox.Session):
+    session.install(str(SETUPTOOLS_RUST), "setuptools", "pytest")
+    session.install("--no-build-isolation", "--editable", ".")
+    try:
+        session.run("pytest", "tests", *session.posargs)
+    finally:
+        # Clear out any data files that _did_ exist
+        session.run(
+            "python",
+            "-c",
+            """
+import os
+import shutil
+from pathlib import Path
+import generated_files
+
+try:
+    os.remove(Path(generated_files.__file__).parent / "my_file.txt")
+except FileNotFoundError:
+    pass
+shutil.rmtree(Path(generated_files.__file__).parent / "_data", ignore_errors=True)""",
+        )

examples/generated-files/pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["setuptools", "setuptools-rust"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "generated-files"
+version = "1.0"
+
+[tool.setuptools.packages]
+# Pure Python packages/modules
+find = { where = ["python"] }
+
+[[tool.setuptools-rust.ext-modules]]
+target = "generated_files._lib"
+[tool.setuptools-rust.ext-modules.generated-files]
+# Keys correspond to files/directories in the Rust extension's build directory `OUT_DIR`.
+# Values are Python packages that the corresponding file or directory should be placed inside.
+"my_file.txt" = "generated_files"
+"dir" = "generated_files._data"

examples/generated-files/python/generated_files/__init__.py

@@ -0,0 +1,11 @@
+__all__ = ["library_ok", "data_files_content"]
+
+from pathlib import Path
+from ._lib import library_ok
+
+
+def data_files_content() -> dict[Path, str]:
+    us = Path(__file__).parent
+    paths = [us / "my_file.txt"]
+    paths.extend((us / "_data" / "dir").glob("*.txt"))
+    return {path.relative_to(us): path.read_text() for path in paths}

examples/generated-files/rust/lib.rs

@@ -0,0 +1,9 @@
+#[pyo3::pymodule]
+mod _lib {
+    use pyo3::prelude::*;
+
+    #[pyfunction]
+    fn library_ok() -> bool {
+        true
+    }
+}

examples/generated-files/tests/test_lib.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+import generated_files
+
+
+def test_rust_extension():
+    assert generated_files.library_ok()
+
+
+def test_data_file_content():
+    expected = {
+        Path("my_file.txt"): "Generated by a build script.\n",
+        Path("_data") / "dir" / "a.txt": "This is file A.\n",
+        Path("_data") / "dir" / "b.txt": "This is file B.\n",
+    }
+    assert generated_files.data_files_content() == expected

setuptools_rust/build.py

@@ -24,6 +24,7 @@
 from setuptools import Distribution
 from setuptools.command.build_ext import build_ext as CommandBuildExt
 from setuptools.command.build_ext import get_abi3_suffix
+from setuptools.command.build_py import build_py as setuptools_build_py
 from setuptools.command.install_scripts import install_scripts as CommandInstallScripts
 
 from ._utils import check_subprocess_output, format_called_process_error, Env
@@ -124,10 +125,12 @@ def run_for_extension(self, ext: RustExtension) -> None:
         assert self.plat_name is not None
         if self.target is _Platform.CARGO_DEFAULT:
             self.target = _override_cargo_default_target(self.plat_name, ext.env)
-        dylib_paths = self.build_extension(ext)
-        self.install_extension(ext, dylib_paths)
+        dylib_paths, artifact_dirs = self.build_extension(ext)
+        self.install_extension(ext, dylib_paths, artifact_dirs)
 
-    def build_extension(self, ext: RustExtension) -> List["_BuiltModule"]:
+    def build_extension(
+        self, ext: RustExtension
+    ) -> Tuple[List["_BuiltModule"], List[Path]]:
         env = _prepare_build_environment(ext.env, ext)
 
         if not os.path.exists(ext.path):
@@ -201,13 +204,21 @@ def build_extension(self, ext: RustExtension) -> List["_BuiltModule"]:
             targets: List[Optional[str]] = [None]
         elif self.target is _Platform.UNIVERSAL2:
             targets = list(_UNIVERSAL2_TARGETS)
+            if ext.generated_files:
+                raise PlatformError(
+                    "generated files are not supported for universal2 wheels"
+                )
         else:
             targets = [self.target]
 
-        cargo_messages = ""
+        cargo_messages: Dict[str, List[str]] = {}
         for target in targets:
             target_command = command.copy()
-            if target is not None:
+            if target is None:
+                # Normalize the entries in `cargo_messages` to always be in terms of the
+                # actual target triple.
+                target = get_rust_host(ext.env)
+            else:
                 target_command += ["--target", target]
             if rustc_args:
                 target_command += ["--"]
@@ -221,12 +232,12 @@ def build_extension(self, ext: RustExtension) -> List["_BuiltModule"]:
                 # If quiet, capture all output and only show it in the exception
                 # If not quiet, forward all cargo output to stderr
                 stderr = subprocess.PIPE if quiet else None
-                cargo_messages += check_subprocess_output(
+                cargo_messages[target] = check_subprocess_output(
                     target_command,
                     env=env,
                     stderr=stderr,
                     text=True,
-                )
+                ).splitlines()
             except subprocess.CalledProcessError as e:
                 # Don't include stdout in the formatted error as it is a huge dump
                 # of cargo json lines which aren't helpful for the end user.
@@ -246,7 +257,7 @@ def build_extension(self, ext: RustExtension) -> List["_BuiltModule"]:
         if ext._uses_exec_binding():
             # Find artifact from cargo messages
             artifacts = _find_cargo_artifacts(
-                cargo_messages.splitlines(),
+                [line for messages in cargo_messages.values() for line in messages],
                 package_id=package_id,
                 kinds={"bin"},
             )
@@ -276,7 +287,7 @@ def build_extension(self, ext: RustExtension) -> List["_BuiltModule"]:
         else:
             # Find artifact from cargo messages
             artifacts = _find_cargo_artifacts(
-                cargo_messages.splitlines(),
+                [line for messages in cargo_messages.values() for line in messages],
                 package_id=package_id,
                 kinds={"cdylib", "dylib"},
             )
@@ -300,10 +311,28 @@ def build_extension(self, ext: RustExtension) -> List["_BuiltModule"]:
 
             # guaranteed to be just one element after checks above
             dylib_paths.append(_BuiltModule(ext.name, artifact_path))
-        return dylib_paths
+
+        if not ext.generated_files:
+            return dylib_paths, []
+
+        out_dirs = [
+            out_dir
+            for target, messages in cargo_messages.items()
+            if (out_dir := _find_cargo_out_dir(messages, package_id)) is not None
+        ]
+        if not out_dirs:
+            raise ExecError(
+                f"extension {ext.name} requests data files, but no corresponding"
+                " build-script out directories could be found"
+            )
+
+        return dylib_paths, out_dirs
 
     def install_extension(
-        self, ext: RustExtension, dylib_paths: List["_BuiltModule"]
+        self,
+        ext: RustExtension,
+        dylib_paths: List["_BuiltModule"],
+        build_artifact_dirs: List[Path],
     ) -> None:
         debug_build = self._is_debug_build(ext)
 
@@ -400,6 +429,39 @@ def install_extension(
             mode |= (mode & 0o444) >> 2  # copy R bits to X
             os.chmod(ext_path, mode)
 
+        if not ext.generated_files:
+            return
+
+        # We'll delegate the finding of the package directories to Setuptools, so we
+        # can be sure we're handling editable installs and other complex situations
+        # correctly.
+        build_py = cast(setuptools_build_py, self.get_finalized_command("build_py"))
+
+        def get_package_dir(package: str) -> Path:
+            if self.inplace:
+                # If `inplace`, we have to ask `build_py` (like `build_ext` would).
+                return Path(build_py.get_package_dir(package))
+            # ... If not, `build_ext` knows where to put the package.
+            return Path(build_ext.build_lib) / Path(*package.split("."))
+
+        for source, package in ext.generated_files.items():
+            dest = get_package_dir(package)
+            dest.mkdir(mode=0o755, parents=True, exist_ok=True)
+            for artifact_dir in build_artifact_dirs:
+                source_full = artifact_dir / source
+                dest_full = dest / source_full.name
+                if source_full.is_file():
+                    logger.info(
+                        "Copying data file from %s to %s", source_full, dest_full
+                    )
+                    shutil.copy2(source_full, dest_full)
+                elif source_full.is_dir():
+                    logger.info(
+                        "Copying data directory from %s to %s", source_full, dest_full
+                    )
+                    shutil.copytree(source_full, dest_full, dirs_exist_ok=True)
+                # This tacitly makes "no match" a silent non-error.
+
     def get_dylib_ext_path(self, ext: RustExtension, target_fname: str) -> str:
         assert self.plat_name is not None
         build_ext = cast(CommandBuildExt, self.get_finalized_command("build_ext"))
@@ -835,6 +897,20 @@ def _find_cargo_artifacts(
     return artifacts
 
 
+def _find_cargo_out_dir(cargo_messages: List[str], package_id: str) -> Optional[Path]:
+    # Chances are that the line we're looking for will be the third laste line in the
+    # messages.  The last is the completion report, the penultimate is generally the
+    # build of the final artifact.
+    for messsage in reversed(cargo_messages):
+        if "build-script-executed" not in messsage or package_id not in messsage:
+            continue
+        parsed = json.loads(messsage)
+        if parsed.get("package_id") == package_id:
+            out_dir = parsed.get("out_dir")
+            return None if out_dir is None else Path(out_dir)
+    return None
+
+
 def _replace_cross_target_dir(path: str, ext: RustExtension, *, quiet: bool) -> str:
     """Replaces target director from `cross` docker build with the correct
     local path.

setuptools_rust/extension.py

@@ -115,6 +115,15 @@ class RustExtension:
         env: Environment variables to use when calling cargo or rustc (``env=``
             in ``subprocess.Popen``). setuptools-rust may add additional
             variables or modify ``PATH``.
+        generated_files: Mapping of paths inside the extension's compilation ``OUT_DIR``
+            to Python (sub)packages that they should be copied into.  Each source path
+            can be either a file or a directory.  The source will be copied
+            (recursively, in the case of a directory) inside the mapped location.  This
+            option is incompatible with multiple targets.
+
+            If this is populated, the built extension must have a build script that
+            populates its ``OUT_DIR``.  Only the output of the build script of the extension itself
+            will be searched for data files.
     """
 
     def __init__(
@@ -135,6 +144,7 @@ def __init__(
         optional: bool = False,
         py_limited_api: Literal["auto", True, False] = "auto",
         env: Optional[Dict[str, str]] = None,
+        generated_files: Optional[Dict[str, str]] = None,
     ):
         if isinstance(target, dict):
             name = "; ".join("%s=%s" % (key, val) for key, val in target.items())
@@ -158,6 +168,12 @@ def __init__(
         self.optional = optional
         self.py_limited_api = py_limited_api
         self.env = Env(env)
+        self.generated_files = generated_files or {}
+
+        if self.generated_files and len(self.target) > 1:
+            raise ValueError(
+                "using 'generated_files' with multiple targets is not supported"
+            )
 
         if native:
             warnings.warn(