diff --git a/CHANGES.md b/CHANGES.md index 7a20b7f..01792c1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,13 @@ # Release Notes +## 0.13.2 + +This release has injected PEXes using PEP-829 `.start` files to affect `PEX_EXTRA_SYS_PATH` +`sys.path` mutation instead of `.pth` import lines. Although the `.pth` file is still emitted for +maximum compatibility with third party code, the `PEX_EXTRA_SYS_PATH.pth` file will no longer be +emitted for PEX venvs using Python 3.20 and greater. See [PEP-829](https://peps.python.org/pep-0829) +and the [3.15 notes](https://docs.python.org/3.15/whatsnew/3.15.html#whatsnew315-startup-files). + ## 0.13.1 This release eliminates an erroneous application starting cursor that would persist for several diff --git a/Cargo.lock b/Cargo.lock index 1a9b0bb..31ea18c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2040,7 +2040,7 @@ dependencies = [ [[package]] name = "pexrc" -version = "0.13.1" +version = "0.13.2" dependencies = [ "anstream", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 419ba30..021b3c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ cargo-features = ["profile-rustflags"] [package] name = "pexrc" -version = "0.13.1" +version = "0.13.2" edition = { workspace = true } publish = false diff --git a/crates/scripts/src/PEX_EXTRA_SYS_PATH.pth b/crates/scripts/src/PEX_EXTRA_SYS_PATH.pth new file mode 100644 index 0000000..8d4c9d4 --- /dev/null +++ b/crates/scripts/src/PEX_EXTRA_SYS_PATH.pth @@ -0,0 +1 @@ +import PEX_EXTRA_SYS_PATH; PEX_EXTRA_SYS_PATH.extend_sys_path(legacy=True) \ No newline at end of file diff --git a/crates/scripts/src/PEX_EXTRA_SYS_PATH.py b/crates/scripts/src/PEX_EXTRA_SYS_PATH.py new file mode 100644 index 0000000..6d4496d --- /dev/null +++ b/crates/scripts/src/PEX_EXTRA_SYS_PATH.py @@ -0,0 +1,25 @@ +# Copyright 2026 Pex project contributors. +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + + +def _extend_sys_path(path_env_var): + path = os.environ.get(path_env_var) + if path: + entries = path.split(os.pathsep) + sys.path.extend(entries) + return entries + return [] + + +def extend_sys_path(legacy=False): + entries = _extend_sys_path("PEX_EXTRA_SYS_PATH") + entries.extend(_extend_sys_path("__PEX_EXTRA_SYS_PATH__")) + debug = os.environ.pop("__PEX_EXTRA_SYS_PATH_DEBUG__", "") + if debug: + import json + + with open(debug, "w") as fp: + json.dump({"entries": entries, "legacy": legacy}, fp) diff --git a/crates/scripts/src/PEX_EXTRA_SYS_PATH.start b/crates/scripts/src/PEX_EXTRA_SYS_PATH.start new file mode 100644 index 0000000..0ec6757 --- /dev/null +++ b/crates/scripts/src/PEX_EXTRA_SYS_PATH.start @@ -0,0 +1 @@ +PEX_EXTRA_SYS_PATH:extend_sys_path \ No newline at end of file diff --git a/crates/scripts/src/lib.rs b/crates/scripts/src/lib.rs index d40bed1..23d93d7 100644 --- a/crates/scripts/src/lib.rs +++ b/crates/scripts/src/lib.rs @@ -25,6 +25,9 @@ pub enum Script { IdentifyInterpreter, VendoredVirtualenv, VenvPex, + VenvPexExtraSysPathPth, + VenvPexExtraSysPathPy, + VenvPexExtraSysPathStart, VenvPexRepl, } @@ -34,6 +37,9 @@ impl Script { Script::IdentifyInterpreter => "interpreter.py", Script::VendoredVirtualenv => "virtualenv.py", Script::VenvPex => "venv-pex.py", + Script::VenvPexExtraSysPathPth => "PEX_EXTRA_SYS_PATH.pth", + Script::VenvPexExtraSysPathPy => "PEX_EXTRA_SYS_PATH.py", + Script::VenvPexExtraSysPathStart => "PEX_EXTRA_SYS_PATH.start", Script::VenvPexRepl => "venv-pex-repl.py", } } @@ -68,6 +74,9 @@ impl Scripts { Script::IdentifyInterpreter => include_str!("interpreter.py"), Script::VendoredVirtualenv => include_str!(env!("VIRTUALENV_PY")), Script::VenvPex => include_str!("venv-pex.py"), + Script::VenvPexExtraSysPathPth => include_str!("PEX_EXTRA_SYS_PATH.pth"), + Script::VenvPexExtraSysPathPy => include_str!("PEX_EXTRA_SYS_PATH.py"), + Script::VenvPexExtraSysPathStart => include_str!("PEX_EXTRA_SYS_PATH.start"), Script::VenvPexRepl => include_str!("venv-pex-repl.py"), })), Scripts::Loose(base_dir) => { @@ -212,4 +221,7 @@ macro_rules! generate_script_type { generate_script_type!(IdentifyInterpreter); generate_script_type!(VendoredVirtualenv); generate_script_type!(VenvPex); +generate_script_type!(VenvPexExtraSysPathPth); +generate_script_type!(VenvPexExtraSysPathPy); +generate_script_type!(VenvPexExtraSysPathStart); generate_script_type!(VenvPexRepl); diff --git a/crates/venv/src/venv_pex.rs b/crates/venv/src/venv_pex.rs index 9f729e9..4b38a63 100644 --- a/crates/venv/src/venv_pex.rs +++ b/crates/venv/src/venv_pex.rs @@ -32,7 +32,14 @@ use pex::{ }; use platform::{Perms, mark_executable, path_as_bytes, path_as_str, symlink_or_link_or_copy}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; -use scripts::{Scripts, VenvPex, VenvPexRepl}; +use scripts::{ + Scripts, + VenvPex, + VenvPexExtraSysPathPth, + VenvPexExtraSysPathPy, + VenvPexExtraSysPathStart, + VenvPexRepl, +}; use serde_json::Value; use zip::ZipArchive; @@ -900,6 +907,7 @@ pub fn populate<'a>( provenance, )?; if matches!(scope, InstallScope::All | InstallScope::Srcs) { + write_pex_extra_sys_path_support_files(venv, scripts)?; write_main( venv, shebang_interpreter, @@ -1062,6 +1070,45 @@ impl<'a> Display for PythonListTupleStrStr<'a> { } } +fn write_pex_extra_sys_path_support_files( + venv: &Virtualenv, + scripts: &mut Scripts, +) -> anyhow::Result<()> { + // See: https://peps.python.org/pep-0829/ + let mut pex_extra_sys_path_py_fp = + File::create_new(venv.site_packages_path("PEX_EXTRA_SYS_PATH.py"))?; + pex_extra_sys_path_py_fp + .write_all(VenvPexExtraSysPathPy::read(scripts)?.contents().as_bytes())?; + + let python_version = { + let version = venv.interpreter.raw().version; + (version.major, version.minor) + }; + + // Starting with Python 3.15 .start files trump import lines in .path files. + // See: https://peps.python.org/pep-0829/#abstract + if python_version >= (3, 15) { + let mut pex_extra_sys_path_start_fp = + File::create_new(venv.site_packages_path("PEX_EXTRA_SYS_PATH.start"))?; + pex_extra_sys_path_start_fp.write_all( + VenvPexExtraSysPathStart::read(scripts)? + .contents() + .as_bytes(), + )?; + } + + // After ~Python 3.20 .pth import lines will start to raise warnings; so we no longer emit a + // .pth compatibility bridge. See: https://peps.python.org/pep-0829/#abstract + if python_version < (3, 20) { + let mut pex_extra_sys_path_pth_fp = + File::create_new(venv.site_packages_path("PEX_EXTRA_SYS_PATH.pth"))?; + pex_extra_sys_path_pth_fp + .write_all(VenvPexExtraSysPathPth::read(scripts)?.contents().as_bytes())?; + } + + Ok(()) +} + fn write_main( venv: &Virtualenv, shebang_interpreter: &Path, @@ -1070,22 +1117,6 @@ fn write_main( scripts: &mut Scripts, bin_path_override: Option, ) -> anyhow::Result<()> { - let pex_extra_sys_path_pth = venv.site_packages_path("PEX_EXTRA_SYS_PATH.pth"); - let mut pex_extra_sys_path_pth_fp = File::create_new(&pex_extra_sys_path_pth)?; - for env_var in ["PEX_EXTRA_SYS_PATH", "__PEX_EXTRA_SYS_PATH__"].into_iter() { - // # N.B.: .pth import lines must be single lines: - // https://docs.python.org/3/library/site.html - writeln!( - pex_extra_sys_path_pth_fp, - "import os, sys; \ - sys.path.extend(\ - entry \ - for entry in os.environ.get('{env_var}', '').split(os.pathsep) \ - if entry\ - )", - )?; - } - let main_py = venv.prefix().join("__main__.py"); let mut main_py_fp = File::create_new(&main_py)?; write_shebang_bytes(&mut main_py_fp, shebang_interpreter, shebang_arg)?; diff --git a/pyproject.toml b/pyproject.toml index 7451bb0..20a7455 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,10 @@ name = "pexrc" requires-python = ">=2.7" dynamic = ["version"] +[build-system] +requires = ["uv_build>=0.11.16,<0.12"] +build-backend = "uv_build" + [tool.uv] package = false @@ -13,6 +17,7 @@ dev = [ "dev-cmd[old-pythons]; python_version >= '3.8'", "mypy", "pex", + "platformdirs", "psutil", "pytest", "pytest-xdist", @@ -22,9 +27,13 @@ dev = [ ] [[tool.mypy.overrides]] -module = "_pytest.*" +module = ["_pytest.*"] follow_imports = "skip" +[[tool.mypy.overrides]] +module = ["platformdirs"] +follow_untyped_imports = true + [tool.ruff] line-length = 100 diff --git a/python/testing/__init__.py b/python/testing/__init__.py index f53326a..2ccf23b 100644 --- a/python/testing/__init__.py +++ b/python/testing/__init__.py @@ -7,6 +7,8 @@ import platform import subprocess +import platformdirs + IS_CI = os.environ.get("CI", "false") == "true" IS_LINUX = platform.system().lower() == "linux" @@ -41,3 +43,8 @@ def session_dir(): def session_pexrc_root(): # type: () -> str return os.environ["_PEXRC_TEST_SESSION_PEXRC_ROOT"] + + +def testing_cache_root(): + # type: () -> str + return platformdirs.user_cache_dir("pexrc-dev", opinion=False) diff --git a/python/testing/interpreter.py b/python/testing/interpreter.py new file mode 100644 index 0000000..88ae23f --- /dev/null +++ b/python/testing/interpreter.py @@ -0,0 +1,69 @@ +# Copyright 2026 Pex project contributors. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import absolute_import + +import os +import platform +import subprocess + +from testing import IS_MAC, IS_WINDOWS, testing_cache_root + +TYPE_CHECKING = False +if TYPE_CHECKING: + # Ruff doesn't understand Python 2 and thus the type comment usages. + from typing import Text, Tuple # noqa: F401 + + +def get_os(): + # type: () -> str + + if IS_WINDOWS: + return "windows" + elif IS_MAC: + return "macos" + else: + return "linux" + + +def get_arch(): + # type: () -> str + + if IS_WINDOWS: + arch = os.environ.get("PROCESSOR_ARCHITECTURE", platform.machine()).lower() + else: + arch = platform.machine().lower() + + if arch in ("aarch64", "arm64"): + return "aarch64" + elif arch in ("x86_64", "amd64"): + return "x86_64" + else: + return arch + + +def ensure_python( + version, # type: Tuple[int, int] + install_if_missing=True, # type: bool +): + # type: (...) -> Text + + # N.B.: We force arch to get arm64 PBS builds for Windows arm64 machines. + # See: https://github.com/astral-sh/uv/issues/12906 + version_spec = "cpython-{major}.{minor}-{os}-{arch}".format( + major=version[0], minor=version[1], os=get_os(), arch=get_arch() + ) + env = dict(os.environ, UV_PYTHON_INSTALL_DIR=os.path.join(testing_cache_root(), "interpreters")) + try: + return ( + subprocess.check_output(args=["uv", "python", "find", version_spec], env=env) + .decode("utf-8") + .strip() + ) + except subprocess.CalledProcessError: + if not install_if_missing: + raise + subprocess.check_call( + args=["uv", "python", "install", "--managed-python", "--force", version_spec], env=env + ) + return ensure_python(version, install_if_missing=False) diff --git a/python/tests/test_pex_extra_sys_path.py b/python/tests/test_pex_extra_sys_path.py new file mode 100644 index 0000000..9561c9f --- /dev/null +++ b/python/tests/test_pex_extra_sys_path.py @@ -0,0 +1,89 @@ +# Copyright 2026 Pex project contributors. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import absolute_import + +import json +import os +import subprocess +import sys + +import pytest +from testing import pexrc_inject +from testing.interpreter import ensure_python + +TYPE_CHECKING = False +if TYPE_CHECKING: + # Ruff doesn't understand Python 2 and thus the type comment usages. + from typing import Any, Text, Tuple # noqa: F401 + + +@pytest.fixture +def find_pythons(): + # type: () -> Tuple[Text, Text] + + if sys.version_info[:2] < (3, 15): + return sys.executable, ensure_python(version=(3, 15)) + else: + return ensure_python(version=(3, 14)), sys.executable + + +@pytest.fixture +def python_pth(find_pythons): + # type: (Tuple[Text, Text]) -> Text + python_pth, _ = find_pythons + return python_pth + + +@pytest.fixture +def python_start(find_pythons): + # type: (Tuple[Text, Text]) -> Text + _, python_start = find_pythons + return python_start + + +def test_pep_829( + tmpdir, # type: Any + python_pth, # type: Text + python_start, # type: Text +): + # type: (...) -> None + + pex_root = os.path.join(str(tmpdir), "pex-root") + pex = os.path.join(str(tmpdir), "pex") + subprocess.check_call(args=["pex", "--runtime-pex-root", pex_root, "-o", pex]) + + one = os.path.join(str(tmpdir), "entry-one") + two = os.path.join(str(tmpdir), "entry-two") + three = os.path.join(str(tmpdir), "entry-three") + debug_file = os.path.join(str(tmpdir), "debug.json") + + injected_pex = pexrc_inject(pex) + + def assert_px_extra_sys_path( + python, # type: Text + expected_legacy, # type: bool + ): + # type: (...) -> None + output = subprocess.check_output( + args=[ + python, + injected_pex, + "-c", + "import json, sys; json.dump(sys.path, sys.stdout)", + ], + env=dict( + os.environ, + PEX_EXTRA_SYS_PATH=os.pathsep.join((two, one)), + __PEX_EXTRA_SYS_PATH__=three, + __PEX_EXTRA_SYS_PATH_DEBUG__=debug_file, + ), + ) + assert [two, one, three] == json.loads(output)[-3:] + with open(debug_file) as fp: + data = json.load(fp) + assert data["legacy"] == expected_legacy + assert [two, one, three] == data["entries"] + + assert_px_extra_sys_path(python_pth, expected_legacy=True) + assert_px_extra_sys_path(python_start, expected_legacy=False) diff --git a/uv.lock b/uv.lock index 0fe363e..d9ca3d4 100644 --- a/uv.lock +++ b/uv.lock @@ -1235,6 +1235,7 @@ dev = [ { name = "mypy", version = "1.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, { name = "mypy", version = "1.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pex" }, + { name = "platformdirs" }, { name = "psutil", version = "6.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.6'" }, { name = "psutil", version = "7.2.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.6'" }, { name = "pytest", version = "4.6.11", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.5'" }, @@ -1273,6 +1274,7 @@ dev = [ { name = "dev-cmd", extras = ["old-pythons"], marker = "python_full_version >= '3.8'" }, { name = "mypy" }, { name = "pex" }, + { name = "platformdirs" }, { name = "psutil" }, { name = "pytest" }, { name = "pytest-xdist" },