diff --git a/src/ewoks/__main__.py b/src/ewoks/__main__.py index 0c4903c..240b318 100644 --- a/src/ewoks/__main__.py +++ b/src/ewoks/__main__.py @@ -295,6 +295,8 @@ def command_in_environment( ) -> Literal[0, 1]: """Run the command in another python environment.""" environment = Environment.from_location(location) + # Flush so that this line comes before the output of the subprocess + print(f"Running in the python environment {environment.prefix}", flush=True) arguments = _remove_option(argv[1:], "--env") return subprocess.call( # noqa: S603 - Arguments of this command [environment.python, "-m", "ewoks", *arguments] diff --git a/src/ewoks/_requirements/__init__.py b/src/ewoks/_requirements/__init__.py index 475e288..31f9f0b 100644 --- a/src/ewoks/_requirements/__init__.py +++ b/src/ewoks/_requirements/__init__.py @@ -193,8 +193,10 @@ def install_requirements( installed = False if native: + files_environment = manager.files_environment(requirements, environment) try: - manager.install_files(requirements, environment) + manager.install_files(requirements, files_environment) + environment = files_environment installed = True except Exception: logger.exception( @@ -211,7 +213,7 @@ def install_requirements( ) if not installed: - manager.install_distributions(requirements, environment) + manager.install_distributions(requirements, environment, python_version) if ensure_ewoks: manager.ensure_ewoks(requirements, environment) diff --git a/src/ewoks/_requirements/pixi.py b/src/ewoks/_requirements/pixi.py index 6a0b9de..efaeacc 100644 --- a/src/ewoks/_requirements/pixi.py +++ b/src/ewoks/_requirements/pixi.py @@ -1,7 +1,6 @@ import json import logging import os -import platform import re import sys import tempfile @@ -42,6 +41,14 @@ class PixiManagerInfo(BaseManagerInfo): description="Environments described and created by a `pixi` workspace.", examples=["pixi"], ) + pixi_environment_name: str = Field( + default="default", + description=( + "Name of the environment of the pixi workspace in which the " + "requirements were generated." + ), + examples=["default"], + ) class PixiRequirements(BaseRequirements): @@ -65,7 +72,6 @@ class PixiManager(BaseManager): PRIORITY = 5 REQUIREMENTS_MODEL = PixiRequirements COMMAND_EXAMPLE = "pixi" - ENVIRONMENT_SUBDIR = os.path.join(".pixi", "envs", "default") def __init__(self, *command: str) -> None: if not command: @@ -109,16 +115,21 @@ def is_active(self) -> bool: def create_environment( self, location: Path, python_version: Optional[str] = None ) -> Environment: - """ - :raises RuntimeError: creation failed - """ - if not python_version: - python_version = platform.python_version() - manifest = self._manifest([], python_version) - write_files(location, {_MANIFEST_FILENAME: manifest}) - self._install_workspace(location) + """Environment that pixi creates when it installs the workspace.""" return self.environment(location) + def environment(self, location: Union[str, Path]) -> Environment: + """The default environment of the pixi workspace in `location`.""" + return _workspace_environment(location, "default") + + def files_environment( + self, requirements: BaseRequirements, environment: Environment + ) -> Environment: + if not isinstance(requirements, PixiRequirements): + return environment + name = requirements.manager.pixi_environment_name + return _workspace_environment(environment.location, name) + def _gather_files( self, distributions: List[models.Distribution], python_version: str ) -> Dict[str, str]: @@ -135,6 +146,14 @@ def _gather_files( # No pixi workspace: no files return super()._gather_files(distributions, python_version) + def _gather_extra_fields(self) -> Dict[str, str]: + prefix = Path(sys.prefix) + if not _is_workspace_environment(prefix): + # The manager model provides the default environment + return dict() + # A pixi environment looks like /.pixi/envs/ + return {"pixi_environment_name": prefix.name} + def _files_from_distributions( self, distributions: Sequence[models.Distribution], python_version: str ) -> Dict[str, str]: @@ -154,22 +173,30 @@ def _install_files( self, files: Mapping[str, str], environment: Environment ) -> None: write_files(environment.location, files) - self._install_workspace(environment.location) + self._install_workspace(environment.location, environment.prefix.name) def _add_ewoks( self, requirements: BaseRequirements, environment: Environment ) -> None: - # The workspace pins all versions so they cannot change + # pixi keeps the locked versions of the workflow requirements, unless ewoks + # needs different versions of them. self._check_call( "add", "--pypi", "ewoks", "--manifest-path", environment.location, + "--no-install", ) + self._install_workspace(environment.location, environment.prefix.name) + if not environment.distribution_version("ewoks"): + raise RuntimeError( + f"ewoks was not installed in the pixi environment " + f"'{environment.prefix.name}': add ewoks to one of its features" + ) - def _install_workspace(self, location: Path) -> None: - self._check_call("install", "--manifest-path", location) + def _install_workspace(self, location: Path, name: str) -> None: + self._check_call("install", "--manifest-path", location, "--environment", name) def _manifest(self, requirements: Sequence[str], python_version: str) -> str: """Content of a `pixi.toml` file with python distributions as pypi @@ -202,6 +229,21 @@ def _platform(self) -> str: raise RuntimeError("Cannot determine the pixi platform") from None +def _workspace_environment(location: Union[str, Path], name: str) -> Environment: + """Return the environment at `/.pixi/envs/`. + + The environment may not exist yet, so the python path is built instead of + searched for. + """ + location = Path(os.path.abspath(location)) + prefix = location / ".pixi" / "envs" / name + if sys.platform == "win32": + python = prefix / "python.exe" + else: + python = prefix / "bin" / "python" + return Environment(location=location, python=python) + + def _is_workspace_environment(prefix: Path) -> bool: """Environment is managed by a pixi workspace.""" parts = prefix.parts diff --git a/src/ewoks/_requirements/utils/base_manager.py b/src/ewoks/_requirements/utils/base_manager.py index 4c0ba77..bd6df96 100644 --- a/src/ewoks/_requirements/utils/base_manager.py +++ b/src/ewoks/_requirements/utils/base_manager.py @@ -1,5 +1,6 @@ import importlib.metadata import logging +import platform from abc import abstractmethod from pathlib import Path from textwrap import indent @@ -11,6 +12,7 @@ from typing import Union from pydantic import Field +from pydantic import model_serializer from . import process from .environment import Environment @@ -44,6 +46,15 @@ class BaseManagerInfo(models.BaseModel): examples=[{"requirements.txt": "ewoks==7.0.0\nnetworkx==3.4.2\n"}], ) + @model_serializer(mode="wrap") + def _files_last(self, handler): + """The files are long: write them after the other manager fields.""" + data = handler(self) + if "files" in data: + # Removing and adding "files" again moves it to the end of the dict + data["files"] = data.pop("files") + return data + class BaseRequirements(models.BaseModel): ewoks: Optional[models.EwoksInfo] = Field( @@ -120,9 +131,11 @@ class BaseManager: An implementation provides `version`, `is_active`, `create_environment`, `_files_from_distributions`, `_install_files` and `_add_ewoks`. Package managers that can inspect a python environment also override `_gather_files`. Package - managers that keep named environments in a directory of their own also override - `environments_root`. Package managers that are not named after the tool that - installs the distributions also override `installed_distribution`. + managers with extra fields in their manager model also override + `_gather_extra_fields`. Package managers that keep named environments in a + directory of their own also override `environments_root`. Package managers that + are not named after the tool that installs the distributions also override + `installed_distribution`. """ NAME = NotImplemented @@ -177,7 +190,12 @@ def gather_requirements(self) -> BaseRequirements: metadata["distributions"], metadata["python"]["version"] ) - manager = dict(name=self.NAME, version=manager_version, files=files) + manager = dict( + name=self.NAME, + version=manager_version, + files=files, + **self._gather_extra_fields(), + ) ewoks = dict(version=importlib.metadata.version("ewoks")) return self.REQUIREMENTS_MODEL(ewoks=ewoks, manager=manager, **metadata) @@ -185,6 +203,14 @@ def environment(self, location: Union[str, Path]) -> Environment: """Environment the manager creates at this location.""" return Environment.at_location(location, self.ENVIRONMENT_SUBDIR) + def files_environment( + self, requirements: BaseRequirements, environment: Environment + ) -> Environment: + """Environment in which the files of the requirements are installed. + Package managers whose files describe several environments override this. + """ + return environment + def environment_location( self, name: str, root: Optional[Union[str, Path]] = None ) -> Path: @@ -243,10 +269,15 @@ def install_files( raise def install_distributions( - self, requirements: BaseRequirements, environment: Environment + self, + requirements: BaseRequirements, + environment: Environment, + python_version: Optional[str] = None, ) -> None: - """Install the python distributions of the requirements in an existing - environment. + """Install the python distributions of the requirements in an environment. + `python_version` is used when neither the requirements nor the environment + provide a python version, and the version of the running python when + `python_version` is not given. :raises ValueError: no distributions provided to install :raises RuntimeError: installation failed @@ -254,8 +285,13 @@ def install_distributions( if not requirements.distributions: raise ValueError("No distributions provided to install") # Requirements that do not specify a python version are described for the - # python version of the environment they are installed in - python_version = requirements.python.version or environment.python_version() + # python version of the environment they are installed in. + if requirements.python.version: + python_version = requirements.python.version + elif environment.exists(): + python_version = environment.python_version() + elif not python_version: + python_version = platform.python_version() try: files = self._files_from_distributions( requirements.distributions, python_version @@ -295,6 +331,13 @@ def _gather_files( """ return dict() + def _gather_extra_fields(self) -> Dict[str, str]: + """Values of the fields a package manager adds to its manager model, + besides name, version and files. Package managers that add fields + override this. + """ + return dict() + @abstractmethod def _files_from_distributions( self, distributions: Sequence[models.Distribution], python_version: str diff --git a/src/ewoks/_requirements/utils/environment.py b/src/ewoks/_requirements/utils/environment.py index 4f82d31..54b5f01 100644 --- a/src/ewoks/_requirements/utils/environment.py +++ b/src/ewoks/_requirements/utils/environment.py @@ -3,6 +3,7 @@ import sys from dataclasses import dataclass from pathlib import Path +from typing import Dict from typing import Optional from typing import Tuple from typing import Union @@ -14,7 +15,6 @@ ENVIRONMENT_SUBDIRS: Tuple[str, ...] = ( "", # environment prefix, for example created by `python -m venv` ".venv", # uv and poetry project - os.path.join(".pixi", "envs", "default"), # pixi project ) @@ -42,14 +42,42 @@ def at_location(cls, location: Union[str, Path], subdir: str = "") -> "Environme @classmethod def from_location(cls, location: Union[str, Path]) -> "Environment": - """Environment found inside the location. + """Environment found inside the location. A pixi workspace provides its only + environment, or its default environment when it has several. - :raises ValueError: no python interpreter found + :raises ValueError: no python interpreter found or several pixi environments + without a default one """ for subdir in ENVIRONMENT_SUBDIRS: environment = cls.at_location(location, subdir) if environment.python.is_file(): return environment + + environments = _pixi_environments(location) + if len(environments) == 1: + return list(environments.values())[0] + if environments: + env_message = ( + f"Several pixi environments found in '{location}': " + f"{', '.join(environments)}" + ) + # Example of a path to pass, other than the default environment + named_env_paths = [ + environment.prefix + for name, environment in environments.items() + if name != "default" + ] + example = f"for example '--env {named_env_paths[0]}'" + if "default" in environments: + logger.warning( + f"{env_message}\nUsing the default environment. To use another " + f"one, rerun with its path in --env, {example}" + ) + return environments["default"] + raise ValueError( + f"{env_message}\nRerun with the path of one of them in --env, {example}" + ) + raise ValueError(f"No python environment found in '{location}'") @property @@ -87,6 +115,19 @@ def _python_output(self, code: str) -> str: return process.check_output(self.python, "-c", code).strip() +def _pixi_environments(location: Union[str, Path]) -> Dict[str, Environment]: + """Environments of a pixi workspace by name, which look like + /.pixi/envs/. + """ + environments = {} + for prefix in sorted((Path(location) / ".pixi" / "envs").glob("*")): + subdir = os.path.join(".pixi", "envs", prefix.name) + environment = Environment.at_location(location, subdir) + if environment.python.is_file(): + environments[prefix.name] = environment + return environments + + def python_executable(prefix: Path) -> Path: """Python interpreter of an environment prefix. The interpreter of a virtual environment is returned when the environment does not exist yet. diff --git a/src/ewoks/tests/requirements/managers.py b/src/ewoks/tests/requirements/managers.py index 74dd463..6a9e59d 100644 --- a/src/ewoks/tests/requirements/managers.py +++ b/src/ewoks/tests/requirements/managers.py @@ -36,6 +36,9 @@ class ManagerCase: CHANNEL_PYTHON: bool = False """Python comes from a conda channel, which decides the patch version.""" + CREATES_ENVIRONMENT: bool = True + """The environment exists before the requirements are installed.""" + def command(self) -> Tuple[str, ...]: """Command that invokes the package manager (the default when empty).""" return tuple() @@ -132,6 +135,7 @@ class PixiCase(ManagerCase): INSTALLER = "uv-pixi" # pixi installs PyPI distributions with its own uv SLOW = True CHANNEL_PYTHON = True + CREATES_ENVIRONMENT = False def native_files( self, distributions: Sequence[models.Distribution], python_version: str diff --git a/src/ewoks/tests/requirements/test_environment.py b/src/ewoks/tests/requirements/test_environment.py index b2abfa0..da8a565 100644 --- a/src/ewoks/tests/requirements/test_environment.py +++ b/src/ewoks/tests/requirements/test_environment.py @@ -36,3 +36,31 @@ def test_environment_layout(platform, relative, subdir, monkeypatch, tmp_path): def test_no_environment(tmp_path): with pytest.raises(ValueError, match="No python environment found"): Environment.from_location(tmp_path) + + +@pytest.mark.parametrize( + "names, expected", + [ + (["default"], "default"), + (["test"], "test"), + (["default", "test"], "default"), + (["test", "docs"], None), + ], +) +def test_pixi_environments(names, expected, monkeypatch, tmp_path, caplog): + """A pixi workspace provides its only environment, or its default environment + with a warning when it has several. + """ + monkeypatch.setattr(sys, "platform", "linux") + envs = tmp_path / ".pixi" / "envs" + for name in names: + python = envs / name / "bin" / "python" + python.parent.mkdir(parents=True) + python.touch() + + if expected is None: + with pytest.raises(ValueError, match="Several pixi environments"): + Environment.from_location(tmp_path) + else: + assert Environment.from_location(tmp_path).prefix == envs / expected + assert ("Several pixi environments" in caplog.text) == (len(names) > 1) diff --git a/src/ewoks/tests/requirements/test_managers.py b/src/ewoks/tests/requirements/test_managers.py index 07d55b5..04dacb7 100644 --- a/src/ewoks/tests/requirements/test_managers.py +++ b/src/ewoks/tests/requirements/test_managers.py @@ -93,13 +93,17 @@ def test_gather_requirements(manager, manager_case, monkeypatch): def test_create_environment(manager_case, environment): - assert environment.exists() - if manager_case.CHANNEL_PYTHON: - assert environment.python_version().startswith(f"{PYTHON_MINOR_VERSION}.") + if manager_case.CREATES_ENVIRONMENT: + assert environment.exists() + if manager_case.CHANNEL_PYTHON: + assert environment.python_version().startswith(f"{PYTHON_MINOR_VERSION}.") + else: + assert environment.python_version() == PYTHON_VERSION + assert Environment.from_location(environment.location) == environment + assert_not_installed(environment, DISTRIBUTIONS) else: - assert environment.python_version() == PYTHON_VERSION - assert Environment.from_location(environment.location) == environment - assert_not_installed(environment, DISTRIBUTIONS) + # The environment is created when the requirements are installed + assert not environment.exists() def test_install_native(manager, manager_case, environment): diff --git a/src/ewoks/tests/requirements/test_pixi.py b/src/ewoks/tests/requirements/test_pixi.py index 6128389..a63d82f 100644 --- a/src/ewoks/tests/requirements/test_pixi.py +++ b/src/ewoks/tests/requirements/test_pixi.py @@ -5,11 +5,13 @@ import pytest from ..._requirements.pixi import PixiManager +from ..._requirements.pixi import PixiRequirements from ..._requirements.utils.conda_channel import python_specifier from ..._requirements.utils.files import write_files from .utils import DISTRIBUTIONS from .utils import PYTHON_VERSION from .utils import REQUIREMENTS +from .utils import manager_requirements def test_workspace_environment(monkeypatch, tmp_path): @@ -57,6 +59,61 @@ def test_is_not_active(monkeypatch, tmp_path): assert not PixiManager().is_active() +def test_pixi_environment_name_not_from_variable(monkeypatch, tmp_path): + """The name comes from the running python, not from an environment that pixi + activated for another interpreter.""" + monkeypatch.setenv("PIXI_ENVIRONMENT_NAME", "other") + monkeypatch.setattr(sys, "prefix", str(tmp_path / ".pixi" / "envs" / "test")) + assert PixiManager()._gather_extra_fields() == {"pixi_environment_name": "test"} + + +def test_pixi_environment_name_from_prefix(monkeypatch, tmp_path): + """The name comes from the environment prefix.""" + monkeypatch.setattr(sys, "prefix", str(tmp_path / ".pixi" / "envs" / "test")) + assert PixiManager()._gather_extra_fields() == {"pixi_environment_name": "test"} + + +def test_pixi_environment_name_default(monkeypatch, tmp_path): + """Outside a pixi workspace the environment is "default".""" + monkeypatch.setattr(sys, "prefix", str(tmp_path)) + assert PixiManager()._gather_extra_fields() == {} + requirements = PixiRequirements(**manager_requirements("pixi")) + assert requirements.manager.pixi_environment_name == "default" + + +def test_gather_pixi_environment_name(monkeypatch, tmp_path): + """The gathered requirements contain the name of the environment.""" + monkeypatch.setattr(PixiManager, "version", lambda self: "0.82.1") + monkeypatch.setattr(sys, "prefix", str(tmp_path / ".pixi" / "envs" / "test")) + requirements = PixiManager().gather_requirements() + assert requirements.manager.pixi_environment_name == "test" + + +def test_files_environment(tmp_path): + """The files are installed in the environment of the requirements.""" + data = manager_requirements("pixi") + data["manager"]["pixi_environment_name"] = "test" + requirements = PixiRequirements(**data) + manager = PixiManager() + environment = manager.files_environment(requirements, manager.environment(tmp_path)) + assert environment.location == tmp_path + assert environment.prefix == tmp_path / ".pixi" / "envs" / "test" + + +def test_windows_interpreter(monkeypatch, tmp_path): + """On Windows the interpreter is in the prefix, also before pixi creates the + environment.""" + monkeypatch.setattr(sys, "platform", "win32") + data = manager_requirements("pixi") + data["manager"]["pixi_environment_name"] = "test" + requirements = PixiRequirements(**data) + manager = PixiManager() + environment = manager.environment(tmp_path) + assert environment.python == tmp_path / ".pixi" / "envs" / "default" / "python.exe" + environment = manager.files_environment(requirements, environment) + assert environment.python == tmp_path / ".pixi" / "envs" / "test" / "python.exe" + + @pytest.mark.skipif(PixiManager().version() is None, reason="pixi is not installed") def test_manifest(): manifest = PixiManager()._manifest(REQUIREMENTS, PYTHON_VERSION)