From 85eb2e38132f694c3cb75de4a1f5cc4dc81bfeae Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 11 Nov 2026 17:02:22 +0100 Subject: [PATCH 1/4] Remove the workflow requirements that moved to ewokscore Co-Authored-By: Claude Opus 5.5 --- src/ewoks/_requirements/__init__.py | 281 ------------- src/ewoks/_requirements/conda.py | 202 --------- src/ewoks/_requirements/pip_venv.py | 147 ------- src/ewoks/_requirements/pixi.py | 261 ------------ src/ewoks/_requirements/poetry.py | 255 ------------ src/ewoks/_requirements/utils/__init__.py | 0 src/ewoks/_requirements/utils/_supported.py | 22 - src/ewoks/_requirements/utils/base_manager.py | 384 ------------------ .../_requirements/utils/conda_channel.py | 13 - src/ewoks/_requirements/utils/detect.py | 167 -------- src/ewoks/_requirements/utils/environment.py | 180 -------- src/ewoks/_requirements/utils/files.py | 34 -- .../_requirements/utils/metadata/__init__.py | 0 .../_requirements/utils/metadata/_unknown.py | 31 -- .../utils/metadata/from_python/__init__.py | 117 ------ .../utils/metadata/from_python/_git.py | 73 ---- .../utils/metadata/from_requirements_txt.py | 30 -- .../utils/metadata/last_resort.py | 48 --- .../_requirements/utils/metadata/models.py | 102 ----- src/ewoks/_requirements/utils/parse.py | 25 -- src/ewoks/_requirements/utils/process.py | 53 --- .../_requirements/utils/requirements_txt.py | 370 ----------------- .../_requirements/utils/toml_dependencies.py | 41 -- src/ewoks/_requirements/uv.py | 224 ---------- src/ewoks/tests/requirements/conftest.py | 126 ------ src/ewoks/tests/requirements/managers.py | 162 -------- src/ewoks/tests/requirements/test_conda.py | 96 ----- .../tests/requirements/test_cross_manager.py | 40 -- .../tests/requirements/test_environment.py | 66 --- src/ewoks/tests/requirements/test_managers.py | 279 ------------- src/ewoks/tests/requirements/test_pip_venv.py | 56 --- src/ewoks/tests/requirements/test_pixi.py | 128 ------ src/ewoks/tests/requirements/test_poetry.py | 111 ----- .../requirements/test_requirements_txt.py | 218 ---------- .../requirements/test_toml_dependencies.py | 27 -- src/ewoks/tests/requirements/test_uv.py | 94 ----- src/ewoks/tests/requirements/utils.py | 144 ------- 37 files changed, 4607 deletions(-) delete mode 100644 src/ewoks/_requirements/__init__.py delete mode 100644 src/ewoks/_requirements/conda.py delete mode 100644 src/ewoks/_requirements/pip_venv.py delete mode 100644 src/ewoks/_requirements/pixi.py delete mode 100644 src/ewoks/_requirements/poetry.py delete mode 100644 src/ewoks/_requirements/utils/__init__.py delete mode 100644 src/ewoks/_requirements/utils/_supported.py delete mode 100644 src/ewoks/_requirements/utils/base_manager.py delete mode 100644 src/ewoks/_requirements/utils/conda_channel.py delete mode 100644 src/ewoks/_requirements/utils/detect.py delete mode 100644 src/ewoks/_requirements/utils/environment.py delete mode 100644 src/ewoks/_requirements/utils/files.py delete mode 100644 src/ewoks/_requirements/utils/metadata/__init__.py delete mode 100644 src/ewoks/_requirements/utils/metadata/_unknown.py delete mode 100644 src/ewoks/_requirements/utils/metadata/from_python/__init__.py delete mode 100644 src/ewoks/_requirements/utils/metadata/from_python/_git.py delete mode 100644 src/ewoks/_requirements/utils/metadata/from_requirements_txt.py delete mode 100644 src/ewoks/_requirements/utils/metadata/last_resort.py delete mode 100644 src/ewoks/_requirements/utils/metadata/models.py delete mode 100644 src/ewoks/_requirements/utils/parse.py delete mode 100644 src/ewoks/_requirements/utils/process.py delete mode 100644 src/ewoks/_requirements/utils/requirements_txt.py delete mode 100644 src/ewoks/_requirements/utils/toml_dependencies.py delete mode 100644 src/ewoks/_requirements/uv.py delete mode 100644 src/ewoks/tests/requirements/conftest.py delete mode 100644 src/ewoks/tests/requirements/managers.py delete mode 100644 src/ewoks/tests/requirements/test_conda.py delete mode 100644 src/ewoks/tests/requirements/test_cross_manager.py delete mode 100644 src/ewoks/tests/requirements/test_environment.py delete mode 100644 src/ewoks/tests/requirements/test_managers.py delete mode 100644 src/ewoks/tests/requirements/test_pip_venv.py delete mode 100644 src/ewoks/tests/requirements/test_pixi.py delete mode 100644 src/ewoks/tests/requirements/test_poetry.py delete mode 100644 src/ewoks/tests/requirements/test_requirements_txt.py delete mode 100644 src/ewoks/tests/requirements/test_toml_dependencies.py delete mode 100644 src/ewoks/tests/requirements/test_uv.py delete mode 100644 src/ewoks/tests/requirements/utils.py diff --git a/src/ewoks/_requirements/__init__.py b/src/ewoks/_requirements/__init__.py deleted file mode 100644 index 31f9f0b..0000000 --- a/src/ewoks/_requirements/__init__.py +++ /dev/null @@ -1,281 +0,0 @@ -"""Workflow requirements.""" - -import logging -import re -import shutil -from pathlib import Path -from typing import Callable -from typing import Dict -from typing import Optional -from typing import Tuple -from typing import Union - -from ewokscore.graph import TaskGraph -from ewokscore.hashing import uhash - -from .utils import parse -from .utils._supported import get_supported_managers -from .utils.base_manager import BaseRequirements -from .utils.detect import get_in_place_installer -from .utils.detect import get_installer -from .utils.detect import get_manager -from .utils.environment import Environment -from .utils.metadata import last_resort -from .utils.metadata.models import EngineInfo - -logger = logging.getLogger(__file__) - -_NO_GRAPH_ID = "notspecified" -"""Identifier of a workflow that does not have one.""" - - -def supported_managers() -> Dict[str, str]: - """Names of the supported package managers with an example command for each.""" - return { - name: manager_cls.COMMAND_EXAMPLE - for name, manager_cls in get_supported_managers().items() - } - - -def managers_supporting_in_place() -> Tuple[str, ...]: - """Names of the package managers that can install in the current environment.""" - return tuple( - name - for name, manager_cls in get_supported_managers().items() - if manager_cls.CAN_INSTALL_IN_PLACE - ) - - -def add_requirements( - graph: TaskGraph, - manager_name: Optional[str] = None, - manager_command: Tuple[str, ...] = tuple(), - engine: Optional[Tuple[str, str]] = None, -) -> str: - """Add requirements to a workflow definition in-place. Returns the name of the - package manager that generated them. `engine` is the name and version of the - distribution of the engine that saves the workflow. - """ - manager = get_manager(manager_name=manager_name, manager_command=manager_command) - requirements = manager.gather_requirements() - if requirements.ewoks and engine: - requirements.ewoks.engine = EngineInfo(name=engine[0], version=engine[1]) - graph.graph.graph["requirements"] = requirements.model_dump() - return manager.NAME - - -def get_requirements(graph: TaskGraph) -> BaseRequirements: - """Extract requirements from a workflow definition.""" - requirements = graph.graph.graph.get("requirements", None) - no_requirements = not requirements - - if no_requirements: - logger.warning( - "BaseRequirements field is empty. Trying to extract requirements automatically..." - ) - requirements = last_resort.last_resort_requirements(graph) - - requirements = parse.parse_requirements(requirements) - - if no_requirements: - logger.info(f"Extracted the following requirements: {requirements.__info__()}") - - return requirements - - -def environment_name(graph: TaskGraph) -> str: - """Name of the python environment of a workflow: its identifier, or a hash of - the workflow when it does not have one. - """ - name = str(graph.graph_id) - if name == _NO_GRAPH_ID: - name = str(uhash(graph.serialize()))[:16] - return re.sub(r"[^\w.-]", "_", name)[:64] or _NO_GRAPH_ID - - -def install_requirements( - requirements: BaseRequirements, - env_name: Optional[str] = None, - env_root: Optional[Union[str, Path]] = None, - manager_name: Optional[str] = None, - manager_command: Tuple[str, ...] = tuple(), - python_version: Optional[str] = None, - ensure_ewoks: bool = False, - clean: bool = False, - confirm: Optional[Callable[[str], None]] = None, -) -> Environment: - """Install workflow requirements in a python environment named `env_name`, - created inside `env_root` or inside the root directory of the package manager, - or in the current python environment when no name is provided. An environment - that already exists is installed in, or removed first when `clean`. - The files of the package manager are installed when the requirements provide - them, the installed python distributions otherwise or when that installation - failed. `ensure_ewoks` adds ewoks itself when the requirements do not contain it. - `confirm` is called with a description of an installation or a removal before - it starts. - - :raises ValueError: no environment name, the package manager cannot install in - the current environment or the environment to remove is not - a python environment - :raises RuntimeError: environment creation or installation failed - """ - if env_name is None: - if env_root: - raise ValueError("An environment root requires an environment name") - if clean: - raise ValueError("Cleaning requires an environment name") - - manager = get_installer( - requirements, manager_name=manager_name, manager_command=manager_command - ) - - if env_name is None: - if not manager.CAN_INSTALL_IN_PLACE: - if manager_name: - raise ValueError( - f"Package manager {manager_name!r} cannot install in the current " - "python environment" - ) - manager = get_in_place_installer() - location = None - remove = False - environment = Environment.current() - destination = "the current python environment" - else: - location = manager.environment_location(env_name, env_root) - if python_version is None: - python_version = requirements.python.version or None - environment = manager.environment(location) - remove = clean and location.exists() - if remove: - _assert_environment(location) - if location.exists() and not remove: - logger.warning( - "The python environment in '%s' already exists: the requirements " - "are installed in it", - location, - ) - destination = f"the existing python environment in '{location}'" - else: - destination = f"a new python environment in '{location}'" - - # The environment is created after the confirmation of the first installation - native = manager.is_native(requirements, environment) - if native: - _confirm( - confirm, - requirements.__files_info__(), - f"This will install the files of the {manager.NAME} package manager " - f"above in {destination}.", - ) - else: - logger.warning( - "The requirements do not provide files that the %r package manager can " - "install. The installed python distributions are installed instead.", - manager.NAME, - ) - _confirm( - confirm, - requirements.__distributions_info__(), - f"The requirements do not provide files that the {manager.NAME} package " - "manager can install. This will install the python distributions above " - f"in {destination} instead.", - ) - - if location is not None: - if remove: - if confirm is not None: - confirm( - f"This will remove the existing python environment in '{location}'." - ) - shutil.rmtree(location) - environment = manager.create_environment(location, python_version) - - installed = False - if native: - files_environment = manager.files_environment(requirements, environment) - try: - manager.install_files(requirements, files_environment) - environment = files_environment - installed = True - except Exception: - logger.exception( - "Continue with the installed python distributions after failure to " - "install the files of the %r package manager", - manager.NAME, - ) - _confirm( - confirm, - requirements.__distributions_info__(), - f"Installing the files of the {manager.NAME} package manager failed. " - "This will install the python distributions above in " - f"{destination} instead.", - ) - - if not installed: - manager.install_distributions(requirements, environment, python_version) - - if ensure_ewoks: - manager.ensure_ewoks(requirements, environment) - - return environment - - -def _assert_environment(location: Path) -> None: - """Only a python environment is removed to create a new one. - - :raises ValueError: the location is not a python environment - """ - try: - Environment.from_location(location) - except ValueError: - raise ValueError( - f"'{location}' is not a python environment: remove it yourself to " - "create an environment there" - ) from None - - -def _confirm( - confirm: Optional[Callable[[str], None]], content: str, action: str -) -> None: - """Describe an installation and let `confirm` accept or refuse it.""" - if confirm is None: - return - confirm(f"{content}\n\n{action}") - - -if __name__ == "__main__": - logging.basicConfig(level=logging.DEBUG) - import time - from pprint import pprint - - from .utils import requirements_txt - - t0 = time.perf_counter() - - try: - manager = get_manager(manager_name="pip-venv") - requirements = manager.gather_requirements() - - print() - print("Model:") - pprint(requirements.model_dump()) - finally: - print("Freeze time:", time.perf_counter() - t0) - - pip_freeze = requirements.manager.files[ - requirements_txt.REQUIREMENTS_FILENAME - ].splitlines() - - dists_freeze = requirements_txt.distributions_requirements( - requirements.distributions - ) - dists_freeze = [s for s in dists_freeze if not s.startswith("#")] - - print() - print("pip freeze has these extra's:") - pprint(set(pip_freeze) - set(dists_freeze)) - - print() - print("native freeze has these extra's:") - pprint(set(dists_freeze) - set(pip_freeze)) diff --git a/src/ewoks/_requirements/conda.py b/src/ewoks/_requirements/conda.py deleted file mode 100644 index da7b0cb..0000000 --- a/src/ewoks/_requirements/conda.py +++ /dev/null @@ -1,202 +0,0 @@ -import json -import logging -import os -import platform -import re -import shutil -import sys -from pathlib import Path -from typing import Dict -from typing import List -from typing import Literal -from typing import Mapping -from typing import Optional -from typing import Sequence -from typing import Tuple - -from pydantic import Field - -from .utils import conda_channel -from .utils import requirements_txt -from .utils.base_manager import BaseManager -from .utils.base_manager import BaseManagerInfo -from .utils.base_manager import BaseRequirements -from .utils.environment import Environment -from .utils.files import temporary_files -from .utils.metadata import models - -logger = logging.getLogger(__name__) - -_ENVIRONMENT_FILENAME = "environment.yml" - -_IMPLEMENTATIONS = ("micromamba", "mamba", "conda") - - -class CondaManagerInfo(BaseManagerInfo): - name: Literal["conda"] = Field( - default="conda", - description="Environments described and created by `conda`.", - examples=["conda"], - ) - - -class CondaRequirements(BaseRequirements): - manager: CondaManagerInfo - - -class CondaManager(BaseManager): - """Uses `conda env export` to describe an environment with an - `environment.yml` file and `conda env update` to reproduce it. Unlike the - python-only package managers this includes non-python dependencies. - - The command invokes conda. For example `CondaManager("micromamba")`. The - fastest implementation available is used when no command is provided. - """ - - NAME = "conda" - PRIORITY = 4 - REQUIREMENTS_MODEL = CondaRequirements - COMMAND_EXAMPLE = "conda" - - def __init__(self, *command: str) -> None: - if not command: - command = _detect_implementation() - super().__init__(*command) - - def version(self) -> Optional[str]: - """Returns None when this manager is not available.""" - try: - output = self._check_output("--version") - except RuntimeError: - return None - # For example "conda 25.11.1" or "2.4.0" for mamba - match = re.search(r"(\d+\.\d+(?:\.\d+)*)", output) - if not match: - return None - return match.group(1) - - def is_active(self) -> bool: - """Manager is explicitly active.""" - return "CONDA_PREFIX" in os.environ or _is_conda_environment(Path(sys.prefix)) - - def create_environment( - self, location: Path, python_version: Optional[str] = None - ) -> Environment: - """ - :raises RuntimeError: creation failed - """ - if not python_version: - python_version = platform.python_version() - # 'pip' is needed for the pip section of a conda environment file - self._check_call( - "create", - "--yes", - "--quiet", - "--prefix", - location, - f"python={conda_channel.python_specifier(python_version)}", - "pip", - ) - return self.environment(location) - - def environments_root(self) -> Path: - """First environment directory of conda, in which `conda create --name` - creates environments so they can be activated by name. - """ - try: - # 'conda config --show envs_dirs' is not supported by mamba - info = json.loads(self._check_output("info", "--json")) - # 'envs_dirs' for conda, 'envs directories' for mamba and micromamba - directories = info.get("envs_dirs") or info.get("envs directories") - return Path(directories[0]) - except (RuntimeError, ValueError, LookupError, TypeError) as ex: - logger.warning("Cannot determine where conda creates environments (%s)", ex) - return super().environments_root() - - def _gather_files( - self, distributions: List[models.Distribution], python_version: str - ) -> Dict[str, str]: - if not _is_conda_environment(Path(sys.prefix)): - logger.warning( - "The current python environment is not a conda environment. It will " - "be reproduced from the installed python distributions." - ) - return dict() - output = self._check_output("env", "export", "--prefix", sys.prefix) - return {_ENVIRONMENT_FILENAME: _without_environment_location(output)} - - def _files_from_distributions( - self, distributions: Sequence[models.Distribution], python_version: str - ) -> Dict[str, str]: - requirements = requirements_txt.manifest_requirements(distributions) - return { - _ENVIRONMENT_FILENAME: _environment_yml(requirements, python_version), - } - - def _install_files( - self, files: Mapping[str, str], environment: Environment - ) -> None: - with temporary_files(files) as directory: - self._check_call( - "env", - "update", - "--quiet", - "--prefix", - environment.location, - "--file", - directory / _ENVIRONMENT_FILENAME, - ) - - def _add_ewoks( - self, requirements: BaseRequirements, environment: Environment - ) -> None: - # The environment file pins all versions so they cannot change - pinned = requirements_txt.manifest_requirements(requirements.distributions) - files = { - _ENVIRONMENT_FILENAME: _environment_yml( - [*pinned, "ewoks"], requirements.python.version - ) - } - self._install_files(files, environment) - - -def _environment_yml(requirements: Sequence[str], python_version: str) -> str: - """Content of a conda `environment.yml` file. Python distributions are declared - in the `pip` section, which is how the conda environment file format expresses - requirements that come from the python package index. - """ - dependencies = "" - if python_version: - dependencies += f" - python={conda_channel.python_specifier(python_version)}\n" - dependencies += " - pip\n" - if requirements: - dependencies += " - pip:\n" - dependencies += "".join(f" - {req}\n" for req in requirements) - return f"""channels: - - conda-forge -dependencies: -{dependencies}""" - - -def _detect_implementation() -> Tuple[str, ...]: - """The command is needed before knowing whether conda is available, so this - does not execute anything.""" - for name in _IMPLEMENTATIONS: - if shutil.which(name): - return (name,) - return (_IMPLEMENTATIONS[-1],) - - -def _without_environment_location(environment_yml: str) -> str: - """Remove the name and the prefix of the exported environment: the location - of the environment to create is provided separately. - """ - return "".join( - f"{line}\n" - for line in environment_yml.splitlines() - if not line.startswith(("name:", "prefix:")) - ) - - -def _is_conda_environment(prefix: Path) -> bool: - return (prefix / "conda-meta").is_dir() diff --git a/src/ewoks/_requirements/pip_venv.py b/src/ewoks/_requirements/pip_venv.py deleted file mode 100644 index d231210..0000000 --- a/src/ewoks/_requirements/pip_venv.py +++ /dev/null @@ -1,147 +0,0 @@ -import importlib.metadata -import sys -from pathlib import Path -from typing import Dict -from typing import List -from typing import Literal -from typing import Mapping -from typing import Optional -from typing import Sequence - -from pydantic import Field - -from .utils import process -from .utils import requirements_txt -from .utils.base_manager import BaseManager -from .utils.base_manager import BaseManagerInfo -from .utils.base_manager import BaseRequirements -from .utils.environment import Environment -from .utils.environment import create_venv -from .utils.files import temporary_files -from .utils.metadata import models - -_CONSTRAINTS_FILENAME = "constraints.txt" - - -class PipVenvManagerInfo(BaseManagerInfo): - name: Literal["pip-venv"] = Field( - default="pip-venv", - description="Environments created with `venv`, requirements installed with `pip`.", - examples=["pip-venv"], - ) - - -class PipVenvRequirements(BaseRequirements): - manager: PipVenvManagerInfo - - -class PipVenvManager(BaseManager): - """Creates environments with the `venv` module and installs in them with the - `pip` module. - - The command is the python interpreter that creates environments. For example - `PipVenvManager("/path/to/python")`. The current python interpreter is used - when no command is provided. Unlike other package managers, `venv` cannot - provide a python version other than the one of this interpreter. - """ - - NAME = "pip-venv" - PRIORITY = 0 - REQUIREMENTS_MODEL = PipVenvRequirements - COMMAND_EXAMPLE = "python" - CAN_INSTALL_IN_PLACE = True - - def __init__(self, *command: str) -> None: - if not command: - command = (sys.executable,) - super().__init__(*command) - - def version(self) -> Optional[str]: - """Returns None when this manager is not available.""" - try: - return importlib.metadata.version("pip") - except importlib.metadata.PackageNotFoundError: - return None - - def is_active(self) -> bool: - """Manager is explicitly active.""" - return False - - @classmethod - def installed_distribution(cls, distribution: models.Distribution) -> bool: - """The distribution was installed by this package manager.""" - return "pip" in cls._installer(distribution) - - def create_environment( - self, location: Path, python_version: Optional[str] = None - ) -> Environment: - """ - :raises RuntimeError: creation failed - """ - environment = self.environment(location) - create_venv(self._cmd_args[0], environment.prefix, python_version) - return environment - - def _gather_files( - self, distributions: List[models.Distribution], python_version: str - ) -> Dict[str, str]: - freeze = process.check_output(sys.executable, "-m", "pip", "freeze") - return {requirements_txt.REQUIREMENTS_FILENAME: freeze} - - def _files_from_distributions( - self, distributions: Sequence[models.Distribution], python_version: str - ) -> Dict[str, str]: - requirements = requirements_txt.distributions_requirements(distributions) - return {requirements_txt.REQUIREMENTS_FILENAME: "\n".join(requirements)} - - def _install_files( - self, files: Mapping[str, str], environment: Environment - ) -> None: - """ - :raises ValueError: no requirements to install - :raises RuntimeError: installation failed - """ - requirements = files[requirements_txt.REQUIREMENTS_FILENAME].splitlines() - self._pip_install(requirements, environment) - - def _add_ewoks( - self, requirements: BaseRequirements, environment: Environment - ) -> None: - self._pip_install( - ["ewoks"], - environment, - constraints=requirements_txt.version_constraints( - requirements.distributions - ), - ) - - def _pip_install( - self, - requirements: Sequence[str], - environment: Environment, - constraints: Optional[Sequence[str]] = None, - ) -> None: - """ - :raises ValueError: no requirements to install - :raises RuntimeError: installation failed - """ - arguments = requirements_txt.sanitize(requirements) - if not arguments: - raise ValueError("No distributions provided to install") - - files = {requirements_txt.REQUIREMENTS_FILENAME: "\n".join(arguments)} - if constraints: - files[_CONSTRAINTS_FILENAME] = "\n".join(constraints) - - with temporary_files(files) as directory: - options = ["-r", directory / requirements_txt.REQUIREMENTS_FILENAME] - if constraints: - options += ["--constraint", directory / _CONSTRAINTS_FILENAME] - process.check_call( - environment.python, - "-m", - "pip", - "install", - "--no-cache-dir", - *options, - ) diff --git a/src/ewoks/_requirements/pixi.py b/src/ewoks/_requirements/pixi.py deleted file mode 100644 index efaeacc..0000000 --- a/src/ewoks/_requirements/pixi.py +++ /dev/null @@ -1,261 +0,0 @@ -import json -import logging -import os -import re -import sys -import tempfile -from pathlib import Path -from typing import Dict -from typing import List -from typing import Literal -from typing import Mapping -from typing import Optional -from typing import Sequence -from typing import Union - -from pydantic import Field - -from .utils import conda_channel -from .utils import requirements_txt -from .utils import toml_dependencies -from .utils.base_manager import BaseManager -from .utils.base_manager import BaseManagerInfo -from .utils.base_manager import BaseRequirements -from .utils.environment import Environment -from .utils.files import read_files -from .utils.files import temporary_files -from .utils.files import write_files -from .utils.metadata import models - -logger = logging.getLogger(__name__) - -_LOCK_FILENAME = "pixi.lock" -_MANIFEST_FILENAME = "pixi.toml" -_PYPROJECT_FILENAME = "pyproject.toml" -_WORKSPACE_NAME = "ewoks-workflow-requirements" - - -class PixiManagerInfo(BaseManagerInfo): - name: Literal["pixi"] = Field( - default="pixi", - description="Environments described and created by a `pixi` workspace.", - examples=["pixi"], - ) - pixi_environment_name: str = Field( - default="default", - description=( - "Name of the environment of the pixi workspace in which the " - "requirements were generated." - ), - examples=["default"], - ) - - -class PixiRequirements(BaseRequirements): - manager: PixiManagerInfo - - -class PixiManager(BaseManager): - """Uses a pixi workspace: `pixi lock` describes an environment with a - `pixi.toml` and a `pixi.lock` file, `pixi install` reproduces the environment - from those files. Unlike the python-only package managers this includes - non-python dependencies. - - When the current environment belongs to a pixi workspace, its manifest and - `pixi.lock` are stored as the manager files. Otherwise no files are stored and - the stored python distributions describe the environment. - - The command invokes pixi. For example `PixiManager("/path/to/pixi")`. - """ - - NAME = "pixi" - PRIORITY = 5 - REQUIREMENTS_MODEL = PixiRequirements - COMMAND_EXAMPLE = "pixi" - - def __init__(self, *command: str) -> None: - if not command: - command = ("pixi",) - super().__init__(*command) - - def _check_output( - self, - *args: Union[str, Path], - extra_env: Optional[Mapping[str, Optional[str]]] = None, - ) -> str: - return super()._check_output(*args, extra_env=_with_cache_dir(extra_env)) - - def _check_call( - self, - *args: Union[str, Path], - extra_env: Optional[Mapping[str, Optional[str]]] = None, - ) -> None: - super()._check_call(*args, extra_env=_with_cache_dir(extra_env)) - - def version(self) -> Optional[str]: - """Returns None when this manager is not available.""" - try: - output = self._check_output("--version") - except RuntimeError: - return None - # For example "pixi 0.63.2" - match = re.search(r"(\d+\.\d+(?:\.\d+)*)", output) - if not match: - return None - return match.group(1) - - def is_active(self) -> bool: - """Manager is explicitly active.""" - return ( - "PIXI_PROJECT_ROOT" in os.environ - or "PIXI_ENVIRONMENT_NAME" in os.environ - or _is_workspace_environment(Path(sys.prefix)) - ) - - def create_environment( - self, location: Path, python_version: Optional[str] = None - ) -> Environment: - """Environment that pixi creates when it installs the workspace.""" - return self.environment(location) - - def environment(self, location: Union[str, Path]) -> Environment: - """The default environment of the pixi workspace in `location`.""" - return _workspace_environment(location, "default") - - def files_environment( - self, requirements: BaseRequirements, environment: Environment - ) -> Environment: - if not isinstance(requirements, PixiRequirements): - return environment - name = requirements.manager.pixi_environment_name - return _workspace_environment(environment.location, name) - - def _gather_files( - self, distributions: List[models.Distribution], python_version: str - ) -> Dict[str, str]: - prefix = Path(sys.prefix) - if _is_workspace_environment(prefix): - # A pixi environment looks like /.pixi/envs/ - project = prefix.parents[2] - # pixi.toml takes precedence over pyproject.toml - project_files = (_MANIFEST_FILENAME, _PYPROJECT_FILENAME) - for project_file in project_files: - files = read_files(project, project_file, _LOCK_FILENAME) - if len(files) == 2: - return files - # No pixi workspace: no files - return super()._gather_files(distributions, python_version) - - def _gather_extra_fields(self) -> Dict[str, str]: - prefix = Path(sys.prefix) - if not _is_workspace_environment(prefix): - # The manager model provides the default environment - return dict() - # A pixi environment looks like /.pixi/envs/ - return {"pixi_environment_name": prefix.name} - - def _files_from_distributions( - self, distributions: Sequence[models.Distribution], python_version: str - ) -> Dict[str, str]: - """ - :raises RuntimeError: the requirements cannot be resolved - """ - requirements = requirements_txt.manifest_requirements(distributions) - files = {_MANIFEST_FILENAME: self._manifest(requirements, python_version)} - - with temporary_files(files) as directory: - self._check_call("lock", "--manifest-path", directory / _MANIFEST_FILENAME) - lock = read_files(directory, _LOCK_FILENAME) - - return {**files, **lock} - - def _install_files( - self, files: Mapping[str, str], environment: Environment - ) -> None: - write_files(environment.location, files) - self._install_workspace(environment.location, environment.prefix.name) - - def _add_ewoks( - self, requirements: BaseRequirements, environment: Environment - ) -> None: - # pixi keeps the locked versions of the workflow requirements, unless ewoks - # needs different versions of them. - self._check_call( - "add", - "--pypi", - "ewoks", - "--manifest-path", - environment.location, - "--no-install", - ) - self._install_workspace(environment.location, environment.prefix.name) - if not environment.distribution_version("ewoks"): - raise RuntimeError( - f"ewoks was not installed in the pixi environment " - f"'{environment.prefix.name}': add ewoks to one of its features" - ) - - def _install_workspace(self, location: Path, name: str) -> None: - self._check_call("install", "--manifest-path", location, "--environment", name) - - def _manifest(self, requirements: Sequence[str], python_version: str) -> str: - """Content of a `pixi.toml` file with python distributions as pypi - dependencies. - """ - pypi_dependencies = "".join( - f"{line}\n" for line in toml_dependencies.dependency_lines(requirements) - ) - python = conda_channel.python_specifier(python_version) - return f"""[workspace] -name = {json.dumps(_WORKSPACE_NAME)} -channels = ["conda-forge"] -platforms = [{json.dumps(self._platform())}] - -[dependencies] -python = {json.dumps(python)} - -[pypi-dependencies] -{pypi_dependencies}""" - - def _platform(self) -> str: - """Platform of the current environment, for example `linux-64`. - - :raises RuntimeError: platform unknown - """ - output = self._check_output("info", "--json") - try: - return json.loads(output)["platform"] - except (ValueError, KeyError): - raise RuntimeError("Cannot determine the pixi platform") from None - - -def _workspace_environment(location: Union[str, Path], name: str) -> Environment: - """Return the environment at `/.pixi/envs/`. - - The environment may not exist yet, so the python path is built instead of - searched for. - """ - location = Path(os.path.abspath(location)) - prefix = location / ".pixi" / "envs" / name - if sys.platform == "win32": - python = prefix / "python.exe" - else: - python = prefix / "bin" / "python" - return Environment(location=location, python=python) - - -def _is_workspace_environment(prefix: Path) -> bool: - """Environment is managed by a pixi workspace.""" - parts = prefix.parts - return len(parts) > 2 and parts[-3] == ".pixi" and parts[-2] == "envs" - - -def _with_cache_dir( - extra_env: Optional[Mapping[str, Optional[str]]], -) -> Dict[str, Optional[str]]: - env = dict(extra_env) if extra_env else dict() - if not os.environ.get("PIXI_CACHE_DIR") and not os.environ.get("RATTLER_CACHE_DIR"): - env.setdefault( - "PIXI_CACHE_DIR", str(Path(tempfile.gettempdir()) / "pixi-cache") - ) - return env diff --git a/src/ewoks/_requirements/poetry.py b/src/ewoks/_requirements/poetry.py deleted file mode 100644 index 3ca1f35..0000000 --- a/src/ewoks/_requirements/poetry.py +++ /dev/null @@ -1,255 +0,0 @@ -import json -import logging -import os -import platform -import re -import shutil -import sys -from pathlib import Path -from typing import Dict -from typing import List -from typing import Literal -from typing import Mapping -from typing import Optional -from typing import Sequence -from typing import Tuple -from typing import Union - -from pydantic import Field - -from .utils import requirements_txt -from .utils import toml_dependencies -from .utils.base_manager import BaseManager -from .utils.base_manager import BaseManagerInfo -from .utils.base_manager import BaseRequirements -from .utils.environment import Environment -from .utils.environment import interpreter_version -from .utils.files import read_files -from .utils.files import temporary_files -from .utils.files import write_files -from .utils.metadata import models - -logger = logging.getLogger(__name__) - -_MINIMUM_VERSION = (1, 8) - -_PYPROJECT_FILENAME = "pyproject.toml" -_LOCK_FILENAME = "poetry.lock" - - -class PoetryManagerInfo(BaseManagerInfo): - name: Literal["poetry"] = Field( - default="poetry", - description="Environments described and created by a `poetry` project.", - examples=["poetry"], - ) - - -class PoetryRequirements(BaseRequirements): - manager: PoetryManagerInfo - - -class PoetryManager(BaseManager): - """Uses a poetry project: `poetry lock` describes an environment with a - `pyproject.toml` and a `poetry.lock` file, `poetry install` reproduces the - environment from those files. - - When the current environment is the `.venv` of a poetry project, its - `pyproject.toml` and `poetry.lock` are stored as the manager files. Otherwise - no files are stored and the stored python distributions describe the - environment. - - Environments are created with `poetry env use`, which needs an interpreter of - the requested python version because poetry cannot install one. - - The command invokes poetry. For example `PoetryManager("/path/to/poetry")`. - """ - - NAME = "poetry" - PRIORITY = 2 - REQUIREMENTS_MODEL = PoetryRequirements - COMMAND_EXAMPLE = "poetry" - ENVIRONMENT_SUBDIR = ".venv" - - def __init__(self, *command: str) -> None: - if not command: - command = ("poetry",) - super().__init__(*command) - - def version(self) -> Optional[str]: - """Returns None when this manager is not available.""" - try: - output = self._check_output( - "--version", extra_env=_python_env(sys.executable) - ) - except RuntimeError: - return None - # For example "Poetry (version 2.4.1)" - match = re.search(r"(\d+\.\d+(?:\.\d+)*)", output) - if not match: - return None - version = match.group(1) - if _version_tuple(version) < _MINIMUM_VERSION: - # Older versions have no project without a package to build - logger.warning( - "Poetry %s is not supported: version %s or later is required", - version, - ".".join(str(part) for part in _MINIMUM_VERSION), - ) - return None - return version - - def is_active(self) -> bool: - """Manager is explicitly active.""" - return "POETRY_ACTIVE" in os.environ or _is_project(Path(sys.prefix).parent) - - def create_environment( - self, location: Path, python_version: Optional[str] = None - ) -> Environment: - """ - :raises RuntimeError: creation failed - """ - # The project that poetry attaches the environment to. Its python version - # is not constrained yet: the requirements provide it when they are installed. - write_files(location, {_PYPROJECT_FILENAME: _pyproject([], "")}) - self._check_call( - "-C", - location, - "env", - "use", - _base_python(python_version), - extra_env=_python_env(sys.executable), - ) - return self.environment(location) - - def environments_root(self) -> Path: - """Directory in which poetry creates the environments of projects that do - not contain them (`virtualenvs.path`). - """ - try: - root = Path( - self._check_output( - "config", "virtualenvs.path", extra_env=_python_env(sys.executable) - ).strip() - ) - if not root.is_absolute(): - raise RuntimeError(f"{str(root)!r} is not a directory") - return root - except RuntimeError as ex: - logger.warning( - "Cannot determine where poetry creates environments (%s)", ex - ) - return super().environments_root() - - def _gather_files( - self, distributions: List[models.Distribution], python_version: str - ) -> Dict[str, str]: - project = Path(sys.prefix).parent - files = read_files(project, _PYPROJECT_FILENAME, _LOCK_FILENAME) - if len(files) == 2: - return files - # No poetry project: no files - return super()._gather_files(distributions, python_version) - - def _files_from_distributions( - self, distributions: Sequence[models.Distribution], python_version: str - ) -> Dict[str, str]: - """ - :raises RuntimeError: the requirements cannot be resolved - """ - pyproject = _pyproject( - requirements_txt.manifest_requirements(distributions), python_version - ) - files = {_PYPROJECT_FILENAME: pyproject} - - with temporary_files(files) as directory: - self._check_call( - "-C", directory, "lock", extra_env=_python_env(sys.executable) - ) - lock = read_files(directory, _LOCK_FILENAME) - - return {**files, **lock} - - def _install_files( - self, files: Mapping[str, str], environment: Environment - ) -> None: - write_files(environment.location, files) - self._check_call( - "-C", - environment.location, - "install", - "--no-root", - extra_env=_python_env(environment.python), - ) - - def _add_ewoks( - self, requirements: BaseRequirements, environment: Environment - ) -> None: - # The project pins all versions so they cannot change - self._check_call( - "-C", - environment.location, - "add", - "ewoks", - extra_env=_python_env(environment.python), - ) - - -def _pyproject(requirements: Sequence[str], python_version: str) -> str: - """Content of a `pyproject.toml` file for a project that is not a package. - - The dependencies are declared in the `tool.poetry` section instead of a PEP 621 - `project` section, which poetry only understands since version 2. - """ - python = f"=={python_version}" if python_version else "*" - dependencies = "".join( - f"{line}\n" for line in toml_dependencies.dependency_lines(requirements) - ) - return f"""[tool.poetry] -package-mode = false - -[tool.poetry.dependencies] -python = {json.dumps(python)} -{dependencies}""" - - -def _base_python(python_version: Optional[str]) -> Path: - """Interpreter poetry creates an environment with.""" - if not python_version or python_version == platform.python_version(): - return Path(sys.executable) - major_minor = ".".join(python_version.split(".")[:2]) - python = Path(shutil.which(f"python{major_minor}") or sys.executable) - version = interpreter_version(python) - if version != python_version: - logger.warning( - "poetry cannot provide python %s: creating the environment with " - "python %s instead", - python_version, - version, - ) - return python - - -def _version_tuple(version: str) -> Tuple[int, ...]: - return tuple(int(part) for part in version.split(".") if part.isdigit()) - - -def _python_env(python: Union[str, Path]) -> Dict[str, Optional[str]]: - """Poetry searches `PATH` for a python interpreter, which is not necessarily - a working one. Make sure it finds this interpreter first. The environment of - a project is the `.venv` directory inside it. - """ - path = os.environ.get("PATH", "") - return { - "PATH": os.pathsep.join([str(Path(python).parent), path]), - "POETRY_VIRTUALENVS_IN_PROJECT": "true", - # Poetry uses an active virtual or conda environment, for example the one - # of `poetry run`, instead of the environment of the project - "VIRTUAL_ENV": None, - "CONDA_PREFIX": None, - } - - -def _is_project(directory: Path) -> bool: - """Directory is a poetry project.""" - return (directory / _LOCK_FILENAME).is_file() diff --git a/src/ewoks/_requirements/utils/__init__.py b/src/ewoks/_requirements/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/ewoks/_requirements/utils/_supported.py b/src/ewoks/_requirements/utils/_supported.py deleted file mode 100644 index 11fc6e6..0000000 --- a/src/ewoks/_requirements/utils/_supported.py +++ /dev/null @@ -1,22 +0,0 @@ -from functools import lru_cache -from typing import Dict -from typing import Type - -from ..conda import CondaManager -from ..pip_venv import PipVenvManager -from ..pixi import PixiManager -from ..poetry import PoetryManager -from ..uv import UvManager -from .base_manager import BaseManager - - -@lru_cache(1) -def get_supported_managers() -> Dict[str, Type[BaseManager]]: - managers = [ - PipVenvManager, - UvManager, - PoetryManager, - CondaManager, - PixiManager, - ] - return {manager_cls.NAME: manager_cls for manager_cls in managers} diff --git a/src/ewoks/_requirements/utils/base_manager.py b/src/ewoks/_requirements/utils/base_manager.py deleted file mode 100644 index bd6df96..0000000 --- a/src/ewoks/_requirements/utils/base_manager.py +++ /dev/null @@ -1,384 +0,0 @@ -import importlib.metadata -import logging -import platform -from abc import abstractmethod -from pathlib import Path -from textwrap import indent -from typing import Dict -from typing import List -from typing import Mapping -from typing import Optional -from typing import Sequence -from typing import Union - -from pydantic import Field -from pydantic import model_serializer - -from . import process -from .environment import Environment -from .metadata import models -from .metadata.from_python import current_requirements - -logger = logging.getLogger(__name__) - -EWOKS_ENVIRONMENTS_ROOT = Path("~", ".ewoks", "envs") -"""Root directory of named environments for package managers that do not create -them in a directory of their own.""" - - -class BaseManagerInfo(models.BaseModel): - name: str = Field( - description="Package manager that generated the requirements.", - examples=["pip-venv", "uv", "poetry", "conda", "pixi"], - ) - version: str = Field( - description="Version of the package manager.", examples=["25.0.1"] - ) - files: Dict[str, str] = Field( - default_factory=dict, - description=( - "Content of the files the package manager needs to reproduce the " - "environment: 'requirements.txt' for pip-venv, 'pyproject.toml' and " - "'uv.lock' for uv, 'pyproject.toml' and 'poetry.lock' for poetry, " - "'environment.yml' for conda, 'pixi.toml' and 'pixi.lock' for pixi. " - "Empty when the package manager could not generate them." - ), - examples=[{"requirements.txt": "ewoks==7.0.0\nnetworkx==3.4.2\n"}], - ) - - @model_serializer(mode="wrap") - def _files_last(self, handler): - """The files are long: write them after the other manager fields.""" - data = handler(self) - if "files" in data: - # Removing and adding "files" again moves it to the end of the dict - data["files"] = data.pop("files") - return data - - -class BaseRequirements(models.BaseModel): - ewoks: Optional[models.EwoksInfo] = Field( - default=None, - description=( - "Ewoks that generated the requirements. Not set for requirements that " - "ewoks did not generate." - ), - ) - system: models.SystemInfo = Field( - description="Operating system on which the requirements were generated." - ) - python: models.PythonInfo = Field( - description="Python interpreter for which the requirements were generated." - ) - distributions: List[models.Distribution] = Field( - description=( - "Installed python distributions. Any package manager can reproduce the " - "environment from this list, also one that does not understand the " - "files of the package manager that generated the requirements." - ), - examples=[[{"name": "networkx", "version": "3.4.2"}]], - ) - manager: BaseManagerInfo = Field( - description="Package manager that generated the requirements." - ) - - def __info__(self) -> str: - return f"{self.__files_info__()}\n{self.__distributions_info__()}" - - def __files_info__(self) -> str: - """The package manager with the content of the files it needs to reproduce - the environment.""" - info = ( - f"Manager: {self.manager.name} ({self.manager.version}) " - f"python={self.python.version} " - f"distributions={len(self.distributions)}" - ) - for filename in sorted(self.manager.files): - content = indent(self.manager.files[filename].strip(), " ") - info = f"{info}\n\n{filename}:\n{content}" - return info - - def __distributions_info__(self) -> str: - """The installed python distributions.""" - distributions = "\n ".join( - f"{dist.name}=={dist.version}" for dist in self.distributions - ) - return f"Distributions:\n {distributions}" - - -class BaseManager: - """Defines the interface all package managers must implement. - - If `MyManager` is an implementation of this interface then the requirements - of the current python environment can be obtained like this: - - .. code-block:: python - - manager = MyManager() - requirements = manager.gather_requirements() - - Those requirements can be reproduced in another python environment like this: - - .. code-block:: python - - manager = MyManager() - environment = manager.create_environment(Path("/path/to/environment")) - if manager.is_native(requirements, environment): - manager.install_files(requirements, environment) - else: - manager.install_distributions(requirements, environment) - - An implementation provides `version`, `is_active`, `create_environment`, - `_files_from_distributions`, `_install_files` and `_add_ewoks`. Package managers - that can inspect a python environment also override `_gather_files`. Package - managers with extra fields in their manager model also override - `_gather_extra_fields`. Package managers that keep named environments in a - directory of their own also override `environments_root`. Package managers that - are not named after the tool that installs the distributions also override - `installed_distribution`. - """ - - NAME = NotImplemented - PRIORITY = NotImplemented - REQUIREMENTS_MODEL = NotImplemented - COMMAND_EXAMPLE = NotImplemented # example of an associated shell command - ENVIRONMENT_SUBDIR = "" # environment prefix relative to the location - CAN_INSTALL_IN_PLACE = False # can install in the current environment - - def __init__(self, *command: str) -> None: - if not command: - raise ValueError(f"{type(self).__name__} needs an associated shell command") - self._cmd_args = command - - def __repr__(self): - return f"{type(self).__name__}({', '.join(self._cmd_args)})" - - @abstractmethod - def version(self) -> Optional[str]: - """Returns None when this manager is not available.""" - pass - - @abstractmethod - def is_active(self) -> bool: - """Manager is explicitly active.""" - pass - - @classmethod - def installed_distribution(cls, distribution: models.Distribution) -> bool: - """The distribution was installed by this package manager.""" - return cls.NAME in cls._installer(distribution) - - @staticmethod - def _installer(distribution: models.Distribution) -> str: - """Tool that installed the distribution. It can contain more than the name - of the tool: poetry writes "Poetry 1.8.5". - """ - return (distribution.installer or "").lower() - - def gather_requirements(self) -> BaseRequirements: - """ - Return requirements associated to the current python environment. - - :raises RuntimeError: package manager not available - """ - manager_version = self.version() - if manager_version is None: - raise RuntimeError(f"{self.NAME!r} is not available") - - metadata = current_requirements() - files = self._gather_files( - metadata["distributions"], metadata["python"]["version"] - ) - - manager = dict( - name=self.NAME, - version=manager_version, - files=files, - **self._gather_extra_fields(), - ) - ewoks = dict(version=importlib.metadata.version("ewoks")) - return self.REQUIREMENTS_MODEL(ewoks=ewoks, manager=manager, **metadata) - - def environment(self, location: Union[str, Path]) -> Environment: - """Environment the manager creates at this location.""" - return Environment.at_location(location, self.ENVIRONMENT_SUBDIR) - - def files_environment( - self, requirements: BaseRequirements, environment: Environment - ) -> Environment: - """Environment in which the files of the requirements are installed. - Package managers whose files describe several environments override this. - """ - return environment - - def environment_location( - self, name: str, root: Optional[Union[str, Path]] = None - ) -> Path: - """Location of the environment with this name, inside a root directory of - the user or inside the root directory of this package manager. - """ - return Path(root or self.environments_root()).expanduser() / name - - def environments_root(self) -> Path: - f"""Root directory in which the package manager creates named environments. - - Package managers venv, uv and pixi allow the environment location to be - specified. For those managers the default location for environments created - by `ewoks` is `{EWOKS_ENVIRONMENTS_ROOT}`. - """ - return EWOKS_ENVIRONMENTS_ROOT - - @abstractmethod - def create_environment( - self, location: Path, python_version: Optional[str] = None - ) -> Environment: - """Create an empty python environment. - - :raises RuntimeError: creation failed - """ - pass - - def is_native( - self, requirements: BaseRequirements, environment: Environment - ) -> bool: - """The files of the requirements are the files of this package manager and - the environment has the layout this package manager creates. - """ - return ( - isinstance(requirements, self.REQUIREMENTS_MODEL) - and bool(requirements.manager.files) - and self._owns(environment) - ) - - def install_files( - self, requirements: BaseRequirements, environment: Environment - ) -> None: - """Install the files of the requirements in an existing environment. The - requirements must be native (see `is_native`). - - :raises RuntimeError: installation failed - """ - try: - self._install_files(requirements.manager.files, environment) - except Exception as ex: - logger.error( - "%s: failed to install the requirement files (%s)", - type(self).__name__, - ex, - ) - raise - - def install_distributions( - self, - requirements: BaseRequirements, - environment: Environment, - python_version: Optional[str] = None, - ) -> None: - """Install the python distributions of the requirements in an environment. - `python_version` is used when neither the requirements nor the environment - provide a python version, and the version of the running python when - `python_version` is not given. - - :raises ValueError: no distributions provided to install - :raises RuntimeError: installation failed - """ - if not requirements.distributions: - raise ValueError("No distributions provided to install") - # Requirements that do not specify a python version are described for the - # python version of the environment they are installed in. - if requirements.python.version: - python_version = requirements.python.version - elif environment.exists(): - python_version = environment.python_version() - elif not python_version: - python_version = platform.python_version() - try: - files = self._files_from_distributions( - requirements.distributions, python_version - ) - self._install_files(files, environment) - except Exception as ex: - logger.error( - "%s: failed to install the python distributions (%s)", - type(self).__name__, - ex, - ) - raise - - def ensure_ewoks( - self, requirements: BaseRequirements, environment: Environment - ) -> None: - """Install ewoks in an existing environment without changing the versions - of the requirements. - - :raises RuntimeError: installation failed - """ - if environment.distribution_version("ewoks"): - return - logger.info("Add ewoks to %s", environment.location) - self._add_ewoks(requirements, environment) - - def _owns(self, environment: Environment) -> bool: - """The environment has the layout this package manager creates.""" - return environment.python == self.environment(environment.location).python - - def _gather_files( - self, distributions: List[models.Distribution], python_version: str - ) -> Dict[str, str]: - """Files that describe the current python environment. Package managers - that can inspect an environment override this. Without files, the stored - python distributions describe the environment. - """ - return dict() - - def _gather_extra_fields(self) -> Dict[str, str]: - """Values of the fields a package manager adds to its manager model, - besides name, version and files. Package managers that add fields - override this. - """ - return dict() - - @abstractmethod - def _files_from_distributions( - self, distributions: Sequence[models.Distribution], python_version: str - ) -> Dict[str, str]: - """Files the package manager needs to install these python distributions. - - :raises RuntimeError: the files cannot be generated - """ - pass - - @abstractmethod - def _install_files( - self, files: Mapping[str, str], environment: Environment - ) -> None: - """Install the files of the package manager in an existing environment. - - :raises RuntimeError: installation failed - """ - pass - - @abstractmethod - def _add_ewoks( - self, requirements: BaseRequirements, environment: Environment - ) -> None: - """Install ewoks in an existing environment without changing the versions - of the requirements. - - :raises RuntimeError: installation failed - """ - pass - - def _check_output( - self, - *args: Union[str, Path], - extra_env: Optional[Mapping[str, Optional[str]]] = None, - ) -> str: - return process.check_output(*self._cmd_args, *args, extra_env=extra_env) - - def _check_call( - self, - *args: Union[str, Path], - extra_env: Optional[Mapping[str, Optional[str]]] = None, - ) -> None: - process.check_call(*self._cmd_args, *args, extra_env=extra_env) diff --git a/src/ewoks/_requirements/utils/conda_channel.py b/src/ewoks/_requirements/utils/conda_channel.py deleted file mode 100644 index fa873d6..0000000 --- a/src/ewoks/_requirements/utils/conda_channel.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Requirements of the package managers that install from conda channels.""" - - -def python_specifier(python_version: str) -> str: - """Conda match specification for a python version. - - Only the major and minor version are pinned because a conda channel does not - build every patch release of python (for example none after end-of-life). - """ - if not python_version: - return "*" - major_minor = ".".join(python_version.split(".")[:2]) - return f"{major_minor}.*" diff --git a/src/ewoks/_requirements/utils/detect.py b/src/ewoks/_requirements/utils/detect.py deleted file mode 100644 index b6516e2..0000000 --- a/src/ewoks/_requirements/utils/detect.py +++ /dev/null @@ -1,167 +0,0 @@ -import logging -from collections import Counter -from functools import lru_cache -from typing import Dict -from typing import Optional -from typing import Tuple - -from ._supported import get_supported_managers -from .base_manager import BaseManager -from .base_manager import BaseRequirements -from .metadata.from_python import current_requirements - -logger = logging.getLogger(__name__) - - -def get_manager( - manager_name: Optional[str] = None, - manager_command: Tuple[str, ...] = tuple(), -) -> BaseManager: - """Package manager that describes the current python environment. - - :raise ValueError: package manager not supported or not available - :raise RuntimeError: no package manager available - """ - if manager_name: - return _select_manager(manager_name, manager_command) - - if manager_command: - raise ValueError( - f"Provide 'manager_name' associated to command {manager_command}" - ) - - manager = _detect_manager() - - if manager is None: - raise RuntimeError("No known package manager installed or available") - - return manager - - -def get_installer( - requirements: BaseRequirements, - manager_name: Optional[str] = None, - manager_command: Tuple[str, ...] = tuple(), -) -> BaseManager: - """Package manager that installs requirements. The package manager that - generated the requirements is used when available. - - :raise ValueError: package manager not supported or not available - :raise RuntimeError: no package manager available - """ - if manager_name: - return _select_manager(manager_name, manager_command) - - if manager_command: - raise ValueError( - f"Provide 'manager_name' associated to command {manager_command}" - ) - - try: - return _select_manager(requirements.manager.name, tuple()) - except ValueError: - logger.debug( - "Package manager %r that generated the requirements is not available", - requirements.manager.name, - ) - - managers = _managers() - scores = {name: (manager.PRIORITY,) for name, manager in managers.items()} - manager = _first_available(managers, scores) - if manager is None: - raise RuntimeError("No known package manager installed or available") - - return manager - - -def get_in_place_installer() -> BaseManager: - """Package manager that can install in the current python environment. - - :raise RuntimeError: no package manager available - """ - managers = { - name: manager - for name, manager in _managers().items() - if manager.CAN_INSTALL_IN_PLACE - } - scores = {name: (manager.PRIORITY,) for name, manager in managers.items()} - manager = _first_available(managers, scores) - if manager is None: - raise RuntimeError( - "No package manager available that can install in the current python " - "environment" - ) - - return manager - - -def _select_manager(manager_name: str, manager_command: Tuple[str, ...]) -> BaseManager: - managers = get_supported_managers() - - manager_cls = managers.get(manager_name) - if manager_cls is None: - raise ValueError(f"Package manager {manager_name!r} is not supported") - - manager = manager_cls(*manager_command) - if manager.version() is None: - raise ValueError(f"Package manager {manager_name!r} is not available") - - return manager - - -def _managers() -> Dict[str, BaseManager]: - return { - name: manager_cls() for name, manager_cls in get_supported_managers().items() - } - - -def _first_available( - managers: Dict[str, BaseManager], scores: Dict[str, Tuple[int, ...]] -) -> Optional[BaseManager]: - """Available package manager with the highest score. Availability is checked - in score order because it may be an expensive check. - """ - for name in sorted(scores, key=lambda name: scores[name], reverse=True): - manager = managers[name] - if manager.version(): - logger.debug("Package manager %r selected\n scores = %s", name, scores) - return manager - return None - - -def _detect_manager() -> Optional[BaseManager]: - managers = _managers() - - # Select the active manager with the highest priority - active_managers = { - name: manager for name, manager in managers.items() if manager.is_active() - } - if active_managers: - scores = { - name: (manager.PRIORITY,) for name, manager in active_managers.items() - } - manager = _first_available(active_managers, scores) - if manager is not None: - return manager - - # Infer most likely package manager - counts = _manager_distribution_count() - scores = { - name: (counts.get(name, -1), manager.PRIORITY) - for name, manager in managers.items() - } - return _first_available(managers, scores) - - -@lru_cache(1) -def _manager_distribution_count() -> Dict[str, int]: - """Number of installed python distributions per package manager. Distributions - installed by a tool that is not a supported package manager are not counted. - """ - managers = get_supported_managers() - counts: Counter = Counter() - for dist in current_requirements()["distributions"]: - for name, manager_cls in managers.items(): - if manager_cls.installed_distribution(dist): - counts[name] += 1 - return dict(counts) diff --git a/src/ewoks/_requirements/utils/environment.py b/src/ewoks/_requirements/utils/environment.py deleted file mode 100644 index 54b5f01..0000000 --- a/src/ewoks/_requirements/utils/environment.py +++ /dev/null @@ -1,180 +0,0 @@ -import logging -import os -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Dict -from typing import Optional -from typing import Tuple -from typing import Union - -from . import process - -logger = logging.getLogger(__name__) - -ENVIRONMENT_SUBDIRS: Tuple[str, ...] = ( - "", # environment prefix, for example created by `python -m venv` - ".venv", # uv and poetry project -) - - -@dataclass(frozen=True) -class Environment: - """Python environment a package manager creates, installs into and runs.""" - - location: Path - """Directory provided by the user: an environment prefix or a project.""" - - python: Path - """Python interpreter of the environment.""" - - @classmethod - def current(cls) -> "Environment": - """Environment of the running python interpreter.""" - return cls(location=Path(sys.prefix), python=Path(sys.executable)) - - @classmethod - def at_location(cls, location: Union[str, Path], subdir: str = "") -> "Environment": - """Environment with a prefix at a fixed position inside the location.""" - location = Path(os.path.abspath(location)) - prefix = location / subdir if subdir else location - return cls(location=location, python=python_executable(prefix)) - - @classmethod - def from_location(cls, location: Union[str, Path]) -> "Environment": - """Environment found inside the location. A pixi workspace provides its only - environment, or its default environment when it has several. - - :raises ValueError: no python interpreter found or several pixi environments - without a default one - """ - for subdir in ENVIRONMENT_SUBDIRS: - environment = cls.at_location(location, subdir) - if environment.python.is_file(): - return environment - - environments = _pixi_environments(location) - if len(environments) == 1: - return list(environments.values())[0] - if environments: - env_message = ( - f"Several pixi environments found in '{location}': " - f"{', '.join(environments)}" - ) - # Example of a path to pass, other than the default environment - named_env_paths = [ - environment.prefix - for name, environment in environments.items() - if name != "default" - ] - example = f"for example '--env {named_env_paths[0]}'" - if "default" in environments: - logger.warning( - f"{env_message}\nUsing the default environment. To use another " - f"one, rerun with its path in --env, {example}" - ) - return environments["default"] - raise ValueError( - f"{env_message}\nRerun with the path of one of them in --env, {example}" - ) - - raise ValueError(f"No python environment found in '{location}'") - - @property - def prefix(self) -> Path: - """Environment prefix, which is the location itself unless the package - manager creates the environment inside the location. - """ - directory = self.python.parent - if directory.name in ("bin", "Scripts"): - return directory.parent - return directory - - def exists(self) -> bool: - return self.python.is_file() - - def python_version(self) -> str: - """ - :raises RuntimeError: environment has no python interpreter - """ - return interpreter_version(self.python) - - def distribution_version(self, name: str) -> Optional[str]: - """Returns None when the distribution is not installed. - - :raises RuntimeError: environment has no python interpreter - """ - code = ( - "import importlib.metadata as m\n" - f"try: print(m.version({name!r}))\n" - "except m.PackageNotFoundError: pass" - ) - return self._python_output(code) or None - - def _python_output(self, code: str) -> str: - return process.check_output(self.python, "-c", code).strip() - - -def _pixi_environments(location: Union[str, Path]) -> Dict[str, Environment]: - """Environments of a pixi workspace by name, which look like - /.pixi/envs/. - """ - environments = {} - for prefix in sorted((Path(location) / ".pixi" / "envs").glob("*")): - subdir = os.path.join(".pixi", "envs", prefix.name) - environment = Environment.at_location(location, subdir) - if environment.python.is_file(): - environments[prefix.name] = environment - return environments - - -def python_executable(prefix: Path) -> Path: - """Python interpreter of an environment prefix. The interpreter of a virtual - environment is returned when the environment does not exist yet. - """ - relative_paths = _interpreters() - for relative in relative_paths: - python = prefix / relative - if python.is_file(): - return python - return prefix / relative_paths[0] - - -def _interpreters() -> Tuple[Path, ...]: - """Interpreter locations inside an environment prefix, the first one being where - a virtual environment has it. On Windows the other environments (a python - installation, conda and pixi) have it in the prefix itself. - """ - if sys.platform == "win32": - return (Path("Scripts", "python.exe"), Path("python.exe")) - return (Path("bin", "python"),) - - -def interpreter_version(python: Union[str, Path]) -> str: - """Version of a python interpreter. - - :raises RuntimeError: not a python interpreter - """ - return process.check_output( - python, "-c", "import platform; print(platform.python_version())" - ).strip() - - -def create_venv( - base_python: Union[str, Path], prefix: Path, python_version: Optional[str] = None -) -> None: - """Create a virtual environment with the `venv` module, which cannot provide - another python version than the one of the interpreter that creates it. - - :raises RuntimeError: creation failed - """ - if python_version: - base_version = interpreter_version(base_python) - if python_version != base_version: - logger.warning( - "'venv' cannot provide python %s: creating the environment with " - "python %s instead", - python_version, - base_version, - ) - process.check_call(base_python, "-m", "venv", prefix) diff --git a/src/ewoks/_requirements/utils/files.py b/src/ewoks/_requirements/utils/files.py deleted file mode 100644 index 172a440..0000000 --- a/src/ewoks/_requirements/utils/files.py +++ /dev/null @@ -1,34 +0,0 @@ -import tempfile -from contextlib import contextmanager -from pathlib import Path -from typing import Dict -from typing import Generator -from typing import Mapping - - -def write_files(directory: Path, files: Mapping[str, str]) -> None: - """Write files, relative to a directory that is created when missing.""" - for name, content in files.items(): - filename = directory / name - filename.parent.mkdir(parents=True, exist_ok=True) - filename.write_text(content, encoding="utf-8") - - -def read_files(directory: Path, *names: str) -> Dict[str, str]: - """Read files, relative to a directory, skipping the ones that do not exist.""" - files = {} - for name in names: - try: - files[name] = (directory / name).read_text(encoding="utf-8") - except OSError: - continue - return files - - -@contextmanager -def temporary_files(files: Mapping[str, str]) -> Generator[Path, None, None]: - """Write files in a temporary directory which is yielded.""" - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) - write_files(path, files) - yield path diff --git a/src/ewoks/_requirements/utils/metadata/__init__.py b/src/ewoks/_requirements/utils/metadata/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/ewoks/_requirements/utils/metadata/_unknown.py b/src/ewoks/_requirements/utils/metadata/_unknown.py deleted file mode 100644 index 892da64..0000000 --- a/src/ewoks/_requirements/utils/metadata/_unknown.py +++ /dev/null @@ -1,31 +0,0 @@ -from functools import lru_cache -from typing import Any -from typing import Dict - - -@lru_cache(1) -def unknown_requirements() -> Dict[str, Any]: - return dict( - system=_unknown_system_metadata(), - python=_unknown_python_metadata(), - distributions=list(), - ) - - -def _unknown_system_metadata() -> Dict[str, Any]: - return dict( - system="", - release="", - version="", - machine="", - processor="", - ) - - -def _unknown_python_metadata() -> Dict[str, Any]: - return dict( - version="", - implementation="", - compiler="", - build="", - ) diff --git a/src/ewoks/_requirements/utils/metadata/from_python/__init__.py b/src/ewoks/_requirements/utils/metadata/from_python/__init__.py deleted file mode 100644 index faaf394..0000000 --- a/src/ewoks/_requirements/utils/metadata/from_python/__init__.py +++ /dev/null @@ -1,117 +0,0 @@ -import json -import logging -import platform -from functools import lru_cache -from importlib import metadata -from pathlib import Path -from typing import Any -from typing import Dict - -from .. import models -from . import _git - -logger = logging.getLogger(__name__) - - -@lru_cache(1) -def current_requirements() -> Dict[str, Any]: - """ - Return installed python distributions. - """ - distributions = [ - _distribution_from_metadata(dist) for dist in metadata.distributions() - ] - return dict( - system=_system_metadata(), - python=_python_metadata(), - distributions=distributions, - ) - - -def _system_metadata() -> Dict[str, Any]: - uname = platform.uname() - return dict( - system=uname.system, - release=uname.release, - version=uname.version, - machine=uname.machine, - processor=uname.processor, - ) - - -def _python_metadata() -> Dict[str, Any]: - return dict( - version=platform.python_version(), - implementation=platform.python_implementation(), - compiler=platform.python_compiler(), - build=", ".join(platform.python_build()), - ) - - -def _distribution_from_metadata( - dist: metadata.Distribution, -) -> models.Distribution: - name = dist.metadata["Name"] - version = dist.version - - try: - installer = dist.read_text("INSTALLER") or "" - installer = installer.strip() - if not installer: - installer = None - except Exception: - installer = None - - # PEP 610: Direct URL Origin of installed distributions - # https://packaging.python.org/en/latest/specifications/direct-url-data-structure/ - git_info = None - archive_info = None - try: - direct_url_text = dist.read_text("direct_url.json") - direct_url = json.loads(direct_url_text) if direct_url_text else {} - except Exception: - direct_url = {} - - url = direct_url.get("url", "") - has_info = False - - if not has_info and "vcs_info" in direct_url: - vcs_info = direct_url["vcs_info"] - if vcs_info.get("vcs") == "git": - commit_id = vcs_info.get("commit_id") - if commit_id: - git_info = models.GitInfo( - commit=commit_id, remote=url, uncommitted_changes=False - ) - has_info = True - - if not has_info and "dir_info" in direct_url: - if url.startswith("file://") or "://" not in url: - path = Path(url.replace("file://", "")) - if path.exists(): - git_info = _git.git_info_from_path(path) - has_info = git_info is not None - - if not has_info and "archive_info" in direct_url: - if not url.startswith("file://") and "://" in url: - archive = direct_url["archive_info"] - - if "hashes" in archive: - hashes = archive["hashes"] - elif "hash" in archive: - # deprecated - algo, value = archive["hash"].split("=", 1) - hashes = {algo: value} - else: - hashes = {} - - archive_info = models.ArchiveInfo(url=url, hashes=hashes) - has_info = True - - return models.Distribution( - name=name, - version=version, - git=git_info, - archive=archive_info, - installer=installer, - ) diff --git a/src/ewoks/_requirements/utils/metadata/from_python/_git.py b/src/ewoks/_requirements/utils/metadata/from_python/_git.py deleted file mode 100644 index d0675f9..0000000 --- a/src/ewoks/_requirements/utils/metadata/from_python/_git.py +++ /dev/null @@ -1,73 +0,0 @@ -import subprocess -from pathlib import Path -from typing import List -from typing import Optional - -from .. import models - - -def git_info_from_path(path: Path) -> Optional[models.GitInfo]: - """ - Return GitInfo for a local path, or None if not a git repository. - """ - try: - commit = _git(["rev-parse", "HEAD"], path) - except Exception: - return None - - try: - uncommitted_changes = bool(_git(["status", "--porcelain"], path)) - except Exception: - uncommitted_changes = False - - remote_name = _find_remote_for_commit(path, commit) - if remote_name: - try: - remote_url = _git(["remote", "get-url", remote_name], path) - except Exception: - remote_url = None - else: - remote_url = None - - return models.GitInfo( - commit=commit, remote=remote_url, uncommitted_changes=uncommitted_changes - ) - - -def _find_remote_for_commit(path: Path, commit: str) -> Optional[str]: - """ - Return the name of a remote that contains the given commit. - """ - try: - remotes = _git_lines(["remote"], path) - except Exception: - return None - - if not remotes: - return None - - if "origin" in remotes: - remotes = ["origin"] + [r for r in remotes if r != "origin"] - - try: - branches = _git_lines(["branch", "-r", "--contains", commit], path) - except Exception: - return None - - for remote in remotes: - prefix = f"{remote}/" - if any(b.startswith(prefix) for b in branches): - return remote - - return None - - -def _git(cmd: List[str], repo: Path) -> str: - return subprocess.check_output( # noqa: S603 - Internal git helper - ["git"] + cmd, cwd=repo, stderr=subprocess.DEVNULL, text=True - ).strip() - - -def _git_lines(cmd: List[str], repo: Path) -> List[str]: - out = _git(cmd, repo) - return [line.strip() for line in out.splitlines() if line.strip()] diff --git a/src/ewoks/_requirements/utils/metadata/from_requirements_txt.py b/src/ewoks/_requirements/utils/metadata/from_requirements_txt.py deleted file mode 100644 index 1e9e3e9..0000000 --- a/src/ewoks/_requirements/utils/metadata/from_requirements_txt.py +++ /dev/null @@ -1,30 +0,0 @@ -from typing import Any -from typing import Dict -from typing import List - -from .. import requirements_txt -from . import _unknown - - -def requirements_txt_metadata(requirements: List[str]) -> Dict[str, Any]: - """Requirements for a legacy requirements list in `requirements.txt` format. - - The requirements are parsed into python distributions so that any package - manager can reproduce the environment, not only the one that installs a - `requirements.txt` file. - """ - metadata = dict(_unknown.unknown_requirements()) - metadata["distributions"] = [ - distribution - for distribution in map( - requirements_txt.distribution_from_requirement, - requirements_txt.sanitize(requirements), - ) - if distribution is not None - ] - metadata["manager"] = dict( - name="pip-venv", - version="", - files={requirements_txt.REQUIREMENTS_FILENAME: "\n".join(requirements)}, - ) - return metadata diff --git a/src/ewoks/_requirements/utils/metadata/last_resort.py b/src/ewoks/_requirements/utils/metadata/last_resort.py deleted file mode 100644 index 99e78fe..0000000 --- a/src/ewoks/_requirements/utils/metadata/last_resort.py +++ /dev/null @@ -1,48 +0,0 @@ -import logging -from typing import Any -from typing import Dict -from typing import Set - -from ewokscore.graph import TaskGraph - -from .from_requirements_txt import requirements_txt_metadata - -logger = logging.getLogger(__name__) - - -def last_resort_requirements(graph: TaskGraph) -> Dict[str, Any]: - """Last resort when installing a workflow that does not have requirements: - guess the requirements from the workflow nodes. - """ - requirements: Set[str] = set() - - for node_id, node in graph.graph.nodes.items(): - task_identifier = node["task_identifier"] - task_type = node["task_type"] - - if task_type in ("class", "method", "ppfmethod", "ppfport"): - package = task_identifier.split(".")[0] - if package in ("__main__", ""): - logger.warning( - f"Could not extract requirements for node {node_id}: the task identifier is a relative import or an import from __main__." - ) - continue - - requirements.add(package) - - elif task_type == "notebook": - logger.warning( - f"Requirement extraction may be incomplete for node {node_id}: {task_type} is only partially supported." - ) - requirements.add("ewokscore[notebook]") - - elif task_type == "script": - logger.warning( - f"Requirement extraction cannot be done for scripts (node {node_id})." - ) - else: - logger.warning( - f"Could not extract requirements for node {node_id}: unsupported task type {task_type}." - ) - - return requirements_txt_metadata(sorted(requirements)) diff --git a/src/ewoks/_requirements/utils/metadata/models.py b/src/ewoks/_requirements/utils/metadata/models.py deleted file mode 100644 index 8c333fe..0000000 --- a/src/ewoks/_requirements/utils/metadata/models.py +++ /dev/null @@ -1,102 +0,0 @@ -from typing import Dict -from typing import Optional - -from pydantic import BaseModel -from pydantic import Field - - -class EngineInfo(BaseModel): - name: str = Field(description="Distribution name.", examples=["ewoksorange"]) - version: str = Field(description="Distribution version.", examples=["2.2.0"]) - - -class EwoksInfo(BaseModel): - version: str = Field(description="Ewoks version.", examples=["7.0.0"]) - engine: Optional[EngineInfo] = Field( - default=None, - description=( - "Distribution of the engine that saved the workflow, which is needed " - "to load it." - ), - ) - - -class SystemInfo(BaseModel): - system: str = Field(description="Operating system name.", examples=["Linux"]) - release: str = Field( - description="Operating system release.", examples=["5.15.0-139-generic"] - ) - version: str = Field( - description="Operating system version.", - examples=["#149-Ubuntu SMP Fri Apr 11 19:19:52 UTC 2025"], - ) - machine: str = Field(description="Machine type.", examples=["x86_64"]) - processor: str = Field(description="Processor name.", examples=["x86_64"]) - - -class PythonInfo(BaseModel): - version: str = Field(description="Python version.", examples=["3.12.11"]) - implementation: str = Field( - description="Python implementation.", examples=["CPython"] - ) - compiler: str = Field( - description="Compiler used to build python.", examples=["GCC 11.4.0"] - ) - build: str = Field( - description="Build number and date of the python interpreter.", - examples=["main, Jun 11 2025 10:57:12"], - ) - - -class GitInfo(BaseModel): - commit: str = Field( - description="Commit from which the distribution was installed.", - examples=["249ea97730a0134de4ecd1b7f136cc48bc0ec2de"], - ) - remote: Optional[str] = Field( - default=None, - description=( - "Repository from which the distribution was installed. Without a " - "repository the distribution cannot be installed elsewhere." - ), - examples=["https://github.com/ewoks-kit/ewokscore.git"], - ) - uncommitted_changes: bool = Field( - default=False, - description=( - "The repository had uncommitted changes, so the commit does not " - "describe the installed distribution completely." - ), - examples=[False], - ) - - -class ArchiveInfo(BaseModel): - url: str = Field( - description="Archive from which the distribution was installed.", - examples=["https://host/ewokscore-5.1.0-py3-none-any.whl"], - ) - hashes: Dict[str, str] = Field( - description="Hashes of the archive, by algorithm name.", - examples=[{"sha256": "5f8e9c1a"}], - ) - - -class Distribution(BaseModel): - name: str = Field(description="Distribution name.", examples=["ewokscore"]) - version: str = Field( - description="Distribution version, empty when unknown.", examples=["5.1.0"] - ) - git: Optional[GitInfo] = Field( - default=None, - description="Set when the distribution was installed from a git repository.", - ) - archive: Optional[ArchiveInfo] = Field( - default=None, - description="Set when the distribution was installed from an archive.", - ) - installer: Optional[str] = Field( - default=None, - description="Tool that installed the distribution.", - examples=["pip", "uv", "conda"], - ) diff --git a/src/ewoks/_requirements/utils/parse.py b/src/ewoks/_requirements/utils/parse.py deleted file mode 100644 index 0ef4330..0000000 --- a/src/ewoks/_requirements/utils/parse.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import List -from typing import Union - -from ._supported import get_supported_managers -from .base_manager import BaseRequirements -from .metadata.from_requirements_txt import requirements_txt_metadata - - -def parse_requirements(requirements: Union[dict, List[str]]) -> BaseRequirements: - if isinstance(requirements, list): - # Legacy 'pip freeze' list - requirements = requirements_txt_metadata(requirements) - - if not isinstance(requirements, dict): - raise TypeError( - f"Graph requirements must be a list or dictionary (type: {type(requirements)})" - ) - - manager_name = requirements.get("manager", dict()).get("name") - - managers = get_supported_managers() - if manager_name not in managers: - raise ValueError(f"{manager_name!r} is not a valid package manager") - - return managers[manager_name].REQUIREMENTS_MODEL(**requirements) diff --git a/src/ewoks/_requirements/utils/process.py b/src/ewoks/_requirements/utils/process.py deleted file mode 100644 index 1143589..0000000 --- a/src/ewoks/_requirements/utils/process.py +++ /dev/null @@ -1,53 +0,0 @@ -import logging -import os -import subprocess -from pathlib import Path -from typing import Dict -from typing import Mapping -from typing import Optional -from typing import Union - -logger = logging.getLogger(__name__) - - -def check_output( - *args: Union[str, Path], extra_env: Optional[Mapping[str, Optional[str]]] = None -) -> str: - """`extra_env` adds environment variables to the current ones, or removes them - when their value is `None`. - - :raises RuntimeError: command failed - """ - logger.debug("Capture output of %s", args) - try: - return subprocess.check_output( # noqa: S603 - Internal manager call - args, text=True, env=_environment(extra_env) - ) - except Exception as ex: - raise RuntimeError(f"Command failed: {args}") from ex - - -def check_call( - *args: Union[str, Path], extra_env: Optional[Mapping[str, Optional[str]]] = None -) -> None: - """`extra_env` adds environment variables to the current ones, or removes them - when their value is `None`. - - :raises RuntimeError: command failed - """ - logger.debug("Execute %s", args) - try: - subprocess.check_call( # noqa: S603 - Internal manager call - args, env=_environment(extra_env) - ) - except Exception as ex: - raise RuntimeError(f"Command failed: {args}") from ex - - -def _environment( - extra_env: Optional[Mapping[str, Optional[str]]], -) -> Optional[Dict[str, str]]: - if not extra_env: - return None - environment = {**os.environ, **extra_env} - return {name: value for name, value in environment.items() if value is not None} diff --git a/src/ewoks/_requirements/utils/requirements_txt.py b/src/ewoks/_requirements/utils/requirements_txt.py deleted file mode 100644 index 3478844..0000000 --- a/src/ewoks/_requirements/utils/requirements_txt.py +++ /dev/null @@ -1,370 +0,0 @@ -"""The `requirements.txt` format, as generated by `pip freeze`. Package managers -use it to describe python distributions, either to install them or to declare them -in a manifest. -""" - -import logging -import os -import re -from typing import Dict -from typing import List -from typing import Optional -from typing import Sequence -from typing import Tuple -from urllib.parse import ParseResult -from urllib.parse import parse_qs -from urllib.parse import urlparse -from urllib.parse import urlunparse - -from packaging.requirements import InvalidRequirement -from packaging.requirements import Requirement -from packaging.utils import NormalizedName -from packaging.utils import canonicalize_name - -from .metadata import models - -logger = logging.getLogger(__name__) - -REQUIREMENTS_FILENAME = "requirements.txt" -"""Name under which requirements in `requirements.txt` format are stored.""" - -_GIT_URL_PREFIX = "git+" - -_URL_HASH_ALGORITHMS = ("sha512", "sha384", "sha256", "sha224", "sha1", "md5") -"""Hash algorithms accepted in the fragment of a direct reference URL, strongest -first.""" - - -def sanitize(requirements: Sequence[str]) -> List[str]: - """Requirements in `requirements.txt` format, ready to be installed.""" - return _log(*_sanitize(requirements)) - - -def distributions_requirements( - distributions: Sequence[models.Distribution], -) -> List[str]: - """Requirements in `requirements.txt` format for a list of distributions.""" - requirements = [] - warnings = [] - for dist in distributions: - dist_requirements, dist_warnings = _distribution_requirements(dist) - requirements.extend(dist_requirements) - warnings.extend(dist_warnings) - return _log(requirements, warnings) - - -def manifest_requirements( - distributions: Sequence[models.Distribution], -) -> List[str]: - """PEP 508 requirements for a list of distributions, for use in a package - manager manifest. Distributions installed from a local directory are skipped - because they cannot be resolved elsewhere. Names are unique because a manifest - cannot contain the same dependency twice. - """ - requirements: Dict[NormalizedName, str] = dict() - for requirement in sanitize(distributions_requirements(distributions)): - if requirement.startswith("-e "): - continue - try: - name = canonicalize_name(Requirement(requirement).name) - except InvalidRequirement: - logger.warning("Skip invalid requirement %r", requirement) - continue - if name in requirements: - logger.warning("Skip duplicate requirement %r", requirement) - continue - requirements[name] = requirement - return list(requirements.values()) - - -def version_constraints(distributions: Sequence[models.Distribution]) -> List[str]: - """Version constraints for a list of distributions. - - Only versions are constrained: direct references are not allowed in a - constraints file. - """ - return [f"{dist.name}=={dist.version}" for dist in distributions if dist.version] - - -def distribution_from_requirement(requirement: str) -> Optional[models.Distribution]: - """Distribution described by a single requirement. Returns None when the - requirement cannot be parsed. - """ - try: - parsed = Requirement(requirement) - except InvalidRequirement: - logger.warning("Skip invalid requirement %r", requirement) - return None - - version = "" - for specifier in parsed.specifier: - if specifier.operator in ("==", "==="): - version = specifier.version - break - - if not parsed.url: - return models.Distribution(name=parsed.name, version=version) - - if parsed.url.startswith(_GIT_URL_PREFIX): - url, separator, commit = parsed.url[len(_GIT_URL_PREFIX) :].rpartition("@") - if separator and commit and "/" not in commit: - return models.Distribution( - name=parsed.name, - version=version, - git=models.GitInfo(commit=commit, remote=url), - ) - - # A URL without git revision is kept as it is - return models.Distribution( - name=parsed.name, version=version, archive=_archive_from_url(parsed.url) - ) - - -def _archive_url(archive: models.ArchiveInfo) -> str: - """Archive URL with one hash in its fragment, which pip verifies when it - downloads the archive. The hashes are not provided as `--hash` options because - pip then requires a hash for every requirement. - """ - for algorithm in _URL_HASH_ALGORITHMS: - value = archive.hashes.get(algorithm) - if value: - return f"{archive.url}#{algorithm}={value}" - return archive.url - - -def _archive_from_url(url: str) -> models.ArchiveInfo: - """Archive of a direct reference URL, with the hashes in its fragment (see - `_archive_url`).""" - base, separator, fragment = url.partition("#") - if not separator: - return models.ArchiveInfo(url=url, hashes=dict()) - - hashes = dict() - others = [] - for part in fragment.split("&"): - algorithm, _, value = part.partition("=") - if value and algorithm in _URL_HASH_ALGORITHMS: - hashes[algorithm] = value - elif part: - others.append(part) - - if not hashes: - return models.ArchiveInfo(url=url, hashes=hashes) - if others: - base = f"{base}#{'&'.join(others)}" - return models.ArchiveInfo(url=base, hashes=hashes) - - -def _log(requirements: List[str], warnings: List[str]) -> List[str]: - for warning in warnings: - logger.warning(warning) - return requirements - - -def _distribution_requirements( - dist: models.Distribution, -) -> Tuple[List[str], List[str]]: - """ - Return the requirements corresponding to the distribution with associated - warnings regarding reproducibility. - """ - lines = [] - warnings = [] - - if dist.version: - pypi_req = f"{dist.name}=={dist.version}" - else: - pypi_req = dist.name - - if dist.archive: - lines.append(f"{dist.name} @ {_archive_url(dist.archive)}") - return lines, warnings - - if dist.git: - warning_fmt = "Non-reproducible Ewoks workflow requirement: {}@{} {}" - - if dist.git.remote: - url = _normalize_git_url(dist.git.remote) - git_req = f"{dist.name} @ {url}@{dist.git.commit}" - else: - git_req = pypi_req - warning = warning_fmt.format( - dist.name, dist.git.commit, "has no remote repository" - ) - warnings.append(warning) - lines.append(f"# {warning}") - - if dist.git.uncommitted_changes: - warning = warning_fmt.format( - dist.name, dist.git.commit, "has uncommited changes" - ) - warnings.append(warning) - lines.append(f"# {warning}") - - lines.append(git_req) - return lines, warnings - - lines.append(pypi_req) - return lines, warnings - - -def _normalize_git_url(url: str, preserve_ssh: bool = False) -> str: - """ - Return a PEP 508-compatible VCS URL, prefixed with 'git+'. - """ - # SCP-like SSH syntax: git@host:group/repo.git - if url.startswith("git@"): - host, path = url[len("git@") :].split(":", 1) - if preserve_ssh: - return f"git+ssh://git@{host}/{path}" - return f"git+https://{host}/{path}" - - # Explicit SSH URL - if url.startswith("ssh://"): - if preserve_ssh: - return f"git+{url}" - - # ssh://git@host/group/repo.git -> https://host/group/repo.git - without_scheme = url[len("ssh://") :] - if without_scheme.startswith("git@"): - without_scheme = without_scheme[len("git@") :] - return f"git+https://{without_scheme}" - - # HTTP(S) - if url.startswith("http://") or url.startswith("https://"): - return f"git+{url}" - - # Unknown / already normalized - return url - - -def _sanitize(requirements: Sequence[str]) -> Tuple[List[str], List[str]]: - """Sanitize a list of requirements in `requirements.txt` format. - - Returns a sanitized list with warnings regarding applied changes. - """ - sanitized = [] - warnings = [] - previous_comment = "" - - for requirement in requirements: - requirement = requirement.strip() - if not requirement: - continue - if requirement.startswith("#"): - previous_comment = requirement - continue - - editable = requirement.startswith("-e ") - if editable: - requirement = requirement[2:].strip() - - if editable and _is_path(requirement): - requirement, warning = _handle_editable_path(requirement, previous_comment) - if warning: - warnings.append(warning) - if _is_path(requirement): - sanitized.append(f"-e {requirement}") - continue - - requirement, warning = _normalize_vcs_url(requirement) - if warning: - warnings.append(warning) - - requirement, warning = _normalize_requirement(requirement) - if warning: - warnings.append(warning) - - sanitized.append(requirement) - previous_comment = "" - - return sanitized, warnings - - -def _is_path(requirement: str) -> bool: - """Return True if requirement looks like a local path (not a URL).""" - return "://" not in requirement and ("/" in requirement or "\\" in requirement) - - -def _normalize_requirement(requirement: str) -> Tuple[str, str]: - """Normalize the requirement or pass-through with warning when not valid.""" - try: - req = Requirement(requirement) - except InvalidRequirement: - warning = f"Possibly invalid requirement format: {requirement!r}" - return requirement, warning - - replacement = _format_requirement(req) - if requirement.replace(" ", "") != replacement.replace(" ", ""): - warning = f"Normalize requirement {requirement!r} to {replacement!r}" - return replacement, warning - - return requirement, "" - - -def _format_requirement(req: Requirement) -> str: - """Convert requirement to a string and replace ssh with https""" - if req.url and req.url.startswith("git+ssh://"): - https_url = re.sub(r"^git\+ssh://git@", "git+https://", req.url) - return f"{req.name} @ {https_url}" - return str(req) - - -def _normalize_vcs_url(requirement: str) -> Tuple[str, str]: - """Normalize VCS URLs to the PEP 508 spec (name @ url).""" - parsed = urlparse(requirement) - - if not parsed.fragment or "egg=" not in parsed.fragment: - return requirement, "" - - qs = parse_qs(parsed.fragment) - egg_names = qs.get("egg") - if not egg_names: - return requirement, "" - - egg_name = egg_names[0] - base_url = _remove_url_fragment(parsed) - new_requirement = f"{egg_name} @ {base_url}" - warning = f"Normalize VCS requirement '{requirement}' to '{new_requirement}'" - return new_requirement, warning - - -def _remove_url_fragment(parsed: ParseResult) -> str: - return urlunparse( - ParseResult( - scheme=parsed.scheme, - netloc=parsed.netloc, - path=parsed.path, - params=parsed.params, - query=parsed.query, - fragment="", - ) - ) - - -def _handle_editable_path(requirement: str, comment: str) -> Tuple[str, str]: - """Replace editable path with comment substitution if available.""" - replacement = _requirement_from_comment(comment) - if replacement: - warning = f"Replaced editable install '{requirement}' with '{replacement}'." - return replacement, warning - if os.path.exists(requirement): - warning = f"Editable path exists locally: '{requirement}'" - else: - warning = f"Editable path does not exist locally: '{requirement}'" - return requirement, warning - - -_REQUIREMENT_COMMENT_RE = re.compile(r"\(([\w\.\-\[\]]+[=> str: - """Extract requirement from a 'pip freeze' comment.""" - # When doing "pip freeze" in the same envrinment where "git status" gives this - # fatal: detected dubious ownership in repository at '/path/to/repo' - # the requirements list will contain "-e /path/to/repo" with a warning that - # contains a requirement message. - match = _REQUIREMENT_COMMENT_RE.search(comment) - if match: - return match.group(1) - return "" diff --git a/src/ewoks/_requirements/utils/toml_dependencies.py b/src/ewoks/_requirements/utils/toml_dependencies.py deleted file mode 100644 index 2591992..0000000 --- a/src/ewoks/_requirements/utils/toml_dependencies.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Dependencies of a TOML manifest: `name = spec` entries in which the -specification is a version constraint, an URL or a git reference. Poetry and pixi -use the same syntax. -""" - -import json -import logging -from typing import List -from typing import Sequence - -from packaging.requirements import InvalidRequirement -from packaging.requirements import Requirement - -logger = logging.getLogger(__name__) - - -def dependency_lines(requirements: Sequence[str]) -> List[str]: - """Dependency entries for requirements in PEP 508 format.""" - lines = [] - for requirement in requirements: - try: - parsed = Requirement(requirement) - except InvalidRequirement: - logger.warning("Skip invalid requirement %r", requirement) - continue - lines.append(f"{json.dumps(parsed.name)} = {_specification(parsed)}") - return lines - - -def _specification(requirement: Requirement) -> str: - """Version, URL or git reference of a dependency.""" - if requirement.url: - if requirement.url.startswith("git+"): - url, _, revision = requirement.url[len("git+") :].rpartition("@") - if url: - return f"{{ git = {json.dumps(url)}, rev = {json.dumps(revision)} }}" - return f"{{ git = {json.dumps(revision)} }}" - return f"{{ url = {json.dumps(requirement.url)} }}" - if requirement.specifier: - return json.dumps(str(requirement.specifier)) - return '"*"' diff --git a/src/ewoks/_requirements/uv.py b/src/ewoks/_requirements/uv.py deleted file mode 100644 index 7ccb493..0000000 --- a/src/ewoks/_requirements/uv.py +++ /dev/null @@ -1,224 +0,0 @@ -import json -import os -import re -import sys -from pathlib import Path -from typing import Dict -from typing import List -from typing import Literal -from typing import Mapping -from typing import Optional -from typing import Sequence - -from pydantic import Field - -from .utils import requirements_txt -from .utils.base_manager import BaseManager -from .utils.base_manager import BaseManagerInfo -from .utils.base_manager import BaseRequirements -from .utils.environment import Environment -from .utils.files import read_files -from .utils.files import temporary_files -from .utils.files import write_files -from .utils.metadata import models - -_PYPROJECT_FILENAME = "pyproject.toml" -_LOCK_FILENAME = "uv.lock" -_CONSTRAINTS_FILENAME = "constraints.txt" -_PROJECT_NAME = "ewoks-workflow-requirements" - - -class UvManagerInfo(BaseManagerInfo): - name: Literal["uv"] = Field( - default="uv", - description="Environments described and created by a `uv` project.", - examples=["uv"], - ) - - -class UvRequirements(BaseRequirements): - manager: UvManagerInfo - - -class UvManager(BaseManager): - """Uses the `uv` project interface: `uv lock` describes an environment with a - `pyproject.toml` and a `uv.lock` file, `uv sync` reproduces the environment - from those files. - - When the current environment is the `.venv` of a uv project, its - `pyproject.toml` and `uv.lock` are stored as the manager files. Otherwise no - files are stored and the stored python distributions describe the - environment. - - The command invokes uv. For example `UvManager("/path/to/uv")`. - """ - - NAME = "uv" - PRIORITY = 1 - REQUIREMENTS_MODEL = UvRequirements - COMMAND_EXAMPLE = "uv" - ENVIRONMENT_SUBDIR = ".venv" - CAN_INSTALL_IN_PLACE = True - - def __init__(self, *command: str) -> None: - if not command: - command = ("uv",) - super().__init__(*command) - - def version(self) -> Optional[str]: - """Returns None when this manager is not available.""" - try: - output = self._check_output("--version") - except RuntimeError: - return None - # For example "uv 0.12.1 (x86_64-unknown-linux-gnu)" - match = re.search(r"(\d+\.\d+(?:\.\d+)*)", output) - if not match: - return None - return match.group(1) - - def is_active(self) -> bool: - """Manager is explicitly active.""" - return ( - "UV" in os.environ - or _created_by_uv(Path(sys.prefix)) - or _is_project(Path(sys.prefix).parent) - ) - - @classmethod - def installed_distribution(cls, distribution: models.Distribution) -> bool: - """The distribution was installed by this package manager.""" - installer = cls._installer(distribution) - # PyPI distributions of a pixi workspace are installed by the uv that pixi - # vendors ("uv-pixi") - return "uv" in installer and "pixi" not in installer - - def create_environment( - self, location: Path, python_version: Optional[str] = None - ) -> Environment: - """ - :raises RuntimeError: creation failed - """ - environment = self.environment(location) - # Without a python request, uv selects one of its own interpreters - python = python_version or sys.executable - self._check_call("venv", "--python", python, environment.prefix) - return environment - - def _gather_files( - self, distributions: List[models.Distribution], python_version: str - ) -> Dict[str, str]: - project = Path(sys.prefix).parent - files = read_files(project, _PYPROJECT_FILENAME, _LOCK_FILENAME) - if len(files) == 2: - return files - # No uv project: no files - return super()._gather_files(distributions, python_version) - - def _files_from_distributions( - self, distributions: Sequence[models.Distribution], python_version: str - ) -> Dict[str, str]: - """ - :raises RuntimeError: the requirements cannot be resolved - """ - pyproject = _pyproject( - requirements_txt.manifest_requirements(distributions), python_version - ) - files = {_PYPROJECT_FILENAME: pyproject} - - with temporary_files(files) as directory: - self._check_call("lock", "--project", directory) - lock = read_files(directory, _LOCK_FILENAME) - - return {**files, **lock} - - def _install_files( - self, files: Mapping[str, str], environment: Environment - ) -> None: - if self._owns(environment): - write_files(environment.location, files) - self._sync(environment.location) - return - - # An environment that is not a uv project can only be filled through the - # pip-compatible interface of uv: `uv sync` always targets '/.venv' - with temporary_files(files) as directory: - filename = directory / requirements_txt.REQUIREMENTS_FILENAME - self._check_call( - "export", - "--frozen", - "--no-hashes", - "--quiet", - "--project", - directory, - "--output-file", - filename, - ) - self._pip_install(environment, "-r", filename) - - def _add_ewoks( - self, requirements: BaseRequirements, environment: Environment - ) -> None: - if self._owns(environment): - # The project pins all versions so they cannot change - self._check_call("add", "--project", environment.location, "ewoks") - return - - constraints = requirements_txt.version_constraints(requirements.distributions) - if not constraints: - self._pip_install(environment, "ewoks") - return - with temporary_files({_CONSTRAINTS_FILENAME: "\n".join(constraints)}) as tmpdir: - self._pip_install( - environment, - "ewoks", - "--constraint", - tmpdir / _CONSTRAINTS_FILENAME, - ) - - def _sync(self, location: Path) -> None: - self._check_call( - "sync", "--frozen", "--no-install-project", "--project", location - ) - - def _pip_install(self, environment: Environment, *arguments: str) -> None: - self._check_call( - "pip", - "install", - "--no-cache", - "--python", - environment.python, - *arguments, - ) - - -def _pyproject(requirements: Sequence[str], python_version: str) -> str: - """Content of a `pyproject.toml` file for a project that is not a package.""" - dependencies = "".join(f" {json.dumps(req)},\n" for req in requirements) - if python_version: - python = f'requires-python = "=={python_version}"\n' - else: - python = "" - return f"""[project] -name = {json.dumps(_PROJECT_NAME)} -version = "0.0.0" -{python}dependencies = [ -{dependencies}] - -[tool.uv] -package = false -""" - - -def _is_project(directory: Path) -> bool: - """Directory is a uv project.""" - return (directory / _LOCK_FILENAME).is_file() - - -def _created_by_uv(prefix: Path) -> bool: - """Environment is a virtual environment created by uv.""" - try: - with open(prefix / "pyvenv.cfg", "r", encoding="utf-8") as fh: - return any(line.split("=")[0].strip() == "uv" for line in fh) - except OSError: - return False diff --git a/src/ewoks/tests/requirements/conftest.py b/src/ewoks/tests/requirements/conftest.py deleted file mode 100644 index 92c3a5d..0000000 --- a/src/ewoks/tests/requirements/conftest.py +++ /dev/null @@ -1,126 +0,0 @@ -import shutil -from pathlib import Path -from typing import Dict -from typing import Iterator - -import pytest - -from ..._requirements.utils.base_manager import BaseManager -from ..._requirements.utils.environment import Environment -from .managers import FAST_MANAGER_CASES -from .managers import MANAGER_CASES -from .managers import ManagerCase - -_IDS = [case.NAME for case in MANAGER_CASES] -_FAST_IDS = [case.NAME for case in FAST_MANAGER_CASES] - - -@pytest.fixture(scope="package", autouse=True) -def isolated_home(tmp_path_factory) -> Iterator[Path]: - """Home directory of the package managers, in which they cache their downloads - and store their configuration. It is inside the pytest temporary directory so a - test run does not write anywhere else. The downside is that the caches are - empty at the start of every test run. - """ - home = tmp_path_factory.mktemp("home") - with pytest.MonkeyPatch.context() as monkeypatch: - for name, path in _home_variables(home).items(): - monkeypatch.setenv(name, str(path)) - yield home - # The caches are not reused by the next test run and fill up the disk - shutil.rmtree(home, ignore_errors=True) - - -def _home_variables(home: Path) -> Dict[str, Path]: - """Environment variables that move everything a package manager writes - outside a python environment to this home directory. - """ - condarc = home / ".condarc" - with open(condarc, "w", encoding="utf-8") as fh: - fh.write("channels:\n - conda-forge\n") - return { - # Home directory on Linux, macOS and Windows - "HOME": home, - "USERPROFILE": home, - "APPDATA": home, - "LOCALAPPDATA": home, - "XDG_CACHE_HOME": home / ".cache", - "XDG_CONFIG_HOME": home / ".config", - "XDG_DATA_HOME": home / ".local" / "share", - # Conda downloads packages next to its own installation instead - "CONDA_PKGS_DIRS": home / "conda" / "pkgs", - # Directories in which package managers create named environments - "CONDA_ENVS_DIRS": home / "conda" / "envs", - "POETRY_VIRTUALENVS_PATH": home / "poetry" / "virtualenvs", - # Pixi cannot find its cache directory without a user profile, like on CI - # runners that run as a service - "PIXI_CACHE_DIR": home / "pixi" / "cache", - # Channels are no longer provided by the home directory of the user - "CONDARC": condarc, - } - - -@pytest.fixture -def env_root(tmp_path) -> Iterator[Path]: - """Root directory of the environments created by a test.""" - yield tmp_path - # Environments are large and there are many tests - shutil.rmtree(tmp_path, ignore_errors=True) - - -@pytest.fixture(params=MANAGER_CASES, ids=_IDS) -def manager_case(request) -> ManagerCase: - """Repeats the test for every package manager.""" - return _manager_case(request) - - -@pytest.fixture -def manager(manager_case) -> BaseManager: - """Package manager under test.""" - return manager_case.manager() - - -@pytest.fixture -def environment(manager_case, tmp_path) -> Iterator[Environment]: - """Empty python environment created by the package manager under test.""" - yield from _environment(manager_case, tmp_path) - - -@pytest.fixture(params=FAST_MANAGER_CASES, ids=_FAST_IDS) -def fast_manager_case(request) -> ManagerCase: - """Repeats the test for every package manager that is fast. Use this when the - package manager is not the subject of the test.""" - return _manager_case(request) - - -@pytest.fixture -def fast_manager(fast_manager_case) -> BaseManager: - """Package manager under test, one that is fast.""" - return fast_manager_case.manager() - - -@pytest.fixture -def fast_environment(fast_manager_case, tmp_path) -> Iterator[Environment]: - """Empty python environment created by a package manager that is fast.""" - yield from _environment(fast_manager_case, tmp_path) - - -@pytest.fixture(params=FAST_MANAGER_CASES, ids=_FAST_IDS) -def source_manager_case(request) -> ManagerCase: - """Repeats the test for every package manager that could have generated the - requirements to be installed.""" - return _manager_case(request) - - -def _manager_case(request) -> ManagerCase: - case: ManagerCase = request.param - if case.manager().version() is None: - pytest.skip(f"{case.NAME} is not installed") - return case - - -def _environment(case: ManagerCase, tmp_path) -> Iterator[Environment]: - location = tmp_path / "environment" - yield case.manager().create_environment(location) - # Environments are large and there are many tests - shutil.rmtree(location, ignore_errors=True) diff --git a/src/ewoks/tests/requirements/managers.py b/src/ewoks/tests/requirements/managers.py deleted file mode 100644 index 6a9e59d..0000000 --- a/src/ewoks/tests/requirements/managers.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Package manager specific knowledge needed by the package manager tests. - -Add a `ManagerCase` to `MANAGER_CASES` to include a package manager in the -tests that apply to all package managers. The tests use the `manager_case`, -`manager` and `environment` fixtures (see `conftest.py`). -""" - -from typing import Dict -from typing import List -from typing import Sequence -from typing import Tuple -from typing import Type - -from ..._requirements.conda import CondaManager -from ..._requirements.pip_venv import PipVenvManager -from ..._requirements.pixi import PixiManager -from ..._requirements.poetry import PoetryManager -from ..._requirements.utils import conda_channel -from ..._requirements.utils.base_manager import BaseManager -from ..._requirements.utils.metadata import models -from ..._requirements.utils.requirements_txt import REQUIREMENTS_FILENAME -from ..._requirements.uv import UvManager - - -class ManagerCase: - NAME: str = NotImplemented - - MANAGER_CLS: Type[BaseManager] = NotImplemented - - INSTALLER: str = NotImplemented - """`INSTALLER` metadata of a distribution installed by the package manager.""" - - SLOW: bool = False - """Creating an environment takes tens of seconds.""" - - CHANNEL_PYTHON: bool = False - """Python comes from a conda channel, which decides the patch version.""" - - CREATES_ENVIRONMENT: bool = True - """The environment exists before the requirements are installed.""" - - def command(self) -> Tuple[str, ...]: - """Command that invokes the package manager (the default when empty).""" - return tuple() - - def manager(self) -> BaseManager: - return self.MANAGER_CLS(*self.command()) - - def cli_command(self) -> str: - """Value for the `--package-manager-command` CLI argument.""" - return " ".join(self.command()) - - def native_files( - self, distributions: Sequence[models.Distribution], python_version: str - ) -> Dict[str, str]: - """Files the package manager generates to reproduce an environment with - these distributions. Empty when they cannot be generated. - """ - raise NotImplementedError - - -class PipVenvCase(ManagerCase): - NAME = "pip-venv" - MANAGER_CLS = PipVenvManager - INSTALLER = "pip" - - def native_files( - self, distributions: Sequence[models.Distribution], python_version: str - ) -> Dict[str, str]: - requirements = "\n".join( - f"{dist.name}=={dist.version}" for dist in distributions - ) - return {REQUIREMENTS_FILENAME: requirements} - - -class UvCase(ManagerCase): - NAME = "uv" - MANAGER_CLS = UvManager - INSTALLER = "uv" - - def native_files( - self, distributions: Sequence[models.Distribution], python_version: str - ) -> Dict[str, str]: - # Resolving a lock file requires the package index - try: - return UvManager()._files_from_distributions(distributions, python_version) - except RuntimeError: - return dict() - - -class PoetryCase(ManagerCase): - NAME = "poetry" - MANAGER_CLS = PoetryManager - INSTALLER = "Poetry 1.8.5" # poetry adds its version - - def native_files( - self, distributions: Sequence[models.Distribution], python_version: str - ) -> Dict[str, str]: - # Resolving a lock file requires the package index - try: - return PoetryManager()._files_from_distributions( - distributions, python_version - ) - except RuntimeError: - return dict() - - -class CondaCase(ManagerCase): - NAME = "conda" - MANAGER_CLS = CondaManager - INSTALLER = "conda" - SLOW = True - CHANNEL_PYTHON = True - - def native_files( - self, distributions: Sequence[models.Distribution], python_version: str - ) -> Dict[str, str]: - pip_dependencies = "".join( - f" - {dist.name}=={dist.version}\n" for dist in distributions - ) - return { - "environment.yml": f"""channels: - - conda-forge -dependencies: - - python={conda_channel.python_specifier(python_version)} - - pip - - pip: -{pip_dependencies}""" - } - - -class PixiCase(ManagerCase): - NAME = "pixi" - MANAGER_CLS = PixiManager - INSTALLER = "uv-pixi" # pixi installs PyPI distributions with its own uv - SLOW = True - CHANNEL_PYTHON = True - CREATES_ENVIRONMENT = False - - def native_files( - self, distributions: Sequence[models.Distribution], python_version: str - ) -> Dict[str, str]: - # Resolving a lock file requires the package index - try: - return PixiManager()._files_from_distributions( - distributions, python_version - ) - except RuntimeError: - return dict() - - -MANAGER_CASES: List[ManagerCase] = [ - PipVenvCase(), - UvCase(), - PoetryCase(), - CondaCase(), - PixiCase(), -] - -FAST_MANAGER_CASES: List[ManagerCase] = [ - case for case in MANAGER_CASES if not case.SLOW -] diff --git a/src/ewoks/tests/requirements/test_conda.py b/src/ewoks/tests/requirements/test_conda.py deleted file mode 100644 index 8040221..0000000 --- a/src/ewoks/tests/requirements/test_conda.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Tests specific to the conda package manager.""" - -import sys - -import pytest - -from ..._requirements.conda import CondaManager -from ..._requirements.conda import _environment_yml -from ..._requirements.utils.base_manager import EWOKS_ENVIRONMENTS_ROOT -from ..._requirements.utils.conda_channel import python_specifier -from .utils import PYTHON_VERSION -from .utils import REQUIREMENTS - - -def test_environment_file(): - """Python distributions are declared in the pip section of a conda - environment file.""" - environment_yml = _environment_yml(REQUIREMENTS, PYTHON_VERSION) - - pip_dependencies = "".join(f" - {requirement}\n" for requirement in REQUIREMENTS) - assert ( - environment_yml - == f"""channels: - - conda-forge -dependencies: - - python={python_specifier(PYTHON_VERSION)} - - pip - - pip: -{pip_dependencies}""" - ) - - -def test_environment_file_without_requirements(): - assert ( - _environment_yml([], "") - == """channels: - - conda-forge -dependencies: - - pip -""" - ) - - -def test_default_command(): - """The fastest implementation available is used.""" - manager = CondaManager() - assert manager._cmd_args[0] in ("micromamba", "mamba", "conda") - - -def test_version_not_available(): - assert CondaManager("conda-does-not-exist").version() is None - - -def test_is_active(monkeypatch, tmp_path): - monkeypatch.delenv("CONDA_PREFIX", raising=False) - monkeypatch.setattr(sys, "prefix", str(tmp_path)) - assert not CondaManager().is_active() - - (tmp_path / "conda-meta").mkdir() - assert CondaManager().is_active() - - -def test_is_active_conda_prefix(monkeypatch, tmp_path): - monkeypatch.setattr(sys, "prefix", str(tmp_path)) - monkeypatch.setenv("CONDA_PREFIX", str(tmp_path)) - assert CondaManager().is_active() - - -def test_environments_root(): - """Named environments are created where conda creates them itself.""" - manager = CondaManager() - if manager.version() is None: - pytest.skip("conda is not installed") - - root = manager.environments_root() - - assert root.is_absolute() - assert root != EWOKS_ENVIRONMENTS_ROOT - - -def test_environments_root_not_available(caplog): - """Environments of ewoks when conda cannot provide its environment directory.""" - manager = CondaManager("conda-does-not-exist") - - assert manager.environments_root() == EWOKS_ENVIRONMENTS_ROOT - - assert "Cannot determine where conda creates environments" in caplog.text - - -def test_files_not_available(monkeypatch, tmp_path, caplog): - """Only a conda environment can be exported.""" - monkeypatch.setattr(sys, "prefix", str(tmp_path)) - - assert CondaManager()._gather_files([], PYTHON_VERSION) == {} - - assert "not a conda environment" in caplog.text diff --git a/src/ewoks/tests/requirements/test_cross_manager.py b/src/ewoks/tests/requirements/test_cross_manager.py deleted file mode 100644 index dc37b43..0000000 --- a/src/ewoks/tests/requirements/test_cross_manager.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Tests of installing requirements generated by another package manager.""" - -import pytest - -from ..._requirements.utils.parse import parse_requirements -from .utils import DISTRIBUTIONS -from .utils import PYTHON_VERSION -from .utils import assert_installed -from .utils import manager_requirements - - -def test_install_foreign_requirements( - source_manager_case, manager, manager_case, environment -): - """Requirements generated by one package manager, installed by another one. - - The files of the package manager that generated the requirements are ignored: - the installed python distributions are used to generate the files of the - package manager that installs them. - """ - if source_manager_case.NAME == manager_case.NAME: - pytest.skip("not a foreign package manager") - - files = source_manager_case.native_files(DISTRIBUTIONS, PYTHON_VERSION) - if not files: - pytest.skip(f"{source_manager_case.NAME} cannot generate its requirement files") - requirements = parse_requirements( - manager_requirements( - source_manager_case.NAME, - files=files, - distributions=DISTRIBUTIONS, - python_version=PYTHON_VERSION, - ) - ) - - assert not manager.is_native(requirements, environment) - - manager.install_distributions(requirements, environment) - - assert_installed(environment, DISTRIBUTIONS) diff --git a/src/ewoks/tests/requirements/test_environment.py b/src/ewoks/tests/requirements/test_environment.py deleted file mode 100644 index da8a565..0000000 --- a/src/ewoks/tests/requirements/test_environment.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Tests of the python environment layouts.""" - -import sys -from pathlib import Path - -import pytest - -from ..._requirements.utils.environment import ENVIRONMENT_SUBDIRS -from ..._requirements.utils.environment import Environment - -_INTERPRETERS = [ - ("linux", Path("bin", "python")), - ("win32", Path("Scripts", "python.exe")), - ("win32", Path("python.exe")), -] -"""Interpreter locations inside an environment prefix, per platform.""" - - -@pytest.mark.parametrize("subdir", ENVIRONMENT_SUBDIRS) -@pytest.mark.parametrize("platform, relative", _INTERPRETERS) -def test_environment_layout(platform, relative, subdir, monkeypatch, tmp_path): - """Every interpreter location is found in every environment layout.""" - monkeypatch.setattr(sys, "platform", platform) - prefix = tmp_path / subdir if subdir else tmp_path - python = prefix / relative - python.parent.mkdir(parents=True, exist_ok=True) - python.touch() - - environment = Environment.from_location(tmp_path) - - assert environment.location == tmp_path - assert environment.prefix == prefix - assert environment.python == python - - -def test_no_environment(tmp_path): - with pytest.raises(ValueError, match="No python environment found"): - Environment.from_location(tmp_path) - - -@pytest.mark.parametrize( - "names, expected", - [ - (["default"], "default"), - (["test"], "test"), - (["default", "test"], "default"), - (["test", "docs"], None), - ], -) -def test_pixi_environments(names, expected, monkeypatch, tmp_path, caplog): - """A pixi workspace provides its only environment, or its default environment - with a warning when it has several. - """ - monkeypatch.setattr(sys, "platform", "linux") - envs = tmp_path / ".pixi" / "envs" - for name in names: - python = envs / name / "bin" / "python" - python.parent.mkdir(parents=True) - python.touch() - - if expected is None: - with pytest.raises(ValueError, match="Several pixi environments"): - Environment.from_location(tmp_path) - else: - assert Environment.from_location(tmp_path).prefix == envs / expected - assert ("Several pixi environments" in caplog.text) == (len(names) > 1) diff --git a/src/ewoks/tests/requirements/test_managers.py b/src/ewoks/tests/requirements/test_managers.py deleted file mode 100644 index 04dacb7..0000000 --- a/src/ewoks/tests/requirements/test_managers.py +++ /dev/null @@ -1,279 +0,0 @@ -"""Tests that apply to all package managers.""" - -import importlib.metadata -from typing import Iterator - -import pytest - -from ..._requirements import install_requirements -from ..._requirements.utils import base_manager -from ..._requirements.utils import detect -from ..._requirements.utils._supported import get_supported_managers -from ..._requirements.utils.detect import get_in_place_installer -from ..._requirements.utils.detect import get_manager -from ..._requirements.utils.environment import Environment -from ..._requirements.utils.parse import parse_requirements -from ...errors import AbortException -from .managers import ManagerCase -from .utils import DISTRIBUTIONS -from .utils import EWOKS_DEPENDENCIES -from .utils import PYTHON_MINOR_VERSION -from .utils import PYTHON_VERSION -from .utils import assert_installed -from .utils import assert_not_installed -from .utils import current_metadata -from .utils import manager_requirements - - -def test_available(manager): - version = manager.version() - assert version - assert version[0].isdigit() - - -def test_not_available(manager_case, monkeypatch): - monkeypatch.setattr(manager_case.MANAGER_CLS, "version", lambda self: None) - with pytest.raises(ValueError, match="not available"): - get_manager(manager_name=manager_case.NAME) - - -def test_select_manager(manager_case): - manager = get_manager( - manager_name=manager_case.NAME, manager_command=manager_case.command() - ) - assert isinstance(manager, manager_case.MANAGER_CLS) - - -def test_is_active(manager): - assert isinstance(manager.is_active(), bool) - - -def test_detect_manager(manager_case, uncached_distribution_count, monkeypatch): - """The package manager that installed the distributions is detected when no - package manager is active.""" - for manager_cls in get_supported_managers().values(): - monkeypatch.setattr(manager_cls, "is_active", lambda self: False) - distributions = [ - dist.model_copy(update={"installer": manager_case.INSTALLER}) - for dist in DISTRIBUTIONS - ] - monkeypatch.setattr( - detect, - "current_requirements", - lambda: current_metadata(distributions, PYTHON_VERSION), - ) - - manager = get_manager() - - assert isinstance(manager, manager_case.MANAGER_CLS) - - -@pytest.fixture -def uncached_distribution_count() -> Iterator[None]: - """The distributions of the current python environment are counted once.""" - detect._manager_distribution_count.cache_clear() - yield - detect._manager_distribution_count.cache_clear() - - -def test_gather_requirements(manager, manager_case, monkeypatch): - """Requirements are gathered from the current python environment.""" - metadata = current_metadata(DISTRIBUTIONS, PYTHON_VERSION) - monkeypatch.setattr(base_manager, "current_requirements", lambda: metadata) - - requirements = manager.gather_requirements() - - assert isinstance(requirements, manager_case.MANAGER_CLS.REQUIREMENTS_MODEL) - assert requirements.ewoks.version == importlib.metadata.version("ewoks") - assert requirements.manager.name == manager_case.NAME - assert requirements.manager.version - assert requirements.python.version == PYTHON_VERSION - assert requirements.distributions == DISTRIBUTIONS - assert isinstance(requirements.manager.files, dict) - - -def test_create_environment(manager_case, environment): - if manager_case.CREATES_ENVIRONMENT: - assert environment.exists() - if manager_case.CHANNEL_PYTHON: - assert environment.python_version().startswith(f"{PYTHON_MINOR_VERSION}.") - else: - assert environment.python_version() == PYTHON_VERSION - assert Environment.from_location(environment.location) == environment - assert_not_installed(environment, DISTRIBUTIONS) - else: - # The environment is created when the requirements are installed - assert not environment.exists() - - -def test_install_native(manager, manager_case, environment): - """Install with the files generated by the package manager itself.""" - requirements = _requirements(manager_case, native=True) - assert manager.is_native(requirements, environment) - - manager.install_files(requirements, environment) - - assert_installed(environment, DISTRIBUTIONS) - - -def test_install_distributions(manager, manager_case, environment): - """Without files, the python distributions are installed.""" - requirements = _requirements(manager_case, native=False) - assert not manager.is_native(requirements, environment) - - manager.install_distributions(requirements, environment) - - assert_installed(environment, DISTRIBUTIONS) - - -def test_install_nothing(manager, manager_case, tmp_path): - """Nothing is installed so the environment is not needed.""" - requirements = parse_requirements(manager_requirements(manager_case.NAME)) - environment = Environment.at_location(tmp_path) - - with pytest.raises(ValueError, match="No distributions provided to install"): - manager.install_distributions(requirements, environment) - - -def test_install_files_failure(fast_manager_case, env_root, monkeypatch): - """The python distributions are installed when the files of the package manager - cannot be installed. Both installations are confirmed.""" - requirements = _requirements(fast_manager_case, native=True) - - def install_files(self, requirements, environment) -> None: - raise RuntimeError("cannot install the files") - - monkeypatch.setattr(fast_manager_case.MANAGER_CLS, "install_files", install_files) - confirmations = [] - - environment = install_requirements( - requirements, - env_name="environment", - env_root=env_root, - manager_name=fast_manager_case.NAME, - manager_command=fast_manager_case.command(), - confirm=confirmations.append, - ) - - assert_installed(environment, DISTRIBUTIONS) - assert len(confirmations) == 2 - assert "Manager:" in confirmations[0] - assert "Distributions:" in confirmations[1] - assert "package manager failed" in confirmations[1] - - -def test_install_not_confirmed(fast_manager_case, env_root): - """Nothing is created when an installation is not confirmed. The confirmation - mentions why the python distributions are installed.""" - requirements = _requirements(fast_manager_case, native=False) - confirmations = [] - - def confirm(description: str) -> None: - confirmations.append(description) - raise AbortException() - - with pytest.raises(AbortException): - install_requirements( - requirements, - env_name="environment", - env_root=env_root, - manager_name=fast_manager_case.NAME, - manager_command=fast_manager_case.command(), - confirm=confirm, - ) - - assert not (env_root / "environment").exists() - assert "do not provide files" in confirmations[0] - - -def test_install_without_name(fast_manager_case): - """A root directory and cleaning do not define an environment on their own.""" - requirements = parse_requirements(manager_requirements(fast_manager_case.NAME)) - - with pytest.raises(ValueError, match="requires an environment name"): - install_requirements(requirements, env_root="/root/of/the/environments") - - with pytest.raises(ValueError, match="requires an environment name"): - install_requirements(requirements, clean=True) - - -def test_environment_location(manager, tmp_path): - """The package manager provides the location of a named environment.""" - default = manager.environment_location("myenv") - assert default.is_absolute() - assert default.name == "myenv" - assert default.parent == manager.environments_root().expanduser() - - in_root = manager.environment_location("myenv", tmp_path) - assert in_root == tmp_path / "myenv" - - -def test_install_in_place_not_supported(manager_case, monkeypatch): - """A package manager that is explicitly requested is not replaced.""" - monkeypatch.setattr(manager_case.MANAGER_CLS, "CAN_INSTALL_IN_PLACE", False) - requirements = parse_requirements(manager_requirements(manager_case.NAME)) - - with pytest.raises(ValueError, match="cannot install in the current"): - install_requirements(requirements, manager_name=manager_case.NAME) - - -def test_in_place_installer(manager_case, monkeypatch): - """A package manager that cannot install in the current environment is not - used to do so, even if that means none is available.""" - monkeypatch.setattr(manager_case.MANAGER_CLS, "CAN_INSTALL_IN_PLACE", False) - - try: - manager = get_in_place_installer() - except RuntimeError as e: - err_msg = ( - "No package manager available that can " - "install in the current python environment" - ) - assert str(e) == err_msg - return - - assert manager.CAN_INSTALL_IN_PLACE - assert not isinstance(manager, manager_case.MANAGER_CLS) - - -def test_ewoks_already_installed(manager, manager_case): - """Ewoks is installed in the current python environment.""" - requirements = parse_requirements(manager_requirements(manager_case.NAME)) - - manager.ensure_ewoks(requirements, Environment.current()) - - -def test_add_ewoks(fast_manager, fast_manager_case, fast_environment): - """Ewoks is added without changing the versions of the requirements it depends - on.""" - requirements = parse_requirements( - manager_requirements( - fast_manager_case.NAME, - distributions=EWOKS_DEPENDENCIES, - python_version=PYTHON_VERSION, - ) - ) - fast_manager.install_distributions(requirements, fast_environment) - - fast_manager.ensure_ewoks(requirements, fast_environment) - - assert fast_environment.distribution_version("ewoks") - assert_installed(fast_environment, EWOKS_DEPENDENCIES) - - -def _requirements(manager_case: ManagerCase, native: bool): - python_version = PYTHON_VERSION - if native: - files = manager_case.native_files(DISTRIBUTIONS, python_version) - if not files: - pytest.skip(f"{manager_case.NAME} cannot generate its requirement files") - else: - files = None - return parse_requirements( - manager_requirements( - manager_case.NAME, - files=files, - distributions=DISTRIBUTIONS, - python_version=python_version, - ) - ) diff --git a/src/ewoks/tests/requirements/test_pip_venv.py b/src/ewoks/tests/requirements/test_pip_venv.py deleted file mode 100644 index 09a0528..0000000 --- a/src/ewoks/tests/requirements/test_pip_venv.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Tests specific to the pip-venv package manager.""" - -import importlib.metadata -import sys - -import pytest - -from ..._requirements.pip_venv import PipVenvManager -from ..._requirements.utils.requirements_txt import REQUIREMENTS_FILENAME -from .utils import PYTHON_VERSION - -pytestmark = pytest.mark.skipif( - PipVenvManager().version() is None, reason="pip is not installed" -) - - -def test_default_command(): - manager = PipVenvManager() - assert manager._cmd_args == (sys.executable,) - assert repr(manager) == f"PipVenvManager({sys.executable})" - - -def test_version(): - assert PipVenvManager().version() == importlib.metadata.version("pip") - - -def test_version_not_available(monkeypatch): - def not_found(name: str) -> str: - raise importlib.metadata.PackageNotFoundError(name) - - monkeypatch.setattr(importlib.metadata, "version", not_found) - assert PipVenvManager().version() is None - - -def test_never_active(): - """Pip is used when no other package manager is detected.""" - assert PipVenvManager().is_active() is False - - -def test_gather_requirements_txt(): - """The current environment is described in `requirements.txt` format.""" - requirements = PipVenvManager().gather_requirements() - - lines = requirements.manager.files[REQUIREMENTS_FILENAME].splitlines() - assert lines - assert any("ewoks" in requirement for requirement in lines) - - -def test_create_environment_python_version(env_root, caplog): - """`venv` cannot provide another python version than the one it runs on.""" - location = env_root / "environment" - - environment = PipVenvManager().create_environment(location, "1.2.3") - - assert "cannot provide python 1.2.3" in caplog.text - assert environment.python_version() == PYTHON_VERSION diff --git a/src/ewoks/tests/requirements/test_pixi.py b/src/ewoks/tests/requirements/test_pixi.py deleted file mode 100644 index a63d82f..0000000 --- a/src/ewoks/tests/requirements/test_pixi.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Tests specific to the pixi package manager.""" - -import sys - -import pytest - -from ..._requirements.pixi import PixiManager -from ..._requirements.pixi import PixiRequirements -from ..._requirements.utils.conda_channel import python_specifier -from ..._requirements.utils.files import write_files -from .utils import DISTRIBUTIONS -from .utils import PYTHON_VERSION -from .utils import REQUIREMENTS -from .utils import manager_requirements - - -def test_workspace_environment(monkeypatch, tmp_path): - """The environment of a pixi workspace is inside the `.pixi` directory.""" - location = tmp_path / "workspace" - prefix = location / ".pixi" / "envs" / "default" - - assert PixiManager().environment(location).prefix == prefix - - monkeypatch.setattr(sys, "prefix", str(prefix)) - assert PixiManager().is_active() - - -def test_files_from_pixi_toml(monkeypatch, tmp_path): - """The files of the pixi workspace are reused. The pixi.toml takes precedence - over the pyproject.toml, like pixi itself.""" - files = {"pixi.toml": "# manifest\n", "pixi.lock": "# lock\n"} - write_files(tmp_path, {**files, "pyproject.toml": "# project\n"}) - monkeypatch.setattr(sys, "prefix", str(tmp_path / ".pixi" / "envs" / "default")) - - assert PixiManager()._gather_files([], PYTHON_VERSION) == files - - -def test_files_from_pyproject(monkeypatch, tmp_path): - """Without pixi.toml the pyproject.toml is the manifest of the workspace.""" - files = {"pyproject.toml": "# project\n", "pixi.lock": "# lock\n"} - write_files(tmp_path, files) - monkeypatch.setattr(sys, "prefix", str(tmp_path / ".pixi" / "envs" / "default")) - - assert PixiManager()._gather_files([], PYTHON_VERSION) == files - - -def test_files_from_lock_only(monkeypatch, tmp_path): - """A lock file without manifest is not reused: no files are stored.""" - write_files(tmp_path, {"pixi.lock": "# lock\n"}) - monkeypatch.setattr(sys, "prefix", str(tmp_path / ".pixi" / "envs" / "default")) - - assert PixiManager()._gather_files([], PYTHON_VERSION) == {} - - -def test_is_not_active(monkeypatch, tmp_path): - monkeypatch.delenv("PIXI_PROJECT_ROOT", raising=False) - monkeypatch.delenv("PIXI_ENVIRONMENT_NAME", raising=False) - monkeypatch.setattr(sys, "prefix", str(tmp_path)) - assert not PixiManager().is_active() - - -def test_pixi_environment_name_not_from_variable(monkeypatch, tmp_path): - """The name comes from the running python, not from an environment that pixi - activated for another interpreter.""" - monkeypatch.setenv("PIXI_ENVIRONMENT_NAME", "other") - monkeypatch.setattr(sys, "prefix", str(tmp_path / ".pixi" / "envs" / "test")) - assert PixiManager()._gather_extra_fields() == {"pixi_environment_name": "test"} - - -def test_pixi_environment_name_from_prefix(monkeypatch, tmp_path): - """The name comes from the environment prefix.""" - monkeypatch.setattr(sys, "prefix", str(tmp_path / ".pixi" / "envs" / "test")) - assert PixiManager()._gather_extra_fields() == {"pixi_environment_name": "test"} - - -def test_pixi_environment_name_default(monkeypatch, tmp_path): - """Outside a pixi workspace the environment is "default".""" - monkeypatch.setattr(sys, "prefix", str(tmp_path)) - assert PixiManager()._gather_extra_fields() == {} - requirements = PixiRequirements(**manager_requirements("pixi")) - assert requirements.manager.pixi_environment_name == "default" - - -def test_gather_pixi_environment_name(monkeypatch, tmp_path): - """The gathered requirements contain the name of the environment.""" - monkeypatch.setattr(PixiManager, "version", lambda self: "0.82.1") - monkeypatch.setattr(sys, "prefix", str(tmp_path / ".pixi" / "envs" / "test")) - requirements = PixiManager().gather_requirements() - assert requirements.manager.pixi_environment_name == "test" - - -def test_files_environment(tmp_path): - """The files are installed in the environment of the requirements.""" - data = manager_requirements("pixi") - data["manager"]["pixi_environment_name"] = "test" - requirements = PixiRequirements(**data) - manager = PixiManager() - environment = manager.files_environment(requirements, manager.environment(tmp_path)) - assert environment.location == tmp_path - assert environment.prefix == tmp_path / ".pixi" / "envs" / "test" - - -def test_windows_interpreter(monkeypatch, tmp_path): - """On Windows the interpreter is in the prefix, also before pixi creates the - environment.""" - monkeypatch.setattr(sys, "platform", "win32") - data = manager_requirements("pixi") - data["manager"]["pixi_environment_name"] = "test" - requirements = PixiRequirements(**data) - manager = PixiManager() - environment = manager.environment(tmp_path) - assert environment.python == tmp_path / ".pixi" / "envs" / "default" / "python.exe" - environment = manager.files_environment(requirements, environment) - assert environment.python == tmp_path / ".pixi" / "envs" / "test" / "python.exe" - - -@pytest.mark.skipif(PixiManager().version() is None, reason="pixi is not installed") -def test_manifest(): - manifest = PixiManager()._manifest(REQUIREMENTS, PYTHON_VERSION) - - assert f'python = "{python_specifier(PYTHON_VERSION)}"' in manifest - for dist in DISTRIBUTIONS: - assert f'"{dist.name}" = "=={dist.version}"' in manifest - assert "platforms = [" in manifest - - -def test_version_not_available(): - assert PixiManager("pixi-does-not-exist").version() is None diff --git a/src/ewoks/tests/requirements/test_poetry.py b/src/ewoks/tests/requirements/test_poetry.py deleted file mode 100644 index 3995285..0000000 --- a/src/ewoks/tests/requirements/test_poetry.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Tests specific to the poetry package manager.""" - -import sys -import venv - -import pytest - -from ..._requirements.poetry import PoetryManager -from ..._requirements.poetry import _pyproject -from ..._requirements.utils.base_manager import EWOKS_ENVIRONMENTS_ROOT -from ..._requirements.utils.files import write_files -from .utils import DISTRIBUTIONS -from .utils import PYTHON_VERSION -from .utils import REQUIREMENTS - - -def test_pyproject(): - """The project is not a package: there is nothing to build. Dependencies are - declared in the legacy format, which every supported poetry version reads.""" - pyproject = _pyproject(REQUIREMENTS, PYTHON_VERSION) - - assert "package-mode = false" in pyproject - assert f'python = "=={PYTHON_VERSION}"' in pyproject - for dist in DISTRIBUTIONS: - assert f'"{dist.name}" = "=={dist.version}"' in pyproject - assert "[project]" not in pyproject - - -def test_project_environment(monkeypatch, tmp_path): - """The environment of a poetry project is the `.venv` directory.""" - location = tmp_path / "project" - prefix = location / ".venv" - - assert PoetryManager().environment(location).prefix == prefix - - monkeypatch.delenv("POETRY_ACTIVE", raising=False) - monkeypatch.setattr(sys, "prefix", str(prefix)) - assert not PoetryManager().is_active() - - location.mkdir() - with open(location / "poetry.lock", "w", encoding="utf-8") as fh: - fh.write("# lock\n") - assert PoetryManager().is_active() - - -def test_files_from_project(monkeypatch, tmp_path): - """The files of the poetry project are reused.""" - files = {"pyproject.toml": "# project\n", "poetry.lock": "# lock\n"} - write_files(tmp_path, files) - monkeypatch.setattr(sys, "prefix", str(tmp_path / ".venv")) - - assert PoetryManager()._gather_files([], PYTHON_VERSION) == files - - -def test_files_from_lock_only(monkeypatch, tmp_path): - """A lock file without pyproject.toml is not reused: no files are stored.""" - write_files(tmp_path, {"poetry.lock": "# lock\n"}) - monkeypatch.setattr(sys, "prefix", str(tmp_path / ".venv")) - - assert PoetryManager()._gather_files([], PYTHON_VERSION) == {} - - -def test_is_active_poetry_run(monkeypatch, tmp_path): - monkeypatch.setattr(sys, "prefix", str(tmp_path)) - monkeypatch.setenv("POETRY_ACTIVE", "1") - assert PoetryManager().is_active() - - -def test_version_not_available(): - assert PoetryManager("poetry-does-not-exist").version() is None - - -@pytest.mark.skipif(PoetryManager().version() is None, reason="poetry is not installed") -def test_version(): - version = PoetryManager().version() - assert version[0].isdigit() - - -@pytest.mark.skipif(PoetryManager().version() is None, reason="poetry is not installed") -def test_environments_root(monkeypatch, tmp_path): - """Named environments are created where poetry creates them itself.""" - monkeypatch.setenv("POETRY_VIRTUALENVS_PATH", str(tmp_path)) - - assert PoetryManager().environments_root() == tmp_path - - -@pytest.mark.skipif(PoetryManager().version() is None, reason="poetry is not installed") -@pytest.mark.parametrize("variable", ["VIRTUAL_ENV", "CONDA_PREFIX"]) -def test_create_environment_with_active_environment(variable, monkeypatch, tmp_path): - """The environment is the one of the project, also when another environment is - active, for example the one of `poetry run`.""" - active = tmp_path / "active" - venv.create(active) - monkeypatch.setenv(variable, str(active)) - # Poetry ignores the base environment of conda - monkeypatch.setenv("CONDA_DEFAULT_ENV", "active") - - environment = PoetryManager().create_environment( - tmp_path / "project", PYTHON_VERSION - ) - - assert environment.exists() - - -def test_environments_root_not_available(caplog): - """Environments of ewoks when poetry cannot provide its environment directory.""" - manager = PoetryManager("poetry-does-not-exist") - - assert manager.environments_root() == EWOKS_ENVIRONMENTS_ROOT - - assert "Cannot determine where poetry creates environments" in caplog.text diff --git a/src/ewoks/tests/requirements/test_requirements_txt.py b/src/ewoks/tests/requirements/test_requirements_txt.py deleted file mode 100644 index 4efca7b..0000000 --- a/src/ewoks/tests/requirements/test_requirements_txt.py +++ /dev/null @@ -1,218 +0,0 @@ -"""Tests of the `requirements.txt` format.""" - -import pytest - -from ..._requirements.utils import requirements_txt -from ..._requirements.utils.metadata import models -from .utils import DISTRIBUTIONS -from .utils import REQUIREMENTS - - -def test_normal_requirement(caplog): - assert requirements_txt.sanitize(["ewoks==1.1.0"]) == ["ewoks==1.1.0"] - assert not caplog.text - - -def test_editable_ssh_vcs_url_normalized(caplog): - ssh_project_name = "querypool" - ssh_project_url = "gitlab.esrf.fr/dau/querypool.git" - ssh_project_commit = "ab6acc7e140ed33eb896b1336a5a5aac6b60cc0f" - requirement = f"-e git+ssh://git@{ssh_project_url}@{ssh_project_commit}#egg={ssh_project_name}" - - sanitized = requirements_txt.sanitize([requirement]) - - assert sanitized == [ - f"{ssh_project_name} @ git+https://{ssh_project_url}@{ssh_project_commit}" - ] - assert ( - f"Normalize VCS requirement 'git+ssh://git@{ssh_project_url}@{ssh_project_commit}" - f"#egg={ssh_project_name}'" in caplog.text - ) - assert "to 'querypool @ git+https" in caplog.text - - -@pytest.mark.parametrize("exists", [True, False]) -def test_editable_local_path_with_comment_replacement(tmp_path, caplog, exists): - path = tmp_path / "repo_name" - if exists: - path.mkdir() - - comment = "# Editable Git install with no remote (project_name==1.0.0)" - replacement = "project_name==1.0.0" - - sanitized = requirements_txt.sanitize([comment, f"-e {path}"]) - - assert sanitized == [replacement] - assert f"Replaced editable install '{path}' with '{replacement}'." in caplog.text - - -@pytest.mark.parametrize("exists", [True, False]) -def test_editable_local_path_without_comment_replacement(tmp_path, caplog, exists): - path = tmp_path / "repo_name" - if exists: - path.mkdir() - warning = f"Editable path exists locally: '{path}'" - else: - warning = f"Editable path does not exist locally: '{path}'" - - sanitized = requirements_txt.sanitize([f"-e {path}"]) - - assert sanitized == [f"-e {path}"] - assert warning in caplog.text - - -def test_branch_specified_requirement(caplog): - project_name = "ewoksutils" - project_url = "github.com/ewoks-kit/ewoksutils.git" - requirement = f"{project_name}@ git+https://{project_url}@main" - - assert requirements_txt.sanitize([requirement]) == [requirement] - assert not caplog.text - - -def test_invalid_requirement_warning(caplog): - requirement = "git+https://github.com/ewoks-kit/ewoksutils.git" - - assert requirements_txt.sanitize([requirement]) == [requirement] - assert "Possibly invalid requirement format" in caplog.text - - -def test_distributions_requirements(): - distributions = [ - *DISTRIBUTIONS, - models.Distribution( - name="mypackage1", - version="1.0", - git=models.GitInfo(commit="123", remote="https://host/group/repo.git"), - ), - models.Distribution( - name="mypackage2", - version="1.0", - archive=models.ArchiveInfo( - url="https://host/mypackage2-1.0-py3-none-any.whl", - hashes={"sha256": "123"}, - ), - ), - ] - - assert requirements_txt.distributions_requirements(distributions) == [ - *REQUIREMENTS, - "mypackage1 @ git+https://host/group/repo.git@123", - # The hash is a URL fragment: pip requires a hash for every requirement as - # soon as one is provided as a '--hash' option - "mypackage2 @ https://host/mypackage2-1.0-py3-none-any.whl#sha256=123", - ] - assert requirements_txt.manifest_requirements( - distributions - ) == requirements_txt.distributions_requirements(distributions) - assert requirements_txt.version_constraints(distributions) == [ - *REQUIREMENTS, - "mypackage1==1.0", - "mypackage2==1.0", - ] - - -def test_manifest_requirements_unique(): - """A package manager manifest cannot contain the same dependency twice.""" - # The same distributions, but named differently and with another version - duplicates = [ - dist.model_copy(update={"name": dist.name.upper(), "version": "0.0.1"}) - for dist in DISTRIBUTIONS - ] - distributions = DISTRIBUTIONS + duplicates - - assert requirements_txt.manifest_requirements(distributions) == REQUIREMENTS - - -def test_manifest_requirements_without_comments(): - """Warnings that `pip freeze` adds as comments are not requirements.""" - distributions = [ - models.Distribution( - name="mypackage", version="1.0", git=models.GitInfo(commit="123") - ) - ] - - requirements = requirements_txt.distributions_requirements(distributions) - assert requirements[0].startswith("# ") - - assert requirements_txt.manifest_requirements(distributions) == ["mypackage==1.0"] - - -@pytest.mark.parametrize( - "requirement,expected", - [ - *( - ( - f"{dist.name}=={dist.version}", - {"name": dist.name, "version": dist.version}, - ) - for dist in DISTRIBUTIONS - ), - ("mypackage", {"name": "mypackage", "version": ""}), - ("mypackage>=1.0", {"name": "mypackage", "version": ""}), - ( - "mypackage @ git+https://host/repo.git@123", - { - "name": "mypackage", - "version": "", - "git": { - "commit": "123", - "remote": "https://host/repo.git", - "uncommitted_changes": False, - }, - }, - ), - ( - "mypackage @ https://host/mypackage-1.0.tar.gz", - { - "name": "mypackage", - "version": "", - "archive": {"url": "https://host/mypackage-1.0.tar.gz", "hashes": {}}, - }, - ), - ( - "mypackage @ https://host/mypackage-1.0.tar.gz#sha256=123", - { - "name": "mypackage", - "version": "", - "archive": { - "url": "https://host/mypackage-1.0.tar.gz", - "hashes": {"sha256": "123"}, - }, - }, - ), - ( - # Only the hashes are taken out of the fragment - "mypackage @ https://host/repo.tar.gz#sha256=123&subdirectory=sub", - { - "name": "mypackage", - "version": "", - "archive": { - "url": "https://host/repo.tar.gz#subdirectory=sub", - "hashes": {"sha256": "123"}, - }, - }, - ), - ( - # A git URL without revision cannot be split - "mypackage @ git+https://host/repo.git", - { - "name": "mypackage", - "version": "", - "archive": {"url": "git+https://host/repo.git", "hashes": {}}, - }, - ), - ], -) -def test_distribution_from_requirement(requirement, expected): - distribution = requirements_txt.distribution_from_requirement(requirement) - - assert distribution is not None - assert distribution.model_dump(exclude_none=True) == expected - - -def test_distribution_from_invalid_requirement(caplog): - assert ( - requirements_txt.distribution_from_requirement("git+https://host/repo") is None - ) - assert "Skip invalid requirement" in caplog.text diff --git a/src/ewoks/tests/requirements/test_toml_dependencies.py b/src/ewoks/tests/requirements/test_toml_dependencies.py deleted file mode 100644 index 18357e0..0000000 --- a/src/ewoks/tests/requirements/test_toml_dependencies.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Tests of the dependencies of a TOML manifest.""" - -from ..._requirements.utils import toml_dependencies -from .utils import DISTRIBUTIONS -from .utils import REQUIREMENTS - - -def test_dependency_lines(): - """Dependencies of a TOML manifest are not PEP 508 requirements.""" - requirements = [ - *REQUIREMENTS, - "mypackag1", - "mypackage2 @ git+https://host/group/repo.git@123", - "mypackage3 @ https://host/mypackage3-1.0-py3-none-any.whl", - ] - - assert toml_dependencies.dependency_lines(requirements) == [ - *(f'"{dist.name}" = "=={dist.version}"' for dist in DISTRIBUTIONS), - '"mypackag1" = "*"', - '"mypackage2" = { git = "https://host/group/repo.git", rev = "123" }', - '"mypackage3" = { url = "https://host/mypackage3-1.0-py3-none-any.whl" }', - ] - - -def test_invalid_dependency(caplog): - assert toml_dependencies.dependency_lines(["git+https://host/repo"]) == [] - assert "Skip invalid requirement" in caplog.text diff --git a/src/ewoks/tests/requirements/test_uv.py b/src/ewoks/tests/requirements/test_uv.py deleted file mode 100644 index 48fe2f6..0000000 --- a/src/ewoks/tests/requirements/test_uv.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Tests specific to the uv package manager.""" - -import os -import shutil -import sys -from pathlib import Path -from typing import Iterator - -import pytest - -from ..._requirements.utils.environment import Environment -from ..._requirements.utils.files import write_files -from ..._requirements.uv import UvManager -from .utils import PYTHON_VERSION - - -@pytest.fixture -def uv_environment(tmp_path) -> Iterator[Environment]: - """Environment of a uv project.""" - if UvManager().version() is None: - pytest.skip("uv is not installed") - location = tmp_path / "environment" - yield UvManager().create_environment(location) - shutil.rmtree(location, ignore_errors=True) - - -@pytest.fixture -def no_uv_environment(monkeypatch, tmp_path) -> Iterator[Path]: - """Python environment that was not created by uv and is not a uv project.""" - monkeypatch.delenv("UV", raising=False) - prefix = tmp_path / "project" / "prefix" - prefix.mkdir(parents=True) - with open(prefix / "pyvenv.cfg", "w", encoding="utf-8") as fh: - fh.write(f"home = {os.path.dirname(sys.executable)}\n") - monkeypatch.setattr(sys, "prefix", str(prefix)) - yield prefix - - -def test_default_command(): - manager = UvManager() - assert manager._cmd_args == ("uv",) - assert repr(manager) == "UvManager(uv)" - - -def test_version_not_available(): - assert UvManager("uv-does-not-exist").version() is None - - -def test_files_from_project(monkeypatch, tmp_path): - """The files of the uv project are reused.""" - files = {"pyproject.toml": "# project\n", "uv.lock": "# lock\n"} - write_files(tmp_path, files) - monkeypatch.setattr(sys, "prefix", str(tmp_path / ".venv")) - - assert UvManager()._gather_files([], PYTHON_VERSION) == files - - -def test_files_from_lock_only(monkeypatch, tmp_path): - """A lock file without pyproject.toml is not reused: no files are stored.""" - write_files(tmp_path, {"uv.lock": "# lock\n"}) - monkeypatch.setattr(sys, "prefix", str(tmp_path / ".venv")) - - assert UvManager()._gather_files([], PYTHON_VERSION) == {} - - -def test_project_environment(uv_environment): - """The environment of a uv project is the `.venv` directory.""" - assert uv_environment.prefix == uv_environment.location / ".venv" - assert Environment.from_location(uv_environment.location) == uv_environment - - -def test_is_active_uv_environment(uv_environment, monkeypatch): - monkeypatch.delenv("UV", raising=False) - monkeypatch.setattr(sys, "prefix", str(uv_environment.prefix)) - assert UvManager().is_active() - - -def test_is_active_uv_run(no_uv_environment, monkeypatch): - """`uv run` provides the uv executable through the environment.""" - assert not UvManager().is_active() - monkeypatch.setenv("UV", "/path/to/uv") - assert UvManager().is_active() - - -def test_is_active_uv_project(no_uv_environment, monkeypatch): - """The environment is inside a uv project.""" - assert not UvManager().is_active() - with open(no_uv_environment.parent / "uv.lock", "w", encoding="utf-8") as fh: - fh.write("version = 1\n") - assert UvManager().is_active() - - -def test_is_not_active(no_uv_environment): - assert not UvManager().is_active() diff --git a/src/ewoks/tests/requirements/utils.py b/src/ewoks/tests/requirements/utils.py deleted file mode 100644 index 16295bd..0000000 --- a/src/ewoks/tests/requirements/utils.py +++ /dev/null @@ -1,144 +0,0 @@ -import platform -import sys -from typing import Any -from typing import Dict -from typing import List -from typing import Mapping -from typing import Optional -from typing import Tuple - -from ewokscore.graph import TaskGraph - -from ..._requirements.utils.environment import Environment -from ..._requirements.utils.metadata import models -from ..._requirements.utils.parse import parse_requirements - -PYTHON_VERSION = platform.python_version() -"""Python version of the environments created by the tests. The version at -runtime is used because a package manager can only provide a python version that -is available on the machine.""" - -PYTHON_MINOR_VERSION = ".".join(PYTHON_VERSION.split(".")[:2]) -"""Major and minor version of `PYTHON_VERSION`.""" - - -def _pinned(name: str, versions: Mapping[Tuple[int, int], str]) -> models.Distribution: - """Distribution pinned to a version the running python supports. The versions - are keyed by the lowest python version they support.""" - version = next( - version - for python_version, version in sorted(versions.items(), reverse=True) - if python_version <= sys.version_info[:2] - ) - return models.Distribution(name=name, version=version) - - -DISTRIBUTIONS = [ - _pinned("six", {(3, 8): "1.17.0"}), - _pinned("iniconfig", {(3, 8): "2.0.0"}), -] -"""Small distributions used to verify that an installation took place. Unlike the -python version, the versions cannot be derived at runtime: they must be exact -versions that a package manager would not install by itself.""" - -REQUIREMENTS = [f"{dist.name}=={dist.version}" for dist in DISTRIBUTIONS] -"""`DISTRIBUTIONS` as requirements.""" - -EWOKS_DEPENDENCIES = [ - _pinned("networkx", {(3, 8): "3.1", (3, 10): "3.4.2"}), - _pinned("packaging", {(3, 8): "24.1"}), -] -"""Older versions of distributions that ewoks depends on, to verify that adding -ewoks to an environment does not change the versions of the requirements.""" - - -def assert_installed( - environment: Environment, distributions: List[models.Distribution] -) -> None: - """Every distribution of `distributions` is installed with its exact version.""" - assert _installed_versions(environment, distributions) == { - dist.name: dist.version for dist in distributions - } - - -def assert_not_installed( - environment: Environment, distributions: List[models.Distribution] -) -> None: - """No distribution of `distributions` is installed.""" - assert _installed_versions(environment, distributions) == { - dist.name: None for dist in distributions - } - - -def _installed_versions( - environment: Environment, distributions: List[models.Distribution] -) -> Dict[str, Optional[str]]: - return { - dist.name: environment.distribution_version(dist.name) for dist in distributions - } - - -def assert_in_graph_requirements(graph: TaskGraph, *distribution_names) -> None: - assert distribution_names, "no names provides" - assert "requirements" in graph.graph.graph, "no requirements" - requirements = parse_requirements(graph.graph.graph["requirements"]) - - existing = {distribution.name for distribution in requirements.distributions} - not_existing = set(distribution_names) - existing - assert not not_existing, f"{sorted(not_existing)} not in requirements" - - -def current_metadata( - distributions: List[models.Distribution], - python_version: str, -) -> Dict[str, Any]: - """Metadata of a python environment with few distributions: the current - environment can have hundreds, which some package managers take minutes to - resolve. - """ - return { - "system": { - "system": "", - "release": "", - "version": "", - "machine": "", - "processor": "", - }, - "python": { - "version": python_version, - "implementation": "", - "compiler": "", - "build": "", - }, - "distributions": distributions, - } - - -def manager_requirements( - manager_name: str, - files: Optional[Dict[str, str]] = None, - distributions: Optional[List[models.Distribution]] = None, - python_version: str = "", -) -> Dict[str, Any]: - """Workflow requirements as generated by a package manager.""" - return { - "system": { - "system": "", - "release": "", - "version": "", - "machine": "", - "processor": "", - }, - "python": { - "version": python_version, - "implementation": "", - "compiler": "", - "build": "", - }, - "distributions": [dist.model_dump() for dist in distributions or list()], - "manager": { - "name": manager_name, - "version": "", - "files": files or dict(), - }, - } From 9a47d1fd920f00272b157634841abb3db0b79a07 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 11 Nov 2026 17:02:22 +0100 Subject: [PATCH 2/4] Use the workflow requirements of ewokscore Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 2 +- src/ewoks/__main__.py | 2 +- src/ewoks/bindings.py | 20 +++++++++++++++---- src/ewoks/cli_utils/cli_arguments.py | 2 +- src/ewoks/cli_utils/cli_install_utils.py | 4 ++-- src/ewoks/tests/requirements/conftest.py | 4 ++++ .../tests/requirements/test_bootstrap.py | 2 +- .../tests/requirements/test_install_cli.py | 16 +++++++-------- src/ewoks/tests/test_convert_cli.py | 20 ++++++++++--------- src/ewoks/tests/test_execute_cli.py | 2 +- 10 files changed, 46 insertions(+), 28 deletions(-) create mode 100644 src/ewoks/tests/requirements/conftest.py diff --git a/pyproject.toml b/pyproject.toml index 6b60671..0ee2c8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ ] requires-python = ">=3.8" dependencies = [ - "ewokscore >=5,<6", + "ewokscore >=5.1.0rc7,<6", "ewoksutils >=2.0.0rc1,<3", "tabulate", "importlib_resources; python_version < '3.9'", diff --git a/src/ewoks/__main__.py b/src/ewoks/__main__.py index 240b318..04b518e 100644 --- a/src/ewoks/__main__.py +++ b/src/ewoks/__main__.py @@ -16,12 +16,12 @@ from typing import Optional from typing import Union +from ewokscore.requirements.environment import Environment from ewoksutils.cli_utils import cli_cancel_utils from ewoksutils.cli_utils import cli_execute_utils from ewoksutils.cli_utils import cli_submit_utils from ewoksutils.cli_utils.cli_argparse import add_to_parser -from ._requirements.utils.environment import Environment from .bindings import _load_graph from .bindings import convert_graph from .bindings import execute_graph diff --git a/src/ewoks/bindings.py b/src/ewoks/bindings.py index 37f5a1a..fb65ae5 100644 --- a/src/ewoks/bindings.py +++ b/src/ewoks/bindings.py @@ -1,4 +1,5 @@ import datetime +import importlib.metadata import logging import os import re @@ -15,18 +16,19 @@ from typing import Tuple from typing import Union +from ewokscore import requirements as _requirements +from ewokscore.engine_interface import WorkflowEngine from ewokscore.events.contexts import RawExecInfoType from ewokscore.events.contexts import job_context from ewokscore.graph import TaskGraph from ewokscore.graph.inputs import graph_inputs_as_table from ewokscore.graph.schema.model import EwoksGraph +from ewokscore.requirements.environment import Environment from pydantic import ValidationError from tabulate import tabulate from . import _engines -from . import _requirements from . import graph_cache -from ._requirements.utils.environment import Environment from .errors import AbortException try: @@ -229,7 +231,7 @@ def convert_graph( graph, manager_name=package_manager_name, manager_command=package_manager_command, - engine=_engines.get_engine_distribution(engine), + bootstrap=_bootstrap_distributions(engine), ) logger.info("Requirements generated with the %r package manager", manager_name) else: @@ -242,6 +244,16 @@ def convert_graph( return save_graph(graph, destination, **save_options) +def _bootstrap_distributions(engine: WorkflowEngine) -> Dict[str, str]: + """Version of ewoks and of the distribution of the engine that saves a workflow.""" + distributions = {"ewoks": importlib.metadata.version("ewoks")} + engine_distribution = _engines.get_engine_distribution(engine) + if engine_distribution: + name, version = engine_distribution + distributions[name] = version + return distributions + + def show_graph( source, inputs: Optional[List[dict]] = None, @@ -329,7 +341,7 @@ def install_graph( manager_name=package_manager_name, manager_command=package_manager_command, python_version=python_version, - ensure_ewoks=ensure_ewoks, + extra_distributions=("ewoks",) if ensure_ewoks else tuple(), clean=clean, confirm=None if skip_prompt else _confirm_installation, ) diff --git a/src/ewoks/cli_utils/cli_arguments.py b/src/ewoks/cli_utils/cli_arguments.py index 59e42a8..6a050f2 100644 --- a/src/ewoks/cli_utils/cli_arguments.py +++ b/src/ewoks/cli_utils/cli_arguments.py @@ -1,9 +1,9 @@ from typing import List +from ewokscore.requirements import supported_managers from ewoksutils.cli_utils.cli_arguments import CLIArg from .._engines import get_graph_representations -from .._requirements import supported_managers def package_manager_arguments(action: str) -> List[CLIArg]: diff --git a/src/ewoks/cli_utils/cli_install_utils.py b/src/ewoks/cli_utils/cli_install_utils.py index 63fa2de..847767e 100644 --- a/src/ewoks/cli_utils/cli_install_utils.py +++ b/src/ewoks/cli_utils/cli_install_utils.py @@ -2,13 +2,13 @@ from argparse import Namespace from typing import List +from ewokscore.requirements import managers_supporting_in_place +from ewokscore.requirements.environment import EWOKS_ENVIRONMENTS_ROOT from ewoksutils.cli_utils import cli_arguments from ewoksutils.cli_utils import cli_log_utils from ewoksutils.cli_utils import cli_parse from ewoksutils.cli_utils.cli_spec import CLIArg -from .._requirements import managers_supporting_in_place -from .._requirements.utils.base_manager import EWOKS_ENVIRONMENTS_ROOT from .cli_arguments import package_manager_arguments logger = logging.getLogger(__name__) diff --git a/src/ewoks/tests/requirements/conftest.py b/src/ewoks/tests/requirements/conftest.py new file mode 100644 index 0000000..5c484ba --- /dev/null +++ b/src/ewoks/tests/requirements/conftest.py @@ -0,0 +1,4 @@ +from ewokscore.tests.requirements.conftest import env_root # noqa: F401 +from ewokscore.tests.requirements.conftest import fast_manager_case # noqa: F401 +from ewokscore.tests.requirements.conftest import isolated_home # noqa: F401 +from ewokscore.tests.requirements.conftest import package_manager_home # noqa: F401 diff --git a/src/ewoks/tests/requirements/test_bootstrap.py b/src/ewoks/tests/requirements/test_bootstrap.py index da33173..f3338bf 100644 --- a/src/ewoks/tests/requirements/test_bootstrap.py +++ b/src/ewoks/tests/requirements/test_bootstrap.py @@ -19,10 +19,10 @@ from typing import Optional import pytest +from ewokscore.tests.requirements.utils import PYTHON_MINOR_VERSION from ...bindings import convert_graph from ...bindings import load_graph -from .utils import PYTHON_MINOR_VERSION _SCRIPTS = Path(__file__).parents[2] / "_bootstrap" _INSTALLABLE_MANAGERS = ["uv", "pixi", "poetry", "conda"] diff --git a/src/ewoks/tests/requirements/test_install_cli.py b/src/ewoks/tests/requirements/test_install_cli.py index 3082fd0..22d6282 100644 --- a/src/ewoks/tests/requirements/test_install_cli.py +++ b/src/ewoks/tests/requirements/test_install_cli.py @@ -12,18 +12,18 @@ from typing import Optional import pytest +from ewokscore.requirements import environment_name +from ewokscore.requirements.environment import Environment +from ewokscore.tests.requirements.managers import ManagerCase +from ewokscore.tests.requirements.utils import DISTRIBUTIONS +from ewokscore.tests.requirements.utils import PYTHON_VERSION +from ewokscore.tests.requirements.utils import REQUIREMENTS +from ewokscore.tests.requirements.utils import assert_installed +from ewokscore.tests.requirements.utils import manager_requirements from ...__main__ import _remove_command from ...__main__ import _shell_command -from ..._requirements import environment_name -from ..._requirements.utils.environment import Environment from ...bindings import load_graph -from .managers import ManagerCase -from .utils import DISTRIBUTIONS -from .utils import PYTHON_VERSION -from .utils import REQUIREMENTS -from .utils import assert_installed -from .utils import manager_requirements _GRAPH_ID = "test_install" """Identifier of the workflows to install, which is also the name of the diff --git a/src/ewoks/tests/test_convert_cli.py b/src/ewoks/tests/test_convert_cli.py index 911e331..c2b3edd 100644 --- a/src/ewoks/tests/test_convert_cli.py +++ b/src/ewoks/tests/test_convert_cli.py @@ -6,14 +6,14 @@ import pytest from defusedxml import ElementTree from ewokscore import load_graph +from ewokscore import requirements from ewokscore.task import Task from ewokscore.tests.examples.graphs import graph_names +from ewokscore.tests.requirements.utils import assert_in_graph_requirements from ewoksutils.import_utils import import_qualname from orangewidget.widget import OWBaseWidget -from .. import _requirements from ..__main__ import main -from .requirements.utils import assert_in_graph_requirements from .utils import has_default_input from .utils import no_widget_registry @@ -36,9 +36,11 @@ def test_convert_to_json(graph_name, tmpdir): graph = load_graph(destination) assert_in_graph_requirements(graph, "ewokscore") - assert graph.graph.graph["requirements"]["ewoks"]["engine"] == { - "name": "ewokscore", - "version": importlib.metadata.version("ewokscore"), + assert graph.graph.graph["requirements"]["bootstrap"] == { + "distributions": { + "ewoks": importlib.metadata.version("ewoks"), + "ewokscore": importlib.metadata.version("ewokscore"), + } } @@ -92,7 +94,7 @@ def test_convert_requirements_failure(tmpdir, monkeypatch): def add_requirements(*args, **kwargs) -> None: raise RuntimeError("no package manager") - monkeypatch.setattr(_requirements, "add_requirements", add_requirements) + monkeypatch.setattr(requirements, "add_requirements", add_requirements) destination = str(tmpdir / "demo.json") argv = [ @@ -140,9 +142,9 @@ def test_convert_to_ows(graph_name, tmpdir): root = tree.getroot() graph_attrs = json.loads(root.find("./ewoks_graph_attrs").text) - assert graph_attrs["requirements"]["ewoks"]["engine"] == { - "name": "ewoksorange", - "version": importlib.metadata.version("ewoksorange"), + assert graph_attrs["requirements"]["bootstrap"]["distributions"] == { + "ewoks": importlib.metadata.version("ewoks"), + "ewoksorange": importlib.metadata.version("ewoksorange"), } for node in root.findall("./nodes/node"): diff --git a/src/ewoks/tests/test_execute_cli.py b/src/ewoks/tests/test_execute_cli.py index 5fd0bba..5b29017 100644 --- a/src/ewoks/tests/test_execute_cli.py +++ b/src/ewoks/tests/test_execute_cli.py @@ -6,10 +6,10 @@ from ewokscore import load_graph from ewokscore.tests.examples.graphs import get_graph from ewokscore.tests.examples.graphs import graph_names +from ewokscore.tests.requirements.utils import assert_in_graph_requirements from ewokscore.tests.utils.results import assert_execute_graph_default_result from ..__main__ import main -from .requirements.utils import assert_in_graph_requirements from .utils import has_default_input From 643897c22d39c353d4f2657a249ddb99a3436b15 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 11 Nov 2026 16:41:03 +0100 Subject: [PATCH 3/4] Bootstrap scripts: install the bootstrap distributions of the workflow requirements Co-Authored-By: Claude Opus 5.5 --- doc/conf.py | 2 +- doc/howtoguides/requirements.rst | 6 +- doc/tutorials/install.rst | 16 +- src/ewoks/_bootstrap/ewoks-install.ps1 | 152 +++++++-------- src/ewoks/_bootstrap/ewoks-install.sh | 181 ++++++++++-------- .../tests/requirements/test_bootstrap.py | 14 +- 6 files changed, 197 insertions(+), 174 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 2ad8f91..0f6b405 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -111,7 +111,7 @@ def _save_demo_workflow(app: Sphinx, exception: Optional[Exception]) -> None: graph, _ = get_graph("demo") workflow = convert_graph(graph, None) requirements = workflow["graph"]["requirements"] - names = ["ewoks", requirements["ewoks"]["engine"]["name"]] + names = list(requirements["bootstrap"]["distributions"]) requirements["distributions"] = [ {"name": name, "version": importlib.metadata.version(name)} for name in names ] diff --git a/doc/howtoguides/requirements.rst b/doc/howtoguides/requirements.rst index 755a4f0..32b0b0d 100644 --- a/doc/howtoguides/requirements.rst +++ b/doc/howtoguides/requirements.rst @@ -13,9 +13,9 @@ What is stored ``ewoks convert`` and ``ewoks execute -o convert_destination=...`` store -* ``ewoks``: the version of :term:`ewoks` that generated the requirements and the engine - that saved the :term:`workflow`. The - :ref:`ewoks-install script ` installs both to run ``ewoks install``. +* ``bootstrap``: the versions of :term:`ewoks` and of the engine that saved the + :term:`workflow`. The :ref:`ewoks-install script ` installs them to run + ``ewoks install``. * ``python`` and ``system``: the python interpreter and the operating system. * ``distributions``: every installed python package with its version and, when it was not installed from the python package index, the git commit or the archive it came from. Any diff --git a/doc/tutorials/install.rst b/doc/tutorials/install.rst index f9f3117..9f00d7e 100644 --- a/doc/tutorials/install.rst +++ b/doc/tutorials/install.rst @@ -40,7 +40,7 @@ Store the requirements .. code-block:: json { - "ewoks": {"version": "7.0.0", "engine": {"name": "ewokscore", "version": "5.1.0"}}, + "bootstrap": {"distributions": {"ewoks": "7.0.0", "ewokscore": "5.1.0"}}, "python": {"version": "3.12.11", "implementation": "CPython", "...": "..."}, "system": {"system": "Linux", "machine": "x86_64", "...": "..."}, "distributions": [ @@ -54,9 +54,9 @@ Store the requirements } } -* ``ewoks`` is the version of :term:`ewoks` that generated the requirements and the engine that - saved the :term:`workflow`: ``ewokscore`` for a ``.json`` or ``.yaml`` file and - ``ewoksorange`` for an ``.ows`` file. +* ``bootstrap`` are the versions of :term:`ewoks` and of the engine that saved the + :term:`workflow`: ``ewokscore`` for a ``.json`` or ``.yaml`` file and ``ewoksorange`` for an + ``.ows`` file. * ``distributions`` are the installed python packages. Any :term:`package manager` can recreate the environment from this list. * ``manager`` is the :term:`package manager` that generated the requirements. Its ``files`` @@ -90,9 +90,9 @@ Create the environment without ewoks ++++++++++++++++++++++++++++++++++++ ``ewoks install`` needs a Python environment with :term:`ewoks`. The ``ewoks-install`` script -only needs a :term:`package manager`: it installs the version of :term:`ewoks` that generated -the requirements, and the engine that saved the :term:`workflow`, in a bootstrap environment -and runs ``ewoks install`` from there. All +only needs a :term:`package manager`: it installs the ``bootstrap`` packages of the +requirements, :term:`ewoks` and the engine that saved the :term:`workflow`, in a bootstrap +environment and runs ``ewoks install`` from there. All arguments are passed to ``ewoks install``, except for the options of the script itself .. tabs:: @@ -170,7 +170,7 @@ Alternatively, download the script first, for example to read it before running :term:`package manager` that creates the bootstrap environment. Without them it is the first one that is available of uv, pixi, conda, poetry and pip-venv. * ``--bootstrap-requirement`` installs a package in the bootstrap environment instead of the - version of :term:`ewoks` and the engine of the requirements, for example + ``bootstrap`` packages of the requirements, for example ``--bootstrap-requirement "ewoks>=7"``. Repeat it to install an engine as well, for example ``--bootstrap-requirement ewoks --bootstrap-requirement ewoksorange``. The latest version of :term:`ewoks` is installed when the requirements do not provide one. diff --git a/src/ewoks/_bootstrap/ewoks-install.ps1 b/src/ewoks/_bootstrap/ewoks-install.ps1 index 097c1b5..665de37 100644 --- a/src/ewoks/_bootstrap/ewoks-install.ps1 +++ b/src/ewoks/_bootstrap/ewoks-install.ps1 @@ -12,8 +12,8 @@ # # 1. Split the arguments in options of this script and arguments of `ewoks install`. # 2. Select the package manager. -# 3. Read the ewoks, engine and python versions from the workflow requirements. -# 4. Create a bootstrap environment with that ewoks and engine, unless it exists. +# 3. Read the bootstrap distributions and python version from the workflow requirements. +# 4. Create a bootstrap environment with those distributions, unless it exists. # 5. Run `ewoks install` in the bootstrap environment. # # The bootstrap environments are cached in ~/.ewoks/bootstrap// @@ -29,9 +29,9 @@ $usage = @' Usage: ewoks-install.ps1 [OPTIONS] WORKFLOW [ARGS ...] Install the python environment of an ewoks workflow with nothing but a package -manager. The ewoks version that generated the workflow requirements, and the -engine that saved the workflow, are installed in a bootstrap environment, which -then runs +manager. The bootstrap distributions of the workflow requirements, ewoks and +the engine that saved the workflow, are installed in a bootstrap environment, +which then runs ewoks install WORKFLOW ARGS @@ -45,7 +45,7 @@ directory with '--install-package-manager'. Options: --bootstrap-requirement REQ Install REQ in the bootstrap environment instead of - the ewoks version and engine of the workflow, for + the bootstrap distributions of the workflow, for example 'ewoks>=7' or a wheel file. Repeat it to install an engine as well, for example '--bootstrap-requirement ewoks @@ -195,52 +195,53 @@ function Get-DistributionVersion([string]$python, [string]$name) { # Workflow requirements # ============================================================================= -# Value in the requirements of a workflow file: ewoks.version, ewoks.engine.name, -# ewoks.engine.version or python.version. The requirements are JSON, which can be +# Items of the object at a key path of the requirements of a workflow file: python +# or bootstrap.distributions, sorted by key. The requirements are JSON, which can be # embedded in another format like the XML of .ows files, or YAML. -function Get-RequirementValue([string]$content, [string]$key) { - $value = Get-JsonRequirementValue $content $key - if (-not $value) { - $value = Get-YamlRequirementValue $content $key +function Get-RequirementItems([string]$content, [string]$path) { + $items = Get-JsonRequirementItems $content $path + if ($items.Count -eq 0) { + $items = Get-YamlRequirementItems $content $path } - return $value + return ,$items } -# Value in JSON without a JSON parser. For example `ewoks.version` in +function New-Items { + return ,([System.Collections.Generic.SortedDictionary[string, string]]::new([StringComparer]::Ordinal)) +} + +# Items in JSON without a JSON parser. For example `bootstrap.distributions` in # -# "ewoks": {"engine": {"name": "ewokscore", "version": "5.1.0"}, "version": "7.0.0"} -function Get-JsonRequirementValue([string]$content, [string]$key) { +# "bootstrap": {"distributions": {"ewoks": "7.0.0", "ewokscore": "5.1.0"}} +function Get-JsonRequirementItems([string]$content, [string]$path) { $q = "[""']" - $value = "$q\s*:\s*$q([^""']*)$q" - # The members of an object, which can contain objects themselves - $members = "[^{}]*(?:\{[^{}]*\}[^{}]*)*" - switch ($key) { - "ewoks.version" { $pattern = "${q}ewoks$q\s*:\s*\{$members${q}version$value" } - "ewoks.engine.name" { $pattern = "${q}ewoks$q\s*:\s*\{[^{}]*${q}engine$q\s*:\s*\{[^{}]*${q}name$value" } - "ewoks.engine.version" { $pattern = "${q}ewoks$q\s*:\s*\{[^{}]*${q}engine$q\s*:\s*\{[^{}]*${q}version$value" } - "python.version" { $pattern = "${q}python$q\s*:\s*\{[^{}]*${q}version$value" } - } + $pattern = ($path.Split(".") | ForEach-Object { "$q$_$q\s*:\s*\{" }) -join "[^{}]*" + $items = New-Items # With the quotes that XML escapes - $found = [regex]::Matches($content.Replace(""", '"'), $pattern) + $found = [regex]::Matches($content.Replace(""", '"'), "$pattern([^{}]*)\}") if ($found.Count -eq 0) { - return "" + return ,$items } - return $found[$found.Count - 1].Groups[1].Value + $members = $found[$found.Count - 1].Groups[1].Value + foreach ($pair in [regex]::Matches($members, "$q([^""']*)$q\s*:\s*$q([^""']*)$q")) { + $items[$pair.Groups[1].Value] = $pair.Groups[2].Value + } + return ,$items } -# Value in YAML without a YAML parser: the last `key: value` line whose key path -# ends with requirements. -function Get-YamlRequirementValue([string]$content, [string]$key) { - $wanted = "requirements.$key" +# Items in YAML without a YAML parser: the `key: value` lines whose parent key path +# ends with requirements. +function Get-YamlRequirementItems([string]$content, [string]$path) { + $wanted = "requirements.$path" $indents = New-Object System.Collections.Generic.List[int] $keys = New-Object System.Collections.Generic.List[string] - $found = "" + $items = New-Items foreach ($line in ($content -split "\r?\n")) { if ($line -match '^\s*(#|$)') { continue } # The dash of a list item belongs to the indentation of its first key - $match = [regex]::Match($line, '^([ -]*)([A-Za-z0-9_]+):\s*(.*)$') + $match = [regex]::Match($line, '^([ -]*)([A-Za-z0-9_.-]+):\s*(.*)$') if (-not $match.Success) { continue } @@ -252,13 +253,13 @@ function Get-YamlRequirementValue([string]$content, [string]$key) { } $indents.Add($indent) $keys.Add($match.Groups[2].Value) - $path = $keys -join "." + $parent = $keys.GetRange(0, $keys.Count - 1) -join "." $value = $match.Groups[3].Value - if ($value -and (($path -eq $wanted) -or $path.EndsWith(".$wanted"))) { - $found = $value -replace "^[""']|[""']$", "" + if ($value -and (($parent -eq $wanted) -or $parent.EndsWith(".$wanted"))) { + $items[$match.Groups[2].Value] = $value -replace "^[""']|[""']$", "" } } - return $found + return ,$items } # ============================================================================= @@ -472,19 +473,19 @@ function Select-Manager([hashtable]$state) { # Set `PythonVersion` (major.minor), `Requirements` of the bootstrap environment, # `Tag` (the name of the bootstrap environment without the python version) and, to -# check whether an existing bootstrap environment is up to date, `Version`, `Engine` -# and `EngineVersion` +# check whether an existing bootstrap environment is up to date, `Pinned` (versions +# by name) and `Latest` (names without a version) function Read-WorkflowRequirements([hashtable]$state) { $content = [System.IO.File]::ReadAllText((Resolve-Path -LiteralPath $state.Workflow)) $state.PythonVersion = "" - if ((Get-RequirementValue $content "python.version") -match '^(\d+\.\d+)') { + $python = Get-RequirementItems $content "python" + if ($python.ContainsKey("version") -and ($python["version"] -match '^(\d+\.\d+)')) { $state.PythonVersion = $Matches[1] } # The `@` and `=` separators of the tag cannot be part of a distribution name or version - $state.Version = "" - $state.Engine = "" - $state.EngineVersion = "" + $state.Pinned = New-Items + $state.Latest = @() if ($state.CustomRequirements) { $state.Requirements = $state.CustomRequirements $sha = [System.Security.Cryptography.SHA256]::Create() @@ -493,38 +494,36 @@ function Read-WorkflowRequirements([hashtable]$state) { return } - $version = Get-RequirementValue $content "ewoks.version" - $engine = Get-RequirementValue $content "ewoks.engine.name" - $engineVersion = Get-RequirementValue $content "ewoks.engine.version" - if (-not $engineVersion) { - $engine = "" + $pinned = Get-RequirementItems $content "bootstrap.distributions" + $latest = @() + if (-not $pinned.ContainsKey("ewoks")) { + Write-Info "the workflow does not provide the ewoks version: using the latest one" + $latest += "ewoks" } - if (-not $engine -and $state.Workflow.EndsWith(".ows")) { + if ($state.Workflow.EndsWith(".ows") -and -not $pinned.ContainsKey("ewoksorange")) { # Workflows saved before ewoks recorded the engine Write-Info "the workflow does not provide its engine: using the latest ewoksorange" - $engine = "ewoksorange" + $latest += "ewoksorange" } - if ($version) { - $requirements = @("ewoks==$version") - $tag = "ewoks=$version" - } else { - Write-Info "the workflow does not provide the ewoks version: using the latest one" - $requirements = @("ewoks>=7") - $tag = "ewoks=latest" - } - if ($engineVersion) { - $requirements += "$engine==$engineVersion" - $tag = "$tag@$engine=$engineVersion" - } elseif ($engine) { - $requirements += $engine - $tag = "$tag@$engine" - } - $state.Version = $version - $state.Engine = $engine - $state.EngineVersion = $engineVersion + $requirements = @() + $tags = @() + foreach ($item in $pinned.GetEnumerator()) { + $requirements += "$($item.Key)==$($item.Value)" + $tags += "$($item.Key)=$($item.Value)" + } + foreach ($name in $latest) { + if ($name -eq "ewoks") { + $requirements += "ewoks>=7" + } else { + $requirements += $name + } + $tags += "$name=latest" + } + $state.Pinned = $pinned + $state.Latest = $latest $state.Requirements = $requirements - $state.Tag = $tag + $state.Tag = $tags -join "@" } # Install the package manager in the bootstrap directory when it is not installed. @@ -639,12 +638,15 @@ function New-BootstrapEnvironment([hashtable]$state) { # Install the requirements in the bootstrap environment unless it has their pinned versions function Install-BootstrapRequirements([hashtable]$state) { $python = $state.Python - if ($state.Version -and ((Get-DistributionVersion $python "ewoks") -eq $state.Version)) { - if (-not $state.Engine) { - return + if ($state.Pinned.Count -gt 0 -and $state.Latest.Count -eq 0) { + $installed = $true + foreach ($item in $state.Pinned.GetEnumerator()) { + if ((Get-DistributionVersion $python $item.Key) -ne $item.Value) { + $installed = $false + break + } } - $installed = Get-DistributionVersion $python $state.Engine - if ($installed -and (-not $state.EngineVersion -or $installed -eq $state.EngineVersion)) { + if ($installed) { return } } diff --git a/src/ewoks/_bootstrap/ewoks-install.sh b/src/ewoks/_bootstrap/ewoks-install.sh index 7dbc9b0..207b8d9 100644 --- a/src/ewoks/_bootstrap/ewoks-install.sh +++ b/src/ewoks/_bootstrap/ewoks-install.sh @@ -13,8 +13,8 @@ # # 1. Split the arguments in options of this script and arguments of `ewoks install`. # 2. Select the package manager. -# 3. Read the ewoks, engine and python versions from the workflow requirements. -# 4. Create a bootstrap environment with that ewoks and engine, unless it exists. +# 3. Read the bootstrap distributions and python version from the workflow requirements. +# 4. Create a bootstrap environment with those distributions, unless it exists. # 5. Run `ewoks install` in the bootstrap environment. # # The bootstrap environments are cached in ~/.ewoks/bootstrap// @@ -26,9 +26,9 @@ usage() { Usage: ewoks-install.sh [OPTIONS] WORKFLOW [ARGS ...] Install the python environment of an ewoks workflow with nothing but a package -manager. The ewoks version that generated the workflow requirements, and the -engine that saved the workflow, are installed in a bootstrap environment, which -then runs +manager. The bootstrap distributions of the workflow requirements, ewoks and +the engine that saved the workflow, are installed in a bootstrap environment, +which then runs ewoks install WORKFLOW ARGS @@ -42,7 +42,7 @@ directory with '--install-package-manager'. Options: --bootstrap-requirement REQ Install REQ in the bootstrap environment instead of - the ewoks version and engine of the workflow, for + the bootstrap distributions of the workflow, for example 'ewoks>=7' or a wheel file. Repeat it to install an engine as well, for example '--bootstrap-requirement ewoks @@ -133,45 +133,52 @@ download() { # Workflow requirements # ============================================================================= -# Value in the requirements of a workflow file: ewoks.version, ewoks.engine.name, -# ewoks.engine.version or python.version. The requirements are JSON, which can be +# `key value` lines of the object at a key path of the requirements of a workflow +# file: python or bootstrap.distributions. The requirements are JSON, which can be # embedded in another format like the XML of .ows files, or YAML. -requirement_value() { - value=$(json_requirement_value "$1" "$2") - [ -n "$value" ] || value=$(yaml_requirement_value "$1" "$2") - printf '%s' "$value" +requirement_items() { + items=$(json_requirement_items "$1" "$2") + [ -n "$items" ] || items=$(yaml_requirement_items "$1" "$2") + printf '%s\n' "$items" } -# Value in JSON without a JSON parser. For example `ewoks.version` in +# Items in JSON without a JSON parser. For example `bootstrap.distributions` in # -# "ewoks": {"engine": {"name": "ewokscore", "version": "5.1.0"}, "version": "7.0.0"} -json_requirement_value() { +# "bootstrap": {"distributions": {"ewoks": "7.0.0", "ewokscore": "5.1.0"}} +json_requirement_items() { q="[\"']" s='[[:space:]]*' - value="${q}$s:$s${q}\([^\"']*\)${q}" - # The members of an object, which can contain objects themselves - members='[^{}]*\({[^{}]*}[^{}]*\)*' - case $2 in - ewoks.version) - pattern="${q}ewoks${q}$s:$s{$members${q}version$value" group=2 - ;; - ewoks.engine.name) - pattern="${q}ewoks${q}$s:$s{[^{}]*${q}engine${q}$s:$s{[^{}]*${q}name$value" group=1 - ;; - ewoks.engine.version) - pattern="${q}ewoks${q}$s:$s{[^{}]*${q}engine${q}$s:$s{[^{}]*${q}version$value" group=1 - ;; - python.version) - pattern="${q}python${q}$s:$s{[^{}]*${q}version$value" group=1 - ;; - esac + pattern="" + old_ifs=$IFS + IFS=. + for key in $2; do + pattern="$pattern${pattern:+[^{\}]*}${q}$key${q}$s:$s{" + done + IFS=$old_ifs # On one line, with the quotes that XML escapes - tr -d '\n\r' <"$1" | sed 's/"/"/g' | sed -n "s/.*$pattern.*/\\$group/p" + tr -d '\n\r' <"$1" | sed 's/"/"/g' | sed -n "s/.*$pattern\([^{}]*\)}.*/\1/p" | + awk -v sq="'" ' + { + quoted = "[\"" sq "][^\"" sq "]*[\"" sq "]" + quotes = "^[ \t]*[\"" sq "]|[\"" sq "][ \t]*$" + rest = $0 + while (match(rest, quoted "[ \t]*:[ \t]*" quoted)) { + pair = substr(rest, RSTART, RLENGTH) + rest = substr(rest, RSTART + RLENGTH) + colon = index(pair, ":") + key = substr(pair, 1, colon - 1) + value = substr(pair, colon + 1) + gsub(quotes, "", key) + gsub(quotes, "", value) + print key " " value + } + } + ' } -# Value in YAML without a YAML parser: the last `key: value` line whose key path -# ends with requirements. -yaml_requirement_value() { +# Items in YAML without a YAML parser: the `key: value` lines whose parent key +# path ends with requirements. +yaml_requirement_items() { awk -v wanted="requirements.$2" -v sq="'" ' { sub(/\r$/, "") @@ -180,7 +187,7 @@ yaml_requirement_value() { match($0, /^[ -]*/) indent = RLENGTH rest = substr($0, indent + 1) - if (rest !~ /^[A-Za-z0-9_]+:/) next + if (rest !~ /^[A-Za-z0-9_.-]+:/) next key = rest sub(/:.*/, "", key) value = rest @@ -190,15 +197,15 @@ yaml_requirement_value() { depth++ indents[depth] = indent keys[depth] = key - path = keys[1] - for (i = 2; i <= depth; i++) path = path "." keys[i] - n = length(path) - length(wanted) - if (value == "" || n < 0 || substr(path, n + 1) != wanted) next - if (n > 0 && substr(path, n, 1) != ".") next + if (value == "" || depth < 2) next + parent = keys[1] + for (i = 2; i < depth; i++) parent = parent "." keys[i] + n = length(parent) - length(wanted) + if (n < 0 || substr(parent, n + 1) != wanted) next + if (n > 0 && substr(parent, n, 1) != ".") next gsub("^[\"" sq "]|[\"" sq "]$", "", value) - found = value + print key " " value } - END { printf "%s", found } ' "$1" } @@ -372,46 +379,59 @@ select_manager() { # Set `python_version` (major.minor), `requirements` of the bootstrap environment # (quoted for the shell), `tag` (the name of the bootstrap environment without the # python version) and, to check whether an existing bootstrap environment is up to -# date, `version`, `engine` and `engine_version` +# date, `pinned` (`name version` lines) and `latest` (names without a version) read_workflow_requirements() { - python_version=$(requirement_value "$workflow" python.version) + python_version=$(requirement_items "$workflow" python | sed -n 's/^version //p' | tail -n 1) python_version=$(printf '%s' "$python_version" | sed -n 's/^\([0-9]*\.[0-9]*\).*/\1/p') # The `@` and `=` separators of the tag cannot be part of a distribution name or version if [ -n "$custom_requirements" ]; then - version="" - engine="" + pinned="" + latest="" requirements=$custom_requirements tag=custom=$(printf '%s' "$requirements" | cksum | cut -d ' ' -f 1) return fi - version=$(requirement_value "$workflow" ewoks.version) - engine=$(requirement_value "$workflow" ewoks.engine.name) - engine_version=$(requirement_value "$workflow" ewoks.engine.version) - [ -n "$engine_version" ] || engine="" - if [ -z "$engine" ]; then - case $workflow in - *.ows) + pinned=$(requirement_items "$workflow" bootstrap.distributions | sed '/^$/d' | LC_ALL=C sort) + latest="" + if ! pins ewoks; then + say "the workflow does not provide the ewoks version: using the latest one" + latest=ewoks + fi + case $workflow in + *.ows) + if ! pins ewoksorange; then # Workflows saved before ewoks recorded the engine say "the workflow does not provide its engine: using the latest ewoksorange" - engine=ewoksorange - ;; - esac - fi + latest="$latest ewoksorange" + fi + ;; + esac - if [ -n "$version" ]; then - requirements=$(quote "ewoks==$version") - tag=ewoks=$version - else - say "the workflow does not provide the ewoks version: using the latest one" - requirements=$(quote "ewoks>=7") - tag=ewoks=latest - fi - if [ -n "$engine" ]; then - requirements="$requirements $(quote "$engine${engine_version:+==$engine_version}")" - tag=$tag@$engine${engine_version:+=$engine_version} - fi + requirements="" + tag="" + while read -r name dist_version; do + [ -n "$name" ] || continue + requirements="$requirements $(quote "$name==$dist_version")" + tag=${tag:+$tag@}$name=$dist_version + done <=7")" + else + requirements="$requirements $(quote "$name")" + fi + tag=${tag:+$tag@}$name=latest + done + requirements=${requirements# } +} + +# The bootstrap distributions of the workflow have a version of this distribution +pins() { + printf '%s\n' "$pinned" | grep -q "^$1 " } # Install the package manager in the bootstrap directory when it is not installed. @@ -490,16 +510,19 @@ create_bootstrap_environment() { esac } +# The bootstrap environment has the pinned versions +pinned_installed() { + while read -r name dist_version; do + [ "$(distribution_version "$python" "$name" Date: Wed, 11 Nov 2026 16:41:03 +0100 Subject: [PATCH 4/4] ewoks install --with-ewoks: test it and document that versions can change Co-Authored-By: Claude Opus 5.5 --- doc/reference/cli.rst | 3 ++- doc/tutorials/install.rst | 5 +++-- src/ewoks/tests/requirements/test_install_cli.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/doc/reference/cli.rst b/doc/reference/cli.rst index a68fe21..b2fa9d6 100644 --- a/doc/reference/cli.rst +++ b/doc/reference/cli.rst @@ -28,7 +28,8 @@ ewoks install Use ``--env-root`` for another directory to create the environment in and ``--env-name`` for another name than the workflow identifier. An environment that already exists is installed in, unless ``--clean`` is provided to remove it first. The :term:`workflow` can be executed in that environment when it contains **ewoks** itself, which is the case when the ``requirements`` contain it. - Use ``--with-ewoks`` to add **ewoks** when the ``requirements`` do not contain it, without changing any of the versions they do contain: + Use ``--with-ewoks`` to add **ewoks** when the ``requirements`` do not contain it. + The versions they contain are kept, unless they come from project files that allow other versions: a warning lists the versions that changed. .. code-block:: bash diff --git a/doc/tutorials/install.rst b/doc/tutorials/install.rst index 9f00d7e..35d8eaa 100644 --- a/doc/tutorials/install.rst +++ b/doc/tutorials/install.rst @@ -194,8 +194,9 @@ Execute the workflow ``--env`` executes the :term:`workflow` with the python interpreter of that environment instead of the current one. This works because the requirements contain ``ewoks`` itself: it was installed in the environment in which the :term:`workflow` was converted. When they do -not, add ``--with-ewoks`` to ``ewoks install`` to install it without changing any of the -versions coming from the :term:`workflow`. +not, add ``--with-ewoks`` to ``ewoks install`` to install it. The versions coming from the +:term:`workflow` are kept, unless they come from project files that allow other versions: a +warning lists the versions that changed. Remove the environment ---------------------- diff --git a/src/ewoks/tests/requirements/test_install_cli.py b/src/ewoks/tests/requirements/test_install_cli.py index 22d6282..38b519a 100644 --- a/src/ewoks/tests/requirements/test_install_cli.py +++ b/src/ewoks/tests/requirements/test_install_cli.py @@ -124,6 +124,20 @@ def test_install_without_requirements(fast_manager_case, env_root): assert Environment.from_location(location).distribution_version("six") +def test_install_with_ewoks(fast_manager_case, env_root): + """`--with-ewoks` adds ewoks to the environment of the workflow.""" + graph = _graph(REQUIREMENTS) + argv = _argv(fast_manager_case, env_root) + ["--with-ewoks", json.dumps(graph)] + + subprocess.check_call(argv) # noqa: S603 - Trusted test command + + environment = Environment.from_location( + _location(graph, fast_manager_case, env_root) + ) + assert environment.distribution_version("ewoks") + assert_installed(environment, DISTRIBUTIONS) + + def test_install_env_name(fast_manager_case, env_root): """The environment name replaces the workflow identifier.""" graph = _graph(REQUIREMENTS)