From 2c517e38fadd5989bb3ee708a2fea327701f94cf Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Sat, 24 Oct 2026 15:13:49 +0200 Subject: [PATCH 1/3] Add the ewoks-install.sh script that installs a workflow environment without ewoks Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test.yml | 19 + CHANGELOG.md | 1 + doc/conf.py | 3 + doc/howtoguides/requirements.rst | 3 +- doc/tutorials/install.rst | 84 +++ doc/tutorials/install/conda.rst | 20 + doc/tutorials/install/pip_venv.rst | 20 + doc/tutorials/install/pixi.rst | 20 + doc/tutorials/install/poetry.rst | 20 + doc/tutorials/install/uv.rst | 20 + pyproject.toml | 1 + src/ewoks/__main__.py | 64 +- src/ewoks/_bootstrap/ewoks-install.sh | 567 ++++++++++++++++++ src/ewoks/tests/conftest.py | 26 + .../tests/requirements/test_bootstrap.py | 384 ++++++++++++ .../tests/requirements/test_install_cli.py | 28 + 16 files changed, 1257 insertions(+), 23 deletions(-) create mode 100644 src/ewoks/_bootstrap/ewoks-install.sh create mode 100644 src/ewoks/tests/requirements/test_bootstrap.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 10a0c13..20b197a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,6 +79,25 @@ jobs: enable-coverage: "false" extra-pytest-warnings: "" + # Run the tests that install package managers, on runners without them + tests-package-manager-install: + needs: build + uses: ewoks-kit/.github/.github/workflows/python-tests.yml@main + with: + os: ${{ matrix.os }} + python-version: "3.12" + pytest-args: >- + -v -ra -W error -m package_manager_install + -W ignore::DeprecationWarning:jupyter_client.session + -W ignore::DeprecationWarning:orangecanvas.utils.localization + -W ignore::DeprecationWarning:orangecanvas.utils.localization.si + -W ignore::DeprecationWarning:matplotlib._fontconfig_pattern + -W ignore::DeprecationWarning:matplotlib._mathtext + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + # Run linter / checks checks: uses: ewoks-kit/.github/.github/workflows/python-check.yml@main diff --git a/CHANGELOG.md b/CHANGELOG.md index 713efd6..fec6b22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - add `--in-place`, `--python-version` and `--with-ewoks` arguments. - `ewoks convert`: add `--package-manager-name` and `--package-manager-command` arguments. - `ewoks execute`: add `--env` argument. +- Add `ewoks-install.sh` script to install the environment of a workflow without ewoks. - Add `ewoks lint` CLI command to check if a workflow is conform to the Ewoks specification. ### Changed diff --git a/doc/conf.py b/doc/conf.py index 72e024d..2ad8f91 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -85,6 +85,9 @@ } +# The ewoks-install scripts +html_extra_path = ["../src/ewoks/_bootstrap"] + # Root of the documentation version that is built: on Read the Docs, for example # .../en/stable, or on GitLab Pages _DOCS_URL = ( diff --git a/doc/howtoguides/requirements.rst b/doc/howtoguides/requirements.rst index f7230b4..755a4f0 100644 --- a/doc/howtoguides/requirements.rst +++ b/doc/howtoguides/requirements.rst @@ -14,7 +14,8 @@ What is stored ``ewoks convert`` and ``ewoks execute -o convert_destination=...`` store * ``ewoks``: the version of :term:`ewoks` that generated the requirements and the engine - that saved the :term:`workflow`. + that saved the :term:`workflow`. The + :ref:`ewoks-install script ` installs both to run ``ewoks install``. * ``python`` and ``system``: the python interpreter and the operating system. * ``distributions``: every installed python package with its version and, when it was not installed from the python package index, the git commit or the archive it came from. Any diff --git a/doc/tutorials/install.rst b/doc/tutorials/install.rst index e3d5c8d..d45d06b 100644 --- a/doc/tutorials/install.rst +++ b/doc/tutorials/install.rst @@ -84,6 +84,90 @@ Without ``--yes`` you are asked to confirm after the packages have been listed. walk-throughs use ``--env-root`` to create the environment in the working directory. Without it the environment is created where the :term:`package manager` creates named environments. +.. _install_bootstrap: + +Create the environment without ewoks +++++++++++++++++++++++++++++++++++++ + +``ewoks install`` needs a python environment with :term:`ewoks`. The ``ewoks-install`` script +only needs a :term:`package manager`: it installs the version of :term:`ewoks` that generated +the requirements, and the engine that saved the :term:`workflow`, in a bootstrap environment +and runs ``ewoks install`` from there. All +arguments are passed to ``ewoks install``, except for the options of the script itself + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + +Without a :term:`workflow` at hand, try it with the demo :term:`workflow` of this +documentation, which only requires :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSfO |docs_url|/demo.json + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSfO |docs_url|/demo.json + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + +Alternatively, download the script first, for example to read it before running it + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf -o ewoks-install.sh |docs_url|/ewoks-install.sh + sh ewoks-install.sh demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf -o ewoks-install.sh |docs_url|/ewoks-install.sh + sh ewoks-install.sh demo.json + +* ``--package-manager-name`` and ``--package-manager-command`` select the + :term:`package manager` that creates the bootstrap environment. Without them it is the first + one that is available of uv, pixi, conda, poetry and pip-venv. +* ``--bootstrap-requirement`` installs a package in the bootstrap environment instead of the + version of :term:`ewoks` and the engine of the requirements, for example + ``--bootstrap-requirement "ewoks>=7"``. Repeat it to install an engine as well, for example + ``--bootstrap-requirement ewoks --bootstrap-requirement ewoksorange``. The latest version of + :term:`ewoks` is installed when the requirements do not provide one. +* ``--bootstrap-dir`` is the directory of the bootstrap environments. +* ``--install-package-manager`` installs the :term:`package manager` of + ``--package-manager-name`` in the bootstrap directory when it is not installed. Conda + cannot be installed in a directory with spaces. + +The script keeps the bootstrap environment in ``~/.ewoks/bootstrap`` and uses it again when +it installs another :term:`workflow` whose requirements were generated by the same version of +:term:`ewoks`. The bootstrap environment only runs ``ewoks install``: +it is not the environment of the :term:`workflow`. The script prints the ``ewoks install`` +command before it runs it. Before the environment of the +:term:`workflow`, the script prints the python interpreter of the bootstrap environment, the +command to run ``ewoks install`` again and the command to remove the bootstrap environment. When the script fails to create the bootstrap +environment, it prints how to install without it. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/conda.rst b/doc/tutorials/install/conda.rst index 71f3722..59978a0 100644 --- a/doc/tutorials/install/conda.rst +++ b/doc/tutorials/install/conda.rst @@ -147,6 +147,26 @@ The environment of the :term:`workflow` is a conda environment in ``ewoks_envs/d ``--env-root`` it is created in the first environment directory of conda, where ``conda activate demo`` finds it. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + +It selects the :term:`package manager` itself and prints the command to execute the +:term:`workflow` in the environment it creates. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/pip_venv.rst b/doc/tutorials/install/pip_venv.rst index 73ec0e3..49823a2 100644 --- a/doc/tutorials/install/pip_venv.rst +++ b/doc/tutorials/install/pip_venv.rst @@ -106,6 +106,26 @@ Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` The environment of the :term:`workflow` is a virtual environment in ``ewoks_envs/demo``. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + +It selects the :term:`package manager` itself and prints the command to execute the +:term:`workflow` in the environment it creates. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/pixi.rst b/doc/tutorials/install/pixi.rst index 81098ae..2ce7309 100644 --- a/doc/tutorials/install/pixi.rst +++ b/doc/tutorials/install/pixi.rst @@ -142,6 +142,26 @@ Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` The environment of the :term:`workflow` is a pixi workspace in ``ewoks_envs/demo`` with the environment in its ``.pixi/envs/default`` directory. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + +It selects the :term:`package manager` itself and prints the command to execute the +:term:`workflow` in the environment it creates. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/poetry.rst b/doc/tutorials/install/poetry.rst index f54ac60..5b5e628 100644 --- a/doc/tutorials/install/poetry.rst +++ b/doc/tutorials/install/poetry.rst @@ -136,6 +136,26 @@ Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` The environment of the :term:`workflow` is a poetry project in ``ewoks_envs/demo`` with the virtual environment in its ``.venv`` directory. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + +It selects the :term:`package manager` itself and prints the command to execute the +:term:`workflow` in the environment it creates. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/uv.rst b/doc/tutorials/install/uv.rst index 49276cc..1e2ffe7 100644 --- a/doc/tutorials/install/uv.rst +++ b/doc/tutorials/install/uv.rst @@ -128,6 +128,26 @@ Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` The environment of the :term:`workflow` is a uv project in ``ewoks_envs/demo`` with the virtual environment in its ``.venv`` directory. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + +It selects the :term:`package manager` itself and prints the command to execute the +:term:`workflow` in the environment it creates. + Execute the workflow -------------------- diff --git a/pyproject.toml b/pyproject.toml index cebf15e..9527b51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ ewoks = "ewoks.__main__:main" [tool.setuptools.package-data] "ewoks.tests.notebooks" = ["*.ipynb"] +"ewoks._bootstrap" = ["*.sh"] [tool.ruff.lint] select = [ diff --git a/src/ewoks/__main__.py b/src/ewoks/__main__.py index b136400..be0dfa0 100644 --- a/src/ewoks/__main__.py +++ b/src/ewoks/__main__.py @@ -185,10 +185,16 @@ def command_convert( return None +# Environment variable with text that is printed before the summary, like the +# ewoks-install scripts do for their bootstrap environment +_SUMMARY_HEADER_VARIABLE = "EWOKS_INSTALL_SUMMARY_HEADER" + + def command_install( cli_args: Namespace, shell: bool = False ) -> Optional[Literal[0, 1]]: cli_install_utils.parse_install_arguments(cli_args, shell=shell) + header = os.environ.get(_SUMMARY_HEADER_VARIABLE) for workflow, source in zip(cli_args.workflows, cli_args.graphs): try: environment = install_graph( @@ -205,35 +211,49 @@ def command_install( ) except (CalledProcessError, RuntimeError, ValueError): traceback.print_exc() - print(f"Install failed for {workflow}") + summary = [f"Install failed for {workflow}"] except AbortException: - print(f"Install aborted for {workflow}") + summary = [f"Install aborted for {workflow}"] else: - print(f"Installed requirements for {workflow}") - print(f" Python : {environment.python}") - if environment.distribution_version("ewoks"): - execute = _shell_command( - [ - str(environment.python), - "-m", - "ewoks", - "execute", - _absolute_workflow(workflow), - ] - ) - print(f" Execute: {execute}") - else: - print( - " Execute: the environment has no ewoks to execute the " - "workflow with (use '--with-ewoks')" - ) - if not cli_args.in_place: - print(f" Remove : {_remove_command(environment.location)}") + summary = _install_summary(workflow, environment, cli_args.in_place) + if header: + print(header) + header = None + print("\n".join(summary)) if shell: return 0 return None +def _install_summary( + workflow: str, environment: Environment, in_place: bool +) -> List[str]: + """Lines that describe the python environment in which a workflow is installed.""" + summary = [ + f"Installed requirements for {workflow}", + f" Python : {environment.python}", + ] + if environment.distribution_version("ewoks"): + execute = _shell_command( + [ + str(environment.python), + "-m", + "ewoks", + "execute", + _absolute_workflow(workflow), + ] + ) + summary.append(f" Execute: {execute}") + else: + summary.append( + " Execute: the environment has no ewoks to execute the workflow with " + "(use '--with-ewoks')" + ) + if not in_place: + summary.append(f" Remove : {_remove_command(environment.location)}") + return summary + + def _remove_command(location: Path) -> str: """Shell command that removes a python environment: PowerShell on Windows.""" if sys.platform == "win32": diff --git a/src/ewoks/_bootstrap/ewoks-install.sh b/src/ewoks/_bootstrap/ewoks-install.sh new file mode 100644 index 0000000..7dbc9b0 --- /dev/null +++ b/src/ewoks/_bootstrap/ewoks-install.sh @@ -0,0 +1,567 @@ +#!/bin/sh +# Install the python environment of an ewoks workflow with nothing but a package +# manager. For example +# +# curl -LsSf https://ewoks.readthedocs.io/en/stable/ewoks-install.sh | sh -s -- demo.json --package-manager-name uv +# +# or download the script first +# +# curl -LsSf -o ewoks-install.sh https://ewoks.readthedocs.io/en/stable/ewoks-install.sh +# sh ewoks-install.sh demo.json --package-manager-name uv +# +# Steps (see `main` at the end): +# +# 1. Split the arguments in options of this script and arguments of `ewoks install`. +# 2. Select the package manager. +# 3. Read the ewoks, engine and python versions from the workflow requirements. +# 4. Create a bootstrap environment with that ewoks and engine, unless it exists. +# 5. Run `ewoks install` in the bootstrap environment. +# +# The bootstrap environments are cached in ~/.ewoks/bootstrap// +# where the name is, for example, `ewoks=7.0.0@ewokscore=5.1.0@python=3.12`. +set -eu + +usage() { + cat <<'EOF' +Usage: ewoks-install.sh [OPTIONS] WORKFLOW [ARGS ...] + +Install the python environment of an ewoks workflow with nothing but a package +manager. The ewoks version that generated the workflow requirements, and the +engine that saved the workflow, are installed in a bootstrap environment, which +then runs + + ewoks install WORKFLOW ARGS + +See 'ewoks install --help' for ARGS. The package manager that creates the +bootstrap environment is selected with '--package-manager-name' and +'--package-manager-command', which are passed to 'ewoks install' as well. +Without them it is the first available of uv, pixi, conda, poetry and pip-venv. +A package manager that is not installed can be installed in the bootstrap +directory with '--install-package-manager'. + +Options: + --bootstrap-requirement REQ + Install REQ in the bootstrap environment instead of + the ewoks version and engine of the workflow, for + example 'ewoks>=7' or a wheel file. Repeat it to + install an engine as well, for example + '--bootstrap-requirement ewoks + --bootstrap-requirement ewoksorange'. + --bootstrap-dir DIR Directory of the bootstrap environments. + Default: ~/.ewoks/bootstrap + --install-package-manager + Install the package manager of + '--package-manager-name' when it is not installed. + -h, --help Show this help. +EOF +} + +# ============================================================================= +# Helpers +# ============================================================================= + +say() { + printf 'ewoks-install: %s\n' "$*" >&2 +} + +die() { + say "error: $*" + exit 1 +} + +# Argument quoted for the shell when needed +quote() { + case $1 in + '' | *[!A-Za-z0-9_./=:@%+,-]*) + printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")" + ;; + *) + printf '%s' "$1" + ;; + esac +} + +# Arguments quoted for the shell and joined with spaces +quote_all() { + for arg in "$@"; do + printf '%s ' "$(quote "$arg")" + done | sed 's/ $//' +} + +# Absolute path of a file +absolute_path() { + case $1 in + /*) printf '%s' "$1" ;; + *) printf '%s/%s' "$(pwd)" "$1" ;; + esac +} + +# Python interpreter with this major.minor version, or the default one +find_python() { + if [ -n "$1" ] && command -v "python$1" >/dev/null 2>&1; then + printf 'python%s' "$1" + elif command -v python3 >/dev/null 2>&1; then + [ -z "$1" ] || say "warning: python$1 of the workflow is not found: using python3" + printf 'python3' + else + die "no python interpreter found (see --package-manager-name uv --install-package-manager)" + fi +} + +# major.minor version of a python interpreter +python_major_minor() { + "$@" -c 'import sys; print("%d.%d" % sys.version_info[:2])' +} + +# Version of an installed distribution, or nothing when it is not installed +distribution_version() { + "$1" -c 'import importlib.metadata as m, sys; print(m.version(sys.argv[1]))' "$2" 2>/dev/null || true +} + +# Download a URL to a file +download() { + if command -v curl >/dev/null 2>&1; then + curl -LsSf -o "$2" "$1" || die "cannot download $1" + elif command -v wget >/dev/null 2>&1; then + wget -q -O "$2" "$1" || die "cannot download $1" + else + die "curl or wget is needed to download $1" + fi +} + +# ============================================================================= +# Workflow requirements +# ============================================================================= + +# Value in the requirements of a workflow file: ewoks.version, ewoks.engine.name, +# ewoks.engine.version or python.version. The requirements are JSON, which can be +# embedded in another format like the XML of .ows files, or YAML. +requirement_value() { + value=$(json_requirement_value "$1" "$2") + [ -n "$value" ] || value=$(yaml_requirement_value "$1" "$2") + printf '%s' "$value" +} + +# Value in JSON without a JSON parser. For example `ewoks.version` in +# +# "ewoks": {"engine": {"name": "ewokscore", "version": "5.1.0"}, "version": "7.0.0"} +json_requirement_value() { + q="[\"']" + s='[[:space:]]*' + value="${q}$s:$s${q}\([^\"']*\)${q}" + # The members of an object, which can contain objects themselves + members='[^{}]*\({[^{}]*}[^{}]*\)*' + case $2 in + ewoks.version) + pattern="${q}ewoks${q}$s:$s{$members${q}version$value" group=2 + ;; + ewoks.engine.name) + pattern="${q}ewoks${q}$s:$s{[^{}]*${q}engine${q}$s:$s{[^{}]*${q}name$value" group=1 + ;; + ewoks.engine.version) + pattern="${q}ewoks${q}$s:$s{[^{}]*${q}engine${q}$s:$s{[^{}]*${q}version$value" group=1 + ;; + python.version) + pattern="${q}python${q}$s:$s{[^{}]*${q}version$value" group=1 + ;; + esac + # On one line, with the quotes that XML escapes + tr -d '\n\r' <"$1" | sed 's/"/"/g' | sed -n "s/.*$pattern.*/\\$group/p" +} + +# Value in YAML without a YAML parser: the last `key: value` line whose key path +# ends with requirements. +yaml_requirement_value() { + awk -v wanted="requirements.$2" -v sq="'" ' + { + sub(/\r$/, "") + if ($0 ~ /^[ \t]*(#|$)/) next + # The dash of a list item belongs to the indentation of its first key + match($0, /^[ -]*/) + indent = RLENGTH + rest = substr($0, indent + 1) + if (rest !~ /^[A-Za-z0-9_]+:/) next + key = rest + sub(/:.*/, "", key) + value = rest + sub(/^[^:]*:[ \t]*/, "", value) + # Key path of this line, from the keys of the lines with less indentation + while (depth > 0 && indents[depth] >= indent) depth-- + depth++ + indents[depth] = indent + keys[depth] = key + path = keys[1] + for (i = 2; i <= depth; i++) path = path "." keys[i] + n = length(path) - length(wanted) + if (value == "" || n < 0 || substr(path, n + 1) != wanted) next + if (n > 0 && substr(path, n, 1) != ".") next + gsub("^[\"" sq "]|[\"" sq "]$", "", value) + found = value + } + END { printf "%s", found } + ' "$1" +} + +# ============================================================================= +# Package managers +# ============================================================================= + +# Run the package manager command `cmd`, which is quoted for the shell like the +# '--package-manager-command' argument of `ewoks install` +run_manager() { + eval "set -- $cmd \"\$@\"" + "$@" +} + +# Executable of a package manager installed in a directory by this script +manager_executable() { + case $1 in + uv) printf '%s/bin/uv' "$2" ;; + pixi) printf '%s/bin/pixi' "$2" ;; + poetry) printf '%s/bin/poetry' "$2" ;; + conda) printf '%s/bin/conda' "$2" ;; + esac +} + +# Install a package manager in a directory with its official installer +install_manager() { + if [ "$1" = conda ]; then + case $2 in + *" "*) die "Miniforge cannot be installed in a directory with spaces: $2 (see --bootstrap-dir)" ;; + esac + fi + say "install $1 in $2" + rm -rf "$2" + mkdir -p "$(dirname "$2")" + # Removed by `on_exit` when the installation fails. + # The Miniforge installer requires the '.sh' suffix. + installer_dir=$(mktemp -d) + installer=$installer_dir/installer.sh + case $1 in + uv) + download https://astral.sh/uv/install.sh "$installer" + UV_INSTALL_DIR=$2/bin UV_NO_MODIFY_PATH=1 sh "$installer" >&2 || + die "the uv installer failed" + ;; + pixi) + download https://pixi.sh/install.sh "$installer" + PIXI_HOME=$2 PIXI_NO_PATH_UPDATE=1 sh "$installer" >&2 || + die "the pixi installer failed" + ;; + poetry) + download https://install.python-poetry.org "$installer" + POETRY_HOME=$2 "$(find_python "")" "$installer" >&2 || + die "the poetry installer failed" + ;; + conda) + system=$(uname -s) + [ "$system" != Darwin ] || system=MacOSX + download "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$system-$(uname -m).sh" "$installer" + bash "$installer" -b -p "$2" >&2 || + die "the Miniforge installer failed" + ;; + esac + rm -rf "$installer_dir" + installer_dir="" +} + +# ============================================================================= +# Steps of `main`, which share their variables +# ============================================================================= + +# Set the options of this script and `forwarded`: the other arguments, which are +# for `ewoks install`, quoted for the shell +parse_arguments() { + install_package_manager=0 + custom_requirements="" # Quoted for the shell + root=$HOME/.ewoks/bootstrap + manager="" + manager_command="" + workflow="" + forwarded="" + while [ $# -gt 0 ]; do + arg=$1 + shift + case $arg in + -h | --help) + usage + exit 0 + ;; + --install-package-manager) + install_package_manager=1 + ;; + --bootstrap-requirement) + [ $# -gt 0 ] || die "$arg requires a value" + custom_requirements="$custom_requirements $(quote "$1")" + shift + ;; + --bootstrap-requirement=*) + custom_requirements="$custom_requirements $(quote "${arg#*=}")" + ;; + --bootstrap-dir) + [ $# -gt 0 ] || die "$arg requires a value" + root=$1 + shift + ;; + --bootstrap-dir=*) + root=${arg#*=} + ;; + --package-manager-name | --package-manager-command) + [ $# -gt 0 ] || die "$arg requires a value" + if [ "$arg" = --package-manager-name ]; then + manager=$1 + else + manager_command=$1 + fi + forwarded="$forwarded $arg $(quote "$1")" + shift + ;; + --package-manager-name=*) + manager=${arg#*=} + forwarded="$forwarded $(quote "$arg")" + ;; + --package-manager-command=*) + manager_command=${arg#*=} + forwarded="$forwarded $(quote "$arg")" + ;; + *) + # The workflow is the first argument that is an existing file. + # Its absolute path makes the printed commands work from any directory. + if [ -z "$workflow" ] && [ -f "$arg" ]; then + arg=$(absolute_path "$arg") + workflow=$arg + fi + forwarded="$forwarded $(quote "$arg")" + ;; + esac + done + custom_requirements=${custom_requirements# } + forwarded=${forwarded# } + + if [ -z "$workflow" ]; then + usage >&2 + die "provide the file of the workflow (no existing file in the arguments)" + fi +} + +# Set `manager` and `cmd`: the command of the package manager, quoted for the shell. +# `cmd` is empty for pip-venv without '--package-manager-command'. +select_manager() { + manager=$(printf '%s' "$manager" | tr '[:upper:]' '[:lower:]') + if [ "$install_package_manager" = 1 ] && [ -z "$manager" ]; then + die "--install-package-manager requires --package-manager-name" + fi + if [ -z "$manager" ]; then + [ -z "$manager_command" ] || die "--package-manager-command requires --package-manager-name" + for name in uv pixi conda poetry; do + if command -v "$name" >/dev/null 2>&1; then + manager=$name + break + fi + done + manager=${manager:-pip-venv} + fi + + case $manager in + uv | pixi | conda | poetry) cmd=${manager_command:-$manager} ;; + pip-venv) cmd=$manager_command ;; + *) die "package manager '$manager' is not supported" ;; + esac +} + +# Set `python_version` (major.minor), `requirements` of the bootstrap environment +# (quoted for the shell), `tag` (the name of the bootstrap environment without the +# python version) and, to check whether an existing bootstrap environment is up to +# date, `version`, `engine` and `engine_version` +read_workflow_requirements() { + python_version=$(requirement_value "$workflow" python.version) + python_version=$(printf '%s' "$python_version" | sed -n 's/^\([0-9]*\.[0-9]*\).*/\1/p') + + # The `@` and `=` separators of the tag cannot be part of a distribution name or version + if [ -n "$custom_requirements" ]; then + version="" + engine="" + requirements=$custom_requirements + tag=custom=$(printf '%s' "$requirements" | cksum | cut -d ' ' -f 1) + return + fi + + version=$(requirement_value "$workflow" ewoks.version) + engine=$(requirement_value "$workflow" ewoks.engine.name) + engine_version=$(requirement_value "$workflow" ewoks.engine.version) + [ -n "$engine_version" ] || engine="" + if [ -z "$engine" ]; then + case $workflow in + *.ows) + # Workflows saved before ewoks recorded the engine + say "the workflow does not provide its engine: using the latest ewoksorange" + engine=ewoksorange + ;; + esac + fi + + if [ -n "$version" ]; then + requirements=$(quote "ewoks==$version") + tag=ewoks=$version + else + say "the workflow does not provide the ewoks version: using the latest one" + requirements=$(quote "ewoks>=7") + tag=ewoks=latest + fi + if [ -n "$engine" ]; then + requirements="$requirements $(quote "$engine${engine_version:+==$engine_version}")" + tag=$tag@$engine${engine_version:+=$engine_version} + fi +} + +# Install the package manager in the bootstrap directory when it is not installed. +# `ewoks install` needs it as well. +ensure_manager() { + if [ -z "$cmd" ] || [ -n "$manager_command" ] || command -v "$cmd" >/dev/null 2>&1; then + return + fi + manager_dir=$root/package-managers/$manager + executable=$(manager_executable "$manager" "$manager_dir") + if [ ! -x "$executable" ]; then + [ "$install_package_manager" = 1 ] || + die "$manager is not installed (see --install-package-manager)" + install_manager "$manager" "$manager_dir" + [ -x "$executable" ] || die "cannot install $manager in $manager_dir" + fi + cmd=$(quote "$executable") + forwarded="$forwarded --package-manager-command $(quote "$cmd")" +} + +# Set `python`: the interpreter of the bootstrap environment, which is created when +# it does not exist +create_bootstrap_environment() { + # pip-venv and poetry cannot provide a python version: the interpreter decides + env_python_version=$python_version + case $manager in + pip-venv) + [ -n "$cmd" ] || cmd=$(find_python "$python_version") + env_python_version=$(python_major_minor run_manager) + ;; + poetry) + base_python=$(find_python "$python_version") + env_python_version=$(python_major_minor "$base_python") + ;; + esac + + location=$root/$manager/$tag${env_python_version:+@python=$env_python_version} + case $manager in + pixi) python=$location/.pixi/envs/default/bin/python ;; + poetry) python=$location/.venv/bin/python ;; + *) python=$location/bin/python ;; + esac + if [ -x "$python" ]; then + return + fi + + say "create the bootstrap environment $location" + # A previous attempt failed + rm -rf "$location" + mkdir -p "$location" + case $manager in + uv) + run_manager venv --quiet ${env_python_version:+--python "$env_python_version"} "$location" >&2 + ;; + pixi) + run_manager init "$location" >&2 + run_manager add --manifest-path "$location/pixi.toml" "python${env_python_version:+=$env_python_version}" pip >&2 + ;; + conda) + run_manager create --yes --quiet --prefix "$location" "python${env_python_version:+=$env_python_version}" pip >&2 + ;; + poetry) + printf '[tool.poetry]\npackage-mode = false\n\n[tool.poetry.dependencies]\npython = "*"\n' >"$location/pyproject.toml" + interpreter=$("$base_python" -c 'import sys; print(sys.executable)') + # Poetry uses an active virtual or conda environment instead of the one of the project. + # Without its own python it runs the `python` command, which can be python 2. + ( + unset VIRTUAL_ENV CONDA_PREFIX + export POETRY_VIRTUALENVS_IN_PROJECT=true POETRY_VIRTUALENVS_USE_POETRY_PYTHON=true + run_manager -C "$location" env use "$interpreter" >&2 + ) + ;; + pip-venv) + run_manager -m venv "$location" >&2 + ;; + esac +} + +# Install the requirements in the bootstrap environment unless it has their pinned versions +install_bootstrap_requirements() { + if [ -n "$version" ] && [ "$(distribution_version "$python" ewoks)" = "$version" ]; then + if [ -z "$engine" ]; then + return + fi + installed=$(distribution_version "$python" "$engine") + if [ -n "$installed" ] && [ "${engine_version:-$installed}" = "$installed" ]; then + return + fi + fi + + say "install $requirements in $location" + if [ "$manager" = uv ]; then + eval "run_manager pip install --quiet --upgrade --python \"\$python\" $requirements" >&2 || + die "cannot install $requirements (see --bootstrap-requirement)" + else + eval "\"\$python\" -m pip install --quiet --disable-pip-version-check --upgrade $requirements" >&2 || + die "cannot install $requirements (see --bootstrap-requirement)" + fi +} + +# Print and run `ewoks install` in the bootstrap environment +run_ewoks_install() { + eval "set -- \"\$python\" -m ewoks install $forwarded" + say "run $(quote_all "$@")" + + # Printed by `ewoks install` before its summary + EWOKS_INSTALL_SUMMARY_HEADER=$( + printf 'Bootstrap environment\n' + printf ' Python : %s\n' "$python" + printf ' Install: %s\n' "$(quote_all "$@")" + printf ' Remove : rm -rf %s' "$(quote "$location")" + ) + export EWOKS_INSTALL_SUMMARY_HEADER + if [ ! -t 0 ] && (exec /dev/null; then + # Confirmation prompts do not read stdin, which is this script with `curl | sh` + exec "$@" None: + config.addinivalue_line( + "markers", + "package_manager_install: installs package managers from the internet, " + "only runs when selected with '-m package_manager_install'", + ) + + +_PACKAGE_MANAGER_INSTALL = "package_manager_install" + + +def pytest_collection_modifyitems( + config: pytest.Config, items: List[pytest.Item] +) -> None: + """Skip the tests that install package managers unless they are selected.""" + if _PACKAGE_MANAGER_INSTALL in config.getoption("markexpr", ""): + return + skip = pytest.mark.skip( + reason=f"installs package managers: select with '-m {_PACKAGE_MANAGER_INSTALL}'" + ) + for item in items: + if _PACKAGE_MANAGER_INSTALL in item.keywords: + item.add_marker(skip) + + @pytest.fixture def mock_icat_client(monkeypatch): mock_defaults = MagicMock() diff --git a/src/ewoks/tests/requirements/test_bootstrap.py b/src/ewoks/tests/requirements/test_bootstrap.py new file mode 100644 index 0000000..2123c5d --- /dev/null +++ b/src/ewoks/tests/requirements/test_bootstrap.py @@ -0,0 +1,384 @@ +"""Tests of the scripts that run `ewoks install` with nothing but a package manager. + +The workflows are saved by `ewoks convert`. A fake bootstrap environment is +prepared in advance so the scripts do not install anything: its python interpreter +runs the ewoks under test, which is not on the python package index. Except for +the tests marked `package_manager_install`, which install package managers and +ewoks from the internet. +""" + +import os +import shlex +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Callable +from typing import Dict +from typing import List +from typing import Optional + +import pytest + +from ...bindings import convert_graph +from ...bindings import load_graph +from .utils import PYTHON_MINOR_VERSION + +_SCRIPTS = Path(__file__).parents[2] / "_bootstrap" +_INSTALLABLE_MANAGERS = ["uv", "pixi", "poetry", "conda"] +_GRAPH_ID = "bootstrap" + + +@pytest.fixture +def script() -> List[str]: + """Command that runs the bootstrap script.""" + if sys.platform == "win32": + pytest.skip("sh is not available on Windows") + return ["sh", str(_SCRIPTS / "ewoks-install.sh")] + + +@pytest.fixture +def no_managers_env(tmp_path) -> Dict[str, str]: + """Environment variables with a PATH that only provides python and the + system tools, which can include package managers (see `_skip_if_installed`).""" + bin_dir = tmp_path / "python-bin" + bin_dir.mkdir() + for name in ("python3", f"python{PYTHON_MINOR_VERSION}"): + _python_wrapper(bin_dir / name) + directories = [str(bin_dir), "/usr/bin", "/bin", "/usr/sbin", "/sbin"] + path = os.pathsep.join(directories) + env = { + name: value + for name, value in os.environ.items() + if name not in ("VIRTUAL_ENV", "CONDA_PREFIX") + } + env["PATH"] = path + return env + + +@pytest.fixture(scope="module") +def saved_workflow(tmp_path_factory) -> Callable[[str], Path]: + """Workflow saved by `ewoks convert` in a format: json, yaml or ows. Every + format is saved once because generating the requirements takes seconds.""" + directory = tmp_path_factory.mktemp("workflows") + workflows: Dict[str, Path] = {} + + def get(workflow_format: str) -> Path: + if workflow_format not in workflows: + workflows[workflow_format] = _save_workflow(directory, workflow_format) + return workflows[workflow_format] + + return get + + +@pytest.fixture +def workflow_format() -> str: + """Format of the workflow: json, yaml or ows.""" + return "json" + + +@pytest.fixture +def workflow(saved_workflow, workflow_format) -> Path: + """Workflow saved by `ewoks convert`.""" + return saved_workflow(workflow_format) + + +@pytest.fixture +def bootstrap_dir(tmp_path) -> Path: + """Directory of the bootstrap environments, with a space like the home + directory of some Windows users.""" + return tmp_path / "boot strap" + + +@pytest.fixture +def bootstrap_python(bootstrap_dir, workflow) -> Path: + """Python interpreter of a fake pip-venv bootstrap environment of the workflow.""" + return _fake_bootstrap_environment(bootstrap_dir, workflow) + + +@pytest.fixture +def failing_install(tmp_path) -> List[str]: + """Arguments that make `ewoks install` fail before it installs anything: the + environment of the workflow is a directory that is not a python environment.""" + env_root = tmp_path / "envs" + (env_root / _GRAPH_ID).mkdir(parents=True) + return ["--yes", "--clean", "--env-root", str(env_root)] + + +def test_run_ewoks_install( + script, workflow, bootstrap_dir, bootstrap_python, failing_install +): + """The bootstrap environment of the ewoks version of the workflow runs + `ewoks install` with the arguments that are not for the script. The command is + printed before it runs and again before the summary of `ewoks install`.""" + arguments = [str(workflow), *_pip_venv_arguments(), *failing_install] + + result = _run(script, arguments, bootstrap_dir) + + assert "is not a python environment" in result.stderr + command = _printed_command(result) + assert str(bootstrap_python) in command + assert command.endswith(f"-m ewoks install {' '.join(arguments)}") + header, _ = result.stdout.split("Install failed for") + assert "Bootstrap environment" in header + assert f"Install: {command}" in header + + +def test_manager_command_with_spaces( + script, workflow, bootstrap_dir, bootstrap_python, failing_install, tmp_path +): + """The package manager command is quoted like the one of `ewoks install`.""" + python = tmp_path / "python bin" / "python" + python.parent.mkdir() + _python_wrapper(python) + arguments = [ + str(workflow), + "--package-manager-name", + "pip-venv", + "--package-manager-command", + shlex.quote(str(python)), + ] + + result = _run(script, [*arguments, *failing_install], bootstrap_dir) + + assert "@python=" in _printed_command(result) + + +def test_installed_manager_with_spaces( + workflow, no_managers_env, bootstrap_dir, failing_install +): + """The package manager installed by the script is passed to `ewoks install` + quoted for the shell.""" + _skip_if_installed("uv", no_managers_env) + if sys.platform == "win32": + pytest.skip("sh is not available on Windows") + uv = bootstrap_dir / "package-managers" / "uv" / "bin" / "uv" + uv.parent.mkdir(parents=True) + uv.write_text("#!/bin/sh\nexit 1\n") + uv.chmod(0o755) + _fake_bootstrap_environment(bootstrap_dir, workflow, manager="uv") + + result = _run( + ["sh", str(_SCRIPTS / "ewoks-install.sh")], + [str(workflow), "--package-manager-name", "uv", *failing_install], + bootstrap_dir, + env=no_managers_env, + ) + + command = shlex.split(_printed_command(result)) + manager_command = command[command.index("--package-manager-command") + 1] + assert shlex.split(manager_command) == [str(uv)] + + +def test_relative_workflow( + script, workflow, bootstrap_dir, bootstrap_python, failing_install +): + """`ewoks install` gets the absolute path of the workflow.""" + result = _run( + script, + [workflow.name, *_pip_venv_arguments(), *failing_install], + bootstrap_dir, + cwd=workflow.parent, + ) + + command = _printed_command(result).replace("'", "") + assert f"-m ewoks install {workflow}" in command + + +def test_no_workflow(script, bootstrap_dir): + result = _run(script, ["--test", "demo"], bootstrap_dir) + + assert result.returncode != 0 + assert "provide the file of the workflow" in result.stderr + + +def test_unsupported_manager(script, workflow, bootstrap_dir): + result = _run( + script, [str(workflow), "--package-manager-name", "unknown"], bootstrap_dir + ) + + assert result.returncode != 0 + assert "package manager 'unknown' is not supported" in result.stderr + + +def test_install_manager_requires_name(script, workflow, bootstrap_dir): + result = _run(script, [str(workflow), "--install-package-manager"], bootstrap_dir) + + assert result.returncode != 0 + assert "--install-package-manager requires --package-manager-name" in ( + result.stderr + ) + + +def test_manager_not_installed(script, workflow, no_managers_env, bootstrap_dir): + _skip_if_installed("pixi", no_managers_env) + + result = _run( + script, + [str(workflow), "--package-manager-name", "pixi"], + bootstrap_dir, + env=no_managers_env, + ) + + assert result.returncode != 0 + assert "pixi is not installed (see --install-package-manager)" in result.stderr + + +def test_install_conda_in_directory_with_spaces( + script, workflow, no_managers_env, bootstrap_dir +): + _skip_if_installed("conda", no_managers_env) + + result = _run( + script, + [ + "--install-package-manager", + str(workflow), + "--package-manager-name", + "conda", + ], + bootstrap_dir, + env=no_managers_env, + ) + + assert result.returncode != 0 + assert "Miniforge cannot be installed in a directory with spaces" in result.stderr + assert "install conda in" not in result.stderr + + +@pytest.mark.package_manager_install +@pytest.mark.parametrize("manager", _INSTALLABLE_MANAGERS) +def test_install_manager( + script, workflow, no_managers_env, bootstrap_dir, failing_install, manager +): + """The package manager is installed in the bootstrap directory, used again + by the next run and passed to `ewoks install`.""" + _skip_if_installed(manager, no_managers_env) + if manager == "conda": + # Miniforge cannot be installed in a directory with spaces + bootstrap_dir = bootstrap_dir.with_name("bootstrap") + arguments = [ + "--bootstrap-requirement", + "ewoks", + str(workflow), + "--package-manager-name", + manager, + *failing_install, + ] + + result = _run( + script, + ["--install-package-manager", *arguments], + bootstrap_dir, + env=no_managers_env, + timeout=1800, + ) + + assert f"install {manager} in" in result.stderr + manager_dir = bootstrap_dir / "package-managers" / manager + command = _printed_command(result) + assert str(manager_dir) in command + + result = _run(script, arguments, bootstrap_dir, env=no_managers_env) + + assert f"install {manager} in" not in result.stderr + assert _printed_command(result) == command + + +@pytest.mark.parametrize("workflow_format", ["json", "yaml", "ows"]) +def test_requirements_of_any_format( + script, workflow, bootstrap_dir, bootstrap_python, failing_install +): + """The bootstrap environment of the ewoks version and the engine of the + workflow is used, whatever the format of the workflow.""" + result = _run( + script, + [str(workflow), *_pip_venv_arguments(), *failing_install], + bootstrap_dir, + ) + + assert "install ewoks" not in result.stderr + assert str(bootstrap_python) in _printed_command(result) + + +def _skip_if_installed(manager: str, env: Dict[str, str]) -> None: + """Skip a test that needs a package manager to be missing from the PATH.""" + if shutil.which(manager, path=env["PATH"]): + pytest.skip(f"{manager} is installed in a system directory") + + +def _fake_bootstrap_environment( + root: Path, workflow: Path, manager: str = "pip-venv" +) -> Path: + """Bootstrap environment of a workflow, as the scripts cache it, whose python + interpreter is the one of the tests. The scripts find the ewoks under test + installed in it, so they do not install anything. Returns the python + interpreter.""" + ewoks = load_graph(str(workflow)).graph.graph["requirements"]["ewoks"] + engine = ewoks["engine"] + name = ( + f"ewoks={ewoks['version']}@{engine['name']}={engine['version']}" + f"@python={PYTHON_MINOR_VERSION}" + ) + python = root / manager / name / "bin" / "python" + python.parent.mkdir(parents=True) + _python_wrapper(python) + return python + + +def _python_wrapper(path: Path) -> None: + """POSIX shell script that runs the python interpreter of the tests.""" + with open(path, "w", encoding="utf-8") as fh: + fh.write(f'#!/bin/sh\nexec "{sys.executable}" "$@"\n') + path.chmod(0o755) + + +def _save_workflow(directory: Path, workflow_format: str) -> Path: + """Workflow saved by `ewoks convert` with its requirements.""" + if workflow_format == "ows": + pytest.importorskip("ewoksorange") + filename = directory / f"workflow.{workflow_format}" + convert_graph( + {"graph": {"id": _GRAPH_ID}, "nodes": [], "links": []}, + str(filename), + package_manager_name="pip-venv", + package_manager_command=(sys.executable,), + ) + return filename + + +def _pip_venv_arguments() -> List[str]: + return [ + "--package-manager-name", + "pip-venv", + "--package-manager-command", + sys.executable, + ] + + +def _printed_command(result: "subprocess.CompletedProcess[str]") -> str: + """`ewoks install` command that a script printed before running it.""" + prefix = "ewoks-install: run " + for line in result.stderr.splitlines(): + if line.startswith(prefix): + return line[len(prefix) :] + raise AssertionError(f"no command printed:\n{result.stderr}") + + +def _run( + script: List[str], + arguments: List[str], + bootstrap_dir: Path, + env: Optional[Dict[str, str]] = None, + timeout: float = 300, + cwd: Optional[Path] = None, +) -> "subprocess.CompletedProcess[str]": + return subprocess.run( # noqa: S603 - Script under test + [*script, "--bootstrap-dir", str(bootstrap_dir), *arguments], + capture_output=True, + text=True, + env=env, + cwd=cwd, + stdin=subprocess.DEVNULL, + timeout=timeout, + ) diff --git a/src/ewoks/tests/requirements/test_install_cli.py b/src/ewoks/tests/requirements/test_install_cli.py index 3305b46..3082fd0 100644 --- a/src/ewoks/tests/requirements/test_install_cli.py +++ b/src/ewoks/tests/requirements/test_install_cli.py @@ -1,6 +1,7 @@ """Tests of `ewoks install`, which does not depend on the package manager.""" import json +import os import shutil import subprocess import sys @@ -41,6 +42,33 @@ def test_printed_commands_with_spaces(tmp_path): assert _remove_command(location) == f"rm -rf '{location}'" +def test_summary_header(tmp_path): + """The text of the environment variable is printed before the summary.""" + graph = _graph(manager_requirements("pip-venv", python_version=PYTHON_VERSION)) + # A directory that is not a python environment, so nothing is installed + (tmp_path / _GRAPH_ID).mkdir() + argv = [ + sys.executable, + "-m", + "ewoks", + "install", + json.dumps(graph), + "--yes", + "--clean", + "--env-root", + str(tmp_path), + "--package-manager-name", + "pip-venv", + ] + env = {**os.environ, "EWOKS_INSTALL_SUMMARY_HEADER": "Header\n line"} + + result = subprocess.run( # noqa: S603 - Trusted test command + argv, capture_output=True, text=True, env=env + ) + + assert result.stdout.startswith("Header\n line\nInstall failed for") + + def test_install_with_files(fast_manager_case, env_root): files = fast_manager_case.native_files(DISTRIBUTIONS, PYTHON_VERSION) if not files: From 93a6bfaed87d6e273c7aa195fd72f852b6467424 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Sat, 24 Oct 2026 15:13:49 +0200 Subject: [PATCH 2/3] Add the ewoks-install.ps1 script for Windows and PowerShell Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test.yml | 2 +- CHANGELOG.md | 2 +- doc/tutorials/install.rst | 20 + doc/tutorials/install/conda.rst | 6 + doc/tutorials/install/pip_venv.rst | 6 + doc/tutorials/install/pixi.rst | 6 + doc/tutorials/install/poetry.rst | 6 + doc/tutorials/install/uv.rst | 6 + pyproject.toml | 2 +- src/ewoks/_bootstrap/ewoks-install.ps1 | 722 ++++++++++++++++++ src/ewoks/tests/requirements/conftest.py | 3 + .../tests/requirements/test_bootstrap.py | 55 +- 12 files changed, 821 insertions(+), 15 deletions(-) create mode 100644 src/ewoks/_bootstrap/ewoks-install.ps1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 20b197a..954795e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -96,7 +96,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] # Run linter / checks checks: diff --git a/CHANGELOG.md b/CHANGELOG.md index fec6b22..45b9347 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - add `--in-place`, `--python-version` and `--with-ewoks` arguments. - `ewoks convert`: add `--package-manager-name` and `--package-manager-command` arguments. - `ewoks execute`: add `--env` argument. -- Add `ewoks-install.sh` script to install the environment of a workflow without ewoks. +- Add `ewoks-install.sh` and `ewoks-install.ps1` scripts to install the environment of a workflow without ewoks. - Add `ewoks lint` CLI command to check if a workflow is conform to the Ewoks specification. ### Changed diff --git a/doc/tutorials/install.rst b/doc/tutorials/install.rst index d45d06b..b2a2145 100644 --- a/doc/tutorials/install.rst +++ b/doc/tutorials/install.rst @@ -109,6 +109,12 @@ arguments are passed to ``ewoks install``, except for the options of the script curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json + Without a :term:`workflow` at hand, try it with the demo :term:`workflow` of this documentation, which only requires :term:`ewoks` @@ -128,6 +134,13 @@ documentation, which only requires :term:`ewoks` curl -LsSfO |docs_url|/demo.json curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + .. group-tab:: Windows + + .. code-block:: powershell + + irm |docs_url|/demo.json -OutFile demo.json + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json + Alternatively, download the script first, for example to read it before running it .. tabs:: @@ -146,6 +159,13 @@ Alternatively, download the script first, for example to read it before running curl -LsSf -o ewoks-install.sh |docs_url|/ewoks-install.sh sh ewoks-install.sh demo.json + .. group-tab:: Windows + + .. code-block:: powershell + + irm |docs_url|/ewoks-install.ps1 -OutFile ewoks-install.ps1 + powershell -ExecutionPolicy Bypass -File ewoks-install.ps1 demo.json + * ``--package-manager-name`` and ``--package-manager-command`` select the :term:`package manager` that creates the bootstrap environment. Without them it is the first one that is available of uv, pixi, conda, poetry and pip-venv. diff --git a/doc/tutorials/install/conda.rst b/doc/tutorials/install/conda.rst index 59978a0..2325f3c 100644 --- a/doc/tutorials/install/conda.rst +++ b/doc/tutorials/install/conda.rst @@ -164,6 +164,12 @@ environment without an environment with :term:`ewoks` curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json + It selects the :term:`package manager` itself and prints the command to execute the :term:`workflow` in the environment it creates. diff --git a/doc/tutorials/install/pip_venv.rst b/doc/tutorials/install/pip_venv.rst index 49823a2..63a4b72 100644 --- a/doc/tutorials/install/pip_venv.rst +++ b/doc/tutorials/install/pip_venv.rst @@ -123,6 +123,12 @@ environment without an environment with :term:`ewoks` curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json + It selects the :term:`package manager` itself and prints the command to execute the :term:`workflow` in the environment it creates. diff --git a/doc/tutorials/install/pixi.rst b/doc/tutorials/install/pixi.rst index 2ce7309..6b21ba2 100644 --- a/doc/tutorials/install/pixi.rst +++ b/doc/tutorials/install/pixi.rst @@ -159,6 +159,12 @@ environment without an environment with :term:`ewoks` curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json + It selects the :term:`package manager` itself and prints the command to execute the :term:`workflow` in the environment it creates. diff --git a/doc/tutorials/install/poetry.rst b/doc/tutorials/install/poetry.rst index 5b5e628..1dfb636 100644 --- a/doc/tutorials/install/poetry.rst +++ b/doc/tutorials/install/poetry.rst @@ -153,6 +153,12 @@ environment without an environment with :term:`ewoks` curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json + It selects the :term:`package manager` itself and prints the command to execute the :term:`workflow` in the environment it creates. diff --git a/doc/tutorials/install/uv.rst b/doc/tutorials/install/uv.rst index 1e2ffe7..2ef1305 100644 --- a/doc/tutorials/install/uv.rst +++ b/doc/tutorials/install/uv.rst @@ -145,6 +145,12 @@ environment without an environment with :term:`ewoks` curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json + It selects the :term:`package manager` itself and prints the command to execute the :term:`workflow` in the environment it creates. diff --git a/pyproject.toml b/pyproject.toml index 9527b51..6b60671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ ewoks = "ewoks.__main__:main" [tool.setuptools.package-data] "ewoks.tests.notebooks" = ["*.ipynb"] -"ewoks._bootstrap" = ["*.sh"] +"ewoks._bootstrap" = ["*.sh", "*.ps1"] [tool.ruff.lint] select = [ diff --git a/src/ewoks/_bootstrap/ewoks-install.ps1 b/src/ewoks/_bootstrap/ewoks-install.ps1 new file mode 100644 index 0000000..097c1b5 --- /dev/null +++ b/src/ewoks/_bootstrap/ewoks-install.ps1 @@ -0,0 +1,722 @@ +# Install the python environment of an ewoks workflow with nothing but a package +# manager. For example +# +# & ([scriptblock]::Create((irm https://ewoks.readthedocs.io/en/stable/ewoks-install.ps1))) demo.json --package-manager-name uv +# +# or download the script first +# +# irm https://ewoks.readthedocs.io/en/stable/ewoks-install.ps1 -OutFile ewoks-install.ps1 +# powershell -ExecutionPolicy Bypass -File ewoks-install.ps1 demo.json --package-manager-name uv +# +# Steps (see `Invoke-EwoksInstall` at the end), like ewoks-install.sh: +# +# 1. Split the arguments in options of this script and arguments of `ewoks install`. +# 2. Select the package manager. +# 3. Read the ewoks, engine and python versions from the workflow requirements. +# 4. Create a bootstrap environment with that ewoks and engine, unless it exists. +# 5. Run `ewoks install` in the bootstrap environment. +# +# The bootstrap environments are cached in ~/.ewoks/bootstrap// +# where the name is, for example, `ewoks=7.0.0@ewokscore=5.1.0@python=3.12`. +# +# Errors are thrown instead of calling `exit`, which would close the PowerShell +# session that runs the script block. Native commands run with the "Continue" +# error action: Windows PowerShell 5.1 turns their stderr into errors. + +$ErrorActionPreference = "Stop" + +$usage = @' +Usage: ewoks-install.ps1 [OPTIONS] WORKFLOW [ARGS ...] + +Install the python environment of an ewoks workflow with nothing but a package +manager. The ewoks version that generated the workflow requirements, and the +engine that saved the workflow, are installed in a bootstrap environment, which +then runs + + ewoks install WORKFLOW ARGS + +See 'ewoks install --help' for ARGS. The package manager that creates the +bootstrap environment is selected with '--package-manager-name' and +'--package-manager-command', which are passed to 'ewoks install' as well. +Without them it is the first available of uv, pixi, conda, poetry and pip-venv. +A package manager that is not installed can be installed in the bootstrap +directory with '--install-package-manager'. + +Options: + --bootstrap-requirement REQ + Install REQ in the bootstrap environment instead of + the ewoks version and engine of the workflow, for + example 'ewoks>=7' or a wheel file. Repeat it to + install an engine as well, for example + '--bootstrap-requirement ewoks + --bootstrap-requirement ewoksorange'. + --bootstrap-dir DIR Directory of the bootstrap environments. + Default: ~/.ewoks/bootstrap + --install-package-manager + Install the package manager of + '--package-manager-name' when it is not installed. + -h, --help Show this help. +'@ + +$onWindows = ($PSVersionTable.PSEdition -eq "Desktop") -or $IsWindows + +# ============================================================================= +# Helpers +# ============================================================================= + +function Write-Info([string]$message) { + [Console]::Error.WriteLine("ewoks-install: $message") +} + +# Argument quoted for PowerShell when needed +function Format-Argument([string]$argument) { + if ($argument -match '^[A-Za-z0-9_./=:+\\-]+$') { + return $argument + } + return "'" + $argument.Replace("'", "''") + "'" +} + +# Arguments quoted for PowerShell and joined with spaces +function Format-Arguments([string[]]$arguments) { + return @($arguments | ForEach-Object { Format-Argument $_ }) -join " " +} + +function Test-Command([string]$name) { + return [bool](Get-Command $name -CommandType Application -ErrorAction SilentlyContinue) +} + +# Run a native command with its output on stderr instead of in the pipeline +function Invoke-Native([string[]]$command) { + $ErrorActionPreference = "Continue" + $arguments = @() + if ($command.Count -gt 1) { + $arguments = $command[1..($command.Count - 1)] + } + & $command[0] @arguments 2>&1 | ForEach-Object { [Console]::Error.WriteLine($_) } + if ($LASTEXITCODE -ne 0) { + throw "command failed: $($command -join ' ')" + } +} + +# Run a native command with additional environment variables +function Invoke-NativeWithEnv([hashtable]$variables, [string[]]$command) { + $previous = @{} + foreach ($name in $variables.Keys) { + $previous[$name] = [Environment]::GetEnvironmentVariable($name) + Set-EnvironmentVariable $name $variables[$name] + } + try { + Invoke-Native $command + } finally { + foreach ($name in $variables.Keys) { + Set-EnvironmentVariable $name $previous[$name] + } + } +} + +# Set an environment variable, or remove it when the value is $null. +# [Environment]::SetEnvironmentVariable gets "" for $null, which PowerShell 7 keeps. +function Set-EnvironmentVariable([string]$name, [object]$value) { + if ($null -eq $value) { + Remove-Item -LiteralPath "Env:$name" -ErrorAction SilentlyContinue + } else { + Set-Item -LiteralPath "Env:$name" -Value $value + } +} + +# Output of a native command +function Get-NativeOutput([string[]]$command) { + $ErrorActionPreference = "Continue" + $arguments = @() + if ($command.Count -gt 1) { + $arguments = $command[1..($command.Count - 1)] + } + $output = & $command[0] @arguments + if ($LASTEXITCODE -ne 0) { + throw "command failed: $($command -join ' ')" + } + return "$output".Trim() +} + +# Python interpreter with this major.minor version, or the default one +function Find-Python([string]$majorMinor) { + $ErrorActionPreference = "Continue" + if ($onWindows) { + # The python of the PATH, like the one of an active environment, when it has this version + if ($majorMinor -and (Test-Command "python")) { + try { + $pathMajorMinor = Get-PythonMajorMinor @("python") + } catch { + $pathMajorMinor = "" + } + if ($pathMajorMinor -eq $majorMinor) { + return @("python") + } + } + if (Test-Command "py") { + if ($majorMinor) { + & py "-$majorMinor" -c "pass" 2>$null | Out-Null + if ($LASTEXITCODE -eq 0) { + return @("py", "-$majorMinor") + } + Write-Info "warning: python $majorMinor of the workflow is not found: using py -3" + } + return @("py", "-3") + } + } elseif ($majorMinor -and (Test-Command "python$majorMinor")) { + return @("python$majorMinor") + } elseif (Test-Command "python3") { + if ($majorMinor) { Write-Info "warning: python$majorMinor of the workflow is not found: using python3" } + return @("python3") + } + if (Test-Command "python") { + if ($majorMinor) { Write-Info "warning: python $majorMinor of the workflow is not found: using python" } + return @("python") + } + throw "no python interpreter found (see --package-manager-name uv --install-package-manager)" +} + +# major.minor version of a python interpreter +function Get-PythonMajorMinor([string[]]$python) { + return Get-NativeOutput ($python + @("-c", 'import sys; print(''%d.%d'' % sys.version_info[:2])')) +} + +# Version of an installed distribution, or nothing when it is not installed +function Get-DistributionVersion([string]$python, [string]$name) { + $ErrorActionPreference = "Continue" + try { + return "$(& $python -c 'import importlib.metadata as m, sys; print(m.version(sys.argv[1]))' $name 2>$null)".Trim() + } catch { + return "" + } +} + +# ============================================================================= +# Workflow requirements +# ============================================================================= + +# Value in the requirements of a workflow file: ewoks.version, ewoks.engine.name, +# ewoks.engine.version or python.version. The requirements are JSON, which can be +# embedded in another format like the XML of .ows files, or YAML. +function Get-RequirementValue([string]$content, [string]$key) { + $value = Get-JsonRequirementValue $content $key + if (-not $value) { + $value = Get-YamlRequirementValue $content $key + } + return $value +} + +# Value in JSON without a JSON parser. For example `ewoks.version` in +# +# "ewoks": {"engine": {"name": "ewokscore", "version": "5.1.0"}, "version": "7.0.0"} +function Get-JsonRequirementValue([string]$content, [string]$key) { + $q = "[""']" + $value = "$q\s*:\s*$q([^""']*)$q" + # The members of an object, which can contain objects themselves + $members = "[^{}]*(?:\{[^{}]*\}[^{}]*)*" + switch ($key) { + "ewoks.version" { $pattern = "${q}ewoks$q\s*:\s*\{$members${q}version$value" } + "ewoks.engine.name" { $pattern = "${q}ewoks$q\s*:\s*\{[^{}]*${q}engine$q\s*:\s*\{[^{}]*${q}name$value" } + "ewoks.engine.version" { $pattern = "${q}ewoks$q\s*:\s*\{[^{}]*${q}engine$q\s*:\s*\{[^{}]*${q}version$value" } + "python.version" { $pattern = "${q}python$q\s*:\s*\{[^{}]*${q}version$value" } + } + # With the quotes that XML escapes + $found = [regex]::Matches($content.Replace(""", '"'), $pattern) + if ($found.Count -eq 0) { + return "" + } + return $found[$found.Count - 1].Groups[1].Value +} + +# Value in YAML without a YAML parser: the last `key: value` line whose key path +# ends with requirements. +function Get-YamlRequirementValue([string]$content, [string]$key) { + $wanted = "requirements.$key" + $indents = New-Object System.Collections.Generic.List[int] + $keys = New-Object System.Collections.Generic.List[string] + $found = "" + foreach ($line in ($content -split "\r?\n")) { + if ($line -match '^\s*(#|$)') { + continue + } + # The dash of a list item belongs to the indentation of its first key + $match = [regex]::Match($line, '^([ -]*)([A-Za-z0-9_]+):\s*(.*)$') + if (-not $match.Success) { + continue + } + # Key path of this line, from the keys of the lines with less indentation + $indent = $match.Groups[1].Length + while ($indents.Count -gt 0 -and $indents[$indents.Count - 1] -ge $indent) { + $indents.RemoveAt($indents.Count - 1) + $keys.RemoveAt($keys.Count - 1) + } + $indents.Add($indent) + $keys.Add($match.Groups[2].Value) + $path = $keys -join "." + $value = $match.Groups[3].Value + if ($value -and (($path -eq $wanted) -or $path.EndsWith(".$wanted"))) { + $found = $value -replace "^[""']|[""']$", "" + } + } + return $found +} + +# ============================================================================= +# Package managers +# ============================================================================= + +# Arguments of a '--package-manager-command' argument, split like `ewoks install` does +function Split-Command([string]$command) { + if ($onWindows) { + # Spaces escaped with a backslash are part of an argument + return @([regex]::Split($command, '(?=7") + $tag = "ewoks=latest" + } + if ($engineVersion) { + $requirements += "$engine==$engineVersion" + $tag = "$tag@$engine=$engineVersion" + } elseif ($engine) { + $requirements += $engine + $tag = "$tag@$engine" + } + $state.Version = $version + $state.Engine = $engine + $state.EngineVersion = $engineVersion + $state.Requirements = $requirements + $state.Tag = $tag +} + +# Install the package manager in the bootstrap directory when it is not installed. +# `ewoks install` needs it as well. +function Initialize-Manager([hashtable]$state) { + if (-not $state.Command -or $state.ManagerCommand -or (Test-Command $state.Command[0])) { + return + } + $manager = $state.Manager + $managerDir = Join-Path (Join-Path $state.Root "package-managers") $manager + $executable = Get-ManagerExecutable $manager $managerDir + if (-not (Test-Path -LiteralPath $executable -PathType Leaf)) { + if (-not $state.InstallPackageManager) { + throw "$manager is not installed (see --install-package-manager)" + } + Install-Manager $manager $managerDir + if (-not (Test-Path -LiteralPath $executable -PathType Leaf)) { + throw "cannot install $manager in $managerDir" + } + } + $state.Command = @($executable) + $state.Forwarded += @("--package-manager-command", (Format-Command $executable)) +} + +# Set `Python` and `Location`: the interpreter and directory of the bootstrap +# environment, which is created when it does not exist +function New-BootstrapEnvironment([hashtable]$state) { + $manager = $state.Manager + $command = $state.Command + + # pip-venv and poetry cannot provide a python version: the interpreter decides + $pythonVersion = $state.PythonVersion + $basePython = @() + if ($manager -eq "pip-venv") { + if (-not $command) { + $command = @(Find-Python $pythonVersion) + $state.Command = $command + } + $pythonVersion = Get-PythonMajorMinor $command + } elseif ($manager -eq "poetry") { + $basePython = @(Find-Python $pythonVersion) + $pythonVersion = Get-PythonMajorMinor $basePython + } + + $name = $state.Tag + if ($pythonVersion) { + $name = "$name@python=$pythonVersion" + } + $location = Join-Path (Join-Path $state.Root $manager) $name + if ($onWindows) { + $venvPython = Join-Path "Scripts" "python.exe" + $prefixPython = "python.exe" + } else { + $venvPython = Join-Path "bin" "python" + $prefixPython = $venvPython + } + switch ($manager) { + "pixi" { $python = Join-Path $location (Join-Path ".pixi/envs/default" $prefixPython) } + "conda" { $python = Join-Path $location $prefixPython } + "poetry" { $python = Join-Path $location (Join-Path ".venv" $venvPython) } + default { $python = Join-Path $location $venvPython } + } + $state.Location = $location + $state.Python = $python + if (Test-Path -LiteralPath $python -PathType Leaf) { + return + } + + Write-Info "create the bootstrap environment $location" + # A previous attempt failed + if (Test-Path -LiteralPath $location) { + Remove-Item -LiteralPath $location -Recurse -Force + } + New-Item -ItemType Directory -Path $location -Force | Out-Null + $pythonSpec = "python" + if ($pythonVersion) { + $pythonSpec = "python=$pythonVersion" + } + switch ($manager) { + "uv" { + $create = $command + @("venv", "--quiet") + if ($pythonVersion) { $create += @("--python", $pythonVersion) } + Invoke-Native ($create + @($location)) + } + "pixi" { + Invoke-Native ($command + @("init", $location)) + Invoke-Native ($command + @("add", "--manifest-path", (Join-Path $location "pixi.toml"), $pythonSpec, "pip")) + } + "conda" { + Invoke-Native ($command + @("create", "--yes", "--quiet", "--prefix", $location, $pythonSpec, "pip")) + } + "poetry" { + $pyproject = "[tool.poetry]`npackage-mode = false`n`n[tool.poetry.dependencies]`npython = `"*`"`n" + [System.IO.File]::WriteAllText((Join-Path $location "pyproject.toml"), $pyproject) + $interpreter = Get-NativeOutput ($basePython + @("-c", "import sys; print(sys.executable)")) + # Poetry uses an active virtual or conda environment instead of the one of the project. + # Without its own python it runs the `python` command, which can be python 2. + $variables = @{ + VIRTUAL_ENV = $null + CONDA_PREFIX = $null + POETRY_VIRTUALENVS_IN_PROJECT = "true" + POETRY_VIRTUALENVS_USE_POETRY_PYTHON = "true" + } + Invoke-NativeWithEnv $variables ($command + @("-C", $location, "env", "use", $interpreter)) + } + "pip-venv" { + Invoke-Native ($command + @("-m", "venv", $location)) + } + } +} + +# Install the requirements in the bootstrap environment unless it has their pinned versions +function Install-BootstrapRequirements([hashtable]$state) { + $python = $state.Python + if ($state.Version -and ((Get-DistributionVersion $python "ewoks") -eq $state.Version)) { + if (-not $state.Engine) { + return + } + $installed = Get-DistributionVersion $python $state.Engine + if ($installed -and (-not $state.EngineVersion -or $installed -eq $state.EngineVersion)) { + return + } + } + + $requirements = $state.Requirements + Write-Info "install $(Format-Arguments $requirements) in $($state.Location)" + try { + if ($state.Manager -eq "uv") { + Invoke-Native ($state.Command + @("pip", "install", "--quiet", "--upgrade", "--python", $python) + $requirements) + } else { + Invoke-Native (@($python, "-m", "pip", "install", "--quiet", "--disable-pip-version-check", "--upgrade") + $requirements) + } + } catch { + throw "cannot install $(Format-Arguments $requirements) (see --bootstrap-requirement)" + } +} + +# Print and run `ewoks install` in the bootstrap environment +function Invoke-EwoksInstallCommand([hashtable]$state) { + $install = @("-m", "ewoks", "install") + $state.Forwarded + $command = "& " + (Format-Arguments (@($state.Python) + $install)) + Write-Info "run $command" + # Printed by `ewoks install` before its summary + $header = @( + "Bootstrap environment" + " Python : $($state.Python)" + " Install: $command" + " Remove : Remove-Item -Recurse -Force $(Format-Argument $state.Location)" + ) -join "`n" + $previous = $env:EWOKS_INSTALL_SUMMARY_HEADER + $env:EWOKS_INSTALL_SUMMARY_HEADER = $header + try { + $ErrorActionPreference = "Continue" + & $state.Python @install + } finally { + # Not left in the PowerShell session that runs the script block + $env:EWOKS_INSTALL_SUMMARY_HEADER = $previous + } +} + +# Explain how to install without this script +function Write-ManualRoute([hashtable]$state) { + $python = "python" + if ($state.PythonVersion) { $python = "python $($state.PythonVersion)" } + Write-Info "the bootstrap failed. To install without this script:" + Write-Info " 1. create a $python environment and run in it: pip install $(Format-Arguments $state.Requirements)" + Write-Info " 2. run in that environment: python -m ewoks install $(Format-Arguments $state.Forwarded)" +} + +function Invoke-EwoksInstall([object[]]$arguments) { + $state = Read-Arguments $arguments + if ($state.Help) { + Write-Output $usage + return + } + if (-not $state.Workflow) { + Write-Info $usage + throw "provide the file of the workflow (no existing file in the arguments)" + } + Select-Manager $state + Read-WorkflowRequirements $state + + try { + Initialize-Manager $state + New-BootstrapEnvironment $state + Install-BootstrapRequirements $state + } catch { + # An interruption is not caught + Write-ManualRoute $state + throw + } + Invoke-EwoksInstallCommand $state +} + +Invoke-EwoksInstall $args diff --git a/src/ewoks/tests/requirements/conftest.py b/src/ewoks/tests/requirements/conftest.py index cb50a1a..92c3a5d 100644 --- a/src/ewoks/tests/requirements/conftest.py +++ b/src/ewoks/tests/requirements/conftest.py @@ -52,6 +52,9 @@ def _home_variables(home: Path) -> Dict[str, Path]: # Directories in which package managers create named environments "CONDA_ENVS_DIRS": home / "conda" / "envs", "POETRY_VIRTUALENVS_PATH": home / "poetry" / "virtualenvs", + # Pixi cannot find its cache directory without a user profile, like on CI + # runners that run as a service + "PIXI_CACHE_DIR": home / "pixi" / "cache", # Channels are no longer provided by the home directory of the user "CONDARC": condarc, } diff --git a/src/ewoks/tests/requirements/test_bootstrap.py b/src/ewoks/tests/requirements/test_bootstrap.py index 2123c5d..da33173 100644 --- a/src/ewoks/tests/requirements/test_bootstrap.py +++ b/src/ewoks/tests/requirements/test_bootstrap.py @@ -29,23 +29,46 @@ _GRAPH_ID = "bootstrap" -@pytest.fixture -def script() -> List[str]: - """Command that runs the bootstrap script.""" - if sys.platform == "win32": - pytest.skip("sh is not available on Windows") - return ["sh", str(_SCRIPTS / "ewoks-install.sh")] +@pytest.fixture(params=["sh", "pwsh", "powershell"]) +def script(request) -> List[str]: + """Command that runs a bootstrap script. `powershell` is Windows PowerShell 5.1.""" + if request.param == "sh": + if sys.platform == "win32": + pytest.skip("sh is not available on Windows") + return ["sh", str(_SCRIPTS / "ewoks-install.sh")] + if request.param == "powershell" and sys.platform != "win32": + pytest.skip("Windows PowerShell is only available on Windows") + powershell = shutil.which(request.param) + if powershell is None: + pytest.skip(f"{request.param} is not installed") + return [ + powershell, + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(_SCRIPTS / "ewoks-install.ps1"), + ] @pytest.fixture def no_managers_env(tmp_path) -> Dict[str, str]: """Environment variables with a PATH that only provides python and the system tools, which can include package managers (see `_skip_if_installed`).""" - bin_dir = tmp_path / "python-bin" - bin_dir.mkdir() - for name in ("python3", f"python{PYTHON_MINOR_VERSION}"): - _python_wrapper(bin_dir / name) - directories = [str(bin_dir), "/usr/bin", "/bin", "/usr/sbin", "/sbin"] + if sys.platform == "win32": + system_root = os.environ.get("SystemRoot", r"C:\Windows") + directories = [ + os.path.dirname(sys.executable), + os.path.join(system_root, "System32"), + system_root, + os.path.join(system_root, "System32", "WindowsPowerShell", "v1.0"), + ] + else: + bin_dir = tmp_path / "python-bin" + bin_dir.mkdir() + for name in ("python3", f"python{PYTHON_MINOR_VERSION}"): + _python_wrapper(bin_dir / name) + directories = [str(bin_dir), "/usr/bin", "/bin", "/usr/sbin", "/sbin"] path = os.pathsep.join(directories) env = { name: value @@ -128,6 +151,8 @@ def test_manager_command_with_spaces( script, workflow, bootstrap_dir, bootstrap_python, failing_install, tmp_path ): """The package manager command is quoted like the one of `ewoks install`.""" + if script[0] != "sh": + pytest.skip("PowerShell splits the command like `ewoks install` on Windows") python = tmp_path / "python bin" / "python" python.parent.mkdir() _python_wrapper(python) @@ -277,7 +302,11 @@ def test_install_manager( assert f"install {manager} in" in result.stderr manager_dir = bootstrap_dir / "package-managers" / manager command = _printed_command(result) - assert str(manager_dir) in command + if sys.platform == "win32": + # `ewoks install` splits the command on spaces that are not escaped + assert str(manager_dir).replace(" ", "\\ ") in command + else: + assert str(manager_dir) in command result = _run(script, arguments, bootstrap_dir, env=no_managers_env) @@ -314,6 +343,8 @@ def _fake_bootstrap_environment( interpreter is the one of the tests. The scripts find the ewoks under test installed in it, so they do not install anything. Returns the python interpreter.""" + if sys.platform == "win32": + pytest.skip("the fake bootstrap environment is a POSIX shell script") ewoks = load_graph(str(workflow)).graph.graph["requirements"]["ewoks"] engine = ewoks["engine"] name = ( From 07d65347e29af5fc6265065ec37421d7d4c0b81e Mon Sep 17 00:00:00 2001 From: Wout De Nolf Date: Tue, 3 Nov 2026 05:53:42 +0100 Subject: [PATCH 3/3] Update doc/tutorials/install.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Loïc Huder <42204205+loichuder@users.noreply.github.com> --- doc/tutorials/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/install.rst b/doc/tutorials/install.rst index d45d06b..96b47de 100644 --- a/doc/tutorials/install.rst +++ b/doc/tutorials/install.rst @@ -89,7 +89,7 @@ it the environment is created where the :term:`package manager` creates named en Create the environment without ewoks ++++++++++++++++++++++++++++++++++++ -``ewoks install`` needs a python environment with :term:`ewoks`. The ``ewoks-install`` script +``ewoks install`` needs a Python environment with :term:`ewoks`. The ``ewoks-install`` script only needs a :term:`package manager`: it installs the version of :term:`ewoks` that generated the requirements, and the engine that saved the :term:`workflow`, in a bootstrap environment and runs ``ewoks install`` from there. All