Use the workflow requirements of ewokscore

doc/conf.py

@@ -111,7 +111,7 @@ def _save_demo_workflow(app: Sphinx, exception: Optional[Exception]) -> None:
     graph, _ = get_graph("demo")
     workflow = convert_graph(graph, None)
     requirements = workflow["graph"]["requirements"]
-    names = ["ewoks", requirements["ewoks"]["engine"]["name"]]
+    names = list(requirements["bootstrap"]["distributions"])
     requirements["distributions"] = [
         {"name": name, "version": importlib.metadata.version(name)} for name in names
     ]

doc/howtoguides/requirements.rst

@@ -13,9 +13,9 @@ What is stored
 
 ``ewoks convert`` and ``ewoks execute -o convert_destination=...`` store
 
-* ``ewoks``: the version of :term:`ewoks` that generated the requirements and the engine
-  that saved the :term:`workflow`. The
-  :ref:`ewoks-install script <install_bootstrap>` installs both to run ``ewoks install``.
+* ``bootstrap``: the versions of :term:`ewoks` and of the engine that saved the
+  :term:`workflow`. The :ref:`ewoks-install script <install_bootstrap>` installs them to run
+  ``ewoks install``.
 * ``python`` and ``system``: the python interpreter and the operating system.
 * ``distributions``: every installed python package with its version and, when it was not
   installed from the python package index, the git commit or the archive it came from. Any

doc/reference/cli.rst

@@ -28,7 +28,8 @@ ewoks install
     Use ``--env-root`` for another directory to create the environment in and ``--env-name`` for another name than the workflow identifier.
     An environment that already exists is installed in, unless ``--clean`` is provided to remove it first.
     The :term:`workflow` can be executed in that environment when it contains **ewoks** itself, which is the case when the ``requirements`` contain it.
-    Use ``--with-ewoks`` to add **ewoks** when the ``requirements`` do not contain it, without changing any of the versions they do contain:
+    Use ``--with-ewoks`` to add **ewoks** when the ``requirements`` do not contain it.
+    The versions they contain are kept, unless they come from project files that allow other versions: a warning lists the versions that changed.
 
     .. code-block:: bash
 

doc/tutorials/install.rst

@@ -40,7 +40,7 @@ Store the requirements
 .. code-block:: json
 
     {
-        "ewoks": {"version": "7.0.0", "engine": {"name": "ewokscore", "version": "5.1.0"}},
+        "bootstrap": {"distributions": {"ewoks": "7.0.0", "ewokscore": "5.1.0"}},
         "python": {"version": "3.12.11", "implementation": "CPython", "...": "..."},
         "system": {"system": "Linux", "machine": "x86_64", "...": "..."},
         "distributions": [
@@ -54,9 +54,9 @@ Store the requirements
         }
     }
 
-* ``ewoks`` is the version of :term:`ewoks` that generated the requirements and the engine that
-  saved the :term:`workflow`: ``ewokscore`` for a ``.json`` or ``.yaml`` file and
-  ``ewoksorange`` for an ``.ows`` file.
+* ``bootstrap`` are the versions of :term:`ewoks` and of the engine that saved the
+  :term:`workflow`: ``ewokscore`` for a ``.json`` or ``.yaml`` file and ``ewoksorange`` for an
+  ``.ows`` file.
 * ``distributions`` are the installed python packages. Any :term:`package manager` can
   recreate the environment from this list.
 * ``manager`` is the :term:`package manager` that generated the requirements. Its ``files``
@@ -90,9 +90,9 @@ Create the environment without ewoks
 ++++++++++++++++++++++++++++++++++++
 
 ``ewoks install`` needs a Python environment with :term:`ewoks`. The ``ewoks-install`` script
-only needs a :term:`package manager`: it installs the version of :term:`ewoks` that generated
-the requirements, and the engine that saved the :term:`workflow`, in a bootstrap environment
-and runs ``ewoks install`` from there. All
+only needs a :term:`package manager`: it installs the ``bootstrap`` packages of the
+requirements, :term:`ewoks` and the engine that saved the :term:`workflow`, in a bootstrap
+environment and runs ``ewoks install`` from there. All
 arguments are passed to ``ewoks install``, except for the options of the script itself
 
 .. tabs::
@@ -170,7 +170,7 @@ Alternatively, download the script first, for example to read it before running
   :term:`package manager` that creates the bootstrap environment. Without them it is the first
   one that is available of uv, pixi, conda, poetry and pip-venv.
 * ``--bootstrap-requirement`` installs a package in the bootstrap environment instead of the
-  version of :term:`ewoks` and the engine of the requirements, for example
+  ``bootstrap`` packages of the requirements, for example
   ``--bootstrap-requirement "ewoks>=7"``. Repeat it to install an engine as well, for example
   ``--bootstrap-requirement ewoks --bootstrap-requirement ewoksorange``. The latest version of
   :term:`ewoks` is installed when the requirements do not provide one.
@@ -194,8 +194,9 @@ Execute the workflow
 ``--env`` executes the :term:`workflow` with the python interpreter of that environment
 instead of the current one. This works because the requirements contain ``ewoks`` itself: it
 was installed in the environment in which the :term:`workflow` was converted. When they do
-not, add ``--with-ewoks`` to ``ewoks install`` to install it without changing any of the
-versions coming from the :term:`workflow`.
+not, add ``--with-ewoks`` to ``ewoks install`` to install it. The versions coming from the
+:term:`workflow` are kept, unless they come from project files that allow other versions: a
+warning lists the versions that changed.
 
 Remove the environment
 ----------------------

pyproject.toml

@@ -16,7 +16,7 @@ classifiers = [
 ]
 requires-python = ">=3.8"
 dependencies = [
-    "ewokscore >=5,<6",
+    "ewokscore >=5.1.0rc7,<6",
     "ewoksutils >=2.0.0rc1,<3",
     "tabulate",
     "importlib_resources; python_version < '3.9'",

src/ewoks/__main__.py

@@ -16,12 +16,12 @@
 from typing import Optional
 from typing import Union
 
+from ewokscore.requirements.environment import Environment
 from ewoksutils.cli_utils import cli_cancel_utils
 from ewoksutils.cli_utils import cli_execute_utils
 from ewoksutils.cli_utils import cli_submit_utils
 from ewoksutils.cli_utils.cli_argparse import add_to_parser
 
-from ._requirements.utils.environment import Environment
 from .bindings import _load_graph
 from .bindings import convert_graph
 from .bindings import execute_graph

src/ewoks/_bootstrap/ewoks-install.ps1

@@ -12,8 +12,8 @@
 #
 #   1. Split the arguments in options of this script and arguments of `ewoks install`.
 #   2. Select the package manager.
-#   3. Read the ewoks, engine and python versions from the workflow requirements.
-#   4. Create a bootstrap environment with that ewoks and engine, unless it exists.
+#   3. Read the bootstrap distributions and python version from the workflow requirements.
+#   4. Create a bootstrap environment with those distributions, unless it exists.
 #   5. Run `ewoks install` in the bootstrap environment.
 #
 # The bootstrap environments are cached in ~/.ewoks/bootstrap/<manager>/<name>
@@ -29,9 +29,9 @@ $usage = @'
 Usage: ewoks-install.ps1 [OPTIONS] WORKFLOW [ARGS ...]
 
 Install the python environment of an ewoks workflow with nothing but a package
-manager. The ewoks version that generated the workflow requirements, and the
-engine that saved the workflow, are installed in a bootstrap environment, which
-then runs
+manager. The bootstrap distributions of the workflow requirements, ewoks and
+the engine that saved the workflow, are installed in a bootstrap environment,
+which then runs
 
     ewoks install WORKFLOW ARGS
 
@@ -45,7 +45,7 @@ directory with '--install-package-manager'.
 Options:
   --bootstrap-requirement REQ
                            Install REQ in the bootstrap environment instead of
-                           the ewoks version and engine of the workflow, for
+                           the bootstrap distributions of the workflow, for
                            example 'ewoks>=7' or a wheel file. Repeat it to
                            install an engine as well, for example
                            '--bootstrap-requirement ewoks
@@ -195,52 +195,53 @@ function Get-DistributionVersion([string]$python, [string]$name) {
 # Workflow requirements
 # =============================================================================
 
-# Value in the requirements of a workflow file: ewoks.version, ewoks.engine.name,
-# ewoks.engine.version or python.version. The requirements are JSON, which can be
+# Items of the object at a key path of the requirements of a workflow file: python
+# or bootstrap.distributions, sorted by key. The requirements are JSON, which can be
 # embedded in another format like the XML of .ows files, or YAML.
-function Get-RequirementValue([string]$content, [string]$key) {
-    $value = Get-JsonRequirementValue $content $key
-    if (-not $value) {
-        $value = Get-YamlRequirementValue $content $key
+function Get-RequirementItems([string]$content, [string]$path) {
+    $items = Get-JsonRequirementItems $content $path
+    if ($items.Count -eq 0) {
+        $items = Get-YamlRequirementItems $content $path
     }
-    return $value
+    return ,$items
 }
 
-# Value in JSON without a JSON parser. For example `ewoks.version` in
+function New-Items {
+    return ,([System.Collections.Generic.SortedDictionary[string, string]]::new([StringComparer]::Ordinal))
+}
+
+# Items in JSON without a JSON parser. For example `bootstrap.distributions` in
 #
-#   "ewoks": {"engine": {"name": "ewokscore", "version": "5.1.0"}, "version": "7.0.0"}
-function Get-JsonRequirementValue([string]$content, [string]$key) {
+#   "bootstrap": {"distributions": {"ewoks": "7.0.0", "ewokscore": "5.1.0"}}
+function Get-JsonRequirementItems([string]$content, [string]$path) {
     $q = "[""']"
-    $value = "$q\s*:\s*$q([^""']*)$q"
-    # The members of an object, which can contain objects themselves
-    $members = "[^{}]*(?:\{[^{}]*\}[^{}]*)*"
-    switch ($key) {
-        "ewoks.version" { $pattern = "${q}ewoks$q\s*:\s*\{$members${q}version$value" }
-        "ewoks.engine.name" { $pattern = "${q}ewoks$q\s*:\s*\{[^{}]*${q}engine$q\s*:\s*\{[^{}]*${q}name$value" }
-        "ewoks.engine.version" { $pattern = "${q}ewoks$q\s*:\s*\{[^{}]*${q}engine$q\s*:\s*\{[^{}]*${q}version$value" }
-        "python.version" { $pattern = "${q}python$q\s*:\s*\{[^{}]*${q}version$value" }
-    }
+    $pattern = ($path.Split(".") | ForEach-Object { "$q$_$q\s*:\s*\{" }) -join "[^{}]*"
+    $items = New-Items
     # With the quotes that XML escapes
-    $found = [regex]::Matches($content.Replace("&quot;", '"'), $pattern)
+    $found = [regex]::Matches($content.Replace("&quot;", '"'), "$pattern([^{}]*)\}")
     if ($found.Count -eq 0) {
-        return ""
+        return ,$items
     }
-    return $found[$found.Count - 1].Groups[1].Value
+    $members = $found[$found.Count - 1].Groups[1].Value
+    foreach ($pair in [regex]::Matches($members, "$q([^""']*)$q\s*:\s*$q([^""']*)$q")) {
+        $items[$pair.Groups[1].Value] = $pair.Groups[2].Value
+    }
+    return ,$items
 }
 
-# Value in YAML without a YAML parser: the last `key: value` line whose key path
-# ends with requirements.<key>
-function Get-YamlRequirementValue([string]$content, [string]$key) {
-    $wanted = "requirements.$key"
+# Items in YAML without a YAML parser: the `key: value` lines whose parent key path
+# ends with requirements.<path>
+function Get-YamlRequirementItems([string]$content, [string]$path) {
+    $wanted = "requirements.$path"
     $indents = New-Object System.Collections.Generic.List[int]
     $keys = New-Object System.Collections.Generic.List[string]
-    $found = ""
+    $items = New-Items
     foreach ($line in ($content -split "\r?\n")) {
         if ($line -match '^\s*(#|$)') {
             continue
         }
         # The dash of a list item belongs to the indentation of its first key
-        $match = [regex]::Match($line, '^([ -]*)([A-Za-z0-9_]+):\s*(.*)$')
+        $match = [regex]::Match($line, '^([ -]*)([A-Za-z0-9_.-]+):\s*(.*)$')
         if (-not $match.Success) {
             continue
         }
@@ -252,13 +253,13 @@ function Get-YamlRequirementValue([string]$content, [string]$key) {
         }
         $indents.Add($indent)
         $keys.Add($match.Groups[2].Value)
-        $path = $keys -join "."
+        $parent = $keys.GetRange(0, $keys.Count - 1) -join "."
         $value = $match.Groups[3].Value
-        if ($value -and (($path -eq $wanted) -or $path.EndsWith(".$wanted"))) {
-            $found = $value -replace "^[""']|[""']$", ""
+        if ($value -and (($parent -eq $wanted) -or $parent.EndsWith(".$wanted"))) {
+            $items[$match.Groups[2].Value] = $value -replace "^[""']|[""']$", ""
         }
     }
-    return $found
+    return ,$items
 }
 
 # =============================================================================
@@ -472,19 +473,19 @@ function Select-Manager([hashtable]$state) {
 
 # Set `PythonVersion` (major.minor), `Requirements` of the bootstrap environment,
 # `Tag` (the name of the bootstrap environment without the python version) and, to
-# check whether an existing bootstrap environment is up to date, `Version`, `Engine`
-# and `EngineVersion`
+# check whether an existing bootstrap environment is up to date, `Pinned` (versions
+# by name) and `Latest` (names without a version)
 function Read-WorkflowRequirements([hashtable]$state) {
     $content = [System.IO.File]::ReadAllText((Resolve-Path -LiteralPath $state.Workflow))
     $state.PythonVersion = ""
-    if ((Get-RequirementValue $content "python.version") -match '^(\d+\.\d+)') {
+    $python = Get-RequirementItems $content "python"
+    if ($python.ContainsKey("version") -and ($python["version"] -match '^(\d+\.\d+)')) {
         $state.PythonVersion = $Matches[1]
     }
 
     # The `@` and `=` separators of the tag cannot be part of a distribution name or version
-    $state.Version = ""
-    $state.Engine = ""
-    $state.EngineVersion = ""
+    $state.Pinned = New-Items
+    $state.Latest = @()
     if ($state.CustomRequirements) {
         $state.Requirements = $state.CustomRequirements
         $sha = [System.Security.Cryptography.SHA256]::Create()
@@ -493,38 +494,36 @@ function Read-WorkflowRequirements([hashtable]$state) {
         return
     }
 
-    $version = Get-RequirementValue $content "ewoks.version"
-    $engine = Get-RequirementValue $content "ewoks.engine.name"
-    $engineVersion = Get-RequirementValue $content "ewoks.engine.version"
-    if (-not $engineVersion) {
-        $engine = ""
+    $pinned = Get-RequirementItems $content "bootstrap.distributions"
+    $latest = @()
+    if (-not $pinned.ContainsKey("ewoks")) {
+        Write-Info "the workflow does not provide the ewoks version: using the latest one"
+        $latest += "ewoks"
     }
-    if (-not $engine -and $state.Workflow.EndsWith(".ows")) {
+    if ($state.Workflow.EndsWith(".ows") -and -not $pinned.ContainsKey("ewoksorange")) {
         # Workflows saved before ewoks recorded the engine
         Write-Info "the workflow does not provide its engine: using the latest ewoksorange"
-        $engine = "ewoksorange"
+        $latest += "ewoksorange"
     }
 
-    if ($version) {
-        $requirements = @("ewoks==$version")
-        $tag = "ewoks=$version"
-    } else {
-        Write-Info "the workflow does not provide the ewoks version: using the latest one"
-        $requirements = @("ewoks>=7")
-        $tag = "ewoks=latest"
-    }
-    if ($engineVersion) {
-        $requirements += "$engine==$engineVersion"
-        $tag = "$tag@$engine=$engineVersion"
-    } elseif ($engine) {
-        $requirements += $engine
-        $tag = "$tag@$engine"
-    }
-    $state.Version = $version
-    $state.Engine = $engine
-    $state.EngineVersion = $engineVersion
+    $requirements = @()
+    $tags = @()
+    foreach ($item in $pinned.GetEnumerator()) {
+        $requirements += "$($item.Key)==$($item.Value)"
+        $tags += "$($item.Key)=$($item.Value)"
+    }
+    foreach ($name in $latest) {
+        if ($name -eq "ewoks") {
+            $requirements += "ewoks>=7"
+        } else {
+            $requirements += $name
+        }
+        $tags += "$name=latest"
+    }
+    $state.Pinned = $pinned
+    $state.Latest = $latest
     $state.Requirements = $requirements
-    $state.Tag = $tag
+    $state.Tag = $tags -join "@"
 }
 
 # Install the package manager in the bootstrap directory when it is not installed.
@@ -639,12 +638,15 @@ function New-BootstrapEnvironment([hashtable]$state) {
 # Install the requirements in the bootstrap environment unless it has their pinned versions
 function Install-BootstrapRequirements([hashtable]$state) {
     $python = $state.Python
-    if ($state.Version -and ((Get-DistributionVersion $python "ewoks") -eq $state.Version)) {
-        if (-not $state.Engine) {
-            return
+    if ($state.Pinned.Count -gt 0 -and $state.Latest.Count -eq 0) {
+        $installed = $true
+        foreach ($item in $state.Pinned.GetEnumerator()) {
+            if ((Get-DistributionVersion $python $item.Key) -ne $item.Value) {
+                $installed = $false
+                break
+            }
         }
-        $installed = Get-DistributionVersion $python $state.Engine
-        if ($installed -and (-not $state.EngineVersion -or $installed -eq $state.EngineVersion)) {
+        if ($installed) {
             return
         }
     }