Skip to content

Commit 2dda047

Browse files
mnriemCopilot
andauthored
feat(workflows): select exact step catalog releases (#4840)
* feat(workflows): support exact step catalog releases Preserve current step entries while resolving historical releases from the winning catalog with per-file SHA-256 and manifest identity checks. Document release selection and cover malformed records and install policy. Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * test(workflows): format rebased step release cases Keep the new release tests consistent with upstream command test imports after reconciling the installer refactor. Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * test(workflows): align step command test imports with lint Keep the rebased step test file clean under Ruff while retaining upstream cases. Assisted-by: GitHub Copilot (model: GPT-6 Sol, autonomous) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent 0f0e3d0 commit 2dda047

8 files changed

Lines changed: 834 additions & 16 deletions

File tree

‎docs/reference/workflows.md‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -664,6 +664,54 @@ Catalog installs resolve individual file URLs from the active step catalogs and
664664
then go through the same validation and commit path as `--dev` and `--from`.
665665
Discovery-only catalogs cannot be installed from.
666666

667+
##### Catalog release history
668+
669+
`specify workflow step info <id> --versions` lists the current and historical
670+
releases in the winning catalog. `specify workflow step add <id> --version <v>`
671+
selects the exact catalog release; without `--version`, `add` installs the
672+
advertised current release. The requested version never falls back to a
673+
lower-priority catalog, and discovery-only sources remain non-installable.
674+
`--version` cannot be combined with direct `--dev` or `--from` installs.
675+
676+
Existing single-version catalog entries continue to work. Keep the current
677+
release's `version`, `step_yml_url` (or `url`), `init_url`, and optional files
678+
at the top level. An optional `releases` mapping adds historical versions:
679+
680+
```json
681+
{
682+
"steps": {
683+
"my-step": {
684+
"version": "2.0",
685+
"step_yml_url": "https://example.com/my-step/2.0/step.yml",
686+
"releases": {
687+
"1.0": {
688+
"step_yml_url": "https://example.com/my-step/1.0/step.yml",
689+
"init_url": "https://example.com/my-step/1.0/__init__.py",
690+
"extra_files": {"helper.py": "https://example.com/my-step/1.0/helper.py"},
691+
"sha256": {
692+
"step.yml": "<64 hex digits>",
693+
"__init__.py": "<64 hex digits>",
694+
"helper.py": "<64 hex digits>"
695+
}
696+
}
697+
}
698+
}
699+
}
700+
}
701+
```
702+
703+
Each historical release must supply its own step URL and SHA-256 digest for
704+
every downloaded file. `init_url` can be omitted when derived from a
705+
`step_yml_url` ending in `step.yml`. To select the current version explicitly,
706+
its top-level entry must likewise supply the per-file `sha256` mapping.
707+
Release-specific files and requirements are not inherited from the current
708+
release. Repeated or malformed versions are rejected; equivalent PEP 440
709+
spellings such as `v1.0` and `1.0` select the same advertised release. The
710+
downloaded `step.yml` must declare the selected step ID and version. An
711+
unqualified legacy catalog install does not require digests or a step version.
712+
Only one version of each step ID can be installed at a time; use `--force` to
713+
replace a previous installation after reviewing the selected package.
714+
667715
#### Replacement and force
668716

669717
```bash

‎src/specify_cli/workflows/step/catalog/_domain.py‎

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,19 @@ class StepValidationError(StepCatalogError):
4040
"""Validation error for step catalog config or step data."""
4141

4242

43+
class _DuplicateCatalogField(StepCatalogError):
44+
"""An ambiguous JSON object in a step catalog."""
45+
46+
47+
def _unique_json_fields(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
48+
result: dict[str, Any] = {}
49+
for key, value in pairs:
50+
if key in result:
51+
raise _DuplicateCatalogField(f"Duplicate field '{key}' in step catalog.")
52+
result[key] = value
53+
return result
54+
55+
4356
# ---------------------------------------------------------------------------
4457
# StepCatalogEntry
4558
# ---------------------------------------------------------------------------
@@ -470,9 +483,11 @@ def _fetch_single_catalog(
470483
if cache_safe and not force_refresh and self._is_url_cache_valid(entry.url):
471484
try:
472485
with open(cache_file, encoding="utf-8") as f:
473-
cached = json.load(f)
486+
cached = json.load(f, object_pairs_hook=_unique_json_fields)
474487
if isinstance(cached, dict):
475488
return cached
489+
except _DuplicateCatalogField:
490+
raise
476491
except (UnicodeDecodeError, json.JSONDecodeError, OSError):
477492
# Ignore invalid/unreadable cache and fall back to fetching from source.
478493
pass
@@ -530,15 +545,20 @@ def _validate_redirect(_old_url: str, new_url: str) -> None:
530545
max_bytes=_max_json_catalog_bytes(),
531546
error_type=StepCatalogError,
532547
label="step catalog",
533-
).decode("utf-8")
548+
).decode("utf-8"),
549+
object_pairs_hook=_unique_json_fields,
534550
)
551+
except _DuplicateCatalogField:
552+
raise
535553
except Exception as exc:
536554
if cache_safe and cache_file.exists():
537555
try:
538556
with open(cache_file, encoding="utf-8") as f:
539-
cached = json.load(f)
557+
cached = json.load(f, object_pairs_hook=_unique_json_fields)
540558
if isinstance(cached, dict):
541559
return cached
560+
except _DuplicateCatalogField:
561+
raise
542562
except (json.JSONDecodeError, ValueError, OSError):
543563
# Stale-cache read failed; let the original fetch error propagate.
544564
pass
@@ -574,6 +594,8 @@ def _get_merged_steps(
574594
for entry in reversed(catalogs):
575595
try:
576596
data = self._fetch_single_catalog(entry, force_refresh)
597+
except _DuplicateCatalogField:
598+
raise
577599
except StepCatalogError:
578600
fetch_errors += 1
579601
continue
@@ -586,6 +608,7 @@ def _get_merged_steps(
586608
step_data["_install_allowed"] = entry.install_allowed
587609
merged[step_id] = step_data
588610
elif isinstance(steps, list):
611+
seen_in_source: set[str] = set()
589612
for step_data in steps:
590613
if not isinstance(step_data, dict):
591614
continue
@@ -594,6 +617,11 @@ def _get_merged_steps(
594617
continue
595618
step_id = str(raw_step_id).strip()
596619
if step_id:
620+
if step_id in seen_in_source:
621+
raise StepCatalogError(
622+
f"Duplicate step ID '{step_id}' in catalog '{entry.name}'."
623+
)
624+
seen_in_source.add(step_id)
597625
step_data["id"] = step_id
598626
step_data["_catalog_name"] = entry.name
599627
step_data["_install_allowed"] = entry.install_allowed
@@ -609,10 +637,13 @@ def search(
609637
query: str | None = None,
610638
) -> list[dict[str, Any]]:
611639
"""Search step types across all configured catalogs."""
640+
from ._versions import available_versions
641+
612642
merged = self._get_merged_steps()
613643
results: list[dict[str, Any]] = []
614644

615645
for step_id, step_data in merged.items():
646+
available_versions(step_data, step_id)
616647
step_data.setdefault("id", step_id)
617648
if query:
618649
q = query.lower()
@@ -628,13 +659,18 @@ def search(
628659
results.append(step_data)
629660
return results
630661

631-
def get_step_info(self, step_id: str) -> dict[str, Any] | None:
632-
"""Get details for a specific step from the catalog."""
662+
def get_step_info(
663+
self, step_id: str, version: str | None = None
664+
) -> dict[str, Any] | None:
665+
"""Get the current or an exact release from the winning catalog."""
666+
from ._versions import select_release
667+
633668
merged = self._get_merged_steps()
634669
step = merged.get(step_id)
635670
if step:
636671
step.setdefault("id", step_id)
637-
return step
672+
return select_release(step, step_id, version)
673+
return None
638674

639675
def get_catalog_configs(self) -> list[dict[str, Any]]:
640676
"""Return current catalog configuration as a list of dicts."""
Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
"""Release selection for step catalog entries."""
2+
3+
from __future__ import annotations
4+
5+
import re
6+
from pathlib import PurePosixPath
7+
from typing import Any
8+
9+
from packaging.version import InvalidVersion, Version
10+
11+
from ._domain import StepCatalogError
12+
13+
_DIGEST = re.compile(r"[0-9a-fA-F]{64}\Z")
14+
_CURRENT_ONLY = frozenset(
15+
{
16+
"version",
17+
"url",
18+
"step_yml_url",
19+
"init_url",
20+
"extra_files",
21+
"sha256",
22+
"requires",
23+
"provides",
24+
"releases",
25+
}
26+
)
27+
28+
29+
def validate_checksums(entry: dict[str, Any], step_id: str, *, required: bool) -> None:
30+
"""Check that a release pins exactly the files its installer downloads."""
31+
hashes = entry.get("sha256")
32+
if hashes is None and not required:
33+
return
34+
files = entry.get("extra_files", {})
35+
if not isinstance(files, dict) or any(
36+
not isinstance(path, str)
37+
or not path.strip()
38+
or "\\" in path
39+
or any(part in ("", ".", "..") for part in path.split("/"))
40+
or PurePosixPath(path).is_absolute()
41+
or path.casefold() in ("step.yml", "__init__.py")
42+
or not isinstance(url, str)
43+
or not url.strip()
44+
for path, url in files.items()
45+
):
46+
raise StepCatalogError(f"Step '{step_id}' has invalid extra_files.")
47+
expected = {"step.yml", "__init__.py", *files}
48+
if (
49+
not isinstance(hashes, dict)
50+
or hashes.keys() != expected
51+
or any(
52+
not isinstance(value, str) or not _DIGEST.fullmatch(value)
53+
for value in hashes.values()
54+
)
55+
):
56+
raise StepCatalogError(
57+
f"Step '{step_id}' needs SHA-256 digests for exactly {sorted(expected)}."
58+
)
59+
60+
61+
def _validated_releases(
62+
entry: dict[str, Any], step_id: str
63+
) -> dict[str, dict[str, Any]]:
64+
if "releases" not in entry:
65+
return {}
66+
releases = entry["releases"]
67+
if not isinstance(releases, dict):
68+
raise StepCatalogError(f"Step '{step_id}' has an invalid releases mapping.")
69+
if entry.get("id", step_id) != step_id:
70+
raise StepCatalogError(f"Step '{step_id}' has a mismatched catalog ID.")
71+
current = entry.get("version")
72+
if not isinstance(current, str) or not current.strip():
73+
raise StepCatalogError(f"Step '{step_id}' has releases but no current version.")
74+
try:
75+
seen = {Version(current)}
76+
except InvalidVersion:
77+
raise StepCatalogError(
78+
f"Step '{step_id}' has invalid current version '{current}'."
79+
) from None
80+
for release_version, record in releases.items():
81+
if not isinstance(release_version, str) or not release_version.strip():
82+
raise StepCatalogError(
83+
f"Step '{step_id}' has an invalid release version key."
84+
)
85+
try:
86+
normalized = Version(release_version)
87+
except InvalidVersion:
88+
raise StepCatalogError(
89+
f"Step '{step_id}' has invalid release version '{release_version}'."
90+
) from None
91+
if normalized in seen:
92+
raise StepCatalogError(
93+
f"Step '{step_id}' repeats release version '{release_version}'."
94+
)
95+
seen.add(normalized)
96+
if not isinstance(record, dict):
97+
raise StepCatalogError(
98+
f"Step '{step_id}' release '{release_version}' must be an object."
99+
)
100+
if any(
101+
key in record
102+
for key in (
103+
"id",
104+
"version",
105+
"releases",
106+
"_catalog_name",
107+
"_install_allowed",
108+
)
109+
):
110+
raise StepCatalogError(
111+
f"Step '{step_id}' release '{release_version}' "
112+
"contains reserved fields."
113+
)
114+
url = record.get("step_yml_url", record.get("url"))
115+
if (
116+
not isinstance(url, str)
117+
or not url.strip()
118+
or (
119+
"url" in record
120+
and "step_yml_url" in record
121+
and record["url"] != record["step_yml_url"]
122+
)
123+
):
124+
raise StepCatalogError(
125+
f"Step '{step_id}' release '{release_version}' needs a step.yml URL."
126+
)
127+
init_url = record.get("init_url")
128+
if (
129+
init_url is not None
130+
and (not isinstance(init_url, str) or not init_url.strip())
131+
) or (init_url is None and not url.endswith("step.yml")):
132+
raise StepCatalogError(
133+
f"Step '{step_id}' release '{release_version}' "
134+
"needs an __init__.py URL."
135+
)
136+
for field in ("requires", "provides"):
137+
if field in record and not isinstance(record[field], dict):
138+
raise StepCatalogError(
139+
f"Step '{step_id}' release '{release_version}' has invalid {field}."
140+
)
141+
validate_checksums(record, step_id, required=True)
142+
return releases
143+
144+
145+
def select_release(
146+
entry: dict[str, Any], step_id: str, version: str | None
147+
) -> dict[str, Any] | None:
148+
"""Select a release from the winning entry; never consult another source."""
149+
releases = _validated_releases(entry, step_id)
150+
current = entry.get("version")
151+
if version is None or version == current:
152+
return entry
153+
try:
154+
requested = Version(version)
155+
except InvalidVersion:
156+
return None
157+
if isinstance(current, str):
158+
try:
159+
if requested == Version(current):
160+
return entry
161+
except InvalidVersion:
162+
pass # Legacy versions still support exact spelling above.
163+
for advertised, record in releases.items():
164+
if requested == Version(advertised):
165+
common = {
166+
key: value for key, value in entry.items() if key not in _CURRENT_ONLY
167+
}
168+
return {**common, **record, "version": advertised}
169+
return None
170+
171+
172+
def available_versions(entry: dict[str, Any], step_id: str) -> list[str]:
173+
"""List the advertised current version before historical versions."""
174+
releases = _validated_releases(entry, step_id)
175+
current = entry.get("version")
176+
if not isinstance(current, str) or not current:
177+
return []
178+
return [current, *sorted(releases, key=Version, reverse=True)]

0 commit comments

Comments
 (0)