From 9780e0e0744048dbcc838b3b147ec810009d22d5 Mon Sep 17 00:00:00 2001 From: ArturSepp Date: Tue, 8 Sep 2026 07:14:23 +0200 Subject: [PATCH 1/2] Align public documentation, dependency checks and optional-release publishing --- .gitattributes | 23 +++ .github/scripts/build_docs_redirects.py | 158 +++++++++++++++ .github/scripts/check_stack_imports.py | 252 ++++++++++++++++++++++++ .github/scripts/publish_tag.py | 46 +++++ .github/scripts/release_guard.py | 251 +++++++++++++++++++++++ .github/scripts/test_docs_redirects.py | 134 +++++++++++++ .github/stack-policy.json | 30 +++ .github/workflows/ci.yml | 44 +++++ .github/workflows/docs.yml | 19 +- .github/workflows/release.yml | 134 +++++++++++++ .gitignore | 4 + AGENTS.md | 53 ++--- CHANGELOG.md | 21 ++ CITATION.cff | 4 +- README.md | 40 ++-- RELEASING.md | 23 ++- SECURITY.md | 6 +- docs/conf.py | 16 +- docs/index.md | 12 +- docs/robots.txt | 4 - docs/sitemap.xml | 10 - pyproject.toml | 4 +- tests/test_project_metadata.py | 49 +++-- uv.lock | 2 +- 24 files changed, 1243 insertions(+), 96 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/scripts/build_docs_redirects.py create mode 100644 .github/scripts/check_stack_imports.py create mode 100644 .github/scripts/publish_tag.py create mode 100644 .github/scripts/release_guard.py create mode 100644 .github/scripts/test_docs_redirects.py create mode 100644 .github/stack-policy.json create mode 100644 .github/workflows/release.yml delete mode 100644 docs/robots.txt delete mode 100644 docs/sitemap.xml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d7c79ff --- /dev/null +++ b/.gitattributes @@ -0,0 +1,23 @@ +# Keep repository text portable; never normalise binary assets. +* text=auto eol=lf +*.py text eol=lf +*.md text eol=lf +*.rst text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.toml text eol=lf +*.json text eol=lf +*.png binary +*.PNG binary +*.jpg binary +*.jpeg binary +*.gif binary +*.pdf binary +*.npz binary +*.npy binary +*.xlsx binary +*.xls binary +*.docx binary +*.pkl binary +*.pickle binary +*.parquet binary diff --git a/.github/scripts/build_docs_redirects.py b/.github/scripts/build_docs_redirects.py new file mode 100644 index 0000000..97c4626 --- /dev/null +++ b/.github/scripts/build_docs_redirects.py @@ -0,0 +1,158 @@ +"""Build a small GitHub Pages redirect site from the verified Sphinx HTML paths. + +SHARED CI CORE v1.2. Keep this file identical in the four legacy Pages repositories. +Only HTTPS Read the Docs destinations are accepted. No documentation or assets are copied. +""" + +import argparse +import html +import json +from pathlib import Path, PurePosixPath +import re +from urllib.parse import quote, urlsplit + + +def validate_base_url(value: str) -> str: + """Accept an absolute RTD version root without credentials, query, or fragment.""" + parsed = urlsplit(value) + hostname = parsed.hostname or "" + if ( + parsed.scheme != "https" + or not hostname.endswith(".readthedocs.io") + or parsed.netloc != hostname + or parsed.query + or parsed.fragment + or not re.fullmatch(r"(?:/[A-Za-z0-9_-]+)*/", parsed.path) + ): + raise ValueError("base URL must be an HTTPS readthedocs.io root ending in /") + return value + + +def validate_project_prefix(value: str) -> str: + """Accept one GitHub project path segment, with leading and trailing slashes.""" + if not re.fullmatch(r"/[A-Za-z0-9_-][A-Za-z0-9_.-]*/", value): + raise ValueError("project prefix must be one repository name between slashes") + return value + + +def destination(base_url: str, relative: str) -> str: + """Map an HTML filename to its fixed RTD destination, encoding path characters.""" + validate_base_url(base_url) + path = PurePosixPath(relative) + if path.is_absolute() or ".." in path.parts or "\\" in relative: + raise ValueError("HTML paths must stay beneath the source directory") + if relative == "index.html": + return base_url + return base_url + "/".join(quote(part, safe="-._~") for part in path.parts) + + +def script_json(value: str) -> str: + """Quote strings for an inline script without permitting an HTML closing tag.""" + return ( + json.dumps(value, ensure_ascii=True) + .replace("<", "\\u003c") + .replace(">", "\\u003e") + .replace("&", "\\u0026") + ) + + +def redirect_script(target: str, project_prefix: str, fallback: bool = False) -> str: + """Preserve the browser query and fragment; keep fallback paths on the RTD host.""" + prefix = script_json(validate_project_prefix(project_prefix)) + if not fallback: + return ( + f"window.location.replace({script_json(target)}" + " + window.location.search + window.location.hash);" + ) + return f"""(() => {{ + const base = {script_json(validate_base_url(target))}; + const prefix = {prefix}; + const pathname = window.location.pathname; + let relative = pathname.startsWith(prefix) ? pathname.slice(prefix.length) : ""; + let suffix = ""; + try {{ + const parts = relative.split("/").filter(Boolean).map(decodeURIComponent); + const unsafe = parts.some(part => + part === "." || part === ".." || /[\\\\/\\x00-\\x1f]/.test(part)); + if (!unsafe) {{ + suffix = parts.map(encodeURIComponent).join("/"); + if (relative.endsWith("/") && suffix) suffix += "/"; + }} + }} catch (_) {{ + suffix = ""; + }} + if (suffix === "index.html") suffix = ""; + window.location.replace(base + suffix + window.location.search + window.location.hash); +}})();""" + + +def render_redirect(target: str, project_prefix: str, fallback: bool = False) -> str: + """Render an accessible redirect with canonical, no-JavaScript, and link fallbacks.""" + escaped = html.escape(target, quote=True) + script = redirect_script(target, project_prefix, fallback=fallback) + return f""" + + + + + + Documentation moved + + + + + +

Documentation moved

+

Continue to the documentation on Read the Docs.

+ + +""" + + +def build_redirects(source: Path, output: Path, base_url: str, project_prefix: str) -> int: + """Write redirects to an empty output directory without modifying source content.""" + validate_base_url(base_url) + validate_project_prefix(project_prefix) + source, output = source.resolve(), output.resolve() + if source == output or source in output.parents or output in source.parents: + raise ValueError("source and output directories must not overlap") + if not (source / "index.html").is_file(): + raise ValueError("source must contain a rendered Sphinx index.html") + if output.exists() and any(output.iterdir()): + raise ValueError("output must be empty; use a dedicated build directory") + pages = sorted(source.rglob("*.html")) + for page in pages: + if page.is_symlink() or source not in page.resolve().parents: + raise ValueError("source HTML must not link outside the rendered documentation") + output.mkdir(parents=True, exist_ok=True) + count = 0 + for page in pages: + relative = page.relative_to(source) + if relative.as_posix() == "404.html": + continue + target = destination(base_url, relative.as_posix()) + path = output / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(render_redirect(target, project_prefix), encoding="utf-8") + count += 1 + (output / "404.html").write_text( + render_redirect(base_url, project_prefix, fallback=True), encoding="utf-8" + ) + (output / ".nojekyll").touch() + return count + + +def main() -> None: + """Build the redirect payload using explicit repository and destination arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--base-url", required=True) + parser.add_argument("--project-prefix", required=True) + args = parser.parse_args() + count = build_redirects(args.source, args.output, args.base_url, args.project_prefix) + print(f"Built {count} page redirects and a 404 fallback to {args.base_url}") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/check_stack_imports.py b/.github/scripts/check_stack_imports.py new file mode 100644 index 0000000..44a3113 --- /dev/null +++ b/.github/scripts/check_stack_imports.py @@ -0,0 +1,252 @@ +"""Check stack dependencies and optional import boundaries without importing source.""" + +from __future__ import annotations + +import argparse +import ast +import fnmatch +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +def module_imports(tree): + """Yield imports executed by a module body, including class/try/if bodies.""" + + def walk(node): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + return + if isinstance(node, ast.If) and ( + isinstance(node.test, ast.Name) + and node.test.id == "TYPE_CHECKING" + or isinstance(node.test, ast.Attribute) + and node.test.attr == "TYPE_CHECKING" + ): + for child in node.orelse: + yield from walk(child) + return + if isinstance(node, (ast.Import, ast.ImportFrom)): + yield node + for child in ast.iter_child_nodes(node): + yield from walk(child) + + yield from walk(tree) + + +def names(node): + """Return absolute import names; relative imports are internal graph edges.""" + if isinstance(node, ast.Import): + return [alias.name for alias in node.names] + return [node.module or ""] if not node.level else [] + + +def permitted(path, module, rules, function=None): + """Exceptions name both a source path and the optional module it may load.""" + return any( + fnmatch.fnmatchcase(path, rule["path"]) + and module in rule["modules"] + and ("function" not in rule or rule["function"] == function) + for rule in rules + ) + + +def audit(root, policy): + """Return concrete violations and the number of parsed package modules.""" + package_root = root / "src" / policy["import"] + modules = {} + for path in package_root.rglob("*.py"): + relative = path.relative_to(root).as_posix() + name = ".".join(path.relative_to(root / "src").with_suffix("").parts) + if name.endswith(".__init__"): + name = name[:-9] + modules[name] = ( + relative, + ast.parse(path.read_text(encoding="utf-8-sig"), filename=relative), + ) + errors, graph, isolated = [], {}, set() + forbidden = set(policy["forbidden"]) + optional = set(policy["optional"]) + for module, (path, tree) in modules.items(): + parents = { + child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent) + } + for node in ast.walk(tree): + if isinstance(node, (ast.Import, ast.ImportFrom)): + parent, function = node, None + while parent in parents: + parent = parents[parent] + if isinstance(parent, (ast.FunctionDef, ast.AsyncFunctionDef)): + function = parent.name + break + for imported in names(node): + top = imported.split(".")[0] + if top in forbidden and not permitted( + path, top, policy["development_imports"], function + ): + errors.append(f"{path}:{node.lineno}: forbidden stack import {top}") + top_level = list(module_imports(tree)) + edges = set() + for node in top_level: + for imported in names(node): + top = imported.split(".")[0] + if top in optional: + if permitted(path, top, policy["optional_adapters"]): + isolated.add(module) + else: + errors.append( + f"{path}:{node.lineno}: optional stack import {top} at module level" + ) + targets = [] + if isinstance(node, ast.Import): + targets = [alias.name for alias in node.names] + elif not node.level: + targets = [node.module or ""] + [ + f"{node.module}.{alias.name}" for alias in node.names + ] + else: + package = module if path.endswith("/__init__.py") else module.rpartition(".")[0] + parts = package.split(".") + base = ".".join(parts[: len(parts) - node.level + 1]) + target = ".".join(part for part in (base, node.module) if part) + targets = [target] + [f"{target}.{alias.name}" for alias in node.names] + for target in targets: + pieces = target.split(".") + edges.update( + ".".join(pieces[:i]) + for i in range(1, len(pieces) + 1) + if ".".join(pieces[:i]) in modules + ) + graph[module] = edges + pending, visited = [policy["import"]], set() + while pending: + module = pending.pop() + if module in visited: + continue + visited.add(module) + pending.extend(graph.get(module, set()) - visited) + for module in sorted(isolated & visited): + errors.append( + f"{modules[module][0]}: optional adapter is reachable during package-root import" + ) + return sorted(set(errors)), len(modules) + + +def check_root(root, policy): + """Test root import in a fresh process while recording blocked optional imports.""" + if not policy["optional"]: + return + probe = """import importlib.abc, json, sys +optional, source, package = json.loads(sys.argv[1]) +attempts = [] +class BlockOptional(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] in optional: + attempts.append(fullname) + raise ModuleNotFoundError('optional stack import blocked: ' + fullname) +sys.meta_path.insert(0, BlockOptional()) +sys.path.insert(0, source) +__import__(package) +if attempts: + raise SystemExit('package-root import attempted optional modules: ' + ', '.join(attempts)) +""" + subprocess.run( + [ + sys.executable, + "-B", + "-c", + probe, + json.dumps([policy["optional"], str(root / "src"), policy["import"]]), + ], + cwd=root, + check=True, + ) + + +class BoundaryTests(unittest.TestCase): + """Known-bad imports, allowed local loads, and adapter leakage regressions.""" + + def check_source(self, source, adapter=None, exception=False): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + package = root / "src/pkg" + package.mkdir(parents=True) + (package / "__init__.py").write_text(source, encoding="utf-8") + if adapter is not None: + (package / "adapter.py").write_text(adapter, encoding="utf-8") + policy = { + "import": "pkg", + "optional": ["optional_stack"], + "forbidden": ["forbidden_stack"], + "development_imports": [], + "optional_adapters": [{"path": "src/pkg/adapter.py", "modules": ["optional_stack"]}] + if exception + else [], + } + return audit(root, policy)[0] + + def test_module_scope_is_rejected(self): + self.assertTrue(self.check_source("import optional_stack\n")) + + def test_function_local_import_is_allowed(self): + self.assertFalse(self.check_source("def load():\n import optional_stack\n")) + + def test_guarded_module_import_is_still_rejected(self): + self.assertTrue( + self.check_source("try:\n import optional_stack\nexcept ImportError:\n pass\n") + ) + + def test_type_checking_only_import_is_allowed(self): + self.assertFalse( + self.check_source( + "from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n import optional_stack\n" + ) + ) + + def test_isolated_named_adapter_is_allowed(self): + self.assertFalse(self.check_source("", "import optional_stack\n", True)) + + def test_root_reachable_adapter_is_rejected(self): + self.assertTrue( + self.check_source("from . import adapter\n", "import optional_stack\n", True) + ) + + def test_forbidden_import_is_not_hidden_by_local_scope(self): + self.assertTrue(self.check_source("def load():\n import forbidden_stack\n")) + + def test_maintainer_exception_requires_named_function_and_module(self): + rules = [ + {"path": "src/pkg/universe.py", "function": "generate_data", "modules": ["bbg_fetch"]} + ] + self.assertTrue(permitted("src/pkg/universe.py", "bbg_fetch", rules, "generate_data")) + self.assertFalse(permitted("src/pkg/universe.py", "bbg_fetch", rules, "other")) + self.assertFalse(permitted("src/pkg/universe.py", "qis", rules, "generate_data")) + + +def main(): + """Run static policy, optional root-import probe, or the self-contained tests.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[2]) + parser.add_argument("--self-test", action="store_true") + parser.add_argument("--check-root", action="store_true") + args = parser.parse_args() + if args.self_test: + result = unittest.TextTestRunner(verbosity=2).run( + unittest.defaultTestLoader.loadTestsFromTestCase(BoundaryTests) + ) + return 0 if result.wasSuccessful() else 1 + policy = json.loads((args.root / ".github/stack-policy.json").read_text(encoding="utf-8")) + errors, count = audit(args.root, policy) + if errors: + print("\n".join(errors)) + return 1 + if args.check_root: + check_root(args.root, policy) + print(f"Stack import policy passed ({count} modules)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/publish_tag.py b/.github/scripts/publish_tag.py new file mode 100644 index 0000000..0bde0dd --- /dev/null +++ b/.github/scripts/publish_tag.py @@ -0,0 +1,46 @@ +"""Validate a prepared release; --push creates/pushes only its named tag. + +Run after the release metadata commit is on origin/main. This does not build locally, +create an environment, upload a package, or create a GitHub Release page. The pushed +tag triggers release.yml. Omitting --push is a read-only dry run. +""" +import argparse +import subprocess +from pathlib import Path + +from release_guard import run, validate_metadata + + +def main() -> None: + """Require a clean main commit and matching remote identity before a named tag push.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("tag") + parser.add_argument("--push", action="store_true") + args = parser.parse_args() + root = Path(run("git", "rev-parse", "--show-toplevel")) + project = validate_metadata(root, args.tag) + if run("git", "status", "--porcelain", cwd=root): + raise SystemExit("Commit the intended changes first; the release checkout must be clean") + if run("git", "branch", "--show-current", cwd=root) != "main": + raise SystemExit("Run this helper from main after merging the release metadata") + sha = run("git", "rev-parse", "HEAD", cwd=root) + print(f"{project['name']} {project['version']}: {args.tag} -> {sha}") + if not args.push: + print("Dry run. --push verifies remote main, creates the named tag if absent, and pushes it.") + return + remote = run("git", "ls-remote", "origin", "refs/heads/main", cwd=root).split() + if not remote or remote[0] != sha: + raise SystemExit("Push the verified main commit before requesting its release tag") + existing = subprocess.run(["git", "rev-parse", "--verify", f"refs/tags/{args.tag}^{{commit}}"], + cwd=root, capture_output=True, text=True) + if existing.returncode == 0: + if existing.stdout.strip() != sha: + raise SystemExit("Existing tag points elsewhere; refusing to move it") + else: + run("git", "tag", "-a", args.tag, "-m", f"Release {project['version']}", cwd=root) + run("git", "push", "origin", f"refs/tags/{args.tag}:refs/tags/{args.tag}", cwd=root) + print("Named tag pushed. Follow the Publish package workflow; GitHub Release page is optional.") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/release_guard.py b/.github/scripts/release_guard.py new file mode 100644 index 0000000..c2b6528 --- /dev/null +++ b/.github/scripts/release_guard.py @@ -0,0 +1,251 @@ +"""Release-v1: validate one tag, inspect built artifacts and plan digest-safe uploads. + +This helper never publishes packages or creates GitHub Release pages. GitHub builds +are isolated from the OIDC publishing job. All subprocess arguments are structured. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import email.parser +import hashlib +import json +import os +import re +import shutil +import subprocess +import tarfile +import tomllib +import urllib.error +import urllib.request +import zipfile +from pathlib import Path, PurePosixPath + +TAG_RE = re.compile(r"v(\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?(?:\.post\d+)?(?:\.dev\d+)?)") + + +def version_from_tag(tag: str) -> str: + """Reject refs, shell fragments, wildcard tags and ambiguous version syntax.""" + match = TAG_RE.fullmatch(tag) + if not match: + raise ValueError("Expected a named tag such as v1.2.3, v1.2.3rc1 or v1.2.3.dev1") + return match[1] + + +def run(*args: str, cwd: Path | None = None) -> str: + """Run an explicit command and return stdout with a bounded runtime.""" + return subprocess.run(list(args), cwd=cwd, check=True, text=True, + encoding="utf-8", capture_output=True, timeout=1800).stdout.strip() + + +def cff_scalar(source: str, key: str) -> str: + """Read a simple scalar and reject duplicate/multiline ambiguous release values.""" + values = re.findall(rf"^{re.escape(key)}:\s*([^\n]+)$", source, flags=re.MULTILINE) + if len(values) != 1: + raise ValueError(f"CITATION.cff requires one {key}") + value = values[0].strip().split(" #", 1)[0].strip() + if value.startswith('"'): + return json.loads(value) + if value.startswith("'") and value.endswith("'"): + return value[1:-1].replace("''", "'") + if not re.fullmatch(r"[0-9A-Za-z.+-]+", value): + raise ValueError(f"Unsupported CFF {key} scalar") + return value + + +def validate_metadata(root: Path, tag: str) -> dict: + """Validate source release identity and the recorded intended release date.""" + version = version_from_tag(tag) + project = tomllib.loads((root / "pyproject.toml").read_text(encoding="utf-8"))["project"] + if project["version"] != version: + raise ValueError(f"Tag {tag} differs from project version {project['version']}") + cff = (root / "CITATION.cff").read_text(encoding="utf-8") + if cff_scalar(cff, "version") != version: + raise ValueError("CITATION.cff version differs from tag") + intended = dt.date.fromisoformat(cff_scalar(cff, "date-released")) + if intended > dt.datetime.now(dt.timezone.utc).date(): + raise ValueError("CITATION.cff intended release date is in the future") + changelog = (root / "CHANGELOG.md").read_text(encoding="utf-8") + pattern = rf"^##\s+(?:\[{re.escape(version)}\]|{re.escape(version)})(?:\s+.*)?$" + if not re.search(pattern, changelog, flags=re.MULTILINE): + raise ValueError("CHANGELOG.md has no heading for the tagged version") + return project + + +def checkout_tag(root: Path, tag: str, expected_sha: str | None = None) -> dict: + """Resolve and check ancestry before checking out exactly one release commit.""" + version_from_tag(tag) + sha = run("git", "rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}", cwd=root) + if expected_sha is not None: + if not re.fullmatch(r"[0-9a-f]{40}", expected_sha): + raise ValueError("Invalid triggering object SHA") + triggered = run("git", "rev-parse", "--verify", f"{expected_sha}^{{commit}}", cwd=root) + if triggered != sha: + raise ValueError("Tag moved after the triggering push; refusing a different commit") + run("git", "merge-base", "--is-ancestor", sha, "refs/remotes/origin/main", cwd=root) + run("git", "checkout", "--detach", sha, cwd=root) + if run("git", "rev-parse", "HEAD", cwd=root) != sha: + raise ValueError("Checkout does not match the resolved tag") + project = validate_metadata(root, tag) + epoch = run("git", "show", "-s", "--format=%ct", sha, cwd=root) + return {"sha": sha, "version": project["version"], "epoch": epoch} + + +def normalized(name: str) -> str: + """Normalize a PyPI distribution name.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def requirement_key(requirement: str) -> str: + """Canonicalize names and specifier order for the stack's core requirements.""" + match = re.fullmatch(r"([A-Za-z0-9_.-]+)(\[[^]]+\])?([^;]*)(?:;(.*))?", requirement.strip()) + if not match: + raise ValueError(f"Unsupported requirement metadata: {requirement}") + specifiers = ",".join(sorted(x.strip() for x in match[3].split(",") if x.strip())) + marker = re.sub(r"\s+", "", match[4] or "").replace("'", '"') + return normalized(match[1]) + (match[2] or "") + specifiers + (";" + marker if marker else "") + + +def inspect_artifacts(dist: Path, project: dict, import_name: str) -> dict[str, str]: + """Check one wheel plus one sdist and compute immutable filename/hash identity.""" + wheels, sdists = list(dist.glob("*.whl")), list(dist.glob("*.tar.gz")) + if len(wheels) != 1 or len(sdists) != 1: + raise ValueError("Expected exactly one wheel and one source distribution") + with zipfile.ZipFile(wheels[0]) as wheel: + metadata_names = [n for n in wheel.namelist() if n.endswith(".dist-info/METADATA")] + if len(metadata_names) != 1 or f"{import_name}/__init__.py" not in wheel.namelist(): + raise ValueError("Wheel lacks the expected package or unique METADATA") + wheel_metadata = wheel.read(metadata_names[0]).decode("utf-8") + wheel_paths = set(wheel.namelist()) + wheel_metadata_root = metadata_names[0].rsplit("/", 1)[0] + with tarfile.open(sdists[0], "r:gz") as archive: + candidates = [m for m in archive.getmembers() if m.name.count("/") == 1 and m.name.endswith("/PKG-INFO")] + if len(candidates) != 1: + raise ValueError("Sdist lacks unique top-level PKG-INFO") + sdist_metadata = archive.extractfile(candidates[0]).read().decode("utf-8") + sdist_paths = set(archive.getnames()) + sdist_root = candidates[0].name.split("/", 1)[0] + source_paths = {name.removeprefix(sdist_root + "/") for name in sdist_paths} + if f"src/{import_name}/__init__.py" not in source_paths: + raise ValueError("Sdist lacks the expected source package") + prohibited = {".idea", ".git", ".venv", "venv", "__pycache__", ".pytest_cache", + ".ruff_cache", ".mypy_cache", "run_local"} + for archive_name, paths in (("Wheel", wheel_paths), ("Sdist", source_paths)): + for name in paths: + parts = PurePosixPath(name).parts + runner = import_name in {"privateassets", "goal_based_allocation"} and import_name in parts and "run" in parts + if prohibited.intersection(parts) or name.endswith((".pyc", ".pyo", ".nbc", ".nbi")) or runner: + raise ValueError(f"{archive_name} contains a development runner, environment or cache: {name}") + for metadata in (wheel_metadata, sdist_metadata): + parsed = email.parser.Parser().parsestr(metadata) + if normalized(parsed["Name"]) != normalized(project["name"]) or parsed["Version"] != project["version"]: + raise ValueError("Built artifact identity differs from source/tag") + if parsed["Summary"] != project["description"]: + raise ValueError("Built artifact summary differs from source") + urls = dict(value.split(", ", 1) for value in parsed.get_all("Project-URL", [])) + if urls != project.get("urls", {}): + raise ValueError("Built artifact project URLs differ from source") + if parsed["Requires-Python"] != project.get("requires-python"): + raise ValueError("Built artifact Requires-Python differs from source") + core = {requirement_key(value) for value in parsed.get_all("Requires-Dist", []) + if not re.search(r"\bextra\s*==", value)} + if core != {requirement_key(value) for value in project.get("dependencies", [])}: + raise ValueError("Built artifact core dependency metadata differs from source") + if set(parsed.get_all("Provides-Extra", [])) != set(project.get("optional-dependencies", {})): + raise ValueError("Built artifact extra names differ from source") + if isinstance(project.get("license"), str): + if parsed["License-Expression"] != project["license"]: + raise ValueError("Built artifact License-Expression differs from source") + licenses = parsed.get_all("License-File", []) + if not licenses: + raise ValueError("Built artifact declares no shipped license file") + for license_file in licenses: + if f"{wheel_metadata_root}/licenses/{license_file}" not in wheel_paths or f"{sdist_root}/{license_file}" not in sdist_paths: + raise ValueError(f"Declared license file is missing from wheel or sdist: {license_file}") + return {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in wheels + sdists} + + +def pypi_files(name: str, version: str) -> list[dict] | None: + """A confirmed 404 means new version; every other API error blocks publication.""" + request = urllib.request.Request(f"https://pypi.org/pypi/{name}/{version}/json", + headers={"User-Agent": "ArturSepp-release-v1"}) + try: + with urllib.request.urlopen(request, timeout=30) as response: + data = json.load(response) + except urllib.error.HTTPError as error: + if error.code == 404: + return None + raise + if not isinstance(data, dict) or not isinstance(data.get("urls"), list) or not data["urls"]: + raise ValueError("PyPI returned an unexpected or empty artifact list") + return data["urls"] + + +def pending_uploads(hashes: dict[str, str], existing: list[dict] | None, + retry_existing: bool = False) -> list[str]: + """Require matching digests; backfill tag pushes never append historical files.""" + if existing is None: + return sorted(hashes) + remote = {f["filename"]: f["digests"]["sha256"] for f in existing} + if len(remote) != len(existing): + raise ValueError("Duplicate PyPI artifact filenames") + if not remote.keys() <= hashes.keys(): + raise ValueError("Existing release has unexpected artifact filenames") + for name, digest in remote.items(): + if hashes[name] != digest: + raise ValueError(f"Immutable PyPI artifact differs: {name}; never skip this mismatch") + pending = sorted(hashes.keys() - remote.keys()) + if pending and not retry_existing: + raise ValueError("Existing version is incomplete: use explicit retry_existing dispatch after digest review") + return pending + + +def output(values: dict) -> None: + """Emit simple validated GitHub outputs and a readable local result.""" + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as stream: + for key, value in values.items(): + if "\n" in str(value): + raise ValueError("Multiline output is not allowed") + stream.write(f"{key}={value}\n") + print(json.dumps(values, indent=2)) + + +def main() -> None: + """Run only the explicit stage selected by the workflow or local maintainer.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["checkout", "check", "artifacts"]) + parser.add_argument("--tag", required=True) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--dist", type=Path) + parser.add_argument("--import-name") + parser.add_argument("--pending", type=Path) + parser.add_argument("--retry-existing", action="store_true") + args = parser.parse_args() + if args.command == "checkout": + if os.environ.get("GITHUB_EVENT_NAME") == "workflow_dispatch" and os.environ.get("GITHUB_REF") != "refs/heads/main": + raise ValueError("Dispatch publishing from the main workflow only") + expected = os.environ.get("GITHUB_SHA") if os.environ.get("GITHUB_EVENT_NAME") == "push" else None + result = checkout_tag(args.root, args.tag, expected) + if os.environ.get("GITHUB_ENV"): + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as stream: + stream.write(f"SOURCE_DATE_EPOCH={result['epoch']}\n") + output(result) + elif args.command == "check": + project = validate_metadata(args.root, args.tag) + output({"name": project["name"], "version": project["version"]}) + else: + if not args.dist or not args.pending or not args.import_name: + parser.error("artifacts requires --dist, --pending, --import-name") + project = validate_metadata(args.root, args.tag) + hashes = inspect_artifacts(args.dist, project, args.import_name) + pending = pending_uploads(hashes, pypi_files(project["name"], project["version"]), args.retry_existing) + args.pending.mkdir(parents=True, exist_ok=False) + for filename in pending: + shutil.copy2(args.dist / filename, args.pending / filename) + (args.dist / "release-manifest.json").write_text(json.dumps({"tag": args.tag, "sha": run("git", "rev-parse", "HEAD", cwd=args.root), "sha256": hashes, "pending": pending}, indent=2) + "\n", encoding="utf-8") + output({"publish": "true" if pending else "false", "pending_count": len(pending)}) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/test_docs_redirects.py b/.github/scripts/test_docs_redirects.py new file mode 100644 index 0000000..fc3e34d --- /dev/null +++ b/.github/scripts/test_docs_redirects.py @@ -0,0 +1,134 @@ +"""Exercise static URL mapping and the actual browser JavaScript without a network.""" + +import json +from pathlib import Path +import subprocess +import tempfile +import unittest + +from build_docs_redirects import ( + build_redirects, + destination, + redirect_script, + render_redirect, + validate_base_url, + validate_project_prefix, +) + + +BASE = "https://example.readthedocs.io/en/latest/" +PREFIX = "/Example/" + + +def run_browser_script(script: str, path: str, query: str = "", fragment: str = "") -> str: + """Evaluate the emitted JavaScript with a minimal window.location object in Node.""" + harness = """ +const fs = require('node:fs'); +const vm = require('node:vm'); +const input = JSON.parse(fs.readFileSync(0, 'utf8')); +let result; +const location = {...input.location, replace: value => { result = value; }}; +vm.runInNewContext(input.script, {window: {location}}, {timeout: 1000}); +process.stdout.write(JSON.stringify(result)); +""" + result = subprocess.run( + ["node", "-e", harness], + input=json.dumps( + { + "script": script, + "location": { + "pathname": path, + "search": query, + "hash": fragment, + }, + } + ), + capture_output=True, + text=True, + check=True, + timeout=10, + ) + return json.loads(result.stdout) + + +class RedirectTests(unittest.TestCase): + """Catch broken deep links, lost fragments, unsafe paths, and source-copy regressions.""" + + def test_homepage_and_encoded_deep_page(self): + self.assertEqual(destination(BASE, "index.html"), BASE) + self.assertEqual(destination(BASE, "guide/schema.html"), BASE + "guide/schema.html") + self.assertEqual(destination(BASE, "a #b.html"), BASE + "a%20%23b.html") + for value in ["../outside.html", "/outside.html", "..\\outside.html"]: + with self.subTest(value=value), self.assertRaises(ValueError): + destination(BASE, value) + + def test_only_explicit_rtd_destinations_and_project_prefixes(self): + for value in [ + "http://example.readthedocs.io/en/latest/", + "https://evil.test/", + "https://example.readthedocs.io.evil.test/", + "https://user@example.readthedocs.io/", + BASE + "?next=evil", + BASE + "#fragment", + BASE + "../other/", + ]: + with self.subTest(value=value), self.assertRaises(ValueError): + validate_base_url(value) + for value in ["//evil.test/", "/../", "Example", "/a/b/"]: + with self.subTest(value=value), self.assertRaises(ValueError): + validate_project_prefix(value) + + def test_browser_preserves_query_and_fragment(self): + target = destination(BASE, "schema.html") + actual = run_browser_script( + redirect_script(target, PREFIX), "/Example/schema.html", "?q=expiry", "#strike-grid" + ) + self.assertEqual(actual, BASE + "schema.html?q=expiry#strike-grid") + + def test_404_strips_only_the_exact_project_prefix(self): + script = redirect_script(BASE, PREFIX, fallback=True) + self.assertEqual( + run_browser_script(script, "/Example/old/page.html", "", "#rules"), + BASE + "old/page.html#rules", + ) + self.assertEqual(run_browser_script(script, "/Example/index.html"), BASE) + self.assertEqual(run_browser_script(script, "/Example/guide/"), BASE + "guide/") + self.assertEqual(run_browser_script(script, "/ExampleOther/page.html"), BASE) + self.assertEqual(run_browser_script(script, "/Example/%2e%2e/elsewhere"), BASE) + self.assertEqual(run_browser_script(script, "/Example/%2f%2fevil.test"), BASE) + self.assertEqual(run_browser_script(script, "/Example/%5cevil.test"), BASE) + self.assertEqual(run_browser_script(script, "/Example/%broken"), BASE) + + def test_html_and_script_escape_untrusted_characters(self): + target = BASE + 'x?test="&' + rendered = render_redirect(target, PREFIX) + self.assertNotIn("