diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e2c890d1..03e83e55 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -74,6 +74,17 @@ repos: pass_filenames: false always_run: true stages: [pre-commit, pre-push] + # 锁集交叉约束(上界 / ==X.* 型冲突,check_pin_floors 的下界检查抓不到): + # 只在依赖清单变更时跑 —— 它会按需查 PyPI requires_dist,不做无谓联网。 + # 判据见 issue #97:#87 曾把 mpmath/antlr4/tokenizers 顶到违反 sympy/hydra/ + # transformers 自身约束的位置,便携包解析当场不可行。 + - id: check-pin-crossconflicts + name: Pin Cross-Constraint Consistency Check + entry: python scripts/check_pin_crossconflicts.py + language: system + pass_filenames: false + files: ^(requirements\.txt|requirements-lock\.txt|launcher/requirements-small\.txt|pyproject\.toml)$ + stages: [pre-commit, pre-push] # P1 安全修复:禁止提交包含 PRIVATE KEY 标记的 .env 或其他文件 - id: forbid-private-key-in-env name: Forbid PRIVATE KEY in .env files diff --git a/scripts/check_pin_crossconflicts.py b/scripts/check_pin_crossconflicts.py new file mode 100644 index 00000000..cc8024bb --- /dev/null +++ b/scripts/check_pin_crossconflicts.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""校验锁文件内部是否自相矛盾:按 PyPI 声明的交叉约束逐条比对。 + +与 `check_pin_floors.py` 的分工:那个只看「钉版 < 本项目声明的下界」(下界方向), +本脚本看「锁里 A==x 与 B==y 的互相约束是否成立」(上界 / == / ~= 方向)。 +2026-09-20 的 issue #97 就是这类:`transformers==4.52.1` 要 `tokenizers<0.22` +而锁里是 0.23.2、`sympy` 要 `mpmath<1.4` 而锁里是 1.4.1、`hydra-core`/`omegaconf` +要 `antlr4-python3-runtime==4.9.*` 而锁里是 4.13.2 —— 下界检查器一条都抓不到。 + +为什么要跑这个而不是「CI 装一次看看」:便携包锁集的真实解析要 WinPython + 数 GB 轮子, +本地/CI 都不便每次改动都装;而 PyPI 的 `requires_dist` 元数据足以证伪。 + +网络失败时**不静默跳过**:受影响依赖计入 unchecked 并以退出码 2 报出。 + +用法: + python scripts/check_pin_crossconflicts.py # 全部锁文件 + python scripts/check_pin_crossconflicts.py --quiet # 只报冲突与 unchecked 数 +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import urllib.error +import urllib.request +from pathlib import Path + +_ROOT = Path(__file__).resolve().parent.parent +LOCK_FILES = ("requirements-lock.txt", "launcher/requirements-small.txt") +_CACHE: dict[str, list[str]] = {} + + +def _ver(s: str) -> tuple: + """版本 → 定长 4 元组;剥掉本地版本段与预发布尾标(与 check_pin_floors 同一口径)。""" + core = re.split(r"[+]", s, maxsplit=1)[0] + core = re.split(r"(?<=[\d.])(?:a|b|rc|dev|pre|post)\d*$", core)[0] + nums = [int(n) for n in re.findall(r"\d+", core)[:4]] + return tuple(nums + [0] * (4 - len(nums))) + + +def _norm(name: str) -> str: + return name.lower().replace("_", "-") + + +def read_pins(path: Path) -> dict[str, tuple[str, str]]: + """{包名: (版本, 出现它的锁文件列表)};只收 `==` 钉版。""" + pins: dict[str, tuple[str, str]] = {} + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + line = line.split("#", 1)[0].strip() + m = re.match(r"^([A-Za-z0-9._-]+)==([0-9][0-9a-zA-Z.+-]*)$", line) + if m: + key = _norm(m.group(1)) + if key in pins and pins[key][0] != m.group(2): + pins[key] = (pins[key][0], f"{pins[key][1]}、{m.group(2)}@{path.name}") + else: + pins.setdefault(key, (m.group(2), path.name)) + return pins + + +def requires_dist(name: str, ver: str) -> list[str] | None: + """取该版本的 requires_dist;网络/不存在返回 None(区别于「确实无依赖」的 [])。""" + key = f"{name}=={ver}" + if key in _CACHE: + return _CACHE[key] + url = f"https://pypi.org/pypi/{name}/{ver}/json" + try: + with urllib.request.urlopen(url, timeout=25) as resp: + data = json.load(resp) + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, ValueError): + return None + out = data.get("info", {}).get("requires_dist") or [] + _CACHE[key] = out + return out + + +_SPEC_RE = re.compile(r"^\s*([A-Za-z0-9._-]+)\s*(?:\[[^\]]*\])?\s*([^;]*)") +_OP_RE = re.compile(r"(>=|<=|==|~=|!=|>|<)\s*([0-9][0-9A-Za-z.*+-]*)") + + +def _nums(s: str) -> list[int]: + core = re.split(r"[+]", s, maxsplit=1)[0] + core = re.split(r"(?<=[\d.])(?:a|b|rc|dev|pre|post)\d*$", core)[0] + return [int(n) for n in re.findall(r"\d+", core)] + + +def _satisfies(op: str, got: str, want: str) -> bool: + if op == ">=": + return _ver(got) >= _ver(want) + if op == ">": + return _ver(got) > _ver(want) + if op == "<=": + return _ver(got) <= _ver(want) + if op == "<": + return _ver(got) < _ver(want) + if op == "==": + if want.endswith(".*"): + prefix = _nums(want[:-2]) # "4.9.*" -> [4, 9] + return _nums(got)[: len(prefix)] == prefix + return got == want + if op == "!=": + return got != want + if op == "~=": + # 只按下界判:兼容版本对「钉低了」的检出已足够,且不引入 PEP 440 前缀语义分歧。 + return _ver(got) >= _ver(want) + return True + + +def check(path: Path) -> tuple[list[str], list[str]]: + """返回 (冲突列表, 无法核验列表)。""" + pins = read_pins(path) + conflicts: list[str] = [] + unchecked: list[str] = [] + + for name, (ver, _) in sorted(pins.items()): + dist = requires_dist(name, ver) + if dist is None: + unchecked.append(f"{name}=={ver}(PyPI 元数据取不到,未核验其约束)") + continue + for raw in dist: + if "extra" in raw: + continue + m = _SPEC_RE.match(raw) + if not m: + continue + dep = _norm(m.group(1)) + if dep not in pins or not m.group(2).strip(): + continue + got = pins[dep][0] + for op, want in _OP_RE.findall(m.group(2)): + if not _satisfies(op, got, want): + conflicts.append(f"{path.name}: {name}=={ver} 要求 {dep}{op}{want},锁里是 {dep}=={got}") + return conflicts, unchecked + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="校验锁文件内部交叉约束") + ap.add_argument("--quiet", action="store_true") + args = ap.parse_args(argv) + + all_conflicts: list[str] = [] + all_unchecked: list[str] = [] + for rel in LOCK_FILES: + p = _ROOT / rel + if not p.exists(): + continue + c, u = check(p) + all_conflicts += c + all_unchecked += u + + # 两份锁应一致;分别报出的同一行会重复,去重但记录来源 + uniq = list(dict.fromkeys(all_conflicts)) + if not args.quiet: + print(f"[pin-cross] 缓存 PyPI 版本 {len(_CACHE)} 个,冲突 {len(uniq)} 条,未核验 {len(all_unchecked)} 条") + for c in uniq: + print(f" [FAIL] {c}") + for u in all_unchecked: + print(f" [SKIP] {u}") + + if uniq: + print( + "\n[pin-cross] 锁集内部自相矛盾:pip 解析必然失败(历史上表现为便携包 " + "ResolutionImpossible)。要么回退被自动化顶掉的手工钉版,要么整体换到互相兼容的一组版本。", + file=sys.stderr, + ) + return 1 + if all_unchecked: + print("\n[pin-cross] 有依赖未能核验(网络/版本不存在),不当作通过。", file=sys.stderr) + return 2 + print("[pin-cross] PASS 未发现交叉约束冲突") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_check_pin_crossconflicts.py b/tests/test_check_pin_crossconflicts.py new file mode 100644 index 00000000..00625b6b --- /dev/null +++ b/tests/test_check_pin_crossconflicts.py @@ -0,0 +1,102 @@ +"""`scripts/check_pin_crossconflicts.py` 的判定逻辑(不联网)。 + +网络部分用 monkeypatch 喂假元数据;重点锁两类历史踩点: +① `==X.*` 通配(曾被解析成 `==X.` 造成 httpx 假阳性); +② 上界/`==` 型冲突(`check_pin_floors.py` 那类下界检查抓不到)。 +""" + +import sys +from pathlib import Path + +_PROJECT_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_PROJECT_ROOT / "scripts")) + +import check_pin_crossconflicts as ccc # noqa: E402 + + +class TestSatisfies: + def test_wildcard_equal_spec(self): + assert ccc._satisfies("==", "1.0.9", "1.*") is True + assert ccc._satisfies("==", "4.13.2", "4.9.*") is False + assert ccc._satisfies("==", "4.9.3", "4.9.*") is True + assert ccc._satisfies("==", "4.10.0", "4.9.*") is False + + def test_bounds(self): + assert ccc._satisfies("<", "0.23.2", "0.22") is False + assert ccc._satisfies("<", "0.21.4", "0.22") is True + assert ccc._satisfies(">=", "1.32.0", "1.5.0") is True + assert ccc._satisfies("<", "1.4.1", "1.4") is False + + def test_local_version_and_prerelease_do_not_confuse(self): + assert ccc._satisfies(">=", "2.13.0+cu132", "2.5.1") is True + # 已知近似(脚本里明确记了这条偏差):rc/dev 按基线参与比较, + # 所以 `1.4.0rc1` 相对 `<1.4` 判为「不满足」→ 会被报成冲突。 + # PEP 440 真语义下 rc < 正式版,本应满足;这里刻意取保守方向: + # 预发布版进锁本身就该被看见,宁可多报一条人工确认。 + assert ccc._satisfies("<", "1.4.0rc1", "1.4") is False + assert ccc._satisfies("<", "1.3.0", "1.4") is True + + +class TestReadPins: + def test_only_double_equals_and_comments(self, tmp_path): + f = tmp_path / "requirements-lock.txt" + f.write_text( + "# comment ==fake==1\ntransformers==4.52.1\ndatasets>=2.0\nmpmath==1.3.0 # 手工降定\n", + encoding="utf-8", + ) + pins = ccc.read_pins(f) + assert set(pins) == {"transformers", "mpmath"} + assert pins["mpmath"][0] == "1.3.0" + + +class TestCheckEndToEnd: + """喂假 PyPI 元数据,复现 #97 的四类冲突形状。""" + + META = { + "transformers==4.52.1": ["tokenizers<0.22,>=0.21", "huggingface-hub<1.0,>=0.30.0"], + "tokenizers==0.23.2": [], + "huggingface-hub==0.36.2": [], + "sympy==1.14.0": ["mpmath<1.4,>=1.1.0"], + "mpmath==1.4.1": [], + "hydra-core==1.3.7": ["antlr4-python3-runtime==4.9.*"], + "antlr4-python3-runtime==4.13.2": [], + "httpx==0.28.1": ["httpcore==1.*", "anyio"], + "httpcore==1.0.9": [], + "anyio==4.14.2": [], + } + + def _run(self, tmp_path, monkeypatch): + f = tmp_path / "requirements-lock.txt" + f.write_text("\n".join(f"{k.split('==')[0]}=={k.split('==')[1]}" for k in self.META) + "\n", encoding="utf-8") + monkeypatch.setattr(ccc, "requires_dist", lambda name, ver: self.META.get(f"{name}=={ver}", [])) + return ccc.check(f) + + def test_finds_ceiling_and_wildcard_conflicts(self, tmp_path, monkeypatch): + conflicts, unchecked = self._run(tmp_path, monkeypatch) + joined = "\n".join(conflicts) + assert "transformers==4.52.1 要求 tokenizers<0.22" in joined + assert "sympy==1.14.0 要求 mpmath<1.4" in joined + assert "hydra-core==1.3.7 要求 antlr4-python3-runtime==4.9.*" in joined + assert unchecked == [] + # 关键:httpcore==1.* 对 1.0.9 是满足的,不得成为第 4 条假阳性 + assert "httpcore" not in joined + + def test_clean_lock_yields_no_conflicts(self, tmp_path, monkeypatch): + meta = dict(self.META) + meta["tokenizers==0.21.4"] = meta.pop("tokenizers==0.23.2") + meta["mpmath==1.3.0"] = meta.pop("mpmath==1.4.1") + meta["antlr4-python3-runtime==4.9.3"] = meta.pop("antlr4-python3-runtime==4.13.2") + src = "\n".join(f"{k.split('==')[0]}=={k.split('==')[1]}" for k in meta) + "\n" + f = tmp_path / "requirements-lock.txt" + f.write_text(src, encoding="utf-8") + monkeypatch.setattr(ccc, "requires_dist", lambda name, ver: meta.get(f"{name}=={ver}", [])) + conflicts, unchecked = ccc.check(f) + assert conflicts == [] and unchecked == [] + + def test_unreachable_metadata_is_reported_not_swallowed(self, tmp_path, monkeypatch): + f = tmp_path / "requirements-lock.txt" + f.write_text("weirdpkg==1.0\nother==2.0\n", encoding="utf-8") + monkeypatch.setattr(ccc, "requires_dist", lambda name, ver: None) + conflicts, unchecked = ccc.check(f) + assert conflicts == [] + assert len(unchecked) == 2, "取不到元数据必须计入未核验,不能当成通过"