From a88e409cbb98925e62b39d958a5b2ca8bc7cbb25 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Thu, 12 Nov 2026 14:23:36 -0800 Subject: [PATCH 1/8] ci(reach-matrix): install the main lock with --no-deps The four reach-matrix jobs installed requirements.lock.txt without --no-deps, so pip refused the filelock override (SECURITY.md) and every nightly run failed at its first step. A test now finds every install of the main lock in the workflows, Dockerfile, Makefiles, mkdocs.yml and docs, and fails without --no-deps. mkdocs.yml's preview note, which showed the install without it, now points to CONTRIBUTING.md. Signed-off-by: Alexander Rybkin --- .github/workflows/reach-matrix.yml | 8 ++++---- dokimi/mkdocs.yml | 6 ++---- dokimi/tests/test_dependency_locks.py | 21 +++++++++++++++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/reach-matrix.yml b/.github/workflows/reach-matrix.yml index cad34fee8..7e8e9a054 100644 --- a/.github/workflows/reach-matrix.yml +++ b/.github/workflows/reach-matrix.yml @@ -37,7 +37,7 @@ jobs: cache-dependency-path: dokimi/requirements.lock.txt - name: Install dokimi working-directory: dokimi - run: pip install --require-hashes -r requirements.lock.txt + run: pip install --no-deps --require-hashes -r requirements.lock.txt - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '26' @@ -90,7 +90,7 @@ jobs: cache-dependency-path: dokimi/requirements.lock.txt - name: Install dokimi working-directory: dokimi - run: pip install --require-hashes -r requirements.lock.txt + run: pip install --no-deps --require-hashes -r requirements.lock.txt - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '26' @@ -135,7 +135,7 @@ jobs: cache-dependency-path: dokimi/requirements.lock.txt - name: Install dokimi working-directory: dokimi - run: pip install --require-hashes -r requirements.lock.txt + run: pip install --no-deps --require-hashes -r requirements.lock.txt - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '26' @@ -182,7 +182,7 @@ jobs: cache-dependency-path: dokimi/requirements.lock.txt - name: Install dokimi working-directory: dokimi - run: pip install --require-hashes -r requirements.lock.txt + run: pip install --no-deps --require-hashes -r requirements.lock.txt - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: reach-matrix-* diff --git a/dokimi/mkdocs.yml b/dokimi/mkdocs.yml index 4a69b3f74..442f2de6c 100644 --- a/dokimi/mkdocs.yml +++ b/dokimi/mkdocs.yml @@ -1,10 +1,8 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Documentation site, published to GitHub Pages by .github/workflows/docs.yml. -# Local preview, from this directory (`make dev` already installs both locks): -# pip install -r requirements.lock.txt -r requirements-docs.lock.txt -# mkdocs serve # or: mkdocs build --strict (what CI runs) +# Documentation site, published to GitHub Pages by .github/workflows/docs.yml, which runs `mkdocs build --strict`. +# Local preview from this directory: `make dev`, then `mkdocs serve`; CONTRIBUTING.md ("Docs build") installs by hand. site_name: Dokimi site_description: Measure whether a skill, MCP doc source, documentation set or agent profile actually improves AI-agent answers and task outcomes site_url: https://intel.github.io/dokimi/ diff --git a/dokimi/tests/test_dependency_locks.py b/dokimi/tests/test_dependency_locks.py index 2a331426c..3579da55e 100644 --- a/dokimi/tests/test_dependency_locks.py +++ b/dokimi/tests/test_dependency_locks.py @@ -253,3 +253,24 @@ def test_a_replacement_update_is_exempt_from_the_release_age_wait() -> None: assert len(exempt) == 1, exempt assert 'matchManagers: ["github-actions"]' in exempt[0] assert 'matchUpdateTypes: ["replacement"]' in exempt[0] + + +# Files whose shell lines install a lock; changelog fragments are history, and tests quote installs. +INSTALL_SOURCES = (".github/**/*.yml", ".agent-eval/Dockerfile", "Makefile", "dokimi/Makefile", "dokimi/mkdocs.yml", + "*.md", "dokimi/*.md", "dokimi/docs/**/*.md") + + +def _install_lines(path: Path) -> list[str]: + """``path``'s lines that install ``requirements.lock.txt``, a backslash continuation joined to its line.""" + text = re.sub(r"\\\n\s*", " ", path.read_text(encoding="utf-8")) + return [line for line in text.splitlines() + if re.search(r"\binstall\b", line) and re.search(r"-r\s+\S*\brequirements\.lock\.txt\b", line)] + + +def test_every_install_of_the_main_lock_passes_no_deps() -> None: + """SECURITY.md: the main lock overrides litellm's filelock cap, so an install without --no-deps cannot resolve.""" + found = {path.relative_to(ROOT.parent).as_posix(): _install_lines(path) + for pattern in INSTALL_SOURCES for path in sorted(ROOT.parent.glob(pattern)) if path.is_file()} + assert sum(len(lines) for lines in found.values()) >= 10, found + missing = [(name, line.strip()) for name, lines in found.items() for line in lines if "--no-deps" not in line] + assert missing == [] From 8641053f42013cb5edbbd0e224a7b8709d05ec07 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Thu, 12 Nov 2026 19:43:19 -0800 Subject: [PATCH 2/8] test: the main-lock guard reads every requirement form and checks each command The guard matched only `-r ` and looked for --no-deps anywhere on the line, so `--requirement `, `-r` and `--no-deps x && pip install -r ` passed. It now reads the forms scripts/ci_local.py reads and checks --no-deps per command; a parametrized test pins each form and the chain. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_dependency_locks.py | 45 ++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/dokimi/tests/test_dependency_locks.py b/dokimi/tests/test_dependency_locks.py index 3579da55e..31ccad5ff 100644 --- a/dokimi/tests/test_dependency_locks.py +++ b/dokimi/tests/test_dependency_locks.py @@ -260,17 +260,46 @@ def test_a_replacement_update_is_exempt_from_the_release_age_wait() -> None: "*.md", "dokimi/*.md", "dokimi/docs/**/*.md") -def _install_lines(path: Path) -> list[str]: - """``path``'s lines that install ``requirements.lock.txt``, a backslash continuation joined to its line.""" - text = re.sub(r"\\\n\s*", " ", path.read_text(encoding="utf-8")) - return [line for line in text.splitlines() - if re.search(r"\binstall\b", line) and re.search(r"-r\s+\S*\brequirements\.lock\.txt\b", line)] +# pip's requirement-file forms, as scripts/ci_local.py reads them: -r f, -rf, --requirement f, --requirement=f +_MAIN_LOCK = re.compile(r"(?:^|\s)(?:-r\s*|--requirement[=\s]\s*)(?:\S*/)?requirements\.lock\.txt\b") + + +def _installs_of_the_main_lock(text: str) -> list[str]: + """Each command in ``text`` that installs ``requirements.lock.txt``; a backslash continuation joins its line + and ``&&``, ``||``, ``;`` and ``|`` separate commands, so a flag of one command never covers another.""" + joined = re.sub(r"\\\r?\n\s*", " ", text) + return [part.strip() for line in joined.splitlines() for part in re.split(r"&&|\|\||;|\|", line) + if re.search(r"\binstall\b", part) and _MAIN_LOCK.search(part)] + + +@pytest.mark.parametrize(("text", "unguarded"), [ + ("pip install -r requirements.lock.txt", True), + ("pip install -rrequirements.lock.txt", True), + ("pip install --requirement requirements.lock.txt", True), + ("pip install --requirement=dokimi/requirements.lock.txt", True), + ("pip install --no-deps wheel && pip install --require-hashes -r requirements.lock.txt", True), + ("uv pip install --python x \\\n -r requirements.lock.txt", True), + ("pip install --no-deps --require-hashes -r requirements.lock.txt -r requirements-ci.lock.txt", False), + ("pip install --no-deps --requirement=/opt/req/requirements.lock.txt", False), + ("uv pip install --python x --no-deps \\\n -r requirements.lock.txt", False), +], ids=["short", "joined", "long", "long-equals", "chained", "continued", "guarded", "guarded-long", + "guarded-continued"]) +def test_an_install_of_the_main_lock_without_no_deps_is_found(text: str, unguarded: bool) -> None: + installs = _installs_of_the_main_lock(text) + assert len(installs) == 1, installs + assert ("--no-deps" not in installs[0]) is unguarded + + +@pytest.mark.parametrize("text", ["pip install --require-hashes -r requirements-ci.lock.txt", + "cache-dependency-path: dokimi/requirements.lock.txt"]) +def test_other_locks_and_mentions_of_the_main_lock_are_not_installs_of_it(text: str) -> None: + assert _installs_of_the_main_lock(text) == [] def test_every_install_of_the_main_lock_passes_no_deps() -> None: """SECURITY.md: the main lock overrides litellm's filelock cap, so an install without --no-deps cannot resolve.""" - found = {path.relative_to(ROOT.parent).as_posix(): _install_lines(path) + found = {path.relative_to(ROOT.parent).as_posix(): _installs_of_the_main_lock(path.read_text(encoding="utf-8")) for pattern in INSTALL_SOURCES for path in sorted(ROOT.parent.glob(pattern)) if path.is_file()} - assert sum(len(lines) for lines in found.values()) >= 10, found - missing = [(name, line.strip()) for name, lines in found.items() for line in lines if "--no-deps" not in line] + assert sum(len(installs) for installs in found.values()) >= 10, found + missing = [(name, install) for name, installs in found.items() for install in installs if "--no-deps" not in install] assert missing == [] From 9d8311f824f24d6b2efb399dcff53877d94b465e Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Thu, 12 Nov 2026 19:58:21 -0800 Subject: [PATCH 3/8] test: read main-lock installs as the shell does The guard's regex missed a quoted lock path (-r "requirements.lock.txt") and took a commented "# --no-deps" for the option. It now reads each pip invocation with shlex (quotes removed, comments dropped, &&/||/;/| separating commands; a Markdown code span ends at its backtick), reads the requirement options as scripts/ci_local.py does, and needs --no-deps as a token of the command that installs the lock. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_dependency_locks.py | 73 +++++++++++++++++++++------ 1 file changed, 58 insertions(+), 15 deletions(-) diff --git a/dokimi/tests/test_dependency_locks.py b/dokimi/tests/test_dependency_locks.py index 31ccad5ff..e33916b3e 100644 --- a/dokimi/tests/test_dependency_locks.py +++ b/dokimi/tests/test_dependency_locks.py @@ -260,16 +260,51 @@ def test_a_replacement_update_is_exempt_from_the_release_age_wait() -> None: "*.md", "dokimi/*.md", "dokimi/docs/**/*.md") -# pip's requirement-file forms, as scripts/ci_local.py reads them: -r f, -rf, --requirement f, --requirement=f -_MAIN_LOCK = re.compile(r"(?:^|\s)(?:-r\s*|--requirement[=\s]\s*)(?:\S*/)?requirements\.lock\.txt\b") - - -def _installs_of_the_main_lock(text: str) -> list[str]: - """Each command in ``text`` that installs ``requirements.lock.txt``; a backslash continuation joins its line - and ``&&``, ``||``, ``;`` and ``|`` separate commands, so a flag of one command never covers another.""" - joined = re.sub(r"\\\r?\n\s*", " ", text) - return [part.strip() for line in joined.splitlines() for part in re.split(r"&&|\|\||;|\|", line) - if re.search(r"\binstall\b", part) and _MAIN_LOCK.search(part)] +_MAIN_LOCK = "requirements.lock.txt" +_SEPARATORS = {"&&", "||", ";", "|", "&"} +# A pip invocation, as scripts/ci_local.py's PIP_INSTALL, with uv's and python -m's forms. +_PIP_INSTALL = re.compile(r"\b(?:python3? -m |uv )?pip3? install\b") + + +def _requirement_files(tokens: list[str]) -> list[str]: + """The files of pip's requirement options, as scripts/ci_local.py reads them: -r f, -rf, --requirement f|=f.""" + files, args = [], iter(tokens) + for token in args: + if token in ("-r", "--requirement"): + files.append(next(args, "")) + elif token.startswith("--requirement="): + files.append(token.partition("=")[2]) + elif token.startswith("-r") and not token.startswith("--"): + files.append(token[2:]) + return [name.rstrip("`.,:)").rsplit("/", 1)[-1] for name in files] + + +def _installs_of_the_main_lock(text: str) -> list[list[str]]: + """The tokens of each command in ``text`` that installs the main lock, read as a POSIX shell reads it: + quotes removed, ``#`` comments dropped, a backslash continuation joined, ``&&``/``||``/``;``/``|`` separating.""" + commands = [] + for line in re.sub(r"\\\r?\n", " ", text).splitlines(): + start = _PIP_INSTALL.search(line) + if _MAIN_LOCK not in line or start is None: + continue + span = line[start.start():] + if line[:start.start()].count("`") % 2: # inside a Markdown code span: up to its closing backtick + span = span.split("`", 1)[0] + lexer = shlex.shlex(span, posix=True, punctuation_chars=True) + lexer.whitespace_split = True + try: + tokens = list(lexer) + except ValueError as exc: + raise AssertionError(f"cannot read as a shell command: {span.strip()!r} ({exc})") from exc + command: list[str] = [] + for token in [*tokens, ";"]: + if token not in _SEPARATORS: + command.append(token) + continue + if "install" in command and _MAIN_LOCK in _requirement_files(command): + commands.append(command) + command = [] + return commands @pytest.mark.parametrize(("text", "unguarded"), [ @@ -277,13 +312,18 @@ def _installs_of_the_main_lock(text: str) -> list[str]: ("pip install -rrequirements.lock.txt", True), ("pip install --requirement requirements.lock.txt", True), ("pip install --requirement=dokimi/requirements.lock.txt", True), + ('pip install --require-hashes -r "requirements.lock.txt"', True), + ("pip install --requirement='requirements.lock.txt'", True), + ("pip install --require-hashes -r requirements.lock.txt # --no-deps", True), ("pip install --no-deps wheel && pip install --require-hashes -r requirements.lock.txt", True), ("uv pip install --python x \\\n -r requirements.lock.txt", True), ("pip install --no-deps --require-hashes -r requirements.lock.txt -r requirements-ci.lock.txt", False), - ("pip install --no-deps --requirement=/opt/req/requirements.lock.txt", False), + ("Run `pip install -r requirements.lock.txt` if you don't use make.", True), + ("Run `pip install --no-deps -r requirements.lock.txt` if you don't use make.", False), + ('pip install --no-deps --requirement="/opt/req/requirements.lock.txt"', False), ("uv pip install --python x --no-deps \\\n -r requirements.lock.txt", False), -], ids=["short", "joined", "long", "long-equals", "chained", "continued", "guarded", "guarded-long", - "guarded-continued"]) +], ids=["short", "joined", "long", "long-equals", "quoted", "single-quoted", "commented-flag", "chained", + "continued", "guarded", "prose-span", "prose-span-guarded", "guarded-quoted", "guarded-continued"]) def test_an_install_of_the_main_lock_without_no_deps_is_found(text: str, unguarded: bool) -> None: installs = _installs_of_the_main_lock(text) assert len(installs) == 1, installs @@ -291,7 +331,9 @@ def test_an_install_of_the_main_lock_without_no_deps_is_found(text: str, unguard @pytest.mark.parametrize("text", ["pip install --require-hashes -r requirements-ci.lock.txt", - "cache-dependency-path: dokimi/requirements.lock.txt"]) + "cache-dependency-path: dokimi/requirements.lock.txt", + "Dependencies install from `requirements.lock.txt`; the docs build's from", + "pip install --no-deps-x -r requirements-docs.lock.txt"]) def test_other_locks_and_mentions_of_the_main_lock_are_not_installs_of_it(text: str) -> None: assert _installs_of_the_main_lock(text) == [] @@ -301,5 +343,6 @@ def test_every_install_of_the_main_lock_passes_no_deps() -> None: found = {path.relative_to(ROOT.parent).as_posix(): _installs_of_the_main_lock(path.read_text(encoding="utf-8")) for pattern in INSTALL_SOURCES for path in sorted(ROOT.parent.glob(pattern)) if path.is_file()} assert sum(len(installs) for installs in found.values()) >= 10, found - missing = [(name, install) for name, installs in found.items() for install in installs if "--no-deps" not in install] + missing = [(name, shlex.join(install)) for name, installs in found.items() for install in installs + if "--no-deps" not in install] assert missing == [] From 3962c3d47006ccdf40c4a839f0363b0c26f4fc33 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Thu, 12 Nov 2026 21:08:00 -0800 Subject: [PATCH 4/8] test: guard the main-lock installs that run, read as YAML and shell The guard now reads what runs: each workflow and action run: step decoded by YAML (a quoted run: value no longer trips it), the eval Dockerfile and the Makefiles, and pins that it sees each kind, the setup action included, which no version read before. Commands are lexed with shlex on ;&| only, so spacing, tabs and a subshell's ")" no longer hide a command. Markdown examples and commands built at run time are out of scope, as the docstring says. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_dependency_locks.py | 108 ++++++++++++++------------ 1 file changed, 60 insertions(+), 48 deletions(-) diff --git a/dokimi/tests/test_dependency_locks.py b/dokimi/tests/test_dependency_locks.py index e33916b3e..03a8efd4d 100644 --- a/dokimi/tests/test_dependency_locks.py +++ b/dokimi/tests/test_dependency_locks.py @@ -255,15 +255,7 @@ def test_a_replacement_update_is_exempt_from_the_release_age_wait() -> None: assert 'matchUpdateTypes: ["replacement"]' in exempt[0] -# Files whose shell lines install a lock; changelog fragments are history, and tests quote installs. -INSTALL_SOURCES = (".github/**/*.yml", ".agent-eval/Dockerfile", "Makefile", "dokimi/Makefile", "dokimi/mkdocs.yml", - "*.md", "dokimi/*.md", "dokimi/docs/**/*.md") - - _MAIN_LOCK = "requirements.lock.txt" -_SEPARATORS = {"&&", "||", ";", "|", "&"} -# A pip invocation, as scripts/ci_local.py's PIP_INSTALL, with uv's and python -m's forms. -_PIP_INSTALL = re.compile(r"\b(?:python3? -m |uv )?pip3? install\b") def _requirement_files(tokens: list[str]) -> list[str]: @@ -276,38 +268,51 @@ def _requirement_files(tokens: list[str]) -> list[str]: files.append(token.partition("=")[2]) elif token.startswith("-r") and not token.startswith("--"): files.append(token[2:]) - return [name.rstrip("`.,:)").rsplit("/", 1)[-1] for name in files] + return [name.rstrip(")").rsplit("/", 1)[-1] for name in files] -def _installs_of_the_main_lock(text: str) -> list[list[str]]: - """The tokens of each command in ``text`` that installs the main lock, read as a POSIX shell reads it: - quotes removed, ``#`` comments dropped, a backslash continuation joined, ``&&``/``||``/``;``/``|`` separating.""" - commands = [] - for line in re.sub(r"\\\r?\n", " ", text).splitlines(): - start = _PIP_INSTALL.search(line) - if _MAIN_LOCK not in line or start is None: +def _installs_of_the_main_lock(script: str) -> list[list[str]]: + """The tokens of each pip command in a shell ``script`` that installs the main lock: POSIX quoting and + ``#`` comments, a backslash continuation joined, and a run of ``;``/``&``/``|`` separating commands.""" + installs = [] + for line in re.sub(r"\\\r?\n", " ", script).splitlines(): + if _MAIN_LOCK not in line: continue - span = line[start.start():] - if line[:start.start()].count("`") % 2: # inside a Markdown code span: up to its closing backtick - span = span.split("`", 1)[0] - lexer = shlex.shlex(span, posix=True, punctuation_chars=True) + lexer = shlex.shlex(line, posix=True, punctuation_chars=";&|") lexer.whitespace_split = True - try: - tokens = list(lexer) - except ValueError as exc: - raise AssertionError(f"cannot read as a shell command: {span.strip()!r} ({exc})") from exc command: list[str] = [] - for token in [*tokens, ";"]: - if token not in _SEPARATORS: + for token in [*lexer, ";"]: + if token.strip(";&|"): command.append(token) continue - if "install" in command and _MAIN_LOCK in _requirement_files(command): - commands.append(command) + pip = [i for i, word in enumerate(command[:-1]) if word.strip("()").rsplit("/", 1)[-1] in ("pip", "pip3")] + if any(command[i + 1] == "install" for i in pip) and _MAIN_LOCK in _requirement_files(command): + installs.append(command) command = [] - return commands + return installs -@pytest.mark.parametrize(("text", "unguarded"), [ +def _install_scripts() -> dict[str, str]: + """The scripts that run: each workflow and action ``run:`` step (YAML-decoded), the eval image's Dockerfile + and the Makefiles. Markdown examples are prose for a reader, not run, so they are not here.""" + scripts = {} + workflows = sorted((ROOT.parent / ".github").glob("**/*.yml")) + for path in [*workflows, *sorted((ROOT.parent / "actions").glob("**/action.yml"))]: + doc = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(doc, dict): + continue + steps = list((doc.get("runs") or {}).get("steps", [])) # a composite action + for job in (doc.get("jobs") or {}).values(): + steps += job.get("steps", []) + for i, step in enumerate(steps): + if isinstance(step.get("run"), str): + scripts[f"{path.relative_to(ROOT.parent).as_posix()} step {i}"] = step["run"] + for name in (".agent-eval/Dockerfile", "Makefile", "dokimi/Makefile"): + scripts[name] = (ROOT.parent / name).read_text(encoding="utf-8") + return scripts + + +@pytest.mark.parametrize(("script", "unguarded"), [ ("pip install -r requirements.lock.txt", True), ("pip install -rrequirements.lock.txt", True), ("pip install --requirement requirements.lock.txt", True), @@ -315,34 +320,41 @@ def _installs_of_the_main_lock(text: str) -> list[list[str]]: ('pip install --require-hashes -r "requirements.lock.txt"', True), ("pip install --requirement='requirements.lock.txt'", True), ("pip install --require-hashes -r requirements.lock.txt # --no-deps", True), + ("pip install\t--require-hashes -r requirements.lock.txt", True), ("pip install --no-deps wheel && pip install --require-hashes -r requirements.lock.txt", True), + ("(pip install -r requirements.lock.txt); pip install --no-deps wheel", True), ("uv pip install --python x \\\n -r requirements.lock.txt", True), ("pip install --no-deps --require-hashes -r requirements.lock.txt -r requirements-ci.lock.txt", False), - ("Run `pip install -r requirements.lock.txt` if you don't use make.", True), - ("Run `pip install --no-deps -r requirements.lock.txt` if you don't use make.", False), - ('pip install --no-deps --requirement="/opt/req/requirements.lock.txt"', False), - ("uv pip install --python x --no-deps \\\n -r requirements.lock.txt", False), -], ids=["short", "joined", "long", "long-equals", "quoted", "single-quoted", "commented-flag", "chained", - "continued", "guarded", "prose-span", "prose-span-guarded", "guarded-quoted", "guarded-continued"]) -def test_an_install_of_the_main_lock_without_no_deps_is_found(text: str, unguarded: bool) -> None: - installs = _installs_of_the_main_lock(text) + ('python -m pip install --no-deps --requirement="/opt/req/requirements.lock.txt"', False), + ("uv pip install --python $(VPY) --no-deps \\\n -r requirements.lock.txt", False), +], ids=["short", "joined", "long", "long-equals", "quoted", "single-quoted", "commented-flag", "spacing", "chained", + "subshell", "continued", "guarded", "guarded-quoted", "guarded-continued"]) +def test_an_install_of_the_main_lock_without_no_deps_is_found(script: str, unguarded: bool) -> None: + installs = _installs_of_the_main_lock(script) assert len(installs) == 1, installs assert ("--no-deps" not in installs[0]) is unguarded -@pytest.mark.parametrize("text", ["pip install --require-hashes -r requirements-ci.lock.txt", - "cache-dependency-path: dokimi/requirements.lock.txt", - "Dependencies install from `requirements.lock.txt`; the docs build's from", - "pip install --no-deps-x -r requirements-docs.lock.txt"]) -def test_other_locks_and_mentions_of_the_main_lock_are_not_installs_of_it(text: str) -> None: - assert _installs_of_the_main_lock(text) == [] +@pytest.mark.parametrize("script", ["pip install --require-hashes -r requirements-ci.lock.txt", + "COPY dokimi/requirements.lock.txt /opt/dokimi-req/", + "pip install --no-deps-x -r requirements-docs.lock.txt"]) +def test_other_locks_and_mentions_of_the_main_lock_are_not_installs_of_it(script: str) -> None: + assert _installs_of_the_main_lock(script) == [] + + +def test_a_quoted_run_value_is_read_as_the_command_it_decodes_to() -> None: + """A workflow may quote its run: value; YAML decodes it before the shell sees it.""" + step = yaml.safe_load('run: "pip install --no-deps -r requirements.lock.txt"')["run"] + assert _installs_of_the_main_lock(step) == [["pip", "install", "--no-deps", "-r", "requirements.lock.txt"]] def test_every_install_of_the_main_lock_passes_no_deps() -> None: - """SECURITY.md: the main lock overrides litellm's filelock cap, so an install without --no-deps cannot resolve.""" - found = {path.relative_to(ROOT.parent).as_posix(): _installs_of_the_main_lock(path.read_text(encoding="utf-8")) - for pattern in INSTALL_SOURCES for path in sorted(ROOT.parent.glob(pattern)) if path.is_file()} - assert sum(len(installs) for installs in found.values()) >= 10, found + """SECURITY.md: the main lock overrides litellm's filelock cap, so an install without --no-deps cannot resolve. + Not covered: a command built at run time (a variable holding the file name, eval) and Markdown examples.""" + found = {name: _installs_of_the_main_lock(script) for name, script in _install_scripts().items()} + sources = {name.split(" step ")[0] for name, installs in found.items() if installs} + assert {".github/workflows/reach-matrix.yml", "actions/setup/action.yml", ".agent-eval/Dockerfile", + "dokimi/Makefile"} <= sources, sources missing = [(name, shlex.join(install)) for name, installs in found.items() for install in installs if "--no-deps" not in install] assert missing == [] From 1c21c6484128ebc19de7b9694decfedd50769667 Mon Sep 17 00:00:00 2001 From: vchamarthi <123653452+vchamarthi@users.noreply.github.com> Date: Fri, 13 Nov 2026 07:08:54 -0600 Subject: [PATCH 5/8] fix(tests): read the whole install script when guarding the main lock The main-lock guard lexed line by line and matched only a bare `pip install`. - `(pip install -r requirements.lock.txt --no-deps)` is read as guarded - a global option, `python -m pip -q`, `pip3.11` and a Makefile `@` prefix are read - a quoted word spanning lines, or holding `\"`, no longer raises - the word after `>` or `<` is not a pip flag; a backslash path is read - `.yaml` workflows and `action.yaml` are scanned Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com> --- dokimi/tests/test_dependency_locks.py | 95 ++++++++++++++++++++------- 1 file changed, 73 insertions(+), 22 deletions(-) diff --git a/dokimi/tests/test_dependency_locks.py b/dokimi/tests/test_dependency_locks.py index 03a8efd4d..dc04787bd 100644 --- a/dokimi/tests/test_dependency_locks.py +++ b/dokimi/tests/test_dependency_locks.py @@ -7,6 +7,7 @@ import re import shlex +import sys import tomllib from pathlib import Path @@ -268,36 +269,56 @@ def _requirement_files(tokens: list[str]) -> list[str]: files.append(token.partition("=")[2]) elif token.startswith("-r") and not token.startswith("--"): files.append(token[2:]) - return [name.rstrip(")").rsplit("/", 1)[-1] for name in files] + return [re.split(r"[/\\]", name)[-1] for name in files] + + +def _commands(script: str) -> list[list[str]]: + """The commands of a shell ``script`` as word lists: POSIX quoting (a quoted word may span lines), ``#`` + comments, a backslash continuation joined, ``;``/``&``/``|``/newline separating commands, ``(`` and ``)`` + dropped from the words, and the word after a redirection (``>``, ``<``) left out.""" + commands: list[list[str]] = [] + command: list[str] = [] + script = re.sub(r"\\\r?\n", " ", script).replace('\\"', "\0") # an escaped quote is a letter, not a quote + script = re.sub(r"""('[^']*'|"[^"]*")|(?\n") + lexer.whitespace, lexer.whitespace_split, lexer.escape, lexer.commenters = " \t\r", True, "", "" + redirected = False + while True: + token = lexer.get_token() + if token is None: + break + if not token.strip(";&|\n"): + commands.append(command) + command = [] + elif token.strip("<>&") != token and not token.strip("<>&"): + redirected = True + elif redirected: + redirected = False + elif token.strip("()"): + command.append(token.strip("()").replace("\0", '"')) + return [*commands, command] def _installs_of_the_main_lock(script: str) -> list[list[str]]: - """The tokens of each pip command in a shell ``script`` that installs the main lock: POSIX quoting and - ``#`` comments, a backslash continuation joined, and a run of ``;``/``&``/``|`` separating commands.""" - installs = [] - for line in re.sub(r"\\\r?\n", " ", script).splitlines(): - if _MAIN_LOCK not in line: - continue - lexer = shlex.shlex(line, posix=True, punctuation_chars=";&|") - lexer.whitespace_split = True - command: list[str] = [] - for token in [*lexer, ";"]: - if token.strip(";&|"): - command.append(token) - continue - pip = [i for i, word in enumerate(command[:-1]) if word.strip("()").rsplit("/", 1)[-1] in ("pip", "pip3")] - if any(command[i + 1] == "install" for i in pip) and _MAIN_LOCK in _requirement_files(command): - installs.append(command) - command = [] - return installs + """The words of each pip command in a shell ``script`` that installs the main lock: ``pip``, ``pip3.11``, + ``python -m pip``, ``uv pip``, with global options before ``install`` and a Makefile ``@``/``-``/``+`` prefix.""" + if _MAIN_LOCK not in script: + return [] + return [ + command for command in _commands(script) + if any(re.fullmatch(r"pip[\d.]*", word.lstrip("@+-").rsplit("/", 1)[-1]) and "install" in command[i + 1:] + for i, word in enumerate(command)) + and _MAIN_LOCK in _requirement_files(command) + ] def _install_scripts() -> dict[str, str]: """The scripts that run: each workflow and action ``run:`` step (YAML-decoded), the eval image's Dockerfile and the Makefiles. Markdown examples are prose for a reader, not run, so they are not here.""" scripts = {} - workflows = sorted((ROOT.parent / ".github").glob("**/*.yml")) - for path in [*workflows, *sorted((ROOT.parent / "actions").glob("**/action.yml"))]: + paths = [path for ext in ("yml", "yaml") for path in sorted((ROOT.parent / ".github").glob(f"**/*.{ext}"))] + paths += [path for ext in ("yml", "yaml") for path in sorted((ROOT.parent / "actions").glob(f"**/action.{ext}"))] + for path in paths: doc = yaml.safe_load(path.read_text(encoding="utf-8")) if not isinstance(doc, dict): continue @@ -324,11 +345,23 @@ def _install_scripts() -> dict[str, str]: ("pip install --no-deps wheel && pip install --require-hashes -r requirements.lock.txt", True), ("(pip install -r requirements.lock.txt); pip install --no-deps wheel", True), ("uv pip install --python x \\\n -r requirements.lock.txt", True), + ("pip --disable-pip-version-check install -r requirements.lock.txt", True), + ("python -m pip -q install -r requirements.lock.txt", True), + ("pip3.11 install -r requirements.lock.txt", True), + ("all:\n\t@pip install -r requirements.lock.txt", True), + ("pip install -r requirements.lock.txt > --no-deps", True), + ("pip install -r dokimi\\requirements.lock.txt", True), + ("pip install -r requirements.lock.txt # --no-deps\npip install --no-deps wheel", True), ("pip install --no-deps --require-hashes -r requirements.lock.txt -r requirements-ci.lock.txt", False), ('python -m pip install --no-deps --requirement="/opt/req/requirements.lock.txt"', False), ("uv pip install --python $(VPY) --no-deps \\\n -r requirements.lock.txt", False), + ("(pip install -r requirements.lock.txt --no-deps)", False), + ('pip install --no-deps -r requirements.lock.txt --log "install\nlog.txt"', False), + ('pip install --no-deps -r requirements.lock.txt --log "build\\"log.txt"', False), ], ids=["short", "joined", "long", "long-equals", "quoted", "single-quoted", "commented-flag", "spacing", "chained", - "subshell", "continued", "guarded", "guarded-quoted", "guarded-continued"]) + "subshell", "continued", "global-option", "python-m-pip", "versioned", "recipe-prefix", "redirected", + "backslash-path", "comment-line", "guarded", "guarded-quoted", "guarded-continued", "guarded-subshell", + "guarded-multiline", "guarded-escaped-quote"]) def test_an_install_of_the_main_lock_without_no_deps_is_found(script: str, unguarded: bool) -> None: installs = _installs_of_the_main_lock(script) assert len(installs) == 1, installs @@ -348,6 +381,24 @@ def test_a_quoted_run_value_is_read_as_the_command_it_decodes_to() -> None: assert _installs_of_the_main_lock(step) == [["pip", "install", "--no-deps", "-r", "requirements.lock.txt"]] +@pytest.mark.parametrize(("workflow", "action"), [("example.yaml", "action.yml"), ("example.yml", "action.yaml")]) +def test_yaml_and_yml_workflows_and_actions_are_scanned(tmp_path: Path, monkeypatch, workflow: str, action: str) -> None: + step = "run: pip install --require-hashes -r requirements.lock.txt" + (tmp_path / "dokimi").mkdir() + for name in (".agent-eval/Dockerfile", "Makefile", "dokimi/Makefile"): + (tmp_path / name).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / name).write_text("all:\n\ttrue\n", encoding="utf-8") + (tmp_path / ".github" / "workflows").mkdir(parents=True) + (tmp_path / ".github" / "workflows" / workflow).write_text( + f"jobs:\n j:\n steps:\n - {step}\n", encoding="utf-8") + (tmp_path / "actions" / "setup").mkdir(parents=True) + (tmp_path / "actions" / "setup" / action).write_text( + f"runs:\n using: composite\n steps:\n - {step}\n shell: bash\n", encoding="utf-8") + monkeypatch.setattr(sys.modules[__name__], "ROOT", tmp_path / "dokimi") + scripts = _install_scripts() + assert f".github/workflows/{workflow} step 0" in scripts and f"actions/setup/{action} step 0" in scripts, scripts + + def test_every_install_of_the_main_lock_passes_no_deps() -> None: """SECURITY.md: the main lock overrides litellm's filelock cap, so an install without --no-deps cannot resolve. Not covered: a command built at run time (a variable holding the file name, eval) and Markdown examples.""" From 81e34fc8effe7d8bc8931c7a909bddfbf42ba686 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 13 Nov 2026 19:02:34 -0800 Subject: [PATCH 6/8] test: a quoted operator is a word in the main-lock guard The lexer took a quoted ";" for a separator and stripped the parentheses of a quoted "(--no-deps)", so `--log ";"` hid an install and `--log "(--no-deps)"` passed one as guarded. Operators inside quotes are now letters until the word is built; the three cases fail without it. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_dependency_locks.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/dokimi/tests/test_dependency_locks.py b/dokimi/tests/test_dependency_locks.py index dc04787bd..c88512342 100644 --- a/dokimi/tests/test_dependency_locks.py +++ b/dokimi/tests/test_dependency_locks.py @@ -257,6 +257,8 @@ def test_a_replacement_update_is_exempt_from_the_release_age_wait() -> None: _MAIN_LOCK = "requirements.lock.txt" +_QUOTED = str.maketrans(";&|<>()", "\ue000\ue001\ue002\ue003\ue004\ue005\ue006") +_UNQUOTED = {letter: operator for operator, letter in _QUOTED.items()} def _requirement_files(tokens: list[str]) -> list[str]: @@ -275,11 +277,12 @@ def _requirement_files(tokens: list[str]) -> list[str]: def _commands(script: str) -> list[list[str]]: """The commands of a shell ``script`` as word lists: POSIX quoting (a quoted word may span lines), ``#`` comments, a backslash continuation joined, ``;``/``&``/``|``/newline separating commands, ``(`` and ``)`` - dropped from the words, and the word after a redirection (``>``, ``<``) left out.""" + dropped from the words, the word after a redirection (``>``, ``<``) left out; a quoted operator is a letter.""" commands: list[list[str]] = [] command: list[str] = [] script = re.sub(r"\\\r?\n", " ", script).replace('\\"', "\0") # an escaped quote is a letter, not a quote - script = re.sub(r"""('[^']*'|"[^"]*")|(?\n") lexer.whitespace, lexer.whitespace_split, lexer.escape, lexer.commenters = " \t\r", True, "", "" redirected = False @@ -295,7 +298,7 @@ def _commands(script: str) -> list[list[str]]: elif redirected: redirected = False elif token.strip("()"): - command.append(token.strip("()").replace("\0", '"')) + command.append(token.strip("()").translate(_UNQUOTED).replace("\0", '"')) return [*commands, command] @@ -352,16 +355,20 @@ def _install_scripts() -> dict[str, str]: ("pip install -r requirements.lock.txt > --no-deps", True), ("pip install -r dokimi\\requirements.lock.txt", True), ("pip install -r requirements.lock.txt # --no-deps\npip install --no-deps wheel", True), + ('pip install --log ";" -r requirements.lock.txt', True), + ('pip install -r requirements.lock.txt --log "(--no-deps)"', True), ("pip install --no-deps --require-hashes -r requirements.lock.txt -r requirements-ci.lock.txt", False), ('python -m pip install --no-deps --requirement="/opt/req/requirements.lock.txt"', False), ("uv pip install --python $(VPY) --no-deps \\\n -r requirements.lock.txt", False), ("(pip install -r requirements.lock.txt --no-deps)", False), ('pip install --no-deps -r requirements.lock.txt --log "install\nlog.txt"', False), ('pip install --no-deps -r requirements.lock.txt --log "build\\"log.txt"', False), + ("pip install --log '>' --no-deps -r requirements.lock.txt", False), ], ids=["short", "joined", "long", "long-equals", "quoted", "single-quoted", "commented-flag", "spacing", "chained", "subshell", "continued", "global-option", "python-m-pip", "versioned", "recipe-prefix", "redirected", - "backslash-path", "comment-line", "guarded", "guarded-quoted", "guarded-continued", "guarded-subshell", - "guarded-multiline", "guarded-escaped-quote"]) + "backslash-path", "comment-line", "quoted-separator", "quoted-parentheses", "guarded", "guarded-quoted", + "guarded-continued", "guarded-subshell", "guarded-multiline", "guarded-escaped-quote", + "guarded-quoted-redirection"]) def test_an_install_of_the_main_lock_without_no_deps_is_found(script: str, unguarded: bool) -> None: installs = _installs_of_the_main_lock(script) assert len(installs) == 1, installs From 037c9ff2ba94e32e64f0407f3d63bfa4ff6bac33 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 13 Nov 2026 19:02:52 -0800 Subject: [PATCH 7/8] test: a comment ends at its newline in the main-lock guard Continuations were joined before comments were dropped, so a comment ending in a backslash swallowed the install on the next line, which the shell runs. Comments are now dropped first; the comment-backslash case fails without it. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_dependency_locks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dokimi/tests/test_dependency_locks.py b/dokimi/tests/test_dependency_locks.py index c88512342..ed0cebc52 100644 --- a/dokimi/tests/test_dependency_locks.py +++ b/dokimi/tests/test_dependency_locks.py @@ -280,9 +280,10 @@ def _commands(script: str) -> list[list[str]]: dropped from the words, the word after a redirection (``>``, ``<``) left out; a quoted operator is a letter.""" commands: list[list[str]] = [] command: list[str] = [] - script = re.sub(r"\\\r?\n", " ", script).replace('\\"', "\0") # an escaped quote is a letter, not a quote + script = script.replace('\\"', "\0") # an escaped quote is a letter, not a quote script = re.sub(r"""('[^']*'|"[^"]*")|(?\n") lexer.whitespace, lexer.whitespace_split, lexer.escape, lexer.commenters = " \t\r", True, "", "" redirected = False @@ -356,6 +357,7 @@ def _install_scripts() -> dict[str, str]: ("pip install -r dokimi\\requirements.lock.txt", True), ("pip install -r requirements.lock.txt # --no-deps\npip install --no-deps wheel", True), ('pip install --log ";" -r requirements.lock.txt', True), + ("# note \\\npip install --require-hashes -r requirements.lock.txt", True), ('pip install -r requirements.lock.txt --log "(--no-deps)"', True), ("pip install --no-deps --require-hashes -r requirements.lock.txt -r requirements-ci.lock.txt", False), ('python -m pip install --no-deps --requirement="/opt/req/requirements.lock.txt"', False), @@ -366,7 +368,7 @@ def _install_scripts() -> dict[str, str]: ("pip install --log '>' --no-deps -r requirements.lock.txt", False), ], ids=["short", "joined", "long", "long-equals", "quoted", "single-quoted", "commented-flag", "spacing", "chained", "subshell", "continued", "global-option", "python-m-pip", "versioned", "recipe-prefix", "redirected", - "backslash-path", "comment-line", "quoted-separator", "quoted-parentheses", "guarded", "guarded-quoted", + "backslash-path", "comment-line", "quoted-separator", "comment-backslash", "quoted-parentheses", "guarded", "guarded-quoted", "guarded-continued", "guarded-subshell", "guarded-multiline", "guarded-escaped-quote", "guarded-quoted-redirection"]) def test_an_install_of_the_main_lock_without_no_deps_is_found(script: str, unguarded: bool) -> None: From a2d83ae83ccaa235abc29011dd2128ee55c4da7b Mon Sep 17 00:00:00 2001 From: vchamarthi <123653452+vchamarthi@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:59:06 -0600 Subject: [PATCH 8/8] fix(tests): read more spellings of the main-lock install The guard missed -qr, --requirem, pip.exe, a JSON-form Dockerfile RUN and a PowerShell backtick continuation, and raised ValueError on an escaped apostrophe or on '...\' . A comment ending in a backtick hid the next line's install. The --no-deps verdict is now a helper with its own test. Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com> --- dokimi/tests/test_dependency_locks.py | 61 +++++++++++++++++++++------ 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/dokimi/tests/test_dependency_locks.py b/dokimi/tests/test_dependency_locks.py index 82468b7da..18cbbf9bd 100644 --- a/dokimi/tests/test_dependency_locks.py +++ b/dokimi/tests/test_dependency_locks.py @@ -5,6 +5,7 @@ from __future__ import annotations +import json import re import shlex import sys @@ -293,28 +294,43 @@ def test_a_replacement_update_is_exempt_from_the_release_age_wait() -> None: def _requirement_files(tokens: list[str]) -> list[str]: - """The files of pip's requirement options, as scripts/ci_local.py reads them: -r f, -rf, --requirement f|=f.""" + """The files of pip's requirement options, as scripts/ci_local.py reads them (-r f, -rf, --requirement f|=f), + plus a clustered short flag (-qr f) and an abbreviated long one (--requirem f).""" files, args = [], iter(tokens) for token in args: - if token in ("-r", "--requirement"): + if token == "-r" or re.fullmatch(r"-[A-Za-z]*r", token) or ( + token.startswith("--requirem") and "--requirement".startswith(token)): files.append(next(args, "")) - elif token.startswith("--requirement="): + elif token.partition("=")[0].startswith("--requirem") and "=" in token and "--requirement".startswith( + token.partition("=")[0]): files.append(token.partition("=")[2]) elif token.startswith("-r") and not token.startswith("--"): files.append(token[2:]) return [re.split(r"[/\\]", name)[-1] for name in files] +def _exec_form(match: re.Match[str]) -> str: + """A Dockerfile ``RUN ["pip", "install", ...]`` line as the shell line it runs; other text is kept.""" + try: + words = json.loads(match.group(1)) + except ValueError: + return match.group(0) + return shlex.join(words) if isinstance(words, list) and all(isinstance(w, str) for w in words) else match.group(0) + + def _commands(script: str) -> list[list[str]]: """The commands of a shell ``script`` as word lists: POSIX quoting (a quoted word may span lines), ``#`` comments, a backslash continuation joined, ``;``/``&``/``|``/newline separating commands, ``(`` and ``)`` dropped from the words, the word after a redirection (``>``, ``<``) left out; a quoted operator is a letter.""" commands: list[list[str]] = [] command: list[str] = [] - script = script.replace('\\"', "\0") # an escaped quote is a letter, not a quote + script = re.sub(r"^\s*RUN\s+(\[.*\])\s*$", _exec_form, script, flags=re.MULTILINE) + script = re.sub(r"""('[^']*')|\\(["'])""", # an escaped quote is a letter, not a quote; '...\' is a whole word + lambda m: m.group(1) or {'"': "\0", "'": "\ue007"}[m.group(2)], script) script = re.sub(r"""('[^']*'|"[^"]*")|(?\n") lexer.whitespace, lexer.whitespace_split, lexer.escape, lexer.commenters = " \t\r", True, "", "" redirected = False @@ -330,7 +346,7 @@ def _commands(script: str) -> list[list[str]]: elif redirected: redirected = False elif token.strip("()"): - command.append(token.strip("()").translate(_UNQUOTED).replace("\0", '"')) + command.append(token.strip("()").translate(_UNQUOTED).replace("\0", '"').replace("\ue007", "'")) return [*commands, command] @@ -341,7 +357,7 @@ def _installs_of_the_main_lock(script: str) -> list[list[str]]: return [] return [ command for command in _commands(script) - if any(re.fullmatch(r"pip[\d.]*", word.lstrip("@+-").rsplit("/", 1)[-1]) and "install" in command[i + 1:] + if any(re.fullmatch(r"pip[\d.]*(\.exe)?", word.lstrip("@+-").rsplit("/", 1)[-1]) and "install" in command[i + 1:] for i, word in enumerate(command)) and _MAIN_LOCK in _requirement_files(command) ] @@ -371,6 +387,15 @@ def _install_scripts() -> dict[str, str]: @pytest.mark.parametrize(("script", "unguarded"), [ ("pip install -r requirements.lock.txt", True), ("pip install -rrequirements.lock.txt", True), + ("pip install -qr requirements.lock.txt", True), + ("# setup `\npip install -r requirements.lock.txt", True), + ("pip install --no-deps -r requirements.lock.txt --log 'build\\'", False), + ("pip install --requirem requirements.lock.txt", True), + ("pip install --requirem=requirements.lock.txt", True), + ("pip.exe install --require-hashes -r requirements.lock.txt", True), + ('RUN ["pip", "install", "--require-hashes", "-r", "/opt/dokimi-req/requirements.lock.txt"]', True), + ("python -m pip install `\n --require-hashes -r requirements.lock.txt", True), + ("echo don\\'t\npip install --require-hashes -r requirements.lock.txt", True), ("pip install --requirement requirements.lock.txt", True), ("pip install --requirement=dokimi/requirements.lock.txt", True), ('pip install --require-hashes -r "requirements.lock.txt"', True), @@ -397,7 +422,8 @@ def _install_scripts() -> dict[str, str]: ('pip install --no-deps -r requirements.lock.txt --log "install\nlog.txt"', False), ('pip install --no-deps -r requirements.lock.txt --log "build\\"log.txt"', False), ("pip install --log '>' --no-deps -r requirements.lock.txt", False), -], ids=["short", "joined", "long", "long-equals", "quoted", "single-quoted", "commented-flag", "spacing", "chained", +], ids=["short", "joined", "clustered", "comment-backtick", "guarded-single-quoted-backslash", "abbreviated", "abbreviated-equals", "exe", "exec-form", "powershell-backtick", + "escaped-apostrophe", "long", "long-equals", "quoted", "single-quoted", "commented-flag", "spacing", "chained", "subshell", "continued", "global-option", "python-m-pip", "versioned", "recipe-prefix", "redirected", "backslash-path", "comment-line", "quoted-separator", "comment-backslash", "quoted-parentheses", "guarded", "guarded-quoted", "guarded-continued", "guarded-subshell", "guarded-multiline", "guarded-escaped-quote", @@ -439,13 +465,24 @@ def test_yaml_and_yml_workflows_and_actions_are_scanned(tmp_path: Path, monkeypa assert f".github/workflows/{workflow} step 0" in scripts and f"actions/setup/{action} step 0" in scripts, scripts +def _unguarded(found: dict[str, list[list[str]]]) -> list[tuple[str, str]]: + """The installs of the main lock, by script name, that lack ``--no-deps``.""" + return [(name, shlex.join(install)) for name, installs in found.items() for install in installs + if "--no-deps" not in install] + + +def test_an_install_without_no_deps_is_reported_and_one_with_it_is_not() -> None: + install = ["pip", "install", "-r", "requirements.lock.txt"] + assert _unguarded({"x": [install]}) == [("x", "pip install -r requirements.lock.txt")] + assert _unguarded({"x": [[*install, "--no-deps"]], "y": []}) == [] + + def test_every_install_of_the_main_lock_passes_no_deps() -> None: """SECURITY.md: the main lock overrides litellm's filelock cap, so an install without --no-deps cannot resolve. - Not covered: a command built at run time (a variable holding the file name, eval) and Markdown examples.""" + Not covered: a command built at run time (a variable holding the file name or the pip command, eval) + and Markdown examples.""" found = {name: _installs_of_the_main_lock(script) for name, script in _install_scripts().items()} sources = {name.split(" step ")[0] for name, installs in found.items() if installs} assert {".github/workflows/reach-matrix.yml", "actions/setup/action.yml", ".agent-eval/Dockerfile", "dokimi/Makefile"} <= sources, sources - missing = [(name, shlex.join(install)) for name, installs in found.items() for install in installs - if "--no-deps" not in install] - assert missing == [] + assert _unguarded(found) == []