From 66b603c3f4db6f2d15c29cf377ae3b2bb24fe1fb Mon Sep 17 00:00:00 2001 From: Alex Kantor Date: Mon, 28 Sep 2026 11:38:58 +0100 Subject: [PATCH 1/4] test: check the Rego policies the docs tell people to copy A reader copies these blocks. One that does not compile wastes their time. One that compiles but denies everything is worse, because it looks like it worked. The test holds a block to that bar when it is presented as a whole file: either it declares its own package, or its fence names a .rego file, as in ```rego pr-compliant.rego. Name the file whenever the page tells someone to create it, and this holds the block to compiling from then on. Blocks with neither are fragments showing one rule. They cannot compile alone because their inputs are unbound, so there is nothing to check. `default allow := false` on its own does not satisfy the allow check. A policy can carry that line, have its real rule misspelled, compile, and deny every input forever. That is the failure this exists to catch, so the check wants a rule that can actually become true. It found one. tutorials/linking_trails_across_branches.md had three faults in nine lines: `package main`, which Kosli never reads because it queries data.policy.allow; pre-1.0 syntax that no longer compiles; and input.trail.compliance, where the field is input.trail.compliance_status. Fixed here. CI installs opa, pinned to v1.9.0 and checked against its published sha256, matching how this workflow pins its actions. Without opa the compile checks skip locally but fail outright when CI is set, so the suite cannot go quietly green. verified: pytest tests/ -> 58 passed. mint broken-links reports one break, in tutorials/working_with_controls.mdx, pre-existing on main. mutation: misspell the allow rule but keep the default line -> "defines no allow rule that can be true". Drop the package from a named block -> "is shown as a file to create but declares no package". Rename the package -> "so Kosli never reads it". Break the syntax -> opa's rego_parse_error. Each fails by name. A fence carrying attributes rather than a filename stays a fragment. search: counted every fenced block on the site, 881 of them. bash, shell, yaml and json were measured and left out: their failures are placeholders like , CloudFormation !Ref and `...` ellipses, so a check there reports conventions, not defects. Co-Authored-By: Claude Opus 5 --- .github/workflows/pr-quality.yml | 9 ++ tests/test_rego_examples.py | 94 +++++++++++++++++++++ tutorials/linking_trails_across_branches.md | 10 ++- 3 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 tests/test_rego_examples.py diff --git a/.github/workflows/pr-quality.yml b/.github/workflows/pr-quality.yml index b6cc2d77..e73962bd 100644 --- a/.github/workflows/pr-quality.yml +++ b/.github/workflows/pr-quality.yml @@ -27,6 +27,15 @@ jobs: - name: Install dependencies run: pip install requests pytest + - name: Install opa + run: | + curl -fsSL -o /tmp/opa \ + https://openpolicyagent.org/downloads/v1.9.0/opa_linux_amd64_static + echo "66fa66f3b730b2fb086003863428b382b2898d343adb4b5dfab5598b4d739eed /tmp/opa" \ + | sha256sum -c - + install -m 0755 /tmp/opa /usr/local/bin/opa + opa version + - name: Run tests run: pytest tests/ diff --git a/tests/test_rego_examples.py b/tests/test_rego_examples.py new file mode 100644 index 00000000..bc7824e5 --- /dev/null +++ b/tests/test_rego_examples.py @@ -0,0 +1,94 @@ +"""Every Rego policy shown in the docs must compile and follow Kosli's contract. + +A reader copies these. A block that does not compile wastes their time; one that +compiles but never defines `allow` denies everything, which is worse because it +looks like it worked. +""" + +import os +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + +import pytest + +DOCS_ROOT = Path(__file__).resolve().parent.parent +FENCE = re.compile(r"^[ \t]*```([a-zA-Z0-9_+-]*)([^\n]*)\n(.*?)^[ \t]*```", re.S | re.M) +PACKAGE = re.compile(r"^\s*package\s+([A-Za-z_][\w.]*)", re.M) +# A rule that can become true, so `default allow := false` on its own does not count. +ALLOW_RULE = re.compile(r"^\s*allow\b(?!\s*:?=\s*false)", re.M) + + +def _dedent(body): + indent = min( + (len(l) - len(l.lstrip()) for l in body.splitlines() if l.strip()), default=0 + ) + return "\n".join(l[indent:] if len(l) >= indent else l for l in body.splitlines()) + + +def rego_policies(): + """Rego blocks a reader is meant to copy whole. + + Two ways a block says that: it declares its own package, or its fence carries a + filename, as in ```rego pr-compliant.rego. Give a block a filename whenever the + page tells someone to create the file, and this will hold it to compiling. + + Anything else is a fragment illustrating one rule. It cannot compile alone + because its inputs are unbound, so there is nothing here to check. + """ + for path in sorted(DOCS_ROOT.rglob("*")): + if path.suffix not in (".md", ".mdx"): + continue + if "node_modules" in path.parts: + continue + text = path.read_text(encoding="utf-8") + for match in FENCE.finditer(text): + if match.group(1).lower() != "rego": + continue + filename = match.group(2).strip() + if filename and not filename.endswith(".rego"): + filename = "" # a fence attribute, not a file the reader creates + body = _dedent(match.group(3)) + package = PACKAGE.search(body) + if not package and not filename: + continue + line = text[: match.start()].count("\n") + 1 + yield pytest.param( + body, + package.group(1) if package else None, + id=f"{path.relative_to(DOCS_ROOT)}:{line}", + ) + + +def _opa(): + opa = shutil.which("opa") + if opa: + return opa + if os.environ.get("CI"): + pytest.fail("opa is not installed, so the Rego examples went unchecked") + pytest.skip("opa is not installed; install it to check the Rego examples") + + +@pytest.mark.parametrize("body,package", rego_policies()) +def test_rego_policy_compiles(body, package): + opa = _opa() + with tempfile.TemporaryDirectory() as tmp: + policy = Path(tmp) / "policy.rego" + policy.write_text(body, encoding="utf-8") + result = subprocess.run( + [opa, "check", str(policy)], capture_output=True, text=True + ) + assert result.returncode == 0, (result.stdout + result.stderr).strip() + + +@pytest.mark.parametrize("body,package", rego_policies()) +def test_rego_policy_follows_the_kosli_contract(body, package): + # policy-reference/rego_policy.mdx: Kosli reads data.policy.allow, and exits 0 + # when it is true and 1 when it is false. + assert package is not None, "is shown as a file to create but declares no package" + assert package == "policy", f"declares package {package}, so Kosli never reads it" + assert ALLOW_RULE.search(body), ( + "defines no allow rule that can be true, so it denies everything" + ) diff --git a/tutorials/linking_trails_across_branches.md b/tutorials/linking_trails_across_branches.md index 3294fe03..8dce9097 100644 --- a/tutorials/linking_trails_across_branches.md +++ b/tutorials/linking_trails_across_branches.md @@ -45,12 +45,14 @@ Use `kosli evaluate trail` on the main-branch build to evaluate the PR trail aga Use `kosli evaluate trail` to check the PR trail against a Rego policy. Create a policy file (e.g. `pr-compliant.rego`): ```rego pr-compliant.rego - package main + package policy - default allow = false + import rego.v1 - allow { - input.trail.compliance.is_compliant == true + default allow := false + + allow if { + input.trail.compliance_status.is_compliant == true } ``` From 6378434299619f7b58935c5b6c4898044503820f Mon Sep 17 00:00:00 2001 From: Alex Kantor Date: Mon, 28 Sep 2026 14:42:50 +0100 Subject: [PATCH 2/4] docs: read compliance_status, not compliance, in the jq tab too The Rego fix in the previous commit corrected input.trail.compliance to compliance_status. The same wrong field is read twice more on the same page, in the jq tab, and I did not look. `kosli get trail --output json` has no compliance key. So jq printed null and the next step ran with --compliant="null". verified: extracted the documented get trail sample from client_reference/kosli_get_trail.md and ran both expressions against it. .compliance.is_compliant gives null, .compliance_status.is_compliant gives true. A top-level is_compliant exists too, so .is_compliant would work. The nested path matches the sentence above the block and the Rego block on the same page. search: grep -rnE "\.compliance\." over every md and mdx -> these two lines and nothing else. Same for jq reads of trail JSON. no mutation: prose and shell, no test. pytest tests/ still 58 passed. Raised by the review bot on #433. Co-Authored-By: Claude Opus 5 --- tutorials/linking_trails_across_branches.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tutorials/linking_trails_across_branches.md b/tutorials/linking_trails_across_branches.md index 8dce9097..7ebae6ce 100644 --- a/tutorials/linking_trails_across_branches.md +++ b/tutorials/linking_trails_across_branches.md @@ -72,7 +72,7 @@ Use `kosli evaluate trail` on the main-branch build to evaluate the PR trail aga ```bash IS_COMPLIANT=$(kosli get trail "$PR_TRAIL" \ --flow app-pr \ - --output json | jq -r '.compliance.is_compliant') + --output json | jq -r '.compliance_status.is_compliant') ``` This is simpler but does not use a formal policy. You will need `$IS_COMPLIANT` in the next step. @@ -260,7 +260,7 @@ Below is a simplified GitHub Actions workflow for a main-branch build that links run: | IS_COMPLIANT=$(kosli get trail "${{ steps.pr.outputs.trail }}" \ --flow app-pr \ - --output json | jq -r '.compliance.is_compliant') + --output json | jq -r '.compliance_status.is_compliant') kosli attest generic \ --name pr-build-compliance \ From 9c588e5f0b3d9f3349a228f1f4a2fc7177d82daf Mon Sep 17 00:00:00 2001 From: Alex Kantor Date: Tue, 29 Sep 2026 10:10:17 +0100 Subject: [PATCH 3/4] ci: install opa with the official action Co-Authored-By: Claude Opus 5 --- .github/workflows/pr-quality.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-quality.yml b/.github/workflows/pr-quality.yml index e73962bd..95364c96 100644 --- a/.github/workflows/pr-quality.yml +++ b/.github/workflows/pr-quality.yml @@ -28,13 +28,9 @@ jobs: run: pip install requests pytest - name: Install opa - run: | - curl -fsSL -o /tmp/opa \ - https://openpolicyagent.org/downloads/v1.9.0/opa_linux_amd64_static - echo "66fa66f3b730b2fb086003863428b382b2898d343adb4b5dfab5598b4d739eed /tmp/opa" \ - | sha256sum -c - - install -m 0755 /tmp/opa /usr/local/bin/opa - opa version + uses: open-policy-agent/setup-opa@b2b258e089860efaadaaf71bf6e3aecb4a3eeff1 # v2.4.0 + with: + version: 1.9.0 - name: Run tests run: pytest tests/ From 7a34e5fae0228979ec7998c0a870e12e9d0d33b2 Mon Sep 17 00:00:00 2001 From: Alex Kantor Date: Tue, 29 Sep 2026 10:15:26 +0100 Subject: [PATCH 4/4] test: fail when no Rego policies are found An empty parametrize list is 0 tests and exit 0, so the whole check disappeared silently if the fence pattern stopped matching. Co-Authored-By: Claude Opus 5 --- tests/test_rego_examples.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/test_rego_examples.py b/tests/test_rego_examples.py index bc7824e5..5613e856 100644 --- a/tests/test_rego_examples.py +++ b/tests/test_rego_examples.py @@ -62,6 +62,16 @@ def rego_policies(): ) +POLICIES = list(rego_policies()) + + +def test_the_docs_still_contain_rego_policies(): + assert POLICIES, ( + "no Rego policies found, so the checks below all pass without checking " + "anything; the fence pattern has stopped matching the docs" + ) + + def _opa(): opa = shutil.which("opa") if opa: @@ -71,7 +81,7 @@ def _opa(): pytest.skip("opa is not installed; install it to check the Rego examples") -@pytest.mark.parametrize("body,package", rego_policies()) +@pytest.mark.parametrize("body,package", POLICIES) def test_rego_policy_compiles(body, package): opa = _opa() with tempfile.TemporaryDirectory() as tmp: @@ -83,7 +93,7 @@ def test_rego_policy_compiles(body, package): assert result.returncode == 0, (result.stdout + result.stderr).strip() -@pytest.mark.parametrize("body,package", rego_policies()) +@pytest.mark.parametrize("body,package", POLICIES) def test_rego_policy_follows_the_kosli_contract(body, package): # policy-reference/rego_policy.mdx: Kosli reads data.policy.allow, and exits 0 # when it is true and 1 when it is false.