diff --git a/.github/workflows/f7las-ci.yml b/.github/workflows/f7las-ci.yml index 55799ec..3898c2e 100644 --- a/.github/workflows/f7las-ci.yml +++ b/.github/workflows/f7las-ci.yml @@ -35,6 +35,9 @@ jobs: - name: Validate supply-chain invariants run: python scripts/validate-supply-chain.py + - name: Validate documentation + run: python scripts/validate-documentation.py + - name: Audit dependencies and generate CycloneDX SBOM run: | mkdir -p "$RUNNER_TEMP/f7las-sbom" @@ -88,6 +91,12 @@ jobs: echo "OPA_BIN=$RUNNER_TEMP/f7las-bin/opa" >> "$GITHUB_ENV" "$RUNNER_TEMP/f7las-bin/opa" version + - name: Run clean-user acceptance + env: + OPA_BIN: ${{ runner.temp }}/f7las-bin/opa + PYTHON_BIN: python + run: scripts/run-clean-user-acceptance.sh + - name: Verify src.policy package imports run: python -c "import src.policy" @@ -110,7 +119,7 @@ jobs: run: opa check --strict config/policies/canonical-workflow.rego - name: Run behavioral scenarios - run: pytest -q tests/test_behavioral_scenarios.py + run: python -m pytest -q tests/test_behavioral_scenarios.py - name: Generate canonical evidence run: | @@ -133,7 +142,7 @@ jobs: --opa-binary "$OPA_BIN" - name: Run unit tests - run: pytest -q --ignore=tests/test_behavioral_scenarios.py + run: python -m pytest -q --ignore=tests/test_behavioral_scenarios.py - name: Run golden dataset evaluation run: | diff --git a/README.md b/README.md index 006ed70..657567b 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,10 @@ Software supply-chain security is a cross-cutting supplemental domain, **Layer S - Architecture diagrams and engineering review material - Canonical v1.0.0 data contracts and one synthetic, offline Python + OPA workflow - Illustrative prompts, additional policies, validators, and runtime stubs -- Structural CI checks and prototype tests +- Supply-chain CI, behavioral tests, evidence verification, deterministic replay, + and a clean-user acceptance gate for the canonical path -The canonical workflow provides one deliberately constrained executable Layers 1–7 path. It does not make the other examples executable or production-ready. The existing golden-dataset runner validates scenario structure; it does not prove the described allow/deny behavior. Placeholder tests and incomplete examples are being replaced as part of the [overhaul roadmap](ROADMAP.md). +The canonical workflow provides one deliberately constrained executable Layers 1–7 path. It does not make the other examples executable or production-ready. The behavioral scenario matrix exercises the canonical enforcement path; the separate golden-dataset runner validates scenario structure and does not prove the described allow/deny behavior. Non-canonical examples remain illustrative unless they are explicitly reclassified and tested. ## Executable versus illustrative @@ -45,7 +46,7 @@ The canonical workflow provides one deliberately constrained executable Layers 1 | Other OPA/PDP/PEP code | Partial prototype | | Planner, tools, sandbox, telemetry | Illustrative prototypes | | Other policy-engine examples | Illustrative, non-canonical patterns | -| Other end-to-end workflows | Planned; not yet implemented | +| Other end-to-end workflows | Not provided | | Production integrations or actions | Not provided | Nothing in this repository should be connected to production data, identities, cloud resources, security platforms, or remediation systems without independent engineering and security review. @@ -61,12 +62,14 @@ python -m venv .venv source .venv/bin/activate python -m pip install --require-hashes -r requirements-ci.lock python scripts/validate-supply-chain.py +python scripts/validate-documentation.py python scripts/validate-prompts.py python scripts/validate-policies.py +python scripts/allowlist-validator.py python scripts/validate-settings.py config/settings.yaml python scripts/validate-contracts.py opa check --strict config/policies/canonical-workflow.rego -pytest -q +python -m pytest -q ``` These commands validate the **current prototype and repository structure**. They are not evidence of production readiness or full behavioral enforcement. @@ -82,6 +85,7 @@ These commands validate the **current prototype and repository structure**. They - [Canonical data contracts v1.0.0](schemas/contracts/README.md) - [Canonical offline workflow](examples/canonical-workflow/README.md) - [Supply-chain and CI controls](docs/supply-chain-and-ci.md) +- [Clean-user acceptance](docs/clean-user-acceptance.md) - [Roadmap](ROADMAP.md) - [Security policy](SECURITY.md) diff --git a/ROADMAP.md b/ROADMAP.md index 236261a..be26541 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -17,7 +17,7 @@ The target is an **Executable Reference Implementation**: one offline, determini 5. **Behavioral scenarios** — an executable canonical matrix tests permitted, denied, malformed, unauthorized, unavailable, timed-out, tampered, expired, obligation, and recovery paths. 6. **Evidence and replay** — canonical evidence can be independently verified for complete correlation and tampering, then replayed from the admitted input to reproduce the exact canonical outcome. 7. **Supply chain and CI** — implement pinned and hashed dependencies, immutable action references, verified tool downloads, known-vulnerability and Git-history secret gates, and a retained CI SBOM. -8. **Documentation reconciliation** — execute every documented command in a clean environment. +8. **Documentation reconciliation** — execute every supported user command in a clean environment; classify maintainer-only and illustrative material explicitly. 9. **F7-LAS-specific diagrams** — replace diagrams only after execution semantics are frozen. 10. **Independent review** — review the complete branch diff and CI evidence before any merge, release, or Zenodo update. diff --git a/docs/F7-LAS-QA.md b/docs/F7-LAS-QA.md index 1d5e0d0..5d93947 100644 --- a/docs/F7-LAS-QA.md +++ b/docs/F7-LAS-QA.md @@ -17,7 +17,8 @@ executable or production-ready. The existing golden-dataset evaluator validates scenario and rubric structure; it does not execute or prove the stated security behavior. -The canonical path has automated behavioral checks, but repository-wide control +The canonical path has automated behavioral checks, evidence verification, +deterministic replay, and a clean-user acceptance gate. Repository-wide control status is not yet machine-readable or evidence-linked. ## Is the code production-ready? @@ -86,6 +87,16 @@ prove that dependencies are non-malicious, guarantee that no secret exists, or create a signed release SBOM or provenance attestation. See [Supply-chain and CI controls](supply-chain-and-ci.md) for the exact boundary. +## Has the documented canonical path been tested as a clean user? + +Yes, within a deliberately narrow boundary. The +[clean-user acceptance gate](clean-user-acceptance.md) copies the repository +without Git metadata, removes inherited Python import settings, creates a new +Python 3.12.14 virtual environment, performs a hash-locked installation, and +runs the supported validation and canonical walkthrough commands with the +checksum-verified OPA 1.20.2 binary. It does not test the illustrative layer +examples or establish production readiness. + ## How should practitioners use the repository today? Use it as a design-review lens, threat-modeling aid, draft control catalog, and source of clearly labeled examples. Verify each claimed outcome independently before adapting any pattern. diff --git a/docs/README.md b/docs/README.md index 0d7d37d..5376eb6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,7 @@ This directory contains the governed documentation for the F7-LAS seven-layer re - [Canonical data contracts v1.0.0](../schemas/contracts/README.md) — machine-validated request-through-audit definitions used by the canonical executable path. - [Canonical offline workflow](../examples/canonical-workflow/README.md) — the bounded synthetic Python + OPA Layers 1–7 demonstration. - [Supply-chain and CI controls](supply-chain-and-ci.md) — implemented dependency, action, download, vulnerability, secret, and SBOM checks and their limits. +- [Clean-user acceptance](clean-user-acceptance.md) — the exact supported environment, complete walkthrough, command inventory, and assurance boundary. ## Architecture graphics @@ -21,6 +22,6 @@ Existing diagrams remain under `docs/` and `docs/images/` while their references ## Status and use -The model is established; the repository implementation is a prototype. Documentation may specify desired controls that are not yet implemented. Consult the control catalog status and future traceability matrix before treating a control as verified. +The model is established; the repository implementation is a prototype. Documentation may specify desired controls that are not implemented. Treat only the canonical behavior covered by tests and correlated evidence as verified; control-to-evidence traceability remains a separate release gate. See the root [license map](../LICENSE.md) and [security policy](../SECURITY.md). diff --git a/docs/clean-user-acceptance.md b/docs/clean-user-acceptance.md new file mode 100644 index 0000000..4aea81b --- /dev/null +++ b/docs/clean-user-acceptance.md @@ -0,0 +1,54 @@ +# Clean-user acceptance + +Milestone 8 defines one supported clean-user path for the bounded F7-LAS +reference implementation. It starts from repository files without Git metadata, +creates a new virtual environment, installs only the reviewed hash-locked graph, +and runs the documented validators, tests, canonical walkthrough, evidence +verification, deterministic replay, golden structural evaluation, vulnerability +audit, and SBOM generation. + +## Exact prerequisites + +- A clean checkout of this repository on Linux x86_64. +- Python **3.12.14**. +- The OPA **1.20.2** Linux x86_64 static binary with SHA-256 + `69da5179ee403d10fa11bab6cfb4ffb0d23dba5f9b682fa977db772a1da5670f`. +- Network access to the configured Python package index and vulnerability + service during dependency installation and `pip-audit`. + +The repository's canonical code supports Python 3.10 or later, but the +acceptance environment is intentionally narrower and matches CI exactly. + +## Run the complete gate + +From the repository root, point `OPA_BIN` at the checksum-verified binary: + +```bash +OPA_BIN=/absolute/path/to/opa scripts/run-clean-user-acceptance.sh +``` + +The script rejects a different Python patch release, OPA version, or OPA binary +digest. It removes inherited `PYTHONPATH`, copies the working tree without Git +metadata or local environments, and performs the acceptance run in a temporary +directory. Set `KEEP_CLEAN_USER_WORKDIR=1` only when the temporary evidence and +SBOM are needed for diagnosis. + +Success ends with `Clean-user acceptance PASSED`. A nonzero status means the +repository has not passed this gate. + +## Command inventory + +| Documentation | Classification | Clean-user evidence | +|---|---|---| +| Root validation commands | Supported | Validators and complete tests run in the isolated environment | +| Canonical workflow commands | Supported | Execution, evidence verification, and replay all run | +| Contract validation commands | Supported | Contract validator and complete tests run | +| Supply-chain reproduction commands | Supported | Locked installation, invariant validation, audit, SBOM, and supply-chain tests run | +| Non-canonical layer examples | Illustrative only | No runnable walkthrough is claimed or accepted | +| Dependency-lock regeneration | Maintainer operation | Not represented as a clean-user command; any update requires separate dependency-diff review | +| Full-history Gitleaks scan | CI-only | Requires Git history and remains enforced by the normal GitHub Actions job | + +This gate proves reproducibility for one synthetic, offline workflow on the +specified platform. It does not prove production readiness, external-system +behavior, OS/container isolation, artifact provenance, or trustworthiness of +the audited dependencies. diff --git a/docs/supply-chain-and-ci.md b/docs/supply-chain-and-ci.md index 5c0ca34..dfcc5cc 100644 --- a/docs/supply-chain-and-ci.md +++ b/docs/supply-chain-and-ci.md @@ -28,12 +28,13 @@ python -m venv .venv source .venv/bin/activate python -m pip install --require-hashes -r requirements-ci.lock python scripts/validate-supply-chain.py +python scripts/validate-documentation.py pip-audit --require-hashes --disable-pip --strict \ --progress-spinner off \ --requirement requirements-ci.lock \ --format cyclonedx-json \ --output f7las-python.cdx.json -pytest -q tests/test_supply_chain.py +python -m pytest -q tests/test_supply_chain.py ``` Gitleaks and OPA are installed in CI from the exact versioned URLs and digests @@ -42,19 +43,12 @@ running the corresponding workflow commands on Linux x64. ## Updating the Python lock -Change exact top-level pins deliberately, then regenerate the lock with the -same resolver command and review all transitive changes: - -```bash -python -m pip install pip-tools==7.6.1 -pip-compile --allow-unsafe --generate-hashes --resolver=backtracking \ - --strip-extras --output-file requirements-ci.lock requirements-ci.in -python scripts/validate-supply-chain.py -``` - -An update is incomplete until the known-vulnerability audit, full tests, and -the complete dependency diff have been reviewed. Dependabot proposals are not -auto-merged. +Dependency-lock regeneration is a maintainer operation, not a supported +clean-user command. Change exact top-level pins deliberately, regenerate the +lock in a separately reviewed maintenance environment, and review every +transitive version and hash change. An update is incomplete until the +known-vulnerability audit, full tests, and complete dependency diff have been +reviewed. Dependabot proposals are not auto-merged. ## Assurance boundary diff --git a/examples/README.md b/examples/README.md index 73bce84..f350343 100644 --- a/examples/README.md +++ b/examples/README.md @@ -34,4 +34,6 @@ A minimal simulated execution wrapper and container example. Containment is not ### Layer 7 — Monitoring & Telemetry A simple event schema + Python logger for auditability. -Each layer folder contains its own README with instructions. +Each layer folder contains its own README with its classification and +limitations. Only [`canonical-workflow/`](canonical-workflow/README.md) provides +a supported runnable walkthrough. diff --git a/examples/canonical-workflow/README.md b/examples/canonical-workflow/README.md index 299c432..3cfbfe2 100644 --- a/examples/canonical-workflow/README.md +++ b/examples/canonical-workflow/README.md @@ -57,7 +57,7 @@ effects. Admission refusals must emit no workflow records. Run the matrix with the pinned OPA CLI available: ```bash -OPA_BIN=opa pytest -q tests/test_behavioral_scenarios.py +OPA_BIN=opa python -m pytest -q tests/test_behavioral_scenarios.py ``` These are deterministic reference-workflow scenarios, not claims of production diff --git a/examples/layer5-policy-engines/opa-rego/README.md b/examples/layer5-policy-engines/opa-rego/README.md index 77770de..0e25052 100644 --- a/examples/layer5-policy-engines/opa-rego/README.md +++ b/examples/layer5-policy-engines/opa-rego/README.md @@ -1,124 +1,21 @@ -# OPA Rego PDP Demo (Layer 5) +# Illustrative OPA/Rego Pattern (Layer 5) -This folder provides a runnable demonstration of the **F7-LAS Layer 5 Policy Decision Point (PDP)** using **Open Policy Agent (OPA)** and the Rego policy located at: +This directory is **illustrative, non-canonical, and unsupported as a runnable walkthrough**. +It preserves an early HTTP-based PDP/PEP pattern for design comparison only. -``` -config/policies/l5/opa/agent_security_enforcement.rego -``` +The local files are not a coherent supported deployment: -The demo shows how the PDP evaluates agent actions such as: +- `policy.rego` uses an older Rego style and is not the policy executed by the + canonical workflow. +- `pep_opa.py` targets a service hostname and package path that require external + orchestration and configuration. +- `docker-compose.yml` is retained as historical prototype material; it uses a + mutable image reference and does not mount the policy from this directory. -- `terminate_instance` -- `describe_instance` -- `list_instances` +Do not infer execution, security, or compatibility from these files and do not +connect them to production tools or data. The only supported OPA execution path +is the checksum-verified CLI workflow documented in the +[canonical workflow](../../canonical-workflow/README.md). Its executable policy +is [`config/policies/canonical-workflow.rego`](../../../config/policies/canonical-workflow.rego). -…and how the **PEP (Policy Enforcement Point)** queries the PDP before tools are executed. - ---- - -## Run the OPA PDP Locally - -From the **repo root**, start the OPA container using: - -```bash -docker compose -f examples/layer5-policy-engines/opa-rego/docker-compose.yml up -``` - -This will: - -- Start OPA on `http://localhost:8181` -- Load the Layer-5 Rego policy automatically -- Watch policy files for changes - ---- - -## Test the Policy Manually - -### 1. Check the `allow` rule - -```bash -curl -s -X POST http://localhost:8181/v1/data/f7las/l5/enforcement/allow -H 'Content-Type: application/json' -d '{ - "input": { - "action": "terminate_instance", - "environment": "production", - "current_time_ok_for_change": false - } - }' -``` - -**Expected Output:** - -```json -{"result": false} -``` - ---- - -### 2. Retrieve the denial reason - -```bash -curl -s -X POST http://localhost:8181/v1/data/f7las/l5/enforcement/deny_message -H 'Content-Type: application/json' -d '{ - "input": { - "action": "terminate_instance", - "environment": "production", - "current_time_ok_for_change": false - } - }' -``` - -**Example Output:** - -```json -{"result": "Agent action denied by L5 Policy: Production modification outside of approved maintenance window or environment."} -``` - ---- - -## Using the PDP from the Python PEP - -If you are using the OPA-backed PEP module located at: - -``` -src/policy/pep_opa.py -``` - -Example wiring: - -```python -from src.policy.pep_opa import OPAPEP - -pep = OPAPEP() - -tool_call = { - "tool_name": "aws_ec2_client", - "action": "terminate_instance", - "arguments": {"instance_id": "i-prod-1234"}, -} - -context = { - "agent_id": "OpsAgent-v1", - "target_environment": "production", - "initiating_user_role": "devops_engineer", -} - -decision = pep.authorize(tool_call, context) - -if decision.is_allowed: - print("L5: ALLOW → call tool") -else: - print(f"L5: DENY → {decision.reason}") -``` - ---- - -## Notes - -This is a **Stage‑1 demo PDP**, not a production deployment. - -Future stages will add: - -- Input validation -- Authenticated PDP queries -- Layer‑7 telemetry integration -- Structured logs for every authorization request -- Secure bundle mounting +This example may be removed or rebuilt in a later, separately reviewed change. diff --git a/examples/walkthrough_false_positive.md b/examples/walkthrough_false_positive.md deleted file mode 100644 index c3d2491..0000000 --- a/examples/walkthrough_false_positive.md +++ /dev/null @@ -1 +0,0 @@ -# Walkthrough: Handling a False Positive Alert (Placeholder) diff --git a/examples/walkthrough_ransomware_case.md b/examples/walkthrough_ransomware_case.md deleted file mode 100644 index 0b229e7..0000000 --- a/examples/walkthrough_ransomware_case.md +++ /dev/null @@ -1 +0,0 @@ -# Walkthrough: Ransomware Case Investigation (Placeholder) diff --git a/schemas/contracts/README.md b/schemas/contracts/README.md index c108aeb..b2642c6 100644 --- a/schemas/contracts/README.md +++ b/schemas/contracts/README.md @@ -95,5 +95,5 @@ reviewed input and repository policy, but it is not proof of provenance. ```bash python scripts/validate-contracts.py -pytest -q +python -m pytest -q ``` diff --git a/scripts/run-clean-user-acceptance.sh b/scripts/run-clean-user-acceptance.sh new file mode 100755 index 0000000..c1fee22 --- /dev/null +++ b/scripts/run-clean-user-acceptance.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +python_bin="${PYTHON_BIN:-python}" +opa_candidate="${OPA_BIN:-opa}" +expected_python="3.12.14" +expected_opa="1.20.2" +expected_opa_sha256="69da5179ee403d10fa11bab6cfb4ffb0d23dba5f9b682fa977db772a1da5670f" + +fail() { + echo "clean-user acceptance failed: $*" >&2 + exit 1 +} + +command -v "$python_bin" >/dev/null 2>&1 || fail "Python executable not found: $python_bin" +opa_bin="$(command -v "$opa_candidate" 2>/dev/null || true)" +[[ -n "$opa_bin" ]] || fail "OPA executable not found: $opa_candidate" +opa_bin="$(cd "$(dirname "$opa_bin")" && pwd)/$(basename "$opa_bin")" + +python_version="$($python_bin -c 'import platform; print(platform.python_version())')" +[[ "$python_version" == "$expected_python" ]] || \ + fail "Python $expected_python is required; found $python_version" + +opa_version="$($opa_bin version | awk '/^Version:/ {print $2; exit}')" +[[ "$opa_version" == "$expected_opa" ]] || \ + fail "OPA $expected_opa is required; found ${opa_version:-unknown}" + +opa_sha256="$($python_bin -c 'import hashlib, pathlib, sys; print(hashlib.sha256(pathlib.Path(sys.argv[1]).read_bytes()).hexdigest())' "$opa_bin")" +[[ "$opa_sha256" == "$expected_opa_sha256" ]] || \ + fail "OPA binary digest does not match the reviewed Linux x86_64 static binary" + +acceptance_root="$(mktemp -d "${TMPDIR:-/tmp}/f7las-clean-user.XXXXXX")" +if [[ "${KEEP_CLEAN_USER_WORKDIR:-0}" != "1" ]]; then + trap 'rm -rf "$acceptance_root"' EXIT +else + echo "Retaining clean-user workspace: $acceptance_root" +fi + +checkout="$acceptance_root/F7-LAS" +venv="$acceptance_root/venv" +mkdir -p "$checkout" + +tar \ + --exclude='./.git' \ + --exclude='./.venv' \ + --exclude='./.pytest_cache' \ + --exclude='*/__pycache__' \ + -cf - -C "$repo_root" . | tar -xf - -C "$checkout" + +cd "$checkout" +unset PYTHONPATH + +echo "[clean-user] Create isolated Python environment" +"$python_bin" -m venv "$venv" +python="$venv/bin/python" +pip_audit="$venv/bin/pip-audit" + +echo "[clean-user] Install the hash-locked dependency graph" +"$python" -m pip install --require-hashes -r requirements-ci.lock + +echo "[clean-user] Validate documentation and repository invariants" +"$python" scripts/validate-documentation.py +"$python" scripts/validate-supply-chain.py +"$python" scripts/validate-prompts.py config/prompts +"$python" scripts/validate-policies.py +"$python" scripts/allowlist-validator.py +"$python" scripts/validate-settings.py config/settings.yaml +"$python" scripts/validate-contracts.py +"$opa_bin" check --strict config/policies/canonical-workflow.rego + +echo "[clean-user] Run the complete automated test suite" +OPA_BIN="$opa_bin" "$python" -m pytest -q + +evidence="$acceptance_root/canonical-evidence.json" +replayed="$acceptance_root/replayed-evidence.json" +golden="$acceptance_root/golden-results.json" +sbom="$acceptance_root/f7las-python.cdx.json" + +echo "[clean-user] Execute, verify, and replay the canonical walkthrough" +"$python" -m src.canonical.cli \ + --input examples/canonical-workflow/request.json \ + --output "$evidence" \ + --opa-binary "$opa_bin" +"$python" -m src.canonical.evidence --evidence "$evidence" +"$python" -m src.canonical.replay \ + --input examples/canonical-workflow/request.json \ + --evidence "$evidence" \ + --output "$replayed" \ + --opa-binary "$opa_bin" + +echo "[clean-user] Run and enforce the golden structural evaluation" +"$python" -m src.demo_runner.run_golden_dataset \ + --scenarios tests/golden_dataset/scenarios.json \ + --rubric tests/golden_dataset/rubric.json \ + --output "$golden" \ + --strict +"$python" scripts/check_golden_thresholds.py "$golden" + +echo "[clean-user] Audit the locked graph and generate a CycloneDX SBOM" +"$pip_audit" \ + --require-hashes \ + --disable-pip \ + --strict \ + --progress-spinner off \ + --requirement requirements-ci.lock \ + --format cyclonedx-json \ + --output "$sbom" +test -s "$sbom" || fail "CycloneDX SBOM was not generated" + +echo "Clean-user acceptance PASSED: Python $python_version, OPA $opa_version." diff --git a/scripts/validate-documentation.py b/scripts/validate-documentation.py new file mode 100755 index 0000000..cacbac2 --- /dev/null +++ b/scripts/validate-documentation.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Validate repository Markdown links, fences, and runnable-command boundaries.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path +from urllib.parse import unquote, urlsplit + + +ROOT = Path(__file__).resolve().parents[1] +LINK_RE = re.compile(r"!?\[[^\]]*\]\(([^)]+)\)") +FENCE_RE = re.compile(r"^\s*```([^`]*)\s*$") +SHELL_LANGUAGES = {"bash", "sh", "shell", "console", "powershell", "cmd"} +RUNNABLE_COMMAND_DOCS = { + Path("README.md"), + Path("docs/clean-user-acceptance.md"), + Path("docs/supply-chain-and-ci.md"), + Path("examples/canonical-workflow/README.md"), + Path("schemas/contracts/README.md"), +} +IGNORED_DIRECTORIES = {".git", ".venv", "venv", "node_modules", "__pycache__"} + + +class DocumentationError(ValueError): + """Raised when documentation cannot be followed from a clean checkout.""" + + +def markdown_files(root: Path) -> list[Path]: + return sorted( + path + for path in root.rglob("*.md") + if not IGNORED_DIRECTORIES.intersection(path.relative_to(root).parts) + ) + + +def _link_target(raw_target: str) -> str: + target = raw_target.strip() + if target.startswith("<") and ">" in target: + return target[1 : target.index(">")] + return target.split(maxsplit=1)[0] + + +def validate_links(path: Path, root: Path) -> None: + text = path.read_text(encoding="utf-8") + for raw_target in LINK_RE.findall(text): + target = _link_target(raw_target) + parsed = urlsplit(target) + if parsed.scheme or parsed.netloc or target.startswith(("#", "mailto:")): + continue + relative = unquote(parsed.path) + if not relative: + continue + resolved = (path.parent / relative).resolve() + try: + resolved.relative_to(root.resolve()) + except ValueError as exc: + raise DocumentationError( + f"{path.relative_to(root)}: link escapes repository: {target}" + ) from exc + if not resolved.exists(): + raise DocumentationError( + f"{path.relative_to(root)}: unresolved relative link: {target}" + ) + + +def shell_blocks(path: Path, root: Path) -> list[tuple[int, str]]: + blocks: list[tuple[int, str]] = [] + open_line: int | None = None + language = "" + content: list[str] = [] + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + match = FENCE_RE.match(line) + if match: + if open_line is None: + open_line = number + language = match.group(1).strip().lower() + content = [] + else: + if language in SHELL_LANGUAGES: + blocks.append((open_line, "\n".join(content))) + open_line = None + language = "" + content = [] + continue + if open_line is not None: + content.append(line) + if open_line is not None: + raise DocumentationError( + f"{path.relative_to(root)}:{open_line}: unclosed Markdown fence" + ) + return blocks + + +def validate_command_boundaries(path: Path, root: Path) -> None: + relative = path.relative_to(root) + blocks = shell_blocks(path, root) + if blocks and relative not in RUNNABLE_COMMAND_DOCS: + raise DocumentationError( + f"{relative}: shell commands appear outside supported command documentation" + ) + for line_number, block in blocks: + for offset, line in enumerate(block.splitlines()): + if re.search(r"(? None: + for path in markdown_files(root): + validate_links(path, root) + validate_command_boundaries(path, root) + + illustrative_opa = ( + root / "examples" / "layer5-policy-engines" / "opa-rego" / "README.md" + ).read_text(encoding="utf-8") + required_notice = "illustrative, non-canonical, and unsupported as a runnable walkthrough" + if required_notice not in illustrative_opa: + raise DocumentationError( + "the non-canonical OPA example must retain its unsupported-walkthrough notice" + ) + + +def main() -> int: + try: + validate_repository() + except (DocumentationError, OSError, UnicodeError) as exc: + print(f"documentation validation failed: {exc}", file=sys.stderr) + return 1 + print("F7-LAS documentation validation PASSED.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_agents_basic.py b/tests/test_agents_basic.py deleted file mode 100644 index 9520456..0000000 --- a/tests/test_agents_basic.py +++ /dev/null @@ -1,3 +0,0 @@ -def test_f7_las_placeholder(): - """Placeholder test to keep CI green until real tests are added.""" - assert True diff --git a/tests/test_documentation.py b/tests/test_documentation.py new file mode 100644 index 0000000..7780d8d --- /dev/null +++ b/tests/test_documentation.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import importlib.util +import os +from pathlib import Path +import subprocess +import sys + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "validate-documentation.py" +SPEC = importlib.util.spec_from_file_location("validate_documentation", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def test_repository_documentation_is_consistent() -> None: + MODULE.validate_repository(ROOT) + + +def test_local_virtual_environment_is_not_documentation(tmp_path: Path) -> None: + ignored = tmp_path / ".venv" / "dependency" / "README.md" + ignored.parent.mkdir(parents=True) + ignored.write_text("```bash\necho not-repository-docs\n```\n", encoding="utf-8") + assert MODULE.markdown_files(tmp_path) == [] + + +def test_unresolved_relative_link_is_rejected(tmp_path: Path) -> None: + document = tmp_path / "README.md" + document.write_text("[missing](missing.md)\n", encoding="utf-8") + with pytest.raises(MODULE.DocumentationError, match="unresolved relative link"): + MODULE.validate_links(document, tmp_path) + + +def test_unclosed_fence_is_rejected(tmp_path: Path) -> None: + document = tmp_path / "README.md" + document.write_text("```bash\npython -m pytest -q\n", encoding="utf-8") + with pytest.raises(MODULE.DocumentationError, match="unclosed Markdown fence"): + MODULE.shell_blocks(document, tmp_path) + + +def test_shell_commands_in_illustrative_doc_are_rejected(tmp_path: Path) -> None: + document = tmp_path / "illustrative.md" + document.write_text("```bash\necho unsupported\n```\n", encoding="utf-8") + with pytest.raises(MODULE.DocumentationError, match="outside supported"): + MODULE.validate_command_boundaries(document, tmp_path) + + +def test_bare_pytest_command_is_rejected(tmp_path: Path) -> None: + document = tmp_path / "README.md" + document.write_text("```bash\npytest -q\n```\n", encoding="utf-8") + with pytest.raises(MODULE.DocumentationError, match="python -m pytest"): + MODULE.validate_command_boundaries(document, tmp_path) + + +def test_clean_user_gate_rejects_substituted_opa(tmp_path: Path) -> None: + fake_opa = tmp_path / "opa" + fake_opa.write_text("#!/bin/sh\necho 'Version: 1.20.2'\n", encoding="utf-8") + fake_opa.chmod(0o755) + result = subprocess.run( + [str(ROOT / "scripts" / "run-clean-user-acceptance.sh")], + cwd=ROOT, + env={ + **os.environ, + "OPA_BIN": str(fake_opa), + "PYTHON_BIN": sys.executable, + }, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 1 + assert "OPA binary digest does not match" in result.stderr