fix(compliance): make framework reports evidence-based and non-certifying

.github/scripts/validate_mapping_pack.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""Validate compliance mapping-pack semantics for every framework file.
+
+Every control must carry evidence-based mapping metadata, not just exist in
+the file (issue #302) - this is what stops a future PR from force-mapping a
+rule into a framework with no rationale for the relationship, or leaving a
+synthetic non-mapping (a rule this repository could not actually tie to a
+numbered framework control) misclassified as if it were direct technical
+evidence.
+
+Runnable standalone (invoked from CI) or imported for unit testing against
+fixture directories.
+"""
+
+import json
+import re
+import sys
+from datetime import date
+from pathlib import Path
+from typing import Any, Dict, List
+
+VALID_MAPPING_TYPES = {"direct", "supporting", "organizational", "not_applicable"}
+VALID_REVIEW_STATUSES = {"pending_review", "reviewed"}
+VALID_PACK_STATUSES = {"current", "legacy"}
+REQUIRED_PACK_FIELDS = (
+    "mapping_pack_version",
+    "mapping_pack_status",
+    "mapping_pack_source",
+    "mapping_pack_published",
+)
+
+_SEMVER_RE = re.compile(r"^\d+\.\d+\.\d+$")
+
+# A control ID this repository itself invented because the framework has no
+# corresponding numbered control ("N/A-<category>-<number>"), or a control
+# whose own name/description/rationale explicitly disclaims a real mapping,
+# cannot be reported as "direct" technical evidence for that framework - a
+# scan of a rule with no target control is not evidence for a control that
+# does not exist. Checking substrings rather than only the ID prefix catches
+# the same disclaimer being made in prose without the N/A- convention.
+_NON_MAPPING_ID_PREFIX = "n/a"
+_NON_MAPPING_PHRASES = ("not mapped", "not directly mapped", "no direct mapping", "not applicable")
+
+
+def _is_disclaimed_non_mapping(control: Dict[str, Any]) -> bool:
+    control_id = str(control.get("control_id", "")).strip().lower()
+    if control_id.startswith(_NON_MAPPING_ID_PREFIX):
+        return True
+    haystack = " ".join(str(control.get(field, "")) for field in ("control_name", "description", "rationale")).lower()
+    return any(phrase in haystack for phrase in _NON_MAPPING_PHRASES)
+
+
+def _is_valid_iso_date(value: Any) -> bool:
+    if not isinstance(value, str):
+        return False
+    try:
+        date.fromisoformat(value)
+        return True
+    except ValueError:
+        return False
+
+
+def validate_framework_dir(framework_dir: "str | Path") -> List[str]:
+    """Return a list of human-readable failure strings; empty means valid."""
+    failures: List[str] = []
+    framework_dir = Path(framework_dir)
+
+    for fpath in sorted(framework_dir.glob("*.json")):
+        fname = fpath.name
+        try:
+            with open(fpath) as f:
+                data = json.load(f)
+        except (json.JSONDecodeError, OSError) as e:
+            failures.append(f"{fname}: could not parse - {e}")
+            continue
+
+        for field in REQUIRED_PACK_FIELDS:
+            if not data.get(field):
+                failures.append(f"{fname}: missing or empty top-level '{field}'")
+
+        version = data.get("mapping_pack_version")
+        if version is not None and not (isinstance(version, str) and _SEMVER_RE.match(version)):
+            failures.append(f"{fname}: mapping_pack_version '{version}' is not a valid semantic version (X.Y.Z)")
+
+        published = data.get("mapping_pack_published")
+        if published is not None and not _is_valid_iso_date(published):
+            failures.append(f"{fname}: mapping_pack_published '{published}' is not a valid ISO date (YYYY-MM-DD)")
+
+        status = data.get("mapping_pack_status")
+        if status is not None and status not in VALID_PACK_STATUSES:
+            failures.append(f"{fname}: mapping_pack_status '{status}' not in {sorted(VALID_PACK_STATUSES)}")
+
+        for rule_id, control in data.get("controls", {}).items():
+            prefix = f"{fname}:{rule_id}"
+
+            mapping_type = control.get("mapping_type")
+            if mapping_type not in VALID_MAPPING_TYPES:
+                failures.append(f"{prefix}: mapping_type '{mapping_type}' not in {sorted(VALID_MAPPING_TYPES)}")
+
+            for field in ("evidence_type", "primary_source", "rationale"):
+                value = control.get(field)
+                if not isinstance(value, str) or not value.strip():
+                    failures.append(f"{prefix}: '{field}' must be a non-empty string")
+
+            review_status = control.get("review_status")
+            if review_status not in VALID_REVIEW_STATUSES:
+                failures.append(f"{prefix}: review_status '{review_status}' not in {sorted(VALID_REVIEW_STATUSES)}")
+
+            owner = control.get("owner")
+            if owner is not None and not (isinstance(owner, str) and owner.strip()):
+                failures.append(f"{prefix}: 'owner' must be null or a non-empty string")
+
+            review_date = control.get("review_date")
+            if review_date is not None and not (isinstance(review_date, str) and review_date.strip()):
+                failures.append(f"{prefix}: 'review_date' must be null or a non-empty ISO date string")
+
+            # A control cannot be marked reviewed without an accountable owner
+            # and a review date - otherwise "reviewed" is unverifiable.
+            if review_status == "reviewed" and (not owner or not review_date):
+                failures.append(f"{prefix}: review_status is 'reviewed' but owner and/or review_date is missing")
+
+            # not_applicable/organizational controls must not claim to be
+            # measured by an automated scan.
+            if (
+                mapping_type in ("not_applicable", "organizational")
+                and control.get("evidence_type") == "automated_configuration_scan"
+            ):
+                failures.append(
+                    f"{prefix}: mapping_type '{mapping_type}' cannot have evidence_type 'automated_configuration_scan'"
+                )
+
+            # A control this repository itself marked as having no real
+            # framework counterpart (synthetic N/A-* ID, or its own text
+            # disclaiming a direct mapping) cannot be reported as direct
+            # technical evidence - that is exactly the overstatement issue
+            # #302 exists to close.
+            if mapping_type == "direct" and _is_disclaimed_non_mapping(control):
+                failures.append(
+                    f"{prefix}: mapping_type is 'direct' but control_id/name/description/rationale disclaims a "
+                    "real mapping (N/A-* ID or 'not mapped'/'no direct mapping' text) - must be 'not_applicable'"
+                )
+
+    return failures
+
+
+def main() -> int:
+    framework_dir = sys.argv[1] if len(sys.argv) > 1 else "compliance/frameworks"
+    print(f"=== Validating compliance mapping-pack semantics in {framework_dir} ===")
+    failures = validate_framework_dir(framework_dir)
+    if failures:
+        print("COMPLIANCE MAPPING SEMANTICS FAILURES:")
+        for f in failures:
+            print(f"  - {f}")
+        return 1
+    print("All compliance mappings carry valid mapping-pack semantics.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/ci.yml

@@ -367,6 +367,19 @@ jobs:
               print(f"All compliance controls map to existing rule files. ({len(existing_ids)} rules checked)")
           PYEOF
 
+      # ── CHECK 8: Compliance mapping-pack semantics validation ─────────
+      # Every control must carry evidence-based mapping metadata, not just
+      # exist in the file (issue #302). This is what actually stops a
+      # future PR from force-mapping a rule into a framework with no
+      # rationale for the relationship, or leaving a synthetic non-mapping
+      # misclassified as direct technical evidence. Logic lives in
+      # .github/scripts/validate_mapping_pack.py so it's unit-testable
+      # (tests/test_mapping_pack_validation.py) rather than only exercised
+      # here, embedded in workflow YAML.
+      - name: Compliance mapping semantics validation
+        id: mapping_semantics_check
+        run: python .github/scripts/validate_mapping_pack.py
+
   # ── Secret scanning (Gitleaks CLI, no gitleaks-action license needed) ─────
   secret-scan:
     name: Secret Scan (Gitleaks)
@@ -645,6 +658,25 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
 
+      - name: Check Alembic revision graph has exactly one head
+        env:
+          DATABASE_URL: "postgresql://ci:ci@localhost:5432/ci_db"
+        # A migration branched from the same down_revision as another
+        # unmerged PR's migration (a fork) is invisible to a single PR's own
+        # CI run - each PR only has its own migration file, so `alembic
+        # upgrade head` succeeds in isolation. It's only detectable once both
+        # land on the same branch. This check catches that state explicitly
+        # instead of leaving `alembic upgrade head` to fail unhelpfully on
+        # whichever branch merges second.
+        run: |
+          HEAD_COUNT=$(alembic heads | grep -c '(head)')
+          if [ "$HEAD_COUNT" -ne 1 ]; then
+            echo "Alembic revision graph has $HEAD_COUNT heads, expected exactly 1:"
+            alembic heads
+            exit 1
+          fi
+          echo "Alembic revision graph has exactly one head."
+
       - name: Apply database migrations
         env:
           DATABASE_URL: "postgresql://ci:ci@localhost:5432/ci_db"
@@ -689,6 +721,9 @@ jobs:
       - name: Run severity contract tests
         run: npm run test:severity
 
+      - name: Run score/compliance null-state tests
+        run: node src/utils/api.test.mjs
+
       - name: Run accessibility and internationalization checks
         run: npm run test:a11y && npm run test:i18n
 

CHANGELOG.md

@@ -1,9 +1,5 @@
 # Changelog
 
-## Unreleased
-
-- Add all ten evidence-rich enterprise network and perimeter controls `AZ-NET-018` through `AZ-NET-027` for issue #253, preserving API failures and incomplete data as indeterminate.
-
 All notable changes to OpenShield are documented in this file.
 
 The format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
@@ -13,6 +9,7 @@ OpenShield uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### Added
 
+- Ten evidence-rich enterprise network and perimeter controls `AZ-NET-018` through `AZ-NET-027` for issue #253, preserving API failures and incomplete data as indeterminate
 - Azure Network Layer Assurance API with 20-domain coverage, network-rule classification, and authoritative IP forwarding and direct Internet route checks
 - Azure Resource Graph inventory snapshots as the first OpenShield Evidence Graph foundation
 - Azure Data Link Layer Assurance API with LLC and MAC coverage plus ExpressRoute Direct MACsec checks

README.md

@@ -94,6 +94,7 @@ Project policies and assurance evidence:
 - [Support and upgrade policy](SUPPORT.md)
 - [Security requirements](docs/security-requirements.md) and [security assurance case](docs/security-assurance-case.md)
 - [Release security](docs/release-security.md) and [accessibility/i18n policy](docs/accessibility-and-i18n.md)
+- [Compliance mapping pack](docs/compliance-mapping-pack.md) — supported framework editions, mapping-pack versioning, and why compliance reports are evidence coverage, not certification
 - [OpenSSF Silver evidence register](docs/openssf-silver-evidence.md)
 
 ---

alembic/versions/3a76ff935bf6_add_scan_compliance_mapping_snapshot.py

@@ -0,0 +1,48 @@
+"""Add compliance_mapping_snapshot to scans.
+
+Revision ID: 3a76ff935bf6
+Revises: d8e4f6a1b2c3
+Create Date: 2026-08-22 00:00:00.000000
+
+This migration and PR #308's (severity contract v1, d8e4f6a1b2c3) were both
+cut from the same parent, c7a2e9f1b3d4, which would have forked the Alembic
+revision graph if both merged independently. Pointing down_revision at
+d8e4f6a1b2c3 ahead of time was tried and reverted earlier in this PR's
+history: Alembic resolves the full revision map from the files present in
+the branch it's run against, so a revision that only existed on #308's
+still-unmerged branch broke `alembic upgrade head` in this PR's own CI.
+#308 has since merged, so d8e4f6a1b2c3 now exists on `dev` and this chains
+onto it correctly - `alembic heads` returns exactly one head again.
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# Revision identifiers, used by Alembic.
+revision: str = "3a76ff935bf6"
+down_revision: Union[str, Sequence[str], None] = "d8e4f6a1b2c3"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Add a nullable JSONB snapshot of each framework's mapping-pack identity.
+
+    Populated by DatabaseManager.save_scan() at scan-completion time, so a
+    historical compliance report can show the framework name, edition,
+    mapping-pack version and source that were actually in effect for that
+    scan instead of reinterpreting it with whatever mapping pack is deployed
+    now (issue #302).
+    """
+    op.add_column(
+        "scans",
+        sa.Column("compliance_mapping_snapshot", postgresql.JSONB(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    """Drop the compliance mapping snapshot column."""
+    op.drop_column("scans", "compliance_mapping_snapshot")