diff --git a/.github/scripts/update_learn_page.py b/.github/scripts/update_learn_page.py deleted file mode 100644 index 747b6d1f..00000000 --- a/.github/scripts/update_learn_page.py +++ /dev/null @@ -1,400 +0,0 @@ -#!/usr/bin/env python3 -"""Refresh the rule/playbook/severity statistics in the Learn page and README. - -docs/learn/index.html hardcodes rule, playbook, severity and category counts -in five places: the headline metric tiles, the hero terminal line, the -pipeline step, the rules-section title/intro, the severity-box grid, and the -"coverage by category" chart. README.md hardcodes the same rule and playbook -counts in its feature table and in two nodes of its Mermaid architecture -diagram. This script recomputes every count from scanner/rules/ and -playbooks/cli/ and rewrites both files in place, so neither can drift as -rules are added. - -Every substitution is tracked. If a pattern matches zero times — because the -surrounding wording changed — the script fails loudly instead of silently -leaving the file unchanged and exiting 0. - -Running the script against already-current files produces no changes, which -lets CI commit only on a real diff. -""" - -import html -import re -import sys -from pathlib import Path -from typing import Dict, List, Tuple - -REPO_ROOT = Path(__file__).resolve().parents[2] -RULES_DIR = REPO_ROOT / "scanner" / "rules" -PLAYBOOKS_DIR = REPO_ROOT / "playbooks" / "cli" -LEARN_PAGE = REPO_ROOT / "docs" / "learn" / "index.html" -README_PATH = REPO_ROOT / "README.md" - -# Rule modules are named az__.py. Matching on that prefix is -# the same convention .github/workflows/ci.yml uses to discover rules, and it -# correctly excludes __init__.py and the shared _*_common.py helpers. -RULE_GLOB = "az_*.py" - -SEVERITY_PATTERN = re.compile(r"^SEVERITY\s*=\s*[\"']([^\"']+)[\"']", re.MULTILINE) -CATEGORY_PATTERN = re.compile(r"^CATEGORY\s*=\s*[\"']([^\"']+)[\"']", re.MULTILINE) -RULE_ID_PATTERN = re.compile(r"^RULE_ID\s*=\s*[\"']([^\"']+)[\"']", re.MULTILINE) - -# Counts that are not derived from the filesystem. -COMPLIANCE_FRAMEWORK_COUNT = 4 # CIS, NIST, ISO 27001, SOC 2 -AI_SECURITY_SKILL_COUNT = 8 - - -def count_rules() -> int: - """Return the number of scanner rule modules that declare a RULE_ID.""" - if not RULES_DIR.is_dir(): - return 0 - return sum(1 for path in RULES_DIR.glob(RULE_GLOB) if RULE_ID_PATTERN.search(path.read_text(encoding="utf-8"))) - - -def find_matching_playbooks() -> Tuple[int, List[str], List[str]]: - """Return (playbook_count, missing_for_rules, orphan_playbook_files). - - Counting every *.sh file in playbooks/cli/ (the previous approach) also - picks up shared helper scripts - e.g. review_enterprise_resilience.sh, - which several fix_*.sh wrappers `exec` into rather than a playbook any - single rule owns - silently inflating the count past rule_count and - breaking the "every rule ships with a matching playbook" claim. - - Instead this mirrors the naming convention ci.yml's playbook_check step - already enforces: playbooks/cli/fix_.sh for every - counted rule module. missing_for_rules lists rule files whose expected - playbook is absent; orphan_playbook_files lists *.sh files on disk that - no counted rule expects (informational - shared helpers are expected to - show up here and are not themselves an error). - """ - if not RULES_DIR.is_dir() or not PLAYBOOKS_DIR.is_dir(): - return 0, [], [] - - expected_names = set() - missing_for_rules: List[str] = [] - for path in sorted(RULES_DIR.glob(RULE_GLOB)): - if not RULE_ID_PATTERN.search(path.read_text(encoding="utf-8")): - continue - expected_name = f"fix_{path.stem}.sh" - expected_names.add(expected_name) - if not (PLAYBOOKS_DIR / expected_name).is_file(): - missing_for_rules.append(path.name) - - actual_names = {p.name for p in PLAYBOOKS_DIR.glob("*.sh")} - orphan_playbook_files = sorted(actual_names - expected_names) - playbook_count = len(expected_names) - len(missing_for_rules) - return playbook_count, missing_for_rules, orphan_playbook_files - - -def collect_rule_stats() -> Tuple[Dict[str, int], Dict[str, int], List[str], List[str]]: - """Parse every rule file once for its SEVERITY and CATEGORY. - - Returns (severity_counts, category_counts, files_missing_severity, - files_missing_category). Severities and categories outside the values - seen so far are still counted, keyed by whatever string the rule - declares, so a new value is never silently dropped. - - Files with no parseable RULE_ID are skipped entirely, matching - count_rules() - otherwise a non-rule file (excluded from rule_count) - could still be counted into these totals, making the severity boxes - disagree with the headline rule count. - """ - severities: Dict[str, int] = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0} - categories: Dict[str, int] = {} - missing_severity: List[str] = [] - missing_category: List[str] = [] - - if not RULES_DIR.is_dir(): - return severities, categories, missing_severity, missing_category - - for path in sorted(RULES_DIR.glob(RULE_GLOB)): - content = path.read_text(encoding="utf-8") - if not RULE_ID_PATTERN.search(content): - continue - - sev_match = SEVERITY_PATTERN.search(content) - if sev_match: - severity = sev_match.group(1).strip().upper() - severities[severity] = severities.get(severity, 0) + 1 - else: - missing_severity.append(path.name) - - cat_match = CATEGORY_PATTERN.search(content) - if cat_match: - category = cat_match.group(1).strip() - categories[category] = categories.get(category, 0) + 1 - else: - missing_category.append(path.name) - - return severities, categories, missing_severity, missing_category - - -def render_category_rows(categories: Dict[str, int]) -> str: - """Build the '
...' lines for the category chart. - - Rows are sorted by descending count (alphabetical tiebreak) so a newly - added category appears automatically instead of requiring a code change. - Bar width is each category's count as a percentage of the largest - category, matching the original hand-authored chart's convention. - """ - if not categories: - return "" - - max_count = max(categories.values()) - rows = [] - for name, count in sorted(categories.items(), key=lambda item: (-item[1], item[0])): - width = round(count / max_count * 100) - rows.append( - f'
{html.escape(name)}' - f'
' - f"{count}
" - ) - return "\n".join(rows) - - -def _metric(label: str) -> str: - """Build the pattern for one headline metric tile on the Learn page.""" - return rf'(
)\d+({re.escape(label)}
)' - - -def apply_replacements(content: str, replacements: Tuple[Tuple[str, str, int], ...]) -> Tuple[str, List[str]]: - """Apply each (name, pattern, value) substitution, tracking zero-match failures. - - A pattern that matches zero times means the surrounding wording no longer - matches what this script expects — that is reported as a failure rather - than silently leaving the file unchanged. - """ - failures: List[str] = [] - for name, pattern, value in replacements: - content, count = re.subn(pattern, rf"\g<1>{value}\g<2>", content) - if count == 0: - failures.append(name) - return content, failures - - -def render( - content: str, - rule_count: int, - playbook_count: int, - critical_count: int, - high_count: int, - medium_count: int, - low_count: int, - category_rows: str, -) -> Tuple[str, List[str]]: - """Return (updated_content, failed_pattern_names) for docs/learn/index.html.""" - intro = ( - r'(

\s*OpenShield currently has )\d+' - r"( dynamic rules\. The strongest contributor work improves rule " - r"accuracy, reduces false positives,)" - ) - pipeline = ( - r'(

Rule Evaluation)' - r"\d+( dynamic checks
)" - ) - section_title = r'(

)\d+( Azure security rules

)' - hero_terminal = r'(loading rules: )\d+( dynamic checks

)' - severity_critical = r'(
)\d+(CRITICAL
)' - severity_high = r'(
)\d+(HIGH
)' - severity_medium = r'(
)\d+(MEDIUM
)' - severity_low = r'(
)\d+(LOW
)' - - if not re.search(severity_critical, content): - severity_start = ( - r'(
\n)' - r'(\s*
)' - ) - critical_box = ( - f'
{critical_count}' - "CRITICAL
\n" - ) - content, count = re.subn( - severity_start, - lambda match: match.group(1) + critical_box + match.group(2), - content, - ) - if count == 0: - return content, ["severity box: CRITICAL"] - - replacements: Tuple[Tuple[str, str, int], ...] = ( - ("headline metric: Azure scan rules", _metric("Azure scan rules"), rule_count), - ("headline metric: CLI remediation playbooks", _metric("CLI remediation playbooks"), playbook_count), - ("headline metric: Compliance frameworks", _metric("Compliance frameworks"), COMPLIANCE_FRAMEWORK_COUNT), - ("headline metric: AI security skills", _metric("AI security skills"), AI_SECURITY_SKILL_COUNT), - ("headline metric: High-severity checks", _metric("High-severity checks"), high_count), - ("pipeline step: Rule Evaluation", pipeline, rule_count), - ("rules section title", section_title, rule_count), - ("rules section intro paragraph", intro, rule_count), - ("hero terminal: dynamic checks line", hero_terminal, rule_count), - ("severity box: CRITICAL", severity_critical, critical_count), - ("severity box: HIGH", severity_high, high_count), - ("severity box: MEDIUM", severity_medium, medium_count), - ("severity box: LOW", severity_low, low_count), - ) - - content, failures = apply_replacements(content, replacements) - - category_block = r'(
\n)(.*?)(\n {10}
)' - content, count = re.subn( - category_block, - lambda m: m.group(1) + category_rows + m.group(3), - content, - flags=re.DOTALL, - ) - if count == 0: - failures.append("coverage-by-category chart") - - return content, failures - - -def render_readme(content: str, rule_count: int, playbook_count: int) -> Tuple[str, List[str]]: - """Return (updated_content, failed_pattern_names) for README.md.""" - feature_row = ( - r"(\| \*\*Misconfiguration Scanner\*\* \| Runs )\d+" - r"( Azure security rules across storage, network, identity, database, " - r"compute, Key Vault, AKS, Kubernetes workloads, post-quantum cryptography, backup, serverless, " - r"private endpoint, and supply chain posture \|)" - ) - playbook_row = ( - r"(\| \*\*Remediation Playbooks\*\* \| Every documented rule ships with a matching " - r"review-gated remediation script \()\d+( playbooks\) \|)" - ) - mermaid_scanner = r'(C\["Scanner Engine\\n)\d+( Python rules"\])' - mermaid_playbooks = r'(G\["Azure CLI Playbooks\\n)\d+( remediation scripts"\])' - - replacements: Tuple[Tuple[str, str, int], ...] = ( - ("feature table: Misconfiguration Scanner row", feature_row, rule_count), - ("feature table: Remediation Playbooks row", playbook_row, playbook_count), - ("Mermaid diagram: Scanner Engine node", mermaid_scanner, rule_count), - ("Mermaid diagram: Azure CLI Playbooks node", mermaid_playbooks, playbook_count), - ) - - return apply_replacements(content, replacements) - - -def main() -> int: - """Rewrite the Learn page and README statistics; return a process exit code.""" - for path in (LEARN_PAGE, README_PATH): - if not path.is_file(): - print(f"Error: {path} not found", file=sys.stderr) - return 1 - - rule_count = count_rules() - playbook_count, missing_playbooks, orphan_playbooks = find_matching_playbooks() - - if rule_count == 0 or playbook_count == 0: - print( - "Error: found no rules or no playbooks; refusing to write zeroes into the docs.", - file=sys.stderr, - ) - return 1 - - if missing_playbooks: - print( - f"Error: {len(missing_playbooks)} rule(s) have no matching playbook file: {', '.join(missing_playbooks)}", - file=sys.stderr, - ) - return 1 - - # Belt-and-suspenders: find_matching_playbooks() only counts playbooks - # that resolve to a counted rule, so this should be unreachable once - # missing_playbooks is empty. Failing loudly here catches a future bug - # in that derivation rather than silently writing mismatched docs - - # this is exactly the drift class (rules and playbooks disagreeing) - # this script exists to catch. - if rule_count != playbook_count: - print( - f"Error: rule/playbook count mismatch (rules: {rule_count}, playbooks: " - f"{playbook_count}); refusing to write inconsistent docs.", - file=sys.stderr, - ) - return 1 - - if orphan_playbooks: - print( - f"Warning: {len(orphan_playbooks)} playbook file(s) in playbooks/cli/ are not " - f"any rule's matching playbook and are excluded from the playbook count " - f"(expected for shared helpers other playbooks exec into): {', '.join(orphan_playbooks)}", - file=sys.stderr, - ) - - severities, categories, missing_severity, missing_category = collect_rule_stats() - - if missing_severity: - print( - f"Warning: {len(missing_severity)} rule file(s) have no parseable SEVERITY " - f"and are excluded from the severity counts: {', '.join(missing_severity)}", - file=sys.stderr, - ) - if missing_category: - print( - f"Warning: {len(missing_category)} rule file(s) have no parseable CATEGORY " - f"and are excluded from the coverage-by-category chart: {', '.join(missing_category)}", - file=sys.stderr, - ) - - chart_severities = {"CRITICAL", "HIGH", "MEDIUM", "LOW"} - excluded_severities = { - severity: count for severity, count in severities.items() if severity not in chart_severities and count - } - if excluded_severities: - excluded_detail = ", ".join(f"{severity}: {count}" for severity, count in sorted(excluded_severities.items())) - excluded_total = sum(excluded_severities.values()) - print( - f"Warning: {excluded_total} rule(s) with severities outside the " - f"CRITICAL/HIGH/MEDIUM/LOW chart are excluded from the severity boxes: {excluded_detail}", - file=sys.stderr, - ) - - category_rows = render_category_rows(categories) - - learn_original = LEARN_PAGE.read_text(encoding="utf-8") - learn_updated, learn_failures = render( - learn_original, - rule_count, - playbook_count, - severities["CRITICAL"], - severities["HIGH"], - severities["MEDIUM"], - severities["LOW"], - category_rows, - ) - - readme_original = README_PATH.read_text(encoding="utf-8") - readme_updated, readme_failures = render_readme(readme_original, rule_count, playbook_count) - - failures = [f"docs/learn/index.html -> {name}" for name in learn_failures] - failures += [f"README.md -> {name}" for name in readme_failures] - - if failures: - print( - "Error: the following patterns matched zero times. The surrounding wording " - "has likely changed and this script needs updating to match:", - file=sys.stderr, - ) - for name in failures: - print(f" - {name}", file=sys.stderr) - return 1 - - changed: List[str] = [] - if learn_updated != learn_original: - LEARN_PAGE.write_text(learn_updated, encoding="utf-8") - changed.append(str(LEARN_PAGE.relative_to(REPO_ROOT))) - if readme_updated != readme_original: - README_PATH.write_text(readme_updated, encoding="utf-8") - changed.append(str(README_PATH.relative_to(REPO_ROOT))) - - if not changed: - print("Learn page and README statistics already current; nothing to do.") - return 0 - - print( - f"Updated {', '.join(changed)} - rules: {rule_count}, playbooks: {playbook_count}, " - f"severity CRITICAL: {severities['CRITICAL']}, HIGH: {severities['HIGH']}, " - f"MEDIUM: {severities['MEDIUM']}, LOW: {severities['LOW']}" - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.github/scripts/update_readme_stats.py b/.github/scripts/update_readme_stats.py new file mode 100644 index 00000000..28c58973 --- /dev/null +++ b/.github/scripts/update_readme_stats.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Refresh the rule and playbook counts hardcoded in README.md. + +README.md states the rule and playbook counts in its feature table and in two +nodes of its Mermaid architecture diagram. This script recomputes both counts +from scanner/rules/ and playbooks/cli/ and rewrites those four spots in place. + +The website (including the Learn page) no longer needs this: it derives every +count from the repository at build time (website/src/lib/repoData.ts), so it +cannot drift. README.md is plain Markdown rendered by GitHub, so it is the one +place a number still has to be written down. + +Usage: + python .github/scripts/update_readme_stats.py # rewrite README.md + python .github/scripts/update_readme_stats.py --check # report drift only + +Nothing commits the result automatically: `dev` only accepts changes through +pull requests. The workflow runs --check after each merge and surfaces drift +as a warning, and whoever next touches README.md (or a maintainer) runs the +script and includes the diff in their PR. + +Every substitution is tracked. If a pattern matches zero times - because the +surrounding wording changed - the script fails loudly instead of silently +leaving the file unchanged and exiting 0. +""" + +import argparse +import re +import sys +from pathlib import Path +from typing import List, Tuple + +REPO_ROOT = Path(__file__).resolve().parents[2] +RULES_DIR = REPO_ROOT / "scanner" / "rules" +PLAYBOOKS_DIR = REPO_ROOT / "playbooks" / "cli" +README_PATH = REPO_ROOT / "README.md" + +# Rule modules are named az__.py. Matching on that prefix is +# the same convention .github/workflows/ci.yml uses to discover rules, and it +# correctly excludes __init__.py and the shared _*_common.py helpers. +RULE_GLOB = "az_*.py" + +RULE_ID_PATTERN = re.compile(r"^RULE_ID\s*=\s*[\"']([^\"']+)[\"']", re.MULTILINE) + +# Exit code for --check when README.md is stale, distinct from real errors. +EXIT_STALE = 3 + + +def count_rules() -> int: + """Return the number of scanner rule modules that declare a RULE_ID.""" + if not RULES_DIR.is_dir(): + return 0 + return sum(1 for path in RULES_DIR.glob(RULE_GLOB) if RULE_ID_PATTERN.search(path.read_text(encoding="utf-8"))) + + +def find_matching_playbooks() -> Tuple[int, List[str], List[str]]: + """Return (playbook_count, missing_for_rules, orphan_playbook_files). + + Counting every *.sh file in playbooks/cli/ would also pick up shared helper + scripts - e.g. review_enterprise_resilience.sh, which several fix_*.sh + wrappers `exec` into rather than a playbook any single rule owns - silently + inflating the count past rule_count and breaking the "every rule ships with + a matching playbook" claim. + + Instead this mirrors the naming convention ci.yml's playbook_check step + already enforces: playbooks/cli/fix_.sh for every + counted rule module. missing_for_rules lists rule files whose expected + playbook is absent; orphan_playbook_files lists *.sh files on disk that + no counted rule expects (informational - shared helpers are expected to + show up here and are not themselves an error). + """ + if not RULES_DIR.is_dir() or not PLAYBOOKS_DIR.is_dir(): + return 0, [], [] + + expected_names = set() + missing_for_rules: List[str] = [] + for path in sorted(RULES_DIR.glob(RULE_GLOB)): + if not RULE_ID_PATTERN.search(path.read_text(encoding="utf-8")): + continue + expected_name = f"fix_{path.stem}.sh" + expected_names.add(expected_name) + if not (PLAYBOOKS_DIR / expected_name).is_file(): + missing_for_rules.append(path.name) + + actual_names = {p.name for p in PLAYBOOKS_DIR.glob("*.sh")} + orphan_playbook_files = sorted(actual_names - expected_names) + playbook_count = len(expected_names) - len(missing_for_rules) + return playbook_count, missing_for_rules, orphan_playbook_files + + +def apply_replacements(content: str, replacements: Tuple[Tuple[str, str, int], ...]) -> Tuple[str, List[str]]: + """Apply each (name, pattern, value) substitution, tracking zero-match failures. + + A pattern that matches zero times means the surrounding wording no longer + matches what this script expects - that is reported as a failure rather + than silently leaving the file unchanged. + """ + failures: List[str] = [] + for name, pattern, value in replacements: + content, count = re.subn(pattern, rf"\g<1>{value}\g<2>", content) + if count == 0: + failures.append(name) + return content, failures + + +def render_readme(content: str, rule_count: int, playbook_count: int) -> Tuple[str, List[str]]: + """Return (updated_content, failed_pattern_names) for README.md.""" + # Anchored on the row label and the unit that follows the number, not on + # the full prose. The wording of these rows (the category list, how the + # scripts are described) is edited independently of the counts, and pinning + # the whole sentence made this script fail on every unrelated reword while + # the counts silently went stale. Losing the row itself, or the "Azure + # security rules"/"(N playbooks)" shape, still fails loudly. + feature_row = r"(\| \*\*Misconfiguration Scanner\*\* \| Runs )\d+( Azure security rules)" + playbook_row = r"(\| \*\*Remediation Playbooks\*\* \|[^|]*\()\d+( playbooks\) \|)" + mermaid_scanner = r'(C\["Scanner Engine\\n)\d+( Python rules"\])' + mermaid_playbooks = r'(G\["Azure CLI Playbooks\\n)\d+( remediation scripts"\])' + + replacements: Tuple[Tuple[str, str, int], ...] = ( + ("feature table: Misconfiguration Scanner row", feature_row, rule_count), + ("feature table: Remediation Playbooks row", playbook_row, playbook_count), + ("Mermaid diagram: Scanner Engine node", mermaid_scanner, rule_count), + ("Mermaid diagram: Azure CLI Playbooks node", mermaid_playbooks, playbook_count), + ) + + return apply_replacements(content, replacements) + + +def main(argv: List[str] | None = None) -> int: + """Rewrite (or, with --check, verify) README.md counts; return an exit code.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--check", + action="store_true", + help=f"do not write; exit {EXIT_STALE} if README.md counts are stale", + ) + args = parser.parse_args(argv) + + if not README_PATH.is_file(): + print(f"Error: {README_PATH} not found", file=sys.stderr) + return 1 + + rule_count = count_rules() + playbook_count, missing_playbooks, orphan_playbooks = find_matching_playbooks() + + if rule_count == 0 or playbook_count == 0: + print("Error: found no rules or no playbooks; refusing to write zeroes into the docs.", file=sys.stderr) + return 1 + + if missing_playbooks: + print( + f"Error: {len(missing_playbooks)} rule(s) have no matching playbook file: {', '.join(missing_playbooks)}", + file=sys.stderr, + ) + return 1 + + # Belt-and-suspenders: find_matching_playbooks() only counts playbooks + # that resolve to a counted rule, so this should be unreachable once + # missing_playbooks is empty. Failing loudly here catches a future bug in + # that derivation rather than silently writing mismatched docs. + if rule_count != playbook_count: + print( + f"Error: rule/playbook count mismatch (rules: {rule_count}, playbooks: " + f"{playbook_count}); refusing to write inconsistent docs.", + file=sys.stderr, + ) + return 1 + + if orphan_playbooks: + print( + f"Note: {len(orphan_playbooks)} playbook file(s) in playbooks/cli/ are not any rule's " + f"matching playbook and are excluded from the count (expected for shared helpers " + f"other playbooks exec into): {', '.join(orphan_playbooks)}", + file=sys.stderr, + ) + + original = README_PATH.read_text(encoding="utf-8") + updated, failures = render_readme(original, rule_count, playbook_count) + + if failures: + print( + "Error: the following README.md patterns matched zero times. The surrounding " + "wording has likely changed and this script needs updating to match:", + file=sys.stderr, + ) + for name in failures: + print(f" - {name}", file=sys.stderr) + return 1 + + if updated == original: + print(f"README.md statistics already current (rules: {rule_count}, playbooks: {playbook_count}).") + return 0 + + if args.check: + print( + f"README.md statistics are stale; repository now has {rule_count} rules and " + f"{playbook_count} playbooks. Run: python .github/scripts/update_readme_stats.py" + ) + return EXIT_STALE + + README_PATH.write_text(updated, encoding="utf-8") + print(f"Updated README.md - rules: {rule_count}, playbooks: {playbook_count}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/readme-stats.yml b/.github/workflows/readme-stats.yml new file mode 100644 index 00000000..a9b7a61f --- /dev/null +++ b/.github/workflows/readme-stats.yml @@ -0,0 +1,61 @@ +name: README Stats Drift + +# The website (Learn page included) computes its statistics at build time from +# website/src/lib/repoData.ts, so it never needs a commit. README.md is the one +# file that still states counts, and `dev` only accepts changes through pull +# requests, so this workflow never pushes: it reports drift as a warning and +# the fix lands in an ordinary PR (python .github/scripts/update_readme_stats.py). +on: + push: + branches: [dev] + paths: + - "scanner/rules/**" + - "playbooks/cli/**" + - "README.md" + - ".github/scripts/update_readme_stats.py" + - ".github/workflows/readme-stats.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: readme-stats-${{ github.ref }} + cancel-in-progress: true + +jobs: + readme-stats: + name: Check README statistics + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - name: Set up Python 3.11 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + + - name: Check README statistics + run: | + set +e + output="$(python .github/scripts/update_readme_stats.py --check 2>&1)" + status=$? + set -e + echo "$output" + if [ "$status" -eq 3 ]; then + # Stale counts are a docs follow-up, not a broken build. + message="$(echo "$output" | tail -n 1)" + echo "::warning title=README statistics are stale::${message}" + { + echo "### README statistics are stale" + echo + echo "${message}" + echo + echo "Open a PR with the output of \`python .github/scripts/update_readme_stats.py\`." + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + exit "$status" diff --git a/.github/workflows/update-learn-page.yml b/.github/workflows/update-learn-page.yml deleted file mode 100644 index 173323eb..00000000 --- a/.github/workflows/update-learn-page.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Update Learn Page and README Stats - -on: - push: - branches: [dev] - -# dev is protected, so the refreshed statistics are proposed through a pull -# request instead of being pushed to dev directly (issue #298). contents:write -# covers the bot branch; pull-requests:write opens or updates the PR. -permissions: - contents: write - pull-requests: write - -concurrency: - group: update-learn-page-${{ github.ref }} - cancel-in-progress: true - -jobs: - update-learn-page: - name: Refresh Learn page and README statistics - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - # The bot branch push must use the same token as the PR so CI starts. - token: ${{ secrets.STATS_BOT_TOKEN || github.token }} - - - name: Set up Python 3.11 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.11" - - - name: Refresh statistics - run: python .github/scripts/update_learn_page.py - - - name: Detect changes - id: diff - run: | - if git diff --quiet -- docs/learn/index.html README.md; then - echo "changed=false" >> "$GITHUB_OUTPUT" - else - echo "changed=true" >> "$GITHUB_OUTPUT" - fi - - - name: Propose refreshed statistics - if: steps.diff.outputs.changed == 'true' - env: - BASE_REF: ${{ github.ref_name }} - BOT_BRANCH: docs/refresh-learn-page-stats - # Pull requests opened with the default GITHUB_TOKEN do not start - # workflows, so required checks never report. A maintainer-provided - # STATS_BOT_TOKEN (GitHub App or fine-grained token with contents and - # pull-requests write) lets CI run on the bot PR like any other. - GH_TOKEN: ${{ secrets.STATS_BOT_TOKEN || github.token }} - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git switch -c "$BOT_BRANCH" - git add docs/learn/index.html README.md - git commit -s -m "docs: refresh learn page and README statistics" - git push --force origin "HEAD:refs/heads/$BOT_BRANCH" - if [ -z "$(gh pr list --head "$BOT_BRANCH" --base "$BASE_REF" --state open --json number --jq '.[].number')" ]; then - gh pr create --base "$BASE_REF" --head "$BOT_BRANCH" \ - --title "docs: refresh learn page and README statistics" \ - --body "Automated refresh generated by \`.github/scripts/update_learn_page.py\` from \`$BASE_REF\` at $GITHUB_SHA. Review and merge through the normal protected-branch flow." - fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e63b48d..6d5bc3c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ OpenShield uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixed +- Learn page statistics are computed at site build time, and the post-merge stats workflow no longer pushes to the protected `dev` branch (it failed on every merge); README count drift is now reported as a warning - High-severity CodeQL findings in Python and JavaScript code - Security findings identified during Semgrep analysis - Sensitive identity metadata removed from scanner debug logging diff --git a/README.md b/README.md index 48ece101..0795e115 100644 --- a/README.md +++ b/README.md @@ -74,10 +74,10 @@ Findings map to NIST FIPS 203 (ML-KEM), FIPS 204 (ML-DSA), and FIPS 205 (SLH-DSA | Feature | Description | |---|---| -| **Misconfiguration Scanner** | Runs 95 Azure security rules across storage, network, identity, database, compute, Key Vault, AKS, Kubernetes workloads, post-quantum cryptography, backup, serverless, private endpoint, and supply chain posture | +| **Misconfiguration Scanner** | Runs 144 Azure security rules across storage, network, identity, database, compute, Key Vault, AKS, Kubernetes workloads, post-quantum cryptography, backup, serverless, private endpoint, and supply chain posture | | **Compliance Mapper** | Maps findings to CIS Benchmarks, NIST CSF, ISO 27001, and SOC 2 framework JSON files | | **Scan History API** | Stores scans and findings in PostgreSQL and exposes findings, score, scan history, compliance posture, drift, and resource inventory over REST | -| **Remediation Playbooks** | Every documented rule ships with a matching review-gated remediation script (95 playbooks) | +| **Remediation Playbooks** | Every documented rule ships with a matching review-gated remediation script (144 playbooks) | | **Security Dashboard** | Full React dashboard deployed on Vercel - live monitoring, findings, compliance, drift, prioritization, and AI-layer views | | **Project Website** | Documentation and reference site at [owasp.github.io/openshield](https://owasp.github.io/openshield/) - blog, rules gallery, architecture, evidence guides, roadmap, and releases | | **Sentinel Integration** | Normalises findings and pushes them into Microsoft Sentinel via a Log Analytics custom table and KQL analytics rules | @@ -119,11 +119,11 @@ Project policies and assurance evidence: flowchart TD A["React Dashboard\nVercel · Live"] B["Flask REST API\nJWT · CORS · Blueprints"] - C["Scanner Engine\n95 Python rules"] + C["Scanner Engine\n144 Python rules"] D["Azure Subscription\nScanned via Azure SDK + Graph"] E["Compliance Framework JSON\nCIS · NIST · ISO 27001 · SOC 2"] F["PostgreSQL Database\nFindings · Scans"] - G["Azure CLI Playbooks\n95 remediation scripts"] + G["Azure CLI Playbooks\n144 remediation scripts"] H["sentinel/ingest.py\nNormalise + HMAC upload"] I["Microsoft Sentinel\nOpenShieldFindings_CL · KQL rules"] @@ -330,7 +330,7 @@ Learn OpenShield covers: - Contributor onboarding - Documentation navigation -Live Learning Portal: https://openshieldlearn.netlify.app/learn/ +Live Learning Portal: https://owasp.github.io/openshield/learn/ Full documentation, the security rules gallery, architecture guide, evidence guide, and blog are available at the project website: **[owasp.github.io/openshield](https://owasp.github.io/openshield/)** diff --git a/docs/_redirects b/docs/_redirects deleted file mode 100644 index fe0b625a..00000000 --- a/docs/_redirects +++ /dev/null @@ -1 +0,0 @@ -/ /learn/ 302 diff --git a/docs/ci-pipeline.md b/docs/ci-pipeline.md index d02d2d66..ce4da191 100644 --- a/docs/ci-pipeline.md +++ b/docs/ci-pipeline.md @@ -213,7 +213,7 @@ There is no standing bypass. If a production incident requires merging without t CI and CodeQL also run on every push to `dev` and `main`, so each merged commit has its own result instead of relying only on a PR run against an older base. Post-merge runs are never cancelled. -The **Update Learn Page and README Stats** workflow no longer pushes to `dev`. When statistics change it force-updates the `docs/refresh-learn-page-stats` branch and opens (or refreshes) a pull request, which goes through the same protected flow. Pull requests opened with the default `GITHUB_TOKEN` do not start workflows, so administrators should provide a `STATS_BOT_TOKEN` secret (GitHub App token or fine-grained token with contents and pull-requests write) and allow GitHub Actions to create pull requests in repository settings. +Nothing pushes statistics to `dev` or opens bot pull requests. The website, including the Learn page, computes every rule, playbook, severity and category count at build time from `website/src/lib/repoData.ts`, so it cannot drift. `README.md` is the only file that still states counts: the **README Stats Drift** workflow (`readme-stats.yml`, `contents: read`) runs `python .github/scripts/update_readme_stats.py --check` after rule or playbook changes land and reports stale counts as a warning with a job summary. The fix is an ordinary PR with the script's output, reviewed through the same protected flow. No `STATS_BOT_TOKEN` secret is needed. --- diff --git a/docs/learn/index.html b/docs/learn/index.html deleted file mode 100644 index 745f276e..00000000 --- a/docs/learn/index.html +++ /dev/null @@ -1,1043 +0,0 @@ - - - - - - - OpenShield Learn - - - - - - - -
-
-
-
Open-source Azure CSPM platform
-

Learn Azure security posture with OpenShield.

-

- OpenShield scans Azure subscriptions for misconfigurations, enriches findings with CVE intelligence, - maps risks to compliance frameworks, stores scan history, exposes a Flask API, and presents results through - a React dashboard with demo and live modes. -

- -

Static learning hub. No backend, no login, no fake upload flows.

-
- - -
- -
-
95Azure scan rules
-
95CLI remediation playbooks
-
4Compliance frameworks
-
8AI security skills
-
58High-severity checks
-
-
- -
-
-
-
-

Overview

-

What OpenShield does

-

- OpenShield is built to help users identify risky Azure configurations, understand the impact, connect findings - to compliance controls, and follow practical remediation guidance. It is not a cloud provider replacement or a SIEM; - it is a focused Azure CSPM platform for posture visibility and learning. -

-
-
- -
-
-

Misconfiguration scanning

-

Dynamic Python rule modules inspect Azure resources through Azure SDK clients and return normalized security findings.

-
ScannerAzure SDKRules
-
-
-

CVE enrichment

-

Findings can be enriched with NVD/CVE context so security issues are easier to prioritize and explain.

-
NVDCVERisk context
-
-
-

Compliance mapping

-

Technical findings are mapped to CIS Azure, NIST CSF, ISO 27001, and SOC 2 for governance-oriented reporting.

-
CISNISTISO 27001SOC 2
-
-
-

Remediation guidance

-

Each rule is paired with a CLI playbook so contributors and users can move from detection to manual remediation.

-
Azure CLIPlaybooksValidation
-
-
-
- -
-

Architecture

-

Production-shaped, MVP-friendly architecture

-

- The platform follows a simple pipeline: Azure credentials are resolved by DefaultAzureCredential, the scan engine loads - rule files from scanner/rules/*.py, findings are enriched and stored, then exposed through the API and dashboard. -

- -
-
Azure SubscriptionResources and configuration
-
Scanner EnginePython rule execution
-
Rule Evaluation95 dynamic checks
-
CVE EnrichmentNVD risk context
-
PostgreSQLFindings and scan history
-
Flask APIJWT-protected REST routes
-
React DashboardDemo and live modes
-
Sentinel / AIKQL, RAG, insights
-
- -
-

Scanner

Core engine, Azure SDK wrapper, NVD/CVE enrichment, and auto-loaded rule files.

-

API

Flask REST API with JWT authentication, CORS, migrations, scans, findings, score, compliance, and AI routes.

-

Frontend

Vite, React, and Tailwind dashboard covering monitoring, discovery, prioritization, compliance, drift, and AI.

-

AI

RAG knowledge pipeline, ChromaDB vector store builder, retriever, and cloud-security knowledge skills.

-

Sentinel

Optional Log Analytics ingestion plus KQL analytics rules for detection workflows.

-

CI and docs

Checks syntax, secrets, rule structure, playbooks, compliance JSON, API syntax, and cross-references.

-
-
- -
-

Rule coverage

-

95 Azure security rules

-

- OpenShield currently has 95 dynamic rules. The strongest contributor work improves rule accuracy, reduces false positives, - strengthens validation, or improves remediation quality. -

- -
-
-

Coverage by category

-
-
Network
23
-
Identity
15
-
Security Operations
10
-
Supply Chain
8
-
KeyVault
6
-
Kubernetes
6
-
Serverless
5
-
Storage
5
-
Backup
4
-
Compute
4
-
Database
4
-
PostQuantum
3
-
Data Link
2
-
-
- -
-

Severity distribution

-

Most checks are high severity. That makes validation important: high-severity false positives damage trust quickly.

-
-
0CRITICAL
-
58HIGH
-
31MEDIUM
-
4LOW
-
-
-
-
- -
-

Learning roadmap

-

Recommended learning path

-

- Follow this path if you are new to OpenShield or preparing to contribute. Learn the security problem before touching code. -

- - -
- - - -
-

Contributors

-

Where contributors can help

-

- Good contributions should improve detection accuracy, correctness of findings, remediation quality, documentation clarity, - or system reliability. Cosmetic work is useful only when it supports those goals. -

- -
-

Rules

Add or improve Azure checks with accurate metadata, safe SDK usage, realistic test cases, and clear findings.

-

Playbooks

Keep remediation scripts aligned with rules. Every fix should include validation and avoid unsafe blanket changes.

-

Compliance

Improve CIS, NIST, ISO 27001, and SOC 2 mappings. Do not map controls just to inflate coverage.

-

Frontend

Connect live API flows carefully. Do not leave mock-backed UI pretending to be production data.

-

Backend

Implement missing endpoints consistently with JWT auth, error handling, data contracts, and PostgreSQL models.

-

AI and Sentinel

Improve RAG quality, knowledge loading, KQL rules, and ingestion without exposing sensitive findings unnecessarily.

-
-
- -
-

Known gaps

-

Current cleanup items

-

- These are not failures; they are useful follow-up targets. Documenting them prevents contributors from pretending the platform is more complete than it is. -

- -
-
-

Documentation drift

-
    -
  • Rule, playbook, and severity statistics on this page and in README.md are generated from scanner/rules/ and playbooks/cli/ by .github/scripts/update_learn_page.py on every push to dev.
  • -
  • Some startup commands assume python, but local environments may only expose python3.
  • -
  • API docs and implementation should stay aligned, especially score response shape.
  • -
-
-
-

Implementation gaps

-
    -
  • Some frontend live pages depend on endpoints that may still be mock-backed.
  • -
  • Examples include resources, drift, prioritization, and finding-specific playbook routes.
  • -
  • Fix syntax issues before claiming AI pipeline readiness.
  • -
-
-
-
- -
-

Documentation

-

Useful repo documents

-

- These relative links are intentionally static-hosting friendly when this file is served from the docs learning folder. - Adjust paths if the Learn page is moved. -

- -
-
-
ArchitectureSystem design, scanner flow, platform components, and storage/API structure.
- Open -
-
-
API ReferenceBackend routes for scans, findings, score, compliance, and AI-related data.
- Open -
-
-
Azure SetupEnvironment variables, Azure credentials, and setup requirements for live scans.
- Open -
-
-
Rules ReferenceRule metadata, categories, severity, expected output, and implementation guidance.
- Open -
-
-
Adding a RuleContributor workflow for implementing, testing, and documenting a new check.
- Open -
-
-
CI PipelineLocal and GitHub Actions checks for rules, playbooks, compliance JSON, and API syntax.
- Open -
-
-
CVE CorrelationNVD enrichment, CVSS scoring, exploit availability, and dashboard-ready CVE fields.
- Open -
-
-
Sentinel SetupLog Analytics ingestion, OpenShield findings table setup, and KQL analytics rules.
- Open -
-
-
API Render DeployRender deployment test plan, smoke testing, and production JWT requirements.
- Open -
-
-
AZ-STOR-003 Test PlanLifecycle management policy rule test setup, execution, remediation, and validation.
- Open -
-
-
- -
- Note: OpenShield Learn is a documentation and learning portal. Features such as authentication, file uploads, scan execution, and data persistence require backend services and are intentionally not implemented in this static site. -
-
- -
- -
- - - - diff --git a/tests/test_update_readme_stats.py b/tests/test_update_readme_stats.py new file mode 100644 index 00000000..3c75dc48 --- /dev/null +++ b/tests/test_update_readme_stats.py @@ -0,0 +1,83 @@ +"""Focused tests for the README statistics script.""" + +import importlib.util +from pathlib import Path + + +_SCRIPT = Path(__file__).parents[1] / ".github" / "scripts" / "update_readme_stats.py" +_SPEC = importlib.util.spec_from_file_location("update_readme_stats", _SCRIPT) +assert _SPEC and _SPEC.loader +stats = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(stats) + + +def _readme(scanner_prose: str, playbook_prose: str, count: int = 5) -> str: + return ( + f"| **Misconfiguration Scanner** | Runs {count}{scanner_prose} |\n" + f"| **Remediation Playbooks** | {playbook_prose} ({count} playbooks) |\n" + f' C["Scanner Engine\\n{count} Python rules"]\n' + f' G["Azure CLI Playbooks\\n{count} remediation scripts"]\n' + ) + + +def test_render_readme_survives_reworded_feature_rows(): + """The counts, not the surrounding prose, are what this script owns. + + Both rows are edited independently of the counts (the category list grows, + the scripts get re-described). Pinning the whole sentence made the script + fail on an unrelated reword while the counts silently went stale, which is + exactly the drift it exists to prevent. + """ + reworded = _readme( + " Azure security rules across storage, network, and a category added later", + "Every documented rule ships with a matching review-gated remediation script", + ) + content, failures = stats.render_readme(reworded, 143, 143) + assert failures == [] + assert "Runs 143 Azure security rules" in content + assert "(143 playbooks)" in content + assert 'C["Scanner Engine\\n143 Python rules"]' in content + assert 'G["Azure CLI Playbooks\\n143 remediation scripts"]' in content + + +def test_render_readme_still_fails_when_a_row_disappears(): + missing = _readme(" Azure security rules across storage", "Every rule ships with a script").replace( + "| **Remediation Playbooks** |", "| **Removed** |" + ) + _, failures = stats.render_readme(missing, 143, 143) + assert "feature table: Remediation Playbooks row" in failures + + +def _point_at(tmp_path, monkeypatch, readme_count: int) -> Path: + rules = tmp_path / "scanner" / "rules" + playbooks = tmp_path / "playbooks" / "cli" + rules.mkdir(parents=True) + playbooks.mkdir(parents=True) + for n in (1, 2): + (rules / f"az_net_00{n}.py").write_text(f'RULE_ID = "AZ-NET-00{n}"\n', encoding="utf-8") + (playbooks / f"fix_az_net_00{n}.sh").write_text("#!/bin/sh\n", encoding="utf-8") + readme = tmp_path / "README.md" + readme.write_text(_readme(" Azure security rules", "Every rule ships a script", readme_count), encoding="utf-8") + monkeypatch.setattr(stats, "RULES_DIR", rules) + monkeypatch.setattr(stats, "PLAYBOOKS_DIR", playbooks) + monkeypatch.setattr(stats, "README_PATH", readme) + return readme + + +def test_check_reports_stale_readme_without_writing(tmp_path, monkeypatch): + readme = _point_at(tmp_path, monkeypatch, readme_count=1) + before = readme.read_text(encoding="utf-8") + assert stats.main(["--check"]) == stats.EXIT_STALE + assert readme.read_text(encoding="utf-8") == before + + +def test_check_passes_when_readme_is_current(tmp_path, monkeypatch): + _point_at(tmp_path, monkeypatch, readme_count=2) + assert stats.main(["--check"]) == 0 + + +def test_default_mode_rewrites_stale_counts(tmp_path, monkeypatch): + readme = _point_at(tmp_path, monkeypatch, readme_count=1) + assert stats.main([]) == 0 + assert "Runs 2 Azure security rules" in readme.read_text(encoding="utf-8") + assert stats.main(["--check"]) == 0 diff --git a/website/scripts/verify-site.mjs b/website/scripts/verify-site.mjs index 61a61cdb..efea4bfe 100644 --- a/website/scripts/verify-site.mjs +++ b/website/scripts/verify-site.mjs @@ -56,6 +56,19 @@ const index = fs.existsSync(path.join(dist, 'index.html')) : ''; if (!index.includes('Illustrative output')) failures.push('homepage does not label sample scan output'); +const learnPath = path.join(dist, 'learn', 'index.html'); +if (!fs.existsSync(learnPath)) { + failures.push('dist does not contain the /openshield/learn/ route'); +} else { + const learn = fs.readFileSync(learnPath, 'utf8'); + if (!//.test(learn)) { + failures.push('Learn route does not have the /openshield/learn/ canonical URL'); + } + if (!learn.includes('Learn security posture from the source.')) { + failures.push('Learn route does not render the Learn experience'); + } +} + const cmsConfigPath = path.join(dist, 'admin', 'config.yml'); const adminPath = path.join(dist, 'admin', 'index.html'); const hasCmsConfig = fs.existsSync(cmsConfigPath); diff --git a/website/src/components/JourneySection.astro b/website/src/components/JourneySection.astro index 34d5de13..33e7a0db 100644 --- a/website/src/components/JourneySection.astro +++ b/website/src/components/JourneySection.astro @@ -7,7 +7,7 @@ const GITHUB = 'https://github.com/OWASP/openshield';

New here? Run it in this order.

- Learn portal → + Learn portal →
01

Set up read-only access

Authenticate, choose the subscription scope and understand where findings will be stored.

On-site quickstart →
diff --git a/website/src/components/RulesSection.astro b/website/src/components/RulesSection.astro index a92919cc..b9958520 100644 --- a/website/src/components/RulesSection.astro +++ b/website/src/components/RulesSection.astro @@ -5,8 +5,9 @@ import type { Rule } from '../lib/repoData'; const { rules, ruleCount } = repoData; -const SEV_TEXT: Record = { HIGH: '#a03a1e', MEDIUM: '#7a5310', LOW: '#3d5a75' }; -const SEV_DOT: Record = { HIGH: '#c73a17', MEDIUM: '#a06f10', LOW: '#5d6c80' }; +const SEV_TEXT: Record = { CRITICAL: '#8f1414', HIGH: '#a03a1e', MEDIUM: '#7a5310', LOW: '#3d5a75', INFO: '#4b5563' }; +const SEV_LABEL: Record = { CRITICAL: 'Critical', HIGH: 'High', MEDIUM: 'Medium', LOW: 'Low', INFO: 'Info' }; +const SEV_DOT: Record = { CRITICAL: '#b91c1c', HIGH: '#c73a17', MEDIUM: '#a06f10', LOW: '#5d6c80', INFO: '#6b7280' }; /* representative picks, one per pastel swatch; fall back to the first rule of the domain so the section never renders an empty card */ @@ -34,7 +35,7 @@ const featured: { rule: Rule; swatch: string }[] = picks.map((id, i) => ({
- {rule.severity === 'HIGH' ? 'High' : rule.severity === 'MEDIUM' ? 'Medium' : 'Low'} + {SEV_LABEL[rule.severity] ?? rule.severity} {rule.id}
diff --git a/website/src/layouts/Base.astro b/website/src/layouts/Base.astro index ff9f1f60..6359fcc7 100644 --- a/website/src/layouts/Base.astro +++ b/website/src/layouts/Base.astro @@ -13,7 +13,7 @@ import '../styles/global.css'; interface Props { title?: string; description?: string; - /** which nav tab is active: start | rules | architecture | docs | blog | community | evidence */ + /** which nav tab is active: start | learn | rules | architecture | docs | blog | community | evidence */ active?: string; /** page-level structured data (SoftwareApplication, BlogPosting, ...) */ jsonLd?: Record; @@ -117,6 +117,7 @@ const siteJsonLd = {