diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index c0e1f8e..86605aa 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -115,7 +115,7 @@ jobs: # Checks for: zero-width spaces, zero-width joiners, BOM, soft hyphens, # non-breaking spaces, null bytes, and other invisible Unicode in source files. set +e - PATTERNS='\xc2\xa0|\xe2\x80\x8b|\xe2\x80\x8c|\xe2\x80\x8d|\xef\xbb\xbf|\xc2\xad|\xe2\x80\x8e|\xe2\x80\x8f|\xe2\x80\xaa|\xe2\x80\xab|\xe2\x80\xac|\xe2\x80\xad|\xe2\x80\xae|\x00' + PATTERNS='(*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}]' find "$GITHUB_WORKSPACE" \ -not -path '*/.git/*' -not -path '*/node_modules/*' \ -not -path '*/.deno/*' -not -path '*/target/*' \ @@ -126,7 +126,7 @@ jobs: -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \) \ - -exec grep -Prl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null + -exec grep -aPrl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null EL_EXIT=$? set -e @@ -135,12 +135,40 @@ jobs: echo "exit_code=$EL_EXIT" >> "$GITHUB_OUTPUT" echo "ready=true" >> "$GITHUB_OUTPUT" + # Blocking subset: C0 controls and NUL only (owner ruling 2026-08-28). + # Invisible Unicode (NBSP/BOM/zero-width) stays ADVISORY - about 2,100 + # estate files carry it as legitimate typography in prose. + blocking=0 + while IFS= read -r bf; do + [ -z "$bf" ] && continue + if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "$bf"; then + blocking=$((blocking+1)) + echo "::error file=${bf#$GITHUB_WORKSPACE/}::C0 control characters or NUL bytes - file corruption, blocks the gate" + fi + done < /tmp/empty-lint-results.txt + echo "blocking=$blocking" >> "$GITHUB_OUTPUT" + # Emit annotations for each file with invisible chars while IFS= read -r filepath; do [ -z "$filepath" ] && continue REL_PATH="${filepath#$GITHUB_WORKSPACE/}" echo "::warning file=${REL_PATH}::Invisible Unicode characters detected (zero-width space, BOM, NBSP, etc.)" done < /tmp/empty-lint-results.txt + + # Enforce (owner ruling 2026-08-28): C0/NUL corruption BLOCKS; other + # invisible Unicode stays advisory. Enforcement lives inside this step + # so a crash above fails the job directly - counts can never arrive + # empty into a separate check that then passes silently. + if [ "$EL_EXIT" -ne 0 ]; then + echo "::warning::invisible-character scan exited $EL_EXIT - results may be incomplete" + fi + if [ "${blocking:-0}" -gt 0 ]; then + echo "## Empty-linter: BLOCKED - $blocking file(s) with C0/NUL corruption" >> "$GITHUB_STEP_SUMMARY" + echo "::error::$blocking file(s) contain C0 control characters or NUL bytes - corruption, not typography. See file annotations." + exit 1 + elif [ "${FINDINGS:-0}" -gt 0 ]; then + echo "::notice::$FINDINGS file(s) carry invisible Unicode (NBSP/BOM/zero-width) - advisory only" + fi - name: Write summary run: | if [ "${{ steps.lint.outputs.ready }}" = "true" ]; then diff --git a/config.ncl b/config.ncl new file mode 100644 index 0000000..3f7f1b0 --- /dev/null +++ b/config.ncl @@ -0,0 +1,281 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# config.ncl — Nickel configuration for byte detection and validation +# +# This file provides structured configuration for byte-level corruption detection, +# matching the patterns defined in stdlib/ByteDetector.affine. +# +# Language: Nickel (NCL) - https://nickel-lang.org/ + +{ + # Byte detection configuration + byte_detection = { + # Version of this configuration schema + version = "1.0.0", + + # BLOCKING patterns - cause gate failure + blocking = { + # C0 control characters (ISO C0: 0x00-0x1F) + # Excludes: 0x09 (tab), 0x0A (LF), 0x0D (CR) + c0_control = { + description = "C0 control characters and NUL bytes - file corruption", + + # PCRE pattern for detection + pattern = "\\x00|[\\x01-\\x08\\x0B\\x0C\\x0E-\\x1F]", + + # Detailed character ranges + ranges = { + nul = { + hex = "0x00", + char = "NUL", + description = "Null byte - file corruption indicator" + }, + soh_to_bs = { + hex_range = "0x01-0x08", + chars = ["SOH", "STX", "ETX", "EOT", "ENQ", "ACK", "BEL", "BS"], + description = "Start of heading through backspace" + }, + vt = { + hex = "0x0B", + char = "VT", + description = "Vertical tab" + }, + ff = { + hex = "0x0C", + char = "FF", + description = "Form feed" + }, + so_to_us = { + hex_range = "0x0E-0x1F", + chars = ["SO", "SI", "DLE", "DC1", "DC2", "DC3", "DC4", "NAK", "SYN", "ETB", "CAN", "EM", "SUB", "ESC", "FS", "GS", "RS", "US"], + description = "Shift out through unit separator" + } + }, + + # Enforcement + enforcement = { + action = "block", + exit_code = 1, + severity = "error", + message = "C0 control characters or NUL bytes - file corruption, blocks the gate" + }, + + # Rationale + rationale = "These characters indicate file corruption or binary data in text files. They are NEVER legitimate in source code." + } + }, + + # ADVISORY patterns - warning only + advisory = { + # Invisible Unicode characters + invisible_unicode = { + description = "Invisible Unicode characters (typography, legitimate in prose)", + + # PCRE pattern with UTF-8 mode + # Combines C0 controls (for completeness) with invisible Unicode + pattern = "(*UTF)[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F\\x{a0}\\x{ad}\\x{200b}-\\x{200f}\\x{202a}-\\x{202f}\\x{2060}\\x{2066}-\\x{2069}\\x{feff}]", + + # Character categories + categories = { + nbsp = { + unicode = "U+00A0", + hex = "\\x{a0}", + name = "Non-Breaking Space", + legitimate_use = "Typography in documentation" + }, + soft_hyphen = { + unicode = "U+00AD", + hex = "\\x{ad}", + name = "Soft Hyphen", + legitimate_use = "Word breaking in documentation" + }, + zero_width_spaces = { + unicode_range = "U+200B-U+200F", + hex = "\\x{200b}-\\x{200f}", + names = ["ZWSP", "ZWNJ", "ZWJ", "LRM", "RLM"], + legitimate_use = "Complex text layout, RTL languages" + }, + bidi_formatting = { + unicode_range = "U+202A-U+202F", + hex = "\\x{202a}-\\x{202f}", + names = ["LRE", "RLE", "PDF", "LRO", "RLO", "NNBSP"], + legitimate_use = "Bidirectional text formatting" + }, + word_joiner = { + unicode = "U+2060", + hex = "\\x{2060}", + name = "Word Joiner", + legitimate_use = "Typography" + }, + bidi_isolates = { + unicode_range = "U+2066-U+2069", + hex = "\\x{2066}-\\x{2069}", + names = ["LRI", "RLI", "FSI", "PDI"], + legitimate_use = "Modern bidirectional text" + }, + bom = { + unicode = "U+FEFF", + hex = "\\x{feff}", + name = "Byte Order Mark (BOM) / Zero Width No-Break Space", + legitimate_use = "Legacy: zero-width no-break space (deprecated)", + note = "In modern Unicode, ZWNBSP is deprecated; U+2060 (WJ) should be used instead" + } + }, + + # Enforcement + enforcement = { + action = "warn", + exit_code = 0, + severity = "warning", + message = "Invisible Unicode characters detected (zero-width space, BOM, NBSP, etc.)" + }, + + # Rationale + rationale = "About 2,100 estate files carry invisible Unicode as legitimate typography in prose (AsciiDoc/Markdown). Advisory only, not blocking." + }, + + # Leading BOM check (separate from general BOM detection) + leading_bom = { + description = "Byte Order Mark at file start (should use UTF-8 without BOM)", + + # Hex patterns for different BOM types + patterns = { + utf8 = { + name = "UTF-8 BOM", + hex_bytes = "EF BB BF", + pattern = "ef bb bf" + }, + utf16_le = { + name = "UTF-16 Little Endian BOM", + hex_bytes = "FF FE", + pattern = "ff fe" + }, + utf16_be = { + name = "UTF-16 Big Endian BOM", + hex_bytes = "FE FF", + pattern = "fe ff" + }, + utf32_le = { + name = "UTF-32 Little Endian BOM", + hex_bytes = "FF FE 00 00", + pattern = "ff fe 00 00" + }, + utf32_be = { + name = "UTF-32 Big Endian BOM", + hex_bytes = "00 00 FE FF", + pattern = "00 00 fe ff" + } + }, + + # Combined regex for detection + combined_pattern = "ef bb bf|ff fe|fe ff|00 00 fe ff|ff fe 00 00", + + # Detection method + detection = { + command = "head -c 3 FILE | od -An -tx1 | grep -qE '^ *(ef bb bf|ff fe|fe ff|00 00 fe ff|ff fe 00 00)'", + description = "Check first 3-4 bytes of file for BOM" + }, + + # Enforcement + enforcement = { + action = "warn", + exit_code = 0, + severity = "warning", + message = "Leading BOM (Byte Order Mark) detected - should use UTF-8 without BOM" + }, + + # Rationale + rationale = "Source files should use UTF-8 without BOM. BOM causes issues with parsers, compilers, and version control. Unix convention is no BOM." + } + }, + + # File types to scan + file_types = [ + "*.rs", # Rust + "*.ex", # Elixir + "*.exs", # Elixir scripts + "*.res", # ReScript + "*.js", # JavaScript + "*.ts", # TypeScript + "*.json", # JSON + "*.toml", # TOML + "*.yml", # YAML + "*.yaml", # YAML + "*.md", # Markdown + "*.adoc", # AsciiDoc + "*.idr", # Idris + "*.zig", # Zig + "*.v", # V / Coq + "*.jl", # Julia + "*.gleam", # Gleam + "*.hs", # Haskell + "*.ml", # OCaml + "*.sh", # Shell scripts + "*.affine", # Affine patterns + "*.ncl" # Nickel config + ], + + # Directories to exclude from scanning + exclude_paths = [ + ".git", + "node_modules", + ".deno", + "target", + "_build", + "deps", + "external_corpora", + ".lake" + ], + + # Grep options + grep_options = { + pcre_mode = true, # Use -P for PCRE + text_mode = true, # Use -a to treat binary as text + recursive = true, # Use -r for recursive + files_with_matches = true, # Use -l for file names only + quiet = true # Use -q for exit code only (in blocking check) + } + }, + + # Enforcement policy + enforcement_policy = { + version = "1.0.0", + owner_ruling = "2026-08-28", + owner = "Jonathan D.A. Jewell", + + summary = { + blocking = "C0 control characters and NUL bytes only", + advisory = "Invisible Unicode (NBSP, BOM, zero-width spaces)", + rationale = "About 2,100 estate files carry legitimate invisible Unicode in prose. Only C0/NUL indicates corruption." + } + }, + + # Tool compatibility + compatibility = { + pcre = { + version = "PCRE 8.0+", + required_features = ["UTF-8 mode", "hex escapes"], + notes = "(*UTF) prefix for UTF-8 mode is PCRE-specific" + }, + grep = { + version = "GNU grep 2.5+", + required_flags = ["-P", "-a"], + notes = "Requires PCRE support (grep -P)" + }, + locale = { + independence = true, + notes = "Uses hex escapes instead of character classes for locale independence" + } + }, + + # References + references = { + c0_standard = "ISO/IEC 6429:1992 (C0 control characters)", + unicode_standard = "Unicode Standard, Chapter 16 (Special Areas and Format Characters)", + utf8_rfc = "RFC 3629 (UTF-8, Section 6: Byte Order Mark)", + posix = "POSIX.1-2017 (grep, regular expressions)", + nickel = "https://nickel-lang.org/", + source = "stdlib/ByteDetector.affine (canonical pattern definitions)" + } +} diff --git a/stdlib/ByteDetector.affine b/stdlib/ByteDetector.affine new file mode 100644 index 0000000..157a337 --- /dev/null +++ b/stdlib/ByteDetector.affine @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# ByteDetector.affine — Byte-level corruption and encoding detection patterns +# +# This file defines the canonical patterns for detecting byte-level issues +# in source files, including C0 control characters, NUL corruption, and +# invisible Unicode characters. +# +# Used by: CI/CD workflows (dogfood-gate.yml), linters, and validation tools. + +## BLOCKING PATTERNS (file corruption - fails gate) + +### C0 Control Characters +# ISO C0 control range: 0x00-0x1F +# EXCLUDES legitimate whitespace: +# - 0x09 (HT, horizontal tab) +# - 0x0A (LF, line feed) +# - 0x0D (CR, carriage return) +# +# Detection pattern (PCRE): +# \x00|[\x01-\x08\x0B\x0C\x0E-\x1F] +# +# Breakdown: +# \x00 - NUL byte (0x00) - file corruption +# \x01-\x08 - SOH, STX, ETX, EOT, ENQ, ACK, BEL, BS +# \x0B - VT (vertical tab) +# \x0C - FF (form feed) +# \x0E-\x1F - SO, SI, DLE, DC1-4, NAK, SYN, ETB, CAN, EM, SUB, ESC, FS, GS, RS, US +# +# Rationale: These characters indicate file corruption or binary data in text files. +# They are NEVER legitimate in source code and BLOCK the gate. + +pattern c0_control_blocking = "\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]" + +## ADVISORY PATTERNS (invisible Unicode - warning only) + +### Invisible Unicode Characters +# These are legitimate typography characters used in prose (especially +# in AsciiDoc/Markdown documentation) but should be flagged for awareness. +# +# NBSP (Non-Breaking Space): \x{a0} +# Soft Hyphen: \x{ad} +# Zero-Width Spaces: \x{200b}-\x{200f} +# Bidirectional formatting: \x{202a}-\x{202f} +# Other zero-width: \x{2060}, \x{2066}-\x{2069} +# BOM (Byte Order Mark): \x{feff} +# +# Detection pattern (PCRE with UTF-8 mode): +# (*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}] +# +# Note: The (*UTF) prefix enables UTF-8 mode in PCRE for proper Unicode matching. +# This pattern includes BOTH blocking (C0) and advisory (Unicode) characters. + +pattern invisible_unicode_advisory = "(*UTF)[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F\\x{a0}\\x{ad}\\x{200b}-\\x{200f}\\x{202a}-\\x{202f}\\x{2060}\\x{2066}-\\x{2069}\\x{feff}]" + +## LEADING BOM CHECK (separate check for file start) + +### Byte Order Mark (BOM) at file start +# UTF-8 BOM: EF BB BF +# UTF-16 LE: FF FE +# UTF-16 BE: FE FF +# UTF-32 LE: FF FE 00 00 +# UTF-32 BE: 00 00 FE FF +# +# Detection: Check first 3-4 bytes of file +# Shell command: head -c 3 FILE | od -An -tx1 | grep -qE '^ *(ef bb bf|ff fe|fe ff|00 00 fe ff|ff fe 00 00)' +# +# Rationale: Source files should use UTF-8 without BOM. BOM causes issues +# with parsers, compilers, and version control. Unix convention is no BOM. + +pattern leading_bom_hex = "ef bb bf|ff fe|fe ff|00 00 fe ff|ff fe 00 00" + +## ENFORCEMENT POLICY + +# BLOCKING (exit 1): +# - C0 control characters (\x00-\x1F excluding tab/LF/CR) +# - NUL bytes (\x00) +# Rationale: File corruption, not typography. About ~2,100 estate files +# carry legitimate invisible Unicode in prose, so only C0/NUL blocks. +# +# ADVISORY (warning only): +# - Invisible Unicode (NBSP, zero-width spaces, soft hyphens) +# - BOM anywhere in file (as part of invisible Unicode pattern) +# - Leading BOM (separate check, emits warning) +# +# Owner ruling: 2026-08-28 (Jonathan D.A. Jewell) + +## USAGE EXAMPLES + +# Grep for C0 control characters (blocking): +# grep -aPrl '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' FILES +# +# Grep for all invisible characters (advisory): +# grep -aPrl '(*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}]' FILES +# +# Check for leading BOM: +# head -c 3 FILE | od -An -tx1 | grep -qE '^ *(ef bb bf|ff fe|fe ff)' + +## COMPATIBILITY + +# - PCRE (Perl-Compatible Regular Expressions): Used by grep -P +# - Requires grep with -P flag (GNU grep, PCRE support) +# - (*UTF) prefix for UTF-8 mode is PCRE-specific +# - Locale-independent: Uses hex escapes, not character classes + +## REFERENCES + +# - ISO/IEC 6429:1992 (C0 control characters) +# - Unicode Standard, Chapter 16 (Special Areas and Format Characters) +# - RFC 3629 (UTF-8, Section 6: Byte Order Mark) +# - POSIX.1-2017 (grep, regular expressions) +# - Owner ruling: Repository issue/PR discussing enforcement policy