From 5dc67efa5abe6529f1ef8701c3b8b12894a653a1 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:07:22 +0100 Subject: [PATCH 1/4] fix(ci): the invisible-character gate never matched anything MEASURED 2026-08-27: this gate's pattern caught 0 OF 6 invisible-character test cases. It has never detected an NBSP, zero-width space, BOM, soft hyphen, bidi override or word joiner. ROOT CAUSE: the pattern used UTF-8 BYTE sequences (\xc2\xa0) while grep -P matches CHARACTERS. Bytes c2 a0 are ONE character U+00A0; \xc2\xa0 asks for TWO characters, U+00C2 then U+00A0, which is never present. grep -P '\xc2\xa0' -> miss grep -P '\x{a0}' -> MATCH Only \x00 worked, being single-byte in both readings. FIXED: codepoint escapes; C0 control characters \x01-\x08,\x0B,\x0C,\x0E-\x1F added (TAB/LF/CR excluded); and grep -a, without which grep skips any NUL-bearing file as binary. The C0 range matters: a stray BACKSPACE byte made a workflow unparseable in developer-ecosystem, so it never ran, and this linter called it clean. Canonical fix: hyperpolymath/empty-linter#70. 1 file(s) here. VERIFIED: YAML re-parsed, and the corrected pattern was confirmed to catch a real NBSP before the change was kept. --- .github/workflows/dogfood-gate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index 8452311..2bdb9bc 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -127,7 +127,7 @@ jobs: # Checks for: zero-width spaces, zero-width joiners, BOM, soft hyphens, # non-breaking spaces, null bytes, and other invisible Unicode in source files. set +e - PATTERNS='\xc2\xa0|\xe2\x80\x8b|\xe2\x80\x8c|\xe2\x80\x8d|\xef\xbb\xbf|\xc2\xad|\xe2\x80\x8e|\xe2\x80\x8f|\xe2\x80\xaa|\xe2\x80\xab|\xe2\x80\xac|\xe2\x80\xad|\xe2\x80\xae|\x00' + PATTERNS='\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]|\x{a0}|\x{ad}|\x{200b}|\x{200c}|\x{200d}|\x{200e}|\x{200f}|\x{202a}|\x{202b}|\x{202c}|\x{202d}|\x{202e}|\x{2060}|\x{feff}' find "$GITHUB_WORKSPACE" \ -not -path '*/.git/*' -not -path '*/node_modules/*' \ -not -path '*/.deno/*' -not -path '*/target/*' \ @@ -138,7 +138,7 @@ jobs: -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \) \ - -exec grep -Prl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null + -exec grep -aPrl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null EL_EXIT=$? set -e From 87efa93a94ea0413898873b9ecd0ab60828ecfa6 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 28 Aug 2026 06:31:40 +0100 Subject: [PATCH 2/4] fix(ci): enforce C0/NUL corruption in-step; warn on invisible Unicode Second layer of the empty-linter fix, scoped by an owner ruling after a census. DETECTION (layer 1, earlier commit on this branch) sees everything the pattern covers. ENFORCEMENT (this commit) distinguishes two classes: BLOCKING C0 control characters and NUL. Never legitimate; proven damage - a backspace byte made a workflow unloadable (it never ran once), and LaTeX maths in wiki files was silently mangled where a generation step turned backslash-b commands into backspaces. ADVISORY NBSP, BOM, zero-width marks. A gate-lens census found ~2,100 first-party files carry these as legitimate typography in prose; blocking would fail 2,333 files estate-wide for no safety gain. Enforcement lives INSIDE the scan step: if the scanner crashes, the step fails the job directly, so empty counts can never drift into a separate check that passes silently (review finding). The blocking count re-greps only the files the full pattern already flagged, so the find expression is not duplicated and cannot drift. 1 file(s). YAML re-parsed per edit; reverted on any mis-apply. --- .github/workflows/dogfood-gate.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index 2bdb9bc..cce2245 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -147,6 +147,19 @@ jobs: echo "exit_code=$EL_EXIT" >> "$GITHUB_OUTPUT" echo "ready=true" >> "$GITHUB_OUTPUT" + # Blocking subset: C0 controls and NUL only (owner ruling 2026-08-28). + # Invisible Unicode (NBSP/BOM/zero-width) stays ADVISORY - about 2,100 + # estate files carry it as legitimate typography in prose. + blocking=0 + while IFS= read -r bf; do + [ -z "$bf" ] && continue + if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "$bf"; then + blocking=$((blocking+1)) + echo "::error file=${bf#$GITHUB_WORKSPACE/}::C0 control characters or NUL bytes - file corruption, blocks the gate" + fi + done < /tmp/empty-lint-results.txt + echo "blocking=$blocking" >> "$GITHUB_OUTPUT" + # Emit annotations for each file with invisible chars while IFS= read -r filepath; do [ -z "$filepath" ] && continue @@ -154,6 +167,21 @@ jobs: echo "::warning file=${REL_PATH}::Invisible Unicode characters detected (zero-width space, BOM, NBSP, etc.)" done < /tmp/empty-lint-results.txt + # Enforce (owner ruling 2026-08-28): C0/NUL corruption BLOCKS; other + # invisible Unicode stays advisory. Enforcement lives inside this step + # so a crash above fails the job directly - counts can never arrive + # empty into a separate check that then passes silently. + if [ "$EL_EXIT" -ne 0 ]; then + echo "::warning::invisible-character scan exited $EL_EXIT - results may be incomplete" + fi + if [ "${blocking:-0}" -gt 0 ]; then + echo "## Empty-linter: BLOCKED - $blocking file(s) with C0/NUL corruption" >> "$GITHUB_STEP_SUMMARY" + echo "::error::$blocking file(s) contain C0 control characters or NUL bytes - corruption, not typography. See file annotations." + exit 1 + elif [ "${FINDINGS:-0}" -gt 0 ]; then + echo "::notice::$FINDINGS file(s) carry invisible Unicode (NBSP/BOM/zero-width) - advisory only" + fi + - name: Write summary run: | if [ "${{ steps.lint.outputs.ready }}" = "true" ]; then From 5be2f33470f9f06584d5fed4c53ac8f56d97bf11 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:06:54 +0100 Subject: [PATCH 3/4] fix(ci): make invisible-character PCRE locale-independent --- .github/workflows/dogfood-gate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index cce2245..990e3c0 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -127,7 +127,7 @@ jobs: # Checks for: zero-width spaces, zero-width joiners, BOM, soft hyphens, # non-breaking spaces, null bytes, and other invisible Unicode in source files. set +e - PATTERNS='\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]|\x{a0}|\x{ad}|\x{200b}|\x{200c}|\x{200d}|\x{200e}|\x{200f}|\x{202a}|\x{202b}|\x{202c}|\x{202d}|\x{202e}|\x{2060}|\x{feff}' + PATTERNS='(*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}]' find "$GITHUB_WORKSPACE" \ -not -path '*/.git/*' -not -path '*/node_modules/*' \ -not -path '*/.deno/*' -not -path '*/target/*' \ From 1f0209c40279ee6ad0bff1332a6fe04df7b8a18f Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:26:36 +0000 Subject: [PATCH 4/4] fix: apply CodeRabbit auto-fixes Fixed 7 file(s) based on 1 failed pre-merge check. Co-authored-by: CodeRabbit --- tests/empty-lint/integration-test.sh | 127 +++++++++++++++++++++++++ tests/empty-lint/test-bom-leading.txt | 1 + tests/empty-lint/test-c0-backspace.txt | 1 + tests/empty-lint/test-c0-nul.txt | Bin 0 -> 33 bytes tests/empty-lint/test-clean.txt | 1 + tests/empty-lint/test-nbsp.txt | 1 + tests/empty-lint/test-runner.sh | 92 ++++++++++++++++++ 7 files changed, 223 insertions(+) create mode 100755 tests/empty-lint/integration-test.sh create mode 100644 tests/empty-lint/test-bom-leading.txt create mode 100644 tests/empty-lint/test-c0-backspace.txt create mode 100644 tests/empty-lint/test-c0-nul.txt create mode 100644 tests/empty-lint/test-clean.txt create mode 100644 tests/empty-lint/test-nbsp.txt create mode 100755 tests/empty-lint/test-runner.sh diff --git a/tests/empty-lint/integration-test.sh b/tests/empty-lint/integration-test.sh new file mode 100755 index 0000000..e20943e --- /dev/null +++ b/tests/empty-lint/integration-test.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Integration test that simulates the dogfood-gate.yml workflow steps +# +# This script runs the exact same logic as the CI workflow to verify +# that the patterns work correctly in the actual CI environment context. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +GITHUB_WORKSPACE="$SCRIPT_DIR" + +echo "=====================================================================" +echo "Empty-Linter Integration Test (CI Workflow Simulation)" +echo "=====================================================================" +echo "" + +# Clean up any previous results +rm -f /tmp/bom-results.txt /tmp/empty-lint-results.txt + +echo "Step 1: Check for leading BOM (Byte Order Mark)" +echo "---------------------------------------------------------------------" + +set +e +find "$GITHUB_WORKSPACE" \ + -not -path '*/.git/*' -not -path '*/node_modules/*' \ + -not -path '*/.deno/*' -not -path '*/target/*' \ + -not -path '*/_build/*' -not -path '*/deps/*' \ + -not -path '*/external_corpora/*' -not -path '*/.lake/*' \ + -type f \( -name '*.rs' -o -name '*.ex' -o -name '*.exs' -o -name '*.res' \ + -o -name '*.js' -o -name '*.ts' -o -name '*.json' -o -name '*.toml' \ + -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ + -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ + -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \ + -o -name '*.txt' \) \ + -exec grep -aPl '^\xef\xbb\xbf' {} \; > /tmp/bom-results.txt 2>/dev/null +BOM_EXIT=$? +set -e + +BOM_COUNT=$(wc -l < /tmp/bom-results.txt 2>/dev/null || echo 0) + +echo "BOM check exit code: $BOM_EXIT" +echo "Files with leading BOM: $BOM_COUNT" + +while IFS= read -r filepath; do + [ -z "$filepath" ] && continue + REL_PATH="${filepath#$GITHUB_WORKSPACE/}" + echo " [WARNING] $REL_PATH: File starts with UTF-8 BOM (U+FEFF)" +done < /tmp/bom-results.txt + +if [ "${BOM_COUNT:-0}" -gt 0 ]; then + echo " [NOTICE] $BOM_COUNT file(s) have leading BOM - advisory only" +fi +echo "" + +echo "Step 2: Scan for invisible characters" +echo "---------------------------------------------------------------------" + +set +e +PATTERNS='(*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}]' +find "$GITHUB_WORKSPACE" \ + -not -path '*/.git/*' -not -path '*/node_modules/*' \ + -not -path '*/.deno/*' -not -path '*/target/*' \ + -not -path '*/_build/*' -not -path '*/deps/*' \ + -not -path '*/external_corpora/*' -not -path '*/.lake/*' \ + -type f \( -name '*.rs' -o -name '*.ex' -o -name '*.exs' -o -name '*.res' \ + -o -name '*.js' -o -name '*.ts' -o -name '*.json' -o -name '*.toml' \ + -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ + -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ + -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \ + -o -name '*.txt' \) \ + -exec grep -aPrl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null +EL_EXIT=$? +set -e + +FINDINGS=$(wc -l < /tmp/empty-lint-results.txt 2>/dev/null || echo 0) + +echo "Invisible character scan exit code: $EL_EXIT" +echo "Files with invisible characters: $FINDINGS" + +# Check for blocking C0/NUL corruption +blocking=0 +while IFS= read -r bf; do + [ -z "$bf" ] && continue + if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "$bf"; then + blocking=$((blocking+1)) + REL_PATH="${bf#$GITHUB_WORKSPACE/}" + echo " [ERROR] $REL_PATH: C0 control characters or NUL bytes - file corruption, blocks the gate" + fi +done < /tmp/empty-lint-results.txt + +# Emit warnings for non-blocking invisible chars +while IFS= read -r filepath; do + [ -z "$filepath" ] && continue + # Only warn if not already reported as blocking + if ! grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "$filepath"; then + REL_PATH="${filepath#$GITHUB_WORKSPACE/}" + echo " [WARNING] $REL_PATH: Invisible Unicode characters detected (zero-width space, BOM, NBSP, etc.)" + fi +done < /tmp/empty-lint-results.txt + +if [ "$EL_EXIT" -ne 0 ]; then + echo " [WARNING] invisible-character scan exited $EL_EXIT - results may be incomplete" +fi + +if [ "${blocking:-0}" -gt 0 ]; then + echo " [ERROR] $blocking file(s) contain C0 control characters or NUL bytes - corruption, not typography" +elif [ "${FINDINGS:-0}" -gt 0 ]; then + echo " [NOTICE] $FINDINGS file(s) carry invisible Unicode (NBSP/BOM/zero-width) - advisory only" +fi +echo "" + +echo "=====================================================================" +echo "Summary" +echo "=====================================================================" +echo "Leading BOM files: $BOM_COUNT (advisory)" +echo "Invisible character files: $FINDINGS (total)" +echo "C0/NUL corruption files: $blocking (blocking)" +echo "" + +if [ "${blocking:-0}" -gt 0 ]; then + echo "RESULT: FAIL - Blocking corruption detected" + exit 1 +else + echo "RESULT: PASS - No blocking issues (advisory findings are acceptable)" + exit 0 +fi diff --git a/tests/empty-lint/test-bom-leading.txt b/tests/empty-lint/test-bom-leading.txt new file mode 100644 index 0000000..a2a0a59 --- /dev/null +++ b/tests/empty-lint/test-bom-leading.txt @@ -0,0 +1 @@ +This file starts with a UTF-8 BOM (should be detected) diff --git a/tests/empty-lint/test-c0-backspace.txt b/tests/empty-lint/test-c0-backspace.txt new file mode 100644 index 0000000..3ed613e --- /dev/null +++ b/tests/empty-lint/test-c0-backspace.txt @@ -0,0 +1 @@ +This file has a backspace:  here diff --git a/tests/empty-lint/test-c0-nul.txt b/tests/empty-lint/test-c0-nul.txt new file mode 100644 index 0000000000000000000000000000000000000000..1a6cd70be59f995fa2f24a192a339c78df574546 GIT binary patch literal 33 ocmWH^$ShV!%gjkt$Vez>% literal 0 HcmV?d00001 diff --git a/tests/empty-lint/test-clean.txt b/tests/empty-lint/test-clean.txt new file mode 100644 index 0000000..74edc39 --- /dev/null +++ b/tests/empty-lint/test-clean.txt @@ -0,0 +1 @@ +Clean file with no invisible characters diff --git a/tests/empty-lint/test-nbsp.txt b/tests/empty-lint/test-nbsp.txt new file mode 100644 index 0000000..86cf30d --- /dev/null +++ b/tests/empty-lint/test-nbsp.txt @@ -0,0 +1 @@ +This file has NBSP: here diff --git a/tests/empty-lint/test-runner.sh b/tests/empty-lint/test-runner.sh new file mode 100755 index 0000000..c882b14 --- /dev/null +++ b/tests/empty-lint/test-runner.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Test runner for empty-linter inline logic validation +# +# This script validates the invisible character detection patterns +# used in .github/workflows/dogfood-gate.yml + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_DIR="$SCRIPT_DIR" + +# The pattern from dogfood-gate.yml (post-fix, with (*UTF) locale-independent mode) +PATTERNS='(*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}]' + +# C0 control pattern (blocking subset) +C0_PATTERN='\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' + +# Leading BOM pattern (separate check as required by issue #70) +# U+FEFF BOM in UTF-8 is ef bb bf bytes +LEADING_BOM_PATTERN='^\xef\xbb\xbf' + +echo "=====================================================================" +echo "Empty-Linter Test Suite" +echo "=====================================================================" +echo "" + +PASS=0 +FAIL=0 + +test_file() { + local file="$1" + local should_detect="$2" + local pattern_name="${3:-invisible}" + local test_pattern="${4:-$PATTERNS}" + + local basename="$(basename "$file")" + + if grep -aPq "$test_pattern" "$file" 2>/dev/null; then + if [ "$should_detect" = "yes" ]; then + echo "[PASS] $basename: correctly detected $pattern_name" + PASS=$((PASS + 1)) + else + echo "[FAIL] $basename: false positive - detected $pattern_name when it should be clean" + FAIL=$((FAIL + 1)) + fi + else + if [ "$should_detect" = "no" ]; then + echo "[PASS] $basename: correctly clean (no $pattern_name)" + PASS=$((PASS + 1)) + else + echo "[FAIL] $basename: false negative - missed $pattern_name" + FAIL=$((FAIL + 1)) + fi + fi +} + +echo "Test 1: General invisible character detection" +echo "---------------------------------------------------------------------" +test_file "$TEST_DIR/test-clean.txt" "no" "invisible characters" +test_file "$TEST_DIR/test-bom-leading.txt" "yes" "BOM" +test_file "$TEST_DIR/test-c0-nul.txt" "yes" "C0/NUL" +test_file "$TEST_DIR/test-c0-backspace.txt" "yes" "C0/backspace" +test_file "$TEST_DIR/test-nbsp.txt" "yes" "NBSP" +echo "" + +echo "Test 2: C0 control character detection (blocking subset)" +echo "---------------------------------------------------------------------" +test_file "$TEST_DIR/test-clean.txt" "no" "C0 controls" "$C0_PATTERN" +test_file "$TEST_DIR/test-c0-nul.txt" "yes" "C0/NUL" "$C0_PATTERN" +test_file "$TEST_DIR/test-c0-backspace.txt" "yes" "C0/backspace" "$C0_PATTERN" +test_file "$TEST_DIR/test-nbsp.txt" "no" "C0 controls (NBSP is not C0)" "$C0_PATTERN" +test_file "$TEST_DIR/test-bom-leading.txt" "no" "C0 controls (BOM is not C0)" "$C0_PATTERN" +echo "" + +echo "Test 3: Leading BOM detection (separate check per issue #70)" +echo "---------------------------------------------------------------------" +test_file "$TEST_DIR/test-clean.txt" "no" "leading BOM" "$LEADING_BOM_PATTERN" +test_file "$TEST_DIR/test-bom-leading.txt" "yes" "leading BOM" "$LEADING_BOM_PATTERN" +test_file "$TEST_DIR/test-c0-nul.txt" "no" "leading BOM" "$LEADING_BOM_PATTERN" +test_file "$TEST_DIR/test-nbsp.txt" "no" "leading BOM" "$LEADING_BOM_PATTERN" +echo "" + +echo "=====================================================================" +echo "Results: $PASS passed, $FAIL failed" +echo "=====================================================================" + +if [ "$FAIL" -gt 0 ]; then + exit 1 +fi + +echo "All tests passed!"