diff --git a/.github/workflows/dogfood-gate.yml b/.github/workflows/dogfood-gate.yml index 8452311..990e3c0 100644 --- a/.github/workflows/dogfood-gate.yml +++ b/.github/workflows/dogfood-gate.yml @@ -127,7 +127,7 @@ jobs: # Checks for: zero-width spaces, zero-width joiners, BOM, soft hyphens, # non-breaking spaces, null bytes, and other invisible Unicode in source files. set +e - PATTERNS='\xc2\xa0|\xe2\x80\x8b|\xe2\x80\x8c|\xe2\x80\x8d|\xef\xbb\xbf|\xc2\xad|\xe2\x80\x8e|\xe2\x80\x8f|\xe2\x80\xaa|\xe2\x80\xab|\xe2\x80\xac|\xe2\x80\xad|\xe2\x80\xae|\x00' + PATTERNS='(*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}]' find "$GITHUB_WORKSPACE" \ -not -path '*/.git/*' -not -path '*/node_modules/*' \ -not -path '*/.deno/*' -not -path '*/target/*' \ @@ -138,7 +138,7 @@ jobs: -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \) \ - -exec grep -Prl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null + -exec grep -aPrl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null EL_EXIT=$? set -e @@ -147,6 +147,19 @@ jobs: echo "exit_code=$EL_EXIT" >> "$GITHUB_OUTPUT" echo "ready=true" >> "$GITHUB_OUTPUT" + # Blocking subset: C0 controls and NUL only (owner ruling 2026-08-28). + # Invisible Unicode (NBSP/BOM/zero-width) stays ADVISORY - about 2,100 + # estate files carry it as legitimate typography in prose. + blocking=0 + while IFS= read -r bf; do + [ -z "$bf" ] && continue + if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "$bf"; then + blocking=$((blocking+1)) + echo "::error file=${bf#$GITHUB_WORKSPACE/}::C0 control characters or NUL bytes - file corruption, blocks the gate" + fi + done < /tmp/empty-lint-results.txt + echo "blocking=$blocking" >> "$GITHUB_OUTPUT" + # Emit annotations for each file with invisible chars while IFS= read -r filepath; do [ -z "$filepath" ] && continue @@ -154,6 +167,21 @@ jobs: echo "::warning file=${REL_PATH}::Invisible Unicode characters detected (zero-width space, BOM, NBSP, etc.)" done < /tmp/empty-lint-results.txt + # Enforce (owner ruling 2026-08-28): C0/NUL corruption BLOCKS; other + # invisible Unicode stays advisory. Enforcement lives inside this step + # so a crash above fails the job directly - counts can never arrive + # empty into a separate check that then passes silently. + if [ "$EL_EXIT" -ne 0 ]; then + echo "::warning::invisible-character scan exited $EL_EXIT - results may be incomplete" + fi + if [ "${blocking:-0}" -gt 0 ]; then + echo "## Empty-linter: BLOCKED - $blocking file(s) with C0/NUL corruption" >> "$GITHUB_STEP_SUMMARY" + echo "::error::$blocking file(s) contain C0 control characters or NUL bytes - corruption, not typography. See file annotations." + exit 1 + elif [ "${FINDINGS:-0}" -gt 0 ]; then + echo "::notice::$FINDINGS file(s) carry invisible Unicode (NBSP/BOM/zero-width) - advisory only" + fi + - name: Write summary run: | if [ "${{ steps.lint.outputs.ready }}" = "true" ]; then diff --git a/tests/empty-lint/integration-test.sh b/tests/empty-lint/integration-test.sh new file mode 100755 index 0000000..e20943e --- /dev/null +++ b/tests/empty-lint/integration-test.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Integration test that simulates the dogfood-gate.yml workflow steps +# +# This script runs the exact same logic as the CI workflow to verify +# that the patterns work correctly in the actual CI environment context. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +GITHUB_WORKSPACE="$SCRIPT_DIR" + +echo "=====================================================================" +echo "Empty-Linter Integration Test (CI Workflow Simulation)" +echo "=====================================================================" +echo "" + +# Clean up any previous results +rm -f /tmp/bom-results.txt /tmp/empty-lint-results.txt + +echo "Step 1: Check for leading BOM (Byte Order Mark)" +echo "---------------------------------------------------------------------" + +set +e +find "$GITHUB_WORKSPACE" \ + -not -path '*/.git/*' -not -path '*/node_modules/*' \ + -not -path '*/.deno/*' -not -path '*/target/*' \ + -not -path '*/_build/*' -not -path '*/deps/*' \ + -not -path '*/external_corpora/*' -not -path '*/.lake/*' \ + -type f \( -name '*.rs' -o -name '*.ex' -o -name '*.exs' -o -name '*.res' \ + -o -name '*.js' -o -name '*.ts' -o -name '*.json' -o -name '*.toml' \ + -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ + -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ + -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \ + -o -name '*.txt' \) \ + -exec grep -aPl '^\xef\xbb\xbf' {} \; > /tmp/bom-results.txt 2>/dev/null +BOM_EXIT=$? +set -e + +BOM_COUNT=$(wc -l < /tmp/bom-results.txt 2>/dev/null || echo 0) + +echo "BOM check exit code: $BOM_EXIT" +echo "Files with leading BOM: $BOM_COUNT" + +while IFS= read -r filepath; do + [ -z "$filepath" ] && continue + REL_PATH="${filepath#$GITHUB_WORKSPACE/}" + echo " [WARNING] $REL_PATH: File starts with UTF-8 BOM (U+FEFF)" +done < /tmp/bom-results.txt + +if [ "${BOM_COUNT:-0}" -gt 0 ]; then + echo " [NOTICE] $BOM_COUNT file(s) have leading BOM - advisory only" +fi +echo "" + +echo "Step 2: Scan for invisible characters" +echo "---------------------------------------------------------------------" + +set +e +PATTERNS='(*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}]' +find "$GITHUB_WORKSPACE" \ + -not -path '*/.git/*' -not -path '*/node_modules/*' \ + -not -path '*/.deno/*' -not -path '*/target/*' \ + -not -path '*/_build/*' -not -path '*/deps/*' \ + -not -path '*/external_corpora/*' -not -path '*/.lake/*' \ + -type f \( -name '*.rs' -o -name '*.ex' -o -name '*.exs' -o -name '*.res' \ + -o -name '*.js' -o -name '*.ts' -o -name '*.json' -o -name '*.toml' \ + -o -name '*.yml' -o -name '*.yaml' -o -name '*.md' -o -name '*.adoc' \ + -o -name '*.idr' -o -name '*.zig' -o -name '*.v' -o -name '*.jl' \ + -o -name '*.gleam' -o -name '*.hs' -o -name '*.ml' -o -name '*.sh' \ + -o -name '*.txt' \) \ + -exec grep -aPrl "$PATTERNS" {} \; > /tmp/empty-lint-results.txt 2>/dev/null +EL_EXIT=$? +set -e + +FINDINGS=$(wc -l < /tmp/empty-lint-results.txt 2>/dev/null || echo 0) + +echo "Invisible character scan exit code: $EL_EXIT" +echo "Files with invisible characters: $FINDINGS" + +# Check for blocking C0/NUL corruption +blocking=0 +while IFS= read -r bf; do + [ -z "$bf" ] && continue + if grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "$bf"; then + blocking=$((blocking+1)) + REL_PATH="${bf#$GITHUB_WORKSPACE/}" + echo " [ERROR] $REL_PATH: C0 control characters or NUL bytes - file corruption, blocks the gate" + fi +done < /tmp/empty-lint-results.txt + +# Emit warnings for non-blocking invisible chars +while IFS= read -r filepath; do + [ -z "$filepath" ] && continue + # Only warn if not already reported as blocking + if ! grep -qaP '\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' "$filepath"; then + REL_PATH="${filepath#$GITHUB_WORKSPACE/}" + echo " [WARNING] $REL_PATH: Invisible Unicode characters detected (zero-width space, BOM, NBSP, etc.)" + fi +done < /tmp/empty-lint-results.txt + +if [ "$EL_EXIT" -ne 0 ]; then + echo " [WARNING] invisible-character scan exited $EL_EXIT - results may be incomplete" +fi + +if [ "${blocking:-0}" -gt 0 ]; then + echo " [ERROR] $blocking file(s) contain C0 control characters or NUL bytes - corruption, not typography" +elif [ "${FINDINGS:-0}" -gt 0 ]; then + echo " [NOTICE] $FINDINGS file(s) carry invisible Unicode (NBSP/BOM/zero-width) - advisory only" +fi +echo "" + +echo "=====================================================================" +echo "Summary" +echo "=====================================================================" +echo "Leading BOM files: $BOM_COUNT (advisory)" +echo "Invisible character files: $FINDINGS (total)" +echo "C0/NUL corruption files: $blocking (blocking)" +echo "" + +if [ "${blocking:-0}" -gt 0 ]; then + echo "RESULT: FAIL - Blocking corruption detected" + exit 1 +else + echo "RESULT: PASS - No blocking issues (advisory findings are acceptable)" + exit 0 +fi diff --git a/tests/empty-lint/test-bom-leading.txt b/tests/empty-lint/test-bom-leading.txt new file mode 100644 index 0000000..a2a0a59 --- /dev/null +++ b/tests/empty-lint/test-bom-leading.txt @@ -0,0 +1 @@ +This file starts with a UTF-8 BOM (should be detected) diff --git a/tests/empty-lint/test-c0-backspace.txt b/tests/empty-lint/test-c0-backspace.txt new file mode 100644 index 0000000..3ed613e --- /dev/null +++ b/tests/empty-lint/test-c0-backspace.txt @@ -0,0 +1 @@ +This file has a backspace:  here diff --git a/tests/empty-lint/test-c0-nul.txt b/tests/empty-lint/test-c0-nul.txt new file mode 100644 index 0000000..1a6cd70 Binary files /dev/null and b/tests/empty-lint/test-c0-nul.txt differ diff --git a/tests/empty-lint/test-clean.txt b/tests/empty-lint/test-clean.txt new file mode 100644 index 0000000..74edc39 --- /dev/null +++ b/tests/empty-lint/test-clean.txt @@ -0,0 +1 @@ +Clean file with no invisible characters diff --git a/tests/empty-lint/test-nbsp.txt b/tests/empty-lint/test-nbsp.txt new file mode 100644 index 0000000..86cf30d --- /dev/null +++ b/tests/empty-lint/test-nbsp.txt @@ -0,0 +1 @@ +This file has NBSP: here diff --git a/tests/empty-lint/test-runner.sh b/tests/empty-lint/test-runner.sh new file mode 100755 index 0000000..c882b14 --- /dev/null +++ b/tests/empty-lint/test-runner.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Test runner for empty-linter inline logic validation +# +# This script validates the invisible character detection patterns +# used in .github/workflows/dogfood-gate.yml + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_DIR="$SCRIPT_DIR" + +# The pattern from dogfood-gate.yml (post-fix, with (*UTF) locale-independent mode) +PATTERNS='(*UTF)[\x00-\x08\x0B\x0C\x0E-\x1F\x{a0}\x{ad}\x{200b}-\x{200f}\x{202a}-\x{202f}\x{2060}\x{2066}-\x{2069}\x{feff}]' + +# C0 control pattern (blocking subset) +C0_PATTERN='\x00|[\x01-\x08\x0B\x0C\x0E-\x1F]' + +# Leading BOM pattern (separate check as required by issue #70) +# U+FEFF BOM in UTF-8 is ef bb bf bytes +LEADING_BOM_PATTERN='^\xef\xbb\xbf' + +echo "=====================================================================" +echo "Empty-Linter Test Suite" +echo "=====================================================================" +echo "" + +PASS=0 +FAIL=0 + +test_file() { + local file="$1" + local should_detect="$2" + local pattern_name="${3:-invisible}" + local test_pattern="${4:-$PATTERNS}" + + local basename="$(basename "$file")" + + if grep -aPq "$test_pattern" "$file" 2>/dev/null; then + if [ "$should_detect" = "yes" ]; then + echo "[PASS] $basename: correctly detected $pattern_name" + PASS=$((PASS + 1)) + else + echo "[FAIL] $basename: false positive - detected $pattern_name when it should be clean" + FAIL=$((FAIL + 1)) + fi + else + if [ "$should_detect" = "no" ]; then + echo "[PASS] $basename: correctly clean (no $pattern_name)" + PASS=$((PASS + 1)) + else + echo "[FAIL] $basename: false negative - missed $pattern_name" + FAIL=$((FAIL + 1)) + fi + fi +} + +echo "Test 1: General invisible character detection" +echo "---------------------------------------------------------------------" +test_file "$TEST_DIR/test-clean.txt" "no" "invisible characters" +test_file "$TEST_DIR/test-bom-leading.txt" "yes" "BOM" +test_file "$TEST_DIR/test-c0-nul.txt" "yes" "C0/NUL" +test_file "$TEST_DIR/test-c0-backspace.txt" "yes" "C0/backspace" +test_file "$TEST_DIR/test-nbsp.txt" "yes" "NBSP" +echo "" + +echo "Test 2: C0 control character detection (blocking subset)" +echo "---------------------------------------------------------------------" +test_file "$TEST_DIR/test-clean.txt" "no" "C0 controls" "$C0_PATTERN" +test_file "$TEST_DIR/test-c0-nul.txt" "yes" "C0/NUL" "$C0_PATTERN" +test_file "$TEST_DIR/test-c0-backspace.txt" "yes" "C0/backspace" "$C0_PATTERN" +test_file "$TEST_DIR/test-nbsp.txt" "no" "C0 controls (NBSP is not C0)" "$C0_PATTERN" +test_file "$TEST_DIR/test-bom-leading.txt" "no" "C0 controls (BOM is not C0)" "$C0_PATTERN" +echo "" + +echo "Test 3: Leading BOM detection (separate check per issue #70)" +echo "---------------------------------------------------------------------" +test_file "$TEST_DIR/test-clean.txt" "no" "leading BOM" "$LEADING_BOM_PATTERN" +test_file "$TEST_DIR/test-bom-leading.txt" "yes" "leading BOM" "$LEADING_BOM_PATTERN" +test_file "$TEST_DIR/test-c0-nul.txt" "no" "leading BOM" "$LEADING_BOM_PATTERN" +test_file "$TEST_DIR/test-nbsp.txt" "no" "leading BOM" "$LEADING_BOM_PATTERN" +echo "" + +echo "=====================================================================" +echo "Results: $PASS passed, $FAIL failed" +echo "=====================================================================" + +if [ "$FAIL" -gt 0 ]; then + exit 1 +fi + +echo "All tests passed!"