test(tooling): gate the whole scripts/ tree against a cp1252 console abort (BACKLOG #1030)

scripts/docs/backlog_status_check.py

@@ -196,6 +196,19 @@ def scan(
 
 
 def main(argv: list[str] | None = None) -> int:
+    # THIS MODULE MUST CARRY NON-cp1252 CHARACTERS, so it hardens the stream instead of losing them
+    # (BACKLOG #1030). The docstring below is argparse's description and quotes the machine-parsed
+    # banner alphabet CLAUDE.md section 11 protects; remediation text that cannot show an author the
+    # character it wants added is not actionable. On a stock Windows cp1252 console `--help` therefore
+    # raised UnicodeEncodeError on U+2705 before this line -- measured, not theorised.
+    #
+    # `errors="replace"` is deliberate and is NOT a way of tolerating mangled text: the codec is what
+    # was wrong, and it is fixed here to UTF-8. Replacement is the backstop for a stream that cannot
+    # be reconfigured at all, so one exotic codepoint can never again truncate a gate's output
+    # mid-sentence. Scoped to the CLI entry point: importers get their own stdout untouched.
+    if hasattr(sys.stdout, "reconfigure"):
+        sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+
     root = Path(__file__).resolve().parents[2]
     ap = argparse.ArgumentParser(
         description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter

tests/test_cp1252_console_safety.py

@@ -0,0 +1,201 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 MessageFoundry Organization and contributors
+"""No script under ``scripts/`` can abort on a stock Windows console (BACKLOG #1030).
+
+THE DEFECT THIS REPLACES. Enforcement was per-file and hand-placed: ``tests/test_cli.py`` asserts one
+STRING is cp1252-encodable, ``tests/test_announce_hook.py`` asserts one FILE is ASCII, and
+``tests/test_session_mail.py`` names five mail scripts in a literal list. None generalises, so a glyph
+reaching ``print()`` from any other script was caught only by a human reading the diff -- and the
+class recurred at least three times.
+
+WHAT IS GATED, AND WHY IT IS NOT BARE ENCODABILITY. The failure is a character reaching a stream that
+can RAISE, not a character existing. ``sys.stdout`` carries ``errors='surrogateescape'``, which
+round-trips only lone surrogates in DC80-DCFF; every other unencodable codepoint still raises.
+``sys.stderr`` carries ``backslashreplace`` and never raises -- that asymmetry, not a strict/non-strict
+split, is why the same text survives on stderr and aborts on stdout.
+
+So a file may carry non-cp1252 characters IF IT HARDENS ITS OWN STDOUT. That is not an exemption list:
+it is a property of the file, checked mechanically, and it is the actual remedy rather than a promise
+about one. ``scripts/docs/backlog_status_check.py`` is exactly why the distinction is load-bearing --
+its argparse description quotes the machine-parsed banner alphabet CLAUDE.md section 11 protects, and
+remediation text that cannot show an author the character it wants added is not actionable. A gate
+that could not express that would fire on correct code and be switched off.
+
+WHAT A SCRUBBING GATE WOULD DESTROY IN THIS REPOSITORY, MEASURED RATHER THAN IMAGINED. Besides the
+banner alphabet, ``backlog_status_check.py`` carries one further non-cp1252 character: a lone U+FE0F
+inside the banner regex, as ``[<class>]\\uFE0F?\\s``. That is an OPTIONAL VS-16, letting a banner be
+written with or without the selector -- exactly the handling CLAUDE.md section 11 mandates for any
+regex touching that alphabet. It is invisible at the point of use and looks like lint.
+
+Delete it and the ``?`` binds to the CHARACTER CLASS instead. The pattern STILL COMPILES, so nothing
+at author time objects. It then matches an indented continuation line (``^>\\s\\s``, which the ledger
+is full of), ``b.group("emoji")`` returns ``None``, and the dispatch below it evaluates
+``None in _CLOSED`` where ``_CLOSED`` is a ``str`` -- ``TypeError``, on any run that touches the real
+ledger. Two failure modes, and the second is the dangerous one:
+
+  * LOUDLY, TODAY -- every gate that calls ``parse_items`` dies, which is most of them.
+  * SILENTLY, LATER -- the first banner authored WITH a selector stops matching. No banner carries
+    one today, so nothing would catch that regression on the day it arrives.
+
+That is the case for hardening the stream rather than scrubbing the file, and it is why the exemption
+had to be expressible: one invisible character, removed by a well-meaning gate, takes out the reader
+every ledger gate depends on.
+
+NO COUNT IS PINNED IN THAT ARGUMENT, DELIBERATELY. The number of qualifying lines is ref-relative and
+grows with every filed item, so a figure would be stale the moment it was written -- and re-reading
+it would reproduce it, which reads as verification. That is the same hazard the banner in
+``conftest.py`` refuses for the same reason. The mechanism above needs no number and is re-derivable
+in one command on any ref.
+
+THREE PROPERTIES THIS KEEPS, each of which the item names:
+
+  * IT PRINTS WHAT IT SCANNED. A filtered scan that skips a file type reads as clean when it never
+    looked. The inventory is asserted, not merely emitted, so a collapse to zero files fails here
+    instead of passing silently.
+  * IT READS THE WHOLE FILE, never line by line. ``str.splitlines()`` splits on U+2028 and U+2029 and
+    consumes them, so a line-oriented scan is structurally blind to the two separators most likely to
+    break a terminal.
+  * IT NEVER SILENTLY DROPS A FILE. A file that will not decode as UTF-8 is a FAILURE, not a skip.
+
+SCOPE, STATED RATHER THAN IMPLIED. ``scripts/**/*.py`` only. The engine already hardens both streams
+at ``messagefoundry/__main__.py``, and the ``.ps1`` surface has no equivalent reconfigure, so
+generalising to PowerShell is a different decision and is left to the existing per-file gates.
+``docs/`` is deliberately out: ``docs/BACKLOG.md`` is a sanctioned holdout for that same alphabet.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+
+_ROOT = Path(__file__).resolve().parents[1]
+_SCRIPTS = _ROOT / "scripts"
+
+#: The remedy, detected as a property of the file. A call that rebinds stdout's codec is the only
+#: thing that actually stops the abort, which is why it -- and not a promissory comment -- is the
+#: signal. Matched loosely on the call itself so a keyword reordering does not silently un-exempt.
+_HARDENS_STDOUT = re.compile(r"stdout\s*\.\s*reconfigure\s*\(", re.MULTILINE)
+
+
+def _python_scripts() -> list[Path]:
+    return sorted(p for p in _SCRIPTS.rglob("*.py") if "__pycache__" not in p.parts)
+
+
+def _unencodable(text: str) -> list[str]:
+    """Distinct characters cp1252 cannot represent, in codepoint order.
+
+    Whole-string, deliberately: see the module docstring on ``splitlines`` eating U+2028/U+2029.
+    """
+    bad: set[str] = set()
+    for ch in set(text):
+        try:
+            ch.encode("cp1252")
+        except UnicodeEncodeError:
+            bad.add(ch)
+    return sorted(bad)
+
+
+def test_the_scan_actually_covers_something() -> None:
+    """PRINT AND PIN WHAT WAS SCANNED. A scan whose file list collapses to nothing reports a clean
+    result forever; this is the positive control that stops that being indistinguishable from green.
+    """
+    found = _python_scripts()
+    print(f"scanned {len(found)} python files under scripts/")
+    assert len(found) >= 25, (
+        f"only {len(found)} files under scripts/ -- the walk is not finding them"
+    )
+    assert (_SCRIPTS / "docs" / "backlog_status_check.py") in found
+
+
+def test_every_script_file_decodes_as_utf8() -> None:
+    """A file that will not decode is a FAILURE, never a silent skip -- an undecodable file is the
+    one most likely to carry the bytes this gate exists to find."""
+    undecodable: list[str] = []
+    for path in _python_scripts():
+        try:
+            path.read_text(encoding="utf-8")
+        except UnicodeDecodeError as exc:
+            undecodable.append(f"{path.relative_to(_ROOT)}: {exc}")
+    assert not undecodable, "not decodable as UTF-8:\n  " + "\n  ".join(undecodable)
+
+
+def test_no_script_can_abort_a_cp1252_console() -> None:
+    """The gate itself: a script may carry non-cp1252 characters only if it hardens its own stdout."""
+    offenders: list[str] = []
+    exempted: list[str] = []
+    for path in _python_scripts():
+        text = path.read_text(encoding="utf-8")
+        bad = _unencodable(text)
+        if not bad:
+            continue
+        rel = path.relative_to(_ROOT)
+        shown = " ".join(f"U+{ord(c):04X}" for c in bad[:6])
+        if _HARDENS_STDOUT.search(text):
+            exempted.append(f"{rel} ({len(bad)} distinct: {shown})")
+            continue
+        offenders.append(
+            f"{rel} carries {len(bad)} non-cp1252 character(s) [{shown}] and does NOT "
+            f"reconfigure sys.stdout -- printing any of them aborts on a stock Windows console"
+        )
+    print(f"carrying non-cp1252 characters, hardened and therefore allowed: {exempted or 'none'}")
+    assert not offenders, "\n  ".join(["scripts that can abort a cp1252 console:", *offenders])
+
+
+# --- the detector's own controls, so a green above is evidence rather than a pattern that quietly
+# --- stopped matching ----------------------------------------------------------------------------
+
+#: Built with chr(), never literals. This file must stay cp1252-clean itself -- a gate whose own
+#: test could abort the console it defends would be the joke version of this item -- and chr() also
+#: keeps it inside CLAUDE.md section 11, naming a character without adopting one. Every entry has a
+#: recorded failure behind it; none is hypothetical.
+_BROKE_SOMETHING = [
+    chr(0x2192),  # broke `messagefoundry --help` (the adr-analyze arrow)
+    chr(0x2705),  # banner alphabet; broke this repo's own backlog gate --help
+    chr(0x26D4),  # banner alphabet
+    chr(0x1F522),  # banner alphabet; the documented cp1252 console crash
+    chr(0x2194),  # crashed a scanner mid-scan this session, TRUNCATING its output
+    chr(0x2028),  # line separator: invisible to any splitlines()-based scan
+    chr(0x2029),  # paragraph separator: same
+]
+
+
+@pytest.mark.parametrize("ch", _BROKE_SOMETHING, ids=lambda c: f"U+{ord(c):04X}")
+def test_the_detector_sees_every_character_that_has_actually_broken_something(ch: str) -> None:
+    assert _unencodable(f"x{ch}y") == [ch]
+
+
+def test_the_detector_does_not_fire_on_representable_text() -> None:
+    """U+2014 and U+00A3 ARE cp1252-representable and must not be flagged. The item calls this out:
+    a gate that fires on an em dash gets switched off within a day."""
+    text = "plain ASCII, an em dash " + chr(0x2014) + ", and a pound sign " + chr(0x00A3)
+    assert _unencodable(text) == []
+
+
+def test_the_line_oriented_blindness_is_real_and_this_scan_avoids_it() -> None:
+    """Demonstrates the mechanism instead of asserting it: ``splitlines()`` CONSUMES U+2028, so a
+    line-oriented scan is structurally unable to see it. The whole-string scan does."""
+    sep = chr(0x2028)
+    text = "before" + sep + "after"
+    assert sep not in "".join(text.splitlines()), "splitlines would have hidden it"
+    assert _unencodable(text) == [sep]
+
+
+def test_the_hardening_signal_is_detected_and_is_not_vacuous() -> None:
+    """The exemption must be the REMEDY itself, not a promise about one."""
+    assert _HARDENS_STDOUT.search('sys.stdout.reconfigure(encoding="utf-8", errors="replace")')
+    assert _HARDENS_STDOUT.search("sys . stdout . reconfigure ( encoding='utf-8' )")
+    assert not _HARDENS_STDOUT.search("# we should probably reconfigure stdout one day")
+    assert not _HARDENS_STDOUT.search("sys.stderr.reconfigure(encoding='utf-8')")
+
+
+def test_a_synthetic_offender_is_caught_and_a_hardened_one_is_not() -> None:
+    """The gate proved in BOTH directions, on files it has never seen."""
+    glyph = chr(0x2705)
+    bare = f'print("{glyph} done")'
+    hardened = 'import sys; sys.stdout.reconfigure(encoding="utf-8"); ' + bare
+    assert _unencodable(bare) == [glyph]
+    assert not _HARDENS_STDOUT.search(bare)
+    assert _unencodable(hardened) == [glyph]
+    assert _HARDENS_STDOUT.search(hardened)