fix(core-engine): scheme globals decided by enclosing form, not indentation (#2674)

gitgalaxy/core/detector.py

@@ -1816,6 +1816,19 @@ def coding_analysis(
 
             seg_len = len(seg_code)
 
+            # #2674: a registry may declare `_scope_filters: {rule_name: filter_name}`
+            # for rules whose regex can match a construct that only *sometimes*
+            # means what the rule counts, and where the deciding context is the
+            # ENCLOSING FORM rather than anything a flat pattern can see (scheme's
+            # `(define x v)` is a global at module level and a local binding
+            # inside any lambda/let/procedure body, at identical indentation).
+            # The filter runs over the same segment the regex ran over and only
+            # ever REMOVES matches, so counts, spatial_map and threat_locations
+            # stay mutually consistent. Scans are cached per segment because a
+            # filter's structural pass is independent of which rule asks for it.
+            scope_filters: dict[str, str] = rules.get("_scope_filters") or {}
+            scope_cache: dict[str, set[int]] = {}
+
             # ---> NEW: Spatial Map for this segment <---
             spatial_map: dict[str, list[int]] = {}
 
@@ -1845,6 +1858,11 @@ def coding_analysis(
                     # ---> THE UPGRADE: Spatial Mapping instead of raw counting <---
                     if hasattr(pattern, "finditer"):
                         matches = list(pattern.finditer(seg_code))
+                        scope_filter_name = scope_filters.get(rule_name)
+                        if scope_filter_name and matches:
+                            matches = self._apply_scope_filter(
+                                scope_filter_name, seg_lang, rule_name, seg_code, matches, scope_cache
+                            )
                         hit_indices = [m.start() for m in matches]
 
                         # ---> NEW: Offset to LOC Conversion <---
@@ -5506,6 +5524,197 @@ def _matching_paren_end(self, text: str, open_idx: int) -> int:
             i += 1
         return len(text)
 
+    # ------------------------------------------------------------------
+    # #2674: registry-declared scope filters (see `_scope_filters` in
+    # coding_analysis). One filter exists today; add new ones here, keyed
+    # by the name a language definition uses, so the registry stays data.
+    # ------------------------------------------------------------------
+
+    # Forms whose body is a LOCAL scope: a `(define ...)` whose nearest
+    # classifying ancestor is one of these is an internal definition (R7RS
+    # 5.3.2 "Internal definitions"), not a global. `define` itself is here
+    # because a define nested inside another define's body is internal by
+    # construction, and every `define-*` form (`define-syntax` templates,
+    # `define-record-type` / nanopass `define-pass` bodies) is treated the
+    # same way in the walk. Unknown heads (`begin`, `if`, a `cond` clause's
+    # own `[...]`, quoted data) are TRANSPARENT: the walk keeps climbing, so a
+    # top-level `(begin (define x 1))` still counts (begin splices into the
+    # enclosing context, R7RS 5.6.1). `let-syntax` / `letrec-syntax` are
+    # transparent too: R6RS 11.18 splices their bodies into the surrounding
+    # context, and Chez's io.ss wraps its whole file in one.
+    _LISP_BODY_FORMS: ClassVar[frozenset[str]] = frozenset(
+        {
+            "lambda",
+            "case-lambda",
+            "let",
+            "let*",
+            "letrec",
+            "letrec*",
+            "let-values",
+            "let*-values",
+            "when",
+            "unless",
+            "cond",
+            "case",
+            "parameterize",
+            "fluid-let",
+            "dynamic-wind",
+            "with-output-language",
+            "meta-cond",
+            "guard",
+            "do",
+            "define",
+        }
+    )
+    # Forms whose body IS the module scope when nothing above them is a body
+    # form: R6RS `library`, Racket `module`, R7RS `define-library`. Chez also
+    # allows `(module ...)` wherever a definition may appear (a LOCAL module),
+    # so one nested inside a lambda/let body is body scope, not module scope.
+    _LISP_MODULE_FORMS: ClassVar[frozenset[str]] = frozenset(
+        {"library", "module", "define-library", "top-level-program"}
+    )
+    # Every token that can open/close a form or hide a paren from the scan.
+    # Comments are already stripped by Prism before coding_analysis, so the
+    # `;` / `#|` branches are defensive only (the scanner is also usable
+    # on raw source). Every alternative is anchored on a distinct first
+    # character and bounded or single-pass, so the tokenizer is linear
+    # (see test_scheme_strict.py's timing check). An unterminated `"` is
+    # swallowed to end of line rather than to end of file so a typo can
+    # only desync one line, not blind the whole scan.
+    _LISP_SCOPE_TOKEN: ClassVar[re.Pattern[str]] = re.compile(
+        r'"(?:\\.|[^"\\])*"'
+        r'|"[^"\n]*'
+        r"|;[^\n]*"
+        r"|#\|"
+        r"|\|#"
+        r"|#\\(?:[a-zA-Z0-9][a-zA-Z0-9-]{0,31}|[^\s])"
+        r"|[()\[\]]",
+        re.S,
+    )
+    _LISP_FORM_HEAD: ClassVar[re.Pattern[str]] = re.compile(r'[ \t\r\n]*([^\s()\[\];"]{1,200})')
+    _LISP_LET_FAMILY: ClassVar[frozenset[str]] = frozenset({"let", "let*", "letrec", "letrec*"})
+
+    def _apply_scope_filter(
+        self,
+        filter_name: str,
+        seg_lang: str,
+        rule_name: str,
+        code: str,
+        matches: list[re.Match[str]],
+        cache: dict[str, set[int]],
+    ) -> list[re.Match[str]]:
+        """
+        Drop the matches of `rule_name` that the named structural filter
+        rejects. Returns `matches` untouched (with a diagnostic) for a filter
+        name the engine doesn't implement, so a registry typo can only ever
+        restore the pre-filter count, never zero a metric.
+        """
+        if filter_name == "lisp_body_position":
+            if filter_name not in cache:
+                cache[filter_name] = self._lisp_module_level_define_offsets(code)
+            keep = cache[filter_name]
+            kept: list[re.Match[str]] = []
+            for m in matches:
+                # The rules that opt in all begin `^[ \t]*\(` so the first "("
+                # inside the match is the define's own opening paren.
+                paren = code.find("(", m.start(), m.end())
+                if paren != -1 and paren in keep:
+                    kept.append(m)
+            return kept
+        self.logger.warning(
+            f"[DIAGNOSTIC] Unknown scope filter '{filter_name}' declared for '{seg_lang}::{rule_name}'. Ignoring."
+        )
+        return matches
+
+    def _lisp_module_level_define_offsets(self, code: str) -> set[int]:
+        """
+        Offsets of the "(" of every `(define ...)` in `code` whose nearest
+        classifying enclosing form makes it MODULE-LEVEL (#2674).
+
+        In Scheme indentation says nothing about scope -- the enclosing form
+        does. `(define y 5)` inside `(define (f x) ...)` is a local binding;
+        `(define y 5)` inside a file-wrapping `(let () ...)` (the standard
+        Chez/R6RS idiom: cpnanopass.ss has 682 indented defines and none at
+        column 0) is a real global. So this walks the paren structure with a
+        stack of form frames, skipping strings / char literals / comments,
+        and classifies each define by climbing the stack to the nearest
+        non-transparent frame:
+
+          * MODULE frame -> module-level. A `_LISP_MODULE_FORMS` head, or the
+            file wrapper: a bindings-less let-family form. Either is a module
+            frame only if no BODY frame sits above it (a local module inside
+            a lambda is local); the wrapper additionally needs no other
+            wrapper above it (a nested `(let () (define who ...))` is a block).
+          * BODY frame -> internal. A `_LISP_BODY_FORMS` head, any `define-*`
+            form, a let-family form with bindings, or a module/let that failed
+            the test above.
+          * everything else is transparent; running out of stack -> module.
+
+        Linear in `len(code)`: one tokenizer pass, and each define's climb is
+        bounded by nesting depth.
+
+        Measured on the language-crucible golden master (Prism code streams,
+        which is what coding_analysis actually sees): the `globals` regex
+        matches 45 defines across cpnanopass.ss / io.ss / schemify.rkt /
+        thread.rkt; this pass keeps 8 (io.ss's six wrapper-level buffer
+        constants and `open-files`, thread.rkt's one top-level callback,
+        cpnanopass.ss's one wrapper-module binding) and drops 37, every one
+        of them inside a procedure body, a nanopass `define-pass`, a `cond`
+        clause, or a local `(module ...)` under a block-scope `(let () ...)`.
+        Against #2674's raw-source oracle the three deliberate differences
+        are: `(begin ...)` and `let-syntax` bodies inside the wrapper are
+        spliced (kept), a `module` inside a `define-pass` is pass-local
+        (dropped), and a `module` under a nested block let is block-local
+        (dropped) -- the same rule the oracle applied to `(define who ...)`.
+        """
+        module_forms = self._LISP_MODULE_FORMS
+        body_forms = self._LISP_BODY_FORMS
+        let_family = self._LISP_LET_FAMILY
+        head_re = self._LISP_FORM_HEAD
+        # Frame kinds: "module" / "body" / "" (transparent).
+        stack: list[str] = []
+        keep: set[int] = set()
+        comment_depth = 0
+        for tok in self._LISP_SCOPE_TOKEN.finditer(code):
+            text = tok.group(0)
+            first = text[0]
+            if comment_depth:
+                if text == "#|":
+                    comment_depth += 1
+                elif text == "|#":
+                    comment_depth -= 1
+                continue
+            if text == "#|":
+                comment_depth = 1
+                continue
+            if first in ")]":
+                if stack:
+                    stack.pop()
+                continue
+            if first not in "([":
+                continue  # string, char literal, line comment, stray |#
+            head_m = head_re.match(code, tok.end())
+            head = head_m.group(1) if head_m else ""
+            if head == "define":
+                kind = "module"
+                for enclosing in reversed(stack):
+                    if enclosing:
+                        kind = enclosing
+                        break
+                if kind == "module":
+                    keep.add(tok.start())
+            frame = ""
+            if head in module_forms:
+                frame = "body" if "body" in stack else "module"
+            elif head in let_family and head_m is not None:
+                after = code[head_m.end() : head_m.end() + 64].lstrip(" \t\r\n")
+                empty_bindings = after.startswith("()") or after.startswith("[]")
+                frame = "module" if empty_bindings and not any(stack) else "body"
+            elif head in body_forms or head.startswith("define-"):
+                frame = "body"
+            stack.append(frame)
+        return keep
+
     @staticmethod
     def _count_space_separated_args(args_str: str) -> int:
         """

gitgalaxy/core/prism.py

@@ -1011,11 +1011,21 @@ def _strip_nested_comments(self, text: str, family: str = "recursive_block") ->
         # match start -- claims the entire line before the scanner ever reaches an
         # apostrophe/backtick inside it, regardless of how far away an unrelated
         # real quote/backtick happens to sit.
+        # #2674: Scheme's char literals are `#\x` -- and `#\;` / `#\"` / `#\(` are
+        # all legal (cpnanopass.ss: `(write-char #\; p)`). With `;` as the family's
+        # line-comment token, the comment branch below claimed `#\;` and ate the rest
+        # of the line INCLUDING its closing parens, so every paren-balanced scan
+        # downstream (the Mode-B function slicer, the #2674 scope filter) was one
+        # level deep for the rest of the file. Claim the literal atomically first;
+        # it goes through the same mask/unmask path as a string so the code stream
+        # keeps it verbatim. Bounded exactly like detector.py's _LISP_SCOPE_TOKEN.
+        lisp_char_literal = r"#\\(?:[a-zA-Z0-9][a-zA-Z0-9-]{0,31}|[^\s])|" if family == "recursive_block_lisp" else ""
         combined_pattern = re.compile(
-            r'(?<!\\)"(?:\\.|[^"\\])*"'
-            r"|(?<!\\)'(?![a-zA-Z_]\w*[=<>(),&|\]\s])(?:\\.|[^'\\]){0,10}'"
-            r"|(?<!\\)`(?:\\.|[^`\\]){0,200}`"
-            rf"|{re.escape(s_line)}[^\n]*",
+            lisp_char_literal
+            + r'(?<!\\)"(?:\\.|[^"\\])*"'
+            + r"|(?<!\\)'(?![a-zA-Z_]\w*[=<>(),&|\]\s])(?:\\.|[^'\\]){0,10}'"
+            + r"|(?<!\\)`(?:\\.|[^`\\]){0,200}`"
+            + rf"|{re.escape(s_line)}[^\n]*",
             re.S | re.M,
         )
         string_cache: dict[str, str] = {}

gitgalaxy/standards/how_to_add_a_language.md

@@ -97,6 +97,9 @@ This dictionary defines the **Structural Signatures** used by an AST-free parsin
 16. **Identifier Capture Classes Must Match the Language's Real Grammar:** A capture class like `[a-zA-Z0-9_!?-]+` for a function/type name assumes a narrow, C-like identifier grammar. Many languages (Lisp-family especially, but any language with idiomatic naming conventions using extra punctuation) allow far more characters in identifiers than that. Because the capture typically feeds a required trailing lookahead, a truncated capture doesn't just capture less — it can break the lookahead entirely, turning a partial-match bug into a complete non-match for the whole rule.
     * ❌ `[a-zA-Z0-9_!?-]+` for Scheme identifiers — excludes `> < = * + / . ~ $ % ^ &`, so idiomatic names like `list->vector`, `1+`, and SRFI-9's `<TypeName>` record-naming convention never matched AT ALL, because the truncated capture broke the trailing lookahead requiring whitespace/`)` right after.
     * ✅ Check the language's actual identifier grammar (e.g. R7RS's special-initial/special-subsequent character sets) before picking the capture class, and verify against real idiomatic names from that language's own standard library — not just simple ASCII test names.
+17. **When the Discriminator Is the Enclosing Form, Don't Fake It With a Column Anchor:** A flat regex sees one line; if the same line means different things depending on what it is nested in, no anchor fixes that — it only trades one error for another. Declare a structural scope filter on the rule instead: add `"_scope_filters": {"<rule>": "<filter_name>"}` to the language's `rules` and implement (or reuse) the named filter in `detector.py`'s `_apply_scope_filter`. The filter runs after the regex over the same segment and only ever removes matches, so counts, spatial maps and threat locations stay consistent, and an unknown filter name is ignored (never zeroes a metric).
+    * ❌ Scheme `globals`: `(define y 5)` is a local binding inside `(define (f x) ...)` and a real global inside a file-wrapping `(let () ...)`, at identical indentation. The `^(?![ \t])` column-0 anchor (#2651) deleted 67 false positives AND 32 real globals on the language-crucible corpus (#2674).
+    * ✅ `"_scope_filters": {"globals": "lisp_body_position"}` — the `lisp_body_position` filter walks the paren structure and keeps only defines whose nearest classifying enclosing form is module scope. Measure any such filter against real files with a hand-checkable oracle before shipping it; the rosetta corpus plants at top level and cannot see this class of defect.
 
 ### THE LEXICAL PARSING FAMILIES
 You must assign the language to one of these 5 lexical parsing families based on how it handles comments and non-executable text:

gitgalaxy/standards/language_standards/languages/scheme.py

@@ -176,6 +176,21 @@
         # a top-level binding using the "X->Y" convention (e.g.
         # `default->value`) failed to match at all.
         "globals": re.compile(r"^[ \t]*\([ \t]*define\s+[a-zA-Z0-9_!?*+/<>=.~$%^&:-]+\s+[^(\s]", re.M),
+        # #2674: the regex above matches BOTH a module-level `(define x v)` (a
+        # real global) and an internal define inside a lambda/let/procedure
+        # body (a local binding, R7RS 5.3.2) -- and in Scheme indentation
+        # doesn't separate them: the whole of Chez's cpnanopass.ss sits inside
+        # a `(let () ...)` wrapper, so its 682 defines are ALL indented and
+        # the #2651 column-0 anchor would have zeroed them. Measured over the
+        # language-crucible Chez/Racket sources the bare regex is 42.7%
+        # precise (53 globals / 71 locals). The discriminator is the
+        # ENCLOSING FORM, which no flat pattern can see, so detector.py's
+        # coding_analysis runs a paren-depth scope pass over the segment and
+        # keeps only the matches whose define is module-level (top level,
+        # `library`/`module`/`define-library`, or a bindings-less let-family
+        # file wrapper). See `_lisp_module_level_define_offsets` in
+        # detector.py; on the same corpus it keeps 53 / drops 71.
+        "_scope_filters": {"globals": "lisp_body_position"},
         # 19. decorators
         "decorators": None,
         # 20. generics

tests/core_engine/test_prism.py

@@ -1201,6 +1201,31 @@ def test_prism_scheme_string_literal_shielding():
     assert "this ; is not a comment" in result["code_stream"]
 
 
+def test_prism_scheme_char_literal_semicolon_is_not_a_comment():
+    """
+    #2674: `#\\;` is a char literal (cpnanopass.ss: `(write-char #\\; p)`), but
+    the family's `;` line-comment branch used to claim it and swallow the
+    rest of the line -- closing parens included -- so every paren-balanced
+    scan downstream ran one level deep for the rest of the file. The char
+    literal must survive verbatim, the real comment after it must still be
+    stripped, and `#\\"` must not open a string.
+    """
+    from gitgalaxy.standards.gitgalaxy_config import LEXICAL_FAMILY_HEURISTICS
+    from gitgalaxy.standards.language_standards import LANGUAGE_DEFINITIONS
+
+    real_prism = Prism(LEXICAL_FAMILY_HEURISTICS, LANGUAGE_DEFINITIONS)
+    src = '(write-char #\\; p) ; trailing comment\n(define q #\\") ; c2\n(define c #\\space)\n(display "a;b")\n'
+    result = real_prism.split_streams(src, "scheme")
+    code, comments = result["code_stream"], result["comment_stream"]
+    assert "(write-char #\\; p)" in code
+    assert '(define q #\\")' in code
+    assert "(define c #\\space)" in code
+    assert '"a;b"' in code
+    assert "trailing comment" not in code and "c2" not in code
+    assert "; trailing comment" in comments and "; c2" in comments
+    assert code.count("(") == code.count(")"), "char literals must not desync the paren balance"
+
+
 def test_prism_line_exclusive_no_longer_lists_scheme_block_tokens():
     """
     #| and |# were removed from line_exclusive's shared delimiter list