feat(core-engine): jcl dead_code + jcl/yaml spec_exposure comment-stream rules (#2732)

gitgalaxy/standards/language_standards/languages/jcl.py

@@ -157,5 +157,43 @@
         # same as cobol.py.
         "planned_debt": GLOBAL_PLANNED_DEBT,
         "fragile_debt": GLOBAL_FRAGILE_DEBT,
+        # #2732: the other half of the #2610 leftover. Turning a statement's `//`
+        # into `//*` is THE way JCL comments a step out -- 13 of the 443 licensed
+        # .jcl/.prc files in the pool do it (16 occurrences), including whole JOB
+        # cards -- but until #2610 routed `//*` lines into comment_analysis the
+        # stream was structurally empty, so #2610 added the debt/ownership rules
+        # and never came back for this one.
+        # The operand guard (a keyword must be followed by operand-shaped text,
+        # not prose) is load-bearing, not decoration: banner comments in real
+        # decks open with these same keywords as English words. Measured in
+        # cics-genapp's CICSTS56.jcl, which contains BOTH shapes -- `//* SET THE
+        # RETURN CODE TO CONTROL...` and `//* EXECUTE DUMP UTILITY PROGRAM...`
+        # (prose, excluded: "THE" is not `NAME=` and "EXECUTE" leaves no space
+        # after `EXEC`) sitting a few lines from `//*        DD DSN=CSQ901.
+        # SCSQLOAD,DISP=SHR` (a genuinely commented-out DD, matched). A bare
+        # `(?:EXEC|DD|JOB|SET|INCLUDE)\b` would have counted all of them.
+        # ReDoS: the name class excludes space/tab, so `[A-Za-z0-9_#$@]*[ \t]+`
+        # partitions at exactly one position -- no ambiguity to backtrack over.
+        "dead_code": re.compile(
+            r"^//\*[A-Za-z0-9_#$@]*[ \t]+"
+            r"(?:EXEC[ \t]+\S|DD[ \t]+\S|JOB[ \t]*[,(]|SET[ \t]+[A-Za-z0-9_#$@]+=|INCLUDE[ \t]+MEMBER=)",
+            re.M | re.I,
+        ),
+        # #2732: the generic `[SPEC-n]`/`[spec]`/`[audit]` traceability tag, but
+        # anchored to `//*` rather than copied bare from python/go/java/js.
+        # Anchoring matters because comment rules are NOT comment-stream-only:
+        # coding_analysis applies every non-underscore rule to the code stream
+        # and comment_analysis then adds a second pass over the comments, so an
+        # unanchored bracket rule also scores brackets in code -- here, whatever
+        # sits in an inline `//SYSIN DD *` payload, which is arbitrary non-JCL
+        # text. `//*` can never appear in jcl's code stream (prism strips those
+        # lines out), so the anchor makes the rule structurally comment-only.
+        # Same anchoring precedent as this file's own `ownership` rule.
+        # `\b` after the alternation keeps the bare `spec` branch off "specified"
+        # / "species" -- see yaml.py's copy of this rule for the pool evidence.
+        "spec_exposure": re.compile(
+            r"^//\*[^\n\[]{0,200}\[(?:[ \t]*SPEC[ \t]*-[ \t]*\d{1,10}|spec|audit)\b[^\]\n]{0,300}\]",
+            re.M | re.I,
+        ),
     },
 }

gitgalaxy/standards/language_standards/languages/yaml.py

@@ -158,7 +158,34 @@
             r"\b(?:password|secret|token|api[_-]?key|client[_-]?secret|private[_-]?key)[ \t]*:[ \t]*[\"'][A-Za-z0-9\-_+/=]{16,}[\"']",
             re.I,
         ),
-        "spec_exposure": None,
+        # 29. spec_exposure (Spec / Audit Traceability)
+        # #2732 asked for the generic bracket-tag rule python/go/java/js share,
+        # verbatim, on the reasoning that "spec_exposure never sees the code
+        # stream, so YAML's [a, b] flow-sequence syntax cannot FP against it."
+        # That premise is wrong, and measuring it is what produced this shape:
+        # coding_analysis applies EVERY non-underscore rule to the code stream,
+        # and comment_analysis then adds a second pass over the comments -- it
+        # supplements the code-stream pass, it does not replace it. Dropped in
+        # verbatim, the generic rule scores spec_exposure=1 on a workflow with
+        # no comments at all, off `needs: [audit, lint]` alone.
+        # YAML is the language where that actually bites: a bracket holding bare
+        # unquoted words is ordinary syntax here (flow sequences), not a tag, so
+        # the absence was never as arbitrary as it looked. Anchored instead to
+        # the comment marker, exactly as this file's own `dead_code` rule is --
+        # prism strips `#` comments out of the code stream, so the anchor makes
+        # the rule structurally comment-only and the flow-sequence FP impossible.
+        # The `\b` after the alternation is a second measured fix: bare `spec`
+        # has no boundary in the generic rule, so it matches "specified" and
+        # "species" -- 2 of the 3 code-stream hits across 41,815 pool .yml/.yaml
+        # files were exactly that (`[specified\n per-machine]` in meson's docs,
+        # `[species]` in an elasticsearch test fixture).
+        # ReDoS: the prefix class excludes `[` and the body class excludes `]`,
+        # so each bounded run has exactly one landing site -- nothing to
+        # backtrack over.
+        "spec_exposure": re.compile(
+            r"^[ \t]*#[^\n\[]{0,200}\[(?:[ \t]*SPEC[ \t]*-[ \t]*\d{1,10}|spec|audit)\b[^\]\n]{0,300}\]",
+            re.M | re.I,
+        ),
         "ssr_boundaries": None,
         "events": re.compile(
             r"^[ \t]*repository_dispatch:|^[ \t]*schedule:|^[ \t]*-?[ \t]*cron:",

tests/extraction/languages/test_jcl_strict.py

@@ -98,6 +98,17 @@
     # meaningful now that prism routes //* lines to the comment stream)
     ("planned_debt", "//* TODO wire the FTP step", "//* all wired up here"),
     ("fragile_debt", "//* HACK: overrides the region size", "//* routine banner comment"),
+    # #2732: dead_code = a statement commented out by turning `//` into `//*`.
+    # Each negative is a real prose-banner shape from the pool corpus that a
+    # bare `(?:EXEC|DD|JOB|SET|INCLUDE)\b` keyword rule would have counted.
+    ("dead_code", "//*STEP1   EXEC PGM=IEFBR14", "//* EXECUTE DUMP UTILITY PROGRAM TO PRINT THE"),
+    ("dead_code", "//*        DD DSN=OLD.FILE,DISP=SHR", "//* SET THE RETURN CODE TO CONTROL IF CICS"),
+    ("dead_code", "//*CREL005 JOB ,,CLASS=A,MSGCLASS=H,", "//* PROC statements are documented in the runbook"),
+    ("dead_code", "//*        SET COUNTER=1", "//* JOB scheduling notes live in the runbook"),
+    ("dead_code", "//*        INCLUDE MEMBER=OLDPROC", "//* INCLUDE the operations team on any change"),
+    # #2732: spec_exposure = the generic traceability tag, `//*`-anchored
+    ("spec_exposure", "//* [SPEC-4412] see the change request", "//* nothing traceable here"),
+    ("spec_exposure", "//* raised under [audit] last quarter", "//* the behaviour is [specified] upstream"),
 ]
 
 
@@ -352,6 +363,71 @@ def test_jcl_cond_bypass_redos_immunity():
     assert_redos_immune(pattern, "//X EXEC PGM=Y,COND=(" + "A" * 100000, timeout_sec=3.0)
 
 
+def test_jcl_dead_code_counts_through_the_real_comment_stream():
+    """
+    #2732: end-to-end proof that jcl's two new comment-stream rules actually
+    reach counts, not just that the regexes match a string.
+
+    This is the shape #2610 fixed for the debt rules and left half-done: a
+    `//*` line is stripped OUT of the code stream by prism._strip_jcl_comments,
+    so a rule anchored to `//*` can ONLY ever score via comment_analysis. The
+    sample interleaves the three real pool shapes -- commented-out statements,
+    English prose banners opening on the same keywords, and a JES3 control verb
+    (`//*MAIN`, which JCL_COMMENT_LINE_PATTERN deliberately leaves in the CODE
+    stream) -- so a regression in either direction shows up as a count change.
+    """
+    from gitgalaxy.core.detector import StructuralExtractor
+    from gitgalaxy.core.prism import Prism
+    from gitgalaxy.standards.gitgalaxy_config import LEXICAL_FAMILY_HEURISTICS
+
+    sample = (
+        "//CREL005  JOB ,,CLASS=A,MSGCLASS=H\n"
+        "//*CREL005 JOB ,,CLASS=A,MSGCLASS=H,\n"  # commented-out JOB card
+        "//*STEP1   EXEC PGM=IEFBR14\n"  # commented-out EXEC
+        "//*        DD DSN=OLD.FILE,DISP=SHR\n"  # commented-out DD
+        "//*        SET COUNTER=1\n"  # commented-out SET
+        "//*        INCLUDE MEMBER=OLDPROC\n"  # commented-out INCLUDE
+        "//* SET THE RETURN CODE TO CONTROL IF CICS SHOULD BE\n"  # prose
+        "//* EXECUTE DUMP UTILITY PROGRAM TO PRINT THE\n"  # prose
+        "//* [SPEC-77] see change request\n"
+        "//*MAIN SYSTEM=SY1\n"  # JES3 verb: stays in the code stream
+        "//STEP1    EXEC PGM=IEFBR14\n"
+    )
+
+    prism = Prism(LEXICAL_FAMILY_HEURISTICS, LANGUAGE_DEFINITIONS)
+    streams = prism.split_streams(sample, "jcl")
+
+    # the anchor is only meaningful because `//*` never survives into code
+    assert "//*CREL005" not in streams["code_stream"]
+    assert "//*MAIN SYSTEM=SY1" in streams["code_stream"]
+    assert not JCL_RULES["dead_code"].search(streams["code_stream"])
+
+    equations = StructuralExtractor("jcl", LANGUAGE_DEFINITIONS).splice(
+        streams["code_stream"], streams["comment_stream"], raw_content=sample
+    )["equations"]
+
+    assert equations["dead_code"] == 5, "one per commented-out statement, no prose banners"
+    assert equations["spec_exposure"] == 1
+
+
+def test_jcl_new_comment_rules_redos_immunity():
+    """
+    #2732: both new rules use bounded runs whose character class excludes the
+    delimiter that must follow it (`[^\\n\\[]{0,200}` before a literal `[`,
+    `[^\\]\\n]{0,300}` before a literal `]`, the name charset before `[ \\t]+`),
+    so each quantifier has exactly one landing site and there is no ambiguous
+    partition to backtrack over. Payloads are long unterminated runs of exactly
+    the character each bounded class accepts.
+    """
+    assert_redos_immune(JCL_RULES["dead_code"], "//*" + "A" * 100000, timeout_sec=3.0)
+    assert_redos_immune(JCL_RULES["dead_code"], "//*" + "A \t" * 40000, timeout_sec=3.0)
+    assert_redos_immune(JCL_RULES["spec_exposure"], "//*" + "a" * 100000, timeout_sec=3.0)
+    assert_redos_immune(JCL_RULES["spec_exposure"], "//* [spec" + "a" * 100000, timeout_sec=3.0)
+
+    assert JCL_RULES["dead_code"].search("//*STEP1   EXEC PGM=IEFBR14")
+    assert JCL_RULES["spec_exposure"].search("//* [SPEC-4412] traceable")
+
+
 def test_jcl_lexical_family_no_block_terminator_state_to_confuse():
     """
     Lexical-family audit: jcl is `line_exclusive` -- no block comment

tests/extraction/languages/test_yaml_strict.py

@@ -92,6 +92,12 @@
         "uses: actions/checkout@v4",
     ),
     ("listeners", "webhook: http://example.com/hook", "endpoint: http://example.com/hook"),
+    # #2732: spec_exposure, comment-anchored. Every negative is a bare flow
+    # sequence -- the shape that makes YAML different from the languages
+    # sharing the unanchored generic rule (see the dedicated test below).
+    ("spec_exposure", "  # [SPEC-4412] pinned per the release spec", "    needs: [audit, lint]"),
+    ("spec_exposure", "      # see [audit] trail", "    branches: [spec, main]"),
+    ("spec_exposure", "# raised in [spec] review", "  # the value is [specified per-machine]"),
     ("test_skip", "run: npm test -- --passWithNoTests", "run: npm test"),
     # --- DEEP ADVERSARIAL CASES FOR HIGH-AMBIGUITY SIGNATURES ---
     # args: tolerating comments and blank lines between 'with:' and args
@@ -597,3 +603,98 @@ def test_yaml_cleanup_redos_immunity():
     assert cleanup.search("run: rm -rf /tmp/cache")
     assert cleanup.search("run: docker-compose down")
     assert cleanup.search("run: kill 1234")
+
+
+def test_yaml_spec_exposure_is_comment_anchored_not_the_generic_bracket_rule():
+    """
+    #2732 proposed giving yaml the generic `[SPEC-n]|[spec]|[audit]` bracket
+    rule verbatim from python/go/java/js, arguing that "spec_exposure never
+    sees the code stream, so YAML's [a, b] flow-sequence syntax cannot FP
+    against it."
+
+    The premise is false, and this test pins the correction. `coding_analysis`
+    applies EVERY non-underscore rule to the code stream; `comment_analysis`
+    then runs the comment-stream rules a SECOND time over the comments. It
+    supplements the code-stream pass rather than replacing it -- so an
+    unanchored bracket rule scores YAML flow sequences, which is ordinary
+    syntax in this language rather than a traceability tag.
+
+    Measured before anchoring: the workflow below has no comments at all and
+    still scored spec_exposure=1, entirely from `needs: [audit, lint]`.
+    """
+    from gitgalaxy.core.detector import StructuralExtractor
+    from gitgalaxy.core.prism import Prism
+    from gitgalaxy.standards.gitgalaxy_config import LEXICAL_FAMILY_HEURISTICS
+
+    generic_rule = re.compile(r"\[(?:\s*SPEC\s*-\s*\d{1,10}|spec|audit)[^\]]{0,300}\]", re.I)
+    comment_free_workflow = (
+        "name: CI\non:\n  push:\njobs:\n"
+        "  audit:\n    steps:\n      - run: npm audit\n"
+        "  build:\n    needs: [audit, lint]\n    steps:\n      - run: npm run build\n"
+    )
+
+    # the rule that was asked for would have counted the flow sequence ...
+    assert generic_rule.search(comment_free_workflow)
+    # ... the rule that shipped cannot, because `#` never survives into code
+    assert not YAML_RULES["spec_exposure"].search(comment_free_workflow)
+
+    prism = Prism(LEXICAL_FAMILY_HEURISTICS, LANGUAGE_DEFINITIONS)
+    streams = prism.split_streams(comment_free_workflow, "yaml")
+    assert streams["comment_stream"] == ""
+    equations = StructuralExtractor("yaml", LANGUAGE_DEFINITIONS).splice(
+        streams["code_stream"], streams["comment_stream"], raw_content=comment_free_workflow
+    )["equations"]
+    assert equations["spec_exposure"] == 0
+
+    # and a real tagged comment still counts, via the comment stream
+    tagged = "# [SPEC-4412] pinned per the release spec\njobs:\n  build:\n    needs: [audit, lint]\n"
+    streams = prism.split_streams(tagged, "yaml")
+    equations = StructuralExtractor("yaml", LANGUAGE_DEFINITIONS).splice(
+        streams["code_stream"], streams["comment_stream"], raw_content=tagged
+    )["equations"]
+    assert equations["spec_exposure"] == 1
+
+    # ... including a TRAILING tag, which the `^[ \t]*#` anchor only reaches
+    # because prism re-emits an end-of-line comment on its own `#`-led line.
+    # That normalization is why the anchor costs no recall (yaml's own
+    # `dead_code` rule relies on exactly the same thing).
+    trailing = "jobs:\n  build:\n    steps:\n      - run: npm ci  # see [audit] trail\n"
+    streams = prism.split_streams(trailing, "yaml")
+    assert streams["comment_stream"] == "# see [audit] trail"
+    equations = StructuralExtractor("yaml", LANGUAGE_DEFINITIONS).splice(
+        streams["code_stream"], streams["comment_stream"], raw_content=trailing
+    )["equations"]
+    assert equations["spec_exposure"] == 1
+
+
+def test_yaml_spec_exposure_bare_spec_branch_is_word_bounded():
+    """
+    #2732: the generic rule's bare `spec` alternative has no trailing
+    boundary, so it matches any word starting "spec". That was not
+    hypothetical -- 2 of the 3 code-stream hits across the 41,815 .yml/.yaml
+    files in the pool corpus were `[specified\\n  per-machine]` (meson's docs)
+    and `[species]` (an elasticsearch test fixture), not tags at all.
+    """
+    spec_exposure = YAML_RULES["spec_exposure"]
+    assert not spec_exposure.search("# the value is [specified per-machine]")
+    assert not spec_exposure.search("# see the [species] list")
+    assert not spec_exposure.search("# the [auditor] signs off")
+
+    assert spec_exposure.search("# [spec] review pending")
+    assert spec_exposure.search("# [SPEC-77] change request")
+    assert spec_exposure.search("# [audit] trail retained")
+
+
+def test_yaml_spec_exposure_redos_immunity():
+    """
+    #2732: `[^\\n\\[]{0,200}` must be followed by a literal `[` and
+    `[^\\]\\n]{0,300}` by a literal `]`, so each bounded run has exactly one
+    landing site -- no ambiguous partition to backtrack over. Payloads are
+    long unterminated runs of exactly what each class accepts.
+    """
+    spec_exposure = YAML_RULES["spec_exposure"]
+    assert_redos_immune(spec_exposure, "# " + "a" * 100000, timeout_sec=3.0)
+    assert_redos_immune(spec_exposure, "# [spec" + "a" * 100000, timeout_sec=3.0)
+    assert_redos_immune(spec_exposure, "# " + "[" * 100000, timeout_sec=3.0)
+
+    assert spec_exposure.search("# [SPEC-4412] traceable")