Skip to content

fix(core-engine): shell reflection counts dispatch, not command substitution (#2722) #2725

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
squid-protocol merged 1 commit into from
Sep 4, 2026
Merged

fix(core-engine): shell reflection counts dispatch, not command substitution (#2722) #2725

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 21 additions & 8 deletions gitgalaxy/standards/language_standards/languages/shell.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,15 +209,28 @@
# 22. scientific (Numerical / Compute Libraries)
"scientific": re.compile(r"\b(bc|awk|dc|expr|jq|RANDOM|SRANDOM)\b|\$\(\("),
# 23. heat_triggers (Metaprogramming & Reflection)
# Sub-languages and indirect expansion. (ReDoS Shielded)
# QUADRATIC BLOWUP + NESTING FIX: `$(...)`'s flat `[^)]+` was
# unbounded and unanchored -- O(n^2) on a long run of unclosed
# `$(` (confirmed ~4x slowdown per input-size doubling). Upgraded
# to the one-level-nesting form so the common nested command
# substitution idiom (e.g. `DIR=$(cd "$(dirname "$0")" && pwd)`)
# is captured in full instead of truncating at the inner `)`.
# Sub-languages and indirect expansion -- the code whose behaviour is
# decided at runtime by a name, not written in the file. (ReDoS Shielded)
# VOCABULARY LEAK FIX (#2722): two alternatives counted ordinary shell
# as dynamism. (a) `\$\{!?...\}` made the indirection marker OPTIONAL,
# so plain `${var}` matched -- 4,117 of the crucible's 4,863 shell hits
# (85%) were variable expansions, while `${!var}` itself never occurs.
# (b) `$(...)` and backticks counted every command substitution, one per
# 33 lines of real shell: running a program yields data, not code, and no
# other language's rule counts invocation (python's does not fire on
# `subprocess.run`). Both dropped. `eval` is now unanchored -- the old
# `\beval\s+\$` missed `eval "$cmd"` -- and namerefs plus `source`/`.`
# of a computed path are added, which is what dynamic dispatch in shell
# actually looks like. Since #2719 this count IS the file's dynamism,
# read by documentation risk and cognitive load, so the leak was
# structural rather than cosmetic.
"reflection_metaprogramming": re.compile(
r'\$\((?:[^()]|\([^()]*\))+\)|`[^`]+`|\b(?:awk|sed|perl|python[23]?|ruby)\s+[\'"][^\'"]{0,500}|\beval\s+\$|\$\{!?[a-zA-Z0-9_]+\}'
r"\beval\b"
r"|\$\{![a-zA-Z0-9_]+\}"
r"|\b(?:declare|typeset|local)[ \t]+-n\b"
r"|(?:^|[ \t;|&])(?:source\b|\.(?=[ \t]))[ \t]+[\"']?\$"
r"|\b(?:awk|sed|perl|python[23]?|ruby)\s+['\"][^'\"]{0,500}",
re.M,
),
# 24. import (Dependency Inclusions)
"import": re.compile(r"(?:^|[ \t;|&])(?:source\b|\.(?=[ \t]))[ \t]+[^\s;]+", re.M),
Expand Down
92 changes: 67 additions & 25 deletions tests/extraction/languages/test_shell_strict.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ def test_shell_state_mutation_arithmetic_redos_immunity():
("globals", "echo $HOME", "echo $myvar"),
("comprehensions", "for i in {1..10}; do", "for i in 1 2 3; do"),
("scientific", "result=$(( 1 + 2 ))", "result=1"),
("reflection_metaprogramming", "output=$(date)", "output=static"),
("reflection_metaprogramming", 'eval "$cmd"', "output=$(date)"),
("import", "source ./lib.sh", "echo lib.sh"),
("ownership", "# Author: Jane Doe", "# just a note"),
# --- PHASE 4 ---
Expand Down Expand Up @@ -188,8 +188,9 @@ def test_shell_lexical_family_no_block_terminator_state_to_confuse():
payload -- there is nothing for a stray `}`/`fi` to falsely "close".
The one place shell rules DO track a nesting depth is delimiter
matching for `$(...)`, `<(...)`/`>(...)`, and `${...}` (safety,
concurrency, reflection_metaprogramming) -- covered by the dedicated
nested-delimiter regression tests below, not by comment-state tracking.
concurrency) -- covered by the dedicated nested-delimiter regression
tests below, not by comment-state tracking. (reflection_metaprogramming
used to be in that list; #2722 dropped command substitution from it.)
"""
branch = SHELL_RULES["branch"]
heredoc_body_with_fi = "cat <<EOF\nif true; then\n echo hi\nfi\nEOF\n"
Expand Down Expand Up @@ -306,35 +307,76 @@ def test_shell_concurrency_process_substitution_redos_immunity():
assert pattern.search("diff <(sort a) <(sort b)")


def test_shell_reflection_metaprogramming_nested_command_substitution_regression():
def test_shell_reflection_metaprogramming_is_dispatch_not_vocabulary():
"""
Nested-delimiter regression (Rule 11): `$(...)` (command substitution)
used a flat `[^)]+` delimiter matcher, which cannot represent one level
of nesting. A realistic nested command substitution -- e.g.
`DIR=$(cd "$(dirname "$0")" && pwd)`, the canonical "find my own script
directory" idiom -- truncated at the first (inner) `)` instead of
capturing the full outer substitution. Upgraded to the one-level-nesting
form.
"""
pattern = SHELL_RULES["reflection_metaprogramming"]
m = pattern.search('DIR=$(cd "$(dirname "$0")" && pwd)')
assert m and m.group() == '$(cd "$(dirname "$0")" && pwd)', (
f"nested command substitution truncated: {m.group() if m else None!r}"
)
assert pattern.search("echo $(date)"), "non-nested form regressed"
Vocabulary-leak regression (#2722). The rule counted two pieces of
ordinary shell as metaprogramming:

* `\\$\\{!?[a-zA-Z0-9_]+\\}` made the indirection marker OPTIONAL, so plain
`${var}` matched. On the crucible that was 4,117 of 4,863 shell hits
(85%) -- while `${!var}`, the construct the alternative exists for,
never occurs in the corpus at all.
* `$(...)` and backticks counted every command substitution, one hit per
~33 lines of real shell. Running a program yields data, not code, and
no other language's rule counts invocation (python's does not fire on
`subprocess.run`).

def test_shell_reflection_metaprogramming_command_substitution_redos_immunity():
Since #2719 this count IS the file's dynamism, feeding documentation risk
and cognitive load, so the leak was structural. What remains is dispatch
decided at runtime by a name: `eval`, indirect expansion, namerefs,
`source`/`.` of a computed path, and inline sub-language programs.

A nested-delimiter regression used to live here: `$(...)` had a flat
`[^)]+` matcher that truncated `DIR=$(cd "$(dirname "$0")" && pwd)` at
the inner `)`. That alternative is gone, so the idiom is now asserted as
a NEGATIVE. `safety` and `concurrency` keep their own nesting-aware
delimiter matchers and their own regressions.
"""
Regression test for a confirmed real O(n^2) ReDoS: `$(...)`'s flat
`[^)]+` was unbounded and unanchored -- quadratic on a long run of
unclosed `$(` (confirmed ~4x per doubling at n=2k/4k/8k/16k/32k, e.g.
0.006s/0.024s/0.094s/0.38s/1.5s) before being upgraded to the
one-level-nesting form, which is linear (~2x per doubling).
pattern = SHELL_RULES["reflection_metaprogramming"]
for src in (
'eval "$cmd"',
"eval :",
"echo ${!name}",
"declare -n ref=x",
"local -n out=$1",
'source "$dir/lib.sh"',
". $HOME/.env",
"awk '{print $1}' file",
):
assert pattern.search(src), f"runtime dispatch not counted: {src!r}"
for src in (
"echo ${var}",
'echo "${HOME}/bin"',
"output=$(date)",
'DIR=$(cd "$(dirname "$0")" && pwd)',
"files=`ls`",
"medieval=1",
"source ./lib.sh",
):
m = pattern.search(src)
assert not m, f"shell vocabulary counted as dynamism: {src!r} -> {m.group()!r}"


def test_shell_reflection_metaprogramming_redos_immunity():
"""
ReDoS coverage for the rule's surviving alternatives (#2722).

The historical bug was `$(...)`'s flat `[^)]+`: unbounded and unanchored,
quadratic on a long run of unclosed `$(` (confirmed ~4x per doubling at
n=2k/4k/8k/16k/32k, e.g. 0.006s/0.024s/0.094s/0.38s/1.5s). That
alternative no longer exists, so the input that provoked it is now
asserted to be immune *and* unmatched, and the check moves to the
alternatives that are still here: the unclosed indirect expansion
`${!`, the computed `source $`, and the inline sub-language program,
whose string run is bounded at {0,500}.
"""
pattern = SHELL_RULES["reflection_metaprogramming"]
assert_redos_immune(pattern, "${!" * 20000, timeout_sec=3.0)
assert_redos_immune(pattern, "source $" * 20000, timeout_sec=3.0)
assert_redos_immune(pattern, "awk '" + "a" * 100000, timeout_sec=3.0)
assert_redos_immune(pattern, "$(" * 20000, timeout_sec=3.0)
assert pattern.search("echo $(date)")
assert pattern.search("echo ${!ref}")
assert not pattern.search("echo $(date)")


def test_shell_spec_exposure_redos_immunity():
Expand Down
Loading
Loading