Skip to content

Commit f08eb62

Browse files
fix(truthbench): keep the faulthandler stack in sandbox crash failures (#366)
The generated-code sandbox reported a failed child with the last 800 characters of its stderr. faulthandler writes the crashing thread's stack first and a long "Extension modules: ..." line last, so a native crash was reported with only the module list and without the frame that located it. The finance/copilot case on PR #360 failed this way. Keep a faulthandler dump from its header, drop the module list line and cap it at 1500 characters. Other failures still keep the traceback tail.
1 parent f4d22bd commit f08eb62

2 files changed

Lines changed: 61 additions & 1 deletion

File tree

‎benchmarks/truthbench/generated_code.py‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,29 @@
6767

6868
_INPUT_BASENAME = "your_data.csv"
6969

70+
#: How much of a failed child's stderr a failure message carries.
71+
_STDERR_TAIL_CHARS = 800
72+
_CRASH_DUMP_CHARS = 1500
73+
74+
75+
def _stderr_excerpt(stderr: str) -> str:
76+
"""The useful part of a failed child's stderr for its failure message.
77+
78+
A normal traceback ends at the bottom, so keep the tail. A faulthandler
79+
dump is the opposite: the stack that locates a native crash comes first and
80+
a long ``Extension modules: ...`` line comes last, so a tail keeps only the
81+
module list. For a dump, keep it from its header and drop that line.
82+
"""
83+
start = stderr.find("Fatal Python error:")
84+
if start == -1:
85+
return stderr[-_STDERR_TAIL_CHARS:]
86+
dump = "\n".join(
87+
line
88+
for line in stderr[start:].splitlines()
89+
if not line.startswith("Extension modules:")
90+
)
91+
return dump[:_CRASH_DUMP_CHARS]
92+
7093

7194
@dataclass(frozen=True)
7295
class GeneratedCodeResult:
@@ -228,7 +251,8 @@ def leaks(label: str, payload: Any) -> None:
228251
if scanner is not None:
229252
safe_stderr = scanner.redact(stderr)
230253
failures.append(
231-
f"generated code exited {proc.returncode}: {str(safe_stderr)[-800:]}"
254+
f"generated code exited {proc.returncode}: "
255+
f"{_stderr_excerpt(str(safe_stderr))}"
232256
)
233257

234258
produced = tuple(

‎tests/truthbench/test_generated_code.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,3 +182,39 @@ def crashed_child(args, **kwargs):
182182
[failure] = [f for f in result.failures if "exited" in f]
183183
assert f"exited {-signal.SIGSEGV}" in failure
184184
assert "Segmentation fault" in failure
185+
186+
187+
def test_native_crash_failure_keeps_stack_not_module_list(monkeypatch):
188+
# faulthandler writes the stack first and a long "Extension modules" line
189+
# last. Keeping only the tail of stderr reported just the module list and
190+
# lost the frame that located the crash (seen in CI on a copilot case).
191+
modules = ", ".join(f"pandas._libs.module_{i}" for i in range(53))
192+
dump = (
193+
"Fatal Python error: Segmentation fault\n\n"
194+
"Current thread 0x0000000000000001 (most recent call first):\n"
195+
' File "pandas/core/tools/numeric.py", line 235 in to_numeric\n'
196+
' File "generated_pipeline.py", line 20 in <module>\n\n'
197+
f"Extension modules: {modules} (total: 53)\n"
198+
)
199+
assert len(dump) > 800
200+
201+
def crashed_child(args, **kwargs):
202+
return subprocess.CompletedProcess(args, -signal.SIGSEGV, "", dump)
203+
204+
monkeypatch.setattr(gc.subprocess, "run", crashed_child)
205+
result = verify_generated_code(GOOD, _fixture())
206+
[failure] = [f for f in result.failures if "exited" in f]
207+
assert "line 235 in to_numeric" in failure
208+
assert "Extension modules" not in failure
209+
210+
211+
def test_ordinary_failure_keeps_traceback_tail(monkeypatch):
212+
stderr = "noise\n" * 300 + "ValueError: bad column\n"
213+
214+
def failed_child(args, **kwargs):
215+
return subprocess.CompletedProcess(args, 1, "", stderr)
216+
217+
monkeypatch.setattr(gc.subprocess, "run", failed_child)
218+
result = verify_generated_code(GOOD, _fixture())
219+
[failure] = [f for f in result.failures if "exited" in f]
220+
assert failure.endswith("ValueError: bad column\n")

0 commit comments

Comments
 (0)