From 0eaf52f3f8d7e24320b7345c7821cb8411f425d0 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 00:53:19 +0000 Subject: [PATCH 1/6] Retain bounded native fault metadata for unit workers Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 13 + docs/development/testing.md | 22 ++ tests/_native_gdb.py | 212 +++++++++++++ tests/_native_worker.py | 293 ++++++++++++++++++ tests/_worker_status.py | 58 +++- tests/unit/test_native_worker.py | 517 +++++++++++++++++++++++++++++++ 6 files changed, 1112 insertions(+), 3 deletions(-) create mode 100644 tests/_native_gdb.py create mode 100644 tests/_native_worker.py create mode 100644 tests/unit/test_native_worker.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5201b0d62..d69e5208d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,6 +190,14 @@ jobs: clickhouse local --query "SELECT version()" # Random order over one worker per core, so a test that depends on another # fails here; reproduce a failure as tests/README.md describes. + - name: Install the native worker debugger + if: matrix.part == 3 + run: | + sudo apt-get update -qq + sudo apt-get install --no-install-recommends -y gdb + - name: Check native worker transport and termination + if: matrix.part == 3 + run: python -m tests.unit.test_native_worker --mechanism - name: pytest unit (this part, random order) env: LOOM_TEST_SHARD: ${{ matrix.part }}/${{ strategy.job-total }} @@ -197,7 +205,12 @@ jobs: run: | SEED="${{ github.run_id }}${{ github.run_attempt }}" echo "pytest-randomly seed: $SEED" + NATIVE_ARGS=() + if [ "${{ matrix.part }}" = 3 ]; then + NATIVE_ARGS=(--native-worker-signatures) + fi pytest tests/unit -vv -n auto -p randomly -p tests._worker_status --randomly-seed="$SEED" --durations=15 \ + "${NATIVE_ARGS[@]}" \ -o faulthandler_timeout=300 \ --cov=server --cov=sdk/python/loom --cov-branch --cov-report= \ --junitxml=junit-unit-${{ matrix.part }}.xml diff --git a/docs/development/testing.md b/docs/development/testing.md index d1c4729681..46052569e3 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -49,6 +49,28 @@ Keep CI's system tools beside the locked Python environment. Release-script test example, call `jq`. Record missing tools or an unsupported platform separately from a failure introduced by the diff. +### Native worker diagnostics + +The third unit part runs local workers under an ancestor debugger. Their interpreter, +arguments, environment, parallelism, random seed, coverage and timeout settings stay +the same. The launcher preserves normal exit codes and terminal signals separately. +GDB is a CI test tool; it is not a server dependency. The controlled transport check is +`python -m tests.unit.test_native_worker --mechanism`. + +`-p tests._worker_status --native-worker-signatures` opts into this collection on Linux +with GDB and an existing zero core-size limit. Missing prerequisites or a tracing +refusal stop startup. No tracing policy or capability changes are made. Each helper +sets its own parent-death signal so forced launcher termination also stops its debugger +and worker. Registrations never authorize controller signals to another process. + +Private session artifacts under `.pytest_cache/worker-status/` contain bounded native +PCs, symbols, module identities and offsets for up to eight threads and 24 frames per +thread. Up to eight observed fatal-signal stops are retained. A stop may be handled by +the worker; the separate terminal result records whether it eventually exited or died +from a signal. A bounded fault-thread summary also appears in the job log. Missing +symbols or mappings remain unresolved. Raw debugger output is discarded; core dumps, +arguments, locals, environment values and memory bytes are not collected. + ## Integration tests The integration suite runs against a live server, the way CI's `integration` job does: diff --git a/tests/_native_gdb.py b/tests/_native_gdb.py new file mode 100644 index 0000000000..c27fa237a5 --- /dev/null +++ b/tests/_native_gdb.py @@ -0,0 +1,212 @@ +"""GDB entry point: retain native metadata, then deliver the original signal.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import signal +import sys +from pathlib import Path +from typing import Any + +import gdb + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _native_worker as native + +MAX_THREADS = 8 +MAX_FRAMES = 24 +FATAL = {"SIGSEGV", "SIGBUS", "SIGILL", "SIGFPE", "SIGABRT"} + + +def integer(expression: str) -> int | None: + value = gdb.parse_and_eval(expression) + return None if value.type.code == gdb.TYPE_CODE_VOID else int(value) + + +def symbol_name(name: str | None) -> str | None: + if name is None: + return None + if len(name) <= 160 and re.fullmatch(r"[A-Za-z_][A-Za-z_0-9.:<>~]*", name): + return name + return "sha256:" + hashlib.sha256(name.encode()).hexdigest() + + +def module_name(path: str) -> str: + name = Path(path).name + if re.fullmatch( + r"(?:python[0-9.]*|lib(?:python|c|m|pthread|dl|gcc_s|stdc\+\+)[0-9.]*\.so[0-9.]*)", name + ): + return name + return "sha256:" + hashlib.sha256(path.encode()).hexdigest() + + +def mappings() -> list[tuple[int, int, int, str]]: + # Read a bounded mapping prefix privately; never retain paths or raw map text. + with open(f"/proc/{gdb.selected_inferior().pid}/maps", "rb") as source: + data = source.read(262144) + result = [] + for line in data.decode(errors="replace").splitlines()[:-1]: + columns = line.split(maxsplit=5) + if len(columns) == 6: + start, end = (int(value, 16) for value in columns[0].split("-")) + result.append((start, end, int(columns[2], 16), columns[5])) + return result + + +def frame_record(frame: Any, regions: list[tuple[int, int, int, str]]) -> dict[str, Any]: + pc = int(frame.pc()) + result: dict[str, Any] = {"pc": pc, "symbol": symbol_name(frame.name())} + for start, end, offset, filename in regions: + if start <= pc < end: + result["module"] = module_name(filename) + result["file_offset"] = pc - start + offset + for objfile in gdb.objfiles(): + if os.path.realpath(objfile.filename) == os.path.realpath(filename): + build_id = objfile.build_id + if build_id and re.fullmatch(r"[0-9a-fA-F]{1,128}", build_id): + result["build_id"] = build_id + break + break + try: + block = gdb.block_for_pc(pc) + while block is not None and block.function is None: + block = block.superblock + if block is not None: + result["symbol_offset"] = pc - block.start + except gdb.error: + pass # Stripped modules still have their PC and build identity when available. + return result + + +def signature(received: str) -> dict[str, Any]: + faulting = gdb.selected_thread() + regions = mappings() + result: dict[str, Any] = {"signal": received, "threads": [], "unwind_complete": True} + for field, expression in (("si_code", "$_siginfo.si_code"),): + try: + result[field] = integer(expression) + except gdb.error: + result[field] = None + if result["si_code"] is not None and result["si_code"] > 0: + try: + result["si_addr"] = integer("$_siginfo._sifields._sigfault.si_addr") + except gdb.error: + result["si_addr"] = None + threads = [faulting, *(t for t in gdb.selected_inferior().threads() if t != faulting)] + result["threads_truncated"] = len(threads) > MAX_THREADS + try: + for thread in threads[:MAX_THREADS]: + thread.switch() + entry: dict[str, Any] = { + "faulting": thread == faulting, + "lwp": thread.ptid[1], + "frames": [], + } + result["threads"].append(entry) + try: + frame = gdb.newest_frame() + for _ in range(MAX_FRAMES): + if frame is None: + break + entry["frames"].append(frame_record(frame, regions)) + frame = frame.older() + entry["frames_truncated"] = frame is not None + except Exception: # noqa: BLE001 — unwinding must never consume the original signal + result["unwind_complete"] = False + finally: + faulting.switch() + while len(json.dumps(result, ensure_ascii=True).encode()) > native.MAX_BYTES - 1024: + # Preserve the faulting thread first when symbol names fill the byte budget. + for entry in reversed(result["threads"]): + if entry["frames"]: + entry["frames"].pop() + entry["frames_truncated"] = True + break + return result + + +def run() -> None: + context = json.loads(os.environ[native.CONTEXT]) + directory = Path(context["directory"]) + stops = 0 + trap = False + trap_stops = 0 + + def on_stop(event: Any) -> None: + nonlocal stops, trap, trap_stops + trap = isinstance(event, gdb.SignalEvent) and event.stop_signal == "SIGTRAP" + if not isinstance(event, gdb.SignalEvent): + # Some targets report a user SIGTRAP as StopEvent rather than SignalEvent. + try: + trap = integer("$_siginfo.si_signo") == signal.SIGTRAP + except gdb.error: + pass + if trap: + trap_stops += 1 + if isinstance(event, gdb.SignalEvent) and event.stop_signal in FATAL: + stops += 1 + if stops > native.MAX_STOPS: + return + path = directory / f"signature-{stops}.json" + try: + native.write_record(path, {"observed_stop": stops, **signature(event.stop_signal)}) + except Exception: # noqa: BLE001 — a collector failure must still deliver the signal + try: + native.write_record(path, {"observed_stop": stops, "capture": "unavailable"}) + except OSError: + pass + + gdb.events.stop.connect(on_stop) + # Preserve ASLR and all delivered signals, including GDB's special defaults. + for command in ( + "set pagination off", + "set confirm off", + "set follow-fork-mode parent", + "set detach-on-fork on", + "set follow-exec-mode same", + "handle all nostop noprint pass", + "handle SIGINT nostop noprint pass", + # Internal exec traps stay with GDB; the loop delivers actual inferior SIGTRAPs. + "handle SIGTRAP stop print nopass", + *(f"handle {name} stop print pass" for name in sorted(FATAL)), + ): + gdb.execute(command, to_string=True) + result: dict[str, Any] + try: + gdb.execute("run", to_string=False) + while gdb.selected_inferior().pid: + # stop handlers only collect; this resumes with the original signal policy. + command = "signal SIGTRAP" if trap else "continue" + trap = False + gdb.execute(command, to_string=False) + exit_code, exit_signal = integer("$_exitcode"), integer("$_exitsignal") + if not (directory / "inferior.json").exists(): + result = { + "kind": "unavailable", + "reason": "inferior_not_started", + "exit_code": exit_code, + "exit_signal": exit_signal, + } + elif exit_signal is not None: + result = {"kind": "signal", "signal": exit_signal} + elif exit_code is not None: + result = {"kind": "exit", "code": exit_code} + else: + result = {"kind": "unavailable", "reason": "terminal_status_unknown"} + except gdb.error as error: + refused = any( + token in str(error).lower() + for token in ("ptrace", "not permitted", "permission denied") + ) + result = {"kind": "unavailable", "reason": "trace_refused" if refused else "debugger_error"} + result["observed_fatal_stops"] = stops + result["captured_stops"] = min(stops, native.MAX_STOPS) + result["trap_stops"] = trap_stops + result["gdb_version"] = gdb.VERSION[:64] + native.write_record(directory / "result.json", result) + + +run() diff --git a/tests/_native_worker.py b/tests/_native_worker.py new file mode 100644 index 0000000000..26e34f0cba --- /dev/null +++ b/tests/_native_worker.py @@ -0,0 +1,293 @@ +"""Opt-in ancestor debugger transport and exact owned-process cleanup for xdist.""" + +from __future__ import annotations + +import ctypes +import json +import os +import shlex +import shutil +import signal +import stat +import subprocess +import sys +from pathlib import Path +from typing import Any + +CONTEXT = "_PYTEST_NATIVE_CONTEXT" +MAX_WORKERS = 64 +MAX_BYTES = 65536 +MAX_STOPS = 8 + + +def write_record(path: Path, record: dict[str, Any]) -> None: + data = json.dumps(record, ensure_ascii=True, separators=(",", ":")).encode() + if len(data) > MAX_BYTES: + raise ValueError("native record too large") + # A hard link publishes a complete record and refuses an existing destination. + temporary = path.with_suffix(".tmp") + created = False + try: + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + created = True + with os.fdopen(descriptor, "wb") as output: + output.write(data) + os.link(temporary, path) + finally: + if created: + temporary.unlink(missing_ok=True) + + +def read_record(path: Path) -> dict[str, Any]: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(descriptor, "rb") as source: + info = os.fstat(source.fileno()) + if ( + not stat.S_ISREG(info.st_mode) + or info.st_uid != os.getuid() + or info.st_mode & 0o077 + or info.st_size > MAX_BYTES + ): + raise ValueError("unsafe native record") + result = json.loads(source.read(MAX_BYTES + 1)) + if not isinstance(result, dict): + raise ValueError("invalid native record") + return result + + +def identity(pid: int) -> dict[str, int]: + base = Path("/proc") / str(pid) + fields = (base / "stat").read_text().rsplit(")", 1)[1].split() + return { + "pid": pid, + "ppid": int(fields[1]), + "start": int(fields[19]), + "uid": base.stat().st_uid, + "zombie": int(fields[0] == "Z"), + } + + +def parent_death_signal(expected_parent: int) -> None: + """Tie this helper's lifetime to its creator without granting tracing permissions.""" + libc = ctypes.CDLL(None, use_errno=True) + prctl = libc.prctl + prctl.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong] + prctl.restype = ctypes.c_int + # PR_SET_PDEATHSIG affects only this process; no tracing or access policy changes. + if prctl(1, signal.SIGKILL, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), "native helper parent-death registration refused") + if os.getppid() != expected_parent: + os.kill(os.getpid(), signal.SIGKILL) + + +class NativeSession: + def __init__(self, root: Path) -> None: + if sys.platform != "linux": + raise ValueError("native diagnostics require Linux") + import resource + + if resource.getrlimit(resource.RLIMIT_CORE)[0] != 0: + raise ValueError("native diagnostics require an existing zero core limit") + if CONTEXT in os.environ: + raise ValueError("native diagnostic environment is already in use") + debugger = shutil.which("gdb") + if debugger is None: + raise ValueError("native diagnostics require gdb") + self.root = root / "native" + self.root.mkdir(mode=0o700) + self.finished = False + write_record(self.root / "config.json", {"debugger": debugger}) + + def configure(self, specs: list[Any], max_restarts: int | None = None) -> None: + restarts = 4 * len(specs) if max_restarts is None else max_restarts + if restarts < 0 or len(specs) + restarts > MAX_WORKERS: + raise ValueError("native diagnostic worker limit exceeded") + for spec in specs: + if not spec.popen or spec.via or spec.python: + raise ValueError("native diagnostics require local default-interpreter workers") + for spec in specs: + spec.python = shlex.join( + [sys.executable, str(Path(__file__).resolve()), "--launch", str(self.root)] + ) + + def observe(self, pid: int) -> dict[str, Any]: + result: dict[str, Any] = {"transport": "ancestor_debugger_launcher"} + directory = self.root / str(pid) + try: + result["inferior_pid"] = read_record(directory / "inferior.json")["pid"] + except (OSError, ValueError, KeyError): + pass + for filename in ("result", *(f"signature-{i}" for i in range(1, MAX_STOPS + 1))): + try: + result["signature" if filename.startswith("signature-") else filename] = ( + read_record(directory / f"{filename}.json") + ) + except (OSError, ValueError): + pass # The launcher may still be running or may have been killed. + if "signature" in result: + signature = result["signature"] + threads = signature.get("threads", []) + result["signature"] = { + key: signature[key] + for key in ("observed_stop", "signal", "si_code", "si_addr", "capture") + if key in signature + } + result["signature"]["faulting_thread"] = ( + {**threads[0], "frames": threads[0]["frames"][:3]} if threads else None + ) + return result + + def finish(self) -> dict[str, Any]: + if self.finished: + return {"event": "native_cleanup", "already_finished": True} + self.finished = True + remaining = unavailable = 0 + reasons: dict[str, int] = {} + # Read only bounded private registrations. They never authorize a signal. + for slot in range(MAX_WORKERS): + try: + launcher = read_record(self.root / f"slot-{slot}.json") + if type(launcher["pid"]) is not int or launcher["pid"] < 2: + raise ValueError("invalid launcher") + directory = self.root / str(launcher["pid"]) + except FileNotFoundError: + continue + except (OSError, ValueError, KeyError, TypeError): + unavailable += 1 + continue + for filename in ("inferior", "debugger"): + try: + expected = read_record(directory / f"{filename}.json") + found = identity(expected["pid"]) + remaining += not found["zombie"] and all( + found[key] == expected[key] for key in ("pid", "start", "uid") + ) + except FileNotFoundError: + continue + except (OSError, ValueError, KeyError, TypeError): + unavailable += 1 + try: + reason = read_record(directory / "result.json").get("reason") + if reason in ( + "trace_refused", + "debugger_error", + "inferior_not_started", + "terminal_status_unknown", + "helper_refused", + ): + reasons[reason] = reasons.get(reason, 0) + 1 + except (OSError, ValueError): + unavailable += 1 + return { + "event": "native_cleanup", + "remaining": remaining, + "unavailable": unavailable, + "reasons": reasons, + } + + +def bootstrap() -> None: + context = json.loads(os.environ.pop(CONTEXT)) + directory = Path(context["directory"]) + try: + parent_death_signal(read_record(directory / "debugger.json")["pid"]) + except OSError: + write_record(directory / "result.json", {"kind": "unavailable", "reason": "helper_refused"}) + raise SystemExit(125) from None + write_record(directory / "inferior.json", identity(os.getpid())) + for target, saved in enumerate(context["stdio"]): + os.dup2(saved, target) + os.close(saved) + # GDB may add terminal variables; restore exactly the original worker environment. + os.environ.clear() + os.environ.update(context["environment"]) + os.execv(context["python"], [context["python"], *context["arguments"]]) + + +def launch(root: Path, arguments: list[str]) -> int: + directory = root / str(os.getpid()) + directory.mkdir(mode=0o700) + for slot in range(MAX_WORKERS): + try: + write_record(root / f"slot-{slot}.json", identity(os.getpid())) + break + except FileExistsError: + continue + else: + return 125 + descriptors = [os.dup(fd) for fd in range(3)] + context = { + "directory": str(directory), + "launcher_pid": os.getpid(), + "stdio": descriptors, + "python": sys.executable, + "arguments": arguments, + "environment": dict(os.environ), + } + environment = {**os.environ, CONTEXT: json.dumps(context)} + debugger = read_record(root / "config.json")["debugger"] + command = [ + debugger, + "-q", + "-nx", + "-nh", + "-batch", + "-iex", + "set auto-load off", + "-iex", + "set debuginfod enabled off", + "-ex", + "set disable-randomization off", + "-ex", + "set startup-with-shell off", + "-x", + str(Path(__file__).with_name("_native_gdb.py")), + "--args", + sys.executable, + str(Path(__file__).resolve()), + "--bootstrap", + ] + try: + with subprocess.Popen( + [sys.executable, str(Path(__file__).resolve()), "--debugger-bootstrap", *command], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + pass_fds=descriptors, + env=environment, + ) as process: + for descriptor in descriptors: + os.close(descriptor) + descriptors.clear() + process.wait() + result = read_record(directory / "result.json") + if result.get("kind") == "exit": + return result["code"] + if result.get("kind") == "signal" and result.get("signal") in signal.valid_signals(): + received = result["signal"] + if received not in (signal.SIGKILL, signal.SIGSTOP): + signal.signal(received, signal.SIG_DFL) + os.kill(os.getpid(), received) + return 125 + finally: + for descriptor in descriptors: + os.close(descriptor) + + +if __name__ == "__main__": + if sys.argv[1] == "--bootstrap": + bootstrap() + elif sys.argv[1] == "--debugger-bootstrap": + context = json.loads(os.environ[CONTEXT]) + directory = Path(context["directory"]) + try: + parent_death_signal(context["launcher_pid"]) + write_record(directory / "debugger.json", identity(os.getpid())) + os.execv(sys.argv[2], sys.argv[2:]) + except OSError: + write_record( + directory / "result.json", {"kind": "unavailable", "reason": "helper_refused"} + ) + raise SystemExit(125) from None + else: + raise SystemExit(launch(Path(sys.argv[2]), sys.argv[3:])) diff --git a/tests/_worker_status.py b/tests/_worker_status.py index c7bebc9ac7..a5e36402bc 100644 --- a/tests/_worker_status.py +++ b/tests/_worker_status.py @@ -12,6 +12,8 @@ import pytest +from tests._native_worker import NativeSession + if TYPE_CHECKING: from collections.abc import Generator @@ -39,6 +41,7 @@ def __init__(self, config: pytest.Config) -> None: self.config = config self.handles: dict[WorkerController, Any] = {} self.records = 0 + self.max_records = MAX_RECORDS self.path: Path | None = None self.dropped_handles = 0 try: @@ -47,12 +50,39 @@ def __init__(self, config: pytest.Config) -> None: self.path = Path(tempfile.mkdtemp(prefix="session-", dir=root)) / "status.jsonl" except OSError: pass # The job log still carries observations when artifact creation fails. + self.native: NativeSession | None = None + if config.getoption("native_worker_signatures", default=False): + if self.path is None: + raise pytest.UsageError("native diagnostics require a private artifact directory") + try: + self.native = NativeSession(self.path.parent) + except (OSError, ValueError) as error: + raise pytest.UsageError(str(error)) from None + self.max_records = 2 * MAX_RECORDS + config.add_cleanup(self.native_cleanup) + + @pytest.hookimpl(optionalhook=True) + def pytest_xdist_setupnodes(self, specs: list[Any]) -> None: + if self.native is not None: + try: + self.native.configure( + specs, self.config.getoption("maxworkerrestart", default=None) + ) + except ValueError as error: + raise pytest.UsageError(str(error)) from None + + def native_cleanup(self) -> None: + if self.native is not None: + try: + self.emit(self.native.finish()) + except Exception: # noqa: BLE001 — cleanup diagnostics must preserve pytest's result + self.emit({"event": "native_cleanup", "capture": "unavailable"}) def emit(self, record: dict[str, Any]) -> None: - if self.records >= MAX_RECORDS: + if self.records >= self.max_records: return - if self.records == MAX_RECORDS - 1: - record = {"event": "record_limit", "limit": MAX_RECORDS} + if self.records == self.max_records - 1: + record = {"event": "record_limit", "limit": self.max_records} self.records += 1 record = {**record, "sequence": self.records, "artifact_available": self.path is not None} line = _encode(record) @@ -124,12 +154,26 @@ def observe(self, node: WorkerController, stage: str, error: object | None = Non except Exception as e: # noqa: BLE001 — diagnostic failure must not replace pytest's outcome state = type(e).__name__[:64] output = getattr(node, "workeroutput", {}) + if self.native is not None and process is not None: + try: + native = self.native.observe(process.pid) + except Exception: # noqa: BLE001 — diagnostic failure must preserve worker status + native = {"capture": "unavailable"} + self.emit( + { + "event": "native_worker", + "worker": node.gateway.id[:64], + "pid": process.pid, + **native, + } + ) self.emit( { "event": "worker_status", "stage": stage, "worker": node.gateway.id[:64], "pid": getattr(process, "pid", None), + "process_role": "native_launcher" if self.native is not None else "pytest_worker", "os_state": state, "os_returncode": returncode, "os_signal": -returncode if returncode is not None and returncode < 0 else None, @@ -162,6 +206,14 @@ def pytest_configure(config: pytest.Config) -> None: config.pluginmanager.register(WorkerStatus(config), "worker-terminal-status") +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--native-worker-signatures", + action="store_true", + help="Capture bounded native worker fault metadata with an ancestor debugger", + ) + + def _encode(record: dict[str, Any]) -> str: line = json.dumps(record, ensure_ascii=True, separators=(",", ":")) + "\n" if len(line.encode()) > MAX_RECORD_BYTES: diff --git a/tests/unit/test_native_worker.py b/tests/unit/test_native_worker.py new file mode 100644 index 0000000000..65463b0f9d --- /dev/null +++ b/tests/unit/test_native_worker.py @@ -0,0 +1,517 @@ +"""Native diagnostics preserve signals, private transport and process ownership.""" + +from __future__ import annotations + +import hashlib +import json +import os +import resource +import runpy +import select +import signal +import stat +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from xml.etree import ElementTree + +import pytest + +from tests import _native_worker as native + + +@pytest.fixture +def session(tmp_path, monkeypatch): + monkeypatch.setattr(resource, "getrlimit", lambda _: (0, 0)) + monkeypatch.setattr(native.shutil, "which", lambda _: "/synthetic/gdb") + monkeypatch.delenv(native.CONTEXT, raising=False) + return native.NativeSession(tmp_path) + + +def test_records_are_private_bounded_and_never_overwrite(tmp_path): + path = tmp_path / "record.json" + native.write_record(path, {"kind": "exit", "code": 139}) + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + with pytest.raises(FileExistsError): + native.write_record(path, {"kind": "signal", "signal": 11}) + assert native.read_record(path) == {"kind": "exit", "code": 139} + with pytest.raises(ValueError, match="too large"): + native.write_record(tmp_path / "large.json", {"value": "x" * native.MAX_BYTES}) + assert not (tmp_path / "large.json").exists() + link = tmp_path / "link.json" + link.symlink_to(path) + with pytest.raises(OSError): + native.read_record(link) + path.chmod(0o644) + with pytest.raises(ValueError, match="unsafe"): + native.read_record(path) + assert not list(tmp_path.glob("*.tmp")) + + +@pytest.mark.parametrize("reason", ["core", "debugger", "environment"]) +def test_unavailable_capture_refuses_before_creating_session(tmp_path, monkeypatch, reason): + monkeypatch.setattr(resource, "getrlimit", lambda _: (1 if reason == "core" else 0, 1)) + monkeypatch.setattr(native.shutil, "which", lambda _: None if reason == "debugger" else "gdb") + monkeypatch.delenv(native.CONTEXT, raising=False) + if reason == "environment": + monkeypatch.setenv(native.CONTEXT, "occupied") + monkeypatch.setattr( + subprocess, "Popen", lambda *a, **k: pytest.fail("refusal must not start a fallback") + ) + with pytest.raises(ValueError): + native.NativeSession(tmp_path) + assert not (tmp_path / "native").exists() + + +def test_worker_configuration_rejects_unsupported_specs_without_partial_changes(session): + valid = SimpleNamespace(popen=True, via=None, python=None) + invalid = SimpleNamespace(popen=True, via=None, python="custom-python") + with pytest.raises(ValueError): + session.configure([valid, invalid]) + assert valid.python is None + with pytest.raises(ValueError, match="limit"): + session.configure([valid] * (native.MAX_WORKERS + 1)) + assert valid.python is None + + +@pytest.mark.parametrize("state", ["registered", "refused", "parent_changed"]) +def test_parent_death_registration_never_grants_access_or_signals_another_process( + monkeypatch, state +): + calls = [] + + class Prctl: + def __call__(self, *arguments): + calls.append(arguments) + return -1 if state == "refused" else 0 + + monkeypatch.setattr(native.ctypes, "CDLL", lambda *a, **k: SimpleNamespace(prctl=Prctl())) + monkeypatch.setattr(native.ctypes, "get_errno", lambda: 1) + monkeypatch.setattr(os, "getppid", lambda: 123 if state != "parent_changed" else 321) + monkeypatch.setattr(os, "getpid", lambda: 456) + signals = [] + monkeypatch.setattr(os, "kill", lambda *arguments: signals.append(arguments)) + if state == "refused": + with pytest.raises(OSError) as refused: + native.parent_death_signal(123) + assert refused.value.errno == 1 + else: + native.parent_death_signal(123) + assert calls == [(1, signal.SIGKILL, 0, 0, 0)] + assert signals == ([(456, signal.SIGKILL)] if state == "parent_changed" else []) + + +@pytest.mark.parametrize("code", [0, 7, 139]) +def test_launcher_preserves_normal_exit_codes_and_keeps_arguments_private( + session, monkeypatch, code +): + canary = "private-argument-canary" + + class Debugger: + def __init__(self, command, **kwargs): + assert kwargs["stdout"] is subprocess.DEVNULL + assert kwargs["stderr"] is subprocess.DEVNULL + assert canary not in command + context = json.loads(kwargs["env"][native.CONTEXT]) + assert context["arguments"] == ["-c", "raise SystemExit(139)", canary] + self.directory = Path(context["directory"]) + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def wait(self): + native.write_record(self.directory / "result.json", {"kind": "exit", "code": code}) + return 0 + + monkeypatch.setattr(subprocess, "Popen", Debugger) + assert native.launch(session.root, ["-c", "raise SystemExit(139)", canary]) == code + assert all(canary not in path.read_text() for path in session.root.rglob("*.json")) + + +def test_exhausted_launch_slots_never_start_an_unobserved_worker(session, monkeypatch): + for slot in range(native.MAX_WORKERS): + native.write_record(session.root / f"slot-{slot}.json", {"pid": slot + 2}) + monkeypatch.setattr(subprocess, "Popen", lambda *a, **k: pytest.fail("slot exhaustion")) + assert native.launch(session.root, ["-c", "raise SystemExit(0)"]) == 125 + assert len(list(session.root.glob("slot-*.json"))) == native.MAX_WORKERS + + +@pytest.mark.parametrize("start_offset", [0, 1]) +def test_forged_same_uid_records_cannot_authorize_cleanup(session, monkeypatch, start_offset): + launcher = {"pid": 100001, "ppid": 2, "start": 100, "uid": os.getuid()} + debugger = {**launcher, "pid": 100002, "ppid": launcher["pid"], "start": 101} + inferior = {**launcher, "pid": 100003, "ppid": debugger["pid"], "start": 102} + native.write_record(session.root / "slot-0.json", launcher) + directory = session.root / str(launcher["pid"]) + directory.mkdir(mode=0o700) + native.write_record(directory / "debugger.json", debugger) + native.write_record(directory / "inferior.json", inferior) + owners = {debugger["pid"]: debugger, inferior["pid"]: inferior} + monkeypatch.setattr( + native, + "identity", + lambda pid: {**owners[pid], "start": owners[pid]["start"] + start_offset, "zombie": 0}, + ) + monkeypatch.setattr( + os, "pidfd_open", lambda *args: pytest.fail("disk records cannot authorize cleanup") + ) + monkeypatch.setattr( + os, "kill", lambda *args: pytest.fail("disk records cannot authorize signals") + ) + monkeypatch.setattr( + signal, + "pidfd_send_signal", + lambda *args: pytest.fail("disk records cannot authorize signals"), + ) + result = session.finish() + assert result["remaining"] == (2 if start_offset == 0 else 0) + assert session.finish() == {"event": "native_cleanup", "already_finished": True} + + +def test_debugger_permission_refusal_is_reported_without_fallback(session, monkeypatch): + class DebuggerError(Exception): + pass + + commands = [] + + def execute(command, **kwargs): + commands.append(command) + if command == "run": + raise DebuggerError("ptrace: Operation not permitted private-error-canary") + return "" + + fake = SimpleNamespace( + VERSION="synthetic-gdb", + error=DebuggerError, + SignalEvent=type("SignalEvent", (), {}), + events=SimpleNamespace(stop=SimpleNamespace(connect=lambda callback: None)), + execute=execute, + ) + directory = session.root / "synthetic-debugger" + directory.mkdir(mode=0o700) + monkeypatch.setenv(native.CONTEXT, json.dumps({"directory": str(directory)})) + monkeypatch.setitem(sys.modules, "gdb", fake) + monkeypatch.setitem(sys.modules, "_native_worker", native) + monkeypatch.syspath_prepend(str(Path(native.__file__).parent)) + runpy.run_path(str(Path(native.__file__).with_name("_native_gdb.py"))) + assert commands[-1] == "run" + result = native.read_record(directory / "result.json") + assert result == { + "kind": "unavailable", + "reason": "trace_refused", + "observed_fatal_stops": 0, + "captured_stops": 0, + "trap_stops": 0, + "gdb_version": "synthetic-gdb", + } + assert all("private-error-canary" not in path.read_text() for path in directory.iterdir()) + + +@pytest.mark.parametrize("failure", ["unwind", "write"]) +def test_collector_failure_still_continues_and_preserves_terminal_signal( + session, monkeypatch, failure +): + class DebuggerError(Exception): + pass + + class SignalEvent: + stop_signal = "SIGSEGV" + + class Value: + def __init__(self, value): + self.value = value + self.type = SimpleNamespace(code=0 if value is None else 1) + + def __int__(self): + return self.value + + commands, callbacks = [], [] + inferior = SimpleNamespace(pid=1) + + def execute(command, **kwargs): + commands.append(command) + if command == "run": + callbacks[0](SignalEvent()) + elif command == "continue": + inferior.pid = 0 + return "" + + def no_unwind(): + raise DebuggerError("private-unwind-canary") + + directory = session.root / "failed-collector" + directory.mkdir(mode=0o700) + native.write_record(directory / "inferior.json", {"pid": 1}) + original_write = native.write_record + + def write(path, record): + if failure == "write" and path.name.startswith("signature-"): + raise OSError("private-write-canary") + original_write(path, record) + + fake = SimpleNamespace( + VERSION="synthetic-gdb", + error=DebuggerError, + SignalEvent=SignalEvent, + TYPE_CODE_VOID=0, + events=SimpleNamespace(stop=SimpleNamespace(connect=callbacks.append)), + execute=execute, + selected_inferior=lambda: inferior, + selected_thread=no_unwind, + parse_and_eval=lambda expression: Value(None if expression == "$_exitcode" else 11), + ) + monkeypatch.setattr(native, "write_record", write) + monkeypatch.setenv(native.CONTEXT, json.dumps({"directory": str(directory)})) + monkeypatch.setitem(sys.modules, "gdb", fake) + monkeypatch.setitem(sys.modules, "_native_worker", native) + monkeypatch.syspath_prepend(str(Path(native.__file__).parent)) + runpy.run_path(str(Path(native.__file__).with_name("_native_gdb.py"))) + result = native.read_record(directory / "result.json") + assert result["kind"] == "signal" and result["signal"] == 11 + assert result["observed_fatal_stops"] == 1 + assert "handle SIGSEGV stop print pass" in commands + assert commands[-1] == "continue" + assert all("canary" not in path.read_text() for path in directory.iterdir()) + + +def _mechanism_start(root, source, arguments=(), *, environment=None): + root.mkdir(mode=0o700) + session = native.NativeSession(root) + program = root / "inferior.py" + program.write_text(source) + process = subprocess.Popen( + [ + sys.executable, + str(Path(native.__file__).resolve()), + "--launch", + str(session.root), + str(program), + *arguments, + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + ) + return session, process + + +def _mechanism_case(root, source, arguments=(), *, payload=b"", environment=None): + session, process = _mechanism_start(root, source, arguments, environment=environment) + try: + output, error = process.communicate(payload, timeout=20) + records = [native.read_record(path) for path in session.root.rglob("*.json")] + return process.returncode, output, error, records + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + session.finish() + + +def _mechanism_parent_death(root): + session, process = _mechanism_start( + root, "import os, signal\nos.write(1, b'ready\\n')\nsignal.pause()\n" + ) + descriptors = [] + try: + assert select.select([process.stdout], [], [], 20)[0], "worker did not start" + assert process.stdout.readline() == b"ready\n" + directory = session.root / str(process.pid) + debugger = native.read_record(directory / "debugger.json") + inferior = native.read_record(directory / "inferior.json") + assert debugger["ppid"] == process.pid + assert inferior["ppid"] == debugger["pid"] + for expected in (debugger, inferior): + descriptor = os.pidfd_open(expected["pid"]) + descriptors.append(descriptor) + found = native.identity(expected["pid"]) + assert all(found[key] == expected[key] for key in ("pid", "start", "uid")) + process.kill() # Only our direct Popen; the kernel owns both descendant edges. + assert process.wait(timeout=5) == -signal.SIGKILL + for descriptor in descriptors: + observer = select.poll() + observer.register(descriptor, select.POLLIN) + assert observer.poll(5000), "owned descendant survived launcher death" + process.communicate(timeout=5) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + for descriptor in descriptors: + os.close(descriptor) + session.finish() + + +def _mechanism_xdist(root): + repository = str(Path(native.__file__).resolve().parents[1]) + environment = { + **os.environ, + "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", + "PYTEST_ADDOPTS": "", + "PYTHONPATH": repository, + } + for crash in (False, True): + outcomes = [] + for enabled in (False, True): + directory = root / f"xdist-{crash}-{enabled}" + directory.mkdir(mode=0o700) + (directory / "pytest.ini").write_text("[pytest]\n") + source = ( + "import os, signal\ndef test_crash(): os.kill(os.getpid(), signal.SIGSEGV)\n" + if crash + else "def test_pass(): assert 1 == 1\n" + "def test_assertion(): assert False, 'original assertion'\n" + "def test_exception(): raise ValueError('original exception')\n" + ) + (directory / "test_case.py").write_text(source) + command = [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "xdist.plugin", + "-n", + "1", + "--max-worker-restart=0", + "--junitxml=result.xml", + ] + if enabled: + command += ["-p", "tests._worker_status", "--native-worker-signatures"] + process = subprocess.run( + command, cwd=directory, env=environment, capture_output=True, timeout=25 + ) + assert process.returncode == 1, ( + crash, + enabled, + process.returncode, + process.stdout, + process.stderr, + ) + cases = ElementTree.parse(directory / "result.xml").findall(".//testcase") + outcomes.append( + sorted((case.get("name"), case.find("failure") is not None) for case in cases) + ) + if enabled: + files = list((directory / ".pytest_cache/worker-status").glob("*/status.jsonl")) + assert len(files) == 1 + records = [json.loads(line) for line in files[0].read_text().splitlines()] + statuses = [record for record in records if record["event"] == "worker_status"] + after = next(record for record in statuses if record["stage"] == "after_teardown") + assert after["os_returncode"] == (-11 if crash else 0) + observations = [record for record in records if record["event"] == "native_worker"] + terminal = next( + record["result"] for record in reversed(observations) if "result" in record + ) + assert terminal["kind"] == ("signal" if crash else "exit") + assert terminal["signal" if crash else "code"] == (11 if crash else 0) + if crash: + assert any( + record.get("signature", {}).get("signal") == "SIGSEGV" + for record in observations + ) + assert outcomes[0] == outcomes[1], outcomes + + +def mechanism_checks(root): + """Run explicitly under GDB; absence or refusal fails instead of skipping.""" + canary = "private-argv-and-environment-canary" + environment = {**os.environ, "NATIVE_TEST_CANARY": canary} + for code in (0, 7, 139): + result, _, error, records = _mechanism_case( + root / f"exit-{code}", f"raise SystemExit({code})\n" + ) + assert result == code, (result, error, records) + terminals = [record for record in records if record.get("kind") == "exit"] + assert len(terminals) == 1 and terminals[0]["code"] == code + assert terminals[0]["observed_fatal_stops"] == 0 + result, _, error, records = _mechanism_case( + root / "caught-then-native-fault", + "import ctypes, os, signal\n" + "signal.signal(signal.SIGSEGV, lambda *_: None)\n" + "os.kill(os.getpid(), signal.SIGSEGV)\n" + "signal.signal(signal.SIGSEGV, signal.SIG_DFL)\n" + "ctypes.string_at(0)\n", + (canary,), + environment=environment, + ) + assert result == -signal.SIGSEGV, (result, error, records) + signatures = sorted( + (record for record in records if record.get("signal") == "SIGSEGV"), + key=lambda record: record["observed_stop"], + ) + assert [record["observed_stop"] for record in signatures] == [1, 2] + assert signatures[0]["si_code"] <= 0 < signatures[1]["si_code"] + assert signatures[1]["si_addr"] == 0 + assert canary not in json.dumps(records) + result, _, error, records = _mechanism_case( + root / "bounded-stops", + "import os, signal\n" + "signal.signal(signal.SIGSEGV, lambda *_: None)\n" + f"for _ in range({native.MAX_STOPS + 2}): os.kill(os.getpid(), signal.SIGSEGV)\n" + "raise SystemExit(7)\n", + ) + assert result == 7, (result, error, records) + assert sum(record.get("signal") == "SIGSEGV" for record in records) == native.MAX_STOPS + terminal = next(record for record in records if record.get("kind") == "exit") + assert terminal["observed_fatal_stops"] == native.MAX_STOPS + 2 + assert terminal["captured_stops"] == native.MAX_STOPS + payload = b"wire\x00\xff" + environment_hash = hashlib.sha256(json.dumps(environment, sort_keys=True).encode()).hexdigest() + result, output, error, records = _mechanism_case( + root / "transport", + "import hashlib, json, os, sys\n" + f"assert hashlib.sha256(json.dumps(dict(os.environ), sort_keys=True).encode()).hexdigest() == {environment_hash!r}, 'worker environment mismatch'\n" + f"assert sys.argv[1] == {canary!r}\n" + "assert os.environ['NATIVE_TEST_CANARY'] == sys.argv[1]\n" + f"assert {native.CONTEXT!r} not in os.environ\n" + "os.write(1, os.read(0, 6))\n", + (canary,), + payload=payload, + environment=environment, + ) + assert (result, output, error) == (0, payload, b""), (result, output, error, records) + assert canary not in json.dumps(records) + _mechanism_parent_death(root / "parent-death") + _mechanism_xdist(root) + for received in (signal.SIGSEGV, signal.SIGTRAP): + for caught in (False, True): + source = "import os, signal\n" + if caught: + source += f"signal.signal({int(received)}, lambda *_: None)\n" + source += f"os.kill(os.getpid(), {int(received)})\nraise SystemExit(139)\n" + result, _, error, records = _mechanism_case( + root / f"signal-{received}-{caught}", source, (canary,), environment=environment + ) + assert result == (139 if caught else -received), ( + int(received), + caught, + result, + error, + records, + ) + terminals = [record for record in records if "kind" in record] + assert len(terminals) == 1 + expected = ( + {"kind": "exit", "code": 139} if caught else {"kind": "signal", "signal": received} + ) + assert all(terminals[0].get(key) == value for key, value in expected.items()) + assert canary not in json.dumps(records) + if received == signal.SIGSEGV: + signatures = [record for record in records if record.get("signal") == "SIGSEGV"] + assert len(signatures) == 1 and signatures[0]["threads"], records + print("native mechanism cases passed") + + +if __name__ == "__main__": + import tempfile + + if sys.argv[1:] != ["--mechanism"]: + raise SystemExit("expected --mechanism") + with tempfile.TemporaryDirectory(prefix="native-worker-mechanism-") as temporary: + mechanism_checks(Path(temporary)) From cdfae3512297258b378cea11014f54d361012048 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 00:57:35 +0000 Subject: [PATCH 2/6] Pass private worker context through an anonymous descriptor Signed-off-by: zlareb1 --- tests/_native_gdb.py | 2 +- tests/_native_worker.py | 26 ++++++++++-- tests/unit/test_native_worker.py | 68 +++++++++++++++++++++++--------- 3 files changed, 74 insertions(+), 22 deletions(-) diff --git a/tests/_native_gdb.py b/tests/_native_gdb.py index c27fa237a5..c6801105de 100644 --- a/tests/_native_gdb.py +++ b/tests/_native_gdb.py @@ -129,7 +129,7 @@ def signature(received: str) -> dict[str, Any]: def run() -> None: - context = json.loads(os.environ[native.CONTEXT]) + context = native.read_context() directory = Path(context["directory"]) stops = 0 trap = False diff --git a/tests/_native_worker.py b/tests/_native_worker.py index 26e34f0cba..8368ed5cad 100644 --- a/tests/_native_worker.py +++ b/tests/_native_worker.py @@ -17,6 +17,7 @@ CONTEXT = "_PYTEST_NATIVE_CONTEXT" MAX_WORKERS = 64 MAX_BYTES = 65536 +MAX_CONTEXT_BYTES = 64 * 1024 * 1024 MAX_STOPS = 8 @@ -55,6 +56,17 @@ def read_record(path: Path) -> dict[str, Any]: return result +def read_context() -> dict[str, Any]: + descriptor = int(os.environ[CONTEXT]) + size = os.fstat(descriptor).st_size + if not 0 < size <= MAX_CONTEXT_BYTES: + raise ValueError("native context size is unsupported") + result = json.loads(os.pread(descriptor, size, 0)) + if not isinstance(result, dict): + raise ValueError("invalid native context") + return result + + def identity(pid: int) -> dict[str, int]: base = Path("/proc") / str(pid) fields = (base / "stat").read_text().rsplit(")", 1)[1].split() @@ -187,7 +199,8 @@ def finish(self) -> dict[str, Any]: def bootstrap() -> None: - context = json.loads(os.environ.pop(CONTEXT)) + context = read_context() + os.close(int(os.environ.pop(CONTEXT))) directory = Path(context["directory"]) try: parent_death_signal(read_record(directory / "debugger.json")["pid"]) @@ -224,7 +237,14 @@ def launch(root: Path, arguments: list[str]) -> int: "arguments": arguments, "environment": dict(os.environ), } - environment = {**os.environ, CONTEXT: json.dumps(context)} + data = json.dumps(context, ensure_ascii=True).encode() + if len(data) > MAX_CONTEXT_BYTES: + raise ValueError("native context size is unsupported") + context_fd = os.memfd_create("pytest-native-context", os.MFD_CLOEXEC) + descriptors.append(context_fd) + with os.fdopen(os.dup(context_fd), "wb") as output: + output.write(data) + environment = {**os.environ, CONTEXT: str(context_fd)} debugger = read_record(root / "config.json")["debugger"] command = [ debugger, @@ -278,7 +298,7 @@ def launch(root: Path, arguments: list[str]) -> int: if sys.argv[1] == "--bootstrap": bootstrap() elif sys.argv[1] == "--debugger-bootstrap": - context = json.loads(os.environ[CONTEXT]) + context = read_context() directory = Path(context["directory"]) try: parent_death_signal(context["launcher_pid"]) diff --git a/tests/unit/test_native_worker.py b/tests/unit/test_native_worker.py index 65463b0f9d..e082616721 100644 --- a/tests/unit/test_native_worker.py +++ b/tests/unit/test_native_worker.py @@ -29,6 +29,21 @@ def session(tmp_path, monkeypatch): return native.NativeSession(tmp_path) +def test_context_uses_bounded_offset_independent_anonymous_fd(monkeypatch): + descriptor = os.memfd_create("native-context-test", os.MFD_CLOEXEC) + try: + data = b'{"arguments":["private-context-canary"]}' + os.write(descriptor, data) + monkeypatch.setenv(native.CONTEXT, str(descriptor)) + assert native.read_context() == {"arguments": ["private-context-canary"]} + assert os.lseek(descriptor, 0, os.SEEK_CUR) == len(data) + os.ftruncate(descriptor, native.MAX_CONTEXT_BYTES + 1) + with pytest.raises(ValueError, match="size"): + native.read_context() + finally: + os.close(descriptor) + + def test_records_are_private_bounded_and_never_overwrite(tmp_path): path = tmp_path / "record.json" native.write_record(path, {"kind": "exit", "code": 139}) @@ -113,7 +128,9 @@ def __init__(self, command, **kwargs): assert kwargs["stdout"] is subprocess.DEVNULL assert kwargs["stderr"] is subprocess.DEVNULL assert canary not in command - context = json.loads(kwargs["env"][native.CONTEXT]) + descriptor = int(kwargs["env"][native.CONTEXT]) + assert descriptor in kwargs["pass_fds"] + context = json.loads(os.pread(descriptor, os.fstat(descriptor).st_size, 0)) assert context["arguments"] == ["-c", "raise SystemExit(139)", canary] self.directory = Path(context["directory"]) @@ -193,7 +210,7 @@ def execute(command, **kwargs): ) directory = session.root / "synthetic-debugger" directory.mkdir(mode=0o700) - monkeypatch.setenv(native.CONTEXT, json.dumps({"directory": str(directory)})) + monkeypatch.setattr(native, "read_context", lambda: {"directory": str(directory)}) monkeypatch.setitem(sys.modules, "gdb", fake) monkeypatch.setitem(sys.modules, "_native_worker", native) monkeypatch.syspath_prepend(str(Path(native.__file__).parent)) @@ -265,7 +282,7 @@ def write(path, record): parse_and_eval=lambda expression: Value(None if expression == "$_exitcode" else 11), ) monkeypatch.setattr(native, "write_record", write) - monkeypatch.setenv(native.CONTEXT, json.dumps({"directory": str(directory)})) + monkeypatch.setattr(native, "read_context", lambda: {"directory": str(directory)}) monkeypatch.setitem(sys.modules, "gdb", fake) monkeypatch.setitem(sys.modules, "_native_worker", native) monkeypatch.syspath_prepend(str(Path(native.__file__).parent)) @@ -462,21 +479,36 @@ def mechanism_checks(root): assert terminal["observed_fatal_stops"] == native.MAX_STOPS + 2 assert terminal["captured_stops"] == native.MAX_STOPS payload = b"wire\x00\xff" - environment_hash = hashlib.sha256(json.dumps(environment, sort_keys=True).encode()).hexdigest() - result, output, error, records = _mechanism_case( - root / "transport", - "import hashlib, json, os, sys\n" - f"assert hashlib.sha256(json.dumps(dict(os.environ), sort_keys=True).encode()).hexdigest() == {environment_hash!r}, 'worker environment mismatch'\n" - f"assert sys.argv[1] == {canary!r}\n" - "assert os.environ['NATIVE_TEST_CANARY'] == sys.argv[1]\n" - f"assert {native.CONTEXT!r} not in os.environ\n" - "os.write(1, os.read(0, 6))\n", - (canary,), - payload=payload, - environment=environment, - ) - assert (result, output, error) == (0, payload, b""), (result, output, error, records) - assert canary not in json.dumps(records) + large_environment = { + **environment, + "NATIVE_LARGE_A": "private-large-a-" + "a" * (70 * 1024), + "NATIVE_LARGE_B": "private-large-b-" + "b" * (70 * 1024), + "NATIVE_NON_UTF8": os.fsdecode(b"\xff"), + } + for label, transport_environment in ( + ("transport", environment), + ("large-environment", large_environment), + ): + environment_hash = hashlib.sha256( + json.dumps(transport_environment, sort_keys=True).encode() + ).hexdigest() + result, output, error, records = _mechanism_case( + root / label, + "import hashlib, json, os, sys\n" + f"assert hashlib.sha256(json.dumps(dict(os.environ), sort_keys=True).encode()).hexdigest() == {environment_hash!r}, 'worker environment mismatch'\n" + f"assert sys.argv[1] == {canary!r}\n" + "assert os.environ['NATIVE_TEST_CANARY'] == sys.argv[1]\n" + f"assert {native.CONTEXT!r} not in os.environ\n" + "os.write(1, os.read(0, 6))\n", + (canary,), + payload=payload, + environment=transport_environment, + ) + assert (result, output, error) == (0, payload, b""), (label, result, output, error, records) + serialized = json.dumps(records) + assert all( + value not in serialized for value in (canary, "private-large-a-", "private-large-b-") + ) _mechanism_parent_death(root / "parent-death") _mechanism_xdist(root) for received in (signal.SIGSEGV, signal.SIGTRAP): From 5f30e87dc9ac8ffeb2fec1620204fcefedabf8df Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 01:00:20 +0000 Subject: [PATCH 3/6] Accept the worker restart option as parsed by xdist Signed-off-by: zlareb1 --- tests/_native_worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/_native_worker.py b/tests/_native_worker.py index 8368ed5cad..36b987f45f 100644 --- a/tests/_native_worker.py +++ b/tests/_native_worker.py @@ -110,8 +110,8 @@ def __init__(self, root: Path) -> None: self.finished = False write_record(self.root / "config.json", {"debugger": debugger}) - def configure(self, specs: list[Any], max_restarts: int | None = None) -> None: - restarts = 4 * len(specs) if max_restarts is None else max_restarts + def configure(self, specs: list[Any], max_restarts: int | str | None = None) -> None: + restarts = 4 * len(specs) if max_restarts is None else int(max_restarts) if restarts < 0 or len(specs) + restarts > MAX_WORKERS: raise ValueError("native diagnostic worker limit exceeded") for spec in specs: From 4a2855283c65c34d723ef9096a9e15a15f76e3fa Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 01:06:25 +0000 Subject: [PATCH 4/6] Record bounded debugger errors during terminal collection Signed-off-by: zlareb1 --- tests/_native_gdb.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/tests/_native_gdb.py b/tests/_native_gdb.py index c6801105de..db7806812d 100644 --- a/tests/_native_gdb.py +++ b/tests/_native_gdb.py @@ -197,11 +197,34 @@ def on_stop(event: Any) -> None: else: result = {"kind": "unavailable", "reason": "terminal_status_unknown"} except gdb.error as error: - refused = any( - token in str(error).lower() - for token in ("ptrace", "not permitted", "permission denied") - ) + detail = str(error).lower() + refused = any(token in detail for token in ("ptrace", "not permitted", "permission denied")) result = {"kind": "unavailable", "reason": "trace_refused" if refused else "debugger_error"} + result["error_categories"] = [ + category + for category in ( + "register", + "no such process", + "not being run", + "thread", + "running", + "breakpoint", + "memory", + "signal", + "exited", + ) + if category in detail + ] + result["error_digest"] = hashlib.sha256(detail.encode()).hexdigest() + try: + result["inferior_pid"] = gdb.selected_inferior().pid + except gdb.error: + result["inferior_pid"] = None + for key, expression in (("exit_code", "$_exitcode"), ("exit_signal", "$_exitsignal")): + try: + result[key] = integer(expression) + except (gdb.error, TypeError, ValueError): + result[key] = None result["observed_fatal_stops"] = stops result["captured_stops"] = min(stops, native.MAX_STOPS) result["trap_stops"] = trap_stops From 9f230933a187468777fb14b3c4da7be616e70478 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 01:15:25 +0000 Subject: [PATCH 5/6] Drain native debugger exit events before finalizing worker status Signed-off-by: zlareb1 --- docs/development/testing.md | 5 +- tests/_native_gdb.py | 184 +++++++++++++---------- tests/_native_worker.py | 3 +- tests/unit/test_native_worker.py | 249 ++++++++++++++++++++++--------- 4 files changed, 284 insertions(+), 157 deletions(-) diff --git a/docs/development/testing.md b/docs/development/testing.md index 46052569e3..205741eaca 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -63,12 +63,13 @@ refusal stop startup. No tracing policy or capability changes are made. Each hel sets its own parent-death signal so forced launcher termination also stops its debugger and worker. Registrations never authorize controller signals to another process. -Private session artifacts under `.pytest_cache/worker-status/` contain bounded native +Session artifacts under `.pytest_cache/worker-status/` contain bounded native PCs, symbols, module identities and offsets for up to eight threads and 24 frames per thread. Up to eight observed fatal-signal stops are retained. A stop may be handled by the worker; the separate terminal result records whether it eventually exited or died from a signal. A bounded fault-thread summary also appears in the job log. Missing -symbols or mappings remain unresolved. Raw debugger output is discarded; core dumps, +symbols or mappings remain unresolved. CI retains these sanitized records in its +existing test artifacts. Raw debugger output is discarded; core dumps, arguments, locals, environment values and memory bytes are not collected. ## Integration tests diff --git a/tests/_native_gdb.py b/tests/_native_gdb.py index db7806812d..a49afdc4f7 100644 --- a/tests/_native_gdb.py +++ b/tests/_native_gdb.py @@ -128,15 +128,84 @@ def signature(received: str) -> dict[str, Any]: return result -def run() -> None: - context = native.read_context() - directory = Path(context["directory"]) - stops = 0 - trap = False - trap_stops = 0 - - def on_stop(event: Any) -> None: - nonlocal stops, trap, trap_stops +class NativeDebugger: + def __init__(self) -> None: + self.directory = Path(native.read_context()["directory"]) + self.inferior = gdb.selected_inferior() + self.stops = 0 + self.trap_stops = 0 + self.finalized = False + self.terminal_seen = False + gdb.events.stop.connect(self.on_stop) + gdb.events.exited.connect(self.on_exit) + # Native async execution leaves exit-event dispatch with GDB's main loop. + gdb.post_event(lambda: self.execute("run &")) + + def finish(self, result: dict[str, Any]) -> None: + if self.finalized: + return + self.finalized = True + result.update( + observed_fatal_stops=self.stops, + captured_stops=min(self.stops, native.MAX_STOPS), + trap_stops=self.trap_stops, + gdb_version=gdb.VERSION[:64], + ) + try: + native.write_record(self.directory / "result.json", result) + finally: + gdb.post_event(lambda: gdb.execute("quit", to_string=False)) + + def on_exit(self, event: Any) -> None: + if event.inferior == self.inferior: + self.terminal_seen = True + # Convenience variables are final only after the native exit event completes. + gdb.post_event(self.terminal_result) + + def terminal_result(self) -> None: + try: + exit_code, exit_signal = integer("$_exitcode"), integer("$_exitsignal") + except (gdb.error, ValueError, TypeError): + self.finish({"kind": "unavailable", "reason": "terminal_status_unknown"}) + return + if not (self.directory / "inferior.json").exists(): + result = {"kind": "unavailable", "reason": "inferior_not_started"} + elif exit_signal is not None: + result = {"kind": "signal", "signal": exit_signal} + elif exit_code is not None: + result = {"kind": "exit", "code": exit_code} + else: + result = {"kind": "unavailable", "reason": "terminal_status_unknown"} + self.finish(result) + + def execute(self, command: str) -> None: + if self.finalized or self.terminal_seen: + return + try: + gdb.execute(command, to_string=False) + except gdb.error as error: + detail = str(error).lower() + if self.stops and "register" in detail and "no such process" in detail: + # A dead thread can precede the queued native process-exit event. + # This marker proves no terminal outcome; never retry or infer a signal. + try: + native.write_record( + self.directory / "awaiting-native-exit.json", + {"kind": "incomplete", "reason": "register_esrch"}, + ) + except OSError: + pass + return + refused = any( + token in detail for token in ("ptrace", "not permitted", "permission denied") + ) + self.finish( + {"kind": "unavailable", "reason": "trace_refused" if refused else "debugger_error"} + ) + + def on_stop(self, event: Any) -> None: + if self.finalized or self.terminal_seen: + return trap = isinstance(event, gdb.SignalEvent) and event.stop_signal == "SIGTRAP" if not isinstance(event, gdb.SignalEvent): # Some targets report a user SIGTRAP as StopEvent rather than SignalEvent. @@ -145,22 +214,30 @@ def on_stop(event: Any) -> None: except gdb.error: pass if trap: - trap_stops += 1 - if isinstance(event, gdb.SignalEvent) and event.stop_signal in FATAL: - stops += 1 - if stops > native.MAX_STOPS: - return - path = directory / f"signature-{stops}.json" + self.trap_stops += 1 + try: + if isinstance(event, gdb.SignalEvent) and event.stop_signal in FATAL: + self.capture(event.stop_signal) + finally: + command = "signal SIGTRAP" if trap else "continue &" + gdb.post_event(lambda: self.execute(command)) + + def capture(self, received: str) -> None: + self.stops += 1 + if self.stops > native.MAX_STOPS: + return + path = self.directory / f"signature-{self.stops}.json" + try: + native.write_record(path, {"observed_stop": self.stops, **signature(received)}) + except Exception: # noqa: BLE001 — a collector failure must still deliver the signal try: - native.write_record(path, {"observed_stop": stops, **signature(event.stop_signal)}) - except Exception: # noqa: BLE001 — a collector failure must still deliver the signal - try: - native.write_record(path, {"observed_stop": stops, "capture": "unavailable"}) - except OSError: - pass + native.write_record(path, {"observed_stop": self.stops, "capture": "unavailable"}) + except OSError: + pass + - gdb.events.stop.connect(on_stop) - # Preserve ASLR and all delivered signals, including GDB's special defaults. +def install() -> NativeDebugger: + # Preserve ASLR and delivered signals, including GDB's special defaults. for command in ( "set pagination off", "set confirm off", @@ -169,67 +246,12 @@ def on_stop(event: Any) -> None: "set follow-exec-mode same", "handle all nostop noprint pass", "handle SIGINT nostop noprint pass", - # Internal exec traps stay with GDB; the loop delivers actual inferior SIGTRAPs. + # Internal exec traps stay with GDB; on_stop delivers actual inferior SIGTRAPs. "handle SIGTRAP stop print nopass", *(f"handle {name} stop print pass" for name in sorted(FATAL)), ): gdb.execute(command, to_string=True) - result: dict[str, Any] - try: - gdb.execute("run", to_string=False) - while gdb.selected_inferior().pid: - # stop handlers only collect; this resumes with the original signal policy. - command = "signal SIGTRAP" if trap else "continue" - trap = False - gdb.execute(command, to_string=False) - exit_code, exit_signal = integer("$_exitcode"), integer("$_exitsignal") - if not (directory / "inferior.json").exists(): - result = { - "kind": "unavailable", - "reason": "inferior_not_started", - "exit_code": exit_code, - "exit_signal": exit_signal, - } - elif exit_signal is not None: - result = {"kind": "signal", "signal": exit_signal} - elif exit_code is not None: - result = {"kind": "exit", "code": exit_code} - else: - result = {"kind": "unavailable", "reason": "terminal_status_unknown"} - except gdb.error as error: - detail = str(error).lower() - refused = any(token in detail for token in ("ptrace", "not permitted", "permission denied")) - result = {"kind": "unavailable", "reason": "trace_refused" if refused else "debugger_error"} - result["error_categories"] = [ - category - for category in ( - "register", - "no such process", - "not being run", - "thread", - "running", - "breakpoint", - "memory", - "signal", - "exited", - ) - if category in detail - ] - result["error_digest"] = hashlib.sha256(detail.encode()).hexdigest() - try: - result["inferior_pid"] = gdb.selected_inferior().pid - except gdb.error: - result["inferior_pid"] = None - for key, expression in (("exit_code", "$_exitcode"), ("exit_signal", "$_exitsignal")): - try: - result[key] = integer(expression) - except (gdb.error, TypeError, ValueError): - result[key] = None - result["observed_fatal_stops"] = stops - result["captured_stops"] = min(stops, native.MAX_STOPS) - result["trap_stops"] = trap_stops - result["gdb_version"] = gdb.VERSION[:64] - native.write_record(directory / "result.json", result) + return NativeDebugger() -run() +collector = install() diff --git a/tests/_native_worker.py b/tests/_native_worker.py index 36b987f45f..8f03946f4d 100644 --- a/tests/_native_worker.py +++ b/tests/_native_worker.py @@ -251,7 +251,6 @@ def launch(root: Path, arguments: list[str]) -> int: "-q", "-nx", "-nh", - "-batch", "-iex", "set auto-load off", "-iex", @@ -270,7 +269,7 @@ def launch(root: Path, arguments: list[str]) -> int: try: with subprocess.Popen( [sys.executable, str(Path(__file__).resolve()), "--debugger-bootstrap", *command], - stdin=subprocess.DEVNULL, + stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, pass_fds=descriptors, diff --git a/tests/unit/test_native_worker.py b/tests/unit/test_native_worker.py index e082616721..6b444a62c5 100644 --- a/tests/unit/test_native_worker.py +++ b/tests/unit/test_native_worker.py @@ -12,6 +12,7 @@ import stat import subprocess import sys +from collections import deque from pathlib import Path from types import SimpleNamespace from xml.etree import ElementTree @@ -125,6 +126,7 @@ def test_launcher_preserves_normal_exit_codes_and_keeps_arguments_private( class Debugger: def __init__(self, command, **kwargs): + assert kwargs["stdin"] is subprocess.PIPE assert kwargs["stdout"] is subprocess.DEVNULL assert kwargs["stderr"] is subprocess.DEVNULL assert canary not in command @@ -189,34 +191,98 @@ def test_forged_same_uid_records_cannot_authorize_cleanup(session, monkeypatch, assert session.finish() == {"event": "native_cleanup", "already_finished": True} -def test_debugger_permission_refusal_is_reported_without_fallback(session, monkeypatch): - class DebuggerError(Exception): +class FakeGDB: + VERSION = "synthetic-gdb" + TYPE_CODE_VOID = 0 + + class error(Exception): pass - commands = [] + class SignalEvent: + def __init__(self, received): + self.stop_signal = received + + class Value: + def __init__(self, value): + self.value = value + self.type = SimpleNamespace(code=0 if value is None else 1) - def execute(command, **kwargs): - commands.append(command) - if command == "run": - raise DebuggerError("ptrace: Operation not permitted private-error-canary") + def __int__(self): + return self.value + + def __init__(self): + self.commands, self.value_reads = [], [] + self.pending = deque() + self.stop_handlers, self.exit_handlers = [], [] + self.events = SimpleNamespace( + stop=SimpleNamespace(connect=self.stop_handlers.append), + exited=SimpleNamespace(connect=self.exit_handlers.append), + ) + self.inferior = SimpleNamespace(pid=1) + self.on_command = {} + self.exit_code = self.exit_signal = None + + def post_event(self, callback): + self.pending.append(callback) + + def execute(self, command, **kwargs): + self.commands.append(command) + action = self.on_command.get(command) + if isinstance(action, Exception): + raise action + if action is not None: + action() return "" - fake = SimpleNamespace( - VERSION="synthetic-gdb", - error=DebuggerError, - SignalEvent=type("SignalEvent", (), {}), - events=SimpleNamespace(stop=SimpleNamespace(connect=lambda callback: None)), - execute=execute, - ) + def selected_inferior(self): + return self.inferior + + def selected_thread(self): + raise self.error("private-unwind-canary") + + def parse_and_eval(self, expression): + self.value_reads.append(expression) + return self.Value( + {"$_exitcode": self.exit_code, "$_exitsignal": self.exit_signal}[expression] + ) + + def stop(self, received="SIGSEGV"): + for callback in self.stop_handlers: + callback(self.SignalEvent(received)) + + def exit(self, *, code=None, received=None): + self.inferior.pid = 0 + for callback in self.exit_handlers: + callback(SimpleNamespace(inferior=self.inferior)) + # GDB updates terminal convenience variables after the native callback returns. + self.exit_code, self.exit_signal = code, received + + def drain(self): + while self.pending: + self.pending.popleft()() + + +@pytest.fixture +def debugger(session, monkeypatch): + fake = FakeGDB() directory = session.root / "synthetic-debugger" directory.mkdir(mode=0o700) + native.write_record(directory / "inferior.json", {"pid": 1}) monkeypatch.setattr(native, "read_context", lambda: {"directory": str(directory)}) monkeypatch.setitem(sys.modules, "gdb", fake) monkeypatch.setitem(sys.modules, "_native_worker", native) monkeypatch.syspath_prepend(str(Path(native.__file__).parent)) - runpy.run_path(str(Path(native.__file__).with_name("_native_gdb.py"))) - assert commands[-1] == "run" - result = native.read_record(directory / "result.json") + module = runpy.run_path(str(Path(native.__file__).with_name("_native_gdb.py"))) + return SimpleNamespace(api=fake, collector=module["collector"], directory=directory) + + +def test_debugger_permission_refusal_is_reported_without_fallback(debugger): + fake = debugger.api + fake.on_command["run &"] = fake.error("ptrace: Operation not permitted private-error-canary") + fake.drain() + assert fake.commands[-2:] == ["run &", "quit"] + assert fake.value_reads == [] + result = native.read_record(debugger.directory / "result.json") assert result == { "kind": "unavailable", "reason": "trace_refused", @@ -225,44 +291,19 @@ def execute(command, **kwargs): "trap_stops": 0, "gdb_version": "synthetic-gdb", } - assert all("private-error-canary" not in path.read_text() for path in directory.iterdir()) + assert all( + "private-error-canary" not in path.read_text() for path in debugger.directory.iterdir() + ) @pytest.mark.parametrize("failure", ["unwind", "write"]) -def test_collector_failure_still_continues_and_preserves_terminal_signal( - session, monkeypatch, failure +@pytest.mark.parametrize("exit_code,exit_signal", [(None, 11), (139, None)]) +def test_collector_failure_still_continues_and_preserves_terminal_event( + debugger, monkeypatch, failure, exit_code, exit_signal ): - class DebuggerError(Exception): - pass - - class SignalEvent: - stop_signal = "SIGSEGV" - - class Value: - def __init__(self, value): - self.value = value - self.type = SimpleNamespace(code=0 if value is None else 1) - - def __int__(self): - return self.value - - commands, callbacks = [], [] - inferior = SimpleNamespace(pid=1) - - def execute(command, **kwargs): - commands.append(command) - if command == "run": - callbacks[0](SignalEvent()) - elif command == "continue": - inferior.pid = 0 - return "" - - def no_unwind(): - raise DebuggerError("private-unwind-canary") - - directory = session.root / "failed-collector" - directory.mkdir(mode=0o700) - native.write_record(directory / "inferior.json", {"pid": 1}) + fake = debugger.api + fake.on_command["run &"] = fake.stop + fake.on_command["continue &"] = lambda: fake.exit(code=exit_code, received=exit_signal) original_write = native.write_record def write(path, record): @@ -270,29 +311,93 @@ def write(path, record): raise OSError("private-write-canary") original_write(path, record) - fake = SimpleNamespace( - VERSION="synthetic-gdb", - error=DebuggerError, - SignalEvent=SignalEvent, - TYPE_CODE_VOID=0, - events=SimpleNamespace(stop=SimpleNamespace(connect=callbacks.append)), - execute=execute, - selected_inferior=lambda: inferior, - selected_thread=no_unwind, - parse_and_eval=lambda expression: Value(None if expression == "$_exitcode" else 11), - ) monkeypatch.setattr(native, "write_record", write) - monkeypatch.setattr(native, "read_context", lambda: {"directory": str(directory)}) - monkeypatch.setitem(sys.modules, "gdb", fake) - monkeypatch.setitem(sys.modules, "_native_worker", native) - monkeypatch.syspath_prepend(str(Path(native.__file__).parent)) - runpy.run_path(str(Path(native.__file__).with_name("_native_gdb.py"))) - result = native.read_record(directory / "result.json") - assert result["kind"] == "signal" and result["signal"] == 11 + fake.drain() + result = native.read_record(debugger.directory / "result.json") + if exit_signal is not None: + assert result["kind"] == "signal" and result["signal"] == exit_signal + else: + assert result["kind"] == "exit" and result["code"] == exit_code assert result["observed_fatal_stops"] == 1 - assert "handle SIGSEGV stop print pass" in commands - assert commands[-1] == "continue" - assert all("canary" not in path.read_text() for path in directory.iterdir()) + assert "handle SIGSEGV stop print pass" in fake.commands + assert fake.commands[-3:] == ["run &", "continue &", "quit"] + assert all("canary" not in path.read_text() for path in debugger.directory.iterdir()) + + +def test_queued_resume_is_ignored_after_native_exit_event(debugger): + fake = debugger.api + fake.drain() # Dispatch startup, then deliver a stop followed by the native exit event. + fake.stop() + fake.exit(received=11) + fake.drain() + assert "continue &" not in fake.commands + assert fake.commands[-1] == "quit" + assert debugger.collector.terminal_seen is True + result = native.read_record(debugger.directory / "result.json") + assert result["kind"] == "signal" and result["signal"] == 11 + fake.stop() + fake.drain() + assert fake.commands.count("quit") == 1 + assert "continue &" not in fake.commands + + +@pytest.mark.parametrize("failure", ["gdb", "value"]) +def test_failed_terminal_variable_read_stays_unknown_and_quits(debugger, monkeypatch, failure): + fake = debugger.api + fake.drain() + + def unavailable(expression): + error = fake.error if failure == "gdb" else ValueError + raise error("private-terminal-read-canary") + + monkeypatch.setattr(fake, "parse_and_eval", unavailable) + fake.exit(received=11) + fake.drain() + result = native.read_record(debugger.directory / "result.json") + assert result["kind"] == "unavailable" + assert result["reason"] == "terminal_status_unknown" + assert "signal" not in result and "code" not in result + assert debugger.collector.finalized is True + assert fake.commands[-1] == "quit" + assert "continue &" not in fake.commands + assert all( + "private-terminal-read-canary" not in path.read_text() + for path in debugger.directory.iterdir() + ) + + +def test_other_inferior_exit_event_does_not_finalize_worker(debugger): + fake = debugger.api + fake.drain() + for callback in fake.exit_handlers: + callback(SimpleNamespace(inferior=SimpleNamespace(pid=2))) + assert not fake.pending + assert fake.value_reads == [] + assert debugger.collector.terminal_seen is False + assert not (debugger.directory / "result.json").exists() + + +def test_register_esrch_without_exit_event_is_incomplete_not_a_terminal_signal(debugger): + fake = debugger.api + fake.on_command["run &"] = fake.stop + fake.on_command["continue &"] = fake.error( + "Could not read registers: No such process private-error-canary" + ) + fake.drain() + assert fake.commands[-2:] == ["run &", "continue &"] + assert fake.commands.count("continue &") == 1 + assert not fake.pending + assert fake.value_reads == [] + assert debugger.collector.finalized is False + assert debugger.collector.terminal_seen is False + assert not (debugger.directory / "result.json").exists() + assert native.read_record(debugger.directory / "awaiting-native-exit.json") == { + "kind": "incomplete", + "reason": "register_esrch", + } + assert all( + "private-error-canary" not in path.read_text() for path in debugger.directory.iterdir() + ) def _mechanism_start(root, source, arguments=(), *, environment=None): From aeb1f66a62838965550ff46d6304f4a4bb5d3deb Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Sat, 21 Nov 2026 01:22:50 +0000 Subject: [PATCH 6/6] Require core dump privacy before native worker diagnostics Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 5 ++++- docs/development/testing.md | 10 +++++---- tests/_native_worker.py | 3 +++ tests/unit/test_native_worker.py | 38 ++++++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d69e5208d8..fc73a5740e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -197,7 +197,9 @@ jobs: sudo apt-get install --no-install-recommends -y gdb - name: Check native worker transport and termination if: matrix.part == 3 - run: python -m tests.unit.test_native_worker --mechanism + run: | + ulimit -S -c 0 + python -m tests.unit.test_native_worker --mechanism - name: pytest unit (this part, random order) env: LOOM_TEST_SHARD: ${{ matrix.part }}/${{ strategy.job-total }} @@ -207,6 +209,7 @@ jobs: echo "pytest-randomly seed: $SEED" NATIVE_ARGS=() if [ "${{ matrix.part }}" = 3 ]; then + ulimit -S -c 0 NATIVE_ARGS=(--native-worker-signatures) fi pytest tests/unit -vv -n auto -p randomly -p tests._worker_status --randomly-seed="$SEED" --durations=15 \ diff --git a/docs/development/testing.md b/docs/development/testing.md index 205741eaca..728098bbb6 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -58,10 +58,12 @@ GDB is a CI test tool; it is not a server dependency. The controlled transport c `python -m tests.unit.test_native_worker --mechanism`. `-p tests._worker_status --native-worker-signatures` opts into this collection on Linux -with GDB and an existing zero core-size limit. Missing prerequisites or a tracing -refusal stop startup. No tracing policy or capability changes are made. Each helper -sets its own parent-death signal so forced launcher termination also stops its debugger -and worker. Registrations never authorize controller signals to another process. +with GDB, a zero core-size limit and no piped core-dump handler. CI restricts only each +diagnostic command's soft core-size limit to zero. A piped handler, unreadable core +configuration, missing prerequisites or a tracing refusal stop startup. No tracing +policy or capability changes are made. Each helper sets its own parent-death signal so +forced launcher termination also stops its debugger and worker. Registrations never +authorize controller signals to another process. Session artifacts under `.pytest_cache/worker-status/` contain bounded native PCs, symbols, module identities and offsets for up to eight threads and 24 frames per diff --git a/tests/_native_worker.py b/tests/_native_worker.py index 8f03946f4d..c25c9e81cc 100644 --- a/tests/_native_worker.py +++ b/tests/_native_worker.py @@ -100,6 +100,9 @@ def __init__(self, root: Path) -> None: if resource.getrlimit(resource.RLIMIT_CORE)[0] != 0: raise ValueError("native diagnostics require an existing zero core limit") + # Piped kernel handlers receive raw dumps even when the core limit is zero. + if Path("/proc/sys/kernel/core_pattern").read_text().startswith("|"): + raise ValueError("native diagnostics refuse piped core dumps") if CONTEXT in os.environ: raise ValueError("native diagnostic environment is already in use") debugger = shutil.which("gdb") diff --git a/tests/unit/test_native_worker.py b/tests/unit/test_native_worker.py index 6b444a62c5..2d5b02b495 100644 --- a/tests/unit/test_native_worker.py +++ b/tests/unit/test_native_worker.py @@ -22,8 +22,22 @@ from tests import _native_worker as native +def _mock_core_pattern(monkeypatch, value): + original = Path.read_text + + def read_text(path, *args, **kwargs): + if path == Path("/proc/sys/kernel/core_pattern"): + if isinstance(value, OSError): + raise value + return value + return original(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read_text) + + @pytest.fixture def session(tmp_path, monkeypatch): + _mock_core_pattern(monkeypatch, "core.%p\n") monkeypatch.setattr(resource, "getrlimit", lambda _: (0, 0)) monkeypatch.setattr(native.shutil, "which", lambda _: "/synthetic/gdb") monkeypatch.delenv(native.CONTEXT, raising=False) @@ -67,6 +81,7 @@ def test_records_are_private_bounded_and_never_overwrite(tmp_path): @pytest.mark.parametrize("reason", ["core", "debugger", "environment"]) def test_unavailable_capture_refuses_before_creating_session(tmp_path, monkeypatch, reason): + _mock_core_pattern(monkeypatch, "core.%p\n") monkeypatch.setattr(resource, "getrlimit", lambda _: (1 if reason == "core" else 0, 1)) monkeypatch.setattr(native.shutil, "which", lambda _: None if reason == "debugger" else "gdb") monkeypatch.delenv(native.CONTEXT, raising=False) @@ -80,6 +95,29 @@ def test_unavailable_capture_refuses_before_creating_session(tmp_path, monkeypat assert not (tmp_path / "native").exists() +@pytest.mark.parametrize( + "pattern", + ["core.%p\n", "|/usr/bin/core-collector %p\n", PermissionError("core pattern unreadable")], +) +def test_core_dump_policy_is_checked_before_creating_session(tmp_path, monkeypatch, pattern): + _mock_core_pattern(monkeypatch, pattern) + monkeypatch.setattr(resource, "getrlimit", lambda _: (0, 0)) + monkeypatch.setattr(native.shutil, "which", lambda _: "/synthetic/gdb") + monkeypatch.delenv(native.CONTEXT, raising=False) + if isinstance(pattern, OSError): + with pytest.raises(OSError) as refused: + native.NativeSession(tmp_path) + assert refused.value is pattern + assert not (tmp_path / "native").exists() + elif pattern.startswith("|"): + with pytest.raises(ValueError, match="piped core dumps"): + native.NativeSession(tmp_path) + assert not (tmp_path / "native").exists() + else: + session = native.NativeSession(tmp_path) + assert session.root.is_dir() + + def test_worker_configuration_rejects_unsupported_specs_without_partial_changes(session): valid = SimpleNamespace(popen=True, via=None, python=None) invalid = SimpleNamespace(popen=True, via=None, python="custom-python")