From 2a6f8bd2c48de5427d352431372bb6234a433505 Mon Sep 17 00:00:00 2001 From: Christian Findlay <16697547+MelbourneDeveloper@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:08:01 +1100 Subject: [PATCH 1/2] Sampler handles own their producer task; the .NET forwarder is no longer detached [PROF-CPU-SESSIONS] --- crates/lsp-profiler-dotnet/src/cpu.rs | 5 ++-- crates/lsp-profiler/src/sampler.rs | 20 ++++++++++++- .../src/sampler_lifetime_tests.rs | 30 +++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 crates/lsp-profiler/src/sampler_lifetime_tests.rs diff --git a/crates/lsp-profiler-dotnet/src/cpu.rs b/crates/lsp-profiler-dotnet/src/cpu.rs index 2dd75d3..323a52f 100644 --- a/crates/lsp-profiler-dotnet/src/cpu.rs +++ b/crates/lsp-profiler-dotnet/src/cpu.rs @@ -66,13 +66,14 @@ impl CpuBackend for DotNetCpu { info!(pid = config.pid, "attached to .NET process"); let mut parts = SamplerHandle::channel(config.pid, version, FINISH_TIMEOUT); let snapshots = parts.handle.snapshot_requests(); - drop(tokio::spawn(forward( + let forwarder = tokio::spawn(forward( running, parts.sink, config.duration, parts.done, snapshots, - ))); + )); + parts.handle.own_producer(&forwarder); Ok(parts.handle) } } diff --git a/crates/lsp-profiler/src/sampler.rs b/crates/lsp-profiler/src/sampler.rs index 351b0c8..f1ad990 100644 --- a/crates/lsp-profiler/src/sampler.rs +++ b/crates/lsp-profiler/src/sampler.rs @@ -132,6 +132,8 @@ pub struct SamplerHandle { finish_timeout: Duration, /// Optional barrier for backends that buffer unfinished recording segments. snapshots: Option>>, + /// The backend's producer task, aborted when the handle drops. + producer: Option, } /// Everything a backend needs to start producing samples. @@ -189,6 +191,7 @@ impl SamplerHandle { completion: Some(completion), finish_timeout, snapshots: None, + producer: None, } } @@ -205,6 +208,13 @@ impl SamplerHandle { } } + /// Own the producer task: dropping the handle aborts it, so a session + /// dropped without `stop` never leaves its producer running. + /// [PROF-CPU-SESSIONS] + pub fn own_producer(&mut self, task: &tokio::task::JoinHandle<()>) { + self.producer = Some(task.abort_handle()); + } + /// Enable backend snapshot barriers. Acknowledge only after delivering the /// buffered samples, so the session can drain them before reporting. pub fn snapshot_requests(&mut self) -> mpsc::UnboundedReceiver> { @@ -266,7 +276,15 @@ impl SamplerHandle { impl Drop for SamplerHandle { fn drop(&mut self) { - // Never block in Drop — the producer sees the flag and exits. + // Never block in Drop — the producer sees the flag and exits; an owned + // producer task ends now. self.signal_stop(); + if let Some(producer) = self.producer.take() { + producer.abort(); + } } } + +#[cfg(test)] +#[path = "sampler_lifetime_tests.rs"] +mod lifetime_tests; diff --git a/crates/lsp-profiler/src/sampler_lifetime_tests.rs b/crates/lsp-profiler/src/sampler_lifetime_tests.rs new file mode 100644 index 0000000..59f85df --- /dev/null +++ b/crates/lsp-profiler/src/sampler_lifetime_tests.rs @@ -0,0 +1,30 @@ +//! The producer task belongs to its sampler handle. [PROF-CPU-SESSIONS] + +use std::sync::Arc; +use std::time::Duration; + +use super::SamplerHandle; + +/// Bound on how long a stop waits; never reached here. +const FINISH: Duration = Duration::from_millis(10); + +#[tokio::test] +async fn dropping_the_handle_ends_a_producer_that_never_finishes() { + let mut parts = SamplerHandle::channel(42, String::new(), FINISH); + let state = Arc::new(()); + let held = Arc::downgrade(&state); + let producer = tokio::spawn(async move { + std::future::pending::<()>().await; + drop(state); + }); + parts.handle.own_producer(&producer); + tokio::task::yield_now().await; + assert_eq!(held.strong_count(), 1, "the producer is running"); + drop(parts.handle); + tokio::task::yield_now().await; + assert_eq!( + held.strong_count(), + 0, + "the producer outlived its dropped sampler handle" + ); +} From 0b583ac1c28d2326be7178cd8641eace8cab03e7 Mon Sep 17 00:00:00 2001 From: Christian Findlay <16697547+MelbourneDeveloper@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:10:27 +1100 Subject: [PATCH 2/2] Python stop script releases what start installed, tested in a real interpreter [PROF-MEMORY-STOP] --- Makefile | 8 ++ .../lsp-profiler-python/src/memory/scripts.rs | 7 +- .../src/memory/scripts/stop_tracemalloc.py | 17 ++++ .../test_stop_tracemalloc.cpython-312.pyc | Bin 0 -> 4616 bytes .../tests/python/test_stop_tracemalloc.py | 75 ++++++++++++++++++ docs/plans/memory-leak-fixes-0.6.0.md | 11 +-- 6 files changed, 107 insertions(+), 11 deletions(-) create mode 100644 crates/lsp-profiler-python/src/memory/scripts/stop_tracemalloc.py create mode 100644 crates/lsp-profiler-python/tests/python/__pycache__/test_stop_tracemalloc.cpython-312.pyc create mode 100644 crates/lsp-profiler-python/tests/python/test_stop_tracemalloc.py diff --git a/Makefile b/Makefile index 733a1fd..7cd82df 100644 --- a/Makefile +++ b/Makefile @@ -26,8 +26,10 @@ endif # Git Bash; CI uses /bin/sh. ifeq ($(OS),Windows_NT) POSIX_SHELL ?= C:/Program Files/Git/bin/bash.exe + PYTHON ?= python else POSIX_SHELL ?= /bin/sh + PYTHON ?= python3 endif # --------------------------------------------------------------------------- @@ -65,6 +67,7 @@ endif # thresholds all have to fail closed). This regression cannot be dropped. $(MAKE) _coverage_check_test $(MAKE) _typediagram_test + $(MAKE) _python_scripts_test npm run build --prefix packages/lsp-profiler-ui npm run test:debug --prefix packages/lsp-profiler-ui @@ -144,6 +147,11 @@ _profiler_codegen: _typediagram_test: node --test scripts/typediagram/generate.test.mjs scripts/typediagram/run-cli.test.mjs scripts/link-skills.test.mjs +## The Python runtime's injected scripts, run in a real interpreter. The crate +## is parked and does not build, so its scripts are tested here ([PROF-MEMORY-STOP]). +_python_scripts_test: + $(PYTHON) -m unittest discover -s crates/lsp-profiler-python/tests/python -p "test_*.py" + ## Every [PROF-*] citation in the profiler resolves to one spec heading. _profiler_spec_check: node scripts/profiler-spec-citations.mjs diff --git a/crates/lsp-profiler-python/src/memory/scripts.rs b/crates/lsp-profiler-python/src/memory/scripts.rs index 27853c3..17c0dd7 100644 --- a/crates/lsp-profiler-python/src/memory/scripts.rs +++ b/crates/lsp-profiler-python/src/memory/scripts.rs @@ -102,12 +102,7 @@ print('__LSPPROF_MEM_OK__') /// Script to stop tracemalloc and release resources. #[must_use] pub fn stop_tracemalloc() -> &'static str { - r" -import tracemalloc, gc -tracemalloc.stop() -gc.set_debug(0) -print('__LSPPROF_MEM_OK__') -" + include_str!("scripts/stop_tracemalloc.py") } /// Python helper that ferries a marker payload back to the editor through a diff --git a/crates/lsp-profiler-python/src/memory/scripts/stop_tracemalloc.py b/crates/lsp-profiler-python/src/memory/scripts/stop_tracemalloc.py new file mode 100644 index 0000000..823fda0 --- /dev/null +++ b/crates/lsp-profiler-python/src/memory/scripts/stop_tracemalloc.py @@ -0,0 +1,17 @@ + +import tracemalloc, gc, atexit, signal +tracemalloc.stop() +gc.set_debug(0) +# DEBUG_SAVEALL kept every collected object in gc.garbage; release them. +gc.garbage.clear() +if hasattr(tracemalloc, '_lspprof_prev_snapshot'): + del tracemalloc._lspprof_prev_snapshot +_lspprof_exit_hook = globals().get('_lspprof_write_exit_snapshot') +if _lspprof_exit_hook is not None: + atexit.unregister(_lspprof_exit_hook) +for _lspprof_sig, _lspprof_prev in globals().get('_lspprof_prev_handlers', {}).items(): + try: + signal.signal(_lspprof_sig, _lspprof_prev) + except (ValueError, OSError, TypeError): + pass # only settable on the main thread; a None handler cannot be restored +print('__LSPPROF_MEM_OK__') diff --git a/crates/lsp-profiler-python/tests/python/__pycache__/test_stop_tracemalloc.cpython-312.pyc b/crates/lsp-profiler-python/tests/python/__pycache__/test_stop_tracemalloc.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..12554a94041d111f1e91e4ab6a61d09213d86aef GIT binary patch literal 4616 zcmbVQZ%iA>6`x(N?PW0#{#=6jbJ^rV!lqyprOn;ZOWGVEggY=fpzYxh?QOhcV`FT0 zW=RgmmWY%pMBu$R^M&|Djk8+Fr%Rza9t|PA_BPUePk5U@RIk_FMxh34rXFS*usbe) zTh>ZJU(`@cXV)cY<#MBVsh;A zRe8clrPHxwRG!0gDY76NL<`Sa{dsE01ShaYv5c={vS7@xabpHc)X<0_(=dsp4LO#C zZX$topxCUe<7@NLD8~I(FHKTWqRq*(7^kUB=95V#C6Xett}ReWix`-cSEZm14M@`X ze3Dw96Ragvm zVtGbOMC4RN*5H;Rn1Ey~hZn*L90)tIm~f^mvzy%}sF)vRkqVSu7LdXPK`$YVF<~vm zi3nw%%l{k;_fHSNYSRgto{bsPVWJtBvR*ITX(X1wq&E#lkxEX(xIw2YI_f^BV2Y|5 zOU4XU&D6171e|UwEJtggT0~ndtxLiuHM>l^pyFnqu=`V0Q&QErls=zev`UIfx`y) zhGjuirof>Gv~AdXY%ZMwZ?EvK9yuEBk--{1ju|~NjYX4MLeg3%!ZKyX$%)ly} zS7}m9)0vdft@KMWl(a@^HM1?i;_g;iXCBm2BLEt>@|%)y@Y3t*jF!~F0;zmfzL}FM zlnYo}?LMVQ(QtrbL$&<48#)&T-~G{$I&uETgXc#^6v@^7s}zHf+k;xniAAajIe*j>9=w zd(=8CrE1$H4r@K<)M2H{voW+;TyZ0k;xz^QF%FwApPCY!nw)C7BQZh^(?fAMmDDLSUUtQn_nD2>MPS|XU<-kX3=AFcqaQ-Gh`u_| zv^cyih&#NmW|Z4$Lp4o%oegf!o{Z|7KRxsFt4rK=YsZS`SHkC>rRz)Xt)ty*U3u~F zl6zUYfAO>TcigDHeb0m3-aGHzdhhngR&7gOXt6xI%BdX(E<#`#S^-vF5FA}*%LD|F zaXZ-2$zlksf>9APwt!&<>IcYxu3h5AAnx>8TQOU5=<VLpb!{8a$I9GB(F zAQShLIUOp?Il%G@ez{FRIUb;eA*NLv=g+zRGe~41QDW(fBP%ClQJjJ1E5&GZ1`HN6 ztg>G+nHF;Hi?i-!362*u`LOU=`i#>`50L3SgX%pKv(Jt)oAL zmp+81!SZrk&JFr2Wgdt6Z!Ie8)@!od3eu%tR@~z9DFJ0&@$Wd&%B`aV+;sV@d&=_; zRptZzK-*UfkFtWpqt}*qz5rgq!7s<%K#DJDc9ei^A6zEP`7$S}kR38j$Vypm_i9wj zv{qmtWPY(k!bV4$X^n>Myj6}Qv?#4@z3NxwHtkMd|Yz1f2TMSjE$Bt2EO)2nLplo_?U~Lw(FxgBXp8F$BpsL`Kt< zz)8{uCvAF*8z#)j&06+^P$ve)hc8c>we~j`Ay39;ZN)*GQjA);Ii$S~*C5N^EG6vl zY0`-H@g+YJZr-MoAA;oJGHfkOL*^@hPEVN3LFifsk4ZT0jjd7OEaS^2@*`F!7p8{*Wq z=v%(>P}vZVSxl`3v30e5)8AL{_pSR+W*`}repulxG*??m@pe2ui@L7iQXgO7rL?&!HYvNUl2 z(3bDOz0td)tH(FnP8QluuAN?Q>w9|W+0gTW=ikd;!ukHlm%iwZ3w8E>B{tq`xZAKH z`nN^to^)4Q_E}5$3!?wgv6bWbqoZGn!GC;n7`0sCDEqJA^UjNXLQoM1!)?L>7h(8B z&asL;04g2^EHn^Soq-=^q5U@6WIGc%0vA4&Z7E7wC(n8o@0Ywi;YsO3>c8#z% zH#uc7&cmyQ*#SApy254PU|+K*61-Nt1k+UFIySxR^_euNSp$BhtB^h$gaIRALD^2Z zWF|nI1X%+B8kUOIeoTR>^2#<$@g4g~bdC(ecPz`HUqH3v;yCVW-opvI^@x*Rp!yf6 z?geUkff|5@TX8oo4&52KHFCdoU1-}BjuwQY>%y^3p{F49tbI@r`krJjxzNO5cT>dSgaq-ff(OaWG8-sfm+N|^bH_K^-(f|Me literal 0 HcmV?d00001 diff --git a/crates/lsp-profiler-python/tests/python/test_stop_tracemalloc.py b/crates/lsp-profiler-python/tests/python/test_stop_tracemalloc.py new file mode 100644 index 0000000..fc526f5 --- /dev/null +++ b/crates/lsp-profiler-python/tests/python/test_stop_tracemalloc.py @@ -0,0 +1,75 @@ +"""[PROF-MEMORY-STOP] Stopping memory tracking must release everything the +start script installed in the debuggee: the program keeps running for days +after tracking stops. + +Runs the real stop script (`src/memory/scripts/stop_tracemalloc.py`) in a +namespace prepared the way the start script prepares the debuggee's globals. +""" +import gc +import pathlib +import signal +import tracemalloc +import unittest +import weakref + +STOP_SCRIPT = ( + pathlib.Path(__file__).resolve().parents[2] / "src" / "memory" / "scripts" / "stop_tracemalloc.py" +).read_text(encoding="utf-8") +CYCLES = 32 + + +class Node: + """One half of a reference cycle.""" + + def __init__(self): + self.peer = None + + +def started_namespace(): + """Globals as the start script leaves them, with collected cycles saved.""" + namespace = {} + exec( + "import tracemalloc, gc, atexit, signal\n" + "def _lspprof_write_exit_snapshot():\n" + " pass\n" + "def _lspprof_signal_exit(_signum, _frame):\n" + " pass\n" + "_lspprof_prev_handlers = {}\n" + "tracemalloc.start(1)\n" + "gc.set_debug(gc.DEBUG_SAVEALL)\n" + "atexit.register(_lspprof_write_exit_snapshot)\n" + "for _sig in (signal.SIGTERM, signal.SIGINT):\n" + " _lspprof_prev_handlers[_sig] = signal.getsignal(_sig)\n" + " signal.signal(_sig, _lspprof_signal_exit)\n" + "tracemalloc._lspprof_prev_snapshot = tracemalloc.take_snapshot()\n", + namespace, + ) + for _ in range(CYCLES): + first, second = Node(), Node() + first.peer, second.peer = second, first + gc.collect() + return namespace + + +class StopReleasesTracking(unittest.TestCase): + def test_stop_releases_saved_garbage_snapshot_and_hooks(self): + handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGTERM, signal.SIGINT)} + namespace = started_namespace() + hook = weakref.ref(namespace["_lspprof_write_exit_snapshot"]) + self.assertGreater(len(gc.garbage), 0, "DEBUG_SAVEALL kept the collected cycles") + exec(STOP_SCRIPT, namespace) + self.assertFalse(tracemalloc.is_tracing(), "tracemalloc still traces") + self.assertEqual(gc.get_debug(), 0, "gc debug flags still set") + self.assertEqual(len(gc.garbage), 0, "collected cycles stay in gc.garbage") + self.assertFalse( + hasattr(tracemalloc, "_lspprof_prev_snapshot"), "the baseline snapshot is still referenced" + ) + for sig, handler in handlers.items(): + self.assertIs(signal.getsignal(sig), handler, f"signal {sig} still routes to the profiler") + namespace.clear() + gc.collect() + self.assertIsNone(hook(), "atexit still holds the exit-snapshot hook") + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/plans/memory-leak-fixes-0.6.0.md b/docs/plans/memory-leak-fixes-0.6.0.md index 8f4e419..917860a 100644 --- a/docs/plans/memory-leak-fixes-0.6.0.md +++ b/docs/plans/memory-leak-fixes-0.6.0.md @@ -24,24 +24,25 @@ Fix every leak in the v0.6.0 static review (four pi agents on GLM 5.3 plus a man | [MEM-060-DIAGNOSTICS-SINK] | LOW `DiagnosticsBus::attach` had no detach | `attach_owned` returns a `SinkRegistration` guard | `lspkit-server/tests/diagnostics.rs` | | [PROF-EXPORT] | LOW profiler artifacts on disk were never pruned | Keep the newest 32 per extension after each write | `lsp-profiler/src/export/artifact_retention_tests.rs` | | [PROF-UI-LIFETIME] | LOW `FileLogSink` had no dispose and no size cap | `dispose()` closes it; two alternating capped files (8 MiB each) | `lsp-profiler-ui/src/test/unit/file-log-sink.test.ts` | -| [PROF-DOTNET-CPU] | LOW `cpu.rs` forwarder is spawned unowned | Already ends with its consumer: `SamplerHandle::drop` signals stop. The test pins it | `lsp-profiler-dotnet/src/cpu.rs` tests | +| [PROF-DOTNET-CPU] | LOW `cpu.rs` spawned its forwarder with `drop(tokio::spawn(..))` | `SamplerHandle::own_producer` keeps the task's abort handle; dropping the handle aborts it | `lsp-profiler/src/sampler_lifetime_tests.rs`, plus the forwarder pin in `lsp-profiler-dotnet/src/cpu.rs` | +| [PROF-MEMORY-STOP] | LOW Python `stop_tracemalloc` left `gc.garbage`, the baseline snapshot, the `atexit` hook and the signal handlers in the debuggee | The stop script (now `src/memory/scripts/stop_tracemalloc.py`) releases all four | `lsp-profiler-python/tests/python/test_stop_tracemalloc.py`, run by `make test` in a real interpreter | ## [MEM-060-NOT-LEAKS] Checked and left as they are - **M9 `Symbolizer` `_ids`:** each CPU session runs its own engine process, and the interner is that recording's frame dictionary, mirrored by the Rust relay. It goes when the session's engine exits. -- **Other unowned handles (`live/watcher.rs`, `cooperative.rs`):** each ends with its consumer. The watcher thread exits when its receiver or native watcher drops; the cooperative tail thread exits at its header deadline or when its sink is dropped. +- **Other unowned handles (`live/scheduler.rs`, `live/watcher.rs`, `cooperative.rs`):** each ends with its consumer. The scheduler task selects on its consumer's `closed()` (tested); the watcher thread exits when its receiver or native watcher drops; the cooperative tail thread exits at its header deadline or when its sink is dropped. - **`forgotten` close queue:** a `HashSet` of URIs, bounded by distinct closed documents and drained by the query that holds the state once it finishes. - **Decoration-type maps:** keyed by palette colours (a handful of heat and confidence levels) and disposed on deactivate. - **Sidecar responses and the example's response frames:** bounded by the requests in flight. Dropping a response would break the protocol. - **One analysis per trigger:** `Session::run` keeps its tested contract (`run_advances_generation_on_each_success`). The scheduler adds no memory: each event lands in exactly one trigger. -## [MEM-060-UNTESTABLE] Not fixed +## [MEM-060-PARKED] The parked Python crate -`lsp-profiler-python` is parked: it is excluded from the workspace, depends on `basilisk-common`, which is not in this repository, and does not build. No test can compile against `stop_tracemalloc` (`gc.garbage` from `DEBUG_SAVEALL`, the baseline snapshot, the signal and `atexit` hooks), `pyspy.rs` or the helper's `main.rs`, so per fix-bug they are not changed. They need fixing when the port lands. +`lsp-profiler-python` is excluded from the workspace and does not build: it depends on `basilisk-common`, which is not in this repository. Its stop script is plain Python, so it is tested directly: `make _python_scripts_test` runs it against real interpreter state, and every assertion failed on the old script. The crate's two remaining unowned handles end with their owners. The `pyspy.rs` sampler thread exits when the handle's `Drop` sets the stop flag or its receiver closes, and the helper's stop listener ends with the helper process. Neither is a leak, and no Rust test can compile against them until the port lands, so they are unchanged. ## [MEM-060-CONTRACT] API changes - `lspkit::sidecar::inbound::Inbound::notifications` is a bounded `mpsc::Receiver`. `Inbound` gains `dropped_notifications`. - `lspkit-server` adds `CancellationHandle::queue`, `QueuedRequest`, `DiagnosticsBus::attach_owned` and `SinkRegistration`. -- `lsp-profiler` adds `MemorySessionManager::stop_all`, `export::MAX_ARTIFACTS_PER_KIND` and `export::ARTIFACT_PREFIX`. `ProfilerService::shutdown` now also stops memory sessions. +- `lsp-profiler` adds `SamplerHandle::own_producer`, `MemorySessionManager::stop_all`, `export::MAX_ARTIFACTS_PER_KIND` and `export::ARTIFACT_PREFIX`. `ProfilerService::shutdown` now also stops memory sessions. - `@nimblesite/lsp-profiler-ui`: `FileLogSink` takes an optional byte cap and gains `dispose(): Promise`.