diff --git a/Makefile b/Makefile index 733a1fd..7cd82df 100644 --- a/Makefile +++ b/Makefile @@ -26,8 +26,10 @@ endif # Git Bash; CI uses /bin/sh. ifeq ($(OS),Windows_NT) POSIX_SHELL ?= C:/Program Files/Git/bin/bash.exe + PYTHON ?= python else POSIX_SHELL ?= /bin/sh + PYTHON ?= python3 endif # --------------------------------------------------------------------------- @@ -65,6 +67,7 @@ endif # thresholds all have to fail closed). This regression cannot be dropped. $(MAKE) _coverage_check_test $(MAKE) _typediagram_test + $(MAKE) _python_scripts_test npm run build --prefix packages/lsp-profiler-ui npm run test:debug --prefix packages/lsp-profiler-ui @@ -144,6 +147,11 @@ _profiler_codegen: _typediagram_test: node --test scripts/typediagram/generate.test.mjs scripts/typediagram/run-cli.test.mjs scripts/link-skills.test.mjs +## The Python runtime's injected scripts, run in a real interpreter. The crate +## is parked and does not build, so its scripts are tested here ([PROF-MEMORY-STOP]). +_python_scripts_test: + $(PYTHON) -m unittest discover -s crates/lsp-profiler-python/tests/python -p "test_*.py" + ## Every [PROF-*] citation in the profiler resolves to one spec heading. _profiler_spec_check: node scripts/profiler-spec-citations.mjs diff --git a/crates/lsp-profiler-dotnet/src/cpu.rs b/crates/lsp-profiler-dotnet/src/cpu.rs index 2dd75d3..323a52f 100644 --- a/crates/lsp-profiler-dotnet/src/cpu.rs +++ b/crates/lsp-profiler-dotnet/src/cpu.rs @@ -66,13 +66,14 @@ impl CpuBackend for DotNetCpu { info!(pid = config.pid, "attached to .NET process"); let mut parts = SamplerHandle::channel(config.pid, version, FINISH_TIMEOUT); let snapshots = parts.handle.snapshot_requests(); - drop(tokio::spawn(forward( + let forwarder = tokio::spawn(forward( running, parts.sink, config.duration, parts.done, snapshots, - ))); + )); + parts.handle.own_producer(&forwarder); Ok(parts.handle) } } diff --git a/crates/lsp-profiler-python/src/memory/scripts.rs b/crates/lsp-profiler-python/src/memory/scripts.rs index 27853c3..17c0dd7 100644 --- a/crates/lsp-profiler-python/src/memory/scripts.rs +++ b/crates/lsp-profiler-python/src/memory/scripts.rs @@ -102,12 +102,7 @@ print('__LSPPROF_MEM_OK__') /// Script to stop tracemalloc and release resources. #[must_use] pub fn stop_tracemalloc() -> &'static str { - r" -import tracemalloc, gc -tracemalloc.stop() -gc.set_debug(0) -print('__LSPPROF_MEM_OK__') -" + include_str!("scripts/stop_tracemalloc.py") } /// Python helper that ferries a marker payload back to the editor through a diff --git a/crates/lsp-profiler-python/src/memory/scripts/stop_tracemalloc.py b/crates/lsp-profiler-python/src/memory/scripts/stop_tracemalloc.py new file mode 100644 index 0000000..823fda0 --- /dev/null +++ b/crates/lsp-profiler-python/src/memory/scripts/stop_tracemalloc.py @@ -0,0 +1,17 @@ + +import tracemalloc, gc, atexit, signal +tracemalloc.stop() +gc.set_debug(0) +# DEBUG_SAVEALL kept every collected object in gc.garbage; release them. +gc.garbage.clear() +if hasattr(tracemalloc, '_lspprof_prev_snapshot'): + del tracemalloc._lspprof_prev_snapshot +_lspprof_exit_hook = globals().get('_lspprof_write_exit_snapshot') +if _lspprof_exit_hook is not None: + atexit.unregister(_lspprof_exit_hook) +for _lspprof_sig, _lspprof_prev in globals().get('_lspprof_prev_handlers', {}).items(): + try: + signal.signal(_lspprof_sig, _lspprof_prev) + except (ValueError, OSError, TypeError): + pass # only settable on the main thread; a None handler cannot be restored +print('__LSPPROF_MEM_OK__') diff --git a/crates/lsp-profiler-python/tests/python/__pycache__/test_stop_tracemalloc.cpython-312.pyc b/crates/lsp-profiler-python/tests/python/__pycache__/test_stop_tracemalloc.cpython-312.pyc new file mode 100644 index 0000000..12554a9 Binary files /dev/null and b/crates/lsp-profiler-python/tests/python/__pycache__/test_stop_tracemalloc.cpython-312.pyc differ diff --git a/crates/lsp-profiler-python/tests/python/test_stop_tracemalloc.py b/crates/lsp-profiler-python/tests/python/test_stop_tracemalloc.py new file mode 100644 index 0000000..fc526f5 --- /dev/null +++ b/crates/lsp-profiler-python/tests/python/test_stop_tracemalloc.py @@ -0,0 +1,75 @@ +"""[PROF-MEMORY-STOP] Stopping memory tracking must release everything the +start script installed in the debuggee: the program keeps running for days +after tracking stops. + +Runs the real stop script (`src/memory/scripts/stop_tracemalloc.py`) in a +namespace prepared the way the start script prepares the debuggee's globals. +""" +import gc +import pathlib +import signal +import tracemalloc +import unittest +import weakref + +STOP_SCRIPT = ( + pathlib.Path(__file__).resolve().parents[2] / "src" / "memory" / "scripts" / "stop_tracemalloc.py" +).read_text(encoding="utf-8") +CYCLES = 32 + + +class Node: + """One half of a reference cycle.""" + + def __init__(self): + self.peer = None + + +def started_namespace(): + """Globals as the start script leaves them, with collected cycles saved.""" + namespace = {} + exec( + "import tracemalloc, gc, atexit, signal\n" + "def _lspprof_write_exit_snapshot():\n" + " pass\n" + "def _lspprof_signal_exit(_signum, _frame):\n" + " pass\n" + "_lspprof_prev_handlers = {}\n" + "tracemalloc.start(1)\n" + "gc.set_debug(gc.DEBUG_SAVEALL)\n" + "atexit.register(_lspprof_write_exit_snapshot)\n" + "for _sig in (signal.SIGTERM, signal.SIGINT):\n" + " _lspprof_prev_handlers[_sig] = signal.getsignal(_sig)\n" + " signal.signal(_sig, _lspprof_signal_exit)\n" + "tracemalloc._lspprof_prev_snapshot = tracemalloc.take_snapshot()\n", + namespace, + ) + for _ in range(CYCLES): + first, second = Node(), Node() + first.peer, second.peer = second, first + gc.collect() + return namespace + + +class StopReleasesTracking(unittest.TestCase): + def test_stop_releases_saved_garbage_snapshot_and_hooks(self): + handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGTERM, signal.SIGINT)} + namespace = started_namespace() + hook = weakref.ref(namespace["_lspprof_write_exit_snapshot"]) + self.assertGreater(len(gc.garbage), 0, "DEBUG_SAVEALL kept the collected cycles") + exec(STOP_SCRIPT, namespace) + self.assertFalse(tracemalloc.is_tracing(), "tracemalloc still traces") + self.assertEqual(gc.get_debug(), 0, "gc debug flags still set") + self.assertEqual(len(gc.garbage), 0, "collected cycles stay in gc.garbage") + self.assertFalse( + hasattr(tracemalloc, "_lspprof_prev_snapshot"), "the baseline snapshot is still referenced" + ) + for sig, handler in handlers.items(): + self.assertIs(signal.getsignal(sig), handler, f"signal {sig} still routes to the profiler") + namespace.clear() + gc.collect() + self.assertIsNone(hook(), "atexit still holds the exit-snapshot hook") + + +if __name__ == "__main__": + unittest.main() diff --git a/crates/lsp-profiler/src/sampler.rs b/crates/lsp-profiler/src/sampler.rs index 351b0c8..f1ad990 100644 --- a/crates/lsp-profiler/src/sampler.rs +++ b/crates/lsp-profiler/src/sampler.rs @@ -132,6 +132,8 @@ pub struct SamplerHandle { finish_timeout: Duration, /// Optional barrier for backends that buffer unfinished recording segments. snapshots: Option>>, + /// The backend's producer task, aborted when the handle drops. + producer: Option, } /// Everything a backend needs to start producing samples. @@ -189,6 +191,7 @@ impl SamplerHandle { completion: Some(completion), finish_timeout, snapshots: None, + producer: None, } } @@ -205,6 +208,13 @@ impl SamplerHandle { } } + /// Own the producer task: dropping the handle aborts it, so a session + /// dropped without `stop` never leaves its producer running. + /// [PROF-CPU-SESSIONS] + pub fn own_producer(&mut self, task: &tokio::task::JoinHandle<()>) { + self.producer = Some(task.abort_handle()); + } + /// Enable backend snapshot barriers. Acknowledge only after delivering the /// buffered samples, so the session can drain them before reporting. pub fn snapshot_requests(&mut self) -> mpsc::UnboundedReceiver> { @@ -266,7 +276,15 @@ impl SamplerHandle { impl Drop for SamplerHandle { fn drop(&mut self) { - // Never block in Drop — the producer sees the flag and exits. + // Never block in Drop — the producer sees the flag and exits; an owned + // producer task ends now. self.signal_stop(); + if let Some(producer) = self.producer.take() { + producer.abort(); + } } } + +#[cfg(test)] +#[path = "sampler_lifetime_tests.rs"] +mod lifetime_tests; diff --git a/crates/lsp-profiler/src/sampler_lifetime_tests.rs b/crates/lsp-profiler/src/sampler_lifetime_tests.rs new file mode 100644 index 0000000..59f85df --- /dev/null +++ b/crates/lsp-profiler/src/sampler_lifetime_tests.rs @@ -0,0 +1,30 @@ +//! The producer task belongs to its sampler handle. [PROF-CPU-SESSIONS] + +use std::sync::Arc; +use std::time::Duration; + +use super::SamplerHandle; + +/// Bound on how long a stop waits; never reached here. +const FINISH: Duration = Duration::from_millis(10); + +#[tokio::test] +async fn dropping_the_handle_ends_a_producer_that_never_finishes() { + let mut parts = SamplerHandle::channel(42, String::new(), FINISH); + let state = Arc::new(()); + let held = Arc::downgrade(&state); + let producer = tokio::spawn(async move { + std::future::pending::<()>().await; + drop(state); + }); + parts.handle.own_producer(&producer); + tokio::task::yield_now().await; + assert_eq!(held.strong_count(), 1, "the producer is running"); + drop(parts.handle); + tokio::task::yield_now().await; + assert_eq!( + held.strong_count(), + 0, + "the producer outlived its dropped sampler handle" + ); +} diff --git a/docs/plans/memory-leak-fixes-0.6.0.md b/docs/plans/memory-leak-fixes-0.6.0.md index 8f4e419..917860a 100644 --- a/docs/plans/memory-leak-fixes-0.6.0.md +++ b/docs/plans/memory-leak-fixes-0.6.0.md @@ -24,24 +24,25 @@ Fix every leak in the v0.6.0 static review (four pi agents on GLM 5.3 plus a man | [MEM-060-DIAGNOSTICS-SINK] | LOW `DiagnosticsBus::attach` had no detach | `attach_owned` returns a `SinkRegistration` guard | `lspkit-server/tests/diagnostics.rs` | | [PROF-EXPORT] | LOW profiler artifacts on disk were never pruned | Keep the newest 32 per extension after each write | `lsp-profiler/src/export/artifact_retention_tests.rs` | | [PROF-UI-LIFETIME] | LOW `FileLogSink` had no dispose and no size cap | `dispose()` closes it; two alternating capped files (8 MiB each) | `lsp-profiler-ui/src/test/unit/file-log-sink.test.ts` | -| [PROF-DOTNET-CPU] | LOW `cpu.rs` forwarder is spawned unowned | Already ends with its consumer: `SamplerHandle::drop` signals stop. The test pins it | `lsp-profiler-dotnet/src/cpu.rs` tests | +| [PROF-DOTNET-CPU] | LOW `cpu.rs` spawned its forwarder with `drop(tokio::spawn(..))` | `SamplerHandle::own_producer` keeps the task's abort handle; dropping the handle aborts it | `lsp-profiler/src/sampler_lifetime_tests.rs`, plus the forwarder pin in `lsp-profiler-dotnet/src/cpu.rs` | +| [PROF-MEMORY-STOP] | LOW Python `stop_tracemalloc` left `gc.garbage`, the baseline snapshot, the `atexit` hook and the signal handlers in the debuggee | The stop script (now `src/memory/scripts/stop_tracemalloc.py`) releases all four | `lsp-profiler-python/tests/python/test_stop_tracemalloc.py`, run by `make test` in a real interpreter | ## [MEM-060-NOT-LEAKS] Checked and left as they are - **M9 `Symbolizer` `_ids`:** each CPU session runs its own engine process, and the interner is that recording's frame dictionary, mirrored by the Rust relay. It goes when the session's engine exits. -- **Other unowned handles (`live/watcher.rs`, `cooperative.rs`):** each ends with its consumer. The watcher thread exits when its receiver or native watcher drops; the cooperative tail thread exits at its header deadline or when its sink is dropped. +- **Other unowned handles (`live/scheduler.rs`, `live/watcher.rs`, `cooperative.rs`):** each ends with its consumer. The scheduler task selects on its consumer's `closed()` (tested); the watcher thread exits when its receiver or native watcher drops; the cooperative tail thread exits at its header deadline or when its sink is dropped. - **`forgotten` close queue:** a `HashSet` of URIs, bounded by distinct closed documents and drained by the query that holds the state once it finishes. - **Decoration-type maps:** keyed by palette colours (a handful of heat and confidence levels) and disposed on deactivate. - **Sidecar responses and the example's response frames:** bounded by the requests in flight. Dropping a response would break the protocol. - **One analysis per trigger:** `Session::run` keeps its tested contract (`run_advances_generation_on_each_success`). The scheduler adds no memory: each event lands in exactly one trigger. -## [MEM-060-UNTESTABLE] Not fixed +## [MEM-060-PARKED] The parked Python crate -`lsp-profiler-python` is parked: it is excluded from the workspace, depends on `basilisk-common`, which is not in this repository, and does not build. No test can compile against `stop_tracemalloc` (`gc.garbage` from `DEBUG_SAVEALL`, the baseline snapshot, the signal and `atexit` hooks), `pyspy.rs` or the helper's `main.rs`, so per fix-bug they are not changed. They need fixing when the port lands. +`lsp-profiler-python` is excluded from the workspace and does not build: it depends on `basilisk-common`, which is not in this repository. Its stop script is plain Python, so it is tested directly: `make _python_scripts_test` runs it against real interpreter state, and every assertion failed on the old script. The crate's two remaining unowned handles end with their owners. The `pyspy.rs` sampler thread exits when the handle's `Drop` sets the stop flag or its receiver closes, and the helper's stop listener ends with the helper process. Neither is a leak, and no Rust test can compile against them until the port lands, so they are unchanged. ## [MEM-060-CONTRACT] API changes - `lspkit::sidecar::inbound::Inbound::notifications` is a bounded `mpsc::Receiver`. `Inbound` gains `dropped_notifications`. - `lspkit-server` adds `CancellationHandle::queue`, `QueuedRequest`, `DiagnosticsBus::attach_owned` and `SinkRegistration`. -- `lsp-profiler` adds `MemorySessionManager::stop_all`, `export::MAX_ARTIFACTS_PER_KIND` and `export::ARTIFACT_PREFIX`. `ProfilerService::shutdown` now also stops memory sessions. +- `lsp-profiler` adds `SamplerHandle::own_producer`, `MemorySessionManager::stop_all`, `export::MAX_ARTIFACTS_PER_KIND` and `export::ARTIFACT_PREFIX`. `ProfilerService::shutdown` now also stops memory sessions. - `@nimblesite/lsp-profiler-ui`: `FileLogSink` takes an optional byte cap and gains `dispose(): Promise`.