Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@ endif
# Git Bash; CI uses /bin/sh.
ifeq ($(OS),Windows_NT)
POSIX_SHELL ?= C:/Program Files/Git/bin/bash.exe
PYTHON ?= python
else
POSIX_SHELL ?= /bin/sh
PYTHON ?= python3
endif

# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -65,6 +67,7 @@ endif
# thresholds all have to fail closed). This regression cannot be dropped.
$(MAKE) _coverage_check_test
$(MAKE) _typediagram_test
$(MAKE) _python_scripts_test
npm run build --prefix packages/lsp-profiler-ui
npm run test:debug --prefix packages/lsp-profiler-ui

Expand Down Expand Up @@ -144,6 +147,11 @@ _profiler_codegen:
_typediagram_test:
node --test scripts/typediagram/generate.test.mjs scripts/typediagram/run-cli.test.mjs scripts/link-skills.test.mjs

## The Python runtime's injected scripts, run in a real interpreter. The crate
## is parked and does not build, so its scripts are tested here ([PROF-MEMORY-STOP]).
_python_scripts_test:
$(PYTHON) -m unittest discover -s crates/lsp-profiler-python/tests/python -p "test_*.py"

## Every [PROF-*] citation in the profiler resolves to one spec heading.
_profiler_spec_check:
node scripts/profiler-spec-citations.mjs
Expand Down
5 changes: 3 additions & 2 deletions crates/lsp-profiler-dotnet/src/cpu.rs
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,14 @@ impl CpuBackend for DotNetCpu {
info!(pid = config.pid, "attached to .NET process");
let mut parts = SamplerHandle::channel(config.pid, version, FINISH_TIMEOUT);
let snapshots = parts.handle.snapshot_requests();
drop(tokio::spawn(forward(
let forwarder = tokio::spawn(forward(
running,
parts.sink,
config.duration,
parts.done,
snapshots,
)));
));
parts.handle.own_producer(&forwarder);
Ok(parts.handle)
}
}
Expand Down
7 changes: 1 addition & 6 deletions crates/lsp-profiler-python/src/memory/scripts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -102,12 +102,7 @@ print('__LSPPROF_MEM_OK__')
/// Script to stop tracemalloc and release resources.
#[must_use]
pub fn stop_tracemalloc() -> &'static str {
r"
import tracemalloc, gc
tracemalloc.stop()
gc.set_debug(0)
print('__LSPPROF_MEM_OK__')
"
include_str!("scripts/stop_tracemalloc.py")
}

/// Python helper that ferries a marker payload back to the editor through a
Expand Down
17 changes: 17 additions & 0 deletions crates/lsp-profiler-python/src/memory/scripts/stop_tracemalloc.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@

import tracemalloc, gc, atexit, signal
tracemalloc.stop()
gc.set_debug(0)
# DEBUG_SAVEALL kept every collected object in gc.garbage; release them.
gc.garbage.clear()
if hasattr(tracemalloc, '_lspprof_prev_snapshot'):
del tracemalloc._lspprof_prev_snapshot
_lspprof_exit_hook = globals().get('_lspprof_write_exit_snapshot')
if _lspprof_exit_hook is not None:
atexit.unregister(_lspprof_exit_hook)
for _lspprof_sig, _lspprof_prev in globals().get('_lspprof_prev_handlers', {}).items():
try:
signal.signal(_lspprof_sig, _lspprof_prev)
except (ValueError, OSError, TypeError):
pass # only settable on the main thread; a None handler cannot be restored
print('__LSPPROF_MEM_OK__')
Binary file not shown.
75 changes: 75 additions & 0 deletions crates/lsp-profiler-python/tests/python/test_stop_tracemalloc.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
"""[PROF-MEMORY-STOP] Stopping memory tracking must release everything the
start script installed in the debuggee: the program keeps running for days
after tracking stops.

Runs the real stop script (`src/memory/scripts/stop_tracemalloc.py`) in a
namespace prepared the way the start script prepares the debuggee's globals.
"""
import gc
import pathlib
import signal
import tracemalloc
import unittest
import weakref

STOP_SCRIPT = (
pathlib.Path(__file__).resolve().parents[2] / "src" / "memory" / "scripts" / "stop_tracemalloc.py"
).read_text(encoding="utf-8")
CYCLES = 32


class Node:
"""One half of a reference cycle."""

def __init__(self):
self.peer = None


def started_namespace():
"""Globals as the start script leaves them, with collected cycles saved."""
namespace = {}
exec(
"import tracemalloc, gc, atexit, signal\n"
"def _lspprof_write_exit_snapshot():\n"
" pass\n"
"def _lspprof_signal_exit(_signum, _frame):\n"
" pass\n"
"_lspprof_prev_handlers = {}\n"
"tracemalloc.start(1)\n"
"gc.set_debug(gc.DEBUG_SAVEALL)\n"
"atexit.register(_lspprof_write_exit_snapshot)\n"
"for _sig in (signal.SIGTERM, signal.SIGINT):\n"
" _lspprof_prev_handlers[_sig] = signal.getsignal(_sig)\n"
" signal.signal(_sig, _lspprof_signal_exit)\n"
"tracemalloc._lspprof_prev_snapshot = tracemalloc.take_snapshot()\n",
namespace,
)
for _ in range(CYCLES):
first, second = Node(), Node()
first.peer, second.peer = second, first
gc.collect()
return namespace


class StopReleasesTracking(unittest.TestCase):
def test_stop_releases_saved_garbage_snapshot_and_hooks(self):
handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGTERM, signal.SIGINT)}
namespace = started_namespace()
hook = weakref.ref(namespace["_lspprof_write_exit_snapshot"])
self.assertGreater(len(gc.garbage), 0, "DEBUG_SAVEALL kept the collected cycles")
exec(STOP_SCRIPT, namespace)
self.assertFalse(tracemalloc.is_tracing(), "tracemalloc still traces")
self.assertEqual(gc.get_debug(), 0, "gc debug flags still set")
self.assertEqual(len(gc.garbage), 0, "collected cycles stay in gc.garbage")
self.assertFalse(
hasattr(tracemalloc, "_lspprof_prev_snapshot"), "the baseline snapshot is still referenced"
)
for sig, handler in handlers.items():
self.assertIs(signal.getsignal(sig), handler, f"signal {sig} still routes to the profiler")
namespace.clear()
gc.collect()
self.assertIsNone(hook(), "atexit still holds the exit-snapshot hook")


if __name__ == "__main__":
unittest.main()
20 changes: 19 additions & 1 deletion crates/lsp-profiler/src/sampler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,8 @@ pub struct SamplerHandle {
finish_timeout: Duration,
/// Optional barrier for backends that buffer unfinished recording segments.
snapshots: Option<mpsc::UnboundedSender<oneshot::Sender<()>>>,
/// The backend's producer task, aborted when the handle drops.
producer: Option<tokio::task::AbortHandle>,
}

/// Everything a backend needs to start producing samples.
Expand Down Expand Up @@ -189,6 +191,7 @@ impl SamplerHandle {
completion: Some(completion),
finish_timeout,
snapshots: None,
producer: None,
}
}

Expand All @@ -205,6 +208,13 @@ impl SamplerHandle {
}
}

/// Own the producer task: dropping the handle aborts it, so a session
/// dropped without `stop` never leaves its producer running.
/// [PROF-CPU-SESSIONS]
pub fn own_producer(&mut self, task: &tokio::task::JoinHandle<()>) {
self.producer = Some(task.abort_handle());
}

/// Enable backend snapshot barriers. Acknowledge only after delivering the
/// buffered samples, so the session can drain them before reporting.
pub fn snapshot_requests(&mut self) -> mpsc::UnboundedReceiver<oneshot::Sender<()>> {
Expand Down Expand Up @@ -266,7 +276,15 @@ impl SamplerHandle {

impl Drop for SamplerHandle {
fn drop(&mut self) {
// Never block in Drop — the producer sees the flag and exits.
// Never block in Drop — the producer sees the flag and exits; an owned
// producer task ends now.
self.signal_stop();
if let Some(producer) = self.producer.take() {
producer.abort();
}
}
}

#[cfg(test)]
#[path = "sampler_lifetime_tests.rs"]
mod lifetime_tests;
30 changes: 30 additions & 0 deletions crates/lsp-profiler/src/sampler_lifetime_tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
//! The producer task belongs to its sampler handle. [PROF-CPU-SESSIONS]

use std::sync::Arc;
use std::time::Duration;

use super::SamplerHandle;

/// Bound on how long a stop waits; never reached here.
const FINISH: Duration = Duration::from_millis(10);

#[tokio::test]
async fn dropping_the_handle_ends_a_producer_that_never_finishes() {
let mut parts = SamplerHandle::channel(42, String::new(), FINISH);
let state = Arc::new(());
let held = Arc::downgrade(&state);
let producer = tokio::spawn(async move {
std::future::pending::<()>().await;
drop(state);
});
parts.handle.own_producer(&producer);
tokio::task::yield_now().await;
assert_eq!(held.strong_count(), 1, "the producer is running");
drop(parts.handle);
tokio::task::yield_now().await;
assert_eq!(
held.strong_count(),
0,
"the producer outlived its dropped sampler handle"
);
}
11 changes: 6 additions & 5 deletions docs/plans/memory-leak-fixes-0.6.0.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,24 +24,25 @@ Fix every leak in the v0.6.0 static review (four pi agents on GLM 5.3 plus a man
| [MEM-060-DIAGNOSTICS-SINK] | LOW `DiagnosticsBus::attach` had no detach | `attach_owned` returns a `SinkRegistration` guard | `lspkit-server/tests/diagnostics.rs` |
| [PROF-EXPORT] | LOW profiler artifacts on disk were never pruned | Keep the newest 32 per extension after each write | `lsp-profiler/src/export/artifact_retention_tests.rs` |
| [PROF-UI-LIFETIME] | LOW `FileLogSink` had no dispose and no size cap | `dispose()` closes it; two alternating capped files (8 MiB each) | `lsp-profiler-ui/src/test/unit/file-log-sink.test.ts` |
| [PROF-DOTNET-CPU] | LOW `cpu.rs` forwarder is spawned unowned | Already ends with its consumer: `SamplerHandle::drop` signals stop. The test pins it | `lsp-profiler-dotnet/src/cpu.rs` tests |
| [PROF-DOTNET-CPU] | LOW `cpu.rs` spawned its forwarder with `drop(tokio::spawn(..))` | `SamplerHandle::own_producer` keeps the task's abort handle; dropping the handle aborts it | `lsp-profiler/src/sampler_lifetime_tests.rs`, plus the forwarder pin in `lsp-profiler-dotnet/src/cpu.rs` |
| [PROF-MEMORY-STOP] | LOW Python `stop_tracemalloc` left `gc.garbage`, the baseline snapshot, the `atexit` hook and the signal handlers in the debuggee | The stop script (now `src/memory/scripts/stop_tracemalloc.py`) releases all four | `lsp-profiler-python/tests/python/test_stop_tracemalloc.py`, run by `make test` in a real interpreter |

## [MEM-060-NOT-LEAKS] Checked and left as they are

- **M9 `Symbolizer` `_ids`:** each CPU session runs its own engine process, and the interner is that recording's frame dictionary, mirrored by the Rust relay. It goes when the session's engine exits.
- **Other unowned handles (`live/watcher.rs`, `cooperative.rs`):** each ends with its consumer. The watcher thread exits when its receiver or native watcher drops; the cooperative tail thread exits at its header deadline or when its sink is dropped.
- **Other unowned handles (`live/scheduler.rs`, `live/watcher.rs`, `cooperative.rs`):** each ends with its consumer. The scheduler task selects on its consumer's `closed()` (tested); the watcher thread exits when its receiver or native watcher drops; the cooperative tail thread exits at its header deadline or when its sink is dropped.
- **`forgotten` close queue:** a `HashSet` of URIs, bounded by distinct closed documents and drained by the query that holds the state once it finishes.
- **Decoration-type maps:** keyed by palette colours (a handful of heat and confidence levels) and disposed on deactivate.
- **Sidecar responses and the example's response frames:** bounded by the requests in flight. Dropping a response would break the protocol.
- **One analysis per trigger:** `Session::run` keeps its tested contract (`run_advances_generation_on_each_success`). The scheduler adds no memory: each event lands in exactly one trigger.

## [MEM-060-UNTESTABLE] Not fixed
## [MEM-060-PARKED] The parked Python crate

`lsp-profiler-python` is parked: it is excluded from the workspace, depends on `basilisk-common`, which is not in this repository, and does not build. No test can compile against `stop_tracemalloc` (`gc.garbage` from `DEBUG_SAVEALL`, the baseline snapshot, the signal and `atexit` hooks), `pyspy.rs` or the helper's `main.rs`, so per fix-bug they are not changed. They need fixing when the port lands.
`lsp-profiler-python` is excluded from the workspace and does not build: it depends on `basilisk-common`, which is not in this repository. Its stop script is plain Python, so it is tested directly: `make _python_scripts_test` runs it against real interpreter state, and every assertion failed on the old script. The crate's two remaining unowned handles end with their owners. The `pyspy.rs` sampler thread exits when the handle's `Drop` sets the stop flag or its receiver closes, and the helper's stop listener ends with the helper process. Neither is a leak, and no Rust test can compile against them until the port lands, so they are unchanged.

## [MEM-060-CONTRACT] API changes

- `lspkit::sidecar::inbound::Inbound::notifications` is a bounded `mpsc::Receiver`. `Inbound` gains `dropped_notifications`.
- `lspkit-server` adds `CancellationHandle::queue`, `QueuedRequest`, `DiagnosticsBus::attach_owned` and `SinkRegistration`.
- `lsp-profiler` adds `MemorySessionManager::stop_all`, `export::MAX_ARTIFACTS_PER_KIND` and `export::ARTIFACT_PREFIX`. `ProfilerService::shutdown` now also stops memory sessions.
- `lsp-profiler` adds `SamplerHandle::own_producer`, `MemorySessionManager::stop_all`, `export::MAX_ARTIFACTS_PER_KIND` and `export::ARTIFACT_PREFIX`. `ProfilerService::shutdown` now also stops memory sessions.
- `@nimblesite/lsp-profiler-ui`: `FileLogSink` takes an optional byte cap and gains `dispose(): Promise<void>`.
Loading