WIP: Memory leak fix for cuda_context

AGENTS.md

@@ -85,7 +85,8 @@ Be sure to update the directory layout in this file if the directory layout chan
 |------|-------------|
 | `include/livekit/` | Public API headers (what SDK consumers include) |
 | `src/` | Implementation files and internal-only headers (`ffi_client.h`, `lk_log.h`, etc.) |
-| `src/tests/` | Google Test integration and stress tests |
+| `src/tests/` | Google Test unit, integration, and stress tests |
+| `src/tests/manual/` | Standalone testers built with the test targets but not registered with CTest |
 | `examples/` | In-tree example applications |
 | `client-sdk-rust/` | Git submodule holding the Rust core of the SDK|
 | `cpp-tools/` | Git submodule holding shared LiveKit C++ engineering guidance, clang-format / clang-tidy configs, scripts, docs, and CI workflow |

scripts/track_process_memory.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+#
+# Copyright 2026 LiveKit
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Run a command and report its resident memory usage."""
+
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+import time
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Run a command and report its initial, final, and peak RSS."
+    )
+    parser.add_argument(
+        "--interval",
+        type=float,
+        default=0.1,
+        help="seconds between RSS samples (default: 0.1)",
+    )
+    parser.add_argument(
+        "command",
+        nargs=argparse.REMAINDER,
+        help="command and its arguments; prefix it with -- when needed",
+    )
+    args = parser.parse_args()
+    if args.interval <= 0:
+        parser.error("--interval must be greater than zero")
+    if not args.command:
+        parser.error("a command is required")
+    return args
+
+
+def process_rss_kib(pid: int) -> int | None:
+    result = subprocess.run(
+        ["ps", "-o", "rss=", "-p", str(pid)],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        return None
+
+    rss = result.stdout.strip()
+    if not rss:
+        return None
+
+    rss_kib = int(rss)
+    # macOS reports 0 RSS for a child that has exited but has not yet been
+    # reaped. Do not overwrite the last live-process sample with that value.
+    return rss_kib if rss_kib > 0 else None
+
+
+def format_rss(rss_kib: int) -> str:
+    return f"{rss_kib:,} KiB ({rss_kib / 1024:.2f} MiB)"
+
+
+def main() -> int:
+    args = parse_args()
+    command = args.command
+    if command[0] == "--":
+        command = command[1:]
+    if not command:
+        print("error: a command is required after --", file=sys.stderr)
+        return 2
+
+    try:
+        process = subprocess.Popen(command)
+    except OSError as error:
+        print(f"error: could not start {command[0]!r}: {error}", file=sys.stderr)
+        return 127
+
+    started_at = time.monotonic()
+    initial_rss_kib: int | None = None
+    final_rss_kib: int | None = None
+    peak_rss_kib: int | None = None
+
+    while process.poll() is None:
+        rss_kib = process_rss_kib(process.pid)
+        if rss_kib is not None:
+            if initial_rss_kib is None:
+                initial_rss_kib = rss_kib
+            final_rss_kib = rss_kib
+            peak_rss_kib = max(peak_rss_kib or rss_kib, rss_kib)
+        time.sleep(args.interval)
+
+    elapsed_s = time.monotonic() - started_at
+    exit_code = process.wait()
+    print(f"command: {' '.join(command)}")
+    print(f"exit code: {exit_code}")
+    print(f"elapsed: {elapsed_s:.2f} s")
+    if initial_rss_kib is None:
+        print("RSS: no samples collected; the command exited before sampling began")
+    else:
+        assert final_rss_kib is not None
+        assert peak_rss_kib is not None
+        print(f"RSS initial: {format_rss(initial_rss_kib)}")
+        print(f"RSS final observed: {format_rss(final_rss_kib)}")
+        print(f"RSS peak: {format_rss(peak_rss_kib)}")
+        print(f"RSS change: {format_rss(final_rss_kib - initial_rss_kib)}")
+
+    return exit_code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/tests/CMakeLists.txt

@@ -348,3 +348,49 @@ endif()
 if(TARGET livekit_stress_tests)
   add_dependencies(run_all_tests livekit_stress_tests)
 endif()
+
+# ============================================================================
+# Manual testers (built with the test targets, not registered with CTest)
+# ============================================================================
+
+add_executable(livekit_memory_lifecycle_tester
+  "${CMAKE_CURRENT_SOURCE_DIR}/manual/memory_lifecycle_tester/main.cpp"
+)
+
+target_link_libraries(livekit_memory_lifecycle_tester PRIVATE livekit)
+target_include_directories(livekit_memory_lifecycle_tester PRIVATE ${LIVEKIT_ROOT_DIR}/include)
+target_compile_definitions(livekit_memory_lifecycle_tester PRIVATE
+  $<$<PLATFORM_ID:Windows>:_USE_MATH_DEFINES>
+)
+
+if(WIN32)
+  add_custom_command(TARGET livekit_memory_lifecycle_tester POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      $<TARGET_FILE:livekit>
+      $<TARGET_FILE_DIR:livekit_memory_lifecycle_tester>
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      "$<TARGET_FILE_DIR:livekit>/livekit_ffi.dll"
+      $<TARGET_FILE_DIR:livekit_memory_lifecycle_tester>
+    COMMENT "Copying DLLs next to livekit_memory_lifecycle_tester"
+  )
+elseif(APPLE)
+  add_custom_command(TARGET livekit_memory_lifecycle_tester POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      $<TARGET_FILE:livekit>
+      $<TARGET_FILE_DIR:livekit_memory_lifecycle_tester>
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      "$<TARGET_FILE_DIR:livekit>/liblivekit_ffi.dylib"
+      $<TARGET_FILE_DIR:livekit_memory_lifecycle_tester>
+    COMMENT "Copying dylibs next to livekit_memory_lifecycle_tester"
+  )
+else()
+  add_custom_command(TARGET livekit_memory_lifecycle_tester POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      $<TARGET_FILE:livekit>
+      $<TARGET_FILE_DIR:livekit_memory_lifecycle_tester>
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      "$<TARGET_FILE_DIR:livekit>/liblivekit_ffi.so"
+      $<TARGET_FILE_DIR:livekit_memory_lifecycle_tester>
+    COMMENT "Copying shared libraries next to livekit_memory_lifecycle_tester"
+  )
+endif()

src/tests/manual/memory_lifecycle_tester/README.md

@@ -0,0 +1,51 @@
+# Memory lifecycle tester
+
+This standalone application repeatedly exercises the public C++ SDK lifecycle
+to expose retained lower-level Rust/WebRTC resources.
+
+Each iteration:
+
+- calls `livekit::initialize()`;
+- creates an `AudioSource` and `LocalAudioTrack`;
+- creates a 1280x720 `VideoSource` and `LocalVideoTrack`;
+- creates a representative `DataTrackFrame`;
+- destroys those objects before calling `livekit::shutdown()`.
+
+The video source intentionally receives no captured frame. This exercises
+teardown of the Rust keepalive task that runs until the first raw video frame
+arrives and previously retained roughly one 720p frame per lifecycle.
+
+`LocalDataTrack` itself cannot be created offline: its public factory publishes
+through a connected `LocalParticipant`. The `DataTrackFrame` allocation covers
+the offline data API surface but does not create a Rust data-track handle.
+
+The executable is built with the normal test targets but is not registered with
+CTest, so it only runs when invoked manually. It does not connect to a server
+and needs no LiveKit credentials.
+
+## Build
+
+```bash
+./build.sh release-tests
+```
+
+## Run
+
+The default is 1,000 iterations. An alternate iteration count may be supplied
+as the only argument:
+
+```bash
+./build-release/bin/livekit_memory_lifecycle_tester
+./build-release/bin/livekit_memory_lifecycle_tester 100
+```
+
+To compare memory behavior before and after a lifecycle fix:
+
+```bash
+python3 scripts/track_process_memory.py --interval 0.01 -- \
+  ./build-release/bin/livekit_memory_lifecycle_tester
+```
+
+Use identical iteration counts and build configurations when comparing results.
+Allocator caching means final RSS need not return to the initial value; the
+useful regression signal is sustained or iteration-proportional growth.

src/tests/manual/memory_lifecycle_tester/main.cpp

@@ -0,0 +1,108 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <livekit/livekit.h>
+
+#include <cstdint>
+#include <exception>
+#include <iostream>
+#include <memory>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+namespace {
+
+constexpr int kDefaultIterations = 1'000;
+constexpr int kAudioSampleRate = 48'000;
+constexpr int kAudioChannels = 1;
+constexpr int kAudioQueueSizeMs = 100;
+constexpr int kVideoWidth = 1'280;
+constexpr int kVideoHeight = 720;
+constexpr std::size_t kDataPayloadSize = 1'024;
+
+int parseIterationCount(const char* value) {
+  try {
+    const int parsed = std::stoi(value);
+    if (parsed <= 0) {
+      throw std::runtime_error("iteration count must be greater than zero");
+    }
+    return parsed;
+  } catch (const std::invalid_argument&) {
+    throw std::runtime_error("iteration count must be an integer");
+  } catch (const std::out_of_range&) {
+    throw std::runtime_error("iteration count is out of range");
+  }
+}
+
+void exerciseCommonFeatures() {
+  auto audio_source = std::make_shared<livekit::AudioSource>(kAudioSampleRate, kAudioChannels, kAudioQueueSizeMs);
+  auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("lifecycle-audio", audio_source);
+  if (!audio_track) {
+    throw std::runtime_error("failed to create local audio track");
+  }
+  // Deliberately do not capture a frame. This covers teardown of the Rust
+  // keepalive task used before a raw video source receives its first frame.
+  auto video_source = std::make_shared<livekit::VideoSource>(kVideoWidth, kVideoHeight);
+  auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("lifecycle-video", video_source);
+  if (!video_track) {
+    throw std::runtime_error("failed to create local video track");
+  }
+
+  // A LocalDataTrack requires a connected LocalParticipant. Constructing the
+  // public frame type still covers the common offline data allocation surface.
+  livekit::DataTrackFrame data_frame(std::vector<std::uint8_t>(kDataPayloadSize, 0x5a));
+  if (data_frame.payload.size() != kDataPayloadSize) {
+    throw std::runtime_error("failed to create data track frame");
+  }
+}
+
+} // namespace
+
+int main(int argc, char* argv[]) {
+  if (argc > 2) {
+    std::cerr << "usage: " << argv[0] << " [iteration-count]\n";
+    return 2;
+  }
+
+  try {
+    const int iteration_count = argc == 2 ? parseIterationCount(argv[1]) : kDefaultIterations;
+    std::cout << "Running " << iteration_count << " LiveKit initialize/shutdown cycles\n";
+
+    for (int iteration = 1; iteration <= iteration_count; ++iteration) {
+      if (!livekit::initialize(livekit::LogLevel::Warn)) {
+        throw std::runtime_error("initialize failed at iteration " + std::to_string(iteration));
+      }
+
+      try {
+        exerciseCommonFeatures();
+      } catch (...) {
+        livekit::shutdown();
+        throw;
+      }
+      livekit::shutdown();
+
+      if (iteration % 100 == 0 || iteration == iteration_count) {
+        std::cout << "Completed " << iteration << "/" << iteration_count << " cycles\n";
+      }
+    }
+  } catch (const std::exception& error) {
+    std::cerr << "memory lifecycle tester failed: " << error.what() << '\n';
+    return 1;
+  }
+
+  return 0;
+}