From fdd67893d1580d0d9730e8b8210da09a37b17d04 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 2 Sep 2026 14:37:06 -0600 Subject: [PATCH 1/4] Fix cuda context lifecycle leak --- client-sdk-rust | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client-sdk-rust b/client-sdk-rust index 1a477bc4..6bff3273 160000 --- a/client-sdk-rust +++ b/client-sdk-rust @@ -1 +1 @@ -Subproject commit 1a477bc422c6890537b3bcdb017f0ac094d49661 +Subproject commit 6bff3273ffe020ca4898f4f801f17d8a9c324eb7 From 8f4b9c45053ed44e8ae314e43b19742b67fd1ef1 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 2 Sep 2026 14:53:23 -0600 Subject: [PATCH 2/4] Maybe fix linux --- .github/workflows/builds.yml | 8 ++++++++ .github/workflows/tests.yml | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index f3fda0da..e572f06c 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -125,6 +125,14 @@ jobs: libcurl4-openssl-dev \ libwayland-dev libdecor-0-dev + # The libwebrtc artifact ships a hermetic libc++ built from LLVM trunk. + # Distro clang (14 on Ubuntu 22.04, 18 on 24.04) cannot compile its + # headers; the submodule script installs LLVM 21 and exports CC/CXX. + - name: Install clang for libwebrtc + if: runner.os == 'Linux' + shell: bash + run: client-sdk-rust/.github/scripts/install-clang.sh + - name: Install deps (macOS) if: runner.os == 'macOS' run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 92f04bf5..47dac142 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -229,6 +229,14 @@ jobs: libwayland-dev libdecor-0-dev \ gdb jq + # The libwebrtc artifact ships a hermetic libc++ built from LLVM trunk. + # Distro clang (14 on Ubuntu 22.04, 18 on 24.04) cannot compile its + # headers; the submodule script installs LLVM 21 and exports CC/CXX. + - name: Install clang for libwebrtc + if: runner.os == 'Linux' + shell: bash + run: client-sdk-rust/.github/scripts/install-clang.sh + - name: Install deps (macOS) if: runner.os == 'macOS' run: | @@ -511,6 +519,12 @@ jobs: libwayland-dev libdecor-0-dev pip install --break-system-packages gcovr + # The coverage build compiles the Rust FFI too, so it needs the same + # compiler floor as the normal Linux build/test matrices. + - name: Install clang for libwebrtc + shell: bash + run: client-sdk-rust/.github/scripts/install-clang.sh + # ---------- Rust toolchain ---------- - name: Install Rust (stable) uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 From dc214de059f172a21077920379fcfebf9082032d Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 3 Sep 2026 16:17:19 -0600 Subject: [PATCH 3/4] Add regression test for dispose Rust changes --- client-sdk-rust | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client-sdk-rust b/client-sdk-rust index 6bff3273..e5fe5dab 160000 --- a/client-sdk-rust +++ b/client-sdk-rust @@ -1 +1 @@ -Subproject commit 6bff3273ffe020ca4898f4f801f17d8a9c324eb7 +Subproject commit e5fe5dab84e49320df58115d37a2a1a9478261d7 From 393044f48c0fd8e9207cffa32a355a5db866e549 Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 3 Sep 2026 23:07:17 -0600 Subject: [PATCH 4/4] Track process memory/memory lifecycle tester --- AGENTS.md | 3 +- scripts/track_process_memory.py | 122 ++++++++++++++++++ src/tests/CMakeLists.txt | 46 +++++++ .../manual/memory_lifecycle_tester/README.md | 51 ++++++++ .../manual/memory_lifecycle_tester/main.cpp | 108 ++++++++++++++++ 5 files changed, 329 insertions(+), 1 deletion(-) create mode 100755 scripts/track_process_memory.py create mode 100644 src/tests/manual/memory_lifecycle_tester/README.md create mode 100644 src/tests/manual/memory_lifecycle_tester/main.cpp diff --git a/AGENTS.md b/AGENTS.md index 55da0430..b4c9d8ba 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,7 +85,8 @@ Be sure to update the directory layout in this file if the directory layout chan |------|-------------| | `include/livekit/` | Public API headers (what SDK consumers include) | | `src/` | Implementation files and internal-only headers (`ffi_client.h`, `lk_log.h`, etc.) | -| `src/tests/` | Google Test integration and stress tests | +| `src/tests/` | Google Test unit, integration, and stress tests | +| `src/tests/manual/` | Standalone testers built with the test targets but not registered with CTest | | `examples/` | In-tree example applications | | `client-sdk-rust/` | Git submodule holding the Rust core of the SDK| | `cpp-tools/` | Git submodule holding shared LiveKit C++ engineering guidance, clang-format / clang-tidy configs, scripts, docs, and CI workflow | diff --git a/scripts/track_process_memory.py b/scripts/track_process_memory.py new file mode 100755 index 00000000..0c4b760d --- /dev/null +++ b/scripts/track_process_memory.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# +# Copyright 2026 LiveKit +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run a command and report its resident memory usage.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import time + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run a command and report its initial, final, and peak RSS." + ) + parser.add_argument( + "--interval", + type=float, + default=0.1, + help="seconds between RSS samples (default: 0.1)", + ) + parser.add_argument( + "command", + nargs=argparse.REMAINDER, + help="command and its arguments; prefix it with -- when needed", + ) + args = parser.parse_args() + if args.interval <= 0: + parser.error("--interval must be greater than zero") + if not args.command: + parser.error("a command is required") + return args + + +def process_rss_kib(pid: int) -> int | None: + result = subprocess.run( + ["ps", "-o", "rss=", "-p", str(pid)], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + + rss = result.stdout.strip() + if not rss: + return None + + rss_kib = int(rss) + # macOS reports 0 RSS for a child that has exited but has not yet been + # reaped. Do not overwrite the last live-process sample with that value. + return rss_kib if rss_kib > 0 else None + + +def format_rss(rss_kib: int) -> str: + return f"{rss_kib:,} KiB ({rss_kib / 1024:.2f} MiB)" + + +def main() -> int: + args = parse_args() + command = args.command + if command[0] == "--": + command = command[1:] + if not command: + print("error: a command is required after --", file=sys.stderr) + return 2 + + try: + process = subprocess.Popen(command) + except OSError as error: + print(f"error: could not start {command[0]!r}: {error}", file=sys.stderr) + return 127 + + started_at = time.monotonic() + initial_rss_kib: int | None = None + final_rss_kib: int | None = None + peak_rss_kib: int | None = None + + while process.poll() is None: + rss_kib = process_rss_kib(process.pid) + if rss_kib is not None: + if initial_rss_kib is None: + initial_rss_kib = rss_kib + final_rss_kib = rss_kib + peak_rss_kib = max(peak_rss_kib or rss_kib, rss_kib) + time.sleep(args.interval) + + elapsed_s = time.monotonic() - started_at + exit_code = process.wait() + print(f"command: {' '.join(command)}") + print(f"exit code: {exit_code}") + print(f"elapsed: {elapsed_s:.2f} s") + if initial_rss_kib is None: + print("RSS: no samples collected; the command exited before sampling began") + else: + assert final_rss_kib is not None + assert peak_rss_kib is not None + print(f"RSS initial: {format_rss(initial_rss_kib)}") + print(f"RSS final observed: {format_rss(final_rss_kib)}") + print(f"RSS peak: {format_rss(peak_rss_kib)}") + print(f"RSS change: {format_rss(final_rss_kib - initial_rss_kib)}") + + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 9583af73..47161f1f 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -348,3 +348,49 @@ endif() if(TARGET livekit_stress_tests) add_dependencies(run_all_tests livekit_stress_tests) endif() + +# ============================================================================ +# Manual testers (built with the test targets, not registered with CTest) +# ============================================================================ + +add_executable(livekit_memory_lifecycle_tester + "${CMAKE_CURRENT_SOURCE_DIR}/manual/memory_lifecycle_tester/main.cpp" +) + +target_link_libraries(livekit_memory_lifecycle_tester PRIVATE livekit) +target_include_directories(livekit_memory_lifecycle_tester PRIVATE ${LIVEKIT_ROOT_DIR}/include) +target_compile_definitions(livekit_memory_lifecycle_tester PRIVATE + $<$:_USE_MATH_DEFINES> +) + +if(WIN32) + add_custom_command(TARGET livekit_memory_lifecycle_tester POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$/livekit_ffi.dll" + $ + COMMENT "Copying DLLs next to livekit_memory_lifecycle_tester" + ) +elseif(APPLE) + add_custom_command(TARGET livekit_memory_lifecycle_tester POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$/liblivekit_ffi.dylib" + $ + COMMENT "Copying dylibs next to livekit_memory_lifecycle_tester" + ) +else() + add_custom_command(TARGET livekit_memory_lifecycle_tester POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$/liblivekit_ffi.so" + $ + COMMENT "Copying shared libraries next to livekit_memory_lifecycle_tester" + ) +endif() diff --git a/src/tests/manual/memory_lifecycle_tester/README.md b/src/tests/manual/memory_lifecycle_tester/README.md new file mode 100644 index 00000000..dfe7c916 --- /dev/null +++ b/src/tests/manual/memory_lifecycle_tester/README.md @@ -0,0 +1,51 @@ +# Memory lifecycle tester + +This standalone application repeatedly exercises the public C++ SDK lifecycle +to expose retained lower-level Rust/WebRTC resources. + +Each iteration: + +- calls `livekit::initialize()`; +- creates an `AudioSource` and `LocalAudioTrack`; +- creates a 1280x720 `VideoSource` and `LocalVideoTrack`; +- creates a representative `DataTrackFrame`; +- destroys those objects before calling `livekit::shutdown()`. + +The video source intentionally receives no captured frame. This exercises +teardown of the Rust keepalive task that runs until the first raw video frame +arrives and previously retained roughly one 720p frame per lifecycle. + +`LocalDataTrack` itself cannot be created offline: its public factory publishes +through a connected `LocalParticipant`. The `DataTrackFrame` allocation covers +the offline data API surface but does not create a Rust data-track handle. + +The executable is built with the normal test targets but is not registered with +CTest, so it only runs when invoked manually. It does not connect to a server +and needs no LiveKit credentials. + +## Build + +```bash +./build.sh release-tests +``` + +## Run + +The default is 1,000 iterations. An alternate iteration count may be supplied +as the only argument: + +```bash +./build-release/bin/livekit_memory_lifecycle_tester +./build-release/bin/livekit_memory_lifecycle_tester 100 +``` + +To compare memory behavior before and after a lifecycle fix: + +```bash +python3 scripts/track_process_memory.py --interval 0.01 -- \ + ./build-release/bin/livekit_memory_lifecycle_tester +``` + +Use identical iteration counts and build configurations when comparing results. +Allocator caching means final RSS need not return to the initial value; the +useful regression signal is sustained or iteration-proportional growth. diff --git a/src/tests/manual/memory_lifecycle_tester/main.cpp b/src/tests/manual/memory_lifecycle_tester/main.cpp new file mode 100644 index 00000000..bf92b2dd --- /dev/null +++ b/src/tests/manual/memory_lifecycle_tester/main.cpp @@ -0,0 +1,108 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr int kDefaultIterations = 1'000; +constexpr int kAudioSampleRate = 48'000; +constexpr int kAudioChannels = 1; +constexpr int kAudioQueueSizeMs = 100; +constexpr int kVideoWidth = 1'280; +constexpr int kVideoHeight = 720; +constexpr std::size_t kDataPayloadSize = 1'024; + +int parseIterationCount(const char* value) { + try { + const int parsed = std::stoi(value); + if (parsed <= 0) { + throw std::runtime_error("iteration count must be greater than zero"); + } + return parsed; + } catch (const std::invalid_argument&) { + throw std::runtime_error("iteration count must be an integer"); + } catch (const std::out_of_range&) { + throw std::runtime_error("iteration count is out of range"); + } +} + +void exerciseCommonFeatures() { + auto audio_source = std::make_shared(kAudioSampleRate, kAudioChannels, kAudioQueueSizeMs); + auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("lifecycle-audio", audio_source); + if (!audio_track) { + throw std::runtime_error("failed to create local audio track"); + } + // Deliberately do not capture a frame. This covers teardown of the Rust + // keepalive task used before a raw video source receives its first frame. + auto video_source = std::make_shared(kVideoWidth, kVideoHeight); + auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("lifecycle-video", video_source); + if (!video_track) { + throw std::runtime_error("failed to create local video track"); + } + + // A LocalDataTrack requires a connected LocalParticipant. Constructing the + // public frame type still covers the common offline data allocation surface. + livekit::DataTrackFrame data_frame(std::vector(kDataPayloadSize, 0x5a)); + if (data_frame.payload.size() != kDataPayloadSize) { + throw std::runtime_error("failed to create data track frame"); + } +} + +} // namespace + +int main(int argc, char* argv[]) { + if (argc > 2) { + std::cerr << "usage: " << argv[0] << " [iteration-count]\n"; + return 2; + } + + try { + const int iteration_count = argc == 2 ? parseIterationCount(argv[1]) : kDefaultIterations; + std::cout << "Running " << iteration_count << " LiveKit initialize/shutdown cycles\n"; + + for (int iteration = 1; iteration <= iteration_count; ++iteration) { + if (!livekit::initialize(livekit::LogLevel::Warn)) { + throw std::runtime_error("initialize failed at iteration " + std::to_string(iteration)); + } + + try { + exerciseCommonFeatures(); + } catch (...) { + livekit::shutdown(); + throw; + } + livekit::shutdown(); + + if (iteration % 100 == 0 || iteration == iteration_count) { + std::cout << "Completed " << iteration << "/" << iteration_count << " cycles\n"; + } + } + } catch (const std::exception& error) { + std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; + return 1; + } + + return 0; +}