diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt index 99041a02b..8545dc087 100644 --- a/electron/native/wgc-capture/CMakeLists.txt +++ b/electron/native/wgc-capture/CMakeLists.txt @@ -46,8 +46,12 @@ add_executable(wgc-capture src/mf_encoder.h src/monitor_utils.cpp src/monitor_utils.h + src/wasapi_device_watcher.cpp + src/wasapi_device_watcher.h src/wasapi_loopback_capture.cpp src/wasapi_loopback_capture.h + src/wasapi_render_keepalive.cpp + src/wasapi_render_keepalive.h src/webcam_capture.cpp src/webcam_capture.h src/wgc_session.cpp diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index 223211c9f..88c0d1572 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -2,7 +2,9 @@ #include "dpi_awareness.h" #include "mf_encoder.h" #include "monitor_utils.h" +#include "wasapi_device_watcher.h" #include "wasapi_loopback_capture.h" +#include "wasapi_render_keepalive.h" #include "webcam_capture.h" #include "wgc_session.h" @@ -796,6 +798,38 @@ int wmain(int argc, wchar_t* argv[]) { WasapiLoopbackCapture loopbackCapture; WasapiLoopbackCapture microphoneCapture; + // getopenscreen/openscreen#724, diagnostic only: logs render/capture endpoint + // state transitions for the duration of the recording, so a report of a headset + // dropping mid-take can be correlated against a real timestamped Windows event + // instead of guessed at. Does not change recording behavior in any way. + const bool deviceWatchEnabled = readEnvInt("OPENSCREEN_WGC_LOG_AUDIO_DEVICE_EVENTS", 0) == 1; + WasapiDeviceWatcher deviceWatcher; + bool deviceWatchActive = false; + const auto stopDeviceWatchIfActive = [&]() { + if (deviceWatchActive) { + deviceWatcher.stop(); + deviceWatchActive = false; + } + }; + // getopenscreen/openscreen#724: confirmed on real hardware that a mic-only + // recording lets a wireless headset's own idle timer fire and drop it, while + // the same recording with system audio (which reads the render endpoint via + // loopback) does not. Only needed when system audio is off: loopback capture + // already keeps the endpoint busy with real content on its own, so running + // this alongside it would be redundant and would additionally get captured + // into the recording's system-audio track, which mic-only never touches. + // Non-fatal: a recording with no output device to keep alive, or one where + // another app holds it exclusively, is unaffected either way. + const bool renderKeepAliveEnabled = + !config.captureSystemAudio && readEnvInt("OPENSCREEN_WGC_DISABLE_AUDIO_KEEPALIVE", 0) != 1; + WasapiRenderKeepAlive renderKeepAlive; + bool renderKeepAliveActive = false; + const auto stopRenderKeepAliveIfActive = [&]() { + if (renderKeepAliveActive) { + renderKeepAlive.stop(); + renderKeepAliveActive = false; + } + }; const AudioInputFormat* audioFormat = nullptr; AudioInputFormat encoderAudioFormat{}; AudioInputFormat systemAudioFormat{}; @@ -1378,13 +1412,33 @@ int wmain(int argc, wchar_t* argv[]) { return true; }; + if (deviceWatchEnabled) { + deviceWatchActive = deviceWatcher.start(); + if (!deviceWatchActive) { + std::cerr << "WARNING: Failed to start audio device watcher; continuing without it" + << std::endl; + } + } + + if (renderKeepAliveEnabled) { + renderKeepAliveActive = renderKeepAlive.start(); + if (!renderKeepAliveActive) { + std::cerr << "WARNING: Failed to start render keep-alive stream; continuing without it" + << std::endl; + } + } + if (!startAudioCaptures()) { + stopDeviceWatchIfActive(); + stopRenderKeepAliveIfActive(); return 1; } if (config.webcamEnabled) { if (!webcamCapture.start()) { microphoneCapture.stop(); loopbackCapture.stop(); + stopDeviceWatchIfActive(); + stopRenderKeepAliveIfActive(); if (audioMixer) { audioMixer->stop(); } @@ -1416,6 +1470,8 @@ int wmain(int argc, wchar_t* argv[]) { webcamCapture.stop(); microphoneCapture.stop(); loopbackCapture.stop(); + stopDeviceWatchIfActive(); + stopRenderKeepAliveIfActive(); if (audioMixer) { audioMixer->stop(); } @@ -1459,6 +1515,8 @@ int wmain(int argc, wchar_t* argv[]) { stopVideoWriter(); microphoneCapture.stop(); loopbackCapture.stop(); + stopDeviceWatchIfActive(); + stopRenderKeepAliveIfActive(); webcamCapture.stop(); if (audioMixer) { audioMixer->stop(); @@ -1594,6 +1652,16 @@ int wmain(int argc, wchar_t* argv[]) { beginStopStep("loopback", stepBudgetMs); loopbackCapture.stop(); logStopStep("loopback"); + if (deviceWatchActive) { + beginStopStep("device-watcher", stepBudgetMs); + stopDeviceWatchIfActive(); + logStopStep("device-watcher"); + } + if (renderKeepAliveActive) { + beginStopStep("render-keepalive", stepBudgetMs); + stopRenderKeepAliveIfActive(); + logStopStep("render-keepalive"); + } beginStopStep("webcam", stepBudgetMs); webcamCapture.stop(); logStopStep("webcam"); diff --git a/electron/native/wgc-capture/src/wasapi_device_watcher.cpp b/electron/native/wgc-capture/src/wasapi_device_watcher.cpp new file mode 100644 index 000000000..8cd73c9b6 --- /dev/null +++ b/electron/native/wgc-capture/src/wasapi_device_watcher.cpp @@ -0,0 +1,314 @@ +#include "wasapi_device_watcher.h" + +#include +#include + +#include +#include +#include +#include + +namespace { + +std::string wideToUtf8(const std::wstring& value) { + if (value.empty()) { + return {}; + } + const int size = WideCharToMultiByte( + CP_UTF8, 0, value.data(), static_cast(value.size()), nullptr, 0, nullptr, nullptr); + std::string result(static_cast(size), '\0'); + WideCharToMultiByte( + CP_UTF8, 0, value.data(), static_cast(value.size()), result.data(), size, nullptr, nullptr); + return result; +} + +std::string jsonEscape(const std::string& value) { + std::string result; + result.reserve(value.size()); + for (const char c : value) { + switch (c) { + case '\\': + result += "\\\\"; + break; + case '"': + result += "\\\""; + break; + case '\n': + result += "\\n"; + break; + default: + result += c; + } + } + return result; +} + +std::string deviceStateLabel(DWORD state) { + switch (state) { + case DEVICE_STATE_ACTIVE: + return "active"; + case DEVICE_STATE_DISABLED: + return "disabled"; + case DEVICE_STATE_NOTPRESENT: + return "not-present"; + case DEVICE_STATE_UNPLUGGED: + return "unplugged"; + default: + return "unknown"; + } +} + +int64_t nowUnixMillis() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); +} + +// Only ever called from the worker thread or from start() (before the worker +// exists), never from an IMMNotificationClient callback -- this does a property +// store round trip and must not run on the callback thread. +std::wstring friendlyNameForDevice(IMMDeviceEnumerator* enumerator, const std::wstring& deviceId) { + if (!enumerator || deviceId.empty()) { + return {}; + } + Microsoft::WRL::ComPtr device; + if (FAILED(enumerator->GetDevice(deviceId.c_str(), &device)) || !device) { + return {}; + } + Microsoft::WRL::ComPtr properties; + if (FAILED(device->OpenPropertyStore(STGM_READ, &properties)) || !properties) { + return {}; + } + PROPVARIANT value; + PropVariantInit(&value); + std::wstring name; + if (SUCCEEDED(properties->GetValue(PKEY_Device_FriendlyName, &value)) && value.vt == VT_LPWSTR && + value.pwszVal) { + name = value.pwszVal; + } + PropVariantClear(&value); + return name; +} + +} // namespace + +WasapiDeviceWatcher::~WasapiDeviceWatcher() { + stop(); +} + +bool WasapiDeviceWatcher::start() { + HRESULT hr = CoCreateInstance( + __uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, IID_PPV_ARGS(&deviceEnumerator_)); + if (FAILED(hr) || !deviceEnumerator_) { + std::cerr << "WARNING: [device-watcher] CoCreateInstance(MMDeviceEnumerator) failed (hr=0x" + << std::hex << hr << std::dec << ")" << std::endl; + return false; + } + + hr = deviceEnumerator_->RegisterEndpointNotificationCallback(this); + if (FAILED(hr)) { + std::cerr << "WARNING: [device-watcher] RegisterEndpointNotificationCallback failed (hr=0x" + << std::hex << hr << std::dec << ")" << std::endl; + deviceEnumerator_.Reset(); + return false; + } + registered_ = true; + + workerStopRequested_ = false; + worker_ = std::thread([this] { + workerLoop(); + }); + + enqueueBaseline(eRender, L"render"); + enqueueBaseline(eCapture, L"capture"); + return true; +} + +void WasapiDeviceWatcher::stop() { + if (registered_ && deviceEnumerator_) { + deviceEnumerator_->UnregisterEndpointNotificationCallback(this); + } + registered_ = false; + + // The worker dereferences deviceEnumerator_ (writeBaseline calls + // GetDefaultAudioEndpoint through it), and stop() can run while events are + // still queued -- main.cpp calls this on every early-failure path, potentially + // right after start() enqueued the baselines. Drain and join first, release + // the enumerator last, or the worker wakes up holding a dangling pointer. + if (worker_.joinable()) { + { + std::lock_guard lock(queueMutex_); + workerStopRequested_ = true; + } + queueCv_.notify_one(); + worker_.join(); + } + deviceEnumerator_.Reset(); +} + +void WasapiDeviceWatcher::workerLoop() { + // This thread owns all name resolution and all event writes for this watcher, + // so nothing here runs on an IMMNotificationClient callback thread. + while (true) { + PendingEvent event; + { + std::unique_lock lock(queueMutex_); + queueCv_.wait(lock, [this] { return !queue_.empty() || workerStopRequested_; }); + if (queue_.empty()) { + if (workerStopRequested_) { + return; + } + continue; + } + event = std::move(queue_.front()); + queue_.pop(); + } + + if (event.needsBaselineLookup) { + writeBaseline(event); + } else { + writeDeviceEvent(event); + } + } +} + +void WasapiDeviceWatcher::enqueueBaseline(EDataFlow flow, const wchar_t* flowLabel) { + PendingEvent event; + event.needsBaselineLookup = true; + event.baselineFlow = flow; + event.baselineFlowLabel = flowLabel; + { + std::lock_guard lock(queueMutex_); + queue_.push(std::move(event)); + } + queueCv_.notify_one(); +} + +void WasapiDeviceWatcher::writeBaseline(const PendingEvent& event) { + Microsoft::WRL::ComPtr device; + HRESULT hr = deviceEnumerator_->GetDefaultAudioEndpoint(event.baselineFlow, eConsole, &device); + if (FAILED(hr) || !device) { + return; + } + + LPWSTR rawId = nullptr; + std::wstring id; + if (SUCCEEDED(device->GetId(&rawId)) && rawId) { + id = rawId; + CoTaskMemFree(rawId); + } + + DWORD state = 0; + device->GetState(&state); + const std::wstring name = friendlyNameForDevice(deviceEnumerator_.Get(), id); + + // Built as one complete string before the write, and emitted with a single + // stream operation. Events from helper threads go to stderr -- the + // microphone-defaulted warning set that precedent -- so stdout protocol + // lines stay owned by the main thread alone. + std::ostringstream line; + line << "{\"event\":\"audio-device-watch\",\"schemaVersion\":1,\"type\":\"baseline\"," + "\"flow\":\"" + << wideToUtf8(event.baselineFlowLabel) << "\",\"deviceId\":\"" << jsonEscape(wideToUtf8(id)) + << "\",\"deviceName\":\"" << jsonEscape(wideToUtf8(name)) << "\",\"state\":\"" + << deviceStateLabel(state) << "\",\"timestampMs\":" << nowUnixMillis() << "}\n"; + + std::cerr << line.str() << std::flush; +} + +void WasapiDeviceWatcher::enqueue(const wchar_t* eventName, LPCWSTR deviceId, const char* extraJson) { + PendingEvent event; + event.eventName = eventName ? eventName : L""; + event.deviceId = deviceId ? deviceId : L""; + event.extraJson = extraJson ? extraJson : ""; + { + std::lock_guard lock(queueMutex_); + queue_.push(std::move(event)); + } + queueCv_.notify_one(); +} + +void WasapiDeviceWatcher::writeDeviceEvent(const PendingEvent& event) { + const std::wstring name = friendlyNameForDevice(deviceEnumerator_.Get(), event.deviceId); + + std::ostringstream line; + line << "{\"event\":\"audio-device-watch\",\"schemaVersion\":1,\"type\":\"" + << wideToUtf8(event.eventName) << "\",\"deviceId\":\"" + << jsonEscape(wideToUtf8(event.deviceId)) << "\",\"deviceName\":\"" + << jsonEscape(wideToUtf8(name)) << "\""; + if (!event.extraJson.empty()) { + line << "," << event.extraJson; + } + line << ",\"timestampMs\":" << nowUnixMillis() << "}\n"; + + std::cerr << line.str() << std::flush; +} + +ULONG STDMETHODCALLTYPE WasapiDeviceWatcher::AddRef() { + return ++refCount_; +} + +ULONG STDMETHODCALLTYPE WasapiDeviceWatcher::Release() { + // This object's lifetime is owned by main.cpp, not by COM: it lives on the + // stack for the duration of the recording and unregisters in stop() before + // destruction, so a ref count reaching zero here must not delete `this`. + const ULONG count = --refCount_; + return count; +} + +HRESULT STDMETHODCALLTYPE WasapiDeviceWatcher::QueryInterface(REFIID riid, void** ppvObject) { + if (!ppvObject) { + return E_POINTER; + } + if (riid == __uuidof(IUnknown) || riid == __uuidof(IMMNotificationClient)) { + *ppvObject = static_cast(this); + AddRef(); + return S_OK; + } + *ppvObject = nullptr; + return E_NOINTERFACE; +} + +// Every method below runs on a COM callback thread and must be nonblocking per +// IMMNotificationClient's documented contract: no name resolution, no I/O, no +// lock that can wait. Each one only copies its arguments and enqueues them -- +// see workerLoop() for where the real work happens. + +HRESULT STDMETHODCALLTYPE WasapiDeviceWatcher::OnDeviceStateChanged(LPCWSTR deviceId, DWORD newState) { + char extra[64]; + snprintf(extra, sizeof(extra), "\"state\":\"%s\"", deviceStateLabel(newState).c_str()); + enqueue(L"state-changed", deviceId, extra); + return S_OK; +} + +HRESULT STDMETHODCALLTYPE WasapiDeviceWatcher::OnDeviceAdded(LPCWSTR deviceId) { + enqueue(L"added", deviceId, nullptr); + return S_OK; +} + +HRESULT STDMETHODCALLTYPE WasapiDeviceWatcher::OnDeviceRemoved(LPCWSTR deviceId) { + enqueue(L"removed", deviceId, nullptr); + return S_OK; +} + +HRESULT STDMETHODCALLTYPE WasapiDeviceWatcher::OnDefaultDeviceChanged( + EDataFlow flow, ERole role, LPCWSTR defaultDeviceId) { + // Only eConsole is what this app's capture paths use (GetDefaultAudioEndpoint + // calls elsewhere all pass eConsole); the other roles fire independently and + // would just be noise here. + if (role != eConsole) { + return S_OK; + } + // Sized for the longest payload: "flow":"capture" is 16 chars plus the + // terminator, and 16 bytes truncates the closing quote into a malformed line. + char extra[32]; + snprintf(extra, sizeof(extra), "\"flow\":\"%s\"", flow == eRender ? "render" : "capture"); + enqueue(L"default-changed", defaultDeviceId, extra); + return S_OK; +} + +HRESULT STDMETHODCALLTYPE WasapiDeviceWatcher::OnPropertyValueChanged(LPCWSTR, const PROPERTYKEY) { + // Not interesting for this diagnosis: fires on things like a renamed endpoint, + // not on power/connection state. + return S_OK; +} diff --git a/electron/native/wgc-capture/src/wasapi_device_watcher.h b/electron/native/wgc-capture/src/wasapi_device_watcher.h new file mode 100644 index 000000000..47301d733 --- /dev/null +++ b/electron/native/wgc-capture/src/wasapi_device_watcher.h @@ -0,0 +1,112 @@ +#pragma once + +// DIAGNOSTIC ONLY, opt-in via OPENSCREEN_WGC_LOG_AUDIO_DEVICE_EVENTS=1 (see main.cpp). +// +// getopenscreen/openscreen#724: before committing to a fix, we need a real, +// timestamped signal on WHY a headset's render/capture endpoints change state +// mid-recording -- USB selective suspend, WASAPI endpoint idle, and the headset's +// own firmware auto-off all look the same to the user ("headphones turned off"), +// but only some of them are fixable from inside this process. This watcher answers +// that by registering an IMMNotificationClient for the duration of the recording +// and emitting a structured JSON event on every state transition of the default +// render and capture endpoints, so it can be correlated against the moment a user +// hears their headset drop. +// +// This does not attempt to fix anything -- it only observes and reports. See the +// issue for the three-way split this is meant to distinguish: +// 1. USB selective suspend (device level): the endpoint would go NOTPRESENT/ +// UNPLUGGED, i.e. the whole device disappears, not just the audio state. +// 2. WASAPI endpoint idle (audio-engine level): the endpoint would typically stay +// ACTIVE while going quiet -- unlikely to show anything here at all, since nothing +// in the device's own docs treats "unwritten render buffer" as a state change. +// 3. Headset firmware auto-off: same observable shape as (1) from Windows' point of +// view (the endpoint disappears), but no amount of keeping the render endpoint +// "busy" in software prevents a keyed hardware timer from firing. +// +// A DEVICE_STATE_NOTPRESENT/UNPLUGGED transition on the render or capture endpoint, +// correlated with the moment the user hears the drop, points at (1) or (3). No event +// at all around the drop, with the endpoint remaining ACTIVE throughout, would point +// at (2) instead -- but the converse does not hold: the confirmed case in #724 +// (Corsair Void Wireless) produced zero events across every live drop, because the +// USB dongle stays enumerated and ACTIVE the whole time and only the RF link to the +// earcups drops, which is invisible to IMMNotificationClient. So silence here does +// not rule out a firmware timer; before concluding (2), check the vendor's own +// power management (e.g. iCUE) and a with-loopback vs mic-only comparison. +// +// Events are written to stderr, as one complete line per write: thread-emitted +// events go to stderr in this helper (the microphone-defaulted warning set that +// precedent), leaving stdout protocol lines owned by the main thread alone. Both +// streams end up merged in the drained helper log. +// +// IMMNotificationClient callbacks must be nonblocking (never resolve names, take a +// lock that can wait, or do I/O) per Microsoft's documented contract, so the +// callbacks here only copy their arguments into a PendingEvent and hand it to a +// worker thread, which does the (possibly slow) name lookup and the actual write. + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +class WasapiDeviceWatcher : public IMMNotificationClient { +public: + WasapiDeviceWatcher() = default; + ~WasapiDeviceWatcher(); + + WasapiDeviceWatcher(const WasapiDeviceWatcher&) = delete; + WasapiDeviceWatcher& operator=(const WasapiDeviceWatcher&) = delete; + + // Registers for notifications and enqueues one baseline event per endpoint + // (render + capture) with their state at the moment the recording starts, so a + // report has a starting point even if nothing changes afterward. + bool start(); + // Unregisters callbacks, drains the queue, and joins the worker before + // returning, so every event queued before stop() is guaranteed to be written. + void stop(); + + // IUnknown + ULONG STDMETHODCALLTYPE AddRef() override; + ULONG STDMETHODCALLTYPE Release() override; + HRESULT STDMETHODCALLTYPE QueryInterface(REFIID riid, void** ppvObject) override; + + // IMMNotificationClient + HRESULT STDMETHODCALLTYPE OnDeviceStateChanged(LPCWSTR deviceId, DWORD newState) override; + HRESULT STDMETHODCALLTYPE OnDeviceAdded(LPCWSTR deviceId) override; + HRESULT STDMETHODCALLTYPE OnDeviceRemoved(LPCWSTR deviceId) override; + HRESULT STDMETHODCALLTYPE OnDefaultDeviceChanged(EDataFlow flow, ERole role, LPCWSTR defaultDeviceId) override; + HRESULT STDMETHODCALLTYPE OnPropertyValueChanged(LPCWSTR deviceId, const PROPERTYKEY key) override; + +private: + struct PendingEvent { + std::wstring eventName; + std::wstring deviceId; + std::string extraJson; + // Baseline events resolve the endpoint themselves (they run on start(), not + // a callback thread, so there is no blocking concern) and need no lookup. + bool needsBaselineLookup = false; + EDataFlow baselineFlow = eRender; + std::wstring baselineFlowLabel; + }; + + void enqueue(const wchar_t* eventName, LPCWSTR deviceId, const char* extraJson); + void enqueueBaseline(EDataFlow flow, const wchar_t* flowLabel); + void workerLoop(); + void writeDeviceEvent(const PendingEvent& event); + void writeBaseline(const PendingEvent& event); + + std::atomic refCount_ = 1; + Microsoft::WRL::ComPtr deviceEnumerator_; + bool registered_ = false; + + std::thread worker_; + std::mutex queueMutex_; + std::condition_variable queueCv_; + std::queue queue_; + bool workerStopRequested_ = false; +}; diff --git a/electron/native/wgc-capture/src/wasapi_render_keepalive.cpp b/electron/native/wgc-capture/src/wasapi_render_keepalive.cpp new file mode 100644 index 000000000..038a648e5 --- /dev/null +++ b/electron/native/wgc-capture/src/wasapi_render_keepalive.cpp @@ -0,0 +1,212 @@ +#include "wasapi_render_keepalive.h" + +#include +#include +#include +#include +#include + +namespace { + +constexpr REFERENCE_TIME BufferDurationHns = 10'000'000; +// A 1kHz tone at 1% amplitude was clearly audible in testing -- 1kHz sits right +// in the most sensitive part of human hearing, so "quiet" in raw amplitude terms +// was still loud in perceived loudness. 19kHz is above what the large majority of +// adults can hear at all (upper hearing limit typically drops well below 20kHz +// with age), so the actual amplitude can afford to be smaller still and remain +// robustly non-silent to the audio engine. +constexpr double ToneAmplitude = 0.003; +constexpr double ToneFrequencyHz = 19000.0; + +bool isFloatFormat(const WAVEFORMATEX* format) { + if (format->wFormatTag == WAVE_FORMAT_IEEE_FLOAT) { + return true; + } + if (format->wFormatTag == WAVE_FORMAT_EXTENSIBLE && + format->cbSize >= sizeof(WAVEFORMATEXTENSIBLE) - sizeof(WAVEFORMATEX)) { + const auto* extensible = reinterpret_cast(format); + return extensible->SubFormat == KSDATAFORMAT_SUBTYPE_IEEE_FLOAT; + } + return false; +} + +constexpr double TwoPi = 2.0 * 3.14159265358979323846; + +// The minimum sample rate at which ToneFrequencyHz stays comfortably below +// Nyquist. Below this, generating the tone at all would mean either aliasing it +// into an audible range or silently picking a lower, audible frequency instead +// -- both worse than not running the keep-alive at all, so start() refuses to +// run rather than risk a perceptible tone. +constexpr double MinSampleRateForTone = ToneFrequencyHz / 0.9 * 2.0; + +// Fills `frameCount` frames of `data` with a quiet sine tone at the format +// described by `format`, starting at `phase` radians and returning the phase to +// continue from on the next call, so the waveform stays continuous across +// separate GetBuffer/ReleaseBuffer calls instead of clicking at each boundary. +// Caller (start()) has already verified the sample rate supports ToneFrequencyHz +// with margin, so this always uses it directly rather than silently degrading. +double writeToneFrames(BYTE* data, UINT32 frameCount, const WAVEFORMATEX* format, double phase) { + // Zeroed up front so an unsupported bit depth (anything but 16/32-bit) falls + // back to real silence for that packet instead of playing back GetBuffer's + // uninitialized memory. + std::memset(data, 0, static_cast(frameCount) * format->nBlockAlign); + + const double phaseStep = TwoPi * ToneFrequencyHz / format->nSamplesPerSec; + const bool isFloat = isFloatFormat(format); + const UINT16 bitsPerSample = format->wBitsPerSample; + + for (UINT32 frame = 0; frame < frameCount; ++frame) { + const double sampleValue = std::sin(phase) * ToneAmplitude; + phase += phaseStep; + if (phase >= TwoPi) { + // Wrap rather than let phase grow unboundedly over a long recording, + // which would eventually lose precision in the sine's argument. + phase -= TwoPi; + } + + for (UINT16 channel = 0; channel < format->nChannels; ++channel) { + BYTE* sampleData = data + frame * format->nBlockAlign + channel * (bitsPerSample / 8); + if (isFloat && bitsPerSample == 32) { + const float value = static_cast(sampleValue); + std::memcpy(sampleData, &value, sizeof(value)); + } else if (bitsPerSample == 16) { + const int16_t value = static_cast(sampleValue * 32767.0); + std::memcpy(sampleData, &value, sizeof(value)); + } else if (bitsPerSample == 32) { + // 32-bit integer PCM. + const int32_t value = static_cast(sampleValue * 2147483647.0); + std::memcpy(sampleData, &value, sizeof(value)); + } + // Any other bit depth is left zeroed (already cleared above) rather + // than risking a malformed write -- this is a best-effort keep-alive, + // not a guarantee of coverage for every possible device format. + } + } + + return phase; +} + +} // namespace + +WasapiRenderKeepAlive::~WasapiRenderKeepAlive() { + stop(); + if (mixFormat_) { + CoTaskMemFree(mixFormat_); + mixFormat_ = nullptr; + } +} + +bool WasapiRenderKeepAlive::start() { + Microsoft::WRL::ComPtr deviceEnumerator; + HRESULT hr = CoCreateInstance( + __uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, IID_PPV_ARGS(&deviceEnumerator)); + if (FAILED(hr)) { + return false; + } + + Microsoft::WRL::ComPtr device; + hr = deviceEnumerator->GetDefaultAudioEndpoint(eRender, eConsole, &device); + if (FAILED(hr)) { + // No default render device to keep alive -- nothing to do. + return false; + } + + hr = device->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, &audioClient_); + if (FAILED(hr)) { + return false; + } + + hr = audioClient_->GetMixFormat(&mixFormat_); + if (FAILED(hr) || !mixFormat_) { + return false; + } + + if (mixFormat_->nSamplesPerSec < MinSampleRateForTone) { + // Generating the tone here would mean either aliasing 19kHz into an + // audible frequency or silently picking a lower, audible one -- both + // worse than not running the keep-alive at all. The caller already + // treats this as non-fatal to the recording. + return false; + } + + // Shared mode: mixes into whatever else may be playing rather than requesting + // exclusive control, so it can't block another app from using the device, and + // it fails cleanly (non-fatal to the caller) if another app already holds it + // exclusively. + hr = audioClient_->Initialize(AUDCLNT_SHAREMODE_SHARED, 0, BufferDurationHns, 0, mixFormat_, nullptr); + if (FAILED(hr)) { + return false; + } + + hr = audioClient_->GetBufferSize(&bufferFrameCount_); + if (FAILED(hr)) { + return false; + } + + hr = audioClient_->GetService(IID_PPV_ARGS(&renderClient_)); + if (FAILED(hr)) { + return false; + } + + // Prime the full buffer with the tone before Start() so there's no gap for the + // audio engine to glitch on. + BYTE* data = nullptr; + hr = renderClient_->GetBuffer(bufferFrameCount_, &data); + if (FAILED(hr)) { + return false; + } + tonePhase_ = writeToneFrames(data, bufferFrameCount_, mixFormat_, tonePhase_); + renderClient_->ReleaseBuffer(bufferFrameCount_, 0); + + stopRequested_ = false; + hr = audioClient_->Start(); + if (FAILED(hr)) { + return false; + } + + thread_ = std::thread([this] { + // GetBuffer/ReleaseBuffer/GetCurrentPadding are called from this thread, + // which is otherwise never COM-initialized. Both this and the wmain thread + // are MTA (see winrt::init_apartment in main.cpp), so no marshaling is + // needed -- this only satisfies the "calling thread must be initialized" + // requirement. + const HRESULT comInit = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + renderLoop(); + if (SUCCEEDED(comInit)) { + CoUninitialize(); + } + }); + return true; +} + +void WasapiRenderKeepAlive::stop() { + stopRequested_ = true; + if (thread_.joinable()) { + thread_.join(); + } + if (audioClient_) { + audioClient_->Stop(); + } + renderClient_.Reset(); + audioClient_.Reset(); +} + +void WasapiRenderKeepAlive::renderLoop() { + while (!stopRequested_) { + UINT32 paddingFrames = 0; + if (FAILED(audioClient_->GetCurrentPadding(&paddingFrames))) { + break; + } + + const UINT32 framesAvailable = bufferFrameCount_ - paddingFrames; + if (framesAvailable > 0) { + BYTE* data = nullptr; + if (SUCCEEDED(renderClient_->GetBuffer(framesAvailable, &data))) { + tonePhase_ = writeToneFrames(data, framesAvailable, mixFormat_, tonePhase_); + renderClient_->ReleaseBuffer(framesAvailable, 0); + } + } + + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } +} diff --git a/electron/native/wgc-capture/src/wasapi_render_keepalive.h b/electron/native/wgc-capture/src/wasapi_render_keepalive.h new file mode 100644 index 000000000..de08b606f --- /dev/null +++ b/electron/native/wgc-capture/src/wasapi_render_keepalive.h @@ -0,0 +1,79 @@ +#pragma once + +// getopenscreen/openscreen#724: a headset can be idled by Windows partway through a +// mic-only recording, because nothing in that path ever writes to the render +// (playback) endpoint -- WasapiLoopbackCapture's system-audio path only ever READS +// from it via AUDCLNT_STREAMFLAGS_LOOPBACK. Confirmed on real hardware: the drop +// reproduces with mic-only capture and does NOT reproduce with system audio (i.e. +// loopback) also enabled, so touching the render endpoint at all is what keeps it +// alive, and loopback capture already does that as a side effect when it runs. +// +// This opens an ordinary render stream on the same default output device and +// writes a near-inaudible tone to it for as long as a recording is running. It +// only runs when system audio is NOT being captured: loopback capture already +// fills the endpoint with real content of its own, so there is nothing to keep +// alive, and anything written here alongside it would end up in the recording's +// system-audio track. +// +// Real signal, not digital silence: confirmed on real hardware that writing +// AUDCLNT_BUFFERFLAGS_SILENT packets does not reliably stop a wireless headset's +// own idle timer from firing, while writing an actual (if very quiet) tone does. +// This matches the earlier finding that system-audio loopback capture (which +// reads real audio, when something is playing) prevents the drop but mic-only +// capture (which touches nothing) does not -- some headsets' firmware appears to +// require genuine signal, not merely a live but silent stream, to register as +// activity. +// +// A first attempt at 1kHz / 1% amplitude was clearly audible in testing -- 1kHz +// sits in the most sensitive part of human hearing, so "quiet" in raw amplitude +// terms was still perceptibly loud. The tone is now 19kHz (above what the large +// majority of adults can hear at all) at 0.3% amplitude. It is not adapted to the +// device: on a mix format whose sample rate cannot represent 19kHz with margin, +// start() refuses to run at all rather than alias it into an audible frequency. +// The tone is never captured into the recording itself, because this stream only +// exists when loopback capture is off. +// +// Known limitation: the stream targets whatever the default render endpoint is at +// start and never re-targets. If the user switches output devices mid-take, the +// new default is not protected for the rest of the recording (the device watcher, +// if enabled, logs the switch). +// +// Kill-switch: set OPENSCREEN_WGC_DISABLE_AUDIO_KEEPALIVE=1 to turn this off. + +#include +#include +#include +#include + +#include +#include + +class WasapiRenderKeepAlive { +public: + WasapiRenderKeepAlive() = default; + ~WasapiRenderKeepAlive(); + + WasapiRenderKeepAlive(const WasapiRenderKeepAlive&) = delete; + WasapiRenderKeepAlive& operator=(const WasapiRenderKeepAlive&) = delete; + + // Opens the default render endpoint in shared mode and starts writing the + // keep-alive tone. Returns false on any failure -- including another app + // holding the device exclusively -- which callers must treat as non-fatal to + // the recording itself. + bool start(); + void stop(); + +private: + void renderLoop(); + + Microsoft::WRL::ComPtr audioClient_; + Microsoft::WRL::ComPtr renderClient_; + WAVEFORMATEX* mixFormat_ = nullptr; + UINT32 bufferFrameCount_ = 0; + std::thread thread_; + std::atomic stopRequested_ = false; + // Carries the tone's phase across GetBuffer/ReleaseBuffer calls so the + // waveform is continuous instead of restarting (and clicking) at each one. + // Only ever touched from the render thread, so no synchronization needed. + double tonePhase_ = 0.0; +}; diff --git a/technical-documentation/architecture/recording.md b/technical-documentation/architecture/recording.md index a191ec0c7..af1c6b101 100644 --- a/technical-documentation/architecture/recording.md +++ b/technical-documentation/architecture/recording.md @@ -72,6 +72,8 @@ A session writes a screen video and a `.session.json` manifest. Windows normally The Windows helper mixes system loopback and microphone into one track, and timestamps it from a running count of emitted frames. That count is advanced by a clock rather than by the arrival of samples: a chunk goes out every 10 ms for as long as the recording runs, filled from whichever source has data and with silence where neither does. Advancing it only when a queue held samples is what made a take that began in silence emit nothing at all — WASAPI loopback delivers no packets while nothing is playing — so the first sound landed at timestamp zero and the track came out shorter than the take. A working microphone concealed it by streaming continuously, which is why it appeared as a system-audio desync on a machine whose microphone had failed. `npm run test:wgc-audio-timeline:win` measures where a tone played at a known instant actually lands. +What a mic-only recording does to the render (output) endpoint is: nothing. System-audio capture is the only path that even opens it, and it reads rather than writes — which reads as idle to some wireless headsets' own firmware idle timers, and they power the set down mid-take (#724: a Corsair Void dropped reliably seven to ten minutes in, as a full disconnect the OS never observes; the USB dongle stays enumerated and the WASAPI endpoints stay ACTIVE throughout, so `IMMNotificationClient` sees nothing, and neither do the device's PnP properties). The helper therefore keeps the endpoint fed: when system audio is not being captured, it opens a second, ordinary shared-mode render stream on the default output device and writes a 19 kHz tone at 0.3% amplitude for as long as the take runs — above what the large majority of adults can hear, and real signal rather than packets flagged `AUDCLNT_BUFFERFLAGS_SILENT`, because the same hardware testing that found the timer also found that digital silence does not hold it off while an actual waveform does. The gate is not an optimization: loopback already fills the endpoint with real content when it runs, and anything this stream wrote beside it would be mixed straight into the recording's own system-audio track. It is a workaround, not a fix — the timer lives in the headset's firmware, below everything an application can observe or configure, and the per-vendor setting (iCUE for Corsair, equivalents elsewhere) is the only way to turn it off; "inaudible" is likewise per listener, since hearing range and driver harmonics vary. `OPENSCREEN_WGC_DISABLE_AUDIO_KEEPALIVE=1` turns the stream off. `OPENSCREEN_WGC_LOG_AUDIO_DEVICE_EVENTS=1` runs a diagnostic device watcher alongside the take, logging endpoint state transitions as JSON events to stderr — with the expectation, learned from #724, that a real firmware drop logs nothing at all, for exactly the reason above. + The macOS helper mixes the same two sources into one AAC track (`AudioTrackMixer`) and runs the same discipline, arrived at from the opposite direction. There the emit cursor is anchored to the writer session start and advanced by a monotonic clock read off `CMClockGetHostTimeClock` — the domain ScreenCaptureKit already timestamps its buffers in — with a 10 ms timer on the sample queue moving it while nothing is arriving to move it. The bug it replaces was the anchor, not the counter: `anchor` was set lazily on the first decoded buffer to `max(firstPresentationTime, sessionStart)`, so audio that first arrived four seconds in made frame zero of the track *be* four seconds in. The trailing half was worse still — `drain()` ran only from `ingest` and `finish`, and the helper never called `writer.endSession(atSourceTime:)` at all, so the file ended at the last sample anything happened to deliver. Both halves and any mid-take gap are now one mechanism: chunks go out for as long as the take runs, from whichever source covers them and from silence where none does, and a source is written off for a chunk only once it has gone 250 ms without delivering anything — a decision about when to stop waiting, which shifts nothing, and not a correction, which would. Measured from that source's own last delivery rather than from how far behind the clock its coverage sits, because those differ for a source that is alive but arriving late: a capture path delivering a fixed delay behind the audio it describes would be "behind" forever, so writing it off for that would emit silence and then drop its real samples, every chunk, for the whole take. A pause freezes the clock rather than resetting it, so what follows keeps the position it would have had. Whether ScreenCaptureKit's system-audio output is silence-gapped the way WASAPI loopback is has never been recorded anywhere, and it decides how much of a take the clock is carrying alone rather than how the mixer should work; the helper now reports it per take as `audio-timeline`, whose `undeliveredSeconds` is the whole track if the tap is gapped and near zero if it streams silence. That figure counts what each source handed over rather than what came out of the mixer — holes under two seconds are zero-filled into the source's own buffer, so a hole read off the mixed output would come back covered, which is the exact case it exists to catch — and `droppedSeconds` beside it is the cost side, counting audio that arrived too late to place and reading zero unless the grace is too short for that tap. `swift test` runs the mixer against a collecting sink and a hand-moved clock on every pull request (`swift-macos-helper` in ci.yml, the first Swift job this repository has had); `npm run test:sck-audio-timeline:mac` measures a real tone through a real device, comparing the audio track's length against the video track's in the same file. The Linux helper mixes the same two sources into one AAC track as well (`AudioMix` in `electron/native/pipewire-capture/src/capture.rs`), and its emit cursor is the data-driven one Windows had: `AudioEncoder::next_pts` is a running count of the samples handed to the encoder, advanced only when a block is encoded, and `AudioMix::pump` returns early when no input has any. Nothing in that code makes a chunk go out for every chunk of real time — what does is the tap. System audio here is the default sink's MONITOR, and PipeWire keeps delivering its buffers whether or not anything is playing, silence included, where WASAPI loopback simply stops. So the count advances at real time for as long as the take runs and the cursor cannot stall. That is a property of the capture source and not of the timeline code, which is the thing to know before moving Linux audio off the sink monitor: a source that gates silence would reproduce the Windows failure above exactly (#406). The anchor half is a decision rather than an accident — the first video frame sets the epoch and clears the audio rings at that instant ("video frame 0 is now, so audio sample 0 is now too"), which is what keeps the pre-roll captured while the portal picker was up, an interval with no upper bound, from shifting the whole track earlier; `cargo test` covers it. The tail is covered by `Capture::finish`, which drains the rings and flushes the AAC encoder before the video flush, so the file ends with the take rather than with the last sound in it. What a mid-take overflow costs is bounded the same way: the ring drops its oldest samples and owes back silence of exactly that length, so the encoder falling two seconds behind leaves a gap where it happened instead of pulling every later sound earlier for the rest of the recording, and it is reported as `audio-dropped`. `npm run test:pw-audio-timeline:linux` measures it end to end — a tone played at a known instant, then the audio track's length against the video track's in the same file. Before it, the whole record that any of this held was one sentence in a commit message: a live 29-second capture whose 18-second tone came back as 18 seconds of steady signal, with the two streams ending 62 ms apart (video 29.100 s, audio 29.162 s). That 62 ms is also why the script allows 250 ms between the tracks where the macOS one allows 100: those two come out of a single writer session, and these two do not — the video track ends at the last `advance()` and the audio track at everything the rings still held, plus AAC's final padded block.