From 36ebf9795357a4ce0eb36f30008b788c045e07dc Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 18 Sep 2026 12:32:12 +0200 Subject: [PATCH 1/4] fix(macos): realign a microphone stamped on a foreign clock ScreenCaptureKit does not promise the microphone output is stamped on the host clock the writer runs on. The mixer placed every buffer by its raw presentation timestamp, so a microphone whose time base sits seconds or hours away from the session landed wholly before frame zero (trimmed as pre-roll) or past the end of the take (never reached): a silent mic track, with no error. It went unnoticed because the only test Mac has no input device, and there the microphone output mirrors system audio, clock included. The first buffer of each source now decides its offset: within 1 s of the clock it is left alone (ordinary latency); beyond, it is mapped so that buffer ends where the clock stands and later buffers keep their spacing. The offset is reported in the audio-timeline summary and warned about once. --- .../AudioTrackMixer.swift | 76 ++++++++++++++++++- .../AudioTrackMixerTests.swift | 76 +++++++++++++++++++ 2 files changed, 149 insertions(+), 3 deletions(-) diff --git a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift index 042347394..8b4551a8d 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift @@ -95,6 +95,11 @@ public final class AudioTrackMixer { static let maxPendingChunks = 500 /// How long the final flush waits for the input to accept the tail before giving up. static let finalFlushTimeout = 5.0 + /// How far a source's first buffer may sit from the clock before its timestamps are + /// taken to be on some other clock entirely. A live ScreenCaptureKit output hands + /// over audio tens of milliseconds after capturing it; a whole second apart is not + /// latency, it is a different time base. See `clockOffsets`. + static let foreignClockThreshold = CMTime(value: 1, timescale: 1) } private let input: MixedAudioSink @@ -124,6 +129,22 @@ public final class AudioTrackMixer { private var pending: [CMSampleBuffer] = [] private var didWarnAboutBacklog = false private var didWarnAboutDecode: Set = [] + /// Per source, what is added to every presentation timestamp before placement — fixed on + /// that source's first delivery, nil until then. + /// + /// Zero for a source whose timestamps are on the writer's clock, which is what placement + /// by timestamp assumes. ScreenCaptureKit does not promise that for the microphone output + /// (Apple: the microphone and app audio are on independent clocks), and on a Mac with no + /// input device at all the microphone output mirrors system audio, clock included — so + /// the only machine the mixer had been proven on was the one place the assumption held. + /// Placed raw, a microphone stamped seconds or hours off the session lands entirely + /// before frame zero (all trimmed) or beyond the end of the take (never reached), and the + /// track comes out silent with every buffer accounted as delivered. + /// + /// A foreign time base is recognised on the first buffer and mapped onto the clock once: + /// that buffer is taken to end where the clock stands, and every later one keeps its own + /// spacing from it, so the source's timing is preserved and only its origin moves. + private var clockOffsets = [CMTime?](repeating: nil, count: Source.allCases.count) public init( input: MixedAudioSink, @@ -164,8 +185,8 @@ public final class AudioTrackMixer { guard includes(source), let anchor else { return } - let presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) - guard presentationTime.isValid, presentationTime.isNumeric else { + let capturedTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + guard capturedTime.isValid, capturedTime.isNumeric else { return } guard let frames = decodeInterleavedStereo(sampleBuffer, gain: gain(for: source)), @@ -178,6 +199,15 @@ public final class AudioTrackMixer { } let now = clock() + let presentationTime = CMTimeAdd( + capturedTime, + clockOffset( + for: source, + capturedAt: capturedTime, + frameCount: frames.count / MixFormat.channelCount, + now: now + ) + ) let startFrame = frameIndex(of: presentationTime, from: anchor) noteDelivery(source, from: startFrame, frameCount: Int64(frames.count / MixFormat.channelCount)) sources[source.rawValue].ingest(frames, atFrame: startFrame) @@ -212,6 +242,41 @@ public final class AudioTrackMixer { flushPending(force: true) } + /// The offset `clockOffsets` documents, fixed on the source's first delivery. + private func clockOffset( + for source: Source, + capturedAt capturedTime: CMTime, + frameCount: Int, + now: CMTime + ) -> CMTime { + if let offset = clockOffsets[source.rawValue] { + return offset + } + guard now.isValid, now.isNumeric else { + return .zero + } + + // Judged on where the buffer starts: a live buffer is a few milliseconds long and + // arrives just after it, so start and arrival differ only by latency. + let skew = CMTimeSubtract(now, capturedTime) + guard CMTimeCompare(CMTimeAbsoluteValue(skew), MixFormat.foreignClockThreshold) > 0 else { + clockOffsets[source.rawValue] = .zero + return .zero + } + // Realigned so that the buffer ends where the clock stands — it was just captured. + let offset = CMTimeSubtract( + skew, + CMTime(value: CMTimeValue(frameCount), timescale: CMTimeScale(MixFormat.sampleRate)) + ) + clockOffsets[source.rawValue] = offset + emit([ + "event": "warning", + "code": "audio-source-clock-rebased", + "message": "\(source == .system ? "System" : "Microphone") audio is timestamped \(String(format: "%.3f", CMTimeGetSeconds(skew))) s off the capture clock; it was realigned to arrival.", + ]) + return offset + } + private func warnAboutDecodeFailure(_ source: Source, _ sampleBuffer: CMSampleBuffer) { guard !didWarnAboutDecode.contains(source.rawValue) else { return @@ -308,6 +373,7 @@ public final class AudioTrackMixer { "longestHoleSeconds": report.longestHoleSeconds, "droppedSeconds": report.droppedSeconds, "trimmedSeconds": report.trimmedSeconds, + "clockOffsetSeconds": report.clockOffsetSeconds, ] } emit(fields) @@ -321,6 +387,9 @@ public final class AudioTrackMixer { public let longestHoleSeconds: Double public let droppedSeconds: Double public let trimmedSeconds: Double + /// What `clockOffsets` added to this source's timestamps; 0 when it was already on the + /// capture clock (or never delivered). + public let clockOffsetSeconds: Double } public func deliveryReport(for source: Source) -> DeliveryReport { @@ -329,7 +398,8 @@ public final class AudioTrackMixer { undeliveredSeconds: seconds(undeliveredFrames[source.rawValue]), longestHoleSeconds: seconds(longestHoleFrames[source.rawValue]), droppedSeconds: seconds(sources[source.rawValue].droppedFrames), - trimmedSeconds: seconds(sources[source.rawValue].trimmedFrames) + trimmedSeconds: seconds(sources[source.rawValue].trimmedFrames), + clockOffsetSeconds: clockOffsets[source.rawValue].map(CMTimeGetSeconds) ?? 0 ) } diff --git a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift index 5b8f7ff11..d5b1f0a11 100644 --- a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift +++ b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift @@ -218,6 +218,82 @@ final class AudioTrackMixerTests: XCTestCase { XCTAssertEqual((microphoneOnset ?? 0) - (systemOnset ?? 0), 0.090, accuracy: 0.002) } + /// The microphone output is not guaranteed to be stamped on the writer's clock. On this + /// project's only test Mac it was — with no input device, ScreenCaptureKit's microphone + /// output mirrors system audio, clock included — while on a MacBook with a real microphone + /// every take came out with a silent track. Placed raw, audio stamped far behind the + /// session is trimmed as pre-roll and audio stamped far ahead is never reached; either way + /// the file holds silence. Both directions must come out audible, where they were spoken. + func testAMicrophoneOnAForeignClockIsStillHeard() { + for foreignOffset in [-3_600.0, -47.5, 9_000.0] { + let (frames, mixer) = recordLiveTake(microphoneClockOffset: foreignOffset) + let label = "offset \(foreignOffset)" + XCTAssertEqual(seconds(ofFrames: frames), 2.0, accuracy: 0.011, label) + // Heard from its first delivery on (the span before that is the ordinary head trim + // every source gets) and still heard at the end of the take. + XCTAssertLessThanOrEqual(firstAudibleSecond(frames, channel: 1) ?? 99, 0.111, label) + XCTAssertGreaterThan(peak(frames, from: 1.7, to: 1.9, channel: 1), audibleThreshold, label) + // System audio, on the right clock, is untouched by the realignment. + XCTAssertGreaterThan(peak(frames, from: 0.1, to: 1.9, channel: 0), audibleThreshold, label) + XCTAssertEqual(mixer.deliveryReport(for: .system).clockOffsetSeconds, 0, label) + // The microphone was moved by its foreign offset and nothing else: its first buffer + // ends where the clock stood when it arrived, which is where it was captured. + XCTAssertEqual( + mixer.deliveryReport(for: .microphone).clockOffsetSeconds, -foreignOffset, accuracy: 0.001, label) + XCTAssertEqual(mixer.deliveryReport(for: .microphone).droppedSeconds, 0, label) + } + } + + /// The realignment must not touch a microphone that is on the clock: its buffers arrive a + /// little after the audio they describe, and that latency is not a foreign time base. + func testAMicrophoneOnTheCaptureClockIsNotRealigned() { + let (frames, mixer) = recordLiveTake(microphoneClockOffset: 0, microphoneLatency: 0.3) + XCTAssertEqual(mixer.deliveryReport(for: .microphone).clockOffsetSeconds, 0) + XCTAssertGreaterThan(peak(frames, from: 1.2, to: 1.4, channel: 1), audibleThreshold) + } + + /// Two seconds of live capture, delivered the way ScreenCaptureKit does: each 100 ms + /// buffer describes the 100 ms that just elapsed. System audio on the left, stamped on the + /// writer's clock; the microphone on the right, stamped `microphoneClockOffset` seconds off + /// it and handed over `microphoneLatency` seconds after its span ends. + private func recordLiveTake( + microphoneClockOffset: Double, + microphoneLatency: Double = 0 + ) -> ([Int16], AudioTrackMixer) { + let sink = RecordingSink() + let clock = TestClock(CMTime(value: 50, timescale: 1)) + let mixer = makeMixer(sink: sink, clock: clock, includesMicrophone: true, microphoneGain: 1) + mixer.beginTimeline(at: clock.now) + let sessionStart = clock.now + let step = 0.1 + var pendingMicrophone: [(deliverAt: Double, buffer: CMSampleBuffer)] = [] + + for index in 0..<20 { + let spanStart = CMTimeAdd(sessionStart, CMTime(seconds: Double(index) * step, preferredTimescale: 48_000)) + clock.advance(seconds: step) + mixer.ingest( + makeSourceBuffer(burst(seconds: step, left: 0.5, right: 0), at: spanStart, nonInterleaved: true), + from: .system + ) + pendingMicrophone.append(( + deliverAt: Double(index + 1) * step + microphoneLatency, + buffer: makeSourceBuffer( + burst(seconds: step, left: 0, right: 0.5), + at: CMTimeAdd(spanStart, CMTime(seconds: microphoneClockOffset, preferredTimescale: 48_000)), + nonInterleaved: false + ) + )) + let elapsed = Double(index + 1) * step + while let next = pendingMicrophone.first, next.deliverAt <= elapsed + 1e-9 { + mixer.ingest(next.buffer, from: .microphone) + pendingMicrophone.removeFirst() + } + mixer.tick() + } + mixer.finish(atSourceTime: clock.now) + return (mixedFrames(sink), mixer) + } + /// A dead microphone must not hold the track back — the whole reason the mixer marks a /// source stalled rather than waiting for it. Now that lateness is measured against the /// clock, this works even when the *other* source is silent too, which is the case the old From a90a7f1b60453d921759a31844690d6dbc7e680c Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 18 Sep 2026 14:31:35 +0200 Subject: [PATCH 2/4] Revert "fix(macos): realign a microphone stamped on a foreign clock" This reverts commit 36ebf9795357a4ce0eb36f30008b788c045e07dc. --- .../AudioTrackMixer.swift | 76 +------------------ .../AudioTrackMixerTests.swift | 76 ------------------- 2 files changed, 3 insertions(+), 149 deletions(-) diff --git a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift index 8b4551a8d..042347394 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift @@ -95,11 +95,6 @@ public final class AudioTrackMixer { static let maxPendingChunks = 500 /// How long the final flush waits for the input to accept the tail before giving up. static let finalFlushTimeout = 5.0 - /// How far a source's first buffer may sit from the clock before its timestamps are - /// taken to be on some other clock entirely. A live ScreenCaptureKit output hands - /// over audio tens of milliseconds after capturing it; a whole second apart is not - /// latency, it is a different time base. See `clockOffsets`. - static let foreignClockThreshold = CMTime(value: 1, timescale: 1) } private let input: MixedAudioSink @@ -129,22 +124,6 @@ public final class AudioTrackMixer { private var pending: [CMSampleBuffer] = [] private var didWarnAboutBacklog = false private var didWarnAboutDecode: Set = [] - /// Per source, what is added to every presentation timestamp before placement — fixed on - /// that source's first delivery, nil until then. - /// - /// Zero for a source whose timestamps are on the writer's clock, which is what placement - /// by timestamp assumes. ScreenCaptureKit does not promise that for the microphone output - /// (Apple: the microphone and app audio are on independent clocks), and on a Mac with no - /// input device at all the microphone output mirrors system audio, clock included — so - /// the only machine the mixer had been proven on was the one place the assumption held. - /// Placed raw, a microphone stamped seconds or hours off the session lands entirely - /// before frame zero (all trimmed) or beyond the end of the take (never reached), and the - /// track comes out silent with every buffer accounted as delivered. - /// - /// A foreign time base is recognised on the first buffer and mapped onto the clock once: - /// that buffer is taken to end where the clock stands, and every later one keeps its own - /// spacing from it, so the source's timing is preserved and only its origin moves. - private var clockOffsets = [CMTime?](repeating: nil, count: Source.allCases.count) public init( input: MixedAudioSink, @@ -185,8 +164,8 @@ public final class AudioTrackMixer { guard includes(source), let anchor else { return } - let capturedTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) - guard capturedTime.isValid, capturedTime.isNumeric else { + let presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + guard presentationTime.isValid, presentationTime.isNumeric else { return } guard let frames = decodeInterleavedStereo(sampleBuffer, gain: gain(for: source)), @@ -199,15 +178,6 @@ public final class AudioTrackMixer { } let now = clock() - let presentationTime = CMTimeAdd( - capturedTime, - clockOffset( - for: source, - capturedAt: capturedTime, - frameCount: frames.count / MixFormat.channelCount, - now: now - ) - ) let startFrame = frameIndex(of: presentationTime, from: anchor) noteDelivery(source, from: startFrame, frameCount: Int64(frames.count / MixFormat.channelCount)) sources[source.rawValue].ingest(frames, atFrame: startFrame) @@ -242,41 +212,6 @@ public final class AudioTrackMixer { flushPending(force: true) } - /// The offset `clockOffsets` documents, fixed on the source's first delivery. - private func clockOffset( - for source: Source, - capturedAt capturedTime: CMTime, - frameCount: Int, - now: CMTime - ) -> CMTime { - if let offset = clockOffsets[source.rawValue] { - return offset - } - guard now.isValid, now.isNumeric else { - return .zero - } - - // Judged on where the buffer starts: a live buffer is a few milliseconds long and - // arrives just after it, so start and arrival differ only by latency. - let skew = CMTimeSubtract(now, capturedTime) - guard CMTimeCompare(CMTimeAbsoluteValue(skew), MixFormat.foreignClockThreshold) > 0 else { - clockOffsets[source.rawValue] = .zero - return .zero - } - // Realigned so that the buffer ends where the clock stands — it was just captured. - let offset = CMTimeSubtract( - skew, - CMTime(value: CMTimeValue(frameCount), timescale: CMTimeScale(MixFormat.sampleRate)) - ) - clockOffsets[source.rawValue] = offset - emit([ - "event": "warning", - "code": "audio-source-clock-rebased", - "message": "\(source == .system ? "System" : "Microphone") audio is timestamped \(String(format: "%.3f", CMTimeGetSeconds(skew))) s off the capture clock; it was realigned to arrival.", - ]) - return offset - } - private func warnAboutDecodeFailure(_ source: Source, _ sampleBuffer: CMSampleBuffer) { guard !didWarnAboutDecode.contains(source.rawValue) else { return @@ -373,7 +308,6 @@ public final class AudioTrackMixer { "longestHoleSeconds": report.longestHoleSeconds, "droppedSeconds": report.droppedSeconds, "trimmedSeconds": report.trimmedSeconds, - "clockOffsetSeconds": report.clockOffsetSeconds, ] } emit(fields) @@ -387,9 +321,6 @@ public final class AudioTrackMixer { public let longestHoleSeconds: Double public let droppedSeconds: Double public let trimmedSeconds: Double - /// What `clockOffsets` added to this source's timestamps; 0 when it was already on the - /// capture clock (or never delivered). - public let clockOffsetSeconds: Double } public func deliveryReport(for source: Source) -> DeliveryReport { @@ -398,8 +329,7 @@ public final class AudioTrackMixer { undeliveredSeconds: seconds(undeliveredFrames[source.rawValue]), longestHoleSeconds: seconds(longestHoleFrames[source.rawValue]), droppedSeconds: seconds(sources[source.rawValue].droppedFrames), - trimmedSeconds: seconds(sources[source.rawValue].trimmedFrames), - clockOffsetSeconds: clockOffsets[source.rawValue].map(CMTimeGetSeconds) ?? 0 + trimmedSeconds: seconds(sources[source.rawValue].trimmedFrames) ) } diff --git a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift index d5b1f0a11..5b8f7ff11 100644 --- a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift +++ b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift @@ -218,82 +218,6 @@ final class AudioTrackMixerTests: XCTestCase { XCTAssertEqual((microphoneOnset ?? 0) - (systemOnset ?? 0), 0.090, accuracy: 0.002) } - /// The microphone output is not guaranteed to be stamped on the writer's clock. On this - /// project's only test Mac it was — with no input device, ScreenCaptureKit's microphone - /// output mirrors system audio, clock included — while on a MacBook with a real microphone - /// every take came out with a silent track. Placed raw, audio stamped far behind the - /// session is trimmed as pre-roll and audio stamped far ahead is never reached; either way - /// the file holds silence. Both directions must come out audible, where they were spoken. - func testAMicrophoneOnAForeignClockIsStillHeard() { - for foreignOffset in [-3_600.0, -47.5, 9_000.0] { - let (frames, mixer) = recordLiveTake(microphoneClockOffset: foreignOffset) - let label = "offset \(foreignOffset)" - XCTAssertEqual(seconds(ofFrames: frames), 2.0, accuracy: 0.011, label) - // Heard from its first delivery on (the span before that is the ordinary head trim - // every source gets) and still heard at the end of the take. - XCTAssertLessThanOrEqual(firstAudibleSecond(frames, channel: 1) ?? 99, 0.111, label) - XCTAssertGreaterThan(peak(frames, from: 1.7, to: 1.9, channel: 1), audibleThreshold, label) - // System audio, on the right clock, is untouched by the realignment. - XCTAssertGreaterThan(peak(frames, from: 0.1, to: 1.9, channel: 0), audibleThreshold, label) - XCTAssertEqual(mixer.deliveryReport(for: .system).clockOffsetSeconds, 0, label) - // The microphone was moved by its foreign offset and nothing else: its first buffer - // ends where the clock stood when it arrived, which is where it was captured. - XCTAssertEqual( - mixer.deliveryReport(for: .microphone).clockOffsetSeconds, -foreignOffset, accuracy: 0.001, label) - XCTAssertEqual(mixer.deliveryReport(for: .microphone).droppedSeconds, 0, label) - } - } - - /// The realignment must not touch a microphone that is on the clock: its buffers arrive a - /// little after the audio they describe, and that latency is not a foreign time base. - func testAMicrophoneOnTheCaptureClockIsNotRealigned() { - let (frames, mixer) = recordLiveTake(microphoneClockOffset: 0, microphoneLatency: 0.3) - XCTAssertEqual(mixer.deliveryReport(for: .microphone).clockOffsetSeconds, 0) - XCTAssertGreaterThan(peak(frames, from: 1.2, to: 1.4, channel: 1), audibleThreshold) - } - - /// Two seconds of live capture, delivered the way ScreenCaptureKit does: each 100 ms - /// buffer describes the 100 ms that just elapsed. System audio on the left, stamped on the - /// writer's clock; the microphone on the right, stamped `microphoneClockOffset` seconds off - /// it and handed over `microphoneLatency` seconds after its span ends. - private func recordLiveTake( - microphoneClockOffset: Double, - microphoneLatency: Double = 0 - ) -> ([Int16], AudioTrackMixer) { - let sink = RecordingSink() - let clock = TestClock(CMTime(value: 50, timescale: 1)) - let mixer = makeMixer(sink: sink, clock: clock, includesMicrophone: true, microphoneGain: 1) - mixer.beginTimeline(at: clock.now) - let sessionStart = clock.now - let step = 0.1 - var pendingMicrophone: [(deliverAt: Double, buffer: CMSampleBuffer)] = [] - - for index in 0..<20 { - let spanStart = CMTimeAdd(sessionStart, CMTime(seconds: Double(index) * step, preferredTimescale: 48_000)) - clock.advance(seconds: step) - mixer.ingest( - makeSourceBuffer(burst(seconds: step, left: 0.5, right: 0), at: spanStart, nonInterleaved: true), - from: .system - ) - pendingMicrophone.append(( - deliverAt: Double(index + 1) * step + microphoneLatency, - buffer: makeSourceBuffer( - burst(seconds: step, left: 0, right: 0.5), - at: CMTimeAdd(spanStart, CMTime(seconds: microphoneClockOffset, preferredTimescale: 48_000)), - nonInterleaved: false - ) - )) - let elapsed = Double(index + 1) * step - while let next = pendingMicrophone.first, next.deliverAt <= elapsed + 1e-9 { - mixer.ingest(next.buffer, from: .microphone) - pendingMicrophone.removeFirst() - } - mixer.tick() - } - mixer.finish(atSourceTime: clock.now) - return (mixedFrames(sink), mixer) - } - /// A dead microphone must not hold the track back — the whole reason the mixer marks a /// source stalled rather than waiting for it. Now that lateness is measured against the /// clock, this works even when the *other* source is silent too, which is the case the old From cae2567a70bbfa86a848d48063de5d2dbdc592af Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 18 Sep 2026 14:33:32 +0200 Subject: [PATCH 3/4] fix(macos): decode 24-bit integer microphone audio The ScreenCaptureKit microphone output delivers the input device's own format, not the 48 kHz stereo Float32 the stream is configured for. On a MacBook Pro (M1, macOS 26.6) that is 48 kHz mono 24-bit integer, for both the built-in and a USB microphone. The mixer only read Float32, Int16 and Int32, so it rejected every microphone buffer with audio-source-undecodable and each take recorded a silent microphone (#709). Integers of any width up to 32 bits are now read from their real container (mBytesPerFrame, not bits/8), packed or padded, high- or low-aligned. Big-endian input is still rejected rather than misread. --- .../AudioTrackMixer.swift | 55 ++++++--- .../AudioTrackMixerTests.swift | 114 ++++++++++++++++++ 2 files changed, 155 insertions(+), 14 deletions(-) diff --git a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift index 042347394..d188949f8 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift @@ -470,11 +470,13 @@ public final class AudioTrackMixer { /// Decodes one capture buffer into gain-applied 48 kHz interleaved-stereo Float. /// - /// Both SCStream audio outputs are configured for 48 kHz stereo, so in practice this is a - /// straight Float32 de-interleave. The format-adaptive paths (Int16/Int32, interleaved or - /// not, off-rate sources) exist because the format is the stream's to choose, not ours — - /// and because a resampled source's rounding drift is absorbed by timeline placement - /// rather than accumulating, unlike in a FIFO mixer. + /// Both SCStream audio outputs are configured for 48 kHz stereo, but only system audio + /// honours that reliably: the microphone output can arrive in the input device's own format + /// — 48 kHz mono 24-bit integer on a MacBook Pro's built-in and USB microphones (#709). So + /// the format-adaptive paths (integers of any width up to 32 bits, interleaved or not, + /// off-rate sources) are not defensive extras; the format is the stream's to choose, not + /// ours. A resampled source's rounding drift is absorbed by timeline placement rather than + /// accumulating, unlike in a FIFO mixer. private func decodeInterleavedStereo(_ sampleBuffer: CMSampleBuffer, gain: Float) -> [Float]? { guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer), let streamDescription = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription) @@ -487,11 +489,24 @@ public final class AudioTrackMixer { let sourceChannels = Int(asbd.mChannelsPerFrame) let bitsPerChannel = Int(asbd.mBitsPerChannel) let isFloat = asbd.mFormatFlags & kAudioFormatFlagIsFloat != 0 + let isNonInterleaved = asbd.mFormatFlags & kAudioFormatFlagIsNonInterleaved != 0 + // The container each sample sits in, which is not always `bitsPerChannel / 8`: a + // microphone commonly arrives as 24-bit integer, either packed into 3 bytes or held in 4. + // Measured on a MacBook Pro (M1, macOS 26.6): the ScreenCaptureKit microphone output + // delivered 48 kHz mono 24-bit interleaved, which the Float32/Int16/Int32-only decoder + // rejected buffer by buffer — every take's microphone was silent (issue #709). + let bytesPerSample = sourceChannels > 0 && asbd.mBytesPerFrame > 0 + ? Int(asbd.mBytesPerFrame) / (isNonInterleaved ? 1 : sourceChannels) + : (bitsPerChannel + 7) / 8 guard asbd.mFormatID == kAudioFormatLinearPCM, + asbd.mFormatFlags & kAudioFormatFlagIsBigEndian == 0, sourceChannels > 0, asbd.mSampleRate > 0, sourceFrames > 0, - isFloat ? bitsPerChannel == 32 : (bitsPerChannel == 16 || bitsPerChannel == 32) + isFloat + ? bitsPerChannel == 32 && bytesPerSample == 4 + : (8...32).contains(bitsPerChannel) && (2...4).contains(bytesPerSample) + && bitsPerChannel <= bytesPerSample * 8 else { return nil } @@ -503,7 +518,6 @@ public final class AudioTrackMixer { // outputs disagree here: system audio arrives non-interleaved, the microphone // interleaved, so sizing this off the channel count alone silently drops every // microphone buffer. - let isNonInterleaved = asbd.mFormatFlags & kAudioFormatFlagIsNonInterleaved != 0 let bufferCount = isNonInterleaved ? sourceChannels : 1 let bufferList = AudioBufferList.allocate(maximumBuffers: bufferCount) defer { free(bufferList.unsafeMutablePointer) } @@ -524,7 +538,6 @@ public final class AudioTrackMixer { } return withExtendedLifetime(blockBuffer) { () -> [Float]? in - let bytesPerChannelSample = bitsPerChannel / 8 // `bufferList` is subscripted against its own count, not the format's channel // count: indexing past `count` would trap rather than degrade. guard bufferList.count > 0 else { @@ -545,11 +558,15 @@ public final class AudioTrackMixer { readers.append( ChannelReader( base: UnsafeRawPointer(data), - sampleCount: Int(buffer.mDataByteSize) / bytesPerChannelSample, + sampleCount: Int(buffer.mDataByteSize) / bytesPerSample, stride: isNonInterleaved ? 1 : sourceChannels, start: isNonInterleaved ? 0 : sourceChannel, - bytesPerSample: bytesPerChannelSample, - isFloat: isFloat + bytesPerSample: bytesPerSample, + isFloat: isFloat, + // Where the significant bits sit when they do not fill the container. + // Irrelevant to packed formats, where the two coincide. + padBits: bytesPerSample * 8 - bitsPerChannel, + isAlignedHigh: asbd.mFormatFlags & kAudioFormatFlagIsAlignedHigh != 0 ) ) } @@ -691,6 +708,8 @@ public final class AudioTrackMixer { let start: Int let bytesPerSample: Int let isFloat: Bool + let padBits: Int + let isAlignedHigh: Bool func value(at frame: Int) -> Float { let index = start + frame * stride @@ -702,10 +721,18 @@ public final class AudioTrackMixer { if isFloat { return base.loadUnaligned(fromByteOffset: offset, as: Float.self) } - if bytesPerSample == 2 { - return Float(base.loadUnaligned(fromByteOffset: offset, as: Int16.self)) / 32_768 + // Integer of any width up to 32 bits: assemble the little-endian bytes at the top of + // a 32-bit word, so the sign bit lands on bit 31 and one full scale serves every width. + var word: UInt32 = 0 + for byte in 0.. 0 && !isAlignedHigh { + // Low-aligned: the padding is on top, so shift it out to put the sign bit on 31. + word <<= UInt32(padBits) } - return Float(base.loadUnaligned(fromByteOffset: offset, as: Int32.self)) / 2_147_483_648 + return Float(Int32(bitPattern: word)) / 2_147_483_648 } } diff --git a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift index 5b8f7ff11..56bef6942 100644 --- a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift +++ b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift @@ -455,6 +455,47 @@ final class AudioTrackMixerTests: XCTestCase { XCTAssertLessThan(loudest, 16_500) } + /// A microphone does not have to arrive as Float32. A MacBook Pro (M1, macOS 26.6) handed + /// the ScreenCaptureKit microphone output over as 48 kHz mono 24-bit integer, which the + /// decoder rejected — Float32, Int16 and Int32 were the only formats it read — so every take + /// recorded there had a silent microphone and one `audio-source-undecodable` warning to show + /// for it (issue #709). Every integer container CoreAudio describes must decode, at the right + /// level and with the right sign. + func testIntegerMicrophoneFormatsAreHeardAtTheirLevel() { + let layouts: [(bits: Int, bytes: Int, alignedHigh: Bool, label: String)] = [ + (24, 3, false, "24-bit packed (the reported format)"), + (24, 4, false, "24-bit in 4 bytes, low-aligned"), + (24, 4, true, "24-bit in 4 bytes, high-aligned"), + (16, 2, false, "16-bit"), + (32, 4, false, "32-bit"), + ] + for layout in layouts { + let sink = RecordingSink() + let clock = TestClock(.zero) + let mixer = makeMixer(sink: sink, clock: clock, includesMicrophone: true, microphoneGain: 1) + mixer.beginTimeline(at: clock.now) + mixer.ingest( + makeIntegerMonoBuffer( + alternating: 0.5, frames: sampleRate / 5, at: clock.now, + bits: layout.bits, bytes: layout.bytes, alignedHigh: layout.alignedHigh + ), + from: .microphone + ) + clock.advance(seconds: 0.2) + mixer.finish(atSourceTime: clock.now) + + let frames = mixedFrames(sink) + XCTAssertEqual(seconds(ofFrames: frames), 0.2, accuracy: 0.011, layout.label) + // Mono feeds both sides at half scale, and +0.5 stays positive: a byte-order or + // sign-extension mistake shows up as the wrong level or a flipped first sample. + for lane in 0.. CMSampleBuffer { + var asbd = AudioStreamBasicDescription( + mSampleRate: Float64(sampleRate), + mFormatID: kAudioFormatLinearPCM, + mFormatFlags: kAudioFormatFlagIsSignedInteger + | (bits == bytes * 8 ? kAudioFormatFlagIsPacked : 0) + | (alignedHigh ? kAudioFormatFlagIsAlignedHigh : 0), + mBytesPerPacket: UInt32(bytes), + mFramesPerPacket: 1, + mBytesPerFrame: UInt32(bytes), + mChannelsPerFrame: 1, + mBitsPerChannel: UInt32(bits), + mReserved: 0 + ) + var formatDescription: CMAudioFormatDescription? + XCTAssertEqual( + CMAudioFormatDescriptionCreate( + allocator: kCFAllocatorDefault, asbd: &asbd, layoutSize: 0, layout: nil, + magicCookieSize: 0, magicCookie: nil, extensions: nil, + formatDescriptionOut: &formatDescription + ), + noErr + ) + var sampleBuffer: CMSampleBuffer? + XCTAssertEqual( + CMAudioSampleBufferCreateWithPacketDescriptions( + allocator: kCFAllocatorDefault, dataBuffer: nil, dataReady: false, + makeDataReadyCallback: nil, refcon: nil, formatDescription: formatDescription!, + sampleCount: frameCount, presentationTimeStamp: presentationTime, + packetDescriptions: nil, sampleBufferOut: &sampleBuffer + ), + noErr + ) + + let byteCount = frameCount * bytes + let memory = UnsafeMutableRawPointer.allocate(byteCount: byteCount, alignment: 16) + defer { memory.deallocate() } + let fullScale = Double(Int64(1) << (bits - 1)) + for frame in 0..> UInt32(8 * byte)), + toByteOffset: frame * bytes + byte, as: UInt8.self) + } + } + let bufferList = AudioBufferList.allocate(maximumBuffers: 1) + defer { free(bufferList.unsafeMutablePointer) } + bufferList[0] = AudioBuffer(mNumberChannels: 1, mDataByteSize: UInt32(byteCount), mData: memory) + XCTAssertEqual( + CMSampleBufferSetDataBufferFromAudioBufferList( + sampleBuffer!, blockBufferAllocator: kCFAllocatorDefault, + blockBufferMemoryAllocator: kCFAllocatorDefault, flags: 0, + bufferList: bufferList.unsafePointer + ), + noErr + ) + XCTAssertEqual(CMSampleBufferSetDataReady(sampleBuffer!), noErr) + return sampleBuffer! + } + private func mixedFrames(_ sink: RecordingSink) -> [Int16] { var samples = [Int16]() for buffer in sink.buffers { From 01b39484ce9384c02dd80bef8f7b9933dfad5d5a Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 18 Sep 2026 14:50:10 +0200 Subject: [PATCH 4/4] fix(macos): read unsigned integer PCM around its midpoint Generalising the decoder to any integer width let unsigned (offset-binary) PCM, the usual 8-bit layout, through as if it were signed: full-scale noise instead of a rejection. Unsigned samples now have their midpoint removed, which with the value left-justified is flipping bit 31. --- .../AudioTrackMixer.swift | 10 ++++++- .../AudioTrackMixerTests.swift | 26 ++++++++++++------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift index d188949f8..47d62951e 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/AudioTrackMixer.swift @@ -566,7 +566,8 @@ public final class AudioTrackMixer { // Where the significant bits sit when they do not fill the container. // Irrelevant to packed formats, where the two coincide. padBits: bytesPerSample * 8 - bitsPerChannel, - isAlignedHigh: asbd.mFormatFlags & kAudioFormatFlagIsAlignedHigh != 0 + isAlignedHigh: asbd.mFormatFlags & kAudioFormatFlagIsAlignedHigh != 0, + isSigned: asbd.mFormatFlags & kAudioFormatFlagIsSignedInteger != 0 ) ) } @@ -710,6 +711,9 @@ public final class AudioTrackMixer { let isFloat: Bool let padBits: Int let isAlignedHigh: Bool + /// Unsigned PCM (offset binary, the usual 8-bit layout) centres on the midpoint, not on + /// zero; read as signed it would come out as full-scale noise rather than be rejected. + let isSigned: Bool func value(at frame: Int) -> Float { let index = start + frame * stride @@ -732,6 +736,10 @@ public final class AudioTrackMixer { // Low-aligned: the padding is on top, so shift it out to put the sign bit on 31. word <<= UInt32(padBits) } + if !isSigned { + // With the value left-justified, subtracting the midpoint is flipping bit 31. + word ^= 0x8000_0000 + } return Float(Int32(bitPattern: word)) / 2_147_483_648 } } diff --git a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift index 56bef6942..64492a7ce 100644 --- a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift +++ b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/AudioTrackMixerTests.swift @@ -462,12 +462,14 @@ final class AudioTrackMixerTests: XCTestCase { /// for it (issue #709). Every integer container CoreAudio describes must decode, at the right /// level and with the right sign. func testIntegerMicrophoneFormatsAreHeardAtTheirLevel() { - let layouts: [(bits: Int, bytes: Int, alignedHigh: Bool, label: String)] = [ - (24, 3, false, "24-bit packed (the reported format)"), - (24, 4, false, "24-bit in 4 bytes, low-aligned"), - (24, 4, true, "24-bit in 4 bytes, high-aligned"), - (16, 2, false, "16-bit"), - (32, 4, false, "32-bit"), + let layouts: [(bits: Int, bytes: Int, alignedHigh: Bool, signed: Bool, label: String)] = [ + (24, 3, false, true, "24-bit packed (the reported format)"), + (24, 4, false, true, "24-bit in 4 bytes, low-aligned"), + (24, 4, true, true, "24-bit in 4 bytes, high-aligned"), + (16, 2, false, true, "16-bit"), + (32, 4, false, true, "32-bit"), + (16, 2, false, false, "16-bit unsigned"), + (24, 3, false, false, "24-bit unsigned"), ] for layout in layouts { let sink = RecordingSink() @@ -477,7 +479,8 @@ final class AudioTrackMixerTests: XCTestCase { mixer.ingest( makeIntegerMonoBuffer( alternating: 0.5, frames: sampleRate / 5, at: clock.now, - bits: layout.bits, bytes: layout.bytes, alignedHigh: layout.alignedHigh + bits: layout.bits, bytes: layout.bytes, alignedHigh: layout.alignedHigh, + signed: layout.signed ), from: .microphone ) @@ -637,12 +640,13 @@ final class AudioTrackMixerTests: XCTestCase { at presentationTime: CMTime, bits: Int, bytes: Int, - alignedHigh: Bool + alignedHigh: Bool, + signed: Bool = true ) -> CMSampleBuffer { var asbd = AudioStreamBasicDescription( mSampleRate: Float64(sampleRate), mFormatID: kAudioFormatLinearPCM, - mFormatFlags: kAudioFormatFlagIsSignedInteger + mFormatFlags: (signed ? kAudioFormatFlagIsSignedInteger : 0) | (bits == bytes * 8 ? kAudioFormatFlagIsPacked : 0) | (alignedHigh ? kAudioFormatFlagIsAlignedHigh : 0), mBytesPerPacket: UInt32(bytes), @@ -680,6 +684,10 @@ final class AudioTrackMixerTests: XCTestCase { let sample = Double(frame % 2 == 0 ? amplitude : -amplitude) var value = Int64((sample * fullScale).rounded()) value = min(max(value, -Int64(fullScale)), Int64(fullScale) - 1) + if !signed { + // Offset binary: the midpoint is silence. + value += Int64(fullScale) + } // High-aligned puts the padding below the value; low-aligned sign-extends above it. let word = UInt32(truncatingIfNeeded: alignedHigh ? value << Int64(bytes * 8 - bits) : value) for byte in 0..