diff --git a/electron/native/ScreenCaptureKitRecorder.swift b/electron/native/ScreenCaptureKitRecorder.swift index 6e4b0efdc..6d938633a 100644 --- a/electron/native/ScreenCaptureKitRecorder.swift +++ b/electron/native/ScreenCaptureKitRecorder.swift @@ -59,6 +59,13 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { private var lastVideoDuration: CMTime = .zero private var lastInlineAudioPresentationTime: CMTime = .invalid private var lastInlineAudioDuration: CMTime = .zero + private var lastSystemAudioDuration: CMTime = .zero + private var lastMicrophoneDuration: CMTime = .zero + private var droppedAudioBufferCount = 0 + private var insertedSilenceFrames: Int64 = 0 + /// Audio is delivered on its own queue so a slow video callback (5K crop + + /// encode) can never make ScreenCaptureKit discard microphone buffers. + private let audioQueue = DispatchQueue(label: "recordly.screencapturekit.audio") private var isRecording = false private var isPaused = false private var pauseStartedHostTime: CMTime? @@ -82,6 +89,9 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { private let microphoneOutputTypeRawValue = 2 + /// Configures the ScreenCaptureKit stream and asset writers from the JSON + /// config passed on the command line, then starts capturing. Screen frames + /// are delivered on `queue`; audio outputs are delivered on `audioQueue`. func startCapture(configJSON: String) async throws { guard !isRecording else { throw NSError(domain: "RecordlyCapture", code: 1, userInfo: [NSLocalizedDescriptionKey: "Recording is already in progress"]) @@ -326,7 +336,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { self.stream = stream try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue) if capturesSystemAudio { - try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: queue) + try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: audioQueue) } if capturesMicrophone { guard let microphoneOutputType = SCStreamOutputType(rawValue: microphoneOutputTypeRawValue) else { @@ -336,7 +346,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { userInfo: [NSLocalizedDescriptionKey: "Microphone stream output type is unavailable"] ) } - try stream.addStreamOutput(self, type: microphoneOutputType, sampleHandlerQueue: queue) + try stream.addStreamOutput(self, type: microphoneOutputType, sampleHandlerQueue: audioQueue) } try await stream.startCapture() @@ -392,7 +402,24 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } } + /// ScreenCaptureKit delivery entry point. Screen frames arrive on `queue` and + /// are handled inline; audio arrives on `audioQueue` and is hopped onto + /// `queue` so recorder state is only ever touched from one queue and a slow + /// video callback can never block or drop audio delivery. func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of outputType: SCStreamOutputType) { + if outputType != .screen { + queue.async { [weak self] in + self?.handleSampleBuffer(sampleBuffer, of: outputType) + } + return + } + handleSampleBuffer(sampleBuffer, of: outputType) + } + + /// Retimes one screen, system-audio, or microphone sample onto the recording + /// timeline and appends it to the matching writer inputs. Always runs on + /// `queue`; drops everything once finalization has started. + private func handleSampleBuffer(_ sampleBuffer: CMSampleBuffer, of outputType: SCStreamOutputType) { guard sessionStarted, sampleBuffer.isValid, isRecording else { return } if outputType == .screen { @@ -448,21 +475,21 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { if outputType == .audio { guard let systemAudioInput else { return } - appendAudioSampleBuffer(sampleBuffer, to: systemAudioInput, of: systemAudioWriter, firstSampleTime: &firstSystemAudioSampleTime, lastPresentationTime: &lastSystemAudioPresentationTime, presentationTime: presentationTime) + appendAudioSampleBuffer(sampleBuffer, to: systemAudioInput, of: systemAudioWriter, firstSampleTime: &firstSystemAudioSampleTime, lastPresentationTime: &lastSystemAudioPresentationTime, lastDuration: &lastSystemAudioDuration, presentationTime: presentationTime) // Also write system audio to the inline video track - if let inlineAudioInput, inlineAudioInput.isReadyForMoreMediaData { - appendAudioSampleBuffer(sampleBuffer, to: inlineAudioInput, of: assetWriter, firstSampleTime: &firstInlineAudioSampleTime, lastPresentationTime: &lastInlineAudioPresentationTime, presentationTime: presentationTime) + if let inlineAudioInput { + appendAudioSampleBuffer(sampleBuffer, to: inlineAudioInput, of: assetWriter, firstSampleTime: &firstInlineAudioSampleTime, lastPresentationTime: &lastInlineAudioPresentationTime, lastDuration: &lastInlineAudioDuration, presentationTime: presentationTime) } return } if outputType.rawValue == microphoneOutputTypeRawValue { if let microphoneOnlyInput { - appendAudioSampleBuffer(sampleBuffer, to: microphoneOnlyInput, of: microphoneOnlyWriter, firstSampleTime: &firstMicrophoneSampleTime, lastPresentationTime: &lastMicrophonePresentationTime, presentationTime: presentationTime) + appendAudioSampleBuffer(sampleBuffer, to: microphoneOnlyInput, of: microphoneOnlyWriter, firstSampleTime: &firstMicrophoneSampleTime, lastPresentationTime: &lastMicrophonePresentationTime, lastDuration: &lastMicrophoneDuration, presentationTime: presentationTime) } // Write mic to inline video track only if there's no system audio (avoids double-writing) - if !capturesSystemAudio, let inlineAudioInput, inlineAudioInput.isReadyForMoreMediaData { - appendAudioSampleBuffer(sampleBuffer, to: inlineAudioInput, of: assetWriter, firstSampleTime: &firstInlineAudioSampleTime, lastPresentationTime: &lastInlineAudioPresentationTime, presentationTime: presentationTime) + if !capturesSystemAudio, let inlineAudioInput { + appendAudioSampleBuffer(sampleBuffer, to: inlineAudioInput, of: assetWriter, firstSampleTime: &firstInlineAudioSampleTime, lastPresentationTime: &lastInlineAudioPresentationTime, lastDuration: &lastInlineAudioDuration, presentationTime: presentationTime) } return } @@ -521,7 +548,13 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { /// the recorder queue have drained. Manual stop and automatic window-close /// detection join the same operation instead of racing the asset writers. private func finalizeCapture(interactive: Bool) async -> CaptureFinalizationResult { - await withCheckedContinuation { continuation in + // Audio arrives on audioQueue and hops onto the recorder queue. Drain that + // hop first so every buffer delivered before this stop request is already + // queued ahead of the finalization block instead of being dropped by the + // isRecording guard. finalizeCapture never runs on either queue (it is + // called from the command queue or a Task), so the barrier cannot deadlock. + audioQueue.sync {} + return await withCheckedContinuation { continuation in queue.async { if self.isFinalizing { self.interactiveStopParticipated = self.interactiveStopParticipated || interactive @@ -574,6 +607,10 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } } + /// Stops the stream, gives the last frame its full duration, closes every + /// writer, reports audio-gap diagnostics on stderr, and resets recorder state. + /// Throws if any writer failed so a half-written file is never reported as + /// a successful recording. private func finishCapture() async throws -> String { if let activeStream = stream { @@ -636,6 +673,15 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { : (writer.error ?? unfinalizedWriterError(status: writer.status)) } .first + if droppedAudioBufferCount > 0 || insertedSilenceFrames > 0 { + fputs("AUDIO_GAPS: droppedBuffers=\(droppedAudioBufferCount) silenceFramesInserted=\(insertedSilenceFrames)\n", stderr) + fflush(stderr) + } + droppedAudioBufferCount = 0 + insertedSilenceFrames = 0 + lastSystemAudioDuration = .zero + lastMicrophoneDuration = .zero + let path = outputURL?.path ?? "" assetWriter = nil videoInput = nil @@ -795,28 +841,133 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { return videoEndTime + CMTimeMinimum(tailExtension, maxInlineAudioTailExtension) } - private func appendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer, to input: AVAssetWriterInput, of writer: AVAssetWriter?, firstSampleTime: inout CMTime?, lastPresentationTime: inout CMTime, presentationTime: CMTime) { + /// Appends one audio buffer to `input` at `presentationTime`, first filling + /// any gap since the previous buffer with silence so lost buffers never + /// compact the track. Buffers that arrive while the input is not ready are + /// counted as dropped; `lastPresentationTime`/`lastDuration` track the last + /// sample (real or silence) the writer accepted for this input. + private func appendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer, to input: AVAssetWriterInput, of writer: AVAssetWriter?, firstSampleTime: inout CMTime?, lastPresentationTime: inout CMTime, lastDuration: inout CMTime, presentationTime: CMTime) { // A writer that failed mid-capture (a full disk, say) raises on every // further append, which would abort the helper and lose the whole file. - guard writer?.status == .writing, input.isReadyForMoreMediaData else { return } + guard writer?.status == .writing else { return } + guard input.isReadyForMoreMediaData else { + // Back-pressure from the writer (typically the video encoder lagging on + // a high-resolution capture). The buffer is lost, but the hole is filled + // with silence on the next accepted buffer so the track keeps real time + // instead of compacting and drifting ahead of the video. + droppedAudioBufferCount += 1 + return + } guard !lastPresentationTime.isValid || CMTimeCompare(presentationTime, lastPresentationTime) > 0 else { return } if firstSampleTime == nil { firstSampleTime = presentationTime } + if lastPresentationTime.isValid, lastDuration.isValid, lastDuration > .zero { + let expectedNext = lastPresentationTime + lastDuration + let gap = presentationTime - expectedNext + let tolerance = CMTimeMultiplyByFloat64(lastDuration, multiplier: 0.5) + if CMTimeCompare(gap, tolerance) > 0, + !appendSilence(matching: sampleBuffer, from: expectedNext, to: presentationTime, into: input, lastPresentationTime: &lastPresentationTime, lastDuration: &lastDuration) { + // The writer stopped accepting data mid-fill. The silence already + // committed is recorded in lastPresentationTime/lastDuration, so drop + // this buffer and let the next one resume the fill from there; the + // track never carries an unfilled hole that the muxer could compact. + droppedAudioBufferCount += 1 + return + } + } + + // The fill may have just saturated the input, and appending to an input + // that is not ready raises an uncatchable Objective-C exception. + guard input.isReadyForMoreMediaData else { + droppedAudioBufferCount += 1 + return + } + // presentationTime is already relative to the video's first frame // (computed by adjustedPresentationTime), so use it directly. let timing = CMSampleTimingInfo(duration: sampleBuffer.duration, presentationTimeStamp: presentationTime, decodeTimeStamp: sampleBuffer.decodeTimeStamp) - if let retimedSampleBuffer = try? CMSampleBuffer(copying: sampleBuffer, withNewTiming: [timing]) { - let appended = input.append(retimedSampleBuffer) - if appended { - lastPresentationTime = presentationTime - if input === inlineAudioInput { - lastInlineAudioDuration = sampleBuffer.duration - } + guard let retimedSampleBuffer = try? CMSampleBuffer(copying: sampleBuffer, withNewTiming: [timing]), + input.append(retimedSampleBuffer) else { + // A failed retime or a rejected append loses this buffer as well; count + // it so AUDIO_GAPS reflects every buffer missing from the track. The + // hole is filled by the next accepted buffer like any other drop. + droppedAudioBufferCount += 1 + return + } + lastPresentationTime = presentationTime + lastDuration = sampleBuffer.duration + } + + /// Appends zeroed LPCM covering [start, end) in the same format as `sampleBuffer`, + /// so buffers lost to back-pressure or late delivery leave a silent hole instead + /// of shifting every later sample earlier. + /// Returns false only when the writer stopped accepting data before the + /// (capped) fill completed; an unfillable format is treated as complete. + /// `lastPresentationTime`/`lastDuration` advance to every silence chunk that + /// was actually accepted, so a retry resumes where the fill stopped instead + /// of re-appending earlier timestamps (which would fail the writer). + @discardableResult + private func appendSilence(matching sampleBuffer: CMSampleBuffer, from start: CMTime, to end: CMTime, into input: AVAssetWriterInput, lastPresentationTime: inout CMTime, lastDuration: inout CMTime) -> Bool { + guard let formatDescription = sampleBuffer.formatDescription, + let asbd = formatDescription.audioStreamBasicDescription else { return true } + let sampleRate = asbd.mSampleRate + guard sampleRate > 0, asbd.mBytesPerFrame > 0 else { return true } + let gapSeconds = CMTimeGetSeconds(end - start) + guard gapSeconds.isFinite, gapSeconds > 0 else { return true } + + // Cap a single hole so a stalled device cannot balloon the file. + let totalFrames = Int(min(gapSeconds, 10.0) * sampleRate) + let isNonInterleaved = (asbd.mFormatFlags & kAudioFormatFlagIsNonInterleaved) != 0 + let bytesPerFrameAllChannels = Int(asbd.mBytesPerFrame) * (isNonInterleaved ? Int(max(1, asbd.mChannelsPerFrame)) : 1) + let chunkFrames = 4096 + var framesWritten = 0 + + while framesWritten < totalFrames, input.isReadyForMoreMediaData { + let frames = min(chunkFrames, totalFrames - framesWritten) + let byteCount = frames * bytesPerFrameAllChannels + var blockBuffer: CMBlockBuffer? + guard CMBlockBufferCreateWithMemoryBlock( + allocator: kCFAllocatorDefault, + memoryBlock: nil, + blockLength: byteCount, + blockAllocator: kCFAllocatorDefault, + customBlockSource: nil, + offsetToData: 0, + dataLength: byteCount, + flags: 0, + blockBufferOut: &blockBuffer) == kCMBlockBufferNoErr, + let blockBuffer, + CMBlockBufferFillDataBytes(with: 0, blockBuffer: blockBuffer, offsetIntoDestination: 0, dataLength: byteCount) == kCMBlockBufferNoErr else { + return true + } + + let pts = start + CMTime(value: CMTimeValue(framesWritten), timescale: CMTimeScale(sampleRate)) + var silence: CMSampleBuffer? + guard CMAudioSampleBufferCreateReadyWithPacketDescriptions( + allocator: kCFAllocatorDefault, + dataBuffer: blockBuffer, + formatDescription: formatDescription, + sampleCount: frames, + presentationTimeStamp: pts, + packetDescriptions: nil, + sampleBufferOut: &silence) == noErr, + let silence else { + return true } + guard input.append(silence) else { + insertedSilenceFrames += Int64(framesWritten) + return false + } + lastPresentationTime = pts + lastDuration = CMTime(value: CMTimeValue(frames), timescale: CMTimeScale(sampleRate)) + framesWritten += frames } + + insertedSilenceFrames += Int64(framesWritten) + return framesWritten >= totalFrames } private static func audioOutputSettings(bitRate: Int) -> [String: Any] { diff --git a/electron/native/ScreenCaptureKitRecorder.test.ts b/electron/native/ScreenCaptureKitRecorder.test.ts index 54ae01b51..e991d60a8 100644 --- a/electron/native/ScreenCaptureKitRecorder.test.ts +++ b/electron/native/ScreenCaptureKitRecorder.test.ts @@ -85,6 +85,33 @@ describe("ScreenCaptureKitRecorder window capture", () => { }); }); +describe("ScreenCaptureKitRecorder audio continuity", () => { + it("delivers audio on a dedicated queue and hops onto the recorder queue", () => { + expect(recorderSource).toContain('DispatchQueue(label: "recordly.screencapturekit.audio")'); + expect(recorderSource).toContain("type: .audio, sampleHandlerQueue: audioQueue"); + expect(recorderSource).toContain( + "type: microphoneOutputType, sampleHandlerQueue: audioQueue", + ); + expect(recorderSource).toMatch(/if outputType != \.screen \{\s*queue\.async/); + expect(recorderSource).toContain("audioQueue.sync {}"); + }); + + it("fills audio timestamp gaps with silence instead of compacting the track", () => { + expect(recorderSource).toContain( + "appendSilence(matching: sampleBuffer, from: expectedNext, to: presentationTime, into: input, lastPresentationTime: &lastPresentationTime, lastDuration: &lastDuration)", + ); + expect(recorderSource).toContain("CMAudioSampleBufferCreateReadyWithPacketDescriptions("); + expect(recorderSource).toContain("lastDuration = sampleBuffer.duration"); + expect(recorderSource).toContain( + "lastDuration = CMTime(value: CMTimeValue(frames), timescale: CMTimeScale(sampleRate))", + ); + }); + + it("counts dropped audio buffers and reports gaps at finalization", () => { + expect(recorderSource).toContain("droppedAudioBufferCount += 1"); + expect(recorderSource).toContain("AUDIO_GAPS: droppedBuffers="); + }); +}); describe("ScreenCaptureKitRecorder first frame timing", () => { const callback = recorderSource.slice(recorderSource.indexOf("func stream(_ stream:"), recorderSource.indexOf("func stream(_ stream:") + 5000);