diff --git a/electron/native/screencapturekit/Package.swift b/electron/native/screencapturekit/Package.swift index ec3b1d98..de66e1e1 100644 --- a/electron/native/screencapturekit/Package.swift +++ b/electron/native/screencapturekit/Package.swift @@ -25,6 +25,10 @@ let package = Package( .executableTarget( name: "OpenScreenMacOSCursorHelper", path: "Sources/OpenScreenMacOSCursorHelper" + ), + .testTarget( + name: "OpenScreenScreenCaptureKitHelperTests", + dependencies: ["OpenScreenScreenCaptureKitHelper"] ) ] ) diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/AudioTrackMixer.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/AudioTrackMixer.swift index e193fb67..1861d066 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/AudioTrackMixer.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/AudioTrackMixer.swift @@ -2,6 +2,46 @@ import AVFoundation import CoreMedia import Foundation +/// Rebases each ScreenCaptureKit audio output onto the writer's session start. +/// +/// Screen, system-audio and microphone outputs become live asynchronously. Their PTS values +/// share a clock, but the first buffer from an audio output can arrive well after the first +/// screen frame because that capture branch is still warming up. Carrying that one-time +/// startup offset into the file makes the entire track sound late. Once a source is live its +/// own PTS deltas are authoritative, so remove only a bounded first-buffer offset and preserve +/// every later gap, pause and drift correction. A source that takes longer than the normal +/// warm-up window keeps its original offset so a device failure cannot masquerade as sync. +@available(macOS 13.0, *) +struct AudioStartAlignment { + private static let maximumCompensatedStartupDelay = CMTime(value: 1, timescale: 4) + private var firstPresentationTimes: [CMTime?] + + init(sourceCount: Int) { + firstPresentationTimes = Array(repeating: nil, count: sourceCount) + } + + mutating func align( + _ presentationTime: CMTime, + forSourceAt index: Int, + to sessionStart: CMTime + ) -> CMTime? { + guard presentationTime.isValid, + sessionStart.isValid, + firstPresentationTimes.indices.contains(index) + else { + return nil + } + + let firstPresentationTime = firstPresentationTimes[index] ?? presentationTime + firstPresentationTimes[index] = firstPresentationTime + let startupDelay = CMTimeSubtract(firstPresentationTime, sessionStart) + let isExpectedWarmup = CMTimeCompare(startupDelay, .zero) >= 0 + && CMTimeCompare(startupDelay, Self.maximumCompensatedStartupDelay) <= 0 + let sourceOrigin = isExpectedWarmup ? firstPresentationTime : sessionStart + return CMTimeAdd(sessionStart, CMTimeSubtract(presentationTime, sourceOrigin)) + } +} + /// Sums system audio and the microphone into the single AAC track the helper muxes. /// /// The helper used to give AVAssetWriter one input per source, so a recording with both @@ -52,7 +92,8 @@ final class AudioTrackMixer { static let finalFlushTimeout = 5.0 } - private let input: AVAssetWriterInput + private let isOutputReady: () -> Bool + private let appendOutput: (CMSampleBuffer) -> Void private let includesSystemAudio: Bool private let includesMicrophone: Bool private let microphoneGain: Float @@ -60,6 +101,7 @@ final class AudioTrackMixer { private var sources = [SourceTimeline](repeating: SourceTimeline(), count: Source.allCases.count) private var sessionStart: CMTime? + private var startAlignment = AudioStartAlignment(sourceCount: Source.allCases.count) /// Timeline origin: frame 0 of the mixed track, in the writer's time domain. private var anchor: CMTime? /// Absolute frame index of the next chunk to emit. @@ -74,16 +116,42 @@ final class AudioTrackMixer { includesMicrophone: Bool, microphoneGain: Double ) { - self.input = input + self.isOutputReady = { input.isReadyForMoreMediaData } + self.appendOutput = { input.append($0) } + self.includesSystemAudio = includesSystemAudio + self.includesMicrophone = includesMicrophone + // The request carries MIC_GAIN_BOOST (1.4); Windows applies it unconditionally and so + // does this. A non-finite or negative value would poison every mixed sample. + self.microphoneGain = Self.sanitizeMicrophoneGain(microphoneGain) + self.outputFormatDescription = Self.makeOutputFormatDescription() + } + + /// Test seam for observing mixed PCM without putting an AVAssetWriter into a writing + /// session. Production always uses the AVAssetWriterInput initializer above. + init( + includesSystemAudio: Bool, + includesMicrophone: Bool, + microphoneGain: Double, + isOutputReady: @escaping () -> Bool, + appendOutput: @escaping (CMSampleBuffer) -> Void + ) { + self.isOutputReady = isOutputReady + self.appendOutput = appendOutput self.includesSystemAudio = includesSystemAudio self.includesMicrophone = includesMicrophone // The request carries MIC_GAIN_BOOST (1.4); Windows applies it unconditionally and so // does this. A non-finite or negative value would poison every mixed sample. - let sanitized = microphoneGain.isFinite ? max(0, microphoneGain) : 1 - self.microphoneGain = Float(sanitized) + self.microphoneGain = Self.sanitizeMicrophoneGain(microphoneGain) self.outputFormatDescription = Self.makeOutputFormatDescription() } + private static func sanitizeMicrophoneGain(_ gain: Double) -> Float { + guard gain.isFinite else { + return 1 + } + return Float(min(max(0, gain), Double(Float.greatestFiniteMagnitude))) + } + /// Anchors the mixer to the writer session. Audio delivered before this is dropped — the /// writer would reject anything ahead of its session start anyway. func beginTimeline(at sessionStart: CMTime) { @@ -92,6 +160,11 @@ final class AudioTrackMixer { } self.sessionStart = sessionStart + anchor = CMTimeConvertScale( + sessionStart, + timescale: CMTimeScale(MixFormat.sampleRate), + method: .roundHalfAwayFromZero + ) } func ingest(_ sampleBuffer: CMSampleBuffer, from source: Source) { @@ -99,9 +172,6 @@ final class AudioTrackMixer { return } let presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) - guard presentationTime.isValid else { - return - } guard let frames = decodeInterleavedStereo(sampleBuffer, gain: gain(for: source)), !frames.isEmpty else { @@ -110,20 +180,20 @@ final class AudioTrackMixer { warnAboutDecodeFailure(source, sampleBuffer) return } - - if anchor == nil { - anchor = CMTimeConvertScale( - CMTimeMaximum(presentationTime, sessionStart), - timescale: CMTimeScale(MixFormat.sampleRate), - method: .roundHalfAwayFromZero - ) + guard let alignedPresentationTime = startAlignment.align( + presentationTime, + forSourceAt: source.rawValue, + to: sessionStart + ) else { + return } + guard let anchor else { return } let offset = CMTimeConvertScale( - CMTimeSubtract(presentationTime, anchor), + CMTimeSubtract(alignedPresentationTime, anchor), timescale: CMTimeScale(MixFormat.sampleRate), method: .roundHalfAwayFromZero ) @@ -184,12 +254,30 @@ final class AudioTrackMixer { /// system-audio device that stops delivering — from blocking the whole track. private func drain(flushing: Bool) { while true { - let delivered = sources.indices.filter { sources[$0].hasDelivered } + let enabled = Source.allCases.filter(includes).map(\.rawValue) + let delivered = enabled.filter { sources[$0].hasDelivered } guard let furthest = delivered.map({ sources[$0].endFrame }).max(), furthest > cursor else { break } let chunkEnd = cursor + Int64(MixFormat.chunkFrames) + // Alignment rebases each enabled source's ordinary startup warm-up onto frame zero. + // Do not advance past that frame until every source has had a chance to contribute: + // otherwise a source that arrives second has its newly aligned opening samples + // discarded by SourceTimeline.dropFrames. If a device never delivers, reuse the + // existing bounded stall tolerance and continue with silence for that source. + let awaitingFirstBuffer = enabled.filter { + !sources[$0].hasDelivered && !sources[$0].isStalled + } + if !awaitingFirstBuffer.isEmpty && !flushing { + if furthest < chunkEnd + Int64(MixFormat.stallToleranceFrames) { + break + } + for index in awaitingFirstBuffer { + sources[index].isStalled = true + } + } + let live = delivered.filter { !sources[$0].isStalled } let complete = live.allSatisfy { sources[$0].endFrame >= chunkEnd } if !complete && !flushing { @@ -237,16 +325,16 @@ final class AudioTrackMixer { /// `append` is not advisory backpressure — it raises an NSException when the input is not /// ready — so every path here waits for readiness rather than pushing through it. private func flushPending(force: Bool) { - while !pending.isEmpty && input.isReadyForMoreMediaData { - input.append(pending.removeFirst()) + while !pending.isEmpty && isOutputReady() { + appendOutput(pending.removeFirst()) } if force { // Teardown: this is the tail's last chance, and the writer is still draining, so // give it a bounded moment instead of dropping audio the recording just captured. let deadline = Date().addingTimeInterval(MixFormat.finalFlushTimeout) while !pending.isEmpty { - if input.isReadyForMoreMediaData { - input.append(pending.removeFirst()) + if isOutputReady() { + appendOutput(pending.removeFirst()) continue } if Date() >= deadline { diff --git a/electron/native/screencapturekit/Tests/OpenScreenScreenCaptureKitHelperTests/AudioStartAlignmentTests.swift b/electron/native/screencapturekit/Tests/OpenScreenScreenCaptureKitHelperTests/AudioStartAlignmentTests.swift new file mode 100644 index 00000000..c8fe0dc9 --- /dev/null +++ b/electron/native/screencapturekit/Tests/OpenScreenScreenCaptureKitHelperTests/AudioStartAlignmentTests.swift @@ -0,0 +1,301 @@ +import AVFoundation +import CoreMedia +import XCTest +@testable import OpenScreenScreenCaptureKitHelper + +@available(macOS 13.0, *) +final class AudioStartAlignmentTests: XCTestCase { + func testMixerWaitsForLateEnabledSourceBeforeAdvancingFrameZero() throws { + var output = [CMSampleBuffer]() + let mixer = AudioTrackMixer( + includesSystemAudio: true, + includesMicrophone: true, + microphoneGain: 1, + isOutputReady: { true }, + appendOutput: { output.append($0) } + ) + let sessionStart = CMTime(seconds: 100, preferredTimescale: 48_000) + mixer.beginTimeline(at: sessionStart) + + mixer.ingest( + try makeFloatStereoBuffer(value: 0.1, frames: 480, at: 100.16), + from: .system + ) + XCTAssertTrue(output.isEmpty, "system audio must wait for the enabled microphone") + + mixer.ingest( + try makeFloatStereoBuffer(value: 0.2, frames: 480, at: 100.21), + from: .microphone + ) + + let firstChunk = try XCTUnwrap(output.first) + XCTAssertEqual(CMTimeCompare(CMSampleBufferGetPresentationTimeStamp(firstChunk), sessionStart), 0) + XCTAssertEqual(try firstInt16Sample(in: firstChunk), 9_830, accuracy: 2) + } + + func testMixerFallsBackWhenEnabledSourceNeverDelivers() throws { + var output = [CMSampleBuffer]() + let mixer = AudioTrackMixer( + includesSystemAudio: true, + includesMicrophone: true, + microphoneGain: 1, + isOutputReady: { true }, + appendOutput: { output.append($0) } + ) + mixer.beginTimeline(at: CMTime(seconds: 100, preferredTimescale: 48_000)) + + mixer.ingest( + try makeFloatStereoBuffer(value: 0.1, frames: 12_480, at: 100.16), + from: .system + ) + + XCTAssertFalse(output.isEmpty, "an absent microphone must not stall recording forever") + XCTAssertEqual(try firstInt16Sample(in: XCTUnwrap(output.first)), 3_277, accuracy: 2) + } + + func testMixerClampsFiniteGainBeforeFloatConversion() throws { + var output = [CMSampleBuffer]() + let mixer = AudioTrackMixer( + includesSystemAudio: false, + includesMicrophone: true, + microphoneGain: Double.greatestFiniteMagnitude, + isOutputReady: { true }, + appendOutput: { output.append($0) } + ) + let sessionStart = CMTime(seconds: 100, preferredTimescale: 48_000) + mixer.beginTimeline(at: sessionStart) + + mixer.ingest( + try makeFloatStereoBuffer(value: 0, frames: 480, at: 100), + from: .microphone + ) + mixer.ingest( + try makeFloatStereoBuffer(value: 0.5, frames: 480, at: 100.01), + from: .microphone + ) + + XCTAssertEqual(output.count, 2) + XCTAssertEqual(try firstInt16Sample(in: output[0]), 0) + XCTAssertEqual(try firstInt16Sample(in: output[1]), 32_767) + } + + func testRemovesOneTimeCaptureStartupDelay() throws { + var alignment = AudioStartAlignment(sourceCount: 2) + let sessionStart = CMTime(seconds: 100, preferredTimescale: 48_000) + let firstAudio = CMTime(seconds: 100.16, preferredTimescale: 48_000) + let secondAudio = CMTime(seconds: 100.17, preferredTimescale: 48_000) + + let alignedFirst = try XCTUnwrap( + alignment.align(firstAudio, forSourceAt: 0, to: sessionStart) + ) + let alignedSecond = try XCTUnwrap( + alignment.align(secondAudio, forSourceAt: 0, to: sessionStart) + ) + + XCTAssertEqual(CMTimeCompare(alignedFirst, sessionStart), 0) + XCTAssertEqual( + CMTimeGetSeconds(CMTimeSubtract(alignedSecond, alignedFirst)), + 0.01, + accuracy: 0.000_001 + ) + } + + func testAlignsIndependentAudioOutputsWithoutDiscardingTheirDeltas() throws { + var alignment = AudioStartAlignment(sourceCount: 2) + let sessionStart = CMTime(seconds: 50, preferredTimescale: 48_000) + let systemFirst = CMTime(seconds: 50.12, preferredTimescale: 48_000) + let microphoneFirst = CMTime(seconds: 50.21, preferredTimescale: 48_000) + let microphoneLater = CMTime(seconds: 50.71, preferredTimescale: 48_000) + + let alignedSystem = try XCTUnwrap( + alignment.align(systemFirst, forSourceAt: 0, to: sessionStart) + ) + let alignedMicrophone = try XCTUnwrap( + alignment.align(microphoneFirst, forSourceAt: 1, to: sessionStart) + ) + let alignedMicrophoneLater = try XCTUnwrap( + alignment.align(microphoneLater, forSourceAt: 1, to: sessionStart) + ) + + XCTAssertEqual(CMTimeCompare(alignedSystem, sessionStart), 0) + XCTAssertEqual(CMTimeCompare(alignedMicrophone, sessionStart), 0) + XCTAssertEqual( + CMTimeGetSeconds(CMTimeSubtract(alignedMicrophoneLater, alignedMicrophone)), + 0.5, + accuracy: 0.000_001 + ) + } + + func testRejectsInvalidSourceIndex() { + var alignment = AudioStartAlignment(sourceCount: 1) + let time = CMTime(seconds: 1, preferredTimescale: 48_000) + + XCTAssertNil(alignment.align(time, forSourceAt: 1, to: time)) + } + + func testPreservesDelayOutsideTheCaptureWarmupWindow() throws { + var alignment = AudioStartAlignment(sourceCount: 1) + let sessionStart = CMTime(seconds: 10, preferredTimescale: 48_000) + let firstAudio = CMTime(seconds: 10.5, preferredTimescale: 48_000) + let secondAudio = CMTime(seconds: 10.6, preferredTimescale: 48_000) + + let alignedFirst = try XCTUnwrap( + alignment.align(firstAudio, forSourceAt: 0, to: sessionStart) + ) + let alignedSecond = try XCTUnwrap( + alignment.align(secondAudio, forSourceAt: 0, to: sessionStart) + ) + + XCTAssertEqual(CMTimeCompare(alignedFirst, firstAudio), 0) + XCTAssertEqual(CMTimeCompare(alignedSecond, secondAudio), 0) + } + + func testPreservesZeroStartupDelayAndLaterDeltas() throws { + var alignment = AudioStartAlignment(sourceCount: 1) + let sessionStart = CMTime(seconds: 10, preferredTimescale: 48_000) + let laterAudio = CMTime(seconds: 10.1, preferredTimescale: 48_000) + + let alignedFirst = try XCTUnwrap( + alignment.align(sessionStart, forSourceAt: 0, to: sessionStart) + ) + let alignedLater = try XCTUnwrap( + alignment.align(laterAudio, forSourceAt: 0, to: sessionStart) + ) + + XCTAssertEqual(CMTimeCompare(alignedFirst, sessionStart), 0) + XCTAssertEqual(CMTimeCompare(alignedLater, laterAudio), 0) + } + + func testPreservesNegativeStartupDelay() throws { + var alignment = AudioStartAlignment(sourceCount: 1) + let sessionStart = CMTime(seconds: 10, preferredTimescale: 48_000) + let firstAudio = CMTime(seconds: 9.9, preferredTimescale: 48_000) + + let alignedFirst = try XCTUnwrap( + alignment.align(firstAudio, forSourceAt: 0, to: sessionStart) + ) + + XCTAssertEqual(CMTimeCompare(alignedFirst, firstAudio), 0) + } + + func testCompensatesStartupDelayAtWarmupBoundary() throws { + var alignment = AudioStartAlignment(sourceCount: 1) + let sessionStart = CMTime(seconds: 10, preferredTimescale: 48_000) + let firstAudio = CMTime(seconds: 10.25, preferredTimescale: 48_000) + let laterAudio = CMTime(seconds: 10.35, preferredTimescale: 48_000) + + let alignedFirst = try XCTUnwrap( + alignment.align(firstAudio, forSourceAt: 0, to: sessionStart) + ) + let alignedLater = try XCTUnwrap( + alignment.align(laterAudio, forSourceAt: 0, to: sessionStart) + ) + + XCTAssertEqual(CMTimeCompare(alignedFirst, sessionStart), 0) + XCTAssertEqual( + CMTimeGetSeconds(CMTimeSubtract(alignedLater, alignedFirst)), + 0.1, + accuracy: 0.000_001 + ) + } + + private func makeFloatStereoBuffer( + value: Float, + frames: Int, + at seconds: Double + ) throws -> CMSampleBuffer { + var format = AudioStreamBasicDescription( + mSampleRate: 48_000, + mFormatID: kAudioFormatLinearPCM, + mFormatFlags: kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked, + mBytesPerPacket: 8, + mFramesPerPacket: 1, + mBytesPerFrame: 8, + mChannelsPerFrame: 2, + mBitsPerChannel: 32, + mReserved: 0 + ) + var description: CMAudioFormatDescription? + XCTAssertEqual( + CMAudioFormatDescriptionCreate( + allocator: kCFAllocatorDefault, + asbd: &format, + layoutSize: 0, + layout: nil, + magicCookieSize: 0, + magicCookie: nil, + extensions: nil, + formatDescriptionOut: &description + ), + noErr + ) + + let samples = [Float](repeating: value, count: frames * 2) + let byteCount = samples.count * MemoryLayout.size + var blockBuffer: CMBlockBuffer? + XCTAssertEqual( + CMBlockBufferCreateWithMemoryBlock( + allocator: kCFAllocatorDefault, + memoryBlock: nil, + blockLength: byteCount, + blockAllocator: kCFAllocatorDefault, + customBlockSource: nil, + offsetToData: 0, + dataLength: byteCount, + flags: kCMBlockBufferAssureMemoryNowFlag, + blockBufferOut: &blockBuffer + ), + kCMBlockBufferNoErr + ) + let resolvedBlockBuffer = try XCTUnwrap(blockBuffer) + XCTAssertEqual( + samples.withUnsafeBytes { bytes in + CMBlockBufferReplaceDataBytes( + with: bytes.baseAddress!, + blockBuffer: resolvedBlockBuffer, + offsetIntoDestination: 0, + dataLength: byteCount + ) + }, + kCMBlockBufferNoErr + ) + + var timing = CMSampleTimingInfo( + duration: CMTime(value: 1, timescale: 48_000), + presentationTimeStamp: CMTime(seconds: seconds, preferredTimescale: 48_000), + decodeTimeStamp: .invalid + ) + var sampleSize = 8 + var sampleBuffer: CMSampleBuffer? + XCTAssertEqual( + CMSampleBufferCreateReady( + allocator: kCFAllocatorDefault, + dataBuffer: resolvedBlockBuffer, + formatDescription: try XCTUnwrap(description), + sampleCount: frames, + sampleTimingEntryCount: 1, + sampleTimingArray: &timing, + sampleSizeEntryCount: 1, + sampleSizeArray: &sampleSize, + sampleBufferOut: &sampleBuffer + ), + noErr + ) + return try XCTUnwrap(sampleBuffer) + } + + private func firstInt16Sample(in sampleBuffer: CMSampleBuffer) throws -> Int16 { + let blockBuffer = try XCTUnwrap(CMSampleBufferGetDataBuffer(sampleBuffer)) + var value: Int16 = 0 + XCTAssertEqual( + CMBlockBufferCopyDataBytes( + blockBuffer, + atOffset: 0, + dataLength: MemoryLayout.size, + destination: &value + ), + kCMBlockBufferNoErr + ) + return value + } +}