From ac6b5f7a7cd18927b0bef1ba691fba0135e8eff0 Mon Sep 17 00:00:00 2001 From: Andre Encarnacao Date: Tue, 13 Oct 2026 09:58:50 +0200 Subject: [PATCH] fix(playback): report the software host's actual video decoder --- CHANGELOG.md | 3 + .../AetherEngine/AetherEngine+Loading.swift | 3 +- Sources/AetherEngine/AetherEngine+Probe.swift | 3 +- Sources/AetherEngine/AetherEngine.swift | 3 +- .../Native/SoftwarePlaybackHost.swift | 4 ++ .../ActiveVideoDecoderTests.swift | 65 +++++++++++++++++++ docs/api.md | 2 +- 7 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 Tests/AetherEngineTests/ActiveVideoDecoderTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 356566aef..0bbf26499 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ the public-API contract. ### Fixed +- **`activeVideoDecoder` names the decoder used by the software host (#753).** HEVC decoded by + VideoToolbox now reports `VideoToolbox HEVC (HW)` on initial load and reload, instead of + `libavcodec HEVC (SW)`. Decoder selection and playback routing are unchanged. - **A live GOP longer than the sealed target duration no longer breaks the playlist.** RFC 8216 keeps `TARGETDURATION` fixed for a playlist's life, so a later segment above it made the playlist invalid for AVPlayer (-12642). The engine now retires that playlist URL with its original target and an diff --git a/Sources/AetherEngine/AetherEngine+Loading.swift b/Sources/AetherEngine/AetherEngine+Loading.swift index 077b2ced6..eaa33f1ae 100644 --- a/Sources/AetherEngine/AetherEngine+Loading.swift +++ b/Sources/AetherEngine/AetherEngine+Loading.swift @@ -2562,7 +2562,8 @@ extension AetherEngine { playbackBackend = .software activeAudioTrackIndex = audioStreamIndex.map { Int($0) } activeVideoDecoder = Self.videoDecoderLabel( - codecID: preservedVideoCodec, isSoftware: true + codecID: preservedVideoCodec, + isSoftware: !(softwareHost?.usesHardwareVideoDecoder ?? false) ) // AE#462: the rebuilt host's own resolved index (see the load site). activeAudioDecoder = Self.softwareAudioDecoderLabel( diff --git a/Sources/AetherEngine/AetherEngine+Probe.swift b/Sources/AetherEngine/AetherEngine+Probe.swift index ecadadc6f..4a2a279a5 100644 --- a/Sources/AetherEngine/AetherEngine+Probe.swift +++ b/Sources/AetherEngine/AetherEngine+Probe.swift @@ -804,7 +804,8 @@ extension AetherEngine { // MARK: - Decoder identity helpers - /// User-facing label for the active video decoder. nil when no video track (AV_CODEC_ID_NONE). Native = VideoToolbox HW; SW = dav1d (AV1) or libavcodec (VP9, MPEG-2, VC-1). + /// User-facing label for the active video decoder. nil when no video track (AV_CODEC_ID_NONE). + /// isSoftware describes the decoder, not the playback path: the software host can use VideoToolbox. static func videoDecoderLabel(codecID: AVCodecID, isSoftware: Bool) -> String? { guard codecID != AV_CODEC_ID_NONE else { return nil } let name: String = { diff --git a/Sources/AetherEngine/AetherEngine.swift b/Sources/AetherEngine/AetherEngine.swift index 5a5629242..39463ec3b 100644 --- a/Sources/AetherEngine/AetherEngine.swift +++ b/Sources/AetherEngine/AetherEngine.swift @@ -5262,7 +5262,8 @@ public final class AetherEngine: ObservableObject { ) } activeVideoDecoder = Self.videoDecoderLabel( - codecID: detectedCodecID, isSoftware: true + codecID: detectedCodecID, + isSoftware: !(softwareHost?.usesHardwareVideoDecoder ?? false) ) // AE#462: the host's own resolved index, not the engine's pick. The pick says which // track was ASKED for; only the host knows whether a decoder opened for it, and this diff --git a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift index 9817a8f6e..2c92ad60f 100644 --- a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift +++ b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift @@ -111,6 +111,10 @@ final class SoftwarePlaybackHost { /// point it uses for `AVPlayerLayer`. var displayLayer: AVSampleBufferDisplayLayer { renderer.displayLayer } + /// Whether load selected VideoToolbox rather than libavcodec / dav1d for video decoding. + /// Only meaningful after load has returned successfully. + var usesHardwareVideoDecoder: Bool { videoDecoder is HardwareVideoDecoder } + /// AE#711 follow-up: the picture on screen now, read back from the renderer before a rebuild's /// `stop()` flushes it, so the engine can hold it over the new host's startup. Falls back to the /// newest frame enqueued, which a layer that is not on screen is all there is of. diff --git a/Tests/AetherEngineTests/ActiveVideoDecoderTests.swift b/Tests/AetherEngineTests/ActiveVideoDecoderTests.swift new file mode 100644 index 000000000..8c17146a4 --- /dev/null +++ b/Tests/AetherEngineTests/ActiveVideoDecoderTests.swift @@ -0,0 +1,65 @@ +import Foundation +import Testing +import VideoToolbox +@testable import AetherEngine + +/// AE#753: the software playback host can use VideoToolbox. Exercise the public label after +/// both initial load and a session rebuild, rather than testing the label formatter alone. +/// These synthetic clips are local-only; generate them with Scripts/fetch-fixtures.sh. +@Suite("Active video decoder identity", .serialized) +@MainActor +struct ActiveVideoDecoderTests { + private nonisolated static func fixture(_ name: String) -> URL { + URL(fileURLWithPath: #filePath) + .deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent() + .appendingPathComponent("Fixtures").appendingPathComponent(name) + } + + private nonisolated static func hasFixture(_ name: String) -> Bool { + FileManager.default.fileExists(atPath: fixture(name).path) + } + + private func checkLoadAndReload(_ name: String, label: String) async throws { + let engine = try AetherEngine() + defer { engine.stop() } + var options = LoadOptions() + options.preferredDecodePath = .software + options.suppressDisplayCriteria = true + // A seekable custom source makes reloadAtCurrentPosition exercise the separate + // reloadWithAudioOverride path, rather than calling the initial URL load again. + let reader = DataIOReader(data: try Data(contentsOf: Self.fixture(name))) + _ = try await engine.load(source: .custom(reader), options: options) + engine.pause() + #expect(engine.playbackBackend == .software) + #expect(engine.activeVideoDecoder == label, "initial load must name the actual decoder") + + _ = try await engine.reloadAtCurrentPosition() + #expect(engine.playbackBackend == .software) + #expect(engine.activeVideoDecoder == label, "reload must name the newly opened decoder") + } + + @Test("HEVC decoded by VideoToolbox on the software path (#753)", + .enabled(if: hasFixture("hdr10-hevc.mp4") && VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)), + .timeLimit(.minutes(1))) + func hardwareHEVC() async throws { + try await checkLoadAndReload("hdr10-hevc.mp4", label: "VideoToolbox HEVC (HW)") + } + + @Test("HEVC with in-band parameter sets still reports libavcodec", + .enabled(if: hasFixture("hev1-inband-xps.mp4")), .timeLimit(.minutes(1))) + func softwareHEVC() async throws { + try await checkLoadAndReload("hev1-inband-xps.mp4", label: "libavcodec HEVC (SW)") + } + + @Test("VP9 still reports libavcodec", + .enabled(if: hasFixture("vp9.webm")), .timeLimit(.minutes(1))) + func softwareVP9() async throws { + try await checkLoadAndReload("vp9.webm", label: "libavcodec VP9 (SW)") + } + + @Test("AV1 still reports dav1d", + .enabled(if: hasFixture("av1.mp4")), .timeLimit(.minutes(1))) + func softwareAV1() async throws { + try await checkLoadAndReload("av1.mp4", label: "dav1d AV1 (SW)") + } +} diff --git a/docs/api.md b/docs/api.md index c4d62d9ce..6503bc7d2 100644 --- a/docs/api.md +++ b/docs/api.md @@ -682,7 +682,7 @@ suppressing `AVPlayerItemLegibleOutput` to keep the measurement running. | `$sourceContainerFormat` | The container libavformat opened ("matroska,webm", "mpegts"), nil on the remote-HLS bypass, where AVFoundation opens the source and there is no libav context to ask. This is the container that ARRIVED, which on a remux or transcode session is not the one a host's library metadata describes. | | `$sourceVideoStreamFormat` | A `VideoStreamFormat?` (AE#658): the source video stream's `pixelFormat` ("yuv420p10le"), `bitDepth`, `colorPrimaries`, `transfer`, `matrix`, `range` and `profile` ("Main 10"), in libav's names as the container and the probe's decoder declared them. A field the stream leaves unspecified is nil, never a guessed BT.709. `colorPrimariesLabel`, `transferLabel`, `matrixLabel` and `rangeLabel` give the names a viewer reads ("BT.2020", "PQ (SMPTE ST 2084)", "BT.2020 NCL", "Limited"). nil before load and without video. On the probe-free remote-HLS bypass it is the DELIVERED stream, read back from AVPlayer's item video track once it resolves: colour and range from the format description, `profile`, `bitDepth` and `pixelFormat` from its avcC / hvcC record (nil where the codec has none of those, AV1 and VP9 included). Under a server-side transcode that is the transcode, not the library's file, which is the point of reading it there. | | `$decodedVideoFormat` | A `DecodedVideoFormat?` (AE#658): what the engine's own decoder actually produced. `frame` is a `VideoStreamFormat` for the decoded picture (libavcodec's output pixel format, or the libav name of the VideoToolbox buffer on the software host's hardware decoder), `pixelBufferFormat` the CoreVideo buffer it was displayed from (`420v`, `x420`; `pixelBufferLabel` reads "P010 (x420)"). Republished when either changes mid-stream. **nil on every native route**: AVPlayer decodes there and its frames never pass through the engine, so a panel shows `$sourceVideoStreamFormat` and names AVPlayer as the decoder instead of inferring a pixel format it never saw. | -| `$activeVideoDecoder`, `$activeAudioDecoder` | The decoder names actually in use, for a stats overlay. This is the honest "what is decoding this" surface; `playbackBackend` is not. Show `$activeAudioDecoder`, classify on `$audioDelivery`: a label cannot separate a source without audio from a source whose audio was dropped. | +| `$activeVideoDecoder`, `$activeAudioDecoder` | The decoder names actually in use, for a stats overlay. This is the honest "what is decoding this" surface; `playbackBackend` is not. The software playback host can use VideoToolbox for HEVC, in which case `$activeVideoDecoder` reports `VideoToolbox HEVC (HW)` on load and reload even though `playbackBackend` is `.software`. Show `$activeAudioDecoder`, classify on `$audioDelivery`: a label cannot separate a source without audio from a source whose audio was dropped. | | `$metadata` | `MediaMetadata` parsed at load (title / artist / album / cover). | | `$mediaChapters`, `$discChapters`, `$discTitles`, `$selectedDiscTitle` | Container chapters, and disc titles / chapters for DVD and Blu-ray ISO sources. | | `$currentAVPlayer`, `$currentAVPlayerItem` | The live AVFoundation objects, re-emitted on every reload. Both nil on `.software`, which renders into its own layer. A host that only ever hands `currentAVPlayer` to an `AVPlayerViewController` gets audio over an empty video plane on that route (#298). |