From 02e1f022473722e53fabc4f7e65a330eb8ab2754 Mon Sep 17 00:00:00 2001 From: orut34iop Date: Tue, 13 Oct 2026 17:56:40 +0800 Subject: [PATCH] fix(demux): repair variable-cadence H264 heads and hierarchical partial offsets --- CHANGELOG.md | 6 ++ .../test-h264-partial-composition-controls.sh | 14 ++++ ...4PartialCompositionRuntimeStandalone.swift | 36 +++++++- .../H264PartialCompositionStandalone.swift | 66 +++++++++++++++ Scripts/tests/make-missing-head-fixture.py | 77 +++++++++++++++++ .../Video/H264CompositionOffsetRepair.swift | 22 +++++ .../Video/H264PartialCompositionRepair.swift | 80 ++++++++++++++++-- .../H264PartialCompositionRepairSession.swift | 83 ++++++++++++++++--- .../H264CompositionOffsetRepairTests.swift | 34 ++++++++ docs/formats.md | 25 ++++-- 10 files changed, 412 insertions(+), 31 deletions(-) create mode 100644 Scripts/tests/make-missing-head-fixture.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 751e60c90..a118685f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,12 @@ the public-API contract. ### Fixed +- **H.264 MP4s with missing composition offsets keep display order across cadence changes and + hierarchical B pictures.** A corroborated missing-offset head with a changing decode cadence now + reuses the existing timestamp slots. Later zero-offset regions can prove a forward-coded anchor + farther ahead than the declared backward reorder delay, using a bounded complete-prefix check. + Original DTS, audio, payload and index remain unchanged; unsupported sequences still pass through. + - **A host with its own Objective-C and C++ links again (#752).** Since 7.34.0 LibTrueHD and LibDovi each brought a `rust_eh_personality`, and with the host's `__objc_personality_v0` and `__gxx_personality_v0` that made four personality routines, one more than compact unwind encodes diff --git a/Scripts/test-h264-partial-composition-controls.sh b/Scripts/test-h264-partial-composition-controls.sh index 9e2c82177..4c9bdf6b0 100644 --- a/Scripts/test-h264-partial-composition-controls.sh +++ b/Scripts/test-h264-partial-composition-controls.sh @@ -32,3 +32,17 @@ ffmpeg -hide_banner -loglevel error -n \ ruby "$TASK_ROOT/Scripts/tests/make-partial-ctts-fixture.rb" \ "$TASK_TMP/healthy-scenecut.mp4" "$TASK_TMP/variable.mp4" tail 300 1200 1200 bash "$TASK_ROOT/Scripts/test-h264-partial-composition-runtime.sh" "$TASK_TMP/variable.mp4" 0 12 0 + +# Entire missing-offset head with a decode cadence change, plus its untouched healthy control. +python3 "$TASK_ROOT/Scripts/tests/make-missing-head-fixture.py" "$TASK_TMP" +bash "$TASK_ROOT/Scripts/test-h264-partial-composition-runtime.sh" "$TASK_TMP/missing-head.mp4" 0 1.5 0 20 40 +bash "$TASK_ROOT/Scripts/test-h264-partial-composition-runtime.sh" "$TASK_TMP/healthy-head.mp4" 0 + +# Hierarchical B pictures: forward distance 3, declared backward depth 2. The healthy twin +# supplies an exact PTS oracle for both a damaged tail and a damaged middle with recovery. +python3 "$TASK_ROOT/Scripts/tests/make-missing-head-fixture.py" "$TASK_TMP" --partial-hierarchical +AETHER_TIMESTAMP_HEALTHY_TWIN="$TASK_TMP/healthy-b3.mp4" \ + bash "$TASK_ROOT/Scripts/test-h264-partial-composition-runtime.sh" "$TASK_TMP/tail-b3.mp4" 0 12 0 6 22 +AETHER_TIMESTAMP_HEALTHY_TWIN="$TASK_TMP/healthy-b3.mp4" \ + bash "$TASK_ROOT/Scripts/test-h264-partial-composition-runtime.sh" "$TASK_TMP/middle-b3.mp4" 0 12 0 6 18 22 +bash "$TASK_ROOT/Scripts/test-h264-partial-composition-runtime.sh" "$TASK_TMP/healthy-b3.mp4" 0 diff --git a/Scripts/tests/H264PartialCompositionRuntimeStandalone.swift b/Scripts/tests/H264PartialCompositionRuntimeStandalone.swift index c11ea29ed..bec0e4c3b 100644 --- a/Scripts/tests/H264PartialCompositionRuntimeStandalone.swift +++ b/Scripts/tests/H264PartialCompositionRuntimeStandalone.swift @@ -54,6 +54,7 @@ struct H264PartialCompositionRuntimeTests { enum Failure: Error { case open, decoder, demux, decode, session, boundedRead } static func main() throws { guard CommandLine.arguments.count >= 2 else { throw Failure.open } + EngineLog.handler = { FileHandle.standardError.write(Data(($0 + "\n").utf8)) } av_log_set_level(AV_LOG_QUIET) var format: UnsafeMutablePointer? guard avformat_open_input(&format, CommandLine.arguments[1], nil, nil) >= 0, let format else { throw Failure.open } @@ -62,6 +63,7 @@ struct H264PartialCompositionRuntimeTests { let index = av_find_best_stream(format, AVMEDIA_TYPE_VIDEO, -1, -1, nil, 0) guard index >= 0, let stream = format.pointee.streams[Int(index)], let par = stream.pointee.codecpar, par.pointee.codec_id == AV_CODEC_ID_H264 else { throw Failure.open } + let twinPTS = try ProcessInfo.processInfo.environment["AETHER_TIMESTAMP_HEALTHY_TWIN"].map(referenceTimes) let name = format.pointee.iformat.flatMap { $0.pointee.name }.map(String.init(cString:)) ?? "" let matroska = name.contains("matroska") guard !matroska else { throw Failure.session } @@ -96,6 +98,9 @@ struct H264PartialCompositionRuntimeTests { precondition(actual.duration == expected.duration && actual.flags == expected.flags) precondition(actual.position == expected.position && actual.streamIndex == expected.streamIndex) precondition(actual.dts == expected.dts, "partial repair must preserve the published decode/index axis") + if let twinPTS { + precondition(twinPTS[actual.dts] == actual.pts, "repaired video PTS must exactly match its healthy twin") + } try decode(fixed, packet: packet, into: &fixedPTS) } else { precondition(actual == expected) } delivered += 1 @@ -120,7 +125,8 @@ struct H264PartialCompositionRuntimeTests { try decode(raw, packet: packet, into: &rawPTS) } if !session.ingest(packet) { try emit(packet) } - observedRepair = observedRepair || session.summary.contains("confirmed_partial_composition_offsets") + observedRepair = observedRepair || (session.summary.contains("confirmed_partial_composition_offsets") || session.summary.contains("confirmed_missing_composition_offsets")) + } session.endOfStream() while let packet = session.dequeue() { try emit(packet) } @@ -130,7 +136,7 @@ struct H264PartialCompositionRuntimeTests { let rawRegressions = zip(rawPTS, rawPTS.dropFirst()).filter { $1 <= $0 }.count let fixedRegressions = zip(fixedPTS, fixedPTS.dropFirst()).filter { $1 <= $0 }.count let diagnostic = session.summary - observedRepair = observedRepair || diagnostic.contains("confirmed_partial_composition_offsets") + observedRepair = observedRepair || (diagnostic.contains("confirmed_partial_composition_offsets") || diagnostic.contains("confirmed_missing_composition_offsets")) FileHandle.standardError.write(Data("MEASURE seek=\(position) raw=\(rawRegressions) fixed=\(fixedRegressions) offset=\(session.decodeTimestampOffset ?? 0) state=\(diagnostic) lead=\(0) shift=\(0)\n".utf8)) precondition(rawPTS.count == fixedPTS.count && !fixedPTS.isEmpty) precondition(fixedRegressions == 0) @@ -145,6 +151,29 @@ struct H264PartialCompositionRuntimeTests { } } + /// Optional generated healthy twin checks exact timestamps, beyond monotonic decoded output. + static func referenceTimes(_ path: String) throws -> [Int64: Int64] { + var input: UnsafeMutablePointer? + guard avformat_open_input(&input, path, nil, nil) >= 0, let input else { throw Failure.open } + defer { var owned: UnsafeMutablePointer? = input; avformat_close_input(&owned) } + guard avformat_find_stream_info(input, nil) >= 0 else { throw Failure.open } + let index = av_find_best_stream(input, AVMEDIA_TYPE_VIDEO, -1, -1, nil, 0) + guard index >= 0 else { throw Failure.open } + var result: [Int64: Int64] = [:] + while true { + guard let packet = trackedPacketAlloc() else { throw Failure.demux } + defer { av_packet_free_safe(packet) } + let status = av_read_frame(input, packet) + if status == -541478725 { break } + guard status >= 0 else { throw Failure.demux } + if packet.pointee.stream_index == index { + precondition(result.count < 8000 && result[packet.pointee.dts] == nil) + result[packet.pointee.dts] = packet.pointee.pts + } + } + return result + } + static func lifecycleChecks(session: H264CompositionOffsetRepairSession, format: UnsafeMutablePointer, stream: UnsafeMutablePointer, index: Int32, position: Double) throws { @@ -172,7 +201,7 @@ struct H264PartialCompositionRuntimeTests { var confirmed = false for _ in 0..<4000 { try read() - if session.summary.contains("confirmed_partial_composition_offsets") { confirmed = true; break } + if (session.summary.contains("confirmed_partial_composition_offsets") || session.summary.contains("confirmed_missing_composition_offsets")) { confirmed = true; break } } precondition(confirmed) guard let packet = trackedPacketAlloc() else { throw Failure.demux } @@ -181,6 +210,7 @@ struct H264PartialCompositionRuntimeTests { packet.pointee.pts = malformed.pts if !session.ingest(packet) { av_packet_free_safe(packet) } while let packet = session.dequeue() { av_packet_free_safe(packet) } + precondition(session.summary.contains("partial_composition_sequence_unproven")) precondition(PacketBalanceTracker.alive == 0, "a refusal still owns every packet it took") // A refused sequence streams through the session rather than out of its queue, so // count both ways a packet can come back. diff --git a/Scripts/tests/H264PartialCompositionStandalone.swift b/Scripts/tests/H264PartialCompositionStandalone.swift index bb914f09f..f7088400c 100644 --- a/Scripts/tests/H264PartialCompositionStandalone.swift +++ b/Scripts/tests/H264PartialCompositionStandalone.swift @@ -32,7 +32,73 @@ struct H264PartialCompositionTests { return (times, deepest, !queue.isEmpty) } + static func missingHeadChecks() { + let ticks: [Int64] = [-48000, 0, 48000, 96000, 144000, 192000, 240000, 288000, 336000, 384000, 424040, 464080] + let order: [Int64] = [0, 6, 2, 4, 12, 8, 10, 18, 14, 16, 22, 20] + let keys = [true] + Array(repeating: false, count: 11) + func proof(_ dts: [Int64] = ticks, pts: [Int64]? = nil, pocs: [Int64] = order, + start: Int64 = 0, origin: Int64 = -48000, depth: Int = 1) -> Int? { + Policy.missingHeadForwardDepth(dts: dts, pts: pts ?? dts, pocs: pocs, keys: keys, + streamStart: start, ladderStart: origin, reorderDepth: depth) + } + precondition(proof() == 2) // Forward-coded P differs from backward reorder lead. + precondition(proof(start: -48000) == nil) + precondition(proof(start: 1) == nil) + precondition(proof(origin: 0) == nil) + precondition(proof(depth: 0) == nil) + precondition(proof(depth: 17) == nil) + precondition(proof(pocs: Array(stride(from: Int64(0), to: 24, by: 2))) == nil) + for bad: Int64 in [-1, 3, 6, 1000, Int64.max] { + var changed = order; changed[4] = bad + precondition(proof(pocs: changed) == nil) + } + var offset = ticks; offset[3] += 1 + precondition(proof(pts: offset) == nil) + var flat = ticks; flat[3] = flat[2] + precondition(proof(flat) == nil) + var missing = ticks; missing[3] = Int64.min + precondition(proof(missing) == nil) + precondition(proof(Array(ticks.prefix(11))) == nil) + print("PASS missing head: corroborated lead, variable slots, bounded forward reorder; invalid/ambiguous evidence refused") + } + + static func forwardProofChecks() { + let ranks = [0, 4, 2, 1, 3, 8, 6, 5, 7, 12, 10, 9] + let dts = (0..<12).map { Int64($0) * 3600 } + func proof(_ order: [Int] = ranks, slots: [Int64] = dts, lead: Int64 = 7200, + depth: Int = 2) -> Int? { + Policy.provenForwardDepth(dts: slots, ranks: order, lead: lead, reorderDepth: depth) + } + precondition(proof() == 3) + precondition(proof(slots: dts.map { $0 + 63_000_000 }) == 3) + precondition(proof(lead: 3600) == nil) + precondition(proof(depth: 1) == nil) + precondition(proof(Array(ranks.prefix(11)), slots: Array(dts.prefix(11))) == nil) + precondition(proof(Array(0..<12)) == nil) + for bad in [-1, 0, 1000, Int.max] { + var order = ranks; order[4] = bad + precondition(proof(order) == nil) + } + // Unique bounded ranks can still omit a display slot: no complete prefix, no proof. + precondition(proof([0, 5, 3, 2, 4, 9, 7, 6, 8, 13, 11, 10]) == nil) + var repeated = dts; repeated[4] = repeated[3] + precondition(proof(slots: repeated) == nil) + var missing = dts; missing[4] = Int64.min + precondition(proof(slots: missing) == nil) + precondition(proof(slots: dts.map { Int64.max - 40000 + $0 }) == nil) + var ladder = Policy.SlotLadder() + for value in dts { precondition(ladder.append(dts: value)) } + precondition(ladder.claim(rank: 4, decodeIndex: 1, lead: 7200, reorderDepth: 2) == .refuse) + precondition(ladder.refusalReason == "forward_wait_depth_exceeded") + precondition(ladder.claim(rank: 4, decodeIndex: 1, lead: 7200, reorderDepth: 2, forwardDepth: 3) == .time(21600)) + precondition(ladder.claim(rank: 4, decodeIndex: 1, lead: 7200, reorderDepth: 2, forwardDepth: 3) == .refuse) + precondition(ladder.refusalReason == "duplicate_display_rank") + print("PASS partial forward proof: complete prefix, asymmetric depth, exact slot, invalid evidence refused") + } + static func main() { + missingHeadChecks() + forwardProofChecks() let expected = pocs.map { slots[Int($0 / 2)] + lead } let plain = run() precondition(!plain.refused) diff --git a/Scripts/tests/make-missing-head-fixture.py b/Scripts/tests/make-missing-head-fixture.py new file mode 100644 index 000000000..a1546ee41 --- /dev/null +++ b/Scripts/tests/make-missing-head-fixture.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Generate a public variable-cadence H.264 twin, never consume real media. + +Run the timestamp runtime on missing-head.mp4 at 0 1.5 0 20 40, and on +healthy-head.mp4 at 0. The former must remove frame-PTS regressions while +preserving DTS/audio/payload; the latter must remain byte-for-byte unchanged. +This fixture proves display ordering, not recovery of an arbitrary writer's +lost original composition timestamps or physical A/V synchronization. +""" +import argparse +from pathlib import Path +import struct +import subprocess + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('output', type=Path) +parser.add_argument('--partial-hierarchical', action='store_true', help='Generate healthy/tail/middle B3 twins for the partial-region path') +args = parser.parse_args() +args.output.mkdir(parents=True, exist_ok=True) +if args.partial_hierarchical: + healthy = args.output / 'healthy-b3.mp4' + subprocess.run([ + 'ffmpeg', '-hide_banner', '-loglevel', 'error', '-n', + '-f', 'lavfi', '-i', 'color=c=blue:s=96x64:r=25:d=24', + '-f', 'lavfi', '-i', 'sine=frequency=440:sample_rate=48000:duration=24', + '-c:v', 'libx264', '-preset', 'fast', '-pix_fmt', 'yuv420p', '-g', '49', '-bf', '3', + '-x264-params', 'scenecut=0:b-adapt=0', '-c:a', 'aac', '-video_track_timescale', '90000', + '-metadata', 'comment=Aether synthetic timestamp fixture', '-movflags', '+faststart', str(healthy), + ], check=True) + # 49-picture GOPs align the IDR boundaries with ctts runs; mutate offsets only. + for mode in ('tail', 'middle'): + subprocess.run(['ruby', str(Path(__file__).with_name('make-partial-ctts-fixture.rb')), + str(healthy), str(args.output / (mode + '-b3.mp4')), mode, '196', '392', '600'], check=True) + raise SystemExit(0) + +healthy = args.output / 'healthy-head.mp4' +missing = args.output / 'missing-head.mp4' +if healthy.exists() or missing.exists(): + raise SystemExit('Refusing to replace an existing fixture') +subprocess.run([ + 'ffmpeg', '-hide_banner', '-loglevel', 'error', '-n', + '-f', 'lavfi', '-i', 'testsrc2=s=96x64:r=25:d=60', + '-f', 'lavfi', '-i', 'sine=frequency=440:sample_rate=48000:duration=60', + '-vf', "settb=1/1200000,setpts='if(lt(N,8),N*48000,384000+(N-8)*40040)'", + '-fps_mode', 'passthrough', '-enc_time_base', '1/1200000', + '-c:v', 'libx264', '-preset', 'fast', '-pix_fmt', 'yuv420p', + '-g', '60', '-bf', '2', '-x264-params', 'scenecut=0:b-adapt=0:b-pyramid=none', + '-c:a', 'aac', '-video_track_timescale', '1200000', + '-metadata', 'comment=Aether synthetic timestamp fixture', + '-movflags', '+faststart', str(healthy), +], check=True) +data = bytearray(healthy.read_bytes()) +changed = 0 + + +def zero_offsets(start, end): + global changed + while start < end: + size, kind = struct.unpack_from('>I4s', data, start) + assert size >= 8 and start + size <= end + if kind in (b'moov', b'trak', b'mdia', b'minf', b'stbl'): + zero_offsets(start + 8, start + size) + elif kind == b'ctts': + count = struct.unpack_from('>I', data, start + 12)[0] + assert size == 16 + 8 * count + for index in range(count): + offset = start + 16 + 8 * index + changed += struct.unpack_from('>I', data, offset)[0] + struct.pack_into('>I', data, offset + 4, 0) + start += size + + +zero_offsets(0, len(data)) +assert changed == 1500 +with missing.open('xb') as output: + output.write(data) +print('PASS generated missing-head twin: 1500 frames, retained edit/index/audio/payload') diff --git a/Sources/AetherEngine/Video/H264CompositionOffsetRepair.swift b/Sources/AetherEngine/Video/H264CompositionOffsetRepair.swift index f718bd9d2..1f0329007 100644 --- a/Sources/AetherEngine/Video/H264CompositionOffsetRepair.swift +++ b/Sources/AetherEngine/Video/H264CompositionOffsetRepair.swift @@ -622,6 +622,7 @@ final class H264CompositionOffsetRepairSession { private let partialRepairCandidate: H264PartialCompositionRepairSession? private var monitorsPartialOffsets = false private var samples: [H264CompositionOffsetRepair.Sample] = [] + private var sampledFramePictures = true private var held: [(packet: UnsafeMutablePointer, pictureOrderCount: Int64?)] = [] private var heldBytes = 0 private var verdictDescription = "sampling" @@ -701,6 +702,7 @@ final class H264CompositionOffsetRepairSession { var pictureOrderCount: Int64? if packet.pointee.stream_index == streamIndex { pictureOrderCount = reader?.pictureOrderCount(for: packet) + sampledFramePictures = sampledFramePictures && reader?.isFramePicture == true samples.append( H264CompositionOffsetRepair.Sample( dts: packet.pointee.dts, @@ -801,6 +803,7 @@ final class H264CompositionOffsetRepairSession { case .sampling: // The sample restarts where the source now stands. samples.removeAll(keepingCapacity: true) + sampledFramePictures = true case .repairing: rewriter?.noteSeek() reader?.reset() @@ -882,6 +885,25 @@ final class H264CompositionOffsetRepairSession { verdictDescription = "healthy" disarm() case .inconclusive(let reason): + if reason == "decode ladder is not uniform", sampledFramePictures, + let candidate = partialRepairCandidate, + let forwardDepth = H264PartialCompositionRepair.missingHeadForwardDepth( + dts: samples.map(\.dts), pts: samples.map(\.pts), + pocs: samples.map(\.pictureOrderCount), keys: samples.map(\.isKeyframe), + streamStart: streamStartTime, ladderStart: ladderStart, reorderDepth: videoDelay), + let firstPacket = held.first(where: { $0.packet.pointee.stream_index == streamIndex })?.packet, + candidate.enableMissingHead(forwardDepth: forwardDepth, firstPacket: firstPacket) { + // Reuse #513's streaming slots. Hand over each interleaved packet once; + // leading non-video packets remain ahead of the candidate's queue. + monitorsPartialOffsets = true + disarm() + let input = held + held.removeAll(keepingCapacity: false) + heldBytes = 0 + for entry in input where !candidate.ingest(entry.packet) { held.append(entry) } + samples.removeAll(keepingCapacity: false) + return + } verdictDescription = "inconclusive (\(reason))" // Only worth a line when the stream looked like a candidate: a file that simply carries // composition offsets is the normal case and says nothing. diff --git a/Sources/AetherEngine/Video/H264PartialCompositionRepair.swift b/Sources/AetherEngine/Video/H264PartialCompositionRepair.swift index 6f5f887c9..4ed6cb1e0 100644 --- a/Sources/AetherEngine/Video/H264PartialCompositionRepair.swift +++ b/Sources/AetherEngine/Video/H264PartialCompositionRepair.swift @@ -9,8 +9,8 @@ import Foundation /// slot waits exactly the mini-GOP the reorder created, so nothing is held for the length of a /// sequence and no sequence is too long to repair. enum H264PartialCompositionRepair { - /// A picture cannot sit further from its own slot than the reorder delay the container - /// declares. `video_delay` is the honest bound; this is only the ceiling on believing it. + /// Backward displacement must fit the declared reorder delay. Forward-coded anchors may + /// be further ahead (#700); a separate bounded prefix proof must establish that distance. static let maximumReorderDepth = 16 /// A sequence past this is not a GOP any more, and the ladder is one Int64 per picture. static let maximumPictures = 1 << 16 @@ -23,6 +23,59 @@ enum H264PartialCompositionRepair { /// that reorders at all shows it within its reorder delay; this is slack, not a policy. static let maximumProofPictures = 64 + /// A missing-offset head has no healthy packet to corroborate its lead. Require + /// the container start/index distance to equal the observed reorder head, then + /// dry-run the existing slots. Never fit an average rate to variable timestamps. + static func missingHeadForwardDepth( + dts: [Int64], pts: [Int64], pocs: [Int64], keys: [Bool], + streamStart: Int64, ladderStart: Int64, reorderDepth: Int + ) -> Int? { + guard dts.count >= 12, dts.count <= maximumProofPictures, + pts.count == dts.count, pocs.count == dts.count, keys.count == dts.count, + (1...maximumReorderDepth).contains(reorderDepth), reorderDepth < dts.count, + streamStart != Int64.min, ladderStart != Int64.min, + dts.first == ladderStart, keys.first == true, pocs.first == 0, + !keys.dropFirst().contains(true), pts == dts else { return nil } + let (lead, overflow) = streamStart.subtractingReportingOverflow(ladderStart) + let (observed, headOverflow) = dts[reorderDepth].subtractingReportingOverflow(ladderStart) + guard !overflow, !headOverflow, lead > 0, lead == observed else { return nil } + var ranks: [Int] = [] + for poc in pocs { + guard poc >= 0, poc % 2 == 0, poc / 2 < Int64(maximumPictures) else { return nil } + ranks.append(Int(poc / 2)) + } + return provenForwardDepth(dts: dts, ranks: ranks, lead: lead, reorderDepth: reorderDepth) + } + + /// Reuse #700's closed decode-prefix evidence for #513's original timestamp slots. + /// Uniqueness plus [0...prefix.count-1] coverage proves a complete mini-GOP prefix; + /// the declared backward depth and every observed slot/lead still have to agree. + static func provenForwardDepth(dts: [Int64], ranks: [Int], lead: Int64, + reorderDepth: Int) -> Int? { + guard (12...maximumProofPictures).contains(dts.count), ranks.count == dts.count, + ranks.first == 0, lead > 0, (1...maximumReorderDepth).contains(reorderDepth), + ranks.allSatisfy({ $0 >= 0 && $0 < maximumPictures }), + Set(ranks).count == ranks.count, + zip(ranks, ranks.dropFirst()).contains(where: { $1 < $0 }) else { return nil } + var ladder = SlotLadder() + for timestamp in dts { guard ladder.append(dts: timestamp) else { return nil } } + var high = 0 + var closedPrefix = false + var forward = reorderDepth + for (index, rank) in ranks.enumerated() { + high = max(high, rank) + if index >= 8, high == index { closedPrefix = true } + guard index - rank <= reorderDepth, rank - index <= maximumReorderDepth else { return nil } + forward = max(forward, rank - index) + } + guard closedPrefix else { return nil } + for (index, rank) in ranks.enumerated() { + if ladder.claim(rank: rank, decodeIndex: index, lead: lead, + reorderDepth: reorderDepth, forwardDepth: forward) == .refuse { return nil } + } + return forward + } + enum Claim: Equatable { /// The slot is still ahead of the reader. The picture waits, and so does everything /// behind it, because the container's own order is what leaves this demuxer. @@ -36,6 +89,7 @@ enum H264PartialCompositionRepair { struct SlotLadder { private var slots: [Int64] = [] private var claimed: [Bool] = [] + private(set) var refusalReason = "none" var count: Int { slots.count } var isEmpty: Bool { slots.isEmpty } @@ -52,14 +106,26 @@ enum H264PartialCompositionRepair { /// The presentation time the picture at `decodeIndex` owns, given its display `rank`. /// `.wait` while its slot is still ahead of the reader, which is the mini-GOP. - mutating func claim(rank: Int, decodeIndex: Int, lead: Int64, reorderDepth: Int) -> Claim { + mutating func claim(rank: Int, decodeIndex: Int, lead: Int64, reorderDepth: Int, + forwardDepth: Int? = nil) -> Claim { + refusalReason = "none" + let forward = forwardDepth ?? reorderDepth guard lead > 0, rank >= 0, decodeIndex >= 0, decodeIndex < slots.count, - abs(rank - decodeIndex) <= reorderDepth, rank < Self.limit else { return .refuse } + (1...H264PartialCompositionRepair.maximumReorderDepth).contains(forward), + rank < Self.limit else { refusalReason = "invalid_slot_claim"; return .refuse } + guard decodeIndex - rank <= reorderDepth else { + refusalReason = "backward_reorder_depth_exceeded"; return .refuse + } + guard rank - decodeIndex <= forward else { + refusalReason = "forward_wait_depth_exceeded"; return .refuse + } guard rank < slots.count else { return .wait } - // One slot, one picture. A rank claimed twice is a sequence this policy cannot own. - guard !claimed[rank] else { return .refuse } + guard !claimed[rank] else { refusalReason = "duplicate_display_rank"; return .refuse } let (pts, overflow) = slots[rank].addingReportingOverflow(lead) - guard !overflow, pts >= slots[decodeIndex] else { return .refuse } + guard !overflow else { refusalReason = "presentation_time_overflow"; return .refuse } + guard pts >= slots[decodeIndex] else { + refusalReason = "presentation_before_decode"; return .refuse + } claimed[rank] = true return .time(pts) } diff --git a/Sources/AetherEngine/Video/H264PartialCompositionRepairSession.swift b/Sources/AetherEngine/Video/H264PartialCompositionRepairSession.swift index b69941ec3..825d831be 100644 --- a/Sources/AetherEngine/Video/H264PartialCompositionRepairSession.swift +++ b/Sources/AetherEngine/Video/H264PartialCompositionRepairSession.swift @@ -24,6 +24,12 @@ final class H264PartialCompositionRepairSession { private let timeBase: AVRational private let lead: Int64 private let reorderDepth: Int + private var missingHead = false + private var forwardDepth: Int? + // Numeric evidence only, capped independently of interleaved packet/byte budgets. + private var proofDTS: [Int64] = [] + private var proofRanks: [Int] = [] + private var forwardProofs = 0 private let framing: VideoNALFraming private let reader: H264PictureOrderReader private var pending: [Entry] = [] @@ -55,7 +61,21 @@ final class H264PartialCompositionRepairSession { } deinit { releasePackets() } - var hasDecision: Bool { reason != "composition_offsets_present" } + var hasDecision: Bool { missingHead || reason != "composition_offsets_present" } + + /// Only the parent's head proof enables this. Healthy-head behavior is unchanged. + func enableMissingHead(forwardDepth: Int, firstPacket: UnsafeMutablePointer) -> Bool { + precondition(!inSequence && pending.isEmpty && ready.isEmpty) + guard isIDR(firstPacket) else { return false } + missingHead = true + self.forwardDepth = forwardDepth + reason = "missing_composition_offsets_pending" + return true + } + + private var confirmedReason: String { + missingHead ? "confirmed_missing_composition_offsets" : "confirmed_partial_composition_offsets" + } private func isIDR(_ packet: UnsafeMutablePointer) -> Bool { guard packet.pointee.flags & AV_PKT_FLAG_KEY != 0, @@ -96,16 +116,20 @@ final class H264PartialCompositionRepairSession { inSequence = true } if abandoned { return handBack(packet) } - guard !missing, ladder.append(dts: packet.pointee.dts) else { return failOpen(with: packet) } + guard !missing, ladder.append(dts: packet.pointee.dts) else { return failOpen(with: packet, cause: "missing_or_nonincreasing_dts") } let poc = reader.pictureOrderCount(for: packet) // Complete, closed, progressive sequences only. Fields, open-GOP leading pictures and a // parser that cannot read an order are not guessed at. guard let poc, reader.isFramePicture, poc >= 0, poc % 2 == 0, decodeIndex > 0 || poc == 0, packet.pointee.flags & AV_PKT_FLAG_KEY == 0 || (idr && poc == 0) - else { return failOpen(with: packet) } + else { return failOpen(with: packet, cause: "unsupported_picture_order") } let rank = Int(poc / 2) if rank != decodeIndex { sequenceReordered = true } + if !missingHead && proofRanks.count < H264PartialCompositionRepair.maximumProofPictures { + proofDTS.append(packet.pointee.dts) + proofRanks.append(rank) + } append(packet, rank: rank) decodeIndex += 1 drain() @@ -119,21 +143,43 @@ final class H264PartialCompositionRepairSession { guard confirmed || sequenceReordered else { return } while let first = pending.first { guard let rank = first.rank else { promoteFirst(); continue } - switch ladder.claim(rank: rank, decodeIndex: first.decodeIndex, lead: lead, reorderDepth: reorderDepth) { + if !missingHead, forwardDepth == nil, rank - first.decodeIndex > reorderDepth { + // Keep the existing fast path for ordinary mini-GOPs. Only an anchor beyond the + // old symmetric allowance waits for bounded, complete-prefix evidence. + if let depth = H264PartialCompositionRepair.provenForwardDepth( + dts: proofDTS, ranks: proofRanks, lead: lead, reorderDepth: reorderDepth), + rank - first.decodeIndex <= depth { + forwardDepth = depth + forwardProofs += 1 + if forwardProofs <= 8 { + EngineLog.emit("[Demuxer] H264 slot forward wait proven: rank=\(rank)" + + " decode_index=\(first.decodeIndex) samples=\(proofRanks.count)" + + " reorder_depth=\(reorderDepth) forward_depth=\(depth)", category: .demux) + } + } else if proofRanks.count < H264PartialCompositionRepair.maximumProofPictures { + deepestWait = max(deepestWait, pending.count) + return + } else { + failOpen(cause: "forward_prefix_unproven") + return + } + } + switch ladder.claim(rank: rank, decodeIndex: first.decodeIndex, lead: lead, + reorderDepth: reorderDepth, forwardDepth: forwardDepth) { case .wait: deepestWait = max(deepestWait, pending.count) return case .refuse: - failOpen() + failOpen(cause: ladder.refusalReason) return case .time(let pts): first.packet.pointee.pts = pts repairedCount += 1 if !confirmed { confirmed = true - reason = "confirmed_partial_composition_offsets" + reason = confirmedReason EngineLog.emit( - "[Demuxer] partial H264 composition offsets confirmed: reorder_depth=\(reorderDepth)" + "[Demuxer] H264 slot composition offsets confirmed: kind=\(confirmedReason) reorder_depth=\(reorderDepth)" + " presentation_lead=\(lead) decode_offset=0 time_base=\(timeBase.num)/\(timeBase.den)", category: .demux ) @@ -160,18 +206,25 @@ final class H264PartialCompositionRepairSession { if pending.count >= H264PartialCompositionRepair.maximumHeldPackets || bytes >= H264PartialCompositionRepair.maximumHeldBytes || (!confirmed && !sequenceReordered && decodeIndex > H264PartialCompositionRepair.maximumProofPictures) { - failOpen() + failOpen(cause: "packet_byte_or_proof_limit") } } /// Hands every held packet back untouched and lets the rest of this sequence stream through. /// The next IDR starts a new candidate; nothing about the session is given up. @discardableResult - private func failOpen(with packet: UnsafeMutablePointer? = nil) -> Bool { + private func failOpen(with packet: UnsafeMutablePointer? = nil, cause: String) -> Bool { if !abandoned, inSequence { abandoned = true abandonedSequences += 1 - if reason == "confirmed_partial_composition_offsets" { + // One bounded diagnostic per refused sequence; never log payloads or source URLs. + if abandonedSequences <= 8 { + EngineLog.emit("[Demuxer] H264 slot repair refused: cause=\(cause)" + + " rank=\(pending.first?.rank ?? -1) decode_index=\(pending.first?.decodeIndex ?? decodeIndex)" + + " samples=\(decodeIndex) reorder_depth=\(reorderDepth) forward_depth=\(forwardDepth ?? reorderDepth)" + + " presentation_lead=\(lead) held_packets=\(pending.count) held_bytes=\(bytes)", category: .demux) + } + if reason == confirmedReason { reason = "partial_composition_sequence_unproven_after_repair" } else { reason = "partial_composition_sequence_unproven" @@ -192,7 +245,7 @@ final class H264PartialCompositionRepairSession { private func closeSequence() { drain() // Anything still waiting is waiting for a slot this sequence never carried. - if !pending.isEmpty { failOpen() } + if !pending.isEmpty { failOpen(cause: "incomplete_sequence") } resetSequence() } @@ -202,6 +255,9 @@ final class H264PartialCompositionRepairSession { inSequence = false sequenceReordered = false abandoned = false + proofDTS.removeAll(keepingCapacity: true) + proofRanks.removeAll(keepingCapacity: true) + if !missingHead { forwardDepth = nil } } private func publishPending() { @@ -227,7 +283,7 @@ final class H264PartialCompositionRepairSession { reader.reset() resetSequence() confirmed = false - reason = "composition_offsets_present" + reason = missingHead ? "missing_composition_offsets_pending" : "composition_offsets_present" } private func releasePackets() { @@ -239,7 +295,8 @@ final class H264PartialCompositionRepairSession { var summary: String { "reason=\(reason) repaired=\(repairedCount) abandoned_sequences=\(abandonedSequences)" - + " deepest_wait=\(deepestWait) reorder_depth=\(reorderDepth)" + + " deepest_wait=\(deepestWait) reorder_depth=\(reorderDepth) forward_proofs=\(forwardProofs)" + " presentation_lead=\(lead) decode_offset=0" } + } diff --git a/Tests/AetherEngineTests/H264CompositionOffsetRepairTests.swift b/Tests/AetherEngineTests/H264CompositionOffsetRepairTests.swift index d01688edd..9612b7df7 100644 --- a/Tests/AetherEngineTests/H264CompositionOffsetRepairTests.swift +++ b/Tests/AetherEngineTests/H264CompositionOffsetRepairTests.swift @@ -68,6 +68,40 @@ struct H264CompositionOffsetRepairTests { #expect(verdict(samples) == .inconclusive("decode ladder is not uniform")) } + @Test("a hierarchical anchor proves forward distance separately from backward reorder depth") + func partialForwardPrefix() { + let ranks = [0, 4, 2, 1, 3, 8, 6, 5, 7, 12, 10, 9] + let dts = (0..<12).map { Int64($0) * 3600 } + #expect(H264PartialCompositionRepair.provenForwardDepth( + dts: dts, ranks: ranks, lead: 7200, reorderDepth: 2) == 3) + #expect(H264PartialCompositionRepair.provenForwardDepth( + dts: dts, ranks: ranks, lead: 3600, reorderDepth: 2) == nil) + #expect(H264PartialCompositionRepair.provenForwardDepth( + dts: dts, ranks: ranks, lead: 7200, reorderDepth: 1) == nil) + #expect(H264PartialCompositionRepair.provenForwardDepth( + dts: dts, ranks: [0, 5, 3, 2, 4, 9, 7, 6, 8, 13, 11, 10], + lead: 7200, reorderDepth: 2) == nil) + } + + @Test("a variable-cadence missing head requires an exactly corroborated container lead") + func variableCadenceHeadProof() { + let dts: [Int64] = [-96000, -48000, 0, 48000, 96000, 144000, + 192000, 240000, 288000, 328040, 368080, 408120] + let pocs: [Int64] = [0, 6, 2, 4, 12, 8, 10, 18, 14, 16, 24, 20] + let keys = [true] + Array(repeating: false, count: 11) + #expect(H264PartialCompositionRepair.missingHeadForwardDepth( + dts: dts, pts: dts, pocs: pocs, keys: keys, + streamStart: 0, ladderStart: -96000, reorderDepth: 2) == 2) + #expect(H264PartialCompositionRepair.missingHeadForwardDepth( + dts: dts, pts: dts, pocs: pocs, keys: keys, + streamStart: 1, ladderStart: -96000, reorderDepth: 2) == nil) + var healthy = dts + healthy[0] = 0 + #expect(H264PartialCompositionRepair.missingHeadForwardDepth( + dts: dts, pts: healthy, pocs: pocs, keys: keys, + streamStart: 0, ladderStart: -96000, reorderDepth: 2) == nil) + } + @Test("a sample that does not start on a picture-order origin cannot be anchored") func mustStartOnOrigin() { var samples = malformedSamples() diff --git a/docs/formats.md b/docs/formats.md index 9d8f1e5df..b154e40e6 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -57,10 +57,16 @@ zero). A source is only repaired when every sampled pair is equal, the decode la picture order regresses, and the ranks it produces are distinct and fill the sampled window (or a decode-order prefix of at least nine pictures fills its own range exactly, which covers a hierarchical mini-GOP longer than the reorder delay, #699). Anything -short of that (variable frame timing, a picture order that does not advance one rank per picture, a +short of that (a picture order that does not advance one rank per picture, a sample that starts nowhere it can be anchored) is delivered exactly as the container wrote it. Reported by @orut34iop; the HEVC case (#699) by @ijuniorfu. +For H.264 only, a changing decode cadence can instead use the original-slot policy described below. +This fallback requires a progressive IDR head with equal PTS/DTS, a container start/index lead that +exactly matches the observed reorder head, and a complete, distinct picture-order prefix whose +timestamps pass the slot policy. It assigns existing slots plus that lead rather than fitting a +frame rate; DTS, audio and the keyframe index remain unchanged. An ambiguous head stays untouched. + ### Matroska with presentation slots in coding order Matroska block timestamps are presentation timestamps by specification, and the format has no @@ -104,10 +110,9 @@ offsets. Seeking into that region can produce persistent judder despite normal aggregate FPS and sufficient network buffering. When a healthy origin picture corroborates the container's edit/index lead, the -demuxer keeps a zero-hold healthy path and watches for zero-offset IDRs. One -bounded, complete IDR-to-IDR progressive sequence is parsed for picture order. -If every selected packet has valid equal PTS/DTS and its distinct even POC fills -the complete sequence, a proven permutation assigns its original DTS slots plus +demuxer keeps a zero-hold healthy path and watches for zero-offset IDRs. Progressive +pictures are parsed as they arrive. Valid equal PTS/DTS and distinct even POC let +the streaming slot ladder assign original DTS slots plus the corroborated presentation lead by display rank. Original DTS, audio and the already published keyframe index never move. Actual timestamp slots, rather than an average-FPS clock, preserve interval changes and quantization within a sequence. @@ -115,8 +120,11 @@ an average-FPS clock, preserve interval changes and quantization within a sequen The slot a picture needs is a packet away, not a plan away, exactly as in the Matroska policy above: a picture coded ahead of its own slot waits the mini-GOP the reorder created, so nothing waits for the end of a sequence and no sequence is too long to -repair. The wait is bounded by the reorder delay the container declares, believed up -to **16 pictures**, and the packets held behind it by **1024 interleaved packets and +repair. Backward displacement remains bounded by the declared reorder delay. A forward-coded +anchor farther ahead needs a bounded, complete decode-prefix proof, as in #700: distinct ranks +fill a prefix, the backward delay still fits, and the observed slots plus lead pass a dry run. +Only the measured forward distance is then allowed, limited to **16 pictures**; that proof resets +at each candidate IDR. The packets held behind it remain bounded by **1024 interleaved packets and 32 MiB**, which is a ceiling on a container's interleaving rather than on its GOP. A candidate sequence has to show its reordering within **64 pictures** before a single picture is rewritten. @@ -127,7 +135,8 @@ not guessed at. **Every refusal hands the held packets back exactly as they arri and lets the rest of that sequence stream through, so a shape this policy does not own costs the repair and never the session; the next IDR is a fresh candidate. Seek and teardown release both unpublished input and pending output. No decoder-route or host-UI -change is needed. The existing whole-file missing-offset policy above remains separate. +change is needed. The uniform whole-file policy above is unchanged; its strictly proven +variable-cadence H.264 fallback reuses this same slot ladder. See the [partial-composition regression and reproduction](partial-composition-regression.md) for generated fixtures, original numeric evidence and verification limits.