fix(demux): repair variable-cadence H264 heads and hierarchical partial offsets

CHANGELOG.md

@@ -21,6 +21,12 @@ the public-API contract.
 
 ### Fixed
 
+- **H.264 MP4s with missing composition offsets keep display order across cadence changes and
+  hierarchical B pictures.** A corroborated missing-offset head with a changing decode cadence now
+  reuses the existing timestamp slots. Later zero-offset regions can prove a forward-coded anchor
+  farther ahead than the declared backward reorder delay, using a bounded complete-prefix check.
+  Original DTS, audio, payload and index remain unchanged; unsupported sequences still pass through.
+
 - **A host with its own Objective-C and C++ links again (#752).** Since 7.34.0 LibTrueHD and LibDovi
   each brought a `rust_eh_personality`, and with the host's `__objc_personality_v0` and
   `__gxx_personality_v0` that made four personality routines, one more than compact unwind encodes

Scripts/test-h264-partial-composition-controls.sh

@@ -32,3 +32,17 @@ ffmpeg -hide_banner -loglevel error -n \
 ruby "$TASK_ROOT/Scripts/tests/make-partial-ctts-fixture.rb" \
   "$TASK_TMP/healthy-scenecut.mp4" "$TASK_TMP/variable.mp4" tail 300 1200 1200
 bash "$TASK_ROOT/Scripts/test-h264-partial-composition-runtime.sh" "$TASK_TMP/variable.mp4" 0 12 0
+
+# Entire missing-offset head with a decode cadence change, plus its untouched healthy control.
+python3 "$TASK_ROOT/Scripts/tests/make-missing-head-fixture.py" "$TASK_TMP"
+bash "$TASK_ROOT/Scripts/test-h264-partial-composition-runtime.sh" "$TASK_TMP/missing-head.mp4" 0 1.5 0 20 40
+bash "$TASK_ROOT/Scripts/test-h264-partial-composition-runtime.sh" "$TASK_TMP/healthy-head.mp4" 0
+
+# Hierarchical B pictures: forward distance 3, declared backward depth 2. The healthy twin
+# supplies an exact PTS oracle for both a damaged tail and a damaged middle with recovery.
+python3 "$TASK_ROOT/Scripts/tests/make-missing-head-fixture.py" "$TASK_TMP" --partial-hierarchical
+AETHER_TIMESTAMP_HEALTHY_TWIN="$TASK_TMP/healthy-b3.mp4" \
+  bash "$TASK_ROOT/Scripts/test-h264-partial-composition-runtime.sh" "$TASK_TMP/tail-b3.mp4" 0 12 0 6 22
+AETHER_TIMESTAMP_HEALTHY_TWIN="$TASK_TMP/healthy-b3.mp4" \
+  bash "$TASK_ROOT/Scripts/test-h264-partial-composition-runtime.sh" "$TASK_TMP/middle-b3.mp4" 0 12 0 6 18 22
+bash "$TASK_ROOT/Scripts/test-h264-partial-composition-runtime.sh" "$TASK_TMP/healthy-b3.mp4" 0

Scripts/tests/H264PartialCompositionRuntimeStandalone.swift

@@ -54,6 +54,7 @@ struct H264PartialCompositionRuntimeTests {
     enum Failure: Error { case open, decoder, demux, decode, session, boundedRead }
     static func main() throws {
         guard CommandLine.arguments.count >= 2 else { throw Failure.open }
+        EngineLog.handler = { FileHandle.standardError.write(Data(($0 + "\n").utf8)) }
         av_log_set_level(AV_LOG_QUIET)
         var format: UnsafeMutablePointer<AVFormatContext>?
         guard avformat_open_input(&format, CommandLine.arguments[1], nil, nil) >= 0, let format else { throw Failure.open }
@@ -62,6 +63,7 @@ struct H264PartialCompositionRuntimeTests {
         let index = av_find_best_stream(format, AVMEDIA_TYPE_VIDEO, -1, -1, nil, 0)
         guard index >= 0, let stream = format.pointee.streams[Int(index)],
               let par = stream.pointee.codecpar, par.pointee.codec_id == AV_CODEC_ID_H264 else { throw Failure.open }
+        let twinPTS = try ProcessInfo.processInfo.environment["AETHER_TIMESTAMP_HEALTHY_TWIN"].map(referenceTimes)
         let name = format.pointee.iformat.flatMap { $0.pointee.name }.map(String.init(cString:)) ?? ""
         let matroska = name.contains("matroska")
         guard !matroska else { throw Failure.session }
@@ -96,6 +98,9 @@ struct H264PartialCompositionRuntimeTests {
                     precondition(actual.duration == expected.duration && actual.flags == expected.flags)
                     precondition(actual.position == expected.position && actual.streamIndex == expected.streamIndex)
                     precondition(actual.dts == expected.dts, "partial repair must preserve the published decode/index axis")
+                    if let twinPTS {
+                        precondition(twinPTS[actual.dts] == actual.pts, "repaired video PTS must exactly match its healthy twin")
+                    }
                     try decode(fixed, packet: packet, into: &fixedPTS)
                 } else { precondition(actual == expected) }
                 delivered += 1
@@ -120,7 +125,8 @@ struct H264PartialCompositionRuntimeTests {
                     try decode(raw, packet: packet, into: &rawPTS)
                 }
                 if !session.ingest(packet) { try emit(packet) }
-                observedRepair = observedRepair || session.summary.contains("confirmed_partial_composition_offsets")
+                observedRepair = observedRepair || (session.summary.contains("confirmed_partial_composition_offsets") || session.summary.contains("confirmed_missing_composition_offsets"))
+
             }
             session.endOfStream()
             while let packet = session.dequeue() { try emit(packet) }
@@ -130,7 +136,7 @@ struct H264PartialCompositionRuntimeTests {
             let rawRegressions = zip(rawPTS, rawPTS.dropFirst()).filter { $1 <= $0 }.count
             let fixedRegressions = zip(fixedPTS, fixedPTS.dropFirst()).filter { $1 <= $0 }.count
             let diagnostic = session.summary
-            observedRepair = observedRepair || diagnostic.contains("confirmed_partial_composition_offsets")
+            observedRepair = observedRepair || (diagnostic.contains("confirmed_partial_composition_offsets") || diagnostic.contains("confirmed_missing_composition_offsets"))
             FileHandle.standardError.write(Data("MEASURE seek=\(position) raw=\(rawRegressions) fixed=\(fixedRegressions) offset=\(session.decodeTimestampOffset ?? 0) state=\(diagnostic) lead=\(0) shift=\(0)\n".utf8))
             precondition(rawPTS.count == fixedPTS.count && !fixedPTS.isEmpty)
             precondition(fixedRegressions == 0)
@@ -145,6 +151,29 @@ struct H264PartialCompositionRuntimeTests {
         }
     }
 
+    /// Optional generated healthy twin checks exact timestamps, beyond monotonic decoded output.
+    static func referenceTimes(_ path: String) throws -> [Int64: Int64] {
+        var input: UnsafeMutablePointer<AVFormatContext>?
+        guard avformat_open_input(&input, path, nil, nil) >= 0, let input else { throw Failure.open }
+        defer { var owned: UnsafeMutablePointer<AVFormatContext>? = input; avformat_close_input(&owned) }
+        guard avformat_find_stream_info(input, nil) >= 0 else { throw Failure.open }
+        let index = av_find_best_stream(input, AVMEDIA_TYPE_VIDEO, -1, -1, nil, 0)
+        guard index >= 0 else { throw Failure.open }
+        var result: [Int64: Int64] = [:]
+        while true {
+            guard let packet = trackedPacketAlloc() else { throw Failure.demux }
+            defer { av_packet_free_safe(packet) }
+            let status = av_read_frame(input, packet)
+            if status == -541478725 { break }
+            guard status >= 0 else { throw Failure.demux }
+            if packet.pointee.stream_index == index {
+                precondition(result.count < 8000 && result[packet.pointee.dts] == nil)
+                result[packet.pointee.dts] = packet.pointee.pts
+            }
+        }
+        return result
+    }
+
     static func lifecycleChecks(session: H264CompositionOffsetRepairSession,
         format: UnsafeMutablePointer<AVFormatContext>, stream: UnsafeMutablePointer<AVStream>, index: Int32,
         position: Double) throws {
@@ -172,7 +201,7 @@ struct H264PartialCompositionRuntimeTests {
             var confirmed = false
             for _ in 0..<4000 {
                 try read()
-                if session.summary.contains("confirmed_partial_composition_offsets") { confirmed = true; break }
+                if (session.summary.contains("confirmed_partial_composition_offsets") || session.summary.contains("confirmed_missing_composition_offsets")) { confirmed = true; break }
             }
             precondition(confirmed)
             guard let packet = trackedPacketAlloc() else { throw Failure.demux }
@@ -181,6 +210,7 @@ struct H264PartialCompositionRuntimeTests {
             packet.pointee.pts = malformed.pts
             if !session.ingest(packet) { av_packet_free_safe(packet) }
             while let packet = session.dequeue() { av_packet_free_safe(packet) }
+            precondition(session.summary.contains("partial_composition_sequence_unproven"))
             precondition(PacketBalanceTracker.alive == 0, "a refusal still owns every packet it took")
             // A refused sequence streams through the session rather than out of its queue, so
             // count both ways a packet can come back.

Scripts/tests/H264PartialCompositionStandalone.swift

@@ -32,7 +32,73 @@ struct H264PartialCompositionTests {
         return (times, deepest, !queue.isEmpty)
     }
 
+    static func missingHeadChecks() {
+        let ticks: [Int64] = [-48000, 0, 48000, 96000, 144000, 192000, 240000, 288000, 336000, 384000, 424040, 464080]
+        let order: [Int64] = [0, 6, 2, 4, 12, 8, 10, 18, 14, 16, 22, 20]
+        let keys = [true] + Array(repeating: false, count: 11)
+        func proof(_ dts: [Int64] = ticks, pts: [Int64]? = nil, pocs: [Int64] = order,
+                   start: Int64 = 0, origin: Int64 = -48000, depth: Int = 1) -> Int? {
+            Policy.missingHeadForwardDepth(dts: dts, pts: pts ?? dts, pocs: pocs, keys: keys,
+                streamStart: start, ladderStart: origin, reorderDepth: depth)
+        }
+        precondition(proof() == 2) // Forward-coded P differs from backward reorder lead.
+        precondition(proof(start: -48000) == nil)
+        precondition(proof(start: 1) == nil)
+        precondition(proof(origin: 0) == nil)
+        precondition(proof(depth: 0) == nil)
+        precondition(proof(depth: 17) == nil)
+        precondition(proof(pocs: Array(stride(from: Int64(0), to: 24, by: 2))) == nil)
+        for bad: Int64 in [-1, 3, 6, 1000, Int64.max] {
+            var changed = order; changed[4] = bad
+            precondition(proof(pocs: changed) == nil)
+        }
+        var offset = ticks; offset[3] += 1
+        precondition(proof(pts: offset) == nil)
+        var flat = ticks; flat[3] = flat[2]
+        precondition(proof(flat) == nil)
+        var missing = ticks; missing[3] = Int64.min
+        precondition(proof(missing) == nil)
+        precondition(proof(Array(ticks.prefix(11))) == nil)
+        print("PASS missing head: corroborated lead, variable slots, bounded forward reorder; invalid/ambiguous evidence refused")
+    }
+
+    static func forwardProofChecks() {
+        let ranks = [0, 4, 2, 1, 3, 8, 6, 5, 7, 12, 10, 9]
+        let dts = (0..<12).map { Int64($0) * 3600 }
+        func proof(_ order: [Int] = ranks, slots: [Int64] = dts, lead: Int64 = 7200,
+                   depth: Int = 2) -> Int? {
+            Policy.provenForwardDepth(dts: slots, ranks: order, lead: lead, reorderDepth: depth)
+        }
+        precondition(proof() == 3)
+        precondition(proof(slots: dts.map { $0 + 63_000_000 }) == 3)
+        precondition(proof(lead: 3600) == nil)
+        precondition(proof(depth: 1) == nil)
+        precondition(proof(Array(ranks.prefix(11)), slots: Array(dts.prefix(11))) == nil)
+        precondition(proof(Array(0..<12)) == nil)
+        for bad in [-1, 0, 1000, Int.max] {
+            var order = ranks; order[4] = bad
+            precondition(proof(order) == nil)
+        }
+        // Unique bounded ranks can still omit a display slot: no complete prefix, no proof.
+        precondition(proof([0, 5, 3, 2, 4, 9, 7, 6, 8, 13, 11, 10]) == nil)
+        var repeated = dts; repeated[4] = repeated[3]
+        precondition(proof(slots: repeated) == nil)
+        var missing = dts; missing[4] = Int64.min
+        precondition(proof(slots: missing) == nil)
+        precondition(proof(slots: dts.map { Int64.max - 40000 + $0 }) == nil)
+        var ladder = Policy.SlotLadder()
+        for value in dts { precondition(ladder.append(dts: value)) }
+        precondition(ladder.claim(rank: 4, decodeIndex: 1, lead: 7200, reorderDepth: 2) == .refuse)
+        precondition(ladder.refusalReason == "forward_wait_depth_exceeded")
+        precondition(ladder.claim(rank: 4, decodeIndex: 1, lead: 7200, reorderDepth: 2, forwardDepth: 3) == .time(21600))
+        precondition(ladder.claim(rank: 4, decodeIndex: 1, lead: 7200, reorderDepth: 2, forwardDepth: 3) == .refuse)
+        precondition(ladder.refusalReason == "duplicate_display_rank")
+        print("PASS partial forward proof: complete prefix, asymmetric depth, exact slot, invalid evidence refused")
+    }
+
     static func main() {
+        missingHeadChecks()
+        forwardProofChecks()
         let expected = pocs.map { slots[Int($0 / 2)] + lead }
         let plain = run()
         precondition(!plain.refused)

Scripts/tests/make-missing-head-fixture.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Generate a public variable-cadence H.264 twin, never consume real media.
+
+Run the timestamp runtime on missing-head.mp4 at 0 1.5 0 20 40, and on
+healthy-head.mp4 at 0. The former must remove frame-PTS regressions while
+preserving DTS/audio/payload; the latter must remain byte-for-byte unchanged.
+This fixture proves display ordering, not recovery of an arbitrary writer's
+lost original composition timestamps or physical A/V synchronization.
+"""
+import argparse
+from pathlib import Path
+import struct
+import subprocess
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument('output', type=Path)
+parser.add_argument('--partial-hierarchical', action='store_true', help='Generate healthy/tail/middle B3 twins for the partial-region path')
+args = parser.parse_args()
+args.output.mkdir(parents=True, exist_ok=True)
+if args.partial_hierarchical:
+    healthy = args.output / 'healthy-b3.mp4'
+    subprocess.run([
+        'ffmpeg', '-hide_banner', '-loglevel', 'error', '-n',
+        '-f', 'lavfi', '-i', 'color=c=blue:s=96x64:r=25:d=24',
+        '-f', 'lavfi', '-i', 'sine=frequency=440:sample_rate=48000:duration=24',
+        '-c:v', 'libx264', '-preset', 'fast', '-pix_fmt', 'yuv420p', '-g', '49', '-bf', '3',
+        '-x264-params', 'scenecut=0:b-adapt=0', '-c:a', 'aac', '-video_track_timescale', '90000',
+        '-metadata', 'comment=Aether synthetic timestamp fixture', '-movflags', '+faststart', str(healthy),
+    ], check=True)
+    # 49-picture GOPs align the IDR boundaries with ctts runs; mutate offsets only.
+    for mode in ('tail', 'middle'):
+        subprocess.run(['ruby', str(Path(__file__).with_name('make-partial-ctts-fixture.rb')),
+                        str(healthy), str(args.output / (mode + '-b3.mp4')), mode, '196', '392', '600'], check=True)
+    raise SystemExit(0)
+
+healthy = args.output / 'healthy-head.mp4'
+missing = args.output / 'missing-head.mp4'
+if healthy.exists() or missing.exists():
+    raise SystemExit('Refusing to replace an existing fixture')
+subprocess.run([
+    'ffmpeg', '-hide_banner', '-loglevel', 'error', '-n',
+    '-f', 'lavfi', '-i', 'testsrc2=s=96x64:r=25:d=60',
+    '-f', 'lavfi', '-i', 'sine=frequency=440:sample_rate=48000:duration=60',
+    '-vf', "settb=1/1200000,setpts='if(lt(N,8),N*48000,384000+(N-8)*40040)'",
+    '-fps_mode', 'passthrough', '-enc_time_base', '1/1200000',
+    '-c:v', 'libx264', '-preset', 'fast', '-pix_fmt', 'yuv420p',
+    '-g', '60', '-bf', '2', '-x264-params', 'scenecut=0:b-adapt=0:b-pyramid=none',
+    '-c:a', 'aac', '-video_track_timescale', '1200000',
+    '-metadata', 'comment=Aether synthetic timestamp fixture',
+    '-movflags', '+faststart', str(healthy),
+], check=True)
+data = bytearray(healthy.read_bytes())
+changed = 0
+
+
+def zero_offsets(start, end):
+    global changed
+    while start < end:
+        size, kind = struct.unpack_from('>I4s', data, start)
+        assert size >= 8 and start + size <= end
+        if kind in (b'moov', b'trak', b'mdia', b'minf', b'stbl'):
+            zero_offsets(start + 8, start + size)
+        elif kind == b'ctts':
+            count = struct.unpack_from('>I', data, start + 12)[0]
+            assert size == 16 + 8 * count
+            for index in range(count):
+                offset = start + 16 + 8 * index
+                changed += struct.unpack_from('>I', data, offset)[0]
+                struct.pack_into('>I', data, offset + 4, 0)
+        start += size
+
+
+zero_offsets(0, len(data))
+assert changed == 1500
+with missing.open('xb') as output:
+    output.write(data)
+print('PASS generated missing-head twin: 1500 frames, retained edit/index/audio/payload')

Sources/AetherEngine/Video/H264CompositionOffsetRepair.swift

@@ -622,6 +622,7 @@ final class H264CompositionOffsetRepairSession {
     private let partialRepairCandidate: H264PartialCompositionRepairSession?
     private var monitorsPartialOffsets = false
     private var samples: [H264CompositionOffsetRepair.Sample] = []
+    private var sampledFramePictures = true
     private var held: [(packet: UnsafeMutablePointer<AVPacket>, pictureOrderCount: Int64?)] = []
     private var heldBytes = 0
     private var verdictDescription = "sampling"
@@ -701,6 +702,7 @@ final class H264CompositionOffsetRepairSession {
             var pictureOrderCount: Int64?
             if packet.pointee.stream_index == streamIndex {
                 pictureOrderCount = reader?.pictureOrderCount(for: packet)
+                sampledFramePictures = sampledFramePictures && reader?.isFramePicture == true
                 samples.append(
                     H264CompositionOffsetRepair.Sample(
                         dts: packet.pointee.dts,
@@ -801,6 +803,7 @@ final class H264CompositionOffsetRepairSession {
         case .sampling:
             // The sample restarts where the source now stands.
             samples.removeAll(keepingCapacity: true)
+            sampledFramePictures = true
         case .repairing:
             rewriter?.noteSeek()
             reader?.reset()
@@ -882,6 +885,25 @@ final class H264CompositionOffsetRepairSession {
             verdictDescription = "healthy"
             disarm()
         case .inconclusive(let reason):
+            if reason == "decode ladder is not uniform", sampledFramePictures,
+               let candidate = partialRepairCandidate,
+               let forwardDepth = H264PartialCompositionRepair.missingHeadForwardDepth(
+                   dts: samples.map(\.dts), pts: samples.map(\.pts),
+                   pocs: samples.map(\.pictureOrderCount), keys: samples.map(\.isKeyframe),
+                   streamStart: streamStartTime, ladderStart: ladderStart, reorderDepth: videoDelay),
+               let firstPacket = held.first(where: { $0.packet.pointee.stream_index == streamIndex })?.packet,
+               candidate.enableMissingHead(forwardDepth: forwardDepth, firstPacket: firstPacket) {
+                // Reuse #513's streaming slots. Hand over each interleaved packet once;
+                // leading non-video packets remain ahead of the candidate's queue.
+                monitorsPartialOffsets = true
+                disarm()
+                let input = held
+                held.removeAll(keepingCapacity: false)
+                heldBytes = 0
+                for entry in input where !candidate.ingest(entry.packet) { held.append(entry) }
+                samples.removeAll(keepingCapacity: false)
+                return
+            }
             verdictDescription = "inconclusive (\(reason))"
             // Only worth a line when the stream looked like a candidate: a file that simply carries
             // composition offsets is the normal case and says nothing.