From 4872ea494a607a0744d952bfd5197f1b358320cb Mon Sep 17 00:00:00 2001 From: orut34iop Date: Tue, 25 Aug 2026 19:40:44 +0800 Subject: [PATCH 1/2] fix(video): recover periodic rational H264 offsets --- CHANGELOG.md | 13 +- Sources/AetherEngine/Demuxer/Demuxer.swift | 12 +- .../Video/H264CompositionOffsetRepair.swift | 641 ++++++++++++++++-- .../H264CompositionOffsetRepairTests.swift | 513 ++++++++++++++ 4 files changed, 1129 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae720e8d..a9619306 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,18 @@ the public-API contract. ## [Unreleased] -_Nothing yet._ +### Fixed + +- **The H.264 composition-offset repair still left a periodically quantized constant-rate ladder in + decode order (AE#409 follow-up).** A physical tvOS source reports `PTS == DTS`, `video_delay=1` + and a `1/1200000` time base, but its exact `200202/5`-tick cadence is stored as the repeating + five-picture step cycle `40040,40041,40040,40040,40041`. The first repair required every decode + step to be identical, classified that ladder as nonuniform and repaired no packets. The demuxer + now derives an exact rational cadence and unique sampling phase from two repeated STTS periods; + declared and average frame rates only corroborate the observation and never construct timestamps. + A complete dry run must safely place the sampled packets before repair is armed, while ambiguous, + non-repeating and genuinely variable timing remains untouched. Packet and container-index mapping + share the same rational lattice across IDRs and seeks, including bounded one-tick resynchronization. ## [6.42.0] - 2026-08-25 diff --git a/Sources/AetherEngine/Demuxer/Demuxer.swift b/Sources/AetherEngine/Demuxer/Demuxer.swift index 7dfcc8b6..436f944b 100644 --- a/Sources/AetherEngine/Demuxer/Demuxer.swift +++ b/Sources/AetherEngine/Demuxer/Demuxer.swift @@ -1059,7 +1059,7 @@ public final class Demuxer: @unchecked Sendable { func indexedKeyframes(streamIndex: Int32) -> [Int64] { accessLock.lock() defer { accessLock.unlock() } - let compositionOffset = compositionRepair?.decodeTimestampOffset + let activeCompositionRepair = compositionRepair?.isRepairing == true ? compositionRepair : nil guard let ctx = formatContext, streamIndex >= 0, streamIndex < Int32(ctx.pointee.nb_streams), @@ -1080,7 +1080,15 @@ public final class Demuxer: @unchecked Sendable { // built from these IRAP positions matches the normalized packets (AE#105), then onto // the repaired decode ladder if #409 moved it. let folded = normalizedTimestamp(entry.pointee.timestamp, pos: entry.pointee.pos, timeBase: tb) - result.append(compositionOffset.map { folded &+ $0 } ?? folded) + if let activeCompositionRepair { + // An unplaceable index entry is safer omitted: mixing one raw timestamp into a + // repaired rational ladder can cut a segment one picture away from its keyframe. + if let repaired = activeCompositionRepair.repairedDecodeTimestamp(folded) { + result.append(repaired) + } + } else { + result.append(folded) + } } } return result diff --git a/Sources/AetherEngine/Video/H264CompositionOffsetRepair.swift b/Sources/AetherEngine/Video/H264CompositionOffsetRepair.swift index a81f74cd..7b9f5890 100644 --- a/Sources/AetherEngine/Video/H264CompositionOffsetRepair.swift +++ b/Sources/AetherEngine/Video/H264CompositionOffsetRepair.swift @@ -18,14 +18,15 @@ import Libavutil /// /// The rewrite reproduces what the muxer should have written: /// -/// PTS = (DTS of the picture that opened this coded video sequence) + shift + displayIndex * step -/// DTS = DTS + shift - videoDelay * step +/// raw(i) = firstDTS + round((phase + i) * cadence) - round(phase * cadence) +/// PTS = presentation(sequenceBaseOrdinal + displayIndex) +/// DTS = presentation(decodeOrdinal - videoDelay) /// -/// Both forms are expressed relative to the packet's own timestamps, never to a counter, so a demuxer -/// that starts mid-file (a resume seek) and one that starts at byte 0 produce the same axis for the -/// same picture. Pulling DTS back by the decode lead is what keeps `PTS >= DTS`, the invariant the -/// fMP4 muxer and its output sanitizer enforce; a healthy file carries exactly the same negative head -/// (the twin's first packet is `pts=0 dts=-2002`), so this is the shape the pipeline already handles. +/// The first packet or seek landing is placed exactly on the sampled rational lattice; decode order +/// then advances its ordinal continuously, so one late one-tick container anomaly cannot mix a raw +/// timestamp back into the repaired axis. Pulling DTS back by the decode lead is what keeps +/// `PTS >= DTS`, the invariant the fMP4 muxer and its output sanitizer enforce; a healthy file carries +/// exactly the same negative head (the twin's first packet is `pts=0 dts=-2002`). /// /// Verified against three fixture pairs (432 packets, both edit-list shapes, 7 IDR boundaries): every /// repaired packet matches its healthy twin's PTS and DTS exactly. @@ -39,6 +40,99 @@ enum H264CompositionOffsetRepair { var isKeyframe: Bool } + /// A constant frame cadence expressed exactly in stream-timebase ticks. Integer timestamp + /// ladders quantize this rational with FFmpeg's nearest/away-from-zero rule, so a legitimate + /// CFR stream may alternate between the floor and ceiling tick counts (for example 40040/40041). + struct Cadence: Equatable, Sendable { + let numerator: Int64 + let denominator: Int64 + + init?(numerator: Int64, denominator: Int64) { + guard numerator > 0, denominator > 0 else { return nil } + let divisor = H264CompositionOffsetRepair.greatestCommonDivisor(numerator, denominator) + self.numerator = numerator / divisor + self.denominator = denominator / divisor + } + + init?(timeBase: AVRational, frameRate: AVRational) { + guard timeBase.num > 0, timeBase.den > 0, + frameRate.num > 0, frameRate.den > 0 else { return nil } + let (numerator, numeratorOverflow) = Int64(timeBase.den) + .multipliedReportingOverflow(by: Int64(frameRate.den)) + let (denominator, denominatorOverflow) = Int64(timeBase.num) + .multipliedReportingOverflow(by: Int64(frameRate.num)) + guard !numeratorOverflow, !denominatorOverflow else { return nil } + self.init(numerator: numerator, denominator: denominator) + } + + var floorStep: Int64 { numerator / denominator } + var ceilStep: Int64 { + let quotient = numerator / denominator + return numerator % denominator == 0 ? quotient : quotient + 1 + } + + /// Reduced denominator is the tick-pattern period. Requiring two observed periods before a + /// fractional plan is accepted keeps short VFR/jitter runs from masquerading as quantization. + var period: Int64 { denominator } + + /// Stream rates are advisory here: `r_frame_rate` is commonly a rounded nominal rate and + /// `avg_frame_rate` is duration-derived. The STTS cycle remains the exact authority, but at + /// least one declared rate must agree to within one thousandth of a stream tick per frame. + /// Double is used only for this bounded consistency check, never to construct timestamps. + func isConsistent(with metadata: Cadence) -> Bool { + if self == metadata { return true } + let observed = Double(numerator) / Double(denominator) + let declared = Double(metadata.numerator) / Double(metadata.denominator) + return observed.isFinite && declared.isFinite && abs(observed - declared) <= 0.001 + } + + /// `round_near_away(frameOrdinal * numerator / denominator)`, without overflowing an Int64 + /// intermediate. The sign-symmetric form is important for the negative reorder head. + func timestamp(at frameOrdinal: Int64) -> Int64? { + guard frameOrdinal != Int64.min else { return nil } + let negative = frameOrdinal < 0 + let magnitude = UInt64(negative ? -frameOrdinal : frameOrdinal) + guard let scaled = scaledMagnitude(magnitude), scaled <= UInt64(Int64.max) else { + return nil + } + let value = Int64(scaled) + return negative ? -value : value + } + + /// Exact inverse for timestamps known to lie on this cadence. Used for container-index + /// folding and after seek, so rounding phase is recovered from the global ladder instead of + /// being restarted at each IDR. nil means the timestamp is not on the declared CFR lattice. + func frameOrdinal(forTimestamp timestamp: Int64) -> Int64? { + guard timestamp != Int64.min else { return nil } + if timestamp == 0 { return 0 } + let negative = timestamp < 0 + let magnitude = UInt64(negative ? -timestamp : timestamp) + let product = magnitude.multipliedFullWidth(by: UInt64(denominator)) + let divisor = UInt64(numerator) + guard product.high < divisor else { return nil } + let approximateMagnitude = divisor.dividingFullWidth(product).quotient + guard approximateMagnitude <= UInt64(Int64.max - 3) else { return nil } + let approximate = negative ? -Int64(approximateMagnitude) : Int64(approximateMagnitude) + for adjustment in -2...2 { + let (candidate, overflow) = approximate.addingReportingOverflow(Int64(adjustment)) + guard !overflow else { continue } + if self.timestamp(at: candidate) == timestamp { return candidate } + } + return nil + } + + private func scaledMagnitude(_ magnitude: UInt64) -> UInt64? { + let product = magnitude.multipliedFullWidth(by: UInt64(numerator)) + let divisor = UInt64(denominator) + guard product.high < divisor else { return nil } + let division = divisor.dividingFullWidth(product) + let threshold = (divisor >> 1) + (divisor & 1) + guard division.remainder >= threshold else { return division.quotient } + let (rounded, overflow) = division.quotient.addingReportingOverflow(1) + return overflow ? nil : rounded + } + } + /// What the rewrite needs, all of it derived once at the head of the session. struct Plan: Equatable, Sendable { /// Ticks between two consecutive pictures in decode order. @@ -52,6 +146,186 @@ enum H264CompositionOffsetRepair { /// Ticks a picture order count advances per displayed picture. 2 for frame coding, but /// measured rather than assumed. var pocStep: Int64 + + /// Fractional-CFR fields. nil together for the original exact-integer path. `rawFrameOffset` + /// describes the broken ladder: `-videoDelay` when the edit list retained the decode head, + /// or 0 when the writer left that ladder on the presentation axis. `rawPhase` is independent: + /// it identifies where the sampled first DTS sits in the cadence's quantization period. + var cadence: Cadence? + var presentationOrigin: Int64? + var rawFrameOffset: Int64? + var videoDelay: Int64? + var rawTimestampAnchor: Int64? + var rawPhase: Int64? + + init(step: Int64, decodeLead: Int64, shift: Int64, pocStep: Int64) { + self.step = step + self.decodeLead = decodeLead + self.shift = shift + self.pocStep = pocStep + cadence = nil + presentationOrigin = nil + rawFrameOffset = nil + videoDelay = nil + rawTimestampAnchor = nil + rawPhase = nil + } + + /// Compatibility constructor for a phase-zero presentation lattice. Classification uses the + /// anchor constructor below because a real MP4's STTS phase need not coincide with semantic + /// frame offset (the affected physical file starts at phase 2 with videoDelay 1). + init?( + cadence: Cadence, + presentationOrigin: Int64, + ladderStart: Int64, + rawFrameOffset: Int64, + videoDelay: Int64, + pocStep: Int64 + ) { + let phase = H264CompositionOffsetRepair.positiveModulo( + rawFrameOffset, + modulus: cadence.period + ) + self.init( + cadence: cadence, + rawTimestampAnchor: ladderStart, + rawPhase: phase, + rawFrameOffset: rawFrameOffset, + videoDelay: videoDelay, + pocStep: pocStep + ) + guard self.presentationOrigin == presentationOrigin else { return nil } + } + + init?( + cadence: Cadence, + rawTimestampAnchor: Int64, + rawPhase: Int64, + rawFrameOffset: Int64, + videoDelay: Int64, + pocStep: Int64 + ) { + guard rawPhase >= 0, rawPhase < cadence.period, + rawFrameOffset == 0 || rawFrameOffset == -videoDelay, + videoDelay > 0, + let oneFrame = cadence.timestamp(at: 1), + let presentationOrigin = Self.anchoredTimestamp( + cadence: cadence, + anchor: rawTimestampAnchor, + phase: rawPhase, + decodeOrdinal: -rawFrameOffset + ), + let firstDecodeTimestamp = Self.anchoredTimestamp( + cadence: cadence, + anchor: rawTimestampAnchor, + phase: rawPhase, + decodeOrdinal: -videoDelay - rawFrameOffset + ) else { return nil } + let (decodeLead, leadOverflow) = presentationOrigin + .subtractingReportingOverflow(firstDecodeTimestamp) + let (shift, shiftOverflow) = presentationOrigin + .subtractingReportingOverflow(rawTimestampAnchor) + guard !leadOverflow, !shiftOverflow, decodeLead > 0 else { return nil } + self.step = oneFrame + self.decodeLead = decodeLead + self.shift = shift + self.pocStep = pocStep + self.cadence = cadence + self.presentationOrigin = presentationOrigin + self.rawFrameOffset = rawFrameOffset + self.videoDelay = videoDelay + self.rawTimestampAnchor = rawTimestampAnchor + self.rawPhase = rawPhase + } + + var isRational: Bool { cadence != nil } + + func decodeOrdinal(forRawTimestamp timestamp: Int64) -> Int64? { + guard let cadence, let rawTimestampAnchor, let rawPhase, + let phaseTimestamp = cadence.timestamp(at: rawPhase) else { return nil } + let (relative, relativeOverflow) = timestamp + .subtractingReportingOverflow(rawTimestampAnchor) + let (absolute, absoluteOverflow) = relative + .addingReportingOverflow(phaseTimestamp) + guard !relativeOverflow, !absoluteOverflow, + let cadenceOrdinal = cadence.frameOrdinal(forTimestamp: absolute) else { return nil } + let (decodeOrdinal, ordinalOverflow) = cadenceOrdinal + .subtractingReportingOverflow(rawPhase) + return ordinalOverflow ? nil : decodeOrdinal + } + + /// A seek may land on the same isolated one-tick container defect tolerated during a + /// continuous read. Re-anchor only when exactly one lattice point exists within that bound; + /// a tight cadence that makes the answer ambiguous remains unplaceable. + func decodeOrdinal(forRawTimestampWithinOneTick timestamp: Int64) -> Int64? { + var match: Int64? + for adjustment in -1...1 { + let (candidateTimestamp, overflow) = timestamp + .addingReportingOverflow(Int64(adjustment)) + guard !overflow, + let candidate = decodeOrdinal(forRawTimestamp: candidateTimestamp) else { + continue + } + if let match, match != candidate { return nil } + match = candidate + } + return match + } + + func presentationTimestamp(frameOrdinal: Int64) -> Int64? { + guard let rawFrameOffset else { return nil } + let (decodeOrdinal, overflow) = frameOrdinal + .subtractingReportingOverflow(rawFrameOffset) + return overflow ? nil : rawTimestamp(decodeOrdinal: decodeOrdinal) + } + + /// Maps a packet/index timestamp from the broken raw ladder onto the repaired decode axis. + /// The exact-integer path preserves its historical constant-offset behavior. + func repairedDecodeTimestamp(_ timestamp: Int64) -> Int64? { + guard isRational else { + let (shifted, shiftOverflow) = timestamp.addingReportingOverflow(shift) + guard !shiftOverflow else { return nil } + let (result, leadOverflow) = shifted.subtractingReportingOverflow(decodeLead) + return leadOverflow ? nil : result + } + guard let decodeOrdinal = decodeOrdinal(forRawTimestamp: timestamp) else { + return nil + } + return repairedDecodeTimestamp(decodeOrdinal: decodeOrdinal) + } + + func repairedDecodeTimestamp(decodeOrdinal: Int64) -> Int64? { + guard let videoDelay else { return nil } + let (targetOrdinal, overflow) = decodeOrdinal.subtractingReportingOverflow(videoDelay) + return overflow ? nil : presentationTimestamp(frameOrdinal: targetOrdinal) + } + + func rawTimestamp(decodeOrdinal: Int64) -> Int64? { + guard let cadence, let rawTimestampAnchor, let rawPhase else { return nil } + return Self.anchoredTimestamp( + cadence: cadence, + anchor: rawTimestampAnchor, + phase: rawPhase, + decodeOrdinal: decodeOrdinal + ) + } + + private static func anchoredTimestamp( + cadence: Cadence, + anchor: Int64, + phase: Int64, + decodeOrdinal: Int64 + ) -> Int64? { + let (cadenceOrdinal, ordinalOverflow) = phase + .addingReportingOverflow(decodeOrdinal) + guard !ordinalOverflow, + let phaseTimestamp = cadence.timestamp(at: phase), + let targetTimestamp = cadence.timestamp(at: cadenceOrdinal) else { return nil } + let (relative, relativeOverflow) = targetTimestamp + .subtractingReportingOverflow(phaseTimestamp) + let (timestamp, timestampOverflow) = anchor.addingReportingOverflow(relative) + return relativeOverflow || timestampOverflow ? nil : timestamp + } } enum Verdict: Equatable, Sendable { @@ -93,7 +367,8 @@ enum H264CompositionOffsetRepair { decodeLead: Int64 ) -> Int64 { guard streamStartTime != Int64.min, ladderStart != Int64.min, decodeLead > 0 else { return 0 } - let raw = streamStartTime - ladderStart + let (raw, overflow) = streamStartTime.subtractingReportingOverflow(ladderStart) + guard !overflow else { return 0 } return min(max(raw, 0), decodeLead) } @@ -103,7 +378,8 @@ enum H264CompositionOffsetRepair { samples: [Sample], videoDelay: Int, streamStartTime: Int64, - ladderStart: Int64 + ladderStart: Int64, + cadenceCandidates: [Cadence] = [] ) -> Verdict { guard videoDelay > 0, videoDelay <= 16 else { return .inconclusive("reorder delay \(videoDelay) outside 1...16") @@ -123,17 +399,59 @@ enum H264CompositionOffsetRepair { return .inconclusive("sample does not start on a picture-order origin") } - // A uniform ladder is what lets a rank be turned back into a timestamp without buffering - // packets. Variable frame timing with no composition offsets is not repairable this way, and - // is left alone rather than guessed at. - var step: Int64 = 0 + // An exact integer ladder keeps the original scalar fast path. A fractional constant-rate + // cadence is also repairable, but only when stream metadata predicts the observed adjacent + // tick pattern exactly for at least two full periods. Merely seeing max-min == 1 is not + // enough: a short VFR/jitter run can have the same range. + var decodeSteps: [Int64] = [] + decodeSteps.reserveCapacity(samples.count - 1) for index in 1.. 0 else { return .inconclusive("decode timestamps do not advance") } - if step == 0 { step = delta } - guard delta == step else { return .inconclusive("decode ladder is not uniform") } + decodeSteps.append(delta) + } + guard let minimumStep = decodeSteps.min(), let maximumStep = decodeSteps.max() else { + return .inconclusive("no ladder step") + } + + let fixedStep: Int64? + var rationalPlan: Plan? + if minimumStep == maximumStep { + fixedStep = minimumStep + } else { + fixedStep = nil + let (stepSpread, spreadOverflow) = maximumStep.subtractingReportingOverflow(minimumStep) + guard !spreadOverflow, stepSpread == 1, + streamStartTime != Int64.min, ladderStart != Int64.min, + samples.first?.dts == ladderStart else { + return .inconclusive("decode ladder is not uniform") + } + + guard let observed = observedCadence( + decodeSteps: decodeSteps, + metadataCandidates: cadenceCandidates + ), let firstDTS = samples.first?.dts else { + return .inconclusive("decode ladder is not uniform") + } + let semanticOffsets = [Int64.zero, -Int64(videoDelay)] + let plans = semanticOffsets.compactMap { rawFrameOffset -> Plan? in + guard let plan = Plan( + cadence: observed.cadence, + rawTimestampAnchor: firstDTS, + rawPhase: observed.phase, + rawFrameOffset: rawFrameOffset, + videoDelay: Int64(videoDelay), + pocStep: 1 + ), plan.presentationOrigin == streamStartTime else { return nil } + return plan + } + guard plans.count == 1 else { + return .inconclusive("decode ladder is not uniform") + } + rationalPlan = plans[0] } - guard step > 0 else { return .inconclusive("no ladder step") } // Without a picture-order regression the file presents in decode order and there is nothing // to repair, whatever its reorder delay claims. @@ -168,14 +486,27 @@ enum H264CompositionOffsetRepair { // are then spread over twice the ladder while still being distinct. Ranks have to FILL the // window they came from: the span may exceed the sample only by the pictures still in flight // at its ragged edge, which is the reorder delay. - guard let maxIndex = displayIndices.max(), let minIndex = displayIndices.min(), - maxIndex - minIndex + 1 <= Int64(samples.count + videoDelay + 1) else { + guard let maxIndex = displayIndices.max(), let minIndex = displayIndices.min() else { + return .inconclusive("display indices do not fill the sampled window") + } + let (displaySpan, spanOverflow) = maxIndex.subtractingReportingOverflow(minIndex) + let (inclusiveSpan, inclusiveOverflow) = displaySpan.addingReportingOverflow(1) + guard !spanOverflow, !inclusiveOverflow, + inclusiveSpan <= Int64(samples.count + videoDelay + 1) else { return .inconclusive("display indices do not fill the sampled window") } - let decodeLead = Int64(videoDelay) * step - return .repair( - Plan( + let plan: Plan + if var rationalPlan { + rationalPlan.pocStep = pocStep + plan = rationalPlan + } else { + guard let step = fixedStep, step > 0 else { return .inconclusive("no ladder step") } + let (decodeLead, leadOverflow) = Int64(videoDelay).multipliedReportingOverflow(by: step) + guard !leadOverflow, decodeLead > 0 else { + return .inconclusive("decode ladder is not uniform") + } + plan = Plan( step: step, decodeLead: decodeLead, shift: presentationShift( @@ -185,7 +516,23 @@ enum H264CompositionOffsetRepair { ), pocStep: pocStep ) - ) + } + + // The structural checks above derive a candidate; the held head must also prove that the + // exact Rewriter can place every sampled picture. This catches a malformed/misreported + // videoDelay whose POC ranks look bijective but would make PTS precede DTS for only part of + // the window, which would otherwise mix repaired and raw axes as the held queue drains. + var dryRun = Rewriter(plan: plan) + for sample in samples { + guard dryRun.rewrite( + dts: sample.dts, + pictureOrderCount: sample.pictureOrderCount, + isKeyframe: sample.isKeyframe + ) != nil else { + return .inconclusive("sample cannot be rewritten safely") + } + } + return .repair(plan) } static func greatestCommonDivisor(_ a: Int64, _ b: Int64) -> Int64 { @@ -194,13 +541,73 @@ enum H264CompositionOffsetRepair { return x } - /// Applies a confirmed plan packet by packet. Holds exactly one piece of state, the decode - /// timestamp of the picture that opened the current coded video sequence, because picture order - /// counts restart at every IDR. + static func positiveModulo(_ value: Int64, modulus: Int64) -> Int64 { + guard modulus > 0 else { return 0 } + let remainder = value % modulus + return remainder >= 0 ? remainder : remainder + modulus + } + + private static func observedCadence( + decodeSteps: [Int64], + metadataCandidates: [Cadence] + ) -> (cadence: Cadence, phase: Int64)? { + let maximumPeriod = decodeSteps.count / 2 + guard maximumPeriod >= 2 else { return nil } + var matches: [(cadence: Cadence, phase: Int64)] = [] + + for period in 2...maximumPeriod { + guard decodeSteps.indices.allSatisfy({ index in + decodeSteps[index] == decodeSteps[index % period] + }) else { continue } + var periodTicks: Int64 = 0 + var overflowed = false + for step in decodeSteps.prefix(period) { + let (sum, overflow) = periodTicks.addingReportingOverflow(step) + if overflow { overflowed = true; break } + periodTicks = sum + } + guard !overflowed, + let cadence = Cadence( + numerator: periodTicks, + denominator: Int64(period) + ), cadence.period == Int64(period), + metadataCandidates.contains(where: { cadence.isConsistent(with: $0) }) else { + continue + } + + for phase in 0.. (pts: Int64, dts: Int64)? { guard dts != Int64.min, plan.pocStep > 0 else { unrepairedPictures += 1 return nil } + if plan.isRational { + return rewriteRational( + dts: dts, + pictureOrderCount: pictureOrderCount, + isKeyframe: isKeyframe + ) + } + guard let pictureOrderCount, + pictureOrderCount >= 0, + pictureOrderCount % plan.pocStep == 0 else { + unrepairedPictures += 1 + return nil + } + let displayIndex = pictureOrderCount / plan.pocStep // A picture order of 0 on a keyframe is an IDR: a new coded video sequence starts here // and its first picture is also the first to be displayed. After a seek the landing // keyframe anchors even if its count is not 0, which is the only way an open-GOP entry @@ -234,21 +657,29 @@ enum H264CompositionOffsetRepair { sequenceAnchorDTS = dts awaitingReanchor = false } else if awaitingReanchor, isKeyframe { - guard pictureOrderCount % plan.pocStep == 0 else { + let (anchorOffset, offsetOverflow) = displayIndex + .multipliedReportingOverflow(by: plan.step) + let (anchor, anchorOverflow) = dts.subtractingReportingOverflow(anchorOffset) + guard !offsetOverflow, !anchorOverflow else { unrepairedPictures += 1 return nil } - sequenceAnchorDTS = dts - (pictureOrderCount / plan.pocStep) * plan.step + sequenceAnchorDTS = anchor awaitingReanchor = false } - guard let anchor = sequenceAnchorDTS, !awaitingReanchor, - pictureOrderCount >= 0, pictureOrderCount % plan.pocStep == 0 else { + guard let anchor = sequenceAnchorDTS, !awaitingReanchor else { + unrepairedPictures += 1 + return nil + } + let (presentationOffset, offsetOverflow) = displayIndex + .multipliedReportingOverflow(by: plan.step) + let (shiftedAnchor, shiftOverflow) = anchor.addingReportingOverflow(plan.shift) + let (pts, ptsOverflow) = shiftedAnchor.addingReportingOverflow(presentationOffset) + guard !offsetOverflow, !shiftOverflow, !ptsOverflow, + let newDTS = plan.repairedDecodeTimestamp(dts) else { unrepairedPictures += 1 return nil } - let displayIndex = pictureOrderCount / plan.pocStep - let pts = anchor &+ plan.shift &+ displayIndex &* plan.step - let newDTS = dts &+ plan.shift &- plan.decodeLead // The muxer invariant. A picture that lands before its own decode time means the // arithmetic no longer describes this stream, and passing it through unchanged is // better than handing the muxer something it will silently clamp. @@ -259,6 +690,109 @@ enum H264CompositionOffsetRepair { repairedPictures += 1 return (pts, newDTS) } + + private mutating func rewriteRational( + dts: Int64, + pictureOrderCount: Int64?, + isKeyframe: Bool + ) -> (pts: Int64, dts: Int64)? { + let exactDecodeOrdinal = plan.decodeOrdinal(forRawTimestamp: dts) + let decodeOrdinal: Int64 + var mayAdvanceAfterUnplacedPicture = false + if let lastRationalDecodeOrdinal { + let (expected, overflow) = lastRationalDecodeOrdinal.addingReportingOverflow(1) + guard !overflow else { + unrepairedPictures += 1 + return nil + } + if let exactDecodeOrdinal { + // An exact forward ordinal is stronger than packet counting: it preserves a + // legitimate gap and resynchronizes after a preceding parser miss. A backward + // exact timestamp is a discontinuity this session was not told about. + guard exactDecodeOrdinal >= expected else { + unrepairedPictures += 1 + return nil + } + decodeOrdinal = exactDecodeOrdinal + } else { + // The sampled stream proved a CFR lattice. A later one-tick timestamp defect is + // therefore rectified to the next decode position instead of being emitted raw. + guard let expectedTimestamp = plan.rawTimestamp(decodeOrdinal: expected), + Self.isWithinOneTick(dts, of: expectedTimestamp) else { + unrepairedPictures += 1 + return nil + } + decodeOrdinal = expected + } + mayAdvanceAfterUnplacedPicture = decodeOrdinal == expected + } else { + let landingOrdinal = exactDecodeOrdinal + ?? plan.decodeOrdinal(forRawTimestampWithinOneTick: dts) + guard awaitingReanchor, isKeyframe, let landingOrdinal else { + unrepairedPictures += 1 + return nil + } + decodeOrdinal = landingOrdinal + } + guard let pictureOrderCount, + pictureOrderCount >= 0, + pictureOrderCount % plan.pocStep == 0 else { + // A parser miss on precisely the expected next packet still consumed one decode + // position. A larger exact jump is not committed until full placement succeeds, + // otherwise one bad-but-on-lattice timestamp can poison every packet behind it. + if mayAdvanceAfterUnplacedPicture { + lastRationalDecodeOrdinal = decodeOrdinal + } + if isKeyframe { + sequenceAnchorDTS = nil + sequenceBaseOrdinal = nil + awaitingReanchor = true + } + unrepairedPictures += 1 + return nil + } + let displayIndex = pictureOrderCount / plan.pocStep + if isKeyframe, pictureOrderCount == 0 { + sequenceBaseOrdinal = decodeOrdinal + sequenceAnchorDTS = dts + awaitingReanchor = false + } else if awaitingReanchor, isKeyframe { + let (baseOrdinal, overflow) = decodeOrdinal + .subtractingReportingOverflow(displayIndex) + guard !overflow else { + unrepairedPictures += 1 + return nil + } + sequenceBaseOrdinal = baseOrdinal + sequenceAnchorDTS = dts + awaitingReanchor = false + } + guard let sequenceBaseOrdinal, !awaitingReanchor else { + unrepairedPictures += 1 + return nil + } + let (presentationOrdinal, ordinalOverflow) = sequenceBaseOrdinal + .addingReportingOverflow(displayIndex) + guard !ordinalOverflow, + let pts = plan.presentationTimestamp(frameOrdinal: presentationOrdinal), + let newDTS = plan.repairedDecodeTimestamp(decodeOrdinal: decodeOrdinal), + pts >= newDTS else { + unrepairedPictures += 1 + return nil + } + lastRationalDecodeOrdinal = decodeOrdinal + repairedPictures += 1 + return (pts, newDTS) + } + + private static func isWithinOneTick(_ value: Int64, of expected: Int64) -> Bool { + if value >= expected { + let (difference, overflow) = value.subtractingReportingOverflow(expected) + return !overflow && difference <= 1 + } + let (difference, overflow) = expected.subtractingReportingOverflow(value) + return !overflow && difference <= 1 + } } } @@ -333,6 +867,7 @@ final class H264CompositionOffsetRepairSession { private let videoDelay: Int private let streamStartTime: Int64 private let ladderStart: Int64 + private let cadenceCandidates: [H264CompositionOffsetRepair.Cadence] private var reader: H264PictureOrderReader? private var rewriter: H264CompositionOffsetRepair.Rewriter? private var samples: [H264CompositionOffsetRepair.Sample] = [] @@ -360,6 +895,18 @@ final class H264CompositionOffsetRepairSession { self.videoDelay = Int(codecpar.pointee.video_delay) self.streamStartTime = stream.pointee.start_time self.ladderStart = ladderStart + var cadences: [H264CompositionOffsetRepair.Cadence] = [] + // r_frame_rate is the coded cadence. avg_frame_rate is duration-derived and may be perturbed + // by head/tail quantization, but remains a bounded fallback for containers that omit r_frame_rate; + // classification still requires an exact two-period match before either candidate is trusted. + for frameRate in [stream.pointee.r_frame_rate, stream.pointee.avg_frame_rate] { + guard let cadence = H264CompositionOffsetRepair.Cadence( + timeBase: stream.pointee.time_base, + frameRate: frameRate + ), !cadences.contains(cadence) else { continue } + cadences.append(cadence) + } + cadenceCandidates = cadences self.reader = reader } @@ -376,14 +923,14 @@ final class H264CompositionOffsetRepairSession { /// this demuxer, because the repair may still be about to move it. var isDecided: Bool { phase != .sampling } - /// Ticks every decode timestamp moves by, so the container's own index can be read on the same - /// axis as the packets. nil while sampling or when nothing is repaired. The plan built from the - /// index and the packets that fill it have to agree: measured, a plan on the raw ladder against - /// repaired packets cut segment 2 one picture past its keyframe, which is a segment AVPlayer - /// cannot start at. - var decodeTimestampOffset: Int64? { + var isRepairing: Bool { phase == .repairing } + + /// Maps the container's own keyframe index onto exactly the same decode axis as packets. A + /// scalar offset is sufficient for an integer cadence; fractional cadence must recover the + /// global frame ordinal or an index/packet pair can disagree by one tick at a rounding boundary. + func repairedDecodeTimestamp(_ timestamp: Int64) -> Int64? { guard phase == .repairing, let rewriter else { return nil } - return rewriter.plan.shift - rewriter.plan.decodeLead + return rewriter.plan.repairedDecodeTimestamp(timestamp) } /// Returns true when the packet was taken over by the session and must not be emitted yet. @@ -486,7 +1033,8 @@ final class H264CompositionOffsetRepairSession { samples: samples, videoDelay: videoDelay, streamStartTime: streamStartTime, - ladderStart: ladderStart + ladderStart: ladderStart, + cadenceCandidates: cadenceCandidates ) switch verdict { case .repair(let plan): @@ -539,11 +1087,10 @@ final class H264CompositionOffsetRepairSession { pictureOrderCount: Int64?, using rewriter: inout H264CompositionOffsetRepair.Rewriter ) { - guard let pictureOrderCount, - let repaired = rewriter.rewrite( - dts: packet.pointee.dts, - pictureOrderCount: pictureOrderCount, - isKeyframe: (packet.pointee.flags & AV_PKT_FLAG_KEY) != 0) + guard let repaired = rewriter.rewrite( + dts: packet.pointee.dts, + pictureOrderCount: pictureOrderCount, + isKeyframe: (packet.pointee.flags & AV_PKT_FLAG_KEY) != 0) else { return } packet.pointee.pts = repaired.pts packet.pointee.dts = repaired.dts diff --git a/Tests/AetherEngineTests/H264CompositionOffsetRepairTests.swift b/Tests/AetherEngineTests/H264CompositionOffsetRepairTests.swift index 0abd36a3..e9621797 100644 --- a/Tests/AetherEngineTests/H264CompositionOffsetRepairTests.swift +++ b/Tests/AetherEngineTests/H264CompositionOffsetRepairTests.swift @@ -26,6 +26,47 @@ struct H264CompositionOffsetRepairTests { } } + private var quantizedCadence: H264CompositionOffsetRepair.Cadence { + H264CompositionOffsetRepair.Cadence(numerator: 200202, denominator: 5)! + } + + private func quantizedMalformedSamples(videoDelay: Int64 = 2) + -> [H264CompositionOffsetRepair.Sample] + { + let pocs: [Int64] = [0, 8, 4, 2, 6, 16, 12, 10, 14, 24, 20, 18] + return pocs.enumerated().map { index, poc in + let dts = quantizedCadence.timestamp(at: Int64(index) - videoDelay)! + return H264CompositionOffsetRepair.Sample( + dts: dts, pts: dts, pictureOrderCount: poc, isKeyframe: index == 0) + } + } + + /// Identity-free timing evidence from the affected physical Apple TV source. The DTS ladder is + /// exact; the POC ranks model the logged videoDelay=1 and three regressions without exposing the + /// source's full bitstream order. The nominal coded rate rounds to a 40040-tick integer cadence, + /// while the duration-derived average rate is nearly 200202/5 and the actual STTS pattern repeats + /// that five-frame quantization twice. + private func physicalQuantizedSamples() -> [H264CompositionOffsetRepair.Sample] { + let pocs: [Int64] = [0, 4, 2, 6, 8, 12, 10, 14, 16, 20, 18, 22] + let steps: [Int64] = [ + 40040, 40041, 40040, 40040, 40041, + 40040, 40041, 40040, 40040, 40041, 40040, + ] + var dts: Int64 = -40040 + return pocs.enumerated().map { index, poc in + defer { if index < steps.count { dts += steps[index] } } + return H264CompositionOffsetRepair.Sample( + dts: dts, pts: dts, pictureOrderCount: poc, isKeyframe: index == 0) + } + } + + private var physicalAverageCadence: H264CompositionOffsetRepair.Cadence { + H264CompositionOffsetRepair.Cadence( + numerator: 34_597_562_400_000, + denominator: 864_066_353 + )! + } + private func verdict( _ samples: [H264CompositionOffsetRepair.Sample], videoDelay: Int = 2, @@ -68,6 +109,309 @@ struct H264CompositionOffsetRepairTests { #expect(verdict(samples) == .inconclusive("decode ladder is not uniform")) } + @Test("nearest rounding is symmetric across the negative decode head and exactly invertible") + func rationalCadenceRoundingAndInverse() { + let expected: [(Int64, Int64)] = [ + (-2, -80081), (-1, -40040), (0, 0), (1, 40040), + (2, 80081), (3, 120121), (4, 160162), + ] + for (ordinal, timestamp) in expected { + #expect(quantizedCadence.timestamp(at: ordinal) == timestamp) + #expect(quantizedCadence.frameOrdinal(forTimestamp: timestamp) == ordinal) + } + #expect(quantizedCadence.frameOrdinal(forTimestamp: 40041) == nil) + } + + @Test("an exact two-period adjacent-tick ladder uses the declared rational cadence") + func quantizedLadderClassification() { + let samples = quantizedMalformedSamples() + let result = H264CompositionOffsetRepair.classify( + samples: samples, videoDelay: 2, + streamStartTime: 0, ladderStart: samples[0].dts, + cadenceCandidates: [quantizedCadence]) + if case .repair(let plan) = result { + #expect(plan.cadence == quantizedCadence) + #expect(plan.rawFrameOffset == -2) + #expect(plan.step == 40040) + #expect(plan.decodeLead == 80081) + #expect(plan.shift == 80081) + } else { + #expect(Bool(false), "the exact rational lattice must be repairable") + } + } + + @Test("adjacent steps in the wrong phase remain inconclusive") + func rejectsWrongQuantizationPattern() { + var samples = quantizedMalformedSamples() + let originalSteps = zip(samples, samples.dropFirst()).map { $1.dts - $0.dts } + var wrongSteps = originalSteps + wrongSteps.swapAt(0, 1) + var timestamp = samples[0].dts + for index in 1.. [Int64] { + let data = try #require(Data(base64Encoded: base64, options: .ignoreUnknownCharacters)) + let demuxer = Demuxer() + try demuxer.open(reader: DataIOReader(data: data), formatHint: "mp4") + defer { demuxer.close() } + demuxer.decideCompositionOffsetRepair() + return demuxer.indexedKeyframes(streamIndex: demuxer.videoStreamIndex) + } + @Test("the repaired twin carries the healthy twin's timestamps, packet for packet") func repairedTwinMatchesHealthyTwin() throws { let healthy = try Self.videoTimestamps(base64: Self.healthyCTTSFixtureBase64) @@ -235,6 +627,28 @@ struct H264CompositionOffsetRepairTests { #expect(repaired == healthy) } + /// Physical-device regression: a rational frame cadence may quantize to two adjacent integer + /// DTS steps even though it is constant-frame-rate. This twin uses 30 fps in a 1/1,201,212 + /// timebase, so one frame is exactly 200202/5 ticks and the packet ladder alternates between + /// 40040 and 40041. Treating that normal quantization as VFR leaves the missing-ctts file in + /// decode order and reproduces the visible judder from #409. + @Test("an adjacent-tick rational ladder is repaired exactly, across multiple IDRs") + func quantizedRationalTwinMatchesHealthyTwin() throws { + let healthy = try Self.videoTimestamps(base64: Self.quantizedHealthyCTTSFixtureBase64) + let repaired = try Self.videoTimestamps(base64: Self.quantizedMissingCTTSFixtureBase64) + #expect(healthy.count == 66) + #expect(repaired.count == healthy.count) + #expect(repaired == healthy) + } + + @Test("rational repair folds container keyframe indexes onto the healthy decode axis") + func quantizedRationalIndexesMatchHealthyTwin() throws { + let healthy = try Self.indexedKeyframes(base64: Self.quantizedHealthyCTTSFixtureBase64) + let repaired = try Self.indexedKeyframes(base64: Self.quantizedMissingCTTSFixtureBase64) + #expect(healthy.count == 3) + #expect(repaired == healthy) + } + @Test("the healthy twin is delivered exactly as the container wrote it") func healthyTwinIsUntouched() throws { let healthy = try Self.videoTimestamps(base64: Self.healthyCTTSFixtureBase64) @@ -343,4 +757,103 @@ struct H264CompositionOffsetRepairTests { BwGft0BtBmAAAAAIQZu8NEB9BmAAAAAJQZ/aRRUodQZgAAAABwGf+UBtBmAAAAAHAZ/7QG0GYAAAAAhBm+A0QH0GYAAAAAlBnh5F FSh1BmAAAAAHAZ49QG0GYAAAAAcBnj9AbQZgAAAACEGaITRAfQZg """ + + /// 96x64 H.264 Main, 30 fps, 66 frames, three IDR sequences. The deliberately unusual + /// 1,201,212 track timescale makes one frame 200202/5 ticks, so the valid CFR ladder uses both + /// 40040- and 40041-tick steps. Generated with: + /// + /// ffmpeg -f lavfi -i 'color=c=red:s=96x64:r=30:d=2.2' -frames:v 66 \ + /// -c:v libx264 -preset ultrafast -pix_fmt yuv420p -bf 3 -b_strategy 0 \ + /// -g 22 -keyint_min 22 -sc_threshold 0 -video_track_timescale 1201212 \ + /// -movflags +faststart healthy.mp4 + /// ffmpeg -i healthy.mp4 -map 0:v:0 -c:v copy -bsf:v 'setts=pts=DTS' \ + /// -movflags +faststart missing.mp4 + private static let quantizedHealthyCTTSFixtureBase64 = """ + AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAfjbW9vdgAAAGxtdmhkAAAAAAAAAAAAAAAAAAAD6AAACJgAAQAAAQAAAAAAAAAA + AAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAABw10cmFrAAAAXHRr + aGQAAAADAAAAAAAAAAAAAAABAAAAAAAACJgAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAAGAA + AABAAAAAAAAkZWR0cwAAABxlbHN0AAAAAAAAAAEAAAiYAAE40QABAAAAAAaFbWRpYQAAACBtZGhkAAAAAAAAAAAAAAAAABJUPAAoUupVxAAA + AAAALWhkbHIAAAAAAAAAAHZpZGUAAAAAAAAAAAAAAABWaWRlb0hhbmRsZXIAAAAGMG1pbmYAAAAUdm1oZAAAAAEAAAAAAAAAAAAAACRkaW5m + AAAAHGRyZWYAAAAAAAAAAQAAAAx1cmwgAAAAAQAABfBzdGJsAAAAuHN0c2QAAAAAAAAAAQAAAKhhdmMxAAAAAAAAAAEAAAAAAAAAAAAAAAAA + AAAAAGAAQABIAAAASAAAAAAAAAABFUxhdmM2Mi4yOC4xMDIgbGlieDI2NAAAAAAAAAAAAAAAGP//AAAALmF2Y0MBTUAK/+EAF2dNQArsoxNg + IgAAAwACAAADAHgeJEssAQAEaM4PyAAAABBwYXNwAAAAAQAAAAEAAAAUYnRydAAAAAAAABXKAAAAAAAAAbBzdHRzAAAAAAAAADQAAAABAACc + aQAAAAIAAJxoAAAAAQAAnGkAAAABAACcaAAAAAEAAJxpAAAAAgAAnGgAAAABAACcaQAAAAEAAJxoAAAAAQAAnGkAAAACAACcaAAAAAEAAJxp + AAAAAQAAnGgAAAABAACcaQAAAAIAAJxoAAAAAQAAnGkAAAABAACcaAAAAAEAAJxpAAAAAgAAnGgAAAABAACcaQAAAAEAAJxoAAAAAQAAnGkA + AAACAACcaAAAAAEAAJxpAAAAAQAAnGgAAAABAACcaQAAAAIAAJxoAAAAAQAAnGkAAAABAACcaAAAAAEAAJxpAAAAAgAAnGgAAAABAACcaQAA + AAEAAJxoAAAAAQAAnGkAAAACAACcaAAAAAEAAJxpAAAAAQAAnGgAAAABAACcaQAAAAIAAJxoAAAAAQAAnGkAAAABAACcaAAAAAEAAJxpAAAA + AgAAnGgAAAABAACcaQAAAAEAAJxoAAAAAQAAnGkAAAACAACcaAAAAAEAAJxpAAAAAQAAnGgAAAABAACcaQAAAAIAAJxoAAAAAQAAnGkAAAAC + AACcaAAAABxzdHNzAAAAAAAAAAMAAAABAAAAFwAAAC0AAAIYY3R0cwAAAAAAAABBAAAAAQABONEAAAABAAMOCgAAAAEAATjRAAAAAQAAAAAA + AAABAACcaAAAAAEAAw4KAAAAAQABONAAAAABAAAAAAAAAAEAAJxpAAAAAQADDgoAAAABAAE40QAAAAEAAAAAAAAAAQAAnGgAAAABAAMOCgAA + AAEAATjRAAAAAQAAAAAAAAABAACcaAAAAAEAAw4KAAAAAQABONEAAAABAAAAAAAAAAEAAJxpAAAAAQABONAAAAABAAE40QAAAAEAAw4KAAAA + AQABONEAAAABAAAAAAAAAAEAAJxoAAAAAQADDgoAAAABAAE40QAAAAEAAAAAAAAAAQAAnGkAAAABAAMOCgAAAAEAATjRAAAAAQAAAAAAAAAB + AACcaAAAAAEAAw4KAAAAAQABONAAAAABAAAAAAAAAAEAAJxpAAAAAQADDgoAAAABAAE40QAAAAEAAAAAAAAAAQAAnGgAAAACAAE40QAAAAEA + Aw4KAAAAAQABONAAAAABAAAAAAAAAAEAAJxpAAAAAQADDgoAAAABAAE40QAAAAEAAAAAAAAAAQAAnGgAAAABAAMOCgAAAAEAATjRAAAAAQAA + AAAAAAABAACcaAAAAAEAAw4KAAAAAQABONEAAAABAAAAAAAAAAEAAJxpAAAAAQADDgoAAAABAAE40QAAAAEAAAAAAAAAAQAAnGgAAAABAAE4 + 0QAAABxzdHNjAAAAAAAAAAEAAAABAAAAQgAAAAEAAAEcc3RzegAAAAAAAAAAAAAAQgAAAswAAAALAAAACwAAAAsAAAALAAAADAAAAA0AAAAL + AAAACwAAAAwAAAANAAAACwAAAAsAAAAMAAAADQAAAAsAAAALAAAADAAAAA0AAAALAAAACwAAAAwAAAArAAAACwAAAAsAAAALAAAACwAAAAwA + AAANAAAACwAAAAsAAAAMAAAADQAAAAsAAAALAAAADAAAAA0AAAALAAAACwAAAAwAAAANAAAACwAAAAsAAAAMAAAAKwAAAAsAAAALAAAACwAA + AAsAAAAMAAAADQAAAAsAAAALAAAADAAAAA0AAAALAAAACwAAAAwAAAANAAAACwAAAAsAAAAMAAAADQAAAAsAAAALAAAADAAAABRzdGNvAAAA + AAAAAAEAAAgTAAAAYnVkdGEAAABabWV0YQAAAAAAAAAhaGRscgAAAAAAAAAAbWRpcmFwcGwAAAAAAAAAAAAAAAAtaWxzdAAAACWpdG9vAAAA + HWRhdGEAAAABAAAAAExhdmY2Mi4xMi4xMDIAAAAIZnJlZQAABgZtZGF0AAACngYF//+a3EXpvebZSLeWLNgg2SPu73gyNjQgLSBjb3JlIDE2 + NSByMzIyMiBiMzU2MDVhIC0gSC4yNjQvTVBFRy00IEFWQyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAyNSAtIGh0dHA6Ly93d3cudmlkZW9s + YW4ub3JnL3gyNjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTAgcmVmPTEgZGVibG9jaz0wOjA6MCBhbmFseXNlPTA6MCBtZT1kaWEgc3VibWU9 + MCBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0wIG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MCA4eDhkY3Q9MCBj + cW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0wIHRocmVhZHM9MiBsb29rYWhlYWRfdGhyZWFkcz0x + IHNsaWNlZF90aHJlYWRzPTAgbnI9MCBkZWNpbWF0ZT0xIGludGVybGFjZWQ9MCBibHVyYXlfY29tcGF0PTAgY29uc3RyYWluZWRfaW50cmE9 + MCBiZnJhbWVzPTMgYl9weXJhbWlkPTIgYl9hZGFwdD0wIGJfYmlhcz0wIGRpcmVjdD0xIHdlaWdodGI9MCBvcGVuX2dvcD0wIHdlaWdodHA9 + MCBrZXlpbnQ9MjIga2V5aW50X21pbj0xMiBzY2VuZWN1dD0wIGludHJhX3JlZnJlc2g9MCByYz1jcmYgbWJ0cmVlPTAgY3JmPTIzLjAgcWNv + bXA9MC42MCBxcG1pbj0wIHFwbWF4PTY5IHFwc3RlcD00IGlwX3JhdGlvPTEuNDAgcGJfcmF0aW89MS4zMCBhcT0wAIAAAAAmZYiEAOhGKAAI + Y8cAAQPY4AAh5ScnJycnXXXXXXXXXXXXXXXXXXgAAAAHQZokAOoMwAAAAAdBnkJANoMwAAAABwGeYUBdBmAAAAAHAZ5jQF0GYAAAAAhBmmg0 + QHUGYAAAAAlBnoZFEShtBmAAAAAHAZ6lQGUGYAAAAAcBnqdAZQZgAAAACEGarDRAfQZgAAAACUGeykUVKG0GYAAAAAcBnulAZQZgAAAABwGe + 60BlBmAAAAAIQZrwNEB9BmAAAAAJQZ8ORRUodQZgAAAABwGfLUBlBmAAAAAHAZ8vQG0GYAAAAAhBmzQ0QH0GYAAAAAlBn1JFFSh1BmAAAAAH + AZ9xQG0GYAAAAAcBn3NAbQZgAAAACEGbdTRAfQZgAAAAJ2WIggAEKEYoAAoSxwABGVjgACnhJycnJyddddddddddddddddddeAAAAAdBmiQA + 6gzAAAAAB0GeQkA2gzAAAAAHAZ5hQGUGYAAAAAcBnmNAZQZgAAAACEGaaDRAdQZgAAAACUGehkURKG0GYAAAAAcBnqVAZQZgAAAABwGep0Bl + BmAAAAAIQZqsNEB9BmAAAAAJQZ7KRRUobQZgAAAABwGe6UBlBmAAAAAHAZ7rQGUGYAAAAAhBmvA0QH0GYAAAAAlBnw5FFSh1BmAAAAAHAZ8t + QGUGYAAAAAcBny9AbQZgAAAACEGbNDRAfQZgAAAACUGfUkUVKHUGYAAAAAcBn3FAbQZgAAAABwGfc0BtBmAAAAAIQZt1NEB9BmAAAAAnZYiE + ABChGKAAKEscAARlY4AAp4ScnJycnXXXXXXXXXXXXXXXXXXgAAAAB0GaJADqDMAAAAAHQZ5CQDaDMAAAAAcBnmFAZQZgAAAABwGeY0BlBmAA + AAAIQZpoNEB1BmAAAAAJQZ6GRREobQZgAAAABwGepUBlBmAAAAAHAZ6nQGUGYAAAAAhBmqw0QH0GYAAAAAlBnspFFShtBmAAAAAHAZ7pQGUG + YAAAAAcBnutAZQZgAAAACEGa8DRAfQZgAAAACUGfDkUVKHUGYAAAAAcBny1AZQZgAAAABwGfL0BtBmAAAAAIQZs0NEB9BmAAAAAJQZ9SRRUo + dQZgAAAABwGfcUBtBmAAAAAHAZ9zQG0GYAAAAAhBm3U0QH0GYA== + """ + + private static let quantizedMissingCTTSFixtureBase64 = """ + AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAXLbW9vdgAAAGxtdmhkAAAAAAAAAAAAAAAAAAAD6AAACFYAAQAAAQAAAAAAAAAA + AAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAABPV0cmFrAAAAXHRr + aGQAAAADAAAAAAAAAAAAAAABAAAAAAAACFYAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAAGAA + AABAAAAAAAAkZWR0cwAAABxlbHN0AAAAAAAAAAEAAAhVAAE40QABAAAAAARtbWRpYQAAACBtZGhkAAAAAAAAAAAAAAAAABJUPAAoUupVxAAA + AAAALWhkbHIAAAAAAAAAAHZpZGUAAAAAAAAAAAAAAABWaWRlb0hhbmRsZXIAAAAEGG1pbmYAAAAUdm1oZAAAAAEAAAAAAAAAAAAAACRkaW5m + AAAAHGRyZWYAAAAAAAAAAQAAAAx1cmwgAAAAAQAAA9hzdGJsAAAAuHN0c2QAAAAAAAAAAQAAAKhhdmMxAAAAAAAAAAEAAAAAAAAAAAAAAAAA + AAAAAGAAQABIAAAASAAAAAAAAAABFUxhdmM2Mi4yOC4xMDIgbGlieDI2NAAAAAAAAAAAAAAAGP//AAAALmF2Y0MBTUAK/+EAF2dNQArsoxNg + IgAAAwACAAADAHgeJEssAQAEaM4PyAAAABBwYXNwAAAAAQAAAAEAAAAUYnRydAAAAAAAABXKAAAVygAAAbBzdHRzAAAAAAAAADQAAAABAACc + aQAAAAIAAJxoAAAAAQAAnGkAAAABAACcaAAAAAEAAJxpAAAAAgAAnGgAAAABAACcaQAAAAEAAJxoAAAAAQAAnGkAAAACAACcaAAAAAEAAJxp + AAAAAQAAnGgAAAABAACcaQAAAAIAAJxoAAAAAQAAnGkAAAABAACcaAAAAAEAAJxpAAAAAgAAnGgAAAABAACcaQAAAAEAAJxoAAAAAQAAnGkA + AAACAACcaAAAAAEAAJxpAAAAAQAAnGgAAAABAACcaQAAAAIAAJxoAAAAAQAAnGkAAAABAACcaAAAAAEAAJxpAAAAAgAAnGgAAAABAACcaQAA + AAEAAJxoAAAAAQAAnGkAAAACAACcaAAAAAEAAJxpAAAAAQAAnGgAAAABAACcaQAAAAIAAJxoAAAAAQAAnGkAAAABAACcaAAAAAEAAJxpAAAA + AgAAnGgAAAABAACcaQAAAAEAAJxoAAAAAQAAnGkAAAACAACcaAAAAAEAAJxpAAAAAQAAnGgAAAABAACcaQAAAAIAAJxoAAAAAQAAnGkAAAAC + AACcaAAAABxzdHNzAAAAAAAAAAMAAAABAAAAFwAAAC0AAAAcc3RzYwAAAAAAAAABAAAAAQAAAEIAAAABAAABHHN0c3oAAAAAAAAAAAAAAEIA + AALMAAAACwAAAAsAAAALAAAACwAAAAwAAAANAAAACwAAAAsAAAAMAAAADQAAAAsAAAALAAAADAAAAA0AAAALAAAACwAAAAwAAAANAAAACwAA + AAsAAAAMAAAAKwAAAAsAAAALAAAACwAAAAsAAAAMAAAADQAAAAsAAAALAAAADAAAAA0AAAALAAAACwAAAAwAAAANAAAACwAAAAsAAAAMAAAA + DQAAAAsAAAALAAAADAAAACsAAAALAAAACwAAAAsAAAALAAAADAAAAA0AAAALAAAACwAAAAwAAAANAAAACwAAAAsAAAAMAAAADQAAAAsAAAAL + AAAADAAAAA0AAAALAAAACwAAAAwAAAAUc3RjbwAAAAAAAAABAAAF+wAAAGJ1ZHRhAAAAWm1ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAG1kaXJh + cHBsAAAAAAAAAAAAAAAALWlsc3QAAAAlqXRvbwAAAB1kYXRhAAAAAQAAAABMYXZmNjIuMTIuMTAyAAAACGZyZWUAAAYGbWRhdAAAAp4GBf// + mtxF6b3m2Ui3lizYINkj7u94MjY0IC0gY29yZSAxNjUgcjMyMjIgYjM1NjA1YSAtIEguMjY0L01QRUctNCBBVkMgY29kZWMgLSBDb3B5bGVm + dCAyMDAzLTIwMjUgLSBodHRwOi8vd3d3LnZpZGVvbGFuLm9yZy94MjY0Lmh0bWwgLSBvcHRpb25zOiBjYWJhYz0wIHJlZj0xIGRlYmxvY2s9 + MDowOjAgYW5hbHlzZT0wOjAgbWU9ZGlhIHN1Ym1lPTAgcHN5PTEgcHN5X3JkPTEuMDA6MC4wMCBtaXhlZF9yZWY9MCBtZV9yYW5nZT0xNiBj + aHJvbWFfbWU9MSB0cmVsbGlzPTAgOHg4ZGN0PTAgY3FtPTAgZGVhZHpvbmU9MjEsMTEgZmFzdF9wc2tpcD0xIGNocm9tYV9xcF9vZmZzZXQ9 + MCB0aHJlYWRzPTIgbG9va2FoZWFkX3RocmVhZHM9MSBzbGljZWRfdGhyZWFkcz0wIG5yPTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1 + cmF5X2NvbXBhdD0wIGNvbnN0cmFpbmVkX2ludHJhPTAgYmZyYW1lcz0zIGJfcHlyYW1pZD0yIGJfYWRhcHQ9MCBiX2JpYXM9MCBkaXJlY3Q9 + MSB3ZWlnaHRiPTAgb3Blbl9nb3A9MCB3ZWlnaHRwPTAga2V5aW50PTIyIGtleWludF9taW49MTIgc2NlbmVjdXQ9MCBpbnRyYV9yZWZyZXNo + PTAgcmM9Y3JmIG1idHJlZT0wIGNyZj0yMy4wIHFjb21wPTAuNjAgcXBtaW49MCBxcG1heD02OSBxcHN0ZXA9NCBpcF9yYXRpbz0xLjQwIHBi + X3JhdGlvPTEuMzAgYXE9MACAAAAAJmWIhADoRigACGPHAAED2OAAIeUnJycnJ1111111111111111114AAAAB0GaJADqDMAAAAAHQZ5CQDaD + MAAAAAcBnmFAXQZgAAAABwGeY0BdBmAAAAAIQZpoNEB1BmAAAAAJQZ6GRREobQZgAAAABwGepUBlBmAAAAAHAZ6nQGUGYAAAAAhBmqw0QH0G + YAAAAAlBnspFFShtBmAAAAAHAZ7pQGUGYAAAAAcBnutAZQZgAAAACEGa8DRAfQZgAAAACUGfDkUVKHUGYAAAAAcBny1AZQZgAAAABwGfL0Bt + BmAAAAAIQZs0NEB9BmAAAAAJQZ9SRRUodQZgAAAABwGfcUBtBmAAAAAHAZ9zQG0GYAAAAAhBm3U0QH0GYAAAACdliIIABChGKAAKEscAARlY + 4AAp4ScnJycnXXXXXXXXXXXXXXXXXXgAAAAHQZokAOoMwAAAAAdBnkJANoMwAAAABwGeYUBlBmAAAAAHAZ5jQGUGYAAAAAhBmmg0QHUGYAAA + AAlBnoZFEShtBmAAAAAHAZ6lQGUGYAAAAAcBnqdAZQZgAAAACEGarDRAfQZgAAAACUGeykUVKG0GYAAAAAcBnulAZQZgAAAABwGe60BlBmAA + AAAIQZrwNEB9BmAAAAAJQZ8ORRUodQZgAAAABwGfLUBlBmAAAAAHAZ8vQG0GYAAAAAhBmzQ0QH0GYAAAAAlBn1JFFSh1BmAAAAAHAZ9xQG0G + YAAAAAcBn3NAbQZgAAAACEGbdTRAfQZgAAAAJ2WIhAAQoRigAChLHAAEZWOAAKeEnJycnJ1111111111111111114AAAAAdBmiQA6gzAAAAA + B0GeQkA2gzAAAAAHAZ5hQGUGYAAAAAcBnmNAZQZgAAAACEGaaDRAdQZgAAAACUGehkURKG0GYAAAAAcBnqVAZQZgAAAABwGep0BlBmAAAAAI + QZqsNEB9BmAAAAAJQZ7KRRUobQZgAAAABwGe6UBlBmAAAAAHAZ7rQGUGYAAAAAhBmvA0QH0GYAAAAAlBnw5FFSh1BmAAAAAHAZ8tQGUGYAAA + AAcBny9AbQZgAAAACEGbNDRAfQZgAAAACUGfUkUVKHUGYAAAAAcBn3FAbQZgAAAABwGfc0BtBmAAAAAIQZt1NEB9BmA= + """ } From 1401993acf25e566195c25de89f1719f76353975 Mon Sep 17 00:00:00 2001 From: orut34iop Date: Tue, 25 Aug 2026 20:02:00 +0800 Subject: [PATCH 2/2] fix(video): bound rational cadence evidence --- CHANGELOG.md | 8 +- .../Video/H264CompositionOffsetRepair.swift | 161 ++++++++++++++---- .../H264CompositionOffsetRepairTests.swift | 135 +++++++++++++-- 3 files changed, 251 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9619306..8c234ca9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,10 +18,12 @@ the public-API contract. five-picture step cycle `40040,40041,40040,40040,40041`. The first repair required every decode step to be identical, classified that ladder as nonuniform and repaired no packets. The demuxer now derives an exact rational cadence and unique sampling phase from two repeated STTS periods; - declared and average frame rates only corroborate the observation and never construct timestamps. - A complete dry run must safely place the sampled packets before repair is armed, while ambiguous, + declared and average frame rates only corroborate the observation and never construct timestamps, + and any approximate rate must remain within half a tick across the known full-stream span. A + complete dry run must safely place the sampled packets before repair is armed, while ambiguous, non-repeating and genuinely variable timing remains untouched. Packet and container-index mapping - share the same rational lattice across IDRs and seeks, including bounded one-tick resynchronization. + share the same rational lattice across IDRs and seeks, including unambiguous one-tick + resynchronization. ## [6.42.0] - 2026-08-25 diff --git a/Sources/AetherEngine/Video/H264CompositionOffsetRepair.swift b/Sources/AetherEngine/Video/H264CompositionOffsetRepair.swift index 7b9f5890..bf905ee6 100644 --- a/Sources/AetherEngine/Video/H264CompositionOffsetRepair.swift +++ b/Sources/AetherEngine/Video/H264CompositionOffsetRepair.swift @@ -76,14 +76,58 @@ enum H264CompositionOffsetRepair { var period: Int64 { denominator } /// Stream rates are advisory here: `r_frame_rate` is commonly a rounded nominal rate and - /// `avg_frame_rate` is duration-derived. The STTS cycle remains the exact authority, but at - /// least one declared rate must agree to within one thousandth of a stream tick per frame. - /// Double is used only for this bounded consistency check, never to construct timestamps. - func isConsistent(with metadata: Cadence) -> Bool { + /// `avg_frame_rate` is duration-derived. The STTS cycle remains the exact authority. An + /// approximate declared rate is accepted only when its exact rational error cannot add up + /// to more than half a tick across the known stream span; without that span it fails closed. + func isConsistent(with metadata: Cadence, maximumFrameSpan: Int64?) -> Bool { if self == metadata { return true } - let observed = Double(numerator) / Double(denominator) - let declared = Double(metadata.numerator) / Double(metadata.denominator) - return observed.isFinite && declared.isFinite && abs(observed - declared) <= 0.001 + guard let maximumFrameSpan, maximumFrameSpan > 0 else { return false } + + // 2 * frames * |a/b - c/d| <= 1, evaluated with fixed-width limbs so neither + // cross-multiplication nor the accumulated error can overflow or round toward a pass. + let observedProduct = UInt64(numerator) + .multipliedFullWidth(by: UInt64(metadata.denominator)) + let metadataProduct = UInt64(metadata.numerator) + .multipliedFullWidth(by: UInt64(denominator)) + let difference = Self.absoluteDifference(observedProduct, metadataProduct) + let denominatorProduct = UInt64(denominator) + .multipliedFullWidth(by: UInt64(metadata.denominator)) + let (scale, scaleOverflow) = UInt64(maximumFrameSpan) + .multipliedReportingOverflow(by: 2) + guard !scaleOverflow else { return false } + + let lowProduct = difference.low.multipliedFullWidth(by: scale) + let highProduct = difference.high.multipliedFullWidth(by: scale) + let (middle, middleCarry) = lowProduct.high + .addingReportingOverflow(highProduct.low) + let (top, topOverflow) = highProduct.high + .addingReportingOverflow(middleCarry ? 1 : 0) + guard !topOverflow, top == 0 else { return false } + return Self.isLessThanOrEqual( + (high: middle, low: lowProduct.low), + denominatorProduct + ) + } + + private static func absoluteDifference( + _ lhs: (high: UInt64, low: UInt64), + _ rhs: (high: UInt64, low: UInt64) + ) -> (high: UInt64, low: UInt64) { + let (larger, smaller) = isLessThanOrEqual(lhs, rhs) ? (rhs, lhs) : (lhs, rhs) + let (low, borrow) = larger.low.subtractingReportingOverflow(smaller.low) + let (partialHigh, highUnderflow) = larger.high + .subtractingReportingOverflow(smaller.high) + let (high, borrowUnderflow) = partialHigh + .subtractingReportingOverflow(borrow ? 1 : 0) + precondition(!highUnderflow && !borrowUnderflow) + return (high, low) + } + + private static func isLessThanOrEqual( + _ lhs: (high: UInt64, low: UInt64), + _ rhs: (high: UInt64, low: UInt64) + ) -> Bool { + lhs.high < rhs.high || (lhs.high == rhs.high && lhs.low <= rhs.low) } /// `round_near_away(frameOrdinal * numerator / denominator)`, without overflowing an Int64 @@ -205,7 +249,11 @@ enum H264CompositionOffsetRepair { videoDelay: Int64, pocStep: Int64 ) { - guard rawPhase >= 0, rawPhase < cadence.period, + // With fewer than two ticks per frame, a tolerated one-tick container defect can be the + // exact timestamp of an adjacent ordinal. Packets and indexes cannot distinguish those + // meanings, so this cadence is not safe to repair at all. + guard cadence.floorStep >= 2, + rawPhase >= 0, rawPhase < cadence.period, rawFrameOffset == 0 || rawFrameOffset == -videoDelay, videoDelay > 0, let oneFrame = cadence.timestamp(at: 1), @@ -379,7 +427,9 @@ enum H264CompositionOffsetRepair { videoDelay: Int, streamStartTime: Int64, ladderStart: Int64, - cadenceCandidates: [Cadence] = [] + streamFrameCount: Int64 = 0, + averageCadence: Cadence? = nil, + nominalCadence: Cadence? = nil ) -> Verdict { guard videoDelay > 0, videoDelay <= 16 else { return .inconclusive("reorder delay \(videoDelay) outside 1...16") @@ -431,7 +481,13 @@ enum H264CompositionOffsetRepair { guard let observed = observedCadence( decodeSteps: decodeSteps, - metadataCandidates: cadenceCandidates + maximumFrameSpan: maximumFrameSpan( + streamFrameCount: streamFrameCount, + videoDelay: videoDelay + ), + // avg_frame_rate reflects the complete stream and must veto a short sampled alias. + // r_frame_rate is only a fallback when that stronger evidence is absent. + metadataCadence: averageCadence ?? nominalCadence ), let firstDTS = samples.first?.dts else { return .inconclusive("decode ladder is not uniform") } @@ -547,9 +603,25 @@ enum H264CompositionOffsetRepair { return remainder >= 0 ? remainder : remainder + modulus } + private static func maximumFrameSpan( + streamFrameCount: Int64, + videoDelay: Int + ) -> Int64? { + // AVStream.duration may be estimated, and converting it with a step sampled only from the + // head would assume the very full-stream CFR property this gate is meant to prove. Only the + // container's explicit frame count is strong enough to bound accumulated cadence error. + guard streamFrameCount > 0 else { return nil } + let (withReorderHead, headOverflow) = streamFrameCount + .addingReportingOverflow(Int64(videoDelay)) + let (conservativeSpan, safetyOverflow) = withReorderHead.addingReportingOverflow(2) + guard !headOverflow, !safetyOverflow, conservativeSpan > 0 else { return nil } + return conservativeSpan + } + private static func observedCadence( decodeSteps: [Int64], - metadataCandidates: [Cadence] + maximumFrameSpan: Int64?, + metadataCadence: Cadence? ) -> (cadence: Cadence, phase: Int64)? { let maximumPeriod = decodeSteps.count / 2 guard maximumPeriod >= 2 else { return nil } @@ -571,7 +643,11 @@ enum H264CompositionOffsetRepair { numerator: periodTicks, denominator: Int64(period) ), cadence.period == Int64(period), - metadataCandidates.contains(where: { cadence.isConsistent(with: $0) }) else { + let metadataCadence, + cadence.isConsistent( + with: metadataCadence, + maximumFrameSpan: maximumFrameSpan + ) else { continue } @@ -705,7 +781,21 @@ enum H264CompositionOffsetRepair { unrepairedPictures += 1 return nil } - if let exactDecodeOrdinal { + guard let expectedTimestamp = plan.rawTimestamp(decodeOrdinal: expected) else { + unrepairedPictures += 1 + return nil + } + if Self.isWithinOneTick(dts, of: expectedTimestamp) { + // Near the expected point, tolerance is safe only when the entire +/-1 window + // identifies that one ordinal. A dense cadence may place an adjacent exact + // lattice point in the same window, in which case choosing either would be a + // silent one-frame jump. + guard plan.decodeOrdinal(forRawTimestampWithinOneTick: dts) == expected else { + unrepairedPictures += 1 + return nil + } + decodeOrdinal = expected + } else if let exactDecodeOrdinal { // An exact forward ordinal is stronger than packet counting: it preserves a // legitimate gap and resynchronizes after a preceding parser miss. A backward // exact timestamp is a discontinuity this session was not told about. @@ -715,19 +805,12 @@ enum H264CompositionOffsetRepair { } decodeOrdinal = exactDecodeOrdinal } else { - // The sampled stream proved a CFR lattice. A later one-tick timestamp defect is - // therefore rectified to the next decode position instead of being emitted raw. - guard let expectedTimestamp = plan.rawTimestamp(decodeOrdinal: expected), - Self.isWithinOneTick(dts, of: expectedTimestamp) else { - unrepairedPictures += 1 - return nil - } - decodeOrdinal = expected + unrepairedPictures += 1 + return nil } mayAdvanceAfterUnplacedPicture = decodeOrdinal == expected } else { - let landingOrdinal = exactDecodeOrdinal - ?? plan.decodeOrdinal(forRawTimestampWithinOneTick: dts) + let landingOrdinal = plan.decodeOrdinal(forRawTimestampWithinOneTick: dts) guard awaitingReanchor, isKeyframe, let landingOrdinal else { unrepairedPictures += 1 return nil @@ -867,7 +950,9 @@ final class H264CompositionOffsetRepairSession { private let videoDelay: Int private let streamStartTime: Int64 private let ladderStart: Int64 - private let cadenceCandidates: [H264CompositionOffsetRepair.Cadence] + private let streamFrameCount: Int64 + private let averageCadence: H264CompositionOffsetRepair.Cadence? + private let nominalCadence: H264CompositionOffsetRepair.Cadence? private var reader: H264PictureOrderReader? private var rewriter: H264CompositionOffsetRepair.Rewriter? private var samples: [H264CompositionOffsetRepair.Sample] = [] @@ -895,18 +980,18 @@ final class H264CompositionOffsetRepairSession { self.videoDelay = Int(codecpar.pointee.video_delay) self.streamStartTime = stream.pointee.start_time self.ladderStart = ladderStart - var cadences: [H264CompositionOffsetRepair.Cadence] = [] - // r_frame_rate is the coded cadence. avg_frame_rate is duration-derived and may be perturbed - // by head/tail quantization, but remains a bounded fallback for containers that omit r_frame_rate; - // classification still requires an exact two-period match before either candidate is trusted. - for frameRate in [stream.pointee.r_frame_rate, stream.pointee.avg_frame_rate] { - guard let cadence = H264CompositionOffsetRepair.Cadence( - timeBase: stream.pointee.time_base, - frameRate: frameRate - ), !cadences.contains(cadence) else { continue } - cadences.append(cadence) - } - cadenceCandidates = cadences + self.streamFrameCount = stream.pointee.nb_frames + // avg_frame_rate summarizes the complete stream and can expose a long-period cadence that a + // short STTS prefix aliases. r_frame_rate is a nominal coded rate, so it is only a fallback + // when the average is unavailable; classification still requires two exact sampled periods. + averageCadence = H264CompositionOffsetRepair.Cadence( + timeBase: stream.pointee.time_base, + frameRate: stream.pointee.avg_frame_rate + ) + nominalCadence = H264CompositionOffsetRepair.Cadence( + timeBase: stream.pointee.time_base, + frameRate: stream.pointee.r_frame_rate + ) self.reader = reader } @@ -1034,7 +1119,9 @@ final class H264CompositionOffsetRepairSession { videoDelay: videoDelay, streamStartTime: streamStartTime, ladderStart: ladderStart, - cadenceCandidates: cadenceCandidates + streamFrameCount: streamFrameCount, + averageCadence: averageCadence, + nominalCadence: nominalCadence ) switch verdict { case .repair(let plan): diff --git a/Tests/AetherEngineTests/H264CompositionOffsetRepairTests.swift b/Tests/AetherEngineTests/H264CompositionOffsetRepairTests.swift index e9621797..cb24caf2 100644 --- a/Tests/AetherEngineTests/H264CompositionOffsetRepairTests.swift +++ b/Tests/AetherEngineTests/H264CompositionOffsetRepairTests.swift @@ -67,6 +67,10 @@ struct H264CompositionOffsetRepairTests { )! } + /// A conservative, identity-free upper bound for the affected source. Even across this many + /// frames, the duration-derived rate and the recovered five-frame cadence differ by < 0.5 tick. + private var physicalValidationFrameCount: Int64 { 600_000 } + private func verdict( _ samples: [H264CompositionOffsetRepair.Sample], videoDelay: Int = 2, @@ -128,7 +132,7 @@ struct H264CompositionOffsetRepairTests { let result = H264CompositionOffsetRepair.classify( samples: samples, videoDelay: 2, streamStartTime: 0, ladderStart: samples[0].dts, - cadenceCandidates: [quantizedCadence]) + averageCadence: quantizedCadence) if case .repair(let plan) = result { #expect(plan.cadence == quantizedCadence) #expect(plan.rawFrameOffset == -2) @@ -156,7 +160,7 @@ struct H264CompositionOffsetRepairTests { #expect(H264CompositionOffsetRepair.classify( samples: samples, videoDelay: 2, streamStartTime: 0, ladderStart: samples[0].dts, - cadenceCandidates: [quantizedCadence]) + averageCadence: quantizedCadence) == .inconclusive("decode ladder is not uniform")) } @@ -177,10 +181,12 @@ struct H264CompositionOffsetRepairTests { videoDelay: 1, streamStartTime: 0, ladderStart: -40040, - cadenceCandidates: [ - H264CompositionOffsetRepair.Cadence(numerator: 40040, denominator: 1)!, - physicalAverageCadence, - ] + streamFrameCount: physicalValidationFrameCount, + averageCadence: physicalAverageCadence, + nominalCadence: H264CompositionOffsetRepair.Cadence( + numerator: 40040, + denominator: 1 + )! ) guard case .repair(let physicalPlan) = result else { #expect(Bool(false), "the repeated physical STTS phase must be repairable") @@ -208,6 +214,72 @@ struct H264CompositionOffsetRepairTests { #expect(rewriter.unrepairedPictures == 0) } + @Test("a long-period cadence prefix cannot masquerade as a shorter repeated cycle") + func rejectsLongPeriodCadenceAlias() { + let declaredCadence = H264CompositionOffsetRepair.Cadence( + numerator: 335_344_009, + denominator: 10_000 + )! + let phase: Int64 = 2 + let phaseTimestamp = declaredCadence.timestamp(at: phase)! + let pocs: [Int64] = [0, 4, 2, 6, 8, 12, 10, 14, 16, 20, 18, 22] + let samples = pocs.enumerated().map { index, poc in + let timestamp = declaredCadence.timestamp(at: phase + Int64(index))! - phaseTimestamp + return H264CompositionOffsetRepair.Sample( + dts: timestamp, + pts: timestamp, + pictureOrderCount: poc, + isKeyframe: index == 0 + ) + } + let observedSteps = zip(samples, samples.dropFirst()).map { $1.dts - $0.dts } + #expect(Array(observedSteps.prefix(10)) == [ + 33534, 33535, 33534, 33534, 33535, + 33534, 33535, 33534, 33534, 33535, + ]) + let aliasedNominalCadence = H264CompositionOffsetRepair.Cadence( + numerator: 167_672, + denominator: 5 + )! + #expect(H264CompositionOffsetRepair.classify( + samples: samples, + videoDelay: 1, + streamStartTime: 0, + ladderStart: 0, + streamFrameCount: 2_000, + averageCadence: declaredCadence, + nominalCadence: aliasedNominalCadence + ) == .inconclusive("decode ladder is not uniform")) + } + + @Test("cadence corroboration uses an exact half-tick full-stream drift bound") + func cadenceConsistencyUsesCumulativeBound() { + let shortAlias = H264CompositionOffsetRepair.Cadence( + numerator: 167_672, + denominator: 5 + )! + let longPeriodRate = H264CompositionOffsetRepair.Cadence( + numerator: 335_344_009, + denominator: 10_000 + )! + #expect(shortAlias.isConsistent(with: longPeriodRate, maximumFrameSpan: 555)) + #expect(!shortAlias.isConsistent(with: longPeriodRate, maximumFrameSpan: 556)) + #expect(!shortAlias.isConsistent(with: longPeriodRate, maximumFrameSpan: nil)) + #expect(shortAlias.isConsistent(with: shortAlias, maximumFrameSpan: nil)) + } + + @Test("an approximate cadence without a reliable stream span fails closed") + func approximateCadenceRequiresStreamSpan() { + let samples = physicalQuantizedSamples() + #expect(H264CompositionOffsetRepair.classify( + samples: samples, + videoDelay: 1, + streamStartTime: 0, + ladderStart: -40040, + averageCadence: physicalAverageCadence + ) == .inconclusive("decode ladder is not uniform")) + } + @Test("classification refuses a POC depth that the declared reorder delay cannot rewrite") func rejectsUnsafePhysicalPlanBeforeRepairStarts() { var samples = physicalQuantizedSamples() @@ -220,7 +292,8 @@ struct H264CompositionOffsetRepairTests { videoDelay: 1, streamStartTime: 0, ladderStart: -40040, - cadenceCandidates: [physicalAverageCadence] + streamFrameCount: physicalValidationFrameCount, + averageCadence: physicalAverageCadence ) == .inconclusive("sample cannot be rewritten safely")) } @@ -235,7 +308,8 @@ struct H264CompositionOffsetRepairTests { videoDelay: 1, streamStartTime: 0, ladderStart: -40040, - cadenceCandidates: [physicalAverageCadence] + streamFrameCount: physicalValidationFrameCount, + averageCadence: physicalAverageCadence ) guard case .repair(let physicalPlan) = result else { #expect(Bool(false), "expected the physical rational plan") @@ -270,7 +344,8 @@ struct H264CompositionOffsetRepairTests { videoDelay: 1, streamStartTime: 0, ladderStart: -40040, - cadenceCandidates: [physicalAverageCadence] + streamFrameCount: physicalValidationFrameCount, + averageCadence: physicalAverageCadence ) guard case .repair(let physicalPlan) = result else { #expect(Bool(false), "expected the physical rational plan") @@ -307,7 +382,8 @@ struct H264CompositionOffsetRepairTests { videoDelay: 1, streamStartTime: 0, ladderStart: -40040, - cadenceCandidates: [physicalAverageCadence] + streamFrameCount: physicalValidationFrameCount, + averageCadence: physicalAverageCadence ) guard case .repair(let physicalPlan) = result else { #expect(Bool(false), "expected the physical rational plan") @@ -347,7 +423,8 @@ struct H264CompositionOffsetRepairTests { videoDelay: 1, streamStartTime: 0, ladderStart: -40040, - cadenceCandidates: [physicalAverageCadence] + streamFrameCount: physicalValidationFrameCount, + averageCadence: physicalAverageCadence ) guard case .repair(let physicalPlan) = result else { #expect(Bool(false), "expected the physical rational plan") @@ -388,7 +465,8 @@ struct H264CompositionOffsetRepairTests { videoDelay: 1, streamStartTime: 0, ladderStart: -40040, - cadenceCandidates: [physicalAverageCadence] + streamFrameCount: physicalValidationFrameCount, + averageCadence: physicalAverageCadence ) guard case .repair(let physicalPlan) = result else { #expect(Bool(false), "expected the physical rational plan") @@ -527,7 +605,7 @@ struct H264CompositionOffsetRepairTests { let result = H264CompositionOffsetRepair.classify( samples: samples, videoDelay: 2, streamStartTime: 0, ladderStart: samples[0].dts, - cadenceCandidates: [quantizedCadence]) + averageCadence: quantizedCadence) guard case .repair(let rationalPlan) = result else { #expect(Bool(false), "expected a rational repair plan") return @@ -545,6 +623,37 @@ struct H264CompositionOffsetRepairTests { #expect(rewriter.unrepairedPictures == 0) } + @Test("a dense cadence whose adjacent ordinals overlap one-tick tolerance is not repairable") + func denseCadenceAmbiguityFailsClosed() { + let denseCadence = H264CompositionOffsetRepair.Cadence(numerator: 3, denominator: 2)! + #expect(H264CompositionOffsetRepair.Plan( + cadence: denseCadence, + rawTimestampAnchor: 0, + rawPhase: 0, + rawFrameOffset: 0, + videoDelay: 1, + pocStep: 2 + ) == nil) + + let pocs: [Int64] = [0, 4, 2, 6, 8, 12, 10, 14, 16, 20, 18, 22] + let samples = pocs.enumerated().map { index, poc in + let timestamp = denseCadence.timestamp(at: Int64(index))! + return H264CompositionOffsetRepair.Sample( + dts: timestamp, + pts: timestamp, + pictureOrderCount: poc, + isKeyframe: index == 0 + ) + } + #expect(H264CompositionOffsetRepair.classify( + samples: samples, + videoDelay: 1, + streamStartTime: 0, + ladderStart: 0, + averageCadence: denseCadence + ) == .inconclusive("decode ladder is not uniform")) + } + @Test("a rational presentation-axis ladder maps indexes with the same phase as packets") func rationalIndexTimestampMapping() { let presentationAxisPlan = H264CompositionOffsetRepair.Plan(