Skip to content

kernel: Decode H.264 temporal direct pictures with the right references #111

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
henrikklee merged 1 commit into from
Nov 12, 2026
Merged

kernel: Decode H.264 temporal direct pictures with the right references #111

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 152 additions & 0 deletions .../drivers/media/0010-media-apple-vxd-Resolve-temporal-direct-references-by-timestamp.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
From 3ddce65f7c19a80e93ff87d56efb11761357ebe5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Berkay=20K=C3=B6sebay?= <brkosebay@users.noreply.github.com>
Date: Wed, 11 Nov 2026 14:26:26 +0100
Subject: [PATCH] media: apple-vxd: Resolve temporal direct references by
timestamp

The inverse L0 table is indexed by the DPB slots recorded in the
co-located picture's stored motion. Building it with the current
request's slots assumes that userspace preserves dpb[] ordering, which
the stateless H.264 API does not require. FFmpeg compacts its DPB and
moves the past reference when inserting the future P picture, causing
temporal direct B pictures to use the wrong reference.

Save each picture's slot-to-CAPTURE timestamp map alongside its stored
motion buffer. Resolve the saved slots against the current L0 timestamps
when emitting the inverse table. Leave missing references unmapped and
replace the snapshot whenever the CAPTURE buffer is decoded again.

[Orchard: from orchard PR #111, commit 883f1eaf; unchanged]

Assisted-by: LLM
---
.../platform/apple/vxd/apple-vxd-h264-cmd.c | 25 +++++++++++++++----
.../platform/apple/vxd/apple-vxd-h264-cmd.h | 3 +++
.../media/platform/apple/vxd/apple-vxd-h264.c | 18 +++++++++++++
drivers/media/platform/apple/vxd/apple-vxd.h | 3 +++
4 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.c b/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.c
index c092a3f49..d559b7840 100644
--- a/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.c
+++ b/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.c
@@ -268,7 +268,7 @@ static void vxd_h264_emit_b(struct apple_vxd_cmdbuf *cb,
{
const struct v4l2_ctrl_h264_decode_params *dp = f->dp;
u8 inverse[V4L2_H264_REF_LIST_LEN];
- unsigned int i, n;
+ unsigned int i, j, n;
u8 col = l1[0];

w[0] = vxd_h264_poc(dp->top_field_order_cnt);
@@ -299,11 +299,26 @@ static void vxd_h264_emit_b(struct apple_vxd_cmdbuf *cb,
w[1] = APPLE_VXD_H264_UNUSED_VA;
}

- /* Inverse L0: first position of each slot (bottom fields at +16). */
+ /*
+ * Stored motion names slots from the co-located picture's DPB, not
+ * this request's DPB. Userspace may reorder dpb[] between pictures.
+ * Translate those saved slots to the current L0 by CAPTURE timestamp.
+ * References absent from L0 stay NO_REF; descending order keeps the
+ * first position when a list contains a reference more than once.
+ */
memset(inverse, APPLE_VXD_H264_NO_REF, sizeof(inverse));
- for (i = V4L2_H264_REF_LIST_LEN; i--; )
- if (l0[i] != APPLE_VXD_H264_NO_REF)
- inverse[(l0[i] >> 3 & 0x10) | (l0[i] & 0x1f)] = i;
+ if (col < V4L2_H264_NUM_DPB_ENTRIES && f->dpb_ref_ts[col]) {
+ for (i = V4L2_H264_REF_LIST_LEN; i--; ) {
+ u8 slot = l0[i];
+
+ if (slot >= V4L2_H264_NUM_DPB_ENTRIES)
+ continue;
+ for (j = 0; j < V4L2_H264_NUM_DPB_ENTRIES; j++)
+ if ((f->dpb_ref_valid[col] & BIT(j)) &&
+ f->dpb_ref_ts[col][j] == dp->dpb[slot].reference_ts)
+ inverse[j] = i;
+ }
+ }
vxd_h264_pack_list(inverse, V4L2_H264_REF_LIST_LEN, &w[2]);
n = vxd_h264_pack_list(l1, l1_minus1 + 1, &w[10]);
apple_vxd_cmd_block(cb, VXD_H264_BE_COL_PIC0, 0, w, 10 + n);
diff --git a/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.h b/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.h
index 192678d47..66e836946 100644
--- a/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.h
+++ b/drivers/media/platform/apple/vxd/apple-vxd-h264-cmd.h
@@ -54,6 +54,9 @@ struct apple_vxd_h264_frame {
u32 dpb_chroma[V4L2_H264_NUM_DPB_ENTRIES];
/* Each reference's stored-MB buffer: the B co-located picture's. */
u32 dpb_colocated[V4L2_H264_NUM_DPB_ENTRIES];
+ /* Saved DPB slot identities belonging to each stored-MB buffer. */
+ const u64 *dpb_ref_ts[V4L2_H264_NUM_DPB_ENTRIES];
+ u16 dpb_ref_valid[V4L2_H264_NUM_DPB_ENTRIES];
struct apple_vxd_h264_bufs bufs;
bool first_picture; /* first picture of the context */
};
diff --git a/drivers/media/platform/apple/vxd/apple-vxd-h264.c b/drivers/media/platform/apple/vxd/apple-vxd-h264.c
index ca289e11d..c6c9df417 100644
--- a/drivers/media/platform/apple/vxd/apple-vxd-h264.c
+++ b/drivers/media/platform/apple/vxd/apple-vxd-h264.c
@@ -319,6 +319,7 @@ static int apple_vxd_h264_run(struct apple_vxd_ctx *ctx)
struct apple_vxd_run run;
const char *why = NULL;
struct vb2_queue *cap_q;
+ struct apple_vxd_buffer *dst;
unsigned int i, nslices;
const u8 *src;
size_t src_len;
@@ -406,6 +407,7 @@ static int apple_vxd_h264_run(struct apple_vxd_ctx *ctx)
for (i = 0; i < V4L2_H264_NUM_DPB_ENTRIES; i++) {
const struct v4l2_h264_dpb_entry *e = &dp->dpb[i];
struct vb2_buffer *ref;
+ struct apple_vxd_buffer *buf;

if (!(e->flags & V4L2_H264_DPB_ENTRY_FLAG_VALID))
continue;
@@ -418,6 +420,10 @@ static int apple_vxd_h264_run(struct apple_vxd_ctx *ctx)
f.dpb_luma[i] = apple_vxd_buf_va(to_vb2_v4l2_buffer(ref));
f.dpb_chroma[i] = f.dpb_luma[i] + stride * pix->height;
f.dpb_colocated[i] = apple_vxd_buf_aux_va(to_vb2_v4l2_buffer(ref));
+ buf = container_of(to_vb2_v4l2_buffer(ref), struct apple_vxd_buffer,
+ base.vb);
+ f.dpb_ref_ts[i] = buf->h264_ref_ts;
+ f.dpb_ref_valid[i] = buf->h264_ref_valid;
}
why = apple_vxd_h264_check_refs(h, nslices, &f);
if (why) {
@@ -435,6 +441,18 @@ static int apple_vxd_h264_run(struct apple_vxd_ctx *ctx)
if (ret)
goto out_why;

+ /* Keep the slot numbering used by this picture with its stored motion. */
+ dst = container_of(run.dst, struct apple_vxd_buffer, base.vb);
+ dst->h264_ref_valid = 0;
+ for (i = 0; i < V4L2_H264_NUM_DPB_ENTRIES; i++) {
+ const struct v4l2_h264_dpb_entry *e = &dp->dpb[i];
+
+ dst->h264_ref_ts[i] = e->reference_ts;
+ if ((e->flags & V4L2_H264_DPB_ENTRY_FLAG_VALID) &&
+ (e->flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE))
+ dst->h264_ref_valid |= BIT(i);
+ }
+
dev_dbg(vxd->dev,
"decode: src %u va %#x (%zu bytes, %u slices, first %c NAL %#x+%u, header %u bits), dst %u Y %#x UV %#x stride %u, %zu command bytes in buf %u (va %#x), %u packets\n",
run.src->vb2_buf.index, f.src_va, src_len, nslices,
diff --git a/drivers/media/platform/apple/vxd/apple-vxd.h b/drivers/media/platform/apple/vxd/apple-vxd.h
index 1d3b74382..7922ebb73 100644
--- a/drivers/media/platform/apple/vxd/apple-vxd.h
+++ b/drivers/media/platform/apple/vxd/apple-vxd.h
@@ -250,6 +250,9 @@ struct apple_vxd_buffer {
* register 0x62ec), allocated with the buffer.
*/
struct apple_vxd_dev_buf aux;
+ /* DPB identity when this CAPTURE's H.264 stored motion was written. */
+ u64 h264_ref_ts[V4L2_H264_NUM_DPB_ENTRIES];
+ u16 h264_ref_valid;
};

/*
--
2.49.0
Loading