media: apple: stage AV1 frame ops and preserve open decode

drivers/media/platform/apple/avd/Kconfig

@@ -24,8 +24,7 @@ config VIDEO_APPLE_AVD_KUNIT_TEST
 	default KUNIT_ALL_TESTS
 	help
 	  Build KUnit tests for the Apple video decoder. They exercise the
-	  checks that the H.264 path makes on the slices that userspace
-	  queues, such as finding the slice data after the slice header,
-	  at the limits and one past them.
+	  H.264 slice bounds, frame-slot ownership, and runtime power management,
+	  including suspend gating and resubmission after firmware resume.
 
 	  If unsure, say N.

drivers/media/platform/apple/avd/Makefile

@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
-apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-h264.o avd-vp9.o avd-hevc.o \
-	avd-av1.o avd-av1-entropymode.o
-apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o
+apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-pm.o avd-h264.o avd-vp9.o \
+	avd-hevc.o avd-av1.o avd-av1-fp.o avd-av1-entropymode.o avd-queue.o avd-backend-open.o \
+	avd-backend-apple.o avd-backend-queue.o
+apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o avd-queue-test.o \
+	avd-pm-test.o
 obj-$(CONFIG_VIDEO_APPLE_AVD) += apple-avd.o

drivers/media/platform/apple/avd/avd-apple-protocol.h

@@ -0,0 +1,141 @@
+/* SPDX-License-Identifier: GPL-2.0-only OR MIT */
+#ifndef AVD_APPLE_PROTOCOL_H_
+#define AVD_APPLE_PROTOCOL_H_
+
+#include <linux/errno.h>
+#include <linux/types.h>
+
+#include "avd-queue.h"
+
+#define AVD_APPLE_INBOX_BASE 0x01482cf8
+#define AVD_APPLE_COMMAND_SIZE 0x60
+#define AVD_APPLE_RESULT_WORDS 20
+#define AVD_APPLE_SHADOW_SIZE 64
+
+/*
+ * [C] Expected address notification, from the MIT V3 trace_avd.py precedent.
+ * Keep this rule in one place for correction by the Borage capture.
+ */
+#define AVD_APPLE_NOTIFY_BASE AVD_APPLE_INBOX_BASE
+
+/* SPEC 5.1: only these session opcodes have proved builders. */
+static inline int
+avd_apple_session_command(u32 cmd[24], u32 client, u32 opcode,
+			  u32 property, u32 value)
+{
+	unsigned int i;
+
+	if (client >= 1024 || (opcode != 0 && opcode != 2 && opcode != 5 && opcode != 6))
+		return -EINVAL;
+	for (i = 0; i < 24; i++)
+		cmd[i] = 0;
+	cmd[0] = opcode | (client << 16);
+	if (opcode == 6) {
+		cmd[1] = property;
+		cmd[2] = value;
+	}
+	return 0;
+}
+
+static inline int avd_apple_decode_prefix(struct avd_frame *f, u32 client,
+					  u32 selector, u32 work_sequence)
+{
+	unsigned int i;
+
+	if (f->idx >= AVD_MAX_FRAMES || client >= 1024 || selector >= 16 || work_sequence >= 4)
+		return -EINVAL;
+	for (i = 0; i < 24; i++)
+		f->cmd[i] = 0;
+	f->cmd[0] = 1 | (selector << 10) | (client << 22);
+	f->cmd[1] = f->idx | (work_sequence << 16);
+	f->cmd[3] = f->frame_num;
+	return 0;
+}
+
+struct avd_apple_reply_layout {
+	u8 type_word;
+	u8 type_shift;
+	u32 type_mask;
+	u8 client_word;
+	u8 client_shift;
+	u32 client_mask;
+	u8 frame_word;
+	u8 status_word;
+	u32 done_type;
+	bool verified;
+};
+
+struct avd_apple_reply {
+	u32 type;
+	u32 client;
+	u32 frame;
+	int status;
+};
+
+static inline int
+avd_apple_parse_reply(const u32 *record, size_t words,
+		      const struct avd_apple_reply_layout *layout,
+		      struct avd_apple_reply *reply)
+{
+	if (words != AVD_APPLE_RESULT_WORDS)
+		return -EMSGSIZE;
+	if (!layout->verified)
+		return -EOPNOTSUPP;
+	if (layout->type_word >= words || layout->client_word >= words ||
+	    layout->frame_word >= words || layout->status_word >= words ||
+	    layout->type_shift >= 32 || layout->client_shift >= 32 ||
+	    !layout->type_mask || !layout->client_mask || layout->done_type == 7)
+		return -EINVAL;
+	reply->type = (record[layout->type_word] >> layout->type_shift) & layout->type_mask;
+	reply->client = (record[layout->client_word] >> layout->client_shift) & layout->client_mask;
+	reply->frame = record[layout->frame_word];
+	/* SPEC 4.3: type 7 is parsing/VP done, never final buffer ownership. */
+	if (reply->type == 7)
+		return -EINPROGRESS;
+	if (reply->type != layout->done_type)
+		return -ENOMSG;
+	reply->status = record[layout->status_word] ? -EIO : 0;
+	return 0;
+}
+
+static inline u32 avd_apple_notification(unsigned int slot)
+{
+	return AVD_APPLE_NOTIFY_BASE + AVD_APPLE_COMMAND_SIZE * slot;
+}
+
+struct avd_apple_hw {
+	const char *fw_name;
+	/* All offsets below are relative to a DT-owned "apple" resource. */
+	u32 aperture_origin;
+	u32 imem;
+	u32 imem_size;
+	u32 version_offset;
+	u32 itcm_start;
+	u32 itcm_end;
+	u32 itcm_clear;
+	u32 itcm_clear_value;
+	u32 itcm_enable;
+	u32 itcm_enable_value;
+	u32 link_setup;
+	u32 link_setup_value;
+	u32 link_state;
+	u32 link_up;
+	u32 link_up_value;
+	u32 ready;
+	u32 notify;
+	u32 receive;
+	u32 receive_status;
+	u32 receive_pending;
+	u32 receive_ack;
+	u32 receive_ack_value;
+	u32 inbox_consumer;
+	u32 result_base;
+	struct avd_apple_reply_layout reply;
+	/* Match-data fact, never a module parameter or a userspace setting. */
+	bool verified;
+};
+
+extern const struct avd_apple_hw avd_borage_hw;
+extern const u8 avd_borage_config[64];
+
+#endif /* AVD_APPLE_PROTOCOL_H_ */

drivers/media/platform/apple/avd/avd-av1-fp.c

@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * AV1 Apple frame parameters. Source: Apple AVD protocol SPEC (2026-10-31),
+ * sections 5, 6.1, 6.5, 6.6, 10. No AV1 layout exists in the public V3 code.
+ * Keep unproved fields unnamed and refuse submission until they are settled.
+ */
+#include <linux/unaligned.h>
+#include "avd.h"
+#include "avd-av1-fp.h"
+
+#define BORAGE_FP_SIZE 0x1130d0
+#define BORAGE_WORK_CAP 0x1002
+
+/*
+ * [C] table: source                         destination / unresolved rule
+ * SPEC 6.1 fp+0x18,+0x19                   cmd+0x00 bits 0,1; meaning
+ * SPEC 6.5 fp+0x4f4                        cmd+0x00 bits 2:6,18:19;
+ *                                         selected source bits unspecified
+ * SPEC 6.5 fp+0x4f4                        cmd+0x00 bits 7:11,13:17;
+ *                                         source shifts unspecified
+ * SPEC 6.5 work array at fp+0x1c           cmd+0x1c; array encoding
+ * SPEC 6.5 fp+0x4fc,+0x5e0,0x01481780      cmd+0x3c,+0x44; packed addresses
+ * SPEC 6.5 fp+0x4f8                        cmd+0x4c; source meaning
+ * SPEC 6.6 sequence/frame/tile/grain       syntax/DVA offsets still open
+ * SPEC 10  T8140                          TODO: identify selected packer;
+ *                                         Daybreak per-SoC ledger task
+ *                                         t-3862-daybreak-avd-per-soc-ledger-
+ *                                         t8140-t6000-t8112-pe
+ */
+
+int avd_av1_validate_tiles(const struct v4l2_ctrl_av1_frame *frame,
+			   const struct v4l2_ctrl_av1_tile_group_entry *tiles,
+			   unsigned int count, size_t payload)
+{
+	const struct v4l2_av1_tile_info *info = &frame->tile_info;
+	unsigned int i, n;
+
+	if (!info->tile_cols || !info->tile_rows ||
+	    info->tile_cols > V4L2_AV1_MAX_TILE_COLS ||
+	    info->tile_rows > V4L2_AV1_MAX_TILE_ROWS)
+		return -EINVAL;
+	n = info->tile_cols * info->tile_rows;
+	if (n > V4L2_AV1_MAX_TILE_COUNT || n > BORAGE_WORK_CAP ||
+	    count < n || count > V4L2_AV1_MAX_TILE_COUNT ||
+	    info->context_update_tile_id >= n)
+		return -EINVAL;
+	for (i = 0; i < n; i++) {
+		if (tiles[i].tile_row != i / info->tile_cols ||
+		    tiles[i].tile_col != i % info->tile_cols ||
+		    !tiles[i].tile_size || tiles[i].tile_offset > payload ||
+		    tiles[i].tile_size > payload - tiles[i].tile_offset)
+			return -EINVAL;
+		if (i && tiles[i].tile_offset <
+		    (u64)tiles[i - 1].tile_offset + tiles[i - 1].tile_size)
+			return -EINVAL;
+	}
+	return 0;
+}
+
+int avd_av1_pack_fp(struct avd_ctx *ctx, struct avd_frame *f,
+		    const struct v4l2_ctrl_av1_sequence *seq,
+		    const struct v4l2_ctrl_av1_frame *frame,
+		    const struct v4l2_ctrl_av1_tile_group_entry *tiles,
+		    const struct v4l2_ctrl_av1_film_grain *grain)
+{
+	u8 *fp;
+
+	if (ctx->dev->variant->av1_fp_layout != AVD_AV1_FP_BORAGE)
+		return -EOPNOTSUPP;
+	if (!f->fp || !f->fp->cpu || f->fp->size < BORAGE_FP_SIZE)
+		return -ENOSPC;
+	fp = f->fp->cpu;
+	memset(fp, 0, f->fp->size);
+	put_unaligned_le32(f->frame_num, fp + 0x10); /* SPEC 5.2 */
+	put_unaligned_le16(frame->tile_info.tile_cols, fp + 0x3e4);
+	put_unaligned_le16(frame->tile_info.tile_rows, fp + 0x3e6);
+	/* SPEC 6.6 does not locate seq/frame/tiles/grain syntax or surface DVAs.
+	 * A native V4L2 struct or Linux instruction stream is not an Apple fp. */
+	return -EOPNOTSUPP;
+}
+
+int avd_av1_pack_cmd(struct avd_ctx *ctx, struct avd_frame *f)
+{
+	u8 *cmd = (u8 *)f->cmd;
+	const u8 *fp;
+	u32 work, base;
+
+	if (ctx->dev->variant->av1_fp_layout != AVD_AV1_FP_BORAGE)
+		return -EOPNOTSUPP;
+	if (!f->fp || !f->fp->cpu || f->fp->size < BORAGE_FP_SIZE)
+		return -ENOSPC;
+	fp = f->fp->cpu;
+	/* Work index is the modulo-four sequence from the core prefix. */
+	work = get_unaligned_le16(cmd + 6);
+	if (work >= 4 || !get_unaligned_le16(fp + 0x1a) ||
+	    get_unaligned_le16(fp + 0x1a) > BORAGE_WORK_CAP)
+		return -EINVAL;
+	base = 0x01483118 + 0x1814 * work;
+	put_unaligned_le16(get_unaligned_le16(fp + 0x1a), cmd + 0x04);
+	put_unaligned_le32(base, cmd + 0x0c);
+	put_unaligned_le32(base + 8, cmd + 0x14);
+	put_unaligned_le32(0x014832c0 + 0x1814 * work, cmd + 0x24);
+	put_unaligned_le32(0x014834dc + 0x1814 * work, cmd + 0x28);
+	put_unaligned_le32(base + 0x4d0, cmd + 0x2c);
+	/* Slot-relative offsets; their packed DVA encoding is still [C]. */
+	put_unaligned_le32(0x5f8, cmd + 0x30);
+	put_unaligned_le32(0x4fc, cmd + 0x34);
+	put_unaligned_le32(0x5e0, cmd + 0x38);
+	put_unaligned_le32(get_unaligned_le32(fp + 0x4f8), cmd + 0x4c);
+	return -EOPNOTSUPP;
+}

drivers/media/platform/apple/avd/avd-av1-fp.h

@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0-only OR MIT */
+#ifndef AVD_AV1_FP_H_
+#define AVD_AV1_FP_H_
+
+#include <linux/types.h>
+
+struct avd_ctx;
+struct avd_frame;
+struct v4l2_ctrl_av1_sequence;
+struct v4l2_ctrl_av1_frame;
+struct v4l2_ctrl_av1_tile_group_entry;
+struct v4l2_ctrl_av1_film_grain;
+
+enum avd_av1_fp_layout {
+	AVD_AV1_FP_UNKNOWN,
+	AVD_AV1_FP_BORAGE,
+	AVD_AV1_FP_T8140_TODO,
+};
+
+/* Incoming edge: the primary reference selects the previous HW CDF output. */
+static inline bool avd_av1_cdf_dep(bool uses_primary, u64 primary_timestamp,
+				 bool have_prev, u64 prev_timestamp,
+				 bool prev_updates_cdf)
+{
+	return uses_primary && have_prev && prev_updates_cdf &&
+		primary_timestamp == prev_timestamp;
+}
+
+int avd_av1_validate_tiles(const struct v4l2_ctrl_av1_frame *frame,
+			   const struct v4l2_ctrl_av1_tile_group_entry *tiles,
+			   unsigned int count, size_t payload);
+int avd_av1_pack_fp(struct avd_ctx *ctx, struct avd_frame *f,
+		    const struct v4l2_ctrl_av1_sequence *seq,
+		    const struct v4l2_ctrl_av1_frame *frame,
+		    const struct v4l2_ctrl_av1_tile_group_entry *tiles,
+		    const struct v4l2_ctrl_av1_film_grain *grain);
+int avd_av1_pack_cmd(struct avd_ctx *ctx, struct avd_frame *f);
+
+#endif

drivers/media/platform/apple/avd/avd-av1-frame-ops.rst

@@ -0,0 +1,82 @@
+AV1 frame operations
+====================
+
+AV1 implements avd_codec_frame_ops and retains the legacy run/done/submit and
+pp_kicks callbacks. The current open transports still require legacy segments
+and codec-specific PP kicks. avd_get_frame_ops selects that compatibility path
+when a legacy run callback exists, and selects frame_ops on Apple. Retirement
+uses the operations selected at preparation, including across a backend reset.
+The queue header is core's unmodified shared contract.
+
+The compatibility adapter applies the request controls, invokes the same CPU
+prepare implementation, and copies the recorded header/tile ranges into the
+legacy job. It never finds boundaries by treating arbitrary instruction data
+as opcodes. Scratch and auxiliary DMA allocations stay alive until done calls
+retire. AV1 uses one legacy picture per context until the open transports accept
+native frame_ops; the queued transport gets the exact number of PP kicks.
+Open decode retains aurora-wip's grain-less fallback for APPLY_GRAIN requests.
+Film-grain synthesis is not implemented by the existing instruction builder.
+
+prepare consumes applied controls and the slot's src/dst. It never accesses an
+m2m queue, completes a request, or touches MMIO. The little-endian open program
+contains header EXEC, the existing header words, and one tile payload plus EXEC
+per tile (END on the last tile). prog_words includes EXEC records. Only the
+compatibility adapter accesses the legacy job; PP register writes live in the
+transport. Core supplies fp only for an Apple submission.
+
+Each live frame owns its pipe state and all above/left/MV scratch allocations.
+CDFs and segmentation maps have reference-counted coherent DMA allocations.
+Capture-index records retain output state; live frames retain their inputs and
+outputs independently. Disabled updates inherit an immutable allocation rather
+than copying unfinished hardware output. Reusing the original capture index
+cannot free inherited state. Color/MV data in capture buffers follows stateless
+reference-buffer lifetime rules. Segmentation inherits a map only when MiCols
+and MiRows match, retaining the existing builder's size comparison. Reference
+publication uses OUTPUT's timestamp:
+core copies that metadata to CAPTURE only after prepare has returned.
+
+The incoming dependency is PREV_DONE exactly when the current frame selects the
+previous frame as primary reference and that previous frame enables frame-end
+CDF updates. Intra/error-resilient/PRIMARY_REF_NONE defaults, another primary
+reference, and disabled frame-end updates produce NONE. prepare does not read
+hardware CDF/map output on the CPU. An unfinished older producer which that
+rule cannot express returns EAGAIN. Native integration still needs preparation
+retry and propagation of failed producers to already prepared dependants.
+Legacy decode is serial and retires its producer before the next preparation.
+
+Apple layout evidence and remaining work
+----------------------------------------
+
+The source is the clean-room Apple AVD protocol SPEC dated 2026-10-31, sections
+5, 6.1, 6.5, 6.6 and 10. Public V3 layouts in eiln/avd cannot supply AV1 fields:
+those generations have no AV1 hardware. avd-av1-fp.c keeps an unnamed [C] table
+with sources for the missing fields. It stages proved Borage tile columns/rows
+at 0x3e4/0x3e6, command destinations, work-count cap 0x1002 and the
+fp+0x4f8 -> cmd+0x4c copy. Both Apple pack operations return EOPNOTSUPP:
+the SPEC omits syntax/DVA mappings, source flag shifts and packed-address
+encoding. These facts do not yet describe a usable Apple AV1 payload.
+
+T6040 selects Borage. T8140 has a distinct TODO row tied to Daybreak's per-SoC
+ledger task t-3862. Other Apple layouts stay unsupported. Hardware capability
+masks still govern format exposure; T6000/T8112 have no AV1 capability or VP.
+
+Verification
+------------
+
+Run ``python3 drivers/media/platform/apple/avd/tests/run-host-tests.py`` from a
+Git checkout (sparse checkouts work). A host C compiler with ASan and UBSan is
+required. Tests compile production CPU code and the production backend selector
+with DMA/V4L2 doubles and the checkout's AV1 UAPI definitions. The licensed
+packing oracle is frozen from aurora-wip
+fbc4e7934ca7325f3483b72a51d24c74aa8536a4. Program comparison resolves the same
+DMA addresses for both builders; changed ownership intentionally changes them.
+
+Coverage includes 64 parity vectors (tiles, CDEF, restoration, superres, 8/10-bit,
+inter frames and grain-less fallback), legacy multi-tile submission/PP counts,
+submission failure cleanup, source-timestamp publication, producer errors, the
+dependency table, sixteen live slots, inherited state and capture reuse,
+allocation failure cleanup, payload/program bounds and Apple-layout refusal.
+These are synthetic controls, not SVT-AV1 bitstreams or pixel validation.
+Show-existing is a userspace reference redisplay without a new stateless decode
+control. Native open execution, the missing Apple payload facts, retry/failure
+integration and end-to-end stream qualification remain outstanding.