From 138aa88cfff6cbe1d42eb63feb2e0e58654348ea Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 22:30:50 +1100 Subject: [PATCH 01/73] media: apple: avd: publish shared frame queue contract Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-queue.h | 87 ++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 drivers/media/platform/apple/avd/avd-queue.h diff --git a/drivers/media/platform/apple/avd/avd-queue.h b/drivers/media/platform/apple/avd/avd-queue.h new file mode 100644 index 0000000000000..ca04f908ef359 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_QUEUE_H_ +#define AVD_QUEUE_H_ + +#include +#include +#include + +struct avd_dev; +struct avd_ctx; +struct avd_buf; /* avd.h: { void *cpu; dma_addr_t addr; size_t size; } */ + +#define AVD_MAX_FRAMES 16 /* SPEC 4.1: 16 frame-param slots */ + +enum avd_frame_state { + AVD_FRAME_FREE = 0, /* SPEC: slot state 0 */ + AVD_FRAME_RESERVED, /* 1: codec is filling it on the CPU */ + AVD_FRAME_SUBMITTED, /* 3: owned by firmware */ + AVD_FRAME_DONE, /* completion seen, waiting for in-order retire */ +}; + +/* Codec says what must finish before this frame may be submitted. */ +enum avd_frame_dep { + AVD_DEP_NONE = 0, /* independent: may overlap the previous frame */ + AVD_DEP_PREV_DONE, /* previous frame of this ctx must be DONE first + * (VP9 backward adaptation, AV1 CDF update, ...) */ +}; + +struct avd_frame { + unsigned int idx; /* 0..AVD_MAX_FRAMES-1, also the fp slot */ + u32 tag; /* (seq << 4) | idx, echoed by completion */ + u64 seq; /* global submission order */ + enum avd_frame_state state; + enum avd_frame_dep dep; + int status; /* 0 or -EIO/-ETIMEDOUT from the backend */ + + struct avd_ctx *ctx; + struct vb2_v4l2_buffer *src; /* OUTPUT, held until retire */ + struct vb2_v4l2_buffer *dst; /* CAPTURE, held until retire */ + u32 frame_num; /* SPEC 5.2 +0x0c */ + + /* open backend: instruction program, DMA-visible, built by the codec */ + struct avd_buf *prog; /* per-slot, allocated by core */ + size_t prog_words; + + /* apple backend: frame-params slot (SPEC 0x1130d0 on Borage) */ + struct avd_buf *fp; /* per-slot, allocated by core, size from match data */ + u32 cmd[24]; /* 0x60 Borage command; core fills the 5.2 prefix, + * codec pack_cmd() fills the codec part */ + + void *codec_priv; /* per-frame codec state, owned by the codec */ + unsigned long deadline; /* jiffies, per-frame watchdog */ +}; + +struct avd_backend_ops { + const char *name; + int (*probe)(struct avd_dev *avd); /* apertures, fw source, version */ + int (*boot)(struct avd_dev *avd); /* power-up: fw present/loaded, link up, ready */ + void (*shutdown)(struct avd_dev *avd); + unsigned int (*max_inflight)(struct avd_dev *avd); + int (*session_start)(struct avd_ctx *ctx); /* apple: cmd 0; open: may be NULL */ + void (*session_stop)(struct avd_ctx *ctx); /* apple: cmd 2 */ + /* Non-blocking. -EBUSY means inbox full: core retries on next completion. */ + int (*submit)(struct avd_dev *avd, struct avd_frame *f); + void (*abort)(struct avd_ctx *ctx); /* apple: cmd 5 */ + irqreturn_t (*irq)(struct avd_dev *avd); /* calls avd_frame_complete() */ + int (*suspend)(struct avd_dev *avd); + int (*resume)(struct avd_dev *avd); +}; + +/* Codec half. Replaces avd_coded_fmt_ops.run/submit/done. CPU only, no MMIO. */ +struct avd_codec_frame_ops { + /* Read the request's controls, set f->dep, fill f->prog (open backend) + * or f->fp (apple backend), allocate f->codec_priv if needed. */ + int (*prepare)(struct avd_ctx *ctx, struct avd_frame *f); + /* Apple backend: fill the codec part of f->cmd (SPEC 6.x). */ + int (*pack_cmd)(struct avd_ctx *ctx, struct avd_frame *f); + /* Called in submission order per ctx, after completion or failure. */ + void (*retire)(struct avd_ctx *ctx, struct avd_frame *f, + enum vb2_buffer_state state); +}; + +/* Core API used by backends and codecs. */ +void avd_frame_complete(struct avd_dev *avd, u32 tag, int status); /* any ctx, IRQ-safe */ +bool avd_backend_is_apple(const struct avd_dev *avd); + +#endif /* AVD_QUEUE_H_ */ From 02c13a19d7d64b6926352ad64e0b15cf8e334545 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 22:35:55 +1100 Subject: [PATCH 02/73] media: apple: avd: introduce owned frame slots and backend transports Keep the legacy open firmware serialized while moving program storage and completion ownership into tagged frame slots. Leave Borage activation dependent on the register and completion facts absent from the clean specification. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/Makefile | 5 +- .../platform/apple/avd/avd-apple-protocol.h | 106 +++++ .../platform/apple/avd/avd-backend-apple.c | 305 +++++++++++++ .../platform/apple/avd/avd-backend-open.c | 161 +++++++ drivers/media/platform/apple/avd/avd-drv.c | 369 ++------------- .../platform/apple/avd/avd-queue-internal.h | 61 +++ .../media/platform/apple/avd/avd-queue-test.c | 128 ++++++ drivers/media/platform/apple/avd/avd-queue.c | 421 ++++++++++++++++++ drivers/media/platform/apple/avd/avd-queue.h | 3 + drivers/media/platform/apple/avd/avd-v4l2.c | 35 +- drivers/media/platform/apple/avd/avd.h | 61 ++- 11 files changed, 1268 insertions(+), 387 deletions(-) create mode 100644 drivers/media/platform/apple/avd/avd-apple-protocol.h create mode 100644 drivers/media/platform/apple/avd/avd-backend-apple.c create mode 100644 drivers/media/platform/apple/avd/avd-backend-open.c create mode 100644 drivers/media/platform/apple/avd/avd-queue-internal.h create mode 100644 drivers/media/platform/apple/avd/avd-queue-test.c create mode 100644 drivers/media/platform/apple/avd/avd-queue.c diff --git a/drivers/media/platform/apple/avd/Makefile b/drivers/media/platform/apple/avd/Makefile index 4d1877c414d41..64095b4242133 100644 --- a/drivers/media/platform/apple/avd/Makefile +++ b/drivers/media/platform/apple/avd/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-h264.o avd-vp9.o avd-hevc.o \ - avd-av1.o avd-av1-entropymode.o -apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o + avd-av1.o avd-av1-entropymode.o avd-queue.o avd-backend-open.o \ + avd-backend-apple.o +apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o avd-queue-test.o obj-$(CONFIG_VIDEO_APPLE_AVD) += apple-avd.o diff --git a/drivers/media/platform/apple/avd/avd-apple-protocol.h b/drivers/media/platform/apple/avd/avd-apple-protocol.h new file mode 100644 index 0000000000000..1a497ff2b2bff --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-apple-protocol.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_APPLE_PROTOCOL_H_ +#define AVD_APPLE_PROTOCOL_H_ + +#include +#include + +#include "avd-queue.h" + +#define AVD_APPLE_INBOX_BASE 0x01482cf8 +#define AVD_APPLE_COMMAND_SIZE 0x60 +#define AVD_APPLE_RESULT_WORDS 20 +#define AVD_APPLE_SHADOW_SIZE 64 + +/* + * [C] Expected address notification, from the MIT V3 trace_avd.py precedent. + * Keep this rule in one place for correction by the Borage capture. + */ +#define AVD_APPLE_NOTIFY_BASE AVD_APPLE_INBOX_BASE + +struct avd_apple_reply_layout { + u8 type_word; + u8 type_shift; + u32 type_mask; + u8 client_word; + u8 client_shift; + u32 client_mask; + u8 frame_word; + u8 status_word; + u32 done_type; + bool verified; +}; + +struct avd_apple_reply { + u32 type; + u32 client; + u32 frame; + int status; +}; + +static inline int avd_apple_parse_reply(const u32 *record, size_t words, + const struct avd_apple_reply_layout *layout, + struct avd_apple_reply *reply) +{ + if (words != AVD_APPLE_RESULT_WORDS) + return -EMSGSIZE; + if (!layout->verified) + return -EOPNOTSUPP; + if (layout->type_word >= words || layout->client_word >= words || + layout->frame_word >= words || layout->status_word >= words || + layout->type_shift >= 32 || layout->client_shift >= 32 || + !layout->type_mask || !layout->client_mask || layout->done_type == 7) + return -EINVAL; + reply->type = (record[layout->type_word] >> layout->type_shift) & layout->type_mask; + reply->client = (record[layout->client_word] >> layout->client_shift) & layout->client_mask; + reply->frame = record[layout->frame_word]; + /* SPEC 4.3: type 7 is parsing/VP done, never final buffer ownership. */ + if (reply->type == 7) + return -EINPROGRESS; + if (reply->type != layout->done_type) + return -ENOMSG; + reply->status = record[layout->status_word] ? -EIO : 0; + return 0; +} + +static inline u32 avd_apple_notification(unsigned int slot) +{ + return AVD_APPLE_NOTIFY_BASE + AVD_APPLE_COMMAND_SIZE * slot; +} + +struct avd_apple_hw { + const char *fw_name; + /* All offsets below are relative to a DT-owned "apple" resource. */ + u32 aperture_origin; + u32 imem; + u32 imem_size; + u32 version_offset; + u32 itcm_start; + u32 itcm_end; + u32 itcm_clear; + u32 itcm_clear_value; + u32 itcm_enable; + u32 itcm_enable_value; + u32 link_setup; + u32 link_setup_value; + u32 link_state; + u32 link_up; + u32 link_up_value; + u32 ready; + u32 notify; + u32 receive; + u32 receive_status; + u32 receive_pending; + u32 receive_ack; + u32 receive_ack_value; + u32 inbox_consumer; + u32 result_base; + struct avd_apple_reply_layout reply; + /* Match-data fact, never a module parameter or a userspace setting. */ + bool verified; +}; + +extern const struct avd_apple_hw avd_borage_hw; +extern const u8 avd_borage_config[64]; + +#endif /* AVD_APPLE_PROTOCOL_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-backend-apple.c b/drivers/media/platform/apple/avd/avd-backend-apple.c new file mode 100644 index 0000000000000..fbe035172957d --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-apple.c @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include + +#include "avd.h" +#include "avd-apple-protocol.h" +#include "avd-queue-internal.h" + +/* SPEC 3.2. Unknown field meanings are intentionally not named. */ +const u8 avd_borage_config[64] = { + [0x0a] = 0x55, [0x0b] = 0x01, /* +0x08: 0x01558000 */ + [0x09] = 0x80, + [0x11] = 0x8a, [0x12] = 0x55, [0x13] = 0x01, + [0x19] = 0x8e, [0x1a] = 0x55, [0x1b] = 0x01, + [0x28] = 0x0a, [0x2a] = 0x04, + [0x2c] = 0x04, + [0x38] = 0x08, [0x3a] = 0x10, [0x3b] = 0x01, +}; + +/* + * Source: clean SPEC 3-5, AppleAVD 993.1 / T6040 Borage. The SPEC does + * not supply the ITCM/version register offsets, link register operations or + * reply field layout. Do not substitute V3/open-firmware offsets for them. + * avd-protocol.rst records every unresolved field and its source/capture. + */ +const struct avd_apple_hw avd_borage_hw = { + .fw_name = "apple/avd-apple-t6040.bin", + .verified = false, +}; + +struct avd_apple { + void __iomem *regs; + const struct avd_apple_hw *hw; + const struct firmware *image; + u32 shadow[AVD_APPLE_SHADOW_SIZE]; + unsigned int shadow_read; + unsigned int shadow_write; + unsigned int producer; + u32 version; +}; + +static void avd_apple_release_image(void *data) +{ + struct avd_apple *apple = data; + + release_firmware(apple->image); +} + +static int avd_apple_image_present(struct avd_apple *apple) +{ + const struct avd_apple_hw *hw = apple->hw; + u32 start = readl(apple->regs + hw->itcm_start); + u32 end = readl(apple->regs + hw->itcm_end); + u32 version; + + if (end <= start || end - start > hw->imem_size) + return -ENOENT; + version = readl(apple->regs + hw->imem + hw->version_offset); + if (!version || version == U32_MAX) + return -ENOENT; + apple->version = version; + return 0; +} + +static int avd_apple_probe(struct avd_dev *avd) +{ + const struct avd_apple_hw *hw = avd->variant->apple; + struct avd_apple *apple; + int ret; + + if (!hw || !hw->verified || !hw->reply.verified) { + dev_info(avd->dev, "Apple transport awaits Borage register/reply facts; using open firmware\n"); + return -EOPNOTSUPP; + } + apple = devm_kzalloc(avd->dev, sizeof(*apple), GFP_KERNEL); + if (!apple) + return -ENOMEM; + apple->hw = hw; + apple->regs = devm_platform_ioremap_resource_byname(avd->pdev, "apple"); + if (IS_ERR(apple->regs)) + return PTR_ERR(apple->regs); + /* Firmware bytes remain in vendorfw; this driver never embeds an image. */ + ret = firmware_request_nowarn(&apple->image, hw->fw_name, avd->dev); + if (ret && ret != -ENOENT) + return ret; + ret = devm_add_action_or_reset(avd->dev, avd_apple_release_image, apple); + if (ret) + return ret; + avd->backend_priv = apple; + return 0; +} + +static int avd_apple_load(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + const struct firmware *fw = apple->image; + size_t i, rounded; + + if (!fw) + return -ENOENT; + if (!fw->size || fw->size > hw->imem_size || + hw->imem_size < 4 || hw->version_offset > hw->imem_size - 4) + return -EINVAL; + rounded = ALIGN(fw->size, 4); + if (rounded > hw->imem_size) + return -EFBIG; + writel(hw->itcm_clear_value, apple->regs + hw->itcm_clear); + writel(0, apple->regs + hw->itcm_start); + writel(rounded, apple->regs + hw->itcm_end); + /* Bounded word copies pad the last word without reading past fw->data. */ + for (i = 0; i < rounded; i += 4) { + u8 bytes[4] = {}; + + memcpy(bytes, fw->data + i, min_t(size_t, 4, fw->size - i)); + writel(get_unaligned_le32(bytes), apple->regs + hw->imem + i); + } + writel(hw->itcm_enable_value, apple->regs + hw->itcm_enable); + return avd_apple_image_present(apple); +} + +static int avd_apple_boot(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + u32 value; + int ret; + + ret = avd_apple_image_present(apple); + if (ret) { + ret = avd_apple_load(avd); + if (ret) + return ret; + } + dev_info(avd->dev, "Apple MCPU firmware %u.%u.%u (%08x)\n", + apple->version >> 8, (apple->version >> 4) & 0xf, + apple->version & 0xf, apple->version); + /* SPEC 3.2: decode connection, aperture 1, state 4, then LinkUp. */ + writel(hw->link_setup_value, apple->regs + hw->link_setup); + ret = readl_poll_timeout(apple->regs + hw->link_state, value, + value == 4, 10, 10000); + if (ret) + return ret; + writel(hw->link_up_value, apple->regs + hw->link_up); + ret = readl_poll_timeout(apple->regs + hw->ready, value, + value == 1, 10, 10000); + if (!ret) { + apple->producer = 0; + apple->shadow_read = 0; + apple->shadow_write = 0; + } + return ret; +} + +static void avd_apple_shutdown(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + + /* MCPU disable value must be proved together with enable in the ledger. */ + writel(0, apple->regs + apple->hw->itcm_enable); +} + +static unsigned int avd_apple_max_inflight(struct avd_dev *avd) +{ + return AVD_MAX_FRAMES; +} + +static int avd_apple_send(struct avd_dev *avd, const u32 cmd[24]) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + unsigned int consumer = readl(apple->regs + hw->inbox_consumer) & 7; + u32 address = AVD_APPLE_INBOX_BASE + apple->producer * AVD_APPLE_COMMAND_SIZE; + unsigned int i; + + if (avd_inbox_full(apple->producer, consumer)) + return -EBUSY; + for (i = 0; i < 24; i++) + writel(cmd[i], apple->regs + address - hw->aperture_origin + i * 4); + /* MMIO commands and coherent fp payload precede the four-byte doorbell. */ + wmb(); + writel(avd_apple_notification(apple->producer), apple->regs + hw->notify); + apple->producer = (apple->producer + 1) & 7; + return 0; +} + +static int avd_apple_session_start(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + cmd[0] = ctx->client_id << 16; + return avd_apple_send(ctx->dev, cmd); +} + +static void avd_apple_session_stop(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + cmd[0] = 2 | (ctx->client_id << 16); + if (avd_apple_send(ctx->dev, cmd)) + dev_warn(ctx->dev->dev, "session stop inbox full\n"); +} + +static void avd_apple_abort(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + cmd[0] = 5 | (ctx->client_id << 16); + /* Core follows abort with reset before returning any DMA buffer. */ + avd_apple_send(ctx->dev, cmd); +} + +static int avd_apple_set_property(struct avd_ctx *ctx, u32 property, u32 value) +{ + u32 cmd[24] = {}; + + cmd[0] = 6 | (ctx->client_id << 16); + cmd[1] = property; + cmd[2] = value; + return avd_apple_send(ctx->dev, cmd); +} + +static int avd_apple_submit(struct avd_dev *avd, struct avd_frame *f) +{ + const struct avd_codec_frame_ops *ops = f->ctx->coded_fmt_desc->frame_ops; + u32 selector; + int ret; + + if (!ops || !ops->pack_cmd || !f->fp->cpu || f->ctx->client_id >= 1024) + return -EOPNOTSUPP; + /* SPEC 5.2 prefix precedes the codec-specific builder. */ + memset(f->cmd, 0, sizeof(f->cmd)); + selector = get_unaligned_le32(f->fp->cpu + 4) & 0xf; + f->cmd[0] = 1 | (selector << 10) | (f->ctx->client_id << 22); + f->cmd[1] = f->idx | ((f->seq & 3) << 16); + f->cmd[3] = f->frame_num; + ret = ops->pack_cmd(f->ctx, f); + if (ret) + return ret; + return avd_apple_send(avd, f->cmd); +} + +static irqreturn_t avd_apple_irq(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + unsigned int budget = AVD_APPLE_SHADOW_SIZE; + + while (budget-- && (readl(apple->regs + hw->receive_status) & hw->receive_pending)) { + unsigned int next = (apple->shadow_write + 1) & 63; + + if (next == apple->shadow_read) + break; + apple->shadow[apple->shadow_write] = readl(apple->regs + hw->receive); + apple->shadow_write = next; + writel(hw->receive_ack_value, apple->regs + hw->receive_ack); + } + while (apple->shadow_read != apple->shadow_write) { + u32 record[AVD_APPLE_RESULT_WORDS]; + struct avd_apple_reply reply; + unsigned int i; + int ret; + + /* Result notification indexing must be verified with the layout. */ + u32 index = apple->shadow[apple->shadow_read]; + + apple->shadow_read = (apple->shadow_read + 1) & 63; + if (index >= AVD_APPLE_SHADOW_SIZE) + continue; + for (i = 0; i < AVD_APPLE_RESULT_WORDS; i++) + record[i] = readl(apple->regs + hw->result_base + + index * sizeof(record) + i * 4); + ret = avd_apple_parse_reply(record, ARRAY_SIZE(record), &hw->reply, &reply); + if (ret) + continue; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + + if (f->state == AVD_FRAME_SUBMITTED && f->frame_num == reply.frame && + f->ctx->client_id == reply.client) { + dma_rmb(); + avd_frame_complete(avd, f->tag, reply.status); + break; + } + } + } + mod_delayed_work(system_wq, &avd->queue_work, 0); + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_apple_backend = { + .name = "apple-mcpu", + .probe = avd_apple_probe, + .boot = avd_apple_boot, + .shutdown = avd_apple_shutdown, + .max_inflight = avd_apple_max_inflight, + .session_start = avd_apple_session_start, + .session_stop = avd_apple_session_stop, + .submit = avd_apple_submit, + .abort = avd_apple_abort, + .set_property = avd_apple_set_property, + .irq = avd_apple_irq, +}; diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c new file mode 100644 index 0000000000000..5ea70af085674 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "avd.h" +#include "avd-inst.h" +#include "avd-regs.h" + +/* + * Compatibility transport for firmware without the queued-program ABI. The + * codec's CPU builder emits a per-slot DMA-visible instruction stream. Only + * this backend touches the VP FIFO. Its untagged PP interrupt limits it to one + * frame; the queued avd-fw transport can replace these ops without changing + * buffer ownership in the core. + */ +int avd_open_build_program(struct avd_ctx *ctx) +{ + struct avd_dev *avd = ctx->dev; + struct avd_frame *f = avd->building; + struct avd_job *job = &ctx->job; + size_t count = 1, pos = 0, i, j; + u32 *words, mask, start; + int ret; + + if (!f || !job->segments) + return -EINVAL; + for (i = 0; i <= job->num; i++) { + if (job->segments[i].num > AVD_MAX_INST || + check_add_overflow(count, job->segments[i].num + !!i, &count)) + return -EOVERFLOW; + } + if (count > SIZE_MAX / sizeof(u32)) + return -EOVERFLOW; + if (f->prog->size < count * sizeof(u32)) { + ret = avd_buf_alloc(avd, f->prog, count * sizeof(u32)); + if (ret) + return ret; + } + mask = job->codec == AVD_CODEC_VP9 && avd->variant->revision == 3 ? + AVD_OP_EXEC_REV3_VP9_MASK : 0; + start = AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | + AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3); + ctx->fifo_idx = 0; + words = f->prog->cpu; + words[pos++] = AVD_OP_EXEC | mask | start | AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx); + for (i = 0; i <= job->num; i++) { + for (j = 0; j < job->segments[i].num; j++) + words[pos++] = job->segments[i].instructions[j]; + if (i) + words[pos++] = AVD_OP_EXEC | mask | AVD_OP_EXEC_FLAG_END(i == job->num); + } + f->prog_words = pos; + f->cmd[0] = job->codec; + kvfree(job->segments); + job->segments = NULL; + return 0; +} + +static unsigned int avd_open_max_inflight(struct avd_dev *avd) +{ + return 1; +} + +static int avd_open_submit(struct avd_dev *avd, struct avd_frame *f) +{ + struct avd_ctx *ctx = f->ctx; + u32 codec = f->cmd[0], vp = 0; + u32 *words = f->prog->cpu; + unsigned int i; + void __iomem *reg; + + if (avd->compat_active) + return -EBUSY; + /* Native frame_ops require firmware-side PP chaining and tagged replies. */ + if (ctx->coded_fmt_desc->frame_ops) + return -EOPNOTSUPP; + if (codec > AVD_CODEC_AV1 || !words || !f->prog_words || + f->prog_words > f->prog->size / sizeof(u32)) + return -EINVAL; + for (i = 0; i < codec; i++) + vp += avd->variant->vp_slots[i]; + if (avd->variant->vp_slot_base[codec]) + vp = avd->variant->vp_slot_base[codec]; + avd->variant->configure_stream(avd, ctx->inst.addr, ctx->fifo_idx, vp); + reg = avd->ctrl + avd->variant->vp_slot_offset + vp * 4; + avd->compat_active = f; + for (i = 0; i < f->prog_words; i++) { + /* Preserve the old firmware's segment FIFO pacing in this shim. */ + if (i && (words[i] & GENMASK(31, 24)) == AVD_OP_EXEC && + (!(avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) || + codec == AVD_CODEC_HEVC)) { + u32 max = readl(avd->ctrl + avd->variant->submit_queue_max_offset + vp * 4); + u32 cur = readl(avd->ctrl + + avd->variant->submit_queue_status_offset + vp * 4); + + if (cur >= max) { + /* Already emitted words: only reset may release this slot. */ + f->deadline = jiffies; + return 0; + } + if (cur >= max / 2) { + if (avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) + usleep_range(100, 150); + else + usleep_range(500, 650); + } + } + writel(words[i], reg); + } + return 0; +} + +static irqreturn_t avd_open_irq(struct avd_dev *avd) +{ + struct avd_frame *f = avd->compat_active; + u32 status = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); + + writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); + if (!f) + return IRQ_HANDLED; + if (status & 0x1000) { + avd->compat_active = NULL; + avd_frame_complete(avd, f->tag, 0); + } else if (status & 0x100) { + /* Old firmware only: no request tag and no firmware-side PP kick. */ + if (f->ctx->coded_fmt_desc->ops->submit) + f->ctx->coded_fmt_desc->ops->submit(f->ctx); + } else { + dev_err_ratelimited(avd->dev, "open firmware error %#x\n", status); + f->deadline = jiffies; + mod_delayed_work(system_wq, &avd->queue_work, 0); + } + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_open_backend = { + .name = "open-compat", + .boot = avd_boot, + .shutdown = avd_shutdown, + .max_inflight = avd_open_max_inflight, + .submit = avd_open_submit, + .irq = avd_open_irq, +}; + +int avd_backend_select(struct avd_dev *avd) +{ + int ret; + + /* Apple probe returns -ENOENT only when no usable image is present. */ + if (avd->variant->apple) { + ret = avd_apple_backend.probe(avd); + if (!ret) { + avd->backend = &avd_apple_backend; + return 0; + } + if (ret != -ENOENT && ret != -EOPNOTSUPP) + return ret; + } + avd->backend = &avd_open_backend; + return request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); +} diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index bf5bf96f4c62a..c60f0417c5cd2 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -22,6 +22,7 @@ #include "avd.h" #include "avd-inst.h" #include "avd-regs.h" +#include "avd-apple-protocol.h" #define AVD_CODEC_INFO_MAX_SIZE 0x40 @@ -116,28 +117,6 @@ avd_get_ref_buf(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst, u64 timestamp) return vb2_to_avd_decoded_buf(buf); } -static int avd_wait_submission_queue(struct avd_ctx *ctx, int vp) -{ - struct avd_dev *avd = ctx->dev; - u32 max = readl_relaxed(avd->ctrl + - avd->variant->submit_queue_max_offset + vp * 4); - u32 cur = readl_relaxed( - avd->ctrl + avd->variant->submit_queue_status_offset + vp * 4); - - if (cur == max) { - dev_err(avd->dev, "instruction que full! %d/%d", cur, max); - return 1; - } - - if (cur >= max / 2) { - if (avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) - usleep_range(100, 150); - else - usleep_range(500, 650); - } - return 0; -} - int avd_init_job(struct avd_ctx *ctx, enum avd_codec codec, size_t segments) { int ret = 0; @@ -155,287 +134,19 @@ int avd_init_job(struct avd_ctx *ctx, enum avd_codec codec, size_t segments) int avd_submit_job(struct avd_ctx *ctx) { - struct avd_dev *avd = ctx->dev; - struct avd_job *sub = &ctx->job; - struct avd_segment *segments, *seg; - unsigned long flags; - size_t num; - int i, idx = 0, vp = 0; - void __iomem *reg; - u32 exec_mask = sub->codec == AVD_CODEC_VP9 && - avd->variant->revision == 3 ? - AVD_OP_EXEC_REV3_VP9_MASK : - 0; - u32 exec_rev_flag = - AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | - AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3); - - ctx->fifo_idx = 0; - for (i = 0; i < sub->codec; i++) - vp += avd->variant->vp_slots[i]; - if (avd->variant->vp_slot_base[sub->codec]) - vp = avd->variant->vp_slot_base[sub->codec]; - - /* - * avd_device_run() keeps this job from being finished (and the next one - * from starting) until it returns, and arms the watchdog itself once the - * job belongs to the hardware. Still walk a local copy of the table. - */ - segments = sub->segments; - num = sub->num; - if (WARN_ON_ONCE(!segments)) - return -EINVAL; - - avd->variant->configure_stream(avd, ctx->inst.addr, ctx->fifo_idx, vp); - reg = avd->ctrl + avd->variant->vp_slot_offset + vp * 4; - - /* the first segment is always the header (needs special handling) */ - - writel(AVD_OP_EXEC | exec_mask | exec_rev_flag | - AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx), - reg); - seg = &segments[idx++]; - for (i = 0; i < seg->num; i++) - writel(seg->instructions[i], reg); - for (; idx <= num; idx++) { - seg = &segments[idx]; - for (i = 0; i < seg->num; i++) - writel(seg->instructions[i], reg); - if ((!(avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) || - sub->codec == AVD_CODEC_HEVC) && - avd_wait_submission_queue(ctx, vp)) - break; - if (idx == num) { - /* A completion from here on belongs to this job. */ - spin_lock_irqsave(&avd->job_lock, flags); - avd->job_end_sent = true; - spin_unlock_irqrestore(&avd->job_lock, flags); - } - writel(AVD_OP_EXEC | exec_mask | - AVD_OP_EXEC_FLAG_END(idx == num), - reg); - } - - if (sub->segments == segments) - sub->segments = NULL; - kvfree(segments); - return 0; -} - -static int avd_reset(struct avd_dev *avd) -{ - int ret = 0; - - ret = pm_runtime_resume_and_get(avd->dev); - if (ret < 0) - return ret; - - ret = reset_control_reset(avd->rstc); - if (ret) - dev_err(avd->dev, "reset: failed: %d", ret); - - if (avd->empty_domain) { - iommu_attach_device(avd->empty_domain, avd->dev); - iommu_detach_device(avd->empty_domain, avd->dev); - } - - ret = avd_boot(avd); - if (ret) - dev_err(avd->dev, "reset: failed to boot"); - - pm_runtime_put_autosuspend(avd->dev); - - return ret; -} - -static void avd_watchdog_func(struct work_struct *work) -{ - struct avd_dev *avd; - struct avd_ctx *ctx; - unsigned long flags; - bool claimed; - int ret; - ctx = container_of(to_delayed_work(work), struct avd_ctx, - watchdog_work); - if (!ctx) - return; - - avd = ctx->dev; - - /* - * Time out only the job this watchdog was armed for, and only while the - * hardware still owns it. Otherwise that job already finished, and - * resetting the hardware would break whichever job is running now. - */ - spin_lock_irqsave(&avd->job_lock, flags); - claimed = avd->job_ctx == ctx && avd->job_seq == ctx->wd_seq && - avd->job_state == AVD_JOB_SUBMITTED; - if (claimed) - avd->job_state = AVD_JOB_FINISHING; - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: watchdog seq %llu: %s\n", ctx->wd_seq, - claimed ? "timeout" : "stale, ignored"); - if (!claimed) - return; - - dev_err(avd->dev, "Frame processing timed out!"); - - writel(0, avd->mbox + AVD_REG_MBOX_IRQ_ENABLE); - ret = avd_reset(avd); - if (ret) - dev_err(avd->dev, "failed to reset: %d", ret); - - avd_job_finish(ctx, VB2_BUF_STATE_ERROR); + return avd_open_build_program(ctx); } static irqreturn_t avd_irq_handler(int irq, void *data) { struct avd_dev *avd = data; - struct avd_ctx *ctx = v4l2_m2m_get_curr_priv(avd->m2m_dev); - - enum vb2_buffer_state state; - unsigned long flags; - bool finish = false, stale = false; - u32 status; - if (!ctx) - return IRQ_HANDLED; - - status = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); - - writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); - - if (status & 0x10000) { /* dbg */ - dev_warn(avd->dev, "no handler for IRQ: %3d", status &~0x10000); - writel_relaxed(0, avd->mbox + AVD_REG_MBOX_IRQ_ENABLE); - return IRQ_HANDLED; - } - - if (status & 0x1000) { - /* pp is done ! we are done */ - state = VB2_BUF_STATE_DONE; - } else if (status & 0x100) { - /* a vp is done, kick the pp and hope for the best */ - if(ctx->coded_fmt_desc->ops->submit) - ctx->coded_fmt_desc->ops->submit(ctx); - goto done; - } else { - dev_err(avd->dev, "H%d error", status); - /* - * avd-fw err(n): VP n raised its error interrupt and stopped, - * and the PP only starts on a VP done, so the job cannot - * complete any more. Run the watchdog now instead of after its - * timeout: it resets the AVD and fails only this frame, and it - * ignores a job the hardware does not own yet. - */ - mod_delayed_work(system_wq, &ctx->watchdog_work, 0); - goto done; - } - - /* - * Finish only a job the hardware owns. While avd_device_run() is still - * setting up or submitting, a completion is left over from an older job - * unless this job's END command already went out; then remember it and - * let avd_device_run() finish the job when it returns. - */ - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_ctx != ctx) { - stale = true; - } else if (avd->job_state == AVD_JOB_SUBMITTED) { - avd->job_state = AVD_JOB_FINISHING; - finish = true; - } else if (avd->job_state == AVD_JOB_RUNNING && avd->job_end_sent) { - avd->job_pending = true; - avd->job_pending_result = state; - } else { - stale = true; - } - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: irq %#x: %s\n", status, - finish ? "finish" : stale ? "stale, ignored" : "pending"); - if (stale) - dev_warn_ratelimited(avd->dev, "ignoring stale completion"); - if (finish) { - cancel_delayed_work(&ctx->watchdog_work); - avd_job_finish(ctx, state); - } - -done: - return IRQ_HANDLED; -} -static void avd_device_run(void *priv) -{ - struct avd_ctx *ctx = priv; - struct avd_dev *avd = ctx->dev; - const struct avd_coded_fmt_desc *desc = ctx->coded_fmt_desc; - enum vb2_buffer_state result = VB2_BUF_STATE_ERROR; - const char *outcome = "already finished"; - unsigned long flags; - bool finish = false; - u64 seq; - int ret; + irqreturn_t ret; - if (WARN_ON(!desc)) - return; - - /* - * IRQ completion cannot wait for an already running watchdog. Drain it - * here, before reusing wd_seq, so an old callback cannot time out this - * context's next job. device_run is sleepable; hold no job_lock here. - */ - cancel_delayed_work_sync(&ctx->watchdog_work); - - ret = pm_runtime_resume_and_get(avd->dev); - if (ret < 0) { - avd_job_finish_no_pm(ctx, VB2_BUF_STATE_ERROR); - return; - } - - /* - * While the job is RUNNING neither completion IRQs nor the watchdog may - * finish it: v4l2-mem2mem would start the next job on a kworker while - * this one still builds and submits ctx->job. - */ - spin_lock_irqsave(&avd->job_lock, flags); - seq = ++avd->job_seq; - avd->job_ctx = ctx; - avd->job_state = AVD_JOB_RUNNING; - avd->job_end_sent = false; - avd->job_pending = false; - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: run seq %llu\n", seq); - - ret = desc->ops->run(ctx); - if (ret) { - dev_dbg(avd->dev, "avd-job: run seq %llu failed: %d\n", seq, ret); - avd_job_finish(ctx, VB2_BUF_STATE_ERROR); - return; - } - - /* - * The run either finished the job itself (held capture buffer) or handed - * it to the hardware: arm the watchdog, or finish right away if the - * completion already arrived. - */ - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_seq == seq && avd->job_state == AVD_JOB_RUNNING) { - if (avd->job_pending) { - avd->job_state = AVD_JOB_FINISHING; - result = avd->job_pending_result; - finish = true; - outcome = "completed during submission"; - } else { - avd->job_state = AVD_JOB_SUBMITTED; - ctx->wd_seq = seq; - schedule_delayed_work(&ctx->watchdog_work, msecs_to_jiffies(2000)); - outcome = "submitted"; - } - } - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: run seq %llu: %s\n", seq, outcome); - - if (finish) - avd_job_finish(ctx, result); + mutex_lock(&avd->queue_mutex); + ret = avd->backend->irq(avd); + mutex_unlock(&avd->queue_mutex); + return ret; } /* @@ -444,12 +155,7 @@ static void avd_device_run(void *priv) * userspace decode ahead to ride out bitrate spikes the hardware can't * decode in real time (e.g. 150+ Mbit/s 4K60 H.264). */ -static unsigned int max_capture_buffers = 128; -module_param(max_capture_buffers, uint, 0444); -MODULE_PARM_DESC(max_capture_buffers, - "Maximum number of capture buffers per context (default 128)"); - -static int avd_queue_init(void *priv, struct vb2_queue *src_vq, +static int avd_m2m_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq) { struct avd_ctx *ctx = priv; @@ -486,7 +192,7 @@ static int avd_queue_init(void *priv, struct vb2_queue *src_vq, dst_vq->lock = &ctx->dev->vdev_lock; dst_vq->dev = ctx->dev->v4l2_dev.dev; dst_vq->allow_cache_hints = true; - dst_vq->max_num_buffers = max(max_capture_buffers, VB2_MAX_FRAME); + dst_vq->max_num_buffers = max(128U, VB2_MAX_FRAME); return vb2_queue_init(dst_vq); } @@ -507,14 +213,15 @@ static int avd_open(struct file *filp) if (ret) goto err_free_ctx; - INIT_DELAYED_WORK(&ctx->watchdog_work, avd_watchdog_func); + INIT_LIST_HEAD(&ctx->queue_node); + ctx->next_dep = AVD_DEP_PREV_DONE; avd_reset_coded_fmt(ctx); avd_reset_decoded_fmt(ctx); v4l2_fh_init(&ctx->fh, video_devdata(filp)); - ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(avd->m2m_dev, ctx, avd_queue_init); + ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(avd->m2m_dev, ctx, avd_m2m_queue_init); if (IS_ERR(ctx->fh.m2m_ctx)) { ret = PTR_ERR(ctx->fh.m2m_ctx); goto err_free_ctx; @@ -524,6 +231,7 @@ static int avd_open(struct file *filp) if (ret) goto err_cleanup_m2m_ctx; + avd_queue_add_ctx(ctx); v4l2_fh_add(&ctx->fh, filp); return 0; @@ -543,8 +251,7 @@ static int avd_release(struct file *filp) v4l2_fh_del(&ctx->fh, filp); v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); - /* The job has finished; its watchdog must not run on the freed ctx. */ - cancel_delayed_work_sync(&ctx->watchdog_work); + avd_queue_del_ctx(ctx); v4l2_ctrl_handler_free(&ctx->ctrl_hdl); v4l2_fh_exit(&ctx->fh); avd_buf_free(ctx->dev, &ctx->inst); @@ -564,7 +271,9 @@ static const struct v4l2_file_operations avd_fops = { }; static const struct v4l2_m2m_ops avd_m2m_ops = { - .device_run = avd_device_run, + .device_run = avd_queue_run, + .job_ready = avd_queue_ready, + .job_abort = avd_queue_abort, }; static const struct media_device_ops avd_media_ops = { @@ -811,6 +520,8 @@ static const struct avd_variant avd_t8132_variant = { * verified on the hardware. */ static const struct avd_variant avd_t6040_variant = { + .apple = &avd_borage_hw, + .fp_size = 0x1130d0, .vp_slots = { [AVD_CODEC_HEVC] = 4, [AVD_CODEC_H264] = 4, @@ -886,12 +597,16 @@ static int avd_probe(struct platform_device *pdev) avd->pdev = pdev; mutex_init(&avd->vdev_lock); - spin_lock_init(&avd->job_lock); + ret = avd_queue_init(avd); + if (ret) + return ret; match = of_match_node(avd_of_match, pdev->dev.of_node); avd->variant = match->data; avd->rstc = devm_reset_control_get_exclusive(avd->dev, NULL); + if (IS_ERR(avd->rstc)) + return PTR_ERR(avd->rstc); avd->code = devm_platform_ioremap_resource_byname(pdev, "code"); if (IS_ERR(avd->code)) @@ -912,36 +627,9 @@ static int avd_probe(struct platform_device *pdev) } } - ret = request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); - - if (ret) { - dev_err(avd->dev, "failed to load firmware: %d", ret); + ret = avd_backend_select(avd); + if (ret) return ret; - } - - /* Determine firmware and HW variant capabilities */ - if (avd->fw->size >= AVD_CODEC_INFO_MAX_SIZE) - codec_info = avd->fw->data + avd->fw->size - AVD_CODEC_INFO_MAX_SIZE; - if (codec_info && strncmp(codec_info, "CODECS:", 7) == 0) { - unsigned int caps = 0; - unsigned int variant_caps = avd->variant->capabilities; - - if (strnstr(codec_info, "av1", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_AV1; - if (strnstr(codec_info, "h264", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_H264; - if (strnstr(codec_info, "h265", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_HEVC; - if (strnstr(codec_info, "vp9", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_VP9; - - dev_info(avd->dev, "Firmware codecs: %.27s\n", codec_info + 7); - - avd->capabilities = caps; - } else { - /* Backward compatibility to old firmware, do not upstream. */ - avd->capabilities = avd->variant->capabilities; - } ret = dma_set_mask_and_coherent(avd->dev, DMA_BIT_MASK((avd->variant->quirks & AVD_QUIRK_LSR) ? 38 : 64)); @@ -982,6 +670,7 @@ static void avd_remove(struct platform_device *pdev) { struct avd_dev *avd = platform_get_drvdata(pdev); + avd_queue_cleanup(avd); release_firmware(avd->fw); avd_v4l2_cleanup(avd); @@ -998,7 +687,7 @@ static __maybe_unused int avd_runtime_resume(struct device *dev) int ret; struct avd_dev *avd = platform_get_drvdata(to_platform_device(dev)); - ret = avd_boot(avd); + ret = avd->backend->boot(avd); if (ret) dev_err(dev, "failed to boot"); return ret; @@ -1008,7 +697,7 @@ static __maybe_unused int avd_runtime_suspend(struct device *dev) { struct avd_dev *avd = platform_get_drvdata(to_platform_device(dev)); - avd_shutdown(avd); + avd->backend->shutdown(avd); return 0; } diff --git a/drivers/media/platform/apple/avd/avd-queue-internal.h b/drivers/media/platform/apple/avd/avd-queue-internal.h new file mode 100644 index 0000000000000..7ff919ce3b7ee --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue-internal.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_QUEUE_INTERNAL_H_ +#define AVD_QUEUE_INTERNAL_H_ + +#include +#include + +#include "avd-queue.h" + +/* Caller serializes the pool; tags are never recycled during device lifetime. */ +static inline int avd_slot_reserve(struct avd_frame *f, struct avd_ctx *ctx, + u64 seq) +{ + if (f->state != AVD_FRAME_FREE) + return -EBUSY; + if (!seq || seq > (U32_MAX >> 4)) + return -EOVERFLOW; + f->seq = seq; + f->tag = (seq << 4) | f->idx; + f->ctx = ctx; + f->state = AVD_FRAME_RESERVED; + f->dep = AVD_DEP_NONE; + f->status = 0; + f->prog_words = 0; + f->codec_priv = NULL; + return 0; +} + +static inline bool avd_slot_complete(struct avd_frame *f, u32 tag, int status) +{ + if (f->state != AVD_FRAME_SUBMITTED || f->tag != tag) + return false; + f->status = status; + f->state = AVD_FRAME_DONE; + return true; +} + +static inline bool avd_slot_first(const struct avd_frame *pool, + const struct avd_frame *f) +{ + unsigned int i; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (pool[i].state != AVD_FRAME_FREE && pool[i].ctx == f->ctx && + pool[i].seq < f->seq) + return false; + return true; +} + +static inline bool avd_slot_expired(const struct avd_frame *f, unsigned long now) +{ + return (f->state == AVD_FRAME_SUBMITTED || + f->state == AVD_FRAME_RESERVED) && time_after_eq(now, f->deadline); +} + +static inline bool avd_inbox_full(unsigned int producer, unsigned int consumer) +{ + return ((producer + 1) & 7) == consumer; +} + +#endif /* AVD_QUEUE_INTERNAL_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-queue-test.c b/drivers/media/platform/apple/avd/avd-queue-test.c new file mode 100644 index 0000000000000..938c577cb7824 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue-test.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include + +#include "avd-apple-protocol.h" +#include "avd-queue-internal.h" + +static void avd_slots_and_tags(struct kunit *test) +{ + struct avd_frame *pool; + struct avd_ctx *ctx = (void *)test; + unsigned int i; + + pool = kunit_kcalloc(test, AVD_MAX_FRAMES, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + pool[i].idx = i; + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[i], ctx, i + 1), 0); + KUNIT_EXPECT_EQ(test, pool[i].tag, ((i + 1) << 4) | i); + KUNIT_EXPECT_EQ(test, pool[i].state, AVD_FRAME_RESERVED); + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[i], ctx, 100), -EBUSY); + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag, 0)); + pool[i].state = AVD_FRAME_SUBMITTED; + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag + 16, 0)); + KUNIT_EXPECT_TRUE(test, avd_slot_complete(&pool[i], pool[i].tag, 0)); + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag, -EIO)); + } + pool[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[0], ctx, 0), -EOVERFLOW); + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[0], ctx, (u64)(U32_MAX >> 4) + 1), + -EOVERFLOW); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[0], ctx, 100), 0); + pool[0].state = AVD_FRAME_SUBMITTED; + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[0], 16, 0)); +} + +static void avd_retirement_order(struct kunit *test) +{ + struct avd_frame *pool; + struct avd_ctx *a = (void *)test; + struct avd_ctx *b = (void *)&pool; + + pool = kunit_kcalloc(test, AVD_MAX_FRAMES, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[0], a, 1), 0); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[1], b, 2), 0); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[2], a, 3), 0); + pool[0].state = AVD_FRAME_SUBMITTED; + pool[1].state = AVD_FRAME_SUBMITTED; + pool[2].state = AVD_FRAME_SUBMITTED; + KUNIT_ASSERT_TRUE(test, avd_slot_complete(&pool[2], pool[2].tag, 0)); + KUNIT_EXPECT_FALSE(test, avd_slot_first(pool, &pool[2])); + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[1])); + KUNIT_ASSERT_TRUE(test, avd_slot_complete(&pool[0], pool[0].tag, -ETIMEDOUT)); + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[0])); + pool[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[2])); + KUNIT_EXPECT_EQ(test, pool[0].status, -ETIMEDOUT); +} + +static void avd_watchdog_and_retry(struct kunit *test) +{ + struct avd_frame f = { .state = AVD_FRAME_SUBMITTED, .deadline = ULONG_MAX - 2 }; + unsigned int producer, consumer; + + KUNIT_EXPECT_FALSE(test, avd_slot_expired(&f, ULONG_MAX - 3)); + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, ULONG_MAX - 2)); + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, 1)); + f.state = AVD_FRAME_DONE; + KUNIT_EXPECT_FALSE(test, avd_slot_expired(&f, 1)); + f.state = AVD_FRAME_RESERVED; + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, 1)); + for (producer = 0; producer < 8; producer++) { + for (consumer = 0; consumer < 8; consumer++) + KUNIT_EXPECT_EQ(test, avd_inbox_full(producer, consumer), + ((producer + 1) & 7) == consumer); + consumer = (producer + 1) & 7; + KUNIT_EXPECT_TRUE(test, avd_inbox_full(producer, consumer)); + /* A consumer advance reopens the same pending inbox slot. */ + consumer = (consumer + 1) & 7; + KUNIT_EXPECT_FALSE(test, avd_inbox_full(producer, consumer)); + KUNIT_EXPECT_EQ(test, avd_apple_notification(producer), + 0x01482cf8U + 0x60 * producer); + } +} + +static void avd_completion_parser(struct kunit *test) +{ + /* Synthetic layout for parser testing; never a claimed Borage wire ABI. */ + struct avd_apple_reply_layout layout = { + .type_word = 0, .type_mask = 0x1f, + .client_word = 1, .client_shift = 16, .client_mask = 0x3ff, + .frame_word = 3, .status_word = 4, .done_type = 8, .verified = true, + }; + struct avd_apple_reply reply; + u32 record[AVD_APPLE_RESULT_WORDS] = { 7, 5 << 16, 0, 0x12340 }; + + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 19, &layout, &reply), -EMSGSIZE); + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EINPROGRESS); + record[0] = 8; + KUNIT_ASSERT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), 0); + KUNIT_EXPECT_EQ(test, reply.client, 5U); + KUNIT_EXPECT_EQ(test, reply.frame, 0x12340U); + KUNIT_EXPECT_EQ(test, reply.status, 0); + record[4] = 1; + KUNIT_ASSERT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), 0); + KUNIT_EXPECT_EQ(test, reply.status, -EIO); + record[0] = 3; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -ENOMSG); + layout.frame_word = 20; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EINVAL); + layout.verified = false; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EOPNOTSUPP); +} + +static struct kunit_case avd_queue_cases[] = { + KUNIT_CASE(avd_slots_and_tags), + KUNIT_CASE(avd_retirement_order), + KUNIT_CASE(avd_watchdog_and_retry), + KUNIT_CASE(avd_completion_parser), + {} +}; + +static struct kunit_suite avd_queue_suite = { + .name = "apple-avd-queue", + .test_cases = avd_queue_cases, +}; + +kunit_test_suite(avd_queue_suite); diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c new file mode 100644 index 0000000000000..ba2f5b17f8683 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include + +#include "avd.h" +#include "avd-queue-internal.h" + +#define AVD_FRAME_TIMEOUT msecs_to_jiffies(2000) +#define AVD_RETRY_DELAY msecs_to_jiffies(1) +#define AVD_PROGRAM_SIZE SZ_2M + +bool avd_backend_is_apple(const struct avd_dev *avd) +{ + return avd->backend == &avd_apple_backend; +} + +/* queue_mutex excludes callbacks and context destruction from retirement. */ +static void avd_retire(struct avd_dev *avd) +{ + struct avd_frame *f; + unsigned long flags; + unsigned int i; + bool progress; + + do { + progress = false; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_ctx *ctx; + enum vb2_buffer_state state; + + f = &avd->frames[i]; + spin_lock_irqsave(&avd->queue_lock, flags); + if (f->state != AVD_FRAME_DONE || + !avd_slot_first(avd->frames, f)) { + spin_unlock_irqrestore(&avd->queue_lock, flags); + continue; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + ctx = f->ctx; + state = f->status ? VB2_BUF_STATE_ERROR : VB2_BUF_STATE_DONE; + if (ctx->coded_fmt_desc->frame_ops) { + if (ctx->coded_fmt_desc->frame_ops->retire) + ctx->coded_fmt_desc->frame_ops->retire(ctx, f, state); + } else if (ctx->coded_fmt_desc->ops->done) { + ctx->coded_fmt_desc->ops->done(ctx, f->src, f->dst, state); + } + if (f->src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(f->src->vb2_buf.req_obj.req, + &ctx->ctrl_hdl); + /* CAPTURE first: OUTPUT completion signals the media request. */ + if (f->scheduler_held) { + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, + ctx->fh.m2m_ctx, state); + } else { + v4l2_m2m_buf_done(f->dst, state); + v4l2_m2m_buf_done(f->src, state); + } + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + spin_lock_irqsave(&avd->queue_lock, flags); + f->ctx = NULL; + f->src = NULL; + f->dst = NULL; + f->state = AVD_FRAME_FREE; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + progress = true; + } + } while (progress); +} + +/* A reset invalidates every firmware-owned slot, including other clients. */ +static void avd_reset_queue(struct avd_dev *avd, int status) +{ + struct avd_ctx *ctx; + unsigned long flags; + unsigned int i; + int ret; + + avd->backend->shutdown(avd); + ret = reset_control_reset(avd->rstc); + if (!ret) + ret = avd->backend->boot(avd); + avd->faulted = !!ret; + if (ret) + dev_err(avd->dev, "queue reset failed: %d\n", ret); + avd->compat_active = NULL; + list_for_each_entry(ctx, &avd->contexts, queue_node) + ctx->session_started = false; + spin_lock_irqsave(&avd->queue_lock, flags); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state == AVD_FRAME_FREE) + continue; + avd->frames[i].status = status; + avd->frames[i].state = AVD_FRAME_DONE; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); +} + +static int avd_send_frame(struct avd_dev *avd, struct avd_frame *f) +{ + unsigned long flags; + int ret; + + if (f->dep == AVD_DEP_PREV_DONE && !avd_slot_first(avd->frames, f)) + return -EBUSY; + if (!f->ctx->session_started && avd->backend->session_start) { + ret = avd->backend->session_start(f->ctx); + if (ret) + return ret; + f->ctx->session_started = true; + } + /* Publish ownership before the doorbell; IRQs may arrive in submit(). */ + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_SUBMITTED; + spin_unlock_irqrestore(&avd->queue_lock, flags); + dma_wmb(); + ret = avd->backend->submit(avd, f); + spin_lock_irqsave(&avd->queue_lock, flags); + if (ret == -EBUSY) { + f->state = AVD_FRAME_RESERVED; + } else if (ret) { + f->status = ret; + f->state = AVD_FRAME_DONE; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + return ret; +} + +static void avd_queue_worker(struct work_struct *work) +{ + struct avd_dev *avd = container_of(to_delayed_work(work), + struct avd_dev, queue_work); + struct avd_ctx *ctx; + unsigned long flags; + unsigned int i; + bool pending; + + mutex_lock(&avd->queue_mutex); + if (avd->stopping) + goto unlock; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + if (avd_slot_expired(&avd->frames[i], jiffies)) { + dev_err(avd->dev, "frame tag %#x timed out\n", avd->frames[i].tag); + avd_reset_queue(avd, -ETIMEDOUT); + break; + } + } + avd_retire(avd); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + int ret; + + if (f->state != AVD_FRAME_RESERVED) + continue; + ret = avd_send_frame(avd, f); + if (ret && ret != -EBUSY) { + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + } + } + avd_retire(avd); + /* Freed slots/dependencies must wake contexts whose job_ready was false. */ + list_for_each_entry(ctx, &avd->contexts, queue_node) + v4l2_m2m_try_schedule(ctx->fh.m2m_ctx); + spin_lock_irqsave(&avd->queue_lock, flags); + pending = avd->inflight != 0; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (pending) + mod_delayed_work(system_wq, &avd->queue_work, AVD_RETRY_DELAY); + unlock: + mutex_unlock(&avd->queue_mutex); +} + +void avd_frame_complete(struct avd_dev *avd, u32 tag, int status) +{ + struct avd_frame *f = &avd->frames[tag & (AVD_MAX_FRAMES - 1)]; + unsigned long flags; + bool accepted; + + spin_lock_irqsave(&avd->queue_lock, flags); + accepted = avd_slot_complete(f, tag, status); + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (accepted) + mod_delayed_work(system_wq, &avd->queue_work, 0); +} + +int avd_queue_ready(void *priv) +{ + struct avd_ctx *ctx = priv; + struct avd_dev *avd = ctx->dev; + unsigned long flags; + unsigned int i; + bool ready; + + spin_lock_irqsave(&avd->queue_lock, flags); + ready = !ctx->draining && !avd->stopping && !avd->faulted && + avd->inflight < avd->backend->max_inflight(avd); + for (i = 0; ready && i < AVD_MAX_FRAMES; i++) + if (ctx->next_dep == AVD_DEP_PREV_DONE && + avd->frames[i].state != AVD_FRAME_FREE && avd->frames[i].ctx == ctx) + ready = false; + spin_unlock_irqrestore(&avd->queue_lock, flags); + return ready; +} + +void avd_queue_run(void *priv) +{ + struct avd_ctx *ctx = priv; + struct avd_dev *avd = ctx->dev; + const struct avd_codec_frame_ops *ops = ctx->coded_fmt_desc->frame_ops; + struct avd_frame *f = NULL; + unsigned long flags; + unsigned int i; + int ret; + + mutex_lock(&avd->queue_mutex); + spin_lock_irqsave(&avd->queue_lock, flags); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state == AVD_FRAME_FREE) { + f = &avd->frames[i]; + break; + } + } + ret = f ? avd_slot_reserve(f, ctx, ++avd->frame_seq) : -EBUSY; + if (!ret) + avd->inflight++; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (WARN_ON(ret)) { + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + goto unlock; + } + ret = pm_runtime_resume_and_get(avd->dev); + if (ret < 0) + goto fail_no_pm; + f->src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + f->dst = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); + f->scheduler_held = ctx->fh.m2m_ctx->out_q_ctx.q.subsystem_flags & + VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF; + f->frame_num = f->tag; + f->deadline = jiffies + AVD_FRAME_TIMEOUT; + memset(f->cmd, 0, sizeof(f->cmd)); + if (ops) { + if (!f->prog->cpu) { + ret = avd_buf_alloc(avd, f->prog, AVD_PROGRAM_SIZE); + if (ret) + goto fail; + } + if (avd_backend_is_apple(avd) && !f->fp->cpu) { + ret = avd_buf_alloc(avd, f->fp, avd->variant->fp_size); + if (ret) + goto fail; + } + if (f->src->vb2_buf.req_obj.req) + v4l2_ctrl_request_setup(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + ret = ops->prepare(ctx, f); + } else { + avd->building = f; + avd->compat_held = false; + ret = ctx->coded_fmt_desc->ops->run(ctx); + avd->building = NULL; + } + if (ret) + goto fail; + ctx->next_dep = ops ? f->dep : AVD_DEP_PREV_DONE; + if (f->src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF) { + if (ops && ops->retire) + ops->retire(ctx, f, VB2_BUF_STATE_DONE); + else if (!ops && ctx->coded_fmt_desc->ops->done) + ctx->coded_fmt_desc->ops->done(ctx, f->src, f->dst, VB2_BUF_STATE_DONE); + pm_runtime_put_autosuspend(avd->dev); + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_FREE; + f->ctx = NULL; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_DONE); + goto unlock; + } + v4l2_m2m_buf_copy_metadata(f->src, f->dst); + if (!f->scheduler_held) { + v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); + v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + } + f->dst->is_held = false; + ret = avd_send_frame(avd, f); + if (ret && ret != -EBUSY) { + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + } + /* Buffers now belong to the slot, not to the m2m scheduler. */ + if (!f->scheduler_held) + v4l2_m2m_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx); + mod_delayed_work(system_wq, &avd->queue_work, 0); + goto unlock; + fail: + /* Detach before retirement, including partially prepared codec state. */ + if (!f->scheduler_held) { + v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); + v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + } + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (!f->scheduler_held) + v4l2_m2m_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx); + mod_delayed_work(system_wq, &avd->queue_work, 0); + goto unlock; + fail_no_pm: + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_FREE; + f->ctx = NULL; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + unlock: + mutex_unlock(&avd->queue_mutex); +} + +void avd_queue_drain(struct avd_ctx *ctx) +{ + struct avd_dev *avd = ctx->dev; + unsigned int i; + + mutex_lock(&avd->queue_mutex); + WRITE_ONCE(ctx->draining, true); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state != AVD_FRAME_FREE && avd->frames[i].ctx == ctx) { + if (avd->backend->abort) + avd->backend->abort(ctx); + avd_reset_queue(avd, -ECANCELED); + break; + } + } + avd_retire(avd); + if (ctx->session_started && avd->backend->session_stop) + avd->backend->session_stop(ctx); + ctx->session_started = false; + mutex_unlock(&avd->queue_mutex); +} + +void avd_queue_abort(void *priv) +{ + avd_queue_drain(priv); +} + +void avd_queue_add_ctx(struct avd_ctx *ctx) +{ + struct avd_ctx *other; + u32 id; + + mutex_lock(&ctx->dev->queue_mutex); + /* Borage decode prefix has ten client bits. */ + for (id = 0; id < 1024; id++) { + bool used = false; + + list_for_each_entry(other, &ctx->dev->contexts, queue_node) + used |= other->client_id == id; + if (!used) + break; + } + ctx->client_id = id; + list_add_tail(&ctx->queue_node, &ctx->dev->contexts); + mutex_unlock(&ctx->dev->queue_mutex); +} + +void avd_queue_del_ctx(struct avd_ctx *ctx) +{ + avd_queue_drain(ctx); + mutex_lock(&ctx->dev->queue_mutex); + list_del_init(&ctx->queue_node); + mutex_unlock(&ctx->dev->queue_mutex); +} + +int avd_queue_init(struct avd_dev *avd) +{ + unsigned int i; + + spin_lock_init(&avd->queue_lock); + mutex_init(&avd->queue_mutex); + INIT_LIST_HEAD(&avd->contexts); + INIT_DELAYED_WORK(&avd->queue_work, avd_queue_worker); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + + f->idx = i; + f->prog = devm_kzalloc(avd->dev, sizeof(*f->prog), GFP_KERNEL); + f->fp = devm_kzalloc(avd->dev, sizeof(*f->fp), GFP_KERNEL); + if (!f->prog || !f->fp) + return -ENOMEM; + } + return 0; +} + +void avd_queue_cleanup(struct avd_dev *avd) +{ + unsigned int i; + + mutex_lock(&avd->queue_mutex); + avd->stopping = true; + if (avd->inflight) { + avd_reset_queue(avd, -ESHUTDOWN); + avd_retire(avd); + } + mutex_unlock(&avd->queue_mutex); + cancel_delayed_work_sync(&avd->queue_work); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + avd_buf_free(avd, avd->frames[i].prog); + avd_buf_free(avd, avd->frames[i].fp); + } +} diff --git a/drivers/media/platform/apple/avd/avd-queue.h b/drivers/media/platform/apple/avd/avd-queue.h index ca04f908ef359..438afc17184ec 100644 --- a/drivers/media/platform/apple/avd/avd-queue.h +++ b/drivers/media/platform/apple/avd/avd-queue.h @@ -50,6 +50,8 @@ struct avd_frame { void *codec_priv; /* per-frame codec state, owned by the codec */ unsigned long deadline; /* jiffies, per-frame watchdog */ + /* Legacy m2m HOLD_CAPTURE API retains scheduler ownership until retire. */ + bool scheduler_held; }; struct avd_backend_ops { @@ -66,6 +68,7 @@ struct avd_backend_ops { irqreturn_t (*irq)(struct avd_dev *avd); /* calls avd_frame_complete() */ int (*suspend)(struct avd_dev *avd); int (*resume)(struct avd_dev *avd); + int (*set_property)(struct avd_ctx *ctx, u32 property, u32 value); }; /* Codec half. Replaces avd_coded_fmt_ops.run/submit/done. CPU only, no MMIO. */ diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index 4622e92055486..f61763bf17fe3 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -902,6 +902,7 @@ static int avd_start_streaming(struct vb2_queue *q, unsigned int count) goto err_return_buffers; } + ctx->draining = false; if (desc->ops->start) { ret = desc->ops->start(ctx); if (ret) @@ -917,7 +918,7 @@ static int avd_start_streaming(struct vb2_queue *q, unsigned int count) return ret; } -static void avd_queue_cleanup(struct vb2_queue *vq, u32 state) +static void avd_m2m_queue_cleanup(struct vb2_queue *vq, u32 state) { struct avd_ctx *ctx = vb2_get_drv_priv(vq); @@ -942,6 +943,7 @@ static void avd_stop_streaming(struct vb2_queue *q) { struct avd_ctx *ctx = vb2_get_drv_priv(q); + avd_queue_drain(ctx); if (V4L2_TYPE_IS_OUTPUT(q->type)) { const struct avd_coded_fmt_desc *desc = ctx->coded_fmt_desc; @@ -956,7 +958,7 @@ static void avd_stop_streaming(struct vb2_queue *q) ctx->job.segments = NULL; } - avd_queue_cleanup(q, VB2_BUF_STATE_ERROR); + avd_m2m_queue_cleanup(q, VB2_BUF_STATE_ERROR); } const struct vb2_ops avd_queue_ops = { @@ -970,36 +972,17 @@ const struct vb2_ops avd_queue_ops = { .stop_streaming = avd_stop_streaming, }; +/* Legacy multi-slice codecs finish held OUTPUT requests during prepare. */ void avd_job_finish_no_pm(struct avd_ctx *ctx, enum vb2_buffer_state result) { - struct avd_dev *avd = ctx->dev; - unsigned long flags; - - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_ctx == ctx) { - avd->job_state = AVD_JOB_IDLE; - avd->job_ctx = NULL; - avd->job_pending = false; - } - spin_unlock_irqrestore(&avd->job_lock, flags); - - if (ctx->coded_fmt_desc->ops->done) { - struct vb2_v4l2_buffer *src_buf, *dst_buf; - - src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); - dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); - ctx->coded_fmt_desc->ops->done(ctx, src_buf, dst_buf, result); - } - - v4l2_m2m_buf_done_and_job_finish(ctx->dev->m2m_dev, ctx->fh.m2m_ctx, - result); + if (WARN_ON(!ctx->dev->building)) + return; + ctx->dev->compat_held = true; + ctx->dev->building->status = result == VB2_BUF_STATE_DONE ? 0 : -EIO; } void avd_job_finish(struct avd_ctx *ctx, enum vb2_buffer_state result) { - struct avd_dev *avd = ctx->dev; - - pm_runtime_put_autosuspend(avd->dev); avd_job_finish_no_pm(ctx, result); } diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index ad7accb6e9615..a924629e5a5ed 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -18,6 +18,9 @@ #include #include #include +#include + +#include "avd-queue.h" #include #include @@ -173,13 +176,18 @@ struct avd_coded_fmt_desc { struct v4l2_frmsize_stepwise frmsize; const struct avd_ctrls *ctrls; const struct avd_coded_fmt_ops *ops; + const struct avd_codec_frame_ops *frame_ops; unsigned int num_decoded_fmts; const struct avd_decoded_fmt_desc *decoded_fmts; u32 subsystem_flags; unsigned int capability; }; +struct avd_apple_hw; + struct avd_variant { + const struct avd_apple_hw *apple; + size_t fp_size; unsigned int vp_slots[4]; unsigned int vp_slot_base[4]; unsigned int fifo_slots; @@ -196,14 +204,6 @@ struct avd_variant { unsigned int quirks; }; -/* Lifecycle of the one m2m job the device runs at a time. */ -enum avd_job_state { - AVD_JOB_IDLE, - AVD_JOB_RUNNING, /* inside avd_device_run() */ - AVD_JOB_SUBMITTED, /* handed to the hardware, watchdog armed */ - AVD_JOB_FINISHING, /* claimed by the IRQ handler or the watchdog */ -}; - struct avd_dev { struct device *dev; struct v4l2_device v4l2_dev; @@ -223,14 +223,23 @@ struct avd_dev { struct mutex vdev_lock; - /* The current job; see avd_device_run(). Protected by job_lock. */ - spinlock_t job_lock; - enum avd_job_state job_state; - struct avd_ctx *job_ctx; - u64 job_seq; - bool job_end_sent; - bool job_pending; - enum vb2_buffer_state job_pending_result; + /* queue_lock protects slot state/tag and job_ready observations. */ + spinlock_t queue_lock; + /* Serializes CPU preparation, submission, retirement and reset. */ + struct mutex queue_mutex; + struct delayed_work queue_work; + struct list_head contexts; + struct avd_frame frames[AVD_MAX_FRAMES]; + u64 frame_seq; + unsigned int inflight; + bool stopping; + bool faulted; + const struct avd_backend_ops *backend; + void *backend_priv; + /* Only the open-firmware compatibility path uses these. */ + struct avd_frame *building; + struct avd_frame *compat_active; + bool compat_held; struct reset_control *rstc; @@ -267,9 +276,11 @@ struct avd_ctx { enum avd_image_fmt image_fmt; bool decomp; - struct delayed_work watchdog_work; - /* job_seq of the job watchdog_work was armed for */ - u64 wd_seq; + struct list_head queue_node; + bool draining; + bool session_started; + enum avd_frame_dep next_dep; + u32 client_id; void *priv; @@ -330,6 +341,18 @@ static inline struct avd_ctx *file_to_ctx(struct file *filp) /* hw stuff */ int avd_boot(struct avd_dev *avd); +int avd_queue_init(struct avd_dev *avd); +void avd_queue_cleanup(struct avd_dev *avd); +void avd_queue_run(void *priv); +int avd_queue_ready(void *priv); +void avd_queue_abort(void *priv); +void avd_queue_drain(struct avd_ctx *ctx); +void avd_queue_add_ctx(struct avd_ctx *ctx); +void avd_queue_del_ctx(struct avd_ctx *ctx); +int avd_open_build_program(struct avd_ctx *ctx); +extern const struct avd_backend_ops avd_open_backend; +extern const struct avd_backend_ops avd_apple_backend; +int avd_backend_select(struct avd_dev *avd); void avd_shutdown(struct avd_dev *avd); void avd_status(struct avd_dev *avd, u32 vp); From bf09ea01eb46b80d7da4e8ecf001a4330fe33349 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 22:38:44 +1100 Subject: [PATCH 03/73] media: apple: avd: exercise queue ownership and document transport gaps Add deterministic host tests against production helpers and KUnit coverage of slot reuse, ordered retirement, watchdog wrap, inbox retry, command builders and completion parsing. Track the unproved Borage fields explicitly and preserve session generations over firmware reboot. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../platform/apple/avd/avd-apple-protocol.h | 40 +++++++- .../platform/apple/avd/avd-backend-apple.c | 24 ++--- .../platform/apple/avd/avd-backend-open.c | 16 ++++ drivers/media/platform/apple/avd/avd-drv.c | 4 +- .../media/platform/apple/avd/avd-protocol.rst | 95 +++++++++++++++++++ .../platform/apple/avd/avd-queue-internal.h | 1 + .../media/platform/apple/avd/avd-queue-test.c | 30 ++++++ drivers/media/platform/apple/avd/avd-queue.c | 22 ++++- drivers/media/platform/apple/avd/avd.h | 3 + .../platform/apple/avd/tests/.kunitconfig | 12 +++ .../apple/avd/tests/include/linux/errno.h | 2 + .../apple/avd/tests/include/linux/interrupt.h | 5 + .../apple/avd/tests/include/linux/jiffies.h | 6 ++ .../apple/avd/tests/include/linux/limits.h | 4 + .../apple/avd/tests/include/linux/types.h | 10 ++ .../avd/tests/include/media/videobuf2-v4l2.h | 5 + .../platform/apple/avd/tests/queue-host.c | 79 +++++++++++++++ .../platform/apple/avd/tests/run-host.sh | 8 ++ 18 files changed, 346 insertions(+), 20 deletions(-) create mode 100644 drivers/media/platform/apple/avd/avd-protocol.rst create mode 100644 drivers/media/platform/apple/avd/tests/.kunitconfig create mode 100644 drivers/media/platform/apple/avd/tests/include/linux/errno.h create mode 100644 drivers/media/platform/apple/avd/tests/include/linux/interrupt.h create mode 100644 drivers/media/platform/apple/avd/tests/include/linux/jiffies.h create mode 100644 drivers/media/platform/apple/avd/tests/include/linux/limits.h create mode 100644 drivers/media/platform/apple/avd/tests/include/linux/types.h create mode 100644 drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h create mode 100644 drivers/media/platform/apple/avd/tests/queue-host.c create mode 100755 drivers/media/platform/apple/avd/tests/run-host.sh diff --git a/drivers/media/platform/apple/avd/avd-apple-protocol.h b/drivers/media/platform/apple/avd/avd-apple-protocol.h index 1a497ff2b2bff..77440a760c1e7 100644 --- a/drivers/media/platform/apple/avd/avd-apple-protocol.h +++ b/drivers/media/platform/apple/avd/avd-apple-protocol.h @@ -18,6 +18,39 @@ */ #define AVD_APPLE_NOTIFY_BASE AVD_APPLE_INBOX_BASE +/* SPEC 5.1: only these session opcodes have proved builders. */ +static inline int avd_apple_session_command(u32 cmd[24], u32 client, u32 opcode, + u32 property, u32 value) +{ + unsigned int i; + + if (client >= 1024 || (opcode != 0 && opcode != 2 && opcode != 5 && opcode != 6)) + return -EINVAL; + for (i = 0; i < 24; i++) + cmd[i] = 0; + cmd[0] = opcode | (client << 16); + if (opcode == 6) { + cmd[1] = property; + cmd[2] = value; + } + return 0; +} + +static inline int avd_apple_decode_prefix(struct avd_frame *f, u32 client, + u32 selector, u32 work_sequence) +{ + unsigned int i; + + if (f->idx >= AVD_MAX_FRAMES || client >= 1024 || selector >= 16 || work_sequence >= 4) + return -EINVAL; + for (i = 0; i < 24; i++) + f->cmd[i] = 0; + f->cmd[0] = 1 | (selector << 10) | (client << 22); + f->cmd[1] = f->idx | (work_sequence << 16); + f->cmd[3] = f->frame_num; + return 0; +} + struct avd_apple_reply_layout { u8 type_word; u8 type_shift; @@ -38,9 +71,10 @@ struct avd_apple_reply { int status; }; -static inline int avd_apple_parse_reply(const u32 *record, size_t words, - const struct avd_apple_reply_layout *layout, - struct avd_apple_reply *reply) +static inline int +avd_apple_parse_reply(const u32 *record, size_t words, + const struct avd_apple_reply_layout *layout, + struct avd_apple_reply *reply) { if (words != AVD_APPLE_RESULT_WORDS) return -EMSGSIZE; diff --git a/drivers/media/platform/apple/avd/avd-backend-apple.c b/drivers/media/platform/apple/avd/avd-backend-apple.c index fbe035172957d..3bd4a8e14d3c8 100644 --- a/drivers/media/platform/apple/avd/avd-backend-apple.c +++ b/drivers/media/platform/apple/avd/avd-backend-apple.c @@ -55,7 +55,8 @@ static int avd_apple_image_present(struct avd_apple *apple) u32 end = readl(apple->regs + hw->itcm_end); u32 version; - if (end <= start || end - start > hw->imem_size) + if (end <= start || end - start < 4 || end - start > hw->imem_size || + hw->version_offset > end - start - 4) return -ENOENT; version = readl(apple->regs + hw->imem + hw->version_offset); if (!version || version == U32_MAX) @@ -190,7 +191,8 @@ static int avd_apple_session_start(struct avd_ctx *ctx) { u32 cmd[24] = {}; - cmd[0] = ctx->client_id << 16; + if (avd_apple_session_command(cmd, ctx->client_id, 0, 0, 0)) + return -EINVAL; return avd_apple_send(ctx->dev, cmd); } @@ -198,7 +200,8 @@ static void avd_apple_session_stop(struct avd_ctx *ctx) { u32 cmd[24] = {}; - cmd[0] = 2 | (ctx->client_id << 16); + if (avd_apple_session_command(cmd, ctx->client_id, 2, 0, 0)) + return; if (avd_apple_send(ctx->dev, cmd)) dev_warn(ctx->dev->dev, "session stop inbox full\n"); } @@ -207,7 +210,8 @@ static void avd_apple_abort(struct avd_ctx *ctx) { u32 cmd[24] = {}; - cmd[0] = 5 | (ctx->client_id << 16); + if (avd_apple_session_command(cmd, ctx->client_id, 5, 0, 0)) + return; /* Core follows abort with reset before returning any DMA buffer. */ avd_apple_send(ctx->dev, cmd); } @@ -216,9 +220,8 @@ static int avd_apple_set_property(struct avd_ctx *ctx, u32 property, u32 value) { u32 cmd[24] = {}; - cmd[0] = 6 | (ctx->client_id << 16); - cmd[1] = property; - cmd[2] = value; + if (avd_apple_session_command(cmd, ctx->client_id, 6, property, value)) + return -EINVAL; return avd_apple_send(ctx->dev, cmd); } @@ -231,11 +234,10 @@ static int avd_apple_submit(struct avd_dev *avd, struct avd_frame *f) if (!ops || !ops->pack_cmd || !f->fp->cpu || f->ctx->client_id >= 1024) return -EOPNOTSUPP; /* SPEC 5.2 prefix precedes the codec-specific builder. */ - memset(f->cmd, 0, sizeof(f->cmd)); selector = get_unaligned_le32(f->fp->cpu + 4) & 0xf; - f->cmd[0] = 1 | (selector << 10) | (f->ctx->client_id << 22); - f->cmd[1] = f->idx | ((f->seq & 3) << 16); - f->cmd[3] = f->frame_num; + ret = avd_apple_decode_prefix(f, f->ctx->client_id, selector, f->seq & 3); + if (ret) + return ret; ret = ops->pack_cmd(f->ctx, f); if (ret) return ret; diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 5ea70af085674..4354c78dac9b6 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -159,3 +159,19 @@ int avd_backend_select(struct avd_dev *avd) avd->backend = &avd_open_backend; return request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); } + +int avd_backend_boot(struct avd_dev *avd) +{ + int ret = avd->backend->boot(avd); + + if (ret == -ENOENT && avd_backend_is_apple(avd)) { + ret = request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); + if (ret) + return ret; + avd->backend = &avd_open_backend; + ret = avd->backend->boot(avd); + } + if (!ret) + avd->fw_generation++; + return ret; +} diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index c60f0417c5cd2..918f4397e346a 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -687,7 +687,7 @@ static __maybe_unused int avd_runtime_resume(struct device *dev) int ret; struct avd_dev *avd = platform_get_drvdata(to_platform_device(dev)); - ret = avd->backend->boot(avd); + ret = avd_backend_boot(avd); if (ret) dev_err(dev, "failed to boot"); return ret; @@ -697,6 +697,8 @@ static __maybe_unused int avd_runtime_suspend(struct device *dev) { struct avd_dev *avd = platform_get_drvdata(to_platform_device(dev)); + if (READ_ONCE(avd->inflight)) + return -EBUSY; avd->backend->shutdown(avd); return 0; } diff --git a/drivers/media/platform/apple/avd/avd-protocol.rst b/drivers/media/platform/apple/avd/avd-protocol.rst new file mode 100644 index 0000000000000..69199e984e789 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-protocol.rst @@ -0,0 +1,95 @@ +AVD queued transport implementation status +========================================== + +The pool owns sixteen independently tagged slots. A source/destination pair +and a runtime-PM reference remain owned by the slot until in-order retirement +within its context. A duplicate or obsolete tag cannot complete a reused +slot. Inbox-full leaves a prepared slot reserved and retries without losing +its buffers. Timeout or streamoff resets the whole engine before failing all +slots affected by that reset. Slot deadlines survive inbox retries. + +The first commit adding avd-queue.h is the shared interface. Later additions +only extend it. Codec callbacks own codec-private state and must release that +state from retire(), including when prepare() failed part way through. + +Open firmware compatibility +-------------------------- + +The deployed, untagged open-firmware protocol has a one-frame limit. Existing +codec run() callbacks build a DMA-visible per-slot program; only the backend +writes those instructions to the VP FIFO. That compatibility backend retains +the old host-side PP kick. A native frame_ops program must use the queued +avd-fw transport before it can replace the compatibility path; this backend +returns EOPNOTSUPP for native programs rather than guessing their wire format. + +The V4L2 core currently warns unconditionally when job_finish() is used on a +queue advertising HOLD_CAPTURE_BUF. Such queues retain the scheduler until +retirement and use buf_done_and_job_finish(), including held slice requests. +Other queues release the scheduler as soon as their buffers belong to a slot. +Removing this restriction requires a V4L2 core API change outside this driver's +scope. It is not safe to toggle subsystem_flags around job_finish(). + +Apple Borage transport +---------------------- + +Only T6040 has Borage match data. The Apple backend contains the 64-byte +little-endian connection configuration, session command builders, decode +prefix, eight-entry inbox publishing, a 64-entry completion shadow and bounded +20-word result parsing. Type 7 is VP/parsing completion and never releases a +buffer. The notification address rule has one definition, +AVD_APPLE_NOTIFY_BASE, so a capture can correct it. + +Boot checks a nonzero ITCM span and a nonzero/non-all-ones version word before +accepting a preloaded image. A separately installed firmware image can restore +ITCM after power loss. The last image word is zero padded without reading +past the firmware object. No Apple image is embedded in this source tree. + +Activation remains disabled in match data: the supplied clean specification +has no ITCM register offsets, link programming sequence, result-area address, +result field layout or final-completion value. It would be unsafe to invent +these or reuse the different open-firmware mailbox. The closest executable +path is the open compatibility backend. The Apple register description and +reply layout must be completed together from clean facts; setting verified +alone is insufficient. The DT must then describe the complete Apple aperture +as an owned named resource before probe can map it. + +Unresolved field ledger +----------------------- + +This is the single ledger for fields marked [C] in the common transport. No +semantic names have been assigned to the unknown bits. Codec-specific fields +remain in their codec packers' ledgers. + +============================== ================================================ +Field Source and required evidence +============================== ================================================ +Config +00/+08/+10/+18/+20/ SPEC 3.2: exact values, meanings unproved; +28/+2c/+30/+38 capture connection construction and LinkUp +ITCM start/end/clear/enable SPEC 3.1: operations, offsets/values absent; + clean per-SoC register ledger required +IMEM version-word offset SPEC 3.1: display format proved, offset absent +Mailbox link register writes SPEC 3.2: aperture 1, state 4, LinkUp proved; + offsets and bit operations absent +Mailbox ready register SPEC 3.1: status 1, poll 10 us; offset absent +Mailbox notification word SPEC 4.2; expected inbox-entry address from MIT + aura trace_avd.py V3 precedent; capture Borage +Inbox consumption/full rule SPEC 4.2: host positions modulo eight; + hardware consumer register/doorbell absent +Result-area base/indexing SPEC 4.3: 64 entries, 20 words; base and mailbox + notification interpretation absent +Reply type/client/frame/status SPEC 4.3: correlation proved, word layout absent; + synthetic KUnit layout is not a hardware fact +Final reply type/status values SPEC 4.3: only VP done = 7 proved; capture final + completion and error replies before activation +Decode +00 bit 9 SPEC 5.2: boolean argument, meaning unproved; + left zero until the producer is established +Decode +50..+5f SPEC 5: Borage extension, nonzero use unproved; + initialized to zero before codec packing +Commands 3/4 and other values SPEC 5.1: unproved; never sent +Retention and clock floors SPEC 8: per-SoC capture required; no guessed knobs +============================== ================================================ + +Licensed implementation references: eiln/avd (MIT), avd_emu.py, and +https://github.com/aurora-silicon/aura/blob/main/proxyclient/hv/trace_avd.py +(MIT). The Borage facts above come from the supplied clean SPEC sections 3-5; +no binary or captured Apple command images are used. diff --git a/drivers/media/platform/apple/avd/avd-queue-internal.h b/drivers/media/platform/apple/avd/avd-queue-internal.h index 7ff919ce3b7ee..57184d5cb65a6 100644 --- a/drivers/media/platform/apple/avd/avd-queue-internal.h +++ b/drivers/media/platform/apple/avd/avd-queue-internal.h @@ -4,6 +4,7 @@ #include #include +#include #include "avd-queue.h" diff --git a/drivers/media/platform/apple/avd/avd-queue-test.c b/drivers/media/platform/apple/avd/avd-queue-test.c index 938c577cb7824..855e6630fafe3 100644 --- a/drivers/media/platform/apple/avd/avd-queue-test.c +++ b/drivers/media/platform/apple/avd/avd-queue-test.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT #include +#include #include "avd-apple-protocol.h" #include "avd-queue-internal.h" @@ -112,11 +113,40 @@ static void avd_completion_parser(struct kunit *test) KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EOPNOTSUPP); } +static void avd_command_builders(struct kunit *test) +{ + struct avd_frame f = { .idx = 15, .frame_num = 0x1234 }; + u32 cmd[24]; + const u64 config[] = { + 0, 0x01558000, 0x01558a00, 0x01558e00, 0, + 0x000000040004000aULL, 0, 0x01100008, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(config); i++) + KUNIT_EXPECT_EQ(test, get_unaligned_le64(avd_borage_config + i * 8), config[i]); + KUNIT_ASSERT_EQ(test, avd_apple_decode_prefix(&f, 1023, 15, 3), 0); + KUNIT_EXPECT_EQ(test, f.cmd[0], 1U | (15U << 10) | (1023U << 22)); + KUNIT_EXPECT_EQ(test, f.cmd[1], 15U | (3U << 16)); + KUNIT_EXPECT_EQ(test, f.cmd[3], 0x1234U); + KUNIT_EXPECT_EQ(test, f.cmd[23], 0U); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 1024, 0, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 0, 16, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 0, 0, 4), -EINVAL); + KUNIT_ASSERT_EQ(test, avd_apple_session_command(cmd, 2, 6, 17, 42), 0); + KUNIT_EXPECT_EQ(test, cmd[0], 6U | (2U << 16)); + KUNIT_EXPECT_EQ(test, cmd[1], 17U); + KUNIT_EXPECT_EQ(test, cmd[2], 42U); + KUNIT_EXPECT_EQ(test, cmd[23], 0U); + KUNIT_EXPECT_EQ(test, avd_apple_session_command(cmd, 2, 3, 0, 0), -EINVAL); +} + static struct kunit_case avd_queue_cases[] = { KUNIT_CASE(avd_slots_and_tags), KUNIT_CASE(avd_retirement_order), KUNIT_CASE(avd_watchdog_and_retry), KUNIT_CASE(avd_completion_parser), + KUNIT_CASE(avd_command_builders), {} }; diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index ba2f5b17f8683..60df9a49f7311 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -46,7 +46,7 @@ static void avd_retire(struct avd_dev *avd) } else if (ctx->coded_fmt_desc->ops->done) { ctx->coded_fmt_desc->ops->done(ctx, f->src, f->dst, state); } - if (f->src->vb2_buf.req_obj.req) + if (ctx->coded_fmt_desc->frame_ops && f->src->vb2_buf.req_obj.req) v4l2_ctrl_request_complete(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); /* CAPTURE first: OUTPUT completion signals the media request. */ @@ -82,7 +82,7 @@ static void avd_reset_queue(struct avd_dev *avd, int status) avd->backend->shutdown(avd); ret = reset_control_reset(avd->rstc); if (!ret) - ret = avd->backend->boot(avd); + ret = avd_backend_boot(avd); avd->faulted = !!ret; if (ret) dev_err(avd->dev, "queue reset failed: %d\n", ret); @@ -106,11 +106,14 @@ static int avd_send_frame(struct avd_dev *avd, struct avd_frame *f) if (f->dep == AVD_DEP_PREV_DONE && !avd_slot_first(avd->frames, f)) return -EBUSY; - if (!f->ctx->session_started && avd->backend->session_start) { + if ((!f->ctx->session_started || + f->ctx->session_generation != avd->fw_generation) && + avd->backend->session_start) { ret = avd->backend->session_start(f->ctx); if (ret) return ret; f->ctx->session_started = true; + f->ctx->session_generation = avd->fw_generation; } /* Publish ownership before the doorbell; IRQs may arrive in submit(). */ spin_lock_irqsave(&avd->queue_lock, flags); @@ -273,6 +276,8 @@ void avd_queue_run(void *priv) ops->retire(ctx, f, VB2_BUF_STATE_DONE); else if (!ops && ctx->coded_fmt_desc->ops->done) ctx->coded_fmt_desc->ops->done(ctx, f->src, f->dst, VB2_BUF_STATE_DONE); + if (ops && f->src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); pm_runtime_put_autosuspend(avd->dev); spin_lock_irqsave(&avd->queue_lock, flags); f->state = AVD_FRAME_FREE; @@ -343,8 +348,15 @@ void avd_queue_drain(struct avd_ctx *ctx) } } avd_retire(avd); - if (ctx->session_started && avd->backend->session_stop) - avd->backend->session_stop(ctx); + if (ctx->session_started && avd->backend->session_stop) { + int ret = pm_runtime_resume_and_get(avd->dev); + + if (ret >= 0) { + avd->backend->session_stop(ctx); + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + } + } ctx->session_started = false; mutex_unlock(&avd->queue_mutex); } diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index a924629e5a5ed..f8e29c1e403c2 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -234,6 +234,7 @@ struct avd_dev { unsigned int inflight; bool stopping; bool faulted; + u64 fw_generation; const struct avd_backend_ops *backend; void *backend_priv; /* Only the open-firmware compatibility path uses these. */ @@ -279,6 +280,7 @@ struct avd_ctx { struct list_head queue_node; bool draining; bool session_started; + u64 session_generation; enum avd_frame_dep next_dep; u32 client_id; @@ -353,6 +355,7 @@ int avd_open_build_program(struct avd_ctx *ctx); extern const struct avd_backend_ops avd_open_backend; extern const struct avd_backend_ops avd_apple_backend; int avd_backend_select(struct avd_dev *avd); +int avd_backend_boot(struct avd_dev *avd); void avd_shutdown(struct avd_dev *avd); void avd_status(struct avd_dev *avd, u32 vp); diff --git a/drivers/media/platform/apple/avd/tests/.kunitconfig b/drivers/media/platform/apple/avd/tests/.kunitconfig new file mode 100644 index 0000000000000..43dde13f9e5c1 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/.kunitconfig @@ -0,0 +1,12 @@ +CONFIG_KUNIT=y +CONFIG_COMPILE_TEST=y +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_SUPPORT_FILTER=n +CONFIG_VIDEO_DEV=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_OF=y +CONFIG_VIDEO_APPLE_AVD=y +CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST=y +CONFIG_WERROR=y diff --git a/drivers/media/platform/apple/avd/tests/include/linux/errno.h b/drivers/media/platform/apple/avd/tests/include/linux/errno.h new file mode 100644 index 0000000000000..32a2df3ba8d20 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/errno.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include_next diff --git a/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h b/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h new file mode 100644 index 0000000000000..7639a30041680 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_INTERRUPT_H_ +#define AVD_HOST_INTERRUPT_H_ +typedef int irqreturn_t; +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h b/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h new file mode 100644 index 0000000000000..a73bb63db2c8e --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_JIFFIES_H_ +#define AVD_HOST_JIFFIES_H_ +#include +#define time_after_eq(a, b) ((long)((a) - (b)) >= 0) +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/linux/limits.h b/drivers/media/platform/apple/avd/tests/include/linux/limits.h new file mode 100644 index 0000000000000..304aeeeff0fbd --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/limits.h @@ -0,0 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include +#include +#define U32_MAX UINT32_MAX diff --git a/drivers/media/platform/apple/avd/tests/include/linux/types.h b/drivers/media/platform/apple/avd/tests/include/linux/types.h new file mode 100644 index 0000000000000..c7ae40af9b31c --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/types.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_TYPES_H_ +#define AVD_HOST_TYPES_H_ +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint64_t u64; +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h b/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h new file mode 100644 index 0000000000000..6cd054cc198d3 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_VB2_H_ +#define AVD_HOST_VB2_H_ +enum vb2_buffer_state { VB2_BUF_STATE_DONE, VB2_BUF_STATE_ERROR }; +#endif diff --git a/drivers/media/platform/apple/avd/tests/queue-host.c b/drivers/media/platform/apple/avd/tests/queue-host.c new file mode 100644 index 0000000000000..5983d7ad5d6a3 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/queue-host.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Exercise the production helpers with deterministic shuffled completions. */ +#include +#include +#include + +#include "../avd-apple-protocol.h" +#include "../avd-queue-internal.h" + +int main(void) +{ + struct avd_frame pool[AVD_MAX_FRAMES] = {}; + int clients[3]; + u64 last[3] = {}, seq = 0; + u32 rng = 0x3851, old_tag[AVD_MAX_FRAMES] = {}; + unsigned int i, step, completed = 0; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + pool[i].idx = i; + for (step = 0; step < 100000; step++) { + struct avd_frame *f; + + rng = rng * 1664525 + 1013904223; + f = &pool[(rng >> 16) & 15]; + if (f->state == AVD_FRAME_FREE) { + struct avd_ctx *ctx = (void *)&clients[(rng >> 8) % 3]; + + assert(!avd_slot_reserve(f, ctx, ++seq)); + assert(!avd_slot_complete(f, f->tag, 0)); + f->deadline = step + 100; + f->state = AVD_FRAME_SUBMITTED; + assert(!avd_slot_complete(f, old_tag[f->idx], 0)); + old_tag[f->idx] = f->tag; + } else if (f->state == AVD_FRAME_SUBMITTED) { + assert(avd_slot_complete(f, f->tag, (rng & 1) ? -EIO : 0)); + assert(!avd_slot_complete(f, f->tag, 0)); + } + for (i = 0; i < AVD_MAX_FRAMES; i++) { + unsigned int c; + + f = &pool[i]; + if (f->state != AVD_FRAME_DONE || !avd_slot_first(pool, f)) + continue; + for (c = 0; c < 3; c++) + if (f->ctx == (void *)&clients[c]) + break; + assert(c < 3 && f->seq > last[c]); + last[c] = f->seq; + f->state = AVD_FRAME_FREE; + completed++; + } + } + for (i = 0; i < 8; i++) { + assert(avd_inbox_full(i, (i + 1) & 7)); + assert(!avd_inbox_full(i, (i + 2) & 7)); + assert(avd_apple_notification(i) == 0x01482cf8U + 0x60 * i); + } + { + struct avd_apple_reply_layout layout = { + .type_word = 0, .type_mask = 31, .client_word = 1, + .client_mask = 1023, .frame_word = 3, .status_word = 4, + .done_type = 8, .verified = true, + }; + struct avd_apple_reply reply; + u32 record[20] = { 7, 9, 0, 1234 }; + + assert(avd_apple_parse_reply(record, 20, &layout, &reply) == -EINPROGRESS); + record[0] = 8; + assert(!avd_apple_parse_reply(record, 20, &layout, &reply)); + assert(reply.frame == 1234 && reply.client == 9 && !reply.status); + record[4] = 1; + assert(!avd_apple_parse_reply(record, 20, &layout, &reply)); + assert(reply.status == -EIO); + assert(avd_apple_parse_reply(record, 19, &layout, &reply) == -EMSGSIZE); + } + assert(completed > 1000); + printf("PASS: 100000 shuffled operations, %u ordered retirements\n", completed); + return 0; +} diff --git a/drivers/media/platform/apple/avd/tests/run-host.sh b/drivers/media/platform/apple/avd/tests/run-host.sh new file mode 100755 index 0000000000000..70e287f14732d --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/run-host.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only OR MIT +set -eu +avd_test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +avd_test_out=${1:?usage: run-host.sh OUTPUT_BINARY} +${CC:-cc} -std=gnu11 -Wall -Wextra -Werror -O2 \ + -I"$avd_test_dir/include" "$avd_test_dir/queue-host.c" -o "$avd_test_out" +"$avd_test_out" From e62165841f0175af835542ca914c6e7df587f796 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 22:42:00 +1100 Subject: [PATCH 04/73] media: apple: avd: tighten slot teardown and backend buffer ownership Remove contexts from the scheduling list before releasing their m2m state, reject already-scheduled jobs during teardown, and keep Apple fp pointers absent on the open backend. Preserve tagged slots over inbox-full retries and avoid periodic watchdog wakeups when only submitted frames remain. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../platform/apple/avd/avd-backend-open.c | 1 + drivers/media/platform/apple/avd/avd-drv.c | 16 ++++- .../media/platform/apple/avd/avd-protocol.rst | 2 + .../platform/apple/avd/avd-queue-internal.h | 11 ++++ .../media/platform/apple/avd/avd-queue-test.c | 11 ++++ drivers/media/platform/apple/avd/avd-queue.c | 63 ++++++++++++++----- drivers/media/platform/apple/avd/avd-v4l2.c | 2 +- drivers/media/platform/apple/avd/avd.h | 1 + .../apple/avd/tests/include/linux/interrupt.h | 2 +- .../apple/avd/tests/include/linux/types.h | 8 +-- 10 files changed, 93 insertions(+), 24 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 4354c78dac9b6..4ffd9faac6aa4 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT #include #include +#include #include "avd.h" #include "avd-inst.h" diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 918f4397e346a..8d217404d55e8 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -250,8 +250,8 @@ static int avd_release(struct file *filp) struct avd_ctx *ctx = file_to_ctx(filp); v4l2_fh_del(&ctx->fh, filp); - v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); avd_queue_del_ctx(ctx); + v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); v4l2_ctrl_handler_free(&ctx->ctrl_hdl); v4l2_fh_exit(&ctx->fh); avd_buf_free(ctx->dev, &ctx->inst); @@ -581,6 +581,13 @@ static const struct of_device_id avd_of_match[] = { MODULE_DEVICE_TABLE(of, avd_of_match); +static void avd_release_open_firmware(void *data) +{ + struct avd_dev *avd = data; + + release_firmware(avd->fw); +} + static int avd_probe(struct platform_device *pdev) { struct avd_dev *avd; @@ -628,11 +635,15 @@ static int avd_probe(struct platform_device *pdev) } ret = avd_backend_select(avd); + if (ret) + return ret; + ret = devm_add_action_or_reset(avd->dev, avd_release_open_firmware, avd); if (ret) return ret; ret = dma_set_mask_and_coherent(avd->dev, - DMA_BIT_MASK((avd->variant->quirks & AVD_QUIRK_LSR) ? 38 : 64)); + DMA_BIT_MASK(avd_backend_is_apple(avd) ? 42 : + (avd->variant->quirks & AVD_QUIRK_LSR) ? 38 : 64)); if (ret) { dev_err(avd->dev, "Failed to set DMA mask"); @@ -671,7 +682,6 @@ static void avd_remove(struct platform_device *pdev) struct avd_dev *avd = platform_get_drvdata(pdev); avd_queue_cleanup(avd); - release_firmware(avd->fw); avd_v4l2_cleanup(avd); diff --git a/drivers/media/platform/apple/avd/avd-protocol.rst b/drivers/media/platform/apple/avd/avd-protocol.rst index 69199e984e789..ce9eaf0bf2720 100644 --- a/drivers/media/platform/apple/avd/avd-protocol.rst +++ b/drivers/media/platform/apple/avd/avd-protocol.rst @@ -86,6 +86,8 @@ Decode +00 bit 9 SPEC 5.2: boolean argument, meaning unproved; Decode +50..+5f SPEC 5: Borage extension, nonzero use unproved; initialized to zero before codec packing Commands 3/4 and other values SPEC 5.1: unproved; never sent +PIODMA mapping stream binding SPEC 7: three mapper roles proved, Linux binding + and per-domain fp/program DVA ownership pending Retention and clock floors SPEC 8: per-SoC capture required; no guessed knobs ============================== ================================================ diff --git a/drivers/media/platform/apple/avd/avd-queue-internal.h b/drivers/media/platform/apple/avd/avd-queue-internal.h index 57184d5cb65a6..0e2a668d5a62f 100644 --- a/drivers/media/platform/apple/avd/avd-queue-internal.h +++ b/drivers/media/platform/apple/avd/avd-queue-internal.h @@ -27,6 +27,17 @@ static inline int avd_slot_reserve(struct avd_frame *f, struct avd_ctx *ctx, return 0; } +/* A full inbox owns nothing: keep the prepared slot, tag and deadline. */ +static inline void avd_slot_submit_result(struct avd_frame *f, int status) +{ + if (status == -EBUSY) { + f->state = AVD_FRAME_RESERVED; + } else if (status) { + f->status = status; + f->state = AVD_FRAME_DONE; + } +} + static inline bool avd_slot_complete(struct avd_frame *f, u32 tag, int status) { if (f->state != AVD_FRAME_SUBMITTED || f->tag != tag) diff --git a/drivers/media/platform/apple/avd/avd-queue-test.c b/drivers/media/platform/apple/avd/avd-queue-test.c index 855e6630fafe3..0945c4b8307ce 100644 --- a/drivers/media/platform/apple/avd/avd-queue-test.c +++ b/drivers/media/platform/apple/avd/avd-queue-test.c @@ -62,6 +62,17 @@ static void avd_watchdog_and_retry(struct kunit *test) { struct avd_frame f = { .state = AVD_FRAME_SUBMITTED, .deadline = ULONG_MAX - 2 }; unsigned int producer, consumer; + u32 tag = 0x124; + unsigned long deadline = f.deadline; + + f.tag = tag; + avd_slot_submit_result(&f, -EBUSY); + KUNIT_EXPECT_EQ(test, f.state, AVD_FRAME_RESERVED); + KUNIT_EXPECT_EQ(test, f.tag, tag); + KUNIT_EXPECT_EQ(test, f.deadline, deadline); + f.state = AVD_FRAME_SUBMITTED; + avd_slot_submit_result(&f, 0); + KUNIT_EXPECT_EQ(test, f.state, AVD_FRAME_SUBMITTED); KUNIT_EXPECT_FALSE(test, avd_slot_expired(&f, ULONG_MAX - 3)); KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, ULONG_MAX - 2)); diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index 60df9a49f7311..f1651722c4601 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT +#include #include #include #include @@ -122,12 +123,7 @@ static int avd_send_frame(struct avd_dev *avd, struct avd_frame *f) dma_wmb(); ret = avd->backend->submit(avd, f); spin_lock_irqsave(&avd->queue_lock, flags); - if (ret == -EBUSY) { - f->state = AVD_FRAME_RESERVED; - } else if (ret) { - f->status = ret; - f->state = AVD_FRAME_DONE; - } + avd_slot_submit_result(f, ret); spin_unlock_irqrestore(&avd->queue_lock, flags); return ret; } @@ -140,12 +136,18 @@ static void avd_queue_worker(struct work_struct *work) unsigned long flags; unsigned int i; bool pending; + unsigned long delay = AVD_FRAME_TIMEOUT; mutex_lock(&avd->queue_mutex); if (avd->stopping) goto unlock; for (i = 0; i < AVD_MAX_FRAMES; i++) { - if (avd_slot_expired(&avd->frames[i], jiffies)) { + bool expired; + + spin_lock_irqsave(&avd->queue_lock, flags); + expired = avd_slot_expired(&avd->frames[i], jiffies); + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (expired) { dev_err(avd->dev, "frame tag %#x timed out\n", avd->frames[i].tag); avd_reset_queue(avd, -ETIMEDOUT); break; @@ -172,9 +174,20 @@ static void avd_queue_worker(struct work_struct *work) v4l2_m2m_try_schedule(ctx->fh.m2m_ctx); spin_lock_irqsave(&avd->queue_lock, flags); pending = avd->inflight != 0; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + unsigned long remaining; + + if (f->state == AVD_FRAME_FREE) + continue; + remaining = time_after(f->deadline, jiffies) ? f->deadline - jiffies : 0; + if (f->state == AVD_FRAME_RESERVED) + remaining = min(remaining, AVD_RETRY_DELAY); + delay = min(delay, remaining); + } spin_unlock_irqrestore(&avd->queue_lock, flags); if (pending) - mod_delayed_work(system_wq, &avd->queue_work, AVD_RETRY_DELAY); + mod_delayed_work(system_wq, &avd->queue_work, delay); unlock: mutex_unlock(&avd->queue_mutex); } @@ -201,7 +214,8 @@ int avd_queue_ready(void *priv) bool ready; spin_lock_irqsave(&avd->queue_lock, flags); - ready = !ctx->draining && !avd->stopping && !avd->faulted && + ready = !READ_ONCE(ctx->draining) && !READ_ONCE(avd->stopping) && + !READ_ONCE(avd->faulted) && avd->inflight < avd->backend->max_inflight(avd); for (i = 0; ready && i < AVD_MAX_FRAMES; i++) if (ctx->next_dep == AVD_DEP_PREV_DONE && @@ -222,6 +236,11 @@ void avd_queue_run(void *priv) int ret; mutex_lock(&avd->queue_mutex); + if (avd->stopping || ctx->draining || avd->faulted) { + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + goto unlock; + } spin_lock_irqsave(&avd->queue_lock, flags); for (i = 0; i < AVD_MAX_FRAMES; i++) { if (avd->frames[i].state == AVD_FRAME_FREE) { @@ -254,10 +273,22 @@ void avd_queue_run(void *priv) if (ret) goto fail; } - if (avd_backend_is_apple(avd) && !f->fp->cpu) { - ret = avd_buf_alloc(avd, f->fp, avd->variant->fp_size); - if (ret) + /* NULL fp is also the codec-facing indication of an open program. */ + f->fp = NULL; + if (avd_backend_is_apple(avd)) { + if (!avd->frame_fp[f->idx]) + avd->frame_fp[f->idx] = devm_kzalloc(avd->dev, + sizeof(*f->fp), GFP_KERNEL); + f->fp = avd->frame_fp[f->idx]; + if (!f->fp) { + ret = -ENOMEM; goto fail; + } + if (!f->fp->cpu) { + ret = avd_buf_alloc(avd, f->fp, avd->variant->fp_size); + if (ret) + goto fail; + } } if (f->src->vb2_buf.req_obj.req) v4l2_ctrl_request_setup(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); @@ -358,6 +389,8 @@ void avd_queue_drain(struct avd_ctx *ctx) } } ctx->session_started = false; + if (!avd->stopping) + mod_delayed_work(system_wq, &avd->queue_work, 0); mutex_unlock(&avd->queue_mutex); } @@ -407,8 +440,7 @@ int avd_queue_init(struct avd_dev *avd) f->idx = i; f->prog = devm_kzalloc(avd->dev, sizeof(*f->prog), GFP_KERNEL); - f->fp = devm_kzalloc(avd->dev, sizeof(*f->fp), GFP_KERNEL); - if (!f->prog || !f->fp) + if (!f->prog) return -ENOMEM; } return 0; @@ -428,6 +460,7 @@ void avd_queue_cleanup(struct avd_dev *avd) cancel_delayed_work_sync(&avd->queue_work); for (i = 0; i < AVD_MAX_FRAMES; i++) { avd_buf_free(avd, avd->frames[i].prog); - avd_buf_free(avd, avd->frames[i].fp); + if (avd->frame_fp[i]) + avd_buf_free(avd, avd->frame_fp[i]); } } diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index f61763bf17fe3..661100c2365b7 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -893,6 +893,7 @@ static int avd_start_streaming(struct vb2_queue *q, unsigned int count) struct vb2_v4l2_buffer *vbuf; int ret; + WRITE_ONCE(ctx->draining, false); if (V4L2_TYPE_IS_CAPTURE(q->type)) return 0; @@ -902,7 +903,6 @@ static int avd_start_streaming(struct vb2_queue *q, unsigned int count) goto err_return_buffers; } - ctx->draining = false; if (desc->ops->start) { ret = desc->ops->start(ctx); if (ret) diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index f8e29c1e403c2..4d00f14b24c55 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -230,6 +230,7 @@ struct avd_dev { struct delayed_work queue_work; struct list_head contexts; struct avd_frame frames[AVD_MAX_FRAMES]; + struct avd_buf *frame_fp[AVD_MAX_FRAMES]; u64 frame_seq; unsigned int inflight; bool stopping; diff --git a/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h b/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h index 7639a30041680..ccda48a6de34b 100644 --- a/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h +++ b/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0-only OR MIT */ #ifndef AVD_HOST_INTERRUPT_H_ #define AVD_HOST_INTERRUPT_H_ -typedef int irqreturn_t; +#define irqreturn_t int #endif diff --git a/drivers/media/platform/apple/avd/tests/include/linux/types.h b/drivers/media/platform/apple/avd/tests/include/linux/types.h index c7ae40af9b31c..3d29df0e18138 100644 --- a/drivers/media/platform/apple/avd/tests/include/linux/types.h +++ b/drivers/media/platform/apple/avd/tests/include/linux/types.h @@ -3,8 +3,8 @@ #define AVD_HOST_TYPES_H_ #include #include -#include -typedef uint8_t u8; -typedef uint32_t u32; -typedef uint64_t u64; +#include_next +typedef __u8 u8; +typedef __u32 u32; +typedef __u64 u64; #endif From 0b31a30118a4564d33306924ebf5e72e533b8f65 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 22:43:14 +1100 Subject: [PATCH 05/73] media: apple: avd: polish queue test configuration and formatting Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-apple-protocol.h | 5 +++-- drivers/media/platform/apple/avd/avd-queue.c | 11 +++++++---- drivers/media/platform/apple/avd/tests/.kunitconfig | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-apple-protocol.h b/drivers/media/platform/apple/avd/avd-apple-protocol.h index 77440a760c1e7..d0b90c5d48e4c 100644 --- a/drivers/media/platform/apple/avd/avd-apple-protocol.h +++ b/drivers/media/platform/apple/avd/avd-apple-protocol.h @@ -19,8 +19,9 @@ #define AVD_APPLE_NOTIFY_BASE AVD_APPLE_INBOX_BASE /* SPEC 5.1: only these session opcodes have proved builders. */ -static inline int avd_apple_session_command(u32 cmd[24], u32 client, u32 opcode, - u32 property, u32 value) +static inline int +avd_apple_session_command(u32 cmd[24], u32 client, u32 opcode, + u32 property, u32 value) { unsigned int i; diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index f1651722c4601..a7298ac5ec56a 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -276,10 +276,13 @@ void avd_queue_run(void *priv) /* NULL fp is also the codec-facing indication of an open program. */ f->fp = NULL; if (avd_backend_is_apple(avd)) { - if (!avd->frame_fp[f->idx]) - avd->frame_fp[f->idx] = devm_kzalloc(avd->dev, - sizeof(*f->fp), GFP_KERNEL); - f->fp = avd->frame_fp[f->idx]; + struct avd_buf *fp = avd->frame_fp[f->idx]; + + if (!fp) { + fp = devm_kzalloc(avd->dev, sizeof(*fp), GFP_KERNEL); + avd->frame_fp[f->idx] = fp; + } + f->fp = fp; if (!f->fp) { ret = -ENOMEM; goto fail; diff --git a/drivers/media/platform/apple/avd/tests/.kunitconfig b/drivers/media/platform/apple/avd/tests/.kunitconfig index 43dde13f9e5c1..b9364f3a86fed 100644 --- a/drivers/media/platform/apple/avd/tests/.kunitconfig +++ b/drivers/media/platform/apple/avd/tests/.kunitconfig @@ -1,7 +1,7 @@ CONFIG_KUNIT=y CONFIG_COMPILE_TEST=y CONFIG_MEDIA_SUPPORT=y -CONFIG_MEDIA_SUPPORT_FILTER=n +# CONFIG_MEDIA_SUPPORT_FILTER is not set CONFIG_VIDEO_DEV=y CONFIG_MEDIA_CONTROLLER=y CONFIG_MEDIA_PLATFORM_SUPPORT=y From 8f5f7145d59ce61f4f22e52a42301e1ac87b8be5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 22:44:35 +1100 Subject: [PATCH 06/73] media: apple: avd: preserve open firmware capability filtering Keep the CODECS trailer intersection with the SoC capability table when selecting the open backend. Bound firmware copies by the mapped code resource. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../platform/apple/avd/avd-backend-open.c | 38 ++++++++++++++++++- drivers/media/platform/apple/avd/avd-drv.c | 3 -- drivers/media/platform/apple/avd/avd-hw.c | 3 ++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 4ffd9faac6aa4..9ab090cd44310 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -1,4 +1,5 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT +// SPDX-License-Identifier: GPL-2.0-only +/* Legacy transport split from avd-drv.c; Copyright The Asahi Linux Contributors. */ #include #include #include @@ -78,6 +79,8 @@ static int avd_open_submit(struct avd_dev *avd, struct avd_frame *f) if (codec > AVD_CODEC_AV1 || !words || !f->prog_words || f->prog_words > f->prog->size / sizeof(u32)) return -EINVAL; + if (!(avd->capabilities & BIT(codec))) + return -EOPNOTSUPP; for (i = 0; i < codec; i++) vp += avd->variant->vp_slots[i]; if (avd->variant->vp_slot_base[codec]) @@ -143,6 +146,32 @@ const struct avd_backend_ops avd_open_backend = { .irq = avd_open_irq, }; +static void avd_open_capabilities(struct avd_dev *avd) +{ + static const struct { + const char *name; + u32 bit; + } codecs[] = { + { "h265", AVD_CAPABILITY_HEVC }, + { "h264", AVD_CAPABILITY_H264 }, + { "vp9", AVD_CAPABILITY_VP9 }, + { "av1", AVD_CAPABILITY_AV1 }, + }; + const char *info; + unsigned int i; + + avd->capabilities = avd->variant->capabilities; + if (!avd->fw || avd->fw->size < 0x40) + return; + info = avd->fw->data + avd->fw->size - 0x40; + if (memcmp(info, "CODECS:", 7)) + return; + avd->capabilities = 0; + for (i = 0; i < ARRAY_SIZE(codecs); i++) + if (strnstr(info, codecs[i].name, 0x40)) + avd->capabilities |= avd->variant->capabilities & codecs[i].bit; +} + int avd_backend_select(struct avd_dev *avd) { int ret; @@ -152,13 +181,17 @@ int avd_backend_select(struct avd_dev *avd) ret = avd_apple_backend.probe(avd); if (!ret) { avd->backend = &avd_apple_backend; + avd->capabilities = avd->variant->capabilities; return 0; } if (ret != -ENOENT && ret != -EOPNOTSUPP) return ret; } avd->backend = &avd_open_backend; - return request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); + ret = request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); + if (!ret) + avd_open_capabilities(avd); + return ret; } int avd_backend_boot(struct avd_dev *avd) @@ -170,6 +203,7 @@ int avd_backend_boot(struct avd_dev *avd) if (ret) return ret; avd->backend = &avd_open_backend; + avd_open_capabilities(avd); ret = avd->backend->boot(avd); } if (!ret) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 8d217404d55e8..3266f5b5d6518 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -24,8 +24,6 @@ #include "avd-regs.h" #include "avd-apple-protocol.h" -#define AVD_CODEC_INFO_MAX_SIZE 0x40 - static void calc_tile_meta(u32 w, u32 h, u32 bpb, u32 tile_dim, u32 meta_hdr_bytes, u32 *tile, u32 *meta) { @@ -592,7 +590,6 @@ static int avd_probe(struct platform_device *pdev) { struct avd_dev *avd; const struct of_device_id *match; - const char *codec_info = NULL; int ret, irq; avd = devm_kzalloc(&pdev->dev, sizeof(*avd), GFP_KERNEL); diff --git a/drivers/media/platform/apple/avd/avd-hw.c b/drivers/media/platform/apple/avd/avd-hw.c index a3987ac4510de..84db61a77f6f4 100644 --- a/drivers/media/platform/apple/avd/avd-hw.c +++ b/drivers/media/platform/apple/avd/avd-hw.c @@ -42,9 +42,12 @@ int avd_boot(struct avd_dev *avd) { + struct resource *code = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "code"); u32 val; int ret; + if (!avd->fw || !avd->fw->size || !code || avd->fw->size > resource_size(code)) + return -EINVAL; if (avd->variant->revision != 3) dev_info_once(avd->dev, "booting hw version: %04x", readl_relaxed(avd->ctrl)); From 3f2510865895ebf471d2873fa1515538ca283191 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 22:45:19 +1100 Subject: [PATCH 07/73] media: apple: avd: retain legacy HEVC held-buffer submissions Only the H.264 compatibility builder explicitly completes held slice requests during preparation. Continue submitting legacy HEVC requests that retain their capture buffer. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-backend-open.c | 3 ++- drivers/media/platform/apple/avd/avd-queue.c | 3 ++- drivers/media/platform/apple/avd/avd.h | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 9ab090cd44310..68326f30eae1f 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "avd.h" #include "avd-inst.h" @@ -163,7 +164,7 @@ static void avd_open_capabilities(struct avd_dev *avd) avd->capabilities = avd->variant->capabilities; if (!avd->fw || avd->fw->size < 0x40) return; - info = avd->fw->data + avd->fw->size - 0x40; + info = (const char *)avd->fw->data + avd->fw->size - 0x40; if (memcmp(info, "CODECS:", 7)) return; avd->capabilities = 0; diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index a7298ac5ec56a..670bd0edbd030 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -305,7 +305,8 @@ void avd_queue_run(void *priv) if (ret) goto fail; ctx->next_dep = ops ? f->dep : AVD_DEP_PREV_DONE; - if (f->src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF) { + if ((ops && (f->src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF)) || + (!ops && avd->compat_held)) { if (ops && ops->retire) ops->retire(ctx, f, VB2_BUF_STATE_DONE); else if (!ops && ctx->coded_fmt_desc->ops->done) diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 4d00f14b24c55..a32598c365262 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -212,7 +212,7 @@ struct avd_dev { struct v4l2_m2m_dev *m2m_dev; struct platform_device *pdev; - const struct firmware *fw; /* fw is lost on suspend */ + const struct firmware *fw; /* separately installed open firmware */ void __iomem *code; void __iomem *mbox; From c6f63dabc46e8026d03bf39fcb113d0aa82ab5e1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 22:47:32 +1100 Subject: [PATCH 08/73] media: apple: avd: reserve m2m dispatch capacity and defer CPU preparation Prevent queued contexts from observing the same free capacity. Run preparation on a context work item because m2m try_schedule can invoke device_run synchronously while the queue mutex is held. Drain that work before freeing the context. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-drv.c | 1 + .../media/platform/apple/avd/avd-queue-test.c | 36 +++++++++++++++++++ drivers/media/platform/apple/avd/avd-queue.c | 29 ++++++++++++--- drivers/media/platform/apple/avd/avd.h | 3 ++ .../platform/apple/avd/tests/.kunitconfig | 1 + 5 files changed, 66 insertions(+), 4 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 3266f5b5d6518..4abdc1de5f032 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -250,6 +250,7 @@ static int avd_release(struct file *filp) v4l2_fh_del(&ctx->fh, filp); avd_queue_del_ctx(ctx); v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); + cancel_work_sync(&ctx->submit_work); v4l2_ctrl_handler_free(&ctx->ctrl_hdl); v4l2_fh_exit(&ctx->fh); avd_buf_free(ctx->dev, &ctx->inst); diff --git a/drivers/media/platform/apple/avd/avd-queue-test.c b/drivers/media/platform/apple/avd/avd-queue-test.c index 0945c4b8307ce..eec4761f850e9 100644 --- a/drivers/media/platform/apple/avd/avd-queue-test.c +++ b/drivers/media/platform/apple/avd/avd-queue-test.c @@ -2,6 +2,7 @@ #include #include +#include "avd.h" #include "avd-apple-protocol.h" #include "avd-queue-internal.h" @@ -152,12 +153,47 @@ static void avd_command_builders(struct kunit *test) KUNIT_EXPECT_EQ(test, avd_apple_session_command(cmd, 2, 3, 0, 0), -EINVAL); } +static unsigned int avd_test_max_inflight(struct avd_dev *avd) +{ + return AVD_MAX_FRAMES; +} + +static void avd_dispatch_reservation(struct kunit *test) +{ + static const struct avd_backend_ops backend = { + .max_inflight = avd_test_max_inflight, + }; + struct avd_dev *avd = kunit_kzalloc(test, sizeof(*avd), GFP_KERNEL); + struct avd_ctx *ctx = kunit_kcalloc(test, 2, sizeof(*ctx), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, avd); + KUNIT_ASSERT_NOT_NULL(test, ctx); + spin_lock_init(&avd->queue_lock); + avd->backend = &backend; + ctx[0].dev = avd; + ctx[1].dev = avd; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[0]), 1); + /* The second context must not reserve the same observed capacity. */ + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 0); + avd->queued_ctx = NULL; + avd->inflight = AVD_MAX_FRAMES; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 0); + avd->inflight = 1; + avd->frames[0].state = AVD_FRAME_SUBMITTED; + avd->frames[0].ctx = &ctx[1]; + ctx[1].next_dep = AVD_DEP_PREV_DONE; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 0); + avd->frames[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); +} + static struct kunit_case avd_queue_cases[] = { KUNIT_CASE(avd_slots_and_tags), KUNIT_CASE(avd_retirement_order), KUNIT_CASE(avd_watchdog_and_retry), KUNIT_CASE(avd_completion_parser), KUNIT_CASE(avd_command_builders), + KUNIT_CASE(avd_dispatch_reservation), {} }; diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index 670bd0edbd030..8afc8ad13c711 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -214,20 +214,24 @@ int avd_queue_ready(void *priv) bool ready; spin_lock_irqsave(&avd->queue_lock, flags); - ready = !READ_ONCE(ctx->draining) && !READ_ONCE(avd->stopping) && + ready = !avd->queued_ctx && !READ_ONCE(ctx->draining) && + !READ_ONCE(avd->stopping) && !READ_ONCE(avd->faulted) && avd->inflight < avd->backend->max_inflight(avd); for (i = 0; ready && i < AVD_MAX_FRAMES; i++) if (ctx->next_dep == AVD_DEP_PREV_DONE && avd->frames[i].state != AVD_FRAME_FREE && avd->frames[i].ctx == ctx) ready = false; + /* Prevent several queued m2m contexts from observing the same free slot. */ + if (ready) + avd->queued_ctx = ctx; spin_unlock_irqrestore(&avd->queue_lock, flags); return ready; } -void avd_queue_run(void *priv) +static void avd_queue_submit_work(struct work_struct *work) { - struct avd_ctx *ctx = priv; + struct avd_ctx *ctx = container_of(work, struct avd_ctx, submit_work); struct avd_dev *avd = ctx->dev; const struct avd_codec_frame_ops *ops = ctx->coded_fmt_desc->frame_ops; struct avd_frame *f = NULL; @@ -236,12 +240,15 @@ void avd_queue_run(void *priv) int ret; mutex_lock(&avd->queue_mutex); + spin_lock_irqsave(&avd->queue_lock, flags); + if (avd->queued_ctx == ctx) + avd->queued_ctx = NULL; if (avd->stopping || ctx->draining || avd->faulted) { + spin_unlock_irqrestore(&avd->queue_lock, flags); v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, VB2_BUF_STATE_ERROR); goto unlock; } - spin_lock_irqsave(&avd->queue_lock, flags); for (i = 0; i < AVD_MAX_FRAMES; i++) { if (avd->frames[i].state == AVD_FRAME_FREE) { f = &avd->frames[i]; @@ -367,13 +374,26 @@ void avd_queue_run(void *priv) mutex_unlock(&avd->queue_mutex); } +void avd_queue_run(void *priv) +{ + struct avd_ctx *ctx = priv; + + /* try_schedule() may invoke device_run synchronously under queue_mutex. */ + schedule_work(&ctx->submit_work); +} + void avd_queue_drain(struct avd_ctx *ctx) { struct avd_dev *avd = ctx->dev; + unsigned long flags; unsigned int i; mutex_lock(&avd->queue_mutex); + spin_lock_irqsave(&avd->queue_lock, flags); WRITE_ONCE(ctx->draining, true); + if (avd->queued_ctx == ctx) + avd->queued_ctx = NULL; + spin_unlock_irqrestore(&avd->queue_lock, flags); for (i = 0; i < AVD_MAX_FRAMES; i++) { if (avd->frames[i].state != AVD_FRAME_FREE && avd->frames[i].ctx == ctx) { if (avd->backend->abort) @@ -408,6 +428,7 @@ void avd_queue_add_ctx(struct avd_ctx *ctx) struct avd_ctx *other; u32 id; + INIT_WORK(&ctx->submit_work, avd_queue_submit_work); mutex_lock(&ctx->dev->queue_mutex); /* Borage decode prefix has ten client bits. */ for (id = 0; id < 1024; id++) { diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index a32598c365262..69787064c4eec 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -229,6 +229,8 @@ struct avd_dev { struct mutex queue_mutex; struct delayed_work queue_work; struct list_head contexts; + /* One m2m dispatch reservation; never identifies firmware-owned work. */ + struct avd_ctx *queued_ctx; struct avd_frame frames[AVD_MAX_FRAMES]; struct avd_buf *frame_fp[AVD_MAX_FRAMES]; u64 frame_seq; @@ -279,6 +281,7 @@ struct avd_ctx { bool decomp; struct list_head queue_node; + struct work_struct submit_work; bool draining; bool session_started; u64 session_generation; diff --git a/drivers/media/platform/apple/avd/tests/.kunitconfig b/drivers/media/platform/apple/avd/tests/.kunitconfig index b9364f3a86fed..c079bc44af531 100644 --- a/drivers/media/platform/apple/avd/tests/.kunitconfig +++ b/drivers/media/platform/apple/avd/tests/.kunitconfig @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT CONFIG_KUNIT=y CONFIG_COMPILE_TEST=y CONFIG_MEDIA_SUPPORT=y From a11617279b458d19e7703e545604e2f1d41d55d6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 22:48:30 +1100 Subject: [PATCH 09/73] media: apple: avd: reject failed native request setup Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-backend-open.c | 1 + drivers/media/platform/apple/avd/avd-queue.c | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 68326f30eae1f..13729c9e4a0b9 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* Legacy transport split from avd-drv.c; Copyright The Asahi Linux Contributors. */ #include +#include #include #include #include diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index 8afc8ad13c711..4991b5a38bac7 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -84,7 +84,7 @@ static void avd_reset_queue(struct avd_dev *avd, int status) ret = reset_control_reset(avd->rstc); if (!ret) ret = avd_backend_boot(avd); - avd->faulted = !!ret; + WRITE_ONCE(avd->faulted, !!ret); if (ret) dev_err(avd->dev, "queue reset failed: %d\n", ret); avd->compat_active = NULL; @@ -300,8 +300,11 @@ static void avd_queue_submit_work(struct work_struct *work) goto fail; } } - if (f->src->vb2_buf.req_obj.req) - v4l2_ctrl_request_setup(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + if (f->src->vb2_buf.req_obj.req) { + ret = v4l2_ctrl_request_setup(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + if (ret) + goto fail; + } ret = ops->prepare(ctx, f); } else { avd->building = f; @@ -476,7 +479,7 @@ void avd_queue_cleanup(struct avd_dev *avd) unsigned int i; mutex_lock(&avd->queue_mutex); - avd->stopping = true; + WRITE_ONCE(avd->stopping, true); if (avd->inflight) { avd_reset_queue(avd, -ESHUTDOWN); avd_retire(avd); From 3be54082040ccd36fe1489962c1bdfc0e51b422f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:01:24 +1100 Subject: [PATCH 10/73] media: apple: avd: coordinate frame queue runtime and system power Keep power through slot retirement and context drains, restore firmware before retrying reserved frames, and defer DMA releases while the DART is inactive. Re-enable a retained Apple MCPU image before mailbox link setup, without uploading it again. Pause and drain the queue for system sleep and final teardown. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/Kconfig | 5 +- drivers/media/platform/apple/avd/Makefile | 7 +- .../platform/apple/avd/avd-backend-apple.c | 5 + .../platform/apple/avd/avd-backend-open.c | 2 + drivers/media/platform/apple/avd/avd-drv.c | 67 +--- .../media/platform/apple/avd/avd-pm-test.c | 291 +++++++++++++++++ drivers/media/platform/apple/avd/avd-pm.c | 304 ++++++++++++++++++ drivers/media/platform/apple/avd/avd-pm.h | 60 ++++ .../platform/apple/avd/avd-pm.kunitconfig | 14 + drivers/media/platform/apple/avd/avd-queue.c | 33 +- drivers/media/platform/apple/avd/avd.h | 4 + 11 files changed, 714 insertions(+), 78 deletions(-) create mode 100644 drivers/media/platform/apple/avd/avd-pm-test.c create mode 100644 drivers/media/platform/apple/avd/avd-pm.c create mode 100644 drivers/media/platform/apple/avd/avd-pm.h create mode 100644 drivers/media/platform/apple/avd/avd-pm.kunitconfig diff --git a/drivers/media/platform/apple/avd/Kconfig b/drivers/media/platform/apple/avd/Kconfig index ed5ad44276c31..d3bb9763b6230 100644 --- a/drivers/media/platform/apple/avd/Kconfig +++ b/drivers/media/platform/apple/avd/Kconfig @@ -24,8 +24,7 @@ config VIDEO_APPLE_AVD_KUNIT_TEST default KUNIT_ALL_TESTS help Build KUnit tests for the Apple video decoder. They exercise the - checks that the H.264 path makes on the slices that userspace - queues, such as finding the slice data after the slice header, - at the limits and one past them. + H.264 slice bounds, frame-slot ownership, and runtime power management, + including suspend gating and resubmission after firmware resume. If unsure, say N. diff --git a/drivers/media/platform/apple/avd/Makefile b/drivers/media/platform/apple/avd/Makefile index 64095b4242133..3372a76baa9ee 100644 --- a/drivers/media/platform/apple/avd/Makefile +++ b/drivers/media/platform/apple/avd/Makefile @@ -1,7 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only -apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-h264.o avd-vp9.o avd-hevc.o \ - avd-av1.o avd-av1-entropymode.o avd-queue.o avd-backend-open.o \ +apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-pm.o avd-h264.o avd-vp9.o \ + avd-hevc.o avd-av1.o avd-av1-entropymode.o avd-queue.o avd-backend-open.o \ avd-backend-apple.o -apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o avd-queue-test.o +apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o avd-queue-test.o \ + avd-pm-test.o obj-$(CONFIG_VIDEO_APPLE_AVD) += apple-avd.o diff --git a/drivers/media/platform/apple/avd/avd-backend-apple.c b/drivers/media/platform/apple/avd/avd-backend-apple.c index 3bd4a8e14d3c8..9a6fc01dfa466 100644 --- a/drivers/media/platform/apple/avd/avd-backend-apple.c +++ b/drivers/media/platform/apple/avd/avd-backend-apple.c @@ -134,6 +134,9 @@ static int avd_apple_boot(struct avd_dev *avd) ret = avd_apple_load(avd); if (ret) return ret; + } else { + /* Suspend stopped the MCPU without clearing the retained image. */ + writel(hw->itcm_enable_value, apple->regs + hw->itcm_enable); } dev_info(avd->dev, "Apple MCPU firmware %u.%u.%u (%08x)\n", apple->version >> 8, (apple->version >> 4) & 0xf, @@ -304,4 +307,6 @@ const struct avd_backend_ops avd_apple_backend = { .abort = avd_apple_abort, .set_property = avd_apple_set_property, .irq = avd_apple_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, }; diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 13729c9e4a0b9..476e07517d96b 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -146,6 +146,8 @@ const struct avd_backend_ops avd_open_backend = { .max_inflight = avd_open_max_inflight, .submit = avd_open_submit, .irq = avd_open_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, }; static void avd_open_capabilities(struct avd_dev *avd) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 4abdc1de5f032..0e9a4da6d6eea 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -72,31 +72,6 @@ void fill_comp(struct avd_comp *comp, enum avd_image_fmt image_fmt, u32 width, comp->size = y_meta + y + uv_meta + uv; } -int avd_buf_alloc(struct avd_dev *avd, struct avd_buf *buf, size_t size) -{ - /* Keep size equal to the payload length used by the H.264 slice path. */ - avd_buf_free(avd, buf); - - if (size <= 0) - return -ENOMEM; - - buf->size = size; - buf->cpu = - dma_alloc_coherent(avd->dev, buf->size, &buf->addr, GFP_KERNEL); - if (!buf->cpu) { - memset(buf, 0, sizeof(*buf)); - return -ENOMEM; - } - return 0; -} - -void avd_buf_free(struct avd_dev *avd, struct avd_buf *buf) -{ - if (buf->cpu) - dma_free_coherent(avd->dev, buf->size, buf->cpu, buf->addr); - memset(buf, 0, sizeof(*buf)); -} - struct avd_decoded_buffer * avd_get_ref_buf(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst, u64 timestamp) { @@ -602,6 +577,7 @@ static int avd_probe(struct platform_device *pdev) avd->pdev = pdev; mutex_init(&avd->vdev_lock); + avd_pm_init(avd, NULL, &avd_pm_queue_ops); ret = avd_queue_init(avd); if (ret) return ret; @@ -659,9 +635,7 @@ static int avd_probe(struct platform_device *pdev) return ret; } - pm_runtime_set_autosuspend_delay(avd->dev, 100); - pm_runtime_use_autosuspend(avd->dev); - pm_runtime_enable(avd->dev); + avd_pm_enable(avd); ret = avd_v4l2_init(avd); if (ret) @@ -678,44 +652,19 @@ static int avd_probe(struct platform_device *pdev) static void avd_remove(struct platform_device *pdev) { struct avd_dev *avd = platform_get_drvdata(pdev); + int ret; + v4l2_m2m_suspend(avd->m2m_dev); + ret = avd->pm.queue->drain(avd); + if (ret) + dev_warn(avd->dev, "queue drain failed, resetting before teardown: %d\n", ret); avd_queue_cleanup(avd); - + avd_pm_disable(avd); avd_v4l2_cleanup(avd); - if (avd->empty_domain) iommu_domain_free(avd->empty_domain); - - pm_runtime_disable(avd->dev); - pm_runtime_dont_use_autosuspend(avd->dev); } -static __maybe_unused int avd_runtime_resume(struct device *dev) -{ - int ret; - struct avd_dev *avd = platform_get_drvdata(to_platform_device(dev)); - - ret = avd_backend_boot(avd); - if (ret) - dev_err(dev, "failed to boot"); - return ret; -} - -static __maybe_unused int avd_runtime_suspend(struct device *dev) -{ - struct avd_dev *avd = platform_get_drvdata(to_platform_device(dev)); - - if (READ_ONCE(avd->inflight)) - return -EBUSY; - avd->backend->shutdown(avd); - return 0; -} - -static const struct dev_pm_ops avd_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS( - pm_runtime_force_suspend, - pm_runtime_force_resume) SET_RUNTIME_PM_OPS(avd_runtime_suspend, - avd_runtime_resume, NULL) }; - static struct platform_driver avd_driver = { .probe = avd_probe, .remove = avd_remove, diff --git a/drivers/media/platform/apple/avd/avd-pm-test.c b/drivers/media/platform/apple/avd/avd-pm-test.c new file mode 100644 index 0000000000000..6aa341eff55a4 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm-test.c @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "avd.h" + +struct avd_pm_test { + struct avd_dev avd; + bool queue_initialized; + struct avd_deferred_unmap unmap; + unsigned int draining; + unsigned int step; + unsigned int unmap_step; + unsigned int boot_step; + unsigned int resubmit_step; + unsigned int shutdowns; + unsigned int submissions; + int boot_error; + int suspend_error; + bool unmapped_active; +}; + +static struct avd_pm_test *avd_pm_fixture(struct avd_dev *avd) +{ + return container_of(avd, struct avd_pm_test, avd); +} + +static bool avd_pm_test_idle(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + return avd_pm_can_suspend(fixture->avd.frames, fixture->draining); +} + +static int avd_pm_test_drain(struct avd_dev *avd) +{ + return avd_pm_test_idle(avd) ? 0 : -EBUSY; +} + +static int avd_pm_test_boot(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->boot_step = ++fixture->step; + return fixture->boot_error; +} + +static void avd_pm_test_shutdown(struct avd_dev *avd) +{ + avd_pm_fixture(avd)->shutdowns++; +} + +static int avd_pm_test_suspend(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + if (fixture->suspend_error) + return fixture->suspend_error; + return avd_pm_backend_suspend(avd); +} + +static int avd_pm_test_submit(struct avd_dev *avd, struct avd_frame *frame) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->resubmit_step = ++fixture->step; + fixture->submissions++; + frame->state = AVD_FRAME_SUBMITTED; + return 0; +} + +static void avd_pm_test_resubmit(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + unsigned int i; + + fixture->resubmit_step = ++fixture->step; + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (fixture->avd.frames[i].state == AVD_FRAME_RESERVED) + avd->backend->submit(avd, &fixture->avd.frames[i]); +} + +static void avd_pm_test_unmap(struct avd_dev *avd, + struct avd_deferred_unmap *unmap) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->unmap_step = ++fixture->step; + fixture->unmapped_active = avd->pm.dart_active; +} + +static const struct avd_queue_pm_ops avd_pm_test_queue = { + .idle = avd_pm_test_idle, + .drain = avd_pm_test_drain, + .resubmit = avd_pm_test_resubmit, +}; + +static const struct avd_backend_ops avd_pm_test_backend = { + .name = "test", + .boot = avd_pm_test_boot, + .shutdown = avd_pm_test_shutdown, + .submit = avd_pm_test_submit, + .suspend = avd_pm_test_suspend, + .resume = avd_pm_backend_resume, +}; + +static int avd_pm_test_init(struct kunit *test) +{ + struct avd_pm_test *fixture; + + fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + if (!fixture) + return -ENOMEM; + test->priv = fixture; + avd_pm_init(&fixture->avd, &avd_pm_test_backend, &avd_pm_test_queue); + INIT_LIST_HEAD(&fixture->unmap.node); + fixture->unmap.unmap = avd_pm_test_unmap; + return 0; +} + +static void avd_pm_suspend_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + unsigned int i, state; + + KUNIT_EXPECT_TRUE(test, avd_pm_test_idle(&fixture->avd)); + fixture->draining = 1; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), -EBUSY); + fixture->draining = 0; + + /* An occupied slot anywhere vetoes idle, even after completion IRQ. */ + for (i = 0; i < AVD_MAX_FRAMES; i++) { + for (state = AVD_FRAME_RESERVED; state <= AVD_FRAME_DONE; state++) { + fixture->avd.frames[i].state = state; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), -EBUSY); + } + fixture->avd.frames[i].state = AVD_FRAME_FREE; + } + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), 0); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); +} + +static void avd_pm_resume_order_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_EQ(test, fixture->unmap_step, 0U); + fixture->avd.frames[0].state = AVD_FRAME_RESERVED; + fixture->avd.frames[1].state = AVD_FRAME_SUBMITTED; + fixture->avd.frames[2].state = AVD_FRAME_DONE; + fixture->avd.frames[15].state = AVD_FRAME_RESERVED; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_TRUE(test, fixture->unmapped_active); + KUNIT_EXPECT_LT(test, fixture->unmap_step, fixture->boot_step); + KUNIT_EXPECT_LT(test, fixture->boot_step, fixture->resubmit_step); + KUNIT_EXPECT_EQ(test, fixture->submissions, 2U); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[0].state, AVD_FRAME_SUBMITTED); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[2].state, AVD_FRAME_DONE); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); +} + +static void avd_pm_resume_failure_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + fixture->avd.frames[0].state = AVD_FRAME_RESERVED; + fixture->boot_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_power_on(avd), -ETIMEDOUT); + KUNIT_EXPECT_FALSE(test, avd->pm.dart_active); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[0].state, AVD_FRAME_RESERVED); + + /* A later successful resume retries the still-unposted frame once. */ + fixture->boot_error = 0; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_EQ(test, fixture->submissions, 1U); +} + +static void avd_pm_suspend_failure_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + fixture->suspend_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, avd->pm.dart_active); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); +} + +static void avd_pm_deferred_unmap_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + unsigned int step; + + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_TRUE(test, fixture->unmapped_active); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); + KUNIT_ASSERT_EQ(test, avd_pm_power_off(avd), 0); + step = fixture->unmap_step; + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_EQ(test, fixture->unmap_step, step); + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_GT(test, fixture->unmap_step, step); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); +} + +static void avd_pm_real_queue_resume_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct avd_ctx *ctx; + unsigned int i; + + avd->dev = kunit_device_register(test, "avd-pm-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, avd->dev); + KUNIT_ASSERT_EQ(test, avd_queue_init(avd), 0); + fixture->queue_initialized = true; + avd->backend = &avd_pm_test_backend; + avd->pm.queue = &avd_pm_queue_ops; + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->dev = avd; + for (i = 0; i < 3; i++) { + avd->frames[i].ctx = ctx; + avd->frames[i].seq = i + 1; + avd->frames[i].deadline = jiffies + msecs_to_jiffies(60000); + avd->frames[i].state = AVD_FRAME_RESERVED; + } + /* This one was posted before the power transition; never replay it. */ + avd->frames[1].state = AVD_FRAME_SUBMITTED; + avd->inflight = 3; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + flush_delayed_work(&avd->queue_work); + cancel_delayed_work_sync(&avd->queue_work); + KUNIT_EXPECT_EQ(test, fixture->submissions, 2U); + KUNIT_EXPECT_LT(test, fixture->boot_step, fixture->resubmit_step); + KUNIT_EXPECT_EQ(test, avd->frames[0].state, AVD_FRAME_SUBMITTED); + KUNIT_EXPECT_EQ(test, avd->frames[2].state, AVD_FRAME_SUBMITTED); +} + +static void avd_pm_real_queue_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + spin_lock_init(&avd->queue_lock); + avd->pm.queue = &avd_pm_queue_ops; + avd_pm_drain_begin(avd); + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -EBUSY); + avd_pm_drain_end(avd); + avd->frames[15].state = AVD_FRAME_DONE; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -EBUSY); + avd->frames[15].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), 0); +} + +static void avd_pm_test_exit(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + + if (fixture->queue_initialized) + cancel_delayed_work_sync(&fixture->avd.queue_work); +} + +static struct kunit_case avd_pm_test_cases[] = { + KUNIT_CASE(avd_pm_suspend_gate_test), + KUNIT_CASE(avd_pm_resume_order_test), + KUNIT_CASE(avd_pm_resume_failure_test), + KUNIT_CASE(avd_pm_suspend_failure_test), + KUNIT_CASE(avd_pm_deferred_unmap_test), + KUNIT_CASE(avd_pm_real_queue_resume_test), + KUNIT_CASE(avd_pm_real_queue_gate_test), + {} +}; + +static struct kunit_suite avd_pm_test_suite = { + .name = "apple-avd-pm", + .init = avd_pm_test_init, + .exit = avd_pm_test_exit, + .test_cases = avd_pm_test_cases, +}; + +kunit_test_suite(avd_pm_test_suite); diff --git a/drivers/media/platform/apple/avd/avd-pm.c b/drivers/media/platform/apple/avd/avd-pm.c new file mode 100644 index 0000000000000..32b2f6929fbd6 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include + +#include "avd.h" + +/* [C] Keep Linux's existing default until the idle-timer capture is available. */ +#define AVD_AUTOSUSPEND_MS 100 +#define AVD_DRAIN_TIMEOUT_MS 5000 + +struct avd_dma_release { + struct avd_deferred_unmap unmap; + void *cpu; + dma_addr_t addr; + size_t size; +}; + +static void avd_dma_unmap(struct avd_dev *avd, + struct avd_deferred_unmap *unmap) +{ + struct avd_dma_release *release; + + release = container_of(unmap, struct avd_dma_release, unmap); + dma_free_coherent(avd->dev, release->size, release->cpu, release->addr); + kfree(release); +} + +int avd_buf_alloc(struct avd_dev *avd, struct avd_buf *buf, size_t size) +{ + struct avd_dma_release *release; + + avd_buf_free(avd, buf); + if (!size) + return -ENOMEM; + + /* Allocate the release record now: freeing must never need to allocate. */ + release = kzalloc_obj(*release); + if (!release) + return -ENOMEM; + INIT_LIST_HEAD(&release->unmap.node); + release->unmap.unmap = avd_dma_unmap; + release->size = size; + release->cpu = dma_alloc_coherent(avd->dev, size, &release->addr, GFP_KERNEL); + if (!release->cpu) { + kfree(release); + return -ENOMEM; + } + + /* Payload length, not an allocation rounding, is used by codec packers. */ + buf->cpu = release->cpu; + buf->addr = release->addr; + buf->size = size; + buf->release = release; + return 0; +} + +void avd_buf_free(struct avd_dev *avd, struct avd_buf *buf) +{ + if (buf->release) + avd_pm_unmap(avd, &buf->release->unmap); + /* The record outlives the context when unmapping has to be deferred. */ + memset(buf, 0, sizeof(*buf)); +} + +bool avd_pm_can_suspend(const struct avd_frame *frames, unsigned int draining) +{ + unsigned int i; + + if (draining) + return false; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (frames[i].state != AVD_FRAME_FREE) + return false; + + return true; +} + +static bool avd_pm_queue_idle(struct avd_dev *avd) +{ + unsigned long flags; + bool idle; + + spin_lock_irqsave(&avd->queue_lock, flags); + idle = avd_pm_can_suspend(avd->frames, avd->pm.draining); + spin_unlock_irqrestore(&avd->queue_lock, flags); + return idle; +} + +void avd_pm_queue_changed(struct avd_dev *avd) +{ + wake_up_all(&avd->pm.idle_wait); +} + +void avd_pm_drain_begin(struct avd_dev *avd) +{ + unsigned long flags; + + spin_lock_irqsave(&avd->queue_lock, flags); + avd->pm.draining++; + spin_unlock_irqrestore(&avd->queue_lock, flags); +} + +void avd_pm_drain_end(struct avd_dev *avd) +{ + unsigned long flags; + + spin_lock_irqsave(&avd->queue_lock, flags); + avd->pm.draining--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); +} + +static void avd_pm_resubmit(struct avd_dev *avd) +{ + /* queue_mutex may be held by the caller of pm_runtime_resume_and_get(). */ + mod_delayed_work(system_wq, &avd->queue_work, 0); +} + +static int avd_pm_queue_drain(struct avd_dev *avd) +{ + avd_pm_resubmit(avd); + if (!wait_event_timeout(avd->pm.idle_wait, avd_pm_queue_idle(avd), + msecs_to_jiffies(AVD_DRAIN_TIMEOUT_MS))) + return -EBUSY; + return 0; +} + +const struct avd_queue_pm_ops avd_pm_queue_ops = { + .idle = avd_pm_queue_idle, + .drain = avd_pm_queue_drain, + .resubmit = avd_pm_resubmit, +}; + +void avd_pm_init(struct avd_dev *avd, const struct avd_backend_ops *backend, + const struct avd_queue_pm_ops *queue) +{ + mutex_init(&avd->pm.lock); + INIT_LIST_HEAD(&avd->pm.deferred_unmaps); + init_waitqueue_head(&avd->pm.idle_wait); + avd->pm.dart_active = false; + avd->backend = backend; + avd->pm.queue = queue; +} + +void avd_pm_enable(struct avd_dev *avd) +{ + pm_runtime_set_autosuspend_delay(avd->dev, AVD_AUTOSUSPEND_MS); + pm_runtime_use_autosuspend(avd->dev); + pm_runtime_enable(avd->dev); +} + +void avd_pm_disable(struct avd_dev *avd) +{ + int ret; + + /* Power suppliers before flushing frees queued by cleanup while idle. */ + ret = pm_runtime_resume_and_get(avd->dev); + if (ret >= 0) + ret = pm_runtime_put_sync_suspend(avd->dev); + if (ret < 0) + dev_warn(avd->dev, "final power transition failed: %d\n", ret); + pm_runtime_disable(avd->dev); + pm_runtime_dont_use_autosuspend(avd->dev); + /* Resume may have kicked the already-stopped queue. */ + cancel_delayed_work_sync(&avd->queue_work); + WARN_ON(!list_empty(&avd->pm.deferred_unmaps)); +} + +/* pm.lock held; release records own their storage and may free themselves. */ +static void avd_pm_drain_unmaps(struct avd_dev *avd) +{ + struct avd_deferred_unmap *unmap, *next; + + list_for_each_entry_safe(unmap, next, &avd->pm.deferred_unmaps, node) { + list_del_init(&unmap->node); + unmap->unmap(avd, unmap); + } +} + +void avd_pm_unmap(struct avd_dev *avd, struct avd_deferred_unmap *unmap) +{ + mutex_lock(&avd->pm.lock); + if (avd->pm.dart_active) + unmap->unmap(avd, unmap); + else + list_add_tail(&unmap->node, &avd->pm.deferred_unmaps); + mutex_unlock(&avd->pm.lock); +} + +int avd_pm_backend_suspend(struct avd_dev *avd) +{ + avd->backend->shutdown(avd); + return 0; +} + +int avd_pm_backend_resume(struct avd_dev *avd) +{ + int ret; + + /* Keep initial image-based selection, but never change ABI after use. */ + if (!avd->fw_generation) + return avd_backend_boot(avd); + /* Apple boot re-verifies ITCM and relinks even when the image survived. */ + ret = avd->backend->boot(avd); + + if (!ret) + avd->fw_generation++; + return ret; +} + +int avd_pm_power_on(struct avd_dev *avd) +{ + const struct avd_backend_ops *backend = avd->backend; + int ret; + + /* genpd and the IOMMU suppliers resume before this runtime callback. */ + mutex_lock(&avd->pm.lock); + avd->pm.dart_active = true; + avd_pm_drain_unmaps(avd); + mutex_unlock(&avd->pm.lock); + + /* Do not hold pm.lock: boot may allocate or release DMA buffers. */ + ret = backend->resume ? backend->resume(avd) : avd_pm_backend_resume(avd); + if (ret) { + avd->backend->shutdown(avd); + mutex_lock(&avd->pm.lock); + avd->pm.dart_active = false; + mutex_unlock(&avd->pm.lock); + return ret; + } + + if (avd->pm.queue->resubmit) + avd->pm.queue->resubmit(avd); + + return 0; +} + +int avd_pm_power_off(struct avd_dev *avd) +{ + const struct avd_backend_ops *backend = avd->backend; + int ret; + + /* PM references cover reservation through retirement, including drain. */ + if (!avd->pm.queue->idle(avd)) + return -EBUSY; + + if (backend->suspend) { + ret = backend->suspend(avd); + if (ret) + return ret; + } else { + backend->shutdown(avd); + } + + /* No more firmware DMA; suppliers stay on until this callback returns. */ + mutex_lock(&avd->pm.lock); + avd_pm_drain_unmaps(avd); + avd->pm.dart_active = false; + mutex_unlock(&avd->pm.lock); + return 0; +} + +static int avd_runtime_resume(struct device *dev) +{ + return avd_pm_power_on(dev_get_drvdata(dev)); +} + +static int avd_runtime_suspend(struct device *dev) +{ + return avd_pm_power_off(dev_get_drvdata(dev)); +} + +static int avd_system_suspend(struct device *dev) +{ + struct avd_dev *avd = dev_get_drvdata(dev); + int ret; + + v4l2_m2m_suspend(avd->m2m_dev); + ret = avd->pm.queue->drain(avd); + if (!ret) + ret = pm_runtime_force_suspend(dev); + if (ret) + v4l2_m2m_resume(avd->m2m_dev); + return ret; +} + +static int avd_system_resume(struct device *dev) +{ + struct avd_dev *avd = dev_get_drvdata(dev); + int ret; + + ret = pm_runtime_force_resume(dev); + if (ret) + return ret; + v4l2_m2m_resume(avd->m2m_dev); + return 0; +} + +const struct dev_pm_ops avd_pm_ops = { + SYSTEM_SLEEP_PM_OPS(avd_system_suspend, avd_system_resume) + RUNTIME_PM_OPS(avd_runtime_suspend, avd_runtime_resume, NULL) +}; diff --git a/drivers/media/platform/apple/avd/avd-pm.h b/drivers/media/platform/apple/avd/avd-pm.h new file mode 100644 index 0000000000000..4e95408d3b305 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_PM_H_ +#define AVD_PM_H_ + +#include +#include +#include + +#include "avd-queue.h" + +struct avd_dma_release; + +struct avd_queue_pm_ops { + /* Snapshot under the queue lock: all FREE, no context draining. */ + bool (*idle)(struct avd_dev *avd); + /* Scheduling is paused; finish/retire all accepted frames. May sleep. */ + int (*drain)(struct avd_dev *avd); + /* Kick prepared RESERVED entries only, respecting dependencies. */ + void (*resubmit)(struct avd_dev *avd); +}; + +struct avd_deferred_unmap { + struct list_head node; + void (*unmap)(struct avd_dev *avd, struct avd_deferred_unmap *unmap); +}; + +struct avd_pm { + /* Serializes DMA releases against DART activation and firmware boot. */ + struct mutex lock; + struct list_head deferred_unmaps; + bool dart_active; + wait_queue_head_t idle_wait; + /* Protected by queue_lock; counts drain operations, not stopped streams. */ + unsigned int draining; + const struct avd_queue_pm_ops *queue; +}; + +/* The caller holds the queue lock, including the draining-context count. */ +bool avd_pm_can_suspend(const struct avd_frame *frames, unsigned int draining); + +void avd_pm_init(struct avd_dev *avd, const struct avd_backend_ops *backend, + const struct avd_queue_pm_ops *queue); +void avd_pm_enable(struct avd_dev *avd); +/* After queue cleanup: release deferred DMA mappings before the final off. */ +void avd_pm_disable(struct avd_dev *avd); +/* Called in process context, after the last frame has retired its mapping. */ +void avd_pm_unmap(struct avd_dev *avd, struct avd_deferred_unmap *unmap); +int avd_pm_power_on(struct avd_dev *avd); +int avd_pm_power_off(struct avd_dev *avd); +void avd_pm_drain_begin(struct avd_dev *avd); +void avd_pm_drain_end(struct avd_dev *avd); +void avd_pm_queue_changed(struct avd_dev *avd); +extern const struct avd_queue_pm_ops avd_pm_queue_ops; + +/* Backends may share these when suspend loses all firmware context. */ +int avd_pm_backend_suspend(struct avd_dev *avd); +int avd_pm_backend_resume(struct avd_dev *avd); +extern const struct dev_pm_ops avd_pm_ops; + +#endif /* AVD_PM_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-pm.kunitconfig b/drivers/media/platform/apple/avd/avd-pm.kunitconfig new file mode 100644 index 0000000000000..4d9b37f71ff41 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.kunitconfig @@ -0,0 +1,14 @@ +CONFIG_KUNIT=y +CONFIG_COMPILE_TEST=y +CONFIG_OF=y +CONFIG_PM=y +CONFIG_SUSPEND=y +CONFIG_PM_SLEEP=y +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_VIDEO_DEV=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_VIDEO_APPLE_AVD=y +CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST=y +CONFIG_WERROR=y diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index 4991b5a38bac7..6cb780bd66e32 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -58,8 +58,6 @@ static void avd_retire(struct avd_dev *avd) v4l2_m2m_buf_done(f->dst, state); v4l2_m2m_buf_done(f->src, state); } - pm_runtime_mark_last_busy(avd->dev); - pm_runtime_put_autosuspend(avd->dev); spin_lock_irqsave(&avd->queue_lock, flags); f->ctx = NULL; f->src = NULL; @@ -67,6 +65,9 @@ static void avd_retire(struct avd_dev *avd) f->state = AVD_FRAME_FREE; avd->inflight--; spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); progress = true; } } while (progress); @@ -323,12 +324,14 @@ static void avd_queue_submit_work(struct work_struct *work) ctx->coded_fmt_desc->ops->done(ctx, f->src, f->dst, VB2_BUF_STATE_DONE); if (ops && f->src->vb2_buf.req_obj.req) v4l2_ctrl_request_complete(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); - pm_runtime_put_autosuspend(avd->dev); spin_lock_irqsave(&avd->queue_lock, flags); f->state = AVD_FRAME_FREE; f->ctx = NULL; avd->inflight--; spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, VB2_BUF_STATE_DONE); goto unlock; @@ -371,6 +374,7 @@ static void avd_queue_submit_work(struct work_struct *work) f->ctx = NULL; avd->inflight--; spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, VB2_BUF_STATE_ERROR); unlock: @@ -390,14 +394,18 @@ void avd_queue_drain(struct avd_ctx *ctx) struct avd_dev *avd = ctx->dev; unsigned long flags; unsigned int i; + int ret; + /* Session stop/reset accesses registers even when there are no frames. */ + ret = pm_runtime_resume_and_get(avd->dev); + avd_pm_drain_begin(avd); mutex_lock(&avd->queue_mutex); spin_lock_irqsave(&avd->queue_lock, flags); WRITE_ONCE(ctx->draining, true); if (avd->queued_ctx == ctx) avd->queued_ctx = NULL; spin_unlock_irqrestore(&avd->queue_lock, flags); - for (i = 0; i < AVD_MAX_FRAMES; i++) { + for (i = 0; ret >= 0 && i < AVD_MAX_FRAMES; i++) { if (avd->frames[i].state != AVD_FRAME_FREE && avd->frames[i].ctx == ctx) { if (avd->backend->abort) avd->backend->abort(ctx); @@ -406,19 +414,18 @@ void avd_queue_drain(struct avd_ctx *ctx) } } avd_retire(avd); - if (ctx->session_started && avd->backend->session_stop) { - int ret = pm_runtime_resume_and_get(avd->dev); - - if (ret >= 0) { - avd->backend->session_stop(ctx); - pm_runtime_mark_last_busy(avd->dev); - pm_runtime_put_autosuspend(avd->dev); - } - } + if (ret >= 0 && ctx->session_started && avd->backend->session_stop && + ctx->session_generation == avd->fw_generation) + avd->backend->session_stop(ctx); ctx->session_started = false; if (!avd->stopping) mod_delayed_work(system_wq, &avd->queue_work, 0); mutex_unlock(&avd->queue_mutex); + avd_pm_drain_end(avd); + if (ret >= 0) { + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + } } void avd_queue_abort(void *priv) diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 69787064c4eec..e363e338f7d8b 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -18,6 +18,8 @@ #include #include #include + +#include "avd-pm.h" #include #include "avd-queue.h" @@ -206,6 +208,7 @@ struct avd_variant { struct avd_dev { struct device *dev; + struct avd_pm pm; struct v4l2_device v4l2_dev; struct media_device mdev; struct video_device vdev; @@ -266,6 +269,7 @@ struct avd_buf { void *cpu; dma_addr_t addr; size_t size; + struct avd_dma_release *release; }; struct avd_ctx { From 911061fd4a281eca4ba5a6e0ee1e186fcbd19bc9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:12:27 +1100 Subject: [PATCH 11/73] media: apple: avd: keep the host IRQ closed while powered off Request the mailbox IRQ without automatic enable, open it only after successful firmware boot, and synchronize and close it after backend suspend. Close the host IRQ on teardown even if the final PM transition fails. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-drv.c | 3 ++- drivers/media/platform/apple/avd/avd-pm.c | 18 ++++++++++++++++++ drivers/media/platform/apple/avd/avd-pm.h | 2 ++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 0e9a4da6d6eea..3778f26c338c7 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -627,8 +627,9 @@ static int avd_probe(struct platform_device *pdev) irq = platform_get_irq(pdev, 1); if (irq < 0) return irq; + avd->pm.irq = irq; ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, - avd_irq_handler, IRQF_ONESHOT, + avd_irq_handler, IRQF_ONESHOT | IRQF_NO_AUTOEN, dev_name(&pdev->dev), avd); if (ret) { dev_err(avd->dev, "Could not request IRQ 1"); diff --git a/drivers/media/platform/apple/avd/avd-pm.c b/drivers/media/platform/apple/avd/avd-pm.c index 32b2f6929fbd6..7570f87d214ef 100644 --- a/drivers/media/platform/apple/avd/avd-pm.c +++ b/drivers/media/platform/apple/avd/avd-pm.c @@ -9,6 +9,15 @@ #define AVD_AUTOSUSPEND_MS 100 #define AVD_DRAIN_TIMEOUT_MS 5000 +static void avd_pm_disable_irq(struct avd_dev *avd) +{ + if (avd->pm.irq >= 0 && avd->pm.irq_enabled) { + /* Wait out threaded handlers before the domain can disappear. */ + disable_irq(avd->pm.irq); + avd->pm.irq_enabled = false; + } +} + struct avd_dma_release { struct avd_deferred_unmap unmap; void *cpu; @@ -140,6 +149,8 @@ void avd_pm_init(struct avd_dev *avd, const struct avd_backend_ops *backend, INIT_LIST_HEAD(&avd->pm.deferred_unmaps); init_waitqueue_head(&avd->pm.idle_wait); avd->pm.dart_active = false; + avd->pm.irq = -1; + avd->pm.irq_enabled = false; avd->backend = backend; avd->pm.queue = queue; } @@ -162,6 +173,8 @@ void avd_pm_disable(struct avd_dev *avd) if (ret < 0) dev_warn(avd->dev, "final power transition failed: %d\n", ret); pm_runtime_disable(avd->dev); + /* Also close the host IRQ if the final hardware transition failed. */ + avd_pm_disable_irq(avd); pm_runtime_dont_use_autosuspend(avd->dev); /* Resume may have kicked the already-stopped queue. */ cancel_delayed_work_sync(&avd->queue_work); @@ -230,6 +243,10 @@ int avd_pm_power_on(struct avd_dev *avd) mutex_unlock(&avd->pm.lock); return ret; } + if (avd->pm.irq >= 0 && !avd->pm.irq_enabled) { + enable_irq(avd->pm.irq); + avd->pm.irq_enabled = true; + } if (avd->pm.queue->resubmit) avd->pm.queue->resubmit(avd); @@ -253,6 +270,7 @@ int avd_pm_power_off(struct avd_dev *avd) } else { backend->shutdown(avd); } + avd_pm_disable_irq(avd); /* No more firmware DMA; suppliers stay on until this callback returns. */ mutex_lock(&avd->pm.lock); diff --git a/drivers/media/platform/apple/avd/avd-pm.h b/drivers/media/platform/apple/avd/avd-pm.h index 4e95408d3b305..ceb5dd2e18709 100644 --- a/drivers/media/platform/apple/avd/avd-pm.h +++ b/drivers/media/platform/apple/avd/avd-pm.h @@ -29,6 +29,8 @@ struct avd_pm { struct mutex lock; struct list_head deferred_unmaps; bool dart_active; + int irq; + bool irq_enabled; wait_queue_head_t idle_wait; /* Protected by queue_lock; counts drain operations, not stopped streams. */ unsigned int draining; From 38ef5b91ee0677fd27db6c6d75846e8e56c3718b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:29:43 +1100 Subject: [PATCH 12/73] media: apple: avd: replay PIODMA setup and restore DART on recovery Map only DT-declared PIODMA and SRAM resources and replay the shifted SRAM base before every open CM3 boot. Keep reset, empty-domain attach/detach and firmware reboot ordered; refuse to reboot if the DMA domain cannot be restored. Keep host IRQs closed until old slots are invalidated. Add synthetic-RAM boot replay and recovery-gating KUnit coverage. Queued PIODMA use still requires the DT to describe the DMA stream as well as decode. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-drv.c | 3 + drivers/media/platform/apple/avd/avd-hw.c | 1 + .../media/platform/apple/avd/avd-pm-test.c | 94 +++++++++++++++++++ drivers/media/platform/apple/avd/avd-pm.c | 76 ++++++++++++++- drivers/media/platform/apple/avd/avd-pm.h | 8 ++ drivers/media/platform/apple/avd/avd-queue.c | 7 +- 6 files changed, 181 insertions(+), 8 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 3778f26c338c7..845b0c144d38c 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -598,6 +598,9 @@ static int avd_probe(struct platform_device *pdev) avd->ctrl = devm_platform_ioremap_resource_byname(pdev, "ctrl"); if (IS_ERR(avd->ctrl)) return PTR_ERR(avd->ctrl); + ret = avd_pm_map_piodma(avd); + if (ret) + return ret; avd->domain = iommu_get_domain_for_dev(avd->dev); if (avd->domain) { diff --git a/drivers/media/platform/apple/avd/avd-hw.c b/drivers/media/platform/apple/avd/avd-hw.c index 84db61a77f6f4..a71b147f3ad8d 100644 --- a/drivers/media/platform/apple/avd/avd-hw.c +++ b/drivers/media/platform/apple/avd/avd-hw.c @@ -52,6 +52,7 @@ int avd_boot(struct avd_dev *avd) dev_info_once(avd->dev, "booting hw version: %04x", readl_relaxed(avd->ctrl)); + avd_pm_prepare_piodma(avd); memcpy_toio(avd->code, avd->fw->data, avd->fw->size); writel_relaxed(AVD_MBOX_ENABLE, avd->mbox + AVD_REG_MBOX1_STATUS); diff --git a/drivers/media/platform/apple/avd/avd-pm-test.c b/drivers/media/platform/apple/avd/avd-pm-test.c index 6aa341eff55a4..c48efbcd47407 100644 --- a/drivers/media/platform/apple/avd/avd-pm-test.c +++ b/drivers/media/platform/apple/avd/avd-pm-test.c @@ -1,8 +1,10 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT #include #include +#include #include "avd.h" +#include "avd-regs.h" struct avd_pm_test { struct avd_dev avd; @@ -262,6 +264,95 @@ static void avd_pm_real_queue_gate_test(struct kunit *test) KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), 0); } +static void avd_pm_open_boot_replay_test(struct kunit *test) +{ + static const struct avd_variant variant = { .revision = 3 }; + static const u8 image[] = { 0x12, 0x34, 0x56, 0x78 }; + static const struct firmware fw = { .data = image, .size = sizeof(image) }; + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct resource code = { + .name = "code", .flags = IORESOURCE_MEM, + .start = 0x1000, .end = 0x10ff, + }; + struct platform_device *pdev; + u8 observed[sizeof(image)]; + unsigned int i; + + pdev = kunit_kzalloc(test, sizeof(*pdev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pdev); + pdev->resource = &code; + pdev->num_resources = 1; + avd->pdev = pdev; + avd->variant = &variant; + avd->fw = &fw; + /* Ordinary test RAM, never a mapped target aperture. */ + avd->code = (__force void __iomem *)kunit_kzalloc(test, 0x100, GFP_KERNEL); + avd->mbox = (__force void __iomem *)kunit_kzalloc(test, 0x100, GFP_KERNEL); + avd->pm.piodma = (__force void __iomem *)kunit_kzalloc(test, 0x28, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, avd->code); + KUNIT_ASSERT_NOT_NULL(test, avd->mbox); + KUNIT_ASSERT_NOT_NULL(test, avd->pm.piodma); + avd->pm.sram_phys = 0x123456780ULL; + for (i = 0; i < 2; i++) { + /* Model loss of both ITCM and PIODMA configuration between boots. */ + memset_io(avd->code, 0, sizeof(image)); + writel(0, avd->pm.piodma + 0x24); + /* The fixture supplies firmware readiness; no firmware is executed. */ + writel(1, avd->mbox + AVD_REG_FLAG0_SET); + KUNIT_ASSERT_EQ(test, avd_boot(avd), 0); + KUNIT_EXPECT_EQ(test, readl(avd->pm.piodma + 0x24), 0x12345678U); + memcpy_fromio(observed, avd->code, sizeof(observed)); + KUNIT_EXPECT_MEMEQ(test, observed, image, sizeof(image)); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_RUN_CTRL), + (u32)AVD_RUN_CTRL_UNK_RUN); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX1_STATUS), + (u32)AVD_MBOX_ENABLE); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX_IRQ_ENABLE), + (u32)AVD_MBOX1_NOT_EMPTY); + avd_shutdown(avd); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_RUN_CTRL), + (u32)(AVD_RUN_CTRL_UNK_STOP)); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_FLAG0_CLR), 1U); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX_IRQ_ENABLE), 0U); + } +} + +static void avd_pm_recovery_dart_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct iommu_domain domain = {}; + + /* A stopped device cannot be reset through powered-off registers. */ + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); + avd->pm.dart_active = true; + avd->domain = &domain; + /* No empty domain means no valid DART flush: never reboot anyway. */ + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -ENODEV); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); + KUNIT_EXPECT_EQ(test, fixture->boot_step, 0U); +} + +static void avd_pm_recovery_boot_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + avd->pm.dart_active = true; + fixture->boot_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 2U); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); + fixture->boot_error = 0; + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), 0); + KUNIT_EXPECT_EQ(test, avd->fw_generation, 1ULL); + /* Recovery must fail old slots before core reopens the host IRQ. */ + KUNIT_EXPECT_FALSE(test, avd->pm.irq_enabled); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); +} + static void avd_pm_test_exit(struct kunit *test) { struct avd_pm_test *fixture = test->priv; @@ -278,6 +369,9 @@ static struct kunit_case avd_pm_test_cases[] = { KUNIT_CASE(avd_pm_deferred_unmap_test), KUNIT_CASE(avd_pm_real_queue_resume_test), KUNIT_CASE(avd_pm_real_queue_gate_test), + KUNIT_CASE(avd_pm_open_boot_replay_test), + KUNIT_CASE(avd_pm_recovery_dart_gate_test), + KUNIT_CASE(avd_pm_recovery_boot_test), {} }; diff --git a/drivers/media/platform/apple/avd/avd-pm.c b/drivers/media/platform/apple/avd/avd-pm.c index 7570f87d214ef..0a2b04c790306 100644 --- a/drivers/media/platform/apple/avd/avd-pm.c +++ b/drivers/media/platform/apple/avd/avd-pm.c @@ -1,6 +1,10 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT #include +#include +#include +#include #include +#include #include #include "avd.h" @@ -8,6 +12,15 @@ /* [C] Keep Linux's existing default until the idle-timer capture is available. */ #define AVD_AUTOSUSPEND_MS 100 #define AVD_DRAIN_TIMEOUT_MS 5000 +#define AVD_PIODMA_SRAM_BASE 0x24 + +void avd_pm_enable_irq(struct avd_dev *avd) +{ + if (avd->pm.irq >= 0 && !avd->pm.irq_enabled) { + enable_irq(avd->pm.irq); + avd->pm.irq_enabled = true; + } +} static void avd_pm_disable_irq(struct avd_dev *avd) { @@ -162,6 +175,36 @@ void avd_pm_enable(struct avd_dev *avd) pm_runtime_enable(avd->dev); } +int avd_pm_map_piodma(struct avd_dev *avd) +{ + struct resource *piodma, *sram; + + piodma = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "piodma"); + /* Old DTs support only the legacy transport; never invent a resource. */ + if (!piodma) + return 0; + sram = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "sram"); + if (!sram || !IS_ALIGNED(sram->start, 16) || + (sram->start >> 4) > U32_MAX || + resource_size(piodma) < AVD_PIODMA_SRAM_BASE + sizeof(u32)) + return -EINVAL; + avd->pm.piodma = devm_ioremap_resource(avd->dev, piodma); + if (IS_ERR(avd->pm.piodma)) + return PTR_ERR(avd->pm.piodma); + avd->pm.sram_phys = sram->start; + if (of_count_phandle_with_args(avd->dev->of_node, "iommus", "#iommu-cells") < 2) + dev_warn(avd->dev, "PIODMA transport requires decode and DMA IOMMU streams\n"); + return 0; +} + +void avd_pm_prepare_piodma(struct avd_dev *avd) +{ + /* Public CM3 boot ABI: replay the SRAM base after every power loss/reset. */ + if (avd->pm.piodma) + writel((u32)(avd->pm.sram_phys >> 4), + avd->pm.piodma + AVD_PIODMA_SRAM_BASE); +} + void avd_pm_disable(struct avd_dev *avd) { int ret; @@ -223,6 +266,34 @@ int avd_pm_backend_resume(struct avd_dev *avd) return ret; } +int avd_pm_recover(struct avd_dev *avd) +{ + int ret; + + if (!avd->pm.dart_active) + return -EHOSTDOWN; + avd_pm_disable_irq(avd); + avd->backend->shutdown(avd); + ret = reset_control_reset(avd->rstc); + if (ret) + return ret; + if (avd->domain) { + if (!avd->empty_domain) + return -ENODEV; + ret = iommu_attach_device(avd->empty_domain, avd->dev); + if (ret) + return ret; + iommu_detach_device(avd->empty_domain, avd->dev); + /* Do not reboot against an empty/blocked domain after a failed restore. */ + if (iommu_get_domain_for_dev(avd->dev) != avd->domain) + return -EIO; + } + ret = avd_pm_backend_resume(avd); + if (ret) + avd->backend->shutdown(avd); + return ret; +} + int avd_pm_power_on(struct avd_dev *avd) { const struct avd_backend_ops *backend = avd->backend; @@ -243,10 +314,7 @@ int avd_pm_power_on(struct avd_dev *avd) mutex_unlock(&avd->pm.lock); return ret; } - if (avd->pm.irq >= 0 && !avd->pm.irq_enabled) { - enable_irq(avd->pm.irq); - avd->pm.irq_enabled = true; - } + avd_pm_enable_irq(avd); if (avd->pm.queue->resubmit) avd->pm.queue->resubmit(avd); diff --git a/drivers/media/platform/apple/avd/avd-pm.h b/drivers/media/platform/apple/avd/avd-pm.h index ceb5dd2e18709..62de4c8c3b70a 100644 --- a/drivers/media/platform/apple/avd/avd-pm.h +++ b/drivers/media/platform/apple/avd/avd-pm.h @@ -31,6 +31,8 @@ struct avd_pm { bool dart_active; int irq; bool irq_enabled; + void __iomem *piodma; + phys_addr_t sram_phys; wait_queue_head_t idle_wait; /* Protected by queue_lock; counts drain operations, not stopped streams. */ unsigned int draining; @@ -43,12 +45,18 @@ bool avd_pm_can_suspend(const struct avd_frame *frames, unsigned int draining); void avd_pm_init(struct avd_dev *avd, const struct avd_backend_ops *backend, const struct avd_queue_pm_ops *queue); void avd_pm_enable(struct avd_dev *avd); +int avd_pm_map_piodma(struct avd_dev *avd); +void avd_pm_prepare_piodma(struct avd_dev *avd); /* After queue cleanup: release deferred DMA mappings before the final off. */ void avd_pm_disable(struct avd_dev *avd); /* Called in process context, after the last frame has retired its mapping. */ void avd_pm_unmap(struct avd_dev *avd, struct avd_deferred_unmap *unmap); int avd_pm_power_on(struct avd_dev *avd); int avd_pm_power_off(struct avd_dev *avd); +/* Caller holds a PM reference and queue_mutex; leaves the host IRQ closed. */ +int avd_pm_recover(struct avd_dev *avd); +/* After recovery has invalidated every previously firmware-owned slot. */ +void avd_pm_enable_irq(struct avd_dev *avd); void avd_pm_drain_begin(struct avd_dev *avd); void avd_pm_drain_end(struct avd_dev *avd); void avd_pm_queue_changed(struct avd_dev *avd); diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index 6cb780bd66e32..1c50c0a10964b 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -81,10 +81,7 @@ static void avd_reset_queue(struct avd_dev *avd, int status) unsigned int i; int ret; - avd->backend->shutdown(avd); - ret = reset_control_reset(avd->rstc); - if (!ret) - ret = avd_backend_boot(avd); + ret = avd_pm_recover(avd); WRITE_ONCE(avd->faulted, !!ret); if (ret) dev_err(avd->dev, "queue reset failed: %d\n", ret); @@ -99,6 +96,8 @@ static void avd_reset_queue(struct avd_dev *avd, int status) avd->frames[i].state = AVD_FRAME_DONE; } spin_unlock_irqrestore(&avd->queue_lock, flags); + if (!ret) + avd_pm_enable_irq(avd); } static int avd_send_frame(struct avd_dev *avd, struct avd_frame *f) From 0eaf5955fb6e2ea01d12e18435c07eec1e9cec99 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:31:43 +1100 Subject: [PATCH 13/73] media: apple: avd: centralize boot replay for open transports Replay PIODMA setup in the common boot dispatcher so queued open transports inherit it even with their own mailbox boot implementation. Keep firmware generation handling and initial-only backend fallback in the same dispatcher, and test the real open boot path through it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-backend-open.c | 9 +++++++-- drivers/media/platform/apple/avd/avd-hw.c | 1 - drivers/media/platform/apple/avd/avd-pm-test.c | 3 ++- drivers/media/platform/apple/avd/avd-pm.c | 13 ++----------- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 476e07517d96b..0161bacbf9893 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -200,14 +200,19 @@ int avd_backend_select(struct avd_dev *avd) int avd_backend_boot(struct avd_dev *avd) { - int ret = avd->backend->boot(avd); + int ret; - if (ret == -ENOENT && avd_backend_is_apple(avd)) { + if (!avd_backend_is_apple(avd)) + avd_pm_prepare_piodma(avd); + ret = avd->backend->boot(avd); + /* Never switch command ABIs after frames have used this backend. */ + if (ret == -ENOENT && !avd->fw_generation && avd_backend_is_apple(avd)) { ret = request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); if (ret) return ret; avd->backend = &avd_open_backend; avd_open_capabilities(avd); + avd_pm_prepare_piodma(avd); ret = avd->backend->boot(avd); } if (!ret) diff --git a/drivers/media/platform/apple/avd/avd-hw.c b/drivers/media/platform/apple/avd/avd-hw.c index a71b147f3ad8d..84db61a77f6f4 100644 --- a/drivers/media/platform/apple/avd/avd-hw.c +++ b/drivers/media/platform/apple/avd/avd-hw.c @@ -52,7 +52,6 @@ int avd_boot(struct avd_dev *avd) dev_info_once(avd->dev, "booting hw version: %04x", readl_relaxed(avd->ctrl)); - avd_pm_prepare_piodma(avd); memcpy_toio(avd->code, avd->fw->data, avd->fw->size); writel_relaxed(AVD_MBOX_ENABLE, avd->mbox + AVD_REG_MBOX1_STATUS); diff --git a/drivers/media/platform/apple/avd/avd-pm-test.c b/drivers/media/platform/apple/avd/avd-pm-test.c index c48efbcd47407..6756d036b3f76 100644 --- a/drivers/media/platform/apple/avd/avd-pm-test.c +++ b/drivers/media/platform/apple/avd/avd-pm-test.c @@ -286,6 +286,7 @@ static void avd_pm_open_boot_replay_test(struct kunit *test) avd->pdev = pdev; avd->variant = &variant; avd->fw = &fw; + avd->backend = &avd_open_backend; /* Ordinary test RAM, never a mapped target aperture. */ avd->code = (__force void __iomem *)kunit_kzalloc(test, 0x100, GFP_KERNEL); avd->mbox = (__force void __iomem *)kunit_kzalloc(test, 0x100, GFP_KERNEL); @@ -300,7 +301,7 @@ static void avd_pm_open_boot_replay_test(struct kunit *test) writel(0, avd->pm.piodma + 0x24); /* The fixture supplies firmware readiness; no firmware is executed. */ writel(1, avd->mbox + AVD_REG_FLAG0_SET); - KUNIT_ASSERT_EQ(test, avd_boot(avd), 0); + KUNIT_ASSERT_EQ(test, avd_backend_boot(avd), 0); KUNIT_EXPECT_EQ(test, readl(avd->pm.piodma + 0x24), 0x12345678U); memcpy_fromio(observed, avd->code, sizeof(observed)); KUNIT_EXPECT_MEMEQ(test, observed, image, sizeof(image)); diff --git a/drivers/media/platform/apple/avd/avd-pm.c b/drivers/media/platform/apple/avd/avd-pm.c index 0a2b04c790306..0df13b4396047 100644 --- a/drivers/media/platform/apple/avd/avd-pm.c +++ b/drivers/media/platform/apple/avd/avd-pm.c @@ -253,17 +253,8 @@ int avd_pm_backend_suspend(struct avd_dev *avd) int avd_pm_backend_resume(struct avd_dev *avd) { - int ret; - - /* Keep initial image-based selection, but never change ABI after use. */ - if (!avd->fw_generation) - return avd_backend_boot(avd); - /* Apple boot re-verifies ITCM and relinks even when the image survived. */ - ret = avd->backend->boot(avd); - - if (!ret) - avd->fw_generation++; - return ret; + /* Includes transport setup, ITCM verification/load and a new generation. */ + return avd_backend_boot(avd); } int avd_pm_recover(struct avd_dev *avd) From dcaa34c572b3d6df5e2578cf5bed4fa547feda2c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:54:42 +1100 Subject: [PATCH 14/73] media: apple: avd: cap Apple frames in flight at the FIFO count Each frame in VP to PP flight owns one hardware instruction FIFO, named by EXEC bits 7:4. The hardware has 15 of them on v3 and later tier 1 parts and 7 on T8103 and T8140, so the 16-slot queue must not keep more frames in flight than the variant has FIFOs. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-backend-apple.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-apple.c b/drivers/media/platform/apple/avd/avd-backend-apple.c index 9a6fc01dfa466..8345f107e4c14 100644 --- a/drivers/media/platform/apple/avd/avd-backend-apple.c +++ b/drivers/media/platform/apple/avd/avd-backend-apple.c @@ -168,7 +168,12 @@ static void avd_apple_shutdown(struct avd_dev *avd) static unsigned int avd_apple_max_inflight(struct avd_dev *avd) { - return AVD_MAX_FRAMES; + /* + * Every frame in VP->PP flight owns one hardware instruction FIFO + * (EXEC bits 7:4), so the FIFO count bounds the frames in flight: + * 15 on v3+ tier 1 parts, 7 on T8103 and T8140. + */ + return min_t(unsigned int, AVD_MAX_FRAMES, avd->variant->fifo_slots ?: 1); } static int avd_apple_send(struct avd_dev *avd, const u32 cmd[24]) From 829bcab267ce8ff7d86a1d4fa1ed8ea3c1f8eaf5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:54:42 +1100 Subject: [PATCH 15/73] media: apple: avd: advertise T6040 HEVC with h265-capable firmware T6040 decodes HEVC with the v5 header. Add HEVC to its capabilities but keep it hidden unless the loaded open firmware lists h265 in its CODECS trailer, so older firmware without the trailer behaves as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-backend-open.c | 4 +++- drivers/media/platform/apple/avd/avd-drv.c | 9 ++++++--- drivers/media/platform/apple/avd/avd.h | 2 ++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 0161bacbf9893..302e5907bb643 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -164,7 +164,9 @@ static void avd_open_capabilities(struct avd_dev *avd) const char *info; unsigned int i; - avd->capabilities = avd->variant->capabilities; + /* Without a CODECS trailer, keep codecs that need a newer firmware hidden. */ + avd->capabilities = avd->variant->capabilities & + ~avd->variant->fw_caps_required; if (!avd->fw || avd->fw->size < 0x40) return; info = (const char *)avd->fw->data + avd->fw->size - 0x40; diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 845b0c144d38c..e470287a2b496 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -490,8 +490,8 @@ static const struct avd_variant avd_t8132_variant = { * T6040 (M4 Pro): the T8132 decoder. Its first HEVC job stalled the VP when * it was sent the pre-v5 header (no DMA configuration words, no reserved * scratch slot); T6040 runs the same v5 firmware generation as T8140, so it - * gets the v5 header. HEVC stays hidden from V4L2 clients until that is - * verified on the hardware. + * gets the v5 header. HEVC is advertised only when the loaded firmware + * names h265 in its CODECS trailer. */ static const struct avd_variant avd_t6040_variant = { .apple = &avd_borage_hw, @@ -504,9 +504,12 @@ static const struct avd_variant avd_t6040_variant = { }, .fifo_slots = 15, .configure_stream = t8122_configure_stream, - .capabilities = AVD_CAPABILITY_H264 | + .capabilities = AVD_CAPABILITY_HEVC | + AVD_CAPABILITY_H264 | AVD_CAPABILITY_VP9 | AVD_CAPABILITY_AV1, + /* HEVC only with firmware whose CODECS trailer lists h265. */ + .fw_caps_required = AVD_CAPABILITY_HEVC, .fw_name = "apple/avd-fw-v5-t1.bin", .revision = 4, .vp_slot_offset = 0xc, diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index e363e338f7d8b..36aa5944c2caf 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -194,6 +194,8 @@ struct avd_variant { unsigned int vp_slot_base[4]; unsigned int fifo_slots; unsigned int capabilities; + /* Codecs advertised only when the firmware CODECS trailer lists them. */ + unsigned int fw_caps_required; void (*configure_stream)(struct avd_dev *avd, dma_addr_t addr, u8 fifo_idx, u32 vp_slot); const char* fw_name; From 44f86ac43cf869e43627429db7b71765f30220a7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 23:55:41 +1100 Subject: [PATCH 16/73] media: apple: avd: poll the VP queue level instead of a fixed sleep When the VP instruction queue is at least half full the open transport slept a fixed 500-650 us (100-150 us on parts with v5 pacing) before the next segment. Poll the queue level every 10 us instead and continue as soon as it drops below half, keeping the old sleep as the upper bound. The submission order and queue thresholds do not change. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../media/platform/apple/avd/avd-backend-open.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 302e5907bb643..f1a4733551bb0 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* Legacy transport split from avd-drv.c; Copyright The Asahi Linux Contributors. */ #include +#include #include #include #include @@ -105,10 +106,18 @@ static int avd_open_submit(struct avd_dev *avd, struct avd_frame *f) return 0; } if (cur >= max / 2) { - if (avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) - usleep_range(100, 150); - else - usleep_range(500, 650); + /* + * Wait for the VP to drain below half full, but + * only as long as it takes. The old fixed sleep + * (100-150 us on v5 pacing parts, 500-650 us + * elsewhere) stays the upper bound. + */ + u32 wait_us = avd->variant->quirks & + AVD_QUIRK_V5_QUEUE_PACING ? 150 : 650; + + readl_relaxed_poll_timeout(avd->ctrl + + avd->variant->submit_queue_status_offset + vp * 4, + cur, cur < max / 2, 10, wait_us); } } writel(words[i], reg); From 0a83a8730df065faf7f3645aad0cfb682a6a9e6e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 00:34:47 +1100 Subject: [PATCH 17/73] media: apple: avd: queued transport for the open firmware The open transport writes every instruction word to the VP FIFO register from the CPU, sleeps while the FIFO is half full, and kicks the PP from the VP-done interrupt thread, one frame at a time. avd-fw queue ABI 2 moves all of that into the CM3. The host streams each frame program into a word ring in CM3 SRAM, posts a 64-byte descriptor and rings mailbox 0; the firmware feeds the VP FIFO, writes the PP kick (plus one continuation kick per extra segment for HEVC, VP9 and AV1) when the VP is done, and posts a tagged completion on mailbox 1. No PIODMA is used, so the second DART stream that the current device trees lack is not needed. The transport is chosen after every firmware boot from the image info block: ABI 2 with the SRAM ring, and an SRAM aperture matching the node. Anything else keeps the legacy transport. Each frame in flight gets its own instruction FIFO (the slot index), and the queue keeps at most the FIFO count in flight. Frames of one context stay serialized by the queue core, so different streams overlap; within a stream the gain is the removed CPU word writes, pacing sleeps and interrupt round trip. T6040 now loads avd-fw-v5-t2, whose SRAM aperture matches the node, and falls back to v5-t1 when that image is not installed. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/Makefile | 2 +- drivers/media/platform/apple/avd/avd-av1.c | 8 + .../platform/apple/avd/avd-backend-open.c | 27 +- .../platform/apple/avd/avd-backend-queue.c | 337 ++++++++++++++++++ drivers/media/platform/apple/avd/avd-drv.c | 4 +- drivers/media/platform/apple/avd/avd-hevc.c | 8 + drivers/media/platform/apple/avd/avd-vp9.c | 8 + drivers/media/platform/apple/avd/avd.h | 11 + 8 files changed, 400 insertions(+), 5 deletions(-) create mode 100644 drivers/media/platform/apple/avd/avd-backend-queue.c diff --git a/drivers/media/platform/apple/avd/Makefile b/drivers/media/platform/apple/avd/Makefile index 3372a76baa9ee..60560dbab592a 100644 --- a/drivers/media/platform/apple/avd/Makefile +++ b/drivers/media/platform/apple/avd/Makefile @@ -2,7 +2,7 @@ apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-pm.o avd-h264.o avd-vp9.o \ avd-hevc.o avd-av1.o avd-av1-entropymode.o avd-queue.o avd-backend-open.o \ - avd-backend-apple.o + avd-backend-apple.o avd-backend-queue.o apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o avd-queue-test.o \ avd-pm-test.o obj-$(CONFIG_VIDEO_APPLE_AVD) += apple-avd.o diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index 60d69bab8a342..6bd5b1a9a6227 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -1515,6 +1515,13 @@ static enum avd_image_fmt avd_av1_get_image_fmt(struct avd_ctx *ctx, #undef BIT_DEPTH } +static unsigned int avd_av1_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_av1_ctx *priv = ctx->priv; + + return max_t(int, priv->submit_num, 1); +} + static void avd_av1_submit(struct avd_ctx *ctx) { struct avd_av1_ctx *av1_ctx = ctx->priv; @@ -1570,6 +1577,7 @@ const struct avd_coded_fmt_ops avd_av1_fmt_ops = { .stop = avd_av1_stop, .run = avd_av1_run, .submit = avd_av1_submit, + .pp_kicks = avd_av1_pp_kicks, .try_ctrl = avd_av1_try_ctrl, .get_image_fmt = avd_av1_get_image_fmt, }; diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index f1a4733551bb0..84d59dd57f297 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -45,7 +45,8 @@ int avd_open_build_program(struct avd_ctx *ctx) AVD_OP_EXEC_REV3_VP9_MASK : 0; start = AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3); - ctx->fifo_idx = 0; + /* The queued transport gives every frame in flight its own FIFO. */ + ctx->fifo_idx = avd_backend_is_qfw(avd) ? f->idx : 0; words = f->prog->cpu; words[pos++] = AVD_OP_EXEC | mask | start | AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx); for (i = 0; i <= job->num; i++) { @@ -187,6 +188,20 @@ static void avd_open_capabilities(struct avd_dev *avd) avd->capabilities |= avd->variant->capabilities & codecs[i].bit; } +static int avd_open_request_firmware(struct avd_dev *avd) +{ + int ret; + + if (!avd->variant->fw_name_fallback) + return request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); + ret = firmware_request_nowarn(&avd->fw, avd->variant->fw_name, avd->dev); + if (ret != -ENOENT) + return ret; + dev_info(avd->dev, "%s not installed, using %s\n", + avd->variant->fw_name, avd->variant->fw_name_fallback); + return request_firmware(&avd->fw, avd->variant->fw_name_fallback, avd->dev); +} + int avd_backend_select(struct avd_dev *avd) { int ret; @@ -203,9 +218,11 @@ int avd_backend_select(struct avd_dev *avd) return ret; } avd->backend = &avd_open_backend; - ret = request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); + ret = avd_open_request_firmware(avd); if (!ret) avd_open_capabilities(avd); + if (!ret) + ret = avd_qfw_probe(avd); return ret; } @@ -218,7 +235,9 @@ int avd_backend_boot(struct avd_dev *avd) ret = avd->backend->boot(avd); /* Never switch command ABIs after frames have used this backend. */ if (ret == -ENOENT && !avd->fw_generation && avd_backend_is_apple(avd)) { - ret = request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); + ret = avd_open_request_firmware(avd); + if (!ret) + ret = avd_qfw_probe(avd); if (ret) return ret; avd->backend = &avd_open_backend; @@ -226,6 +245,8 @@ int avd_backend_boot(struct avd_dev *avd) avd_pm_prepare_piodma(avd); ret = avd->backend->boot(avd); } + if (!ret) + avd_qfw_select(avd); if (!ret) avd->fw_generation++; return ret; diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c new file mode 100644 index 0000000000000..b6026fd67832e --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Queued transport for the open AVD firmware (avd-fw queue ABI 2). + * + * The codec's CPU builder still produces the per-frame instruction program. + * Instead of writing it word by word into the VP FIFO register and kicking + * the PP from the VP-done interrupt, the host streams the program into a word + * ring in CM3 SRAM and posts one descriptor per frame. The firmware feeds the + * VP FIFO from SRAM, starts PP itself when the VP is done, and posts a tagged + * completion. No PIODMA (and so no second DART stream) is involved. + */ +#include +#include +#include +#include +#include + +#include "avd.h" +#include "avd-inst.h" +#include "avd-regs.h" + +#define AVDQ_MAGIC 0x41565131 /* "AVQ1" */ +#define AVDQ_VERSION 2 +#define AVDQ_COUNT 16 +#define AVDQ_DESC_BYTES 64 +#define AVDQ_RING_OFFSET 0x8000 +#define AVDQ_INFO 0x7fc0 +/* Firmware private state starts here; the SRAM ring must end before it. */ +#define AVDQ_RING_LIMIT 0x6000 +#define AVDQ_INFO_RP (AVDQ_INFO + 0x38) +#define AVDQ_INFO_WP (AVDQ_INFO + 0x3c) +#define AVDQ_FEAT_SRAM_RING BIT(0) +#define AVDQ_FEAT_MULTI_KICK BIT(1) +#define AVDQ_FLAG_SRAM BIT(0) +#define AVDQ_MAX_PP_EXTRA 255 +#define AVDQ_STATE_SUBMITTED 1 +#define AVDQ_STATE_COMPLETE 2 +#define AVDQ_NOTIFY_MASK 0xffff0000 +#define AVDQ_NOTIFY 0x54530000 +/* SRAM addresses as the CM3 doorbell sees them: 0x01000000 + offset from AVD base. */ +#define AVDQ_CM3_APERTURE(off) (0x01000000 + (off)) +#define AVD_REG_MBOX0_STATUS 0x50 +#define AVD_REG_MBOX0_SEND 0x54 +#define AVDQ_STREAM_TIMEOUT_US (1000 * USEC_PER_MSEC) + +/* Descriptor word offsets (bytes), avd-fw docs/queue.md. */ +enum { + D_MAGIC = 0x00, D_TAG = 0x04, D_CODEC = 0x08, D_VP = 0x0c, + D_PROG_LO = 0x10, D_PROG_HI = 0x14, D_WORDS = 0x18, D_KICK = 0x1c, + D_STATE = 0x20, D_STATUS = 0x24, D_CTAG = 0x28, D_FLAGS = 0x2c, + D_PP_EXTRA = 0x30, D_PP_CONT = 0x34, D_RSVD0 = 0x38, D_RSVD1 = 0x3c, +}; + +struct avd_qfw { + void __iomem *sram; + resource_size_t sram_size; + u32 aperture; /* expected CM3 address of SRAM byte 0 */ + u32 ring_off; + u32 ring_words; + u32 wp; /* host write count, free-running */ + u32 fifo_slots; + bool usable; /* last boot found a matching ABI 2 image */ +}; + +static void __iomem *avd_qfw_desc(struct avd_qfw *q, unsigned int idx) +{ + return q->sram + AVDQ_RING_OFFSET + idx * AVDQ_DESC_BYTES; +} + +int avd_qfw_probe(struct avd_dev *avd) +{ + struct resource *code, *sram; + struct avd_qfw *q; + + code = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "code"); + sram = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "sram"); + if (!code || !sram || sram->start < code->start || + resource_size(sram) < AVDQ_RING_OFFSET + AVDQ_COUNT * AVDQ_DESC_BYTES) + return 0; /* no queue mode on this node: legacy transport only */ + q = devm_kzalloc(avd->dev, sizeof(*q), GFP_KERNEL); + if (!q) + return -ENOMEM; + q->sram = devm_ioremap_resource(avd->dev, sram); + if (IS_ERR(q->sram)) + return PTR_ERR(q->sram); + q->sram_size = resource_size(sram); + /* The code window sits at +0x80000 from the decoder base on every SoC. */ + q->aperture = AVDQ_CM3_APERTURE(0x80000 + (u32)(sram->start - code->start)); + avd->qfw = q; + return 0; +} + +/* Read the firmware's info block after boot; true when ABI 2 queue mode fits. */ +static bool avd_qfw_check(struct avd_dev *avd) +{ + struct avd_qfw *q = avd->qfw; + u32 info[16]; + unsigned int i; + + if (!q) + return false; + for (i = 0; i < ARRAY_SIZE(info); i++) + info[i] = readl(q->sram + AVDQ_INFO + 4 * i); + q->usable = false; + if (info[0] != AVDQ_MAGIC) + return false; + if (info[1] != AVDQ_VERSION || info[2] != AVDQ_COUNT || + info[3] != AVDQ_DESC_BYTES || info[5] != AVDQ_RING_OFFSET || + !(info[11] & AVDQ_FEAT_SRAM_RING) || + !(info[11] & AVDQ_FEAT_MULTI_KICK)) { + dev_info_once(avd->dev, "open firmware queue ABI %u not supported, using legacy transport\n", + info[1]); + return false; + } + if (info[4] != q->aperture) { + dev_warn_once(avd->dev, "firmware SRAM aperture %#x, node needs %#x: wrong image tier, using legacy transport\n", + info[4], q->aperture); + return false; + } + if (info[13] < 1024 || info[12] % 4 || + (u64)info[12] + 4ULL * info[13] > AVDQ_RING_LIMIT || + !info[10] || info[10] > AVDQ_COUNT || info[14] || info[15]) { + dev_warn_once(avd->dev, "firmware SRAM ring geometry invalid, using legacy transport\n"); + return false; + } + q->ring_off = info[12]; + q->ring_words = info[13]; + q->fifo_slots = min_t(u32, info[10], avd->variant->fifo_slots ?: 1); + q->wp = 0; + q->usable = true; + return true; +} + +static int avd_qfw_boot(struct avd_dev *avd) +{ + int ret = avd_boot(avd); + + if (ret) + return ret; + /* AP -> CM3 doorbells use mailbox 0; completions stay on mailbox 1. */ + writel_relaxed(AVD_MBOX_ENABLE, avd->mbox + AVD_REG_MBOX0_STATUS); + return 0; +} + +static unsigned int avd_qfw_max_inflight(struct avd_dev *avd) +{ + /* Each frame in flight owns one hardware instruction FIFO (EXEC 7:4). */ + return min_t(unsigned int, AVD_MAX_FRAMES, avd->qfw->fifo_slots); +} + +static void avd_qfw_fail_all(struct avd_dev *avd) +{ + unsigned int i; + + /* ABI: any fatal word or I/O status needs a whole-queue reset. */ + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (avd->frames[i].state == AVD_FRAME_SUBMITTED) + avd->frames[i].deadline = jiffies; + mod_delayed_work(system_wq, &avd->queue_work, 0); +} + +static int avd_qfw_stream(struct avd_dev *avd, const u32 *words, u32 count) +{ + struct avd_qfw *q = avd->qfw; + unsigned long timeout = jiffies + usecs_to_jiffies(AVDQ_STREAM_TIMEOUT_US); + u32 done = 0; + + while (done < count) { + u32 rp = readl(q->sram + AVDQ_INFO_RP); + u32 used = q->wp - rp, space, pos, chunk; + + if (used > q->ring_words) + return -EIO; + space = q->ring_words - used; + if (!space) { + if (time_after(jiffies, timeout)) + return -ETIMEDOUT; + usleep_range(20, 50); + continue; + } + pos = q->wp % q->ring_words; + chunk = min3(space, count - done, q->ring_words - pos); + memcpy_toio(q->sram + q->ring_off + 4 * pos, words + done, 4 * chunk); + q->wp += chunk; + done += chunk; + /* Words before the count that publishes them. */ + wmb(); + writel(q->wp, q->sram + AVDQ_INFO_WP); + } + return 0; +} + +static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) +{ + struct avd_qfw *q = avd->qfw; + struct avd_ctx *ctx = f->ctx; + const struct avd_coded_fmt_ops *ops = ctx->coded_fmt_desc->ops; + u32 codec = f->cmd[0], vp = 0, kick, cont, mask, extra = 0; + u32 *words = f->prog->cpu; + void __iomem *d = avd_qfw_desc(q, f->idx); + unsigned int i; + int ret; + + if (ctx->coded_fmt_desc->frame_ops) + return -EOPNOTSUPP; + if (codec > AVD_CODEC_AV1 || !words || !f->prog_words || + f->prog_words > f->prog->size / sizeof(u32)) + return -EINVAL; + if (!(avd->capabilities & BIT(codec))) + return -EOPNOTSUPP; + /* The program's EXEC words name FIFO f->idx; it must be a real FIFO. */ + if (f->idx >= q->fifo_slots || ctx->fifo_idx != f->idx) + return -EINVAL; + for (i = 0; i < codec; i++) + vp += avd->variant->vp_slots[i]; + if (avd->variant->vp_slot_base[codec]) + vp = avd->variant->vp_slot_base[codec]; + if (ops->pp_kicks) + extra = ops->pp_kicks(ctx) - 1; + if (extra > AVDQ_MAX_PP_EXTRA) + return -E2BIG; + + mask = codec == AVD_CODEC_VP9 && avd->variant->revision == 3 ? + AVD_OP_EXEC_REV3_VP9_MASK : 0; + cont = AVD_OP_EXEC | mask | AVD_OP_EXEC_FIFO_IDX(f->idx) | + AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots); + kick = cont | AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | + AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3 && + codec != AVD_CODEC_AV1); + + /* FIFO registers of other live frames are never touched. */ + avd->variant->configure_stream(avd, ctx->inst.addr, f->idx, vp); + + writel(AVDQ_MAGIC, d + D_MAGIC); + writel(f->tag, d + D_TAG); + writel(codec, d + D_CODEC); + writel(vp, d + D_VP); + writel(q->wp, d + D_PROG_LO); + writel(0, d + D_PROG_HI); + writel(f->prog_words, d + D_WORDS); + writel(kick, d + D_KICK); + writel(0, d + D_STATUS); + writel(0, d + D_CTAG); + writel(AVDQ_FLAG_SRAM, d + D_FLAGS); + writel(extra, d + D_PP_EXTRA); + writel(extra ? cont : 0, d + D_PP_CONT); + writel(0, d + D_RSVD0); + writel(0, d + D_RSVD1); + writel(AVDQ_STATE_SUBMITTED, d + D_STATE); + /* Descriptor before the doorbell that hands it to the firmware. */ + wmb(); + (void)readl(d + D_STATE); /* flush posted aperture writes */ + writel(q->aperture + AVDQ_RING_OFFSET + f->idx * AVDQ_DESC_BYTES, + avd->mbox + AVD_REG_MBOX0_SEND); + + /* The firmware may start feeding while the rest of the program streams in. */ + ret = avd_qfw_stream(avd, words, f->prog_words); + if (ret) { + dev_err_ratelimited(avd->dev, "queue: program stream failed: %d\n", ret); + /* The descriptor is already posted: only a reset releases it. */ + f->deadline = jiffies; + } + return 0; +} + +static irqreturn_t avd_qfw_irq(struct avd_dev *avd) +{ + struct avd_qfw *q = avd->qfw; + u32 word = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); + struct avd_frame *f; + void __iomem *d; + u32 state, status, ctag; + unsigned int idx; + + writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); + if ((word & AVDQ_NOTIFY_MASK) != AVDQ_NOTIFY) { + dev_err_ratelimited(avd->dev, "queue: firmware reported %#x, resetting\n", word); + avd_qfw_fail_all(avd); + return IRQ_HANDLED; + } + idx = word & 15; + d = avd_qfw_desc(q, idx); + state = readl(d + D_STATE); + status = readl(d + D_STATUS); + ctag = readl(d + D_CTAG); + f = &avd->frames[idx]; + if (state != AVDQ_STATE_COMPLETE || ((ctag ^ word) & 0xfff0) || + f->state != AVD_FRAME_SUBMITTED || ctag != f->tag) { + dev_warn_ratelimited(avd->dev, "queue: stale completion %#x (state %u tag %#x)\n", + word, state, ctag); + return IRQ_HANDLED; + } + writel(0, d + D_STATE); + avd_frame_complete(avd, ctag, status == 0 ? 0 : status == 1 ? -EINVAL : -EIO); + if (status > 1) + avd_qfw_fail_all(avd); + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_qfw_backend = { + .name = "open-queue", + .boot = avd_qfw_boot, + .shutdown = avd_shutdown, + .max_inflight = avd_qfw_max_inflight, + .submit = avd_qfw_submit, + .irq = avd_qfw_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, +}; + +bool avd_backend_is_qfw(const struct avd_dev *avd) +{ + return avd->backend == &avd_qfw_backend; +} + +/* + * Called after every successful open-firmware boot (probe, resume, reset), + * with no frame owned by the firmware. Picks the queued transport when the + * loaded image speaks ABI 2 for this node, else the legacy one. + */ +void avd_qfw_select(struct avd_dev *avd) +{ + const struct avd_backend_ops *old = avd->backend; + + if (old != &avd_open_backend && old != &avd_qfw_backend) + return; + if (avd_qfw_check(avd)) { + /* Legacy boot left mailbox 0 disabled. */ + writel_relaxed(AVD_MBOX_ENABLE, avd->mbox + AVD_REG_MBOX0_STATUS); + avd->backend = &avd_qfw_backend; + } else { + avd->backend = &avd_open_backend; + } + if (avd->backend != old) + dev_info(avd->dev, "using %s transport (%u frames in flight)\n", + avd->backend->name, avd->backend->max_inflight(avd)); +} diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index e470287a2b496..22b6b177d994b 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -510,7 +510,9 @@ static const struct avd_variant avd_t6040_variant = { AVD_CAPABILITY_AV1, /* HEVC only with firmware whose CODECS trailer lists h265. */ .fw_caps_required = AVD_CAPABILITY_HEVC, - .fw_name = "apple/avd-fw-v5-t1.bin", + /* v5-t2 matches T6040's SRAM aperture, which the queued transport needs. */ + .fw_name = "apple/avd-fw-v5-t2.bin", + .fw_name_fallback = "apple/avd-fw-v5-t1.bin", .revision = 4, .vp_slot_offset = 0xc, .submit_offset = 0x40, diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index 9dfa20ab71913..c588e7a295bbc 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -1395,6 +1395,13 @@ static int avd_hevc_try_ctrl(struct avd_ctx *ctx, struct v4l2_ctrl *ctrl) return 0; } +static unsigned int avd_hevc_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_hevc_ctx *priv = ctx->priv; + + return max_t(int, priv->submit_num, 1); +} + static void avd_hevc_submit(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; @@ -1421,6 +1428,7 @@ const struct avd_coded_fmt_ops avd_hevc_fmt_ops = { .stop = avd_hevc_stop, .run = avd_hevc_run, .submit = avd_hevc_submit, + .pp_kicks = avd_hevc_pp_kicks, .try_ctrl = avd_hevc_try_ctrl, .get_image_fmt = avd_hevc_get_image_fmt, }; diff --git a/drivers/media/platform/apple/avd/avd-vp9.c b/drivers/media/platform/apple/avd/avd-vp9.c index 4cdcb6594c4d2..37374b73e8a86 100644 --- a/drivers/media/platform/apple/avd/avd-vp9.c +++ b/drivers/media/platform/apple/avd/avd-vp9.c @@ -1049,6 +1049,13 @@ static enum avd_image_fmt avd_vp9_get_image_fmt(struct avd_ctx *ctx, #undef BIT_DEPTH } +static unsigned int avd_vp9_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_vp9_ctx *priv = ctx->priv; + + return max_t(int, priv->submit_num, 1); +} + static void avd_vp9_submit(struct avd_ctx *ctx) { struct avd_vp9_ctx *vp9_ctx = ctx->priv; @@ -1078,5 +1085,6 @@ const struct avd_coded_fmt_ops avd_vp9_fmt_ops = { .run = avd_vp9_run, .done = avd_vp9_done, .submit = avd_vp9_submit, + .pp_kicks = avd_vp9_pp_kicks, .get_image_fmt = avd_vp9_get_image_fmt, }; diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 36aa5944c2caf..5d8cb1e48b72d 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -149,6 +149,8 @@ struct avd_coded_fmt_ops { void (*adjust_decoded_fmt)(struct avd_ctx *ctx, struct v4l2_pix_format_mplane *pix_mp); void (*submit)(struct avd_ctx *ctx); + /* PP EXEC commands the last built frame needs (submit's count), >= 1. */ + unsigned int (*pp_kicks)(struct avd_ctx *ctx); int (*start)(struct avd_ctx *ctx); void (*stop)(struct avd_ctx *ctx); int (*run)(struct avd_ctx *ctx); @@ -199,6 +201,8 @@ struct avd_variant { void (*configure_stream)(struct avd_dev *avd, dma_addr_t addr, u8 fifo_idx, u32 vp_slot); const char* fw_name; + /* Older image tried when fw_name is not installed (legacy transport). */ + const char *fw_name_fallback; unsigned char revision; /* the same as the device tree */ /* just for convenience */ u32 vp_slot_offset; @@ -247,6 +251,8 @@ struct avd_dev { void *backend_priv; /* Only the open-firmware compatibility path uses these. */ struct avd_frame *building; + /* Open firmware queue ABI 2 state, NULL when the node has no SRAM. */ + struct avd_qfw *qfw; struct avd_frame *compat_active; bool compat_held; @@ -364,6 +370,11 @@ void avd_queue_del_ctx(struct avd_ctx *ctx); int avd_open_build_program(struct avd_ctx *ctx); extern const struct avd_backend_ops avd_open_backend; extern const struct avd_backend_ops avd_apple_backend; +extern const struct avd_backend_ops avd_qfw_backend; +struct avd_qfw; +int avd_qfw_probe(struct avd_dev *avd); +void avd_qfw_select(struct avd_dev *avd); +bool avd_backend_is_qfw(const struct avd_dev *avd); int avd_backend_select(struct avd_dev *avd); int avd_backend_boot(struct avd_dev *avd); void avd_shutdown(struct avd_dev *avd); From 415972f5bedfc65a80a6e580cebe776909f4d9f7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 00:35:07 +1100 Subject: [PATCH 18/73] media: apple: avd: clear the queue info block before each firmware boot CM3 SRAM keeps its contents across a firmware reload. After an ABI 2 image ran, a later boot of an older image would still show the queue info magic and the driver would pick the queued transport for firmware that does not speak it. Clear the magic before every open-firmware boot so only an image that publishes it again selects queue mode. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-backend-open.c | 5 ++++- drivers/media/platform/apple/avd/avd-backend-queue.c | 11 +++++++++++ drivers/media/platform/apple/avd/avd.h | 1 + 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 84d59dd57f297..2df2b68104cc3 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -230,8 +230,10 @@ int avd_backend_boot(struct avd_dev *avd) { int ret; - if (!avd_backend_is_apple(avd)) + if (!avd_backend_is_apple(avd)) { avd_pm_prepare_piodma(avd); + avd_qfw_prepare_boot(avd); + } ret = avd->backend->boot(avd); /* Never switch command ABIs after frames have used this backend. */ if (ret == -ENOENT && !avd->fw_generation && avd_backend_is_apple(avd)) { @@ -243,6 +245,7 @@ int avd_backend_boot(struct avd_dev *avd) avd->backend = &avd_open_backend; avd_open_capabilities(avd); avd_pm_prepare_piodma(avd); + avd_qfw_prepare_boot(avd); ret = avd->backend->boot(avd); } if (!ret) diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c index b6026fd67832e..8e390010f7550 100644 --- a/drivers/media/platform/apple/avd/avd-backend-queue.c +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -313,6 +313,17 @@ bool avd_backend_is_qfw(const struct avd_dev *avd) return avd->backend == &avd_qfw_backend; } +/* + * SRAM survives a CM3 reload, so a stale info block from an earlier ABI 2 + * image could make a legacy image look queue capable. Clear the magic before + * every open-firmware boot; only an image that publishes it again qualifies. + */ +void avd_qfw_prepare_boot(struct avd_dev *avd) +{ + if (avd->qfw) + writel(0, avd->qfw->sram + AVDQ_INFO); +} + /* * Called after every successful open-firmware boot (probe, resume, reset), * with no frame owned by the firmware. Picks the queued transport when the diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 5d8cb1e48b72d..b361304648bb4 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -374,6 +374,7 @@ extern const struct avd_backend_ops avd_qfw_backend; struct avd_qfw; int avd_qfw_probe(struct avd_dev *avd); void avd_qfw_select(struct avd_dev *avd); +void avd_qfw_prepare_boot(struct avd_dev *avd); bool avd_backend_is_qfw(const struct avd_dev *avd); int avd_backend_select(struct avd_dev *avd); int avd_backend_boot(struct avd_dev *avd); From 71a03d2fb7ac06333e46fee11bd694bb29a3d754 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 00:40:49 +1100 Subject: [PATCH 19/73] media: apple: avd: overlap H.264 pictures on the queued transport With the firmware chaining VP to PP, the VP of picture N+1 can run while the PP of picture N is still busy, but the legacy H.264 builder kept one set of scratch buffers, one VP to PP instruction FIFO and one slice table per context, and the queue core held the m2m scheduler until a picture retired. Give H.264 two banks of everything the hardware touches while decoding a picture (instruction FIFO, pipe state, above, loop-filter and MV row buffers) and of the slice copies. A new picture takes a free bank and records it in its CAPTURE buffer; retirement frees that bank. Pictures whose last slice never arrived restart in their own bank. The legacy transport keeps one bank, so nothing changes there. The queue core now lets a legacy codec context hold as many pictures as it has banks, carries each frame's FIFO address to the transport, and only keeps the m2m scheduler for slices that hold their CAPTURE buffer, so the next picture is prepared while the previous one decodes. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../platform/apple/avd/avd-backend-open.c | 2 + .../platform/apple/avd/avd-backend-queue.c | 2 +- drivers/media/platform/apple/avd/avd-h264.c | 253 ++++++++++++------ drivers/media/platform/apple/avd/avd-queue.c | 25 +- drivers/media/platform/apple/avd/avd-queue.h | 2 + drivers/media/platform/apple/avd/avd.h | 6 + 6 files changed, 198 insertions(+), 92 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 2df2b68104cc3..1a2f107262451 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -47,6 +47,8 @@ int avd_open_build_program(struct avd_ctx *ctx) AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3); /* The queued transport gives every frame in flight its own FIFO. */ ctx->fifo_idx = avd_backend_is_qfw(avd) ? f->idx : 0; + f->fifo_iova = ctx->job_fifo ?: ctx->inst.addr; + ctx->job_fifo = 0; words = f->prog->cpu; words[pos++] = AVD_OP_EXEC | mask | start | AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx); for (i = 0; i <= job->num; i++) { diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c index 8e390010f7550..44922f18a030c 100644 --- a/drivers/media/platform/apple/avd/avd-backend-queue.c +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -229,7 +229,7 @@ static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) codec != AVD_CODEC_AV1); /* FIFO registers of other live frames are never touched. */ - avd->variant->configure_stream(avd, ctx->inst.addr, f->idx, vp); + avd->variant->configure_stream(avd, f->fifo_iova ?: ctx->inst.addr, f->idx, vp); writel(AVDQ_MAGIC, d + D_MAGIC); writel(f->tag, d + D_TAG); diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index e8bc8d0066ead..7eb23f42a07a0 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -57,15 +57,18 @@ struct avd_h264_run { u32 slice_data_off; }; -/* state */ -struct avd_h264_ctx { - bool monochrome; - struct avd_h264_reflists { - struct v4l2_h264_reference p[V4L2_H264_REF_LIST_LEN]; - struct v4l2_h264_reference b0[V4L2_H264_REF_LIST_LEN]; - struct v4l2_h264_reference b1[V4L2_H264_REF_LIST_LEN]; - } reflists; +/* + * One bank per picture in flight. With the queued transport two pictures of + * a stream overlap (VP of one, PP of the other), so everything the hardware + * reads or writes while decoding a picture, and the CPU copies of its slices, + * exists once per bank. The legacy transport uses a single bank. + */ +#define AVD_H264_BANKS 2 +struct avd_h264_bank { + bool monochrome; + bool busy; /* a picture of this bank is not retired yet */ + bool submitted; /* ... and its last slice went to the hardware */ struct avd_buf *slices; struct avd_buf *active_slice; size_t slice_num; @@ -74,6 +77,7 @@ struct avd_h264_ctx { size_t slice_bytes; struct avd_h264_bufs { + /* VP to PP instruction FIFO for pictures of this bank */ struct avd_buf inst; struct avd_buf pipe_state; struct avd_buf above_info; @@ -84,6 +88,20 @@ struct avd_h264_ctx { } bufs; }; +/* state */ +struct avd_h264_ctx { + struct avd_h264_reflists { + struct v4l2_h264_reference p[V4L2_H264_REF_LIST_LEN]; + struct v4l2_h264_reference b0[V4L2_H264_REF_LIST_LEN]; + struct v4l2_h264_reference b1[V4L2_H264_REF_LIST_LEN]; + } reflists; + + unsigned int nbanks; + struct avd_h264_bank bank[AVD_H264_BANKS]; + /* bank of the picture being built */ + struct avd_h264_bank *b; +}; + /* ffmpeg submits wrong timestamp, use first_mb_in_slice as a workaround */ #define is_new_frame(sl) \ (sl->first_mb_in_slice == 0) /* (ctx->fh.m2m_ctx->new_frame) */ @@ -118,7 +136,7 @@ static void stream_refs(struct avd_ctx *ctx, struct avd_h264_run *run) dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); push(0, ""); - pusha(h264_ctx->bufs.mv_above_info.addr, "mv_above_info", 7); + pusha(h264_ctx->b->bufs.mv_above_info.addr, "mv_above_info", 7); pusha(run->addresses.mv_color, "mv_color", 0); push(0, ""); @@ -272,7 +290,7 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) if (avd->variant->revision == 3) push(0, "zero"); - pusha(h264_ctx->bufs.above_info.addr, "hdr_9c_pps_tile_addr_lsb8", 0); + pusha(h264_ctx->b->bufs.above_info.addr, "hdr_9c_pps_tile_addr_lsb8", 0); push(0, ""); push(0, ""); @@ -280,11 +298,11 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) if (avd->variant->revision == 3) push(0, "zero"); else if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) - pusha(h264_ctx->bufs.pipe_state.addr, "pipe_state", 0); + pusha(h264_ctx->b->bufs.pipe_state.addr, "pipe_state", 0); - pusha(h264_ctx->bufs.ip_above.addr, "ip_above", 1); - pusha(h264_ctx->bufs.lf_above.addr, "lf_above", 2); - pusha(h264_ctx->bufs.lf_above_info.addr, "lf_above_info", 3); + pusha(h264_ctx->b->bufs.ip_above.addr, "ip_above", 1); + pusha(h264_ctx->b->bufs.lf_above.addr, "lf_above", 2); + pusha(h264_ctx->b->bufs.lf_above_info.addr, "lf_above_info", 3); push(0, ""); push_comp(ctx, run->base.comp_out, ctx->comp.offsets); @@ -437,12 +455,12 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) const struct v4l2_ctrl_h264_sps *sps = run->sps; const struct v4l2_ctrl_h264_slice_params *sl = run->slice_params; struct avd_h264_ctx *h264_ctx = ctx->priv; - u32 payload_len = h264_ctx->active_slice->size; + u32 payload_len = h264_ctx->b->active_slice->size; bool en_mode = (pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE) == 0; u32 off = run->slice_data_off; - dma_addr_t slc_a84 = h264_ctx->active_slice->addr + off; + dma_addr_t slc_a84 = h264_ctx->b->active_slice->addr + off; push(AVD_OP_CODED_DATA | AVD_OP_CODED_DATA_BIT_OFF( @@ -538,10 +556,9 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) } } -static int avd_h264_alloc_bufs(struct avd_ctx *ctx) +static int avd_h264_alloc_bufs(struct avd_ctx *ctx, struct avd_h264_bank *bank) { struct avd_dev *dev = ctx->dev; - struct avd_h264_ctx *h264_ctx = ctx->priv; int ret, w, bit_depth, mb; w = fmt_width(ctx); bit_depth = (ctx->image_fmt == AVD_IMG_FMT_420_10BIT || @@ -549,13 +566,13 @@ static int avd_h264_alloc_bufs(struct avd_ctx *ctx) 10 : 8; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.inst, fifo_size()); + ret = avd_buf_alloc(dev, &bank->bufs.inst, fifo_size()); if (ret) { return ret; } if (!(dev->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) { - ret = avd_buf_alloc(dev, &h264_ctx->bufs.pipe_state, 0x200); + ret = avd_buf_alloc(dev, &bank->bufs.pipe_state, 0x200); if (ret) { return ret; } @@ -563,24 +580,24 @@ static int avd_h264_alloc_bufs(struct avd_ctx *ctx) mb = DIV_ROUND_UP(w, 16); - ret = avd_buf_alloc(dev, &h264_ctx->bufs.above_info, mb * 20); + ret = avd_buf_alloc(dev, &bank->bufs.above_info, mb * 20); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.ip_above, bit_depth * 4 * mb); + ret = avd_buf_alloc(dev, &bank->bufs.ip_above, bit_depth * 4 * mb); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.lf_above, + ret = avd_buf_alloc(dev, &bank->bufs.lf_above, bit_depth * 4 * 4 * mb); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.lf_above_info, 32 * mb); + ret = avd_buf_alloc(dev, &bank->bufs.lf_above_info, 32 * mb); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.mv_above_info, 32 * mb); + ret = avd_buf_alloc(dev, &bank->bufs.mv_above_info, 32 * mb); if (ret) return ret; @@ -620,35 +637,46 @@ static int avd_h264_validate_sps(struct avd_ctx *ctx, return 0; } -/* Free the slices held for the current picture. */ +/* Free the slices held for one bank's picture. */ +static void avd_h264_drop_bank_slices(struct avd_ctx *ctx, struct avd_h264_bank *bank) +{ + for (size_t i = 0; i < bank->slice_num; i++) + avd_buf_free(ctx->dev, &bank->slices[i]); + bank->slice_num = 0; + bank->slice_bytes = 0; +} + +/* Free the slices held for the picture being built. */ static void avd_h264_drop_slices(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; - for (size_t i = 0; i < h264_ctx->slice_num; i++) - avd_buf_free(ctx->dev, &h264_ctx->slices[i]); - h264_ctx->slice_num = 0; - h264_ctx->slice_bytes = 0; + avd_h264_drop_bank_slices(ctx, h264_ctx->b); } static void avd_h264_stop(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; struct avd_dev *dev = ctx->dev; + unsigned int i; + ctx->max_frames = 1; if (!h264_ctx) return; - avd_buf_free(dev, &h264_ctx->bufs.pipe_state); - avd_buf_free(dev, &h264_ctx->bufs.inst); - avd_buf_free(dev, &h264_ctx->bufs.above_info); - avd_buf_free(dev, &h264_ctx->bufs.lf_above_info); - avd_buf_free(dev, &h264_ctx->bufs.lf_above); - avd_buf_free(dev, &h264_ctx->bufs.ip_above); - avd_buf_free(dev, &h264_ctx->bufs.mv_above_info); - - avd_h264_drop_slices(ctx); - kfree(h264_ctx->slices); + for (i = 0; i < AVD_H264_BANKS; i++) { + struct avd_h264_bank *bank = &h264_ctx->bank[i]; + + avd_buf_free(dev, &bank->bufs.pipe_state); + avd_buf_free(dev, &bank->bufs.inst); + avd_buf_free(dev, &bank->bufs.above_info); + avd_buf_free(dev, &bank->bufs.lf_above_info); + avd_buf_free(dev, &bank->bufs.lf_above); + avd_buf_free(dev, &bank->bufs.ip_above); + avd_buf_free(dev, &bank->bufs.mv_above_info); + avd_h264_drop_bank_slices(ctx, bank); + kfree(bank->slices); + } kfree(h264_ctx); ctx->priv = NULL; @@ -673,20 +701,28 @@ static int avd_h264_start(struct avd_ctx *ctx) return -ENOMEM; ctx->priv = h264_ctx; + /* Overlap pictures only where the firmware chains VP to PP itself. */ + h264_ctx->nbanks = avd_backend_is_qfw(ctx->dev) ? AVD_H264_BANKS : 1; + h264_ctx->b = &h264_ctx->bank[0]; + + for (unsigned int i = 0; i < h264_ctx->nbanks; i++) { + struct avd_h264_bank *bank = &h264_ctx->bank[i]; + + /* assume one slice per row */ + bank->alloc_slice_num = + ctrl->p_new.p_h264_sps->pic_height_in_map_units_minus1 + 1; + bank->slices = kcalloc(bank->alloc_slice_num, + sizeof(*bank->slices), GFP_KERNEL); + if (!bank->slices) { + ret = -ENOMEM; + goto err_free_ctx; + } - /* assume one slice per row */ - h264_ctx->alloc_slice_num = - ctrl->p_new.p_h264_sps->pic_height_in_map_units_minus1 + 1; - h264_ctx->slices = kcalloc(h264_ctx->alloc_slice_num, - sizeof(*h264_ctx->slices), GFP_KERNEL); - if (!h264_ctx->slices) { - ret = -ENOMEM; - goto err_free_ctx; + ret = avd_h264_alloc_bufs(ctx, bank); + if (ret) + goto err_free_ctx; } - - ret = avd_h264_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; + ctx->max_frames = h264_ctx->nbanks; return 0; @@ -732,22 +768,22 @@ static int avd_h264_realloc_slices(struct avd_ctx *ctx, size_t max_slices) void *tmp; size_t alloc_slice_num; - tmp = h264_ctx->slices; + tmp = h264_ctx->b->slices; /* grow by half, and by at least one for a one-row picture */ - alloc_slice_num = h264_ctx->alloc_slice_num + - max_t(size_t, h264_ctx->alloc_slice_num / 2, 1); + alloc_slice_num = h264_ctx->b->alloc_slice_num + + max_t(size_t, h264_ctx->b->alloc_slice_num / 2, 1); alloc_slice_num = min(alloc_slice_num, max_slices); - h264_ctx->slices = kcalloc(alloc_slice_num, sizeof(*h264_ctx->slices), + h264_ctx->b->slices = kcalloc(alloc_slice_num, sizeof(*h264_ctx->b->slices), GFP_KERNEL); - if (!h264_ctx->slices) { + if (!h264_ctx->b->slices) { /* make deallocating a little easier */ - h264_ctx->slices = tmp; + h264_ctx->b->slices = tmp; return -ENOMEM; } - h264_ctx->alloc_slice_num = alloc_slice_num; - memcpy(h264_ctx->slices, tmp, - sizeof(*h264_ctx->slices) * h264_ctx->slice_num); + h264_ctx->b->alloc_slice_num = alloc_slice_num; + memcpy(h264_ctx->b->slices, tmp, + sizeof(*h264_ctx->b->slices) * h264_ctx->b->slice_num); kfree(tmp); /* a slice that arrives before the frame's header has no job table yet */ @@ -853,6 +889,33 @@ VISIBLE_IF_KUNIT bool avd_h264_slice_fits(const struct v4l2_ctrl_h264_sps *sps, avd_h264_slice_budget(decoded_size); } +/* + * Start a new picture in a bank no unretired picture uses, and remember the + * bank in its CAPTURE buffer so retirement frees the right one. + */ +static int avd_h264_pick_bank(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst) +{ + struct avd_h264_ctx *h264_ctx = ctx->priv; + struct avd_h264_bank *bank = NULL; + unsigned int i; + + /* A picture whose last slice never came: start over in its bank. */ + if (h264_ctx->b->busy && !h264_ctx->b->submitted) + bank = h264_ctx->b; + for (i = 0; !bank && i < h264_ctx->nbanks; i++) + if (!h264_ctx->bank[i].busy) + bank = &h264_ctx->bank[i]; + if (!bank) + return -EBUSY; + avd_h264_drop_bank_slices(ctx, bank); + bank->busy = true; + bank->submitted = false; + h264_ctx->b = bank; + /* 0 means no bank, so a stale or missing value never frees one. */ + vb2_to_avd_decoded_buf(&dst->vb2_buf)->h264_bank = bank - h264_ctx->bank + 1; + return 0; +} + static int avd_h264_run(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; @@ -862,6 +925,14 @@ static int avd_h264_run(struct avd_ctx *ctx) avd_h264_run_preamble(ctx, &run); + if (run.slice_params && is_new_frame(run.slice_params)) { + ret = avd_h264_pick_bank(ctx, run.base.bufs.dst); + if (ret) { + avd_run_postamble(ctx, &run.base); + return ret; + } + } + struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); u32 payload_len = vb2_get_plane_payload(&src->vb2_buf, 0); const u8 *data = vb2_plane_vaddr(&src->vb2_buf, 0); @@ -875,8 +946,8 @@ static int avd_h264_run(struct avd_ctx *ctx) return -EINVAL; } - if (!avd_h264_slice_fits(run.sps, h264_ctx->slice_num, - h264_ctx->slice_bytes, payload_len, + if (!avd_h264_slice_fits(run.sps, h264_ctx->b->slice_num, + h264_ctx->b->slice_bytes, payload_len, ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].sizeimage)) { dev_err_ratelimited(ctx->dev->dev, "too many slices for one picture, dropping it\n"); @@ -886,20 +957,20 @@ static int avd_h264_run(struct avd_ctx *ctx) goto err_free_jobs; } - if (h264_ctx->slice_num >= h264_ctx->alloc_slice_num) { + if (h264_ctx->b->slice_num >= h264_ctx->b->alloc_slice_num) { ret = avd_h264_realloc_slices(ctx, avd_h264_max_slices(run.sps)); if (ret) goto err_free_jobs; } - h264_ctx->active_slice = &h264_ctx->slices[h264_ctx->slice_num]; - ret = avd_buf_alloc(ctx->dev, h264_ctx->active_slice, payload_len); + h264_ctx->b->active_slice = &h264_ctx->b->slices[h264_ctx->b->slice_num]; + ret = avd_buf_alloc(ctx->dev, h264_ctx->b->active_slice, payload_len); if (ret) goto err_free_jobs; - memcpy(h264_ctx->active_slice->cpu, data, payload_len); - h264_ctx->slice_num++; - h264_ctx->slice_bytes += PAGE_ALIGN((size_t)payload_len); - h264_ctx->monochrome = run.sps->chroma_format_idc == 0; + memcpy(h264_ctx->b->active_slice->cpu, data, payload_len); + h264_ctx->b->slice_num++; + h264_ctx->b->slice_bytes += PAGE_ALIGN((size_t)payload_len); + h264_ctx->b->monochrome = run.sps->chroma_format_idc == 0; /* Build the P/B{0,1} ref lists. */ v4l2_h264_init_reflist_builder(&reflist_builder, run.decode_params, @@ -923,7 +994,7 @@ static int avd_h264_run(struct avd_ctx *ctx) if (is_new_frame(run.slice_params)) { /* the header segment plus one per slice */ ret = avd_init_job(ctx, AVD_CODEC_H264, - h264_ctx->alloc_slice_num + 1); + h264_ctx->b->alloc_slice_num + 1); if (ret) return ret; stream_hdr(ctx, &run); @@ -940,6 +1011,8 @@ static int avd_h264_run(struct avd_ctx *ctx) return 0; } + ctx->job_fifo = h264_ctx->b->bufs.inst.addr; + h264_ctx->b->submitted = true; return avd_submit_job(ctx); err_free_jobs: @@ -954,22 +1027,32 @@ static void avd_h264_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, enum vb2_buffer_state result) { struct avd_h264_ctx *h264_ctx = ctx->priv; + struct avd_decoded_buffer *buf; + struct avd_h264_bank *bank; - if (!(src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF)) { - avd_h264_drop_slices(ctx); + if (src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF || !dst_buf) + return; - /* 4:0:0 has no chroma to decode; the picture is grey */ - if (h264_ctx->monochrome && dst_buf && result == VB2_BUF_STATE_DONE) { - const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; - struct avd_decoded_buffer *dst = - vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); - - /* vb2 must return cache ownership before the CPU writes. */ - dst->grey_chroma_offset = - (size_t)pix->plane_fmt[0].bytesperline * pix->height; - dst->grey_chroma_10bit = - ctx->image_fmt == AVD_IMG_FMT_420_10BIT; - } + buf = vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + if (!buf->h264_bank || buf->h264_bank > h264_ctx->nbanks) + return; + bank = &h264_ctx->bank[buf->h264_bank - 1]; + buf->h264_bank = 0; + avd_h264_drop_bank_slices(ctx, bank); + bank->busy = false; + bank->submitted = false; + + /* 4:0:0 has no chroma to decode; the picture is grey */ + if (bank->monochrome && result == VB2_BUF_STATE_DONE) { + const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; + struct avd_decoded_buffer *dst = + vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + + /* vb2 must return cache ownership before the CPU writes. */ + dst->grey_chroma_offset = + (size_t)pix->plane_fmt[0].bytesperline * pix->height; + dst->grey_chroma_10bit = + ctx->image_fmt == AVD_IMG_FMT_420_10BIT; } } diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index 1c50c0a10964b..5e0c102dbd663 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -218,10 +218,17 @@ int avd_queue_ready(void *priv) !READ_ONCE(avd->stopping) && !READ_ONCE(avd->faulted) && avd->inflight < avd->backend->max_inflight(avd); - for (i = 0; ready && i < AVD_MAX_FRAMES; i++) - if (ctx->next_dep == AVD_DEP_PREV_DONE && - avd->frames[i].state != AVD_FRAME_FREE && avd->frames[i].ctx == ctx) - ready = false; + if (ready && ctx->next_dep == AVD_DEP_PREV_DONE) { + /* Legacy codecs may hold several pictures in separate banks. */ + unsigned int limit = ctx->coded_fmt_desc && !ctx->coded_fmt_desc->frame_ops ? + max(ctx->max_frames, 1U) : 1, held = 0; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (avd->frames[i].state != AVD_FRAME_FREE && + avd->frames[i].ctx == ctx) + held++; + ready = held < limit; + } /* Prevent several queued m2m contexts from observing the same free slot. */ if (ready) avd->queued_ctx = ctx; @@ -269,8 +276,14 @@ static void avd_queue_submit_work(struct work_struct *work) goto fail_no_pm; f->src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); f->dst = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); - f->scheduler_held = ctx->fh.m2m_ctx->out_q_ctx.q.subsystem_flags & - VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF; + /* + * Only a slice that holds its CAPTURE buffer needs the m2m scheduler + * until retirement. The last slice of a picture releases both buffers + * to the slot, so the next picture can be prepared while it decodes. + */ + f->scheduler_held = (ctx->fh.m2m_ctx->out_q_ctx.q.subsystem_flags & + VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF) && + (f->src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF); f->frame_num = f->tag; f->deadline = jiffies + AVD_FRAME_TIMEOUT; memset(f->cmd, 0, sizeof(f->cmd)); diff --git a/drivers/media/platform/apple/avd/avd-queue.h b/drivers/media/platform/apple/avd/avd-queue.h index 438afc17184ec..cd912259c636a 100644 --- a/drivers/media/platform/apple/avd/avd-queue.h +++ b/drivers/media/platform/apple/avd/avd-queue.h @@ -52,6 +52,8 @@ struct avd_frame { unsigned long deadline; /* jiffies, per-frame watchdog */ /* Legacy m2m HOLD_CAPTURE API retains scheduler ownership until retire. */ bool scheduler_held; + /* VP to PP instruction FIFO for this frame (open backends). */ + dma_addr_t fifo_iova; }; struct avd_backend_ops { diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index b361304648bb4..c051faca4099b 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -128,6 +128,8 @@ struct avd_decoded_buffer { /* Pending CPU fill, retained with the buffer until dequeue. */ size_t grey_chroma_offset; bool grey_chroma_10bit; + /* H.264: scratch bank + 1 of the picture decoded into this buffer */ + u8 h264_bank; union { struct avd_vp9_decoded_buffer_info vp9; @@ -304,6 +306,10 @@ struct avd_ctx { struct avd_comp comp; int fifo_idx; + /* Pictures of this context the queue may hold at once (0 or 1: one). */ + unsigned int max_frames; + /* VP to PP FIFO for the job being built; 0 means ctx->inst. */ + dma_addr_t job_fifo; struct avd_job job; struct avd_buf inst; }; From b94a760e737502d6abd8478cee8e3d703aa5424b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 00:50:06 +1100 Subject: [PATCH 20/73] media: apple: avd: overlap HEVC pictures and gate held-back codecs on the queue image HEVC gets the same two-bank treatment as H.264 on the queued transport: the VP to PP instruction FIFO, pipe state and all row/column scratch exist once per bank, a picture records its bank in its CAPTURE buffer, and a new done hook frees it at retirement. The VP of picture N+1 then overlaps the PP of picture N. The legacy transport keeps one bank. The per-buffer bank field is now shared by both codecs. Codecs a variant holds back until a newer firmware (fw_caps_required, T6040 HEVC) are now also enabled when the loaded image runs the queue ABI and lists the codec in its info-block codec mask, not only through a CODECS trailer. Falling back to the legacy transport restores the trailer-derived set. Both the transport choice and the codec list come from what the image reports at boot, and clients enumerate formats and size their contexts before a first decode would power the decoder up, so boot the firmware once at probe and let it autosuspend. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../platform/apple/avd/avd-backend-open.c | 4 +- .../platform/apple/avd/avd-backend-queue.c | 9 ++ drivers/media/platform/apple/avd/avd-drv.c | 11 ++ drivers/media/platform/apple/avd/avd-h264.c | 26 ++-- drivers/media/platform/apple/avd/avd-hevc.c | 144 +++++++++++++----- drivers/media/platform/apple/avd/avd.h | 6 +- 6 files changed, 144 insertions(+), 56 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 1a2f107262451..14a693f932908 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -179,15 +179,17 @@ static void avd_open_capabilities(struct avd_dev *avd) /* Without a CODECS trailer, keep codecs that need a newer firmware hidden. */ avd->capabilities = avd->variant->capabilities & ~avd->variant->fw_caps_required; + avd->fw_capabilities = avd->capabilities; if (!avd->fw || avd->fw->size < 0x40) return; info = (const char *)avd->fw->data + avd->fw->size - 0x40; if (memcmp(info, "CODECS:", 7)) - return; + return; /* fw_capabilities already set above */ avd->capabilities = 0; for (i = 0; i < ARRAY_SIZE(codecs); i++) if (strnstr(info, codecs[i].name, 0x40)) avd->capabilities |= avd->variant->capabilities & codecs[i].bit; + avd->fw_capabilities = avd->capabilities; } static int avd_open_request_firmware(struct avd_dev *avd) diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c index 44922f18a030c..57132acbe1485 100644 --- a/drivers/media/platform/apple/avd/avd-backend-queue.c +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -59,6 +59,7 @@ struct avd_qfw { u32 ring_words; u32 wp; /* host write count, free-running */ u32 fifo_slots; + u32 codec_mask; /* info+0x24, bit n = codec n */ bool usable; /* last boot found a matching ABI 2 image */ }; @@ -126,6 +127,7 @@ static bool avd_qfw_check(struct avd_dev *avd) q->ring_off = info[12]; q->ring_words = info[13]; q->fifo_slots = min_t(u32, info[10], avd->variant->fifo_slots ?: 1); + q->codec_mask = info[9]; q->wp = 0; q->usable = true; return true; @@ -339,8 +341,15 @@ void avd_qfw_select(struct avd_dev *avd) /* Legacy boot left mailbox 0 disabled. */ writel_relaxed(AVD_MBOX_ENABLE, avd->mbox + AVD_REG_MBOX0_STATUS); avd->backend = &avd_qfw_backend; + /* + * Codecs held back for a newer firmware (T6040 HEVC) are + * enabled by a queue image that lists them in its codec mask. + */ + avd->capabilities = avd->fw_capabilities | + (avd->variant->capabilities & avd->qfw->codec_mask); } else { avd->backend = &avd_open_backend; + avd->capabilities = avd->fw_capabilities; } if (avd->backend != old) dev_info(avd->dev, "using %s transport (%u frames in flight)\n", diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 22b6b177d994b..250492679bbb7 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -646,6 +646,17 @@ static int avd_probe(struct platform_device *pdev) avd_pm_enable(avd); + /* + * Boot the firmware once now: the transport and the codec list depend + * on what the loaded image reports, and clients enumerate formats and + * size their contexts before the first decode would power it up. + */ + ret = pm_runtime_resume_and_get(avd->dev); + if (ret < 0) + dev_warn(avd->dev, "firmware probe boot failed: %d\n", ret); + else + pm_runtime_put_autosuspend(avd->dev); + ret = avd_v4l2_init(avd); if (ret) goto err_disable_runtime_pm; diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index 7eb23f42a07a0..5ff0026ef3730 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -65,7 +65,7 @@ struct avd_h264_run { */ #define AVD_H264_BANKS 2 -struct avd_h264_bank { +struct avd_bank { bool monochrome; bool busy; /* a picture of this bank is not retired yet */ bool submitted; /* ... and its last slice went to the hardware */ @@ -97,9 +97,9 @@ struct avd_h264_ctx { } reflists; unsigned int nbanks; - struct avd_h264_bank bank[AVD_H264_BANKS]; + struct avd_bank bank[AVD_H264_BANKS]; /* bank of the picture being built */ - struct avd_h264_bank *b; + struct avd_bank *b; }; /* ffmpeg submits wrong timestamp, use first_mb_in_slice as a workaround */ @@ -556,7 +556,7 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) } } -static int avd_h264_alloc_bufs(struct avd_ctx *ctx, struct avd_h264_bank *bank) +static int avd_h264_alloc_bufs(struct avd_ctx *ctx, struct avd_bank *bank) { struct avd_dev *dev = ctx->dev; int ret, w, bit_depth, mb; @@ -638,7 +638,7 @@ static int avd_h264_validate_sps(struct avd_ctx *ctx, } /* Free the slices held for one bank's picture. */ -static void avd_h264_drop_bank_slices(struct avd_ctx *ctx, struct avd_h264_bank *bank) +static void avd_h264_drop_bank_slices(struct avd_ctx *ctx, struct avd_bank *bank) { for (size_t i = 0; i < bank->slice_num; i++) avd_buf_free(ctx->dev, &bank->slices[i]); @@ -665,7 +665,7 @@ static void avd_h264_stop(struct avd_ctx *ctx) return; for (i = 0; i < AVD_H264_BANKS; i++) { - struct avd_h264_bank *bank = &h264_ctx->bank[i]; + struct avd_bank *bank = &h264_ctx->bank[i]; avd_buf_free(dev, &bank->bufs.pipe_state); avd_buf_free(dev, &bank->bufs.inst); @@ -706,7 +706,7 @@ static int avd_h264_start(struct avd_ctx *ctx) h264_ctx->b = &h264_ctx->bank[0]; for (unsigned int i = 0; i < h264_ctx->nbanks; i++) { - struct avd_h264_bank *bank = &h264_ctx->bank[i]; + struct avd_bank *bank = &h264_ctx->bank[i]; /* assume one slice per row */ bank->alloc_slice_num = @@ -896,7 +896,7 @@ VISIBLE_IF_KUNIT bool avd_h264_slice_fits(const struct v4l2_ctrl_h264_sps *sps, static int avd_h264_pick_bank(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst) { struct avd_h264_ctx *h264_ctx = ctx->priv; - struct avd_h264_bank *bank = NULL; + struct avd_bank *bank = NULL; unsigned int i; /* A picture whose last slice never came: start over in its bank. */ @@ -912,7 +912,7 @@ static int avd_h264_pick_bank(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst) bank->submitted = false; h264_ctx->b = bank; /* 0 means no bank, so a stale or missing value never frees one. */ - vb2_to_avd_decoded_buf(&dst->vb2_buf)->h264_bank = bank - h264_ctx->bank + 1; + vb2_to_avd_decoded_buf(&dst->vb2_buf)->bank = bank - h264_ctx->bank + 1; return 0; } @@ -1028,16 +1028,16 @@ static void avd_h264_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, { struct avd_h264_ctx *h264_ctx = ctx->priv; struct avd_decoded_buffer *buf; - struct avd_h264_bank *bank; + struct avd_bank *bank; if (src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF || !dst_buf) return; buf = vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); - if (!buf->h264_bank || buf->h264_bank > h264_ctx->nbanks) + if (!buf->bank || buf->bank > h264_ctx->nbanks) return; - bank = &h264_ctx->bank[buf->h264_bank - 1]; - buf->h264_bank = 0; + bank = &h264_ctx->bank[buf->bank - 1]; + buf->bank = 0; avd_h264_drop_bank_slices(ctx, bank); bank->busy = false; bank->submitted = false; diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index c588e7a295bbc..b1fb5aa63d1f9 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -79,10 +79,14 @@ struct avd_hevc_run { struct avd_hevc_tile_info tile_info; }; -struct avd_hevc_ctx { - struct v4l2_ctrl_hevc_scaling_matrix scaling_matrix_cache; +/* See avd-h264.c: one bank per picture in flight on the queued transport. */ +#define AVD_HEVC_BANKS 2 +struct avd_hevc_bank { + bool busy; struct avd_h264_bufs { + /* VP to PP instruction FIFO for pictures of this bank */ + struct avd_buf inst; struct avd_buf mv_above_info; struct avd_buf az_above; struct avd_buf ip_above; @@ -91,9 +95,17 @@ struct avd_hevc_ctx { struct avd_buf lf_left; struct avd_buf lf_left_info; struct avd_buf sw_left; - struct avd_buf inst; struct avd_buf pipe_state; } bufs; +}; + +struct avd_hevc_ctx { + struct v4l2_ctrl_hevc_scaling_matrix scaling_matrix_cache; + + unsigned int nbanks; + struct avd_hevc_bank bank[AVD_HEVC_BANKS]; + /* bank of the picture being built */ + struct avd_hevc_bank *b; int submit_num; }; @@ -110,7 +122,7 @@ static void stream_refs(struct avd_ctx *ctx, struct avd_hevc_run *run) push(ctx->dev->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER ? HEVC_DMA_CONFIG : 0, "reference_dma_config"); - pusha(hevc_ctx->bufs.mv_above_info.addr, "mv_above_info", 7); + pusha(hevc_ctx->b->bufs.mv_above_info.addr, "mv_above_info", 7); pusha(run->addresses.mv_color, "mv_color", 0); push(0, ""); @@ -357,30 +369,30 @@ static void set_header(struct avd_ctx *ctx, struct avd_hevc_run *run) if (avd->variant->revision == 3) push(0, "zero"); else if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) - pusha(hevc_ctx->bufs.pipe_state.addr, "pipe_state", 0); + pusha(hevc_ctx->b->bufs.pipe_state.addr, "pipe_state", 0); - pusha(hevc_ctx->bufs.ip_above.addr, "scratch_addr", 0); - pusha(hevc_ctx->bufs.lf_above.addr, "scratch_addr", 1); - pusha(hevc_ctx->bufs.lf_above_info.addr, "scratch_addr", 2); + pusha(hevc_ctx->b->bufs.ip_above.addr, "scratch_addr", 0); + pusha(hevc_ctx->b->bufs.lf_above.addr, "scratch_addr", 1); + pusha(hevc_ctx->b->bufs.lf_above_info.addr, "scratch_addr", 2); if ((pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) || !(avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER)) { - pusha(hevc_ctx->bufs.lf_left.addr, + pusha(hevc_ctx->b->bufs.lf_left.addr, "scratch_addr", 3); - pusha(hevc_ctx->bufs.lf_left_info.addr, + pusha(hevc_ctx->b->bufs.lf_left_info.addr, "scratch_addr", 4); if (avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER) pusha(0, "reserved_scratch", 0); - pusha(hevc_ctx->bufs.az_above.addr, + pusha(hevc_ctx->b->bufs.az_above.addr, "scratch_addr", 8); - pusha(hevc_ctx->bufs.sw_left.addr, + pusha(hevc_ctx->b->bufs.sw_left.addr, "scratch_addr", 9); } else { pusha(0, "", 3); pusha(0, "", 4); if (avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER) pusha(0, "reserved_scratch", 0); - pusha(hevc_ctx->bufs.az_above.addr, + pusha(hevc_ctx->b->bufs.az_above.addr, "scratch_addr", 8); pusha(0, "", 9); } @@ -1056,17 +1068,16 @@ static int avd_hevc_validate_sps(struct avd_ctx *ctx, return 0; } -static int avd_hevc_alloc_bufs(struct avd_ctx *ctx) +static int avd_hevc_alloc_bufs(struct avd_ctx *ctx, struct avd_hevc_bank *bank) { struct avd_dev *avd = ctx->dev; - struct avd_hevc_ctx *hevc_ctx = ctx->priv; int ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.inst, fifo_size()); + ret = avd_buf_alloc(avd, &bank->bufs.inst, fifo_size()); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.pipe_state, 0x200); + ret = avd_buf_alloc(avd, &bank->bufs.pipe_state, 0x200); if (ret) return ret; @@ -1077,20 +1088,26 @@ static void avd_hevc_stop(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; struct avd_dev *avd = ctx->dev; + unsigned int i; + ctx->max_frames = 1; if (!hevc_ctx) return; - avd_buf_free(avd, &hevc_ctx->bufs.pipe_state); - avd_buf_free(avd, &hevc_ctx->bufs.inst); - avd_buf_free(avd, &hevc_ctx->bufs.mv_above_info); - avd_buf_free(avd, &hevc_ctx->bufs.az_above); - avd_buf_free(avd, &hevc_ctx->bufs.ip_above); - avd_buf_free(avd, &hevc_ctx->bufs.lf_above); - avd_buf_free(avd, &hevc_ctx->bufs.lf_above_info); - avd_buf_free(avd, &hevc_ctx->bufs.lf_left); - avd_buf_free(avd, &hevc_ctx->bufs.lf_left_info); - avd_buf_free(avd, &hevc_ctx->bufs.sw_left); + for (i = 0; i < AVD_HEVC_BANKS; i++) { + struct avd_h264_bufs *bufs = &hevc_ctx->bank[i].bufs; + + avd_buf_free(avd, &bufs->pipe_state); + avd_buf_free(avd, &bufs->inst); + avd_buf_free(avd, &bufs->mv_above_info); + avd_buf_free(avd, &bufs->az_above); + avd_buf_free(avd, &bufs->ip_above); + avd_buf_free(avd, &bufs->lf_above); + avd_buf_free(avd, &bufs->lf_above_info); + avd_buf_free(avd, &bufs->lf_left); + avd_buf_free(avd, &bufs->lf_left_info); + avd_buf_free(avd, &bufs->sw_left); + } kfree(hevc_ctx); ctx->priv = NULL; @@ -1099,6 +1116,7 @@ static void avd_hevc_stop(struct avd_ctx *ctx) static int avd_hevc_start(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx; + unsigned int i; int ret; hevc_ctx = kzalloc(sizeof(*hevc_ctx), GFP_KERNEL); @@ -1106,9 +1124,15 @@ static int avd_hevc_start(struct avd_ctx *ctx) return -ENOMEM; ctx->priv = hevc_ctx; - ret = avd_hevc_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; + /* Overlap pictures only where the firmware chains VP to PP itself. */ + hevc_ctx->nbanks = avd_backend_is_qfw(ctx->dev) ? AVD_HEVC_BANKS : 1; + hevc_ctx->b = &hevc_ctx->bank[0]; + for (i = 0; i < hevc_ctx->nbanks; i++) { + ret = avd_hevc_alloc_bufs(ctx, &hevc_ctx->bank[i]); + if (ret) + goto err_free_ctx; + } + ctx->max_frames = hevc_ctx->nbanks; return 0; @@ -1117,6 +1141,38 @@ static int avd_hevc_start(struct avd_ctx *ctx) return ret; } +/* Take a bank no unretired picture uses; record it in the CAPTURE buffer. */ +static int avd_hevc_pick_bank(struct avd_ctx *ctx, struct avd_decoded_buffer *dst) +{ + struct avd_hevc_ctx *hevc_ctx = ctx->priv; + unsigned int i; + + for (i = 0; i < hevc_ctx->nbanks; i++) { + if (hevc_ctx->bank[i].busy) + continue; + hevc_ctx->bank[i].busy = true; + hevc_ctx->b = &hevc_ctx->bank[i]; + dst->bank = i + 1; + return 0; + } + return -EBUSY; +} + +static void avd_hevc_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, + struct vb2_v4l2_buffer *dst_buf, + enum vb2_buffer_state result) +{ + struct avd_hevc_ctx *hevc_ctx = ctx->priv; + struct avd_decoded_buffer *buf; + + if (!dst_buf || !hevc_ctx) + return; + buf = vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + if (buf->bank && buf->bank <= hevc_ctx->nbanks) + hevc_ctx->bank[buf->bank - 1].busy = false; + buf->bank = 0; +} + static int avd_hevc_alloc_scratch(struct avd_ctx *ctx, struct avd_hevc_run *run) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; @@ -1152,51 +1208,51 @@ static int avd_hevc_alloc_scratch(struct avd_ctx *ctx, struct avd_hevc_run *run) tile_info->row_height[i] : max_row; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.ip_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.ip_above, ((max_col * max_cu_width) / 4) * bit_depth); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.mv_above_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.mv_above_info, ((max_col * max_cu_width) / 16) * 20); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_above, DIV_ROUND_UP(w, 16) * bit_depth * (10 + 6) + (cols - 1) * 256); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_above_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_above_info, DIV_ROUND_UP(w + 7, 16) * 36 + cols * 128); if (ret) return ret; if (pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) { - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_left, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_left, ((max_row * max_cu_width) / 4) * 36 + 144 /* why? */); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_left_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_left_info, ((max_row * max_cu_width) / 4) * 9); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.az_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.az_above, /* not sure if its cols or rows */ DIV_ROUND_UP(w, 64) * 144 + (cols - 1) * 128); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.sw_left, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.sw_left, ((max_row * max_cu_width) / 4) * 216); if (ret) return ret; } else { - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.az_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.az_above, DIV_ROUND_UP(w + 7, 16) * 4 * bit_depth); if (ret) return ret; @@ -1345,6 +1401,7 @@ static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) static int avd_hevc_run(struct avd_ctx *ctx) { + struct avd_hevc_ctx *hevc_ctx = ctx->priv; struct avd_hevc_run run = {}; struct avd_decoded_buffer *dst; int ret; @@ -1356,6 +1413,10 @@ static int avd_hevc_run(struct avd_ctx *ctx) dst = vb2_to_avd_decoded_buf(&run.base.bufs.dst->vb2_buf); update_dec_buf_info(dst, &run.sl[0]); + ret = avd_hevc_pick_bank(ctx, dst); + if (ret) + goto complete; + ret = avd_hevc_compute_tiles(ctx, &run); if (ret) goto complete; @@ -1372,8 +1433,10 @@ static int avd_hevc_run(struct avd_ctx *ctx) set_header(ctx, &run); ret = stream_slices(ctx, &run); avd_run_postamble(ctx, &run.base); - if (!ret) + if (!ret) { + ctx->job_fifo = hevc_ctx->b->bufs.inst.addr; ret = avd_submit_job(ctx); + } goto done; complete: @@ -1427,6 +1490,7 @@ const struct avd_coded_fmt_ops avd_hevc_fmt_ops = { .start = avd_hevc_start, .stop = avd_hevc_stop, .run = avd_hevc_run, + .done = avd_hevc_done, .submit = avd_hevc_submit, .pp_kicks = avd_hevc_pp_kicks, .try_ctrl = avd_hevc_try_ctrl, diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index c051faca4099b..44aa9371ef0e0 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -128,8 +128,8 @@ struct avd_decoded_buffer { /* Pending CPU fill, retained with the buffer until dequeue. */ size_t grey_chroma_offset; bool grey_chroma_10bit; - /* H.264: scratch bank + 1 of the picture decoded into this buffer */ - u8 h264_bank; + /* H.264/HEVC: scratch bank + 1 of the picture decoded into this buffer */ + u8 bank; union { struct avd_vp9_decoded_buffer_info vp9; @@ -262,6 +262,8 @@ struct avd_dev { const struct avd_variant *variant; unsigned int capabilities; + /* What the loaded image's CODECS trailer (or its absence) allows. */ + unsigned int fw_capabilities; }; struct avd_segment { From 2d0d23f3c3673c7d1342268a229219de81825736 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 00:50:22 +1100 Subject: [PATCH 21/73] media: apple: avd: give the queue host test a dma_addr_t avd_frame gained a DMA address for its instruction FIFO, and the host build of the queue test has no dma_addr_t. Define it in the host shim. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/tests/include/linux/types.h | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/media/platform/apple/avd/tests/include/linux/types.h b/drivers/media/platform/apple/avd/tests/include/linux/types.h index 3d29df0e18138..8b9638abb9a7f 100644 --- a/drivers/media/platform/apple/avd/tests/include/linux/types.h +++ b/drivers/media/platform/apple/avd/tests/include/linux/types.h @@ -7,4 +7,5 @@ typedef __u8 u8; typedef __u32 u32; typedef __u64 u64; +typedef __u64 dma_addr_t; #endif From 7c878fb4e4575a5ff2162a03f75ee83c481b7900 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 01:48:38 +1100 Subject: [PATCH 22/73] media: apple: avd: never wait for the IRQ thread while it may need queue_mutex The threaded interrupt handler takes queue_mutex before it touches the decoder. Queue reset runs from the queue worker with queue_mutex held, and its recovery called disable_irq(), which waits for that handler thread. When an interrupt arrived around a frame timeout, the handler blocked on queue_mutex and the reset blocked on the handler: the worker and the IRQ thread deadlocked with the mutex held. Runtime suspend had a similar cycle with a submission waiting for it under queue_mutex. Close hardware access with a spinlock instead: the handler runs the backend's interrupt code only under irq_lock, with hw_on set and the interrupt generation it sampled before taking queue_mutex unchanged. Turning the IRQ off clears hw_on and bumps the generation under that lock, so an access in progress finishes first and a handler woken by an older firmware instance leaves the hardware alone, and then uses disable_irq_nosync(). Device removal, which holds no queue_mutex, still synchronizes with the handler. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-drv.c | 5 +-- drivers/media/platform/apple/avd/avd-pm.c | 40 ++++++++++++++++++++-- drivers/media/platform/apple/avd/avd-pm.h | 10 ++++++ 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 250492679bbb7..fc23c0935f36f 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -113,11 +113,12 @@ int avd_submit_job(struct avd_ctx *ctx) static irqreturn_t avd_irq_handler(int irq, void *data) { struct avd_dev *avd = data; - + /* An interrupt raised before a reset or power-off belongs to that instance. */ + unsigned int gen = READ_ONCE(avd->pm.irq_gen); irqreturn_t ret; mutex_lock(&avd->queue_mutex); - ret = avd->backend->irq(avd); + ret = avd_pm_irq(avd, gen); mutex_unlock(&avd->queue_mutex); return ret; } diff --git a/drivers/media/platform/apple/avd/avd-pm.c b/drivers/media/platform/apple/avd/avd-pm.c index 0df13b4396047..bb9014b86f640 100644 --- a/drivers/media/platform/apple/avd/avd-pm.c +++ b/drivers/media/platform/apple/avd/avd-pm.c @@ -16,21 +16,51 @@ void avd_pm_enable_irq(struct avd_dev *avd) { + unsigned long flags; + if (avd->pm.irq >= 0 && !avd->pm.irq_enabled) { + spin_lock_irqsave(&avd->pm.irq_lock, flags); + avd->pm.hw_on = true; + spin_unlock_irqrestore(&avd->pm.irq_lock, flags); enable_irq(avd->pm.irq); avd->pm.irq_enabled = true; } } +/* + * Callers may hold queue_mutex, which the threaded handler takes first, so + * never wait for the handler here (disable_irq() would deadlock). Instead + * close hardware access under irq_lock: a handler already touching the + * decoder finishes before this returns, and any later one sees hw_on clear + * or a new irq_gen and leaves the hardware alone. + */ static void avd_pm_disable_irq(struct avd_dev *avd) { + unsigned long flags; + if (avd->pm.irq >= 0 && avd->pm.irq_enabled) { - /* Wait out threaded handlers before the domain can disappear. */ - disable_irq(avd->pm.irq); + spin_lock_irqsave(&avd->pm.irq_lock, flags); + avd->pm.hw_on = false; + avd->pm.irq_gen++; + spin_unlock_irqrestore(&avd->pm.irq_lock, flags); + disable_irq_nosync(avd->pm.irq); avd->pm.irq_enabled = false; } } +/* Run the backend's interrupt handler only against a live firmware instance. */ +irqreturn_t avd_pm_irq(struct avd_dev *avd, unsigned int gen) +{ + irqreturn_t ret = IRQ_HANDLED; + unsigned long flags; + + spin_lock_irqsave(&avd->pm.irq_lock, flags); + if (avd->pm.hw_on && gen == avd->pm.irq_gen) + ret = avd->backend->irq(avd); + spin_unlock_irqrestore(&avd->pm.irq_lock, flags); + return ret; +} + struct avd_dma_release { struct avd_deferred_unmap unmap; void *cpu; @@ -164,6 +194,9 @@ void avd_pm_init(struct avd_dev *avd, const struct avd_backend_ops *backend, avd->pm.dart_active = false; avd->pm.irq = -1; avd->pm.irq_enabled = false; + spin_lock_init(&avd->pm.irq_lock); + avd->pm.hw_on = false; + avd->pm.irq_gen = 0; avd->backend = backend; avd->pm.queue = queue; } @@ -218,6 +251,9 @@ void avd_pm_disable(struct avd_dev *avd) pm_runtime_disable(avd->dev); /* Also close the host IRQ if the final hardware transition failed. */ avd_pm_disable_irq(avd); + /* No queue_mutex is held here: let a woken handler thread finish. */ + if (avd->pm.irq >= 0) + synchronize_irq(avd->pm.irq); pm_runtime_dont_use_autosuspend(avd->dev); /* Resume may have kicked the already-stopped queue. */ cancel_delayed_work_sync(&avd->queue_work); diff --git a/drivers/media/platform/apple/avd/avd-pm.h b/drivers/media/platform/apple/avd/avd-pm.h index 62de4c8c3b70a..5fe55f595beb2 100644 --- a/drivers/media/platform/apple/avd/avd-pm.h +++ b/drivers/media/platform/apple/avd/avd-pm.h @@ -31,6 +31,15 @@ struct avd_pm { bool dart_active; int irq; bool irq_enabled; + /* + * irq_lock guards hw_on and irq_gen. The threaded handler touches the + * decoder only under irq_lock with hw_on set and an unchanged irq_gen, + * so turning the IRQ off never has to wait for the handler thread + * (which may itself be waiting for queue_mutex). + */ + spinlock_t irq_lock; + bool hw_on; + unsigned int irq_gen; void __iomem *piodma; phys_addr_t sram_phys; wait_queue_head_t idle_wait; @@ -57,6 +66,7 @@ int avd_pm_power_off(struct avd_dev *avd); int avd_pm_recover(struct avd_dev *avd); /* After recovery has invalidated every previously firmware-owned slot. */ void avd_pm_enable_irq(struct avd_dev *avd); +irqreturn_t avd_pm_irq(struct avd_dev *avd, unsigned int gen); void avd_pm_drain_begin(struct avd_dev *avd); void avd_pm_drain_end(struct avd_dev *avd); void avd_pm_queue_changed(struct avd_dev *avd); From 3eac43ba8c455e3b6bc0bb8edb3516f6c126451e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 01:48:38 +1100 Subject: [PATCH 23/73] media: apple: avd: kick PP for every VP DONE on the legacy transport The old driver kicked the PP for every VP DONE message of the open firmware. The legacy shim in the frame queue dropped a VP DONE that came in while no frame was in flight. That happens when a surplus PP DONE of a multi-kick picture completed the next frame early: the next frame's VP DONE then found no active frame, its PP was never started, and it timed out. Kick PP for the last submitting context in that case, as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-backend-open.c | 13 ++++++++++++- drivers/media/platform/apple/avd/avd-queue.c | 2 ++ drivers/media/platform/apple/avd/avd.h | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 14a693f932908..ce2f0d71174cf 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -94,6 +94,7 @@ static int avd_open_submit(struct avd_dev *avd, struct avd_frame *f) avd->variant->configure_stream(avd, ctx->inst.addr, ctx->fifo_idx, vp); reg = avd->ctrl + avd->variant->vp_slot_offset + vp * 4; avd->compat_active = f; + avd->compat_ctx = ctx; for (i = 0; i < f->prog_words; i++) { /* Preserve the old firmware's segment FIFO pacing in this shim. */ if (i && (words[i] & GENMASK(31, 24)) == AVD_OP_EXEC && @@ -134,8 +135,18 @@ static irqreturn_t avd_open_irq(struct avd_dev *avd) u32 status = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); - if (!f) + if (!f) { + /* + * The old driver kicked PP for every VP DONE. A VP DONE with no + * frame in flight (its frame already completed on a surplus PP + * DONE of a multi-kick picture) still needs that kick, or the + * next picture's PP is never started and it times out. + */ + if ((status & 0x11000) == 0 && (status & 0x100) && avd->compat_ctx && + avd->compat_ctx->coded_fmt_desc->ops->submit) + avd->compat_ctx->coded_fmt_desc->ops->submit(avd->compat_ctx); return IRQ_HANDLED; + } if (status & 0x1000) { avd->compat_active = NULL; avd_frame_complete(avd, f->tag, 0); diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index 5e0c102dbd663..a682b04747f61 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -470,6 +470,8 @@ void avd_queue_del_ctx(struct avd_ctx *ctx) { avd_queue_drain(ctx); mutex_lock(&ctx->dev->queue_mutex); + if (ctx->dev->compat_ctx == ctx) + ctx->dev->compat_ctx = NULL; list_del_init(&ctx->queue_node); mutex_unlock(&ctx->dev->queue_mutex); } diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 44aa9371ef0e0..f1e1cfef218f8 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -256,6 +256,8 @@ struct avd_dev { /* Open firmware queue ABI 2 state, NULL when the node has no SRAM. */ struct avd_qfw *qfw; struct avd_frame *compat_active; + /* Context of the last legacy submission, for VP DONE without a frame. */ + struct avd_ctx *compat_ctx; bool compat_held; struct reset_control *rstc; From c5f9ad82219969ea5f94fd4c406014f9df37568b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 01:55:52 +1100 Subject: [PATCH 24/73] media: apple: avd: do not pop an empty firmware mailbox Reading MBOX1_RETRIEVE when the mailbox is empty returns the previous message again. On a spurious or early interrupt that is a repeated PP DONE, which completes the next frame before its PP has run. The next PP then never reports done, the frame times out and the decoder is reset. The legacy shim and the queued transport both popped unconditionally. Check the empty flag in MBOX1_STATUS (bit 17) first, give a message that is not readable yet up to 10 us, and otherwise acknowledge and ignore the interrupt. Both transports share the helper. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../platform/apple/avd/avd-backend-open.c | 5 +++-- .../platform/apple/avd/avd-backend-queue.c | 6 ++--- drivers/media/platform/apple/avd/avd-hw.c | 22 +++++++++++++++++++ drivers/media/platform/apple/avd/avd-regs.h | 2 ++ drivers/media/platform/apple/avd/avd.h | 1 + 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index ce2f0d71174cf..9680dc4ef64e1 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -132,9 +132,10 @@ static int avd_open_submit(struct avd_dev *avd, struct avd_frame *f) static irqreturn_t avd_open_irq(struct avd_dev *avd) { struct avd_frame *f = avd->compat_active; - u32 status = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); + u32 status; - writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); + if (!avd_mbox_pop(avd, &status)) + return IRQ_HANDLED; if (!f) { /* * The old driver kicked PP for every VP DONE. A VP DONE with no diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c index 57132acbe1485..a706602386c17 100644 --- a/drivers/media/platform/apple/avd/avd-backend-queue.c +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -268,13 +268,13 @@ static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) static irqreturn_t avd_qfw_irq(struct avd_dev *avd) { struct avd_qfw *q = avd->qfw; - u32 word = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); struct avd_frame *f; void __iomem *d; - u32 state, status, ctag; + u32 word, state, status, ctag; unsigned int idx; - writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); + if (!avd_mbox_pop(avd, &word)) + return IRQ_HANDLED; if ((word & AVDQ_NOTIFY_MASK) != AVDQ_NOTIFY) { dev_err_ratelimited(avd->dev, "queue: firmware reported %#x, resetting\n", word); avd_qfw_fail_all(avd); diff --git a/drivers/media/platform/apple/avd/avd-hw.c b/drivers/media/platform/apple/avd/avd-hw.c index 84db61a77f6f4..b64c0bf4d78ea 100644 --- a/drivers/media/platform/apple/avd/avd-hw.c +++ b/drivers/media/platform/apple/avd/avd-hw.c @@ -67,6 +67,28 @@ int avd_boot(struct avd_dev *avd) return 0; } +/* + * Pop one firmware message. Reading RETRIEVE on an empty mailbox returns the + * previous message again, and a repeated PP DONE completes the next frame + * before its PP has run. An interrupt can also come just before its message + * is readable, so allow 10 us; after that treat the interrupt as spurious. + * Always acknowledges the interrupt. Returns false when there is no message. + */ +bool avd_mbox_pop(struct avd_dev *avd, u32 *msg) +{ + u32 st; + int ret; + + ret = readl_poll_timeout_atomic(avd->mbox + AVD_REG_MBOX1_STATUS, st, + !(st & AVD_MBOX1_STATUS_EMPTY), 1, 10); + if (!ret) + *msg = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); + writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); + if (ret) + dev_dbg_ratelimited(avd->dev, "spurious interrupt, mailbox empty\n"); + return !ret; +} + void avd_shutdown(struct avd_dev *avd) { writel_relaxed(AVD_RUN_CTRL_UNK_STOP, avd->mbox + AVD_REG_RUN_CTRL); diff --git a/drivers/media/platform/apple/avd/avd-regs.h b/drivers/media/platform/apple/avd/avd-regs.h index 388f3e64895f5..ace0e4c48f40a 100644 --- a/drivers/media/platform/apple/avd/avd-regs.h +++ b/drivers/media/platform/apple/avd/avd-regs.h @@ -13,6 +13,8 @@ #define AVD_MBOX1_NOT_EMPTY BIT(3) #define AVD_REG_MBOX1_STATUS 0x5c +/* Read side: bits 8-11 write pointer, bits 12-15 read pointer, bit 17 empty. */ +#define AVD_MBOX1_STATUS_EMPTY BIT(17) #define AVD_REG_MBOX1_RETRIEVE 0x64 #define AVD_MBOX_ENABLE BIT(0) diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index f1e1cfef218f8..4aeda520ab651 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -387,6 +387,7 @@ void avd_qfw_select(struct avd_dev *avd); void avd_qfw_prepare_boot(struct avd_dev *avd); bool avd_backend_is_qfw(const struct avd_dev *avd); int avd_backend_select(struct avd_dev *avd); +bool avd_mbox_pop(struct avd_dev *avd, u32 *msg); int avd_backend_boot(struct avd_dev *avd); void avd_shutdown(struct avd_dev *avd); From 58e286e75bc715d18a445e55fcbe19e31f95d499 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 01:55:52 +1100 Subject: [PATCH 25/73] media: apple: avd: log each decoder recovery step A frame timeout on the J616s was followed by the machine going down with nothing in the journal, so nothing showed how far the recovery got. Log every recovery step (stop firmware, reset, DART flush, firmware boot) before it runs, with the DART, runtime PM and IRQ state at the start, refuse and warn once if recovery is ever entered with the decoder powered off, and name the codec, slot state, frames in flight and transport in the timeout message. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-pm.c | 37 +++++++++++++++++--- drivers/media/platform/apple/avd/avd-queue.c | 6 +++- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-pm.c b/drivers/media/platform/apple/avd/avd-pm.c index bb9014b86f640..9c141fe549be4 100644 --- a/drivers/media/platform/apple/avd/avd-pm.c +++ b/drivers/media/platform/apple/avd/avd-pm.c @@ -297,28 +297,55 @@ int avd_pm_recover(struct avd_dev *avd) { int ret; + /* + * Recovery is rare and touches the decoder, its reset and its DART: + * leave a step-by-step trail in the log before each step, so a failure + * that takes the machine down still shows how far recovery got. + */ + dev_warn(avd->dev, "recover: start (dart %s, runtime %s, irq %s)\n", + avd->pm.dart_active ? "on" : "off", + pm_runtime_suspended(avd->dev) ? "suspended" : "active", + avd->pm.irq_enabled ? "on" : "off"); if (!avd->pm.dart_active) return -EHOSTDOWN; + /* Every caller holds a runtime PM reference; anything else is a bug. */ + if (WARN_ONCE(pm_runtime_suspended(avd->dev), + "avd: recovery with the decoder powered off\n")) + return -EHOSTDOWN; avd_pm_disable_irq(avd); + dev_warn(avd->dev, "recover: stop firmware\n"); avd->backend->shutdown(avd); + dev_warn(avd->dev, "recover: reset\n"); ret = reset_control_reset(avd->rstc); - if (ret) + if (ret) { + dev_err(avd->dev, "recover: reset failed: %d\n", ret); return ret; + } if (avd->domain) { if (!avd->empty_domain) return -ENODEV; + dev_warn(avd->dev, "recover: DART flush\n"); ret = iommu_attach_device(avd->empty_domain, avd->dev); - if (ret) + if (ret) { + dev_err(avd->dev, "recover: DART empty attach failed: %d\n", ret); return ret; + } iommu_detach_device(avd->empty_domain, avd->dev); /* Do not reboot against an empty/blocked domain after a failed restore. */ - if (iommu_get_domain_for_dev(avd->dev) != avd->domain) + if (iommu_get_domain_for_dev(avd->dev) != avd->domain) { + dev_err(avd->dev, "recover: DART domain not restored\n"); return -EIO; + } } + dev_warn(avd->dev, "recover: boot firmware\n"); ret = avd_pm_backend_resume(avd); - if (ret) + if (ret) { + dev_err(avd->dev, "recover: boot failed: %d\n", ret); avd->backend->shutdown(avd); - return ret; + return ret; + } + dev_warn(avd->dev, "recover: done\n"); + return 0; } int avd_pm_power_on(struct avd_dev *avd) diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index a682b04747f61..4aba8997f38e0 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -148,7 +148,11 @@ static void avd_queue_worker(struct work_struct *work) expired = avd_slot_expired(&avd->frames[i], jiffies); spin_unlock_irqrestore(&avd->queue_lock, flags); if (expired) { - dev_err(avd->dev, "frame tag %#x timed out\n", avd->frames[i].tag); + struct avd_frame *t = &avd->frames[i]; + + dev_err(avd->dev, "frame tag %#x timed out (%p4cc, state %d, %u in flight, %s)\n", + t->tag, t->ctx ? &t->ctx->coded_fmt_desc->fourcc : &(u32){0}, + t->state, avd->inflight, avd->backend->name); avd_reset_queue(avd, -ETIMEDOUT); break; } From 422060fd4999ef8e8c44e5d4b201d01e82dedccc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 02:18:50 +1100 Subject: [PATCH 26/73] media: apple: avd: stop a decoder error storm instead of resetting in a loop The Neo hung after a burst of "H0 error" (VP0, the HEVC VP, raising its error interrupt) with "Frame processing timed out!" for every frame: each error reset the decoder (pmgr reset, DART empty-domain flush, firmware reload), the next frame of the same broken stream failed at once, and the resets ran back to back, up to one per millisecond, until the machine stopped. The frame queue inherited the same loop. Bound it at three levels: - A stream that fails three frames in a row in hardware is stopped: its frames complete with an error, without touching the decoder, until streaming is restarted. - Decoder recoveries are at least 20 ms apart. - More than eight recoveries within 10 s put the decoder in a 10 s cool-down in which new frames fail at once. A faulted decoder (failed recovery) no longer blocks the m2m scheduler, which left clients waiting forever: its frames fail at submission, the request controls are completed so the request fd signals, and the next clean power-up clears the fault. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-pm.c | 2 + drivers/media/platform/apple/avd/avd-queue.c | 50 +++++++++++++++++++- drivers/media/platform/apple/avd/avd-v4l2.c | 3 ++ drivers/media/platform/apple/avd/avd.h | 10 ++++ 4 files changed, 63 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-pm.c b/drivers/media/platform/apple/avd/avd-pm.c index 9c141fe549be4..c65a447f93b00 100644 --- a/drivers/media/platform/apple/avd/avd-pm.c +++ b/drivers/media/platform/apple/avd/avd-pm.c @@ -369,6 +369,8 @@ int avd_pm_power_on(struct avd_dev *avd) return ret; } avd_pm_enable_irq(avd); + /* A clean boot after power-up ends a failed recovery's fault state. */ + WRITE_ONCE(avd->faulted, false); if (avd->pm.queue->resubmit) avd->pm.queue->resubmit(avd); diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index 4aba8997f38e0..45759a0593792 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT +#include #include #include #include @@ -9,6 +10,14 @@ #include "avd-queue-internal.h" #define AVD_FRAME_TIMEOUT msecs_to_jiffies(2000) +/* A stream that fails this many frames in a row in hardware is stopped. */ +#define AVD_CTX_MAX_HW_ERRORS 3 +/* Never reset the decoder more often than this. */ +#define AVD_RECOVER_GAP_MS 20 +/* More than AVD_STORM_MAX recoveries within AVD_STORM_WINDOW: cool down. */ +#define AVD_STORM_MAX 8 +#define AVD_STORM_WINDOW msecs_to_jiffies(10000) +#define AVD_COOLDOWN msecs_to_jiffies(10000) #define AVD_RETRY_DELAY msecs_to_jiffies(1) #define AVD_PROGRAM_SIZE SZ_2M @@ -41,6 +50,13 @@ static void avd_retire(struct avd_dev *avd) spin_unlock_irqrestore(&avd->queue_lock, flags); ctx = f->ctx; state = f->status ? VB2_BUF_STATE_ERROR : VB2_BUF_STATE_DONE; + if (!f->status) { + ctx->hw_errors = 0; + } else if (++ctx->hw_errors >= AVD_CTX_MAX_HW_ERRORS && !ctx->broken) { + ctx->broken = true; + dev_err(avd->dev, "stream stopped after %u failed frames; restart streaming to retry\n", + ctx->hw_errors); + } if (ctx->coded_fmt_desc->frame_ops) { if (ctx->coded_fmt_desc->frame_ops->retire) ctx->coded_fmt_desc->frame_ops->retire(ctx, f, state); @@ -81,7 +97,28 @@ static void avd_reset_queue(struct avd_dev *avd, int status) unsigned int i; int ret; + /* + * A stream whose every frame faults (an H0 error storm) used to reset + * the decoder back to back, about once per millisecond, until the + * machine hung. Space recoveries out, and after a storm stop taking + * new frames for a while instead of resetting again. + */ + if (avd->last_recover && + time_before(jiffies, avd->last_recover + msecs_to_jiffies(AVD_RECOVER_GAP_MS))) + msleep(AVD_RECOVER_GAP_MS); + if (!avd->storm_count || time_after(jiffies, avd->storm_start + AVD_STORM_WINDOW)) { + avd->storm_start = jiffies; + avd->storm_count = 0; + } + if (++avd->storm_count > AVD_STORM_MAX) { + dev_err(avd->dev, "%u decoder recoveries in %u s: failing new frames for %u s\n", + avd->storm_count, jiffies_to_msecs(AVD_STORM_WINDOW) / 1000, + jiffies_to_msecs(AVD_COOLDOWN) / 1000); + WRITE_ONCE(avd->cooldown_until, jiffies + AVD_COOLDOWN); + avd->storm_count = 0; + } ret = avd_pm_recover(avd); + avd->last_recover = jiffies ?: 1; WRITE_ONCE(avd->faulted, !!ret); if (ret) dev_err(avd->dev, "queue reset failed: %d\n", ret); @@ -218,9 +255,12 @@ int avd_queue_ready(void *priv) bool ready; spin_lock_irqsave(&avd->queue_lock, flags); + /* + * A faulted or cooling-down decoder still schedules: submission fails + * those frames at once, so clients see errors instead of waiting. + */ ready = !avd->queued_ctx && !READ_ONCE(ctx->draining) && !READ_ONCE(avd->stopping) && - !READ_ONCE(avd->faulted) && avd->inflight < avd->backend->max_inflight(avd); if (ready && ctx->next_dep == AVD_DEP_PREV_DONE) { /* Legacy codecs may hold several pictures in separate banks. */ @@ -254,8 +294,14 @@ static void avd_queue_submit_work(struct work_struct *work) spin_lock_irqsave(&avd->queue_lock, flags); if (avd->queued_ctx == ctx) avd->queued_ctx = NULL; - if (avd->stopping || ctx->draining || avd->faulted) { + if (avd->stopping || ctx->draining || avd->faulted || ctx->broken || + time_before(jiffies, READ_ONCE(avd->cooldown_until))) { + struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + spin_unlock_irqrestore(&avd->queue_lock, flags); + /* Complete the request's controls too, or its fd never signals. */ + if (src && src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, VB2_BUF_STATE_ERROR); goto unlock; diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index 661100c2365b7..2676744d52c3a 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -896,6 +896,9 @@ static int avd_start_streaming(struct vb2_queue *q, unsigned int count) WRITE_ONCE(ctx->draining, false); if (V4L2_TYPE_IS_CAPTURE(q->type)) return 0; + /* A new stream gets a fresh hardware error budget. */ + ctx->hw_errors = 0; + ctx->broken = false; desc = ctx->coded_fmt_desc; if (WARN_ON(!desc)) { diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 4aeda520ab651..a9d092bebd56a 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -248,6 +248,12 @@ struct avd_dev { unsigned int inflight; bool stopping; bool faulted; + /* Recovery pacing: when the last one ran, and a sliding storm count. */ + unsigned long last_recover; + unsigned long storm_start; + unsigned int storm_count; + /* After a recovery storm, fail new frames until this time. */ + unsigned long cooldown_until; u64 fw_generation; const struct avd_backend_ops *backend; void *backend_priv; @@ -301,6 +307,10 @@ struct avd_ctx { struct list_head queue_node; struct work_struct submit_work; bool draining; + /* Consecutive frames that failed in hardware; reset by a good frame. */ + unsigned int hw_errors; + /* Too many hardware errors: fail every frame until streaming restarts. */ + bool broken; bool session_started; u64 session_generation; enum avd_frame_dep next_dep; From e329d379f1e2bf0d30d319715ee3e7599c91418b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 02:37:17 +1100 Subject: [PATCH 27/73] media: apple: avd: accept FFmpeg's HEVC slice layout The HEVC builder computes each slice's coded data address and size from GStreamer's layout: slices tile the OUTPUT buffer as whole NALs, bit_size covers the whole NAL and data_byte_offset is the slice data's offset in it. FFmpeg's V4L2 request backend gives data_byte_offset from the start of the buffer and a bit_size of the slice data only. Every slice then got a wrong address and a size short by its header, the VP raised its error interrupt on every frame, and the decoder was reset once per frame. This is the "H0 error" storm the Neo hit in an FFmpeg-based benchmark. When the slices describe a complete, unambiguous absolute layout, rewrite the run's private copy into the equivalent NAL-relative description. The GStreamer layout and anything ambiguous are left untouched. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-hevc.c | 47 +++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index b1fb5aa63d1f9..b2ce0f3711390 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -1338,6 +1338,51 @@ static int avd_hevc_compute_tiles(struct avd_ctx *ctx, struct avd_hevc_run *run) return 0; } +/* + * The slice builder below expects the GStreamer layout: the slices tile the + * OUTPUT buffer as whole NALs, bit_size covers the whole NAL and + * data_byte_offset is the slice data's offset inside it. FFmpeg's request + * API backend instead gives each slice's data_byte_offset from the start of + * the buffer and a bit_size of the slice data only, so every slice was given + * a wrong address and size and the VP raised its error interrupt on every + * frame. Detect the absolute layout (complete and unambiguous) and rewrite + * the run's private copy into the equivalent NAL-relative description. + */ +static void avd_hevc_normalize_slices(struct avd_hevc_run *run, u32 payload) +{ + struct v4l2_ctrl_hevc_slice_params *sl = + (struct v4l2_ctrl_hevc_slice_params *)run->sl; + u64 nal_end = 0, data_end = 0, prev_end = 0; + bool nal = true, absolute = true; + int i; + + if (!run->num_slices || !payload) + return; + for (i = 0; i < run->num_slices; i++) { + u64 bytes = DIV_ROUND_UP((u64)sl[i].bit_size, 8); + u64 start = sl[i].data_byte_offset; + + if (!bytes || start >= bytes || nal_end + bytes > payload) + nal = false; + nal_end += bytes; + if (!bytes || start < data_end || start + bytes > payload) + absolute = false; + data_end = start + bytes; + } + nal &= nal_end == payload; + absolute &= data_end == payload; + if (!absolute || nal) + return; + for (i = 0; i < run->num_slices; i++) { + u64 bytes = DIV_ROUND_UP((u64)sl[i].bit_size, 8); + u64 start = sl[i].data_byte_offset; + + sl[i].data_byte_offset = start - prev_end; + sl[i].bit_size = (start - prev_end + bytes) * 8; + prev_end = start + bytes; + } +} + static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) { struct v4l2_ctrl *ctrl, *sl, *ep; @@ -1410,6 +1455,8 @@ static int avd_hevc_run(struct avd_ctx *ctx) if (ret) return ret; + avd_hevc_normalize_slices(&run, + vb2_get_plane_payload(&run.base.bufs.src->vb2_buf, 0)); dst = vb2_to_avd_decoded_buf(&run.base.bufs.dst->vb2_buf); update_dec_buf_info(dst, &run.sl[0]); From f158937ee9b00b2f8ef3d47ea3b8d40f49af45e2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 02:41:50 +1100 Subject: [PATCH 28/73] media: apple: avd: include bitfield.h for the instruction macros avd-inst.h builds every instruction word with FIELD_PREP() but relied on an indirect include. The arm64 build gets it through other headers; the UML KUnit build of the avd tests does not and fails. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-inst.h | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/media/platform/apple/avd/avd-inst.h b/drivers/media/platform/apple/avd/avd-inst.h index e9b9ef908b8d5..dcdb205d79315 100644 --- a/drivers/media/platform/apple/avd/avd-inst.h +++ b/drivers/media/platform/apple/avd/avd-inst.h @@ -3,6 +3,7 @@ #ifndef AVD_INST_H_ #define AVD_INST_H_ /* instruction stream things */ +#include #include #include "avd.h" From 11138e494aab9b21def973f28ebdad376f87abc8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 02:44:09 +1100 Subject: [PATCH 29/73] media: apple: avd: check the DART state before logging recovery The recovery trail queried the runtime PM state before the existing inactive-DART check, which the PM KUnit test exercises with no device. Check the DART first and log that case on its own. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-pm.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-pm.c b/drivers/media/platform/apple/avd/avd-pm.c index c65a447f93b00..3876aec76f2ee 100644 --- a/drivers/media/platform/apple/avd/avd-pm.c +++ b/drivers/media/platform/apple/avd/avd-pm.c @@ -295,6 +295,7 @@ int avd_pm_backend_resume(struct avd_dev *avd) int avd_pm_recover(struct avd_dev *avd) { + bool suspended; int ret; /* @@ -302,14 +303,17 @@ int avd_pm_recover(struct avd_dev *avd) * leave a step-by-step trail in the log before each step, so a failure * that takes the machine down still shows how far recovery got. */ - dev_warn(avd->dev, "recover: start (dart %s, runtime %s, irq %s)\n", - avd->pm.dart_active ? "on" : "off", - pm_runtime_suspended(avd->dev) ? "suspended" : "active", - avd->pm.irq_enabled ? "on" : "off"); - if (!avd->pm.dart_active) + if (!avd->pm.dart_active) { + dev_err(avd->dev, "recover: decoder DART inactive, not touching it\n"); return -EHOSTDOWN; + } + /* The PM KUnit fixture has no struct device. */ + suspended = avd->dev && pm_runtime_suspended(avd->dev); + dev_warn(avd->dev, "recover: start (runtime %s, irq %s)\n", + suspended ? "suspended" : "active", + avd->pm.irq_enabled ? "on" : "off"); /* Every caller holds a runtime PM reference; anything else is a bug. */ - if (WARN_ONCE(pm_runtime_suspended(avd->dev), + if (WARN_ONCE(suspended, "avd: recovery with the decoder powered off\n")) return -EHOSTDOWN; avd_pm_disable_irq(avd); From 01de6ce703a14fc587a7b4526eb34f205ece61b7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 03:29:03 +1100 Subject: [PATCH 30/73] media: apple: avd: dispatch m2m jobs in order instead of reserving one context job_ready() admitted at most one context at a time (queued_ctx) and only while a slot was free. Contexts that queued work while another was admitted were rejected, and were retried only from the queue worker, which walks the context list in order: the first context won every time and later ones could wait for seconds. With four streams the starved stream's OUTPUT buffer was not returned within FFmpeg's two-second poll ("Failed waiting on OUTPUT buffer") and the run failed, with no decoder timeout or error. Drop the reservation and the capacity condition from job_ready() so every context with work enters the m2m job queue, which runs jobs one at a time in arrival order. A job that finds no free slot or no transport capacity is kept as m2m's running job and restarted by the queue worker once a frame retires; draining its context finishes it. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../media/platform/apple/avd/avd-queue-test.c | 7 +-- drivers/media/platform/apple/avd/avd-queue.c | 56 ++++++++++++++----- drivers/media/platform/apple/avd/avd.h | 6 +- 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-queue-test.c b/drivers/media/platform/apple/avd/avd-queue-test.c index eec4761f850e9..1efae38789b57 100644 --- a/drivers/media/platform/apple/avd/avd-queue-test.c +++ b/drivers/media/platform/apple/avd/avd-queue-test.c @@ -172,12 +172,11 @@ static void avd_dispatch_reservation(struct kunit *test) avd->backend = &backend; ctx[0].dev = avd; ctx[1].dev = avd; + /* Every context with work is ready; capacity is checked at dispatch. */ KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[0]), 1); - /* The second context must not reserve the same observed capacity. */ - KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 0); - avd->queued_ctx = NULL; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); avd->inflight = AVD_MAX_FRAMES; - KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 0); + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); avd->inflight = 1; avd->frames[0].state = AVD_FRAME_SUBMITTED; avd->frames[0].ctx = &ctx[1]; diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index 45759a0593792..226de0e4d7574 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -210,6 +210,14 @@ static void avd_queue_worker(struct work_struct *work) } } avd_retire(avd); + /* Restart a job that found no free slot. */ + spin_lock_irqsave(&avd->queue_lock, flags); + ctx = avd->inflight < avd->backend->max_inflight(avd) ? avd->deferred_ctx : NULL; + if (ctx) + avd->deferred_ctx = NULL; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (ctx) + schedule_work(&ctx->submit_work); /* Freed slots/dependencies must wake contexts whose job_ready was false. */ list_for_each_entry(ctx, &avd->contexts, queue_node) v4l2_m2m_try_schedule(ctx->fh.m2m_ctx); @@ -259,9 +267,14 @@ int avd_queue_ready(void *priv) * A faulted or cooling-down decoder still schedules: submission fails * those frames at once, so clients see errors instead of waiting. */ - ready = !avd->queued_ctx && !READ_ONCE(ctx->draining) && - !READ_ONCE(avd->stopping) && - avd->inflight < avd->backend->max_inflight(avd); + /* + * Slot capacity is not a readiness condition: every context with work + * enters the m2m job queue, which runs them in order, and a job that + * finds no free slot is deferred until one retires. Gating readiness on + * one reserved context let the first context in the list win every + * time and starved the others (multi-stream decodes timing out). + */ + ready = !READ_ONCE(ctx->draining) && !READ_ONCE(avd->stopping); if (ready && ctx->next_dep == AVD_DEP_PREV_DONE) { /* Legacy codecs may hold several pictures in separate banks. */ unsigned int limit = ctx->coded_fmt_desc && !ctx->coded_fmt_desc->frame_ops ? @@ -273,9 +286,6 @@ int avd_queue_ready(void *priv) held++; ready = held < limit; } - /* Prevent several queued m2m contexts from observing the same free slot. */ - if (ready) - avd->queued_ctx = ctx; spin_unlock_irqrestore(&avd->queue_lock, flags); return ready; } @@ -292,8 +302,8 @@ static void avd_queue_submit_work(struct work_struct *work) mutex_lock(&avd->queue_mutex); spin_lock_irqsave(&avd->queue_lock, flags); - if (avd->queued_ctx == ctx) - avd->queued_ctx = NULL; + if (avd->deferred_ctx == ctx) + avd->deferred_ctx = NULL; if (avd->stopping || ctx->draining || avd->faulted || ctx->broken || time_before(jiffies, READ_ONCE(avd->cooldown_until))) { struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); @@ -306,13 +316,22 @@ static void avd_queue_submit_work(struct work_struct *work) VB2_BUF_STATE_ERROR); goto unlock; } - for (i = 0; i < AVD_MAX_FRAMES; i++) { - if (avd->frames[i].state == AVD_FRAME_FREE) { - f = &avd->frames[i]; - break; + if (avd->inflight < avd->backend->max_inflight(avd)) { + for (i = 0; i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state == AVD_FRAME_FREE) { + f = &avd->frames[i]; + break; + } } } - ret = f ? avd_slot_reserve(f, ctx, ++avd->frame_seq) : -EBUSY; + if (!f) { + /* Keep the m2m job; the worker restarts it when a slot frees. */ + avd->deferred_ctx = ctx; + spin_unlock_irqrestore(&avd->queue_lock, flags); + mod_delayed_work(system_wq, &avd->queue_work, 0); + goto unlock; + } + ret = avd_slot_reserve(f, ctx, ++avd->frame_seq); if (!ret) avd->inflight++; spin_unlock_irqrestore(&avd->queue_lock, flags); @@ -456,6 +475,7 @@ void avd_queue_drain(struct avd_ctx *ctx) struct avd_dev *avd = ctx->dev; unsigned long flags; unsigned int i; + bool finish_deferred = false; int ret; /* Session stop/reset accesses registers even when there are no frames. */ @@ -464,9 +484,15 @@ void avd_queue_drain(struct avd_ctx *ctx) mutex_lock(&avd->queue_mutex); spin_lock_irqsave(&avd->queue_lock, flags); WRITE_ONCE(ctx->draining, true); - if (avd->queued_ctx == ctx) - avd->queued_ctx = NULL; + if (avd->deferred_ctx == ctx) { + avd->deferred_ctx = NULL; + finish_deferred = true; + } spin_unlock_irqrestore(&avd->queue_lock, flags); + /* A deferred job is still m2m's running job: end it so abort can wait. */ + if (finish_deferred) + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); for (i = 0; ret >= 0 && i < AVD_MAX_FRAMES; i++) { if (avd->frames[i].state != AVD_FRAME_FREE && avd->frames[i].ctx == ctx) { if (avd->backend->abort) diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index a9d092bebd56a..be8579e4b4981 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -241,7 +241,11 @@ struct avd_dev { struct delayed_work queue_work; struct list_head contexts; /* One m2m dispatch reservation; never identifies firmware-owned work. */ - struct avd_ctx *queued_ctx; + /* + * m2m runs one device_run at a time until job_finish; a job that finds + * no free slot waits here and is restarted when a frame retires. + */ + struct avd_ctx *deferred_ctx; struct avd_frame frames[AVD_MAX_FRAMES]; struct avd_buf *frame_fp[AVD_MAX_FRAMES]; u64 frame_seq; From 01e8ce80a4f46d8f2e97320490e86757583eab9e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 03:40:57 +1100 Subject: [PATCH 31/73] media: apple: avd: tolerate backends without max_inflight in the queue The deferred-dispatch restart in the queue worker called the backend's max_inflight() unconditionally. The PM KUnit fixtures use backends without a transport and crashed there. Treat a missing hook as one frame. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-queue.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index 226de0e4d7574..f05d83cc7219f 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -21,6 +21,12 @@ #define AVD_RETRY_DELAY msecs_to_jiffies(1) #define AVD_PROGRAM_SIZE SZ_2M +/* PM test fixtures use backends without a transport. */ +static unsigned int avd_max_inflight(struct avd_dev *avd) +{ + return avd->backend->max_inflight ? avd->backend->max_inflight(avd) : 1; +} + bool avd_backend_is_apple(const struct avd_dev *avd) { return avd->backend == &avd_apple_backend; @@ -212,7 +218,7 @@ static void avd_queue_worker(struct work_struct *work) avd_retire(avd); /* Restart a job that found no free slot. */ spin_lock_irqsave(&avd->queue_lock, flags); - ctx = avd->inflight < avd->backend->max_inflight(avd) ? avd->deferred_ctx : NULL; + ctx = avd->inflight < avd_max_inflight(avd) ? avd->deferred_ctx : NULL; if (ctx) avd->deferred_ctx = NULL; spin_unlock_irqrestore(&avd->queue_lock, flags); @@ -316,7 +322,7 @@ static void avd_queue_submit_work(struct work_struct *work) VB2_BUF_STATE_ERROR); goto unlock; } - if (avd->inflight < avd->backend->max_inflight(avd)) { + if (avd->inflight < avd_max_inflight(avd)) { for (i = 0; i < AVD_MAX_FRAMES; i++) { if (avd->frames[i].state == AVD_FRAME_FREE) { f = &avd->frames[i]; From 9043840b67a0ba05ab2c3b980a7e3a3eb8a4a565 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 04:51:50 +1100 Subject: [PATCH 32/73] media: apple: avd: harden decoder access against machine-wide hangs The J616s hung hard three times during decode benchmarks, each time after frame timeouts and decoder resets, once beyond what the hardware watchdog could recover. A stuck bus access, a reset that cuts DMA in flight, or DMA into memory the driver already gave back can do that. Close each of those paths: - Recovery stops the CM3 firmware, then blocks the decoder's DMA by attaching the empty IOMMU domain and lets in-flight transfers drain into DART faults, and only then asserts the pmgr reset. The domain is restored after the reset and checked before the firmware boots. - A third recovery within a minute, or any failed recovery, retires the decoder: every later frame fails at submission without touching the hardware until the module is reloaded. A stream is stopped at its first frame that fails in hardware. - The legacy transport never writes the VP FIFO register without room in its queue: it checks the level before every word (the old code only looked between segments, and programs exceed the 128-word queue), waits up to 2 ms for room, and otherwise leaves the frame to the watchdog. - A PP DONE completes the active legacy frame only after its VP finished and its PP was kicked. A surplus PP DONE of a multi-kick picture no longer completes the next frame while the engine is still writing it. - The interrupt handler's hardware access lock is now a mutex, so no decoder register is touched with interrupts off. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../platform/apple/avd/avd-backend-open.c | 51 ++++++++------ drivers/media/platform/apple/avd/avd-pm.c | 51 ++++++++------ drivers/media/platform/apple/avd/avd-pm.h | 7 +- drivers/media/platform/apple/avd/avd-queue.c | 67 ++++++++++--------- drivers/media/platform/apple/avd/avd.h | 5 +- 5 files changed, 102 insertions(+), 79 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 9680dc4ef64e1..9781f5647cca1 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -76,6 +76,7 @@ static int avd_open_submit(struct avd_dev *avd, struct avd_frame *f) u32 *words = f->prog->cpu; unsigned int i; void __iomem *reg; + u32 max; if (avd->compat_active) return -EBUSY; @@ -95,34 +96,30 @@ static int avd_open_submit(struct avd_dev *avd, struct avd_frame *f) reg = avd->ctrl + avd->variant->vp_slot_offset + vp * 4; avd->compat_active = f; avd->compat_ctx = ctx; + avd->compat_pp_kicked = false; + max = readl(avd->ctrl + avd->variant->submit_queue_max_offset + vp * 4); for (i = 0; i < f->prog_words; i++) { - /* Preserve the old firmware's segment FIFO pacing in this shim. */ - if (i && (words[i] & GENMASK(31, 24)) == AVD_OP_EXEC && - (!(avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) || - codec == AVD_CODEC_HEVC)) { - u32 max = readl(avd->ctrl + avd->variant->submit_queue_max_offset + vp * 4); + /* + * Never write the VP FIFO register without room in its queue: a + * write the block cannot accept may stall the bus. The old driver + * only looked between segments, and programs exceed the 128-word + * queue. Wait a bounded time for room, then give the frame to + * the watchdog instead of writing. + */ + if (max) { u32 cur = readl(avd->ctrl + avd->variant->submit_queue_status_offset + vp * 4); - if (cur >= max) { - /* Already emitted words: only reset may release this slot. */ + if (cur >= max && + readl_relaxed_poll_timeout(avd->ctrl + + avd->variant->submit_queue_status_offset + vp * 4, + cur, cur < max, 5, 2000)) { + dev_err_ratelimited(avd->dev, + "VP%u queue full (%u/%u), not writing\n", + vp, cur, max); f->deadline = jiffies; return 0; } - if (cur >= max / 2) { - /* - * Wait for the VP to drain below half full, but - * only as long as it takes. The old fixed sleep - * (100-150 us on v5 pacing parts, 500-650 us - * elsewhere) stays the upper bound. - */ - u32 wait_us = avd->variant->quirks & - AVD_QUIRK_V5_QUEUE_PACING ? 150 : 650; - - readl_relaxed_poll_timeout(avd->ctrl + - avd->variant->submit_queue_status_offset + vp * 4, - cur, cur < max / 2, 10, wait_us); - } } writel(words[i], reg); } @@ -149,12 +146,24 @@ static irqreturn_t avd_open_irq(struct avd_dev *avd) return IRQ_HANDLED; } if (status & 0x1000) { + /* + * Only the PP this driver kicked for the active frame can finish + * it. A surplus PP DONE of a multi-kick picture used to complete + * the next frame early, so userspace reused its buffers while the + * engine was still writing them. + */ + if (!avd->compat_pp_kicked) { + dev_dbg_ratelimited(avd->dev, "PP DONE before the frame's VP, ignored\n"); + return IRQ_HANDLED; + } avd->compat_active = NULL; + avd->compat_pp_kicked = false; avd_frame_complete(avd, f->tag, 0); } else if (status & 0x100) { /* Old firmware only: no request tag and no firmware-side PP kick. */ if (f->ctx->coded_fmt_desc->ops->submit) f->ctx->coded_fmt_desc->ops->submit(f->ctx); + avd->compat_pp_kicked = true; } else { dev_err_ratelimited(avd->dev, "open firmware error %#x\n", status); f->deadline = jiffies; diff --git a/drivers/media/platform/apple/avd/avd-pm.c b/drivers/media/platform/apple/avd/avd-pm.c index 3876aec76f2ee..609e47ef98eaf 100644 --- a/drivers/media/platform/apple/avd/avd-pm.c +++ b/drivers/media/platform/apple/avd/avd-pm.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT +#include #include #include #include @@ -16,12 +17,10 @@ void avd_pm_enable_irq(struct avd_dev *avd) { - unsigned long flags; - if (avd->pm.irq >= 0 && !avd->pm.irq_enabled) { - spin_lock_irqsave(&avd->pm.irq_lock, flags); + mutex_lock(&avd->pm.hw_lock); avd->pm.hw_on = true; - spin_unlock_irqrestore(&avd->pm.irq_lock, flags); + mutex_unlock(&avd->pm.hw_lock); enable_irq(avd->pm.irq); avd->pm.irq_enabled = true; } @@ -30,19 +29,17 @@ void avd_pm_enable_irq(struct avd_dev *avd) /* * Callers may hold queue_mutex, which the threaded handler takes first, so * never wait for the handler here (disable_irq() would deadlock). Instead - * close hardware access under irq_lock: a handler already touching the + * close hardware access under hw_lock: a handler already touching the * decoder finishes before this returns, and any later one sees hw_on clear * or a new irq_gen and leaves the hardware alone. */ static void avd_pm_disable_irq(struct avd_dev *avd) { - unsigned long flags; - if (avd->pm.irq >= 0 && avd->pm.irq_enabled) { - spin_lock_irqsave(&avd->pm.irq_lock, flags); + mutex_lock(&avd->pm.hw_lock); avd->pm.hw_on = false; avd->pm.irq_gen++; - spin_unlock_irqrestore(&avd->pm.irq_lock, flags); + mutex_unlock(&avd->pm.hw_lock); disable_irq_nosync(avd->pm.irq); avd->pm.irq_enabled = false; } @@ -52,12 +49,12 @@ static void avd_pm_disable_irq(struct avd_dev *avd) irqreturn_t avd_pm_irq(struct avd_dev *avd, unsigned int gen) { irqreturn_t ret = IRQ_HANDLED; - unsigned long flags; - spin_lock_irqsave(&avd->pm.irq_lock, flags); + /* Caller holds queue_mutex; hw_lock holders never wait for it. */ + mutex_lock(&avd->pm.hw_lock); if (avd->pm.hw_on && gen == avd->pm.irq_gen) ret = avd->backend->irq(avd); - spin_unlock_irqrestore(&avd->pm.irq_lock, flags); + mutex_unlock(&avd->pm.hw_lock); return ret; } @@ -194,7 +191,7 @@ void avd_pm_init(struct avd_dev *avd, const struct avd_backend_ops *backend, avd->pm.dart_active = false; avd->pm.irq = -1; avd->pm.irq_enabled = false; - spin_lock_init(&avd->pm.irq_lock); + mutex_init(&avd->pm.hw_lock); avd->pm.hw_on = false; avd->pm.irq_gen = 0; avd->backend = backend; @@ -319,21 +316,31 @@ int avd_pm_recover(struct avd_dev *avd) avd_pm_disable_irq(avd); dev_warn(avd->dev, "recover: stop firmware\n"); avd->backend->shutdown(avd); - dev_warn(avd->dev, "recover: reset\n"); - ret = reset_control_reset(avd->rstc); - if (ret) { - dev_err(avd->dev, "recover: reset failed: %d\n", ret); - return ret; - } + /* + * Block the decoder's DMA at its DART before resetting the engine: with + * the empty domain attached, any transfer still in flight faults at the + * DART instead of reaching memory that may already be reused, and the + * pmgr reset (which does not wait for the engine to go idle on every + * SoC) never cuts a transfer that is half way through the DART. Then + * give outstanding transfers time to drain into faults. + */ if (avd->domain) { if (!avd->empty_domain) return -ENODEV; - dev_warn(avd->dev, "recover: DART flush\n"); + dev_warn(avd->dev, "recover: block DMA\n"); ret = iommu_attach_device(avd->empty_domain, avd->dev); if (ret) { dev_err(avd->dev, "recover: DART empty attach failed: %d\n", ret); return ret; } + usleep_range(5000, 6000); + } + dev_warn(avd->dev, "recover: reset\n"); + ret = reset_control_reset(avd->rstc); + if (ret) + dev_err(avd->dev, "recover: reset failed: %d\n", ret); + if (avd->domain) { + dev_warn(avd->dev, "recover: restore DMA\n"); iommu_detach_device(avd->empty_domain, avd->dev); /* Do not reboot against an empty/blocked domain after a failed restore. */ if (iommu_get_domain_for_dev(avd->dev) != avd->domain) { @@ -341,6 +348,8 @@ int avd_pm_recover(struct avd_dev *avd) return -EIO; } } + if (ret) + return ret; dev_warn(avd->dev, "recover: boot firmware\n"); ret = avd_pm_backend_resume(avd); if (ret) { @@ -373,8 +382,6 @@ int avd_pm_power_on(struct avd_dev *avd) return ret; } avd_pm_enable_irq(avd); - /* A clean boot after power-up ends a failed recovery's fault state. */ - WRITE_ONCE(avd->faulted, false); if (avd->pm.queue->resubmit) avd->pm.queue->resubmit(avd); diff --git a/drivers/media/platform/apple/avd/avd-pm.h b/drivers/media/platform/apple/avd/avd-pm.h index 5fe55f595beb2..b1146a37fd4da 100644 --- a/drivers/media/platform/apple/avd/avd-pm.h +++ b/drivers/media/platform/apple/avd/avd-pm.h @@ -32,12 +32,13 @@ struct avd_pm { int irq; bool irq_enabled; /* - * irq_lock guards hw_on and irq_gen. The threaded handler touches the - * decoder only under irq_lock with hw_on set and an unchanged irq_gen, + * hw_lock guards hw_on and irq_gen. The threaded handler touches the + * decoder only under hw_lock with hw_on set and an unchanged irq_gen, * so turning the IRQ off never has to wait for the handler thread * (which may itself be waiting for queue_mutex). */ - spinlock_t irq_lock; + /* A mutex: decoder MMIO is never done with interrupts off. */ + struct mutex hw_lock; bool hw_on; unsigned int irq_gen; void __iomem *piodma; diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index f05d83cc7219f..7cb0221c07bda 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -10,14 +10,13 @@ #include "avd-queue-internal.h" #define AVD_FRAME_TIMEOUT msecs_to_jiffies(2000) -/* A stream that fails this many frames in a row in hardware is stopped. */ -#define AVD_CTX_MAX_HW_ERRORS 3 +/* A stream is stopped at its first frame that fails in hardware. */ +#define AVD_CTX_MAX_HW_ERRORS 1 /* Never reset the decoder more often than this. */ #define AVD_RECOVER_GAP_MS 20 -/* More than AVD_STORM_MAX recoveries within AVD_STORM_WINDOW: cool down. */ -#define AVD_STORM_MAX 8 -#define AVD_STORM_WINDOW msecs_to_jiffies(10000) -#define AVD_COOLDOWN msecs_to_jiffies(10000) +/* A third recovery within AVD_STORM_WINDOW is refused: the decoder is retired. */ +#define AVD_STORM_MAX 2 +#define AVD_STORM_WINDOW msecs_to_jiffies(60000) #define AVD_RETRY_DELAY msecs_to_jiffies(1) #define AVD_PROGRAM_SIZE SZ_2M @@ -104,30 +103,37 @@ static void avd_reset_queue(struct avd_dev *avd, int status) int ret; /* - * A stream whose every frame faults (an H0 error storm) used to reset - * the decoder back to back, about once per millisecond, until the - * machine hung. Space recoveries out, and after a storm stop taking - * new frames for a while instead of resetting again. + * Every decoder reset is a risk to the whole machine (three hard hangs + * on the J616s followed timeouts and resets), so recover sparingly: + * at least AVD_RECOVER_GAP_MS apart, at most AVD_STORM_MAX times in + * AVD_STORM_WINDOW, and never again after a recovery fails. A retired + * decoder fails every frame without touching the hardware until the + * module is reloaded. */ - if (avd->last_recover && - time_before(jiffies, avd->last_recover + msecs_to_jiffies(AVD_RECOVER_GAP_MS))) - msleep(AVD_RECOVER_GAP_MS); - if (!avd->storm_count || time_after(jiffies, avd->storm_start + AVD_STORM_WINDOW)) { - avd->storm_start = jiffies; - avd->storm_count = 0; - } - if (++avd->storm_count > AVD_STORM_MAX) { - dev_err(avd->dev, "%u decoder recoveries in %u s: failing new frames for %u s\n", - avd->storm_count, jiffies_to_msecs(AVD_STORM_WINDOW) / 1000, - jiffies_to_msecs(AVD_COOLDOWN) / 1000); - WRITE_ONCE(avd->cooldown_until, jiffies + AVD_COOLDOWN); - avd->storm_count = 0; + if (avd->faulted) { + ret = -ENODEV; + } else { + if (!avd->storm_count || + time_after(jiffies, avd->storm_start + AVD_STORM_WINDOW)) { + avd->storm_start = jiffies; + avd->storm_count = 0; + } + if (++avd->storm_count > AVD_STORM_MAX) { + ret = -EBUSY; + } else { + if (avd->last_recover && + time_before(jiffies, avd->last_recover + + msecs_to_jiffies(AVD_RECOVER_GAP_MS))) + msleep(AVD_RECOVER_GAP_MS); + ret = avd_pm_recover(avd); + avd->last_recover = jiffies ?: 1; + } + if (ret) { + WRITE_ONCE(avd->faulted, true); + dev_err(avd->dev, "decoder retired after %s (%d): reload the module to use it again\n", + ret == -EBUSY ? "repeated resets" : "a failed reset", ret); + } } - ret = avd_pm_recover(avd); - avd->last_recover = jiffies ?: 1; - WRITE_ONCE(avd->faulted, !!ret); - if (ret) - dev_err(avd->dev, "queue reset failed: %d\n", ret); avd->compat_active = NULL; list_for_each_entry(ctx, &avd->contexts, queue_node) ctx->session_started = false; @@ -270,7 +276,7 @@ int avd_queue_ready(void *priv) spin_lock_irqsave(&avd->queue_lock, flags); /* - * A faulted or cooling-down decoder still schedules: submission fails + * A retired decoder still schedules: submission fails * those frames at once, so clients see errors instead of waiting. */ /* @@ -310,8 +316,7 @@ static void avd_queue_submit_work(struct work_struct *work) spin_lock_irqsave(&avd->queue_lock, flags); if (avd->deferred_ctx == ctx) avd->deferred_ctx = NULL; - if (avd->stopping || ctx->draining || avd->faulted || ctx->broken || - time_before(jiffies, READ_ONCE(avd->cooldown_until))) { + if (avd->stopping || ctx->draining || READ_ONCE(avd->faulted) || ctx->broken) { struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); spin_unlock_irqrestore(&avd->queue_lock, flags); diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index be8579e4b4981..878a557996a59 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -251,13 +251,12 @@ struct avd_dev { u64 frame_seq; unsigned int inflight; bool stopping; + /* Retired after a failed or too frequent recovery, until module reload. */ bool faulted; /* Recovery pacing: when the last one ran, and a sliding storm count. */ unsigned long last_recover; unsigned long storm_start; unsigned int storm_count; - /* After a recovery storm, fail new frames until this time. */ - unsigned long cooldown_until; u64 fw_generation; const struct avd_backend_ops *backend; void *backend_priv; @@ -266,6 +265,8 @@ struct avd_dev { /* Open firmware queue ABI 2 state, NULL when the node has no SRAM. */ struct avd_qfw *qfw; struct avd_frame *compat_active; + /* The active legacy frame's VP finished and its PP was kicked. */ + bool compat_pp_kicked; /* Context of the last legacy submission, for VP DONE without a frame. */ struct avd_ctx *compat_ctx; bool compat_held; From 68e1aeeed88b3d32b443bda05d4ad48bf53a91ff Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 20:18:46 +1100 Subject: [PATCH 33/73] accel/ane: restart owned H14 firmware after a command timeout Fence commands and buffer creation, stop only the Linux-owned ASC CPU, and power-cycle every domain through the same runtime-PM transaction. Verify the physical PMGR states and repeat the probe access gates before restoring the cached pristine image and the existing fenced boot flow. Require READY/DONE, ChMan validation, host ACK and CONFIG_GET before accepting work again. Unsupported ownership profiles remain quarantined. Retain metadata for closed firmware-visible BOs and old boot/MALLOC arenas. Reclaim old resources only after validated restart; charge retired arenas to the existing limits and bound failed-boot retention. Release the old CPU pin only after quiescence and acquire its replacement before any boot write. The existing domain-link lifetime pin remains. Use opaque kernel program/process IDs, independent of reused firmware wire IDs. Invalidate the cache across restart without resetting the allocator, so clients reload rather than executing a stale identifier. Add KUnit non-reuse/exhaustion checks and matching reset diagnostics. Carried onto aurora-wip 9ed109df. The conflicts were the include list, the client struct and the wire IDs in the procedure call. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 64 ++- drivers/accel/ane/Kconfig | 5 +- drivers/accel/ane/Makefile | 2 +- drivers/accel/ane/ane_fw_ids.h | 25 ++ drivers/accel/ane/ane_fw_ids_test.c | 59 +++ drivers/accel/ane/ane_t6021.h | 10 +- drivers/accel/ane/ane_t6021_boot.c | 23 +- drivers/accel/ane/ane_t6021_boot.h | 4 +- drivers/accel/ane/ane_t6021_fwload.c | 49 ++- drivers/accel/ane/ane_t6021_rtclient_main.c | 462 ++++++++++++++++++-- include/uapi/drm/ane_accel.h | 12 +- 11 files changed, 647 insertions(+), 68 deletions(-) create mode 100644 drivers/accel/ane/ane_fw_ids.h create mode 100644 drivers/accel/ane/ane_fw_ids_test.c diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index c644b0286078d..793aef9d346b8 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -332,24 +332,46 @@ opening a device. H14 recovery lifecycle ---------------------- -The firmware path (``ane_t6021.ko``) uses one bounded deadline for the command -acknowledgment and its finish event. H14 engine reset is not enabled by this -change. Reusing only the H13 task-manager re-arm would leave the firmware and -its program cache invalid. The corresponding recovery implementation needs to: - -* Serialize firmware commands and stop the RX worker before releasing the - consumer's runtime-PM reference and synchronously suspending its suppliers. -* Verify power-off for the CPU and engine partitions through their providers - before dropping boot pins or reclaiming firmware-visible allocations. -* Retain metadata for every firmware-owned BO and boot allocation, including - objects whose last userspace reference disappeared. The current release - path discards some of this metadata while retaining the backing memory. -* Restore the validated firmware image and run the existing fenced probe boot - sequence, including channel-table validation, host acknowledgment and - CONFIG_GET. A failed boot remains quarantined with its DMA backing held. -* Invalidate the cached program/process IDs across the restart and prevent a - stale userspace ID from aliasing a newly allocated firmware ID. Clients must - reload their programs before resubmitting. - -Those ownership and identifier changes require their own H14 reset test. -Quarantine must not be cleared solely because a power transition returned zero. +The firmware path (``ane_t6021.ko``) gives the command acknowledgment and +finish event one bounded deadline. A failed exchange quarantines submissions +and attempts one reset under the firmware lock. Operator retry uses the same +``wedged`` and ``reset`` attributes as H13. + +The supported reset profile is the Linux-owned 13.5 legacy firmware path: +the complete SoC power-domain binding, an owned staged image, and the fenced +boot sequence. Borrowed firmware, RTKit mode, later generations and incomplete +initial boots remain fenced. A second H14 binding is refused because the +firmware cache and BO pool have one owner. + +Recovery clears only the owned ASC CPU's RUN bit and waits for STOPPED before +suspending the consumer and suppliers through the shared PM transaction. This +prevents firmware power-state writes from racing provider shutdown. It then +verifies every domain and the CPU ACTUAL field off, resumes, repeats the probe +power/PWGATE gates, and requires the CPU to be stopped before changing its +image or boot state. It restores a cached, validated pristine image into the +existing mappings; recovery does not fetch a replacement file or rebuild the +entry aliases. The existing boot helper performs the READY/DONE handshake, +then recovery validates ChMan, writes the host acknowledgment and requires a +valid CONFIG_GET response. An enabled firmware performance property is +restored through its existing command. + +Old boot pools, firmware MALLOC buffers and closed firmware-visible BOs keep +their metadata and backing until the new boot is validated. Failed resets +retain those records and any CPU lifetime pin needed by a running or uncertain +CPU. Retired MALLOC arenas remain charged against the existing byte and record +limits; at most two failed-boot arenas can accumulate. A successful restart +reclaims retired arenas and closed BOs. Live old BO handles and mappings remain +client-owned memory and can be reused for loading a program again. + +Program and process IDs are opaque kernel identifiers. The allocator survives +firmware restarts and refuses exhaustion instead of wrapping. Reset invalidates +the program cache, so an old ID cannot alias a reused firmware wire ID. Clients +must reload their program and create its process before retrying. The +``ane-fw-ids`` KUnit suite checks non-reuse and exhaustion boundaries. + +H14 keeps its existing domain-link lifetime reference and a reference for the +currently running firmware CPU. Recovery releases the old CPU reference only +after verified quiescence and acquires a new one before the first boot write; +it does not accumulate one reference per failure. This change does not provide +normal H14 module unload while firmware is active. H14 restart still requires +silicon qualification; the H13 hang fixture must not be submitted through ABI 2. diff --git a/drivers/accel/ane/Kconfig b/drivers/accel/ane/Kconfig index 2e804e63199ce..19bc539ca1a14 100644 --- a/drivers/accel/ane/Kconfig +++ b/drivers/accel/ane/Kconfig @@ -57,13 +57,14 @@ config DRM_ACCEL_ANE_FW_KUNIT_TEST If unsure, say N. config DRM_ACCEL_ANE_PM_KUNIT_TEST - tristate "KUnit tests for ANE runtime-PM recovery" if !KUNIT_ALL_TESTS + tristate "KUnit tests for ANE recovery" if !KUNIT_ALL_TESTS depends on DRM_ACCEL && KUNIT && PM default KUNIT_ALL_TESTS help Exercise supplier suspension, power-off verification failure and runtime resume errors against the runtime-PM and device-link core. No ANE hardware is required. These tests can run independently of - the ANE driver on non-Apple platforms. + the ANE driver on non-Apple platforms. Also test that firmware + restarts cannot reuse a stale userspace program identifier. If unsure, say N. diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile index 1d830b9af9235..aa80d67b937c9 100644 --- a/drivers/accel/ane/Makefile +++ b/drivers/accel/ane/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_DRM_ACCEL_ANE) += ane.o ane_t6021.o -obj-$(CONFIG_DRM_ACCEL_ANE_PM_KUNIT_TEST) += ane_pm_test.o +obj-$(CONFIG_DRM_ACCEL_ANE_PM_KUNIT_TEST) += ane_pm_test.o ane_fw_ids_test.o obj-$(CONFIG_DRM_ACCEL_ANE_H16_VALIDATION) += ane_h16_hwx.o obj-$(CONFIG_DRM_ACCEL_ANE_KUNIT_TEST) += ane_drv_test.o obj-$(CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST) += ane_fw_platform_test.o \ diff --git a/drivers/accel/ane/ane_fw_ids.h b/drivers/accel/ane/ane_fw_ids.h new file mode 100644 index 0000000000000..9c8fa99d74bd8 --- /dev/null +++ b/drivers/accel/ane/ane_fw_ids.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_FW_IDS_H__ +#define __ANE_FW_IDS_H__ + +#include +#include +#include + +/* Protected by the firmware lock; never reset when firmware restarts. */ +struct ane_fw_ids { + u64 next; +}; + +static inline int ane_fw_id_alloc(struct ane_fw_ids *ids, u32 *id) +{ + if (!ids->next) + ids->next = 1; + /* Keep zero and the protocol's U32_MAX sentinel out of the namespace. */ + if (ids->next >= U32_MAX) + return -ENOSPC; + *id = ids->next++; + return 0; +} + +#endif /* __ANE_FW_IDS_H__ */ diff --git a/drivers/accel/ane/ane_fw_ids_test.c b/drivers/accel/ane/ane_fw_ids_test.c new file mode 100644 index 0000000000000..57635daf46adf --- /dev/null +++ b/drivers/accel/ane/ane_fw_ids_test.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include + +#include "ane_fw_ids.h" + +static void ane_fw_ids_restart_test(struct kunit *test) +{ + struct ane_fw_ids ids = {}; + u32 old, fresh; + + KUNIT_ASSERT_EQ(test, ane_fw_id_alloc(&ids, &old), 0); + /* Firmware may reuse wire ID zero after reset; the kernel namespace + * is deliberately independent and survives cache invalidation. + */ + KUNIT_ASSERT_EQ(test, ane_fw_id_alloc(&ids, &fresh), 0); + KUNIT_EXPECT_NE(test, old, fresh); + KUNIT_EXPECT_NE(test, old, 0U); + KUNIT_EXPECT_NE(test, fresh, U32_MAX); +} + +static void ane_fw_ids_exhaustion_test(struct kunit *test) +{ + struct ane_fw_ids ids = { .next = U32_MAX - 1ULL }; + u32 key = 0; + + KUNIT_ASSERT_EQ(test, ane_fw_id_alloc(&ids, &key), 0); + KUNIT_EXPECT_EQ(test, key, U32_MAX - 1); + key = 123; + KUNIT_EXPECT_EQ(test, ane_fw_id_alloc(&ids, &key), -ENOSPC); + KUNIT_EXPECT_EQ(test, key, 123U); + KUNIT_EXPECT_EQ(test, ane_fw_id_alloc(&ids, &key), -ENOSPC); +} + +static void ane_fw_ids_overflow_test(struct kunit *test) +{ + struct ane_fw_ids ids = { .next = U64_MAX }; + u32 key = 123; + + KUNIT_EXPECT_EQ(test, ane_fw_id_alloc(&ids, &key), -ENOSPC); + KUNIT_EXPECT_EQ(test, key, 123U); + KUNIT_EXPECT_EQ(test, ids.next, U64_MAX); +} + +static struct kunit_case ane_fw_ids_cases[] = { + KUNIT_CASE(ane_fw_ids_restart_test), + KUNIT_CASE(ane_fw_ids_exhaustion_test), + KUNIT_CASE(ane_fw_ids_overflow_test), + {} +}; + +static struct kunit_suite ane_fw_ids_suite = { + .name = "ane-fw-ids", + .test_cases = ane_fw_ids_cases, +}; + +kunit_test_suite(ane_fw_ids_suite); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("ANE firmware recovery identifier tests"); diff --git a/drivers/accel/ane/ane_t6021.h b/drivers/accel/ane/ane_t6021.h index bf6d0f3e44202..105d4e8ed34bb 100644 --- a/drivers/accel/ane/ane_t6021.h +++ b/drivers/accel/ane/ane_t6021.h @@ -98,7 +98,8 @@ enum { * The IOP has not been started and no firmware image is programmed, * so nothing on this transport can answer the host until it is. */ -#define ANE_ASC_CPU_CONTROL 0x1400044 /* RUN = BIT(4) */ +#define ANE_ASC_CPU_CONTROL 0x1400044 +#define ANE_ASC_CPU_CONTROL_RUN BIT(4) #define ANE_ASC_CPU_STATUS 0x1400048 /* m1n1 R_CPU_STATUS shape */ #define ANE_ASC_RVBAR 0x1050000 /* fw entry | valid bit0 */ #define ANE_ASC_EDPRCR 0x1010310 /* read whitelist */ @@ -371,6 +372,7 @@ struct ane_t6021_ep { }; struct reset_control; +struct firmware; struct ane_t6021 { struct device *dev; @@ -397,6 +399,7 @@ struct ane_t6021 { */ bool power_gated; bool cpu_started; + bool boot_pin; bool fw_alive; bool booted; @@ -422,6 +425,8 @@ struct ane_t6021 { void *fw_buf; dma_addr_t fw_iova; u32 fw_size; + /* Validated pristine image retained for bounded, source-stable recovery. */ + const struct firmware *fw_image; /* Entry alias: IOVA the fw pages are aliased at (the latched * RVBAR entry); 0 = no alias mapped. The DMA allocator has no * runtime reservation API on this kernel, so instead of a prose @@ -737,6 +742,9 @@ void ane_t6021_csne_ping_attempt(struct ane_t6021 *ane); */ int ane_t6021_fwload_check(struct device *dev); int ane_t6021_fwload_probe(struct ane_t6021 *ane); +int ane_t6021_fwload_reload(struct ane_t6021 *ane); +bool ane_t6021_fwload_recoverable(struct ane_t6021 *ane); +void ane_t6021_boot_release_pin(struct ane_t6021 *ane); void ane_t6021_fwload_remove(struct ane_t6021 *ane); bool ane_t6021_fwload_options_ok(void); bool ane_t6021_fwload_placement_ok(struct device *dev); diff --git a/drivers/accel/ane/ane_t6021_boot.c b/drivers/accel/ane/ane_t6021_boot.c index 7c59d18fc6223..f6514a09df43b 100644 --- a/drivers/accel/ane/ane_t6021_boot.c +++ b/drivers/accel/ane/ane_t6021_boot.c @@ -414,8 +414,8 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, /* Wedged-pin module lifetime: the ref is acquired * BEFORE the first write — a started CPU can never * outlive the pin, and a dying module refuses the boot before - * any write happens. Retained on CPU start (never released: - * intentional); released only if no CPU start occurred. + * any write happens. Retained on CPU start; released only if no CPU start occurred or + * recovery proves a complete power-off cycle and a stopped CPU. */ if (!try_module_get(THIS_MODULE)) { dev_err(ane->dev, @@ -423,6 +423,8 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, return -EBUSY; } + ane->boot_pin = true; + dev_dbg(ane->dev, "BOOT-PHASE dispatch (stop_after=%d%s)\n", stop_after, stop_after ? " BISECT STOP ARMED" : ""); @@ -443,6 +445,7 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, if (!cs) { /* no CPU start: full release path, normal ownership */ + ane->boot_pin = false; module_put(THIS_MODULE); dev_info(ane->dev, "BOOT-PHASE done r=%d cpu_started=0 (no CPU release: state clean, module unpinned)\n", @@ -456,13 +459,13 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, * DMA surfaces are wedge-held, never freed. */ dev_warn(ane->dev, - "boot: module PINNED until reboot (started CPU; wedged-pin)\n"); + "boot: module pinned while firmware CPU may run\n"); if (r == -ENODATA) return r; /* unreachable: the caller gated */ if (r && cs) { dev_err(ane->dev, - "boot: sequence error %d AFTER CPU start (cpu_started=%u fw_alive=%u booted=%u scratch_result=%016llx) — WEDGED-PIN HOLD: all surfaces/rings/IRQ/links preserved; reboot is the only reclamation; no retry. HANDSHAKE state only: the result word has NO sourced success semantics\n", + "boot: sequence error %d AFTER CPU start (cpu_started=%u fw_alive=%u booted=%u scratch_result=%016llx) — WEDGED-PIN HOLD: all surfaces/rings/IRQ/links preserved; verified recovery or reboot required. HANDSHAKE state only: the result word has NO sourced success semantics\n", r, cs, fa, bo, sres); return 0; /* bind fenced, state held */ } @@ -475,3 +478,15 @@ int ane_t6021_boot_start(struct ane_t6021 *ane, int stop_after, int table_mode, sres); return r; } + +/* Caller has verified a complete domain-off cycle and a stopped CPU. */ +void ane_t6021_boot_release_pin(struct ane_t6021 *ane) +{ + if (ane->boot_pin) { + ane->boot_pin = false; + module_put(THIS_MODULE); + } + ane->cpu_started = false; + ane->fw_alive = false; + ane->booted = false; +} diff --git a/drivers/accel/ane/ane_t6021_boot.h b/drivers/accel/ane/ane_t6021_boot.h index ac0497bf6b010..a3b6c6d857ffa 100644 --- a/drivers/accel/ane/ane_t6021_boot.h +++ b/drivers/accel/ane/ane_t6021_boot.h @@ -300,6 +300,8 @@ ane_t6021_init_struct_fill(u8 *buf, const struct ane_t6021_init_sources *s, * heap requests and over-budget IPC sizes are refused BEFORE any * allocation. */ +#define ANE_T6021_BOOT_POOL_SIZE 0x40000 + struct ane_t6021_boot_allocs { u64 pool_dva, ipc_dva, heap_dva; u64 ipc_size, heap_size; @@ -347,7 +349,7 @@ ane_t6021_boot_prepare_publish(const struct ane_t6021_init_sources *s, a->heap_size = (u64)heap; } - a->pool = alloc(alloc_ctx, 0x40000, &a->pool_dva); + a->pool = alloc(alloc_ctx, ANE_T6021_BOOT_POOL_SIZE, &a->pool_dva); if (!a->pool) return -ENOMEM; a->ipc = alloc(alloc_ctx, a->ipc_size, &a->ipc_dva); diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index f39e4b7e8b05c..21e0f2ce7e9df 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -633,10 +633,55 @@ int ane_t6021_fwload_probe(struct ane_t6021 *ane) return ret; } - release_firmware(fw); + ane->fw_image = fw; return 0; } +bool ane_t6021_fwload_recoverable(struct ane_t6021 *ane) +{ + return fw_load && ane->fw_buf && ane->fw_image && + ane->fw_size >= ANE_FW_BUF_SIZE && + !ane_t6021_fw_alias_is_reserved(ane->dev); +} + +/* Restore the validated image in its existing owned mappings. The caller + * has stopped the old CPU through genpd; no alias is removed or recreated. + */ +int ane_t6021_fwload_reload(struct ane_t6021 *ane) +{ + const struct ane_t602x_soc *soc = of_device_get_match_data(ane->dev); + const struct ane_fw_image *img = soc->fw; + struct ane_fw_seg segs[ANE_FW_NSEGS]; + const struct firmware *fw = ane->fw_image; + const char *reason = NULL; + u8 actual_sha[32]; + u64 entry; + int ret; + + if (!ane_t6021_fwload_recoverable(ane)) + return -EOPNOTSUPP; + sha256(fw->data, fw->size, actual_sha); + ret = ane_fw_validate_blob(fw->data, fw->size, img, actual_sha, + segs, &entry, &reason); + if (ret) + goto out; + for (unsigned int i = 0; i < ANE_FW_NSEGS; i++) { + if (segs[i].vmaddr > ane->fw_size || + segs[i].filesize > ane->fw_size - segs[i].vmaddr) { + ret = -E2BIG; + goto out; + } + } + memset(ane->fw_buf, 0, ane->fw_size); + for (unsigned int i = 0; i < ANE_FW_NSEGS; i++) + if (segs[i].filesize) + memcpy(ane->fw_buf + segs[i].vmaddr, + fw->data + segs[i].fileoff, segs[i].filesize); + ret = ane_t6021_fw_patch(ane, ane->fw_buf); +out: + return ret; +} + void ane_t6021_fwload_remove(struct ane_t6021 *ane) { if (ane->fw_alias_extn) { @@ -655,6 +700,8 @@ void ane_t6021_fwload_remove(struct ane_t6021 *ane) ane->fw_alias_ext_len[i]); ane->fw_alias_extn = 0; } + release_firmware(ane->fw_image); + ane->fw_image = NULL; if (!ane->fw_buf) return; dma_free_coherent(ane->dev, ane->fw_size, ane->fw_buf, ane->fw_iova); diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index ed08c6506b324..e05fb4f2b3b34 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -98,6 +98,8 @@ #include "ane_h16_resources.h" #include "ane_fw_protocol.h" #include "ane_stats.h" +#include "ane_pm.h" +#include "ane_fw_ids.h" #include "ane_t6021.h" #include "ane_t6021_boot.h" @@ -214,10 +216,29 @@ struct ane_h16_process { u32 resource_count; }; +/* Retain every old arena until a fresh firmware boot is validated. */ +struct ane_rtclient_retired { + struct list_head node; + struct ane_legacy_buffer *buffers; + u32 allocated; + size_t bytes; + struct ane_t6021_boot_allocs boot; +}; + struct ane_rtclient { struct device *dev; void __iomem *engine; void __iomem *pmgr; + void __iomem *set; + bool recovering; + bool engine_ps_valid; + u32 engine_ps_off; + bool registered; + bool desired_perf; + struct list_head retired; + u32 retired_count; + u32 retired_allocated; + size_t retired_bytes; const struct ane_t602x_soc *soc; struct apple_rtkit *rtk; struct reset_control *cpu_rst; @@ -279,7 +300,7 @@ struct ane_t6021_fd { /* Per-file handle counter and every per-fd list against same-fd concurrent * ioctls. A BO whose IOVA reached the firmware (fw_ref) never goes back to - * the kernel: a program section is held until reboot, an io BO is parked + * the kernel: a program section is held until a verified reset, an io BO is parked * for reuse (see ane_t6021_bo_release). Every other BO frees at its last * reference. * @@ -289,7 +310,7 @@ struct ane_t6021_fd { * and any H14 softmax/reduction with an 8 MiB table. Total BO bytes * across all fds are capped separately (bo_total_max_mb) by an atomic * counter, enforced at alloc and released when the memory is really - * freed. Program sections stay held until reboot, and one large model's + * freed. Program sections stay held until a verified reset, and one large model's * programs alone can hold gigabytes. * A cap of 0 refuses every BO_INIT with -ENOSPC; there is no unlimited * value. Every BO also needs IOVA below 4 GiB (32-bit DMA mask), so @@ -307,6 +328,7 @@ struct ane_t6021_bo { void *cpu; /* coherent mapping */ dma_addr_t dma; size_t size; + u64 fw_epoch; bool fw_ref; /* the firmware received this IOVA */ bool fw_program; /* a loaded program keeps this IOVA */ struct device *dev; @@ -343,7 +365,7 @@ MODULE_PARM_DESC(bo_total_bytes, /* Mark the device quarantined: a timed-out command left the firmware * queue state unknown. The only safe next step is to refuse further - * ioctls until a reboot reclaims the surfaces (wedged-pin rule). + * ioctls until a verified firmware restart reclaims the old surfaces. */ static atomic_t ane_t6021_quarantined = ATOMIC_INIT(0); @@ -353,31 +375,70 @@ static atomic_t ane_t6021_quarantined = ATOMIC_INIT(0); * growing with every process until the BO cap refuses BO_INIT. */ static LIST_HEAD(ane_t6021_bo_pool); +static LIST_HEAD(ane_t6021_bo_held); +static u64 ane_t6021_bo_epoch = 1; static DEFINE_SPINLOCK(ane_t6021_bo_pool_lock); /* Final put: the last handle or mapping is gone. The firmware must never - * see a freed IOVA, so a fw_ref BO is never freed: a program + * see a freed IOVA, so a current-generation fw_ref BO is retained: a program * section stays held, because a cached firmware program keeps reading * it; an io BO goes to the pool, unless a quarantined firmware may still * write it. Every other BO frees here, so the BO cap bounds only the * memory the firmware may touch. */ +static void ane_t6021_bo_destroy(struct ane_t6021_bo *bo) +{ + atomic64_sub(PAGE_ALIGN(bo->size), &ane_t6021_bo_total_bytes); + dma_free_coherent(bo->dev, bo->size, bo->cpu, bo->dma); + kfree(bo); +} + static void ane_t6021_bo_release(struct kref *ref) { struct ane_t6021_bo *bo = container_of(ref, struct ane_t6021_bo, refcount); + bool release; - if (!bo->fw_ref) { - atomic64_sub(PAGE_ALIGN(bo->size), &ane_t6021_bo_total_bytes); - dma_free_coherent(bo->dev, bo->size, bo->cpu, bo->dma); - kfree(bo); - } else if (bo->fw_program || atomic_read(&ane_t6021_quarantined)) { - kfree(bo); - } else { - spin_lock(&ane_t6021_bo_pool_lock); - list_add(&bo->node, &ane_t6021_bo_pool); - spin_unlock(&ane_t6021_bo_pool_lock); + spin_lock(&ane_t6021_bo_pool_lock); + release = !bo->fw_ref || bo->fw_epoch != ane_t6021_bo_epoch; + if (!release) { + if (bo->fw_program || atomic_read(&ane_t6021_quarantined)) + list_add_tail(&bo->node, &ane_t6021_bo_held); + else + list_add_tail(&bo->node, &ane_t6021_bo_pool); } + spin_unlock(&ane_t6021_bo_pool_lock); + if (release) + ane_t6021_bo_destroy(bo); +} + +/* After verified restart, closed BOs are reclaimable and live old BOs + * become ordinary client-owned memory until published to firmware again. + */ +static void ane_t6021_bo_new_epoch(void) +{ + struct ane_t6021_bo *bo, *next; + LIST_HEAD(reap); + + spin_lock(&ane_t6021_bo_pool_lock); + ane_t6021_bo_epoch++; + list_splice_init(&ane_t6021_bo_pool, &reap); + list_splice_init(&ane_t6021_bo_held, &reap); + spin_unlock(&ane_t6021_bo_pool_lock); + list_for_each_entry_safe(bo, next, &reap, node) { + list_del(&bo->node); + ane_t6021_bo_destroy(bo); + } +} + +/* bo_lock and a live handle protect the BO while its IOVA is published. */ +static void ane_t6021_bo_publish(struct ane_t6021_bo *bo, bool program) +{ + if (bo->fw_epoch != ane_t6021_bo_epoch) + bo->fw_program = false; + bo->fw_epoch = ane_t6021_bo_epoch; + bo->fw_ref = true; + bo->fw_program |= program; } /* A parked io BO of SIZE's page-aligned size, or NULL. */ @@ -426,6 +487,8 @@ static const struct vm_operations_struct ane_t6021_vm_ops = { static DEFINE_MUTEX(ane_t6021_fw_lock); /* Forward declaration (defined below). */ +static int ane_rtclient_recover(struct ane_rtclient *ane); + static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, struct ane_legacy_buffer *command, size_t length, u16 opcode, @@ -442,8 +505,8 @@ static int ane_rtclient_legacy_alloc(struct ane_rtclient *ane, u64 size) struct ane_legacy_buffer *buffer; if (!size || !IS_ALIGNED(size, SZ_16K) || size > SZ_2M || - ane->legacy_allocated == ANE_LEGACY_ALLOCS || - size > ANE_LEGACY_BYTES - ane->legacy_bytes) + ane->legacy_allocated + ane->retired_allocated >= ANE_LEGACY_ALLOCS || + size > ANE_LEGACY_BYTES - ane->legacy_bytes - ane->retired_bytes) return -E2BIG; buffer = &ane->legacy_buffers[ane->legacy_allocated]; buffer->cpu = dma_alloc_coherent(ane->dev, size, &buffer->dma, @@ -1155,6 +1218,8 @@ static int ane_rtclient_command(struct ane_rtclient *ane, dev_info(ane->dev, "EXCH op=%#x failed %d (fw allocs %u, %zu bytes)\n", opcode, ret, ane->legacy_allocated, ane->legacy_bytes); atomic_set(&ane_t6021_quarantined, 1); + if (!ane->recovering) + ane_rtclient_recover(ane); return ret; } if (execute) { @@ -1177,6 +1242,8 @@ static int ane_rtclient_command(struct ane_rtclient *ane, "call completion failed %d within %u ms\n", ret, timeout_ms); atomic_set(&ane_t6021_quarantined, 1); + if (!ane->recovering) + ane_rtclient_recover(ane); return ret; } } @@ -1214,10 +1281,13 @@ struct ane_t6021_prog { u8 digest[SHA256_DIGEST_SIZE]; u32 prog_id; u32 proc_id; /* U32_MAX until a process exists */ + u32 fw_prog_id; + u32 fw_proc_id; }; static struct ane_t6021_prog ane_t6021_progs[ANE_T6021_MAX_PROGRAMS]; static unsigned int ane_t6021_nprogs; +static struct ane_fw_ids ane_t6021_ids; static struct ane_t6021_prog *ane_t6021_prog_find(const u8 *digest) { @@ -1256,6 +1326,7 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, struct ane_legacy_buffer *command; struct ane_t6021_fd *fd = file->driver_priv; struct ane_t6021_prog *cached; + u32 key; u8 digest[SHA256_DIGEST_SIZE]; size_t binds_size; int ret, i, j; @@ -1380,6 +1451,10 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, goto out; } + ret = ane_fw_id_alloc(&ane_t6021_ids, &key); + if (ret) + goto out; + command = ane->cmd_buf; if (!command) { ret = -ENODEV; @@ -1414,8 +1489,7 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, * Under bo_lock the handle reference cannot go * away, so no kref is needed here. */ - bo->fw_ref = true; - bo->fw_program = true; + ane_t6021_bo_publish(bo, true); iova = bo->dma + sections[i].offset; } else { bo = NULL; @@ -1468,7 +1542,9 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, } memcpy(ane_t6021_progs[ane_t6021_nprogs].digest, digest, SHA256_DIGEST_SIZE); - ane_t6021_progs[ane_t6021_nprogs].prog_id = *prog_id; + ane_t6021_progs[ane_t6021_nprogs].fw_prog_id = *prog_id; + *prog_id = key; + ane_t6021_progs[ane_t6021_nprogs].prog_id = key; ane_t6021_progs[ane_t6021_nprogs].proc_id = U32_MAX; ane_t6021_nprogs++; } @@ -1502,7 +1578,7 @@ static int ane_rtclient_create_process(struct ane_rtclient *ane, { u8 *cmd = command->cpu; - *(u32 *)(cmd + 0x08) = cpu_to_le32(prog_id); + *(u32 *)(cmd + 0x08) = cpu_to_le32(prog->fw_prog_id); *(u32 *)(cmd + 0x0c) = cpu_to_le32(U32_MAX); } ret = ane_rtclient_command(ane, command, 0x10, @@ -1512,10 +1588,13 @@ static int ane_rtclient_create_process(struct ane_rtclient *ane, u8 *cmd = command->cpu; *proc_id = le32_to_cpu(*(u32 *)(cmd + 0x0c)); - if (*proc_id == U32_MAX) + if (*proc_id == U32_MAX) { ret = -EPROTO; - else - prog->proc_id = *proc_id; + } else { + prog->fw_proc_id = *proc_id; + prog->proc_id = prog->prog_id; + *proc_id = prog->proc_id; + } } return ret; } @@ -1661,6 +1740,7 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, struct drm_ane_exec_io *ios; struct ane_t6021_bo **exec_bos; struct ane_legacy_buffer *command; + struct ane_t6021_prog *program; struct ane_t6021_fd *fd = file->driver_priv; size_t ios_size; size_t cmd_size; @@ -1705,6 +1785,12 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, ret = -ETIMEDOUT; goto unlock; } + program = ane_t6021_prog_by_id(user->prog_id); + if (!program || program->proc_id == U32_MAX || + program->proc_id != user->proc_id) { + ret = -ESTALE; + goto unlock; + } command = ane->cmd_buf; if (!command) { ret = -ENODEV; @@ -1714,8 +1800,8 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, { u8 *cmd = command->cpu; - *(u32 *)(cmd + 0x08) = cpu_to_le32(user->prog_id); - *(u32 *)(cmd + 0x0c) = cpu_to_le32(user->proc_id); + *(u32 *)(cmd + 0x08) = cpu_to_le32(program->fw_prog_id); + *(u32 *)(cmd + 0x0c) = cpu_to_le32(program->fw_proc_id); *(u64 *)(cmd + 0x10) = cpu_to_le64(0); *(u32 *)(cmd + 0x18) = cpu_to_le32(user->priority); *(u64 *)(cmd + 0x20) = cpu_to_le64(ANE_CALL_COOKIE); @@ -1740,7 +1826,7 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, /* Keep this IO element out of the reuse pool until completion. */ kref_get(&bo->refcount); exec_bos[pinned++] = bo; - bo->fw_ref = true; + ane_t6021_bo_publish(bo, false); iova = bo->dma; mutex_unlock(&ane_t6021_bo_lock); *(u32 *)(cmd + slot_base + 0x00) = cpu_to_le32(1); @@ -1788,15 +1874,15 @@ static struct ane_t6021_drm *to_ane_t6021_drm(struct drm_device *drm) return container_of(drm, struct ane_t6021_drm, drm); } -static int ane_t6021_bo_init_ioctl(struct drm_device *drm, void *data, - struct drm_file *file) +static int ane_t6021_bo_init_locked(struct drm_device *drm, void *data, + struct drm_file *file) { struct drm_ane_bo_init *args = data; struct ane_t6021_fd *fd = file->driver_priv; struct ane_t6021_bo *bo; - struct ane_rtclient *ane; + struct ane_rtclient *ane = to_ane_t6021_drm(drm)->ane; - if (atomic_read(&ane_t6021_quarantined)) + if (atomic_read(&ane_t6021_quarantined) || READ_ONCE(ane->recovering)) return -EIO; if (args->pad || args->size == 0 || args->size > ANE_T6021_BO_MAX || !fd) @@ -1819,7 +1905,6 @@ static int ane_t6021_bo_init_ioctl(struct drm_device *drm, void *data, atomic64_sub(PAGE_ALIGN(args->size), &ane_t6021_bo_total_bytes); return -ENOSPC; } - ane = to_ane_t6021_drm(drm)->ane; bo = kzalloc_obj(*bo); if (!bo) { atomic64_sub(PAGE_ALIGN(args->size), &ane_t6021_bo_total_bytes); @@ -1863,6 +1948,17 @@ static int ane_t6021_bo_init_ioctl(struct drm_device *drm, void *data, return 0; } +static int ane_t6021_bo_init_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + int ret; + + mutex_lock(&ane_t6021_fw_lock); + ret = ane_t6021_bo_init_locked(drm, data, file); + mutex_unlock(&ane_t6021_fw_lock); + return ret; +} + /* Drop one handle owned by this fd. The kref may keep the memory * alive past this call — a user mapping still holds a reference — so * the final ane_t6021_bo_release makes the free-or-hold decision. @@ -2069,6 +2165,7 @@ static int ane_t6021_perf_mode_set(const char *val, ret = ane_rtclient_command(ane, command, 0x14, 0x001f, 1, 3000); if (!ret) { fw_perf_mode = true; + ane->desired_perf = true; dev_info(ane->dev, "fw perf mode set (property 0x10aa = 1)\n"); } mutex_unlock(&ane_t6021_fw_lock); @@ -2289,6 +2386,7 @@ static DEFINE_MUTEX(ane_rtclient_pd_lock); static bool ane_rtclient_pinned; static bool ane_rtclient_probe_failed_dirty; static DEFINE_MUTEX(ane_rtclient_probe_guard_lock); +static struct ane_rtclient *ane_rtclient_active; static void ane_rtclient_pd_free(struct ane_rtclient_pd *pd) { @@ -2509,6 +2607,261 @@ static void ane_rtclient_detach_genpd(struct device *dev) mutex_unlock(&ane_rtclient_pd_lock); } +struct ane_rtclient_cycle { + struct ane_rtclient *ane; + struct ane_rtclient_pd *pd; +}; + +static int ane_rtclient_engine_accessible(struct ane_rtclient *ane) +{ + if (FIELD_GET(ANE_PS_ACTUAL, + readl(ane->pmgr + ane->soc->ps_cpu_off)) != ANE_PS_ON) + return -EIO; + if (ane->soc->pwgate_off && + (IS_ERR_OR_NULL(ane->set) || + readl(ane->set + ane->soc->pwgate_off) & GENMASK(29, 28))) + return -EACCES; + return 0; +} + +static bool ane_rtclient_domains_off(struct ane_rtclient_cycle *cycle) +{ + struct ane_rtclient *ane = cycle->ane; + + if (readl(ane->pmgr + ane->soc->ps_cpu_off) & ANE_PS_ACTUAL) + return false; + /* Firmware controls these seven PMGR words itself. Logical genpd + * state alone cannot prove they stayed off during CPU shutdown. + */ + for (int i = 0; i < 7; i++) + if (readl(ane->pmgr + ane->engine_ps_off + i * 8) & ANE_PS_ACTUAL) + return false; + for (int i = 0; i < cycle->pd->count; i++) + if (dev_pm_genpd_is_on(cycle->pd->pd_dev[i])) + return false; + return true; +} + +static int ane_rtclient_verify_off(void *ctx) +{ + struct ane_rtclient_cycle *cycle = ctx; + bool off; + + return read_poll_timeout(ane_rtclient_domains_off, off, off, + 100, 100000, false, cycle); +} + +static void ane_rtclient_reap_retired(struct ane_rtclient *ane) +{ + struct ane_rtclient_retired *r, *next; + + list_for_each_entry_safe(r, next, &ane->retired, node) { + for (u32 i = 0; i < r->allocated; i++) + if (r->buffers[i].cpu) + dma_free_coherent(ane->dev, r->buffers[i].size, + r->buffers[i].cpu, r->buffers[i].dma); + devm_kfree(ane->dev, r->buffers); + if (r->boot.pool) + dma_free_coherent(ane->dev, ANE_T6021_BOOT_POOL_SIZE, + r->boot.pool, r->boot.pool_dva); + if (r->boot.ipc) + dma_free_coherent(ane->dev, r->boot.ipc_size, + r->boot.ipc, r->boot.ipc_dva); + if (r->boot.heap) + dma_free_coherent(ane->dev, r->boot.heap_size, + r->boot.heap, r->boot.heap_dva); + list_del(&r->node); + kfree(r); + } + ane->retired_bytes = 0; + ane->retired_allocated = 0; + ane->retired_count = 0; +} + +static int ane_rtclient_recover(struct ane_rtclient *ane) +{ + struct ane_t6021 *a = ane->fw; + struct ane_rtclient_cycle cycle = { .ane = ane }; + struct ane_rtclient_pd *pd; + struct ane_rtclient_retired *retired; + struct ane_legacy_buffer *fresh; + u32 status, control; + unsigned int domains = ane->soc->soc == 0x8112 ? 7 : 8; + int ret; + + lockdep_assert_held(&ane_t6021_fw_lock); + if (!ane->registered) + return -EOPNOTSUPP; + if (!atomic_read(&ane_t6021_quarantined)) + return 0; + /* Only the qualified, Linux-owned legacy boot profile can restart. + * Borrowed firmware, RTKit and later-generation ownership stay fenced. + */ + if (ane->soc->platform || !a || ane->rtk || + !legacy_only || !scratch3_ack || !legacy_query || fw_start_rtb_mode || + fw_start_table_mode != 2 || !ane_t6021_fwload_recoverable(a)) + return -EOPNOTSUPP; + /* Keep failed-boot retention bounded and charged against existing caps. */ + if (ane->retired_count >= 2 || ane_t6021_bo_epoch == U64_MAX) + return -ENOSPC; + mutex_lock(&ane_rtclient_pd_lock); + list_for_each_entry(pd, &ane_rtclient_pd_list, list) + if (pd->dev == ane->dev && pd->count == domains) + cycle.pd = pd; + mutex_unlock(&ane_rtclient_pd_lock); + if (!cycle.pd || !ane->engine_ps_valid) + return -EOPNOTSUPP; + retired = kzalloc_obj(*retired); + if (!retired) + return -ENOMEM; + fresh = devm_kcalloc(ane->dev, ANE_LEGACY_ALLOCS, sizeof(*fresh), GFP_KERNEL); + if (!fresh) { + kfree(retired); + return -ENOMEM; + } + + WRITE_ONCE(ane->recovering, true); + cancel_delayed_work_sync(&ane->poll_work); + /* Stop only our owned ASC CPU before supplier shutdown. Clearing RUN + * is the public m1n1 ASC shutdown operation (hw/asc.py, c4c2fed8f894). + * The existing probe boot sequence also clears RUN before starting it. + * Wait for STOPPED so firmware cannot race the PMGR power-off writes. + */ + ret = ane_rtclient_engine_accessible(ane); + if (ret) + goto unchanged; + control = readl(ane->engine + ANE_ASC_CPU_CONTROL); + writel(control & ~ANE_ASC_CPU_CONTROL_RUN, + ane->engine + ANE_ASC_CPU_CONTROL); + ret = readl_poll_timeout(ane->engine + ANE_ASC_CPU_STATUS, status, + (status & ANE_ASC_CPU_STATUS_STOPPED) && + !(status & ANE_ASC_CPU_STATUS_RUNNING), + 100, 100000); + if (ret) + goto unchanged; + /* Probe retains one consumer reference for this firmware lifetime. */ + ret = ane_pm_cycle(ane->dev, cycle.pd->pd_dev, cycle.pd->count, + ane_rtclient_verify_off, &cycle); + if (ret) + goto unchanged; + /* Repeat probe's gates before any engine-aperture read. */ + ret = ane_rtclient_engine_accessible(ane); + if (ret) + goto unchanged; + status = readl(ane->engine + ANE_ASC_CPU_STATUS); + if ((status & ANE_ASC_CPU_STATUS_RUNNING) || + !(status & ANE_ASC_CPU_STATUS_STOPPED)) { + ret = -EBUSY; + goto unchanged; + } + + /* No old allocation is discarded until the new boot is validated. */ + retired->buffers = ane->legacy_buffers; + retired->allocated = ane->legacy_allocated; + retired->bytes = ane->legacy_bytes; + retired->boot = (struct ane_t6021_boot_allocs) { + .pool = a->boot_pool, .pool_dva = a->boot_pool_iova, + .ipc = a->boot_ipc, .ipc_dva = a->boot_ipc_iova, + .ipc_size = a->boot_ipc_size, + .heap = a->boot_heap, .heap_dva = a->boot_heap_iova, + .heap_size = a->boot_heap_size, + }; + list_add_tail(&retired->node, &ane->retired); + ane->retired_count++; + ane->retired_bytes += retired->bytes; + ane->retired_allocated += retired->allocated; + ane_t6021_boot_release_pin(a); + a->boot_pool = NULL; + a->boot_ipc = NULL; + a->boot_heap = NULL; + a->boot_pool_iova = 0; + a->boot_ipc_iova = 0; + a->boot_heap_iova = 0; + a->boot_ipc_size = 0; + a->boot_heap_size = 0; + ane->legacy_buffers = fresh; + ane->legacy_allocated = 0; + ane->legacy_bytes = 0; + ane->legacy_malloc_cursor = 0; + memset(ane->legacy_cmd_cursor, 0, sizeof(ane->legacy_cmd_cursor)); + memset(ane->channels, 0, sizeof(ane->channels)); + memset(ane->channel_snapshot, 0, sizeof(ane->channel_snapshot)); + ane->cmd_buf = NULL; + ane->chman_ok = false; + ane->held = false; + fw_perf_mode = false; + + ret = ane_t6021_fwload_reload(a); + if (ret) + goto failed; + ret = ane_t6021_boot_start(a, 0, fw_start_table_mode, 0); + ane->held = a->cpu_started; + if (ret || !a->fw_alive || !a->booted) { + ret = ret ?: -ETIMEDOUT; + goto failed; + } + ane_rtclient_validate_chman(ane); + if (!ane->chman_ok || + !ane_t6021_chman_host_init(a->boot_ipc, a->boot_ipc_size, + a->boot_ipc_iova)) { + ret = -EPROTO; + goto failed; + } + dma_wmb(); + writel(ANE_T6021_BOOT_ACK, ane->engine + ANE_MBI_SCRATCH0 + 4 * 3); + ret = ane_rtclient_legacy_alloc(ane, SZ_16K); + if (ret) + goto failed; + ane->cmd_buf = &ane->legacy_buffers[ane->legacy_allocated - 1]; + memset(ane->cmd_buf->cpu, 0, SZ_16K); + /* The firmware lock still fences commands; BO_INIT also checks recovering. + * Clearing the old latch lets transport validation detect a new error. + */ + atomic_set(&ane_t6021_quarantined, 0); + ret = ane_rtclient_legacy_exchange(ane, ane->cmd_buf, 16, 0x03, 1, + jiffies + msecs_to_jiffies(3000)); + if (!ret && !READ_ONCE(((u32 *)ane->cmd_buf->cpu)[2])) + ret = -EPROTO; + if (ret) + goto failed; + if (ane->desired_perf) { + u8 *cmd = ane->cmd_buf->cpu; + + memset(cmd, 0, SZ_16K); + *(u32 *)(cmd + 0x0c) = cpu_to_le32(0x10aa); + *(u32 *)(cmd + 0x10) = cpu_to_le32(1); + ret = ane_rtclient_legacy_exchange(ane, ane->cmd_buf, 0x14, 0x001f, 1, + jiffies + msecs_to_jiffies(3000)); + if (ret) + goto failed; + fw_perf_mode = true; + } + if (atomic_read(&ane_t6021_quarantined)) { + ret = -EIO; + goto failed; + } + /* Opaque IDs never alias a new firmware instance. Clients reload programs. + * Advance the BO epoch only after old DMA and the new boot are verified. + */ + ane_t6021_nprogs = 0; + ane_t6021_bo_new_epoch(); + ane_rtclient_reap_retired(ane); + WRITE_ONCE(ane->recovering, false); + dev_info(ane->dev, "firmware recovered; old program IDs invalid, reload and resubmit\n"); + return 0; + +unchanged: + devm_kfree(ane->dev, fresh); + kfree(retired); +failed: + atomic_set(&ane_t6021_quarantined, 1); + WRITE_ONCE(ane->recovering, false); + dev_err(ane->dev, + "firmware recovery failed: %d; DMA backing and any live-CPU pin retained\n", + ret); + return ret; +} + static int ane_rtclient_probe_inner(struct platform_device *pdev, bool *hardware_touched) { @@ -2564,6 +2917,7 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, ane->soc = of_device_get_match_data(dev); platform_set_drvdata(pdev, ane); INIT_DELAYED_WORK(&ane->poll_work, ane_rtclient_post_boot); + INIT_LIST_HEAD(&ane->retired); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res) @@ -2601,6 +2955,16 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, return dev_err_probe(dev, ret, "pmgr window map failed; G1 gate cannot run\n"); } + res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (res) { + u64 first = ane->soc->pmu_pa + ane->soc->ps_off; + + if (first >= res->start && first <= res->end && + res->end - first >= 6 * 8 + sizeof(u32) - 1) { + ane->engine_ps_off = first - res->start; + ane->engine_ps_valid = true; + } + } ps_cpu = readl(ane->pmgr + ane->soc->ps_cpu_off); dev_dbg(dev, "ane_cpu ACTUAL = 0x%x\n", ps_cpu); if (FIELD_GET(ANE_PS_ACTUAL, ps_cpu) != ANE_PS_ON) { @@ -2615,9 +2979,12 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, */ if (ane->soc->pwgate_off) { void __iomem *set = devm_of_iomap(dev, dev->of_node, 2, NULL); + u32 gate = IS_ERR(set) ? U32_MAX : readl(set + ane->soc->pwgate_off); + ane->set = set; + dev_dbg(dev, "PWGATE = 0x%x\n", gate); if (gate & GENMASK(29, 28)) { pm_runtime_put_sync_suspend(dev); @@ -2921,6 +3288,8 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, ane_timeline_show, &ane->stats_ring); drmret = drm_dev_register(&adrm->drm, 0); + if (!drmret) + ane->registered = true; if (drmret) { dev_err_probe(dev, drmret, "drm_dev_register\n"); ret = drmret; @@ -2962,14 +3331,16 @@ static int ane_rtclient_probe(struct platform_device *pdev) return ane_fw_platform_preflight(pdev, soc->platform); mutex_lock(&ane_rtclient_probe_guard_lock); - if (ane_rtclient_probe_failed_dirty) { + if (ane_rtclient_probe_failed_dirty || ane_rtclient_active) { mutex_unlock(&ane_rtclient_probe_guard_lock); dev_err(&pdev->dev, - "previous probe failed after touching the hardware: reboot required\n"); + "ANE already owned or previous probe left hardware dirty; refusing another binding\n"); return -EBUSY; } ret = ane_rtclient_probe_inner(pdev, &hardware_touched); + if (!ret) + ane_rtclient_active = platform_get_drvdata(pdev); if (ret) { struct ane_rtclient *ane = platform_get_drvdata(pdev); @@ -3034,8 +3405,33 @@ static ssize_t ane_stats_show(struct device *dev, } static DEVICE_ATTR_RO(ane_stats); +static ssize_t wedged_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct ane_rtclient *ane = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", atomic_read(&ane_t6021_quarantined) || + READ_ONCE(ane->recovering) || !READ_ONCE(ane->registered)); +} +static DEVICE_ATTR_RO(wedged); + +static ssize_t reset_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ane_rtclient *ane = dev_get_drvdata(dev); + int ret; + + mutex_lock(&ane_t6021_fw_lock); + ret = ane_rtclient_recover(ane); + mutex_unlock(&ane_t6021_fw_lock); + return ret ? ret : count; +} +static DEVICE_ATTR_WO(reset); + static struct attribute *ane_t6021_stats_attrs[] = { &dev_attr_ane_stats.attr, + &dev_attr_wedged.attr, + &dev_attr_reset.attr, NULL, }; diff --git a/include/uapi/drm/ane_accel.h b/include/uapi/drm/ane_accel.h index 4e16e1771b53c..94109d1eead35 100644 --- a/include/uapi/drm/ane_accel.h +++ b/include/uapi/drm/ane_accel.h @@ -171,9 +171,12 @@ struct drm_ane_generic_bind { * @generic_ptr: In. Pointer to struct drm_ane_generic_bind[@generic_count]. * @section_count: In. 1 to %DRM_ANE_MAX_SECTIONS. * @generic_count: In. 0 to %DRM_ANE_MAX_BINDS. - * @prog_id_out: Out. Firmware program id. Identical sections give the same - * program id. + * @prog_id_out: Out. Opaque program id. Identical sections in one firmware + * generation give the same program id. * @pad: Must be zero. + * + * Device reset invalidates program and process ids. Reload the sections and + * create the process again before resubmitting. */ struct drm_ane_prog_load { __u64 sections_ptr; @@ -188,7 +191,7 @@ struct drm_ane_prog_load { * struct drm_ane_proc_create - DRM_IOCTL_ANE_PROC_CREATE argument * (ABI version 2) * @prog_id: In. Program id from PROG_LOAD. - * @proc_id_out: Out. Firmware process id. + * @proc_id_out: Out. Opaque process id. */ struct drm_ane_proc_create { __u32 prog_id; @@ -219,7 +222,8 @@ struct drm_ane_exec_io { * @prog_id: In. Program id from PROG_LOAD. * @proc_id: In. Process id from PROC_CREATE. * @priority: In. 2 to 7. - * @timeout_ms: In. Firmware completion timeout, 0 for 5000 ms. + * @timeout_ms: In. Firmware completion timeout, capped at 5000 ms; 0 uses + * 5000 ms. * @count: In. 1 to %DRM_ANE_MAX_BINDS. * @pad: Must be zero. * @io_ptr: In. Pointer to struct drm_ane_exec_io[@count]. From d826cca0dff6fffdc444e8f900821c424023c3d7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 09:38:08 +1100 Subject: [PATCH 34/73] media: apple: avd: wait for a closing stream's frames instead of resetting Closing a stream (STREAMOFF, close or an m2m job abort) while it still owned a slot reset the whole decoder, cancelling every other stream's frames with their DMA in flight. On the J616s both resets logged before the 04:11 hard hang came from this path (no frame had timed out), each when FFmpeg gave up on one of four concurrent streams. Cancel the closing stream's frames that never reached the decoder, then wait for the ones it already owns. They complete normally, or the queue worker times them out and recovers as for any other stuck frame. Only if frames are still owned a second after the frame timeout does the drain fall back to the old reset, and it says so in the log. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-queue.c | 43 ++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index 7cb0221c07bda..5ca9b10accc1a 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -481,6 +481,21 @@ void avd_queue_run(void *priv) schedule_work(&ctx->submit_work); } +/* True while any slot still belongs to @ctx; the caller holds no queue lock. */ +static bool avd_ctx_has_frames(struct avd_dev *avd, struct avd_ctx *ctx) +{ + unsigned long flags; + unsigned int i; + bool busy = false; + + spin_lock_irqsave(&avd->queue_lock, flags); + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (avd->frames[i].state != AVD_FRAME_FREE && avd->frames[i].ctx == ctx) + busy = true; + spin_unlock_irqrestore(&avd->queue_lock, flags); + return busy; +} + void avd_queue_drain(struct avd_ctx *ctx) { struct avd_dev *avd = ctx->dev; @@ -499,13 +514,41 @@ void avd_queue_drain(struct avd_ctx *ctx) avd->deferred_ctx = NULL; finish_deferred = true; } + /* Prepared but never handed to the decoder: nothing to wait for. */ + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + + if (f->ctx == ctx && f->state == AVD_FRAME_RESERVED) { + f->status = -ECANCELED; + f->state = AVD_FRAME_DONE; + } + } spin_unlock_irqrestore(&avd->queue_lock, flags); /* A deferred job is still m2m's running job: end it so abort can wait. */ if (finish_deferred) v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, VB2_BUF_STATE_ERROR); + avd_retire(avd); + + /* + * Frames the decoder already owns finish on their own, or the queue + * worker times them out and recovers. Closing one stream must not reset + * the decoder under every other stream's frames: that reset, with their + * DMA in flight, preceded the J616s hangs. Wait for this context's own + * frames instead, without the queue mutex so the worker can retire them. + */ + if (ret >= 0 && !avd->stopping && avd_ctx_has_frames(avd, ctx)) { + mod_delayed_work(system_wq, &avd->queue_work, 0); + mutex_unlock(&avd->queue_mutex); + wait_event_timeout(avd->pm.idle_wait, !avd_ctx_has_frames(avd, ctx), + AVD_FRAME_TIMEOUT + msecs_to_jiffies(1000)); + mutex_lock(&avd->queue_mutex); + avd_retire(avd); + } + /* Last resort (worker stopped or wedged): cancel through a reset. */ for (i = 0; ret >= 0 && i < AVD_MAX_FRAMES; i++) { if (avd->frames[i].state != AVD_FRAME_FREE && avd->frames[i].ctx == ctx) { + dev_warn(avd->dev, "stream close: frames still owned by the decoder, resetting\n"); if (avd->backend->abort) avd->backend->abort(ctx); avd_reset_queue(avd, -ECANCELED); From 5a041336365de7bcfbe71ddc60158ffd5a277483 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 09:50:43 +1100 Subject: [PATCH 35/73] accel/ane: free H14 firmware allocations and unload programs on close The H14 firmware never got a program back. Every LOAD_PROGRAM and CREATE_PROCESS stayed in the firmware until reboot, and so did the section buffers the driver kept for it. On an M2, ten or eleven MobileNet-sized programs loaded by short-lived processes were enough for the firmware to stop answering. The next LOAD timed out with 17 firmware allocations (17 MB) outstanding, well inside the driver's own limits, and the device stayed quarantined until reboot. Give every program a lifetime. Each file holds a reference on the programs it loaded, and EXEC and PROC_CREATE refuse a program the file holds none on. When the last reference goes, the driver sends TERMINATE_PROCESS and then DESTROY_PROGRAM, and only after both are acknowledged does it drop its references on the section buffers. A failed unload keeps the program and its buffers until a verified firmware restart. Answer the firmware's FREE requests on the SHAREDMALLOC ring. The firmware posts one when a pool shrinks, for example when a process is terminated, and waits for the answer with its shared memory lock held, sometimes after the command that caused it was acknowledged. The driver used to take such a slot for a malformed MALLOC. It now answers the request, frees the buffer and reuses its table slot, so the table and its byte count follow the live allocations. The ring is served before each command, while a command or call is pending, and for 10 ms of quiet after each unload command. A timed-out command now logs the allocation ring and the T2H rings at the host cursors, and debugfs fw_memory reports the live allocations, bytes and programs. The request classes, the slot reuse rule and the per-file program references follow omarchy-ane by Joshua Warren (MIT). KUnit ane-t6021-lifetime covers the request classes, slot reuse, shared programs, unload order, held programs, a restart during an unload and the table limits. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 26 + drivers/accel/ane/Kconfig | 4 +- drivers/accel/ane/Makefile | 3 +- drivers/accel/ane/ane_t6021_lifetime.h | 380 ++++++++++++++ drivers/accel/ane/ane_t6021_lifetime_test.c | 411 +++++++++++++++ drivers/accel/ane/ane_t6021_rtclient_main.c | 536 +++++++++++++++----- 6 files changed, 1236 insertions(+), 124 deletions(-) create mode 100644 drivers/accel/ane/ane_t6021_lifetime.h create mode 100644 drivers/accel/ane/ane_t6021_lifetime_test.c diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index 793aef9d346b8..4ab0448e4e4ff 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -329,6 +329,32 @@ produce a timeout. The loader pins both fixture hashes and uses ABI 1 ioctls dir shared libane library is needed. ``--check-fixtures`` validates the two containers without opening a device. +H14 program lifetime +-------------------- + +Byte-identical sections share one firmware program and one process. Every +file holds a reference on each program it loaded, and ``PROC_CREATE`` and +``EXEC`` refuse a program the file holds no reference on with ``-ENOENT``. +When the last file holding a program closes, the driver sends +TERMINATE_PROCESS (0x203) for its process and then DESTROY_PROGRAM (0x201). +Only after both are acknowledged does it drop its references on the section +buffers, which are then freed or pooled like any other BO. A failed unload +keeps the program and its buffers until a verified firmware restart. + +The firmware asks for memory on the SHAREDMALLOC ring and gives it back with +a FREE request, which carries the returned IOVA in the slot header and an +empty body. It waits for the answer with its shared-memory lock held, and a +FREE caused by a command can arrive after that command's acknowledgment. The +driver serves the ring before each command, while a command or call is +pending, and for 10 ms of quiet after each unload command. A FREE releases the +buffer and its table slot, so the table and its byte count follow the live +allocations rather than every allocation since boot. + +``fw_memory`` in the device's accel debugfs directory reports the live +firmware allocations, their bytes, the table high-water mark, and the loaded +and held programs. A command timeout also logs the allocation ring and the +target-to-host rings at the host cursors. + H14 recovery lifecycle ---------------------- diff --git a/drivers/accel/ane/Kconfig b/drivers/accel/ane/Kconfig index 19bc539ca1a14..c502238817ee8 100644 --- a/drivers/accel/ane/Kconfig +++ b/drivers/accel/ane/Kconfig @@ -50,8 +50,8 @@ config DRM_ACCEL_ANE_FW_KUNIT_TEST default KUNIT_ALL_TESTS help Check preload ranges, 27.x channel and reply validation, RTKit - management ordering, H16 HWX/resource bounds and UUID retirement - without hardware. + management ordering, H16 HWX/resource bounds, UUID retirement and + the H14 firmware allocation and program lifetime without hardware. These tests can run on architectures other than Apple arm64. If unsure, say N. diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile index aa80d67b937c9..5bbd4746e3cee 100644 --- a/drivers/accel/ane/Makefile +++ b/drivers/accel/ane/Makefile @@ -5,7 +5,8 @@ obj-$(CONFIG_DRM_ACCEL_ANE_PM_KUNIT_TEST) += ane_pm_test.o ane_fw_ids_test.o obj-$(CONFIG_DRM_ACCEL_ANE_H16_VALIDATION) += ane_h16_hwx.o obj-$(CONFIG_DRM_ACCEL_ANE_KUNIT_TEST) += ane_drv_test.o obj-$(CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST) += ane_fw_platform_test.o \ - ane_fw_protocol_test.o ane_h16_tasks_test.o ane_h16_execute_test.o ane_h16_pending_test.o ane_h16_hwx_test.o + ane_fw_protocol_test.o ane_h16_tasks_test.o ane_h16_execute_test.o ane_h16_pending_test.o ane_h16_hwx_test.o \ + ane_t6021_lifetime_test.o ane-y := \ ane_drv.o \ diff --git a/drivers/accel/ane/ane_t6021_lifetime.h b/drivers/accel/ane/ane_t6021_lifetime.h new file mode 100644 index 0000000000000..83ad3a14bb13d --- /dev/null +++ b/drivers/accel/ane/ane_t6021_lifetime.h @@ -0,0 +1,380 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * H14 firmware allocation and program lifetime rules. + * + * Kept free of device access so the KUnit suite (ane_t6021_lifetime_test.c) + * runs the same code as the driver. The caller serializes every function + * here with the firmware command lock. + * + * The SHAREDMALLOC request classes, the slot reuse rule for the firmware + * allocation table and the per-file program references follow the design of + * omarchy-ane by Joshua Warren (MIT), adapted to this driver's opaque program + * IDs and firmware restart. + * + * SHAREDMALLOC (channel 5) carries one request per 64-byte slot as three + * words: header, size, mode. A slot with header bit 0 set is host-owned. + * - MALLOC: header 0, size in bytes, a 32-bit mode. The host answers with + * the buffer IOVA | 1 in the header and a table index in word 2. + * - FREE: header is the IOVA the firmware gives back (16 KiB aligned, so + * bit 0 is clear and the header is not 0), size and mode 0. The firmware + * has already dropped its mapping and waits for the answer without a + * timeout while it holds its shared-memory lock, so the host must answer + * every FREE: header 1, size and mode 0. The request can arrive after the + * command that caused it was acknowledged. + * - header 0 and size 0: nothing posted yet. The slot is read again on + * the next poll and the cursor does not move. + * + * A firmware program is shared by every file that loaded byte-identical + * sections. Each file holds one reference per program it loaded. When the + * last reference goes, the program is unloaded: TERMINATE_PROCESS (0x203) + * for its process, then DESTROY_PROGRAM (0x201). Only after both are + * acknowledged are its section buffers given back. A program whose unload + * failed is HELD: its buffers stay and its slot is never reused until a + * verified firmware restart resets the table. + */ +#ifndef __ANE_T6021_LIFETIME_H__ +#define __ANE_T6021_LIFETIME_H__ + +#include +#include +#include +#include +#include + +/* ---- SHAREDMALLOC requests ---- */ + +enum ane_t6021_mring_kind { + ANE_T6021_MRING_IDLE, /* nothing posted in this slot */ + ANE_T6021_MRING_MALLOC, /* allocate size bytes */ + ANE_T6021_MRING_FREE, /* the firmware returns the IOVA in hdr */ + ANE_T6021_MRING_ANSWERED, /* host-owned slot */ + ANE_T6021_MRING_BAD, /* no request has this shape */ +}; + +static inline enum ane_t6021_mring_kind +ane_t6021_mring_classify(u64 hdr, u64 size, u64 mode) +{ + if (hdr & 1) + return ANE_T6021_MRING_ANSWERED; + if (!hdr) { + /* No size yet: not a request, look again later. */ + if (!size) + return ANE_T6021_MRING_IDLE; + return mode > U32_MAX ? ANE_T6021_MRING_BAD : + ANE_T6021_MRING_MALLOC; + } + if (size || mode || hdr & (SZ_16K - 1)) + return ANE_T6021_MRING_BAD; + return ANE_T6021_MRING_FREE; +} + +/* ---- firmware allocation table ---- */ + +/* One buffer the firmware asked for. A slot is free when cpu is NULL. */ +struct ane_legacy_buffer { + void *cpu; + dma_addr_t dma; + size_t size; +}; + +/* + * The slot a new buffer takes: the first free one below the high-water mark + * @used, else @used itself while the table can grow below @cap, else -1. + * Reusing freed slots keeps the table at the peak of live allocations + * instead of growing with every allocation the firmware makes. + */ +static inline int ane_t6021_legtab_pick(const struct ane_legacy_buffer *t, + u32 used, u32 cap) +{ + u32 i; + + for (i = 0; i < used; i++) + if (!t[i].cpu) + return i; + return used < cap ? (int)used : -1; +} + +/* The live slot holding @dma, never @keep (a buffer the driver itself uses). */ +static inline int ane_t6021_legtab_find(const struct ane_legacy_buffer *t, + u32 used, u64 dma, + const struct ane_legacy_buffer *keep) +{ + u32 i; + + for (i = 0; i < used; i++) + if (t[i].cpu && t[i].dma == dma && &t[i] != keep) + return i; + return -1; +} + +/* ---- program table ---- */ + +/* The firmware table holds 256 programs; the 257th LOAD is refused. */ +#define ANE_T6021_MAX_PROGRAMS 250 +#define ANE_T6021_PROG_SECS 9 +#define ANE_T6021_NO_ID U32_MAX + +#define ANE_T6021_CMD_DESTROY_PROGRAM 0x0201 +#define ANE_T6021_CMD_TERMINATE_PROCESS 0x0203 + +enum ane_t6021_prog_state { + ANE_T6021_PROG_FREE = 0, + ANE_T6021_PROG_LIVE, /* loaded, shared by the files that hold it */ + ANE_T6021_PROG_HELD, /* load or unload uncertain: keep everything */ +}; + +struct ane_t6021_prog { + u8 digest[32]; + u32 prog_id; /* opaque kernel ID, never reused */ + u32 proc_id; /* opaque, ANE_T6021_NO_ID until a process exists */ + u32 fw_prog_id; /* firmware wire IDs */ + u32 fw_proc_id; + u32 users; /* files holding a reference */ + enum ane_t6021_prog_state state; + /* The program's own reference on each section buffer, or NULL. */ + void *sec[ANE_T6021_PROG_SECS]; +}; + +struct ane_t6021_progtab { + struct ane_t6021_prog progs[ANE_T6021_MAX_PROGRAMS]; + unsigned int hi; /* slots below this index may be in use */ + u32 gen; /* bumped by every reset */ +}; + +/* The program IDs one file holds a reference on. */ +struct ane_t6021_fdprogs { + u32 n; + u32 ids[ANE_T6021_MAX_PROGRAMS]; +}; + +struct ane_t6021_progtab_ops { + /* Unload from the firmware; zero only when every command was acked. */ + int (*unload)(void *ctx, struct ane_t6021_prog *p); + /* The firmware no longer reads the sections: drop their references. */ + void (*release)(void *ctx, struct ane_t6021_prog *p); +}; + +/* Only LIVE programs are ever found, so HELD and stale entries are unusable. */ +static inline struct ane_t6021_prog * +ane_t6021_progtab_find(struct ane_t6021_progtab *t, const u8 *digest) +{ + unsigned int i; + + for (i = 0; i < t->hi; i++) + if (t->progs[i].state == ANE_T6021_PROG_LIVE && + !memcmp(t->progs[i].digest, digest, sizeof(t->progs[i].digest))) + return &t->progs[i]; + return NULL; +} + +static inline struct ane_t6021_prog * +ane_t6021_progtab_by_id(struct ane_t6021_progtab *t, u32 prog_id) +{ + unsigned int i; + + if (prog_id == ANE_T6021_NO_ID) + return NULL; + for (i = 0; i < t->hi; i++) + if (t->progs[i].state == ANE_T6021_PROG_LIVE && + t->progs[i].prog_id == prog_id) + return &t->progs[i]; + return NULL; +} + +/* + * Claim a slot for a program about to be loaded. It starts HELD so nothing + * can find it, and it owns the section references taken before the LOAD is + * sent: if the LOAD fails, the buffers stay held with it. + */ +static inline struct ane_t6021_prog * +ane_t6021_progtab_reserve(struct ane_t6021_progtab *t, const u8 *digest) +{ + struct ane_t6021_prog *p = NULL; + unsigned int i; + + for (i = 0; i < t->hi; i++) { + if (t->progs[i].state == ANE_T6021_PROG_FREE) { + p = &t->progs[i]; + break; + } + } + if (!p) { + if (t->hi == ANE_T6021_MAX_PROGRAMS) + return NULL; + p = &t->progs[t->hi++]; + } + memset(p, 0, sizeof(*p)); + memcpy(p->digest, digest, sizeof(p->digest)); + p->prog_id = ANE_T6021_NO_ID; + p->proc_id = ANE_T6021_NO_ID; + p->fw_prog_id = ANE_T6021_NO_ID; + p->fw_proc_id = ANE_T6021_NO_ID; + p->state = ANE_T6021_PROG_HELD; + return p; +} + +/* The firmware acknowledged the LOAD: publish the program. */ +static inline void ane_t6021_progtab_commit(struct ane_t6021_prog *p, + u32 prog_id, u32 fw_prog_id) +{ + p->prog_id = prog_id; + p->fw_prog_id = fw_prog_id; + p->state = ANE_T6021_PROG_LIVE; +} + +static inline bool ane_t6021_fdprogs_has(const struct ane_t6021_fdprogs *f, + u32 prog_id) +{ + u32 i; + + for (i = 0; i < f->n; i++) + if (f->ids[i] == prog_id) + return true; + return false; +} + +/* Forget IDs whose program is gone (a firmware restart reset the table). */ +static inline void ane_t6021_fdprogs_compact(struct ane_t6021_progtab *t, + struct ane_t6021_fdprogs *f) +{ + u32 i, n = 0; + + for (i = 0; i < f->n; i++) + if (ane_t6021_progtab_by_id(t, f->ids[i])) + f->ids[n++] = f->ids[i]; + f->n = n; +} + +/* True when the file can take one more program reference. */ +static inline bool ane_t6021_fdprogs_room(struct ane_t6021_progtab *t, + struct ane_t6021_fdprogs *f) +{ + if (f->n < ANE_T6021_MAX_PROGRAMS) + return true; + ane_t6021_fdprogs_compact(t, f); + return f->n < ANE_T6021_MAX_PROGRAMS; +} + +/* The file takes one reference on a LIVE program, once per file. */ +static inline int ane_t6021_fdprogs_ref(struct ane_t6021_progtab *t, + struct ane_t6021_fdprogs *f, + struct ane_t6021_prog *p) +{ + if (p->state != ANE_T6021_PROG_LIVE) + return -ESTALE; + if (ane_t6021_fdprogs_has(f, p->prog_id)) + return 0; + if (!ane_t6021_fdprogs_room(t, f)) + return -ENOSPC; + f->ids[f->n++] = p->prog_id; + p->users++; + return 0; +} + +/* + * The firmware commands of an unload: the process first, because destroying + * a program that still has a process halts the firmware. @cmd returns zero + * only for an acknowledged command. Nothing after a failure is sent. + */ +static inline int +ane_t6021_prog_unload_cmds(const struct ane_t6021_prog *p, + int (*cmd)(void *ctx, u16 opcode, u32 fw_prog, + u32 fw_proc), + void *ctx) +{ + int ret; + + if (p->fw_proc_id != ANE_T6021_NO_ID) { + ret = cmd(ctx, ANE_T6021_CMD_TERMINATE_PROCESS, p->fw_prog_id, + p->fw_proc_id); + if (ret) + return ret; + } + return cmd(ctx, ANE_T6021_CMD_DESTROY_PROGRAM, p->fw_prog_id, + ANE_T6021_NO_ID); +} + +/* + * The file is closing: drop its references and unload every program whose + * last reference this was. Returns how many programs ended HELD. + */ +static inline unsigned int +ane_t6021_progtab_fd_close(struct ane_t6021_progtab *t, + struct ane_t6021_fdprogs *f, + const struct ane_t6021_progtab_ops *ops, void *ctx) +{ + unsigned int held = 0; + u32 i, gen; + + for (i = 0; i < f->n; i++) { + struct ane_t6021_prog *p = ane_t6021_progtab_by_id(t, f->ids[i]); + + if (!p || !p->users) + continue; + if (--p->users) + continue; + gen = t->gen; + if (ops->unload(ctx, p)) { + /* A restart during the unload already reset the table. */ + if (t->gen == gen) { + p->state = ANE_T6021_PROG_HELD; + held++; + } + continue; + } + ops->release(ctx, p); + memset(p, 0, sizeof(*p)); + } + f->n = 0; + return held; +} + +/* + * A verified firmware restart dropped every program: release all section + * references, LIVE and HELD alike. Files keep their old IDs, which never + * match again because kernel IDs are not reused. + */ +static inline void ane_t6021_progtab_reset(struct ane_t6021_progtab *t, + const struct ane_t6021_progtab_ops *ops, + void *ctx) +{ + unsigned int i; + + for (i = 0; i < t->hi; i++) { + struct ane_t6021_prog *p = &t->progs[i]; + + if (p->state != ANE_T6021_PROG_FREE) + ops->release(ctx, p); + memset(p, 0, sizeof(*p)); + } + t->hi = 0; + t->gen++; +} + +/* Give back a reservation whose LOAD was never sent. */ +static inline void ane_t6021_progtab_cancel(struct ane_t6021_prog *p, + const struct ane_t6021_progtab_ops *ops, + void *ctx) +{ + ops->release(ctx, p); + memset(p, 0, sizeof(*p)); +} + +/* LIVE and HELD program counts, for diagnostics. */ +static inline void ane_t6021_progtab_count(const struct ane_t6021_progtab *t, + unsigned int *live, + unsigned int *held) +{ + unsigned int i; + + *live = 0; + *held = 0; + for (i = 0; i < t->hi; i++) { + if (t->progs[i].state == ANE_T6021_PROG_LIVE) + (*live)++; + else if (t->progs[i].state == ANE_T6021_PROG_HELD) + (*held)++; + } +} + +#endif /* __ANE_T6021_LIFETIME_H__ */ diff --git a/drivers/accel/ane/ane_t6021_lifetime_test.c b/drivers/accel/ane/ane_t6021_lifetime_test.c new file mode 100644 index 0000000000000..45d60213c2d3a --- /dev/null +++ b/drivers/accel/ane/ane_t6021_lifetime_test.c @@ -0,0 +1,411 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include + +#include "ane_t6021_lifetime.h" + +static void mring_classify_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, ane_t6021_mring_classify(0, 0, 0), + ANE_T6021_MRING_IDLE); + KUNIT_EXPECT_EQ(test, ane_t6021_mring_classify(0, SZ_1M, 0x45585050), + ANE_T6021_MRING_MALLOC); + KUNIT_EXPECT_EQ(test, ane_t6021_mring_classify(0, SZ_16K, U32_MAX), + ANE_T6021_MRING_MALLOC); + KUNIT_EXPECT_EQ(test, ane_t6021_mring_classify(0, SZ_16K, 1ULL << 32), + ANE_T6021_MRING_BAD); + /* A FREE gives back an IOVA in the header with an empty body. */ + KUNIT_EXPECT_EQ(test, ane_t6021_mring_classify(0xfbff8000, 0, 0), + ANE_T6021_MRING_FREE); + KUNIT_EXPECT_EQ(test, ane_t6021_mring_classify(0xfbff8000, SZ_16K, 0), + ANE_T6021_MRING_BAD); + KUNIT_EXPECT_EQ(test, ane_t6021_mring_classify(0xfbff8000, 0, 1), + ANE_T6021_MRING_BAD); + KUNIT_EXPECT_EQ(test, ane_t6021_mring_classify(0xfbff8002, 0, 0), + ANE_T6021_MRING_BAD); + /* Answered slots carry bit 0 whatever the body holds. */ + KUNIT_EXPECT_EQ(test, ane_t6021_mring_classify(1, 0, 0), + ANE_T6021_MRING_ANSWERED); + KUNIT_EXPECT_EQ(test, ane_t6021_mring_classify(0xfbff8001, 0, 7), + ANE_T6021_MRING_ANSWERED); +} + +static void legtab_reuse_test(struct kunit *test) +{ + struct ane_legacy_buffer t[4] = {}; + u8 mem[4]; + u32 used = 0; + int i, slot; + + for (i = 0; i < 4; i++) { + slot = ane_t6021_legtab_pick(t, used, 4); + KUNIT_ASSERT_EQ(test, slot, i); + t[slot] = (struct ane_legacy_buffer) { + .cpu = &mem[i], .dma = 0x100000 + i * SZ_16K, .size = SZ_16K, + }; + used++; + } + KUNIT_EXPECT_EQ(test, ane_t6021_legtab_pick(t, used, 4), -1); + /* A FREE of slot 2 makes it the next slot taken; the table does not grow. */ + slot = ane_t6021_legtab_find(t, used, 0x100000 + 2 * SZ_16K, NULL); + KUNIT_ASSERT_EQ(test, slot, 2); + memset(&t[slot], 0, sizeof(t[slot])); + KUNIT_EXPECT_EQ(test, ane_t6021_legtab_find(t, used, 0x100000 + 2 * SZ_16K, + NULL), -1); + KUNIT_EXPECT_EQ(test, ane_t6021_legtab_pick(t, used, 4), 2); + /* The driver's own command buffer never matches a FREE. */ + KUNIT_EXPECT_EQ(test, ane_t6021_legtab_find(t, used, 0x100000, &t[0]), -1); + KUNIT_EXPECT_EQ(test, ane_t6021_legtab_find(t, used, 0x100000, NULL), 0); + /* A freed slot has size 0, so no address lookup can land in it. */ + KUNIT_EXPECT_EQ(test, t[2].size, (size_t)0); +} + +/* A fake firmware: records every unload and release, can fail on demand. */ +struct fake_fw { + unsigned int unloads, releases; + int fail_unload; + int sec_refs; +}; + +static int fake_unload(void *ctx, struct ane_t6021_prog *p) +{ + struct fake_fw *fw = ctx; + + fw->unloads++; + return fw->fail_unload; +} + +static void fake_release(void *ctx, struct ane_t6021_prog *p) +{ + struct fake_fw *fw = ctx; + unsigned int i; + + fw->releases++; + for (i = 0; i < ANE_T6021_PROG_SECS; i++) + if (p->sec[i]) + fw->sec_refs--; +} + +static const struct ane_t6021_progtab_ops fake_ops = { + .unload = fake_unload, + .release = fake_release, +}; + +static struct ane_t6021_prog *load(struct ane_t6021_progtab *t, + struct fake_fw *fw, u8 tag, u32 id) +{ + u8 digest[32]; + struct ane_t6021_prog *p; + + memset(digest, tag, sizeof(digest)); + p = ane_t6021_progtab_reserve(t, digest); + if (!p) + return NULL; + p->sec[0] = fw; + p->sec[6] = fw; + fw->sec_refs += 2; + ane_t6021_progtab_commit(p, id, id + 100); + return p; +} + +static void progtab_shared_unload_test(struct kunit *test) +{ + struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + struct ane_t6021_fdprogs *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); + struct ane_t6021_fdprogs *b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL); + struct fake_fw fw = {}; + struct ane_t6021_prog *p, *q; + u8 digest[32]; + + KUNIT_ASSERT_NOT_NULL(test, t); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + p = load(t, &fw, 1, 10); + KUNIT_ASSERT_NOT_NULL(test, p); + KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), 0); + /* A second load of identical sections finds the same program. */ + memset(digest, 1, sizeof(digest)); + q = ane_t6021_progtab_find(t, digest); + KUNIT_ASSERT_PTR_EQ(test, q, p); + KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, b, q), 0); + /* Taking the same reference twice from one file counts once. */ + KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, b, q), 0); + KUNIT_EXPECT_EQ(test, p->users, 2U); + KUNIT_EXPECT_TRUE(test, ane_t6021_fdprogs_has(b, 10)); + + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 0U); + KUNIT_EXPECT_EQ(test, fw.unloads, 0U); + KUNIT_EXPECT_EQ(test, p->users, 1U); + KUNIT_EXPECT_EQ(test, fw.sec_refs, 2); + + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, b, &fake_ops, &fw), 0U); + KUNIT_EXPECT_EQ(test, fw.unloads, 1U); + KUNIT_EXPECT_EQ(test, fw.releases, 1U); + KUNIT_EXPECT_EQ(test, fw.sec_refs, 0); + KUNIT_EXPECT_NULL(test, ane_t6021_progtab_find(t, digest)); + KUNIT_EXPECT_NULL(test, ane_t6021_progtab_by_id(t, 10)); + /* The slot is reused by the next program. */ + q = load(t, &fw, 2, 11); + KUNIT_EXPECT_PTR_EQ(test, q, p); + KUNIT_EXPECT_EQ(test, t->hi, 1U); +} + +static void progtab_failed_unload_holds_test(struct kunit *test) +{ + struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + struct ane_t6021_fdprogs *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); + struct fake_fw fw = { .fail_unload = -ETIMEDOUT }; + struct ane_t6021_prog *p, *q; + unsigned int live, held; + + KUNIT_ASSERT_NOT_NULL(test, t); + KUNIT_ASSERT_NOT_NULL(test, a); + p = load(t, &fw, 3, 20); + KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), 0); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 1U); + /* The firmware may still read the sections: nothing is released. */ + KUNIT_EXPECT_EQ(test, fw.releases, 0U); + KUNIT_EXPECT_EQ(test, fw.sec_refs, 2); + KUNIT_EXPECT_EQ(test, p->state, ANE_T6021_PROG_HELD); + KUNIT_EXPECT_NULL(test, ane_t6021_progtab_by_id(t, 20)); + ane_t6021_progtab_count(t, &live, &held); + KUNIT_EXPECT_EQ(test, live, 0U); + KUNIT_EXPECT_EQ(test, held, 1U); + /* A held slot is never reused before a restart. */ + q = load(t, &fw, 4, 21); + KUNIT_EXPECT_PTR_NE(test, q, p); + KUNIT_EXPECT_EQ(test, t->hi, 2U); + /* A verified restart releases LIVE and HELD programs alike. */ + ane_t6021_progtab_reset(t, &fake_ops, &fw); + KUNIT_EXPECT_EQ(test, fw.releases, 2U); + KUNIT_EXPECT_EQ(test, fw.sec_refs, 0); + KUNIT_EXPECT_EQ(test, t->hi, 0U); +} + +static void progtab_load_failure_test(struct kunit *test) +{ + struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + struct ane_t6021_fdprogs *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); + struct ane_t6021_prog *p; + u8 digest[32]; + + KUNIT_ASSERT_NOT_NULL(test, t); + KUNIT_ASSERT_NOT_NULL(test, a); + memset(digest, 5, sizeof(digest)); + p = ane_t6021_progtab_reserve(t, digest); + KUNIT_ASSERT_NOT_NULL(test, p); + /* An uncommitted program is invisible and cannot be referenced. */ + KUNIT_EXPECT_NULL(test, ane_t6021_progtab_find(t, digest)); + KUNIT_EXPECT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), -ESTALE); + KUNIT_EXPECT_EQ(test, a->n, 0U); + KUNIT_EXPECT_EQ(test, p->state, ANE_T6021_PROG_HELD); +} + +static void progtab_restart_stale_ids_test(struct kunit *test) +{ + struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + struct ane_t6021_fdprogs *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); + struct ane_t6021_fdprogs *b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL); + struct fake_fw fw = {}; + struct ane_t6021_prog *p, *q; + + KUNIT_ASSERT_NOT_NULL(test, t); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + p = load(t, &fw, 6, 30); + KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), 0); + ane_t6021_progtab_reset(t, &fake_ops, &fw); + /* Another file loads after the restart and reuses the slot. */ + q = load(t, &fw, 7, 31); + KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, b, q), 0); + KUNIT_EXPECT_PTR_EQ(test, q, p); + /* The old file's stale ID neither reaches nor drops the new program. */ + KUNIT_EXPECT_FALSE(test, ane_t6021_fdprogs_has(a, 31)); + KUNIT_EXPECT_NULL(test, ane_t6021_progtab_by_id(t, 30)); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 0U); + KUNIT_EXPECT_EQ(test, fw.unloads, 0U); + KUNIT_EXPECT_EQ(test, q->users, 1U); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, b, &fake_ops, &fw), 0U); + KUNIT_EXPECT_EQ(test, fw.unloads, 1U); + KUNIT_EXPECT_EQ(test, fw.sec_refs, 0); +} + +static void progtab_capacity_test(struct kunit *test) +{ + struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + struct ane_t6021_fdprogs *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); + struct ane_t6021_fdprogs *b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL); + struct fake_fw fw = {}; + struct ane_t6021_prog *p; + u8 digest[32]; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, t); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + for (i = 0; i < ANE_T6021_MAX_PROGRAMS; i++) { + memset(digest, 0, sizeof(digest)); + digest[0] = i; + digest[1] = i >> 8; + p = ane_t6021_progtab_reserve(t, digest); + KUNIT_ASSERT_NOT_NULL(test, p); + ane_t6021_progtab_commit(p, 1000 + i, i); + KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), 0); + } + memset(digest, 0xff, sizeof(digest)); + KUNIT_EXPECT_NULL(test, ane_t6021_progtab_reserve(t, digest)); + KUNIT_EXPECT_FALSE(test, ane_t6021_fdprogs_room(t, a)); + /* Closing the file unloads all of them and frees every slot. */ + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 0U); + KUNIT_EXPECT_EQ(test, fw.unloads, (unsigned int)ANE_T6021_MAX_PROGRAMS); + p = ane_t6021_progtab_reserve(t, digest); + KUNIT_EXPECT_PTR_EQ(test, p, &t->progs[0]); + + /* A full file list drops stale IDs before it refuses a reference. */ + b->n = ANE_T6021_MAX_PROGRAMS; + for (i = 0; i < ANE_T6021_MAX_PROGRAMS; i++) + b->ids[i] = 5000 + i; + ane_t6021_progtab_commit(p, 9000, 1); + KUNIT_EXPECT_EQ(test, ane_t6021_fdprogs_ref(t, b, p), 0); + KUNIT_EXPECT_EQ(test, b->n, 1U); + KUNIT_EXPECT_EQ(test, p->users, 1U); +} + +static struct ane_t6021_progtab *restart_table; + +/* An unload whose failure made the driver restart the firmware. */ +static int restart_unload(void *ctx, struct ane_t6021_prog *p) +{ + struct fake_fw *fw = ctx; + + fw->unloads++; + ane_t6021_progtab_reset(restart_table, &fake_ops, fw); + return -ETIMEDOUT; +} + +static const struct ane_t6021_progtab_ops restart_ops = { + .unload = restart_unload, + .release = fake_release, +}; + +static void progtab_restart_during_unload_test(struct kunit *test) +{ + struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + struct ane_t6021_fdprogs *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); + struct fake_fw fw = {}; + struct ane_t6021_prog *p, *q; + + KUNIT_ASSERT_NOT_NULL(test, t); + KUNIT_ASSERT_NOT_NULL(test, a); + restart_table = t; + p = load(t, &fw, 8, 40); + q = load(t, &fw, 9, 41); + KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), 0); + KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, q), 0); + /* The restart released both programs; nothing may be marked HELD + * afterwards and the second program needs no unload of its own. + */ + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &restart_ops, &fw), 0U); + KUNIT_EXPECT_EQ(test, fw.unloads, 1U); + KUNIT_EXPECT_EQ(test, fw.releases, 2U); + KUNIT_EXPECT_EQ(test, fw.sec_refs, 0); + KUNIT_EXPECT_EQ(test, p->state, ANE_T6021_PROG_FREE); + KUNIT_EXPECT_EQ(test, t->gen, 1U); +} + +static void progtab_cancel_test(struct kunit *test) +{ + struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + struct fake_fw fw = {}; + struct ane_t6021_prog *p; + u8 digest[32]; + + KUNIT_ASSERT_NOT_NULL(test, t); + memset(digest, 10, sizeof(digest)); + p = ane_t6021_progtab_reserve(t, digest); + KUNIT_ASSERT_NOT_NULL(test, p); + p->sec[2] = &fw; + fw.sec_refs = 1; + /* A LOAD that was never sent gives its slot and references back. */ + ane_t6021_progtab_cancel(p, &fake_ops, &fw); + KUNIT_EXPECT_EQ(test, fw.sec_refs, 0); + KUNIT_EXPECT_EQ(test, p->state, ANE_T6021_PROG_FREE); + KUNIT_EXPECT_PTR_EQ(test, ane_t6021_progtab_reserve(t, digest), p); +} + +struct fake_cmds { + u16 ops[4]; + u32 progs[4]; + u32 procs[4]; + unsigned int n; + int fail_at; +}; + +static int fake_cmd(void *ctx, u16 opcode, u32 fw_prog, u32 fw_proc) +{ + struct fake_cmds *c = ctx; + + c->ops[c->n] = opcode; + c->progs[c->n] = fw_prog; + c->procs[c->n] = fw_proc; + return ++c->n == c->fail_at ? -ETIMEDOUT : 0; +} + +static void unload_order_test(struct kunit *test) +{ + struct ane_t6021_prog p = { + .fw_prog_id = 4, .fw_proc_id = 9, + }; + struct fake_cmds c = {}; + + /* The process goes first: destroying a program with a process halts + * the firmware. + */ + KUNIT_EXPECT_EQ(test, ane_t6021_prog_unload_cmds(&p, fake_cmd, &c), 0); + KUNIT_ASSERT_EQ(test, c.n, 2U); + KUNIT_EXPECT_EQ(test, c.ops[0], ANE_T6021_CMD_TERMINATE_PROCESS); + KUNIT_EXPECT_EQ(test, c.progs[0], 4U); + KUNIT_EXPECT_EQ(test, c.procs[0], 9U); + KUNIT_EXPECT_EQ(test, c.ops[1], ANE_T6021_CMD_DESTROY_PROGRAM); + KUNIT_EXPECT_EQ(test, c.progs[1], 4U); + + /* No process: only the program is destroyed. */ + memset(&c, 0, sizeof(c)); + p.fw_proc_id = ANE_T6021_NO_ID; + KUNIT_EXPECT_EQ(test, ane_t6021_prog_unload_cmds(&p, fake_cmd, &c), 0); + KUNIT_ASSERT_EQ(test, c.n, 1U); + KUNIT_EXPECT_EQ(test, c.ops[0], ANE_T6021_CMD_DESTROY_PROGRAM); + + /* A failed terminate stops the sequence. */ + memset(&c, 0, sizeof(c)); + c.fail_at = 1; + p.fw_proc_id = 9; + KUNIT_EXPECT_EQ(test, ane_t6021_prog_unload_cmds(&p, fake_cmd, &c), + -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, c.n, 1U); +} + +static struct kunit_case lifetime_cases[] = { + KUNIT_CASE(mring_classify_test), + KUNIT_CASE(legtab_reuse_test), + KUNIT_CASE(progtab_shared_unload_test), + KUNIT_CASE(progtab_failed_unload_holds_test), + KUNIT_CASE(progtab_load_failure_test), + KUNIT_CASE(progtab_restart_stale_ids_test), + KUNIT_CASE(progtab_capacity_test), + KUNIT_CASE(progtab_restart_during_unload_test), + KUNIT_CASE(progtab_cancel_test), + KUNIT_CASE(unload_order_test), + {} +}; + +static struct kunit_suite lifetime_suite = { + .name = "ane-t6021-lifetime", + .test_cases = lifetime_cases, +}; + +kunit_test_suite(lifetime_suite); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("ANE H14 firmware allocation and program lifetime tests"); diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index e05fb4f2b3b34..00f58b3fecd21 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -72,6 +72,7 @@ #include #include #include +#include #include #include #include @@ -102,6 +103,7 @@ #include "ane_fw_ids.h" #include "ane_t6021.h" #include "ane_t6021_boot.h" +#include "ane_t6021_lifetime.h" #include @@ -191,15 +193,12 @@ module_param(stats, bool, 0444); MODULE_PARM_DESC(stats, "Expose ane_stats sysfs and ane_timeline debugfs (default 1; 0 = no files and the hot path skips the counters)"); +/* Caps on the live firmware allocations (SHAREDMALLOC) and the command + * buffer. A FREE from the firmware releases its slot and bytes. + */ #define ANE_LEGACY_ALLOCS 8192 #define ANE_LEGACY_BYTES SZ_512M -struct ane_legacy_buffer { - void *cpu; - dma_addr_t dma; - size_t size; -}; - struct ane_t6021_fd; /* Populated only by an original-container loader and decoded process reply. @@ -264,7 +263,8 @@ struct ane_rtclient { struct ane_h16_pending *waiting; struct list_head h16_processes; struct ane_legacy_buffer *legacy_buffers; - u32 legacy_allocated; + u32 legacy_allocated; /* high-water mark of the table */ + u32 legacy_live; /* slots in use now */ size_t legacy_bytes; u32 legacy_malloc_cursor; u32 legacy_cmd_cursor[ANE_T6021_CHMAN_COUNT]; @@ -296,6 +296,8 @@ struct ane_rtclient { struct ane_t6021_fd { struct list_head bos; struct list_head h16_processes; + /* H14 programs this file loaded; under ane_t6021_fw_lock. */ + struct ane_t6021_fdprogs progs; }; /* Per-file handle counter and every per-fd list against same-fd concurrent @@ -329,8 +331,10 @@ struct ane_t6021_bo { dma_addr_t dma; size_t size; u64 fw_epoch; + u32 prog_refs; /* program sections using this BO */ bool fw_ref; /* the firmware received this IOVA */ bool fw_program; /* a loaded program keeps this IOVA */ + bool fw_io; /* published as a CALL buffer */ struct device *dev; }; @@ -434,11 +438,16 @@ static void ane_t6021_bo_new_epoch(void) /* bo_lock and a live handle protect the BO while its IOVA is published. */ static void ane_t6021_bo_publish(struct ane_t6021_bo *bo, bool program) { - if (bo->fw_epoch != ane_t6021_bo_epoch) + if (bo->fw_epoch != ane_t6021_bo_epoch) { bo->fw_program = false; + bo->fw_io = false; + } bo->fw_epoch = ane_t6021_bo_epoch; bo->fw_ref = true; - bo->fw_program |= program; + if (program) + bo->fw_program = true; + else + bo->fw_io = true; } /* A parked io BO of SIZE's page-aligned size, or NULL. */ @@ -500,15 +509,25 @@ static bool ane_rtclient_drain_t2h(struct ane_rtclient *ane, /* ---- boot helpers (shared with boot.c / fwload.c) ---- */ +/* Allocate a firmware-visible buffer in the first free table slot. Returns + * the slot index or a negative errno. legacy_allocated is the high-water + * mark; legacy_live and legacy_bytes count what is allocated now. + */ static int ane_rtclient_legacy_alloc(struct ane_rtclient *ane, u64 size) { struct ane_legacy_buffer *buffer; + u32 cap = ANE_LEGACY_ALLOCS - min_t(u32, ane->retired_allocated, + ANE_LEGACY_ALLOCS); + int slot; if (!size || !IS_ALIGNED(size, SZ_16K) || size > SZ_2M || - ane->legacy_allocated + ane->retired_allocated >= ANE_LEGACY_ALLOCS || size > ANE_LEGACY_BYTES - ane->legacy_bytes - ane->retired_bytes) return -E2BIG; - buffer = &ane->legacy_buffers[ane->legacy_allocated]; + slot = ane_t6021_legtab_pick(ane->legacy_buffers, ane->legacy_allocated, + cap); + if (slot < 0) + return -E2BIG; + buffer = &ane->legacy_buffers[slot]; buffer->cpu = dma_alloc_coherent(ane->dev, size, &buffer->dma, GFP_KERNEL); if (!buffer->cpu) @@ -516,14 +535,16 @@ static int ane_rtclient_legacy_alloc(struct ane_rtclient *ane, u64 size) if (!IS_ALIGNED(buffer->dma, SZ_16K) || (ane->fw && !ane_t6021_fw_alias_iova_ok(ane->fw, buffer->dma, size))) { dma_free_coherent(ane->dev, size, buffer->cpu, buffer->dma); - buffer->cpu = NULL; + memset(buffer, 0, sizeof(*buffer)); return -ERANGE; } memset(buffer->cpu, 0, size); buffer->size = size; ane->legacy_bytes += size; - ane->legacy_allocated++; - return 0; + ane->legacy_live++; + if ((u32)slot == ane->legacy_allocated) + ane->legacy_allocated++; + return slot; } /* Validate the chman table the fw published in the 'IPC ' surface. */ @@ -619,6 +640,163 @@ static void __iomem *ane_rtclient_doorbell(struct ane_rtclient *ane) ane->soc->platform->doorbell : ANE_IPI_OFF); } +/* + * Answer the SHAREDMALLOC slot at *cursor if the firmware posted a request + * there (see ane_t6021_lifetime.h). Returns 1 when a request was answered + * and the cursor moved, 0 when the slot holds none, or a negative errno for + * a slot no request has the shape of or an allocation that failed. + */ +static int ane_rtclient_malloc_service(struct ane_rtclient *ane, + unsigned int *cursor) +{ + const struct ane_fw_channel *c = &ane->channels[5]; + u64 *slot = ane->fw->boot_ipc + c->offset + (size_t)*cursor * 64; + void __iomem *ipi = ane_rtclient_doorbell(ane); + u64 hdr = READ_ONCE(slot[0]), size, mode; + struct ane_legacy_buffer *buf; + int idx; + + if (hdr & 1) + return 0; + /* H16 needs its own validated allocation/reply provider. Never answer + * it with the legacy H14 allocation index. + */ + if (ane->soc->platform) + return -EOPNOTSUPP; + dma_rmb(); + size = READ_ONCE(slot[1]); + mode = READ_ONCE(slot[2]); + switch (ane_t6021_mring_classify(hdr, size, mode)) { + case ANE_T6021_MRING_IDLE: + return 0; + case ANE_T6021_MRING_MALLOC: + idx = ane_rtclient_legacy_alloc(ane, size); + if (idx < 0) { + dev_err(ane->dev, + "fw malloc of %#llx bytes (mode %#llx) refused %d (live %u, %zu bytes)\n", + size, mode, idx, ane->legacy_live, ane->legacy_bytes); + return -ENOMEM; + } + buf = &ane->legacy_buffers[idx]; + WRITE_ONCE(slot[1], 0); + WRITE_ONCE(slot[2], idx + 1); + dma_wmb(); + WRITE_ONCE(slot[0], buf->dma | 1); + dev_dbg(ane->dev, + "fw malloc slot %u: %#llx bytes mode %#llx -> %pad index %d (live %u, %zu bytes)\n", + *cursor, size, mode, &buf->dma, idx + 1, ane->legacy_live, + ane->legacy_bytes); + break; + case ANE_T6021_MRING_FREE: + /* The firmware dropped its mapping before it asked and waits + * for this answer holding its shared-memory lock. Answer first, + * then give the buffer back. The command buffer never matches. + */ + WRITE_ONCE(slot[1], 0); + WRITE_ONCE(slot[2], 0); + dma_wmb(); + WRITE_ONCE(slot[0], 1); + idx = ane_t6021_legtab_find(ane->legacy_buffers, + ane->legacy_allocated, hdr, + ane->cmd_buf); + if (idx >= 0) { + buf = &ane->legacy_buffers[idx]; + dma_free_coherent(ane->dev, buf->size, buf->cpu, buf->dma); + ane->legacy_bytes -= buf->size; + ane->legacy_live--; + memset(buf, 0, sizeof(*buf)); + } + if (idx >= 0) + dev_dbg(ane->dev, + "fw free slot %u: iova %#llx released (live %u, %zu bytes)\n", + *cursor, hdr, ane->legacy_live, ane->legacy_bytes); + else + dev_info_ratelimited(ane->dev, + "fw free slot %u: iova %#llx is not a firmware allocation; answered, nothing released\n", + *cursor, hdr); + break; + default: + dev_err(ane->dev, + "fw malloc ring slot %u is not a request: %#llx %#llx %#llx\n", + *cursor, hdr, size, mode); + return -EPROTO; + } + dma_wmb(); + writel(BIT(c->bit), ipi); + *cursor = (*cursor + 1) % c->slots; + return 1; +} + +/* Answer every request already posted, at most one ring's worth. */ +static int ane_rtclient_malloc_drain(struct ane_rtclient *ane) +{ + unsigned int n; + int ret = 0; + + for (n = 0; n < ane->channels[5].slots; n++) { + ret = ane_rtclient_malloc_service(ane, &ane->legacy_malloc_cursor); + if (ret <= 0) + break; + } + return ret < 0 ? ret : 0; +} + +/* + * A FREE caused by a command can land after its acknowledgment. Keep + * answering until the ring was quiet for quiet_ms, at most 64 answers. + */ +static int ane_rtclient_malloc_settle(struct ane_rtclient *ane, + unsigned int quiet_ms) +{ + unsigned long until = jiffies + msecs_to_jiffies(quiet_ms); + unsigned int answered = 0; + int ret; + + while (time_before(jiffies, until)) { + ret = ane_rtclient_malloc_service(ane, &ane->legacy_malloc_cursor); + if (ret < 0) + return ret; + if (ret) { + if (++answered > 64) + return -EBUSY; + until = jiffies + msecs_to_jiffies(quiet_ms); + } else { + usleep_range(200, 400); + } + } + return 0; +} + +/* What the rings held when a command timed out, for the log. */ +static void ane_rtclient_dump_rings(struct ane_rtclient *ane) +{ + static const unsigned int chans[] = { 4, 5, 6 }; + unsigned int i, j; + + for (i = 0; i < ARRAY_SIZE(chans); i++) { + const struct ane_fw_channel *c = &ane->channels[chans[i]]; + unsigned int cur = chans[i] == 5 ? ane->legacy_malloc_cursor : + ane->legacy_cmd_cursor[chans[i]]; + unsigned int n = min_t(unsigned int, c->slots, 8); + + if (!c->slots) + continue; + /* The whole allocation ring; T2H rings from the host cursor. */ + for (j = 0; j < n; j++) { + unsigned int k = chans[i] == 5 ? j : (cur + j) % c->slots; + u64 *slot = ane->fw->boot_ipc + c->offset + (size_t)k * 64; + + dev_info(ane->dev, + "ring ch%u slot %u: %016llx %016llx %016llx%s\n", + chans[i], k, READ_ONCE(slot[0]), + READ_ONCE(slot[1]), READ_ONCE(slot[2]), + k == cur ? " <- host cursor" : ""); + } + } + dev_info(ane->dev, "fw allocations: live %u, high water %u, %zu bytes\n", + ane->legacy_live, ane->legacy_allocated, ane->legacy_bytes); +} + /* Legacy ChMan exchange (post-DONE 13.5 transport). Resends to the same * 64-byte IO slot would wedge the ring; cursor advances after each * completed command (the fw treats bit0 as host-owned). @@ -630,9 +808,8 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, unsigned long deadline) { struct ane_t6021 *a = ane->fw; - u64 *io, *malloc_ring, header; + u64 *io, header; void __iomem *ipi = ane_rtclient_doorbell(ane); - unsigned int cursor = ane->legacy_malloc_cursor; int result = -ETIMEDOUT; if (!ane->held || !ane_rtclient_channels_unchanged(ane)) @@ -644,7 +821,6 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, } io = a->boot_ipc + ane->channels[channel].offset + (size_t)ane->legacy_cmd_cursor[channel] * 64; - malloc_ring = a->boot_ipc + ane->channels[5].offset; if (!(READ_ONCE(io[0]) & 1)) { result = -EBUSY; @@ -652,6 +828,15 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, } if (time_after_eq(jiffies, deadline)) goto out; + /* A FREE posted since the last command (an idle pool shrink) is + * answered before this command goes out. + */ + if (!ane->soc->platform) { + result = ane_rtclient_malloc_drain(ane); + if (result) + goto out; + result = -ETIMEDOUT; + } if (ane->soc->platform) { result = ane_h16_command_prepare(command->cpu, length, opcode, ane->channels[channel].index); @@ -698,47 +883,14 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, header = READ_ONCE(io[0]); if (!(header & 1)) { /* malloc-ring service: the fw may ask for an - * allocation while we wait; service those until - * the command slot flips back. + * allocation, or give one back, while we wait. */ - u64 *slot = malloc_ring + cursor * 8; - u64 alloc_hdr = READ_ONCE(slot[0]); - - if (!(alloc_hdr & 1)) { - u64 size, tag; - struct ane_legacy_buffer *buf; - - /* H16 needs its own validated allocation/reply provider. - * Never answer it with the legacy H14 allocation index. - */ - if (ane->soc->platform) { - result = -EOPNOTSUPP; - goto out; - } - dma_rmb(); - size = READ_ONCE(slot[1]); - tag = READ_ONCE(slot[2]); - /* A valid MALLOC slot carries - * header 0 and a u32 tag; anything else - * is a malformed ring and we stop. - */ - if (alloc_hdr || tag > U32_MAX) { - result = -EPROTO; - goto out; - } - if (ane_rtclient_legacy_alloc(ane, size)) { - result = -ENOMEM; - goto out; - } - buf = &ane->legacy_buffers[ane->legacy_allocated - 1]; - WRITE_ONCE(slot[1], 0); - WRITE_ONCE(slot[2], ane->legacy_allocated); - dma_wmb(); - WRITE_ONCE(slot[0], buf->dma | 1); - dma_wmb(); - writel(BIT(ane->channels[5].bit), ipi); - cursor = (cursor + 1) % - ane->channels[5].slots; + int served = ane_rtclient_malloc_service(ane, + &ane->legacy_malloc_cursor); + + if (served < 0) { + result = served; + goto out; } } /* Allocation service cannot extend the command deadline. */ @@ -773,10 +925,10 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, */ udelay(50); } - dev_info(ane->dev, "LEGACY timeout ch=%u io=%016llx\n", - channel, READ_ONCE(io[0])); + dev_info(ane->dev, "LEGACY timeout ch=%u op=%#x io=%016llx\n", + channel, opcode, READ_ONCE(io[0])); + ane_rtclient_dump_rings(ane); out: - ane->legacy_malloc_cursor = cursor; return result; } @@ -1143,6 +1295,11 @@ static int ane_rtclient_call_wait(struct ane_rtclient *ane, return ane->waiting->status; } else if (ane_rtclient_drain_t2h(ane, 6)) { return 0; + } else if (ane_rtclient_malloc_drain(ane)) { + /* A program waiting on an unanswered request never + * finishes; a malformed ring ends the call now. + */ + return -EPROTO; } if (poll_us && ktime_before(ktime_get(), tight_end)) usleep_range(1, 2); @@ -1267,49 +1424,125 @@ static int ane_rtclient_command(struct ane_rtclient *ane, /* ---- LOAD / CREATE / CALL wrappers ---- */ -/* The firmware never frees a program or a process, and its program table - * holds 256 entries: the 257th LOAD_PROGRAM after a boot is answered - * with a protocol error, which quarantines the device. Two - * loads of byte-identical sections therefore share one firmware program - * and one process. The key is SHA-256 over every section's id, size and - * bytes, so a client can only reach a program whose bytes it also - * supplied. Protected by ane_t6021_fw_lock. +/* The firmware program table holds 256 entries: the 257th LOAD_PROGRAM + * after a boot is answered with a protocol error. Two loads of + * byte-identical sections share one firmware program and one process. The + * key is SHA-256 over every section's id, size and bytes, so a client can + * only reach a program whose bytes it also supplied, and every file holds + * a reference on each program it loaded (ane_t6021_lifetime.h). The last + * file to close unloads the program from the firmware: TERMINATE_PROCESS, + * then DESTROY_PROGRAM, then its section buffers go back. Without that, + * every program and process stayed in the firmware until reboot and the + * firmware stopped answering after about ten MobileNet-sized loads. + * Protected by ane_t6021_fw_lock. */ -#define ANE_T6021_MAX_PROGRAMS 250 - -struct ane_t6021_prog { - u8 digest[SHA256_DIGEST_SIZE]; - u32 prog_id; - u32 proc_id; /* U32_MAX until a process exists */ - u32 fw_prog_id; - u32 fw_proc_id; -}; - -static struct ane_t6021_prog ane_t6021_progs[ANE_T6021_MAX_PROGRAMS]; -static unsigned int ane_t6021_nprogs; +static struct ane_t6021_progtab ane_t6021_progtab; static struct ane_fw_ids ane_t6021_ids; +static_assert(sizeof(ane_t6021_progtab.progs[0].digest) == SHA256_DIGEST_SIZE); + static struct ane_t6021_prog *ane_t6021_prog_find(const u8 *digest) { - unsigned int i; - - for (i = 0; i < ane_t6021_nprogs; i++) - if (!memcmp(ane_t6021_progs[i].digest, digest, - SHA256_DIGEST_SIZE)) - return &ane_t6021_progs[i]; - return NULL; + return ane_t6021_progtab_find(&ane_t6021_progtab, digest); } static struct ane_t6021_prog *ane_t6021_prog_by_id(u32 prog_id) +{ + return ane_t6021_progtab_by_id(&ane_t6021_progtab, prog_id); +} + +/* The LIVE program PROG_ID if this file holds a reference on it. */ +static struct ane_t6021_prog *ane_t6021_prog_owned(struct ane_t6021_fd *fd, + u32 prog_id, int *err) +{ + struct ane_t6021_prog *p; + + if (!fd || !ane_t6021_fdprogs_has(&fd->progs, prog_id)) { + *err = -ENOENT; + return NULL; + } + p = ane_t6021_prog_by_id(prog_id); + /* A reference the file still holds on a program that is gone means + * a firmware restart dropped it: reload and create the process again. + */ + *err = p ? 0 : -ESTALE; + return p; +} + +static int ane_t6021_unload_cmd(void *ctx, u16 opcode, u32 fw_prog, u32 fw_proc) +{ + struct ane_rtclient *ane = ctx; + struct ane_legacy_buffer *command = ane->cmd_buf; + size_t length = opcode == ANE_T6021_CMD_TERMINATE_PROCESS ? 0x10 : 0x0c; + int ret; + + if (!command || atomic_read(&ane_t6021_quarantined)) + return -ENODEV; + memset(command->cpu, 0, SZ_16K); + *(u32 *)((u8 *)command->cpu + 0x08) = cpu_to_le32(fw_prog); + if (opcode == ANE_T6021_CMD_TERMINATE_PROCESS) + *(u32 *)((u8 *)command->cpu + 0x0c) = cpu_to_le32(fw_proc); + /* A failed command has already quarantined and tried a restart. */ + ret = ane_rtclient_command(ane, command, length, opcode, 1, 3000); + if (ret) + return ret; + /* Terminating a process frees its spill buffers; their FREE requests + * can land after the acknowledgment and must be answered before the + * next command or the firmware stays locked. + */ + ret = ane_rtclient_malloc_settle(ane, 10); + if (ret) { + atomic_set(&ane_t6021_quarantined, 1); + if (!ane->recovering) + ane_rtclient_recover(ane); + } + return ret; +} + +/* Firmware side of an unload. Zero only when both commands were acked. */ +static int ane_t6021_prog_unload_op(void *ctx, struct ane_t6021_prog *p) +{ + struct ane_rtclient *ane = ctx; + int ret; + + if (!ane) + return -ENODEV; + ret = ane_t6021_prog_unload_cmds(p, ane_t6021_unload_cmd, ane); + dev_dbg(ane->dev, "unload program %u (fw %u/%d): %d (fw allocations live %u, %zu bytes)\n", + p->prog_id, p->fw_prog_id, (int)p->fw_proc_id, ret, + ane->legacy_live, ane->legacy_bytes); + return ret; +} + +/* The firmware no longer reads the sections: drop the program's references. + * A BO no other program uses stops being a program BO, and is freed or + * pooled by the usual rule at its last put. + */ +static void ane_t6021_prog_release_op(void *ctx, struct ane_t6021_prog *p) { unsigned int i; - for (i = 0; i < ane_t6021_nprogs; i++) - if (ane_t6021_progs[i].prog_id == prog_id) - return &ane_t6021_progs[i]; - return NULL; + mutex_lock(&ane_t6021_bo_lock); + for (i = 0; i < ANE_T6021_PROG_SECS; i++) { + struct ane_t6021_bo *bo = p->sec[i]; + + if (!bo) + continue; + p->sec[i] = NULL; + if (!WARN_ON(!bo->prog_refs) && !--bo->prog_refs) { + bo->fw_program = false; + bo->fw_ref = bo->fw_io; + } + kref_put(&bo->refcount, ane_t6021_bo_release); + } + mutex_unlock(&ane_t6021_bo_lock); } +static const struct ane_t6021_progtab_ops ane_t6021_progtab_ops = { + .unload = ane_t6021_prog_unload_op, + .release = ane_t6021_prog_release_op, +}; + /* Build a LOAD_PROGRAM (0x200) message in a kernel-owned buffer. The * section bytes live in the BOs the user supplied (section_ptr), the * generic binds (bufferId, bo_handle, size) patch the generic section @@ -1325,8 +1558,9 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, struct drm_ane_generic_bind *binds; struct ane_legacy_buffer *command; struct ane_t6021_fd *fd = file->driver_priv; - struct ane_t6021_prog *cached; - u32 key; + struct ane_t6021_prog *cached, *prog = NULL; + bool sent = false; + u32 key, gen; u8 digest[SHA256_DIGEST_SIZE]; size_t binds_size; int ret, i, j; @@ -1442,24 +1676,34 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, } cached = ane_t6021_prog_find(digest); if (cached) { - *prog_id = cached->prog_id; - ret = 0; + ret = ane_t6021_fdprogs_ref(&ane_t6021_progtab, &fd->progs, + cached); + if (!ret) + *prog_id = cached->prog_id; goto out; } - if (ane_t6021_nprogs == ANE_T6021_MAX_PROGRAMS) { + /* Check the file can hold the reference before the firmware loads a + * program nobody could then unload. + */ + if (!ane_t6021_fdprogs_room(&ane_t6021_progtab, &fd->progs)) { ret = -ENOSPC; goto out; } + command = ane->cmd_buf; + if (!command) { + ret = -ENODEV; + goto out; + } ret = ane_fw_id_alloc(&ane_t6021_ids, &key); if (ret) goto out; - - command = ane->cmd_buf; - if (!command) { - ret = -ENODEV; + prog = ane_t6021_progtab_reserve(&ane_t6021_progtab, digest); + if (!prog) { + ret = -ENOSPC; goto out; } + gen = ane_t6021_progtab.gen; memset(command->cpu, 0, SZ_16K); /* One 0x30-byte section record per supplied section, placed at * its IDENTITY slot: slot = id - 1 (tdprop id 7 lands at slot @@ -1487,9 +1731,14 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, * the fw may read the section any time after the * doorbell rings, including during a timeout. * Under bo_lock the handle reference cannot go - * away, so no kref is needed here. + * away. The program takes its own reference, which + * it keeps until the firmware dropped the program + * (or forever if that is uncertain). */ ane_t6021_bo_publish(bo, true); + kref_get(&bo->refcount); + bo->prog_refs++; + prog->sec[sections[i].id - 1] = bo; iova = bo->dma + sections[i].offset; } else { bo = NULL; @@ -1528,6 +1777,7 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, * 0x1b8, so the length is pinned here. */ BUILD_BUG_ON(sizeof(struct ane_csne_cmd_load_program) != 0x1b8); + sent = true; ret = ane_rtclient_command(ane, command, 0x1c0, CSNE_CMD_LOAD_PROGRAM, @@ -1535,38 +1785,49 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, if (!ret) { u8 *cmd = command->cpu; - *prog_id = le32_to_cpu(*(u32 *)(cmd + 0x1b8)); - if (*prog_id == U32_MAX) { + u32 fw_prog = le32_to_cpu(*(u32 *)(cmd + 0x1b8)); + + /* A LOAD acked without an id leaves the program HELD. */ + if (fw_prog == U32_MAX) { ret = -EPROTO; goto out; } - memcpy(ane_t6021_progs[ane_t6021_nprogs].digest, digest, - SHA256_DIGEST_SIZE); - ane_t6021_progs[ane_t6021_nprogs].fw_prog_id = *prog_id; + ane_t6021_progtab_commit(prog, key, fw_prog); + /* Room was checked under the same lock. */ + ret = ane_t6021_fdprogs_ref(&ane_t6021_progtab, &fd->progs, prog); + if (WARN_ON(ret)) + goto out; *prog_id = key; - ane_t6021_progs[ane_t6021_nprogs].prog_id = key; - ane_t6021_progs[ane_t6021_nprogs].proc_id = U32_MAX; - ane_t6021_nprogs++; + dev_dbg(ane->dev, "loaded program %u (fw %u) (fw allocations live %u, %zu bytes)\n", + key, fw_prog, ane->legacy_live, ane->legacy_bytes); + } else if (ane_t6021_progtab.gen != gen) { + /* A firmware restart already released the reservation. */ + prog = NULL; } out: + /* The firmware never saw these IOVAs: give the reservation back. */ + if (prog && !sent) + ane_t6021_progtab_cancel(prog, &ane_t6021_progtab_ops, ane); kfree(sections); kfree(binds); return ret; } static int ane_rtclient_create_process(struct ane_rtclient *ane, + struct drm_file *file, __u32 prog_id, __u32 *proc_id) { struct ane_legacy_buffer *command; - struct ane_t6021_prog *prog = ane_t6021_prog_by_id(prog_id); + struct ane_t6021_prog *prog; int ret; /* H16 uses 0x258 bytes; its body is not the H14 0x10-byte body. */ if (ane->soc->platform) return -EOPNOTSUPP; + prog = ane_t6021_prog_owned(file->driver_priv, prog_id, &ret); if (!prog) - return -ENOENT; + return ret; if (prog->proc_id != U32_MAX) { *proc_id = prog->proc_id; return 0; @@ -1785,9 +2046,10 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, ret = -ETIMEDOUT; goto unlock; } - program = ane_t6021_prog_by_id(user->prog_id); - if (!program || program->proc_id == U32_MAX || - program->proc_id != user->proc_id) { + program = ane_t6021_prog_owned(fd, user->prog_id, &ret); + if (!program) + goto unlock; + if (program->proc_id == U32_MAX || program->proc_id != user->proc_id) { ret = -ESTALE; goto unlock; } @@ -2071,6 +2333,13 @@ static void ane_t6021_postclose(struct drm_device *drm, struct drm_file *file) /* Revoke per-file authority, retaining device-owned DMA allocations. */ list_for_each_entry_safe(process, next, &fd->h16_processes, owner_node) list_del_init(&process->owner_node); + /* Unload every program this file was the last user of. A failed + * unload holds the program and its sections until a verified restart. + */ + if (ane_t6021_progtab_fd_close(&ane_t6021_progtab, &fd->progs, + &ane_t6021_progtab_ops, + to_ane_t6021_drm(drm)->ane)) + dev_warn(drm->dev, "program unload failed; sections held until the firmware restarts\n"); mutex_lock(&ane_t6021_bo_lock); list_for_each_entry_safe(bo, tmp, &fd->bos, node) ane_t6021_bo_drop(bo); @@ -2133,6 +2402,26 @@ static int ane_t6021_prog_load_ioctl(struct drm_device *drm, void *data, static struct ane_rtclient *ane_t6021_perf_ane; static bool fw_perf_mode; +/* debugfs fw_memory: what the firmware holds now. */ +static int ane_t6021_fw_memory_show(struct seq_file *m, void *v) +{ + struct drm_debugfs_entry *entry = m->private; + struct ane_rtclient *ane = entry->file.data; + unsigned int live, held; + int ret; + + ret = mutex_lock_interruptible(&ane_t6021_fw_lock); + if (ret) + return ret; + ane_t6021_progtab_count(&ane_t6021_progtab, &live, &held); + seq_printf(m, "allocations: %u\nbytes: %zu\nhigh_water: %u\n", + ane->legacy_live, ane->legacy_bytes, ane->legacy_allocated); + seq_printf(m, "programs: %u\nheld: %u\nquarantined: %d\n", live, held, + atomic_read(&ane_t6021_quarantined)); + mutex_unlock(&ane_t6021_fw_lock); + return 0; +} + static int ane_t6021_perf_mode_set(const char *val, const struct kernel_param *kp) { @@ -2193,7 +2482,7 @@ static int ane_t6021_proc_create_ioctl(struct drm_device *drm, void *data, return -ETIMEDOUT; } args->proc_id_out = 0; - ret = ane_rtclient_create_process(adrm->ane, args->prog_id, + ret = ane_rtclient_create_process(adrm->ane, file, args->prog_id, &args->proc_id_out); mutex_unlock(&ane_t6021_fw_lock); return ret; @@ -2781,6 +3070,7 @@ static int ane_rtclient_recover(struct ane_rtclient *ane) a->boot_heap_size = 0; ane->legacy_buffers = fresh; ane->legacy_allocated = 0; + ane->legacy_live = 0; ane->legacy_bytes = 0; ane->legacy_malloc_cursor = 0; memset(ane->legacy_cmd_cursor, 0, sizeof(ane->legacy_cmd_cursor)); @@ -2810,9 +3100,9 @@ static int ane_rtclient_recover(struct ane_rtclient *ane) dma_wmb(); writel(ANE_T6021_BOOT_ACK, ane->engine + ANE_MBI_SCRATCH0 + 4 * 3); ret = ane_rtclient_legacy_alloc(ane, SZ_16K); - if (ret) + if (ret < 0) goto failed; - ane->cmd_buf = &ane->legacy_buffers[ane->legacy_allocated - 1]; + ane->cmd_buf = &ane->legacy_buffers[ret]; memset(ane->cmd_buf->cpu, 0, SZ_16K); /* The firmware lock still fences commands; BO_INIT also checks recovering. * Clearing the old latch lets transport validation detect a new error. @@ -2843,8 +3133,8 @@ static int ane_rtclient_recover(struct ane_rtclient *ane) /* Opaque IDs never alias a new firmware instance. Clients reload programs. * Advance the BO epoch only after old DMA and the new boot are verified. */ - ane_t6021_nprogs = 0; ane_t6021_bo_new_epoch(); + ane_t6021_progtab_reset(&ane_t6021_progtab, &ane_t6021_progtab_ops, ane); ane_rtclient_reap_retired(ane); WRITE_ONCE(ane->recovering, false); dev_info(ane->dev, "firmware recovered; old program IDs invalid, reload and resubmit\n"); @@ -3154,12 +3444,13 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, /* One reusable command buffer for the transport * (CONFIG_GET + every ioctl); reuse is legal * because each exchange completes with the slot - * host-owned again. The 128-entry table stays + * host-owned again. The rest of the table is * for fw MALLOC replies only. */ - if (ane_rtclient_legacy_alloc(ane, SZ_16K) == 0) - ane->cmd_buf = - &ane->legacy_buffers[ane->legacy_allocated - 1]; + int slot = ane_rtclient_legacy_alloc(ane, SZ_16K); + + if (slot >= 0) + ane->cmd_buf = &ane->legacy_buffers[slot]; else cfg_err = -ENOMEM; } @@ -3287,6 +3578,9 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, drm_debugfs_add_file(&adrm->drm, "ane_timeline", ane_timeline_show, &ane->stats_ring); + if (!ane->soc->platform) + drm_debugfs_add_file(&adrm->drm, "fw_memory", + ane_t6021_fw_memory_show, ane); drmret = drm_dev_register(&adrm->drm, 0); if (!drmret) ane->registered = true; From 037449efb82e8c9b9b78ad044fe1fd24ddff7329 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 10:22:58 +1100 Subject: [PATCH 36/73] media: apple: avd: stream queued programs into SRAM with 32-bit stores The queued open-firmware transport copied each frame's instruction program into the CM3 data SRAM with memcpy_toio(), which on arm64 issues 64-bit stores for aligned runs and byte stores for unaligned ends. Every other access to that SRAM (the info block, the descriptors, the boot time clear) is a 32-bit readl()/writel(), and the firmware consumes the ring as 32-bit words. The first decode through this transport on an M2 hung the machine hard right after the doorbell, before any completion or log line, which is what an unsupported access width on that bus would do. Use __iowrite32_copy() so the ring is written with the same word stores as the rest of the SRAM. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-backend-queue.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c index a706602386c17..7b9c06269eb11 100644 --- a/drivers/media/platform/apple/avd/avd-backend-queue.c +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -182,7 +182,12 @@ static int avd_qfw_stream(struct avd_dev *avd, const u32 *words, u32 count) } pos = q->wp % q->ring_words; chunk = min3(space, count - done, q->ring_words - pos); - memcpy_toio(q->sram + q->ring_off + 4 * pos, words + done, 4 * chunk); + /* + * Word stores only: memcpy_toio() would use 64-bit and byte + * stores, and the CM3 data SRAM is only ever accessed with + * 32-bit ones elsewhere (info block, descriptors). + */ + __iowrite32_copy(q->sram + q->ring_off + 4 * pos, words + done, chunk); q->wp += chunk; done += chunk; /* Words before the count that publishes them. */ From 9844fa528d1270e08bfa1b5355100c9082ef0850 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:29:35 +1100 Subject: [PATCH 37/73] media: apple: avd: close queued transport ownership and ring gaps Keep a queued frame's FIFO and PP continuation metadata independent of later preparation on its context. Encode its FIFO index in every VP continuation and END, check the mailbox before transferring ownership, and split SRAM copies at both the ring and u32 counter boundaries. Never let a reserved mailbox retry be overtaken by a later picture of that context; already-posted pictures and unrelated contexts still overlap. Legacy VP and PP instruction semantics remain unchanged. Drain tagged replies, retain fatal I/O completions until quiesced reset, and add dynamic-debug descriptor, ring-counter and mailbox snapshots at submission, completion and timeout. A sole fatal frame must not retire before recovery merely because no other frame can arm the watchdog. Add host coverage of the production H.264 builder, queued submit, stream, IRQ and T8112 FIFO configuration. Extend the firmware Unicorn harness with reset-vector boot, CM3/NVIC routing and real mailbox status bits. The companion MIT firmware patch fixes the missing queue IRQ routes and the erroneous use of an IRQ-source bit as the outbox EMPTY status bit. The old harness injected both interrupt delivery and the incorrect status, so arbitrary-word transport tests concealed those firmware failures. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../platform/apple/avd/avd-backend-open.c | 11 +- .../platform/apple/avd/avd-backend-queue.c | 100 +++++-- .../platform/apple/avd/avd-queue-internal.h | 13 + .../media/platform/apple/avd/avd-queue-test.c | 22 ++ drivers/media/platform/apple/avd/avd-queue.c | 3 + drivers/media/platform/apple/avd/avd.h | 1 + .../media/platform/apple/avd/tests/README.rst | 68 +++++ .../apple/avd/tests/avd-fw-queue-fixes.patch | 52 ++++ .../platform/apple/avd/tests/host_source.py | 17 ++ .../platform/apple/avd/tests/program-test.py | 279 ++++++++++++++++++ .../platform/apple/avd/tests/queue-host.c | 19 ++ .../apple/avd/tests/unicorn-queue-test.py | 140 +++++++++ 12 files changed, 690 insertions(+), 35 deletions(-) create mode 100644 drivers/media/platform/apple/avd/tests/README.rst create mode 100644 drivers/media/platform/apple/avd/tests/avd-fw-queue-fixes.patch create mode 100644 drivers/media/platform/apple/avd/tests/host_source.py create mode 100644 drivers/media/platform/apple/avd/tests/program-test.py create mode 100644 drivers/media/platform/apple/avd/tests/unicorn-queue-test.py diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 9781f5647cca1..c766cf12e3b9d 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -24,7 +24,7 @@ int avd_open_build_program(struct avd_ctx *ctx) struct avd_frame *f = avd->building; struct avd_job *job = &ctx->job; size_t count = 1, pos = 0, i, j; - u32 *words, mask, start; + u32 *words, mask, start, fifo; int ret; if (!f || !job->segments) @@ -47,18 +47,23 @@ int avd_open_build_program(struct avd_ctx *ctx) AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3); /* The queued transport gives every frame in flight its own FIFO. */ ctx->fifo_idx = avd_backend_is_qfw(avd) ? f->idx : 0; + fifo = AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx); f->fifo_iova = ctx->job_fifo ?: ctx->inst.addr; ctx->job_fifo = 0; words = f->prog->cpu; - words[pos++] = AVD_OP_EXEC | mask | start | AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx); + words[pos++] = AVD_OP_EXEC | mask | start | fifo; for (i = 0; i <= job->num; i++) { for (j = 0; j < job->segments[i].num; j++) words[pos++] = job->segments[i].instructions[j]; if (i) - words[pos++] = AVD_OP_EXEC | mask | AVD_OP_EXEC_FLAG_END(i == job->num); + words[pos++] = AVD_OP_EXEC | mask | fifo | + AVD_OP_EXEC_FLAG_END(i == job->num); } f->prog_words = pos; f->cmd[0] = job->codec; + /* A mailbox retry may outlive the next preparation on this context. */ + f->cmd[1] = avd_backend_is_qfw(avd) && ctx->coded_fmt_desc->ops->pp_kicks ? + ctx->coded_fmt_desc->ops->pp_kicks(ctx) - 1 : 0; kvfree(job->segments); job->segments = NULL; return 0; diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c index 7b9c06269eb11..f62458d67af7f 100644 --- a/drivers/media/platform/apple/avd/avd-backend-queue.c +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -41,6 +41,7 @@ #define AVDQ_CM3_APERTURE(off) (0x01000000 + (off)) #define AVD_REG_MBOX0_STATUS 0x50 #define AVD_REG_MBOX0_SEND 0x54 +#define AVDQ_MBOX_STATUS_EMPTY BIT(17) #define AVDQ_STREAM_TIMEOUT_US (1000 * USEC_PER_MSEC) /* Descriptor word offsets (bytes), avd-fw docs/queue.md. */ @@ -68,6 +69,24 @@ static void __iomem *avd_qfw_desc(struct avd_qfw *q, unsigned int idx) return q->sram + AVDQ_RING_OFFSET + idx * AVDQ_DESC_BYTES; } +/* Arguments of dev_dbg() are evaluated only when dynamic debug is enabled. */ +void avd_qfw_debug(struct avd_dev *avd, struct avd_frame *f, const char *where) +{ + struct avd_qfw *q = avd->qfw; + void __iomem *d; + + if (!avd_backend_is_qfw(avd)) + return; + d = avd_qfw_desc(q, f->idx); + dev_dbg(avd->dev, + "queue %s: tag %#x slot %u host %u desc %u status %u ctag %#x rp %#x wp %#x mbox0 %#x mbox1 %#x\n", + where, f->tag, f->idx, f->state, readl(d + D_STATE), + readl(d + D_STATUS), readl(d + D_CTAG), + readl(q->sram + AVDQ_INFO_RP), readl(q->sram + AVDQ_INFO_WP), + readl(avd->mbox + AVD_REG_MBOX0_STATUS), + readl(avd->mbox + AVD_REG_MBOX1_STATUS)); +} + int avd_qfw_probe(struct avd_dev *avd) { struct resource *code, *sram; @@ -130,6 +149,8 @@ static bool avd_qfw_check(struct avd_dev *avd) q->codec_mask = info[9]; q->wp = 0; q->usable = true; + dev_dbg(avd->dev, "queue ABI %u: aperture %#x ring %#x/%u words FIFO slots %u codecs %#x\n", + info[1], info[4], q->ring_off, q->ring_words, q->fifo_slots, q->codec_mask); return true; } @@ -182,6 +203,8 @@ static int avd_qfw_stream(struct avd_dev *avd, const u32 *words, u32 count) } pos = q->wp % q->ring_words; chunk = min3(space, count - done, q->ring_words - pos); + /* The free-running u32 counter can wrap before the word ring does. */ + chunk = min_t(u64, chunk, (1ULL << 32) - q->wp); /* * Word stores only: memcpy_toio() would use 64-bit and byte * stores, and the CM3 data SRAM is only ever accessed with @@ -201,8 +224,7 @@ static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) { struct avd_qfw *q = avd->qfw; struct avd_ctx *ctx = f->ctx; - const struct avd_coded_fmt_ops *ops = ctx->coded_fmt_desc->ops; - u32 codec = f->cmd[0], vp = 0, kick, cont, mask, extra = 0; + u32 codec = f->cmd[0], vp = 0, kick, cont, mask, extra = f->cmd[1]; u32 *words = f->prog->cpu; void __iomem *d = avd_qfw_desc(q, f->idx); unsigned int i; @@ -216,14 +238,15 @@ static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) if (!(avd->capabilities & BIT(codec))) return -EOPNOTSUPP; /* The program's EXEC words name FIFO f->idx; it must be a real FIFO. */ - if (f->idx >= q->fifo_slots || ctx->fifo_idx != f->idx) + if (f->idx >= q->fifo_slots) return -EINVAL; + /* A busy inbox owns nothing; retry this same prepared frame later. */ + if (!(readl(avd->mbox + AVD_REG_MBOX0_STATUS) & AVDQ_MBOX_STATUS_EMPTY)) + return -EBUSY; for (i = 0; i < codec; i++) vp += avd->variant->vp_slots[i]; if (avd->variant->vp_slot_base[codec]) vp = avd->variant->vp_slot_base[codec]; - if (ops->pp_kicks) - extra = ops->pp_kicks(ctx) - 1; if (extra > AVDQ_MAX_PP_EXTRA) return -E2BIG; @@ -257,11 +280,15 @@ static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) /* Descriptor before the doorbell that hands it to the firmware. */ wmb(); (void)readl(d + D_STATE); /* flush posted aperture writes */ + avd_qfw_debug(avd, f, "doorbell"); + dev_dbg(avd->dev, "queue program: tag %#x vp %u words %zu kick %#x start %#x end %#x\n", + f->tag, vp, f->prog_words, kick, words[0], words[f->prog_words - 1]); writel(q->aperture + AVDQ_RING_OFFSET + f->idx * AVDQ_DESC_BYTES, avd->mbox + AVD_REG_MBOX0_SEND); /* The firmware may start feeding while the rest of the program streams in. */ ret = avd_qfw_stream(avd, words, f->prog_words); + avd_qfw_debug(avd, f, "streamed"); if (ret) { dev_err_ratelimited(avd->dev, "queue: program stream failed: %d\n", ret); /* The descriptor is already posted: only a reset releases it. */ @@ -273,34 +300,43 @@ static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) static irqreturn_t avd_qfw_irq(struct avd_dev *avd) { struct avd_qfw *q = avd->qfw; - struct avd_frame *f; - void __iomem *d; - u32 word, state, status, ctag; - unsigned int idx; - - if (!avd_mbox_pop(avd, &word)) - return IRQ_HANDLED; - if ((word & AVDQ_NOTIFY_MASK) != AVDQ_NOTIFY) { - dev_err_ratelimited(avd->dev, "queue: firmware reported %#x, resetting\n", word); - avd_qfw_fail_all(avd); - return IRQ_HANDLED; - } - idx = word & 15; - d = avd_qfw_desc(q, idx); - state = readl(d + D_STATE); - status = readl(d + D_STATUS); - ctag = readl(d + D_CTAG); - f = &avd->frames[idx]; - if (state != AVDQ_STATE_COMPLETE || ((ctag ^ word) & 0xfff0) || - f->state != AVD_FRAME_SUBMITTED || ctag != f->tag) { - dev_warn_ratelimited(avd->dev, "queue: stale completion %#x (state %u tag %#x)\n", - word, state, ctag); - return IRQ_HANDLED; + unsigned int n; + u32 word; + + /* At most sixteen completions and one fatal token can be outstanding. */ + for (n = 0; n < AVDQ_COUNT + 1 && avd_mbox_pop(avd, &word); n++) { + struct avd_frame *f; + void __iomem *d; + u32 state, status, ctag; + unsigned int idx; + + if ((word & AVDQ_NOTIFY_MASK) != AVDQ_NOTIFY) { + dev_err_ratelimited(avd->dev, + "queue: firmware reported %#x, resetting\n", word); + avd_qfw_fail_all(avd); + continue; + } + idx = word & 15; + d = avd_qfw_desc(q, idx); + state = readl(d + D_STATE); + status = readl(d + D_STATUS); + ctag = readl(d + D_CTAG); + f = &avd->frames[idx]; + avd_qfw_debug(avd, f, "completion"); + if (state != AVDQ_STATE_COMPLETE || ((ctag ^ word) & 0xfff0) || + f->state != AVD_FRAME_SUBMITTED || ctag != f->tag) { + dev_warn_ratelimited(avd->dev, "queue: stale completion %#x (state %u tag %#x)\n", + word, state, ctag); + continue; + } + if (status > 1) { + /* An I/O error is not proof of quiescence, even for one frame. */ + avd_qfw_fail_all(avd); + continue; + } + writel(0, d + D_STATE); + avd_frame_complete(avd, ctag, status ? -EINVAL : 0); } - writel(0, d + D_STATE); - avd_frame_complete(avd, ctag, status == 0 ? 0 : status == 1 ? -EINVAL : -EIO); - if (status > 1) - avd_qfw_fail_all(avd); return IRQ_HANDLED; } diff --git a/drivers/media/platform/apple/avd/avd-queue-internal.h b/drivers/media/platform/apple/avd/avd-queue-internal.h index 0e2a668d5a62f..2c09a5d85da6b 100644 --- a/drivers/media/platform/apple/avd/avd-queue-internal.h +++ b/drivers/media/platform/apple/avd/avd-queue-internal.h @@ -65,6 +65,19 @@ static inline bool avd_slot_expired(const struct avd_frame *f, unsigned long now f->state == AVD_FRAME_RESERVED) && time_after_eq(now, f->deadline); } +/* Posted frames may overlap, but never pass an earlier reserved frame. */ +static inline bool avd_slot_submit_first(const struct avd_frame *pool, + const struct avd_frame *f) +{ + unsigned int i; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (pool[i].state == AVD_FRAME_RESERVED && pool[i].ctx == f->ctx && + pool[i].seq < f->seq) + return false; + return true; +} + static inline bool avd_inbox_full(unsigned int producer, unsigned int consumer) { return ((producer + 1) & 7) == consumer; diff --git a/drivers/media/platform/apple/avd/avd-queue-test.c b/drivers/media/platform/apple/avd/avd-queue-test.c index 1efae38789b57..8d343b1b93399 100644 --- a/drivers/media/platform/apple/avd/avd-queue-test.c +++ b/drivers/media/platform/apple/avd/avd-queue-test.c @@ -186,6 +186,27 @@ static void avd_dispatch_reservation(struct kunit *test) KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); } +static void avd_submit_order(struct kunit *test) +{ + struct avd_frame *pool; + struct avd_ctx *a = (void *)test, *b = (void *)&pool; + + pool = kunit_kcalloc(test, AVD_MAX_FRAMES, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + pool[1].idx = 1; + pool[2].idx = 2; + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[0], a, 1), 0); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[1], a, 2), 0); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[2], b, 3), 0); + KUNIT_EXPECT_TRUE(test, avd_slot_submit_first(pool, &pool[0])); + KUNIT_EXPECT_FALSE(test, avd_slot_submit_first(pool, &pool[1])); + KUNIT_EXPECT_TRUE(test, avd_slot_submit_first(pool, &pool[2])); + pool[0].state = AVD_FRAME_SUBMITTED; + KUNIT_EXPECT_TRUE(test, avd_slot_submit_first(pool, &pool[1])); + avd_slot_submit_result(&pool[0], -EBUSY); + KUNIT_EXPECT_FALSE(test, avd_slot_submit_first(pool, &pool[1])); +} + static struct kunit_case avd_queue_cases[] = { KUNIT_CASE(avd_slots_and_tags), KUNIT_CASE(avd_retirement_order), @@ -193,6 +214,7 @@ static struct kunit_case avd_queue_cases[] = { KUNIT_CASE(avd_completion_parser), KUNIT_CASE(avd_command_builders), KUNIT_CASE(avd_dispatch_reservation), + KUNIT_CASE(avd_submit_order), {} }; diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index 5ca9b10accc1a..45bc67012bf5b 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -154,6 +154,8 @@ static int avd_send_frame(struct avd_dev *avd, struct avd_frame *f) unsigned long flags; int ret; + if (!avd_slot_submit_first(avd->frames, f)) + return -EBUSY; if (f->dep == AVD_DEP_PREV_DONE && !avd_slot_first(avd->frames, f)) return -EBUSY; if ((!f->ctx->session_started || @@ -202,6 +204,7 @@ static void avd_queue_worker(struct work_struct *work) dev_err(avd->dev, "frame tag %#x timed out (%p4cc, state %d, %u in flight, %s)\n", t->tag, t->ctx ? &t->ctx->coded_fmt_desc->fourcc : &(u32){0}, t->state, avd->inflight, avd->backend->name); + avd_qfw_debug(avd, t, "timeout"); avd_reset_queue(avd, -ETIMEDOUT); break; } diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 878a557996a59..f4e3c1a5f402c 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -400,6 +400,7 @@ struct avd_qfw; int avd_qfw_probe(struct avd_dev *avd); void avd_qfw_select(struct avd_dev *avd); void avd_qfw_prepare_boot(struct avd_dev *avd); +void avd_qfw_debug(struct avd_dev *avd, struct avd_frame *f, const char *where); bool avd_backend_is_qfw(const struct avd_dev *avd); int avd_backend_select(struct avd_dev *avd); bool avd_mbox_pop(struct avd_dev *avd, u32 *msg); diff --git a/drivers/media/platform/apple/avd/tests/README.rst b/drivers/media/platform/apple/avd/tests/README.rst new file mode 100644 index 0000000000000..1699aaf91b62b --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/README.rst @@ -0,0 +1,68 @@ +Queued open-firmware regressions +=============================== + +Run compilation in a controller buildbox slot. These tests use the current +production C function bodies; allocation, buffer addresses, V4L2 scheduling and +MMIO are explicit host fixtures. Linux UAPI headers and an ASan/UBSan-capable C +compiler are required:: + + sh tests/run-host.sh /tmp/avd-queue-host + python3 tests/program-test.py /tmp/avd-program + +The program test runs the H.264 header/slice emitters, open-program builder, +t8112_configure_stream(), queued descriptor submit and SRAM streaming code. +It generates a 1920x1088 IDR program with two I slices and fixture controls/IOVAs, plus +ABI 2 descriptors for FIFO indices 0 through 14. It verifies busy-mailbox +backpressure, the doorbell address, LSR FIFO addressing, decoder IRQ masks, +ring wrap and free-running u32 counter wrap. It checks mailbox draining, +full-tag rejection and fatal completion ownership until quiesced recovery. It also checks that legacy VP +EXEC words retain their existing values. Input SHA256 manifests and generated +C sources accompany the binaries. + +For T8112 (host revision 4, firmware v3), VP EXEC start is 0x2b000200 plus +FIFO_IDX, and END is 0x2b000400 plus FIFO_IDX. VP programs retain the legacy +zero low nibble. PP start is 0x2b00020f plus FIFO_IDX: its low nibble carries +the hardware FIFO mask, 15. Do not replace the VP's low nibble with the PP mask +or confuse host revision IDs with firmware image versions. + +Routed CM3 test +-------------- + +Use avd-fw PR #4's linear ABI 2 implementation (baseline commit +7f2f41369d14e0deccbac9b7d2b7c9070e48e66d). Build its images and install the +optional Unicorn Python dependency. The firmware checkout supplies the existing +transport-test.py harness:: + + python3 tests/unicorn-queue-test.py /path/to/avd-fw \ + /path/to/avd-fw/build/avd-fw-v3-t1.bin /tmp/avd-program --broken + +The baseline must reproduce both missing CM3 decoder interrupt routing and, +when those routes are repaired in the fixture, the incorrect mailbox status +EMPTY test. A firmware fix must program the queue's CM3 IRQ router and use +FIFO status bit 17, rather than IRQ-source bit 2, for outbox EMPTY. With that +firmware and its status-bit-corrected transport harness:: + + git -C /path/to/fixed-avd-fw apply --unidiff-zero \ + /path/to/linux/drivers/media/platform/apple/avd/tests/avd-fw-queue-fixes.patch + # Rebuild the firmware images, then: + python3 tests/unicorn-queue-test.py /path/to/fixed-avd-fw \ + /path/to/fixed-avd-fw/build/avd-fw-v3-t1.bin /tmp/avd-program + +The test boots the actual reset vector, checks NVIC and CM3 router masks before +calling IRQ wrappers, consumes the production Linux descriptor/program, and +raises VP/PP DONE only after the modeled program/kick. It requires descriptor +state COMPLETE and exactly the expected tagged mailbox-1 notification at FIFO +0, 1 and 14. The firmware's original all-image transport tests remain useful +for refill, ordering, backpressure and error behavior. + +Unicorn models decoder events and memory; it does not decode pixels or prove +silicon register behavior, DMA safety, throughput or decoded output correctness. +A controller-approved, single-request hardware decode and software framemd5 +comparison remain necessary. The accompanying firmware patch changes queue +mode only; apply it to the pinned open firmware source and retain its MIT +license. The unmodified firmware image cannot produce tagged completions with +the corrected mailbox-status model. + +The FIFO status and IRQ-router geometry follow the public MIT emulator at +https://github.com/eiln/avd/blob/main/avd_emu.py. The pinned avd-fw source's +Tansy routing independently identifies the same banks and bit positions. diff --git a/drivers/media/platform/apple/avd/tests/avd-fw-queue-fixes.patch b/drivers/media/platform/apple/avd/tests/avd-fw-queue-fixes.patch new file mode 100644 index 0000000000000..e4ba07084ae86 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/avd-fw-queue-fixes.patch @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: MIT +--- a/src/queue-fw.c ++++ b/src/queue-fw.c +@@ -108,0 +109,22 @@ ++/* The CM3 interrupt router is separate from the NVIC and decoder masks. */ ++static void queue_enable_irq(unsigned int irq) ++{ ++ unsigned int bank = (irq - 14u) / 32u; ++ unsigned int mask = 1u << ((irq - 14u) % 32u); ++ ++ reg_write(CM3_MBOX_IRQEN(bank), reg_read(CM3_MBOX_IRQEN(bank)) | mask); ++} ++ ++static void queue_enable_irqs(unsigned int vp) ++{ ++ unsigned int event, first; ++ ++ /* Enable only a validated descriptor's VP, not holes in the VP limit. */ ++ first = (AVD_VER == 2 ? 18u : 78u) + vp * 5u; ++ for (event = 0; event < 3; event++) ++ queue_enable_irq(first + event); /* unknown/error/DONE, not FULL */ ++ first = AVD_VER == 2 ? 38u : 62u; ++ queue_enable_irq(first); /* PP unknown */ ++ queue_enable_irq(first + 2u); /* PP DONE */ ++} ++ +@@ -114,0 +137,5 @@ ++ if (!FW->mode) { ++ /* Queue mode owns these routes; legacy mode before it is unchanged. */ ++ for (unsigned int i = 0; i < 6; i++) ++ reg_write(CM3_MBOX_IRQEN(i), 0); ++ } +@@ -246 +273 @@ +- if (!(reg_read(CM3_MBOX1_STATUS) & CM3_MBOX1_EMPTY)) ++ if (!(reg_read(CM3_MBOX1_STATUS) & CM3_MBOX_STATUS_EMPTY)) +@@ -292,0 +320,2 @@ ++ } else if (FW->q.frame[idx].status == AVDQ_OK) { ++ queue_enable_irqs(d.vp_slot); +@@ -299 +328 @@ +- (reg_read(CM3_MBOX1_STATUS) & CM3_MBOX1_EMPTY)) { ++ (reg_read(CM3_MBOX1_STATUS) & CM3_MBOX_STATUS_EMPTY)) { +--- a/src/hw/reg/reg_common.h ++++ b/src/hw/reg/reg_common.h +@@ -36,0 +37,3 @@ ++/* FIFO status is separate from the mailbox IRQ-source bits above. */ ++#define CM3_MBOX_STATUS_EMPTY BIT(17) ++ +--- a/tests/transport-test.py ++++ b/tests/transport-test.py +@@ -79 +79 @@ +- self.put(addr, 0 if self.busy else 4) ++ self.put(addr, 0x10101 if self.busy else 0x20001) diff --git a/drivers/media/platform/apple/avd/tests/host_source.py b/drivers/media/platform/apple/avd/tests/host_source.py new file mode 100644 index 0000000000000..b473b65e3c4ed --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/host_source.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Extract unchanged production function bodies for host boundary tests.""" +import re + + +def function(source, name): + match = re.search(r"^[A-Za-z_][\w *]*\b" + re.escape(name) + + r"\([^;{}]*?\)\n\{", source, re.M) + if not match: + raise ValueError(name) + start = source.index("{", match.start()) + depth = 1 + end = start + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start():end] + "\n" diff --git a/drivers/media/platform/apple/avd/tests/program-test.py b/drivers/media/platform/apple/avd/tests/program-test.py new file mode 100644 index 0000000000000..b00652d6f5e71 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/program-test.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the production H.264 emitters and open-program builder on a host. + +Only allocation, controls, and buffer addresses are fixtures. No copied hardware +program, firmware payload, or device capture is used. Requires Linux UAPI headers. +The output is a codec-generated 1920x1088 IDR with two I slices, not a +pixel decoder. Run in a build slot: program-test.py OUTPUT_DIRECTORY. +""" +import os +import hashlib +from pathlib import Path +import re +import subprocess +import sys + +from host_source import function + +ROOT = Path(sys.argv[2]).resolve() if len(sys.argv) > 2 else Path(__file__).resolve().parent.parent + + + + +PRELUDE = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef int32_t s32; +typedef uint64_t u64; +typedef uint64_t dma_addr_t; +#define AVD_H_ +#define BIT(n) (1U << (n)) +#define GENMASK(h,l) ((UINT32_MAX >> (31-(h))) & (UINT32_MAX << (l))) +#define FIELD_PREP(m,v) (((u32)(v) << __builtin_ctz(m)) & (m)) +#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) +#define DIV_ROUND_UP(n,d) (((n)+(d)-1)/(d)) +#define VISIBLE_IF_KUNIT +#define AVD_QUIRK_LSR 1 +#define AVD_QUIRK_NO_PIPE_STATE 2 +#define AVD_CODEC_H264 1 +#define AVD_CODEC_VP9 2 +#define AVD_MAX_INST 512 +#define AVD_CODEC_AV1 3 +#define AVD_MAX_FRAMES 16 +#define AVD_FRAME_SUBMITTED 2 +#define AVD_FRAME_DONE 3 +#define IRQ_HANDLED 1 +#define system_wq NULL +typedef int irqreturn_t; +#define dev_warn_ratelimited(d,...) ((void)(d)) +#define __iomem +#define USEC_PER_MSEC 1000 +#define min_t(t,a,b) ((t)(a)<(t)(b)?(t)(a):(t)(b)) +#define min3(a,b,c) ((a)<(b) ? ((a)<(c)?(a):(c)) : ((b)<(c)?(b):(c))) +#define wmb() ((void)0) +#define dev_dbg(d,...) ((void)(d)) +#define dev_err_ratelimited(d,...) ((void)(d)) +static unsigned long jiffies; +static unsigned long usecs_to_jiffies(unsigned long t) { return t/1000; } +static bool time_after(unsigned long a,unsigned long b) { return (long)(b-a)<0; } +static void usleep_range(unsigned int a,unsigned int b) { (void)a;(void)b;jiffies++; } +static u32 readl(void *p) { assert(!((uintptr_t)p&3));return *(u32 *)p; } +static void writel(u32 v,void *p) { assert(!((uintptr_t)p&3));*(u32 *)p=v; } +#define writel_relaxed writel +static void __iowrite32_copy(void *p,const void *s,size_t n) { + const u32 *src=s;for(size_t i=0;iqueued; } +static int avd_buf_alloc(struct avd_dev *d, struct avd_buf *b, size_t n) +{ (void)d; free(b->cpu); b->cpu = calloc(1,n); b->size = n; return b->cpu ? 0 : -ENOMEM; } +static struct avd_decoded_buffer *vb2_to_avd_decoded_buf(struct vb2_buffer *b) +{ return (void *)b; } +static struct avd_decoded_buffer *avd_get_ref_buf(struct avd_ctx *c, + struct vb2_v4l2_buffer *b, u64 t) { (void)c; (void)t; return (void *)b; } +static dma_addr_t vb2_dma_contig_plane_dma_addr(struct vb2_buffer *b, int p) +{ (void)p; return b->dma; } +static struct vb2_buffer *vb2_find_buffer(void *q, u64 t) +{ (void)q; (void)t; return NULL; } +static unsigned int fmt_width(struct avd_ctx *c) { (void)c; return 1920; } +static unsigned int fmt_height(struct avd_ctx *c) { (void)c; return 1088; } +static void avd_qfw_debug(struct avd_dev *d,struct avd_frame *f,const char *w) { (void)d;(void)f;(void)w; } +static unsigned int pp_kicks(struct avd_ctx *c) { return c->pp_total; } +static u32 messages[16];static unsigned int msg_read,msg_count,completed,reset_requested; +static bool avd_mbox_pop(struct avd_dev *d,u32 *word) { + (void)d;if(msg_read==msg_count)return false;*word=messages[msg_read++];return true; +} +static void avd_frame_complete(struct avd_dev *d,u32 tag,int status) { + struct avd_frame *f=&d->frames[tag&15];assert(f->tag==tag&&f->state==AVD_FRAME_SUBMITTED); + f->state=AVD_FRAME_DONE;f->status=status;completed++; +} +static void mod_delayed_work(void *w,void *v,int t) { (void)w;(void)v;(void)t;reset_requested++; } +''' + +MAIN = r''' +int main(int argc, char **argv) +{ + struct avd_variant variant = { .revision=4, .quirks=AVD_QUIRK_LSR, + .fifo_slots=15, .vp_slots={4,4,1,0}, .configure_stream=t8112_configure_stream }; + struct avd_buf program = {}; + struct avd_frame frame = { .prog=&program }; + struct avd_dev dev = { .variant=&variant, .building=&frame, .queued=true }; + struct avd_ctx ctx = { .dev=&dev, .job.codec=AVD_CODEC_H264 }; + struct avd_h264_ctx h264 = {}; + struct avd_qfw q = { .sram=calloc(1,0x10000), .aperture=0x1090000, + .ring_words=0x1800, .fifo_slots=15 }; + static const struct avd_coded_fmt_ops ops={.pp_kicks=pp_kicks}; + static const struct fmt_desc fmt={.ops=&ops}; + dev.qfw=&q;dev.ctrl=calloc(1,0x10000);dev.mbox=calloc(1,0x100);dev.capabilities=7; + ctx.coded_fmt_desc=&fmt;frame.ctx=&ctx; + struct avd_buf slice = { .size=4096, .addr=0x200800000ULL }; + struct v4l2_ctrl_h264_sps sps = { .pic_width_in_mbs_minus1=119, + .pic_height_in_map_units_minus1=67, .chroma_format_idc=1, + .flags=V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY | V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE }; + struct v4l2_ctrl_h264_decode_params decode = { .flags=V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC }; + struct v4l2_ctrl_h264_pps pps = {}; + struct v4l2_ctrl_h264_slice_params sl = { .slice_type=V4L2_H264_SLICE_TYPE_I, .header_bit_size=32 }; + struct avd_h264_run run = { .sps=&sps, .pps=&pps, .decode_params=&decode, + .slice_params=&sl, .slice_data_off=4, + .base={ .comp_out=0x200100000ULL, .y_out=0x200200000ULL, .uv_out=0x200400000ULL } }; + assert(argc==2); + ctx.priv=&h264; ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; + h264.b=&h264.bank[0]; h264.b->active_slice=&slice; + h264.b->bufs.above_info.addr=0x200600000ULL; + h264.b->bufs.pipe_state.addr=0x200610000ULL; + h264.b->bufs.ip_above.addr=0x200620000ULL; + h264.b->bufs.lf_above.addr=0x200630000ULL; + h264.b->bufs.lf_above_info.addr=0x200640000ULL; + for (unsigned int mode=0; mode<2; mode++) { + dev.queued=!mode; + for (unsigned int idx=0; idx<15; idx++) { + char path[4096]; FILE *out; + frame.idx=idx; ctx.job.num=0;ctx.pp_total=1; + ctx.job.segments=calloc(3,sizeof(*ctx.job.segments)); assert(ctx.job.segments); + stream_hdr(&ctx,&run); ctx.job.num=1;sl.first_mb_in_slice=0;slice.addr=0x200800000ULL; + stream_slice(&ctx,&run); + size_t continuation=1+ctx.job.segments[0].num+ctx.job.segments[1].num; + ctx.job.num=2;sl.first_mb_in_slice=4080;slice.addr+=0x1000; + stream_slice(&ctx,&run); + assert(!avd_open_build_program(&ctx)); + u32 fifo=dev.queued ? idx<<4 : 0; + assert(((u32 *)program.cpu)[0]==(0x2b000200U|fifo)); + assert(((u32 *)program.cpu)[continuation]==(0x2b000000U|fifo)); + assert(((u32 *)program.cpu)[frame.prog_words-1]==(0x2b000400U|fifo)); + if (dev.queued) { + frame.tag=0x10|idx;frame.fifo_iova=0x201000000ULL+idx*0x100000; + q.wp=0;memset(q.sram,0,0x10000);memset(dev.ctrl,0,0x10000); + writel(1,dev.mbox+0x50); + assert(avd_qfw_submit(&dev,&frame)==-EBUSY); + assert(!readl(q.sram+0x8000)&&!q.wp&&!readl(dev.ctrl+0x150+idx*4)); + for(unsigned int other=0;other<15;other++) + if(other!=idx)writel(0xabc000+other,dev.ctrl+0x150+other*4); + writel(0x20001,dev.mbox+0x50); + ctx.fifo_idx=99;ctx.job.num=256;ctx.pp_total=256; + /* Retry uses the saved FIFO and PP metadata. */ + assert(!avd_qfw_submit(&dev,&frame)); + for(unsigned int other=0;other<15;other++) + if(other!=idx)assert(readl(dev.ctrl+0x150+other*4)==0xabc000+other); + assert(readl(dev.mbox+0x54)==0x1098000U+idx*64); + assert(readl(q.sram+0x7ffc)==frame.prog_words); + assert(!readl(q.sram+0x8000+idx*64+D_PP_EXTRA)); + assert(readl(dev.ctrl+0x150+idx*4)==frame.fifo_iova>>8); + assert(readl(dev.ctrl+0x10c)==7&&readl(dev.ctrl+0x120)==5); + snprintf(path,sizeof(path),"%s/descriptor-%u.bin",argv[1],idx); + out=fopen(path,"wb");assert(out);assert(fwrite(q.sram+0x8000+idx*64,64,1,out)==1);assert(!fclose(out)); + } + snprintf(path,sizeof(path),"%s/%s-%u.bin",argv[1],dev.queued?"h264":"legacy",idx); + out=fopen(path,"wb"); assert(out); + /* Explicit LE wire format, independent of the host byte order. */ + for (size_t i=0; i>(8*b))&255,out); + } + assert(!fclose(out)); + } + } + /* Both ring-position wrap and u32-counter wrap must split the I/O copy. */ + for (unsigned int test=0;test<2;test++) { + u32 words[32];u32 start=test?UINT32_MAX-10:0x1800-8; + for(unsigned int i=0;i<32;i++)words[i]=0x12340000U+i; + memset(q.sram,0,0x10000);q.wp=start;writel(start,q.sram+0x7ff8); + assert(!avd_qfw_stream(&dev,words,32)); + assert(q.wp==(u32)(start+32)); + for(unsigned int i=0;i<32;i++)assert(readl(q.sram+4*((u32)(start+i)%0x1800))==words[i]); + } + /* A sole fatal completion must keep DMA buffers owned until reset. */ + memset(q.sram,0,0x10000);jiffies=123; + dev.frames[0].tag=0x10;dev.frames[0].state=AVD_FRAME_SUBMITTED; + void *desc=q.sram+0x8000; + writel(2,desc+D_STATE);writel(2,desc+D_STATUS);writel(0x10,desc+D_CTAG); + messages[0]=0x54530010;msg_count=1;msg_read=0; + assert(avd_qfw_irq(&dev)==IRQ_HANDLED); + assert(!completed&&reset_requested&&dev.frames[0].state==AVD_FRAME_SUBMITTED); + assert(dev.frames[0].deadline==jiffies); + /* Drain both valid tagged words and reject a stale full tag. */ + dev.frames[1].tag=0x11;dev.frames[1].state=AVD_FRAME_SUBMITTED; + writel(0,desc+D_STATUS); + writel(2,desc+64+D_STATE);writel(0x11,desc+64+D_CTAG); + messages[1]=0x54530011;msg_count=2;msg_read=0; + avd_qfw_irq(&dev);assert(completed==2&&msg_read==2); + dev.frames[0].state=AVD_FRAME_SUBMITTED;dev.frames[0].tag=0x20; + writel(2,desc+D_STATE);messages[0]=0x54530010;msg_count=1;msg_read=0; + avd_qfw_irq(&dev);assert(completed==2&&dev.frames[0].state==AVD_FRAME_SUBMITTED); + free(program.cpu);free(q.sram);free(dev.ctrl);free(dev.mbox); + puts("PASS: production H.264 IDR program, queued FIFO 0..14 and unchanged legacy EXECs"); +} +''' + + +def main(): + out = Path(sys.argv[1]).resolve() + out.mkdir(parents=True, exist_ok=True) + names = ['avd-h264.c', 'avd-inst.h', 'avd-backend-open.c', 'avd-backend-queue.c', 'avd-hw.c'] + (out / 'inputs.sha256').write_text(''.join(hashlib.sha256((ROOT / n).read_bytes()).hexdigest()+' '+n+'\n' for n in names)) + h264 = (ROOT / 'avd-h264.c').read_text().split('static int avd_h264_alloc_bufs(')[0] + h264 = re.sub(r'^#include.*\n', '', h264, flags=re.M) + inst = (ROOT / 'avd-inst.h').read_text() + inst = re.sub(r'^#include.*\n', '', inst, flags=re.M) + # Use the actual codec push/address/comp helpers, not fixture emitters. + builder = function((ROOT / 'avd-backend-open.c').read_text(), 'avd_open_build_program') + code = out / 'program.c' + qfw = (ROOT / 'avd-backend-queue.c').read_text() + wire = qfw[qfw.index('#define AVDQ_MAGIC'):qfw.index('/* Arguments of dev_dbg()')] + hw = (ROOT / 'avd-hw.c').read_text() + regs = hw[hw.index('#define AVD_V3_'):hw.index('int avd_boot(')] + config = '#define w32(r,v) writel((v),avd->ctrl+(r))\n#define m32(r,v) w32(r,(v)|readl(avd->ctrl+(r)))\n' + config += function(hw, 't8112_configure_stream') + code.write_text(PRELUDE + inst + h264 + builder + wire + regs + config + + function(qfw, 'avd_qfw_stream') + function(qfw, 'avd_qfw_submit') + + function(qfw, 'avd_qfw_fail_all') + function(qfw, 'avd_qfw_irq') + MAIN) + exe = out / 'program-test' + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-Wall', '-Wextra', + '-Werror', '-Wno-sign-compare', '-Wno-unused-function', + '-fsanitize=address,undefined', '-g', str(code), '-o', str(exe)], check=True) + subprocess.run([str(exe), str(out)], check=True) + + +if __name__ == '__main__': + main() diff --git a/drivers/media/platform/apple/avd/tests/queue-host.c b/drivers/media/platform/apple/avd/tests/queue-host.c index 5983d7ad5d6a3..ac47decffe3cf 100644 --- a/drivers/media/platform/apple/avd/tests/queue-host.c +++ b/drivers/media/platform/apple/avd/tests/queue-host.c @@ -74,6 +74,25 @@ int main(void) assert(avd_apple_parse_reply(record, 19, &layout, &reply) == -EMSGSIZE); } assert(completed > 1000); + { + struct avd_frame ordered[AVD_MAX_FRAMES] = {}; + struct avd_ctx *a = (void *)&clients[0], *b = (void *)&clients[1]; + + ordered[1].idx = 1; + ordered[2].idx = 2; + assert(!avd_slot_reserve(&ordered[0], a, 1)); + assert(!avd_slot_reserve(&ordered[1], a, 2)); + assert(!avd_slot_reserve(&ordered[2], b, 3)); + assert(avd_slot_submit_first(ordered, &ordered[0])); + assert(!avd_slot_submit_first(ordered, &ordered[1])); + assert(avd_slot_submit_first(ordered, &ordered[2])); + ordered[0].state = AVD_FRAME_SUBMITTED; + assert(avd_slot_submit_first(ordered, &ordered[1])); + avd_slot_submit_result(&ordered[0], -EBUSY); + assert(!avd_slot_submit_first(ordered, &ordered[1])); + ordered[0].state = AVD_FRAME_DONE; + assert(avd_slot_submit_first(ordered, &ordered[1])); + } printf("PASS: 100000 shuffled operations, %u ordered retirements\n", completed); return 0; } diff --git a/drivers/media/platform/apple/avd/tests/unicorn-queue-test.py b/drivers/media/platform/apple/avd/tests/unicorn-queue-test.py new file mode 100644 index 0000000000000..7ed280bde00da --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/unicorn-queue-test.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run codec-generated Linux descriptors/programs through actual CM3 code. + +Usage: unicorn-queue-test.py FIRMWARE_TREE IMAGE PROGRAM_DIRECTORY [--broken] +The firmware tree provides its existing tests/transport-test.py MMIO harness. +Extend that harness with reset-vector boot, real FIFO status bits, IRQ router +and NVIC checks, and VP/PP events following a complete generated program. +Decoder arithmetic/DMA/pixels remain synthetic; this is a transport regression. +--broken proves the old image stalls both at IRQ routing and at publication. +""" +import importlib.util +from pathlib import Path +import struct +import sys + +from unicorn.arm_const import UC_ARM_REG_SP + +firmware = Path(sys.argv[1]).resolve() +image = Path(sys.argv[2]).resolve() +programs = Path(sys.argv[3]).resolve() +broken = '--broken' in sys.argv[4:] +spec = importlib.util.spec_from_file_location('transport', firmware / 'tests/transport-test.py') +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class RoutedTransport(module.Transport): + def __init__(self, image): + self.booting = False + self.suppressed = set() + super().__init__(image) + self.booting = True + self.uc.reg_write(UC_ARM_REG_SP, self.u32(0)) + self.uc.emu_start(self.u32(4), 0, count=500000) + assert not self.booting, 'reset vector did not publish boot-ready' + assert self.u32(0x50010010) & 2, 'CM3 mailbox-0 not-empty disabled' + # Linux avd_boot() AP mailbox-1 notification routing, separate from CM3. + self.ap_irq_enable = 8 + + def read(self, uc, access, addr, size, value, user): + super().read(uc, access, addr, size, value, user) + if addr == 0x5001005c: + # BIT(2) belongs to IRQ status, not this FIFO status register. + # Empty FIFO has bit 17 set, ENABLE remains bit 0. + self.put(addr, 1 | (0x10100 if self.busy else 1 << 17)) + + def write(self, uc, access, addr, size, value, user): + super().write(uc, access, addr, size, value, user) + if self.booting and addr == 0x50010090 and value == 1: + self.booting = False + uc.emu_stop() + if addr == 0x50010060: + assert self.ap_irq_enable == 8, 'AP mailbox-1 IRQ disabled' + + def route(self, irq): + if irq < 14: + router = self.u32(0x50010010) & (1 << irq) + else: + offset = irq - 14 + router = self.u32(0x50010014 + (offset // 32) * 4) & (1 << (offset % 32)) + nvic = self.u32(0xe000e100 + (irq // 32) * 4) & (1 << (irq % 32)) + if not (router and nvic): + self.suppressed.add(irq) + return False + self.call('irq' + str(irq)) + return True + + def descriptor(self, idx): + data = (programs / f'descriptor-{idx}.bin').read_bytes() + d = struct.unpack('<16I', data) + assert d[0:4] == (0x41565131, 0x10 | idx, 1, 4) + assert d[4:6] == (0, 0) and d[8:12] == (1, 0, 0, 1) + assert d[7] == 0x2b00020f | (idx << 4) + assert d[12:] == (0, 0, 0, 0) + self.uc.mem_write(0x10008000 + idx * 64, data) + # Actual Linux t8112_configure_stream() results are asserted by program-test. + self.put(self.decode + 0x10c, 7) # VP4 decoder-local mask + self.put(self.decode + 0x120, 5) # PP decoder-local mask + self.put(0x50010058, self.ring + idx * 64) + assert self.route(1), 'AP mailbox-0 doorbell was not delivered' + words = list(struct.unpack('<' + 'I' * d[6], (programs / f'h264-{idx}.bin').read_bytes())) + assert words[0] == 0x2b000200 | (idx << 4) + assert words[-1] == 0x2b000400 | (idx << 4) + module.ring_write(self, 0, words) + self.put(0x10007ffc, len(words)) + return words + + def execute(self, words): + vp_done = pp_done = False + for _ in range(40): + self.poll() + if not vp_done and self.stores == words: + # Generate DONE only after the complete real codec program was fed. + assert self.u32(self.decode + 0x10c) & 4 + vp_done = self.route(100) # v3 H.264 VP4 DONE + if self.kicks and not pp_done: + assert self.u32(self.decode + 0x120) & 4 + pp_done = self.route(64) + assert self.stores == words + assert self.u32(0x10007ff8) == self.u32(0x10007ffc) == len(words) + assert not self.fetches and not self.ring_writes + return vp_done, pp_done + + +def force_decoder_routes(t): + # Isolate the independent status-register bug in the unfixed image. + for irq in (100, 64): + offset = irq - 14 + addr = 0x50010014 + (offset // 32) * 4 + t.put(addr, t.u32(addr) | (1 << (offset % 32))) + + +assert image.stem == 'avd-fw-v3-t1', 'this first-light regression targets T8112' +if broken: + t = RoutedTransport(image) + words = t.descriptor(0) + assert t.execute(words) == (False, False) + assert 100 in t.suppressed and t.result(0) == [1, 0, 0] and not t.notifications + print('REPRODUCED: v3 VP4 IRQ100 masked; program consumed but descriptor remains SUBMITTED') + t = RoutedTransport(image) + words = t.descriptor(0) + force_decoder_routes(t) + assert t.execute(words) == (True, True) + assert t.kicks == [0x2b00020f] + assert t.result(0) == [1, 0, 0] and not t.notifications + print('REPRODUCED: VP/PP done with routes repaired; EMPTY bit17 never passes firmware BIT(2) test') +else: + for idx in (0, 1, 14): + t = RoutedTransport(image) + words = t.descriptor(idx) + assert t.execute(words) == (True, True) + assert t.u32(0x50010018) == 0x50000 # PP unknown/DONE + assert t.u32(0x5001001c) == 0x700000 # VP4 unknown/error/DONE only + assert all(t.u32(a) == 0 for a in (0x50010014, 0x50010020, 0x50010024, 0x50010028)) + assert t.kicks == [0x2b00020f | (idx << 4)] + assert t.result(idx) == [2, 0, 0x10 | idx] + assert t.notifications == [0x54530010 | idx] + print(f'PASS: v3-t1 slot {idx}, {len(words)} production H.264 words, ' + f'IRQ1 -> VP4 IRQ100 -> PP IRQ64 -> tagged completion {t.notifications[0]:#x}') From 6e3a36cd298ed36d9962641849a7596181b9171f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 14:34:23 +1100 Subject: [PATCH 38/73] media: v4l2-mem2mem: support detached buffers for pipelined dispatch Plain job_finish() warns for every HOLD_CAPTURE-capable queue, even when the current source is a final slice. buf_done_and_job_finish() releases OUTPUT and final CAPTURE immediately, so neither API can hand both active buffers to an asynchronous frame slot while releasing the scheduler. Add job_finish_detach() to atomically remove the current pair and finish dispatch without completing either buffer. Document the driver's DMA ownership and eventual completion obligations. Reject partial HOLD slices; they continue to use the existing held-buffer completion API. Use this handoff for AVD's final slices to preserve early dispatch and VP/PP overlap. Return slot-owned buffers only after tagged retirement or quiesced recovery. Reject stale deferred submit work whose context no longer owns the running m2m dispatch. Keep existing HOLD warnings intact. Exercise the production core helpers and AVD submit/send/retire with a strict host scheduler: held/final slices, overlapping pictures, two contexts, capacity and inbox retries, stale work and error paths. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../media/platform/apple/avd/avd-protocol.rst | 15 +- drivers/media/platform/apple/avd/avd-queue.c | 26 +- drivers/media/platform/apple/avd/avd-queue.h | 2 +- .../media/platform/apple/avd/tests/README.rst | 7 + .../platform/apple/avd/tests/dispatch-test.py | 272 ++++++++++++++++++ drivers/media/v4l2-core/v4l2-mem2mem.c | 28 ++ include/media/v4l2-mem2mem.h | 18 ++ 7 files changed, 345 insertions(+), 23 deletions(-) create mode 100644 drivers/media/platform/apple/avd/tests/dispatch-test.py diff --git a/drivers/media/platform/apple/avd/avd-protocol.rst b/drivers/media/platform/apple/avd/avd-protocol.rst index ce9eaf0bf2720..ed75e333dd739 100644 --- a/drivers/media/platform/apple/avd/avd-protocol.rst +++ b/drivers/media/platform/apple/avd/avd-protocol.rst @@ -22,12 +22,15 @@ the old host-side PP kick. A native frame_ops program must use the queued avd-fw transport before it can replace the compatibility path; this backend returns EOPNOTSUPP for native programs rather than guessing their wire format. -The V4L2 core currently warns unconditionally when job_finish() is used on a -queue advertising HOLD_CAPTURE_BUF. Such queues retain the scheduler until -retirement and use buf_done_and_job_finish(), including held slice requests. -Other queues release the scheduler as soon as their buffers belong to a slot. -Removing this restriction requires a V4L2 core API change outside this driver's -scope. It is not safe to toggle subsystem_flags around job_finish(). +Partial HOLD_CAPTURE slice requests use buf_done_and_job_finish(), returning +the copied OUTPUT slice while CAPTURE remains held on the ready list. Final +slices use job_finish_detach(): it atomically transfers both ready buffers to +the frame slot and releases m2m dispatch without completing either buffer. +The next picture can prepare while the decoder still owns the earlier frame. +The slot returns those buffers only after its tagged completion or quiesced +recovery. The queue capability flags are never toggled around job_finish(). +This requires the matching V4L2 m2m detach API; an external-module deployment +must install the matching m2m module as well as apple-avd. Apple Borage transport ---------------------- diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index 45bc67012bf5b..166ea61a9ee26 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -316,6 +316,9 @@ static void avd_queue_submit_work(struct work_struct *work) int ret; mutex_lock(&avd->queue_mutex); + /* A queued retry can outlive drain/abort finishing its m2m dispatch. */ + if (v4l2_m2m_get_curr_priv(avd->m2m_dev) != ctx) + goto unlock; spin_lock_irqsave(&avd->queue_lock, flags); if (avd->deferred_ctx == ctx) avd->deferred_ctx = NULL; @@ -360,9 +363,9 @@ static void avd_queue_submit_work(struct work_struct *work) f->src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); f->dst = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); /* - * Only a slice that holds its CAPTURE buffer needs the m2m scheduler - * until retirement. The last slice of a picture releases both buffers - * to the slot, so the next picture can be prepared while it decodes. + * Partial slices retain CAPTURE through the ordinary m2m HOLD API. + * Final slices detach both buffers and release dispatch atomically; + * neither DMA-owned buffer is completed by that early handoff. */ f->scheduler_held = (ctx->fh.m2m_ctx->out_q_ctx.q.subsystem_flags & VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF) && @@ -432,11 +435,6 @@ static void avd_queue_submit_work(struct work_struct *work) goto unlock; } v4l2_m2m_buf_copy_metadata(f->src, f->dst); - if (!f->scheduler_held) { - v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); - v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); - } - f->dst->is_held = false; ret = avd_send_frame(avd, f); if (ret && ret != -EBUSY) { spin_lock_irqsave(&avd->queue_lock, flags); @@ -444,23 +442,19 @@ static void avd_queue_submit_work(struct work_struct *work) f->state = AVD_FRAME_DONE; spin_unlock_irqrestore(&avd->queue_lock, flags); } - /* Buffers now belong to the slot, not to the m2m scheduler. */ + /* Atomically detach both buffers to the slot and release dispatch. */ if (!f->scheduler_held) - v4l2_m2m_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx); + v4l2_m2m_job_finish_detach(avd->m2m_dev, ctx->fh.m2m_ctx); mod_delayed_work(system_wq, &avd->queue_work, 0); goto unlock; fail: - /* Detach before retirement, including partially prepared codec state. */ - if (!f->scheduler_held) { - v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); - v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); - } + /* Failed preparation still transfers its buffers before retirement. */ spin_lock_irqsave(&avd->queue_lock, flags); f->status = ret; f->state = AVD_FRAME_DONE; spin_unlock_irqrestore(&avd->queue_lock, flags); if (!f->scheduler_held) - v4l2_m2m_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx); + v4l2_m2m_job_finish_detach(avd->m2m_dev, ctx->fh.m2m_ctx); mod_delayed_work(system_wq, &avd->queue_work, 0); goto unlock; fail_no_pm: diff --git a/drivers/media/platform/apple/avd/avd-queue.h b/drivers/media/platform/apple/avd/avd-queue.h index cd912259c636a..9aa15c73eeee3 100644 --- a/drivers/media/platform/apple/avd/avd-queue.h +++ b/drivers/media/platform/apple/avd/avd-queue.h @@ -50,7 +50,7 @@ struct avd_frame { void *codec_priv; /* per-frame codec state, owned by the codec */ unsigned long deadline; /* jiffies, per-frame watchdog */ - /* Legacy m2m HOLD_CAPTURE API retains scheduler ownership until retire. */ + /* True while the buffers still use m2m's held-CAPTURE completion path. */ bool scheduler_held; /* VP to PP instruction FIFO for this frame (open backends). */ dma_addr_t fifo_iova; diff --git a/drivers/media/platform/apple/avd/tests/README.rst b/drivers/media/platform/apple/avd/tests/README.rst index 1699aaf91b62b..947dac1bf5a8a 100644 --- a/drivers/media/platform/apple/avd/tests/README.rst +++ b/drivers/media/platform/apple/avd/tests/README.rst @@ -8,6 +8,7 @@ compiler are required:: sh tests/run-host.sh /tmp/avd-queue-host python3 tests/program-test.py /tmp/avd-program + python3 tests/dispatch-test.py /tmp/avd-dispatch The program test runs the H.264 header/slice emitters, open-program builder, t8112_configure_stream(), queued descriptor submit and SRAM streaming code. @@ -25,6 +26,12 @@ zero low nibble. PP start is 0x2b00020f plus FIFO_IDX: its low nibble carries the hardware FIFO mask, 15. Do not replace the VP's low nibble with the PP mask or confuse host revision IDs with firmware image versions. +The dispatch test compiles the production submit/retire functions and V4L2 +core detach/held-buffer helpers. It checks early handoff without completing +DMA-owned buffers, held slices, two contexts, deferred capacity, stale work, +prepare/PM/abort failures and exactly-once scheduler release. The matching +V4L2 m2m core source must be present in the kernel checkout. + Routed CM3 test -------------- diff --git a/drivers/media/platform/apple/avd/tests/dispatch-test.py b/drivers/media/platform/apple/avd/tests/dispatch-test.py new file mode 100644 index 0000000000000..6555905c1bdcc --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/dispatch-test.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise production submission/retirement with a strict fake m2m scheduler. + +The boundary rejects job_finish on HOLD_CAPTURE queues and a finish for any +context other than the running one. PM, work scheduling and codec preparation +are fixtures; avd_queue_submit_work(), avd_retire() and slot helpers are real. +""" +import os +from pathlib import Path +import hashlib +import subprocess +import sys + +HERE = Path(__file__).resolve().parent +ROOT = Path(sys.argv[2]).resolve() if len(sys.argv) > 2 else HERE.parent +CORE = Path(sys.argv[3]).resolve() if len(sys.argv) > 3 else ROOT.parents[4] / 'drivers/media/v4l2-core/v4l2-mem2mem.c' +from host_source import function + +PRELUDE = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define AVD_H_ +#define AVD_QUEUE_H_ +#define AVD_MAX_FRAMES 16 +#define VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF 1 +#define V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF 1 +#define AVD_FRAME_TIMEOUT 2000 +#define AVD_CTX_MAX_HW_ERRORS 1 +#define AVD_PROGRAM_SIZE 4096 +#define GFP_KERNEL 0 +#define dma_wmb() ((void)0) +#define system_wq NULL +#define READ_ONCE(v) (v) +static bool warn_on(bool v) { assert(!v);return v; } +#define WARN_ON(v) warn_on(!!(v)) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define spin_lock_irqsave(l,f) do { (void)(l); (f)=0; } while(0) +#define spin_unlock_irqrestore(l,f) do { (void)(l); (void)(f); } while(0) +#define mutex_lock(l) ((void)(l)) +#define mutex_unlock(l) ((void)(l)) +#define dev_err(d,...) ((void)(d)) +static unsigned long jiffies=1; +struct work_struct { int unused; }; +struct vb2_v4l2_buffer { unsigned int flags; bool is_held; int done; + struct { struct { void *req; } req_obj; } vb2_buf; +}; +struct avd_ctx; +struct avd_dev; +struct avd_buf { void *cpu; size_t size; }; +''' + +BOUNDARY = r''' +struct fake_m2m { + struct { struct { unsigned int subsystem_flags; } q; } out_q_ctx; + struct vb2_v4l2_buffer *src,*dst; + struct avd_ctx *ctx; + void *priv; +}; +#define v4l2_m2m_ctx fake_m2m +struct v4l2_m2m_dev { struct fake_m2m *curr_ctx; int job_spinlock; }; +static struct v4l2_m2m_dev core; +struct avd_codec_frame_ops { + int (*prepare)(struct avd_ctx *,struct avd_frame *); + void (*retire)(struct avd_ctx *,struct avd_frame *,enum vb2_buffer_state); +}; +struct avd_coded_fmt_ops { + int (*run)(struct avd_ctx *); + void (*done)(struct avd_ctx *,struct vb2_v4l2_buffer *,struct vb2_v4l2_buffer *,enum vb2_buffer_state); +}; +struct format { const struct avd_codec_frame_ops *frame_ops; const struct avd_coded_fmt_ops *ops; }; +struct avd_ctx { + struct avd_dev *dev; struct { struct fake_m2m *m2m_ctx; } fh; + const struct format *coded_fmt_desc; struct work_struct submit_work; + bool draining,broken; unsigned int hw_errors; enum avd_frame_dep next_dep; + bool session_started; u64 session_generation; + int ctrl_hdl; +}; +struct avd_variant { size_t fp_size; }; +struct avd_backend_ops { + int (*session_start)(struct avd_ctx *); + int (*submit)(struct avd_dev *,struct avd_frame *); +}; +struct avd_dev { + const struct avd_variant *variant; + const struct avd_backend_ops *backend; u64 fw_generation; + struct avd_frame frames[AVD_MAX_FRAMES]; struct avd_buf *frame_fp[AVD_MAX_FRAMES]; + struct avd_ctx *deferred_ctx; struct avd_frame *building; + unsigned int inflight; u64 frame_seq; bool compat_held,stopping,faulted; + int queue_lock,queue_mutex,queue_work; void *dev,*m2m_dev; +}; +static struct avd_ctx *running; +static unsigned int finishes,raw_finishes,pm_refs; +static int prepare_result,pm_error,send_result; +static u32 send_tags[64];static unsigned int send_count; +static unsigned int capacity=1; +static struct vb2_v4l2_buffer *v4l2_m2m_next_src_buf(struct fake_m2m *m) { return m->src; } +static struct vb2_v4l2_buffer *v4l2_m2m_next_dst_buf(struct fake_m2m *m) { return m->dst; } +static struct vb2_v4l2_buffer *v4l2_m2m_src_buf_remove(struct fake_m2m *m) { + struct vb2_v4l2_buffer *b=m->src;m->src=NULL;return b; +} +static struct vb2_v4l2_buffer *v4l2_m2m_dst_buf_remove(struct fake_m2m *m) { + struct vb2_v4l2_buffer *b=m->dst;m->dst=NULL;return b; +} +static void finish(struct fake_m2m *m) { + assert(running==m->ctx&&core.curr_ctx==m);running=NULL;core.curr_ctx=NULL;finishes++; +} +static bool _v4l2_m2m_job_finish(struct v4l2_m2m_dev *d,struct fake_m2m *m) +{ assert(d==&core);finish(m);return true; } +static void v4l2_m2m_schedule_next_job(struct v4l2_m2m_dev *d,struct fake_m2m *m) +{ (void)d;assert(!m->src&&(!m->dst||m->dst->is_held)); } +#define EXPORT_SYMBOL_GPL(f) +static void __attribute__((unused)) v4l2_m2m_job_finish(void *d,struct fake_m2m *m) { + (void)d; assert(!(m->out_q_ctx.q.subsystem_flags&VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF)); + raw_finishes++; finish(m); +} +static void v4l2_m2m_buf_done(struct vb2_v4l2_buffer *b,enum vb2_buffer_state s) +{ (void)s; assert(b && !b->done); b->done++; } +static void v4l2_m2m_buf_copy_metadata(void *s,void *d) { (void)s;(void)d; } +static void v4l2_ctrl_request_complete(void *r,void *h) { (void)r;(void)h; } +static int v4l2_ctrl_request_setup(void *r,void *h) { (void)r;(void)h; return 0; } +static unsigned int avd_max_inflight(struct avd_dev *d) { (void)d; return capacity; } +static bool avd_backend_is_apple(struct avd_dev *d) { (void)d; return false; } +static int pm_runtime_resume_and_get(void *d) { (void)d; if(pm_error)return pm_error; pm_refs++;return 0; } +static void pm_runtime_mark_last_busy(void *d) { (void)d; } +static void pm_runtime_put_autosuspend(void *d) { (void)d; assert(pm_refs);pm_refs--; } +static void avd_pm_queue_changed(struct avd_dev *d) { (void)d; } +static void mod_delayed_work(void *w,void *v,int t) { (void)w;(void)v;(void)t; } +static int avd_buf_alloc(struct avd_dev *d,struct avd_buf *b,size_t n) { (void)d;(void)b;(void)n;return -ENOMEM; } +static void *devm_kzalloc(void *d,size_t n,int f) { (void)d;(void)f;return calloc(1,n); } +static int transport_submit(struct avd_dev *d,struct avd_frame *f) { + (void)d;if(!send_result)send_tags[send_count++]=f->tag;return send_result; +} +static int codec_prepare(struct avd_ctx *c,struct avd_frame *f) { (void)c;(void)f;return prepare_result; } +void avd_job_finish(struct avd_ctx *c,enum vb2_buffer_state state); +static int codec_run(struct avd_ctx *c) { + if(!prepare_result&&(c->fh.m2m_ctx->src->flags&V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF)) + avd_job_finish(c,VB2_BUF_STATE_DONE); + return prepare_result; +} +''' + +MAIN = r''' +static void submit(struct avd_ctx *c) { assert(!running);running=c;core.curr_ctx=c->fh.m2m_ctx;avd_queue_submit_work(&c->submit_work); } +static struct avd_frame *owned(struct avd_dev *d,struct avd_ctx *c) { + for(unsigned int i=0;i<16;i++)if(d->frames[i].ctx==c&&d->frames[i].state!=AVD_FRAME_FREE)return &d->frames[i]; + return NULL; +} +static void complete(struct avd_dev *d,struct avd_ctx *c,int status) { + struct avd_frame *f=owned(d,c);assert(f);f->state=AVD_FRAME_DONE;f->status=status;avd_retire(d); +} +int main(void) { + static const struct avd_backend_ops backend={.submit=transport_submit}; + struct avd_dev d={.m2m_dev=&core,.backend=&backend};struct avd_buf programs[16]={}; + static const struct avd_coded_fmt_ops ops={.run=codec_run}; + static const struct format fmt={.ops=&ops}; + struct fake_m2m m[2]={};struct avd_ctx c[2]={}; + struct vb2_v4l2_buffer src[20]={},dst[20]={}; + for(unsigned int i=0;i<16;i++){d.frames[i].idx=i;d.frames[i].prog=&programs[i];} + for(unsigned int i=0;i<2;i++){c[i].dev=&d;c[i].fh.m2m_ctx=&m[i];c[i].coded_fmt_desc=&fmt;m[i].ctx=&c[i];m[i].priv=&c[i];} + /* Non-HOLD jobs detach once, finish dispatch once, retire buffers later. */ + m[0].src=&src[0];m[0].dst=&dst[0];submit(&c[0]);assert(!running&&d.inflight==1&&finishes==1); + assert(!src[0].done&&!dst[0].done); + /* The next context keeps the running dispatch while capacity is full. */ + m[1].src=&src[1];m[1].dst=&dst[1];submit(&c[1]); + assert(running==&c[1]&&d.deferred_ctx==&c[1]&&finishes==1); + complete(&d,&c[0],0);assert(src[0].done&&dst[0].done&&!d.inflight); + d.deferred_ctx=NULL;avd_queue_submit_work(&c[1].submit_work); + assert(!running&&finishes==2);complete(&d,&c[1],0);assert(finishes==2); + /* Final slices release dispatch early without returning either DMA buffer. */ + m[0].out_q_ctx.q.subsystem_flags=VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF; + m[0].src=&src[2];m[0].dst=&dst[2];submit(&c[0]); + assert(!running&&finishes==3&&!owned(&d,&c[0])->scheduler_held); + assert(!m[0].src&&!m[0].dst&&!src[2].done&&!dst[2].done); + complete(&d,&c[0],0);assert(finishes==3&&!raw_finishes); + /* A held intermediate slice returns only OUTPUT and keeps CAPTURE. */ + src[3].flags=V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF; + m[0].src=&src[3];m[0].dst=&dst[3];submit(&c[0]); + assert(!d.inflight&&!running&&src[3].done&&!dst[3].done&&dst[3].is_held); + src[13].flags=V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;m[0].src=&src[13];submit(&c[0]); + assert(!d.inflight&&!running&&src[13].done&&!dst[3].done&&dst[3].is_held); + m[0].src=&src[4];submit(&c[0]); + assert(!running&&!src[4].done&&!dst[3].done&&!dst[3].is_held); + complete(&d,&c[0],0);assert(src[4].done&&dst[3].done); + /* Final-slice prepare failures detach early and retire buffers with error. */ + prepare_result=-EINVAL;m[0].src=&src[5];m[0].dst=&dst[5];submit(&c[0]); + assert(!running&&!src[5].done&&!dst[5].done);avd_retire(&d); + assert(src[5].done&&dst[5].done); + c[0].broken=false;prepare_result=0; + /* PM failure and stopped-stream dispatch finish the current job once. */ + pm_error=-EIO;m[0].src=&src[6];m[0].dst=&dst[6];submit(&c[0]);assert(!running&&!d.inflight); + pm_error=0;c[0].draining=true;m[0].src=&src[7];m[0].dst=&dst[7];submit(&c[0]); + assert(!running&&src[7].done&&dst[7].done&&!d.inflight&&!pm_refs); + /* The native prepare failure path detaches and finishes a non-HOLD job. */ + static const struct avd_codec_frame_ops native_ops={.prepare=codec_prepare}; + static const struct format native_fmt={.frame_ops=&native_ops}; + c[1].coded_fmt_desc=&native_fmt;prepare_result=-EINVAL; + programs[0].cpu=&programs[0];m[1].src=&src[8];m[1].dst=&dst[8]; + submit(&c[1]);assert(!running);avd_retire(&d); + assert(src[8].done&&dst[8].done&&!d.inflight&&!pm_refs); + /* The next picture really prepares while the first still owns DMA buffers. */ + c[0].draining=false;c[0].broken=false;prepare_result=0;capacity=2; + m[0].src=&src[9];m[0].dst=&dst[9];submit(&c[0]); + m[0].src=&src[10];m[0].dst=&dst[10];submit(&c[0]); + assert(d.inflight==2&&!running&&!src[9].done&&!dst[9].done&&!src[10].done&&!dst[10].done); + complete(&d,&c[0],0);assert(src[9].done&&dst[9].done&&!src[10].done&&!dst[10].done); + complete(&d,&c[0],0);assert(src[10].done&&dst[10].done&&!d.inflight&&!pm_refs&&!raw_finishes); + unsigned int before=finishes; + avd_queue_submit_work(&c[0].submit_work);assert(finishes==before&&!d.inflight); + capacity=1;c[1].coded_fmt_desc=&fmt;c[1].broken=false; + m[0].src=&src[11];m[0].dst=&dst[11];submit(&c[0]); + m[1].src=&src[12];m[1].dst=&dst[12];submit(&c[1]); + assert(running==&c[1]&&d.deferred_ctx==&c[1]);before=finishes; + avd_queue_submit_work(&c[0].submit_work); + assert(running==&c[1]&&d.deferred_ctx==&c[1]&&finishes==before); + complete(&d,&c[0],0);avd_queue_submit_work(&c[1].submit_work);complete(&d,&c[1],0); + assert(!running&&!d.inflight&&!pm_refs&&!raw_finishes); + /* Inbox pressure cannot reverse two pictures of a context. */ + capacity=3;send_result=-EBUSY;unsigned int sent=send_count; + m[0].src=&src[14];m[0].dst=&dst[14];submit(&c[0]); + u32 first=d.frames[0].tag; + assert(d.frames[0].state==AVD_FRAME_RESERVED&&send_count==sent); + send_result=0;m[0].src=&src[15];m[0].dst=&dst[15];submit(&c[0]); + u32 second=d.frames[1].tag; + assert(d.frames[1].state==AVD_FRAME_RESERVED&&send_count==sent); + m[1].src=&src[16];m[1].dst=&dst[16];submit(&c[1]); + assert(d.frames[2].state==AVD_FRAME_SUBMITTED&&send_count==sent+1); + assert(!avd_send_frame(&d,&d.frames[0]));assert(!avd_send_frame(&d,&d.frames[1])); + assert(send_tags[sent+1]==first&&send_tags[sent+2]==second); + complete(&d,&c[1],0);complete(&d,&c[0],0);complete(&d,&c[0],0); + assert(!running&&!d.inflight&&!pm_refs&&!raw_finishes); + puts("PASS: production dispatch/retire: final/held slices, two contexts, capacity retry, prepare/PM/abort errors, exactly-once job_finish"); +} +''' + + +def main(): + out = Path(sys.argv[1]).resolve() + out.mkdir(parents=True, exist_ok=True) + names = ['avd-queue.c', 'avd-queue-internal.h', 'avd-queue.h', 'avd-v4l2.c'] + (out / 'core.sha256').write_text(hashlib.sha256(CORE.read_bytes()).hexdigest()+' v4l2-mem2mem.c\n') + (out / 'inputs.sha256').write_text(''.join(hashlib.sha256((ROOT / n).read_bytes()).hexdigest()+' '+n+'\n' for n in names)) + qh = (ROOT / 'avd-queue.h').read_text() + frame = qh[qh.index('enum avd_frame_state'):qh.index('struct avd_backend_ops {')] + source = (ROOT / 'avd-queue.c').read_text() + legacy = (ROOT / 'avd-v4l2.c').read_text() + code = out / 'dispatch.c' + code.write_text(PRELUDE + frame + BOUNDARY + '\n#include "avd-queue-internal.h"\n' + + function(CORE.read_text(), 'v4l2_m2m_get_curr_priv') + + function(CORE.read_text(), 'v4l2_m2m_job_finish_detach') + + function(CORE.read_text(), 'v4l2_m2m_buf_done_and_job_finish') + + function(legacy, 'avd_job_finish_no_pm') + function(legacy, 'avd_job_finish') + + function(source, 'avd_send_frame') + + function(source, 'avd_retire') + function(source, 'avd_queue_submit_work') + MAIN) + exe = out / 'dispatch-test' + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-Wall', '-Wextra', '-Werror', + '-fsanitize=address,undefined', '-g', '-I' + str(HERE / 'include'), + '-I' + str(ROOT), str(code), '-o', str(exe)], check=True) + subprocess.run([str(exe)], check=True) + + +if __name__ == '__main__': + main() diff --git a/drivers/media/v4l2-core/v4l2-mem2mem.c b/drivers/media/v4l2-core/v4l2-mem2mem.c index 311e2e08d066d..dd47ccd7f7d71 100644 --- a/drivers/media/v4l2-core/v4l2-mem2mem.c +++ b/drivers/media/v4l2-core/v4l2-mem2mem.c @@ -503,6 +503,34 @@ void v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev, } EXPORT_SYMBOL(v4l2_m2m_job_finish); +void v4l2_m2m_job_finish_detach(struct v4l2_m2m_dev *m2m_dev, + struct v4l2_m2m_ctx *m2m_ctx) +{ + struct vb2_v4l2_buffer *src_buf, *dst_buf; + bool schedule_next = false; + unsigned long flags; + + spin_lock_irqsave(&m2m_dev->job_spinlock, flags); + if (WARN_ON(m2m_dev->curr_ctx != m2m_ctx)) + goto unlock; + src_buf = v4l2_m2m_next_src_buf(m2m_ctx); + dst_buf = v4l2_m2m_next_dst_buf(m2m_ctx); + /* Partial slices must use buf_done_and_job_finish() and keep CAPTURE. */ + if (WARN_ON(!src_buf || !dst_buf || + (src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF))) + goto unlock; + + v4l2_m2m_src_buf_remove(m2m_ctx); + v4l2_m2m_dst_buf_remove(m2m_ctx); + dst_buf->is_held = false; + schedule_next = _v4l2_m2m_job_finish(m2m_dev, m2m_ctx); + unlock: + spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags); + if (schedule_next) + v4l2_m2m_schedule_next_job(m2m_dev, m2m_ctx); +} +EXPORT_SYMBOL_GPL(v4l2_m2m_job_finish_detach); + void v4l2_m2m_buf_done_and_job_finish(struct v4l2_m2m_dev *m2m_dev, struct v4l2_m2m_ctx *m2m_ctx, enum vb2_buffer_state state) diff --git a/include/media/v4l2-mem2mem.h b/include/media/v4l2-mem2mem.h index 31de25d792b98..5284bc506fc59 100644 --- a/include/media/v4l2-mem2mem.h +++ b/include/media/v4l2-mem2mem.h @@ -200,6 +200,24 @@ void v4l2_m2m_try_schedule(struct v4l2_m2m_ctx *m2m_ctx); void v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev, struct v4l2_m2m_ctx *m2m_ctx); +/** + * v4l2_m2m_job_finish_detach() - detach buffers and release a dispatch + * @m2m_dev: mem2mem device + * @m2m_ctx: context of the running job + * + * Remove the current source and destination from their ready lists and finish + * the running job, without completing either buffer. The driver must save both + * buffer pointers before calling, retain ownership until hardware has stopped + * accessing them, and eventually complete them with v4l2_m2m_buf_done(). + * + * This may be used by pipelined drivers on HOLD_CAPTURE-capable queues only + * for a source without V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF. Partial slices must + * instead use v4l2_m2m_buf_done_and_job_finish() to retain the capture buffer. + * The destination is no longer held or visible to subsequent m2m scheduling. + */ +void v4l2_m2m_job_finish_detach(struct v4l2_m2m_dev *m2m_dev, + struct v4l2_m2m_ctx *m2m_ctx); + /** * v4l2_m2m_buf_done_and_job_finish() - return source/destination buffers with * state and inform the framework that a job has been finished and have it From 1faa0604fb9e793dc7530e84ec8d4df784984356 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 15:04:38 +1100 Subject: [PATCH 39/73] accel/ane: H14 program lifetime without multi-task terminate On the M2 the previous commit unloads single-task programs cleanly, but TERMINATE_PROCESS after a program with more than one task has run never completes and the firmware stops answering. The firmware waits for its own flush with no timeout, no host step is missing, and there is no other command that frees such a process. Every useful network is multi-task, so unloading on close cannot be the design. Separate a program from its residency in the firmware. - A program is single-task only when its TD property section (load id 7) and its TD stream (load id 3) both count one task, read from the same snapshot as the cache key. Anything else counts as multi-task, and a multi-task process is never terminated. - At the last close a single-task program, or one without a process, is unloaded as before. A multi-task program stays resident as a cache entry that the next load of the same bytes reuses with no command. - At most eight programs are resident. Each process holds a 2 MiB firmware allocation and a LOAD sent with eight resident never completed. Before every LOAD the driver unloads single-task programs (unreferenced first, then least recently used), and when none is left it restarts the firmware at that idle point. - Programs keep their kernel IDs, section references and LOAD records while files hold them. After an eviction or a restart the next PROC_CREATE or EXEC loads the program again and creates its process, so clients see no change. A restart drops unreferenced cache entries. - The restart is a state machine. A planned restart sends POWER_DOWN (0x0b), waits for its completion and for WFI, clears CPU_CONTROL RUN and then bit 5, and requires CPU_STATUS STOPPED with RUN clear before any domain goes off. The old code waited for RUNNING to clear and STOPPED to set right after clearing RUN, which never happens on T8112 (0x2c idle, 0x28 halted, neither stopped). After a command timeout only the RUN clear is tried. If STOPPED is not seen nothing is powered off and the device stays quarantined with its mappings, power claim and pin. A refused POWER_DOWN leaves the live firmware in service. - debugfs fw_memory also reports resident, cached and single-task programs, the limit, the budget restarts and the firmware generation. The KUnit suite ane-t6021-lifetime covers the task count parsers, the never-terminate rule, the cache, the resident budget and eviction order, reload after a restart with stable IDs, a thirty-program sequence that never exceeds the budget, and the restart stop ordering. The TD walk follows the block walk that omarchy-ane's libane section builder by Joshua Warren (MIT) documents. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 64 +- drivers/accel/ane/ane_t6021_lifetime.h | 441 ++++++++++++-- drivers/accel/ane/ane_t6021_lifetime_test.c | 617 ++++++++++++++++---- drivers/accel/ane/ane_t6021_rtclient_main.c | 589 ++++++++++++------- 4 files changed, 1331 insertions(+), 380 deletions(-) diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index 4ab0448e4e4ff..bda1dd31f6357 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -332,14 +332,35 @@ opening a device. H14 program lifetime -------------------- -Byte-identical sections share one firmware program and one process. Every -file holds a reference on each program it loaded, and ``PROC_CREATE`` and -``EXEC`` refuse a program the file holds no reference on with ``-ENOENT``. -When the last file holding a program closes, the driver sends -TERMINATE_PROCESS (0x203) for its process and then DESTROY_PROGRAM (0x201). -Only after both are acknowledged does it drop its references on the section -buffers, which are then freed or pooled like any other BO. A failed unload -keeps the program and its buffers until a verified firmware restart. +Byte-identical sections share one program and one process. Every file holds +a reference on each program it loaded, and ``PROC_CREATE`` and ``EXEC`` +refuse a program the file holds no reference on with ``-ENOENT``. A program's +kernel IDs stay valid for as long as a file holds it; whether it is loaded in +the firmware right now (resident) is managed by the driver: + +- Every process holds one 2 MiB firmware allocation, and a LOAD sent while + eight programs with processes are resident never completes. At most eight + programs are resident, and room is made before each LOAD. +- TERMINATE_PROCESS (0x203) of a process whose program has more than one task + waits inside the firmware without a timeout and never completes, and no + other command frees such a process. The driver never sends it for one. A + program is single-task only when its TD property section (load id 7) and + its TD stream (load id 3) both count one task; anything else, including a + section it cannot parse, counts as multi-task. +- When the last file holding a program closes, a single-task program (or any + program without a process) is unloaded: TERMINATE_PROCESS, then + DESTROY_PROGRAM (0x201), and only after both are acknowledged are the + section buffers given back. A multi-task program stays resident as a cache + entry, and the next load of the same bytes reuses it without a command. +- Room comes first from unloading single-task programs, unreferenced ones + before the least recently used. When every resident program is a + multi-task process, the driver restarts the firmware at that idle point + (see below). Programs that files still hold become non-resident; the next + ``PROC_CREATE`` or ``EXEC`` loads them again and creates a new process, + with the same IDs. Unreferenced cache entries are dropped. + +A failed load or unload keeps the program and its buffers until a verified +firmware restart. The firmware asks for memory on the SHAREDMALLOC ring and gives it back with a FREE request, which carries the returned IOVA in the slot header and an @@ -351,8 +372,9 @@ buffer and its table slot, so the table and its byte count follow the live allocations rather than every allocation since boot. ``fw_memory`` in the device's accel debugfs directory reports the live -firmware allocations, their bytes, the table high-water mark, and the loaded -and held programs. A command timeout also logs the allocation ring and the +firmware allocations, their bytes, the table high-water mark, the known, +held, resident, cached and single-task programs, the resident limit, the +budget restarts and the firmware generation. A command timeout also logs the allocation ring and the target-to-host rings at the host cursors. H14 recovery lifecycle @@ -369,8 +391,15 @@ boot sequence. Borrowed firmware, RTKit mode, later generations and incomplete initial boots remain fenced. A second H14 binding is refused because the firmware cache and BO pool have one owner. -Recovery clears only the owned ASC CPU's RUN bit and waits for STOPPED before -suspending the consumer and suppliers through the shared PM transaction. This +A restart for room (planned) runs with the firmware idle under the firmware +lock. It sends POWER_DOWN (0x0b), waits for its completion and for the ASC to +report WFI, then clears CPU_CONTROL RUN (bit 4) and bit 5, and requires +CPU_STATUS STOPPED (bit 1) with RUN clear. On T8112 an idle core reads 0x2c +and a halted one 0x28, neither of which is stopped. After a command timeout +the firmware cannot answer POWER_DOWN, so only the RUN clear is tried. If +STOPPED is not seen, nothing is powered off: the device stays quarantined +with its mappings, power claim and pin. Only after STOPPED does the restart +suspend the consumer and suppliers through the shared PM transaction. This prevents firmware power-state writes from racing provider shutdown. It then verifies every domain and the CPU ACTUAL field off, resumes, repeats the probe power/PWGATE gates, and requires the CPU to be stopped before changing its @@ -390,10 +419,13 @@ reclaims retired arenas and closed BOs. Live old BO handles and mappings remain client-owned memory and can be reused for loading a program again. Program and process IDs are opaque kernel identifiers. The allocator survives -firmware restarts and refuses exhaustion instead of wrapping. Reset invalidates -the program cache, so an old ID cannot alias a reused firmware wire ID. Clients -must reload their program and create its process before retrying. The -``ane-fw-ids`` KUnit suite checks non-reuse and exhaustion boundaries. +firmware restarts and refuses exhaustion instead of wrapping. A restart drops +every firmware wire ID; programs that files hold load again with new wire IDs +on their next use, so an old wire ID never aliases a new one and clients keep +their kernel IDs. The ``ane-fw-ids`` KUnit suite checks non-reuse and +exhaustion boundaries, and ``ane-t6021-lifetime`` checks the task count, the +never-terminate rule, the resident budget, eviction order, reload after a +restart and the restart stop sequence. H14 keeps its existing domain-link lifetime reference and a reference for the currently running firmware CPU. Recovery releases the old CPU reference only diff --git a/drivers/accel/ane/ane_t6021_lifetime.h b/drivers/accel/ane/ane_t6021_lifetime.h index 83ad3a14bb13d..69505d7c768ab 100644 --- a/drivers/accel/ane/ane_t6021_lifetime.h +++ b/drivers/accel/ane/ane_t6021_lifetime.h @@ -24,13 +24,31 @@ * - header 0 and size 0: nothing posted yet. The slot is read again on * the next poll and the cursor does not move. * - * A firmware program is shared by every file that loaded byte-identical - * sections. Each file holds one reference per program it loaded. When the - * last reference goes, the program is unloaded: TERMINATE_PROCESS (0x203) - * for its process, then DESTROY_PROGRAM (0x201). Only after both are - * acknowledged are its section buffers given back. A program whose unload - * failed is HELD: its buffers stay and its slot is never reused until a - * verified firmware restart resets the table. + * A program is shared by every file that loaded byte-identical sections and + * outlives the firmware instance it was first loaded into. Each file holds one + * reference per program it loaded, by an opaque kernel ID that never changes. + * Whether the program is in the firmware right now (resident) is separate: + * + * - Every live process holds a 2 MiB firmware allocation, and a LOAD sent + * while eight programs with processes are resident never completes. The + * driver keeps at most ANE_T6021_FW_RESIDENT_MAX programs resident and + * makes room before each LOAD. + * - TERMINATE_PROCESS (0x203) of a process whose program has more than one + * task waits inside the firmware without a timeout and never completes, + * and no other command frees such a process. Such a process is never + * terminated. Only a program proven to have exactly one task (its TD + * property section and its TD stream both count one) is unloaded: + * TERMINATE_PROCESS for its process, then DESTROY_PROGRAM (0x201). + * - An unreferenced program that cannot be unloaded stays resident as a + * cache entry; a later load of the same bytes reuses it with no command. + * - Room comes first from unloading a single-task program (unreferenced + * first, then the least recently used), and when none is left from a + * cold firmware restart at an idle point. A restart drops every cache + * entry; referenced programs become non-resident and are loaded again, + * with a new process, on their next use. Files see no change. + * + * A program whose load or unload failed is HELD: its buffers stay and its + * slot is not reused until a verified firmware restart. */ #ifndef __ANE_T6021_LIFETIME_H__ #define __ANE_T6021_LIFETIME_H__ @@ -107,38 +125,119 @@ static inline int ane_t6021_legtab_find(const struct ane_legacy_buffer *t, return -1; } +/* ---- task count ---- */ + +/* + * The task count of a program, from the two sections that carry it. The + * firmware rejects a LOAD whose counts disagree, so both are read here and + * a program is single-task only when both say one. Anything this code + * cannot parse counts as multi-task: such a process is never terminated. + * The TD walk follows the firmware's block walk as omarchy-ane's libane + * section builder (Joshua Warren, MIT) documents it. + */ +#define ANE_T6021_SEC_TEXT 3 /* the TD stream */ +#define ANE_T6021_SEC_TDPROP 7 /* TD property segments */ +#define ANE_T6021_TDPROP_SEG 0x20 +#define ANE_T6021_TDPROP_MAX_SEGS 64 + +static inline u32 ane_t6021_le32(const u8 *p) +{ + return p[0] | p[1] << 8 | p[2] << 16 | (u32)p[3] << 24; +} + +/* Sum of blockNbr over the TD property segments: u32 count at +0, then + * count 0x20-byte segments from +8, blockNbr at segment +4. + */ +static inline long ane_t6021_tdprop_tasks(const u8 *p, size_t size) +{ + u32 segs, i; + u64 sum = 0; + + if (size < 8) + return -EINVAL; + segs = ane_t6021_le32(p); + if (!segs || segs > ANE_T6021_TDPROP_MAX_SEGS || + size < 8 + (size_t)segs * ANE_T6021_TDPROP_SEG) + return -EINVAL; + for (i = 0; i < segs; i++) + sum += ane_t6021_le32(p + 8 + i * ANE_T6021_TDPROP_SEG + 4); + return sum > INT_MAX ? -EINVAL : (long)sum; +} + +/* TD blocks in the stream: a 0x10-byte header (0x30 when word 0 bit 2 is + * set), then blocks whose u16 at +2 holds the word count in bits 10:0. + */ +static inline long ane_t6021_td_blocks(const u8 *p, size_t size) +{ + size_t cur; + long n = 0; + + if (size < 4) + return -EINVAL; + cur = (ane_t6021_le32(p) & 4) ? 0x30 : 0x10; + while (cur < size) { + u32 words; + + if (size - cur < 4) + return -EINVAL; + words = (p[cur + 2] | p[cur + 3] << 8) & 0x7ff; + if (!words) + return -EINVAL; + cur += ((words << 2) + 0xf) & 0x3ff0; + n++; + } + return n; +} + /* ---- program table ---- */ /* The firmware table holds 256 programs; the 257th LOAD is refused. */ #define ANE_T6021_MAX_PROGRAMS 250 #define ANE_T6021_PROG_SECS 9 #define ANE_T6021_NO_ID U32_MAX +/* + * Programs resident in the firmware at once. Each may own one process, and + * each process one 2 MiB firmware allocation. With eight programs and + * eight processes resident the next LOAD never completed (M2, 13.5 bia + * firmware: 9 baseline + 8 x 2 MiB allocations), so no LOAD is sent while + * this many are resident. + */ +#define ANE_T6021_FW_RESIDENT_MAX 8 #define ANE_T6021_CMD_DESTROY_PROGRAM 0x0201 #define ANE_T6021_CMD_TERMINATE_PROCESS 0x0203 enum ane_t6021_prog_state { ANE_T6021_PROG_FREE = 0, - ANE_T6021_PROG_LIVE, /* loaded, shared by the files that hold it */ + ANE_T6021_PROG_LIVE, /* known; shared by the files that hold it */ ANE_T6021_PROG_HELD, /* load or unload uncertain: keep everything */ }; struct ane_t6021_prog { u8 digest[32]; u32 prog_id; /* opaque kernel ID, never reused */ - u32 proc_id; /* opaque, ANE_T6021_NO_ID until a process exists */ - u32 fw_prog_id; /* firmware wire IDs */ + u32 proc_id; /* opaque, ANE_T6021_NO_ID until a process is asked for */ + u32 fw_prog_id; /* firmware wire IDs, only while resident */ u32 fw_proc_id; u32 users; /* files holding a reference */ enum ane_t6021_prog_state state; - /* The program's own reference on each section buffer, or NULL. */ + bool resident; /* LOADed into the current firmware instance */ + bool single_task; /* proven: its process may be terminated */ + u64 last_use; + /* The program's own reference on each section buffer, or NULL, and + * the IOVA and size its LOAD names for that section. + */ void *sec[ANE_T6021_PROG_SECS]; + u64 sec_iova[ANE_T6021_PROG_SECS]; + u64 sec_size[ANE_T6021_PROG_SECS]; }; struct ane_t6021_progtab { struct ane_t6021_prog progs[ANE_T6021_MAX_PROGRAMS]; unsigned int hi; /* slots below this index may be in use */ - u32 gen; /* bumped by every reset */ + u32 gen; /* bumped by every firmware restart */ + u64 clock; /* last_use source */ + u32 restarts; /* budget restarts, for diagnostics */ }; /* The program IDs one file holds a reference on. */ @@ -152,6 +251,15 @@ struct ane_t6021_progtab_ops { int (*unload)(void *ctx, struct ane_t6021_prog *p); /* The firmware no longer reads the sections: drop their references. */ void (*release)(void *ctx, struct ane_t6021_prog *p); + /* LOAD the program again; on success set fw_prog_id. */ + int (*load)(void *ctx, struct ane_t6021_prog *p); + /* CREATE_PROCESS; on success set fw_proc_id. */ + int (*create)(void *ctx, struct ane_t6021_prog *p); + /* Cold-restart the firmware at an idle point. Zero only when the new + * instance is verified, in which case it has called + * ane_t6021_progtab_restarted(). + */ + int (*restart)(void *ctx); }; /* Only LIVE programs are ever found, so HELD and stale entries are unusable. */ @@ -181,10 +289,34 @@ ane_t6021_progtab_by_id(struct ane_t6021_progtab *t, u32 prog_id) return NULL; } +/* Programs that hold firmware state now: resident, or HELD (uncertain). */ +static inline unsigned int +ane_t6021_progtab_resident(const struct ane_t6021_progtab *t) +{ + unsigned int i, n = 0; + + for (i = 0; i < t->hi; i++) + if (t->progs[i].state == ANE_T6021_PROG_HELD || + (t->progs[i].state == ANE_T6021_PROG_LIVE && + t->progs[i].resident)) + n++; + return n; +} + +/* The firmware can give a resident program back without a restart: it has + * no process, or its process runs a program proven to have one task. + */ +static inline bool ane_t6021_prog_unloadable(const struct ane_t6021_prog *p) +{ + return p->state == ANE_T6021_PROG_LIVE && p->resident && + (p->fw_proc_id == ANE_T6021_NO_ID || p->single_task); +} + /* * Claim a slot for a program about to be loaded. It starts HELD so nothing * can find it, and it owns the section references taken before the LOAD is - * sent: if the LOAD fails, the buffers stay held with it. + * sent: if the LOAD fails, the buffers stay held with it. Make room in the + * firmware before this: a restart frees unreferenced slots, this one too. */ static inline struct ane_t6021_prog * ane_t6021_progtab_reserve(struct ane_t6021_progtab *t, const u8 *digest) @@ -214,11 +346,14 @@ ane_t6021_progtab_reserve(struct ane_t6021_progtab *t, const u8 *digest) } /* The firmware acknowledged the LOAD: publish the program. */ -static inline void ane_t6021_progtab_commit(struct ane_t6021_prog *p, +static inline void ane_t6021_progtab_commit(struct ane_t6021_progtab *t, + struct ane_t6021_prog *p, u32 prog_id, u32 fw_prog_id) { p->prog_id = prog_id; p->fw_prog_id = fw_prog_id; + p->resident = true; + p->last_use = ++t->clock; p->state = ANE_T6021_PROG_LIVE; } @@ -233,7 +368,7 @@ static inline bool ane_t6021_fdprogs_has(const struct ane_t6021_fdprogs *f, return false; } -/* Forget IDs whose program is gone (a firmware restart reset the table). */ +/* Forget IDs whose program is gone. */ static inline void ane_t6021_fdprogs_compact(struct ane_t6021_progtab *t, struct ane_t6021_fdprogs *f) { @@ -274,7 +409,8 @@ static inline int ane_t6021_fdprogs_ref(struct ane_t6021_progtab *t, /* * The firmware commands of an unload: the process first, because destroying * a program that still has a process halts the firmware. @cmd returns zero - * only for an acknowledged command. Nothing after a failure is sent. + * only for an acknowledged command. Nothing after a failure is sent, and a + * process of a program not proven single-task is never terminated. */ static inline int ane_t6021_prog_unload_cmds(const struct ane_t6021_prog *p, @@ -285,6 +421,8 @@ ane_t6021_prog_unload_cmds(const struct ane_t6021_prog *p, int ret; if (p->fw_proc_id != ANE_T6021_NO_ID) { + if (!p->single_task) + return -EBUSY; ret = cmd(ctx, ANE_T6021_CMD_TERMINATE_PROCESS, p->fw_prog_id, p->fw_proc_id); if (ret) @@ -294,9 +432,51 @@ ane_t6021_prog_unload_cmds(const struct ane_t6021_prog *p, ANE_T6021_NO_ID); } +static inline void ane_t6021_progtab_free(struct ane_t6021_prog *p, + const struct ane_t6021_progtab_ops *ops, + void *ctx) +{ + ops->release(ctx, p); + memset(p, 0, sizeof(*p)); +} + /* - * The file is closing: drop its references and unload every program whose - * last reference this was. Returns how many programs ended HELD. + * Give a resident program back to the firmware. The program stays known + * while a file holds it and is loaded again on its next use. A failed + * unload marks it HELD, unless a restart during the unload already reset + * the table. Returns zero when the program left the firmware. + */ +static inline int ane_t6021_progtab_evict(struct ane_t6021_progtab *t, + struct ane_t6021_prog *p, + const struct ane_t6021_progtab_ops *ops, + void *ctx) +{ + u32 gen = t->gen; + int ret; + + if (!ane_t6021_prog_unloadable(p)) + return -EBUSY; + ret = ops->unload(ctx, p); + /* A restart during the unload already dropped every program. */ + if (t->gen != gen) + return 0; + if (ret) { + p->state = ANE_T6021_PROG_HELD; + return ret; + } + p->resident = false; + p->fw_prog_id = ANE_T6021_NO_ID; + p->fw_proc_id = ANE_T6021_NO_ID; + if (!p->users) + ane_t6021_progtab_free(p, ops, ctx); + return 0; +} + +/* + * The file is closing: drop its references. A program whose last reference + * this was is unloaded when the firmware can give it back, freed when it is + * not resident, and otherwise kept resident as a cache entry. Returns how + * many programs ended HELD. */ static inline unsigned int ane_t6021_progtab_fd_close(struct ane_t6021_progtab *t, @@ -304,7 +484,7 @@ ane_t6021_progtab_fd_close(struct ane_t6021_progtab *t, const struct ane_t6021_progtab_ops *ops, void *ctx) { unsigned int held = 0; - u32 i, gen; + u32 i; for (i = 0; i < f->n; i++) { struct ane_t6021_prog *p = ane_t6021_progtab_by_id(t, f->ids[i]); @@ -313,41 +493,142 @@ ane_t6021_progtab_fd_close(struct ane_t6021_progtab *t, continue; if (--p->users) continue; - gen = t->gen; - if (ops->unload(ctx, p)) { - /* A restart during the unload already reset the table. */ - if (t->gen == gen) { - p->state = ANE_T6021_PROG_HELD; - held++; - } - continue; - } - ops->release(ctx, p); - memset(p, 0, sizeof(*p)); + if (!p->resident) + ane_t6021_progtab_free(p, ops, ctx); + else if (ane_t6021_prog_unloadable(p) && + ane_t6021_progtab_evict(t, p, ops, ctx) && + p->state == ANE_T6021_PROG_HELD) + held++; } f->n = 0; return held; } /* - * A verified firmware restart dropped every program: release all section - * references, LIVE and HELD alike. Files keep their old IDs, which never - * match again because kernel IDs are not reused. + * The resident program the firmware can give back most cheaply, never + * @keep: an unreferenced one first, then the least recently used. */ -static inline void ane_t6021_progtab_reset(struct ane_t6021_progtab *t, +static inline struct ane_t6021_prog * +ane_t6021_progtab_victim(struct ane_t6021_progtab *t, + const struct ane_t6021_prog *keep) +{ + struct ane_t6021_prog *best = NULL; + unsigned int i; + + for (i = 0; i < t->hi; i++) { + struct ane_t6021_prog *p = &t->progs[i]; + + if (p == keep || !ane_t6021_prog_unloadable(p)) + continue; + if (!best || (!p->users && best->users) || + (!p->users == !best->users && p->last_use < best->last_use)) + best = p; + } + return best; +} + +/* + * Make room for one more resident program. Evicts single-task programs + * first; when none is left, restarts the firmware, which drops every + * resident program. @keep is never evicted. + */ +static inline int ane_t6021_progtab_make_room(struct ane_t6021_progtab *t, + const struct ane_t6021_prog *keep, + const struct ane_t6021_progtab_ops *ops, + void *ctx) +{ + unsigned int tries = ANE_T6021_FW_RESIDENT_MAX + 2; + int ret; + + while (ane_t6021_progtab_resident(t) >= ANE_T6021_FW_RESIDENT_MAX) { + struct ane_t6021_prog *v = ane_t6021_progtab_victim(t, keep); + + if (!tries--) + return -EBUSY; + if (v) { + ret = ane_t6021_progtab_evict(t, v, ops, ctx); + } else { + ret = ops->restart(ctx); + if (!ret) + t->restarts++; + } + if (ret) + return ret; + } + return 0; +} + +/* + * Put a LIVE program into the firmware, with a process when @proc. The + * caller holds a file reference on it, so a restart on the way keeps it. + */ +static inline int ane_t6021_progtab_ensure(struct ane_t6021_progtab *t, + struct ane_t6021_prog *p, bool proc, const struct ane_t6021_progtab_ops *ops, void *ctx) +{ + int ret; + + if (p->state != ANE_T6021_PROG_LIVE || !p->users) + return -ESTALE; + if (!p->resident) { + ret = ane_t6021_progtab_make_room(t, p, ops, ctx); + if (ret) + return ret; + if (p->state != ANE_T6021_PROG_LIVE) + return -ESTALE; + p->fw_prog_id = ANE_T6021_NO_ID; + p->fw_proc_id = ANE_T6021_NO_ID; + ret = ops->load(ctx, p); + /* Acked without an ID: the firmware may hold it after all. */ + if (ret == -EPROTO) + p->state = ANE_T6021_PROG_HELD; + if (ret) + return ret; + p->resident = true; + } + if (proc && p->fw_proc_id == ANE_T6021_NO_ID) { + u32 gen = t->gen; + + ret = ops->create(ctx, p); + if (ret == -EPROTO && t->gen == gen) + p->state = ANE_T6021_PROG_HELD; + if (ret) + return ret; + if (t->gen != gen || !p->resident) + return -ESTALE; + } + p->last_use = ++t->clock; + return 0; +} + +/* + * A verified firmware restart dropped every program and process. Programs + * no file holds are freed; held ones become LIVE and non-resident, keep + * their kernel IDs and section references, and load again on next use. + */ +static inline void ane_t6021_progtab_restarted(struct ane_t6021_progtab *t, + const struct ane_t6021_progtab_ops *ops, + void *ctx) { unsigned int i; for (i = 0; i < t->hi; i++) { struct ane_t6021_prog *p = &t->progs[i]; - if (p->state != ANE_T6021_PROG_FREE) - ops->release(ctx, p); - memset(p, 0, sizeof(*p)); + if (p->state == ANE_T6021_PROG_FREE) + continue; + if (!p->users || p->prog_id == ANE_T6021_NO_ID) { + ane_t6021_progtab_free(p, ops, ctx); + continue; + } + p->state = ANE_T6021_PROG_LIVE; + p->resident = false; + p->fw_prog_id = ANE_T6021_NO_ID; + p->fw_proc_id = ANE_T6021_NO_ID; } - t->hi = 0; + while (t->hi && t->progs[t->hi - 1].state == ANE_T6021_PROG_FREE) + t->hi--; t->gen++; } @@ -356,25 +637,87 @@ static inline void ane_t6021_progtab_cancel(struct ane_t6021_prog *p, const struct ane_t6021_progtab_ops *ops, void *ctx) { - ops->release(ctx, p); - memset(p, 0, sizeof(*p)); + ane_t6021_progtab_free(p, ops, ctx); } -/* LIVE and HELD program counts, for diagnostics. */ +struct ane_t6021_progtab_stats { + unsigned int live, held, resident, cached, single_task; +}; + static inline void ane_t6021_progtab_count(const struct ane_t6021_progtab *t, - unsigned int *live, - unsigned int *held) + struct ane_t6021_progtab_stats *s) { unsigned int i; - *live = 0; - *held = 0; + memset(s, 0, sizeof(*s)); for (i = 0; i < t->hi; i++) { - if (t->progs[i].state == ANE_T6021_PROG_LIVE) - (*live)++; - else if (t->progs[i].state == ANE_T6021_PROG_HELD) - (*held)++; + const struct ane_t6021_prog *p = &t->progs[i]; + + if (p->state == ANE_T6021_PROG_HELD) { + s->held++; + } else if (p->state == ANE_T6021_PROG_LIVE) { + s->live++; + s->resident += p->resident; + s->cached += p->resident && !p->users; + s->single_task += p->single_task; + } } } +/* ---- firmware restart ---- */ + +/* + * A cold restart of the H14 firmware. The ASC must be proven stopped + * (CPU_STATUS STOPPED) before its power domains go off: on T8112 an idle + * core reads 0x2c and a halted but running one 0x28, neither stopped. A + * planned restart first asks the firmware to shut down (POWER_DOWN) and + * waits for its completion and for WFI. After a command timeout that + * handshake is impossible, so only the RUN clear is tried. If STOPPED is + * not seen nothing is powered off, and the caller keeps the instance + * quarantined with every mapping and its power claim. + */ +enum ane_t6021_restart_stage { + ANE_T6021_RESTART_STOP, /* failed before power removal */ + ANE_T6021_RESTART_CYCLE, /* failed in the power cycle */ + ANE_T6021_RESTART_BOOT, /* failed booting the new instance */ + ANE_T6021_RESTART_DONE, +}; + +struct ane_t6021_restart_ops { + int (*power_down)(void *ctx); /* POWER_DOWN and its completion */ + int (*wait_idle)(void *ctx); /* bounded wait for WFI */ + int (*halt)(void *ctx); /* clear RUN, require STOPPED */ + int (*power_cycle)(void *ctx); /* domains verified off, then on */ + int (*boot)(void *ctx); /* cold boot and handshake */ +}; + +static inline int ane_t6021_restart_run(const struct ane_t6021_restart_ops *ops, + void *ctx, bool planned, + enum ane_t6021_restart_stage *stage) +{ + int ret; + + *stage = ANE_T6021_RESTART_STOP; + if (planned) { + ret = ops->power_down(ctx); + if (!ret) + ret = ops->wait_idle(ctx); + if (ret) + return ret; + } + ret = ops->halt(ctx); + if (ret) + return ret; + *stage = ANE_T6021_RESTART_CYCLE; + ret = ops->power_cycle(ctx); + if (ret) + return ret; + *stage = ANE_T6021_RESTART_BOOT; + ret = ops->boot(ctx); + if (ret) + return ret; + *stage = ANE_T6021_RESTART_DONE; + return 0; +} + #endif /* __ANE_T6021_LIFETIME_H__ */ diff --git a/drivers/accel/ane/ane_t6021_lifetime_test.c b/drivers/accel/ane/ane_t6021_lifetime_test.c index 45d60213c2d3a..14f976b40c531 100644 --- a/drivers/accel/ane/ane_t6021_lifetime_test.c +++ b/drivers/accel/ane/ane_t6021_lifetime_test.c @@ -61,18 +61,88 @@ static void legtab_reuse_test(struct kunit *test) KUNIT_EXPECT_EQ(test, t[2].size, (size_t)0); } -/* A fake firmware: records every unload and release, can fail on demand. */ +static void put32(u8 *p, u32 v) +{ + p[0] = v; + p[1] = v >> 8; + p[2] = v >> 16; + p[3] = v >> 24; +} + +static void task_count_test(struct kunit *test) +{ + u8 tdp[8 + 2 * ANE_T6021_TDPROP_SEG] = {}; + u8 td[0x200] = {}; + + /* One segment of one block: the single-task form a builder emits. */ + put32(tdp, 1); + put32(tdp + 0xc, 1); + KUNIT_EXPECT_EQ(test, ane_t6021_tdprop_tasks(tdp, 40), 1L); + put32(tdp + 0xc, 4); + KUNIT_EXPECT_EQ(test, ane_t6021_tdprop_tasks(tdp, 40), 4L); + /* Segments add up. */ + put32(tdp, 2); + put32(tdp + 0xc, 1); + put32(tdp + 0x2c, 1); + KUNIT_EXPECT_EQ(test, ane_t6021_tdprop_tasks(tdp, sizeof(tdp)), 2L); + /* Truncated, empty or absurd tables do not parse. */ + KUNIT_EXPECT_LT(test, ane_t6021_tdprop_tasks(tdp, 40), 0L); + KUNIT_EXPECT_LT(test, ane_t6021_tdprop_tasks(tdp, 4), 0L); + put32(tdp, 0); + KUNIT_EXPECT_LT(test, ane_t6021_tdprop_tasks(tdp, sizeof(tdp)), 0L); + put32(tdp, ANE_T6021_TDPROP_MAX_SEGS + 1); + KUNIT_EXPECT_LT(test, ane_t6021_tdprop_tasks(tdp, sizeof(tdp)), 0L); + + /* TD stream: 0x10-byte header, then one block of 0x20 words. */ + td[0x10 + 2] = 0x20; + KUNIT_EXPECT_EQ(test, ane_t6021_td_blocks(td, 0x10 + 0x80), 1L); + /* A second block of 0x41 words, padded to 16 bytes. */ + td[0x90 + 2] = 0x41; + KUNIT_EXPECT_EQ(test, ane_t6021_td_blocks(td, 0x90 + 0x110), 2L); + /* Header bit 2 selects a 0x30-byte header. */ + memset(td, 0, sizeof(td)); + td[0] = 4; + td[0x30 + 2] = 0x10; + KUNIT_EXPECT_EQ(test, ane_t6021_td_blocks(td, 0x30 + 0x40), 1L); + /* A zero word count where a block must start does not parse. */ + KUNIT_EXPECT_LT(test, ane_t6021_td_blocks(td, 0x30 + 0x50), 0L); + /* Neither does a block header cut short. */ + KUNIT_EXPECT_LT(test, ane_t6021_td_blocks(td, 0x30 + 0x42), 0L); + KUNIT_EXPECT_LT(test, ane_t6021_td_blocks(td, 2), 0L); +} + +/* + * A fake firmware. Records every command class, fails on demand, and + * refuses (test failure) to terminate a process of a program not proven + * single-task, which on hardware halts the firmware for good. + */ struct fake_fw { - unsigned int unloads, releases; - int fail_unload; + struct kunit *test; + struct ane_t6021_progtab *t; + unsigned int unloads, releases, loads, creates, restarts; + unsigned int terminates, multi_terminates; + int fail_unload, fail_restart; + bool restart_in_unload; int sec_refs; + u32 next_fw; }; +static const struct ane_t6021_progtab_ops fake_ops; + static int fake_unload(void *ctx, struct ane_t6021_prog *p) { struct fake_fw *fw = ctx; fw->unloads++; + if (p->fw_proc_id != ANE_T6021_NO_ID) { + fw->terminates++; + if (!p->single_task) + fw->multi_terminates++; + } + if (fw->restart_in_unload) { + ane_t6021_progtab_restarted(fw->t, &fake_ops, fw); + return -ETIMEDOUT; + } return fw->fail_unload; } @@ -87,111 +157,229 @@ static void fake_release(void *ctx, struct ane_t6021_prog *p) fw->sec_refs--; } +static int fake_load(void *ctx, struct ane_t6021_prog *p) +{ + struct fake_fw *fw = ctx; + + fw->loads++; + p->fw_prog_id = fw->next_fw++; + return 0; +} + +static int fake_create(void *ctx, struct ane_t6021_prog *p) +{ + struct fake_fw *fw = ctx; + + fw->creates++; + p->fw_proc_id = fw->next_fw++; + return 0; +} + +static int fake_restart(void *ctx) +{ + struct fake_fw *fw = ctx; + + fw->restarts++; + if (fw->fail_restart) + return fw->fail_restart; + ane_t6021_progtab_restarted(fw->t, &fake_ops, fw); + return 0; +} + static const struct ane_t6021_progtab_ops fake_ops = { .unload = fake_unload, .release = fake_release, + .load = fake_load, + .create = fake_create, + .restart = fake_restart, }; +static struct ane_t6021_progtab *new_table(struct kunit *test, + struct fake_fw *fw) +{ + struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, t); + fw->test = test; + fw->t = t; + fw->next_fw = 100; + return t; +} + +static struct ane_t6021_fdprogs *new_fd(struct kunit *test) +{ + struct ane_t6021_fdprogs *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, f); + return f; +} + +/* What the driver's LOAD ioctl does for a new program. */ static struct ane_t6021_prog *load(struct ane_t6021_progtab *t, - struct fake_fw *fw, u8 tag, u32 id) + struct fake_fw *fw, u32 tag, u32 id, + bool single) { - u8 digest[32]; + u8 digest[32] = {}; struct ane_t6021_prog *p; - memset(digest, tag, sizeof(digest)); + digest[0] = tag; + digest[1] = tag >> 8; + if (ane_t6021_progtab_make_room(t, NULL, &fake_ops, fw)) + return NULL; p = ane_t6021_progtab_reserve(t, digest); if (!p) return NULL; + p->single_task = single; p->sec[0] = fw; p->sec[6] = fw; fw->sec_refs += 2; - ane_t6021_progtab_commit(p, id, id + 100); + fake_load(fw, p); + ane_t6021_progtab_commit(t, p, id, p->fw_prog_id); + return p; +} + +/* LOAD, PROC_CREATE and one EXEC from a fresh file. */ +static struct ane_t6021_prog *open_run(struct kunit *test, + struct ane_t6021_progtab *t, + struct fake_fw *fw, + struct ane_t6021_fdprogs *f, u32 tag, + bool single) +{ + u8 digest[32] = {}; + struct ane_t6021_prog *p; + + digest[0] = tag; + digest[1] = tag >> 8; + p = ane_t6021_progtab_find(t, digest); + if (!p) + p = load(t, fw, tag, 1000 + tag, single); + KUNIT_ASSERT_NOT_NULL(test, p); + KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, f, p), 0); + KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p, true, &fake_ops, fw), 0); + KUNIT_EXPECT_TRUE(test, p->resident); + KUNIT_EXPECT_NE(test, p->fw_proc_id, ANE_T6021_NO_ID); + KUNIT_EXPECT_LE(test, ane_t6021_progtab_resident(t), + (unsigned int)ANE_T6021_FW_RESIDENT_MAX); return p; } static void progtab_shared_unload_test(struct kunit *test) { - struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); - struct ane_t6021_fdprogs *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); - struct ane_t6021_fdprogs *b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL); struct fake_fw fw = {}; + struct ane_t6021_progtab *t = new_table(test, &fw); + struct ane_t6021_fdprogs *a = new_fd(test), *b = new_fd(test); struct ane_t6021_prog *p, *q; - u8 digest[32]; - KUNIT_ASSERT_NOT_NULL(test, t); - KUNIT_ASSERT_NOT_NULL(test, a); - KUNIT_ASSERT_NOT_NULL(test, b); - p = load(t, &fw, 1, 10); - KUNIT_ASSERT_NOT_NULL(test, p); - KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), 0); - /* A second load of identical sections finds the same program. */ - memset(digest, 1, sizeof(digest)); - q = ane_t6021_progtab_find(t, digest); + p = open_run(test, t, &fw, a, 1, true); + /* A second load of identical sections finds the same program and + * process: no command reaches the firmware. + */ + q = open_run(test, t, &fw, b, 1, true); KUNIT_ASSERT_PTR_EQ(test, q, p); - KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, b, q), 0); /* Taking the same reference twice from one file counts once. */ KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, b, q), 0); KUNIT_EXPECT_EQ(test, p->users, 2U); - KUNIT_EXPECT_TRUE(test, ane_t6021_fdprogs_has(b, 10)); + KUNIT_EXPECT_EQ(test, fw.loads, 1U); + KUNIT_EXPECT_EQ(test, fw.creates, 1U); KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 0U); KUNIT_EXPECT_EQ(test, fw.unloads, 0U); KUNIT_EXPECT_EQ(test, p->users, 1U); KUNIT_EXPECT_EQ(test, fw.sec_refs, 2); + /* A single-task program is unloaded at its last close. */ KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, b, &fake_ops, &fw), 0U); KUNIT_EXPECT_EQ(test, fw.unloads, 1U); + KUNIT_EXPECT_EQ(test, fw.terminates, 1U); KUNIT_EXPECT_EQ(test, fw.releases, 1U); KUNIT_EXPECT_EQ(test, fw.sec_refs, 0); - KUNIT_EXPECT_NULL(test, ane_t6021_progtab_find(t, digest)); - KUNIT_EXPECT_NULL(test, ane_t6021_progtab_by_id(t, 10)); + KUNIT_EXPECT_NULL(test, ane_t6021_progtab_by_id(t, 1001)); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 0U); /* The slot is reused by the next program. */ - q = load(t, &fw, 2, 11); + q = load(t, &fw, 2, 1002, true); KUNIT_EXPECT_PTR_EQ(test, q, p); KUNIT_EXPECT_EQ(test, t->hi, 1U); } +static void multitask_never_terminated_test(struct kunit *test) +{ + struct fake_fw fw = {}; + struct ane_t6021_progtab *t = new_table(test, &fw); + struct ane_t6021_fdprogs *a = new_fd(test), *b = new_fd(test); + struct ane_t6021_prog *p, *q; + struct ane_t6021_progtab_stats st; + + p = open_run(test, t, &fw, a, 3, false); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 0U); + /* No TERMINATE, no DESTROY: the program stays resident as a cache + * entry with its section references. + */ + KUNIT_EXPECT_EQ(test, fw.unloads, 0U); + KUNIT_EXPECT_EQ(test, fw.releases, 0U); + KUNIT_EXPECT_EQ(test, fw.sec_refs, 2); + KUNIT_EXPECT_TRUE(test, p->resident); + ane_t6021_progtab_count(t, &st); + KUNIT_EXPECT_EQ(test, st.cached, 1U); + /* The next opener of the same bytes reuses program and process. */ + q = open_run(test, t, &fw, b, 3, false); + KUNIT_EXPECT_PTR_EQ(test, q, p); + KUNIT_EXPECT_EQ(test, fw.loads, 1U); + KUNIT_EXPECT_EQ(test, fw.creates, 1U); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, b, &fake_ops, &fw), 0U); + /* Nor can an eviction pick it. */ + KUNIT_EXPECT_NULL(test, ane_t6021_progtab_victim(t, NULL)); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_evict(t, p, &fake_ops, &fw), -EBUSY); + KUNIT_EXPECT_EQ(test, fw.multi_terminates, 0U); + + /* A multi-task program that never got a process can be destroyed. */ + p = load(t, &fw, 4, 1004, false); + KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), 0); + KUNIT_EXPECT_TRUE(test, ane_t6021_prog_unloadable(p)); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 0U); + KUNIT_EXPECT_EQ(test, fw.unloads, 1U); + KUNIT_EXPECT_EQ(test, fw.terminates, 0U); +} + static void progtab_failed_unload_holds_test(struct kunit *test) { - struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); - struct ane_t6021_fdprogs *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); struct fake_fw fw = { .fail_unload = -ETIMEDOUT }; + struct ane_t6021_progtab *t = new_table(test, &fw); + struct ane_t6021_fdprogs *a = new_fd(test); + struct ane_t6021_progtab_stats st; struct ane_t6021_prog *p, *q; - unsigned int live, held; - KUNIT_ASSERT_NOT_NULL(test, t); - KUNIT_ASSERT_NOT_NULL(test, a); - p = load(t, &fw, 3, 20); - KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), 0); + p = open_run(test, t, &fw, a, 5, true); KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 1U); /* The firmware may still read the sections: nothing is released. */ KUNIT_EXPECT_EQ(test, fw.releases, 0U); KUNIT_EXPECT_EQ(test, fw.sec_refs, 2); KUNIT_EXPECT_EQ(test, p->state, ANE_T6021_PROG_HELD); - KUNIT_EXPECT_NULL(test, ane_t6021_progtab_by_id(t, 20)); - ane_t6021_progtab_count(t, &live, &held); - KUNIT_EXPECT_EQ(test, live, 0U); - KUNIT_EXPECT_EQ(test, held, 1U); + KUNIT_EXPECT_NULL(test, ane_t6021_progtab_by_id(t, 1005)); + ane_t6021_progtab_count(t, &st); + KUNIT_EXPECT_EQ(test, st.live, 0U); + KUNIT_EXPECT_EQ(test, st.held, 1U); + /* A HELD program still counts against the firmware budget. */ + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 1U); /* A held slot is never reused before a restart. */ - q = load(t, &fw, 4, 21); + q = load(t, &fw, 6, 1006, true); KUNIT_EXPECT_PTR_NE(test, q, p); KUNIT_EXPECT_EQ(test, t->hi, 2U); - /* A verified restart releases LIVE and HELD programs alike. */ - ane_t6021_progtab_reset(t, &fake_ops, &fw); + /* A verified restart frees everything no file holds. */ + ane_t6021_progtab_restarted(t, &fake_ops, &fw); KUNIT_EXPECT_EQ(test, fw.releases, 2U); KUNIT_EXPECT_EQ(test, fw.sec_refs, 0); KUNIT_EXPECT_EQ(test, t->hi, 0U); + KUNIT_EXPECT_EQ(test, t->gen, 1U); } static void progtab_load_failure_test(struct kunit *test) { - struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); - struct ane_t6021_fdprogs *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); + struct fake_fw fw = {}; + struct ane_t6021_progtab *t = new_table(test, &fw); + struct ane_t6021_fdprogs *a = new_fd(test); struct ane_t6021_prog *p; u8 digest[32]; - KUNIT_ASSERT_NOT_NULL(test, t); - KUNIT_ASSERT_NOT_NULL(test, a); memset(digest, 5, sizeof(digest)); p = ane_t6021_progtab_reserve(t, digest); KUNIT_ASSERT_NOT_NULL(test, p); @@ -200,65 +388,179 @@ static void progtab_load_failure_test(struct kunit *test) KUNIT_EXPECT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), -ESTALE); KUNIT_EXPECT_EQ(test, a->n, 0U); KUNIT_EXPECT_EQ(test, p->state, ANE_T6021_PROG_HELD); + /* A restart frees the reservation, it never becomes LIVE. */ + ane_t6021_progtab_restarted(t, &fake_ops, &fw); + KUNIT_EXPECT_EQ(test, p->state, ANE_T6021_PROG_FREE); } -static void progtab_restart_stale_ids_test(struct kunit *test) +static void progtab_restart_keeps_ids_test(struct kunit *test) { - struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); - struct ane_t6021_fdprogs *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); - struct ane_t6021_fdprogs *b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL); struct fake_fw fw = {}; - struct ane_t6021_prog *p, *q; - - KUNIT_ASSERT_NOT_NULL(test, t); - KUNIT_ASSERT_NOT_NULL(test, a); - KUNIT_ASSERT_NOT_NULL(test, b); - p = load(t, &fw, 6, 30); - KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), 0); - ane_t6021_progtab_reset(t, &fake_ops, &fw); - /* Another file loads after the restart and reuses the slot. */ - q = load(t, &fw, 7, 31); - KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, b, q), 0); - KUNIT_EXPECT_PTR_EQ(test, q, p); - /* The old file's stale ID neither reaches nor drops the new program. */ - KUNIT_EXPECT_FALSE(test, ane_t6021_fdprogs_has(a, 31)); - KUNIT_EXPECT_NULL(test, ane_t6021_progtab_by_id(t, 30)); - KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 0U); - KUNIT_EXPECT_EQ(test, fw.unloads, 0U); - KUNIT_EXPECT_EQ(test, q->users, 1U); + struct ane_t6021_progtab *t = new_table(test, &fw); + struct ane_t6021_fdprogs *a = new_fd(test), *b = new_fd(test); + struct ane_t6021_prog *p, *z; + u32 fw_prog, fw_proc; + + p = open_run(test, t, &fw, a, 7, false); + /* An unreferenced cache entry next to it. */ + z = open_run(test, t, &fw, b, 8, false); KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, b, &fake_ops, &fw), 0U); + fw_prog = p->fw_prog_id; + fw_proc = p->fw_proc_id; + + KUNIT_ASSERT_EQ(test, fake_restart(&fw), 0); + /* The held program keeps its kernel ID and sections, not its + * firmware IDs; the cache entry is gone. + */ + KUNIT_EXPECT_PTR_EQ(test, ane_t6021_progtab_by_id(t, 1007), p); + KUNIT_EXPECT_FALSE(test, p->resident); + KUNIT_EXPECT_EQ(test, p->fw_prog_id, ANE_T6021_NO_ID); + KUNIT_EXPECT_EQ(test, p->fw_proc_id, ANE_T6021_NO_ID); + KUNIT_EXPECT_EQ(test, z->state, ANE_T6021_PROG_FREE); + KUNIT_EXPECT_EQ(test, fw.sec_refs, 2); + KUNIT_EXPECT_TRUE(test, ane_t6021_fdprogs_has(a, 1007)); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 0U); + + /* The next use loads it and creates its process again. */ + KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p, true, &fake_ops, &fw), 0); + KUNIT_EXPECT_TRUE(test, p->resident); + KUNIT_EXPECT_NE(test, p->fw_prog_id, fw_prog); + KUNIT_EXPECT_NE(test, p->fw_proc_id, fw_proc); + KUNIT_EXPECT_EQ(test, fw.loads, 3U); + KUNIT_EXPECT_EQ(test, fw.creates, 3U); + /* A program without a file reference is not loaded again. */ + z->state = ANE_T6021_PROG_LIVE; + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_ensure(t, z, true, &fake_ops, &fw), + -ESTALE); + z->state = ANE_T6021_PROG_FREE; +} + +static void budget_eviction_order_test(struct kunit *test) +{ + struct fake_fw fw = {}; + struct ane_t6021_progtab *t = new_table(test, &fw); + struct ane_t6021_fdprogs *f[ANE_T6021_FW_RESIDENT_MAX + 1]; + struct ane_t6021_prog *p[ANE_T6021_FW_RESIDENT_MAX + 1]; + unsigned int i; + + for (i = 0; i <= ANE_T6021_FW_RESIDENT_MAX; i++) + f[i] = new_fd(test); + /* 0..5 multi-task and held, 6 and 7 single-task and held. */ + for (i = 0; i < ANE_T6021_FW_RESIDENT_MAX; i++) + p[i] = open_run(test, t, &fw, f[i], 10 + i, i >= 6); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), + (unsigned int)ANE_T6021_FW_RESIDENT_MAX); + /* 7 was used last, so 6 is the least recently used single-task one. */ + KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p[7], true, &fake_ops, &fw), 0); + KUNIT_EXPECT_PTR_EQ(test, ane_t6021_progtab_victim(t, NULL), p[6]); + KUNIT_EXPECT_PTR_EQ(test, ane_t6021_progtab_victim(t, p[6]), p[7]); + /* A new program takes the least recently used single-task slot. */ + p[8] = open_run(test, t, &fw, f[8], 30, false); + KUNIT_EXPECT_EQ(test, fw.restarts, 0U); KUNIT_EXPECT_EQ(test, fw.unloads, 1U); - KUNIT_EXPECT_EQ(test, fw.sec_refs, 0); + KUNIT_EXPECT_FALSE(test, p[6]->resident); + KUNIT_EXPECT_PTR_EQ(test, ane_t6021_progtab_by_id(t, 1016), p[6]); + /* Using 6 again evicts 7, the last single-task resident one. */ + KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p[6], true, &fake_ops, &fw), 0); + KUNIT_EXPECT_FALSE(test, p[7]->resident); + KUNIT_EXPECT_EQ(test, fw.restarts, 0U); + /* The two single-task programs now take turns in the last slot; the + * multi-task ones are never touched and no restart is needed. + */ + KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p[7], true, &fake_ops, &fw), 0); + KUNIT_EXPECT_EQ(test, fw.restarts, 0U); + KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p[6], true, &fake_ops, &fw), 0); + KUNIT_EXPECT_EQ(test, fw.restarts, 0U); + /* Close the single-task holders; their programs unload. */ + ane_t6021_progtab_fd_close(t, f[6], &fake_ops, &fw); + ane_t6021_progtab_fd_close(t, f[7], &fake_ops, &fw); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 7U); + /* One more multi-task load fits; the next one forces the restart. */ + p[6] = open_run(test, t, &fw, f[6], 40, false); + KUNIT_EXPECT_EQ(test, fw.restarts, 0U); + p[7] = open_run(test, t, &fw, f[7], 41, false); + KUNIT_EXPECT_EQ(test, fw.restarts, 1U); + KUNIT_EXPECT_EQ(test, t->restarts, 1U); + /* After it only the new program is resident; the others are known + * and load on use. + */ + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 1U); + KUNIT_EXPECT_PTR_EQ(test, ane_t6021_progtab_by_id(t, 1010), p[0]); + KUNIT_EXPECT_FALSE(test, p[0]->resident); + KUNIT_EXPECT_EQ(test, fw.multi_terminates, 0U); + /* A failed restart reaches the caller and changes nothing. */ + for (i = 1; i < 8; i++) + KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p[i - 1], true, + &fake_ops, &fw), 0); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 8U); + fw.fail_restart = -EBUSY; + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_ensure(t, p[8], true, &fake_ops, &fw), + -EBUSY); + KUNIT_EXPECT_FALSE(test, p[8]->resident); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 8U); +} + +/* Thirty distinct programs, every third one multi-task, each opened, run + * and closed: the budget holds at every step, no multi-task process is + * ever terminated, and only the multi-task cache entries force restarts. + */ +static void budget_sequence_test(struct kunit *test) +{ + struct fake_fw fw = {}; + struct ane_t6021_progtab *t = new_table(test, &fw); + struct ane_t6021_fdprogs *f = new_fd(test), *keep = new_fd(test); + unsigned int i; + + /* One MobileNet-like program held open across the whole run. */ + open_run(test, t, &fw, keep, 500, false); + for (i = 0; i < 30; i++) { + struct ane_t6021_prog *p = open_run(test, t, &fw, f, 600 + i, + i % 3 != 0); + + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, f, &fake_ops, &fw), + 0U); + if (i % 3) + KUNIT_EXPECT_EQ(test, p->state, ANE_T6021_PROG_FREE); + KUNIT_EXPECT_LE(test, ane_t6021_progtab_resident(t), + (unsigned int)ANE_T6021_FW_RESIDENT_MAX); + } + KUNIT_EXPECT_EQ(test, fw.multi_terminates, 0U); + /* Ten multi-task programs plus the held one: one restart after the + * cache filled, and the held program still loads on use. + */ + KUNIT_EXPECT_EQ(test, fw.restarts, 1U); + KUNIT_ASSERT_NOT_NULL(test, ane_t6021_progtab_by_id(t, 1500)); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_ensure(t, ane_t6021_progtab_by_id(t, 1500), + true, &fake_ops, &fw), 0); + ane_t6021_progtab_fd_close(t, keep, &fake_ops, &fw); + KUNIT_EXPECT_EQ(test, fw.multi_terminates, 0U); } static void progtab_capacity_test(struct kunit *test) { - struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); - struct ane_t6021_fdprogs *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); - struct ane_t6021_fdprogs *b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL); struct fake_fw fw = {}; + struct ane_t6021_progtab *t = new_table(test, &fw); + struct ane_t6021_fdprogs *a = new_fd(test), *b = new_fd(test); struct ane_t6021_prog *p; u8 digest[32]; unsigned int i; - KUNIT_ASSERT_NOT_NULL(test, t); - KUNIT_ASSERT_NOT_NULL(test, a); - KUNIT_ASSERT_NOT_NULL(test, b); + /* Known programs are bounded by the firmware table, not the budget. */ for (i = 0; i < ANE_T6021_MAX_PROGRAMS; i++) { - memset(digest, 0, sizeof(digest)); - digest[0] = i; - digest[1] = i >> 8; - p = ane_t6021_progtab_reserve(t, digest); + p = load(t, &fw, i, 1000 + i, true); KUNIT_ASSERT_NOT_NULL(test, p); - ane_t6021_progtab_commit(p, 1000 + i, i); KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), 0); + KUNIT_ASSERT_LE(test, ane_t6021_progtab_resident(t), + (unsigned int)ANE_T6021_FW_RESIDENT_MAX); } memset(digest, 0xff, sizeof(digest)); KUNIT_EXPECT_NULL(test, ane_t6021_progtab_reserve(t, digest)); KUNIT_EXPECT_FALSE(test, ane_t6021_fdprogs_room(t, a)); - /* Closing the file unloads all of them and frees every slot. */ + KUNIT_EXPECT_EQ(test, fw.restarts, 0U); + /* Closing the file unloads the resident ones and frees every slot. */ KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 0U); - KUNIT_EXPECT_EQ(test, fw.unloads, (unsigned int)ANE_T6021_MAX_PROGRAMS); + KUNIT_EXPECT_EQ(test, fw.releases, (unsigned int)ANE_T6021_MAX_PROGRAMS); + KUNIT_EXPECT_EQ(test, fw.sec_refs, 0); p = ane_t6021_progtab_reserve(t, digest); KUNIT_EXPECT_PTR_EQ(test, p, &t->progs[0]); @@ -266,62 +568,40 @@ static void progtab_capacity_test(struct kunit *test) b->n = ANE_T6021_MAX_PROGRAMS; for (i = 0; i < ANE_T6021_MAX_PROGRAMS; i++) b->ids[i] = 5000 + i; - ane_t6021_progtab_commit(p, 9000, 1); + ane_t6021_progtab_commit(t, p, 9000, 1); KUNIT_EXPECT_EQ(test, ane_t6021_fdprogs_ref(t, b, p), 0); KUNIT_EXPECT_EQ(test, b->n, 1U); KUNIT_EXPECT_EQ(test, p->users, 1U); } -static struct ane_t6021_progtab *restart_table; - -/* An unload whose failure made the driver restart the firmware. */ -static int restart_unload(void *ctx, struct ane_t6021_prog *p) -{ - struct fake_fw *fw = ctx; - - fw->unloads++; - ane_t6021_progtab_reset(restart_table, &fake_ops, fw); - return -ETIMEDOUT; -} - -static const struct ane_t6021_progtab_ops restart_ops = { - .unload = restart_unload, - .release = fake_release, -}; - static void progtab_restart_during_unload_test(struct kunit *test) { - struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); - struct ane_t6021_fdprogs *a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL); struct fake_fw fw = {}; + struct ane_t6021_progtab *t = new_table(test, &fw); + struct ane_t6021_fdprogs *a = new_fd(test); struct ane_t6021_prog *p, *q; - KUNIT_ASSERT_NOT_NULL(test, t); - KUNIT_ASSERT_NOT_NULL(test, a); - restart_table = t; - p = load(t, &fw, 8, 40); - q = load(t, &fw, 9, 41); - KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), 0); - KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, q), 0); - /* The restart released both programs; nothing may be marked HELD - * afterwards and the second program needs no unload of its own. + p = open_run(test, t, &fw, a, 50, true); + q = open_run(test, t, &fw, a, 51, true); + /* The first unload fails and its recovery restarts the firmware. + * Nothing may be marked HELD afterwards, and the second program, + * no longer resident, needs no unload. */ - KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &restart_ops, &fw), 0U); + fw.restart_in_unload = true; + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 0U); KUNIT_EXPECT_EQ(test, fw.unloads, 1U); - KUNIT_EXPECT_EQ(test, fw.releases, 2U); - KUNIT_EXPECT_EQ(test, fw.sec_refs, 0); - KUNIT_EXPECT_EQ(test, p->state, ANE_T6021_PROG_FREE); + KUNIT_EXPECT_NE(test, p->state, ANE_T6021_PROG_HELD); + KUNIT_EXPECT_NE(test, q->state, ANE_T6021_PROG_HELD); KUNIT_EXPECT_EQ(test, t->gen, 1U); } static void progtab_cancel_test(struct kunit *test) { - struct ane_t6021_progtab *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); struct fake_fw fw = {}; + struct ane_t6021_progtab *t = new_table(test, &fw); struct ane_t6021_prog *p; u8 digest[32]; - KUNIT_ASSERT_NOT_NULL(test, t); memset(digest, 10, sizeof(digest)); p = ane_t6021_progtab_reserve(t, digest); KUNIT_ASSERT_NOT_NULL(test, p); @@ -355,7 +635,7 @@ static int fake_cmd(void *ctx, u16 opcode, u32 fw_prog, u32 fw_proc) static void unload_order_test(struct kunit *test) { struct ane_t6021_prog p = { - .fw_prog_id = 4, .fw_proc_id = 9, + .fw_prog_id = 4, .fw_proc_id = 9, .single_task = true, }; struct fake_cmds c = {}; @@ -384,19 +664,124 @@ static void unload_order_test(struct kunit *test) KUNIT_EXPECT_EQ(test, ane_t6021_prog_unload_cmds(&p, fake_cmd, &c), -ETIMEDOUT); KUNIT_EXPECT_EQ(test, c.n, 1U); + + /* A process of a program not proven single-task is never terminated, + * and nothing else is sent for it. + */ + memset(&c, 0, sizeof(c)); + p.single_task = false; + KUNIT_EXPECT_EQ(test, ane_t6021_prog_unload_cmds(&p, fake_cmd, &c), -EBUSY); + KUNIT_EXPECT_EQ(test, c.n, 0U); +} + +/* The restart steps, recorded in order; a step fails when asked to. */ +struct fake_restart { + char log[8]; + unsigned int n; + char fail; +}; + +static int fake_step(struct fake_restart *r, char step) +{ + r->log[r->n++] = step; + return r->fail == step ? -ETIMEDOUT : 0; +} + +static int fr_power_down(void *ctx) +{ + return fake_step(ctx, 'p'); +} + +static int fr_wait_idle(void *ctx) +{ + return fake_step(ctx, 'w'); +} + +static int fr_halt(void *ctx) +{ + return fake_step(ctx, 'h'); +} + +static int fr_power_cycle(void *ctx) +{ + return fake_step(ctx, 'c'); +} + +static int fr_boot(void *ctx) +{ + return fake_step(ctx, 'b'); +} + +static const struct ane_t6021_restart_ops fr_ops = { + .power_down = fr_power_down, + .wait_idle = fr_wait_idle, + .halt = fr_halt, + .power_cycle = fr_power_cycle, + .boot = fr_boot, +}; + +static void restart_state_machine_test(struct kunit *test) +{ + enum ane_t6021_restart_stage stage; + struct fake_restart r = {}; + + /* Planned: shut the firmware down, wait for WFI, stop, then power. */ + KUNIT_EXPECT_EQ(test, ane_t6021_restart_run(&fr_ops, &r, true, &stage), 0); + KUNIT_EXPECT_STREQ(test, r.log, "pwhcb"); + KUNIT_EXPECT_EQ(test, stage, ANE_T6021_RESTART_DONE); + + /* After a timeout the firmware cannot answer: no POWER_DOWN. */ + memset(&r, 0, sizeof(r)); + KUNIT_EXPECT_EQ(test, ane_t6021_restart_run(&fr_ops, &r, false, &stage), 0); + KUNIT_EXPECT_STREQ(test, r.log, "hcb"); + + /* STOPPED not proven: the domains are never touched. */ + memset(&r, 0, sizeof(r)); + r.fail = 'h'; + KUNIT_EXPECT_NE(test, ane_t6021_restart_run(&fr_ops, &r, true, &stage), 0); + KUNIT_EXPECT_STREQ(test, r.log, "pwh"); + KUNIT_EXPECT_EQ(test, stage, ANE_T6021_RESTART_STOP); + + /* No POWER_DOWN completion: not even the RUN clear is tried. */ + memset(&r, 0, sizeof(r)); + r.fail = 'p'; + KUNIT_EXPECT_NE(test, ane_t6021_restart_run(&fr_ops, &r, true, &stage), 0); + KUNIT_EXPECT_STREQ(test, r.log, "p"); + KUNIT_EXPECT_EQ(test, stage, ANE_T6021_RESTART_STOP); + + /* No WFI after POWER_DOWN: same. */ + memset(&r, 0, sizeof(r)); + r.fail = 'w'; + KUNIT_EXPECT_NE(test, ane_t6021_restart_run(&fr_ops, &r, true, &stage), 0); + KUNIT_EXPECT_STREQ(test, r.log, "pw"); + + /* Later failures name their stage. */ + memset(&r, 0, sizeof(r)); + r.fail = 'c'; + KUNIT_EXPECT_NE(test, ane_t6021_restart_run(&fr_ops, &r, true, &stage), 0); + KUNIT_EXPECT_EQ(test, stage, ANE_T6021_RESTART_CYCLE); + memset(&r, 0, sizeof(r)); + r.fail = 'b'; + KUNIT_EXPECT_NE(test, ane_t6021_restart_run(&fr_ops, &r, true, &stage), 0); + KUNIT_EXPECT_EQ(test, stage, ANE_T6021_RESTART_BOOT); } static struct kunit_case lifetime_cases[] = { KUNIT_CASE(mring_classify_test), KUNIT_CASE(legtab_reuse_test), + KUNIT_CASE(task_count_test), KUNIT_CASE(progtab_shared_unload_test), + KUNIT_CASE(multitask_never_terminated_test), KUNIT_CASE(progtab_failed_unload_holds_test), KUNIT_CASE(progtab_load_failure_test), - KUNIT_CASE(progtab_restart_stale_ids_test), + KUNIT_CASE(progtab_restart_keeps_ids_test), + KUNIT_CASE(budget_eviction_order_test), + KUNIT_CASE(budget_sequence_test), KUNIT_CASE(progtab_capacity_test), KUNIT_CASE(progtab_restart_during_unload_test), KUNIT_CASE(progtab_cancel_test), KUNIT_CASE(unload_order_test), + KUNIT_CASE(restart_state_machine_test), {} }; diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 00f58b3fecd21..ac60c0ab36476 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -113,9 +113,21 @@ */ #define ANE_IPI_OFF 0x1844000 -/* CPU_STATUS bits (m1n1 ASCRegs shape) */ +/* CPU_STATUS bits (m1n1 ASCRegs shape). On T8112 RUNNING is clear while + * the firmware is live and IDLE (WFI) is set whenever it waits, so STOPPED + * is the only stop predicate. + */ #define ANE_ASC_CPU_STATUS_RUNNING BIT(0) #define ANE_ASC_CPU_STATUS_STOPPED BIT(1) +#define ANE_ASC_CPU_STATUS_IDLE BIT(5) +/* CPU_CONTROL bit cleared after RUN in the ASC stop sequence. */ +#define ANE_ASC_CPU_CONTROL_B5 BIT(5) + +/* Legacy 13.5 POWER_DOWN: the firmware stops its timers and waits in WFI + * for good. Zero-filled body; the completion arrives before the WFI. + */ +#define ANE_T6021_CMD_POWER_DOWN 0x000b +#define ANE_T6021_POWER_DOWN_LEN 0x10 /* Boot-time module parameters — defaults are the configuration that * boots the firmware, with fw_alias_reserved defaulting to own memory. @@ -1424,16 +1436,17 @@ static int ane_rtclient_command(struct ane_rtclient *ane, /* ---- LOAD / CREATE / CALL wrappers ---- */ -/* The firmware program table holds 256 entries: the 257th LOAD_PROGRAM - * after a boot is answered with a protocol error. Two loads of - * byte-identical sections share one firmware program and one process. The - * key is SHA-256 over every section's id, size and bytes, so a client can - * only reach a program whose bytes it also supplied, and every file holds - * a reference on each program it loaded (ane_t6021_lifetime.h). The last - * file to close unloads the program from the firmware: TERMINATE_PROCESS, - * then DESTROY_PROGRAM, then its section buffers go back. Without that, - * every program and process stayed in the firmware until reboot and the - * firmware stopped answering after about ten MobileNet-sized loads. +/* Two loads of byte-identical sections share one program and one process. + * The key is SHA-256 over every section's id, size and bytes, so a client + * can only reach a program whose bytes it also supplied, and every file + * holds a reference on each program it loaded. Which programs are in the + * firmware at a time, which may be unloaded and when the firmware is + * restarted to make room is in ane_t6021_lifetime.h: a process of a + * multi-task program is never terminated, at most + * ANE_T6021_FW_RESIDENT_MAX programs are resident, and a program a file + * still holds loads again on its next use after an eviction or a restart. + * Without unloads every program and process stayed in the firmware until + * reboot and the firmware stopped answering after eight processes. * Protected by ane_t6021_fw_lock. */ static struct ane_t6021_progtab ane_t6021_progtab; @@ -1462,8 +1475,9 @@ static struct ane_t6021_prog *ane_t6021_prog_owned(struct ane_t6021_fd *fd, return NULL; } p = ane_t6021_prog_by_id(prog_id); - /* A reference the file still holds on a program that is gone means - * a firmware restart dropped it: reload and create the process again. + /* A program a restart or eviction took out of the firmware stays known + * while a file holds it. One that is gone failed in a way that left + * the firmware uncertain: load it again. */ *err = p ? 0 : -ESTALE; return p; @@ -1538,9 +1552,112 @@ static void ane_t6021_prog_release_op(void *ctx, struct ane_t6021_prog *p) mutex_unlock(&ane_t6021_bo_lock); } +/* + * LOAD_PROGRAM (0x200) from the program's own section records: one 0x30-byte + * record per section at its identity slot (slot = id - 1; tdprop id 7 lands + * at slot 6 and slot 5 stays empty when id 6 is absent). flags bit0 = 1, id + * at +0x04, iova at +0x18, size at +0x20. The firmware writes the program ID + * back at +0x1b8. Used for the first load and for every reload after an + * eviction or a restart; the section IOVAs are republished to the current + * firmware generation before the doorbell. + */ +static int ane_rtclient_send_load(struct ane_rtclient *ane, + struct ane_t6021_prog *p) +{ + struct ane_legacy_buffer *command = ane->cmd_buf; + u8 *cmd; + u32 fw_prog; + unsigned int i; + int ret; + + if (!command || atomic_read(&ane_t6021_quarantined)) + return -ENODEV; + cmd = command->cpu; + memset(cmd, 0, SZ_16K); + mutex_lock(&ane_t6021_bo_lock); + for (i = 0; i < ANE_T6021_PROG_SECS; i++) { + u64 slot_base = 0x8 + (u64)i * 0x30; + + if (!p->sec[i]) + continue; + /* The firmware may read the section any time after the + * doorbell, including during a timeout. + */ + ane_t6021_bo_publish(p->sec[i], true); + *(u32 *)(cmd + slot_base + 0x00) = cpu_to_le32(1); + *(u32 *)(cmd + slot_base + 0x04) = cpu_to_le32(i + 1); + *(u64 *)(cmd + slot_base + 0x18) = cpu_to_le64(p->sec_iova[i]); + *(u64 *)(cmd + slot_base + 0x20) = cpu_to_le64(p->sec_size[i]); + } + mutex_unlock(&ane_t6021_bo_lock); + *(u32 *)(cmd + 0x1b8) = cpu_to_le32(U32_MAX); + /* The LOAD command is 0x1C0 bytes: the nine 0x30 records end at + * 0x1b8, ProgramId at +0x1b8, 8 bytes of zero tail. + */ + BUILD_BUG_ON(sizeof(struct ane_csne_cmd_load_program) != 0x1b8); + ret = ane_rtclient_command(ane, command, 0x1c0, CSNE_CMD_LOAD_PROGRAM, + 1, 5000); + if (ret) + return ret; + fw_prog = le32_to_cpu(*(u32 *)(cmd + 0x1b8)); + /* A LOAD acked without an id leaves the program HELD. */ + if (fw_prog == U32_MAX) + return -EPROTO; + p->fw_prog_id = fw_prog; + return 0; +} + +static int ane_t6021_prog_load_op(void *ctx, struct ane_t6021_prog *p) +{ + struct ane_rtclient *ane = ctx; + int ret = ane_rtclient_send_load(ane, p); + + dev_dbg(ane->dev, "reload program %u: %d (fw %u)\n", p->prog_id, ret, + p->fw_prog_id); + return ret; +} + +/* CREATE_PROCESS (0x202): programId at +8, the firmware writes the + * process ID back at +0xc. + */ +static int ane_t6021_prog_create_op(void *ctx, struct ane_t6021_prog *p) +{ + struct ane_rtclient *ane = ctx; + struct ane_legacy_buffer *command = ane->cmd_buf; + u8 *cmd; + u32 fw_proc; + int ret; + + if (!command || atomic_read(&ane_t6021_quarantined)) + return -ENODEV; + cmd = command->cpu; + memset(cmd, 0, SZ_16K); + *(u32 *)(cmd + 0x08) = cpu_to_le32(p->fw_prog_id); + *(u32 *)(cmd + 0x0c) = cpu_to_le32(U32_MAX); + ret = ane_rtclient_command(ane, command, 0x10, CSNE_CMD_CREATE_PROCESS, + 1, 3000); + if (ret) + return ret; + fw_proc = le32_to_cpu(*(u32 *)(cmd + 0x0c)); + if (fw_proc == U32_MAX) + return -EPROTO; + p->fw_proc_id = fw_proc; + return 0; +} + +static int ane_rtclient_restart(struct ane_rtclient *ane); + +static int ane_t6021_prog_restart_op(void *ctx) +{ + return ane_rtclient_restart(ctx); +} + static const struct ane_t6021_progtab_ops ane_t6021_progtab_ops = { .unload = ane_t6021_prog_unload_op, .release = ane_t6021_prog_release_op, + .load = ane_t6021_prog_load_op, + .create = ane_t6021_prog_create_op, + .restart = ane_t6021_prog_restart_op, }; /* Build a LOAD_PROGRAM (0x200) message in a kernel-owned buffer. The @@ -1556,10 +1673,10 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, { struct drm_ane_section *sections; struct drm_ane_generic_bind *binds; - struct ane_legacy_buffer *command; struct ane_t6021_fd *fd = file->driver_priv; struct ane_t6021_prog *cached, *prog = NULL; bool sent = false; + long tdprop_tasks = -EINVAL, td_blocks = -EINVAL; u32 key, gen; u8 digest[SHA256_DIGEST_SIZE]; size_t binds_size; @@ -1658,6 +1775,17 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, goto out; } sha256_update(&sha, (const u8 *)hdr, sizeof(hdr)); + /* The same snapshot as the key decides whether the + * program's process may ever be terminated. + */ + if (sections[i].id == ANE_T6021_SEC_TEXT) + td_blocks = ane_t6021_td_blocks((const u8 *)bo->cpu + + sections[i].offset, + sections[i].size); + else if (sections[i].id == ANE_T6021_SEC_TDPROP) + tdprop_tasks = ane_t6021_tdprop_tasks((const u8 *)bo->cpu + + sections[i].offset, + sections[i].size); left = sections[i].size; while (left) { size_t n = min_t(size_t, left, @@ -1676,6 +1804,9 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, } cached = ane_t6021_prog_find(digest); if (cached) { + /* A resident cache entry or a non-resident program is reused as + * is; it goes back into the firmware on its next use. + */ ret = ane_t6021_fdprogs_ref(&ane_t6021_progtab, &fd->progs, cached); if (!ret) @@ -1689,12 +1820,15 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, ret = -ENOSPC; goto out; } - command = ane->cmd_buf; - if (!command) { + if (!ane->cmd_buf) { ret = -ENODEV; goto out; } - + /* Room first: a restart frees every unreferenced slot. */ + ret = ane_t6021_progtab_make_room(&ane_t6021_progtab, NULL, + &ane_t6021_progtab_ops, ane); + if (ret) + goto out; ret = ane_fw_id_alloc(&ane_t6021_ids, &key); if (ret) goto out; @@ -1703,19 +1837,15 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, ret = -ENOSPC; goto out; } + prog->single_task = tdprop_tasks == 1 && td_blocks == 1; gen = ane_t6021_progtab.gen; - memset(command->cpu, 0, SZ_16K); - /* One 0x30-byte section record per supplied section, placed at - * its IDENTITY slot: slot = id - 1 (tdprop id 7 lands at slot - * 6, slot 5 stays empty — e.g. an image with - * six populated records, ids 1,2,3,4,5 then 7). Unsupplied - * slots stay zero-filled (flags = 0). flags bit0 = 1, id at - * +0x04, iova at +0x18, size at +0x20. + /* The record slot IS the section identity (slot = id - 1). The program + * takes its own reference on each section buffer, kept until the + * firmware dropped the program for good (or forever if uncertain). */ for (i = 0; i < user->section_count; i++) { struct ane_t6021_bo *bo = NULL, *b; - u64 slot_base, iova; - u8 *cmd = command->cpu; + unsigned int slot = sections[i].id - 1; mutex_lock(&ane_t6021_bo_lock); list_for_each_entry(b, &fd->bos, node) { @@ -1726,20 +1856,11 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, } if (bo && sections[i].size <= bo->size && sections[i].offset <= bo->size - sections[i].size) { - /* The IOVA below is about to be published to the - * firmware, so mark the BO before the exchange: - * the fw may read the section any time after the - * doorbell rings, including during a timeout. - * Under bo_lock the handle reference cannot go - * away. The program takes its own reference, which - * it keeps until the firmware dropped the program - * (or forever if that is uncertain). - */ - ane_t6021_bo_publish(bo, true); kref_get(&bo->refcount); bo->prog_refs++; - prog->sec[sections[i].id - 1] = bo; - iova = bo->dma + sections[i].offset; + prog->sec[slot] = bo; + prog->sec_iova[slot] = bo->dma + sections[i].offset; + prog->sec_size[slot] = sections[i].size; } else { bo = NULL; } @@ -1748,58 +1869,25 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, ret = -EINVAL; goto out; } - slot_base = 0x8 + (u64)(sections[i].id - 1) * 0x30; - *(u32 *)(cmd + slot_base + 0x00) = cpu_to_le32(1); - *(u32 *)(cmd + slot_base + 0x04) = - cpu_to_le32(sections[i].id); - *(u64 *)(cmd + slot_base + 0x18) = cpu_to_le64(iova); - *(u64 *)(cmd + slot_base + 0x20) = - cpu_to_le64(sections[i].size); - } - /* Generic binds are accepted for ABI compatibility and unused: - * the LOAD record already carries iova + size, and the generic - * section image is prepared by userspace (it ships inside the - * section bytes, generic.bin). - */ - - /* ProgramId placeholder; the firmware writes the assigned id - * back at +0x1b8. - */ - { - u8 *cmd = command->cpu; - - *(u32 *)(cmd + 0x1b8) = cpu_to_le32(U32_MAX); } - - /* The LOAD command is 0x1C0 bytes: the nine - * 0x30 records end at 0x1b8, ProgramId at +0x1b8, 8 bytes of - * zero tail. sizeof(struct ane_csne_cmd_load_program) is only - * 0x1b8, so the length is pinned here. + /* Generic binds are accepted for ABI compatibility and unused: the + * LOAD record already carries iova + size, and the generic section + * image is prepared by userspace (generic.bin). */ - BUILD_BUG_ON(sizeof(struct ane_csne_cmd_load_program) != 0x1b8); sent = true; - ret = ane_rtclient_command(ane, command, - 0x1c0, - CSNE_CMD_LOAD_PROGRAM, - 1, 5000); + ret = ane_rtclient_send_load(ane, prog); if (!ret) { - u8 *cmd = command->cpu; - - u32 fw_prog = le32_to_cpu(*(u32 *)(cmd + 0x1b8)); - - /* A LOAD acked without an id leaves the program HELD. */ - if (fw_prog == U32_MAX) { - ret = -EPROTO; - goto out; - } - ane_t6021_progtab_commit(prog, key, fw_prog); + ane_t6021_progtab_commit(&ane_t6021_progtab, prog, key, + prog->fw_prog_id); /* Room was checked under the same lock. */ ret = ane_t6021_fdprogs_ref(&ane_t6021_progtab, &fd->progs, prog); if (WARN_ON(ret)) goto out; *prog_id = key; - dev_dbg(ane->dev, "loaded program %u (fw %u) (fw allocations live %u, %zu bytes)\n", - key, fw_prog, ane->legacy_live, ane->legacy_bytes); + dev_dbg(ane->dev, "loaded program %u (fw %u, %s) (fw allocations live %u, %zu bytes)\n", + key, prog->fw_prog_id, + prog->single_task ? "single task" : "multi task or unknown", + ane->legacy_live, ane->legacy_bytes); } else if (ane_t6021_progtab.gen != gen) { /* A firmware restart already released the reservation. */ prog = NULL; @@ -1818,7 +1906,6 @@ static int ane_rtclient_create_process(struct ane_rtclient *ane, struct drm_file *file, __u32 prog_id, __u32 *proc_id) { - struct ane_legacy_buffer *command; struct ane_t6021_prog *prog; int ret; @@ -1828,36 +1915,17 @@ static int ane_rtclient_create_process(struct ane_rtclient *ane, prog = ane_t6021_prog_owned(file->driver_priv, prog_id, &ret); if (!prog) return ret; - if (prog->proc_id != U32_MAX) { - *proc_id = prog->proc_id; - return 0; - } - command = ane->cmd_buf; - if (!command) - return -ENODEV; - memset(command->cpu, 0, SZ_16K); - { - u8 *cmd = command->cpu; - - *(u32 *)(cmd + 0x08) = cpu_to_le32(prog->fw_prog_id); - *(u32 *)(cmd + 0x0c) = cpu_to_le32(U32_MAX); - } - ret = ane_rtclient_command(ane, command, 0x10, - CSNE_CMD_CREATE_PROCESS, - 1, 3000); - if (!ret) { - u8 *cmd = command->cpu; - - *proc_id = le32_to_cpu(*(u32 *)(cmd + 0x0c)); - if (*proc_id == U32_MAX) { - ret = -EPROTO; - } else { - prog->fw_proc_id = *proc_id; - prog->proc_id = prog->prog_id; - *proc_id = prog->proc_id; - } - } - return ret; + /* One firmware process per program, shared by its files. The public + * process ID stays the same when a restart or an eviction replaces + * the firmware process. + */ + ret = ane_t6021_progtab_ensure(&ane_t6021_progtab, prog, true, + &ane_t6021_progtab_ops, ane); + if (ret) + return ret; + prog->proc_id = prog->prog_id; + *proc_id = prog->proc_id; + return 0; } /* H16 consumes only backend-owned process resources. The ABI-2 section @@ -2053,6 +2121,13 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, ret = -ESTALE; goto unlock; } + /* Load it and its process again if an eviction or a restart took + * them out of the firmware. This may restart the firmware. + */ + ret = ane_t6021_progtab_ensure(&ane_t6021_progtab, program, true, + &ane_t6021_progtab_ops, ane); + if (ret) + goto unlock; command = ane->cmd_buf; if (!command) { ret = -ENODEV; @@ -2407,17 +2482,20 @@ static int ane_t6021_fw_memory_show(struct seq_file *m, void *v) { struct drm_debugfs_entry *entry = m->private; struct ane_rtclient *ane = entry->file.data; - unsigned int live, held; + struct ane_t6021_progtab_stats st; int ret; ret = mutex_lock_interruptible(&ane_t6021_fw_lock); if (ret) return ret; - ane_t6021_progtab_count(&ane_t6021_progtab, &live, &held); + ane_t6021_progtab_count(&ane_t6021_progtab, &st); seq_printf(m, "allocations: %u\nbytes: %zu\nhigh_water: %u\n", ane->legacy_live, ane->legacy_bytes, ane->legacy_allocated); - seq_printf(m, "programs: %u\nheld: %u\nquarantined: %d\n", live, held, - atomic_read(&ane_t6021_quarantined)); + seq_printf(m, "programs: %u\nheld: %u\nresident: %u\ncached: %u\nsingle_task: %u\n", + st.live, st.held, st.resident, st.cached, st.single_task); + seq_printf(m, "resident_max: %u\nrestarts: %u\nfw_generation: %u\nquarantined: %d\n", + ANE_T6021_FW_RESIDENT_MAX, ane_t6021_progtab.restarts, + ane_t6021_progtab.gen, atomic_read(&ane_t6021_quarantined)); mutex_unlock(&ane_t6021_fw_lock); return 0; } @@ -2967,82 +3045,104 @@ static void ane_rtclient_reap_retired(struct ane_rtclient *ane) ane->retired_count = 0; } -static int ane_rtclient_recover(struct ane_rtclient *ane) -{ - struct ane_t6021 *a = ane->fw; - struct ane_rtclient_cycle cycle = { .ane = ane }; - struct ane_rtclient_pd *pd; +struct ane_rtclient_restart { + struct ane_rtclient *ane; + struct ane_rtclient_cycle cycle; struct ane_rtclient_retired *retired; struct ane_legacy_buffer *fresh; - u32 status, control; - unsigned int domains = ane->soc->soc == 0x8112 ? 7 : 8; + bool refused; /* POWER_DOWN answered with a refusal */ +}; + +/* Ask the firmware to shut down. Only a live instance can answer, so this + * is never tried after a command timeout. + */ +static int ane_rtclient_rs_power_down(void *ctx) +{ + struct ane_rtclient_restart *rs = ctx; + struct ane_rtclient *ane = rs->ane; int ret; - lockdep_assert_held(&ane_t6021_fw_lock); - if (!ane->registered) - return -EOPNOTSUPP; - if (!atomic_read(&ane_t6021_quarantined)) - return 0; - /* Only the qualified, Linux-owned legacy boot profile can restart. - * Borrowed firmware, RTKit and later-generation ownership stay fenced. - */ - if (ane->soc->platform || !a || ane->rtk || - !legacy_only || !scratch3_ack || !legacy_query || fw_start_rtb_mode || - fw_start_table_mode != 2 || !ane_t6021_fwload_recoverable(a)) - return -EOPNOTSUPP; - /* Keep failed-boot retention bounded and charged against existing caps. */ - if (ane->retired_count >= 2 || ane_t6021_bo_epoch == U64_MAX) - return -ENOSPC; - mutex_lock(&ane_rtclient_pd_lock); - list_for_each_entry(pd, &ane_rtclient_pd_list, list) - if (pd->dev == ane->dev && pd->count == domains) - cycle.pd = pd; - mutex_unlock(&ane_rtclient_pd_lock); - if (!cycle.pd || !ane->engine_ps_valid) - return -EOPNOTSUPP; - retired = kzalloc_obj(*retired); - if (!retired) - return -ENOMEM; - fresh = devm_kcalloc(ane->dev, ANE_LEGACY_ALLOCS, sizeof(*fresh), GFP_KERNEL); - if (!fresh) { - kfree(retired); - return -ENOMEM; - } + if (!ane->cmd_buf) + return -ENODEV; + memset(ane->cmd_buf->cpu, 0, SZ_16K); + ret = ane_rtclient_legacy_exchange(ane, ane->cmd_buf, + ANE_T6021_POWER_DOWN_LEN, + ANE_T6021_CMD_POWER_DOWN, 1, + jiffies + msecs_to_jiffies(3000)); + rs->refused = ret == -EREMOTEIO; + dev_info(ane->dev, "restart: POWER_DOWN %d (CPU_STATUS %#x)\n", ret, + readl(ane->engine + ANE_ASC_CPU_STATUS)); + return ret; +} + +static int ane_rtclient_rs_wait_idle(void *ctx) +{ + struct ane_rtclient_restart *rs = ctx; + u32 status; + + return readl_poll_timeout(rs->ane->engine + ANE_ASC_CPU_STATUS, status, + status & ANE_ASC_CPU_STATUS_IDLE, 100, 100000); +} + +/* Clear RUN, then bit 5, each read back; then require STOPPED with RUN + * clear. Nothing is powered off unless this returns zero. + */ +static int ane_rtclient_rs_halt(void *ctx) +{ + struct ane_rtclient_restart *rs = ctx; + struct ane_rtclient *ane = rs->ane; + void __iomem *ctl = ane->engine + ANE_ASC_CPU_CONTROL; + u32 status; + int ret; - WRITE_ONCE(ane->recovering, true); - cancel_delayed_work_sync(&ane->poll_work); - /* Stop only our owned ASC CPU before supplier shutdown. Clearing RUN - * is the public m1n1 ASC shutdown operation (hw/asc.py, c4c2fed8f894). - * The existing probe boot sequence also clears RUN before starting it. - * Wait for STOPPED so firmware cannot race the PMGR power-off writes. - */ ret = ane_rtclient_engine_accessible(ane); if (ret) - goto unchanged; - control = readl(ane->engine + ANE_ASC_CPU_CONTROL); - writel(control & ~ANE_ASC_CPU_CONTROL_RUN, - ane->engine + ANE_ASC_CPU_CONTROL); + return ret; + writel(readl(ctl) & ~ANE_ASC_CPU_CONTROL_RUN, ctl); + readl(ctl); + writel(readl(ctl) & ~ANE_ASC_CPU_CONTROL_B5, ctl); + readl(ctl); ret = readl_poll_timeout(ane->engine + ANE_ASC_CPU_STATUS, status, (status & ANE_ASC_CPU_STATUS_STOPPED) && !(status & ANE_ASC_CPU_STATUS_RUNNING), 100, 100000); - if (ret) - goto unchanged; + if (!ret && (readl(ctl) & ANE_ASC_CPU_CONTROL_RUN)) + ret = -EBUSY; + dev_info(ane->dev, "restart: ASC stop %s: CPU_STATUS %#x CPU_CONTROL %#x\n", + ret ? "NOT proven" : "proven", status, readl(ctl)); + return ret; +} + +static int ane_rtclient_rs_power_cycle(void *ctx) +{ + struct ane_rtclient_restart *rs = ctx; + struct ane_rtclient *ane = rs->ane; + u32 status; + int ret; + /* Probe retains one consumer reference for this firmware lifetime. */ - ret = ane_pm_cycle(ane->dev, cycle.pd->pd_dev, cycle.pd->count, - ane_rtclient_verify_off, &cycle); + ret = ane_pm_cycle(ane->dev, rs->cycle.pd->pd_dev, rs->cycle.pd->count, + ane_rtclient_verify_off, &rs->cycle); if (ret) - goto unchanged; + return ret; /* Repeat probe's gates before any engine-aperture read. */ ret = ane_rtclient_engine_accessible(ane); if (ret) - goto unchanged; + return ret; status = readl(ane->engine + ANE_ASC_CPU_STATUS); if ((status & ANE_ASC_CPU_STATUS_RUNNING) || - !(status & ANE_ASC_CPU_STATUS_STOPPED)) { - ret = -EBUSY; - goto unchanged; - } + !(status & ANE_ASC_CPU_STATUS_STOPPED)) + return -EBUSY; + return 0; +} + +static int ane_rtclient_rs_boot(void *ctx) +{ + struct ane_rtclient_restart *rs = ctx; + struct ane_rtclient *ane = rs->ane; + struct ane_rtclient_retired *retired = rs->retired; + struct ane_t6021 *a = ane->fw; + int ret; /* No old allocation is discarded until the new boot is validated. */ retired->buffers = ane->legacy_buffers; @@ -3056,6 +3156,7 @@ static int ane_rtclient_recover(struct ane_rtclient *ane) .heap_size = a->boot_heap_size, }; list_add_tail(&retired->node, &ane->retired); + rs->retired = NULL; ane->retired_count++; ane->retired_bytes += retired->bytes; ane->retired_allocated += retired->allocated; @@ -3068,7 +3169,8 @@ static int ane_rtclient_recover(struct ane_rtclient *ane) a->boot_heap_iova = 0; a->boot_ipc_size = 0; a->boot_heap_size = 0; - ane->legacy_buffers = fresh; + ane->legacy_buffers = rs->fresh; + rs->fresh = NULL; ane->legacy_allocated = 0; ane->legacy_live = 0; ane->legacy_bytes = 0; @@ -3083,25 +3185,21 @@ static int ane_rtclient_recover(struct ane_rtclient *ane) ret = ane_t6021_fwload_reload(a); if (ret) - goto failed; + return ret; ret = ane_t6021_boot_start(a, 0, fw_start_table_mode, 0); ane->held = a->cpu_started; - if (ret || !a->fw_alive || !a->booted) { - ret = ret ?: -ETIMEDOUT; - goto failed; - } + if (ret || !a->fw_alive || !a->booted) + return ret ?: -ETIMEDOUT; ane_rtclient_validate_chman(ane); if (!ane->chman_ok || !ane_t6021_chman_host_init(a->boot_ipc, a->boot_ipc_size, - a->boot_ipc_iova)) { - ret = -EPROTO; - goto failed; - } + a->boot_ipc_iova)) + return -EPROTO; dma_wmb(); writel(ANE_T6021_BOOT_ACK, ane->engine + ANE_MBI_SCRATCH0 + 4 * 3); ret = ane_rtclient_legacy_alloc(ane, SZ_16K); if (ret < 0) - goto failed; + return ret; ane->cmd_buf = &ane->legacy_buffers[ret]; memset(ane->cmd_buf->cpu, 0, SZ_16K); /* The firmware lock still fences commands; BO_INIT also checks recovering. @@ -3113,7 +3211,7 @@ static int ane_rtclient_recover(struct ane_rtclient *ane) if (!ret && !READ_ONCE(((u32 *)ane->cmd_buf->cpu)[2])) ret = -EPROTO; if (ret) - goto failed; + return ret; if (ane->desired_perf) { u8 *cmd = ane->cmd_buf->cpu; @@ -3123,35 +3221,128 @@ static int ane_rtclient_recover(struct ane_rtclient *ane) ret = ane_rtclient_legacy_exchange(ane, ane->cmd_buf, 0x14, 0x001f, 1, jiffies + msecs_to_jiffies(3000)); if (ret) - goto failed; + return ret; fw_perf_mode = true; } - if (atomic_read(&ane_t6021_quarantined)) { - ret = -EIO; - goto failed; + return atomic_read(&ane_t6021_quarantined) ? -EIO : 0; +} + +static const struct ane_t6021_restart_ops ane_rtclient_restart_ops = { + .power_down = ane_rtclient_rs_power_down, + .wait_idle = ane_rtclient_rs_wait_idle, + .halt = ane_rtclient_rs_halt, + .power_cycle = ane_rtclient_rs_power_cycle, + .boot = ane_rtclient_rs_boot, +}; + +/* + * Cold-restart the Linux-owned firmware. @planned: the instance is live and + * idle (the caller holds the firmware lock, so no command or call is in + * flight) and is asked to shut down first. Otherwise a command timed out + * and only the RUN clear is tried. The ASC must be proven stopped before + * its domains go off; if it is not, nothing changes and the device stays + * quarantined with every mapping, its power claim and its pin. + */ +static int __ane_rtclient_recover(struct ane_rtclient *ane, bool planned) +{ + struct ane_t6021 *a = ane->fw; + struct ane_rtclient_restart rs = { .ane = ane, .cycle.ane = ane }; + struct ane_rtclient_pd *pd; + enum ane_t6021_restart_stage stage = ANE_T6021_RESTART_STOP; + unsigned int domains = ane->soc->soc == 0x8112 ? 7 : 8; + int ret; + + lockdep_assert_held(&ane_t6021_fw_lock); + if (!ane->registered) + return -EOPNOTSUPP; + if (!planned && !atomic_read(&ane_t6021_quarantined)) + return 0; + /* Only the qualified, Linux-owned legacy boot profile can restart. + * Borrowed firmware, RTKit and later-generation ownership stay fenced. + */ + if (ane->soc->platform || !a || ane->rtk || + !legacy_only || !scratch3_ack || !legacy_query || fw_start_rtb_mode || + fw_start_table_mode != 2 || !ane_t6021_fwload_recoverable(a)) + return -EOPNOTSUPP; + /* Keep failed-boot retention bounded and charged against existing caps. */ + if (ane->retired_count >= 2 || ane_t6021_bo_epoch == U64_MAX) + return -ENOSPC; + mutex_lock(&ane_rtclient_pd_lock); + list_for_each_entry(pd, &ane_rtclient_pd_list, list) + if (pd->dev == ane->dev && pd->count == domains) + rs.cycle.pd = pd; + mutex_unlock(&ane_rtclient_pd_lock); + if (!rs.cycle.pd || !ane->engine_ps_valid) + return -EOPNOTSUPP; + rs.retired = kzalloc_obj(*rs.retired); + if (!rs.retired) + return -ENOMEM; + rs.fresh = devm_kcalloc(ane->dev, ANE_LEGACY_ALLOCS, sizeof(*rs.fresh), + GFP_KERNEL); + if (!rs.fresh) { + kfree(rs.retired); + return -ENOMEM; } - /* Opaque IDs never alias a new firmware instance. Clients reload programs. - * Advance the BO epoch only after old DMA and the new boot are verified. + + WRITE_ONCE(ane->recovering, true); + cancel_delayed_work_sync(&ane->poll_work); + ret = ane_t6021_restart_run(&ane_rtclient_restart_ops, &rs, planned, + &stage); + if (ret) + goto failed; + /* Opaque IDs never alias a new firmware instance; programs files still + * hold load again on their next use. Advance the BO epoch only after + * old DMA and the new boot are verified. */ ane_t6021_bo_new_epoch(); - ane_t6021_progtab_reset(&ane_t6021_progtab, &ane_t6021_progtab_ops, ane); + ane_t6021_progtab_restarted(&ane_t6021_progtab, &ane_t6021_progtab_ops, + ane); ane_rtclient_reap_retired(ane); WRITE_ONCE(ane->recovering, false); - dev_info(ane->dev, "firmware recovered; old program IDs invalid, reload and resubmit\n"); + dev_info(ane->dev, "firmware %s; programs reload on next use\n", + planned ? "restarted" : "recovered"); return 0; -unchanged: - devm_kfree(ane->dev, fresh); - kfree(retired); failed: - atomic_set(&ane_t6021_quarantined, 1); + if (rs.fresh) + devm_kfree(ane->dev, rs.fresh); + kfree(rs.retired); + /* A refused POWER_DOWN is a normal completion: the instance is still + * live and nothing was stopped. + */ + if (!rs.refused) + atomic_set(&ane_t6021_quarantined, 1); WRITE_ONCE(ane->recovering, false); dev_err(ane->dev, - "firmware recovery failed: %d; DMA backing and any live-CPU pin retained\n", + "firmware %s failed at %s: %d; DMA backing and any live-CPU pin retained\n", + planned ? "restart" : "recovery", + stage == ANE_T6021_RESTART_STOP ? "the ASC stop" : + stage == ANE_T6021_RESTART_CYCLE ? "the power cycle" : "the boot", ret); return ret; } +static int ane_rtclient_recover(struct ane_rtclient *ane) +{ + return __ane_rtclient_recover(ane, false); +} + +/* + * The budget restart: every resident program is a multi-task process the + * firmware cannot give back, and a new one must be loaded. + */ +static int ane_rtclient_restart(struct ane_rtclient *ane) +{ + int ret; + + if (atomic_read(&ane_t6021_quarantined)) + return -EIO; + dev_info(ane->dev, "restart: %u programs resident, none can be unloaded\n", + ane_t6021_progtab_resident(&ane_t6021_progtab)); + ret = __ane_rtclient_recover(ane, true); + return ret == -EOPNOTSUPP || ret == -ENOSPC ? -ENOSPC : ret; +} + static int ane_rtclient_probe_inner(struct platform_device *pdev, bool *hardware_touched) { From 7af2409ad59d9b12fbd2a11158d052d2416c5d14 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 15:10:47 +1100 Subject: [PATCH 40/73] accel/ane: post H14 POWER_DOWN without waiting for a completion POWER_DOWN (0x0b) on the 13.5 legacy firmware is the 8-byte header only; any other length asserts. The handler stops the firmware timers, requests its power state and waits in WFI for good, so it never posts a completion and its slot stays firmware-owned. The previous commit sent 16 bytes and waited for a completion, so the first planned restart would have quarantined the device. Post the header without waiting, after answering any request already on the SHAREDMALLOC ring (a core in WFI cannot finish that handshake). The dispatcher clears the completion field on entry, so mark it and wait for the clear, then require CPU_STATUS IDLE on twenty reads 100 us apart before clearing RUN. Drop the extra CPU_CONTROL bit 5 clear: it is not part of the ASC register contract and bit 5 is already clear while the core runs. STOPPED with RUN clear is still required before any power domain goes off. STOP (0x01) is not sent first. It runs the same no-timeout flush that never returns after a multi-task call, and the host already proves the instance idle under the firmware lock. The resident limit stays at eight. Document it as what it is: the firmware's L2 spill pool holds eight 2 MiB buffers and grows on every CREATE_PROCESS while it holds fewer, the program and process tables are far larger (240 and 496 advertised), and a LOAD sent with eight processes resident was seen to never complete. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 21 ++-- drivers/accel/ane/ane_t6021_lifetime.h | 27 +++-- drivers/accel/ane/ane_t6021_rtclient_main.c | 106 +++++++++++++++----- 3 files changed, 113 insertions(+), 41 deletions(-) diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index bda1dd31f6357..5ec14ed5fcab7 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -338,9 +338,12 @@ refuse a program the file holds no reference on with ``-ENOENT``. A program's kernel IDs stay valid for as long as a file holds it; whether it is loaded in the firmware right now (resident) is managed by the driver: -- Every process holds one 2 MiB firmware allocation, and a LOAD sent while - eight programs with processes are resident never completes. At most eight - programs are resident, and room is made before each LOAD. +- The firmware's L2 spill pool holds eight 2 MiB buffers; every + CREATE_PROCESS grows it while it holds fewer than eight, and a LOAD sent + while eight programs with processes were resident never completed. At most + eight programs are resident, and room is made before each LOAD. The + firmware's program and process tables (240 and 496 advertised) are not the + limit. - TERMINATE_PROCESS (0x203) of a process whose program has more than one task waits inside the firmware without a timeout and never completes, and no other command frees such a process. The driver never sends it for one. A @@ -392,11 +395,15 @@ initial boots remain fenced. A second H14 binding is refused because the firmware cache and BO pool have one owner. A restart for room (planned) runs with the firmware idle under the firmware -lock. It sends POWER_DOWN (0x0b), waits for its completion and for the ASC to -report WFI, then clears CPU_CONTROL RUN (bit 4) and bit 5, and requires -CPU_STATUS STOPPED (bit 1) with RUN clear. On T8112 an idle core reads 0x2c +lock. It answers any pending SHAREDMALLOC request and posts POWER_DOWN +(0x0b, the 8-byte header only). That command never completes: the firmware +stops its timers and waits in WFI for good. The driver instead waits for the +dispatcher to clear the completion field it marked, then for the ASC to report +WFI on twenty reads 100 us apart, then clears CPU_CONTROL RUN (bit 4) and +requires CPU_STATUS STOPPED (bit 1) with RUN clear. On T8112 an idle core reads 0x2c and a halted one 0x28, neither of which is stopped. After a command timeout -the firmware cannot answer POWER_DOWN, so only the RUN clear is tried. If +the command worker is blocked and nothing can overtake the stuck command, so +only the RUN clear is tried. If STOPPED is not seen, nothing is powered off: the device stays quarantined with its mappings, power claim and pin. Only after STOPPED does the restart suspend the consumer and suppliers through the shared PM transaction. This diff --git a/drivers/accel/ane/ane_t6021_lifetime.h b/drivers/accel/ane/ane_t6021_lifetime.h index 69505d7c768ab..1a769d3064f55 100644 --- a/drivers/accel/ane/ane_t6021_lifetime.h +++ b/drivers/accel/ane/ane_t6021_lifetime.h @@ -196,11 +196,15 @@ static inline long ane_t6021_td_blocks(const u8 *p, size_t size) #define ANE_T6021_PROG_SECS 9 #define ANE_T6021_NO_ID U32_MAX /* - * Programs resident in the firmware at once. Each may own one process, and - * each process one 2 MiB firmware allocation. With eight programs and - * eight processes resident the next LOAD never completed (M2, 13.5 bia - * firmware: 9 baseline + 8 x 2 MiB allocations), so no LOAD is sent while - * this many are resident. + * Programs resident in the firmware at once, each with at most one process. + * The firmware's L2 spill pool holds eight 2 MiB EXPP buffers: every + * CREATE_PROCESS grows it while it holds fewer than eight, whatever the + * program, and a TERMINATE may shrink it again. Its program and process + * tables are far larger (240 and 496 advertised). On the M2 a LOAD sent with + * eight processes resident (9 baseline + 8 x 2 MiB allocations) never + * completed, and the firmware has no static check that explains it. So no + * LOAD or CREATE is sent while this many programs are resident, which keeps + * one spill buffer per live process and stays below that observed state. */ #define ANE_T6021_FW_RESIDENT_MAX 8 @@ -670,11 +674,12 @@ static inline void ane_t6021_progtab_count(const struct ane_t6021_progtab *t, * A cold restart of the H14 firmware. The ASC must be proven stopped * (CPU_STATUS STOPPED) before its power domains go off: on T8112 an idle * core reads 0x2c and a halted but running one 0x28, neither stopped. A - * planned restart first asks the firmware to shut down (POWER_DOWN) and - * waits for its completion and for WFI. After a command timeout that - * handshake is impossible, so only the RUN clear is tried. If STOPPED is - * not seen nothing is powered off, and the caller keeps the instance - * quarantined with every mapping and its power claim. + * planned restart first asks the idle firmware to shut down (POWER_DOWN, + * which never completes; its dispatch is seen instead) and waits for WFI. + * After a command timeout the command worker is blocked and nothing can + * overtake it, so only the RUN clear is tried. If STOPPED is not seen + * nothing is powered off, and the caller keeps the instance quarantined + * with every mapping and its power claim. */ enum ane_t6021_restart_stage { ANE_T6021_RESTART_STOP, /* failed before power removal */ @@ -684,7 +689,7 @@ enum ane_t6021_restart_stage { }; struct ane_t6021_restart_ops { - int (*power_down)(void *ctx); /* POWER_DOWN and its completion */ + int (*power_down)(void *ctx); /* POWER_DOWN taken by the firmware */ int (*wait_idle)(void *ctx); /* bounded wait for WFI */ int (*halt)(void *ctx); /* clear RUN, require STOPPED */ int (*power_cycle)(void *ctx); /* domains verified off, then on */ diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index ac60c0ab36476..95f3bcbff6d7c 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -120,14 +120,16 @@ #define ANE_ASC_CPU_STATUS_RUNNING BIT(0) #define ANE_ASC_CPU_STATUS_STOPPED BIT(1) #define ANE_ASC_CPU_STATUS_IDLE BIT(5) -/* CPU_CONTROL bit cleared after RUN in the ASC stop sequence. */ -#define ANE_ASC_CPU_CONTROL_B5 BIT(5) -/* Legacy 13.5 POWER_DOWN: the firmware stops its timers and waits in WFI - * for good. Zero-filled body; the completion arrives before the WFI. +/* Legacy 13.5 POWER_DOWN: exactly the 8-byte header (any other length + * asserts). The firmware stops its timers, requests its power state and + * waits in WFI for good. It never posts a completion: its slot stays + * firmware-owned. The dispatcher zeroes the completion field at +6 on + * entry, which is the only sign the command was taken. */ #define ANE_T6021_CMD_POWER_DOWN 0x000b -#define ANE_T6021_POWER_DOWN_LEN 0x10 +#define ANE_T6021_POWER_DOWN_LEN 8 +#define ANE_T6021_DISPATCH_MARK 0xa55a /* Boot-time module parameters — defaults are the configuration that * boots the firmware, with fw_alias_reserved defaulting to own memory. @@ -3050,41 +3052,101 @@ struct ane_rtclient_restart { struct ane_rtclient_cycle cycle; struct ane_rtclient_retired *retired; struct ane_legacy_buffer *fresh; - bool refused; /* POWER_DOWN answered with a refusal */ + bool live; /* POWER_DOWN never reached the firmware */ }; -/* Ask the firmware to shut down. Only a live instance can answer, so this - * is never tried after a command timeout. +/* + * Post a command that never completes (POWER_DOWN). Same slot rules as + * ane_rtclient_legacy_exchange(): only a host-owned slot, and every request + * already on the SHAREDMALLOC ring is answered first, because a core in + * WFI cannot finish that handshake. The completion field carries a mark the + * dispatcher clears when it takes the command. + */ +static int ane_rtclient_legacy_post(struct ane_rtclient *ane, + struct ane_legacy_buffer *command, + size_t length, u16 opcode, + unsigned int channel) +{ + u64 *io; + int ret; + + if (!ane->held || !ane_rtclient_channels_unchanged(ane) || + ane->soc->platform) + return -EPROTO; + if (length < 8 || length > command->size) + return -EINVAL; + io = ane->fw->boot_ipc + ane->channels[channel].offset + + (size_t)ane->legacy_cmd_cursor[channel] * 64; + if (!(READ_ONCE(io[0]) & 1)) + return -EBUSY; + ret = ane_rtclient_malloc_drain(ane); + if (ret) + return ret; + ane_fw_command_header(command->cpu, opcode, false); + put_unaligned_le16(ANE_T6021_DISPATCH_MARK, (u8 *)command->cpu + 6); + WRITE_ONCE(io[1], length); + WRITE_ONCE(io[2], length); + dma_wmb(); + WRITE_ONCE(io[0], command->dma); + dma_wmb(); + writel(BIT(ane->channels[channel].bit), ane_rtclient_doorbell(ane)); + return 0; +} + +static u16 ane_rtclient_cmd_mark(struct ane_legacy_buffer *command) +{ + return le16_to_cpu(READ_ONCE(*(__le16 *)((u8 *)command->cpu + 6))); +} + +/* Ask the firmware to shut down and wait until its dispatcher took the + * command. Only a live, idle instance can take it: after a timeout the + * command worker is blocked and nothing can overtake the stuck command, so + * this is never tried then. */ static int ane_rtclient_rs_power_down(void *ctx) { struct ane_rtclient_restart *rs = ctx; struct ane_rtclient *ane = rs->ane; + u16 mark; int ret; if (!ane->cmd_buf) return -ENODEV; memset(ane->cmd_buf->cpu, 0, SZ_16K); - ret = ane_rtclient_legacy_exchange(ane, ane->cmd_buf, - ANE_T6021_POWER_DOWN_LEN, - ANE_T6021_CMD_POWER_DOWN, 1, - jiffies + msecs_to_jiffies(3000)); - rs->refused = ret == -EREMOTEIO; - dev_info(ane->dev, "restart: POWER_DOWN %d (CPU_STATUS %#x)\n", ret, + ret = ane_rtclient_legacy_post(ane, ane->cmd_buf, + ANE_T6021_POWER_DOWN_LEN, + ANE_T6021_CMD_POWER_DOWN, 1); + /* Nothing was posted: the instance keeps serving. */ + rs->live = ret == -EINVAL; + if (!ret) + ret = read_poll_timeout(ane_rtclient_cmd_mark, mark, !mark, 50, + 100000, false, ane->cmd_buf); + dev_info(ane->dev, "restart: POWER_DOWN %s %d (CPU_STATUS %#x)\n", + ret ? "not taken" : "taken", ret, readl(ane->engine + ANE_ASC_CPU_STATUS)); return ret; } +/* The shutdown path ends in WFI. IDLE is also set at ordinary idle points, + * so require it on twenty reads 100 us apart after the command was taken. + */ static int ane_rtclient_rs_wait_idle(void *ctx) { struct ane_rtclient_restart *rs = ctx; - u32 status; + void __iomem *st = rs->ane->engine + ANE_ASC_CPU_STATUS; + unsigned long until = jiffies + msecs_to_jiffies(100); + unsigned int run = 0; - return readl_poll_timeout(rs->ane->engine + ANE_ASC_CPU_STATUS, status, - status & ANE_ASC_CPU_STATUS_IDLE, 100, 100000); + while (time_before(jiffies, until)) { + run = readl(st) & ANE_ASC_CPU_STATUS_IDLE ? run + 1 : 0; + if (run >= 20) + return 0; + usleep_range(100, 150); + } + return -ETIMEDOUT; } -/* Clear RUN, then bit 5, each read back; then require STOPPED with RUN +/* Clear RUN (read-modify-write, read back), then require STOPPED with RUN * clear. Nothing is powered off unless this returns zero. */ static int ane_rtclient_rs_halt(void *ctx) @@ -3100,8 +3162,6 @@ static int ane_rtclient_rs_halt(void *ctx) return ret; writel(readl(ctl) & ~ANE_ASC_CPU_CONTROL_RUN, ctl); readl(ctl); - writel(readl(ctl) & ~ANE_ASC_CPU_CONTROL_B5, ctl); - readl(ctl); ret = readl_poll_timeout(ane->engine + ANE_ASC_CPU_STATUS, status, (status & ANE_ASC_CPU_STATUS_STOPPED) && !(status & ANE_ASC_CPU_STATUS_RUNNING), @@ -3307,10 +3367,10 @@ static int __ane_rtclient_recover(struct ane_rtclient *ane, bool planned) if (rs.fresh) devm_kfree(ane->dev, rs.fresh); kfree(rs.retired); - /* A refused POWER_DOWN is a normal completion: the instance is still - * live and nothing was stopped. + /* A POWER_DOWN that was never posted left the instance in service. + * Once posted, the firmware is shut down or about to be. */ - if (!rs.refused) + if (!rs.live) atomic_set(&ane_t6021_quarantined, 1); WRITE_ONCE(ane->recovering, false); dev_err(ane->dev, From 64e23b40571458f482244ae36e29dcfc92379dd5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 15:30:14 +1100 Subject: [PATCH 41/73] drm/apple: Switch desktop 26.x external displays to VRR without a DRM mode set GNOME never turned on variable refresh for a fullscreen game on the J616s external displays: mutter flips VRR_ENABLED in a plain page flip, and the external processors made that a mode set, so the flip failed with EINVAL. Their firmware takes adaptive sync only with a mode set: a new minimum rate on a live mode is ignored (an MSI MPG 325CQRXF stayed at its 48 Hz floor with 280, 279 or 240 Hz), and macOS too reselects the same timing to switch such a display between fixed and variable refresh. Accept the VRR_ENABLED change in atomic check and, when it changes on a live VRR timing, reselect the mode from the commit's flush, once. The mode set installs adaptive sync at the timing's minimum when VRR_ENABLED is set and fixed refresh otherwise, as before. A compositor that toggles VRR only for fullscreen clients sees one reselect each time one starts or ends. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 16 +++--------- drivers/gpu/drm/apple/iomfb.c | 25 +++++++++++++++++++ .../drivers/gpu/apple-dcp/lifecycle.py | 13 ++++------ 4 files changed, 36 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 7e234e2bfac43..aed0d95607973 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -274,6 +274,8 @@ struct dcp_desktop_state { * the firmware was last given. Zero when not installed that way. */ u32 vrr_min_on, vrr_min_off, vrr_min_sent; + /* VRR_ENABLED as of the last mode set (external processors). */ + bool vrr_requested; /* The frame's colour matrix, sent after an adaptive sync update. */ struct iomfb_set_matrix_req matrix; /* diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d575a503c6d8b..ca3b47379acbd 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1510,19 +1510,11 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi return -EINVAL; /* - * On desktop 26.6 firmware the panel takes VRR_ENABLED with the next - * frame, without a mode set (see dcp_vrr_update()). The external - * processors ignore a new minimum rate on a live mode, so they still - * change adaptive sync with a mode set. + * On desktop 26.6 firmware VRR_ENABLED takes effect with the next + * frame, without a DRM mode set: the panel through a new minimum + * rate (dcp_vrr_update()), the external processors by reselecting + * their timing (dcp_flush()). */ - if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && crtc_state->enable && - !dcp_vrr_live(dcp)) { - struct drm_crtc_state *old = - drm_atomic_get_old_crtc_state(commit, crtc); - - if (old && old->vrr_enabled != crtc_state->vrr_enabled) - crtc_state->mode_changed = true; - } /* DRM core only caps the blob size; the gamma update copies a full table. */ if (crtc_state->color_mgmt_changed && crtc_state->gamma_lut && diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8c464d03a0508..8147b8782125e 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -2615,6 +2615,8 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, dcp->pq_requested = dcp_crtc_eotf(dcp, crtc, commit) == HDMI_EOTF_SMPTE_ST2084; dcp->output_edr = dcp->pq_requested && dcp_panel_edr_supported(dcp); dcp->pq_requested &= dcp_hdr_output_supported(dcp); + if (dcp->desktop) + dcp->desktop->vrr_requested = crtc_state->vrr_enabled; dcp_hdr_metadata_forget(dcp); dcp_mode_begin(&dcp->mode_state); switch (dcp->fw_compat) { @@ -3158,6 +3160,29 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) dcp_crtc_atomic_modeset(crtc, commit); } } + + /* + * The external processors take adaptive sync only with a mode set; + * macOS too reselects the same timing to switch a display between + * fixed and variable refresh. Compositors flip VRR_ENABLED in plain + * page flips (mutter when a fullscreen client comes and goes), so a + * change on a live VRR timing reselects the mode here, once. + */ + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && !dcp_vrr_live(dcp) && + READ_ONCE(dcp->mode_state.valid) && dcp->connector && + READ_ONCE(dcp->connector->connected)) { + struct drm_crtc_state *cs = drm_atomic_get_new_crtc_state(commit, crtc); + struct dcp_display_mode mode; + + if (cs && cs->active && !drm_atomic_crtc_needs_modeset(cs) && + cs->vrr_enabled != dcp->desktop->vrr_requested && + lookup_mode(dcp, &cs->mode, &mode) && mode.vrr) { + dev_dbg(dcp->dev, "VRR_ENABLED %u: reselecting the mode\n", + cs->vrr_enabled); + dcp_mode_invalidate(&dcp->mode_state); + dcp_crtc_atomic_modeset(crtc, commit); + } + } if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { iomfb_v14_7_flush(dcp, crtc, commit); return; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 89dd1f31f954b..89081523f183d 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -1730,9 +1730,6 @@ def vrr(root): static int dcp_crtc_check_hdr(struct apple_dcp *d, struct drm_crtc *c, struct drm_atomic_commit *s) { (void)d; (void)c; (void)s; return 0; } -/* The panel of a desktop 26.6 machine takes VRR_ENABLED live. */ -static bool dcp_vrr_live(struct apple_dcp *d) -{ return d->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && d->main_display; } static bool iomfb_is_m4(struct apple_dcp *d) { (void)d; return false; } static int iomfb_m4_error(struct apple_dcp *d) { (void)d; assert(false); return -ENODEV; } static int iomfb_m4_get_mode(struct apple_dcp *d,struct drm_display_mode *m) @@ -1791,10 +1788,10 @@ def vrr(root): return 0; } """ - # Desktop 26.6 takes VRR_ENABLED live on the panel (main display), with - # the next frame: compositors toggle it in plain page flips, so there a - # VRR_ENABLED change must never become a mode set. The external - # processors ignore a live minimum rate and still apply it with a mode set. + # Desktop 26.6 takes VRR_ENABLED with the next frame on the panel and the + # external processors alike (the latter reselect their mode in flush): + # compositors toggle it in plain page flips, so a VRR_ENABLED change + # must never become a DRM mode set. desktop = r""" dcp.fw_compat = DCP_FIRMWARE_V_26_6_DESKTOP; dcp.hw.iomfb_method_profile = 0; dcp.mode_state.valid = true; @@ -1812,7 +1809,7 @@ def vrr(root): state.no_old_crtc = old_vrr == 2; state.old_crtc.vrr_enabled = old_vrr == 1; assert(dcp_crtc_atomic_check(&crtc, &state) == 0); - bool expected = !panel && enable && old_vrr != 2 && (old_vrr == 1) != vrr; + bool expected = false; if (state.crtc.mode_changed != expected) { fprintf(stderr, "desktop panel=%u enable=%u old_vrr=%u vrr=%u: mode_changed=%d expected %d\n", panel, enable, old_vrr, vrr, state.crtc.mode_changed, expected); From 9b125f8d1eb21ffa94a5036538d83a79e37c19e5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 15:37:08 +1100 Subject: [PATCH 42/73] media: apple: avd: require the saved queued FIFO and audit bank addresses A queued frame must use the FIFO saved by its program builder. Reject a missing address before configuring registers or posting a descriptor, rather than substituting a mutable context FIFO during a retry. Include the saved DMA address in the existing dynamic-debug program record. Exercise the H.264 bank allocator and selector, IDR/P/B address emitters, all valid slots, and the v3/v5 host register layouts. Inject a zero address into each bank-1 buffer and require the host test to fail. Replay the resulting descriptors, programs and MMIO in a routed Unicorn workload matrix, holding decoder DONE events and checking DMA operands and tags. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../platform/apple/avd/avd-backend-queue.c | 10 +- .../media/platform/apple/avd/tests/README.rst | 43 ++++ .../platform/apple/avd/tests/program-test.py | 144 +++++++++--- .../apple/avd/tests/unicorn-inflight-test.py | 209 ++++++++++++++++++ .../apple/avd/tests/unicorn-queue-test.py | 2 + 5 files changed, 377 insertions(+), 31 deletions(-) create mode 100644 drivers/media/platform/apple/avd/tests/unicorn-inflight-test.py diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c index 0c195879416a4..785716d450e3e 100644 --- a/drivers/media/platform/apple/avd/avd-backend-queue.c +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -240,6 +240,9 @@ static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) /* The program's EXEC words name FIFO f->idx; it must be a real FIFO. */ if (f->idx >= q->fifo_slots) return -EINVAL; + /* The prepared picture owns this FIFO, including across inbox retries. */ + if (!f->fifo_iova) + return -EINVAL; /* A busy inbox owns nothing; retry this same prepared frame later. */ if (!(readl(avd->mbox + AVD_REG_MBOX0_STATUS) & AVDQ_MBOX_STATUS_EMPTY)) return -EBUSY; @@ -259,7 +262,7 @@ static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) codec != AVD_CODEC_AV1); /* FIFO registers of other live frames are never touched. */ - avd->variant->configure_stream(avd, f->fifo_iova ?: ctx->inst.addr, f->idx, vp); + avd->variant->configure_stream(avd, f->fifo_iova, f->idx, vp); writel(AVDQ_MAGIC, d + D_MAGIC); writel(f->tag, d + D_TAG); @@ -281,8 +284,9 @@ static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) wmb(); (void)readl(d + D_STATE); /* flush posted aperture writes */ avd_qfw_debug(avd, f, "doorbell"); - dev_dbg(avd->dev, "queue program: tag %#x vp %u words %zu kick %#x start %#x end %#x\n", - f->tag, vp, f->prog_words, kick, words[0], words[f->prog_words - 1]); + dev_dbg(avd->dev, "queue program: tag %#x vp %u fifo %pad words %zu kick %#x start %#x end %#x\n", + f->tag, vp, &f->fifo_iova, f->prog_words, kick, words[0], + words[f->prog_words - 1]); writel(q->aperture + AVDQ_RING_OFFSET + f->idx * AVDQ_DESC_BYTES, avd->mbox + AVD_REG_MBOX0_SEND); diff --git a/drivers/media/platform/apple/avd/tests/README.rst b/drivers/media/platform/apple/avd/tests/README.rst index dd32f5a103e99..2a6f378fa31bf 100644 --- a/drivers/media/platform/apple/avd/tests/README.rst +++ b/drivers/media/platform/apple/avd/tests/README.rst @@ -74,3 +74,46 @@ the corrected mailbox-status model. The FIFO status and IRQ-router geometry follow the public MIT emulator at https://github.com/eiln/avd/blob/main/avd_emu.py. The pinned avd-fw source's Tansy routing independently identifies the same banks and bit positions. + +Concurrent pictures and DMA addresses +------------------------------------ + +``program-test.py`` also generates per-image subdirectories for v3-t0, v3-t1, +v5-t0 and v5-t2. It runs the production H.264 bank allocator and bank selection, +uses both banks of eight fixture contexts, and emits IDR, P and B pictures. +Every scratch, output, compressed-reference and MV address passed to an address +emitter must be nonzero. The emitted address locations/encodings are exported +alongside the actual descriptor and configure_stream MMIO snapshot for each +slot. Coded-data operands are checked separately by the routed test. Seven +fault-injection runs zero each bank-1 buffer address in turn and must fail. +A missing saved FIFO is rejected at every valid slot before the doorbell, +including when the context contains a different nonzero fallback address. + +Build the firmware with the busy-VP scheduling fix, then run:: + + for image in v3-t0 v3-t1 v5-t0 v5-t2; do + python3 tests/unicorn-inflight-test.py /path/to/avd-fw \ + /path/to/avd-fw/build/avd-fw-$image.bin /tmp/avd-program + done + +This test boots each image's reset vector, checks router and NVIC masks, replays +production host MMIO, and verifies every emitted DMA operand at its recorded +program location. It holds VP DONE, PP DONE and the completion mailbox to test +ownership and backpressure. Workloads range from 2 to 15 pictures. On the +15-FIFO images all requested pictures are posted before any VP DONE. v5-t0 +advertises only seven physical FIFOs: workloads above seven use a seven-slot +window with slot reuse and a new full completion tag. This is not evidence of +8..15 simultaneous hardware FIFOs on that image. H.264 retains two banks per +context; larger windows represent multiple contexts. + +The decoder model asserts that a VP cannot receive the next picture's START +before its current picture's DONE. With the old firmware this reproduces at +slot 1; ``--before`` requires that failure instead of treating it as a pass. +The fixed scheduler waits at that boundary while retaining VP/PP overlap and +strict global SRAM program order. The native firmware test also covers +independent VPs and ordered PP completion. + +The live-VP assertion is a conservative decoder ownership model. It proves the +software feed ordering and its correction, not the undocumented silicon cause +of a DART fault. A hardware retest remains necessary to connect this scheduling +defect to a reported zero-address DMA and to verify pixels and performance. diff --git a/drivers/media/platform/apple/avd/tests/program-test.py b/drivers/media/platform/apple/avd/tests/program-test.py index 6563dcfa05d0a..89613f1665a92 100644 --- a/drivers/media/platform/apple/avd/tests/program-test.py +++ b/drivers/media/platform/apple/avd/tests/program-test.py @@ -11,6 +11,7 @@ import hashlib from pathlib import Path import re +import signal import subprocess import sys @@ -50,6 +51,8 @@ #define AVD_MAX_INST 512 #define AVD_CODEC_AV1 3 #define AVD_MAX_FRAMES 16 +#define AVD_IMG_FMT_420_10BIT 1 +#define AVD_IMG_FMT_422_10BIT 2 #define AVD_FRAME_SUBMITTED 2 #define AVD_FRAME_DONE 3 #define IRQ_HANDLED 1 @@ -93,11 +96,11 @@ struct vb2_v4l2_buffer { struct vb2_buffer vb2_buf; }; struct v4l2_m2m_buffer { struct vb2_v4l2_buffer vb; }; struct avd_comp { size_t start_offset; u32 offsets[4]; }; -struct avd_decoded_buffer { struct v4l2_m2m_buffer base; struct avd_comp comp; }; +struct avd_decoded_buffer { struct v4l2_m2m_buffer base; struct avd_comp comp; unsigned int bank; }; struct avd_ctx; struct avd_coded_fmt_ops { unsigned int (*pp_kicks)(struct avd_ctx *); }; struct fmt_desc { void *frame_ops; const struct avd_coded_fmt_ops *ops; }; -struct avd_ctx { struct avd_dev *dev; void *priv; struct avd_job job; +struct avd_ctx { struct avd_dev *dev; void *priv; struct avd_job job; unsigned int image_fmt; const struct fmt_desc *coded_fmt_desc; struct { struct { struct { int q; } cap_q_ctx; } *m2m_ctx; } fh; struct { struct { struct { struct { u32 bytesperline; } plane_fmt[1]; } pix_mp; } fmt; } decoded_fmt; @@ -106,13 +109,23 @@ }; struct avd_run { struct { struct vb2_v4l2_buffer *dst; } bufs; dma_addr_t comp_out, y_out, uv_out; }; +struct dma_site { unsigned int segment,word,shift,width; dma_addr_t addr; }; +static struct dma_site dma_sites[256];static unsigned int dma_count; +static void record_dma(struct avd_ctx *ctx,dma_addr_t addr,unsigned int shift,unsigned int width) { + assert(addr && (addr>>shift) && dma_countjob.num,ctx->job.segments[ctx->job.num].num,shift,width,addr}; +} static bool avd_backend_is_qfw(struct avd_dev *d) { return d->queued; } +static dma_addr_t next_iova=0x204000000ULL; static int avd_buf_alloc(struct avd_dev *d, struct avd_buf *b, size_t n) -{ (void)d; free(b->cpu); b->cpu = calloc(1,n); b->size = n; return b->cpu ? 0 : -ENOMEM; } +{ (void)d; free(b->cpu); b->cpu = calloc(1,n); b->size = n; + b->addr=next_iova; next_iova+=(n+0xffff)&~0xffffULL; return b->cpu ? 0 : -ENOMEM; } +static void avd_buf_free(struct avd_dev *d, struct avd_buf *b) { (void)d;free(b->cpu);memset(b,0,sizeof(*b)); } static struct avd_decoded_buffer *vb2_to_avd_decoded_buf(struct vb2_buffer *b) { return (void *)b; } +static struct avd_decoded_buffer reference; static struct avd_decoded_buffer *avd_get_ref_buf(struct avd_ctx *c, - struct vb2_v4l2_buffer *b, u64 t) { (void)c; (void)t; return (void *)b; } + struct vb2_v4l2_buffer *b, u64 t) { (void)c; (void)b; (void)t; return &reference; } static dma_addr_t vb2_dma_contig_plane_dma_addr(struct vb2_buffer *b, int p) { (void)p; return b->dma; } static struct vb2_buffer *vb2_find_buffer(void *q, u64 t) @@ -141,68 +154,114 @@ struct avd_frame frame = { .prog=&program }; struct avd_dev dev = { .variant=&variant, .building=&frame, .queued=true }; struct avd_ctx ctx = { .dev=&dev, .job.codec=AVD_CODEC_H264 }; - struct avd_h264_ctx h264 = {}; + typeof(*ctx.fh.m2m_ctx) m2m = {}; + struct avd_h264_ctx h264[8] = {}; struct avd_qfw q = { .sram=calloc(1,0x10000), .aperture=0x1090000, .ring_words=0x1800, .fifo_slots=15 }; static const struct avd_coded_fmt_ops ops={.pp_kicks=pp_kicks}; static const struct fmt_desc fmt={.ops=&ops}; dev.qfw=&q;dev.ctrl=calloc(1,0x10000);dev.mbox=calloc(1,0x100);dev.capabilities=7; - ctx.coded_fmt_desc=&fmt;frame.ctx=&ctx; + ctx.coded_fmt_desc=&fmt;frame.ctx=&ctx;ctx.fh.m2m_ctx=&m2m; struct avd_buf slice = { .size=4096, .addr=0x200800000ULL }; struct v4l2_ctrl_h264_sps sps = { .pic_width_in_mbs_minus1=119, .pic_height_in_map_units_minus1=67, .chroma_format_idc=1, .flags=V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY | V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE }; struct v4l2_ctrl_h264_decode_params decode = { .flags=V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC }; struct v4l2_ctrl_h264_pps pps = {}; + struct v4l2_ctrl_h264_pred_weights weights = {}; + struct avd_decoded_buffer dst = { .base.vb.vb2_buf.dma=0x203000000ULL }; struct v4l2_ctrl_h264_slice_params sl = { .slice_type=V4L2_H264_SLICE_TYPE_I, .header_bit_size=32 }; struct avd_h264_run run = { .sps=&sps, .pps=&pps, .decode_params=&decode, - .slice_params=&sl, .slice_data_off=4, + .slice_params=&sl, .slice_data_off=4, .pred_weights=&weights, + .addresses.mv_color=0x203800000ULL, .num_valid=1, .base={ .comp_out=0x200100000ULL, .y_out=0x200200000ULL, .uv_out=0x200400000ULL } }; - assert(argc==2); - ctx.priv=&h264; ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; - h264.b=&h264.bank[0]; h264.b->active_slice=&slice; - h264.b->bufs.above_info.addr=0x200600000ULL; - h264.b->bufs.pipe_state.addr=0x200610000ULL; - h264.b->bufs.ip_above.addr=0x200620000ULL; - h264.b->bufs.lf_above.addr=0x200630000ULL; - h264.b->bufs.lf_above_info.addr=0x200640000ULL; + assert(argc==3); + bool v5=!strncmp(argv[2],"v5",2); + if (v5) { + variant.quirks=0;variant.configure_stream=t8122_configure_stream; + if (!strcmp(argv[2],"v5-t0")) { variant.fifo_slots=7;variant.vp_slot_base[1]=4; } + } else if (!strcmp(argv[2],"v3-t0")) variant.quirks|=AVD_QUIRK_NO_PIPE_STATE; + q.fifo_slots=variant.fifo_slots; + ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; + run.base.bufs.dst=&dst.base.vb; + for(unsigned int c=0;c<8;c++) { + ctx.priv=&h264[c];h264[c].nbanks=AVD_H264_BANKS;h264[c].b=&h264[c].bank[0]; + for(unsigned int b=0;bb==&hc->bank[idx%AVD_H264_BANKS]);hc->b->active_slice=&slice; + hc->b->submitted=true; + ctx.job_fifo=hc->b->bufs.inst.addr; + assert(ctx.job_fifo && hc->b->bufs.above_info.addr && hc->b->bufs.ip_above.addr && + hc->b->bufs.lf_above.addr && hc->b->bufs.lf_above_info.addr && hc->b->bufs.mv_above_info.addr); + assert((variant.quirks&AVD_QUIRK_NO_PIPE_STATE)||hc->b->bufs.pipe_state.addr); frame.idx=idx; ctx.job.num=0;ctx.pp_total=1; + dma_count=0; + run.base.comp_out=0x230000000ULL+idx*0x1000000ULL; + run.base.y_out=run.base.comp_out+0x100000; + run.base.uv_out=run.base.comp_out+0x300000; + run.addresses.mv_color=run.base.y_out+0x700000; + reference.base.vb.vb2_buf.dma=idx?run.base.comp_out-0x1000000:run.base.comp_out; + decode.flags=idx%3?0:V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC; + decode.dpb[0].flags=idx%3?V4L2_H264_DPB_ENTRY_FLAG_VALID:0; + sl.slice_type=idx%3==0?V4L2_H264_SLICE_TYPE_I:idx%3==1?V4L2_H264_SLICE_TYPE_P:V4L2_H264_SLICE_TYPE_B; ctx.job.segments=calloc(3,sizeof(*ctx.job.segments)); assert(ctx.job.segments); stream_hdr(&ctx,&run); ctx.job.num=1;sl.first_mb_in_slice=0;slice.addr=0x200800000ULL; stream_slice(&ctx,&run); size_t continuation=1+ctx.job.segments[0].num+ctx.job.segments[1].num; ctx.job.num=2;sl.first_mb_in_slice=4080;slice.addr+=0x1000; stream_slice(&ctx,&run); + size_t starts[3]={1,1+ctx.job.segments[0].num,2+ctx.job.segments[0].num+ctx.job.segments[1].num}; assert(!avd_open_build_program(&ctx)); u32 fifo=dev.queued ? idx<<4 : 0; assert(((u32 *)program.cpu)[0]==(0x2b000200U|fifo)); assert(((u32 *)program.cpu)[continuation]==(0x2b000000U|fifo)); assert(((u32 *)program.cpu)[frame.prog_words-1]==(0x2b000400U|fifo)); if (dev.queued) { - frame.tag=0x10|idx;frame.fifo_iova=0x201000000ULL+idx*0x100000; + frame.tag=0x10|idx;assert(frame.fifo_iova==hc->b->bufs.inst.addr && !ctx.job_fifo); q.wp=0;memset(q.sram,0,0x10000);memset(dev.ctrl,0,0x10000); writel(1,dev.mbox+0x50); assert(avd_qfw_submit(&dev,&frame)==-EBUSY); - assert(!readl(q.sram+0x8000)&&!q.wp&&!readl(dev.ctrl+0x150+idx*4)); - for(unsigned int other=0;other<15;other++) - if(other!=idx)writel(0xabc000+other,dev.ctrl+0x150+other*4); + unsigned int lo=v5?0x1d0:0x150; + assert(!readl(q.sram+0x8000)&&!q.wp&&!readl(dev.ctrl+lo+idx*4)); + for(unsigned int other=0;other>8); - assert(readl(dev.ctrl+0x10c)==7&&readl(dev.ctrl+0x120)==5); + assert(readl(dev.ctrl+lo+idx*4)==(v5?(u32)frame.fifo_iova:frame.fifo_iova>>8)); + assert(readl(dev.ctrl+(v5?0x16c:0x10c))==7&&readl(dev.ctrl+(v5?0x190:0x120))==5); + snprintf(path,sizeof(path),"%s/mmio-%u.bin",argv[1],idx); + out=fopen(path,"wb");assert(out);assert(fwrite(dev.ctrl,0x10000,1,out)==1);assert(!fclose(out)); snprintf(path,sizeof(path),"%s/descriptor-%u.bin",argv[1],idx); out=fopen(path,"wb");assert(out);assert(fwrite(q.sram+0x8000+idx*64,64,1,out)==1);assert(!fclose(out)); + snprintf(path,sizeof(path),"%s/dma-sites-%u.txt",argv[1],idx); + out=fopen(path,"w");assert(out); + for(unsigned int a=0;asegment]+site->word,site->width,site->shift,(unsigned long long)site->addr); + } + assert(!fclose(out)); } snprintf(path,sizeof(path),"%s/%s-%u.bin",argv[1],dev.queued?"h264":"legacy",idx); out=fopen(path,"wb"); assert(out); @@ -241,8 +300,20 @@ dev.frames[0].state=AVD_FRAME_SUBMITTED;dev.frames[0].tag=0x20; writel(2,desc+D_STATE);messages[0]=0x54530010;msg_count=1;msg_read=0; avd_qfw_irq(&dev);assert(completed==2&&dev.frames[0].state==AVD_FRAME_SUBMITTED); + /* A missing saved FIFO must fail before any ownership/MMIO changes, even + * when the mutable context happens to contain another nonzero address. */ + dev.queued=true;frame.fifo_iova=0;ctx.inst.addr=0x208000000ULL; + writel(0x20001,dev.mbox+0x50);u32 old_wp=q.wp; + for(unsigned int idx=0;idxinst.cpu);free(buf->pipe_state.cpu);free(buf->above_info.cpu); + free(buf->lf_above_info.cpu);free(buf->lf_above.cpu);free(buf->ip_above.cpu);free(buf->mv_above_info.cpu); + } free(program.cpu);free(q.sram);free(dev.ctrl);free(dev.mbox); - puts("PASS: production H.264 IDR program, queued FIFO 0..14 and unchanged legacy EXECs"); + puts("PASS: production H.264 banks, IDR/P/B DMA operands, saved FIFO addresses and legacy EXECs"); } ''' @@ -264,15 +335,32 @@ def main(): hw = (ROOT / 'avd-hw.c').read_text() regs = hw[hw.index('#define AVD_V3_'):hw.index('int avd_boot(')] config = '#define w32(r,v) writel((v),avd->ctrl+(r))\n#define m32(r,v) w32(r,(v)|readl(avd->ctrl+(r)))\n' - config += function(hw, 't8112_configure_stream') - code.write_text(PRELUDE + inst + h264 + builder + wire + regs + config + + config += function(hw, 't8112_configure_stream') + function(hw, 't8122_configure_stream') + # Address emitters must never consume a zero bank/output/reference address. + inst = inst.replace('static inline void push_address(struct avd_ctx *ctx, dma_addr_t addr)\n{', + 'static inline void push_address(struct avd_ctx *ctx, dma_addr_t addr)\n{\n record_dma(ctx,addr,(ctx->dev->variant->quirks&AVD_QUIRK_LSR)?8:0,(ctx->dev->variant->quirks&AVD_QUIRK_LSR)?1:2);') + inst = inst.replace('u32 offsets[4])\n{', 'u32 offsets[4])\n{\n assert(addr);') + inst = inst.replace('push(ctx, (addr + offsets[i]) >> 7);', '{ record_dma(ctx,addr+offsets[i],7,1); push(ctx, (addr + offsets[i]) >> 7); }') + code.write_text(PRELUDE + inst + h264 + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_alloc_bufs') + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_drop_bank_slices') + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_pick_bank') + builder + wire + regs + config + function(qfw, 'avd_qfw_stream') + function(qfw, 'avd_qfw_submit') + function(qfw, 'avd_qfw_fail_all') + function(qfw, 'avd_qfw_irq') + MAIN) exe = out / 'program-test' subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-Wall', '-Wextra', '-Werror', '-Wno-sign-compare', '-Wno-unused-function', '-fsanitize=address,undefined', '-g', str(code), '-o', str(exe)], check=True) - subprocess.run([str(exe), str(out)], check=True) + for variant in ('v3-t0', 'v3-t1', 'v5-t0', 'v5-t2'): + target = out / variant + target.mkdir(exist_ok=True) + subprocess.run([str(exe), str(target), variant], check=True) + for field in ('inst', 'pipe_state', 'above_info', 'ip_above', 'lf_above', 'lf_above_info', 'mv_above_info'): + target = out / ('zero-' + field) + target.mkdir(exist_ok=True) + with (target / 'expected-failure.log').open('w') as log: + result = subprocess.run([str(exe), str(target), 'v3-t1'], + env=dict(os.environ, AVD_ZERO_BANK_ADDR=field), + stdout=log, stderr=subprocess.STDOUT) + assert result.returncode == -signal.SIGABRT, (field, result.returncode) + print('PASS: host test rejects zero bank-1 ' + field, flush=True) if __name__ == '__main__': diff --git a/drivers/media/platform/apple/avd/tests/unicorn-inflight-test.py b/drivers/media/platform/apple/avd/tests/unicorn-inflight-test.py new file mode 100644 index 0000000000000..1ba0f5d0f121f --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/unicorn-inflight-test.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Routed CM3 queue test using production host programs, DMA sites and MMIO. + +usage: unicorn-inflight-test.py FIRMWARE_TREE IMAGE PROGRAM_DIRECTORY [--before] +The decoder model holds each VP busy until an explicit DONE, checks emitted +addresses, and delays DONE/PP/outbox to exercise overlap. It does not decode +pixels or simulate the decoder's undocumented internal register execution. +""" +import importlib.util +from pathlib import Path +import struct +import sys +from unicorn.arm_const import UC_ARM_REG_SP + +firmware, image, programs = map(lambda p: Path(p).resolve(), sys.argv[1:4]) +spec = importlib.util.spec_from_file_location('transport', firmware / 'tests/transport-test.py') +transport = importlib.util.module_from_spec(spec) +spec.loader.exec_module(transport) + + +class LiveVPError(AssertionError): + pass + + +class Inflight(transport.Transport): + def __init__(self): + self.booting = False + self.live = None + self.feed_idx = 0 + self.feed_word = 0 + self.expected = [] + self.events = [] + self.address_checks = 0 + self.pp_owner = None + self.vp_events = self.pp_events = 0 + super().__init__(image) + self.booting = True + self.uc.reg_write(UC_ARM_REG_SP, self.u32(0)) + self.uc.emu_start(self.u32(4), 0, count=500000) + assert not self.booting, 'reset did not publish boot-ready' + assert self.u32(0x50010010) & 2 + self.lo = 0x150 if self.version == 3 else 0x1d0 + self.hi = None if self.version == 3 else 0x20c + self.mask = 0x18c if self.version == 3 else 0x248 + self.cach = 0x1c8 if self.version == 3 else 0x284 + self.xfer = 0x204 if self.version == 3 else 0x2c0 + + def route(self, irq): + if irq < 14: + router = self.u32(0x50010010) & (1 << irq) + else: + off = irq - 14 + router = self.u32(0x50010014 + (off // 32) * 4) & (1 << (off % 32)) + nvic = self.u32(0xe000e100 + (irq // 32) * 4) & (1 << (irq % 32)) + assert router and nvic, f'masked IRQ{irq}' + self.call(f'irq{irq}') + + def fifo_addr(self, idx): + low = self.u32(self.decode + self.lo + idx * 4) + addr = low << 8 if self.hi is None else low | self.u32(self.decode + self.hi + idx * 4) << 32 + assert addr, f'zero FIFO DMA address at slot {idx}' + self.address_checks += 1 + return addr + + def write(self, uc, access, addr, size, value, user): + if self.booting and addr == 0x50010090 and value == 1: + self.booting = False + uc.emu_stop() + if self.decode + self.fifo <= addr < self.decode + self.fifo + self.vps * 4: + idx, words = self.expected[self.feed_idx] + assert addr == self.decode + self.fifo + 4 * 4, 'wrong H.264 VP' + assert value == words[self.feed_word], (idx, self.feed_word, hex(value)) + if self.feed_word == 0: + if self.live is not None: + raise LiveVPError(f'VP4 receives slot {idx} START while slot {self.live} has no DONE') + self.live = idx + assert (value >> 4) & 15 == idx + fifo = self.fifo_addr(idx) + self.events.append(f'VP4 START slot={idx} fifo={fifo:#x}') + self.feed_word += 1 + if self.feed_word == len(words): + self.feed_idx += 1 + self.feed_word = 0 + if addr == self.decode + self.pp: + idx = (value >> 4) & 15 + assert self.pp_owner is None, 'PP overwritten before DONE' + assert value == 0x2b000200 | idx << 4 | self.info[10] + self.pp_owner = idx + fifo = self.fifo_addr(idx) + self.events.append(f'PP START slot={idx} fifo={fifo:#x}') + if addr == 0x50010060: + self.events.append(f'completion token={value:#x}') + super().write(uc, access, addr, size, value, user) + + def submit_host(self, idx, start, tag): + variant = image.stem.removeprefix('avd-fw-') + source = programs / variant + data = (source / f'descriptor-{idx}.bin').read_bytes() + desc = list(struct.unpack('<16I', data)) + words = list(struct.unpack('<' + 'I' * desc[6], (source / f'h264-{idx}.bin').read_bytes())) + assert desc[:4] == [0x41565131, 0x10 | idx, 1, 4] + assert desc[4:6] == [0, 0] and desc[8:12] == [1, 0, 0, 1] + assert desc[12:] == [0, 0, 0, 0] + assert desc[7] == 0x2b000200 | idx << 4 | self.info[10] + # program_lo is a free-running SRAM word count, not a DMA pointer. + desc[4] = start + desc[1] = tag + assert words[0] == 0x2b000200 | idx << 4 + assert words[-1] == 0x2b000400 | idx << 4 + for line in (source / f'dma-sites-{idx}.txt').read_text().splitlines(): + offset, width, shift, expected = line.split() + offset, width, shift, expected = int(offset), int(width), int(shift), int(expected, 16) + encoded = sum(words[offset + n] << (32 * n) for n in range(width)) + assert encoded and encoded == expected >> shift, f'zero/truncated DMA operand slot {idx} word {offset}' + self.address_checks += 1 + for n, word in enumerate(words[:-1]): + if word >> 20 == 0x2d8: # coded-slice address: high 13 bits + next low word + assert ((word & 0x1fff) << 32) | words[n + 1], f'zero coded-data DMA slot {idx}' + self.address_checks += 1 + snapshot = (source / f'mmio-{idx}.bin').read_bytes() + # Replay only this host submission's writes, preserving live slots. + offsets = [self.lo, self.mask, self.cach, self.xfer] + if self.hi is not None: + offsets.append(self.hi) + for off in offsets: + at = off + idx * 4 + self.put(self.decode + at, struct.unpack_from(' retired: + idx = retired % slots + assert t.result(idx) == [2, 0, tags[retired]] + assert t.notifications[retired] == 0x54530000 | tags[retired] + retired += 1 + t.busy = False + assert t.vp_events == t.pp_events == depth + assert t.feed_idx == depth and not t.feed_word and t.live is t.pp_owner is None + assert t.u32(0x10007ff8) == t.u32(0x10007ffc) == wp + assert not t.fetches and not t.ring_writes + if depth in (2, 15): + print('\n'.join('TRACE ' + event for event in t.events), flush=True) + print(f'PASS {image.stem}: workload {depth}, peak {peak}, {t.address_checks} nonzero DMA checks, ' + f'{depth} routed VP/PP completions, RP=WP={wp}', flush=True) + + +if '--before' in sys.argv[4:]: + try: + check(2) + except LiveVPError as error: + print(f'REPRODUCED {image.stem}: {error}', flush=True) + else: + raise AssertionError('baseline unexpectedly respected VP ownership') +else: + for depth in range(2, 16): + check(depth) diff --git a/drivers/media/platform/apple/avd/tests/unicorn-queue-test.py b/drivers/media/platform/apple/avd/tests/unicorn-queue-test.py index 7ed280bde00da..d1b08ce00f55e 100644 --- a/drivers/media/platform/apple/avd/tests/unicorn-queue-test.py +++ b/drivers/media/platform/apple/avd/tests/unicorn-queue-test.py @@ -19,6 +19,8 @@ firmware = Path(sys.argv[1]).resolve() image = Path(sys.argv[2]).resolve() programs = Path(sys.argv[3]).resolve() +if (programs / 'v3-t1').is_dir(): + programs /= 'v3-t1' broken = '--broken' in sys.argv[4:] spec = importlib.util.spec_from_file_location('transport', firmware / 'tests/transport-test.py') module = importlib.util.module_from_spec(spec) From 6da646dd86cdb4cb2a00858ce546da02a9657c0a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:28:17 +1100 Subject: [PATCH 43/73] pmdomain: apple: pmc: provide rail floors as power domains Some devices only run at full speed while a shared rail is above its lowest state. The M4 Pro video encoder runs on the SoC fabric clock: with the SoC rail at its idle state it encodes 1080p H.264 at 100 fps, with the rail at its highest state at 219 fps, close to what macOS gets (it raises the same rail for every encode session). Let the DT describe such a floor as a child of the PMC with a rail and a performance state, and provide it as a power domain. While the domain is powered, the AP's own voter agent holds the rail at least at that state. Floors on one rail combine to the highest one held, so several devices can share a rail. The DCS rail is left to the DCS limit, which votes through the same agent. Floors are dropped when the PMC goes away. Add KUnit cases for combining floors next to the AP's DCS vote and for a PMC that does not take the new floor. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/power/apple,pmc.yaml | 38 ++++ drivers/pmdomain/apple/pmc-test.c | 67 +++++++ drivers/pmdomain/apple/pmc.c | 183 ++++++++++++++++-- 3 files changed, 268 insertions(+), 20 deletions(-) diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index 36ac4fe142ddb..383b42d560e98 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -112,6 +112,37 @@ patternProperties: - reg - '#power-domain-cells' + '^floor-[a-z0-9-]+$': + type: object + description: | + A rail floor, provided as a power domain. While the domain is powered, + the AP's voter agent holds the rail at least at the given performance + state. Floors on the same rail combine to the highest powered one. The + AP agent must be a static voter, and the DCS rail cannot have a floor. + additionalProperties: false + + properties: + label: + description: Name of the power domain. + + '#power-domain-cells': + const: 0 + + apple,rail: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Rail index. + + apple,level: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 15 + description: Performance state held while the domain is powered. + + required: + - '#power-domain-cells' + - apple,rail + - apple,level + '^performance-controller@[0-9a-f]+$': type: object description: The host performance controller's voter agent. @@ -170,5 +201,12 @@ examples: label = "gfx"; #power-domain-cells = <0>; }; + + floor-soc-max { + label = "soc-max"; + #power-domain-cells = <0>; + apple,rail = <1>; + apple,level = <2>; + }; }; }; diff --git a/drivers/pmdomain/apple/pmc-test.c b/drivers/pmdomain/apple/pmc-test.c index 00954a9189e84..9428e3c4317d2 100644 --- a/drivers/pmdomain/apple/pmc-test.c +++ b/drivers/pmdomain/apple/pmc-test.c @@ -185,6 +185,71 @@ static void apple_pmc_test_pending_floor(struct kunit *test) KUNIT_EXPECT_EQ(test, apple_pmc_floor_update(&dev, 4, 0xff, 0x35), -EINVAL); } +static struct apple_pmc_domain *apple_pmc_test_floor(struct kunit *test, + struct apple_pmc *pmc, + unsigned int rail, + unsigned int level) +{ + struct apple_pmc_domain *dom = kunit_kzalloc(test, sizeof(*dom), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, dom); + dom->pmc = pmc; + dom->floor = true; + dom->rail = rail; + dom->level = level; + list_add_tail(&dom->list, &pmc->domains); + return dom; +} + +static u32 apple_pmc_test_ap_floor(struct apple_pmc *pmc) +{ + return readl(pmc->voters + pmc->hw->ap_agent * pmc->hw->agent_stride); +} + +/* + * Floors on one rail combine to the highest one held, leave other rails and + * the AP's DCS vote alone, and are gone once the last one is released. + */ +static void apple_pmc_test_floor_aggregate(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + struct apple_pmc_domain *mid, *max, *other; + u32 soc = APPLE_PMC_RAIL_MASK(1); + + INIT_LIST_HEAD(&pmc->domains); + pmc->hw = &apple_pmc_hw_t6040; + apple_pmc_test_set_dcs_vote(pmc, pmc->hw->ap_agent, 3); + mid = apple_pmc_test_floor(test, pmc, 1, 1); + max = apple_pmc_test_floor(test, pmc, 1, 2); + other = apple_pmc_test_floor(test, pmc, 3, 1); + + KUNIT_ASSERT_EQ(test, apple_pmc_floor_set(mid, true), 0); + KUNIT_EXPECT_EQ(test, apple_pmc_test_ap_floor(pmc) & soc, 1U << 4); + KUNIT_ASSERT_EQ(test, apple_pmc_floor_set(max, true), 0); + KUNIT_EXPECT_EQ(test, apple_pmc_test_ap_floor(pmc) & soc, 2U << 4); + KUNIT_ASSERT_EQ(test, apple_pmc_floor_set(max, false), 0); + KUNIT_EXPECT_EQ(test, apple_pmc_test_ap_floor(pmc) & soc, 1U << 4); + KUNIT_ASSERT_EQ(test, apple_pmc_floor_set(mid, false), 0); + KUNIT_EXPECT_EQ(test, apple_pmc_test_ap_floor(pmc), 3U); + KUNIT_EXPECT_FALSE(test, other->on); +} + +/* A floor that the PMC does not take back is not left marked as held. */ +static void apple_pmc_test_floor_pending_timeout(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + struct apple_pmc_domain *dom; + void __iomem *reg; + + INIT_LIST_HEAD(&pmc->domains); + pmc->hw = &apple_pmc_hw_t8132; + reg = pmc->voters + pmc->hw->ap_agent * pmc->hw->agent_stride; + writel(BIT(pmc->hw->ap_agent), reg + 4); + dom = apple_pmc_test_floor(test, pmc, 1, 2); + KUNIT_EXPECT_EQ(test, apple_pmc_floor_set(dom, true), -ETIMEDOUT); + KUNIT_EXPECT_FALSE(test, dom->on); +} + static struct kunit_case apple_pmc_test_cases[] = { KUNIT_CASE(apple_pmc_test_partial_enable), KUNIT_CASE(apple_pmc_test_release_restores_ap_vote), @@ -192,6 +257,8 @@ static struct kunit_case apple_pmc_test_cases[] = { KUNIT_CASE(apple_pmc_test_agent_enable), KUNIT_CASE(apple_pmc_test_agent_rails), KUNIT_CASE(apple_pmc_test_dcs_limit_restores_ap_vote), + KUNIT_CASE(apple_pmc_test_floor_aggregate), + KUNIT_CASE(apple_pmc_test_floor_pending_timeout), {} }; diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 6004e23211bc1..85b2578351af1 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -17,6 +17,12 @@ * and models each dynamic agent as a power domain that sits between the * device and its PMGR power state, which gives the same ordering. * + * A rail floor is a power domain too: while any device in it is powered, + * the AP's own agent holds that rail at a given performance state. The AVE + * encoder on the M4 Pro runs on the SoC fabric clock and needs the SoC rail + * at its highest state to encode at full speed; macOS raises the same rail + * for every encode session. Floors on one rail combine to the highest. + * * Under thermal pressure the driver can also limit the DRAM controller * (DCS) rail: it stops honouring the other agents on that rail and votes * on their behalf through the AP's own agent, capped. This bounds the heat @@ -96,6 +102,8 @@ struct apple_pmc { /* Agents enabled on each rail, before any DCS limit is applied. */ u32 enable[APPLE_PMC_MAX_RAILS]; u32 static_agents; + /* Rails with a floor domain. */ + u32 floor_rails; struct list_head domains; struct dentry *debugfs; /* Cooling state of the DCS limit: 0 is none, dcs_max holds DCS at 0. */ @@ -107,12 +115,24 @@ struct apple_pmc { struct delayed_work dcs_work; }; +/** + * struct apple_pmc_domain - a power domain provided by the PMC + * @agent: dynamic voter agent the domain enables (not for a floor) + * @floor: the domain is a rail floor held through the AP's agent + * @rail: with @floor, the rail + * @level: with @floor, the performance state held while powered + * @on: with @floor, whether the floor is held; protected by pmc->lock + */ struct apple_pmc_domain { struct generic_pm_domain genpd; struct apple_pmc *pmc; struct device_node *np; struct list_head list; unsigned int agent; + bool floor; + unsigned int rail; + unsigned int level; + bool on; bool provider; }; @@ -269,6 +289,32 @@ u32 apple_pmc_floor_read(struct device *dev, unsigned int agent) } EXPORT_SYMBOL_GPL(apple_pmc_floor_read); +/* Caller holds pmc->lock. */ +static int __apple_pmc_floor_update(struct apple_pmc *pmc, unsigned int agent, + u32 mask, u32 value) +{ + void __iomem *reg = pmc->voters + agent * pmc->hw->agent_stride; + u32 old, val; + int ret = 0; + + if (pmc->hw->has_pending) { + ret = readl_poll_timeout_atomic(reg + 4, old, !(old & BIT(agent)), 1, 1000); + if (ret) + return ret; + } + old = readl(reg); + val = (old & ~mask) | (value & mask); + if (val != old) { + writel(val, reg); + if ((readl(reg) & mask) != (value & mask)) + ret = -EIO; + } + if (!ret && pmc->hw->has_pending) + ret = readl_poll_timeout_atomic(reg + 4, old, !(old & BIT(agent)), 1, 1000); + + return ret; +} + /** * apple_pmc_floor_update() - update the perf-state floor of an agent * @dev: the PMC device @@ -287,31 +333,14 @@ int apple_pmc_floor_update(struct device *dev, unsigned int agent, u32 mask, u32 value) { struct apple_pmc *pmc = apple_pmc_from_dev(dev); - void __iomem *reg; unsigned long flags; - u32 old, val; - int ret = 0; + int ret; if (!pmc || agent >= pmc->hw->agents) return -EINVAL; - reg = pmc->voters + agent * pmc->hw->agent_stride; raw_spin_lock_irqsave(&pmc->lock, flags); - if (pmc->hw->has_pending) { - ret = readl_poll_timeout_atomic(reg + 4, old, !(old & BIT(agent)), 1, 1000); - if (ret) - goto unlock; - } - old = readl(reg); - val = (old & ~mask) | (value & mask); - if (val != old) { - writel(val, reg); - if ((readl(reg) & mask) != (value & mask)) - ret = -EIO; - } - if (!ret && pmc->hw->has_pending) - ret = readl_poll_timeout_atomic(reg + 4, old, !(old & BIT(agent)), 1, 1000); -unlock: + ret = __apple_pmc_floor_update(pmc, agent, mask, value); raw_spin_unlock_irqrestore(&pmc->lock, flags); return ret; @@ -569,10 +598,48 @@ static int apple_pmc_dcs_init(struct apple_pmc *pmc) return 0; } +/* + * Caller holds pmc->lock. The AP agent's floor on @rail becomes the highest + * level of the floors held there. + */ +static int apple_pmc_floor_apply(struct apple_pmc *pmc, unsigned int rail) +{ + struct apple_pmc_domain *dom; + unsigned int level = 0; + + list_for_each_entry(dom, &pmc->domains, list) + if (dom->floor && dom->rail == rail && dom->on) + level = max(level, dom->level); + + return __apple_pmc_floor_update(pmc, pmc->hw->ap_agent, + APPLE_PMC_RAIL_MASK(rail), + level << APPLE_PMC_RAIL_SHIFT(rail)); +} + +static int apple_pmc_floor_set(struct apple_pmc_domain *dom, bool on) +{ + struct apple_pmc *pmc = dom->pmc; + unsigned long flags; + int ret; + + raw_spin_lock_irqsave(&pmc->lock, flags); + dom->on = on; + ret = apple_pmc_floor_apply(pmc, dom->rail); + if (ret) { + dom->on = !on; + apple_pmc_floor_apply(pmc, dom->rail); + } + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + return ret; +} + static int apple_pmc_domain_power_on(struct generic_pm_domain *genpd) { struct apple_pmc_domain *dom = genpd_to_apple_pmc_domain(genpd); + if (dom->floor) + return apple_pmc_floor_set(dom, true); return apple_pmc_agent_enable(dom->pmc, dom->agent, true); } @@ -580,6 +647,8 @@ static int apple_pmc_domain_power_off(struct generic_pm_domain *genpd) { struct apple_pmc_domain *dom = genpd_to_apple_pmc_domain(genpd); + if (dom->floor) + return apple_pmc_floor_set(dom, false); return apple_pmc_agent_enable(dom->pmc, dom->agent, false); } @@ -671,10 +740,72 @@ static int apple_pmc_domain_add(struct apple_pmc *pmc, struct device_node *np) return 0; } +/* A rail floor held through the AP's agent while the domain is powered. */ +static int apple_pmc_floor_add(struct apple_pmc *pmc, struct device_node *np) +{ + const struct apple_pmc_hw *hw = pmc->hw; + struct device *dev = pmc->dev; + struct apple_pmc_domain *dom; + const char *label; + u32 rail, level; + int ret; + + if (of_property_read_u32(np, "apple,rail", &rail) || + of_property_read_u32(np, "apple,level", &level)) + return dev_err_probe(dev, -EINVAL, "%pOF: missing rail or level\n", np); + /* The DCS rail's AP vote belongs to the DCS limit. */ + if (rail >= hw->rails || rail == hw->dcs_rail || + (hw->agent_rails && !(hw->agent_rails & BIT(rail)))) + return dev_err_probe(dev, -EINVAL, "%pOF: invalid rail %u\n", np, rail); + if (!level || level >= APPLE_PMC_MAX_STATES) + return dev_err_probe(dev, -EINVAL, "%pOF: invalid level %u\n", np, level); + if (!(pmc->static_agents & BIT(hw->ap_agent))) + return dev_err_probe(dev, -EINVAL, + "%pOF: the AP agent is not a static voter\n", np); + + dom = kzalloc_obj(*dom); + if (!dom) + return -ENOMEM; + + dom->pmc = pmc; + dom->np = of_node_get(np); + dom->floor = true; + dom->rail = rail; + dom->level = level; + if (of_property_read_string(np, "label", &label)) + label = np->name; + dom->genpd.name = kasprintf(GFP_KERNEL, "pmc-%s", label); + if (!dom->genpd.name) { + of_node_put(dom->np); + kfree(dom); + return -ENOMEM; + } + dom->genpd.power_on = apple_pmc_domain_power_on; + dom->genpd.power_off = apple_pmc_domain_power_off; + + ret = pm_genpd_init(&dom->genpd, NULL, true); + if (ret) { + of_node_put(dom->np); + kfree(dom->genpd.name); + kfree(dom); + return dev_err_probe(dev, ret, "%pOF: genpd init failed\n", np); + } + list_add_tail(&dom->list, &pmc->domains); + + ret = of_genpd_add_provider_simple(np, &dom->genpd); + if (ret) + return dev_err_probe(dev, ret, "%pOF: provider failed\n", np); + dom->provider = true; + pmc->floor_rails |= BIT(rail); + + return 0; +} + static int apple_pmc_status_show(struct seq_file *s, void *unused) { struct apple_pmc *pmc = s->private; const struct apple_pmc_hw *hw = pmc->hw; + struct apple_pmc_domain *dom; unsigned int i; seq_printf(s, "static agents: %#x\n", pmc->static_agents); @@ -687,6 +818,10 @@ static int apple_pmc_status_show(struct seq_file *s, void *unused) i, apple_pmc_enable_read(pmc, i), pmc->saved_enable[i]); seq_printf(s, "dcs limit: state %lu of %u\n", READ_ONCE(pmc->dcs_state), hw->dcs_max); + list_for_each_entry(dom, &pmc->domains, list) + if (dom->floor) + seq_printf(s, "%s: rail%u level %u %s\n", dom->genpd.name, + dom->rail, dom->level, READ_ONCE(dom->on) ? "held" : "released"); return 0; } @@ -709,6 +844,11 @@ static void apple_pmc_release(void *data) /* The cooling device is gone by now; drop any limit it left. */ cancel_delayed_work_sync(&pmc->dcs_work); raw_spin_lock_irqsave(&pmc->lock, flags); + /* No floor outlives its domains. */ + for (rail = 0; rail < pmc->hw->rails; rail++) + if (pmc->floor_rails & BIT(rail)) + __apple_pmc_floor_update(pmc, pmc->hw->ap_agent, + APPLE_PMC_RAIL_MASK(rail), 0); for (rail = 0; rail < pmc->hw->rails; rail++) writel(pmc->saved_enable[rail], pmc->rails + pmc->hw->rail_enable + 4 * rail); @@ -811,7 +951,10 @@ static int apple_pmc_probe(struct platform_device *pdev) for_each_available_child_of_node(np, child) { if (!of_property_present(child, "#power-domain-cells")) continue; - ret = apple_pmc_domain_add(pmc, child); + if (of_property_present(child, "apple,rail")) + ret = apple_pmc_floor_add(pmc, child); + else + ret = apple_pmc_domain_add(pmc, child); if (ret) { of_node_put(child); return ret; From 90534c57928cd4d3ce9043b883c4f8225a9e57e9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:28:44 +1100 Subject: [PATCH 44/73] media: apple: ave: hold the performance floor domain while powered A chip may need a performance floor for the encoder to run at full speed, such as the SoC rail floor of the PMC on the M4 Pro. Accept an optional sixth power domain named "perf" and attach it with the others, so it is held exactly while the encoder is powered: from the first session until the autosuspend after the last one, and dropped across a firmware recovery power cycle like the codec domains. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/media/apple,t8140-ave.yaml | 9 +++++- drivers/media/platform/apple/ave/ave-drv.c | 13 +++++++-- tools/asahi/ave-probe-pm-tests.py | 28 +++++++++++++------ 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml b/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml index cfc518d4a0237..e95bcfbcd9a74 100644 --- a/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml +++ b/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml @@ -38,15 +38,22 @@ properties: maxItems: 5 power-domains: - maxItems: 5 + minItems: 5 + maxItems: 6 power-domain-names: + minItems: 5 items: - const: dma - const: pipe4 - const: pipe5 - const: me0 - const: me1 + - const: perf + description: | + A performance floor the encoder needs to run at full speed, held + while the encoder is powered (for example the SoC rail floor of a + PMC). iommus: maxItems: 2 diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c index d96267c00bef3..c145e29538c78 100644 --- a/drivers/media/platform/apple/ave/ave-drv.c +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -206,15 +206,23 @@ static void ave_debugfs_init(struct ave_dev *ave) } } +/* + * The encoder pipes, the DMA block and the motion engines, then optionally + * "perf": a performance floor the encoder needs to run at full speed, such + * as the SoC rail floor on the M4 Pro. It is held while the encoder is + * powered, like the other domains, so it ends with the last session. + */ static const char *const ave_pd_names[] = { "dma", "pipe4", "pipe5", "me0", - "me1" }; + "me1", "perf" }; static int ave_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + bool perf = of_property_match_string(dev->of_node, "power-domain-names", + "perf") >= 0; const struct dev_pm_domain_attach_data pd_data = { .pd_names = ave_pd_names, - .num_pd_names = ARRAY_SIZE(ave_pd_names), + .num_pd_names = ARRAY_SIZE(ave_pd_names) - !perf, .pd_flags = PD_FLAG_DEV_LINK_ON, }; struct ave_dev *ave; @@ -247,7 +255,6 @@ static int ave_probe(struct platform_device *pdev) if (ret) return ret; - /* The encoder pipes, the DMA block and the motion engines. */ ret = devm_pm_domain_attach_list(dev, &pd_data, &ave->pd_list); if (ret < 0) return dev_err_probe(dev, ret, "cannot attach power domains\n"); diff --git a/tools/asahi/ave-probe-pm-tests.py b/tools/asahi/ave-probe-pm-tests.py index 417eecdb575df..055c8d03b8ea8 100644 --- a/tools/asahi/ave-probe-pm-tests.py +++ b/tools/asahi/ave-probe-pm-tests.py @@ -25,7 +25,8 @@ #include #include struct ave_dev; -struct device { bool enabled, suspended; unsigned supplier_refs; struct ave_dev *data; }; +struct device { bool enabled, suspended; unsigned supplier_refs; struct ave_dev *data; + const void *of_node; }; struct platform_device { struct device dev; }; struct iommu_domain { unsigned type; }; struct ave_dev { struct device *dev; const void *variant; int hw_lock; @@ -41,7 +42,11 @@ #define DMA_BIT_MASK(x) ((UINT64_C(1) << (x)) - 1) #define IS_ERR(x) ((uintptr_t)(x) >= (uintptr_t)-4095) #define PTR_ERR(x) ((int)(intptr_t)(x)) -static const char *const ave_pd_names[] = {"dma", "pipe4", "pipe5", "me0", "me1"}; +static const char *const ave_pd_names[] = {"dma", "pipe4", "pipe5", "me0", "me1", "perf"}; +/* Whether the node lists the optional "perf" domain (the M4 Pro SoC floor). */ +static bool has_perf; +static int of_property_match_string(const void *np, const char *prop, const char *str) +{ (void)np; (void)prop; (void)str; return has_perf ? 5 : -61; } static struct ave_dev context; static struct iommu_domain domain = {.type = __IOMMU_DOMAIN_PAGING}; static int attach_error, state_error, enable_error, fw_error, register_error; @@ -62,13 +67,14 @@ const struct dev_pm_domain_attach_data *data, void **list) { assert(!d->enabled && d->suspended); - assert(data->num_pd_names == 5 && data->pd_flags == PD_FLAG_DEV_LINK_ON); + assert(data->num_pd_names == (has_perf ? 6U : 5U) && data->pd_flags == PD_FLAG_DEV_LINK_ON); + assert(data->num_pd_names < 6 || data->pd_names[5][0] == 'p'); assert(!*list); if (attach_error) return attach_error; - /* Five virtual devices plus the two preactivated DART supplier links. */ - d->supplier_refs = 7; + /* One virtual device per domain plus the two preactivated DART supplier links. */ + d->supplier_refs = data->num_pd_names + 2; *list = d; - return 5; + return data->num_pd_names; } static int pm_runtime_set_suspended(struct device *d) { @@ -93,7 +99,7 @@ } static int ave_fw_probe(struct ave_dev *a) { - assert(!a->dev->enabled && a->dev->supplier_refs == 7); + assert(!a->dev->enabled && a->dev->supplier_refs == (has_perf ? 8U : 7U)); fw_probes++; return fw_error; } @@ -152,7 +158,13 @@ register_error = -EIO; assert(ave_probe(&p) == -EIO); assert(!p.dev.supplier_refs && fw_removes == 1); - puts("AVE probe PM: idle suppliers, immediate use, 10 cycles and 5 error paths passed"); + /* A node with the optional performance floor attaches it with the rest, and it is idle too. */ + reset(&p); + has_perf = true; + assert(ave_probe(&p) == 0); + assert(!p.dev.supplier_refs); + has_perf = false; + puts("AVE probe PM: idle suppliers, immediate use, 10 cycles, 5 error paths and the perf domain passed"); } ''' From ffc0f46c004adf16aa6f9cbc605912e4c1eb5a84 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:28:44 +1100 Subject: [PATCH 45/73] arm64: dts: apple: t6040: raise the SoC rail while the encoder runs The M4 Pro video encoder runs on the SoC fabric clock and encodes at less than half speed while the SoC rail is at its idle state. Describe a PMC floor that holds the SoC rail at its highest state, and give it to the encoder as its performance domain. Measured on the J616s with the same floor applied by hand (avebench, 600 and 240 frames): 1080p H.264 100 to 219 fps, HEVC 91 to 198, 4K H.264 27 to 59, 4K HEVC 26 to 57. macOS on the same machine: 236, 224, 62 and 59. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6040.dtsi | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index a33ef2e0abf83..c3b18651a4d9b 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -1171,6 +1171,17 @@ reg = <1>; }; + + /* + * The SoC rail at its highest state. The video encoder runs on + * the fabric clock and encodes at less than half speed below it. + */ + pmc_soc_max: floor-soc-max { + label = "soc-max"; + #power-domain-cells = <0>; + apple,rail = <1>; + apple,level = <2>; + }; }; /* @@ -2277,8 +2288,9 @@ , ; power-domains = <&ps_venc0_dma>, <&ps_venc0_pipe4>, - <&ps_venc0_pipe5>, <&ps_venc0_me0>, <&ps_venc0_me1>; - power-domain-names = "dma", "pipe4", "pipe5", "me0", "me1"; + <&ps_venc0_pipe5>, <&ps_venc0_me0>, <&ps_venc0_me1>, + <&pmc_soc_max>; + power-domain-names = "dma", "pipe4", "pipe5", "me0", "me1", "perf"; iommus = <&ave0_dart0 0>, <&ave0_dart1 0>; status = "disabled"; }; From 5c6bfa057dc09bba4b6bb65423674405259e8673 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:48:45 +1100 Subject: [PATCH 46/73] media: apple: avd: hold the PMC performance floor while decoding Linux decodes at about half the speed it could on the M4 Pro because nothing raises the SoC rail while the decoder works; macOS votes a VMAX floor for the decoder whenever frames are queued. Holding the SoC rail at its performance level from the AP's PMC agent took H.264 1080p from 326 to 695 fps, AV1 4K from 198 to 420 and VP9 4K from 82 to 183 on a J616s, every frame bit-exact against software decode. A node can now list that floor as a second power domain, "perf", next to the decoder's own. With more than one domain the core attaches none, so attach them all with runtime PM device links: the floor is held exactly while the decoder is runtime-active (frames in flight plus the 100 ms autosuspend delay) and released when it idles, rests or is retired. The floor domain is shared and refcounted by genpd, so the encoder's vote and this one combine. Nodes without it are unchanged. The domain list is attached with runtime PM device links (the core attaches no domain itself once there are several), and the links' initial active references are settled before runtime PM is enabled, as the encoder does, so the floor is not held from probe until the first decode. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-drv.c | 32 ++++++++++++++++++++++ drivers/media/platform/apple/avd/avd.h | 2 ++ 2 files changed, 34 insertions(+) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 215ac921ce671..4efd1bd5e2558 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -11,6 +11,8 @@ * Copyright (C) 2011 Samsung Electronics Co., Ltd. */ +#include +#include #include #include #include @@ -1074,6 +1076,26 @@ static int avd_probe(struct platform_device *pdev) avd->rstc = devm_reset_control_get_exclusive(avd->dev, NULL); + /* + * A node may list a second power domain, "perf": the PMC floor that + * holds the SoC rail at its performance level while the decoder works + * (decode runs at about half speed on the M4 Pro without it). With more + * than one domain the core attaches none, so attach them all with + * runtime PM device links: the floor is then held exactly while the + * decoder is runtime-active (frames in flight plus the autosuspend + * delay) and released when it idles, rests or is retired. + */ + if (of_count_phandle_with_args(pdev->dev.of_node, "power-domains", + "#power-domain-cells") > 1) { + static const struct dev_pm_domain_attach_data pd_data = { + .pd_flags = PD_FLAG_DEV_LINK_ON, + }; + + ret = devm_pm_domain_attach_list(avd->dev, &pd_data, &avd->pd_list); + if (ret < 0) + return dev_err_probe(avd->dev, ret, "failed to attach power domains\n"); + } + avd->code = devm_platform_ioremap_resource_byname(pdev, "code"); if (IS_ERR(avd->code)) return PTR_ERR(avd->code); @@ -1144,6 +1166,16 @@ static int avd_probe(struct platform_device *pdev) return ret; } + /* + * The domain links were added active, but the decoder starts + * suspended: settle that before enabling runtime PM, so the floor is + * not held until the first decode has come and gone. + */ + if (avd->pd_list) { + ret = pm_runtime_set_suspended(avd->dev); + if (ret) + return ret; + } pm_runtime_set_autosuspend_delay(avd->dev, 100); pm_runtime_use_autosuspend(avd->dev); pm_runtime_enable(avd->dev); diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 1f2fa8e312d92..153a462320c4a 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -220,6 +220,8 @@ struct avd_dev { struct iommu_domain *domain; struct iommu_domain *empty_domain; + /* Core and perf-floor domains when the node lists more than one. */ + struct dev_pm_domain_list *pd_list; struct mutex vdev_lock; From 78d3ff491c849dd231440610ff10e4976e773916 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:48:50 +1100 Subject: [PATCH 47/73] arm64: dts: apple: t6040: give the AVD the PMC SoC performance floor List pmc_soc_max as the decoder's second power domain, "perf", next to its own, so the floor is held while it decodes and the encoder's and decoder's votes combine. The binding gains the optional named domain. With the floor held, a J616s decoded H.264 1080p at 695 fps instead of 326, AV1 4K at 420 instead of 198 and VP9 4K at 183 instead of 82, every frame bit-exact against software decode. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/media/apple,avd.yaml | 11 ++++++++++- arch/arm64/boot/dts/apple/t6040.dtsi | 3 ++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/media/apple,avd.yaml b/Documentation/devicetree/bindings/media/apple,avd.yaml index 59ccb50461181..450af7ffa4559 100644 --- a/Documentation/devicetree/bindings/media/apple,avd.yaml +++ b/Documentation/devicetree/bindings/media/apple,avd.yaml @@ -51,7 +51,16 @@ properties: maxItems: 3 power-domains: - maxItems: 1 + minItems: 1 + items: + - description: decoder power state + - description: performance floor (PMC rail level) held while decoding + + power-domain-names: + minItems: 1 + items: + - const: core + - const: perf resets: maxItems: 1 diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index c3b18651a4d9b..50ad177d4f17d 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -648,7 +648,8 @@ interrupt-parent = <&aic>; interrupts = , ; - power-domains = <&ps_avd_sys>; + power-domains = <&ps_avd_sys>, <&pmc_soc_max>; + power-domain-names = "core", "perf"; resets = <&ps_avd_sys>; iommus = <&avd_dart 0>; }; From be531920c5a28cad06bd4b36312e036de0188677 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 16:02:39 +1100 Subject: [PATCH 48/73] accel/ane: refuse H14 programs with malformed first/last task flags Header word 2 of each H14 TD block marks the first task of a program (0x08) and its last (0x22), 0 between, both (0x2a) on a single task. All 1503 decoded Apple H14 programs follow that. A compiler bug emitted 0x2a on every task of a multi-task layer. On the M2 such a program's call returns, and then the firmware never completes another command until reboot, with this driver and with the stock one. That is the hang the earlier commits read as a TERMINATE_PROCESS wedge after multi-task programs: both repro programs had the malformed flags. Check the flags in the same snapshot that keys the cache and refuse a malformed program with -EINVAL before the firmware sees it, so one client cannot take the ANE away from every other until reboot. A stream that does not parse is left to the firmware's own descriptor check. KUnit td_flags_test covers the single, two- and three-task forms and the broken ones. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 7 ++++ drivers/accel/ane/ane_t6021_lifetime.h | 41 ++++++++++++++++++++ drivers/accel/ane/ane_t6021_lifetime_test.c | 42 +++++++++++++++++++++ drivers/accel/ane/ane_t6021_rtclient_main.c | 16 +++++++- 4 files changed, 105 insertions(+), 1 deletion(-) diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index 5ec14ed5fcab7..571e62ce93880 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -362,6 +362,13 @@ the firmware right now (resident) is managed by the driver: ``PROC_CREATE`` or ``EXEC`` loads them again and creates a new process, with the same IDs. Unreferenced cache entries are dropped. +A program whose TD stream opens or closes on the wrong task is refused with +``-EINVAL`` before the firmware sees it. Header word 2 of each TD block must +be 0x08 on the first task, 0 between and 0x22 on the last (0x2a for a single +task), as in every decoded Apple H14 program. A program with 0x2a on every +task of several returned from its call on the M2 and then left the firmware +unable to complete any further command until reboot. + A failed load or unload keeps the program and its buffers until a verified firmware restart. diff --git a/drivers/accel/ane/ane_t6021_lifetime.h b/drivers/accel/ane/ane_t6021_lifetime.h index 1a769d3064f55..f01179ada3895 100644 --- a/drivers/accel/ane/ane_t6021_lifetime.h +++ b/drivers/accel/ane/ane_t6021_lifetime.h @@ -189,6 +189,47 @@ static inline long ane_t6021_td_blocks(const u8 *p, size_t size) return n; } +/* + * Header word 2 of each TD block: 0x08 opens the program on its first task + * and 0x22 closes it on its last, 0 between, 0x2a for a single task. Every + * decoded Apple H14 program follows this. A program with 0x2a on every task + * of several returned from its call and then left the firmware unable to + * complete any further command until reboot (M2, 13.5 bia firmware), so such + * a program is refused before the firmware sees it. Returns -EINVAL for a + * stream that parses and breaks the rule, 0 otherwise (a stream that does + * not parse is left to the firmware's own descriptor check). + */ +static inline int ane_t6021_td_flags_check(const u8 *p, size_t size) +{ + size_t cur; + u32 w2, last = 0; + long n = 0; + + if (size < 4) + return 0; + cur = (ane_t6021_le32(p) & 4) ? 0x30 : 0x10; + while (cur < size) { + u32 words; + + if (size - cur < 12) + return 0; + words = (p[cur + 2] | p[cur + 3] << 8) & 0x7ff; + if (words < 3) + return 0; + w2 = ane_t6021_le32(p + cur + 8); + if (n == 0 && w2 != 0x08 && w2 != 0x2a) + return -EINVAL; + if (n > 0 && last != (n == 1 ? 0x08 : 0)) + return -EINVAL; + last = w2; + cur += ((words << 2) + 0xf) & 0x3ff0; + n++; + } + if (!n) + return 0; + return last == (n == 1 ? 0x2a : 0x22) ? 0 : -EINVAL; +} + /* ---- program table ---- */ /* The firmware table holds 256 programs; the 257th LOAD is refused. */ diff --git a/drivers/accel/ane/ane_t6021_lifetime_test.c b/drivers/accel/ane/ane_t6021_lifetime_test.c index 14f976b40c531..921958b400b31 100644 --- a/drivers/accel/ane/ane_t6021_lifetime_test.c +++ b/drivers/accel/ane/ane_t6021_lifetime_test.c @@ -111,6 +111,47 @@ static void task_count_test(struct kunit *test) KUNIT_EXPECT_LT(test, ane_t6021_td_blocks(td, 2), 0L); } +static void put_task(u8 *td, size_t at, u32 words, u32 w2) +{ + td[at + 2] = words; + td[at + 3] = words >> 8; + put32(td + at + 8, w2); +} + +static void td_flags_test(struct kunit *test) +{ + u8 td[0x200] = {}; + + /* One task: opens and closes the program. */ + put_task(td, 0x10, 0x20, 0x2a); + KUNIT_EXPECT_EQ(test, ane_t6021_td_flags_check(td, 0x90), 0); + /* Two and three tasks: 0x08 first, 0 between, 0x22 last. */ + put_task(td, 0x10, 0x20, 0x08); + put_task(td, 0x90, 0x20, 0x22); + KUNIT_EXPECT_EQ(test, ane_t6021_td_flags_check(td, 0x110), 0); + put_task(td, 0x90, 0x20, 0); + put_task(td, 0x110, 0x20, 0x22); + KUNIT_EXPECT_EQ(test, ane_t6021_td_flags_check(td, 0x190), 0); + /* The wedging form: every task opens and closes. */ + put_task(td, 0x10, 0x20, 0x2a); + put_task(td, 0x90, 0x20, 0x2a); + KUNIT_EXPECT_EQ(test, ane_t6021_td_flags_check(td, 0x110), -EINVAL); + /* Not closed, closed early, opened twice. */ + put_task(td, 0x10, 0x20, 0x08); + put_task(td, 0x90, 0x20, 0); + KUNIT_EXPECT_EQ(test, ane_t6021_td_flags_check(td, 0x110), -EINVAL); + put_task(td, 0x90, 0x20, 0x22); + put_task(td, 0x110, 0x20, 0x22); + KUNIT_EXPECT_EQ(test, ane_t6021_td_flags_check(td, 0x190), -EINVAL); + put_task(td, 0x90, 0x20, 0x08); + KUNIT_EXPECT_EQ(test, ane_t6021_td_flags_check(td, 0x190), -EINVAL); + /* A lone task without both bits. */ + KUNIT_EXPECT_EQ(test, ane_t6021_td_flags_check(td, 0x90), -EINVAL); + /* A stream that does not parse is left to the firmware. */ + KUNIT_EXPECT_EQ(test, ane_t6021_td_flags_check(td, 0x14), 0); + KUNIT_EXPECT_EQ(test, ane_t6021_td_flags_check(td, 2), 0); +} + /* * A fake firmware. Records every command class, fails on demand, and * refuses (test failure) to terminate a process of a program not proven @@ -770,6 +811,7 @@ static struct kunit_case lifetime_cases[] = { KUNIT_CASE(mring_classify_test), KUNIT_CASE(legtab_reuse_test), KUNIT_CASE(task_count_test), + KUNIT_CASE(td_flags_test), KUNIT_CASE(progtab_shared_unload_test), KUNIT_CASE(multitask_never_terminated_test), KUNIT_CASE(progtab_failed_unload_holds_test), diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 95f3bcbff6d7c..a01e4ff634b74 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -1679,6 +1679,7 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, struct ane_t6021_prog *cached, *prog = NULL; bool sent = false; long tdprop_tasks = -EINVAL, td_blocks = -EINVAL; + int td_flags = 0; u32 key, gen; u8 digest[SHA256_DIGEST_SIZE]; size_t binds_size; @@ -1780,10 +1781,14 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, /* The same snapshot as the key decides whether the * program's process may ever be terminated. */ - if (sections[i].id == ANE_T6021_SEC_TEXT) + if (sections[i].id == ANE_T6021_SEC_TEXT) { td_blocks = ane_t6021_td_blocks((const u8 *)bo->cpu + sections[i].offset, sections[i].size); + td_flags = ane_t6021_td_flags_check((const u8 *)bo->cpu + + sections[i].offset, + sections[i].size); + } else if (sections[i].id == ANE_T6021_SEC_TDPROP) tdprop_tasks = ane_t6021_tdprop_tasks((const u8 *)bo->cpu + sections[i].offset, @@ -1804,6 +1809,15 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, kvfree(scratch); sha256_final(&sha, digest); } + /* A program that opens or closes on the wrong task wedges the firmware + * for every client; refuse it before the firmware sees it. + */ + if (td_flags) { + dev_info_ratelimited(ane->dev, + "LOAD refused: TD first/last task flags malformed\n"); + ret = td_flags; + goto out; + } cached = ane_t6021_prog_find(digest); if (cached) { /* A resident cache entry or a non-resident program is reused as From 6fcb1fbf5d3b6255373cd68b95610c4c31e4e436 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 16:34:03 +1100 Subject: [PATCH 49/73] accel/ane: unload every H14 program at last close The rule that a multi-task process is never terminated came from two M2 repros that both used programs with malformed first/last task flags, which the previous commit refuses at LOAD. On the M2 with well-formed programs, TERMINATE_PROCESS and DESTROY_PROGRAM complete for processes of 2, 4, 8, 10 and 45 tasks (a MobileNetV1), and the firmware allocations go back to the idle baseline every time. Drop the rule and the task count it needed. The last close now unloads every program. Room for a new LOAD comes from unloading the least recently used program a file still holds, which loads again with a new process on its next use. A firmware restart is left only for the case where every resident program is HELD, that is a load or unload whose outcome is unknown. The malformed program refusal, the resident limit, the restart state machine and the POWER_DOWN stop stay as they are. KUnit drops the task count cases and covers unload at last close for any program, LRU eviction of held-open programs, and a restart only when all residents are HELD. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 38 ++- drivers/accel/ane/ane_t6021_lifetime.h | 126 +++------- drivers/accel/ane/ane_t6021_lifetime_test.c | 249 +++++++------------- drivers/accel/ane/ane_t6021_rtclient_main.c | 44 +--- 4 files changed, 139 insertions(+), 318 deletions(-) diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index 571e62ce93880..cc1c69a5d0192 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -338,29 +338,19 @@ refuse a program the file holds no reference on with ``-ENOENT``. A program's kernel IDs stay valid for as long as a file holds it; whether it is loaded in the firmware right now (resident) is managed by the driver: +- When the last file holding a program closes, the driver sends + TERMINATE_PROCESS (0x203) for its process and then DESTROY_PROGRAM (0x201), + and only after both are acknowledged gives the section buffers back. - The firmware's L2 spill pool holds eight 2 MiB buffers; every - CREATE_PROCESS grows it while it holds fewer than eight, and a LOAD sent - while eight programs with processes were resident never completed. At most - eight programs are resident, and room is made before each LOAD. The - firmware's program and process tables (240 and 496 advertised) are not the - limit. -- TERMINATE_PROCESS (0x203) of a process whose program has more than one task - waits inside the firmware without a timeout and never completes, and no - other command frees such a process. The driver never sends it for one. A - program is single-task only when its TD property section (load id 7) and - its TD stream (load id 3) both count one task; anything else, including a - section it cannot parse, counts as multi-task. -- When the last file holding a program closes, a single-task program (or any - program without a process) is unloaded: TERMINATE_PROCESS, then - DESTROY_PROGRAM (0x201), and only after both are acknowledged are the - section buffers given back. A multi-task program stays resident as a cache - entry, and the next load of the same bytes reuses it without a command. -- Room comes first from unloading single-task programs, unreferenced ones - before the least recently used. When every resident program is a - multi-task process, the driver restarts the firmware at that idle point - (see below). Programs that files still hold become non-resident; the next - ``PROC_CREATE`` or ``EXEC`` loads them again and creates a new process, - with the same IDs. Unreferenced cache entries are dropped. + CREATE_PROCESS grows it while it holds fewer than eight. At most eight + programs are resident. Before each LOAD the least recently used program a + file still holds is unloaded for room; its next ``PROC_CREATE`` or ``EXEC`` + loads it again and creates a new process, with the same IDs. The firmware's + program and process tables (240 and 496 advertised) are not the limit. +- Only when every resident program is held (a load or unload whose outcome is + unknown) does the driver restart the firmware at that idle point (see + below). Programs that files still hold become non-resident and load again + on use. A program whose TD stream opens or closes on the wrong task is refused with ``-EINVAL`` before the firmware sees it. Header word 2 of each TD block must @@ -383,8 +373,8 @@ allocations rather than every allocation since boot. ``fw_memory`` in the device's accel debugfs directory reports the live firmware allocations, their bytes, the table high-water mark, the known, -held, resident, cached and single-task programs, the resident limit, the -budget restarts and the firmware generation. A command timeout also logs the allocation ring and the +held, resident and cached programs, the resident limit, the restarts for +room and the firmware generation. A command timeout also logs the allocation ring and the target-to-host rings at the host cursors. H14 recovery lifecycle diff --git a/drivers/accel/ane/ane_t6021_lifetime.h b/drivers/accel/ane/ane_t6021_lifetime.h index f01179ada3895..30a5ca4a8f28d 100644 --- a/drivers/accel/ane/ane_t6021_lifetime.h +++ b/drivers/accel/ane/ane_t6021_lifetime.h @@ -25,27 +25,23 @@ * the next poll and the cursor does not move. * * A program is shared by every file that loaded byte-identical sections and - * outlives the firmware instance it was first loaded into. Each file holds one + * outlives the firmware instance it was loaded into. Each file holds one * reference per program it loaded, by an opaque kernel ID that never changes. * Whether the program is in the firmware right now (resident) is separate: * - * - Every live process holds a 2 MiB firmware allocation, and a LOAD sent - * while eight programs with processes are resident never completes. The - * driver keeps at most ANE_T6021_FW_RESIDENT_MAX programs resident and - * makes room before each LOAD. - * - TERMINATE_PROCESS (0x203) of a process whose program has more than one - * task waits inside the firmware without a timeout and never completes, - * and no other command frees such a process. Such a process is never - * terminated. Only a program proven to have exactly one task (its TD - * property section and its TD stream both count one) is unloaded: - * TERMINATE_PROCESS for its process, then DESTROY_PROGRAM (0x201). - * - An unreferenced program that cannot be unloaded stays resident as a - * cache entry; a later load of the same bytes reuses it with no command. - * - Room comes first from unloading a single-task program (unreferenced - * first, then the least recently used), and when none is left from a - * cold firmware restart at an idle point. A restart drops every cache - * entry; referenced programs become non-resident and are loaded again, - * with a new process, on their next use. Files see no change. + * - When the last file holding a program closes, it is unloaded: + * TERMINATE_PROCESS (0x203) for its process, then DESTROY_PROGRAM (0x201). + * - The firmware's L2 spill pool holds eight 2 MiB buffers and grows on every + * CREATE_PROCESS. At most ANE_T6021_FW_RESIDENT_MAX programs are resident; + * before each LOAD the least recently used one a file still holds is + * unloaded to make room, and loaded again, with a new process, on its + * next use. Files see no change. + * - Only when every resident program is HELD (a load or unload whose outcome + * is unknown) is the firmware restarted, at an idle point. A restart drops + * every program from the firmware; the ones files hold load again on use. + * - A program whose TD stream opens or closes on the wrong task is refused at + * LOAD: such a program leaves the firmware unable to complete another + * command (see ane_t6021_td_flags_check()). * * A program whose load or unload failed is HELD: its buffers stay and its * slot is not reused until a verified firmware restart. @@ -125,70 +121,15 @@ static inline int ane_t6021_legtab_find(const struct ane_legacy_buffer *t, return -1; } -/* ---- task count ---- */ +/* ---- TD stream ---- */ -/* - * The task count of a program, from the two sections that carry it. The - * firmware rejects a LOAD whose counts disagree, so both are read here and - * a program is single-task only when both say one. Anything this code - * cannot parse counts as multi-task: such a process is never terminated. - * The TD walk follows the firmware's block walk as omarchy-ane's libane - * section builder (Joshua Warren, MIT) documents it. - */ -#define ANE_T6021_SEC_TEXT 3 /* the TD stream */ -#define ANE_T6021_SEC_TDPROP 7 /* TD property segments */ -#define ANE_T6021_TDPROP_SEG 0x20 -#define ANE_T6021_TDPROP_MAX_SEGS 64 +#define ANE_T6021_SEC_TEXT 3 /* LOAD section id of the TD stream */ static inline u32 ane_t6021_le32(const u8 *p) { return p[0] | p[1] << 8 | p[2] << 16 | (u32)p[3] << 24; } -/* Sum of blockNbr over the TD property segments: u32 count at +0, then - * count 0x20-byte segments from +8, blockNbr at segment +4. - */ -static inline long ane_t6021_tdprop_tasks(const u8 *p, size_t size) -{ - u32 segs, i; - u64 sum = 0; - - if (size < 8) - return -EINVAL; - segs = ane_t6021_le32(p); - if (!segs || segs > ANE_T6021_TDPROP_MAX_SEGS || - size < 8 + (size_t)segs * ANE_T6021_TDPROP_SEG) - return -EINVAL; - for (i = 0; i < segs; i++) - sum += ane_t6021_le32(p + 8 + i * ANE_T6021_TDPROP_SEG + 4); - return sum > INT_MAX ? -EINVAL : (long)sum; -} - -/* TD blocks in the stream: a 0x10-byte header (0x30 when word 0 bit 2 is - * set), then blocks whose u16 at +2 holds the word count in bits 10:0. - */ -static inline long ane_t6021_td_blocks(const u8 *p, size_t size) -{ - size_t cur; - long n = 0; - - if (size < 4) - return -EINVAL; - cur = (ane_t6021_le32(p) & 4) ? 0x30 : 0x10; - while (cur < size) { - u32 words; - - if (size - cur < 4) - return -EINVAL; - words = (p[cur + 2] | p[cur + 3] << 8) & 0x7ff; - if (!words) - return -EINVAL; - cur += ((words << 2) + 0xf) & 0x3ff0; - n++; - } - return n; -} - /* * Header word 2 of each TD block: 0x08 opens the program on its first task * and 0x22 closes it on its last, 0 between, 0x2a for a single task. Every @@ -267,7 +208,6 @@ struct ane_t6021_prog { u32 users; /* files holding a reference */ enum ane_t6021_prog_state state; bool resident; /* LOADed into the current firmware instance */ - bool single_task; /* proven: its process may be terminated */ u64 last_use; /* The program's own reference on each section buffer, or NULL, and * the IOVA and size its LOAD names for that section. @@ -348,13 +288,10 @@ ane_t6021_progtab_resident(const struct ane_t6021_progtab *t) return n; } -/* The firmware can give a resident program back without a restart: it has - * no process, or its process runs a program proven to have one task. - */ +/* The firmware can give a resident program back without a restart. */ static inline bool ane_t6021_prog_unloadable(const struct ane_t6021_prog *p) { - return p->state == ANE_T6021_PROG_LIVE && p->resident && - (p->fw_proc_id == ANE_T6021_NO_ID || p->single_task); + return p->state == ANE_T6021_PROG_LIVE && p->resident; } /* @@ -454,8 +391,7 @@ static inline int ane_t6021_fdprogs_ref(struct ane_t6021_progtab *t, /* * The firmware commands of an unload: the process first, because destroying * a program that still has a process halts the firmware. @cmd returns zero - * only for an acknowledged command. Nothing after a failure is sent, and a - * process of a program not proven single-task is never terminated. + * only for an acknowledged command. Nothing after a failure is sent. */ static inline int ane_t6021_prog_unload_cmds(const struct ane_t6021_prog *p, @@ -466,8 +402,6 @@ ane_t6021_prog_unload_cmds(const struct ane_t6021_prog *p, int ret; if (p->fw_proc_id != ANE_T6021_NO_ID) { - if (!p->single_task) - return -EBUSY; ret = cmd(ctx, ANE_T6021_CMD_TERMINATE_PROCESS, p->fw_prog_id, p->fw_proc_id); if (ret) @@ -519,9 +453,8 @@ static inline int ane_t6021_progtab_evict(struct ane_t6021_progtab *t, /* * The file is closing: drop its references. A program whose last reference - * this was is unloaded when the firmware can give it back, freed when it is - * not resident, and otherwise kept resident as a cache entry. Returns how - * many programs ended HELD. + * this was is unloaded if resident and freed. Returns how many programs + * ended HELD. */ static inline unsigned int ane_t6021_progtab_fd_close(struct ane_t6021_progtab *t, @@ -540,8 +473,7 @@ ane_t6021_progtab_fd_close(struct ane_t6021_progtab *t, continue; if (!p->resident) ane_t6021_progtab_free(p, ops, ctx); - else if (ane_t6021_prog_unloadable(p) && - ane_t6021_progtab_evict(t, p, ops, ctx) && + else if (ane_t6021_progtab_evict(t, p, ops, ctx) && p->state == ANE_T6021_PROG_HELD) held++; } @@ -549,9 +481,8 @@ ane_t6021_progtab_fd_close(struct ane_t6021_progtab *t, return held; } -/* - * The resident program the firmware can give back most cheaply, never - * @keep: an unreferenced one first, then the least recently used. +/* The resident program to give back for room, never @keep: an + * unreferenced one first (none normally exist), then the least recently used. */ static inline struct ane_t6021_prog * ane_t6021_progtab_victim(struct ane_t6021_progtab *t, @@ -573,9 +504,9 @@ ane_t6021_progtab_victim(struct ane_t6021_progtab *t, } /* - * Make room for one more resident program. Evicts single-task programs - * first; when none is left, restarts the firmware, which drops every - * resident program. @keep is never evicted. + * Make room for one more resident program by unloading the least recently + * used one. Only when none can be unloaded (all HELD) is the firmware + * restarted, which drops every resident program. @keep is never evicted. */ static inline int ane_t6021_progtab_make_room(struct ane_t6021_progtab *t, const struct ane_t6021_prog *keep, @@ -686,7 +617,7 @@ static inline void ane_t6021_progtab_cancel(struct ane_t6021_prog *p, } struct ane_t6021_progtab_stats { - unsigned int live, held, resident, cached, single_task; + unsigned int live, held, resident, cached; }; static inline void ane_t6021_progtab_count(const struct ane_t6021_progtab *t, @@ -704,7 +635,6 @@ static inline void ane_t6021_progtab_count(const struct ane_t6021_progtab *t, s->live++; s->resident += p->resident; s->cached += p->resident && !p->users; - s->single_task += p->single_task; } } } diff --git a/drivers/accel/ane/ane_t6021_lifetime_test.c b/drivers/accel/ane/ane_t6021_lifetime_test.c index 921958b400b31..501e1c75d11cc 100644 --- a/drivers/accel/ane/ane_t6021_lifetime_test.c +++ b/drivers/accel/ane/ane_t6021_lifetime_test.c @@ -69,48 +69,6 @@ static void put32(u8 *p, u32 v) p[3] = v >> 24; } -static void task_count_test(struct kunit *test) -{ - u8 tdp[8 + 2 * ANE_T6021_TDPROP_SEG] = {}; - u8 td[0x200] = {}; - - /* One segment of one block: the single-task form a builder emits. */ - put32(tdp, 1); - put32(tdp + 0xc, 1); - KUNIT_EXPECT_EQ(test, ane_t6021_tdprop_tasks(tdp, 40), 1L); - put32(tdp + 0xc, 4); - KUNIT_EXPECT_EQ(test, ane_t6021_tdprop_tasks(tdp, 40), 4L); - /* Segments add up. */ - put32(tdp, 2); - put32(tdp + 0xc, 1); - put32(tdp + 0x2c, 1); - KUNIT_EXPECT_EQ(test, ane_t6021_tdprop_tasks(tdp, sizeof(tdp)), 2L); - /* Truncated, empty or absurd tables do not parse. */ - KUNIT_EXPECT_LT(test, ane_t6021_tdprop_tasks(tdp, 40), 0L); - KUNIT_EXPECT_LT(test, ane_t6021_tdprop_tasks(tdp, 4), 0L); - put32(tdp, 0); - KUNIT_EXPECT_LT(test, ane_t6021_tdprop_tasks(tdp, sizeof(tdp)), 0L); - put32(tdp, ANE_T6021_TDPROP_MAX_SEGS + 1); - KUNIT_EXPECT_LT(test, ane_t6021_tdprop_tasks(tdp, sizeof(tdp)), 0L); - - /* TD stream: 0x10-byte header, then one block of 0x20 words. */ - td[0x10 + 2] = 0x20; - KUNIT_EXPECT_EQ(test, ane_t6021_td_blocks(td, 0x10 + 0x80), 1L); - /* A second block of 0x41 words, padded to 16 bytes. */ - td[0x90 + 2] = 0x41; - KUNIT_EXPECT_EQ(test, ane_t6021_td_blocks(td, 0x90 + 0x110), 2L); - /* Header bit 2 selects a 0x30-byte header. */ - memset(td, 0, sizeof(td)); - td[0] = 4; - td[0x30 + 2] = 0x10; - KUNIT_EXPECT_EQ(test, ane_t6021_td_blocks(td, 0x30 + 0x40), 1L); - /* A zero word count where a block must start does not parse. */ - KUNIT_EXPECT_LT(test, ane_t6021_td_blocks(td, 0x30 + 0x50), 0L); - /* Neither does a block header cut short. */ - KUNIT_EXPECT_LT(test, ane_t6021_td_blocks(td, 0x30 + 0x42), 0L); - KUNIT_EXPECT_LT(test, ane_t6021_td_blocks(td, 2), 0L); -} - static void put_task(u8 *td, size_t at, u32 words, u32 w2) { td[at + 2] = words; @@ -152,16 +110,12 @@ static void td_flags_test(struct kunit *test) KUNIT_EXPECT_EQ(test, ane_t6021_td_flags_check(td, 2), 0); } -/* - * A fake firmware. Records every command class, fails on demand, and - * refuses (test failure) to terminate a process of a program not proven - * single-task, which on hardware halts the firmware for good. - */ +/* A fake firmware. Records every command class and fails on demand. */ struct fake_fw { struct kunit *test; struct ane_t6021_progtab *t; unsigned int unloads, releases, loads, creates, restarts; - unsigned int terminates, multi_terminates; + unsigned int terminates; int fail_unload, fail_restart; bool restart_in_unload; int sec_refs; @@ -175,11 +129,8 @@ static int fake_unload(void *ctx, struct ane_t6021_prog *p) struct fake_fw *fw = ctx; fw->unloads++; - if (p->fw_proc_id != ANE_T6021_NO_ID) { + if (p->fw_proc_id != ANE_T6021_NO_ID) fw->terminates++; - if (!p->single_task) - fw->multi_terminates++; - } if (fw->restart_in_unload) { ane_t6021_progtab_restarted(fw->t, &fake_ops, fw); return -ETIMEDOUT; @@ -257,8 +208,7 @@ static struct ane_t6021_fdprogs *new_fd(struct kunit *test) /* What the driver's LOAD ioctl does for a new program. */ static struct ane_t6021_prog *load(struct ane_t6021_progtab *t, - struct fake_fw *fw, u32 tag, u32 id, - bool single) + struct fake_fw *fw, u32 tag, u32 id) { u8 digest[32] = {}; struct ane_t6021_prog *p; @@ -270,7 +220,6 @@ static struct ane_t6021_prog *load(struct ane_t6021_progtab *t, p = ane_t6021_progtab_reserve(t, digest); if (!p) return NULL; - p->single_task = single; p->sec[0] = fw; p->sec[6] = fw; fw->sec_refs += 2; @@ -283,8 +232,7 @@ static struct ane_t6021_prog *load(struct ane_t6021_progtab *t, static struct ane_t6021_prog *open_run(struct kunit *test, struct ane_t6021_progtab *t, struct fake_fw *fw, - struct ane_t6021_fdprogs *f, u32 tag, - bool single) + struct ane_t6021_fdprogs *f, u32 tag) { u8 digest[32] = {}; struct ane_t6021_prog *p; @@ -293,7 +241,7 @@ static struct ane_t6021_prog *open_run(struct kunit *test, digest[1] = tag >> 8; p = ane_t6021_progtab_find(t, digest); if (!p) - p = load(t, fw, tag, 1000 + tag, single); + p = load(t, fw, tag, 1000 + tag); KUNIT_ASSERT_NOT_NULL(test, p); KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, f, p), 0); KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p, true, &fake_ops, fw), 0); @@ -311,11 +259,11 @@ static void progtab_shared_unload_test(struct kunit *test) struct ane_t6021_fdprogs *a = new_fd(test), *b = new_fd(test); struct ane_t6021_prog *p, *q; - p = open_run(test, t, &fw, a, 1, true); + p = open_run(test, t, &fw, a, 1); /* A second load of identical sections finds the same program and * process: no command reaches the firmware. */ - q = open_run(test, t, &fw, b, 1, true); + q = open_run(test, t, &fw, b, 1); KUNIT_ASSERT_PTR_EQ(test, q, p); /* Taking the same reference twice from one file counts once. */ KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, b, q), 0); @@ -328,7 +276,7 @@ static void progtab_shared_unload_test(struct kunit *test) KUNIT_EXPECT_EQ(test, p->users, 1U); KUNIT_EXPECT_EQ(test, fw.sec_refs, 2); - /* A single-task program is unloaded at its last close. */ + /* The program is unloaded at its last close. */ KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, b, &fake_ops, &fw), 0U); KUNIT_EXPECT_EQ(test, fw.unloads, 1U); KUNIT_EXPECT_EQ(test, fw.terminates, 1U); @@ -337,48 +285,38 @@ static void progtab_shared_unload_test(struct kunit *test) KUNIT_EXPECT_NULL(test, ane_t6021_progtab_by_id(t, 1001)); KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 0U); /* The slot is reused by the next program. */ - q = load(t, &fw, 2, 1002, true); + q = load(t, &fw, 2, 1002); KUNIT_EXPECT_PTR_EQ(test, q, p); KUNIT_EXPECT_EQ(test, t->hi, 1U); } -static void multitask_never_terminated_test(struct kunit *test) +static void multitask_unloaded_at_close_test(struct kunit *test) { struct fake_fw fw = {}; struct ane_t6021_progtab *t = new_table(test, &fw); - struct ane_t6021_fdprogs *a = new_fd(test), *b = new_fd(test); - struct ane_t6021_prog *p, *q; + struct ane_t6021_fdprogs *a = new_fd(test); + struct ane_t6021_prog *p; struct ane_t6021_progtab_stats st; - p = open_run(test, t, &fw, a, 3, false); - KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 0U); - /* No TERMINATE, no DESTROY: the program stays resident as a cache - * entry with its section references. + /* Any program with a process, whatever its task count, is terminated + * and destroyed at its last close; nothing stays cached. */ - KUNIT_EXPECT_EQ(test, fw.unloads, 0U); - KUNIT_EXPECT_EQ(test, fw.releases, 0U); - KUNIT_EXPECT_EQ(test, fw.sec_refs, 2); - KUNIT_EXPECT_TRUE(test, p->resident); + p = open_run(test, t, &fw, a, 3); + KUNIT_EXPECT_TRUE(test, ane_t6021_prog_unloadable(p)); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 0U); + KUNIT_EXPECT_EQ(test, fw.unloads, 1U); + KUNIT_EXPECT_EQ(test, fw.terminates, 1U); + KUNIT_EXPECT_EQ(test, fw.sec_refs, 0); ane_t6021_progtab_count(t, &st); - KUNIT_EXPECT_EQ(test, st.cached, 1U); - /* The next opener of the same bytes reuses program and process. */ - q = open_run(test, t, &fw, b, 3, false); - KUNIT_EXPECT_PTR_EQ(test, q, p); - KUNIT_EXPECT_EQ(test, fw.loads, 1U); - KUNIT_EXPECT_EQ(test, fw.creates, 1U); - KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, b, &fake_ops, &fw), 0U); - /* Nor can an eviction pick it. */ - KUNIT_EXPECT_NULL(test, ane_t6021_progtab_victim(t, NULL)); - KUNIT_EXPECT_EQ(test, ane_t6021_progtab_evict(t, p, &fake_ops, &fw), -EBUSY); - KUNIT_EXPECT_EQ(test, fw.multi_terminates, 0U); + KUNIT_EXPECT_EQ(test, st.cached, 0U); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 0U); - /* A multi-task program that never got a process can be destroyed. */ - p = load(t, &fw, 4, 1004, false); + /* A program that never got a process is only destroyed. */ + p = load(t, &fw, 4, 1004); KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), 0); - KUNIT_EXPECT_TRUE(test, ane_t6021_prog_unloadable(p)); KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 0U); - KUNIT_EXPECT_EQ(test, fw.unloads, 1U); - KUNIT_EXPECT_EQ(test, fw.terminates, 0U); + KUNIT_EXPECT_EQ(test, fw.unloads, 2U); + KUNIT_EXPECT_EQ(test, fw.terminates, 1U); } static void progtab_failed_unload_holds_test(struct kunit *test) @@ -389,7 +327,7 @@ static void progtab_failed_unload_holds_test(struct kunit *test) struct ane_t6021_progtab_stats st; struct ane_t6021_prog *p, *q; - p = open_run(test, t, &fw, a, 5, true); + p = open_run(test, t, &fw, a, 5); KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, a, &fake_ops, &fw), 1U); /* The firmware may still read the sections: nothing is released. */ KUNIT_EXPECT_EQ(test, fw.releases, 0U); @@ -402,7 +340,7 @@ static void progtab_failed_unload_holds_test(struct kunit *test) /* A HELD program still counts against the firmware budget. */ KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 1U); /* A held slot is never reused before a restart. */ - q = load(t, &fw, 6, 1006, true); + q = load(t, &fw, 6, 1006); KUNIT_EXPECT_PTR_NE(test, q, p); KUNIT_EXPECT_EQ(test, t->hi, 2U); /* A verified restart frees everything no file holds. */ @@ -442,16 +380,16 @@ static void progtab_restart_keeps_ids_test(struct kunit *test) struct ane_t6021_prog *p, *z; u32 fw_prog, fw_proc; - p = open_run(test, t, &fw, a, 7, false); - /* An unreferenced cache entry next to it. */ - z = open_run(test, t, &fw, b, 8, false); + p = open_run(test, t, &fw, a, 7); + /* A second program, closed again: it is unloaded at once. */ + z = open_run(test, t, &fw, b, 8); KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, b, &fake_ops, &fw), 0U); fw_prog = p->fw_prog_id; fw_proc = p->fw_proc_id; KUNIT_ASSERT_EQ(test, fake_restart(&fw), 0); /* The held program keeps its kernel ID and sections, not its - * firmware IDs; the cache entry is gone. + * firmware IDs. */ KUNIT_EXPECT_PTR_EQ(test, ane_t6021_progtab_by_id(t, 1007), p); KUNIT_EXPECT_FALSE(test, p->resident); @@ -486,64 +424,62 @@ static void budget_eviction_order_test(struct kunit *test) for (i = 0; i <= ANE_T6021_FW_RESIDENT_MAX; i++) f[i] = new_fd(test); - /* 0..5 multi-task and held, 6 and 7 single-task and held. */ + /* Eight programs, each held open by its own file. */ for (i = 0; i < ANE_T6021_FW_RESIDENT_MAX; i++) - p[i] = open_run(test, t, &fw, f[i], 10 + i, i >= 6); + p[i] = open_run(test, t, &fw, f[i], 10 + i); KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), (unsigned int)ANE_T6021_FW_RESIDENT_MAX); - /* 7 was used last, so 6 is the least recently used single-task one. */ - KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p[7], true, &fake_ops, &fw), 0); - KUNIT_EXPECT_PTR_EQ(test, ane_t6021_progtab_victim(t, NULL), p[6]); - KUNIT_EXPECT_PTR_EQ(test, ane_t6021_progtab_victim(t, p[6]), p[7]); - /* A new program takes the least recently used single-task slot. */ - p[8] = open_run(test, t, &fw, f[8], 30, false); + /* Using 0 again makes 1 the least recently used. */ + KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p[0], true, &fake_ops, &fw), 0); + KUNIT_EXPECT_PTR_EQ(test, ane_t6021_progtab_victim(t, NULL), p[1]); + KUNIT_EXPECT_PTR_EQ(test, ane_t6021_progtab_victim(t, p[1]), p[2]); + /* A ninth program unloads 1 for room; no restart. */ + p[8] = open_run(test, t, &fw, f[8], 30); KUNIT_EXPECT_EQ(test, fw.restarts, 0U); KUNIT_EXPECT_EQ(test, fw.unloads, 1U); - KUNIT_EXPECT_FALSE(test, p[6]->resident); - KUNIT_EXPECT_PTR_EQ(test, ane_t6021_progtab_by_id(t, 1016), p[6]); - /* Using 6 again evicts 7, the last single-task resident one. */ - KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p[6], true, &fake_ops, &fw), 0); - KUNIT_EXPECT_FALSE(test, p[7]->resident); + KUNIT_EXPECT_EQ(test, fw.terminates, 1U); + KUNIT_EXPECT_FALSE(test, p[1]->resident); + KUNIT_EXPECT_PTR_EQ(test, ane_t6021_progtab_by_id(t, 1011), p[1]); + /* Its file uses it again: 2 goes, 1 loads with a new process. */ + KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p[1], true, &fake_ops, &fw), 0); + KUNIT_EXPECT_FALSE(test, p[2]->resident); + KUNIT_EXPECT_TRUE(test, p[1]->resident); KUNIT_EXPECT_EQ(test, fw.restarts, 0U); - /* The two single-task programs now take turns in the last slot; the - * multi-task ones are never touched and no restart is needed. + /* HELD programs cannot be given back; when all residents are HELD the + * firmware is restarted, and the held programs files still hold come + * back as non-resident LIVE programs. */ - KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p[7], true, &fake_ops, &fw), 0); - KUNIT_EXPECT_EQ(test, fw.restarts, 0U); - KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p[6], true, &fake_ops, &fw), 0); - KUNIT_EXPECT_EQ(test, fw.restarts, 0U); - /* Close the single-task holders; their programs unload. */ - ane_t6021_progtab_fd_close(t, f[6], &fake_ops, &fw); - ane_t6021_progtab_fd_close(t, f[7], &fake_ops, &fw); - KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 7U); - /* One more multi-task load fits; the next one forces the restart. */ - p[6] = open_run(test, t, &fw, f[6], 40, false); - KUNIT_EXPECT_EQ(test, fw.restarts, 0U); - p[7] = open_run(test, t, &fw, f[7], 41, false); + for (i = 0; i <= ANE_T6021_FW_RESIDENT_MAX; i++) + if (p[i]->resident) + p[i]->state = ANE_T6021_PROG_HELD; + KUNIT_EXPECT_NULL(test, ane_t6021_progtab_victim(t, NULL)); + KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p[2], true, &fake_ops, &fw), 0); KUNIT_EXPECT_EQ(test, fw.restarts, 1U); KUNIT_EXPECT_EQ(test, t->restarts, 1U); - /* After it only the new program is resident; the others are known - * and load on use. - */ KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 1U); KUNIT_EXPECT_PTR_EQ(test, ane_t6021_progtab_by_id(t, 1010), p[0]); KUNIT_EXPECT_FALSE(test, p[0]->resident); - KUNIT_EXPECT_EQ(test, fw.multi_terminates, 0U); /* A failed restart reaches the caller and changes nothing. */ - for (i = 1; i < 8; i++) - KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p[i - 1], true, + for (i = 0; i <= ANE_T6021_FW_RESIDENT_MAX; i++) + KUNIT_ASSERT_EQ(test, ane_t6021_progtab_ensure(t, p[i], true, &fake_ops, &fw), 0); - KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 8U); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), + (unsigned int)ANE_T6021_FW_RESIDENT_MAX); + /* Nine programs, eight resident: the one left out is p[0]. */ + KUNIT_EXPECT_FALSE(test, p[0]->resident); + for (i = 1; i <= ANE_T6021_FW_RESIDENT_MAX; i++) + p[i]->state = ANE_T6021_PROG_HELD; fw.fail_restart = -EBUSY; - KUNIT_EXPECT_EQ(test, ane_t6021_progtab_ensure(t, p[8], true, &fake_ops, &fw), + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_ensure(t, p[0], true, &fake_ops, &fw), -EBUSY); - KUNIT_EXPECT_FALSE(test, p[8]->resident); - KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 8U); + KUNIT_EXPECT_FALSE(test, p[0]->resident); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), + (unsigned int)ANE_T6021_FW_RESIDENT_MAX); } -/* Thirty distinct programs, every third one multi-task, each opened, run - * and closed: the budget holds at every step, no multi-task process is - * ever terminated, and only the multi-task cache entries force restarts. +/* Thirty distinct programs, each opened, run and closed, while one more is + * held open throughout: everything unloads at close, the budget holds at + * every step and no restart is ever needed. */ static void budget_sequence_test(struct kunit *test) { @@ -552,29 +488,21 @@ static void budget_sequence_test(struct kunit *test) struct ane_t6021_fdprogs *f = new_fd(test), *keep = new_fd(test); unsigned int i; - /* One MobileNet-like program held open across the whole run. */ - open_run(test, t, &fw, keep, 500, false); + open_run(test, t, &fw, keep, 500); for (i = 0; i < 30; i++) { - struct ane_t6021_prog *p = open_run(test, t, &fw, f, 600 + i, - i % 3 != 0); + struct ane_t6021_prog *p = open_run(test, t, &fw, f, 600 + i); KUNIT_EXPECT_EQ(test, ane_t6021_progtab_fd_close(t, f, &fake_ops, &fw), 0U); - if (i % 3) - KUNIT_EXPECT_EQ(test, p->state, ANE_T6021_PROG_FREE); - KUNIT_EXPECT_LE(test, ane_t6021_progtab_resident(t), - (unsigned int)ANE_T6021_FW_RESIDENT_MAX); + KUNIT_EXPECT_EQ(test, p->state, ANE_T6021_PROG_FREE); + KUNIT_EXPECT_LE(test, ane_t6021_progtab_resident(t), 2U); } - KUNIT_EXPECT_EQ(test, fw.multi_terminates, 0U); - /* Ten multi-task programs plus the held one: one restart after the - * cache filled, and the held program still loads on use. - */ - KUNIT_EXPECT_EQ(test, fw.restarts, 1U); + KUNIT_EXPECT_EQ(test, fw.restarts, 0U); + KUNIT_EXPECT_EQ(test, fw.terminates, 30U); KUNIT_ASSERT_NOT_NULL(test, ane_t6021_progtab_by_id(t, 1500)); - KUNIT_EXPECT_EQ(test, ane_t6021_progtab_ensure(t, ane_t6021_progtab_by_id(t, 1500), - true, &fake_ops, &fw), 0); + KUNIT_EXPECT_TRUE(test, ane_t6021_progtab_by_id(t, 1500)->resident); ane_t6021_progtab_fd_close(t, keep, &fake_ops, &fw); - KUNIT_EXPECT_EQ(test, fw.multi_terminates, 0U); + KUNIT_EXPECT_EQ(test, ane_t6021_progtab_resident(t), 0U); } static void progtab_capacity_test(struct kunit *test) @@ -588,7 +516,7 @@ static void progtab_capacity_test(struct kunit *test) /* Known programs are bounded by the firmware table, not the budget. */ for (i = 0; i < ANE_T6021_MAX_PROGRAMS; i++) { - p = load(t, &fw, i, 1000 + i, true); + p = load(t, &fw, i, 1000 + i); KUNIT_ASSERT_NOT_NULL(test, p); KUNIT_ASSERT_EQ(test, ane_t6021_fdprogs_ref(t, a, p), 0); KUNIT_ASSERT_LE(test, ane_t6021_progtab_resident(t), @@ -622,8 +550,8 @@ static void progtab_restart_during_unload_test(struct kunit *test) struct ane_t6021_fdprogs *a = new_fd(test); struct ane_t6021_prog *p, *q; - p = open_run(test, t, &fw, a, 50, true); - q = open_run(test, t, &fw, a, 51, true); + p = open_run(test, t, &fw, a, 50); + q = open_run(test, t, &fw, a, 51); /* The first unload fails and its recovery restarts the firmware. * Nothing may be marked HELD afterwards, and the second program, * no longer resident, needs no unload. @@ -676,7 +604,7 @@ static int fake_cmd(void *ctx, u16 opcode, u32 fw_prog, u32 fw_proc) static void unload_order_test(struct kunit *test) { struct ane_t6021_prog p = { - .fw_prog_id = 4, .fw_proc_id = 9, .single_task = true, + .fw_prog_id = 4, .fw_proc_id = 9, }; struct fake_cmds c = {}; @@ -705,14 +633,6 @@ static void unload_order_test(struct kunit *test) KUNIT_EXPECT_EQ(test, ane_t6021_prog_unload_cmds(&p, fake_cmd, &c), -ETIMEDOUT); KUNIT_EXPECT_EQ(test, c.n, 1U); - - /* A process of a program not proven single-task is never terminated, - * and nothing else is sent for it. - */ - memset(&c, 0, sizeof(c)); - p.single_task = false; - KUNIT_EXPECT_EQ(test, ane_t6021_prog_unload_cmds(&p, fake_cmd, &c), -EBUSY); - KUNIT_EXPECT_EQ(test, c.n, 0U); } /* The restart steps, recorded in order; a step fails when asked to. */ @@ -810,10 +730,9 @@ static void restart_state_machine_test(struct kunit *test) static struct kunit_case lifetime_cases[] = { KUNIT_CASE(mring_classify_test), KUNIT_CASE(legtab_reuse_test), - KUNIT_CASE(task_count_test), KUNIT_CASE(td_flags_test), KUNIT_CASE(progtab_shared_unload_test), - KUNIT_CASE(multitask_never_terminated_test), + KUNIT_CASE(multitask_unloaded_at_close_test), KUNIT_CASE(progtab_failed_unload_holds_test), KUNIT_CASE(progtab_load_failure_test), KUNIT_CASE(progtab_restart_keeps_ids_test), diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index a01e4ff634b74..ed0735e43852d 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -1441,15 +1441,11 @@ static int ane_rtclient_command(struct ane_rtclient *ane, /* Two loads of byte-identical sections share one program and one process. * The key is SHA-256 over every section's id, size and bytes, so a client * can only reach a program whose bytes it also supplied, and every file - * holds a reference on each program it loaded. Which programs are in the - * firmware at a time, which may be unloaded and when the firmware is - * restarted to make room is in ane_t6021_lifetime.h: a process of a - * multi-task program is never terminated, at most - * ANE_T6021_FW_RESIDENT_MAX programs are resident, and a program a file - * still holds loads again on its next use after an eviction or a restart. - * Without unloads every program and process stayed in the firmware until - * reboot and the firmware stopped answering after eight processes. - * Protected by ane_t6021_fw_lock. + * holds a reference on each program it loaded. The last close unloads a + * program, at most ANE_T6021_FW_RESIDENT_MAX programs are resident, and a + * program a file still holds loads again on its next use after it was + * unloaded for room or a restart (ane_t6021_lifetime.h). Protected by + * ane_t6021_fw_lock. */ static struct ane_t6021_progtab ane_t6021_progtab; static struct ane_fw_ids ane_t6021_ids; @@ -1678,7 +1674,6 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, struct ane_t6021_fd *fd = file->driver_priv; struct ane_t6021_prog *cached, *prog = NULL; bool sent = false; - long tdprop_tasks = -EINVAL, td_blocks = -EINVAL; int td_flags = 0; u32 key, gen; u8 digest[SHA256_DIGEST_SIZE]; @@ -1778,21 +1773,11 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, goto out; } sha256_update(&sha, (const u8 *)hdr, sizeof(hdr)); - /* The same snapshot as the key decides whether the - * program's process may ever be terminated. - */ - if (sections[i].id == ANE_T6021_SEC_TEXT) { - td_blocks = ane_t6021_td_blocks((const u8 *)bo->cpu + - sections[i].offset, - sections[i].size); + /* Checked in the same snapshot as the key. */ + if (sections[i].id == ANE_T6021_SEC_TEXT) td_flags = ane_t6021_td_flags_check((const u8 *)bo->cpu + sections[i].offset, sections[i].size); - } - else if (sections[i].id == ANE_T6021_SEC_TDPROP) - tdprop_tasks = ane_t6021_tdprop_tasks((const u8 *)bo->cpu + - sections[i].offset, - sections[i].size); left = sections[i].size; while (left) { size_t n = min_t(size_t, left, @@ -1853,7 +1838,6 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, ret = -ENOSPC; goto out; } - prog->single_task = tdprop_tasks == 1 && td_blocks == 1; gen = ane_t6021_progtab.gen; /* The record slot IS the section identity (slot = id - 1). The program * takes its own reference on each section buffer, kept until the @@ -1900,10 +1884,8 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, if (WARN_ON(ret)) goto out; *prog_id = key; - dev_dbg(ane->dev, "loaded program %u (fw %u, %s) (fw allocations live %u, %zu bytes)\n", - key, prog->fw_prog_id, - prog->single_task ? "single task" : "multi task or unknown", - ane->legacy_live, ane->legacy_bytes); + dev_dbg(ane->dev, "loaded program %u (fw %u) (fw allocations live %u, %zu bytes)\n", + key, prog->fw_prog_id, ane->legacy_live, ane->legacy_bytes); } else if (ane_t6021_progtab.gen != gen) { /* A firmware restart already released the reservation. */ prog = NULL; @@ -2507,8 +2489,8 @@ static int ane_t6021_fw_memory_show(struct seq_file *m, void *v) ane_t6021_progtab_count(&ane_t6021_progtab, &st); seq_printf(m, "allocations: %u\nbytes: %zu\nhigh_water: %u\n", ane->legacy_live, ane->legacy_bytes, ane->legacy_allocated); - seq_printf(m, "programs: %u\nheld: %u\nresident: %u\ncached: %u\nsingle_task: %u\n", - st.live, st.held, st.resident, st.cached, st.single_task); + seq_printf(m, "programs: %u\nheld: %u\nresident: %u\ncached: %u\n", + st.live, st.held, st.resident, st.cached); seq_printf(m, "resident_max: %u\nrestarts: %u\nfw_generation: %u\nquarantined: %d\n", ANE_T6021_FW_RESIDENT_MAX, ane_t6021_progtab.restarts, ane_t6021_progtab.gen, atomic_read(&ane_t6021_quarantined)); @@ -3402,8 +3384,8 @@ static int ane_rtclient_recover(struct ane_rtclient *ane) } /* - * The budget restart: every resident program is a multi-task process the - * firmware cannot give back, and a new one must be loaded. + * The restart for room: every resident program is HELD (its load or unload + * outcome is unknown), so none can be given back, and a new one must load. */ static int ane_rtclient_restart(struct ane_rtclient *ane) { From f2fff71fd915a7fa0c96c74352116de5a6d9821d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 16:36:43 +1100 Subject: [PATCH 50/73] media: apple: avd: release VP9 probability dependencies at VP DONE Give queued VP9 pictures separate DMA banks and retain them until PP retirement. Publish backward probability adaptation from a tagged VP DONE callback, so the next picture can prepare and run VP while PP is active. Forward-only, non-refreshing and error-resilient producers impose no count wait. Preserve the preceding picture's temporal-MV input across banks. Negotiate an optional queue ABI 2 VP notification, with a separate full-tag record and no early release of descriptor or buffer ownership. Include the matching MIT firmware patch and host/Unicorn regressions. Firmware without the feature retains one VP9 picture per stream; legacy programs and saved probability contexts have an independent before/after comparison harness. Document AV1's CDF and segmentation-tail dependencies pending evidence for their visibility and CPU synchronization at VP DONE. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../platform/apple/avd/avd-backend-queue.c | 30 +- .../platform/apple/avd/avd-queue-internal.h | 2 + drivers/media/platform/apple/avd/avd-queue.c | 41 +++ drivers/media/platform/apple/avd/avd-queue.h | 4 + drivers/media/platform/apple/avd/avd-vp9.c | 340 ++++++++++++------ drivers/media/platform/apple/avd/avd.h | 7 +- .../media/platform/apple/avd/tests/README.rst | 116 ++++++ .../apple/avd/tests/avd-fw-vp-done.patch | 117 ++++++ .../platform/apple/avd/tests/dispatch-test.py | 39 +- .../platform/apple/avd/tests/program-test.py | 22 +- .../apple/avd/tests/unicorn-vp9-test.py | 153 ++++++++ .../platform/apple/avd/tests/vp9-test.py | 279 ++++++++++++++ 12 files changed, 1030 insertions(+), 120 deletions(-) create mode 100644 drivers/media/platform/apple/avd/tests/avd-fw-vp-done.patch create mode 100644 drivers/media/platform/apple/avd/tests/unicorn-vp9-test.py create mode 100644 drivers/media/platform/apple/avd/tests/vp9-test.py diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c index 785716d450e3e..7d21921d17229 100644 --- a/drivers/media/platform/apple/avd/avd-backend-queue.c +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -31,6 +31,9 @@ #define AVDQ_INFO_WP (AVDQ_INFO + 0x3c) #define AVDQ_FEAT_SRAM_RING BIT(0) #define AVDQ_FEAT_MULTI_KICK BIT(1) +#define AVDQ_FEAT_VP_DONE BIT(2) +#define AVDQ_FLAG_VP_DONE BIT(1) +#define AVDQ_VP_NOTIFY 0x54540000 #define AVDQ_FLAG_SRAM BIT(0) #define AVDQ_MAX_PP_EXTRA 255 #define AVDQ_STATE_SUBMITTED 1 @@ -61,6 +64,7 @@ struct avd_qfw { u32 wp; /* host write count, free-running */ u32 fifo_slots; u32 codec_mask; /* info+0x24, bit n = codec n */ + bool vp_done; bool usable; /* last boot found a matching ABI 2 image */ }; @@ -122,6 +126,7 @@ static bool avd_qfw_check(struct avd_dev *avd) for (i = 0; i < ARRAY_SIZE(info); i++) info[i] = readl(q->sram + AVDQ_INFO + 4 * i); q->usable = false; + q->vp_done = false; if (info[0] != AVDQ_MAGIC) return false; if (info[1] != AVDQ_VERSION || info[2] != AVDQ_COUNT || @@ -143,6 +148,7 @@ static bool avd_qfw_check(struct avd_dev *avd) dev_warn_once(avd->dev, "firmware SRAM ring geometry invalid, using legacy transport\n"); return false; } + q->vp_done = info[11] & AVDQ_FEAT_VP_DONE; q->ring_off = info[12]; q->ring_words = info[13]; q->fifo_slots = min_t(u32, info[10], avd->variant->fifo_slots ?: 1); @@ -274,7 +280,8 @@ static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) writel(kick, d + D_KICK); writel(0, d + D_STATUS); writel(0, d + D_CTAG); - writel(AVDQ_FLAG_SRAM, d + D_FLAGS); + writel(AVDQ_FLAG_SRAM | (q->vp_done && ctx->coded_fmt_desc->ops->vp_done ? + AVDQ_FLAG_VP_DONE : 0), d + D_FLAGS); writel(extra, d + D_PP_EXTRA); writel(extra ? cont : 0, d + D_PP_CONT); writel(0, d + D_RSVD0); @@ -307,13 +314,25 @@ static irqreturn_t avd_qfw_irq(struct avd_dev *avd) unsigned int n; u32 word; - /* At most sixteen completions and one fatal token can be outstanding. */ - for (n = 0; n < AVDQ_COUNT + 1 && avd_mbox_pop(avd, &word); n++) { + /* At most one VP and PP token per slot, plus one fatal token. */ + for (n = 0; n < 2 * AVDQ_COUNT + 1 && avd_mbox_pop(avd, &word); n++) { struct avd_frame *f; void __iomem *d; u32 state, status, ctag; unsigned int idx; + if ((word & AVDQ_NOTIFY_MASK) == AVDQ_VP_NOTIFY && q->vp_done) { + idx = word & 15; + d = avd_qfw_desc(q, idx); + /* Separate record: PP completion may already have been published. */ + state = readl(d + D_RSVD1); + ctag = readl(d + D_RSVD0); + f = &avd->frames[idx]; + if (state == 1 && !((ctag ^ word) & 0xfff0) && + f->state == AVD_FRAME_SUBMITTED && ctag == f->tag) + avd_frame_vp_complete(avd, ctag); + continue; + } if ((word & AVDQ_NOTIFY_MASK) != AVDQ_NOTIFY) { dev_err_ratelimited(avd->dev, "queue: firmware reported %#x, resetting\n", word); @@ -360,6 +379,11 @@ bool avd_backend_is_qfw(const struct avd_dev *avd) return avd->backend == &avd_qfw_backend; } +bool avd_qfw_has_vp_done(const struct avd_dev *avd) +{ + return avd_backend_is_qfw(avd) && avd->qfw->vp_done; +} + /* * SRAM survives a CM3 reload, so a stale info block from an earlier ABI 2 * image could make a legacy image look queue capable. Clear the magic before diff --git a/drivers/media/platform/apple/avd/avd-queue-internal.h b/drivers/media/platform/apple/avd/avd-queue-internal.h index 2c09a5d85da6b..c5b9c06836992 100644 --- a/drivers/media/platform/apple/avd/avd-queue-internal.h +++ b/drivers/media/platform/apple/avd/avd-queue-internal.h @@ -22,6 +22,8 @@ static inline int avd_slot_reserve(struct avd_frame *f, struct avd_ctx *ctx, f->state = AVD_FRAME_RESERVED; f->dep = AVD_DEP_NONE; f->status = 0; + f->vp_done = false; + f->vp_retired = false; f->prog_words = 0; f->codec_priv = NULL; return 0; diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index dbb6336d44af4..22e40b631a527 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -31,6 +31,27 @@ bool avd_backend_is_apple(const struct avd_dev *avd) return avd->backend == &avd_apple_backend; } +/* Only the CPU callback releases a probability dependency, never the IRQ. */ +static void avd_retire_vp(struct avd_dev *avd) +{ + unsigned int i; + + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + const struct avd_coded_fmt_ops *ops; + + if (f->state == AVD_FRAME_FREE || !f->vp_done || f->vp_retired) + continue; + ops = f->ctx->coded_fmt_desc->ops; + /* A reset/error must not publish partial symbol counts. */ + if (!f->status && ops && ops->vp_done) { + dma_rmb(); + ops->vp_done(f->ctx, f->dst); + } + f->vp_retired = true; + } +} + /* queue_mutex excludes callbacks and context destruction from retirement. */ static void avd_retire(struct avd_dev *avd) { @@ -215,6 +236,7 @@ static void avd_queue_worker(struct work_struct *work) break; } } + avd_retire_vp(avd); avd_retire(avd); for (i = 0; i < AVD_MAX_FRAMES; i++) { struct avd_frame *f = &avd->frames[i]; @@ -275,6 +297,22 @@ void avd_frame_complete(struct avd_dev *avd, u32 tag, int status) mod_delayed_work(system_dfl_wq, &avd->queue_work, 0); } +/* No buffer, FIFO or descriptor ownership is released by a VP event. */ +void avd_frame_vp_complete(struct avd_dev *avd, u32 tag) +{ + struct avd_frame *f = &avd->frames[tag & (AVD_MAX_FRAMES - 1)]; + unsigned long flags; + bool accepted; + + spin_lock_irqsave(&avd->queue_lock, flags); + accepted = f->state == AVD_FRAME_SUBMITTED && f->tag == tag && !f->vp_done; + if (accepted) + f->vp_done = true; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (accepted) + mod_delayed_work(system_dfl_wq, &avd->queue_work, 0); +} + int avd_queue_ready(void *priv) { struct avd_ctx *ctx = priv; @@ -307,6 +345,9 @@ int avd_queue_ready(void *priv) held++; ready = held < limit; } + if (ready && ctx->coded_fmt_desc && !ctx->coded_fmt_desc->frame_ops && + ctx->coded_fmt_desc->ops->ready) + ready = ctx->coded_fmt_desc->ops->ready(ctx); spin_unlock_irqrestore(&avd->queue_lock, flags); return ready; } diff --git a/drivers/media/platform/apple/avd/avd-queue.h b/drivers/media/platform/apple/avd/avd-queue.h index 9aa15c73eeee3..038c4f911313e 100644 --- a/drivers/media/platform/apple/avd/avd-queue.h +++ b/drivers/media/platform/apple/avd/avd-queue.h @@ -52,6 +52,9 @@ struct avd_frame { unsigned long deadline; /* jiffies, per-frame watchdog */ /* True while the buffers still use m2m's held-CAPTURE completion path. */ bool scheduler_held; + /* Tagged VP event and CPU codec callback, independent of PP retirement. */ + bool vp_done; + bool vp_retired; /* VP to PP instruction FIFO for this frame (open backends). */ dma_addr_t fifo_iova; }; @@ -87,6 +90,7 @@ struct avd_codec_frame_ops { /* Core API used by backends and codecs. */ void avd_frame_complete(struct avd_dev *avd, u32 tag, int status); /* any ctx, IRQ-safe */ +void avd_frame_vp_complete(struct avd_dev *avd, u32 tag); bool avd_backend_is_apple(const struct avd_dev *avd); #endif /* AVD_QUEUE_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-vp9.c b/drivers/media/platform/apple/avd/avd-vp9.c index 37374b73e8a86..fc0d64c760a50 100644 --- a/drivers/media/platform/apple/avd/avd-vp9.c +++ b/drivers/media/platform/apple/avd/avd-vp9.c @@ -160,6 +160,7 @@ struct avd_vp9_frame_info { u32 interpolation_filter : 3; u32 flags; u64 timestamp; + dma_addr_t color; struct v4l2_vp9_segmentation seg; struct v4l2_vp9_loop_filter lf; }; @@ -171,10 +172,14 @@ struct avd_vp9_run { const struct v4l2_ctrl_vp9_compressed_hdr *prob_updates; }; -struct avd_vp9_ctx { +#define AVD_VP9_BANKS 2 + +/* Every DMA scratch mapping stays owned until PP retirement, including counts. */ +struct avd_vp9_bank { + bool busy; + bool updated; struct v4l2_vp9_frame_symbol_counts cnts; struct v4l2_vp9_frame_context probability_tables; - struct v4l2_vp9_frame_context frame_context[4]; struct avd_vp9_bufs { struct avd_buf inst; struct avd_buf az_left; @@ -194,6 +199,33 @@ struct avd_vp9_ctx { u8 submit_num; }; +struct avd_vp9_ctx { + struct v4l2_vp9_frame_context frame_context[4]; + struct avd_vp9_frame_info last; + unsigned int nbanks; + struct avd_vp9_bank bank[AVD_VP9_BANKS]; + struct avd_vp9_bank *b; +}; + +static bool avd_vp9_needs_counts(u32 flags) +{ + return (flags & V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX) && + !(flags & (V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE | + V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT)); +} + +static bool avd_vp9_ready(struct avd_ctx *ctx) +{ + struct avd_vp9_ctx *vp9_ctx = ctx->priv; + + /* Only a producer doing backward adaptation holds the next prepare. */ + for (unsigned int i = 0; i < vp9_ctx->nbanks; i++) + if (READ_ONCE(vp9_ctx->bank[i].busy) && + !READ_ONCE(vp9_ctx->bank[i].updated)) + return false; + return true; +} + static void set_refs(struct avd_ctx *ctx, struct avd_vp9_run *run) { const struct v4l2_ctrl_vp9_frame *frame = run->decode_params; @@ -233,6 +265,15 @@ static void set_refs(struct avd_ctx *ctx, struct avd_vp9_run *run) } } +/* Temporal MV input is the previous shown inter picture, not the new bank. */ +static bool avd_vp9_uses_prev_mv(u32 flags, const struct avd_vp9_frame_info *last) +{ + return !(flags & V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT) && + !(flags & V4L2_VP9_FRAME_FLAG_KEY_FRAME) && last->valid && + (last->flags & V4L2_VP9_FRAME_FLAG_SHOW_FRAME) && + !(last->flags & V4L2_VP9_FRAME_FLAG_KEY_FRAME); +} + static u32 make_flags1(struct avd_ctx *ctx, struct avd_vp9_run *run) { struct avd_vp9_ctx *vp9_ctx = ctx->priv; @@ -284,12 +325,7 @@ static u32 make_flags1(struct avd_ctx *ctx, struct avd_vp9_run *run) flags |= VP9_SEG_ENABLED(frame->seg.flags & V4L2_VP9_SEGMENTATION_FLAG_ENABLED); - flags |= VP9_HAS_REF( - !(frame->flags & V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT) && - !(frame->flags & V4L2_VP9_FRAME_FLAG_KEY_FRAME) && - vp9_ctx->last.valid && - vp9_ctx->last.flags & V4L2_VP9_FRAME_FLAG_SHOW_FRAME && - !(vp9_ctx->last.flags & V4L2_VP9_FRAME_FLAG_KEY_FRAME)); + flags |= VP9_HAS_REF(avd_vp9_uses_prev_mv(frame->flags, &vp9_ctx->last)); flags |= VP9_SEG_RESET( frame->seg.flags & V4L2_VP9_SEGMENTATION_FLAG_ENABLED && @@ -307,7 +343,7 @@ static u32 seg_features(struct avd_ctx *ctx, struct avd_vp9_run *run, unsigned int segid) { struct avd_vp9_ctx *vp9_ctx = ctx->priv; - const struct v4l2_vp9_segmentation *seg = &vp9_ctx->cur.seg; + const struct v4l2_vp9_segmentation *seg = &vp9_ctx->b->cur.seg; u32 feature_val = 0; int feature_id = 0; @@ -381,13 +417,15 @@ static void set_header(struct avd_ctx *ctx, struct avd_vp9_run *run) push(0, ""); push(0, ""); - pusha(vp9_ctx->bufs.counts.addr, "counts", 0); - pusha(vp9_ctx->bufs.probs.addr, "probs", 0); - pusha(vp9_ctx->bufs.above_info.addr, "above_info", 0); - pusha(vp9_ctx->bufs.seg.addr, "seg", 1); - pusha(vp9_ctx->bufs.seg.addr, "seg", 2); - pusha(vp9_ctx->bufs.color.addr, "color", 3); - pusha(vp9_ctx->bufs.color.addr, "color", 4); + pusha(vp9_ctx->b->bufs.counts.addr, "counts", 0); + pusha(vp9_ctx->b->bufs.probs.addr, "probs", 0); + pusha(vp9_ctx->b->bufs.above_info.addr, "above_info", 0); + pusha(vp9_ctx->b->bufs.seg.addr, "seg", 1); + pusha(vp9_ctx->b->bufs.seg.addr, "seg", 2); + pusha(vp9_ctx->b->bufs.color.addr, "color", 3); + pusha(avd_vp9_uses_prev_mv(frame->flags, &vp9_ctx->last) ? + vp9_ctx->last.color : vp9_ctx->b->bufs.color.addr, + "color", 4); push(VP9_Q_IDX(frame->quant.base_q_idx) | VP9_Q_DC_Y(frame->quant.delta_q_y_dc) | @@ -419,15 +457,15 @@ static void set_header(struct avd_ctx *ctx, struct avd_vp9_run *run) if (avd->variant->revision == 3) push(0, ""); if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) - pusha(vp9_ctx->bufs.pipe_state.addr, "pipe_state", 0); + pusha(vp9_ctx->b->bufs.pipe_state.addr, "pipe_state", 0); - pusha(vp9_ctx->bufs.ip_above.addr, "ip_above", 0); - pusha(vp9_ctx->bufs.lf_above.addr, "lf_above", 0); + pusha(vp9_ctx->b->bufs.ip_above.addr, "ip_above", 0); + pusha(vp9_ctx->b->bufs.lf_above.addr, "lf_above", 0); /* no lf_above_info? */ pusha((u64)0, "hdr_e8_sps0_tile_addr_lsb8", 0); - pusha(vp9_ctx->bufs.lf_left.addr, "lf_left", 0); - pusha(vp9_ctx->bufs.lf_left_info.addr, "lf_left_info", 0); - pusha(vp9_ctx->bufs.az_left.addr, "az_left", 0); + pusha(vp9_ctx->b->bufs.lf_left.addr, "lf_left", 0); + pusha(vp9_ctx->b->bufs.lf_left_info.addr, "lf_left_info", 0); + pusha(vp9_ctx->b->bufs.az_left.addr, "az_left", 0); push(0, ""); @@ -507,7 +545,7 @@ static void set_tiles(struct avd_ctx *ctx, struct avd_vp9_run *run) offset += tile_size; size -= tile_size; - vp9_ctx->submit_num++; + vp9_ctx->b->submit_num++; } } @@ -523,18 +561,19 @@ static void update_ctx_cur_info(struct avd_vp9_ctx *vp9_ctx, struct avd_decoded_buffer *buf, const struct v4l2_ctrl_vp9_frame *dec_params) { - vp9_ctx->cur.valid = true; - vp9_ctx->cur.reference_mode = dec_params->reference_mode; - vp9_ctx->cur.interpolation_filter = dec_params->interpolation_filter; - vp9_ctx->cur.flags = dec_params->flags; - vp9_ctx->cur.timestamp = buf->base.vb.vb2_buf.timestamp; - vp9_ctx->cur.seg = dec_params->seg; - vp9_ctx->cur.lf = dec_params->lf; + vp9_ctx->b->cur.valid = true; + vp9_ctx->b->cur.reference_mode = dec_params->reference_mode; + vp9_ctx->b->cur.interpolation_filter = dec_params->interpolation_filter; + vp9_ctx->b->cur.flags = dec_params->flags; + vp9_ctx->b->cur.timestamp = buf->base.vb.vb2_buf.timestamp; + vp9_ctx->b->cur.color = vp9_ctx->b->bufs.color.addr; + vp9_ctx->b->cur.seg = dec_params->seg; + vp9_ctx->b->cur.lf = dec_params->lf; } static void update_ctx_last_info(struct avd_vp9_ctx *vp9_ctx) { - vp9_ctx->last = vp9_ctx->cur; + vp9_ctx->last = vp9_ctx->b->cur; } static void copy_vp9_frame_mv(struct avd_vp9_probs *avd_probs, @@ -563,14 +602,14 @@ static void init_probs(struct avd_ctx *ctx, const struct avd_vp9_run *run) { const struct v4l2_ctrl_vp9_frame *dec_params; struct avd_vp9_ctx *vp9_ctx = ctx->priv; - struct avd_vp9_probs *avd_probs = vp9_ctx->bufs.probs.cpu; + struct avd_vp9_probs *avd_probs = vp9_ctx->b->bufs.probs.cpu; const struct v4l2_vp9_segmentation *seg; const struct v4l2_vp9_frame_context *probs; bool intra_only; int count = 0; dec_params = run->decode_params; - probs = &vp9_ctx->probability_tables; + probs = &vp9_ctx->b->probability_tables; seg = &dec_params->seg; memset(avd_probs, 0, sizeof(*avd_probs)); @@ -651,10 +690,9 @@ static int validate_dec_params(struct avd_ctx *ctx, return 0; } -static int avd_vp9_alloc_bufs(struct avd_ctx *ctx) +static int avd_vp9_alloc_bufs(struct avd_ctx *ctx, struct avd_vp9_bank *bank) { struct avd_dev *avd = ctx->dev; - struct avd_vp9_ctx *vp9_ctx = ctx->priv; int ret, w, h, bit_depth; w = fmt_width(ctx); h = fmt_height(ctx); @@ -663,62 +701,62 @@ static int avd_vp9_alloc_bufs(struct avd_ctx *ctx) 10 : 8; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.inst, fifo_size()); + ret = avd_buf_alloc(avd, &bank->bufs.inst, fifo_size()); if (ret) return ret; if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) { - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.pipe_state, 0x200); + ret = avd_buf_alloc(avd, &bank->bufs.pipe_state, 0x200); if (ret) return ret; } - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.probs, + ret = avd_buf_alloc(avd, &bank->bufs.probs, sizeof(struct avd_vp9_probs)); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.counts, + ret = avd_buf_alloc(avd, &bank->bufs.counts, sizeof(struct avd_vp9_frame_symbol_counts)); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.color, + ret = avd_buf_alloc(avd, &bank->bufs.color, DIV_ROUND_UP(w, 16) * DIV_ROUND_UP(h, 64) * 144); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.ip_above, + ret = avd_buf_alloc(avd, &bank->bufs.ip_above, DIV_ROUND_UP(w, 16) * 4 * bit_depth + (VP9_MAX_TILE_COLS - 1) * 128); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.lf_above, + ret = avd_buf_alloc(avd, &bank->bufs.lf_above, DIV_ROUND_UP(w, 8) * 16 * bit_depth); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.seg, DIV_ROUND_UP(w, 8) * 24); + ret = avd_buf_alloc(avd, &bank->bufs.seg, DIV_ROUND_UP(w, 8) * 24); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.lf_left, + ret = avd_buf_alloc(avd, &bank->bufs.lf_left, DIV_ROUND_UP(h, 8) * 16 * bit_depth); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.lf_left_info, + ret = avd_buf_alloc(avd, &bank->bufs.lf_left_info, DIV_ROUND_UP(h, 64) * 16); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.az_left, + ret = avd_buf_alloc(avd, &bank->bufs.az_left, bit_depth * 36 * DIV_ROUND_UP(h, 8) + bit_depth * 18 * DIV_ROUND_UP(h, 16)); if (ret) return ret; - ret = avd_buf_alloc(avd, &vp9_ctx->bufs.above_info, + ret = avd_buf_alloc(avd, &bank->bufs.above_info, DIV_ROUND_UP(w, 64) * 288 + (VP9_MAX_TILE_COLS - 1) * 128); if (ret) @@ -754,18 +792,40 @@ static int avd_vp9_run_preamble(struct avd_ctx *ctx, struct avd_vp9_run *run) run->decode_params = dec_params; - vp9_ctx->cur.tx_mode = run->prob_updates->tx_mode; + if (avd_backend_is_qfw(ctx->dev)) { + struct avd_vp9_bank *bank = NULL; + + for (unsigned int i = 0; i < vp9_ctx->nbanks; i++) + if (!vp9_ctx->bank[i].busy) { + bank = &vp9_ctx->bank[i]; + break; + } + if (!bank) + return -EBUSY; + vp9_ctx->b = bank; + WRITE_ONCE(bank->busy, true); + WRITE_ONCE(bank->updated, false); + bank->last = vp9_ctx->last; + vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf)->bank = + bank - vp9_ctx->bank + 1; + } + + vp9_ctx->b->cur.tx_mode = run->prob_updates->tx_mode; fctx_idx = v4l2_vp9_reset_frame_ctx(dec_params, vp9_ctx->frame_context); - vp9_ctx->cur.frame_context_idx = fctx_idx; + vp9_ctx->b->cur.frame_context_idx = fctx_idx; - vp9_ctx->probability_tables = vp9_ctx->frame_context[fctx_idx]; - v4l2_vp9_fw_update_probs(&vp9_ctx->probability_tables, + vp9_ctx->b->probability_tables = vp9_ctx->frame_context[fctx_idx]; + v4l2_vp9_fw_update_probs(&vp9_ctx->b->probability_tables, run->prob_updates, dec_params); return 0; } +static void avd_vp9_update_probs(struct avd_ctx *ctx, + struct avd_vp9_bank *bank, + enum vb2_buffer_state result); + static int avd_vp9_run(struct avd_ctx *ctx) { struct avd_vp9_run run; @@ -797,10 +857,17 @@ static int avd_vp9_run(struct avd_ctx *ctx) update_ctx_cur_info(vp9_ctx, dst, run.decode_params); set_header(ctx, &run); - vp9_ctx->submit_num = 0; + vp9_ctx->b->submit_num = 0; set_tiles(ctx, &run); avd_run_postamble(ctx, &run.base); + if (avd_backend_is_qfw(ctx->dev)) { + /* Forward updates need no hardware result; publish before next prepare. */ + if (!avd_vp9_needs_counts(vp9_ctx->b->cur.flags)) + avd_vp9_update_probs(ctx, vp9_ctx->b, VB2_BUF_STATE_DONE); + update_ctx_last_info(vp9_ctx); + ctx->job_fifo = vp9_ctx->b->bufs.inst.addr; + } return avd_submit_job(ctx); } @@ -812,16 +879,19 @@ static int avd_vp9_run(struct avd_ctx *ctx) memcpy((p1)->skip, (p2)->skip, sizeof((p1)->skip)); \ } while (0) -static void avd_vp9_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, - struct vb2_v4l2_buffer *dst_buf, - enum vb2_buffer_state result) +static void avd_vp9_update_probs(struct avd_ctx *ctx, + struct avd_vp9_bank *bank, + enum vb2_buffer_state result) { struct avd_vp9_ctx *vp9_ctx = ctx->priv; unsigned int fctx_idx; + if (bank->updated) + return; + /* v4l2-specific stuff */ if (result == VB2_BUF_STATE_ERROR) - goto out_update_last; + goto out; /* * vp9 stuff @@ -832,19 +902,20 @@ static void avd_vp9_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, * but it makes no sense to perform all the activities from the first "if" * there if we actually are not refreshing the frame context. On top of that, * because of 6.2 uncompressed_header() whenever error_resilient_mode == 1, - * refresh_frame_context == 0. Consequently, if we don't jump to out_update_last + * refresh_frame_context == 0. Consequently, if we don't jump to out * it means error_resilient_mode must be 0. */ - if (!(vp9_ctx->cur.flags & V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX)) - goto out_update_last; + if (!(bank->cur.flags & V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX) || + (bank->cur.flags & V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT)) + goto out; - fctx_idx = vp9_ctx->cur.frame_context_idx; + fctx_idx = bank->cur.frame_context_idx; - if (!(vp9_ctx->cur.flags & V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE)) { + if (avd_vp9_needs_counts(bank->cur.flags)) { /* error_resilient_mode == 0 && frame_parallel_decoding_mode == 0 */ struct v4l2_vp9_frame_context *probs = - &vp9_ctx->probability_tables; - bool frame_is_intra = vp9_ctx->cur.flags & + &bank->probability_tables; + bool frame_is_intra = bank->cur.flags & (V4L2_VP9_FRAME_FLAG_KEY_FRAME | V4L2_VP9_FRAME_FLAG_INTRA_ONLY); struct tx_and_skip { @@ -866,12 +937,12 @@ static void avd_vp9_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, if (frame_is_intra) copy_tx_and_skip(probs, tx_skip); - counts = &vp9_ctx->cnts; + counts = &bank->cnts; v4l2_vp9_adapt_coef_probs( probs, counts, - !vp9_ctx->last.valid || - vp9_ctx->last.flags & + !bank->last.valid || + bank->last.flags & V4L2_VP9_FRAME_FLAG_KEY_FRAME, frame_is_intra); if (!frame_is_intra) { @@ -887,7 +958,7 @@ static void avd_vp9_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, u32 class0_hp[2][2]; u32 hp[2][2]; - cnts = vp9_ctx->bufs.counts.cpu; + cnts = bank->bufs.counts.cpu; for (i = 0; i < ARRAY_SIZE(cnts->tx16p); ++i) memcpy(tx16p[i], cnts->tx16p[i], @@ -922,47 +993,81 @@ static void avd_vp9_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, /* load_probs2() already done */ v4l2_vp9_adapt_noncoef_probs( - &vp9_ctx->probability_tables, counts, - vp9_ctx->cur.reference_mode, - vp9_ctx->cur.interpolation_filter, - vp9_ctx->cur.tx_mode, vp9_ctx->cur.flags); + &bank->probability_tables, counts, + bank->cur.reference_mode, + bank->cur.interpolation_filter, + bank->cur.tx_mode, bank->cur.flags); } } /* 6.1.2 refresh_probs(): save_probs(fctx_idx) */ - vp9_ctx->frame_context[fctx_idx] = vp9_ctx->probability_tables; + vp9_ctx->frame_context[fctx_idx] = bank->probability_tables; -out_update_last: - update_ctx_last_info(vp9_ctx); +out: + WRITE_ONCE(bank->updated, true); } -static noinline_for_stack void -avd_init_v4l2_vp9_count_tbl(struct avd_ctx *ctx) + +static void avd_vp9_vp_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst) +{ + struct avd_vp9_ctx *vp9_ctx = ctx->priv; + unsigned int id = vb2_to_avd_decoded_buf(&dst->vb2_buf)->bank; + + if (id && id <= vp9_ctx->nbanks) + avd_vp9_update_probs(ctx, &vp9_ctx->bank[id - 1], VB2_BUF_STATE_DONE); +} + +static void avd_vp9_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src, + struct vb2_v4l2_buffer *dst, enum vb2_buffer_state result) { struct avd_vp9_ctx *vp9_ctx = ctx->priv; - struct avd_vp9_frame_symbol_counts *cnts = vp9_ctx->bufs.counts.cpu; + struct avd_decoded_buffer *buf = vb2_to_avd_decoded_buf(&dst->vb2_buf); + struct avd_vp9_bank *bank; + + if (!buf->bank) { + if (avd_backend_is_qfw(ctx->dev)) + return; + vp9_ctx->b->last = vp9_ctx->last; + WRITE_ONCE(vp9_ctx->b->updated, false); + avd_vp9_update_probs(ctx, vp9_ctx->b, result); + update_ctx_last_info(vp9_ctx); + return; + } + if (buf->bank > vp9_ctx->nbanks) + return; + bank = &vp9_ctx->bank[buf->bank - 1]; + buf->bank = 0; + /* Also covers old queue firmware and preparation/submission failures. */ + avd_vp9_update_probs(ctx, bank, result); + WRITE_ONCE(bank->busy, false); +} + +static noinline_for_stack void +avd_init_v4l2_vp9_count_tbl(struct avd_vp9_bank *bank) +{ + struct avd_vp9_frame_symbol_counts *cnts = bank->bufs.counts.cpu; int coeff_cnts = 0, eob_cnts = 0; - vp9_ctx->cnts.intra_inter = &cnts->intra_inter; - vp9_ctx->cnts.y_mode = &cnts->y_mode; - vp9_ctx->cnts.uv_mode = &cnts->uv_mode; - vp9_ctx->cnts.comp = &cnts->comp; - vp9_ctx->cnts.comp_ref = &cnts->comp_ref; - vp9_ctx->cnts.single_ref = &cnts->single_ref; - vp9_ctx->cnts.filter = &cnts->filter; - vp9_ctx->cnts.mv_mode = &cnts->mv_mode; - vp9_ctx->cnts.mv_joint = &cnts->mv_joint; + bank->cnts.intra_inter = &cnts->intra_inter; + bank->cnts.y_mode = &cnts->y_mode; + bank->cnts.uv_mode = &cnts->uv_mode; + bank->cnts.comp = &cnts->comp; + bank->cnts.comp_ref = &cnts->comp_ref; + bank->cnts.single_ref = &cnts->single_ref; + bank->cnts.filter = &cnts->filter; + bank->cnts.mv_mode = &cnts->mv_mode; + bank->cnts.mv_joint = &cnts->mv_joint; /* all of the mv use a different structure, so they must all be copied */ - vp9_ctx->cnts.tx8p = &cnts->tx8p; + bank->cnts.tx8p = &cnts->tx8p; /* avd also uses "u32 tx16p[2][3]" instead of "u32 tx16p[2][4]" */ - vp9_ctx->cnts.tx32p = &cnts->tx32p; + bank->cnts.tx32p = &cnts->tx32p; /* it would suck if we need to reverse partition */ - vp9_ctx->cnts.partition = &cnts->partition; + bank->cnts.partition = &cnts->partition; /* these should work */ - vp9_ctx->cnts.skip = &cnts->skip; + bank->cnts.skip = &cnts->skip; for (int t = 0; t < 4; t++) for (int i = 0; i < 2; i++) @@ -970,12 +1075,12 @@ avd_init_v4l2_vp9_count_tbl(struct avd_ctx *ctx) for (int k = 0; k < 6; k++) { int max_l = (k == 0) ? 3 : 6; for (int l = 0; l < max_l; l++) { - vp9_ctx->cnts.coeff[t][i][j][k][l] = + bank->cnts.coeff[t][i][j][k][l] = (u32 (*)[3])&cnts->ref_cnt[coeff_cnts]; coeff_cnts += 3; - vp9_ctx->cnts.eob[t][i][j][k][l][0] = + bank->cnts.eob[t][i][j][k][l][0] = &cnts->eob_0[eob_cnts++]; - vp9_ctx->cnts.eob[t][i][j][k][l][1] = + bank->cnts.eob[t][i][j][k][l][1] = &cnts->ref_cnt[coeff_cnts++]; } } @@ -986,18 +1091,23 @@ static void avd_vp9_stop(struct avd_ctx *ctx) struct avd_vp9_ctx *vp9_ctx = ctx->priv; struct avd_dev *avd = ctx->dev; - avd_buf_free(avd, &vp9_ctx->bufs.pipe_state); - avd_buf_free(avd, &vp9_ctx->bufs.inst); - avd_buf_free(avd, &vp9_ctx->bufs.probs); - avd_buf_free(avd, &vp9_ctx->bufs.counts); - avd_buf_free(avd, &vp9_ctx->bufs.seg); - avd_buf_free(avd, &vp9_ctx->bufs.color); - avd_buf_free(avd, &vp9_ctx->bufs.ip_above); - avd_buf_free(avd, &vp9_ctx->bufs.lf_above); - avd_buf_free(avd, &vp9_ctx->bufs.above_info); - avd_buf_free(avd, &vp9_ctx->bufs.az_left); - avd_buf_free(avd, &vp9_ctx->bufs.lf_left_info); - avd_buf_free(avd, &vp9_ctx->bufs.lf_left); + ctx->max_frames = 1; + for (unsigned int i = 0; i < vp9_ctx->nbanks; i++) { + struct avd_vp9_bank *bank = &vp9_ctx->bank[i]; + + avd_buf_free(avd, &bank->bufs.pipe_state); + avd_buf_free(avd, &bank->bufs.inst); + avd_buf_free(avd, &bank->bufs.probs); + avd_buf_free(avd, &bank->bufs.counts); + avd_buf_free(avd, &bank->bufs.seg); + avd_buf_free(avd, &bank->bufs.color); + avd_buf_free(avd, &bank->bufs.ip_above); + avd_buf_free(avd, &bank->bufs.lf_above); + avd_buf_free(avd, &bank->bufs.above_info); + avd_buf_free(avd, &bank->bufs.az_left); + avd_buf_free(avd, &bank->bufs.lf_left_info); + avd_buf_free(avd, &bank->bufs.lf_left); + } kfree(vp9_ctx); ctx->priv = NULL; @@ -1013,11 +1123,15 @@ static int avd_vp9_start(struct avd_ctx *ctx) return -ENOMEM; ctx->priv = vp9_ctx; - ret = avd_vp9_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; - - avd_init_v4l2_vp9_count_tbl(ctx); + vp9_ctx->nbanks = avd_qfw_has_vp_done(ctx->dev) ? AVD_VP9_BANKS : 1; + vp9_ctx->b = &vp9_ctx->bank[0]; + for (unsigned int i = 0; i < vp9_ctx->nbanks; i++) { + ret = avd_vp9_alloc_bufs(ctx, &vp9_ctx->bank[i]); + if (ret) + goto err_free_ctx; + avd_init_v4l2_vp9_count_tbl(&vp9_ctx->bank[i]); + } + ctx->max_frames = vp9_ctx->nbanks; return 0; @@ -1053,7 +1167,7 @@ static unsigned int avd_vp9_pp_kicks(struct avd_ctx *ctx) { struct avd_vp9_ctx *priv = ctx->priv; - return max_t(int, priv->submit_num, 1); + return max_t(int, priv->b->submit_num, 1); } static void avd_vp9_submit(struct avd_ctx *ctx) @@ -1072,7 +1186,7 @@ static void avd_vp9_submit(struct avd_ctx *ctx) AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx) | AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots), avd->ctrl + avd->variant->submit_offset); - for (int i = 0; i < vp9_ctx->submit_num - 1; i++) + for (int i = 0; i < vp9_ctx->b->submit_num - 1; i++) writel(AVD_OP_EXEC | submit_mask | AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx) | AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots), @@ -1084,6 +1198,8 @@ const struct avd_coded_fmt_ops avd_vp9_fmt_ops = { .stop = avd_vp9_stop, .run = avd_vp9_run, .done = avd_vp9_done, + .vp_done = avd_vp9_vp_done, + .ready = avd_vp9_ready, .submit = avd_vp9_submit, .pp_kicks = avd_vp9_pp_kicks, .get_image_fmt = avd_vp9_get_image_fmt, diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 6af972518f84e..2da75d833b75a 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -128,7 +128,7 @@ struct avd_decoded_buffer { /* Pending CPU fill, retained with the buffer until dequeue. */ size_t grey_chroma_offset; bool grey_chroma_10bit; - /* H.264/HEVC: scratch bank + 1 of the picture decoded into this buffer */ + /* Codec scratch bank + 1 of the picture decoded into this buffer */ u8 bank; union { @@ -159,6 +159,10 @@ struct avd_coded_fmt_ops { void (*done)(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, struct vb2_v4l2_buffer *dst_buf, enum vb2_buffer_state result); + /* Called under queue_lock: CPU preparation may read prior VP results. */ + bool (*ready)(struct avd_ctx *ctx); + /* queue_mutex held, after a tagged VP DONE; DMA banks remain owned. */ + void (*vp_done)(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst); int (*try_ctrl)(struct avd_ctx *ctx, struct v4l2_ctrl *ctrl); enum avd_image_fmt (*get_image_fmt)(struct avd_ctx *ctx, struct v4l2_ctrl *ctrl); @@ -406,6 +410,7 @@ void avd_qfw_select(struct avd_dev *avd); void avd_qfw_prepare_boot(struct avd_dev *avd); void avd_qfw_debug(struct avd_dev *avd, struct avd_frame *f, const char *where); bool avd_backend_is_qfw(const struct avd_dev *avd); +bool avd_qfw_has_vp_done(const struct avd_dev *avd); int avd_backend_select(struct avd_dev *avd); bool avd_mbox_pop(struct avd_dev *avd, u32 *msg); int avd_backend_boot(struct avd_dev *avd); diff --git a/drivers/media/platform/apple/avd/tests/README.rst b/drivers/media/platform/apple/avd/tests/README.rst index 2a6f378fa31bf..4f485a2d84135 100644 --- a/drivers/media/platform/apple/avd/tests/README.rst +++ b/drivers/media/platform/apple/avd/tests/README.rst @@ -117,3 +117,119 @@ The live-VP assertion is a conservative decoder ownership model. It proves the software feed ordering and its correction, not the undocumented silicon cause of a DART fault. A hardware retest remains necessary to connect this scheduling defect to a reported zero-address DMA and to verify pixels and performance. + +VP9 probability dependencies and picture banks +--------------------------------------------- + +``vp9-test.py`` compiles the production VP9 codec and V4L2 probability helpers. +It exercises all combinations of the three dependency flags, real backward +adaptation with synthetic counts, independent forward updates, per-bank count +pointers, bank exhaustion, PP retirement while another bank is current, and +old-firmware fallback. It also emits two VP9 programs and their production queue +descriptors/configure_stream snapshots for ``unicorn-vp9-test.py``:: + + python3 tests/vp9-test.py /tmp/avd-vp9 + python3 tests/unicorn-vp9-test.py /path/to/avd-fw \ + /path/to/avd-fw/build/avd-fw-v3-t1.bin /tmp/avd-vp9 + +For the independent legacy comparison, run in a kernel Git checkout containing +the public pre-change revision:: + + AVD_VP9_BASELINE_REF= python3 tests/vp9-test.py /tmp/avd-vp9 + +This builds separate before/after executables and compares the complete legacy +programs and saved probability contexts, including key, inter, frame-parallel, +non-refreshing and error-resilient frames. No firmware capture is used. + +The flags below describe whether a picture *produces* a backward-adaptation +dependency for the next prepare. A non-refreshing successor still consumes the +saved context of an earlier refreshing picture. No CPU can read that context +before that producer's counts have been adapted. + +================= ============= =============== ============================ +refresh_frame_ctx parallel_mode error_resilient Release next prepare +================= ============= =============== ============================ +0 any any During prepare +1 1 any During prepare (forward only) +1 0 1 During prepare (no refresh) +1 0 0 After tagged VP DONE/adaptation +================= ============= =============== ============================ + +Preparation, VP callbacks and PP retirement run under queue_mutex. Readiness +checks only observe the published dependency state under queue_lock. Counts +come from coherent allocations and the VP callback uses dma_rmb() before reading +them. Each bank has its own count-pointer table, forward probabilities, header +snapshot, previous-frame flags, FIFO and all DMA scratch. Only PP retirement +releases a bank, and it selects the bank saved with CAPTURE rather than the +context's current builder. A duplicate VP event cannot adapt twice. Failure and +reset retirement never adapt partial counts. + +The temporal-MV input color address follows the preceding picture across banks; +the current color output belongs to the new bank. The public MIT VP9 builder at +https://github.com/eiln/avd/blob/main/avid/vp9/halv3.py alternates these operands +(``pps2_tile_addrs``); the single-bank legacy driver used one address for both. +A stream using old queue firmware retains one bank and adapts after full +completion. The ordinary legacy transport and its instruction builder are +unchanged. + +The Unicorn test boots the real reset vector and routes VP8/PP IRQs. It checks +that VP(1) is issued while PP(0) owns its FIFO, and that PP(1) waits for PP(0). +It repeats with a blocked VP notification and holds the outbox across PP DONE. +The original H.264 routed tests exercise descriptors without the optional flag +and must retain their single completion per picture. + +Optional queue ABI 2 VP notifications +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Apply ``avd-fw-vp-done.patch`` to the public MIT ``aurora-silicon/avd-fw`` +revision ``a112b70d4b1e437e4ce9c0353b2a0e805ebf41e5`` and rebuild its images. +This is a backwards-compatible optional ABI 2 feature: + +* Info feature bit 2 advertises tagged VP DONE. Without it the host keeps VP9 + at one picture per stream. Features 0/1 and the ordinary completion stay as + before. +* Host descriptor flag bit 1 requests one VP event. An old host leaves it zero + and receives no additional tokens. Reserved words must still start at zero. +* After VP DONE, firmware writes the full submitted tag at descriptor +0x38, + DMB, writes 1 at +0x3c, DMB, then sends + ``0x54540000 | (tag & 0xfff0) | index``. These records live independently of + the PP completion_tag. The host validates the prefix, marker, short sequence, + full saved tag and submitted ownership before recording the event. +* Outbox backpressure retains the VP event. A successful opted-in picture's PP + completion cannot be notified before its VP event. Errors may finish without + a VP event and require the existing reset/ownership handling. +* Neither the VP token nor its CPU callback permits buffer, descriptor, FIFO or + bank reuse. Only the original tagged PP completion does that. A missing VP + event remains bounded by the original per-picture watchdog. + +AV1 analysis +~~~~~~~~~~~~ + +AV1's current ``avd_av1_set_prob()`` reads the primary reference's CDF tail on +the CPU, and seeds the current frame's output CDF because hardware leaves it +untouched when DISABLE_FRAME_END_UPDATE_CDF is set. ``avd_av1_set_seg()`` can +also copy the primary reference's segmentation map on the CPU. These are +separate dependencies; removing only the CDF wait is insufficient. + +=================================== ========================================= +Case CDF source dependency +=================================== ========================================= +Intra/error-resilient/PRIMARY_NONE Defaults; no primary-reference CDF read +Primary ref with frame-end disabled Its initialized CDF, available at prepare +Primary ref with frame-end enabled Its hardware-written CDF tail +=================================== ========================================= + +``disable_frame_end_update_cdf`` on the *consumer* does not remove its primary +reference input dependency. It controls that consumer's future CDF output. +Likewise, a default CDF does not necessarily remove the segmentation-map input +selected by primary_ref_frame. + +The available implementation does not establish whether the CDF tail *and* +segmentation map are fully visible at VP DONE, or which CPU DMA synchronization +is required for an in-flight CAPTURE allocation while PP writes another part. +This differs from VP9's separate coherent count buffer. AV1 therefore retains +its one-picture path. Enabling it needs evidence for both tail writes and a +safe visibility/synchronization rule, then two scratch banks and a dependency +on the actual primary reference (which need not be the preceding picture). +No AV1 or VP9 pixel correctness or throughput claim follows from these host +and emulator tests; device qualification remains necessary. diff --git a/drivers/media/platform/apple/avd/tests/avd-fw-vp-done.patch b/drivers/media/platform/apple/avd/tests/avd-fw-vp-done.patch new file mode 100644 index 0000000000000..c35e38c5ed0c4 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/avd-fw-vp-done.patch @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: MIT +# Optional tagged VP DONE on avd-fw a112b70d4b1e437e4ce9c0353b2a0e805ebf41e5 +--- a/src/queue.h ++++ b/src/queue.h +@@ -24,6 +24,9 @@ + */ + #define AVDQ_FLAG_SRAM 1u + #define AVDQ_FEAT_SRAM_RING 1u ++#define AVDQ_FEAT_VP_DONE (1u << 2) ++#define AVDQ_FLAG_VP_DONE (1u << 1) ++#define AVDQ_VP_NOTIFY 0x54540000u + #define AVDQ_FEAT_MULTI_KICK 2u + #define AVDQ_SRAM_RING_OFFSET 0x0u + #define AVDQ_SRAM_RING_WORDS 0x1800u +@@ -78,7 +81,7 @@ + AVDQ_PP_READY, AVDQ_PP, AVDQ_DONE }; + struct avdq_frame { + struct avdq_desc desc; +- uint32_t phase, pushed, status; ++ uint32_t phase, pushed, status, vp_notified; + }; + struct avdq { + struct avdq_frame frame[AVDQ_COUNT]; +--- a/src/queue.c ++++ b/src/queue.c +@@ -21,7 +21,7 @@ + d->state != AVDQ_SUBMITTED || d->codec > AVDQ_AV1 || + !(codec_mask & (1u << d->codec)) || d->vp_slot >= vp_count || + !d->program_words || d->program_words > AVDQ_MAX_WORDS || +- !d->pp_kick || (d->flags & ~AVDQ_FLAG_SRAM) || ++ !d->pp_kick || (d->flags & ~(AVDQ_FLAG_SRAM | AVDQ_FLAG_VP_DONE)) || + d->pp_extra > AVDQ_MAX_PP_EXTRA || (d->pp_extra && !d->pp_cont)) + return 0; + if (d->flags & AVDQ_FLAG_SRAM) { +@@ -49,6 +49,7 @@ + f = &q->frame[idx]; + f->desc = *d; + f->pushed = 0; ++ f->vp_notified = 0; + f->status = valid(d, idx, vp_count, codec_mask) ? AVDQ_OK : AVDQ_INVALID; + f->phase = f->status ? AVDQ_DONE : AVDQ_PENDING; + q->order[(q->head + q->count) & (AVDQ_COUNT - 1)] = idx; +--- a/src/queue-fw.c ++++ b/src/queue-fw.c +@@ -99,7 +99,8 @@ + Q_INFO[8] = VP_COUNT; + Q_INFO[9] = AVD_VER <= 3 ? 7u : 15u; + Q_INFO[10] = FIFO_SLOTS; +- Q_INFO[11] = AVDQ_FEAT_SRAM_RING | AVDQ_FEAT_MULTI_KICK; ++ Q_INFO[11] = AVDQ_FEAT_SRAM_RING | AVDQ_FEAT_MULTI_KICK | ++ AVDQ_FEAT_VP_DONE; + Q_INFO[12] = AVDQ_SRAM_RING_OFFSET; + Q_INFO[13] = AVDQ_SRAM_RING_WORDS; + barrier(); +@@ -266,6 +267,32 @@ + reg_write(REG(DECODE_CTRL_BASE + PP_FIFO), d->pp_cont); + } + ++/* VP records do not release any descriptor or DMA storage. */ ++static void notify_vp(void) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < FW->q.count; i++) { ++ unsigned int idx = FW->q.order[(FW->q.head + i) & 15u]; ++ struct avdq_frame *f = &FW->q.frame[idx]; ++ volatile struct avdq_desc *d = Q_DESC(idx); ++ ++ if (!(f->desc.flags & AVDQ_FLAG_VP_DONE) || f->vp_notified || ++ f->status || (f->phase != AVDQ_PP_READY && ++ f->phase != AVDQ_PP && f->phase != AVDQ_DONE)) ++ continue; ++ if (!(reg_read(CM3_MBOX1_STATUS) & CM3_MBOX_STATUS_EMPTY)) ++ return; ++ d->reserved[0] = f->desc.tag; ++ barrier(); ++ d->reserved[1] = 1; ++ barrier(); ++ reg_write(CM3_MBOX1_TX, AVDQ_VP_NOTIFY | (f->desc.tag & 0xfff0u) | idx); ++ f->vp_notified = 1; ++ return; ++ } ++} ++ + static int complete(void *cookie, unsigned int idx, const struct avdq_frame *f) + { + volatile struct avdq_desc *d = Q_DESC(idx); +@@ -323,6 +350,7 @@ + /* Queue mode owns PIODMA; the Tansy transport cannot run concurrently. */ + reg_write(Q_DMA(0x00), reg_read(Q_DMA(0x00)) & ~1u); + } ++ notify_vp(); + avdq_step(&FW->q, &ops, FW); + if (FW->fatal && !FW->q.count && + (reg_read(CM3_MBOX1_STATUS) & CM3_MBOX_STATUS_EMPTY)) { +--- a/tests/queue-test.c ++++ b/tests/queue-test.c +@@ -105,7 +105,7 @@ + case 10: d.program_lo = 0xfffffffcu; d.program_hi = 0xffffffffu; break; + case 11: d.pp_kick = 0; break; + case 12: d.reserved[1] = 1; break; +- case 13: d.flags = 2; break; ++ case 13: d.flags = 4; break; + case 14: d.pp_extra = AVDQ_MAX_PP_EXTRA + 1; d.pp_cont = 1; break; + case 15: d.pp_extra = 1; break; /* continuation word missing */ + case 16: d.flags = AVDQ_FLAG_SRAM; break; /* program_hi must be 0 */ +--- a/tests/transport-test.py ++++ b/tests/transport-test.py +@@ -50,7 +50,7 @@ + self.call("avdq_fw_init") + self.info = [self.u32(0x10007fc0 + i * 4) for i in range(16)] + assert self.info[:4] == [0x41565131, 2, 16, 64] +- assert self.info[11:16] == [3, 0, 0x1800, 0, 0] ++ assert self.info[11:16] == [7, 0, 0x1800, 0, 0] + expected = 0x0108c000 if self.version == 2 else 0x01090000 if self.version == 3 or (self.version == 5 and self.tier == 0) else 0x01094000 if (self.version == 5 and self.tier == 2) else 0x01092000 + assert self.info[4] == expected and self.info[8] == self.vps + assert self.info[10] == (7 if self.version == 2 or (self.version == 5 and self.tier == 0) else 15) diff --git a/drivers/media/platform/apple/avd/tests/dispatch-test.py b/drivers/media/platform/apple/avd/tests/dispatch-test.py index a36b34573b5ab..055f0d544bcae 100755 --- a/drivers/media/platform/apple/avd/tests/dispatch-test.py +++ b/drivers/media/platform/apple/avd/tests/dispatch-test.py @@ -41,10 +41,12 @@ #define AVD_PROGRAM_SIZE 4096 #define GFP_KERNEL 0 #define dma_wmb() ((void)0) +#define dma_rmb() ((void)0) #define system_dfl_wq NULL #define time_before(a,b) ((long)((a)-(b))<0) #define WRITE_ONCE(a,b) ((a)=(b)) #define READ_ONCE(v) (v) +#define max(a,b) ((a)>(b)?(a):(b)) static bool warn_on(bool v) { assert(!v);return v; } #define WARN_ON(v) warn_on(!!(v)) #define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) @@ -79,6 +81,8 @@ void (*retire)(struct avd_ctx *,struct avd_frame *,enum vb2_buffer_state); }; struct avd_coded_fmt_ops { + bool (*ready)(struct avd_ctx *); + void (*vp_done)(struct avd_ctx *,struct vb2_v4l2_buffer *); int (*run)(struct avd_ctx *); void (*done)(struct avd_ctx *,struct vb2_v4l2_buffer *,struct vb2_v4l2_buffer *,enum vb2_buffer_state); }; @@ -86,7 +90,7 @@ struct avd_ctx { struct avd_dev *dev; struct { struct fake_m2m *m2m_ctx; } fh; const struct format *coded_fmt_desc; struct work_struct submit_work; - bool draining,broken; unsigned int hw_errors; enum avd_frame_dep next_dep; + bool draining,broken; unsigned int hw_errors,max_frames; enum avd_frame_dep next_dep; bool session_started; u64 session_generation; int ctrl_hdl; }; @@ -263,6 +267,35 @@ } ''' +VP_MAIN = r''' +static unsigned int vp_calls; +static bool dependency_ready; +static bool fake_ready(struct avd_ctx *c) { (void)c;return dependency_ready; } +static void vp_callback(struct avd_ctx *c,struct vb2_v4l2_buffer *dst) { + (void)c;assert(dst&&!dst->done);vp_calls++;dependency_ready=true; +} +static void check_vp_events(void) { + struct avd_dev d={};struct avd_ctx c={.dev=&d,.next_dep=AVD_DEP_PREV_DONE,.max_frames=2}; + struct vb2_v4l2_buffer dst={}; + const struct avd_coded_fmt_ops ops={.ready=fake_ready,.vp_done=vp_callback}; + const struct format fmt={.ops=&ops};c.coded_fmt_desc=&fmt; + struct avd_frame *f=&d.frames[0];f->idx=0; + assert(!avd_slot_reserve(f,&c,1));f->state=AVD_FRAME_SUBMITTED;f->dst=&dst; + assert(!avd_queue_ready(&c)); + avd_frame_vp_complete(&d,0x10010);assert(!f->vp_done&&!vp_calls); + avd_frame_vp_complete(&d,0x10);assert(f->vp_done&&!vp_calls&&!f->vp_retired); + assert(!avd_queue_ready(&c));avd_retire_vp(&d); + assert(vp_calls==1&&f->vp_retired&&avd_queue_ready(&c)); + assert(f->state==AVD_FRAME_SUBMITTED&&f->dst==&dst&&!dst.done); + avd_frame_vp_complete(&d,0x10);avd_retire_vp(&d);assert(vp_calls==1); + f->state=AVD_FRAME_FREE;assert(!avd_slot_reserve(f,&c,2));assert(!f->vp_done&&!f->vp_retired); + f->state=AVD_FRAME_SUBMITTED;avd_frame_vp_complete(&d,0x10);assert(!f->vp_done); + avd_frame_vp_complete(&d,0x20);f->status=-EIO;f->state=AVD_FRAME_DONE; + avd_retire_vp(&d);assert(vp_calls==1&&f->vp_retired); + puts("PASS: production VP event: stale/duplicate/reset tags, callback dependency and retained PP ownership"); +} +''' + def main(): out = Path(sys.argv[1]).resolve() @@ -281,7 +314,9 @@ def main(): function(CORE.read_text(), 'v4l2_m2m_buf_done_and_job_finish') + function(legacy, 'avd_job_finish_no_pm') + function(legacy, 'avd_job_finish') + function(source, 'avd_send_frame') + - function(source, 'avd_retire') + function(source, 'avd_queue_submit_work') + MAIN) + function(source, 'avd_retire') + function(source, 'avd_queue_submit_work') + + function(source, 'avd_frame_vp_complete') + function(source, 'avd_retire_vp') + + function(source, 'avd_queue_ready') + VP_MAIN + MAIN.replace(' puts("PASS: production dispatch', ' check_vp_events();\n puts("PASS: production dispatch')) exe = out / 'dispatch-test' subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-Wall', '-Wextra', '-Werror', '-fsanitize=address,undefined', '-g', '-I' + str(HERE / 'include'), diff --git a/drivers/media/platform/apple/avd/tests/program-test.py b/drivers/media/platform/apple/avd/tests/program-test.py index 89613f1665a92..ad5d2ec009d95 100755 --- a/drivers/media/platform/apple/avd/tests/program-test.py +++ b/drivers/media/platform/apple/avd/tests/program-test.py @@ -58,6 +58,8 @@ #define IRQ_HANDLED 1 #define system_dfl_wq NULL typedef int irqreturn_t; +#define dev_info_once(d,...) ((void)(d)) +#define dev_warn_once(d,...) ((void)(d)) #define dev_warn_ratelimited(d,...) ((void)(d)) #define __iomem #define USEC_PER_MSEC 1000 @@ -98,7 +100,8 @@ struct avd_comp { size_t start_offset; u32 offsets[4]; }; struct avd_decoded_buffer { struct v4l2_m2m_buffer base; struct avd_comp comp; unsigned int bank; }; struct avd_ctx; -struct avd_coded_fmt_ops { unsigned int (*pp_kicks)(struct avd_ctx *); }; +struct avd_coded_fmt_ops { unsigned int (*pp_kicks)(struct avd_ctx *); + void (*vp_done)(struct avd_ctx *,struct vb2_v4l2_buffer *); }; struct fmt_desc { void *frame_ops; const struct avd_coded_fmt_ops *ops; }; struct avd_ctx { struct avd_dev *dev; void *priv; struct avd_job job; unsigned int image_fmt; const struct fmt_desc *coded_fmt_desc; @@ -138,6 +141,8 @@ static bool avd_mbox_pop(struct avd_dev *d,u32 *word) { (void)d;if(msg_read==msg_count)return false;*word=messages[msg_read++];return true; } +static unsigned int vp_completed; +static void avd_frame_vp_complete(struct avd_dev *d,u32 tag) { assert(d->frames[tag&15].tag==tag);vp_completed++; } static void avd_frame_complete(struct avd_dev *d,u32 tag,int status) { struct avd_frame *f=&d->frames[tag&15];assert(f->tag==tag&&f->state==AVD_FRAME_SUBMITTED); f->state=AVD_FRAME_DONE;f->status=status;completed++; @@ -162,6 +167,12 @@ static const struct fmt_desc fmt={.ops=&ops}; dev.qfw=&q;dev.ctrl=calloc(1,0x10000);dev.mbox=calloc(1,0x100);dev.capabilities=7; ctx.coded_fmt_desc=&fmt;frame.ctx=&ctx;ctx.fh.m2m_ctx=&m2m; + /* Optional VP notifications are enabled only by the matching feature bit. */ + u32 info[16]={AVDQ_MAGIC,2,16,64,0x1090000,0x8000,256,0,9,7,15,3,0,0x1800,0,0}; + memcpy(q.sram+AVDQ_INFO,info,sizeof(info));assert(avd_qfw_check(&dev)&&!q.vp_done); + info[11]=7;memcpy(q.sram+AVDQ_INFO,info,sizeof(info));assert(avd_qfw_check(&dev)&&q.vp_done); + info[1]=3;memcpy(q.sram+AVDQ_INFO,info,sizeof(info));assert(!avd_qfw_check(&dev)&&!q.vp_done); + memset(q.sram,0,0x10000); struct avd_buf slice = { .size=4096, .addr=0x200800000ULL }; struct v4l2_ctrl_h264_sps sps = { .pic_width_in_mbs_minus1=119, .pic_height_in_map_units_minus1=67, .chroma_format_idc=1, @@ -307,6 +318,13 @@ for(unsigned int idx=0;idxinst.cpu);free(buf->pipe_state.cpu);free(buf->above_info.cpu); @@ -342,7 +360,7 @@ def main(): inst = inst.replace('u32 offsets[4])\n{', 'u32 offsets[4])\n{\n assert(addr);') inst = inst.replace('push(ctx, (addr + offsets[i]) >> 7);', '{ record_dma(ctx,addr+offsets[i],7,1); push(ctx, (addr + offsets[i]) >> 7); }') code.write_text(PRELUDE + inst + h264 + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_alloc_bufs') + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_drop_bank_slices') + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_pick_bank') + builder + wire + regs + config + - function(qfw, 'avd_qfw_stream') + function(qfw, 'avd_qfw_submit') + + function(qfw, 'avd_qfw_check') + function(qfw, 'avd_qfw_stream') + function(qfw, 'avd_qfw_submit') + function(qfw, 'avd_qfw_fail_all') + function(qfw, 'avd_qfw_irq') + MAIN) exe = out / 'program-test' subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-Wall', '-Wextra', diff --git a/drivers/media/platform/apple/avd/tests/unicorn-vp9-test.py b/drivers/media/platform/apple/avd/tests/unicorn-vp9-test.py new file mode 100644 index 0000000000000..72234b3d5b1a9 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/unicorn-vp9-test.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""VP9 production programs: VP(N+1) starts after VP(N) DONE, during PP(N). + +Usage: unicorn-vp9-test.py FIRMWARE_TREE v3-t1_IMAGE VP9_HOST_OUTPUT +The symbol counts are a host fixture. CM3 code, descriptors, program words, +PP commands and configure_stream writes are production. No pixels are decoded. +""" +import importlib.util +from pathlib import Path +import struct +import sys +from unicorn.arm_const import UC_ARM_REG_SP + +fw, image, programs = map(lambda p: Path(p).resolve(), sys.argv[1:4]) +spec = importlib.util.spec_from_file_location('transport', fw / 'tests/transport-test.py') +transport = importlib.util.module_from_spec(spec) +spec.loader.exec_module(transport) + + +class VP9(transport.Transport): + def __init__(self): + self.booting = False + self.expected = [] + self.events = [] + self.live = self.pp_owner = None + self.feed = self.word = 0 + super().__init__(image) + assert self.version == 3 and self.tier == 1 + self.booting = True + self.uc.reg_write(UC_ARM_REG_SP, self.u32(0)) + self.uc.emu_start(self.u32(4), 0, count=500000) + assert not self.booting + + def route(self, irq): + if irq < 14: + router = self.u32(0x50010010) & (1 << irq) + else: + off = irq - 14 + router = self.u32(0x50010014 + off // 32 * 4) & (1 << (off % 32)) + nvic = self.u32(0xe000e100 + irq // 32 * 4) & (1 << (irq % 32)) + assert router and nvic, f'IRQ {irq} masked' + self.call(f'irq{irq}') + + def write(self, uc, access, addr, size, value, user): + if self.booting and addr == 0x50010090 and value == 1: + self.booting = False + uc.emu_stop() + if addr == self.decode + self.fifo + 8 * 4: + idx, words = self.expected[self.feed] + assert value == words[self.word], (idx, self.word, value) + if not self.word: + assert self.live is None, 'successor issued before VP DONE' + self.live = idx + self.events.append(f'VP9 VP START {idx}; PP owner={self.pp_owner}') + self.word += 1 + if self.word == len(words): + self.feed += 1 + self.word = 0 + if addr == self.decode + self.pp: + idx = (value >> 4) & 15 + assert self.pp_owner is None, 'successor PP issued before predecessor PP DONE' + assert value == 0x2b00020f | idx << 4 + self.pp_owner = idx + self.events.append(f'PP START {idx}') + if addr == 0x50010060: + self.events.append(f'notification {value:#x}') + super().write(uc, access, addr, size, value, user) + + def submit_host(self, idx, start): + stem = f'vp9-adaptive-{idx}' + data = (programs / f'{stem}.bin').read_bytes() + words = list(struct.unpack('<' + 'I' * (len(data) // 4), data)) + desc = list(struct.unpack('<16I', (programs / f'{stem}-desc.bin').read_bytes())) + assert desc[:4] == [0x41565131, 0x10 | idx, 2, 8] + assert desc[4:6] == [0, 0] and desc[6] == len(words) + assert desc[8:16] == [1, 0, 0, 3, 0, 0, 0, 0] + assert words[0] == 0x2b000200 | idx << 4 + assert words[-1] == 0x2b000400 | idx << 4 + mmio = (programs / f'{stem}-mmio.bin').read_bytes() + for offset in (0x150, 0x18c, 0x1c8, 0x204): + at = offset + idx * 4 + self.put(self.decode + at, struct.unpack_from(' 2 else HERE.parent +KERNEL = Path(sys.argv[3]).resolve() if len(sys.argv) > 3 else AVD.parents[4] +OUT = Path(sys.argv[1]).resolve() +OUT.mkdir(parents=True, exist_ok=True) + +# Share only the explicit MMIO/allocation boundary with the H.264 tests. +spec = importlib.util.spec_from_file_location('program_fixture', HERE / 'program-test.py') +fixture = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fixture) +pre = fixture.PRELUDE +pre = pre.replace('typedef uint8_t u8;', 'typedef uint8_t u8; typedef uint16_t u16;') +pre = pre.replace('#define AVD_MAX_INST 512', '#define AVD_MAX_INST 512\n#define AVD_CODEC_HEVC 0') +pre = pre.replace('struct vb2_buffer { struct { size_t length; } planes[1]; dma_addr_t dma; };', + 'struct vb2_buffer { struct { size_t length; } planes[1]; dma_addr_t dma; void *cpu; size_t payload; u64 timestamp; };') +pre = pre.replace('struct avd_comp comp; unsigned int bank;', 'struct avd_comp comp; unsigned int bank; struct { unsigned int width,height,bit_depth; } vp9;') +pre = pre.replace('struct vb2_v4l2_buffer *dst; } bufs;', 'struct vb2_v4l2_buffer *src,*dst; } bufs;') +pre = pre.replace('unsigned int fifo_idx, pp_total;', 'unsigned int fifo_idx, pp_total, max_frames; int ctrl_hdl;') +pre += r''' +#define round_up(x,a) (((x)+(a)-1)&~((a)-1)) +#define max_t(t,a,b) ((t)(a)>(t)(b)?(t)(a):(t)(b)) +#define min(a,b) ((a)<(b)?(a):(b)) +#define clamp(x,l,h) ((x)<(l)?(l):(x)>(h)?(h):(x)) +#define WARN_ON(v) (assert(!(v)),false) +#define GFP_KERNEL 0 +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define noinline_for_stack __attribute__((noinline)) +#define static_assert _Static_assert +#define EXPORT_SYMBOL_GPL(x) +#define MODULE_LICENSE(x) +#define MODULE_DESCRIPTION(x) +#define MODULE_AUTHOR(x) +#define VB2_BUF_STATE_DONE 0 +#define VB2_BUF_STATE_ERROR 1 +enum vb2_buffer_state { STATE_DUMMY }; +#define dev_err(d,...) ((void)(d)) +#define kzalloc(n,f) calloc(1,n) +#define kfree free +static bool vp_feature = true; +static bool avd_qfw_has_vp_done(const struct avd_dev *d) { return d->queued && vp_feature; } +static void *vb2_plane_vaddr(struct vb2_buffer *b,int p) { (void)p;return b->cpu; } +static size_t vb2_get_plane_payload(struct vb2_buffer *b,int p) { (void)p;return b->payload; } +static struct vb2_v4l2_buffer *current_src,*current_dst; +static struct vb2_v4l2_buffer *v4l2_m2m_next_src_buf(void *m) { (void)m;return current_src; } +static u32 get_unaligned_be32(const void *p) { const u8 *v=p;return (u32)v[0]<<24|(u32)v[1]<<16|(u32)v[2]<<8|v[3]; } +struct v4l2_ctrl { union { void *p; } p_cur; }; +static struct v4l2_ctrl_vp9_frame params; +static struct v4l2_ctrl_vp9_compressed_hdr updates; +static struct v4l2_ctrl frame_ctrl,update_ctrl; +static struct v4l2_ctrl *v4l2_ctrl_find(void *h,u32 id) { + (void)h;frame_ctrl.p_cur.p=¶ms;update_ctrl.p_cur.p=&updates; + return id==V4L2_CID_STATELESS_VP9_FRAME?&frame_ctrl:&update_ctrl; +} +static void avd_run_preamble(struct avd_ctx *ctx,struct avd_run *run) { + (void)ctx;memset(run,0,sizeof(*run));run->bufs.src=current_src;run->bufs.dst=current_dst; + run->coded_in=current_src->vb2_buf.dma;run->comp_out=current_dst->vb2_buf.dma; + run->y_out=run->comp_out+0x100000;run->uv_out=run->comp_out+0x300000; +} +static void avd_run_postamble(struct avd_ctx *ctx,struct avd_run *run) { (void)ctx;(void)run; } +static int avd_init_job(struct avd_ctx *ctx,int codec,size_t segments) { + free(ctx->job.segments);ctx->job.codec=codec;ctx->job.num=0; + ctx->job.segments=calloc(segments,sizeof(*ctx->job.segments));return ctx->job.segments?0:-ENOMEM; +} +static int avd_submit_job(struct avd_ctx *ctx); +''' +pre = pre.replace('dma_addr_t comp_out, y_out, uv_out;', 'dma_addr_t comp_out, y_out, uv_out, coded_in;') + + +def without_includes(s): + return re.sub(r'^#include[^\n]*\n', '', s, flags=re.M) + +vp9 = (AVD / 'avd-vp9.c').read_text() +helper_h = (KERNEL / 'include/media/v4l2-vp9.h').read_text() +helper_c = (KERNEL / 'drivers/media/v4l2-core/v4l2-vp9.c').read_text() +source = pre + '\n' + without_includes(helper_h) + '\n' + without_includes(helper_c) +source += '\n' + without_includes((AVD / 'avd-inst.h').read_text()) +# Include unchanged codec declarations/functions, excluding only the ops table. +source += '\n' + without_includes(vp9[vp9.index('#define VP9_Q_IDX'):vp9.index('static enum avd_image_fmt avd_vp9_get_image_fmt')]) +source += '\n' + function(vp9, 'avd_vp9_pp_kicks') +source += '\n' + function((AVD / 'avd-backend-open.c').read_text(), 'avd_open_build_program') +qfw = (AVD / 'avd-backend-queue.c').read_text() +source += qfw[qfw.index('#define AVDQ_MAGIC'):qfw.index('/* Arguments of dev_dbg()')] +hw = (AVD / 'avd-hw.c').read_text() +source += hw[hw.index('#define AVD_V3_'):hw.index('int avd_boot(')] +source += '#define w32(r,v) writel((v),avd->ctrl+(r))\n#define m32(r,v) w32(r,(v)|readl(avd->ctrl+(r)))\n' +source += function(hw, 't8112_configure_stream') +source += function(qfw, 'avd_qfw_stream') + function(qfw, 'avd_qfw_submit') +source += '\nstatic int avd_submit_job(struct avd_ctx *ctx) { return avd_open_build_program(ctx); }\n' +source += r''' +static void write_program(const char *dir,const char *name,struct avd_frame *f) { + char path[4096];snprintf(path,sizeof(path),"%s/%s.bin",dir,name);FILE *out=fopen(path,"wb");assert(out); + assert(fwrite(f->prog->cpu,4,f->prog_words,out)==f->prog_words);fclose(out); + struct avd_dev *dev=f->ctx->dev;struct avd_qfw *q=dev->qfw; + /* The descriptor, PP command and configure_stream snapshot are production. */ + q->wp=0;memset(q->sram,0,0x10000);f->tag=0x10|f->idx; + assert(!avd_qfw_submit(dev,f)); + snprintf(path,sizeof(path),"%s/%s-desc.bin",dir,name);out=fopen(path,"wb");assert(out); + assert(fwrite(q->sram+0x8000+f->idx*64,1,64,out)==64);fclose(out); + snprintf(path,sizeof(path),"%s/%s-mmio.bin",dir,name);out=fopen(path,"wb");assert(out); + assert(fwrite(dev->ctrl,1,0x10000,out)==0x10000);fclose(out); +} +static unsigned int offset_words[256]; +static void save_dma(const char *dir,const char *name) { + char path[4096];snprintf(path,sizeof(path),"%s/%s-dma.txt",dir,name);FILE *out=fopen(path,"w");assert(out); + for(unsigned int i=0;inbanks==2); + assert(p->bank[0].bufs.counts.addr!=p->bank[1].bufs.counts.addr); + assert(p->bank[0].bufs.probs.addr!=p->bank[1].bufs.probs.addr); + assert(p->bank[0].cnts.skip!=p->bank[1].cnts.skip); + assert(p->bank[0].cnts.coeff[0][0][0][0][0]!=p->bank[1].cnts.coeff[0][0][0][0][0]); + assert(p->bank[0].cnts.eob[0][0][0][0][0][0]!=p->bank[1].cnts.eob[0][0][0][0][0][0]); + /* All header-flag combinations, including malformed error-resilient refresh. */ + for(unsigned int bits=0;bits<8;bits++) { + u32 flags=(bits&1?V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX:0)| + (bits&2?V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE:0)|(bits&4?V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT:0); + assert(avd_vp9_needs_counts(flags)==(bits==1)); + } + params=(struct v4l2_ctrl_vp9_frame){.flags=V4L2_VP9_FRAME_FLAG_KEY_FRAME|V4L2_VP9_FRAME_FLAG_SHOW_FRAME|V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX, + .frame_width_minus_1=1919,.frame_height_minus_1=1087,.bit_depth=8}; + updates.tx_mode=V4L2_VP9_TX_MODE_SELECT; + current_dst=&dst[0].base.vb;frame.idx=0;dma_count=0; + assert(avd_vp9_ready(&ctx));assert(!avd_vp9_run(&ctx)); + assert(dst[0].bank==1&&p->bank[0].busy&&!avd_vp9_ready(&ctx)); + assert(frame.fifo_iova==p->bank[0].bufs.inst.addr); + write_program(argv[1],"vp9-adaptive-0",&frame); + /* Symbol-count fixture must change a saved coefficient probability. */ + struct avd_vp9_frame_symbol_counts *counts=p->bank[0].bufs.counts.cpu; + for(size_t i=0;i<528;i++){counts->eob_0[i]=16;counts->ref_cnt[i*4]=1;counts->ref_cnt[i*4+1]=4;counts->ref_cnt[i*4+2]=11;counts->ref_cnt[i*4+3]=8;} + struct v4l2_vp9_frame_context before=p->frame_context[0]; + avd_vp9_vp_done(&ctx,&dst[0].base.vb); + assert(avd_vp9_ready(&ctx)&&p->bank[0].busy&&p->bank[0].updated&&dst[0].bank==1); + assert(memcmp(&before,&p->frame_context[0],sizeof(before))); + struct v4l2_vp9_frame_context adapted=p->frame_context[0]; + avd_vp9_vp_done(&ctx,&dst[0].base.vb);assert(!memcmp(&adapted,&p->frame_context[0],sizeof(adapted))); + current_dst=&dst[1].base.vb;frame.idx=1;dma_count=0;params.flags=V4L2_VP9_FRAME_FLAG_SHOW_FRAME|V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX; + reference=dst[0];reference.vp9.width=1920;reference.vp9.height=1088; + assert(!avd_vp9_run(&ctx));assert(dst[1].bank==2&&p->bank[0].busy&&p->bank[1].busy); + assert(!memcmp(&adapted,&p->bank[1].probability_tables,sizeof(adapted))); + write_program(argv[1],"vp9-adaptive-1",&frame); + assert(!avd_vp9_ready(&ctx)); + struct avd_vp9_run attempt; + current_dst=&dst[2].base.vb;assert(avd_vp9_run_preamble(&ctx,&attempt)==-EBUSY);assert(!dst[2].bank); + /* PP(0) retires while b points to bank 1: only bank 0 may be released. */ + avd_vp9_done(&ctx,&src,&dst[0].base.vb,VB2_BUF_STATE_DONE); + assert(!p->bank[0].busy&&p->bank[1].busy&&!dst[0].bank); + assert(!memcmp(&adapted,&p->frame_context[0],sizeof(adapted))); + /* Failed VP/prepare must never adapt partial counts. */ + avd_vp9_done(&ctx,&src,&dst[1].base.vb,VB2_BUF_STATE_ERROR); + assert(!p->bank[1].busy&&avd_vp9_ready(&ctx)); + assert(!memcmp(&adapted,&p->frame_context[0],sizeof(adapted))); + for(unsigned int mode=0;mode<3;mode++) { + params.flags=mode==0?0:mode==1?V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX|V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE:V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT; + current_dst=&dst[0].base.vb;frame.idx=0;assert(!avd_vp9_run(&ctx)); + assert(avd_vp9_ready(&ctx)&&p->bank[0].busy); + if(mode==0) { + /* The third inter picture's temporal-MV input stays in the prior bank. */ + assert(p->last.color==p->bank[0].bufs.color.addr); + bool prior_color=false;u32 *words=program.cpu; + for(size_t j=0;jbank[1].bufs.color.addr>>8))prior_color=true; + assert(prior_color); + } + current_dst=&dst[1].base.vb;frame.idx=1;assert(!avd_vp9_run(&ctx)); + assert(p->bank[0].busy&&p->bank[1].busy); + avd_vp9_done(&ctx,&src,&dst[0].base.vb,VB2_BUF_STATE_DONE);avd_vp9_done(&ctx,&src,&dst[1].base.vb,VB2_BUF_STATE_DONE); + } + avd_vp9_stop(&ctx); + /* Old firmware and the legacy transport allocate only one bank. */ + for(unsigned int mode=0;mode<2;mode++) { + dev.queued=mode;vp_feature=false;assert(!avd_vp9_start(&ctx));p=ctx.priv; + assert(p->nbanks==1&&ctx.max_frames==1);avd_vp9_stop(&ctx); + } + free(program.cpu);free(queue.sram);free(dev.ctrl);free(dev.mbox);puts("PASS: VP9 dependency table, bank ownership, real adaptation, early release and legacy fallback"); +} +''' +# Extra pixel-format shape and CPU-reference dimensions in this boundary. +source = source.replace('struct { u32 bytesperline; } plane_fmt[1];', 'u32 width,height; struct { u32 bytesperline; } plane_fmt[1];') +(OUT/'vp9-host.c').write_text(source) +cmd = [os.environ.get('CC','cc'), '-std=gnu11','-g','-O1','-Wall','-Wextra','-Werror', + '-Wno-unused-function','-Wno-unused-parameter','-Wno-sign-compare', + '-fsanitize=address,undefined','-fno-omit-frame-pointer',str(OUT/'vp9-host.c'),'-o',str(OUT/'vp9-host')] +subprocess.run(cmd,check=True) +subprocess.run([str(OUT/'vp9-host'),str(OUT)],check=True) +(OUT/'source.sha256').write_text('\n'.join(hashlib.sha256(p.read_bytes()).hexdigest()+' '+str(p) for p in (AVD/'avd-vp9.c',AVD/'avd-backend-open.c',KERNEL/'include/media/v4l2-vp9.h',KERNEL/'drivers/media/v4l2-core/v4l2-vp9.c'))+'\n') + +# Optional independent before/after executable comparison. The baseline is read +# from public Git history, never from a captured stock firmware program. +base_ref = os.environ.get('AVD_VP9_BASELINE_REF') +if base_ref: + old = subprocess.check_output(['git','-C',str(KERNEL),'show', + base_ref + ':drivers/media/platform/apple/avd/avd-vp9.c'],text=True) + legacy_main = r''' +int main(int argc,char **argv) { + assert(argc==2); + struct avd_variant variant={.revision=4,.quirks=AVD_QUIRK_LSR,.fifo_slots=15}; + struct avd_dev dev={.variant=&variant,.queued=false}; + struct avd_coded_fmt_ops ops={};struct fmt_desc fmt={.ops=&ops}; + struct avd_ctx ctx={.dev=&dev,.coded_fmt_desc=&fmt}; + struct avd_buf program={};struct avd_frame f={.prog=&program,.ctx=&ctx};dev.building=&f; + struct avd_decoded_buffer dst={};struct vb2_v4l2_buffer src={};u8 coded[512]={}; + dst.base.vb.vb2_buf.dma=0x230000000ULL;src.vb2_buf=(struct vb2_buffer){.dma=0x220000000ULL,.cpu=coded,.payload=512}; + current_dst=&dst.base.vb;current_src=&src;ctx.inst.addr=0x205000000ULL; + ctx.decoded_fmt.fmt.pix_mp.width=1920;ctx.decoded_fmt.fmt.pix_mp.height=1088; + ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; + assert(!avd_vp9_start(&ctx));struct avd_vp9_ctx *p=ctx.priv; + params=(struct v4l2_ctrl_vp9_frame){.frame_width_minus_1=1919,.frame_height_minus_1=1087,.bit_depth=8}; + updates.tx_mode=V4L2_VP9_TX_MODE_SELECT; + const u32 flags[]={V4L2_VP9_FRAME_FLAG_KEY_FRAME|V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX, + V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX,V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE|V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX, + 0,V4L2_VP9_FRAME_FLAG_ERROR_RESILIENT|V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE, + V4L2_VP9_FRAME_FLAG_KEY_FRAME|V4L2_VP9_FRAME_FLAG_REFRESH_FRAME_CTX|V4L2_VP9_FRAME_FLAG_PARALLEL_DEC_MODE}; + for(unsigned int i=0;ibufs.counts.cpu; + for(size_t j=0;j<528;j++){c->eob_0[j]=16;c->ref_cnt[j*4]=1;c->ref_cnt[j*4+1]=4;c->ref_cnt[j*4+2]=11;c->ref_cnt[j*4+3]=8;} + avd_vp9_done(&ctx,&src,&dst.base.vb,VB2_BUF_STATE_DONE); + snprintf(path,sizeof(path),"%s/context-%u.bin",argv[1],i);out=fopen(path,"wb");assert(out); + assert(fwrite(p->frame_context,1,sizeof(p->frame_context),out)==sizeof(p->frame_context));fclose(out); + } + avd_vp9_stop(&ctx);free(program.cpu); +} +''' + for name, body in (('before',old),('after',vp9)): + target = OUT / ('legacy-' + name) + target.mkdir(exist_ok=True) + src = pre + without_includes(helper_h) + without_includes(helper_c) + src += without_includes((AVD/'avd-inst.h').read_text()) + src += without_includes(body[body.index('#define VP9_Q_IDX'):body.index('static enum avd_image_fmt avd_vp9_get_image_fmt')]) + src += function((AVD/'avd-backend-open.c').read_text(),'avd_open_build_program') + src += '\nstatic int avd_submit_job(struct avd_ctx *ctx) { return avd_open_build_program(ctx); }\n' + src += '#define WORK(p) ' + ('((p)->b)' if name=='after' else '(p)') + '\n' + legacy_main + src = src.replace('struct { u32 bytesperline; } plane_fmt[1];','u32 width,height; struct { u32 bytesperline; } plane_fmt[1];') + (target/'legacy.c').write_text(src) + subprocess.run(cmd[:-3]+[str(target/'legacy.c'),'-o',str(target/'legacy')],check=True) + subprocess.run([str(target/'legacy'),str(target)],check=True) + for p in sorted((OUT/'legacy-before').glob('*.bin')): + assert p.read_bytes()==(OUT/'legacy-after'/p.name).read_bytes(),p.name + print('PASS: legacy VP9 programs and adapted frame contexts byte-identical to '+base_ref) From 8cc0f398c265b2f85712ae353e903a89f357b26a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 16:36:43 +1100 Subject: [PATCH 51/73] tools: asahi: test the rewritten AVD capability parser Extract avd_open_capabilities from avd-backend-open.c after the transport split. Update the boundary structures and mutations for the current parser, including its saved firmware capability mask and bounded 64-byte trailer. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/avd-codec-tail-tests.py | 38 +++++++++++++++-------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/tools/asahi/avd-codec-tail-tests.py b/tools/asahi/avd-codec-tail-tests.py index 31616efc2651c..a7f2e039957f1 100755 --- a/tools/asahi/avd-codec-tail-tests.py +++ b/tools/asahi/avd-codec-tail-tests.py @@ -17,7 +17,7 @@ import tempfile ROOT = Path(__file__).resolve().parents[2] -SOURCE = "drivers/media/platform/apple/avd/avd-drv.c" +SOURCE = "drivers/media/platform/apple/avd/avd-backend-open.c" HEADER = "drivers/media/platform/apple/avd/avd.h" @@ -38,13 +38,14 @@ def build_source(body, defines): #define dev_info(...) ((void)0) ''' + defines + r''' struct firmware { const unsigned char *data; size_t size; }; -struct variant { unsigned int capabilities; }; +struct variant { unsigned int capabilities, fw_caps_required; }; struct avd_dev { struct firmware *fw; const struct variant *variant; - unsigned int capabilities; + unsigned int capabilities, fw_capabilities; }; -static char *strnstr(const char *s, const char *needle, size_t n) +#define strnstr fixture_strnstr +static char *fixture_strnstr(const char *s, const char *needle, size_t n) { size_t len = strlen(needle); for (size_t i = 0; i + len <= n && s[i]; i++) @@ -52,10 +53,10 @@ def build_source(body, defines): return (char *)s + i; return NULL; } +''' + body + r''' static unsigned int probe_tail(struct avd_dev *avd) { - const char *codec_info = NULL; -''' + body + r''' + avd_open_capabilities(avd); return avd->capabilities; } int main(int argc, char **argv) @@ -67,12 +68,14 @@ def build_source(body, defines): if (!data) return 2; struct firmware fw = {data, size}; - struct variant variant = {strtoul(argv[3], NULL, 0)}; - struct avd_dev avd = {&fw, &variant, 0}; + struct variant variant = {strtoul(argv[3], NULL, 0), 0}; + struct avd_dev avd = {&fw, &variant, 0, 0}; size_t len = strlen(argv[2]); if (size >= AVD_CODEC_INFO_MAX_SIZE && len <= AVD_CODEC_INFO_MAX_SIZE) memcpy(data + size - AVD_CODEC_INFO_MAX_SIZE, argv[2], len); unsigned int actual = probe_tail(&avd); + if (actual != avd.fw_capabilities) + return 1; unsigned int expected = strtoul(argv[4], NULL, 0); printf("size=%zu variant=%u capabilities=%u expected=%u\n", size, variant.capabilities, actual, expected); @@ -122,12 +125,12 @@ def main(): args = parser.parse_args() args.output.mkdir(parents=True, exist_ok=False) source, header = read(SOURCE, args.ref), read(HEADER, args.ref) - start = source.index("\t/* Determine firmware and HW variant capabilities */") - end = source.index("\n\tret = dma_set_mask_and_coherent", start) + start = source.index("static void avd_open_capabilities(") + end = source.index("static int avd_open_request_firmware(", start) body = source[start:end] - wanted = ["AVD_CODEC_INFO_MAX_SIZE", "AVD_CAPABILITY_AV1", "AVD_CAPABILITY_H264", + wanted = ["AVD_CAPABILITY_AV1", "AVD_CAPABILITY_H264", "AVD_CAPABILITY_HEVC", "AVD_CAPABILITY_VP9"] - defines = "" + defines = "#define AVD_CODEC_INFO_MAX_SIZE 0x40\n#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))\n#define u32 unsigned int\n" for macro in wanted: matches = re.findall(r"^#define\s+" + macro + r"\s+[^\n]+$", source + "\n" + header, re.M) @@ -148,16 +151,15 @@ def main(): (128, "legacy", 5, 5)] variants = [("control-before", body)] if args.mutations: - anchor = "\tif (avd->fw->size >= AVD_CODEC_INFO_MAX_SIZE)\n" + anchor = "!avd->fw || avd->fw->size < 0x40" if body.count(anchor) != 1: raise ValueError("size-guard mutation did not match exactly once") - variants += [("no-size-guard", body.replace(anchor, "")), + variants += [("no-size-guard", body.replace(anchor, "!avd->fw")), ("no-variant-intersection", body.replace( - "\t\tunsigned int variant_caps = avd->variant->capabilities;\n", "" - ).replace("caps |= variant_caps & ", "caps |= ")), + "avd->variant->capabilities & codecs[i].bit", "codecs[i].bit")), ("no-legacy-fallback", body.replace( - "avd->capabilities = avd->variant->capabilities;", - "avd->capabilities = 0;"))] + "avd->variant->capabilities &\n\t\t\t ~avd->variant->fw_caps_required", + "0"))] variants.append(("control-after", body)) results = [run_variant(args.output, name, text, defines, cases) for name, text in variants] From a1b768568d780c989f2ca33cf7b1147a853caf7e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 16:52:02 +1100 Subject: [PATCH 52/73] media: apple: avd: publish VP9 banks after initialization Readiness can be checked while STREAMON is still allocating codec storage. Keep ctx->priv unpublished until all VP9 banks and count-pointer tables are initialized, and pair its publication with the readiness callback's acquire load. Failed allocations free their local context without exposing partially initialized banks. Exercise readiness before start, from every bank allocation, and after stop in the production host fixture. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-vp9.c | 18 +++++++++++++----- .../media/platform/apple/avd/tests/vp9-test.py | 16 +++++++++++++--- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-vp9.c b/drivers/media/platform/apple/avd/avd-vp9.c index fc0d64c760a50..58b0106371f90 100644 --- a/drivers/media/platform/apple/avd/avd-vp9.c +++ b/drivers/media/platform/apple/avd/avd-vp9.c @@ -216,8 +216,10 @@ static bool avd_vp9_needs_counts(u32 flags) static bool avd_vp9_ready(struct avd_ctx *ctx) { - struct avd_vp9_ctx *vp9_ctx = ctx->priv; + struct avd_vp9_ctx *vp9_ctx = smp_load_acquire(&ctx->priv); + if (!vp9_ctx) + return false; /* Only a producer doing backward adaptation holds the next prepare. */ for (unsigned int i = 0; i < vp9_ctx->nbanks; i++) if (READ_ONCE(vp9_ctx->bank[i].busy) && @@ -1086,9 +1088,8 @@ avd_init_v4l2_vp9_count_tbl(struct avd_vp9_bank *bank) } } -static void avd_vp9_stop(struct avd_ctx *ctx) +static void avd_vp9_free(struct avd_ctx *ctx, struct avd_vp9_ctx *vp9_ctx) { - struct avd_vp9_ctx *vp9_ctx = ctx->priv; struct avd_dev *avd = ctx->dev; ctx->max_frames = 1; @@ -1110,6 +1111,12 @@ static void avd_vp9_stop(struct avd_ctx *ctx) } kfree(vp9_ctx); +} + +static void avd_vp9_stop(struct avd_ctx *ctx) +{ + /* drain() excludes readiness callbacks before private storage is freed. */ + avd_vp9_free(ctx, ctx->priv); ctx->priv = NULL; } @@ -1122,7 +1129,6 @@ static int avd_vp9_start(struct avd_ctx *ctx) if (!vp9_ctx) return -ENOMEM; - ctx->priv = vp9_ctx; vp9_ctx->nbanks = avd_qfw_has_vp_done(ctx->dev) ? AVD_VP9_BANKS : 1; vp9_ctx->b = &vp9_ctx->bank[0]; for (unsigned int i = 0; i < vp9_ctx->nbanks; i++) { @@ -1132,11 +1138,13 @@ static int avd_vp9_start(struct avd_ctx *ctx) avd_init_v4l2_vp9_count_tbl(&vp9_ctx->bank[i]); } ctx->max_frames = vp9_ctx->nbanks; + /* job_ready can run while STREAMON is still initializing this codec. */ + smp_store_release(&ctx->priv, vp9_ctx); return 0; err_free_ctx: - avd_vp9_stop(ctx); + avd_vp9_free(ctx, vp9_ctx); return ret; } diff --git a/drivers/media/platform/apple/avd/tests/vp9-test.py b/drivers/media/platform/apple/avd/tests/vp9-test.py index 7bceb98752828..7c60bbb168843 100644 --- a/drivers/media/platform/apple/avd/tests/vp9-test.py +++ b/drivers/media/platform/apple/avd/tests/vp9-test.py @@ -33,6 +33,10 @@ pre = pre.replace('struct avd_comp comp; unsigned int bank;', 'struct avd_comp comp; unsigned int bank; struct { unsigned int width,height,bit_depth; } vp9;') pre = pre.replace('struct vb2_v4l2_buffer *dst; } bufs;', 'struct vb2_v4l2_buffer *src,*dst; } bufs;') pre = pre.replace('unsigned int fifo_idx, pp_total;', 'unsigned int fifo_idx, pp_total, max_frames; int ctrl_hdl;') +pre = pre.replace('static int avd_buf_alloc(struct avd_dev *d, struct avd_buf *b, size_t n)', + '#ifdef AVD_TEST_VP_READY\nstatic struct avd_ctx *allocating_ctx;\nstatic int fail_alloc_after;\nstatic bool avd_vp9_ready(struct avd_ctx *);\n#endif\nstatic int avd_buf_alloc(struct avd_dev *d, struct avd_buf *b, size_t n)') +pre = pre.replace('{ (void)d; free(b->cpu); b->cpu = calloc(1,n);', + '{\n#ifdef AVD_TEST_VP_READY\n if(allocating_ctx){assert(!avd_vp9_ready(allocating_ctx));if(fail_alloc_after&&!--fail_alloc_after)return -ENOMEM;}\n#endif\n (void)d; free(b->cpu); b->cpu = calloc(1,n);') pre += r''' #define round_up(x,a) (((x)+(a)-1)&~((a)-1)) #define max_t(t,a,b) ((t)(a)>(t)(b)?(t)(a):(t)(b)) @@ -40,6 +44,8 @@ #define clamp(x,l,h) ((x)<(l)?(l):(x)>(h)?(h):(x)) #define WARN_ON(v) (assert(!(v)),false) #define GFP_KERNEL 0 +#define smp_load_acquire(p) (*(p)) +#define smp_store_release(p,v) (*(p)=(v)) #define READ_ONCE(x) (x) #define WRITE_ONCE(x,v) ((x)=(v)) #define noinline_for_stack __attribute__((noinline)) @@ -90,7 +96,7 @@ def without_includes(s): vp9 = (AVD / 'avd-vp9.c').read_text() helper_h = (KERNEL / 'include/media/v4l2-vp9.h').read_text() helper_c = (KERNEL / 'drivers/media/v4l2-core/v4l2-vp9.c').read_text() -source = pre + '\n' + without_includes(helper_h) + '\n' + without_includes(helper_c) +source = '#define AVD_TEST_VP_READY\n' + pre + '\n' + without_includes(helper_h) + '\n' + without_includes(helper_c) source += '\n' + without_includes((AVD / 'avd-inst.h').read_text()) # Include unchanged codec declarations/functions, excluding only the ops table. source += '\n' + without_includes(vp9[vp9.index('#define VP9_Q_IDX'):vp9.index('static enum avd_image_fmt avd_vp9_get_image_fmt')]) @@ -140,7 +146,8 @@ def without_includes(s): for(unsigned int i=0;i<3;i++)dst[i].base.vb.vb2_buf.dma=0x230000000ULL+i*0x1000000ULL; ctx.decoded_fmt.fmt.pix_mp.width=1920;ctx.decoded_fmt.fmt.pix_mp.height=1088; ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; - assert(!avd_vp9_start(&ctx));struct avd_vp9_ctx *p=ctx.priv; + assert(!avd_vp9_ready(&ctx));allocating_ctx=&ctx; + assert(!avd_vp9_start(&ctx));allocating_ctx=NULL;struct avd_vp9_ctx *p=ctx.priv; assert(ctx.max_frames==2&&p->nbanks==2); assert(p->bank[0].bufs.counts.addr!=p->bank[1].bufs.counts.addr); assert(p->bank[0].bufs.probs.addr!=p->bank[1].bufs.probs.addr); @@ -201,12 +208,15 @@ def without_includes(s): assert(p->bank[0].busy&&p->bank[1].busy); avd_vp9_done(&ctx,&src,&dst[0].base.vb,VB2_BUF_STATE_DONE);avd_vp9_done(&ctx,&src,&dst[1].base.vb,VB2_BUF_STATE_DONE); } - avd_vp9_stop(&ctx); + avd_vp9_stop(&ctx);assert(!avd_vp9_ready(&ctx)); /* Old firmware and the legacy transport allocate only one bank. */ for(unsigned int mode=0;mode<2;mode++) { dev.queued=mode;vp_feature=false;assert(!avd_vp9_start(&ctx));p=ctx.priv; assert(p->nbanks==1&&ctx.max_frames==1);avd_vp9_stop(&ctx); } + dev.queued=true;vp_feature=true;fail_alloc_after=13;allocating_ctx=&ctx; + assert(avd_vp9_start(&ctx)==-ENOMEM);allocating_ctx=NULL; + assert(!ctx.priv&&!avd_vp9_ready(&ctx)&&ctx.max_frames==1); free(program.cpu);free(queue.sram);free(dev.ctrl);free(dev.mbox);puts("PASS: VP9 dependency table, bank ownership, real adaptation, early release and legacy fallback"); } ''' From 88bb7fa7cf0bdf7bae03473f8e68a1dee24c91f7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 15:47:36 +1100 Subject: [PATCH 53/73] accel/ane: stop and restart H14 firmware across system sleep The H14 firmware currently remains live while system PM suspends its power and IOMMU suppliers. The next command can time out after resume, leaving the device unusable until reboot. Fence submitters under the firmware mutex and split the existing lifetime restart transaction at its proven-stop boundary. System suspend uses the same cooperative shutdown and ASC stop helper as planned recovery. Keep the transaction and all old backing across the supplier transition, then use the existing accessibility checks, cold boot, handshake and lifetime commit on resume. Only a verified restart reopens submissions. Runtime recovery continues to perform its existing explicit power cycle. Reuse the shared program table and lazy reload policy, preserving public program IDs; introduce no second replay or allocation-retirement path. Leave failed stops and restarts fenced with the lifetime owner's existing retention policy. Add KUnit coverage for system callback wiring, three sleep cycles, serialization, stop/restart errors and the shared stop-only boundary. Document the M2 platform-test cycles and subsequent s2idle acceptance run. This change depends on the shared H14 lifetime work in linux #737. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 101 +++++++++ drivers/accel/ane/Kconfig | 5 +- drivers/accel/ane/Makefile | 2 +- drivers/accel/ane/ane_t6021_lifetime.h | 21 +- drivers/accel/ane/ane_t6021_lifetime_test.c | 26 ++- drivers/accel/ane/ane_t6021_pm.h | 105 +++++++++ drivers/accel/ane/ane_t6021_pm_test.c | 173 ++++++++++++++ drivers/accel/ane/ane_t6021_rtclient_main.c | 235 ++++++++++++++++---- 8 files changed, 620 insertions(+), 48 deletions(-) create mode 100644 drivers/accel/ane/ane_t6021_pm.h create mode 100644 drivers/accel/ane/ane_t6021_pm_test.c diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index cc1c69a5d0192..0739c351853a0 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -437,3 +437,104 @@ after verified quiescence and acquires a new one before the first boot write; it does not accumulate one reference per failure. This change does not provide normal H14 module unload while firmware is active. H14 restart still requires silicon qualification; the H13 hang fixture must not be submitted through ABI 2. + +H14 system-sleep adapter +======================= + +``ane_t6021_pm.h`` provides the system-sleep PM operations and submission +fencing used by ane_t6021. Stop, firmware reload, channels, allocation ownership +and program replay remain in the shared H14 lifetime implementation from +https://github.com/aurora-silicon/linux/pull/737. System sleep and timeout +recovery use one stop helper, one boot function and one transaction commit. +The shutdown protocol is not implemented a second time by the PM adapter. + +The adapter contract is: + +* Embed ``struct ane_t6021_pm`` in the client, initialized once at probe with + its existing firmware mutex, lifetime callback table, client context and + ``ANE_T6021_PM_ACTIVE`` state. +* Use ``DEFINE_ANE_T6021_SYSTEM_PM_OPS(name, getter)`` to define the device PM + operations. The getter returns the embedded adapter without replacing the + driver's own drvdata. The driver binds ``pm_sleep_ptr(&name)`` to ``driver.pm`` and supplies + the shared lifetime callbacks below. +* Every external firmware-facing submission checks ``ane_t6021_pm_submit_error()`` under + that same firmware mutex. Suspend acquires the mutex, draining the current + synchronous command and its finish event, and fences queued submitters + before calling the shared stop callback. +* The stop callback performs the validated firmware shutdown handshake, drains + RX and proves ASC STOPPED with RUN clear before returning success. Clearing + RUN or observing ordinary WFI/idle is insufficient. Failure must retain + power claims, DMA mappings, firmware backing and the module pin. +* The restart callback is the same cold-start/handshake/program-replay path + used by timeout recovery. Supplier and DART system callbacks must restore + access first. Return success only after channels, CONFIG_GET and desired firmware + properties are restored and the lifetime owner has made public IDs usable + under its normal lazy-reload policy. RX and all firmware-owned resources stay with this one lifetime + implementation. + +System PM disables runtime PM and holds an additional consumer reference. +Consequently suspend stops at ``ane_t6021_restart_stop()`` and keeps the shared +restart transaction on the client across the system power boundary. Genpd and +DART system callbacks perform that power transition. Resume checks provider +access and the stopped-core predicate, then invokes the same boot and commit +functions as runtime recovery. Runtime recovery still uses the shared runtime-PM +cycle between stop and boot. No runtime-PM force/cycle helper is called by the +system-sleep callbacks. + +Failure leaves submissions fenced with EHOSTDOWN and retains firmware-visible +backing and pins. A verified explicit recovery can reopen the PM gate. The +adapter never sends firmware power commands or duplicates the program/allocation +cache. Internal shutdown/restart commands run through the shared lifetime owner +while external work is fenced. + +Device-free tests +----------------- + +The ``ane-h14-sleep`` KUnit suite invokes the generated device callbacks with a +fake shared-lifetime backend. It checks three successive cycles, that callbacks +hold the firmware mutex with submissions fenced, failed stop and restart +propagation, unwind before suspend, and suspend/freeze/poweroff versus +resume/thaw/restore callback mapping. Firmware stop predicates, shutdown command +layouts and program replay are tested by the lifetime owner, not reimplemented +here. Run the PM suites without hardware:: + + ./tools/testing/kunit/kunit.py run \ + --kunitconfig=tools/testing/selftests/drivers/ane/pm.kunitconfig \ + --jobs=8 + +M2 validation plan +------------------ + +Runtime testing requires the shared lifetime implementation and this PM +integration together. Do not infer a hardware pass from the adapter's KUnit results. +Run on a T8112 with the candidate kernel and its matching ane_t6021 module, +through the hardware controller with exclusive device ownership, bounded +console capture and a tested boot fallback. Record the kernel/module revision, +firmware identity, boot ID, initial kernel log and the smoke command. Use the +same known-good H14 smoke and model/input fixtures for the baseline and every +post-resume check. Verify the complete output, including its expected surface +size, and vary inputs or poison outputs so retained output cannot pass. + +1. Run the smoke before suspend. Keep a client with a loaded program, process, + BO handles and mappings open for the duration of the test. +2. Save the current pm_test setting. Repeat the following three times, capturing + the new kernel log and running the smoke after *each* cycle:: + + echo platform > /sys/power/pm_test + echo mem > /sys/power/state + # Run the recorded H14 smoke, checking its exit status and full output. + + Require the shared stop proof and successful restart/handshake/replay receipts + each time. Exercise the surviving client's original handles with new inputs, + and a newly opened client, after each cycle. Require an unchanged boot ID, + correct nonempty outputs, and no LEGACY timeout, failed exchange, DART fault, + crash or PM failure. Confirm identical repeated cycles do not accumulate + module references or retained allocations. +3. Set pm_test to ``none``. If ``/sys/power/mem_sleep`` advertises ``s2idle``, + select it, arrange a supported timed or external wake through the controller, + and run a real ``echo mem > /sys/power/state`` cycle. Run both smokes after + wake and apply the same checks. Record unsupported s2idle or wake support + explicitly; a platform test does not establish real-suspend behavior. +4. Restore pm_test and mem_sleep selections. Preserve failure evidence before + using the approved known-good boot fallback. Do not force-unload retained + firmware memory or treat an unproven stop as permission to remove power. diff --git a/drivers/accel/ane/Kconfig b/drivers/accel/ane/Kconfig index c502238817ee8..1484a95b16a70 100644 --- a/drivers/accel/ane/Kconfig +++ b/drivers/accel/ane/Kconfig @@ -63,8 +63,9 @@ config DRM_ACCEL_ANE_PM_KUNIT_TEST help Exercise supplier suspension, power-off verification failure and runtime resume errors against the runtime-PM and device-link core. + Also check H14 system-sleep callback ordering, serialization and + failed stop/restart submission fencing with a fake lifetime backend. No ANE hardware is required. These tests can run independently of - the ANE driver on non-Apple platforms. Also test that firmware - restarts cannot reuse a stale userspace program identifier. + the ANE driver on non-Apple platforms. If unsure, say N. diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile index 5bbd4746e3cee..767efba5b3e1e 100644 --- a/drivers/accel/ane/Makefile +++ b/drivers/accel/ane/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_DRM_ACCEL_ANE) += ane.o ane_t6021.o -obj-$(CONFIG_DRM_ACCEL_ANE_PM_KUNIT_TEST) += ane_pm_test.o ane_fw_ids_test.o +obj-$(CONFIG_DRM_ACCEL_ANE_PM_KUNIT_TEST) += ane_pm_test.o ane_t6021_pm_test.o ane_fw_ids_test.o obj-$(CONFIG_DRM_ACCEL_ANE_H16_VALIDATION) += ane_h16_hwx.o obj-$(CONFIG_DRM_ACCEL_ANE_KUNIT_TEST) += ane_drv_test.o obj-$(CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST) += ane_fw_platform_test.o \ diff --git a/drivers/accel/ane/ane_t6021_lifetime.h b/drivers/accel/ane/ane_t6021_lifetime.h index 30a5ca4a8f28d..76dd48e538942 100644 --- a/drivers/accel/ane/ane_t6021_lifetime.h +++ b/drivers/accel/ane/ane_t6021_lifetime.h @@ -667,13 +667,14 @@ struct ane_t6021_restart_ops { int (*boot)(void *ctx); /* cold boot and handshake */ }; -static inline int ane_t6021_restart_run(const struct ane_t6021_restart_ops *ops, - void *ctx, bool planned, - enum ane_t6021_restart_stage *stage) +/* The stop boundary is shared by runtime recovery and system sleep. + * The PM owner removes/restores power only after this returns success. + */ +static inline int ane_t6021_restart_stop(const struct ane_t6021_restart_ops *ops, + void *ctx, bool planned) { int ret; - *stage = ANE_T6021_RESTART_STOP; if (planned) { ret = ops->power_down(ctx); if (!ret) @@ -681,7 +682,17 @@ static inline int ane_t6021_restart_run(const struct ane_t6021_restart_ops *ops, if (ret) return ret; } - ret = ops->halt(ctx); + return ops->halt(ctx); +} + +static inline int ane_t6021_restart_run(const struct ane_t6021_restart_ops *ops, + void *ctx, bool planned, + enum ane_t6021_restart_stage *stage) +{ + int ret; + + *stage = ANE_T6021_RESTART_STOP; + ret = ane_t6021_restart_stop(ops, ctx, planned); if (ret) return ret; *stage = ANE_T6021_RESTART_CYCLE; diff --git a/drivers/accel/ane/ane_t6021_lifetime_test.c b/drivers/accel/ane/ane_t6021_lifetime_test.c index 501e1c75d11cc..4732f378780d9 100644 --- a/drivers/accel/ane/ane_t6021_lifetime_test.c +++ b/drivers/accel/ane/ane_t6021_lifetime_test.c @@ -681,6 +681,29 @@ static const struct ane_t6021_restart_ops fr_ops = { .boot = fr_boot, }; +static void restart_stop_boundary_test(struct kunit *test) +{ + struct fake_restart r = {}; + static const char failures[] = "pwh"; + unsigned int i; + + /* System suspend stops here; its PM owner performs the power transition. */ + KUNIT_EXPECT_EQ(test, ane_t6021_restart_stop(&fr_ops, &r, true), 0); + KUNIT_EXPECT_STREQ(test, r.log, "pwh"); + memset(&r, 0, sizeof(r)); + KUNIT_EXPECT_EQ(test, ane_t6021_restart_stop(&fr_ops, &r, false), 0); + KUNIT_EXPECT_STREQ(test, r.log, "h"); + for (i = 0; i < sizeof(failures) - 1; i++) { + memset(&r, 0, sizeof(r)); + r.fail = failures[i]; + KUNIT_EXPECT_EQ(test, ane_t6021_restart_stop(&fr_ops, &r, true), + -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, r.n, i + 1); + KUNIT_EXPECT_NULL(test, strchr(r.log, 'c')); + KUNIT_EXPECT_NULL(test, strchr(r.log, 'b')); + } +} + static void restart_state_machine_test(struct kunit *test) { enum ane_t6021_restart_stage stage; @@ -703,7 +726,7 @@ static void restart_state_machine_test(struct kunit *test) KUNIT_EXPECT_STREQ(test, r.log, "pwh"); KUNIT_EXPECT_EQ(test, stage, ANE_T6021_RESTART_STOP); - /* No POWER_DOWN completion: not even the RUN clear is tried. */ + /* POWER_DOWN not accepted: not even the RUN clear is tried. */ memset(&r, 0, sizeof(r)); r.fail = 'p'; KUNIT_EXPECT_NE(test, ane_t6021_restart_run(&fr_ops, &r, true, &stage), 0); @@ -743,6 +766,7 @@ static struct kunit_case lifetime_cases[] = { KUNIT_CASE(progtab_cancel_test), KUNIT_CASE(unload_order_test), KUNIT_CASE(restart_state_machine_test), + KUNIT_CASE(restart_stop_boundary_test), {} }; diff --git a/drivers/accel/ane/ane_t6021_pm.h b/drivers/accel/ane/ane_t6021_pm.h new file mode 100644 index 0000000000000..3a3f57a4c59bb --- /dev/null +++ b/drivers/accel/ane/ane_t6021_pm.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_T6021_PM_H__ +#define __ANE_T6021_PM_H__ + +#include +#include +#include +#include + +/* Every state access, including submission checks, holds the firmware mutex. */ +enum ane_t6021_pm_state { + ANE_T6021_PM_ACTIVE, + ANE_T6021_PM_SUSPENDED, + ANE_T6021_PM_FAILED, +}; + +/* + * Adapters to the shared H14 lifetime implementation, not another firmware + * restart path. stop must complete protocol shutdown and prove ASC STOPPED + * with RUN clear before system PM may remove power. A failed stop retains + * the power claim, DMA mappings and module pin. restart must use the same + * cold-start/handshake/program-replay path as timeout recovery, returning + * success only once existing public IDs are usable again. + * + * These callbacks hold the existing firmware mutex. Neither callback may use + * runtime-PM force/cycle helpers while system PM has disabled runtime PM. + * Supplier and DART system callbacks perform power removal/restoration around + * these consumer callbacks. RX shutdown and restoration belong to the shared + * lifetime implementation along with the firmware state. + */ +struct ane_t6021_pm_ops { + int (*stop)(void *ctx); + int (*restart)(void *ctx); +}; + +struct ane_t6021_pm { + /* Existing firmware command mutex, shared with submitters. */ + struct mutex *fw_lock; + const struct ane_t6021_pm_ops *ops; + void *ctx; + enum ane_t6021_pm_state state; +}; + +static inline int ane_t6021_pm_submit_error(enum ane_t6021_pm_state state) +{ + return state == ANE_T6021_PM_ACTIVE ? 0 : -EHOSTDOWN; +} + +static inline int ane_t6021_pm_suspend(struct ane_t6021_pm *pm) +{ + int ret = -EHOSTDOWN; + + /* Drains the current synchronous command and its finish event, then + * fences queued submitters before handing off to the shared stop path. + */ + mutex_lock(pm->fw_lock); + if (pm->state != ANE_T6021_PM_ACTIVE) + goto out; + pm->state = ANE_T6021_PM_FAILED; + ret = pm->ops->stop(pm->ctx); + if (!ret) + pm->state = ANE_T6021_PM_SUSPENDED; +out: + mutex_unlock(pm->fw_lock); + return ret; +} + +static inline int ane_t6021_pm_resume(struct ane_t6021_pm *pm) +{ + int ret = -EHOSTDOWN; + + mutex_lock(pm->fw_lock); + /* PM core may unwind before it reached this device's suspend. */ + if (pm->state == ANE_T6021_PM_ACTIVE) { + ret = 0; + goto out; + } + if (pm->state != ANE_T6021_PM_SUSPENDED) + goto out; + pm->state = ANE_T6021_PM_FAILED; + ret = pm->ops->restart(pm->ctx); + if (!ret) + pm->state = ANE_T6021_PM_ACTIVE; +out: + mutex_unlock(pm->fw_lock); + return ret; +} + +/* The getter preserves the driver's own drvdata layout and returns its PM + * adapter, initialized once at probe with the shared lifetime callbacks. + */ +#define DEFINE_ANE_T6021_SYSTEM_PM_OPS(name, get_pm) \ +static int __maybe_unused name##_suspend(struct device *dev) \ +{ \ + return ane_t6021_pm_suspend(get_pm(dev)); \ +} \ +static int __maybe_unused name##_resume(struct device *dev) \ +{ \ + return ane_t6021_pm_resume(get_pm(dev)); \ +} \ +static const struct dev_pm_ops __maybe_unused name = { \ + SYSTEM_SLEEP_PM_OPS(name##_suspend, name##_resume) \ +} + +#endif /* __ANE_T6021_PM_H__ */ diff --git a/drivers/accel/ane/ane_t6021_pm_test.c b/drivers/accel/ane/ane_t6021_pm_test.c new file mode 100644 index 0000000000000..2df11cfb310c6 --- /dev/null +++ b/drivers/accel/ane/ane_t6021_pm_test.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include + +#include "ane_t6021_pm.h" + +struct ane_sleep_test { + struct kunit *test; + struct device dev; + /* Models the driver command mutex. */ + struct mutex fw_lock; + struct ane_t6021_pm pm; + unsigned int stops, restarts; + int stop_error, restart_error; +}; + +static struct ane_t6021_pm *ane_sleep_get_pm(struct device *dev) +{ + return dev_get_drvdata(dev); +} + +DEFINE_ANE_T6021_SYSTEM_PM_OPS(ane_sleep_dev_pm, ane_sleep_get_pm); + +static int ane_sleep_stop(void *data) +{ + struct ane_sleep_test *ctx = data; + + KUNIT_EXPECT_TRUE(ctx->test, mutex_is_locked(&ctx->fw_lock)); + KUNIT_EXPECT_EQ(ctx->test, ane_t6021_pm_submit_error(ctx->pm.state), -EHOSTDOWN); + KUNIT_EXPECT_EQ(ctx->test, ctx->stops, ctx->restarts); + ctx->stops++; + return ctx->stop_error; +} + +static int ane_sleep_restart(void *data) +{ + struct ane_sleep_test *ctx = data; + + KUNIT_EXPECT_TRUE(ctx->test, mutex_is_locked(&ctx->fw_lock)); + KUNIT_EXPECT_EQ(ctx->test, ane_t6021_pm_submit_error(ctx->pm.state), -EHOSTDOWN); + KUNIT_EXPECT_EQ(ctx->test, ctx->stops, ctx->restarts + 1); + ctx->restarts++; + return ctx->restart_error; +} + +static const struct ane_t6021_pm_ops ane_sleep_ops = { + .stop = ane_sleep_stop, + .restart = ane_sleep_restart, +}; + +static int ane_sleep_init(struct kunit *test) +{ + struct ane_sleep_test *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + + if (!ctx) + return -ENOMEM; + ctx->test = test; + mutex_init(&ctx->fw_lock); + ctx->pm = (struct ane_t6021_pm) { + .fw_lock = &ctx->fw_lock, + .ops = &ane_sleep_ops, + .ctx = ctx, + .state = ANE_T6021_PM_ACTIVE, + }; + dev_set_drvdata(&ctx->dev, &ctx->pm); + test->priv = ctx; + return 0; +} + +static void ane_sleep_exit(struct kunit *test) +{ + struct ane_sleep_test *ctx = test->priv; + + mutex_destroy(&ctx->fw_lock); +} + +static void ane_sleep_cycles_test(struct kunit *test) +{ + struct ane_sleep_test *ctx = test->priv; + unsigned int i; + + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_EQ(test, ane_sleep_dev_pm_suspend(&ctx->dev), 0); + KUNIT_EXPECT_FALSE(test, mutex_is_locked(&ctx->fw_lock)); + KUNIT_EXPECT_EQ(test, ctx->pm.state, ANE_T6021_PM_SUSPENDED); + KUNIT_EXPECT_EQ(test, ane_t6021_pm_submit_error(ctx->pm.state), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, ane_sleep_dev_pm_suspend(&ctx->dev), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, ctx->stops, i + 1); + KUNIT_ASSERT_EQ(test, ane_sleep_dev_pm_resume(&ctx->dev), 0); + KUNIT_EXPECT_FALSE(test, mutex_is_locked(&ctx->fw_lock)); + KUNIT_EXPECT_EQ(test, ane_t6021_pm_submit_error(ctx->pm.state), 0); + KUNIT_EXPECT_EQ(test, ctx->restarts, i + 1); + } +} + +static void ane_sleep_stop_failure_test(struct kunit *test) +{ + struct ane_sleep_test *ctx = test->priv; + + ctx->stop_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, ane_sleep_dev_pm_suspend(&ctx->dev), -ETIMEDOUT); + KUNIT_EXPECT_FALSE(test, mutex_is_locked(&ctx->fw_lock)); + KUNIT_EXPECT_EQ(test, ctx->pm.state, ANE_T6021_PM_FAILED); + KUNIT_EXPECT_EQ(test, ane_t6021_pm_submit_error(ctx->pm.state), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, ane_sleep_dev_pm_resume(&ctx->dev), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, ane_sleep_dev_pm_suspend(&ctx->dev), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, ctx->stops, 1U); + KUNIT_EXPECT_EQ(test, ctx->restarts, 0U); +} + +static void ane_sleep_restart_failure_test(struct kunit *test) +{ + struct ane_sleep_test *ctx = test->priv; + + KUNIT_ASSERT_EQ(test, ane_sleep_dev_pm_suspend(&ctx->dev), 0); + ctx->restart_error = -EIO; + KUNIT_EXPECT_EQ(test, ane_sleep_dev_pm_resume(&ctx->dev), -EIO); + KUNIT_EXPECT_FALSE(test, mutex_is_locked(&ctx->fw_lock)); + KUNIT_EXPECT_EQ(test, ctx->pm.state, ANE_T6021_PM_FAILED); + KUNIT_EXPECT_EQ(test, ane_t6021_pm_submit_error(ctx->pm.state), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, ane_sleep_dev_pm_resume(&ctx->dev), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, ctx->restarts, 1U); +} + +static void ane_sleep_abort_test(struct kunit *test) +{ + struct ane_sleep_test *ctx = test->priv; + + KUNIT_EXPECT_EQ(test, ane_sleep_dev_pm_resume(&ctx->dev), 0); + KUNIT_EXPECT_EQ(test, ane_t6021_pm_submit_error(ctx->pm.state), 0); + KUNIT_EXPECT_EQ(test, ctx->stops, 0U); + KUNIT_EXPECT_EQ(test, ctx->restarts, 0U); +} + +static void ane_sleep_callback_mapping_test(struct kunit *test) +{ + if (IS_ENABLED(CONFIG_PM_SLEEP)) { + KUNIT_EXPECT_TRUE(test, ane_sleep_dev_pm.suspend == ane_sleep_dev_pm_suspend); + KUNIT_EXPECT_TRUE(test, ane_sleep_dev_pm.freeze == ane_sleep_dev_pm_suspend); + KUNIT_EXPECT_TRUE(test, ane_sleep_dev_pm.poweroff == ane_sleep_dev_pm_suspend); + KUNIT_EXPECT_TRUE(test, ane_sleep_dev_pm.resume == ane_sleep_dev_pm_resume); + KUNIT_EXPECT_TRUE(test, ane_sleep_dev_pm.thaw == ane_sleep_dev_pm_resume); + KUNIT_EXPECT_TRUE(test, ane_sleep_dev_pm.restore == ane_sleep_dev_pm_resume); + } else { + KUNIT_EXPECT_NULL(test, ane_sleep_dev_pm.suspend); + KUNIT_EXPECT_NULL(test, ane_sleep_dev_pm.freeze); + KUNIT_EXPECT_NULL(test, ane_sleep_dev_pm.poweroff); + KUNIT_EXPECT_NULL(test, ane_sleep_dev_pm.resume); + KUNIT_EXPECT_NULL(test, ane_sleep_dev_pm.thaw); + KUNIT_EXPECT_NULL(test, ane_sleep_dev_pm.restore); + } +} + +static struct kunit_case ane_sleep_cases[] = { + KUNIT_CASE(ane_sleep_cycles_test), + KUNIT_CASE(ane_sleep_stop_failure_test), + KUNIT_CASE(ane_sleep_restart_failure_test), + KUNIT_CASE(ane_sleep_abort_test), + KUNIT_CASE(ane_sleep_callback_mapping_test), + {} +}; + +static struct kunit_suite ane_sleep_suite = { + .name = "ane-h14-sleep", + .test_cases = ane_sleep_cases, + .init = ane_sleep_init, + .exit = ane_sleep_exit, +}; + +kunit_test_suite(ane_sleep_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index ed0735e43852d..e67ff1d42cdf8 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -104,6 +104,7 @@ #include "ane_t6021.h" #include "ane_t6021_boot.h" #include "ane_t6021_lifetime.h" +#include "ane_t6021_pm.h" #include @@ -238,6 +239,8 @@ struct ane_rtclient_retired { struct ane_t6021_boot_allocs boot; }; +struct ane_rtclient_restart; + struct ane_rtclient { struct device *dev; void __iomem *engine; @@ -256,6 +259,8 @@ struct ane_rtclient { struct apple_rtkit *rtk; struct reset_control *cpu_rst; struct delayed_work poll_work; + struct ane_t6021_pm sleep_pm; + struct ane_rtclient_restart *sleep_restart; /* fw_start=1 view over the shared boot/fwload helpers. */ struct ane_t6021 *fw; @@ -1488,6 +1493,8 @@ static int ane_t6021_unload_cmd(void *ctx, u16 opcode, u32 fw_prog, u32 fw_proc) size_t length = opcode == ANE_T6021_CMD_TERMINATE_PROCESS ? 0x10 : 0x0c; int ret; + if (ane_t6021_pm_submit_error(ane->sleep_pm.state)) + return -EHOSTDOWN; if (!command || atomic_read(&ane_t6021_quarantined)) return -ENODEV; memset(command->cpu, 0, SZ_16K); @@ -1950,6 +1957,10 @@ static int ane_rtclient_h16_call(struct ane_rtclient *ane, struct drm_file *file if (IS_ERR(ios)) return PTR_ERR(ios); mutex_lock(&ane_t6021_fw_lock); + if (ane_t6021_pm_submit_error(ane->sleep_pm.state)) { + ret = -EHOSTDOWN; + goto out; + } if (atomic_read(&ane_t6021_quarantined) || ane->pending.quarantined) { ret = -EIO; goto out; @@ -2108,6 +2119,10 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, } mutex_lock(&ane_t6021_fw_lock); + if (ane_t6021_pm_submit_error(ane->sleep_pm.state)) { + ret = -EHOSTDOWN; + goto unlock; + } if (atomic_read(&ane_t6021_quarantined)) { ret = -ETIMEDOUT; goto unlock; @@ -2217,6 +2232,8 @@ static int ane_t6021_bo_init_locked(struct drm_device *drm, void *data, struct ane_t6021_bo *bo; struct ane_rtclient *ane = to_ane_t6021_drm(drm)->ane; + if (ane_t6021_pm_submit_error(ane->sleep_pm.state)) + return -EHOSTDOWN; if (atomic_read(&ane_t6021_quarantined) || READ_ONCE(ane->recovering)) return -EIO; if (args->pad || args->size == 0 || args->size > ANE_T6021_BO_MAX || @@ -2304,8 +2321,8 @@ static void ane_t6021_bo_drop(struct ane_t6021_bo *bo) kref_put(&bo->refcount, ane_t6021_bo_release); } -static int ane_t6021_bo_free_ioctl(struct drm_device *drm, void *data, - struct drm_file *file) +static int ane_t6021_bo_free_locked(struct drm_device *drm, void *data, + struct drm_file *file) { struct drm_ane_bo_free *args = data; struct ane_t6021_fd *fd = file->driver_priv; @@ -2328,6 +2345,20 @@ static int ane_t6021_bo_free_ioctl(struct drm_device *drm, void *data, return bo ? 0 : -ENOENT; } +static int ane_t6021_bo_free_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct ane_t6021_drm *adrm = to_ane_t6021_drm(drm); + int ret; + + mutex_lock(&ane_t6021_fw_lock); + ret = ane_t6021_pm_submit_error(adrm->ane->sleep_pm.state); + if (!ret) + ret = ane_t6021_bo_free_locked(drm, data, file); + mutex_unlock(&ane_t6021_fw_lock); + return ret; +} + /* Resolve the BO a user mmap names (BO_INIT returned offset = handle * << PAGE_SHIFT) and map the coherent buffer cacheably — the device * half coheres through the DART (IOMMU_CACHE), so reads after EXEC @@ -2455,6 +2486,10 @@ static int ane_t6021_prog_load_ioctl(struct drm_device *drm, void *data, int ret; mutex_lock(&ane_t6021_fw_lock); + if (ane_t6021_pm_submit_error(adrm->ane->sleep_pm.state)) { + mutex_unlock(&ane_t6021_fw_lock); + return -EHOSTDOWN; + } if (atomic_read(&ane_t6021_quarantined)) { mutex_unlock(&ane_t6021_fw_lock); return -ETIMEDOUT; @@ -2514,6 +2549,10 @@ static int ane_t6021_perf_mode_set(const char *val, if (!ane) return -ENODEV; mutex_lock(&ane_t6021_fw_lock); + if (ane_t6021_pm_submit_error(ane->sleep_pm.state)) { + mutex_unlock(&ane_t6021_fw_lock); + return -EHOSTDOWN; + } if (atomic_read(&ane_t6021_quarantined)) { mutex_unlock(&ane_t6021_fw_lock); return -ETIMEDOUT; @@ -2553,6 +2592,10 @@ static int ane_t6021_proc_create_ioctl(struct drm_device *drm, void *data, int ret; mutex_lock(&ane_t6021_fw_lock); + if (ane_t6021_pm_submit_error(adrm->ane->sleep_pm.state)) { + mutex_unlock(&ane_t6021_fw_lock); + return -EHOSTDOWN; + } if (atomic_read(&ane_t6021_quarantined)) { mutex_unlock(&ane_t6021_fw_lock); return -ETIMEDOUT; @@ -3169,29 +3212,39 @@ static int ane_rtclient_rs_halt(void *ctx) return ret; } -static int ane_rtclient_rs_power_cycle(void *ctx) +/* Shared post-power-transition admission, including system resume. */ +static int ane_rtclient_rs_ready(void *ctx) { struct ane_rtclient_restart *rs = ctx; struct ane_rtclient *ane = rs->ane; u32 status; int ret; - /* Probe retains one consumer reference for this firmware lifetime. */ - ret = ane_pm_cycle(ane->dev, rs->cycle.pd->pd_dev, rs->cycle.pd->count, - ane_rtclient_verify_off, &rs->cycle); - if (ret) - return ret; - /* Repeat probe's gates before any engine-aperture read. */ ret = ane_rtclient_engine_accessible(ane); if (ret) return ret; status = readl(ane->engine + ANE_ASC_CPU_STATUS); if ((status & ANE_ASC_CPU_STATUS_RUNNING) || - !(status & ANE_ASC_CPU_STATUS_STOPPED)) + !(status & ANE_ASC_CPU_STATUS_STOPPED) || + (readl(ane->engine + ANE_ASC_CPU_CONTROL) & ANE_ASC_CPU_CONTROL_RUN)) return -EBUSY; return 0; } +static int ane_rtclient_rs_power_cycle(void *ctx) +{ + struct ane_rtclient_restart *rs = ctx; + struct ane_rtclient *ane = rs->ane; + int ret; + + /* Probe retains one consumer reference for this firmware lifetime. */ + ret = ane_pm_cycle(ane->dev, rs->cycle.pd->pd_dev, rs->cycle.pd->count, + ane_rtclient_verify_off, &rs->cycle); + if (ret) + return ret; + return ane_rtclient_rs_ready(ctx); +} + static int ane_rtclient_rs_boot(void *ctx) { struct ane_rtclient_restart *rs = ctx; @@ -3291,28 +3344,19 @@ static const struct ane_t6021_restart_ops ane_rtclient_restart_ops = { .boot = ane_rtclient_rs_boot, }; -/* - * Cold-restart the Linux-owned firmware. @planned: the instance is live and - * idle (the caller holds the firmware lock, so no command or call is in - * flight) and is asked to shut down first. Otherwise a command timed out - * and only the RUN clear is tried. The ASC must be proven stopped before - * its domains go off; if it is not, nothing changes and the device stays - * quarantined with every mapping, its power claim and its pin. +/* Allocate the lifetime owner's transaction before either PM path stops it. + * The same transaction survives the system power boundary in sleep_restart. */ -static int __ane_rtclient_recover(struct ane_rtclient *ane, bool planned) +static int ane_rtclient_rs_prepare(struct ane_rtclient_restart *rs) { + struct ane_rtclient *ane = rs->ane; struct ane_t6021 *a = ane->fw; - struct ane_rtclient_restart rs = { .ane = ane, .cycle.ane = ane }; struct ane_rtclient_pd *pd; - enum ane_t6021_restart_stage stage = ANE_T6021_RESTART_STOP; unsigned int domains = ane->soc->soc == 0x8112 ? 7 : 8; - int ret; lockdep_assert_held(&ane_t6021_fw_lock); if (!ane->registered) return -EOPNOTSUPP; - if (!planned && !atomic_read(&ane_t6021_quarantined)) - return 0; /* Only the qualified, Linux-owned legacy boot profile can restart. * Borrowed firmware, RTKit and later-generation ownership stay fenced. */ @@ -3326,24 +3370,32 @@ static int __ane_rtclient_recover(struct ane_rtclient *ane, bool planned) mutex_lock(&ane_rtclient_pd_lock); list_for_each_entry(pd, &ane_rtclient_pd_list, list) if (pd->dev == ane->dev && pd->count == domains) - rs.cycle.pd = pd; + rs->cycle.pd = pd; mutex_unlock(&ane_rtclient_pd_lock); - if (!rs.cycle.pd || !ane->engine_ps_valid) + if (!rs->cycle.pd || !ane->engine_ps_valid) return -EOPNOTSUPP; - rs.retired = kzalloc_obj(*rs.retired); - if (!rs.retired) + rs->retired = kzalloc_obj(*rs->retired); + if (!rs->retired) return -ENOMEM; - rs.fresh = devm_kcalloc(ane->dev, ANE_LEGACY_ALLOCS, sizeof(*rs.fresh), - GFP_KERNEL); - if (!rs.fresh) { - kfree(rs.retired); + rs->fresh = devm_kcalloc(ane->dev, ANE_LEGACY_ALLOCS, sizeof(*rs->fresh), + GFP_KERNEL); + if (!rs->fresh) { + kfree(rs->retired); return -ENOMEM; } WRITE_ONCE(ane->recovering, true); cancel_delayed_work_sync(&ane->poll_work); - ret = ane_t6021_restart_run(&ane_rtclient_restart_ops, &rs, planned, - &stage); + return 0; +} + +/* Commit or retain the lifetime owner's transaction in exactly one place. */ +static int ane_rtclient_rs_finish(struct ane_rtclient_restart *rs, bool planned, + enum ane_t6021_restart_stage stage, int ret) +{ + struct ane_rtclient *ane = rs->ane; + + lockdep_assert_held(&ane_t6021_fw_lock); if (ret) goto failed; /* Opaque IDs never alias a new firmware instance; programs files still @@ -3355,18 +3407,19 @@ static int __ane_rtclient_recover(struct ane_rtclient *ane, bool planned) ane); ane_rtclient_reap_retired(ane); WRITE_ONCE(ane->recovering, false); + ane->sleep_pm.state = ANE_T6021_PM_ACTIVE; dev_info(ane->dev, "firmware %s; programs reload on next use\n", planned ? "restarted" : "recovered"); return 0; failed: - if (rs.fresh) - devm_kfree(ane->dev, rs.fresh); - kfree(rs.retired); + if (rs->fresh) + devm_kfree(ane->dev, rs->fresh); + kfree(rs->retired); /* A POWER_DOWN that was never posted left the instance in service. * Once posted, the firmware is shut down or about to be. */ - if (!rs.live) + if (!rs->live) atomic_set(&ane_t6021_quarantined, 1); WRITE_ONCE(ane->recovering, false); dev_err(ane->dev, @@ -3378,9 +3431,37 @@ static int __ane_rtclient_recover(struct ane_rtclient *ane, bool planned) return ret; } +/* + * Cold-restart the Linux-owned firmware. @planned: the instance is live and + * idle (the caller holds the firmware lock, so no command or call is in + * flight) and is asked to shut down first. Otherwise a command timed out + * and only the RUN clear is tried. The ASC must be proven stopped before + * its domains go off; if it is not, nothing changes and the device stays + * quarantined with every mapping, its power claim and its pin. + */ +static int __ane_rtclient_recover(struct ane_rtclient *ane, bool planned) +{ + struct ane_rtclient_restart rs = { .ane = ane, .cycle.ane = ane }; + enum ane_t6021_restart_stage stage = ANE_T6021_RESTART_STOP; + int ret; + + lockdep_assert_held(&ane_t6021_fw_lock); + if (!ane->registered) + return -EOPNOTSUPP; + if (!planned && !atomic_read(&ane_t6021_quarantined)) + return 0; + ret = ane_rtclient_rs_prepare(&rs); + if (ret) + return ret; + ret = ane_t6021_restart_run(&ane_rtclient_restart_ops, &rs, planned, &stage); + return ane_rtclient_rs_finish(&rs, planned, stage, ret); +} + static int ane_rtclient_recover(struct ane_rtclient *ane) { - return __ane_rtclient_recover(ane, false); + return __ane_rtclient_recover(ane, + !atomic_read(&ane_t6021_quarantined) && + ane->sleep_pm.state == ANE_T6021_PM_FAILED); } /* @@ -3399,6 +3480,74 @@ static int ane_rtclient_restart(struct ane_rtclient *ane) return ret == -EOPNOTSUPP || ret == -ENOSPC ? -ENOSPC : ret; } +/* System PM owns the power transition; it has disabled runtime PM and + * holds an extra consumer reference. Reuse the lifetime stop/boot/commit + * boundaries without invoking the runtime power-cycle transaction. + */ +static int ane_rtclient_sleep_stop(void *ctx) +{ + struct ane_rtclient *ane = ctx; + struct ane_rtclient_restart *rs; + int ret; + + lockdep_assert_held(&ane_t6021_fw_lock); + if (atomic_read(&ane_t6021_quarantined) || ane->sleep_restart) + return -EBUSY; + rs = kzalloc_obj(*rs); + if (!rs) + return -ENOMEM; + rs->ane = ane; + rs->cycle.ane = ane; + ret = ane_rtclient_rs_prepare(rs); + if (ret) + goto free; + ret = ane_t6021_restart_stop(&ane_rtclient_restart_ops, rs, true); + if (ret) { + ane_rtclient_rs_finish(rs, true, ANE_T6021_RESTART_STOP, ret); + goto free; + } + ane->sleep_restart = rs; + return 0; +free: + kfree(rs); + return ret; +} + +static int ane_rtclient_sleep_restart(void *ctx) +{ + struct ane_rtclient *ane = ctx; + struct ane_rtclient_restart *rs = ane->sleep_restart; + enum ane_t6021_restart_stage stage = ANE_T6021_RESTART_CYCLE; + int ret; + + lockdep_assert_held(&ane_t6021_fw_lock); + if (!rs) + return -EIO; + ane->sleep_restart = NULL; + ret = ane_rtclient_rs_ready(rs); + if (!ret) { + stage = ANE_T6021_RESTART_BOOT; + ret = ane_rtclient_rs_boot(rs); + } + ret = ane_rtclient_rs_finish(rs, true, stage, ret); + kfree(rs); + return ret; +} + +static const struct ane_t6021_pm_ops ane_rtclient_sleep_ops = { + .stop = ane_rtclient_sleep_stop, + .restart = ane_rtclient_sleep_restart, +}; + +static struct ane_t6021_pm *ane_rtclient_get_sleep_pm(struct device *dev) +{ + struct ane_rtclient *ane = dev_get_drvdata(dev); + + return &ane->sleep_pm; +} + +DEFINE_ANE_T6021_SYSTEM_PM_OPS(ane_rtclient_sleep_pm, ane_rtclient_get_sleep_pm); + static int ane_rtclient_probe_inner(struct platform_device *pdev, bool *hardware_touched) { @@ -3454,6 +3603,12 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, ane->soc = of_device_get_match_data(dev); platform_set_drvdata(pdev, ane); INIT_DELAYED_WORK(&ane->poll_work, ane_rtclient_post_boot); + ane->sleep_pm = (struct ane_t6021_pm) { + .fw_lock = &ane_t6021_fw_lock, + .ops = &ane_rtclient_sleep_ops, + .ctx = ane, + .state = ANE_T6021_PM_ACTIVE, + }; INIT_LIST_HEAD(&ane->retired); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -3952,7 +4107,8 @@ static ssize_t wedged_show(struct device *dev, struct device_attribute *attr, struct ane_rtclient *ane = dev_get_drvdata(dev); return sysfs_emit(buf, "%d\n", atomic_read(&ane_t6021_quarantined) || - READ_ONCE(ane->recovering) || !READ_ONCE(ane->registered)); + READ_ONCE(ane->recovering) || !READ_ONCE(ane->registered) || + READ_ONCE(ane->sleep_pm.state) == ANE_T6021_PM_FAILED); } static DEVICE_ATTR_RO(wedged); @@ -4002,6 +4158,7 @@ static struct platform_driver ane_rtclient_driver = { .name = "ane_t6021", .of_match_table = ane_rtclient_of_match, .dev_groups = ane_t6021_stats_groups, + .pm = pm_sleep_ptr(&ane_rtclient_sleep_pm), .suppress_bind_attrs = true, }, .probe = ane_rtclient_probe, From 51d33d37fca0badf32047b732bedf89edc5b79f2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 15:54:25 +1100 Subject: [PATCH 54/73] accel/ane: clarify PM phases and describe the sleep tests Runtime PM is disabled during the late/noirq phases, between the normal suspend and resume callbacks. Clarify that ordering and wrap the adapter documentation. The system callbacks still leave the supplier transition to system PM and do not invoke the runtime recovery power cycle. Add the sleep test module description required by modpost. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/accel/apple-ane.rst | 28 +++++++++++---------- drivers/accel/ane/ane_t6021_pm.h | 2 +- drivers/accel/ane/ane_t6021_pm_test.c | 1 + drivers/accel/ane/ane_t6021_rtclient_main.c | 7 +++--- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index 0739c351853a0..2253c72015f1c 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -455,10 +455,10 @@ The adapter contract is: ``ANE_T6021_PM_ACTIVE`` state. * Use ``DEFINE_ANE_T6021_SYSTEM_PM_OPS(name, getter)`` to define the device PM operations. The getter returns the embedded adapter without replacing the - driver's own drvdata. The driver binds ``pm_sleep_ptr(&name)`` to ``driver.pm`` and supplies - the shared lifetime callbacks below. -* Every external firmware-facing submission checks ``ane_t6021_pm_submit_error()`` under - that same firmware mutex. Suspend acquires the mutex, draining the current + driver's own drvdata. The driver binds ``pm_sleep_ptr(&name)`` to + ``driver.pm`` and supplies the shared lifetime callbacks below. +* Every external firmware-facing submission checks + ``ane_t6021_pm_submit_error()`` under that same firmware mutex. Suspend acquires the mutex, draining the current synchronous command and its finish event, and fences queued submitters before calling the shared stop callback. * The stop callback performs the validated firmware shutdown handshake, drains @@ -467,14 +467,15 @@ The adapter contract is: power claims, DMA mappings, firmware backing and the module pin. * The restart callback is the same cold-start/handshake/program-replay path used by timeout recovery. Supplier and DART system callbacks must restore - access first. Return success only after channels, CONFIG_GET and desired firmware - properties are restored and the lifetime owner has made public IDs usable - under its normal lazy-reload policy. RX and all firmware-owned resources stay with this one lifetime - implementation. - -System PM disables runtime PM and holds an additional consumer reference. -Consequently suspend stops at ``ane_t6021_restart_stop()`` and keeps the shared -restart transaction on the client across the system power boundary. Genpd and + access first. Return success only after channels, CONFIG_GET and desired + firmware properties are restored and the lifetime owner has made public IDs + usable under its normal lazy-reload policy. RX and all firmware-owned + resources stay with this one lifetime implementation. + +System PM holds an additional consumer reference and disables runtime PM +during the late/noirq phases. Suspend stops at ``ane_t6021_restart_stop()`` and +keeps the shared restart transaction on the client across the system power +boundary. Genpd and DART system callbacks perform that power transition. Resume checks provider access and the stopped-core predicate, then invokes the same boot and commit functions as runtime recovery. Runtime recovery still uses the shared runtime-PM @@ -506,7 +507,8 @@ M2 validation plan ------------------ Runtime testing requires the shared lifetime implementation and this PM -integration together. Do not infer a hardware pass from the adapter's KUnit results. +integration together. Do not infer a hardware pass from the adapter's KUnit +results. Run on a T8112 with the candidate kernel and its matching ane_t6021 module, through the hardware controller with exclusive device ownership, bounded console capture and a tested boot fallback. Record the kernel/module revision, diff --git a/drivers/accel/ane/ane_t6021_pm.h b/drivers/accel/ane/ane_t6021_pm.h index 3a3f57a4c59bb..ddd169fc01af6 100644 --- a/drivers/accel/ane/ane_t6021_pm.h +++ b/drivers/accel/ane/ane_t6021_pm.h @@ -7,7 +7,7 @@ #include #include -/* Every state access, including submission checks, holds the firmware mutex. */ +/* State transitions and submission checks hold the firmware mutex. */ enum ane_t6021_pm_state { ANE_T6021_PM_ACTIVE, ANE_T6021_PM_SUSPENDED, diff --git a/drivers/accel/ane/ane_t6021_pm_test.c b/drivers/accel/ane/ane_t6021_pm_test.c index 2df11cfb310c6..fe988c5621d21 100644 --- a/drivers/accel/ane/ane_t6021_pm_test.c +++ b/drivers/accel/ane/ane_t6021_pm_test.c @@ -171,3 +171,4 @@ static struct kunit_suite ane_sleep_suite = { kunit_test_suite(ane_sleep_suite); MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("ANE H14 system sleep adapter tests"); diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index e67ff1d42cdf8..d5691ff084fca 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -3480,9 +3480,10 @@ static int ane_rtclient_restart(struct ane_rtclient *ane) return ret == -EOPNOTSUPP || ret == -ENOSPC ? -ENOSPC : ret; } -/* System PM owns the power transition; it has disabled runtime PM and - * holds an extra consumer reference. Reuse the lifetime stop/boot/commit - * boundaries without invoking the runtime power-cycle transaction. +/* System PM owns the supplier transition and holds a consumer reference. + * Reuse the lifetime stop/boot/commit boundaries without invoking the + * runtime power-cycle transaction. Runtime PM is disabled in the late/noirq + * phases, after this stop and before the normal resume callback. */ static int ane_rtclient_sleep_stop(void *ctx) { From e3738f0bff97ed97f179cb0697d6c4bec30fb0ea Mon Sep 17 00:00:00 2001 From: AxxzyWasTaken Date: Fri, 30 Oct 2026 18:29:18 +0000 Subject: [PATCH 55/73] drm/apple: Drive desktop 26.x EDR panels at their peak EDR swaps on a desktop 26.x panel carry the EDR scale (SDR white over the peak) and the headroom, but leave the panel level at SDR white. The firmware shows a PQ surface at (PQ nits / 100) x the EDR scale x the panel level the swap asks for, so with that level the whole PQ range ends up at or below SDR white. On a J616C at 100 nits, full-screen PQ 100, 400, 1000 and 1600 drew 0.30, 0.58, 1.22 and 1.88 W of backlight power, against 1.81 W for SDR white. Ask for the panel at its peak in EDR swaps as well, and keep the scale at SDR white over the peak. PQ 100 then shows at the user's level at any brightness, and highlights reach the peak: SDR level SDR white PQ 100 PQ 400 PQ 1000 (full screen) 100 nits 1.81 W 1.81 W 7.77 W 20.2 W 200 nits 3.53 W 3.59 W 17.1 W 22.9 W 300 nits 5.33 W 5.33 W 22.6 W 23.1 W A swap that carries a level of its own (a brightness change) gets the peak in its place and the scale from its level, so a change on a still HDR frame takes effect at once. Ending EDR puts the panel back at the SDR level, and a swap that turns the panel off keeps its zero. Signed-off-by: AxxzyWasTaken --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/iomfb_template.c | 41 +++++++++++++++++++++----- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index aed0d95607973..a953028c6139f 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -571,6 +571,8 @@ struct apple_dcp { bool output_pq; bool output_edr; bool edr_sent; + /* EDR swaps have asked for the panel at its peak (see dcp_edr_prepare()) */ + bool edr_boosted; struct dcp_set_digital_out_mode_req mode; /* completion for active turning true */ diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index e1cb13e7f8517..f4628dd2a1ef0 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -166,17 +166,41 @@ static void dcp_swap_set_edr(struct DCP_FW_NAME(dcp_swap) *swap, u32 sdr, u32 pe put_unaligned_le64(dcp_binary64_ratio(peak, sdr), &swap->edr_headroom); } -/* An EDR panel swap carries the headroom over the current panel level. */ -static void dcp_edr_prepare(struct apple_dcp *dcp, struct DCP_FW_NAME(dcp_swap) *swap) +/* + * An EDR panel swap carries the headroom over the panel level: @nits when + * the swap itself carries a level (@carries), else the current one. + * + * The firmware shows a PQ surface at (PQ nits / 100) x the EDR scale x the + * panel level the swap asks for. Left at SDR white, that level squeezes the + * whole PQ range below SDR white, so EDR swaps also ask for the panel at its + * peak; the scale (SDR white over the peak) keeps PQ 100 at the user's level. + * The panel goes back to that level when EDR ends, and a swap that turns it + * off keeps its zero. + */ +static void dcp_edr_prepare(struct apple_dcp *dcp, struct DCP_FW_NAME(dcp_swap) *swap, + bool carries, u32 nits) { u32 peak = READ_ONCE(dcp->panel.peak_nits); - u32 sdr = dcp->output_edr ? dcp_backlight_desktop_nits(dcp) : 0; + u32 sdr = 0; + if (dcp->output_edr) + sdr = carries ? nits : dcp_backlight_desktop_nits(dcp); if (sdr && peak > sdr) { dcp_swap_set_edr(swap, sdr, peak); + /* A level of the swap's own would replace the peak. */ + if (carries || !dcp->edr_boosted) + dcp_swap_set_brightness(swap, peak); + dcp->edr_boosted = true; dcp->edr_sent = true; } else if (dcp->edr_sent) { dcp_swap_set_edr(swap, 1, 1); + if (dcp->edr_boosted && !carries) { + u32 level = dcp_backlight_desktop_nits(dcp); + + if (level) + dcp_swap_set_brightness(swap, level); + } + dcp->edr_boosted = false; dcp->edr_sent = false; } } @@ -2795,15 +2819,16 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) #if DCP_FW_DESKTOP_26(DCP_FW_VER) if (dcp_has_panel(dcp)) { - u32 nits; - bool have_surface = false; + u32 nits = 0; + bool have_surface = false, carries; for (unsigned int i = 0; i < SWAP_SURFACES; i++) have_surface |= !DCP_FW_SWAP(dcp).surf_null[i]; - if (dcp_backlight_desktop_prepare(dcp, resp->swap_id, - have_surface, &nits)) + carries = dcp_backlight_desktop_prepare(dcp, resp->swap_id, + have_surface, &nits); + if (carries) dcp_swap_set_brightness(&DCP_FW_SWAP(dcp).swap, nits); - dcp_edr_prepare(dcp, &DCP_FW_SWAP(dcp).swap); + dcp_edr_prepare(dcp, &DCP_FW_SWAP(dcp).swap, carries, nits); } #endif trace_iomfb_swap_submit(dcp, resp->swap_id); From 10c70d091fc5cc436032e1462e40593a8980dfcf Mon Sep 17 00:00:00 2001 From: AxxzyWasTaken Date: Fri, 30 Oct 2026 19:05:20 +0000 Subject: [PATCH 56/73] drm/apple: Show EDR only for BT.2020 PQ on desktop 26.x panels Hyprland 0.56, going from its HDR mode to a wide-gamut SDR mode (cm = "dp3"), sets the connector's Colorspace back to Default but leaves the PQ HDR_OUTPUT_METADATA blob in place. The panel EDR path decides on the metadata alone, so it kept tagging the SDR frames as PQ BT.2020 and driving the panel at its peak. On a J616C at 100 nits the backlight drew 22 W, the image had a green cast, and brightness changes did nothing. Show PQ content as EDR only while BT2020_RGB is asked for as well. A Colorspace change already forces a mode set, so the decision is taken again whenever either property changes. External PQ output is unchanged. Signed-off-by: AxxzyWasTaken --- drivers/gpu/drm/apple/iomfb.c | 37 ++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8147b8782125e..296b8b6b20387 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -2333,14 +2333,14 @@ static bool dcp_panel_edr_supported(struct apple_dcp *dcp) dcp_has_panel(dcp) && READ_ONCE(dcp->panel.peak_nits); } -/* The EOTF the compositor asks of the display on @crtc, an HDMI_EOTF_* value. */ -u8 dcp_crtc_eotf(struct apple_dcp *dcp, struct drm_crtc *crtc, - struct drm_atomic_commit *commit) +/* The state of the connector on @crtc that @commit sets up. */ +static const struct drm_connector_state * +dcp_crtc_connector_state(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *commit) { const struct drm_connector_state *conn_state = NULL; struct drm_connector_state *state; struct drm_connector *connector; - const struct hdr_output_metadata *hdr; int i; for_each_new_connector_in_state(commit, connector, state, i) @@ -2350,12 +2350,38 @@ u8 dcp_crtc_eotf(struct apple_dcp *dcp, struct drm_crtc *crtc, if (!conn_state && dcp->connector && dcp->connector->base.state && dcp->connector->base.state->crtc == crtc) conn_state = dcp->connector->base.state; + return conn_state; +} + +/* The EOTF the compositor asks of the display on @crtc, an HDMI_EOTF_* value. */ +u8 dcp_crtc_eotf(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *commit) +{ + const struct drm_connector_state *conn_state = + dcp_crtc_connector_state(dcp, crtc, commit); + const struct hdr_output_metadata *hdr; + if (!conn_state || !conn_state->hdr_output_metadata) return HDMI_EOTF_TRADITIONAL_GAMMA_SDR; hdr = conn_state->hdr_output_metadata->data; return hdr->hdmi_metadata_type1.eotf; } +/* + * A panel shows PQ content as EDR only while BT.2020 RGB is asked for too. + * Hyprland 0.56, going from HDR to a wide-gamut SDR mode, sets Colorspace back + * to Default but leaves the PQ metadata; on that alone the panel showed SDR + * frames as PQ at its peak. + */ +static bool dcp_crtc_bt2020(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *commit) +{ + const struct drm_connector_state *conn_state = + dcp_crtc_connector_state(dcp, crtc, commit); + + return conn_state && conn_state->colorspace == DRM_MODE_COLORIMETRY_BT2020_RGB; +} + /* * HDR10 static metadata is link state on the external processors: a link * set up again (mode set, power off) starts without it. @@ -2613,7 +2639,8 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, return 0; dcp->pq_requested = dcp_crtc_eotf(dcp, crtc, commit) == HDMI_EOTF_SMPTE_ST2084; - dcp->output_edr = dcp->pq_requested && dcp_panel_edr_supported(dcp); + dcp->output_edr = dcp->pq_requested && dcp_panel_edr_supported(dcp) && + dcp_crtc_bt2020(dcp, crtc, commit); dcp->pq_requested &= dcp_hdr_output_supported(dcp); if (dcp->desktop) dcp->desktop->vrr_requested = crtc_state->vrr_enabled; From 7e6b33aa91d6e180991c84793c361e933de0a0fd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 17:34:20 +1100 Subject: [PATCH 57/73] drm/apple: Drive desktop 26.x panel EDR the way macOS does EDR on the J616s panel asked for the peak in one swap and left it there, with the display EDR scale anchored so that PQ 203 met SDR white, a zero brightness limit in every brightness swap, and a jump straight to the peak again after a power off. After a DPMS cycle with HDR content up, the panel then stayed at about a third of its EDR level (GNOME bt2100: backlight 11-13 W before, 3.9 W after), because a direct jump to the peak does not take after a power on. Follow the sequence macOS uses on this panel (captured on the J616s under macOS 27): with applied headroom H, brightness SDR white x H, display EDR 1 / H and, while the frame uses EDR, headroom used H; H moves linear in stops, one step per swap, about 1 s up and 2 s down, with a brightness limit of 1.3 x the brightness in the middle of a ramp and never a zero limit; headroom used drops to 1.0 as soon as EDR ends; after a power off the ramp starts again from 1.0. PQ 100 now lands at SDR white, as on macOS. On the J616s (fleet14b, DCP 26.6.2) a DPMS cycle with GNOME in bt2100 keeps the panel at 8.7 W instead of dropping to 3.9 W. Full-field SDR white draws less after a PQ session until the next display power cycle (6.1 W to 4.2 W at the same brightness); stock fleet14b does the same, so that predates this change and stays open. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 9 ++- drivers/gpu/drm/apple/dcp.c | 2 + drivers/gpu/drm/apple/iomfb_template.c | 96 ++++++++++++++++---------- drivers/gpu/drm/apple/iomfb_template.h | 13 ++-- 4 files changed, 75 insertions(+), 45 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index a953028c6139f..105ec1d3b1d4f 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -571,8 +571,13 @@ struct apple_dcp { bool output_pq; bool output_edr; bool edr_sent; - /* EDR swaps have asked for the panel at its peak (see dcp_edr_prepare()) */ - bool edr_boosted; + /* + * EDR headroom the panel was last given, x1000 (0: not in use), and + * a power off since, after which the ramp starts again from 1.0 (see + * dcp_edr_prepare()). + */ + u32 edr_hm; + bool edr_restart; struct dcp_set_digital_out_mode_req mode; /* completion for active turning true */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index ca3b47379acbd..97e0ca217ad21 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3827,6 +3827,8 @@ static void _dcp_poweroff(struct apple_dcp *dcp) break; case DCP_FIRMWARE_V_26_6_DESKTOP: iomfb_poweroff_v26_6_desktop(dcp); + if (dcp->edr_hm) + dcp->edr_restart = true; /* The link set up again starts without HDR10 metadata. */ dcp_hdr_metadata_forget(dcp); break; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index f4628dd2a1ef0..0caf3af398a18 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -119,7 +119,6 @@ static void dcp_swap_set_brightness(struct DCP_FW_NAME(dcp_swap) *swap, u32 nits } swap->bl_update = 1; - swap->bl_flags[1] = 1; put_unaligned_le64(bits, &swap->bl_nits); } @@ -154,55 +153,75 @@ static u64 dcp_binary64_ratio(u64 num, u64 den) } /* - * EDR: SDR white stays at @sdr nits while highlights reach @peak. The swap - * carries the display EDR scale, SDR white over the peak, and the headroom - * the frame uses, the peak over SDR white; equal values end EDR. + * EDR on a desktop 26.x panel, as macOS drives it. With applied headroom H + * (x1000 below), a swap asks for brightness SDR white x H, display EDR + * scale 1 / H and, while the frame uses EDR, headroom used H; the firmware + * shows a PQ surface at (PQ nits / 100) x the scale x the panel level, so + * PQ 100 lands at SDR white and highlights reach the panel peak. H moves + * in steps linear in stops, one per swap: about 1 s up when EDR starts and + * 2 s down when it ends, with a brightness limit of 1.3 x the brightness in + * the middle of a ramp. Headroom used drops to 1.0 as soon as EDR ends. + * Unchanged values are left out: the firmware keeps them. */ -static void dcp_swap_set_edr(struct DCP_FW_NAME(dcp_swap) *swap, u32 sdr, u32 peak) -{ - swap->bl_flags[2] = 1; - put_unaligned_le64(dcp_binary64_ratio(sdr, peak), &swap->edr_scale); - swap->bl_flags[4] = 1; - put_unaligned_le64(dcp_binary64_ratio(peak, sdr), &swap->edr_headroom); -} +#define DCP_EDR_STEP_UP 1047 /* x1000 per swap */ +#define DCP_EDR_STEP_DOWN 1023 + +#define dcp_swap_set_ratio(swap, field, flag, num, den) \ + do { \ + (swap)->bl_flags[flag] = 1; \ + put_unaligned_le64(dcp_binary64_ratio(num, den), \ + &(swap)->field); \ + } while (0) /* - * An EDR panel swap carries the headroom over the panel level: @nits when - * the swap itself carries a level (@carries), else the current one. - * - * The firmware shows a PQ surface at (PQ nits / 100) x the EDR scale x the - * panel level the swap asks for. Left at SDR white, that level squeezes the - * whole PQ range below SDR white, so EDR swaps also ask for the panel at its - * peak; the scale (SDR white over the peak) keeps PQ 100 at the user's level. - * The panel goes back to that level when EDR ends, and a swap that turns it - * off keeps its zero. + * @nits is the level a swap carries itself (@carries: a brightness change + * or power on), else the current one stands. A swap that turns the panel + * off keeps its zero. After a power off the ramp starts again from 1.0: + * asking for the peak at once does not take after a power on, on macOS + * either. */ static void dcp_edr_prepare(struct apple_dcp *dcp, struct DCP_FW_NAME(dcp_swap) *swap, bool carries, u32 nits) { u32 peak = READ_ONCE(dcp->panel.peak_nits); - u32 sdr = 0; - - if (dcp->output_edr) - sdr = carries ? nits : dcp_backlight_desktop_nits(dcp); - if (sdr && peak > sdr) { - dcp_swap_set_edr(swap, sdr, peak); - /* A level of the swap's own would replace the peak. */ - if (carries || !dcp->edr_boosted) - dcp_swap_set_brightness(swap, peak); - dcp->edr_boosted = true; + u32 sdr = carries ? nits : dcp_backlight_desktop_nits(dcp); + u32 cur = dcp->edr_hm ?: 1000, target = 1000, next, bri; + bool restart = dcp->edr_restart; + + dcp->edr_restart = false; + if (!dcp->output_edr && !dcp->edr_hm) + return; /* no EDR, and none to wind down */ + if (!sdr) + return; + if (dcp->output_edr && peak > sdr) + target = peak * 1000 / sdr; + if (restart && dcp->output_edr) + cur = 1000; + if (cur == target && !carries && !restart && dcp->edr_sent == dcp->output_edr) + return; + + next = cur; + if (target > cur) + next = min(cur * DCP_EDR_STEP_UP / 1000 + 1, target); + else if (target < cur) + next = max(cur * 1000 / DCP_EDR_STEP_DOWN, target); + + bri = (u64)sdr * next / 1000; + dcp_swap_set_brightness(swap, bri); + dcp_swap_set_ratio(swap, edr_scale, 2, 1000, next); + if (dcp->output_edr) { + dcp_swap_set_ratio(swap, edr_headroom, 4, next, 1000); dcp->edr_sent = true; } else if (dcp->edr_sent) { - dcp_swap_set_edr(swap, 1, 1); - if (dcp->edr_boosted && !carries) { - u32 level = dcp_backlight_desktop_nits(dcp); - - if (level) - dcp_swap_set_brightness(swap, level); - } - dcp->edr_boosted = false; + dcp_swap_set_ratio(swap, edr_headroom, 4, 1, 1); dcp->edr_sent = false; } + if (next != target && next != cur) { + swap->bl_flags[1] = 1; + put_unaligned_le64(dcp_binary64_ratio(bri * 13, 10), &swap->bl_limit); + } + /* Back at 1.0 without EDR: leave the firmware's values alone again. */ + dcp->edr_hm = (!dcp->output_edr && next == 1000) ? 0 : next; } /* @@ -262,6 +281,7 @@ static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), swap_id) == 0x98); static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), bl_update) == 0x354); static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), bl_flags) == 0x358); static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), bl_nits) == 0x35e); +static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), bl_limit) == 0x376); static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), edr_scale) == 0x37e); static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), edr_headroom) == 0x3e6); static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), presents_surface) == 0x51e); diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 8929f2f535c72..10182468a7fbb 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -68,14 +68,17 @@ struct DCP_FW_NAME(dcp_swap) { #if DCP_FW_DESKTOP_26(DCP_FW_VER) /* * The desktop 26.x firmware takes the panel level in nits, as an - * IEEE-754 binary64 at +0x35e (unaligned), flagged by bl_flags[1]. - * The display EDR scale (SDR white over the current peak) is the - * binary64 at +0x37e, flagged by bl_flags[2]; the headroom the frame - * uses is the one at +0x3e6, flagged by bl_flags[4]. + * IEEE-754 binary64 at +0x35e (unaligned), flagged by bl_update. The + * brightness limit is the binary64 at +0x376, flagged by bl_flags[1] + * (a zero limit clears it). The display EDR scale is the binary64 at + * +0x37e, flagged by bl_flags[2]; the headroom the frame uses is the + * one at +0x3e6, flagged by bl_flags[4]. Each value is kept by the + * firmware until a swap carries it again. */ u8 bl_flags[6]; __le64 bl_nits; - u8 unk_366[0x18]; + u8 unk_366[0x10]; + __le64 bl_limit; __le64 edr_scale; u8 unk_386[0xb]; #else From 46861edf850f67c933a64e8173885918af982bb7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 18:07:50 +1100 Subject: [PATCH 58/73] drm/apple: End desktop 26.x panel EDR in one swap The EDR exit ramped H down one step per swap, as macOS does. macOS's compositor keeps presenting until its ramp ends; GNOME on a still desktop may send no further swap, and with headroom used already back at 1.0 a ramp left partway would keep SDR content scaled down. Return H to 1.0 in the swap that ends EDR, which macOS also does for an immediate transition. On the J616s (fleet14b) the GNOME desktop after leaving bt2100 draws the same backlight power with an exit ramp and without one, and a full-field SDR white after EDR settles as on stock. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 0caf3af398a18..8e82e74fc1114 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -157,14 +157,13 @@ static u64 dcp_binary64_ratio(u64 num, u64 den) * (x1000 below), a swap asks for brightness SDR white x H, display EDR * scale 1 / H and, while the frame uses EDR, headroom used H; the firmware * shows a PQ surface at (PQ nits / 100) x the scale x the panel level, so - * PQ 100 lands at SDR white and highlights reach the panel peak. H moves - * in steps linear in stops, one per swap: about 1 s up when EDR starts and - * 2 s down when it ends, with a brightness limit of 1.3 x the brightness in - * the middle of a ramp. Headroom used drops to 1.0 as soon as EDR ends. + * PQ 100 lands at SDR white and highlights reach the panel peak. H rises + * in steps linear in stops, one per swap, about 1 s when EDR starts, with + * a brightness limit of 1.3 x the brightness in the middle of the ramp. + * When EDR ends, headroom used drops to 1.0 and H returns to 1.0. * Unchanged values are left out: the firmware keeps them. */ #define DCP_EDR_STEP_UP 1047 /* x1000 per swap */ -#define DCP_EDR_STEP_DOWN 1023 #define dcp_swap_set_ratio(swap, field, flag, num, den) \ do { \ @@ -200,11 +199,17 @@ static void dcp_edr_prepare(struct apple_dcp *dcp, struct DCP_FW_NAME(dcp_swap) if (cur == target && !carries && !restart && dcp->edr_sent == dcp->output_edr) return; + /* + * Up in steps; down at once. macOS ramps down too, but its compositor + * keeps presenting until the ramp ends. Here a still desktop may send + * no further swap, and a ramp stuck partway leaves SDR content dimmed + * by the scale (headroom used is already 1.0). + */ next = cur; if (target > cur) next = min(cur * DCP_EDR_STEP_UP / 1000 + 1, target); else if (target < cur) - next = max(cur * 1000 / DCP_EDR_STEP_DOWN, target); + next = target; bri = (u64)sdr * next / 1000; dcp_swap_set_brightness(swap, bri); From 9df30b701ab6295d3e83ae1753ed61f8bfdae71f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 18:56:58 +1100 Subject: [PATCH 59/73] media: apple: avd: tests: point at the avd-fw VP DONE commit The optional tagged VP DONE event is now committed in avd-fw (bf49a31b, queue feature bit 2), so the patch kept under tests/ is redundant. Remove it and name the firmware commit instead. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../media/platform/apple/avd/tests/README.rst | 4 +- .../apple/avd/tests/avd-fw-vp-done.patch | 117 ------------------ 2 files changed, 2 insertions(+), 119 deletions(-) delete mode 100644 drivers/media/platform/apple/avd/tests/avd-fw-vp-done.patch diff --git a/drivers/media/platform/apple/avd/tests/README.rst b/drivers/media/platform/apple/avd/tests/README.rst index 4f485a2d84135..b15ae618562d1 100644 --- a/drivers/media/platform/apple/avd/tests/README.rst +++ b/drivers/media/platform/apple/avd/tests/README.rst @@ -181,8 +181,8 @@ and must retain their single completion per picture. Optional queue ABI 2 VP notifications ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Apply ``avd-fw-vp-done.patch`` to the public MIT ``aurora-silicon/avd-fw`` -revision ``a112b70d4b1e437e4ce9c0353b2a0e805ebf41e5`` and rebuild its images. +The feature is in ``aurora-silicon/avd-fw`` from commit +``bf49a31b1036c72be5fdc37eb8232f6c953e0626`` (queue feature bit 2). This is a backwards-compatible optional ABI 2 feature: * Info feature bit 2 advertises tagged VP DONE. Without it the host keeps VP9 diff --git a/drivers/media/platform/apple/avd/tests/avd-fw-vp-done.patch b/drivers/media/platform/apple/avd/tests/avd-fw-vp-done.patch deleted file mode 100644 index c35e38c5ed0c4..0000000000000 --- a/drivers/media/platform/apple/avd/tests/avd-fw-vp-done.patch +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-License-Identifier: MIT -# Optional tagged VP DONE on avd-fw a112b70d4b1e437e4ce9c0353b2a0e805ebf41e5 ---- a/src/queue.h -+++ b/src/queue.h -@@ -24,6 +24,9 @@ - */ - #define AVDQ_FLAG_SRAM 1u - #define AVDQ_FEAT_SRAM_RING 1u -+#define AVDQ_FEAT_VP_DONE (1u << 2) -+#define AVDQ_FLAG_VP_DONE (1u << 1) -+#define AVDQ_VP_NOTIFY 0x54540000u - #define AVDQ_FEAT_MULTI_KICK 2u - #define AVDQ_SRAM_RING_OFFSET 0x0u - #define AVDQ_SRAM_RING_WORDS 0x1800u -@@ -78,7 +81,7 @@ - AVDQ_PP_READY, AVDQ_PP, AVDQ_DONE }; - struct avdq_frame { - struct avdq_desc desc; -- uint32_t phase, pushed, status; -+ uint32_t phase, pushed, status, vp_notified; - }; - struct avdq { - struct avdq_frame frame[AVDQ_COUNT]; ---- a/src/queue.c -+++ b/src/queue.c -@@ -21,7 +21,7 @@ - d->state != AVDQ_SUBMITTED || d->codec > AVDQ_AV1 || - !(codec_mask & (1u << d->codec)) || d->vp_slot >= vp_count || - !d->program_words || d->program_words > AVDQ_MAX_WORDS || -- !d->pp_kick || (d->flags & ~AVDQ_FLAG_SRAM) || -+ !d->pp_kick || (d->flags & ~(AVDQ_FLAG_SRAM | AVDQ_FLAG_VP_DONE)) || - d->pp_extra > AVDQ_MAX_PP_EXTRA || (d->pp_extra && !d->pp_cont)) - return 0; - if (d->flags & AVDQ_FLAG_SRAM) { -@@ -49,6 +49,7 @@ - f = &q->frame[idx]; - f->desc = *d; - f->pushed = 0; -+ f->vp_notified = 0; - f->status = valid(d, idx, vp_count, codec_mask) ? AVDQ_OK : AVDQ_INVALID; - f->phase = f->status ? AVDQ_DONE : AVDQ_PENDING; - q->order[(q->head + q->count) & (AVDQ_COUNT - 1)] = idx; ---- a/src/queue-fw.c -+++ b/src/queue-fw.c -@@ -99,7 +99,8 @@ - Q_INFO[8] = VP_COUNT; - Q_INFO[9] = AVD_VER <= 3 ? 7u : 15u; - Q_INFO[10] = FIFO_SLOTS; -- Q_INFO[11] = AVDQ_FEAT_SRAM_RING | AVDQ_FEAT_MULTI_KICK; -+ Q_INFO[11] = AVDQ_FEAT_SRAM_RING | AVDQ_FEAT_MULTI_KICK | -+ AVDQ_FEAT_VP_DONE; - Q_INFO[12] = AVDQ_SRAM_RING_OFFSET; - Q_INFO[13] = AVDQ_SRAM_RING_WORDS; - barrier(); -@@ -266,6 +267,32 @@ - reg_write(REG(DECODE_CTRL_BASE + PP_FIFO), d->pp_cont); - } - -+/* VP records do not release any descriptor or DMA storage. */ -+static void notify_vp(void) -+{ -+ unsigned int i; -+ -+ for (i = 0; i < FW->q.count; i++) { -+ unsigned int idx = FW->q.order[(FW->q.head + i) & 15u]; -+ struct avdq_frame *f = &FW->q.frame[idx]; -+ volatile struct avdq_desc *d = Q_DESC(idx); -+ -+ if (!(f->desc.flags & AVDQ_FLAG_VP_DONE) || f->vp_notified || -+ f->status || (f->phase != AVDQ_PP_READY && -+ f->phase != AVDQ_PP && f->phase != AVDQ_DONE)) -+ continue; -+ if (!(reg_read(CM3_MBOX1_STATUS) & CM3_MBOX_STATUS_EMPTY)) -+ return; -+ d->reserved[0] = f->desc.tag; -+ barrier(); -+ d->reserved[1] = 1; -+ barrier(); -+ reg_write(CM3_MBOX1_TX, AVDQ_VP_NOTIFY | (f->desc.tag & 0xfff0u) | idx); -+ f->vp_notified = 1; -+ return; -+ } -+} -+ - static int complete(void *cookie, unsigned int idx, const struct avdq_frame *f) - { - volatile struct avdq_desc *d = Q_DESC(idx); -@@ -323,6 +350,7 @@ - /* Queue mode owns PIODMA; the Tansy transport cannot run concurrently. */ - reg_write(Q_DMA(0x00), reg_read(Q_DMA(0x00)) & ~1u); - } -+ notify_vp(); - avdq_step(&FW->q, &ops, FW); - if (FW->fatal && !FW->q.count && - (reg_read(CM3_MBOX1_STATUS) & CM3_MBOX_STATUS_EMPTY)) { ---- a/tests/queue-test.c -+++ b/tests/queue-test.c -@@ -105,7 +105,7 @@ - case 10: d.program_lo = 0xfffffffcu; d.program_hi = 0xffffffffu; break; - case 11: d.pp_kick = 0; break; - case 12: d.reserved[1] = 1; break; -- case 13: d.flags = 2; break; -+ case 13: d.flags = 4; break; - case 14: d.pp_extra = AVDQ_MAX_PP_EXTRA + 1; d.pp_cont = 1; break; - case 15: d.pp_extra = 1; break; /* continuation word missing */ - case 16: d.flags = AVDQ_FLAG_SRAM; break; /* program_hi must be 0 */ ---- a/tests/transport-test.py -+++ b/tests/transport-test.py -@@ -50,7 +50,7 @@ - self.call("avdq_fw_init") - self.info = [self.u32(0x10007fc0 + i * 4) for i in range(16)] - assert self.info[:4] == [0x41565131, 2, 16, 64] -- assert self.info[11:16] == [3, 0, 0x1800, 0, 0] -+ assert self.info[11:16] == [7, 0, 0x1800, 0, 0] - expected = 0x0108c000 if self.version == 2 else 0x01090000 if self.version == 3 or (self.version == 5 and self.tier == 0) else 0x01094000 if (self.version == 5 and self.tier == 2) else 0x01092000 - assert self.info[4] == expected and self.info[8] == self.vps - assert self.info[10] == (7 if self.version == 2 or (self.version == 5 and self.tier == 0) else 15) From 6910946a755f475e2d1c4325c1fa52eb647d3b09 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 18:57:24 +1100 Subject: [PATCH 60/73] media: apple: avd: tests: point at the avd-fw queue IRQ fix commit The CM3 IRQ routing and FIFO EMPTY bit 17 fixes are committed in avd-fw (50d7f869), so the patch kept under tests/ is redundant. Remove it and name the firmware commit instead. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../media/platform/apple/avd/tests/README.rst | 8 ++- .../apple/avd/tests/avd-fw-queue-fixes.patch | 52 ------------------- 2 files changed, 3 insertions(+), 57 deletions(-) delete mode 100644 drivers/media/platform/apple/avd/tests/avd-fw-queue-fixes.patch diff --git a/drivers/media/platform/apple/avd/tests/README.rst b/drivers/media/platform/apple/avd/tests/README.rst index b15ae618562d1..d2b10320f2b75 100644 --- a/drivers/media/platform/apple/avd/tests/README.rst +++ b/drivers/media/platform/apple/avd/tests/README.rst @@ -47,12 +47,10 @@ transport-test.py harness:: The baseline must reproduce both missing CM3 decoder interrupt routing and, when those routes are repaired in the fixture, the incorrect mailbox status EMPTY test. A firmware fix must program the queue's CM3 IRQ router and use -FIFO status bit 17, rather than IRQ-source bit 2, for outbox EMPTY. With that -firmware and its status-bit-corrected transport harness:: +FIFO status bit 17, rather than IRQ-source bit 2, for outbox EMPTY. Both fixes +are in ``aurora-silicon/avd-fw`` from commit ``50d7f869``. With that firmware +built:: - git -C /path/to/fixed-avd-fw apply --unidiff-zero \ - /path/to/linux/drivers/media/platform/apple/avd/tests/avd-fw-queue-fixes.patch - # Rebuild the firmware images, then: python3 tests/unicorn-queue-test.py /path/to/fixed-avd-fw \ /path/to/fixed-avd-fw/build/avd-fw-v3-t1.bin /tmp/avd-program diff --git a/drivers/media/platform/apple/avd/tests/avd-fw-queue-fixes.patch b/drivers/media/platform/apple/avd/tests/avd-fw-queue-fixes.patch deleted file mode 100644 index e4ba07084ae86..0000000000000 --- a/drivers/media/platform/apple/avd/tests/avd-fw-queue-fixes.patch +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-License-Identifier: MIT ---- a/src/queue-fw.c -+++ b/src/queue-fw.c -@@ -108,0 +109,22 @@ -+/* The CM3 interrupt router is separate from the NVIC and decoder masks. */ -+static void queue_enable_irq(unsigned int irq) -+{ -+ unsigned int bank = (irq - 14u) / 32u; -+ unsigned int mask = 1u << ((irq - 14u) % 32u); -+ -+ reg_write(CM3_MBOX_IRQEN(bank), reg_read(CM3_MBOX_IRQEN(bank)) | mask); -+} -+ -+static void queue_enable_irqs(unsigned int vp) -+{ -+ unsigned int event, first; -+ -+ /* Enable only a validated descriptor's VP, not holes in the VP limit. */ -+ first = (AVD_VER == 2 ? 18u : 78u) + vp * 5u; -+ for (event = 0; event < 3; event++) -+ queue_enable_irq(first + event); /* unknown/error/DONE, not FULL */ -+ first = AVD_VER == 2 ? 38u : 62u; -+ queue_enable_irq(first); /* PP unknown */ -+ queue_enable_irq(first + 2u); /* PP DONE */ -+} -+ -@@ -114,0 +137,5 @@ -+ if (!FW->mode) { -+ /* Queue mode owns these routes; legacy mode before it is unchanged. */ -+ for (unsigned int i = 0; i < 6; i++) -+ reg_write(CM3_MBOX_IRQEN(i), 0); -+ } -@@ -246 +273 @@ -- if (!(reg_read(CM3_MBOX1_STATUS) & CM3_MBOX1_EMPTY)) -+ if (!(reg_read(CM3_MBOX1_STATUS) & CM3_MBOX_STATUS_EMPTY)) -@@ -292,0 +320,2 @@ -+ } else if (FW->q.frame[idx].status == AVDQ_OK) { -+ queue_enable_irqs(d.vp_slot); -@@ -299 +328 @@ -- (reg_read(CM3_MBOX1_STATUS) & CM3_MBOX1_EMPTY)) { -+ (reg_read(CM3_MBOX1_STATUS) & CM3_MBOX_STATUS_EMPTY)) { ---- a/src/hw/reg/reg_common.h -+++ b/src/hw/reg/reg_common.h -@@ -36,0 +37,3 @@ -+/* FIFO status is separate from the mailbox IRQ-source bits above. */ -+#define CM3_MBOX_STATUS_EMPTY BIT(17) -+ ---- a/tests/transport-test.py -+++ b/tests/transport-test.py -@@ -79 +79 @@ -- self.put(addr, 0 if self.busy else 4) -+ self.put(addr, 0x10101 if self.busy else 0x20001) From 2f9bd00ed9cc50572b2748febaa077f4c2b3ebcd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 18:57:57 +1100 Subject: [PATCH 61/73] media: apple: avd: overlap AV1 pictures with immutable tail inputs Keep two complete AV1 scratch banks on queued firmware advertising tagged VP DONE. Look ahead at the pending request's frame control and follow its actual primary-reference timestamp. Default CDFs and references outside the in-flight set do not wait for an unrelated picture. The open firmware's VP event does not establish CPU visibility of AV1 hardware-written CDF, segmentation or motion-field tails. Retain PP retirement dependencies for those inputs. When frame-end CDF updates or segmentation-map writes are disabled, copy preparation's immutable input snapshots into the consumer bank instead of reading live CAPTURE storage. Only PP retirement releases the saved picture's bank. Preserve the legacy emitters and tail preparation, add production host flag/ownership tests and a routed AV1 Unicorn overlap test, and document the remaining visibility evidence requirement. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-av1.c | 328 +++++++++++++++-- .../media/platform/apple/avd/tests/README.rst | 127 +++++-- .../platform/apple/avd/tests/av1-test.py | 330 ++++++++++++++++++ .../apple/avd/tests/unicorn-av1-test.py | 144 ++++++++ tools/asahi/avd-codec-tail-tests.py | 4 + 5 files changed, 867 insertions(+), 66 deletions(-) create mode 100644 drivers/media/platform/apple/avd/tests/av1-test.py create mode 100644 drivers/media/platform/apple/avd/tests/unicorn-av1-test.py diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index ba7beea61c842..6a87a4de6fe9b 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -225,12 +225,15 @@ struct avd_av1_run { }; struct avd_av1_ctx { + struct avd_av1_stream *stream; u8 submit_num; struct { struct avd_buf inst; struct avd_buf pipe_state; struct avd_buf probs; struct avd_buf no_prev_seg; + struct avd_buf saved_seg; + struct avd_buf prev_seg; struct avd_buf above_info; struct avd_buf rf_above_info; struct avd_buf az_above; @@ -246,6 +249,24 @@ struct avd_av1_ctx { } bufs; }; +#define AVD_AV1_BANKS 2 + +struct avd_av1_bank { + struct avd_av1_ctx codec; + bool busy; + bool prepared; + struct avd_decoded_buffer *dst; + u64 timestamp; + u32 flags; + u8 seg_flags; +}; + +struct avd_av1_stream { + unsigned int nbanks; + struct avd_av1_bank bank[AVD_AV1_BANKS]; + struct avd_av1_bank *b; +}; + /* * Timestamp of the reference a ref_frame_idx entry names. The entries are * -1 on frames that use no references (FFmpeg fills key frames that way); @@ -259,6 +280,111 @@ static u64 avd_av1_ref_ts(const struct v4l2_ctrl_av1_frame *frame, int idx) return frame->reference_frame_ts[idx]; } +static bool avd_av1_default_cdf(const struct v4l2_ctrl_av1_frame *frame) +{ + return frame->primary_ref_frame == AV1_PRIMARY_REF_NONE || + frame->frame_type == V4L2_AV1_KEY_FRAME || + frame->frame_type == V4L2_AV1_INTRA_ONLY_FRAME || + (frame->flags & V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE); +} + +static bool avd_av1_writes_seg(u8 flags) +{ + return (flags & V4L2_AV1_SEGMENTATION_FLAG_ENABLED) && + (flags & V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP); +} + +/* + * No available hardware contract proves AV1 tails CPU-visible at VP DONE. + * Only preparation's immutable CDF/map snapshots can be consumed before PP + * retirement. The consumer's disable_frame_end_update_cdf does not remove + * its CDF input dependency. Slot timestamps, not refresh order, select it. + */ +static bool avd_av1_depends(const struct v4l2_ctrl_av1_frame *frame, + const struct avd_av1_bank *bank) +{ + if (frame->flags & V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS) + for (unsigned int i = 0; i < V4L2_AV1_REFS_PER_FRAME; i++) + if (avd_av1_ref_ts(frame, frame->ref_frame_idx[i]) == bank->timestamp) + return true; + + if (frame->primary_ref_frame >= AV1_PRIMARY_REF_NONE || + avd_av1_ref_ts(frame, frame->ref_frame_idx[frame->primary_ref_frame]) != + bank->timestamp) + return false; + if (!avd_av1_default_cdf(frame) && + !(bank->flags & V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF)) + return true; + return (frame->segmentation.flags & V4L2_AV1_SEGMENTATION_FLAG_ENABLED) && + avd_av1_writes_seg(bank->seg_flags) && + DIV_ROUND_UP(bank->dst->av1.width + 1, 8) == + DIV_ROUND_UP(frame->frame_width_minus_1 + 1, 8) && + DIV_ROUND_UP(bank->dst->av1.height + 1, 8) == + DIV_ROUND_UP(frame->frame_height_minus_1 + 1, 8); +} + +static bool avd_av1_ready(struct avd_ctx *ctx) +{ + struct avd_av1_ctx *codec = smp_load_acquire(&ctx->priv); + struct avd_av1_stream *av1_ctx; + struct vb2_v4l2_buffer *src; + struct v4l2_ctrl_handler *hdl; + struct v4l2_ctrl_ref *ref; + const struct v4l2_ctrl_av1_frame *frame = NULL; + bool ready = false, free = false, busy = false; + + if (!codec) + return false; + av1_ctx = codec->stream; + for (unsigned int i = 0; i < av1_ctx->nbanks; i++) { + if (!READ_ONCE(av1_ctx->bank[i].busy)) + free = true; + else if (!smp_load_acquire(&av1_ctx->bank[i].prepared)) + return false; + else + busy = true; + } + if (!free || !busy) + return free; + src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + if (!src || !src->vb2_buf.req_obj.req) + return false; + hdl = v4l2_ctrl_request_hdl_find(src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + if (!hdl) + return false; + /* + * The queued request is immutable and cannot complete while its OUTPUT + * is still on the ready queue. Read p_req, never the previous picture's + * shared p_cur, without taking a sleeping control mutex under queue_lock. + */ + list_for_each_entry(ref, &hdl->ctrl_refs, node) + if (ref->ctrl->id == V4L2_CID_STATELESS_AV1_FRAME && ref->p_req_valid) { + frame = ref->p_req.p; + break; + } + if (frame) { + ready = true; + for (unsigned int i = 0; i < av1_ctx->nbanks; i++) + if (READ_ONCE(av1_ctx->bank[i].busy) && + avd_av1_depends(frame, &av1_ctx->bank[i])) + ready = false; + } + v4l2_ctrl_request_hdl_put(hdl); + return ready; +} + +static struct avd_av1_bank *avd_av1_ref_bank(struct avd_av1_stream *av1_ctx, + struct avd_decoded_buffer *ref) +{ + if (av1_ctx->nbanks > 1 && ref->bank && ref->bank <= av1_ctx->nbanks) { + struct avd_av1_bank *bank = &av1_ctx->bank[ref->bank - 1]; + + if (bank->busy && bank->prepared && bank->dst == ref) + return bank; + } + return NULL; +} + /* * from verisilicon/rockchip_vpu981_hw_av1_dec.c */ @@ -1160,6 +1286,71 @@ static void avd_av1_set_seg(struct avd_ctx *ctx, struct avd_av1_run *run) memcpy(seg, prev_seg, dst->av1.seg_size); } +/* Preserve the legacy tail preparation; only queued immutable inputs differ. */ +static void avd_av1_prepare_prob(struct avd_ctx *ctx, struct avd_av1_run *run) +{ + struct avd_av1_ctx *codec = ctx->priv; + struct avd_av1_stream *stream = codec->stream; + struct avd_decoded_buffer *dst, *ref; + struct avd_av1_bank *bank = NULL; + + dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); + if (stream->nbanks > 1 && !avd_av1_default_cdf(run->frame)) { + ref = avd_get_ref_buf(ctx, &dst->base.vb, + avd_av1_ref_ts(run->frame, + run->frame->ref_frame_idx[run->frame->primary_ref_frame])); + bank = avd_av1_ref_bank(stream, ref); + } + if (!bank) { + avd_av1_set_prob(ctx, run); + return; + } + memcpy(codec->bufs.probs.cpu, bank->codec.bufs.probs.cpu, AVD_CDFS_SIZE); + memcpy(vb2_plane_vaddr(&dst->base.vb.vb2_buf, 0) + + AVD_AV1_CDFS_OFFSET(dst->base.vb.vb2_buf.planes[0].length, + dst->av1.color_size), + codec->bufs.probs.cpu, AVD_CDFS_SIZE); +} + +static void avd_av1_prepare_seg(struct avd_ctx *ctx, struct avd_av1_run *run) +{ + struct avd_av1_ctx *codec = ctx->priv; + struct avd_av1_stream *stream = codec->stream; + const struct v4l2_ctrl_av1_frame *frame = run->frame; + struct avd_decoded_buffer *dst, *ref; + struct avd_av1_bank *bank = NULL; + void *seg; + + dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); + seg = vb2_plane_vaddr(&dst->base.vb.vb2_buf, 0) + + AVD_AV1_SEG_OFFSET(dst->base.vb.vb2_buf.planes[0].length, + dst->av1.color_size, dst->av1.seg_size); + if (stream->nbanks > 1 && frame->primary_ref_frame != AV1_PRIMARY_REF_NONE) { + ref = avd_get_ref_buf(ctx, &dst->base.vb, + avd_av1_ref_ts(frame, frame->ref_frame_idx[frame->primary_ref_frame])); + if (ref != dst && + DIV_ROUND_UP(ref->av1.width + 1, 8) == + DIV_ROUND_UP(frame->frame_width_minus_1 + 1, 8) && + DIV_ROUND_UP(ref->av1.height + 1, 8) == + DIV_ROUND_UP(frame->frame_height_minus_1 + 1, 8)) + bank = avd_av1_ref_bank(stream, ref); + } + if (bank && !avd_av1_writes_seg(bank->seg_flags)) { + /* Copy into our bank: producer PP can retire during our VP. */ + memcpy(codec->bufs.prev_seg.cpu, bank->codec.bufs.saved_seg.cpu, + dst->av1.seg_size); + run->addresses.prev_seg = codec->bufs.prev_seg.addr; + if (!(frame->segmentation.flags & V4L2_AV1_SEGMENTATION_FLAG_ENABLED)) + memset(seg, 0, dst->av1.seg_size); + else if (!(frame->segmentation.flags & V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP)) + memcpy(seg, codec->bufs.prev_seg.cpu, dst->av1.seg_size); + } else { + avd_av1_set_seg(ctx, run); + } + if (stream->nbanks > 1 && !avd_av1_writes_seg(frame->segmentation.flags)) + memcpy(codec->bufs.saved_seg.cpu, seg, dst->av1.seg_size); +} + static int avd_av1_seg_size(int w, int h) { /* 3 bits per 4x4 block, in rows of 128x128 superblocks (64x64 fit too) */ @@ -1280,6 +1471,17 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) int h = frame->frame_height_minus_1 + 1; int bit_depth = seq->bit_depth; + if (av1_ctx->stream->nbanks > 1) { + ret = avd_buf_alloc(avd, &av1_ctx->bufs.saved_seg, + avd_av1_seg_size(w, h)); + if (ret) + return ret; + ret = avd_buf_alloc(avd, &av1_ctx->bufs.prev_seg, + avd_av1_seg_size(w, h)); + if (ret) + return ret; + } + for (i = 0; i < tile_info->tile_cols; i++) { sb = tile_info->width_in_sbs_minus_1[i] + 1; max_sb_col = sb > max_sb_col ? sb : max_sb_col; @@ -1380,6 +1582,7 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) static int avd_av1_run(struct avd_ctx *ctx) { struct avd_av1_run run; + struct avd_av1_stream *av1_ctx = ((struct avd_av1_ctx *)ctx->priv)->stream; struct avd_decoded_buffer *dst, *ref; int ret; @@ -1402,6 +1605,28 @@ static int avd_av1_run(struct avd_ctx *ctx) avd_run_postamble(ctx, &run.base); return -ENOMEM; } + if (av1_ctx->nbanks > 1) { + struct avd_av1_bank *bank = NULL; + + for (unsigned int i = 0; i < av1_ctx->nbanks; i++) + if (!av1_ctx->bank[i].busy) { + bank = &av1_ctx->bank[i]; + break; + } + if (!bank) { + avd_run_postamble(ctx, &run.base); + return -EBUSY; + } + av1_ctx->b = bank; + smp_store_release(&ctx->priv, &bank->codec); + WRITE_ONCE(bank->prepared, false); + WRITE_ONCE(bank->busy, true); + bank->dst = dst; + bank->timestamp = run.base.bufs.src->vb2_buf.timestamp; + bank->flags = run.frame->flags; + bank->seg_flags = run.frame->segmentation.flags; + dst->bank = bank - av1_ctx->bank + 1; + } ret = avd_init_job(ctx, AVD_CODEC_AV1, run.frame->tile_info.tile_cols * @@ -1420,33 +1645,51 @@ static int avd_av1_run(struct avd_ctx *ctx) update_dec_buf_info(dst, run.seq, run.frame); - avd_av1_set_prob(ctx, &run); - avd_av1_set_seg(ctx, &run); + avd_av1_prepare_prob(ctx, &run); + avd_av1_prepare_seg(ctx, &run); set_header(ctx, &run); set_tiles(ctx, &run); + if (av1_ctx->nbanks > 1) { + ctx->job_fifo = av1_ctx->b->codec.bufs.inst.addr; + smp_store_release(&av1_ctx->b->prepared, true); + } avd_run_postamble(ctx, &run.base); return avd_submit_job(ctx); } -static int avd_av1_alloc_bufs(struct avd_ctx *ctx) +static void avd_av1_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src, + struct vb2_v4l2_buffer *dst, enum vb2_buffer_state result) +{ + struct avd_av1_stream *av1_ctx = ((struct avd_av1_ctx *)ctx->priv)->stream; + struct avd_decoded_buffer *buf = vb2_to_avd_decoded_buf(&dst->vb2_buf); + + if (buf->bank && buf->bank <= av1_ctx->nbanks) { + struct avd_av1_bank *bank = &av1_ctx->bank[buf->bank - 1]; + + if (bank->dst == buf) + WRITE_ONCE(bank->busy, false); + buf->bank = 0; + } +} + +static int avd_av1_alloc_bufs(struct avd_ctx *ctx, struct avd_av1_bank *bank) { struct avd_dev *avd = ctx->dev; - struct avd_av1_ctx *av1_ctx = ctx->priv; int ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.inst, fifo_size()); + ret = avd_buf_alloc(avd, &bank->codec.bufs.inst, fifo_size()); if (ret) return ret; /* no m1 to worry about :) */ - ret = avd_buf_alloc(avd, &av1_ctx->bufs.pipe_state, 0x200); + ret = avd_buf_alloc(avd, &bank->codec.bufs.pipe_state, 0x200); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.probs, + ret = avd_buf_alloc(avd, &bank->codec.bufs.probs, sizeof(struct avd_av1_cdfs)); if (ret) return ret; @@ -1454,51 +1697,66 @@ static int avd_av1_alloc_bufs(struct avd_ctx *ctx) return 0; } -static void avd_av1_stop(struct avd_ctx *ctx) +static void avd_av1_free(struct avd_ctx *ctx, struct avd_av1_stream *av1_ctx) { - struct avd_av1_ctx *av1_ctx = ctx->priv; struct avd_dev *avd = ctx->dev; - avd_buf_free(avd, &av1_ctx->bufs.above_info); - avd_buf_free(avd, &av1_ctx->bufs.rf_above_info); - avd_buf_free(avd, &av1_ctx->bufs.az_above); - avd_buf_free(avd, &av1_ctx->bufs.ip_above); - avd_buf_free(avd, &av1_ctx->bufs.lf_above); - avd_buf_free(avd, &av1_ctx->bufs.lf_above_info); - avd_buf_free(avd, &av1_ctx->bufs.lf_left); - avd_buf_free(avd, &av1_ctx->bufs.lf_left_info); - avd_buf_free(avd, &av1_ctx->bufs.sr_left); - avd_buf_free(avd, &av1_ctx->bufs.rf_left); - avd_buf_free(avd, &av1_ctx->bufs.rf_left_info); - avd_buf_free(avd, &av1_ctx->bufs.no_prev_seg); - avd_buf_free(avd, &av1_ctx->bufs.mv_above_info); - - avd_buf_free(avd, &av1_ctx->bufs.pipe_state); - avd_buf_free(avd, &av1_ctx->bufs.inst); - avd_buf_free(avd, &av1_ctx->bufs.probs); - + for (unsigned int i = 0; i < av1_ctx->nbanks; i++) { + struct avd_av1_bank *bank = &av1_ctx->bank[i]; + + avd_buf_free(avd, &bank->codec.bufs.above_info); + avd_buf_free(avd, &bank->codec.bufs.rf_above_info); + avd_buf_free(avd, &bank->codec.bufs.az_above); + avd_buf_free(avd, &bank->codec.bufs.ip_above); + avd_buf_free(avd, &bank->codec.bufs.lf_above); + avd_buf_free(avd, &bank->codec.bufs.lf_above_info); + avd_buf_free(avd, &bank->codec.bufs.lf_left); + avd_buf_free(avd, &bank->codec.bufs.lf_left_info); + avd_buf_free(avd, &bank->codec.bufs.sr_left); + avd_buf_free(avd, &bank->codec.bufs.rf_left); + avd_buf_free(avd, &bank->codec.bufs.rf_left_info); + avd_buf_free(avd, &bank->codec.bufs.no_prev_seg); + avd_buf_free(avd, &bank->codec.bufs.mv_above_info); + + avd_buf_free(avd, &bank->codec.bufs.pipe_state); + avd_buf_free(avd, &bank->codec.bufs.inst); + avd_buf_free(avd, &bank->codec.bufs.probs); + avd_buf_free(avd, &bank->codec.bufs.saved_seg); + avd_buf_free(avd, &bank->codec.bufs.prev_seg); + } kfree(av1_ctx); +} + +static void avd_av1_stop(struct avd_ctx *ctx) +{ + avd_av1_free(ctx, ((struct avd_av1_ctx *)ctx->priv)->stream); ctx->priv = NULL; } static int avd_av1_start(struct avd_ctx *ctx) { - struct avd_av1_ctx *av1_ctx; + struct avd_av1_stream *av1_ctx; int ret; av1_ctx = kzalloc(sizeof(*av1_ctx), GFP_KERNEL); if (!av1_ctx) return -ENOMEM; - ctx->priv = av1_ctx; - ret = avd_av1_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; - + av1_ctx->nbanks = avd_qfw_has_vp_done(ctx->dev) ? AVD_AV1_BANKS : 1; + av1_ctx->b = &av1_ctx->bank[0]; + for (unsigned int i = 0; i < av1_ctx->nbanks; i++) { + av1_ctx->bank[i].codec.stream = av1_ctx; + ret = avd_av1_alloc_bufs(ctx, &av1_ctx->bank[i]); + if (ret) + goto err_free_ctx; + } + ctx->max_frames = av1_ctx->nbanks; + /* Readiness must never see partially allocated bank storage. */ + smp_store_release(&ctx->priv, &av1_ctx->bank[0].codec); return 0; err_free_ctx: - avd_av1_stop(ctx); + avd_av1_free(ctx, av1_ctx); return ret; } @@ -1586,6 +1844,8 @@ const struct avd_coded_fmt_ops avd_av1_fmt_ops = { .start = avd_av1_start, .stop = avd_av1_stop, .run = avd_av1_run, + .done = avd_av1_done, + .ready = avd_av1_ready, .submit = avd_av1_submit, .pp_kicks = avd_av1_pp_kicks, .try_ctrl = avd_av1_try_ctrl, diff --git a/drivers/media/platform/apple/avd/tests/README.rst b/drivers/media/platform/apple/avd/tests/README.rst index d2b10320f2b75..82169ce2a47cd 100644 --- a/drivers/media/platform/apple/avd/tests/README.rst +++ b/drivers/media/platform/apple/avd/tests/README.rst @@ -179,8 +179,11 @@ and must retain their single completion per picture. Optional queue ABI 2 VP notifications ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The feature is in ``aurora-silicon/avd-fw`` from commit +Use public MIT ``aurora-silicon/avd-fw`` branch +``avd/queue-abi2-sram-ring`` from commit ``bf49a31b1036c72be5fdc37eb8232f6c953e0626`` (queue feature bit 2). +Rebuild with ``make test && sh tests/build-all.sh``. The former local +VP-DONE patch is superseded by that commit and is no longer shipped. This is a backwards-compatible optional ABI 2 feature: * Info feature bit 2 advertises tagged VP DONE. Without it the host keeps VP9 @@ -200,34 +203,94 @@ This is a backwards-compatible optional ABI 2 feature: bank reuse. Only the original tagged PP completion does that. A missing VP event remains bounded by the original per-picture watchdog. -AV1 analysis -~~~~~~~~~~~~ - -AV1's current ``avd_av1_set_prob()`` reads the primary reference's CDF tail on -the CPU, and seeds the current frame's output CDF because hardware leaves it -untouched when DISABLE_FRAME_END_UPDATE_CDF is set. ``avd_av1_set_seg()`` can -also copy the primary reference's segmentation map on the CPU. These are -separate dependencies; removing only the CDF wait is insufficient. - -=================================== ========================================= -Case CDF source dependency -=================================== ========================================= -Intra/error-resilient/PRIMARY_NONE Defaults; no primary-reference CDF read -Primary ref with frame-end disabled Its initialized CDF, available at prepare -Primary ref with frame-end enabled Its hardware-written CDF tail -=================================== ========================================= - -``disable_frame_end_update_cdf`` on the *consumer* does not remove its primary -reference input dependency. It controls that consumer's future CDF output. -Likewise, a default CDF does not necessarily remove the segmentation-map input -selected by primary_ref_frame. - -The available implementation does not establish whether the CDF tail *and* -segmentation map are fully visible at VP DONE, or which CPU DMA synchronization -is required for an in-flight CAPTURE allocation while PP writes another part. -This differs from VP9's separate coherent count buffer. AV1 therefore retains -its one-picture path. Enabling it needs evidence for both tail writes and a -safe visibility/synchronization rule, then two scratch banks and a dependency -on the actual primary reference (which need not be the preceding picture). -No AV1 or VP9 pixel correctness or throughput claim follows from these host -and emulator tests; device qualification remains necessary. +AV1 conservative picture overlap +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``av1-test.py`` compiles the production AV1 codec, entropy defaults, open +program builder and queued descriptor writer with ASan/UBSan. It is also +called by the registered ``avd-codec-tail-tests.py`` host check. Run:: + + AVD_AV1_BASELINE_REF=a1b76856 python3 tests/av1-test.py /tmp/avd-av1 + python3 tests/unicorn-av1-test.py /path/to/avd-fw \ + /path/to/avd-fw/build/avd-fw-v5-t2.bin /tmp/avd-av1 + +The independent legacy comparison builds before/after executables and compares +complete emitted programs plus initialized CDF/map tails for eight header cases. +The flag table covers primary-reference/default CDF cases, both producer and +consumer disable_frame_end_update_cdf, segmentation enabled/update_map on both +pictures, and all 256 refresh masks. Pending request p_req values are used for +readiness; p_cur still describes an earlier picture and must not be used. + +Visibility evidence and limit +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The public AV1 specification sections 7.7, 7.19 and 7.20 define frame CDF, +motion-field and segmentation storage semantics: +https://aomediacodec.github.io/av1-spec/ . They do not define this decoder's +VP/PP DMA boundary. In our driver ``set_header()`` supplies probs_out, seg and +w_color output addresses to the VP program; ``avd_av1_set_prob()`` seeds CDF +output and ``avd_av1_set_seg()`` initializes non-updated maps. The open firmware +``avdq_vp_done()`` changes VP to PP_READY and ``notify_vp()`` publishes a tag +with barriers. Neither reads the AV1 output memory nor supplies an AV1-specific +DMA visibility guarantee. A CM3 DMB orders its descriptor stores; it does not +prove the decoder's prior external tail writes are CPU-visible. + +There is therefore **no evidence here proving that a hardware-written AV1 CDF, +segmentation map or motion field is visible at VP DONE**. There is also no +proof that they are written only at PP DONE. Unicorn injects DONE and cannot +establish either assertion. The implementation keeps those dependencies until +PP retirement instead of inferring a visibility contract from their operands. +No CPU DMA sync is attempted on a CAPTURE allocation still owned by PP. + +Safe subset +^^^^^^^^^^^ + +Queue firmware advertising VP-DONE feature bit 2 enables two scratch banks; +legacy transport and older queue firmware retain one bank and identical +programs. The CDF output, segmentation output and motion-field/color output +already live in each picture's distinct CAPTURE tail. Each scratch bank owns +its FIFO, CDF input, pipe/filter/tile scratch, a saved initialized map and a +separate map input snapshot. Only PP retirement releases its bank. VP DONE +never releases storage or publishes hardware-written tails for AV1. + +For a pending consumer, each busy producer is tested by the exact timestamp +named through ref_frame_idx/primary_ref_frame, rather than the preceding frame +or a global refresh-context flag. refresh_frame_flags controls which slots +userspace subsequently names; it does not create a dependency by itself. + +=============================== ============================================== +Consumer / producer condition Prepare dependency while producer is in flight +=============================== ============================================== +PRIMARY_REF_NONE No primary CDF or map dependency +Intra or error resilient Default CDF; map dependency still checked +Primary slot names retired ref None on an in-flight picture +Primary CDF, producer end update Enabled: wait for producer PP retirement +Primary CDF, producer end update Disabled: use immutable prepared CDF input copy +Consumer end update disabled Does not remove its CDF input dependency +Consumer segmentation disabled No previous-map read +Consumer segmentation enabled Updated producer map: wait for PP (same Mi size) +Producer map not updated Copy immutable saved map into consumer bank +Different reference Mi size Zero map; no map dependency +USE_REF_FRAME_MVS Wait for PP of any named in-flight reference +refresh_frame_flags 0..255 No additional dependency; named slot decides +=============================== ============================================== + +Both consumer update_map values retain the previous-map input when segmentation +is enabled, as the existing program supplies it in either case. A disabled +producer segmentation map is saved as zeroes. Producer map snapshots are copied +into consumer-owned coherent storage, so producer PP retirement and bank reuse +cannot invalidate a live successor's VP input. CDF snapshots are similarly +copied into the consumer's CDF input. No hardware-written tail is sampled by +these copies. Busy banks publish fully prepared metadata with release/acquire; +allocation failures and PP error/reset retirement release the saved CAPTURE's +bank, independent of the current builder. + +This subset needs no extra VP token: firmware already waits for VP(N) DONE +before feeding VP(N+1). AV1 descriptors retain their ordinary completion-only +flags. The Unicorn run boots the v5-t2 CM3 reset vector, routes AV1 and PP IRQs, +and checks production AV1 program words/descriptors/configuration. It shows +VP(1) issued during PP(0), PP serialization and blocked completion mailboxes. +Host tests poison in-flight CAPTURE tails to ensure snapshot consumers never +read them. Pixels, physical DMA visibility and throughput remain hardware +qualification work for the lead. Full CDF/MV dependency release at VP DONE is +still a separate evidence requirement. diff --git a/drivers/media/platform/apple/avd/tests/av1-test.py b/drivers/media/platform/apple/avd/tests/av1-test.py new file mode 100644 index 0000000000000..a02070c85ffe7 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/av1-test.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Production AV1 bank/dependency/emitters, with explicit host DMA boundaries. + +Run in a controller build slot: av1-test.py OUT [AVD_DIR [KERNEL_ROOT]]. +AVD_AV1_BASELINE_REF optionally compares complete legacy programs/CDF/map bytes. +No decoder arithmetic or DMA visibility is modeled by these host fixtures. +""" +import hashlib +import importlib.util +import os +from pathlib import Path +import re +import subprocess +import sys + +from host_source import function + +HERE = Path(__file__).resolve().parent +AVD = Path(sys.argv[2]).resolve() if len(sys.argv) > 2 else HERE.parent +KERNEL = Path(sys.argv[3]).resolve() if len(sys.argv) > 3 else AVD.parents[4] +OUT = Path(sys.argv[1]).resolve() +OUT.mkdir(parents=True, exist_ok=True) +spec = importlib.util.spec_from_file_location('program_fixture', HERE / 'program-test.py') +fixture = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fixture) +pre = fixture.PRELUDE +pre = pre.replace('typedef uint8_t u8;', 'typedef uint8_t u8; typedef uint16_t u16; typedef int8_t s8; typedef int16_t s16; typedef int64_t s64;') +pre = pre.replace('#define AVD_MAX_INST 512', '#define AVD_MAX_INST 512\n#define AVD_CODEC_HEVC 0') +pre = pre.replace('struct vb2_buffer { struct { size_t length; } planes[1]; dma_addr_t dma; };', + 'struct vb2_buffer { struct { size_t length; } planes[1]; dma_addr_t dma; void *cpu; size_t payload; u64 timestamp; struct {void *req;} req_obj; };') +header = (AVD / 'avd.h').read_text() +a = header.index('struct avd_av1_decoded_buffer_info {') +b = header.index('\n};', a) + 3 +pre = pre.replace('struct avd_decoded_buffer {', header[a:b] + '\nstruct avd_decoded_buffer {') +pre = pre.replace('struct avd_comp comp; unsigned int bank;', 'struct avd_comp comp; unsigned int bank; struct avd_av1_decoded_buffer_info av1;') +pre = pre.replace('struct vb2_v4l2_buffer *dst; } bufs;', 'struct vb2_v4l2_buffer *src,*dst; } bufs;') +pre = pre.replace('unsigned int fifo_idx, pp_total;', 'unsigned int fifo_idx, pp_total, max_frames; int ctrl_hdl;') +pre = pre.replace('dma_addr_t comp_out, y_out, uv_out;', 'dma_addr_t comp_out, y_out, uv_out, coded_in;') +pre = pre.replace('struct { u32 bytesperline; } plane_fmt[1];', 'u32 width,height; struct { u32 bytesperline; } plane_fmt[1];') +pre = pre.replace('static struct avd_decoded_buffer reference;', 'static struct avd_decoded_buffer reference; static struct avd_decoded_buffer *refs[3];') +pre = pre.replace('(void)c; (void)b; (void)t; return &reference;', '(void)c;for(unsigned int i=0;i<3;i++)if(refs[i]&&refs[i]->base.vb.vb2_buf.timestamp==t)return refs[i];return (void *)b;') +pre = pre.replace('static int avd_buf_alloc(struct avd_dev *d, struct avd_buf *b, size_t n)', + 'static struct avd_ctx *allocating_ctx;\nstatic int fail_alloc_after;\nstatic bool avd_av1_ready(struct avd_ctx *);\nstatic int avd_buf_alloc(struct avd_dev *d, struct avd_buf *b, size_t n)') +pre = pre.replace('{ (void)d; free(b->cpu); b->cpu = calloc(1,n);', + '{ if(allocating_ctx){assert(!avd_av1_ready(allocating_ctx));if(fail_alloc_after&&!--fail_alloc_after)return -ENOMEM;} (void)d; free(b->cpu); b->cpu = calloc(1,n);') +pre += r''' +#define AVD_ALIGN 256 +#define S16_MIN INT16_MIN +#define S16_MAX INT16_MAX +#define clamp_val(x,l,h) clamp(x,l,h) +#define U64_MAX UINT64_MAX +#define BIT_ULL(n) (1ULL<<(n)) +#define ALIGN(x,a) (((x)+(a)-1)&~((__typeof__(x))(a)-1)) +#define ALIGN_DOWN(x,a) ((x)&~((__typeof__(x))(a)-1)) +#define max_t(t,a,b) ((t)(a)>(t)(b)?(t)(a):(t)(b)) +#define min(a,b) ((a)<(b)?(a):(b)) +#define max(a,b) ((a)>(b)?(a):(b)) +#define clamp(x,l,h) ((x)<(l)?(l):(x)>(h)?(h):(x)) +#define WARN_ON(v) (v) +#define GFP_KERNEL 0 +#define smp_load_acquire(p) (*(p)) +#define smp_store_release(p,v) (*(p)=(v)) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define static_assert _Static_assert +#define VB2_BUF_STATE_DONE 0 +#define VB2_BUF_STATE_ERROR 1 +enum vb2_buffer_state { STATE_DUMMY }; +#define kzalloc(n,f) calloc(1,n) +#define kfree free +static bool vp_feature=true; +static bool avd_qfw_has_vp_done(const struct avd_dev *d){return d->queued&&vp_feature;} +static void *vb2_plane_vaddr(struct vb2_buffer *b,int p){(void)p;return b->cpu;} +static struct vb2_v4l2_buffer *current_src,*current_dst; +static struct vb2_v4l2_buffer *v4l2_m2m_next_src_buf(void *m){(void)m;return current_src;} +struct v4l2_ctrl {unsigned int id; union {void *p;} p_cur;}; +struct v4l2_ctrl_ref {struct v4l2_ctrl_ref *next; struct v4l2_ctrl *ctrl; bool p_req_valid; union {void *p;} p_req;}; +struct v4l2_ctrl_handler {struct v4l2_ctrl_ref *ctrl_refs;}; +#define list_for_each_entry(ref,head,node) for((ref)=*(head);(ref);(ref)=(ref)->next) +static struct v4l2_ctrl_av1_frame params,request_params; +static struct v4l2_ctrl_av1_sequence seq; +static struct v4l2_ctrl_av1_tile_group_entry tile; +static struct v4l2_ctrl_av1_film_grain grain; +static struct v4l2_ctrl controls[4]; +static struct v4l2_ctrl_ref request_ref; +static struct v4l2_ctrl_handler request_hdl; +static unsigned int request_gets,request_puts; +static struct v4l2_ctrl_handler *v4l2_ctrl_request_hdl_find(void *req,void *h){(void)h;assert(req==&request_hdl);request_gets++;return &request_hdl;} +static void v4l2_ctrl_request_hdl_put(struct v4l2_ctrl_handler *h){assert(h==&request_hdl);request_puts++;} +static struct v4l2_ctrl *v4l2_ctrl_find(void *h,u32 id){ + (void)h;controls[0].p_cur.p=&seq;controls[1].p_cur.p=¶ms;controls[2].p_cur.p=&tile;controls[3].p_cur.p=&grain; + if(id==V4L2_CID_STATELESS_AV1_SEQUENCE)return &controls[0]; + if(id==V4L2_CID_STATELESS_AV1_FRAME)return &controls[1]; + if(id==V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY)return &controls[2]; + return &controls[3]; +} +static void avd_run_preamble(struct avd_ctx *ctx,struct avd_run *run){ + (void)ctx;memset(run,0,sizeof(*run));run->bufs.src=current_src;run->bufs.dst=current_dst; + run->coded_in=current_src->vb2_buf.dma;run->comp_out=current_dst->vb2_buf.dma; + run->y_out=run->comp_out+0x100000;run->uv_out=run->comp_out+0x300000; +} +static void avd_run_postamble(struct avd_ctx *ctx,struct avd_run *run){(void)ctx;(void)run;} +static int avd_init_job(struct avd_ctx *ctx,int codec,size_t segments){ + free(ctx->job.segments);ctx->job.codec=codec;ctx->job.num=0; + ctx->job.segments=calloc(segments,sizeof(*ctx->job.segments));return ctx->job.segments?0:-ENOMEM; +} +static int avd_submit_job(struct avd_ctx *ctx); +''' + + +def without_includes(s): + return re.sub(r'^#include[^\n]*\n', '', s, flags=re.M) + + +av1 = (AVD / 'avd-av1.c').read_text() +common = pre + without_includes((AVD / 'avd-av1-entropymode.h').read_text()) +common += without_includes((AVD / 'avd-av1-entropymode.c').read_text()) +common += without_includes((AVD / 'avd-inst.h').read_text()) +qfw = (AVD / 'avd-backend-queue.c').read_text() +hw = (AVD / 'avd-hw.c').read_text() +post = function((AVD / 'avd-backend-open.c').read_text(), 'avd_open_build_program') +post += qfw[qfw.index('#define AVDQ_MAGIC'):qfw.index('/* Arguments of dev_dbg()')] +post += hw[hw.index('#define AVD_V3_'):hw.index('int avd_boot(')] +post += '#define w32(r,v) writel((v),avd->ctrl+(r))\n#define m32(r,v) w32(r,(v)|readl(avd->ctrl+(r)))\n' +post += function(hw, 't8112_configure_stream') + function(hw, 't8122_configure_stream') +post += function(qfw, 'avd_qfw_stream') + function(qfw, 'avd_qfw_submit') +post += '\nstatic int avd_submit_job(struct avd_ctx *ctx){return avd_open_build_program(ctx);}\n' + + +def codec(s): + return without_includes(s[s.index('#define V4L2_AV1_SEG_LVL_ALT_LF_Y_H'):s.index('static enum avd_image_fmt avd_av1_get_image_fmt')]) + function(s, 'avd_av1_pp_kicks') + + +MAIN = r''' +static void write_program(const char *dir,const char *name,struct avd_frame *f){ + char path[4096];snprintf(path,sizeof(path),"%s/%s.bin",dir,name);FILE *out=fopen(path,"wb");assert(out); + assert(fwrite(f->prog->cpu,4,f->prog_words,out)==f->prog_words);fclose(out); + struct avd_dev *dev=f->ctx->dev;struct avd_qfw *q=dev->qfw; + q->wp=0;memset(q->sram,0,0x10000);f->tag=0x10|f->idx;assert(!avd_qfw_submit(dev,f)); + snprintf(path,sizeof(path),"%s/%s-desc.bin",dir,name);out=fopen(path,"wb");assert(out); + assert(fwrite(q->sram+0x8000+f->idx*64,1,64,out)==64);fclose(out); + snprintf(path,sizeof(path),"%s/%s-mmio.bin",dir,name);out=fopen(path,"wb");assert(out); + assert(fwrite(dev->ctrl,1,0x10000,out)==0x10000);fclose(out); +} +static void queue_request(struct vb2_v4l2_buffer *src){ + request_params=params;controls[1].id=V4L2_CID_STATELESS_AV1_FRAME; + request_ref=(struct v4l2_ctrl_ref){.ctrl=&controls[1],.p_req_valid=true,.p_req.p=&request_params}; + request_hdl.ctrl_refs=&request_ref;src->vb2_buf.req_obj.req=&request_hdl; +} +static void flag_table(struct avd_av1_bank *b){ + /* Actual producer timestamp is slot 5, not the immediately prior slot 0. */ + struct v4l2_ctrl_av1_frame f={.frame_type=V4L2_AV1_INTER_FRAME,.primary_ref_frame=2, + .frame_width_minus_1=1919,.frame_height_minus_1=1087}; + for(unsigned int i=0;i<7;i++)f.ref_frame_idx[i]=i; + f.ref_frame_idx[2]=5;f.reference_frame_ts[5]=b->timestamp; + for(unsigned int defaults=0;defaults<4;defaults++) + for(unsigned int output_disabled=0;output_disabled<2;output_disabled++) + for(unsigned int input_disabled=0;input_disabled<2;input_disabled++) + for(unsigned int producer_seg=0;producer_seg<4;producer_seg++) + for(unsigned int consumer_seg=0;consumer_seg<4;consumer_seg++) + for(unsigned int refresh=0;refresh<256;refresh++){ + f.primary_ref_frame=defaults==1?7:2; + f.frame_type=defaults==2?V4L2_AV1_INTRA_ONLY_FRAME:V4L2_AV1_INTER_FRAME; + f.flags=(defaults==3?V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE:0)| + (output_disabled?V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF:0); + f.refresh_frame_flags=refresh; + f.segmentation.flags=(consumer_seg&1?V4L2_AV1_SEGMENTATION_FLAG_ENABLED:0)| + (consumer_seg&2?V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP:0); + b->flags=input_disabled?V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF:0; + b->seg_flags=(producer_seg&1?V4L2_AV1_SEGMENTATION_FLAG_ENABLED:0)| + (producer_seg&2?V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP:0); + bool expect=defaults!=1&&((!defaults&&!input_disabled)||((consumer_seg&1)&&producer_seg==3)); + assert(avd_av1_depends(&f,b)==expect); + } + for(unsigned int primary=0;primary<7;primary++){ + f.primary_ref_frame=primary;f.flags=0;f.frame_type=V4L2_AV1_INTER_FRAME;f.segmentation.flags=0; + b->flags=0;f.ref_frame_idx[primary]=5;assert(avd_av1_depends(&f,b)); + f.reference_frame_ts[5]++;assert(!avd_av1_depends(&f,b));f.reference_frame_ts[5]--; + } + f.primary_ref_frame=2;f.flags=V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS;f.reference_frame_ts[5]=b->timestamp; + assert(avd_av1_depends(&f,b));f.primary_ref_frame=7;assert(avd_av1_depends(&f,b)); + f.reference_frame_ts[5]++;assert(!avd_av1_depends(&f,b)); + puts("PASS: AV1 flag table (primary, consumer/producer CDF flags, segmentation enabled/update_map, every refresh mask), actual slot and MV dependencies"); +} +int main(int argc,char **argv){ + assert(argc==2); + struct avd_variant variant={.revision=4,.quirks=0,.fifo_slots=15,.vp_slots={4,4,1,3}}; + struct avd_dev dev={.variant=&variant,.queued=true,.capabilities=15}; + struct avd_qfw queue={.sram=calloc(1,0x10000),.aperture=0x1094000,.ring_words=0x1800,.fifo_slots=15,.vp_done=true}; + dev.qfw=&queue;dev.ctrl=calloc(1,0x10000);dev.mbox=calloc(1,0x100);variant.configure_stream=t8122_configure_stream; + writel(0x20001,dev.mbox+0x50); + static const struct avd_coded_fmt_ops ops={.pp_kicks=avd_av1_pp_kicks}; + static const struct fmt_desc fmt={.ops=&ops}; + struct avd_ctx ctx={.dev=&dev,.coded_fmt_desc=&fmt,.pp_total=1}; + struct avd_buf program={};struct avd_frame frame={.prog=&program,.ctx=&ctx};dev.building=&frame; + struct avd_decoded_buffer dst[3]={};struct vb2_v4l2_buffer src={};u8 coded[512]={}; + src.vb2_buf=(struct vb2_buffer){.dma=0x220000000ULL,.cpu=coded,.payload=512,.timestamp=1000};current_src=&src; + for(unsigned int i=0;i<3;i++){ + dst[i].base.vb.vb2_buf=(struct vb2_buffer){.dma=0x230000000ULL+i*0x1000000ULL,.cpu=calloc(1,0x800000),.planes[0].length=0x800000,.timestamp=1000+i};refs[i]=&dst[i]; + } + ctx.decoded_fmt.fmt.pix_mp.width=1920;ctx.decoded_fmt.fmt.pix_mp.height=1088; + ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; + assert(!avd_av1_ready(&ctx));allocating_ctx=&ctx;assert(!avd_av1_start(&ctx));allocating_ctx=NULL; + struct avd_av1_stream *p=((struct avd_av1_ctx *)ctx.priv)->stream;assert(p->nbanks==2&&ctx.max_frames==2); + assert(p->bank[0].codec.bufs.probs.addr!=p->bank[1].codec.bufs.probs.addr); + seq=(struct v4l2_ctrl_av1_sequence){.bit_depth=8,.flags=V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_X|V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_Y}; + params=(struct v4l2_ctrl_av1_frame){.frame_type=V4L2_AV1_KEY_FRAME,.primary_ref_frame=7, + .flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF,.frame_width_minus_1=1919,.frame_height_minus_1=1087,.upscaled_width=1920}; + params.tile_info.tile_cols=params.tile_info.tile_rows=1;params.tile_info.width_in_sbs_minus_1[0]=29;params.tile_info.height_in_sbs_minus_1[0]=16; + tile.tile_size=512;tile.tile_col=tile.tile_row=0; + current_dst=&dst[0].base.vb;frame.idx=0;assert(avd_av1_ready(&ctx));assert(!avd_av1_run(&ctx)); + assert(dst[0].bank==1&&p->bank[0].busy&&p->bank[0].prepared); + assert(frame.fifo_iova==p->bank[0].codec.bufs.inst.addr); + write_program(argv[1],"av1-overlap-0",&frame);flag_table(&p->bank[0]); + p->bank[0].flags=params.flags;p->bank[0].seg_flags=0; + params.frame_type=V4L2_AV1_INTER_FRAME;params.primary_ref_frame=0;params.ref_frame_idx[0]=0;params.reference_frame_ts[0]=1000; + params.segmentation.flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED; + queue_request(&src); + /* p_cur is deliberately stale and different from the pending request. */ + params.flags=0;request_params.flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; + assert(avd_av1_ready(&ctx)); + p->bank[0].flags=0;assert(!avd_av1_ready(&ctx));p->bank[0].flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; + p->bank[0].seg_flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED|V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP; + assert(!avd_av1_ready(&ctx));p->bank[0].seg_flags=0; + request_params.flags|=V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS;assert(!avd_av1_ready(&ctx));request_params.flags&=~V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS; + assert(avd_av1_ready(&ctx));params=request_params; + /* CAPTURE tails are poisoned: successor must consume preparation snapshots. */ + size_t tail=AVD_AV1_CDFS_OFFSET(0x800000,dst[0].av1.color_size); + memset((u8 *)dst[0].base.vb.vb2_buf.cpu+tail,0xee,sizeof(struct avd_av1_cdfs)); + memset(p->bank[0].codec.bufs.saved_seg.cpu,0x35,dst[0].av1.seg_size); + current_dst=&dst[1].base.vb;src.vb2_buf.timestamp=1001;frame.idx=1; + assert(!avd_av1_run(&ctx));assert(dst[1].bank==2&&p->bank[0].busy&&p->bank[1].busy); + assert(!memcmp(p->bank[0].codec.bufs.probs.cpu,p->bank[1].codec.bufs.probs.cpu,sizeof(struct avd_av1_cdfs))); + assert(!memcmp(p->bank[0].codec.bufs.saved_seg.cpu,p->bank[1].codec.bufs.prev_seg.cpu,dst[1].av1.seg_size)); + assert(p->bank[1].codec.bufs.prev_seg.addr!=p->bank[0].codec.bufs.saved_seg.addr); + write_program(argv[1],"av1-overlap-1",&frame);assert(!avd_av1_ready(&ctx)); + current_dst=&dst[2].base.vb;assert(avd_av1_run(&ctx)==-EBUSY);assert(!dst[2].bank); + avd_av1_done(&ctx,&src,&dst[0].base.vb,VB2_BUF_STATE_DONE); + assert(!p->bank[0].busy&&p->bank[1].busy&&dst[1].bank==2); + /* Producer bank reuse must not invalidate the consumer's VP map input. */ + memset(p->bank[0].codec.bufs.saved_seg.cpu,0x98,dst[0].av1.seg_size); + assert(((u8 *)p->bank[1].codec.bufs.prev_seg.cpu)[0]==0x35); + avd_av1_done(&ctx,&src,&dst[0].base.vb,VB2_BUF_STATE_DONE);assert(p->bank[1].busy); + avd_av1_done(&ctx,&src,&dst[1].base.vb,VB2_BUF_STATE_ERROR);assert(!p->bank[1].busy&&avd_av1_ready(&ctx)); + assert(request_gets==request_puts); + avd_av1_stop(&ctx);assert(!avd_av1_ready(&ctx)); + for(unsigned int mode=0;mode<2;mode++){ + dev.queued=mode;vp_feature=false;assert(!avd_av1_start(&ctx));p=((struct avd_av1_ctx *)ctx.priv)->stream; + assert(p->nbanks==1&&ctx.max_frames==1);avd_av1_stop(&ctx); + } + dev.queued=true;vp_feature=true; + for(int n=1;n<=6;n++){ + fail_alloc_after=n;allocating_ctx=&ctx;assert(avd_av1_start(&ctx)==-ENOMEM);allocating_ctx=NULL; + assert(!ctx.priv&&!avd_av1_ready(&ctx)); + } + for(unsigned int i=0;i<3;i++)free(dst[i].base.vb.vb2_buf.cpu); + free(program.cpu);free(queue.sram);free(dev.ctrl);free(dev.mbox); + puts("PASS: AV1 snapshot inheritance, poisoned CAPTURE, request lookahead, banks, PP retirement, exhaustion, failures and legacy fallback"); +} +''' + + +def build_run(name, source): + (OUT / f'{name}.c').write_text(source) + cmd = [os.environ.get('CC', 'cc'), '-std=gnu11', '-g', '-O1', '-Wall', '-Wextra', '-Werror', + '-Wno-unused-function', '-Wno-unused-variable', '-Wno-unused-parameter', '-Wno-sign-compare', '-Wno-pointer-sign', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', str(OUT / f'{name}.c'), '-o', str(OUT / name)] + subprocess.run(cmd, check=True) + subprocess.run([str(OUT / name), str(OUT)], check=True) + + +build_run('av1-host', common + codec(av1) + post + MAIN) +(OUT / 'source.sha256').write_text('\n'.join(hashlib.sha256(p.read_bytes()).hexdigest() + ' ' + str(p) for p in ( + AVD / 'avd-av1.c', AVD / 'avd-av1-entropymode.c', AVD / 'avd-backend-open.c', AVD / 'avd-backend-queue.c')) + '\n') + +base_ref = os.environ.get('AVD_AV1_BASELINE_REF') +if base_ref: + old = subprocess.check_output(['git', '-C', str(KERNEL), 'show', + base_ref + ':drivers/media/platform/apple/avd/avd-av1.c'], text=True) + legacy = r''' +int main(int argc,char **argv){ + assert(argc==2); + struct avd_variant variant={.revision=4,.fifo_slots=15}; + struct avd_dev dev={.variant=&variant,.queued=false}; + struct avd_coded_fmt_ops ops={};struct fmt_desc fmt={.ops=&ops}; + struct avd_ctx ctx={.dev=&dev,.coded_fmt_desc=&fmt}; + struct avd_buf program={};struct avd_frame f={.prog=&program,.ctx=&ctx};dev.building=&f; + struct avd_decoded_buffer dst[2]={};struct vb2_v4l2_buffer src={};u8 coded[512]={}; + current_src=&src;src.vb2_buf=(struct vb2_buffer){.dma=0x220000000ULL,.cpu=coded,.payload=512}; + for(unsigned int i=0;i<2;i++){ + dst[i].base.vb.vb2_buf=(struct vb2_buffer){.dma=0x230000000ULL+i*0x1000000ULL,.cpu=calloc(1,0x800000),.planes[0].length=0x800000,.timestamp=1000+i};refs[i]=&dst[i]; + } + ctx.inst.addr=0x205000000ULL; + ctx.decoded_fmt.fmt.pix_mp.width=1920;ctx.decoded_fmt.fmt.pix_mp.height=1088; + ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; + seq=(struct v4l2_ctrl_av1_sequence){.bit_depth=8,.flags=V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_X|V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_Y}; + params=(struct v4l2_ctrl_av1_frame){.frame_type=V4L2_AV1_KEY_FRAME,.primary_ref_frame=7, + .frame_width_minus_1=1919,.frame_height_minus_1=1087,.upscaled_width=1920}; + params.tile_info.tile_cols=params.tile_info.tile_rows=1;params.tile_info.width_in_sbs_minus_1[0]=29;params.tile_info.height_in_sbs_minus_1[0]=16;tile.tile_size=512; + assert(!avd_av1_start(&ctx)); + for(unsigned int mode=0;mode<8;mode++){ + current_dst=&dst[!!mode].base.vb; + params.frame_type=mode==0?V4L2_AV1_KEY_FRAME:mode==2?V4L2_AV1_INTRA_ONLY_FRAME:V4L2_AV1_INTER_FRAME; + params.primary_ref_frame=mode==0||mode==4?7:0; + params.ref_frame_idx[0]=0;params.reference_frame_ts[0]=1000; + params.flags=(mode==3?V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE:0)|(mode==5?V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF:0); + params.segmentation.flags=mode>=6?V4L2_AV1_SEGMENTATION_FLAG_ENABLED:0; + if(mode==7)params.segmentation.flags|=V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP; + assert(!avd_av1_run(&ctx)); + char path[4096];snprintf(path,sizeof(path),"%s/@NAME@-%u.bin",argv[1],mode);FILE *out=fopen(path,"wb");assert(out); + assert(fwrite(program.cpu,4,f.prog_words,out)==f.prog_words); + struct avd_decoded_buffer *d=&dst[!!mode];size_t off=AVD_AV1_SEG_OFFSET(0x800000,d->av1.color_size,d->av1.seg_size); + assert(fwrite((u8 *)d->base.vb.vb2_buf.cpu+off,1,d->av1.seg_size+ALIGN(AVD_CDFS_SIZE,AVD_ALIGN),out)==d->av1.seg_size+ALIGN(AVD_CDFS_SIZE,AVD_ALIGN));fclose(out); + if(mode==0){ + size_t cdf=AVD_AV1_CDFS_OFFSET(0x800000,dst[0].av1.color_size); + memset((u8 *)dst[0].base.vb.vb2_buf.cpu+cdf,0x57,AVD_CDFS_SIZE); + memset((u8 *)dst[0].base.vb.vb2_buf.cpu+cdf-dst[0].av1.seg_size,0x36,dst[0].av1.seg_size); + } + } + avd_av1_stop(&ctx);free(program.cpu); + for(unsigned int i=0;i<2;i++)free(dst[i].base.vb.vb2_buf.cpu); +} +''' + for name, body in [('legacy-before', old), ('legacy-after', av1)]: + shim = 'static bool avd_av1_ready(struct avd_ctx *ctx){(void)ctx;return false;}\n' if name == 'legacy-before' else '' + build_run(name, common + codec(body) + shim + post + legacy.replace('@NAME@', name)) + for mode in range(8): + assert (OUT / f'legacy-before-{mode}.bin').read_bytes() == (OUT / f'legacy-after-{mode}.bin').read_bytes(), mode + print('PASS: separate baseline/current executables emit byte-identical legacy AV1 programs and CDF/map tails for eight header cases') diff --git a/drivers/media/platform/apple/avd/tests/unicorn-av1-test.py b/drivers/media/platform/apple/avd/tests/unicorn-av1-test.py new file mode 100644 index 0000000000000..93f4651686776 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/unicorn-av1-test.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""AV1 production programs: VP(N+1) starts after VP(N) DONE, during PP(N). + +Usage: unicorn-av1-test.py FIRMWARE_TREE v5-t2_IMAGE AV1_HOST_OUTPUT +CDF/map input snapshots are host fixtures. CM3 code, descriptors, program words, +PP commands and configure_stream writes are production. No pixels are decoded. +""" +import importlib.util +from pathlib import Path +import struct +import sys +from unicorn.arm_const import UC_ARM_REG_SP + +fw, image, programs = map(lambda p: Path(p).resolve(), sys.argv[1:4]) +spec = importlib.util.spec_from_file_location('transport', fw / 'tests/transport-test.py') +transport = importlib.util.module_from_spec(spec) +spec.loader.exec_module(transport) + + +class AV1(transport.Transport): + def __init__(self): + self.booting = False + self.expected = [] + self.events = [] + self.live = self.pp_owner = None + self.feed = self.word = 0 + super().__init__(image) + assert self.version == 5 and self.tier == 2 + self.booting = True + self.uc.reg_write(UC_ARM_REG_SP, self.u32(0)) + self.uc.emu_start(self.u32(4), 0, count=500000) + assert not self.booting + + def route(self, irq): + if irq < 14: + router = self.u32(0x50010010) & (1 << irq) + else: + off = irq - 14 + router = self.u32(0x50010014 + off // 32 * 4) & (1 << (off % 32)) + nvic = self.u32(0xe000e100 + irq // 32 * 4) & (1 << (irq % 32)) + assert router and nvic, f'IRQ {irq} masked' + self.call(f'irq{irq}') + + def write(self, uc, access, addr, size, value, user): + if self.booting and addr == 0x50010090 and value == 1: + self.booting = False + uc.emu_stop() + if addr == self.decode + self.fifo + 9 * 4: + idx, words = self.expected[self.feed] + assert value == words[self.word], (idx, self.word, value) + if not self.word: + assert self.live is None, 'successor issued before VP DONE' + self.live = idx + self.events.append(f'AV1 VP START {idx}; PP owner={self.pp_owner}') + self.word += 1 + if self.word == len(words): + self.feed += 1 + self.word = 0 + if addr == self.decode + self.pp: + idx = (value >> 4) & 15 + assert self.pp_owner is None, 'successor PP issued before predecessor PP DONE' + assert value == 0x2b00020f | idx << 4 + self.pp_owner = idx + self.events.append(f'PP START {idx}') + if addr == 0x50010060: + self.events.append(f'notification {value:#x}') + super().write(uc, access, addr, size, value, user) + + def submit_host(self, idx, start): + stem = f'av1-overlap-{idx}' + data = (programs / f'{stem}.bin').read_bytes() + words = list(struct.unpack('<' + 'I' * (len(data) // 4), data)) + desc = list(struct.unpack('<16I', (programs / f'{stem}-desc.bin').read_bytes())) + assert desc[:4] == [0x41565131, 0x10 | idx, 3, 9] + assert desc[4:6] == [0, 0] and desc[6] == len(words) + assert desc[8:16] == [1, 0, 0, 1, 0, 0, 0, 0] + assert words[0] == 0x2b000200 | idx << 4 + assert words[-1] == 0x2b000400 | idx << 4 + mmio = (programs / f'{stem}-mmio.bin').read_bytes() + for offset in (0x1d0, 0x20c, 0x248, 0x284, 0x2c0): + at = offset + idx * 4 + self.put(self.decode + at, struct.unpack_from(' Date: Sun, 1 Nov 2026 19:10:02 +1100 Subject: [PATCH 62/73] Revert "media: apple: avd: overlap AV1 pictures with immutable tail inputs" This reverts commit 2f9bd00ed9cc50572b2748febaa077f4c2b3ebcd. The AV1 overlap has host and emulator tests but has not run on AV1 hardware (T6040, T8140) yet, and this branch is about to merge with the code that was qualified on the J616s and the M2. The change is kept on avd/rewrite-av1-overlap and comes back once it is bit-exact on hardware. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-av1.c | 328 ++--------------- .../media/platform/apple/avd/tests/README.rst | 127 ++----- .../platform/apple/avd/tests/av1-test.py | 330 ------------------ .../apple/avd/tests/unicorn-av1-test.py | 144 -------- tools/asahi/avd-codec-tail-tests.py | 4 - 5 files changed, 66 insertions(+), 867 deletions(-) delete mode 100644 drivers/media/platform/apple/avd/tests/av1-test.py delete mode 100644 drivers/media/platform/apple/avd/tests/unicorn-av1-test.py diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index 6a87a4de6fe9b..ba7beea61c842 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -225,15 +225,12 @@ struct avd_av1_run { }; struct avd_av1_ctx { - struct avd_av1_stream *stream; u8 submit_num; struct { struct avd_buf inst; struct avd_buf pipe_state; struct avd_buf probs; struct avd_buf no_prev_seg; - struct avd_buf saved_seg; - struct avd_buf prev_seg; struct avd_buf above_info; struct avd_buf rf_above_info; struct avd_buf az_above; @@ -249,24 +246,6 @@ struct avd_av1_ctx { } bufs; }; -#define AVD_AV1_BANKS 2 - -struct avd_av1_bank { - struct avd_av1_ctx codec; - bool busy; - bool prepared; - struct avd_decoded_buffer *dst; - u64 timestamp; - u32 flags; - u8 seg_flags; -}; - -struct avd_av1_stream { - unsigned int nbanks; - struct avd_av1_bank bank[AVD_AV1_BANKS]; - struct avd_av1_bank *b; -}; - /* * Timestamp of the reference a ref_frame_idx entry names. The entries are * -1 on frames that use no references (FFmpeg fills key frames that way); @@ -280,111 +259,6 @@ static u64 avd_av1_ref_ts(const struct v4l2_ctrl_av1_frame *frame, int idx) return frame->reference_frame_ts[idx]; } -static bool avd_av1_default_cdf(const struct v4l2_ctrl_av1_frame *frame) -{ - return frame->primary_ref_frame == AV1_PRIMARY_REF_NONE || - frame->frame_type == V4L2_AV1_KEY_FRAME || - frame->frame_type == V4L2_AV1_INTRA_ONLY_FRAME || - (frame->flags & V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE); -} - -static bool avd_av1_writes_seg(u8 flags) -{ - return (flags & V4L2_AV1_SEGMENTATION_FLAG_ENABLED) && - (flags & V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP); -} - -/* - * No available hardware contract proves AV1 tails CPU-visible at VP DONE. - * Only preparation's immutable CDF/map snapshots can be consumed before PP - * retirement. The consumer's disable_frame_end_update_cdf does not remove - * its CDF input dependency. Slot timestamps, not refresh order, select it. - */ -static bool avd_av1_depends(const struct v4l2_ctrl_av1_frame *frame, - const struct avd_av1_bank *bank) -{ - if (frame->flags & V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS) - for (unsigned int i = 0; i < V4L2_AV1_REFS_PER_FRAME; i++) - if (avd_av1_ref_ts(frame, frame->ref_frame_idx[i]) == bank->timestamp) - return true; - - if (frame->primary_ref_frame >= AV1_PRIMARY_REF_NONE || - avd_av1_ref_ts(frame, frame->ref_frame_idx[frame->primary_ref_frame]) != - bank->timestamp) - return false; - if (!avd_av1_default_cdf(frame) && - !(bank->flags & V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF)) - return true; - return (frame->segmentation.flags & V4L2_AV1_SEGMENTATION_FLAG_ENABLED) && - avd_av1_writes_seg(bank->seg_flags) && - DIV_ROUND_UP(bank->dst->av1.width + 1, 8) == - DIV_ROUND_UP(frame->frame_width_minus_1 + 1, 8) && - DIV_ROUND_UP(bank->dst->av1.height + 1, 8) == - DIV_ROUND_UP(frame->frame_height_minus_1 + 1, 8); -} - -static bool avd_av1_ready(struct avd_ctx *ctx) -{ - struct avd_av1_ctx *codec = smp_load_acquire(&ctx->priv); - struct avd_av1_stream *av1_ctx; - struct vb2_v4l2_buffer *src; - struct v4l2_ctrl_handler *hdl; - struct v4l2_ctrl_ref *ref; - const struct v4l2_ctrl_av1_frame *frame = NULL; - bool ready = false, free = false, busy = false; - - if (!codec) - return false; - av1_ctx = codec->stream; - for (unsigned int i = 0; i < av1_ctx->nbanks; i++) { - if (!READ_ONCE(av1_ctx->bank[i].busy)) - free = true; - else if (!smp_load_acquire(&av1_ctx->bank[i].prepared)) - return false; - else - busy = true; - } - if (!free || !busy) - return free; - src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); - if (!src || !src->vb2_buf.req_obj.req) - return false; - hdl = v4l2_ctrl_request_hdl_find(src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); - if (!hdl) - return false; - /* - * The queued request is immutable and cannot complete while its OUTPUT - * is still on the ready queue. Read p_req, never the previous picture's - * shared p_cur, without taking a sleeping control mutex under queue_lock. - */ - list_for_each_entry(ref, &hdl->ctrl_refs, node) - if (ref->ctrl->id == V4L2_CID_STATELESS_AV1_FRAME && ref->p_req_valid) { - frame = ref->p_req.p; - break; - } - if (frame) { - ready = true; - for (unsigned int i = 0; i < av1_ctx->nbanks; i++) - if (READ_ONCE(av1_ctx->bank[i].busy) && - avd_av1_depends(frame, &av1_ctx->bank[i])) - ready = false; - } - v4l2_ctrl_request_hdl_put(hdl); - return ready; -} - -static struct avd_av1_bank *avd_av1_ref_bank(struct avd_av1_stream *av1_ctx, - struct avd_decoded_buffer *ref) -{ - if (av1_ctx->nbanks > 1 && ref->bank && ref->bank <= av1_ctx->nbanks) { - struct avd_av1_bank *bank = &av1_ctx->bank[ref->bank - 1]; - - if (bank->busy && bank->prepared && bank->dst == ref) - return bank; - } - return NULL; -} - /* * from verisilicon/rockchip_vpu981_hw_av1_dec.c */ @@ -1286,71 +1160,6 @@ static void avd_av1_set_seg(struct avd_ctx *ctx, struct avd_av1_run *run) memcpy(seg, prev_seg, dst->av1.seg_size); } -/* Preserve the legacy tail preparation; only queued immutable inputs differ. */ -static void avd_av1_prepare_prob(struct avd_ctx *ctx, struct avd_av1_run *run) -{ - struct avd_av1_ctx *codec = ctx->priv; - struct avd_av1_stream *stream = codec->stream; - struct avd_decoded_buffer *dst, *ref; - struct avd_av1_bank *bank = NULL; - - dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); - if (stream->nbanks > 1 && !avd_av1_default_cdf(run->frame)) { - ref = avd_get_ref_buf(ctx, &dst->base.vb, - avd_av1_ref_ts(run->frame, - run->frame->ref_frame_idx[run->frame->primary_ref_frame])); - bank = avd_av1_ref_bank(stream, ref); - } - if (!bank) { - avd_av1_set_prob(ctx, run); - return; - } - memcpy(codec->bufs.probs.cpu, bank->codec.bufs.probs.cpu, AVD_CDFS_SIZE); - memcpy(vb2_plane_vaddr(&dst->base.vb.vb2_buf, 0) + - AVD_AV1_CDFS_OFFSET(dst->base.vb.vb2_buf.planes[0].length, - dst->av1.color_size), - codec->bufs.probs.cpu, AVD_CDFS_SIZE); -} - -static void avd_av1_prepare_seg(struct avd_ctx *ctx, struct avd_av1_run *run) -{ - struct avd_av1_ctx *codec = ctx->priv; - struct avd_av1_stream *stream = codec->stream; - const struct v4l2_ctrl_av1_frame *frame = run->frame; - struct avd_decoded_buffer *dst, *ref; - struct avd_av1_bank *bank = NULL; - void *seg; - - dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); - seg = vb2_plane_vaddr(&dst->base.vb.vb2_buf, 0) + - AVD_AV1_SEG_OFFSET(dst->base.vb.vb2_buf.planes[0].length, - dst->av1.color_size, dst->av1.seg_size); - if (stream->nbanks > 1 && frame->primary_ref_frame != AV1_PRIMARY_REF_NONE) { - ref = avd_get_ref_buf(ctx, &dst->base.vb, - avd_av1_ref_ts(frame, frame->ref_frame_idx[frame->primary_ref_frame])); - if (ref != dst && - DIV_ROUND_UP(ref->av1.width + 1, 8) == - DIV_ROUND_UP(frame->frame_width_minus_1 + 1, 8) && - DIV_ROUND_UP(ref->av1.height + 1, 8) == - DIV_ROUND_UP(frame->frame_height_minus_1 + 1, 8)) - bank = avd_av1_ref_bank(stream, ref); - } - if (bank && !avd_av1_writes_seg(bank->seg_flags)) { - /* Copy into our bank: producer PP can retire during our VP. */ - memcpy(codec->bufs.prev_seg.cpu, bank->codec.bufs.saved_seg.cpu, - dst->av1.seg_size); - run->addresses.prev_seg = codec->bufs.prev_seg.addr; - if (!(frame->segmentation.flags & V4L2_AV1_SEGMENTATION_FLAG_ENABLED)) - memset(seg, 0, dst->av1.seg_size); - else if (!(frame->segmentation.flags & V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP)) - memcpy(seg, codec->bufs.prev_seg.cpu, dst->av1.seg_size); - } else { - avd_av1_set_seg(ctx, run); - } - if (stream->nbanks > 1 && !avd_av1_writes_seg(frame->segmentation.flags)) - memcpy(codec->bufs.saved_seg.cpu, seg, dst->av1.seg_size); -} - static int avd_av1_seg_size(int w, int h) { /* 3 bits per 4x4 block, in rows of 128x128 superblocks (64x64 fit too) */ @@ -1471,17 +1280,6 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) int h = frame->frame_height_minus_1 + 1; int bit_depth = seq->bit_depth; - if (av1_ctx->stream->nbanks > 1) { - ret = avd_buf_alloc(avd, &av1_ctx->bufs.saved_seg, - avd_av1_seg_size(w, h)); - if (ret) - return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.prev_seg, - avd_av1_seg_size(w, h)); - if (ret) - return ret; - } - for (i = 0; i < tile_info->tile_cols; i++) { sb = tile_info->width_in_sbs_minus_1[i] + 1; max_sb_col = sb > max_sb_col ? sb : max_sb_col; @@ -1582,7 +1380,6 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) static int avd_av1_run(struct avd_ctx *ctx) { struct avd_av1_run run; - struct avd_av1_stream *av1_ctx = ((struct avd_av1_ctx *)ctx->priv)->stream; struct avd_decoded_buffer *dst, *ref; int ret; @@ -1605,28 +1402,6 @@ static int avd_av1_run(struct avd_ctx *ctx) avd_run_postamble(ctx, &run.base); return -ENOMEM; } - if (av1_ctx->nbanks > 1) { - struct avd_av1_bank *bank = NULL; - - for (unsigned int i = 0; i < av1_ctx->nbanks; i++) - if (!av1_ctx->bank[i].busy) { - bank = &av1_ctx->bank[i]; - break; - } - if (!bank) { - avd_run_postamble(ctx, &run.base); - return -EBUSY; - } - av1_ctx->b = bank; - smp_store_release(&ctx->priv, &bank->codec); - WRITE_ONCE(bank->prepared, false); - WRITE_ONCE(bank->busy, true); - bank->dst = dst; - bank->timestamp = run.base.bufs.src->vb2_buf.timestamp; - bank->flags = run.frame->flags; - bank->seg_flags = run.frame->segmentation.flags; - dst->bank = bank - av1_ctx->bank + 1; - } ret = avd_init_job(ctx, AVD_CODEC_AV1, run.frame->tile_info.tile_cols * @@ -1645,51 +1420,33 @@ static int avd_av1_run(struct avd_ctx *ctx) update_dec_buf_info(dst, run.seq, run.frame); - avd_av1_prepare_prob(ctx, &run); - avd_av1_prepare_seg(ctx, &run); + avd_av1_set_prob(ctx, &run); + avd_av1_set_seg(ctx, &run); set_header(ctx, &run); set_tiles(ctx, &run); - if (av1_ctx->nbanks > 1) { - ctx->job_fifo = av1_ctx->b->codec.bufs.inst.addr; - smp_store_release(&av1_ctx->b->prepared, true); - } avd_run_postamble(ctx, &run.base); return avd_submit_job(ctx); } -static void avd_av1_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src, - struct vb2_v4l2_buffer *dst, enum vb2_buffer_state result) -{ - struct avd_av1_stream *av1_ctx = ((struct avd_av1_ctx *)ctx->priv)->stream; - struct avd_decoded_buffer *buf = vb2_to_avd_decoded_buf(&dst->vb2_buf); - - if (buf->bank && buf->bank <= av1_ctx->nbanks) { - struct avd_av1_bank *bank = &av1_ctx->bank[buf->bank - 1]; - - if (bank->dst == buf) - WRITE_ONCE(bank->busy, false); - buf->bank = 0; - } -} - -static int avd_av1_alloc_bufs(struct avd_ctx *ctx, struct avd_av1_bank *bank) +static int avd_av1_alloc_bufs(struct avd_ctx *ctx) { struct avd_dev *avd = ctx->dev; + struct avd_av1_ctx *av1_ctx = ctx->priv; int ret; - ret = avd_buf_alloc(avd, &bank->codec.bufs.inst, fifo_size()); + ret = avd_buf_alloc(avd, &av1_ctx->bufs.inst, fifo_size()); if (ret) return ret; /* no m1 to worry about :) */ - ret = avd_buf_alloc(avd, &bank->codec.bufs.pipe_state, 0x200); + ret = avd_buf_alloc(avd, &av1_ctx->bufs.pipe_state, 0x200); if (ret) return ret; - ret = avd_buf_alloc(avd, &bank->codec.bufs.probs, + ret = avd_buf_alloc(avd, &av1_ctx->bufs.probs, sizeof(struct avd_av1_cdfs)); if (ret) return ret; @@ -1697,66 +1454,51 @@ static int avd_av1_alloc_bufs(struct avd_ctx *ctx, struct avd_av1_bank *bank) return 0; } -static void avd_av1_free(struct avd_ctx *ctx, struct avd_av1_stream *av1_ctx) +static void avd_av1_stop(struct avd_ctx *ctx) { + struct avd_av1_ctx *av1_ctx = ctx->priv; struct avd_dev *avd = ctx->dev; - for (unsigned int i = 0; i < av1_ctx->nbanks; i++) { - struct avd_av1_bank *bank = &av1_ctx->bank[i]; - - avd_buf_free(avd, &bank->codec.bufs.above_info); - avd_buf_free(avd, &bank->codec.bufs.rf_above_info); - avd_buf_free(avd, &bank->codec.bufs.az_above); - avd_buf_free(avd, &bank->codec.bufs.ip_above); - avd_buf_free(avd, &bank->codec.bufs.lf_above); - avd_buf_free(avd, &bank->codec.bufs.lf_above_info); - avd_buf_free(avd, &bank->codec.bufs.lf_left); - avd_buf_free(avd, &bank->codec.bufs.lf_left_info); - avd_buf_free(avd, &bank->codec.bufs.sr_left); - avd_buf_free(avd, &bank->codec.bufs.rf_left); - avd_buf_free(avd, &bank->codec.bufs.rf_left_info); - avd_buf_free(avd, &bank->codec.bufs.no_prev_seg); - avd_buf_free(avd, &bank->codec.bufs.mv_above_info); - - avd_buf_free(avd, &bank->codec.bufs.pipe_state); - avd_buf_free(avd, &bank->codec.bufs.inst); - avd_buf_free(avd, &bank->codec.bufs.probs); - avd_buf_free(avd, &bank->codec.bufs.saved_seg); - avd_buf_free(avd, &bank->codec.bufs.prev_seg); - } - kfree(av1_ctx); -} + avd_buf_free(avd, &av1_ctx->bufs.above_info); + avd_buf_free(avd, &av1_ctx->bufs.rf_above_info); + avd_buf_free(avd, &av1_ctx->bufs.az_above); + avd_buf_free(avd, &av1_ctx->bufs.ip_above); + avd_buf_free(avd, &av1_ctx->bufs.lf_above); + avd_buf_free(avd, &av1_ctx->bufs.lf_above_info); + avd_buf_free(avd, &av1_ctx->bufs.lf_left); + avd_buf_free(avd, &av1_ctx->bufs.lf_left_info); + avd_buf_free(avd, &av1_ctx->bufs.sr_left); + avd_buf_free(avd, &av1_ctx->bufs.rf_left); + avd_buf_free(avd, &av1_ctx->bufs.rf_left_info); + avd_buf_free(avd, &av1_ctx->bufs.no_prev_seg); + avd_buf_free(avd, &av1_ctx->bufs.mv_above_info); + + avd_buf_free(avd, &av1_ctx->bufs.pipe_state); + avd_buf_free(avd, &av1_ctx->bufs.inst); + avd_buf_free(avd, &av1_ctx->bufs.probs); -static void avd_av1_stop(struct avd_ctx *ctx) -{ - avd_av1_free(ctx, ((struct avd_av1_ctx *)ctx->priv)->stream); + kfree(av1_ctx); ctx->priv = NULL; } static int avd_av1_start(struct avd_ctx *ctx) { - struct avd_av1_stream *av1_ctx; + struct avd_av1_ctx *av1_ctx; int ret; av1_ctx = kzalloc(sizeof(*av1_ctx), GFP_KERNEL); if (!av1_ctx) return -ENOMEM; - av1_ctx->nbanks = avd_qfw_has_vp_done(ctx->dev) ? AVD_AV1_BANKS : 1; - av1_ctx->b = &av1_ctx->bank[0]; - for (unsigned int i = 0; i < av1_ctx->nbanks; i++) { - av1_ctx->bank[i].codec.stream = av1_ctx; - ret = avd_av1_alloc_bufs(ctx, &av1_ctx->bank[i]); - if (ret) - goto err_free_ctx; - } - ctx->max_frames = av1_ctx->nbanks; - /* Readiness must never see partially allocated bank storage. */ - smp_store_release(&ctx->priv, &av1_ctx->bank[0].codec); + ctx->priv = av1_ctx; + ret = avd_av1_alloc_bufs(ctx); + if (ret) + goto err_free_ctx; + return 0; err_free_ctx: - avd_av1_free(ctx, av1_ctx); + avd_av1_stop(ctx); return ret; } @@ -1844,8 +1586,6 @@ const struct avd_coded_fmt_ops avd_av1_fmt_ops = { .start = avd_av1_start, .stop = avd_av1_stop, .run = avd_av1_run, - .done = avd_av1_done, - .ready = avd_av1_ready, .submit = avd_av1_submit, .pp_kicks = avd_av1_pp_kicks, .try_ctrl = avd_av1_try_ctrl, diff --git a/drivers/media/platform/apple/avd/tests/README.rst b/drivers/media/platform/apple/avd/tests/README.rst index 82169ce2a47cd..d2b10320f2b75 100644 --- a/drivers/media/platform/apple/avd/tests/README.rst +++ b/drivers/media/platform/apple/avd/tests/README.rst @@ -179,11 +179,8 @@ and must retain their single completion per picture. Optional queue ABI 2 VP notifications ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Use public MIT ``aurora-silicon/avd-fw`` branch -``avd/queue-abi2-sram-ring`` from commit +The feature is in ``aurora-silicon/avd-fw`` from commit ``bf49a31b1036c72be5fdc37eb8232f6c953e0626`` (queue feature bit 2). -Rebuild with ``make test && sh tests/build-all.sh``. The former local -VP-DONE patch is superseded by that commit and is no longer shipped. This is a backwards-compatible optional ABI 2 feature: * Info feature bit 2 advertises tagged VP DONE. Without it the host keeps VP9 @@ -203,94 +200,34 @@ This is a backwards-compatible optional ABI 2 feature: bank reuse. Only the original tagged PP completion does that. A missing VP event remains bounded by the original per-picture watchdog. -AV1 conservative picture overlap -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``av1-test.py`` compiles the production AV1 codec, entropy defaults, open -program builder and queued descriptor writer with ASan/UBSan. It is also -called by the registered ``avd-codec-tail-tests.py`` host check. Run:: - - AVD_AV1_BASELINE_REF=a1b76856 python3 tests/av1-test.py /tmp/avd-av1 - python3 tests/unicorn-av1-test.py /path/to/avd-fw \ - /path/to/avd-fw/build/avd-fw-v5-t2.bin /tmp/avd-av1 - -The independent legacy comparison builds before/after executables and compares -complete emitted programs plus initialized CDF/map tails for eight header cases. -The flag table covers primary-reference/default CDF cases, both producer and -consumer disable_frame_end_update_cdf, segmentation enabled/update_map on both -pictures, and all 256 refresh masks. Pending request p_req values are used for -readiness; p_cur still describes an earlier picture and must not be used. - -Visibility evidence and limit -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The public AV1 specification sections 7.7, 7.19 and 7.20 define frame CDF, -motion-field and segmentation storage semantics: -https://aomediacodec.github.io/av1-spec/ . They do not define this decoder's -VP/PP DMA boundary. In our driver ``set_header()`` supplies probs_out, seg and -w_color output addresses to the VP program; ``avd_av1_set_prob()`` seeds CDF -output and ``avd_av1_set_seg()`` initializes non-updated maps. The open firmware -``avdq_vp_done()`` changes VP to PP_READY and ``notify_vp()`` publishes a tag -with barriers. Neither reads the AV1 output memory nor supplies an AV1-specific -DMA visibility guarantee. A CM3 DMB orders its descriptor stores; it does not -prove the decoder's prior external tail writes are CPU-visible. - -There is therefore **no evidence here proving that a hardware-written AV1 CDF, -segmentation map or motion field is visible at VP DONE**. There is also no -proof that they are written only at PP DONE. Unicorn injects DONE and cannot -establish either assertion. The implementation keeps those dependencies until -PP retirement instead of inferring a visibility contract from their operands. -No CPU DMA sync is attempted on a CAPTURE allocation still owned by PP. - -Safe subset -^^^^^^^^^^^ - -Queue firmware advertising VP-DONE feature bit 2 enables two scratch banks; -legacy transport and older queue firmware retain one bank and identical -programs. The CDF output, segmentation output and motion-field/color output -already live in each picture's distinct CAPTURE tail. Each scratch bank owns -its FIFO, CDF input, pipe/filter/tile scratch, a saved initialized map and a -separate map input snapshot. Only PP retirement releases its bank. VP DONE -never releases storage or publishes hardware-written tails for AV1. - -For a pending consumer, each busy producer is tested by the exact timestamp -named through ref_frame_idx/primary_ref_frame, rather than the preceding frame -or a global refresh-context flag. refresh_frame_flags controls which slots -userspace subsequently names; it does not create a dependency by itself. - -=============================== ============================================== -Consumer / producer condition Prepare dependency while producer is in flight -=============================== ============================================== -PRIMARY_REF_NONE No primary CDF or map dependency -Intra or error resilient Default CDF; map dependency still checked -Primary slot names retired ref None on an in-flight picture -Primary CDF, producer end update Enabled: wait for producer PP retirement -Primary CDF, producer end update Disabled: use immutable prepared CDF input copy -Consumer end update disabled Does not remove its CDF input dependency -Consumer segmentation disabled No previous-map read -Consumer segmentation enabled Updated producer map: wait for PP (same Mi size) -Producer map not updated Copy immutable saved map into consumer bank -Different reference Mi size Zero map; no map dependency -USE_REF_FRAME_MVS Wait for PP of any named in-flight reference -refresh_frame_flags 0..255 No additional dependency; named slot decides -=============================== ============================================== - -Both consumer update_map values retain the previous-map input when segmentation -is enabled, as the existing program supplies it in either case. A disabled -producer segmentation map is saved as zeroes. Producer map snapshots are copied -into consumer-owned coherent storage, so producer PP retirement and bank reuse -cannot invalidate a live successor's VP input. CDF snapshots are similarly -copied into the consumer's CDF input. No hardware-written tail is sampled by -these copies. Busy banks publish fully prepared metadata with release/acquire; -allocation failures and PP error/reset retirement release the saved CAPTURE's -bank, independent of the current builder. - -This subset needs no extra VP token: firmware already waits for VP(N) DONE -before feeding VP(N+1). AV1 descriptors retain their ordinary completion-only -flags. The Unicorn run boots the v5-t2 CM3 reset vector, routes AV1 and PP IRQs, -and checks production AV1 program words/descriptors/configuration. It shows -VP(1) issued during PP(0), PP serialization and blocked completion mailboxes. -Host tests poison in-flight CAPTURE tails to ensure snapshot consumers never -read them. Pixels, physical DMA visibility and throughput remain hardware -qualification work for the lead. Full CDF/MV dependency release at VP DONE is -still a separate evidence requirement. +AV1 analysis +~~~~~~~~~~~~ + +AV1's current ``avd_av1_set_prob()`` reads the primary reference's CDF tail on +the CPU, and seeds the current frame's output CDF because hardware leaves it +untouched when DISABLE_FRAME_END_UPDATE_CDF is set. ``avd_av1_set_seg()`` can +also copy the primary reference's segmentation map on the CPU. These are +separate dependencies; removing only the CDF wait is insufficient. + +=================================== ========================================= +Case CDF source dependency +=================================== ========================================= +Intra/error-resilient/PRIMARY_NONE Defaults; no primary-reference CDF read +Primary ref with frame-end disabled Its initialized CDF, available at prepare +Primary ref with frame-end enabled Its hardware-written CDF tail +=================================== ========================================= + +``disable_frame_end_update_cdf`` on the *consumer* does not remove its primary +reference input dependency. It controls that consumer's future CDF output. +Likewise, a default CDF does not necessarily remove the segmentation-map input +selected by primary_ref_frame. + +The available implementation does not establish whether the CDF tail *and* +segmentation map are fully visible at VP DONE, or which CPU DMA synchronization +is required for an in-flight CAPTURE allocation while PP writes another part. +This differs from VP9's separate coherent count buffer. AV1 therefore retains +its one-picture path. Enabling it needs evidence for both tail writes and a +safe visibility/synchronization rule, then two scratch banks and a dependency +on the actual primary reference (which need not be the preceding picture). +No AV1 or VP9 pixel correctness or throughput claim follows from these host +and emulator tests; device qualification remains necessary. diff --git a/drivers/media/platform/apple/avd/tests/av1-test.py b/drivers/media/platform/apple/avd/tests/av1-test.py deleted file mode 100644 index a02070c85ffe7..0000000000000 --- a/drivers/media/platform/apple/avd/tests/av1-test.py +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Production AV1 bank/dependency/emitters, with explicit host DMA boundaries. - -Run in a controller build slot: av1-test.py OUT [AVD_DIR [KERNEL_ROOT]]. -AVD_AV1_BASELINE_REF optionally compares complete legacy programs/CDF/map bytes. -No decoder arithmetic or DMA visibility is modeled by these host fixtures. -""" -import hashlib -import importlib.util -import os -from pathlib import Path -import re -import subprocess -import sys - -from host_source import function - -HERE = Path(__file__).resolve().parent -AVD = Path(sys.argv[2]).resolve() if len(sys.argv) > 2 else HERE.parent -KERNEL = Path(sys.argv[3]).resolve() if len(sys.argv) > 3 else AVD.parents[4] -OUT = Path(sys.argv[1]).resolve() -OUT.mkdir(parents=True, exist_ok=True) -spec = importlib.util.spec_from_file_location('program_fixture', HERE / 'program-test.py') -fixture = importlib.util.module_from_spec(spec) -spec.loader.exec_module(fixture) -pre = fixture.PRELUDE -pre = pre.replace('typedef uint8_t u8;', 'typedef uint8_t u8; typedef uint16_t u16; typedef int8_t s8; typedef int16_t s16; typedef int64_t s64;') -pre = pre.replace('#define AVD_MAX_INST 512', '#define AVD_MAX_INST 512\n#define AVD_CODEC_HEVC 0') -pre = pre.replace('struct vb2_buffer { struct { size_t length; } planes[1]; dma_addr_t dma; };', - 'struct vb2_buffer { struct { size_t length; } planes[1]; dma_addr_t dma; void *cpu; size_t payload; u64 timestamp; struct {void *req;} req_obj; };') -header = (AVD / 'avd.h').read_text() -a = header.index('struct avd_av1_decoded_buffer_info {') -b = header.index('\n};', a) + 3 -pre = pre.replace('struct avd_decoded_buffer {', header[a:b] + '\nstruct avd_decoded_buffer {') -pre = pre.replace('struct avd_comp comp; unsigned int bank;', 'struct avd_comp comp; unsigned int bank; struct avd_av1_decoded_buffer_info av1;') -pre = pre.replace('struct vb2_v4l2_buffer *dst; } bufs;', 'struct vb2_v4l2_buffer *src,*dst; } bufs;') -pre = pre.replace('unsigned int fifo_idx, pp_total;', 'unsigned int fifo_idx, pp_total, max_frames; int ctrl_hdl;') -pre = pre.replace('dma_addr_t comp_out, y_out, uv_out;', 'dma_addr_t comp_out, y_out, uv_out, coded_in;') -pre = pre.replace('struct { u32 bytesperline; } plane_fmt[1];', 'u32 width,height; struct { u32 bytesperline; } plane_fmt[1];') -pre = pre.replace('static struct avd_decoded_buffer reference;', 'static struct avd_decoded_buffer reference; static struct avd_decoded_buffer *refs[3];') -pre = pre.replace('(void)c; (void)b; (void)t; return &reference;', '(void)c;for(unsigned int i=0;i<3;i++)if(refs[i]&&refs[i]->base.vb.vb2_buf.timestamp==t)return refs[i];return (void *)b;') -pre = pre.replace('static int avd_buf_alloc(struct avd_dev *d, struct avd_buf *b, size_t n)', - 'static struct avd_ctx *allocating_ctx;\nstatic int fail_alloc_after;\nstatic bool avd_av1_ready(struct avd_ctx *);\nstatic int avd_buf_alloc(struct avd_dev *d, struct avd_buf *b, size_t n)') -pre = pre.replace('{ (void)d; free(b->cpu); b->cpu = calloc(1,n);', - '{ if(allocating_ctx){assert(!avd_av1_ready(allocating_ctx));if(fail_alloc_after&&!--fail_alloc_after)return -ENOMEM;} (void)d; free(b->cpu); b->cpu = calloc(1,n);') -pre += r''' -#define AVD_ALIGN 256 -#define S16_MIN INT16_MIN -#define S16_MAX INT16_MAX -#define clamp_val(x,l,h) clamp(x,l,h) -#define U64_MAX UINT64_MAX -#define BIT_ULL(n) (1ULL<<(n)) -#define ALIGN(x,a) (((x)+(a)-1)&~((__typeof__(x))(a)-1)) -#define ALIGN_DOWN(x,a) ((x)&~((__typeof__(x))(a)-1)) -#define max_t(t,a,b) ((t)(a)>(t)(b)?(t)(a):(t)(b)) -#define min(a,b) ((a)<(b)?(a):(b)) -#define max(a,b) ((a)>(b)?(a):(b)) -#define clamp(x,l,h) ((x)<(l)?(l):(x)>(h)?(h):(x)) -#define WARN_ON(v) (v) -#define GFP_KERNEL 0 -#define smp_load_acquire(p) (*(p)) -#define smp_store_release(p,v) (*(p)=(v)) -#define READ_ONCE(x) (x) -#define WRITE_ONCE(x,v) ((x)=(v)) -#define static_assert _Static_assert -#define VB2_BUF_STATE_DONE 0 -#define VB2_BUF_STATE_ERROR 1 -enum vb2_buffer_state { STATE_DUMMY }; -#define kzalloc(n,f) calloc(1,n) -#define kfree free -static bool vp_feature=true; -static bool avd_qfw_has_vp_done(const struct avd_dev *d){return d->queued&&vp_feature;} -static void *vb2_plane_vaddr(struct vb2_buffer *b,int p){(void)p;return b->cpu;} -static struct vb2_v4l2_buffer *current_src,*current_dst; -static struct vb2_v4l2_buffer *v4l2_m2m_next_src_buf(void *m){(void)m;return current_src;} -struct v4l2_ctrl {unsigned int id; union {void *p;} p_cur;}; -struct v4l2_ctrl_ref {struct v4l2_ctrl_ref *next; struct v4l2_ctrl *ctrl; bool p_req_valid; union {void *p;} p_req;}; -struct v4l2_ctrl_handler {struct v4l2_ctrl_ref *ctrl_refs;}; -#define list_for_each_entry(ref,head,node) for((ref)=*(head);(ref);(ref)=(ref)->next) -static struct v4l2_ctrl_av1_frame params,request_params; -static struct v4l2_ctrl_av1_sequence seq; -static struct v4l2_ctrl_av1_tile_group_entry tile; -static struct v4l2_ctrl_av1_film_grain grain; -static struct v4l2_ctrl controls[4]; -static struct v4l2_ctrl_ref request_ref; -static struct v4l2_ctrl_handler request_hdl; -static unsigned int request_gets,request_puts; -static struct v4l2_ctrl_handler *v4l2_ctrl_request_hdl_find(void *req,void *h){(void)h;assert(req==&request_hdl);request_gets++;return &request_hdl;} -static void v4l2_ctrl_request_hdl_put(struct v4l2_ctrl_handler *h){assert(h==&request_hdl);request_puts++;} -static struct v4l2_ctrl *v4l2_ctrl_find(void *h,u32 id){ - (void)h;controls[0].p_cur.p=&seq;controls[1].p_cur.p=¶ms;controls[2].p_cur.p=&tile;controls[3].p_cur.p=&grain; - if(id==V4L2_CID_STATELESS_AV1_SEQUENCE)return &controls[0]; - if(id==V4L2_CID_STATELESS_AV1_FRAME)return &controls[1]; - if(id==V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY)return &controls[2]; - return &controls[3]; -} -static void avd_run_preamble(struct avd_ctx *ctx,struct avd_run *run){ - (void)ctx;memset(run,0,sizeof(*run));run->bufs.src=current_src;run->bufs.dst=current_dst; - run->coded_in=current_src->vb2_buf.dma;run->comp_out=current_dst->vb2_buf.dma; - run->y_out=run->comp_out+0x100000;run->uv_out=run->comp_out+0x300000; -} -static void avd_run_postamble(struct avd_ctx *ctx,struct avd_run *run){(void)ctx;(void)run;} -static int avd_init_job(struct avd_ctx *ctx,int codec,size_t segments){ - free(ctx->job.segments);ctx->job.codec=codec;ctx->job.num=0; - ctx->job.segments=calloc(segments,sizeof(*ctx->job.segments));return ctx->job.segments?0:-ENOMEM; -} -static int avd_submit_job(struct avd_ctx *ctx); -''' - - -def without_includes(s): - return re.sub(r'^#include[^\n]*\n', '', s, flags=re.M) - - -av1 = (AVD / 'avd-av1.c').read_text() -common = pre + without_includes((AVD / 'avd-av1-entropymode.h').read_text()) -common += without_includes((AVD / 'avd-av1-entropymode.c').read_text()) -common += without_includes((AVD / 'avd-inst.h').read_text()) -qfw = (AVD / 'avd-backend-queue.c').read_text() -hw = (AVD / 'avd-hw.c').read_text() -post = function((AVD / 'avd-backend-open.c').read_text(), 'avd_open_build_program') -post += qfw[qfw.index('#define AVDQ_MAGIC'):qfw.index('/* Arguments of dev_dbg()')] -post += hw[hw.index('#define AVD_V3_'):hw.index('int avd_boot(')] -post += '#define w32(r,v) writel((v),avd->ctrl+(r))\n#define m32(r,v) w32(r,(v)|readl(avd->ctrl+(r)))\n' -post += function(hw, 't8112_configure_stream') + function(hw, 't8122_configure_stream') -post += function(qfw, 'avd_qfw_stream') + function(qfw, 'avd_qfw_submit') -post += '\nstatic int avd_submit_job(struct avd_ctx *ctx){return avd_open_build_program(ctx);}\n' - - -def codec(s): - return without_includes(s[s.index('#define V4L2_AV1_SEG_LVL_ALT_LF_Y_H'):s.index('static enum avd_image_fmt avd_av1_get_image_fmt')]) + function(s, 'avd_av1_pp_kicks') - - -MAIN = r''' -static void write_program(const char *dir,const char *name,struct avd_frame *f){ - char path[4096];snprintf(path,sizeof(path),"%s/%s.bin",dir,name);FILE *out=fopen(path,"wb");assert(out); - assert(fwrite(f->prog->cpu,4,f->prog_words,out)==f->prog_words);fclose(out); - struct avd_dev *dev=f->ctx->dev;struct avd_qfw *q=dev->qfw; - q->wp=0;memset(q->sram,0,0x10000);f->tag=0x10|f->idx;assert(!avd_qfw_submit(dev,f)); - snprintf(path,sizeof(path),"%s/%s-desc.bin",dir,name);out=fopen(path,"wb");assert(out); - assert(fwrite(q->sram+0x8000+f->idx*64,1,64,out)==64);fclose(out); - snprintf(path,sizeof(path),"%s/%s-mmio.bin",dir,name);out=fopen(path,"wb");assert(out); - assert(fwrite(dev->ctrl,1,0x10000,out)==0x10000);fclose(out); -} -static void queue_request(struct vb2_v4l2_buffer *src){ - request_params=params;controls[1].id=V4L2_CID_STATELESS_AV1_FRAME; - request_ref=(struct v4l2_ctrl_ref){.ctrl=&controls[1],.p_req_valid=true,.p_req.p=&request_params}; - request_hdl.ctrl_refs=&request_ref;src->vb2_buf.req_obj.req=&request_hdl; -} -static void flag_table(struct avd_av1_bank *b){ - /* Actual producer timestamp is slot 5, not the immediately prior slot 0. */ - struct v4l2_ctrl_av1_frame f={.frame_type=V4L2_AV1_INTER_FRAME,.primary_ref_frame=2, - .frame_width_minus_1=1919,.frame_height_minus_1=1087}; - for(unsigned int i=0;i<7;i++)f.ref_frame_idx[i]=i; - f.ref_frame_idx[2]=5;f.reference_frame_ts[5]=b->timestamp; - for(unsigned int defaults=0;defaults<4;defaults++) - for(unsigned int output_disabled=0;output_disabled<2;output_disabled++) - for(unsigned int input_disabled=0;input_disabled<2;input_disabled++) - for(unsigned int producer_seg=0;producer_seg<4;producer_seg++) - for(unsigned int consumer_seg=0;consumer_seg<4;consumer_seg++) - for(unsigned int refresh=0;refresh<256;refresh++){ - f.primary_ref_frame=defaults==1?7:2; - f.frame_type=defaults==2?V4L2_AV1_INTRA_ONLY_FRAME:V4L2_AV1_INTER_FRAME; - f.flags=(defaults==3?V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE:0)| - (output_disabled?V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF:0); - f.refresh_frame_flags=refresh; - f.segmentation.flags=(consumer_seg&1?V4L2_AV1_SEGMENTATION_FLAG_ENABLED:0)| - (consumer_seg&2?V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP:0); - b->flags=input_disabled?V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF:0; - b->seg_flags=(producer_seg&1?V4L2_AV1_SEGMENTATION_FLAG_ENABLED:0)| - (producer_seg&2?V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP:0); - bool expect=defaults!=1&&((!defaults&&!input_disabled)||((consumer_seg&1)&&producer_seg==3)); - assert(avd_av1_depends(&f,b)==expect); - } - for(unsigned int primary=0;primary<7;primary++){ - f.primary_ref_frame=primary;f.flags=0;f.frame_type=V4L2_AV1_INTER_FRAME;f.segmentation.flags=0; - b->flags=0;f.ref_frame_idx[primary]=5;assert(avd_av1_depends(&f,b)); - f.reference_frame_ts[5]++;assert(!avd_av1_depends(&f,b));f.reference_frame_ts[5]--; - } - f.primary_ref_frame=2;f.flags=V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS;f.reference_frame_ts[5]=b->timestamp; - assert(avd_av1_depends(&f,b));f.primary_ref_frame=7;assert(avd_av1_depends(&f,b)); - f.reference_frame_ts[5]++;assert(!avd_av1_depends(&f,b)); - puts("PASS: AV1 flag table (primary, consumer/producer CDF flags, segmentation enabled/update_map, every refresh mask), actual slot and MV dependencies"); -} -int main(int argc,char **argv){ - assert(argc==2); - struct avd_variant variant={.revision=4,.quirks=0,.fifo_slots=15,.vp_slots={4,4,1,3}}; - struct avd_dev dev={.variant=&variant,.queued=true,.capabilities=15}; - struct avd_qfw queue={.sram=calloc(1,0x10000),.aperture=0x1094000,.ring_words=0x1800,.fifo_slots=15,.vp_done=true}; - dev.qfw=&queue;dev.ctrl=calloc(1,0x10000);dev.mbox=calloc(1,0x100);variant.configure_stream=t8122_configure_stream; - writel(0x20001,dev.mbox+0x50); - static const struct avd_coded_fmt_ops ops={.pp_kicks=avd_av1_pp_kicks}; - static const struct fmt_desc fmt={.ops=&ops}; - struct avd_ctx ctx={.dev=&dev,.coded_fmt_desc=&fmt,.pp_total=1}; - struct avd_buf program={};struct avd_frame frame={.prog=&program,.ctx=&ctx};dev.building=&frame; - struct avd_decoded_buffer dst[3]={};struct vb2_v4l2_buffer src={};u8 coded[512]={}; - src.vb2_buf=(struct vb2_buffer){.dma=0x220000000ULL,.cpu=coded,.payload=512,.timestamp=1000};current_src=&src; - for(unsigned int i=0;i<3;i++){ - dst[i].base.vb.vb2_buf=(struct vb2_buffer){.dma=0x230000000ULL+i*0x1000000ULL,.cpu=calloc(1,0x800000),.planes[0].length=0x800000,.timestamp=1000+i};refs[i]=&dst[i]; - } - ctx.decoded_fmt.fmt.pix_mp.width=1920;ctx.decoded_fmt.fmt.pix_mp.height=1088; - ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; - assert(!avd_av1_ready(&ctx));allocating_ctx=&ctx;assert(!avd_av1_start(&ctx));allocating_ctx=NULL; - struct avd_av1_stream *p=((struct avd_av1_ctx *)ctx.priv)->stream;assert(p->nbanks==2&&ctx.max_frames==2); - assert(p->bank[0].codec.bufs.probs.addr!=p->bank[1].codec.bufs.probs.addr); - seq=(struct v4l2_ctrl_av1_sequence){.bit_depth=8,.flags=V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_X|V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_Y}; - params=(struct v4l2_ctrl_av1_frame){.frame_type=V4L2_AV1_KEY_FRAME,.primary_ref_frame=7, - .flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF,.frame_width_minus_1=1919,.frame_height_minus_1=1087,.upscaled_width=1920}; - params.tile_info.tile_cols=params.tile_info.tile_rows=1;params.tile_info.width_in_sbs_minus_1[0]=29;params.tile_info.height_in_sbs_minus_1[0]=16; - tile.tile_size=512;tile.tile_col=tile.tile_row=0; - current_dst=&dst[0].base.vb;frame.idx=0;assert(avd_av1_ready(&ctx));assert(!avd_av1_run(&ctx)); - assert(dst[0].bank==1&&p->bank[0].busy&&p->bank[0].prepared); - assert(frame.fifo_iova==p->bank[0].codec.bufs.inst.addr); - write_program(argv[1],"av1-overlap-0",&frame);flag_table(&p->bank[0]); - p->bank[0].flags=params.flags;p->bank[0].seg_flags=0; - params.frame_type=V4L2_AV1_INTER_FRAME;params.primary_ref_frame=0;params.ref_frame_idx[0]=0;params.reference_frame_ts[0]=1000; - params.segmentation.flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED; - queue_request(&src); - /* p_cur is deliberately stale and different from the pending request. */ - params.flags=0;request_params.flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; - assert(avd_av1_ready(&ctx)); - p->bank[0].flags=0;assert(!avd_av1_ready(&ctx));p->bank[0].flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; - p->bank[0].seg_flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED|V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP; - assert(!avd_av1_ready(&ctx));p->bank[0].seg_flags=0; - request_params.flags|=V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS;assert(!avd_av1_ready(&ctx));request_params.flags&=~V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS; - assert(avd_av1_ready(&ctx));params=request_params; - /* CAPTURE tails are poisoned: successor must consume preparation snapshots. */ - size_t tail=AVD_AV1_CDFS_OFFSET(0x800000,dst[0].av1.color_size); - memset((u8 *)dst[0].base.vb.vb2_buf.cpu+tail,0xee,sizeof(struct avd_av1_cdfs)); - memset(p->bank[0].codec.bufs.saved_seg.cpu,0x35,dst[0].av1.seg_size); - current_dst=&dst[1].base.vb;src.vb2_buf.timestamp=1001;frame.idx=1; - assert(!avd_av1_run(&ctx));assert(dst[1].bank==2&&p->bank[0].busy&&p->bank[1].busy); - assert(!memcmp(p->bank[0].codec.bufs.probs.cpu,p->bank[1].codec.bufs.probs.cpu,sizeof(struct avd_av1_cdfs))); - assert(!memcmp(p->bank[0].codec.bufs.saved_seg.cpu,p->bank[1].codec.bufs.prev_seg.cpu,dst[1].av1.seg_size)); - assert(p->bank[1].codec.bufs.prev_seg.addr!=p->bank[0].codec.bufs.saved_seg.addr); - write_program(argv[1],"av1-overlap-1",&frame);assert(!avd_av1_ready(&ctx)); - current_dst=&dst[2].base.vb;assert(avd_av1_run(&ctx)==-EBUSY);assert(!dst[2].bank); - avd_av1_done(&ctx,&src,&dst[0].base.vb,VB2_BUF_STATE_DONE); - assert(!p->bank[0].busy&&p->bank[1].busy&&dst[1].bank==2); - /* Producer bank reuse must not invalidate the consumer's VP map input. */ - memset(p->bank[0].codec.bufs.saved_seg.cpu,0x98,dst[0].av1.seg_size); - assert(((u8 *)p->bank[1].codec.bufs.prev_seg.cpu)[0]==0x35); - avd_av1_done(&ctx,&src,&dst[0].base.vb,VB2_BUF_STATE_DONE);assert(p->bank[1].busy); - avd_av1_done(&ctx,&src,&dst[1].base.vb,VB2_BUF_STATE_ERROR);assert(!p->bank[1].busy&&avd_av1_ready(&ctx)); - assert(request_gets==request_puts); - avd_av1_stop(&ctx);assert(!avd_av1_ready(&ctx)); - for(unsigned int mode=0;mode<2;mode++){ - dev.queued=mode;vp_feature=false;assert(!avd_av1_start(&ctx));p=((struct avd_av1_ctx *)ctx.priv)->stream; - assert(p->nbanks==1&&ctx.max_frames==1);avd_av1_stop(&ctx); - } - dev.queued=true;vp_feature=true; - for(int n=1;n<=6;n++){ - fail_alloc_after=n;allocating_ctx=&ctx;assert(avd_av1_start(&ctx)==-ENOMEM);allocating_ctx=NULL; - assert(!ctx.priv&&!avd_av1_ready(&ctx)); - } - for(unsigned int i=0;i<3;i++)free(dst[i].base.vb.vb2_buf.cpu); - free(program.cpu);free(queue.sram);free(dev.ctrl);free(dev.mbox); - puts("PASS: AV1 snapshot inheritance, poisoned CAPTURE, request lookahead, banks, PP retirement, exhaustion, failures and legacy fallback"); -} -''' - - -def build_run(name, source): - (OUT / f'{name}.c').write_text(source) - cmd = [os.environ.get('CC', 'cc'), '-std=gnu11', '-g', '-O1', '-Wall', '-Wextra', '-Werror', - '-Wno-unused-function', '-Wno-unused-variable', '-Wno-unused-parameter', '-Wno-sign-compare', '-Wno-pointer-sign', - '-fsanitize=address,undefined', '-fno-omit-frame-pointer', str(OUT / f'{name}.c'), '-o', str(OUT / name)] - subprocess.run(cmd, check=True) - subprocess.run([str(OUT / name), str(OUT)], check=True) - - -build_run('av1-host', common + codec(av1) + post + MAIN) -(OUT / 'source.sha256').write_text('\n'.join(hashlib.sha256(p.read_bytes()).hexdigest() + ' ' + str(p) for p in ( - AVD / 'avd-av1.c', AVD / 'avd-av1-entropymode.c', AVD / 'avd-backend-open.c', AVD / 'avd-backend-queue.c')) + '\n') - -base_ref = os.environ.get('AVD_AV1_BASELINE_REF') -if base_ref: - old = subprocess.check_output(['git', '-C', str(KERNEL), 'show', - base_ref + ':drivers/media/platform/apple/avd/avd-av1.c'], text=True) - legacy = r''' -int main(int argc,char **argv){ - assert(argc==2); - struct avd_variant variant={.revision=4,.fifo_slots=15}; - struct avd_dev dev={.variant=&variant,.queued=false}; - struct avd_coded_fmt_ops ops={};struct fmt_desc fmt={.ops=&ops}; - struct avd_ctx ctx={.dev=&dev,.coded_fmt_desc=&fmt}; - struct avd_buf program={};struct avd_frame f={.prog=&program,.ctx=&ctx};dev.building=&f; - struct avd_decoded_buffer dst[2]={};struct vb2_v4l2_buffer src={};u8 coded[512]={}; - current_src=&src;src.vb2_buf=(struct vb2_buffer){.dma=0x220000000ULL,.cpu=coded,.payload=512}; - for(unsigned int i=0;i<2;i++){ - dst[i].base.vb.vb2_buf=(struct vb2_buffer){.dma=0x230000000ULL+i*0x1000000ULL,.cpu=calloc(1,0x800000),.planes[0].length=0x800000,.timestamp=1000+i};refs[i]=&dst[i]; - } - ctx.inst.addr=0x205000000ULL; - ctx.decoded_fmt.fmt.pix_mp.width=1920;ctx.decoded_fmt.fmt.pix_mp.height=1088; - ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; - seq=(struct v4l2_ctrl_av1_sequence){.bit_depth=8,.flags=V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_X|V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_Y}; - params=(struct v4l2_ctrl_av1_frame){.frame_type=V4L2_AV1_KEY_FRAME,.primary_ref_frame=7, - .frame_width_minus_1=1919,.frame_height_minus_1=1087,.upscaled_width=1920}; - params.tile_info.tile_cols=params.tile_info.tile_rows=1;params.tile_info.width_in_sbs_minus_1[0]=29;params.tile_info.height_in_sbs_minus_1[0]=16;tile.tile_size=512; - assert(!avd_av1_start(&ctx)); - for(unsigned int mode=0;mode<8;mode++){ - current_dst=&dst[!!mode].base.vb; - params.frame_type=mode==0?V4L2_AV1_KEY_FRAME:mode==2?V4L2_AV1_INTRA_ONLY_FRAME:V4L2_AV1_INTER_FRAME; - params.primary_ref_frame=mode==0||mode==4?7:0; - params.ref_frame_idx[0]=0;params.reference_frame_ts[0]=1000; - params.flags=(mode==3?V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE:0)|(mode==5?V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF:0); - params.segmentation.flags=mode>=6?V4L2_AV1_SEGMENTATION_FLAG_ENABLED:0; - if(mode==7)params.segmentation.flags|=V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP; - assert(!avd_av1_run(&ctx)); - char path[4096];snprintf(path,sizeof(path),"%s/@NAME@-%u.bin",argv[1],mode);FILE *out=fopen(path,"wb");assert(out); - assert(fwrite(program.cpu,4,f.prog_words,out)==f.prog_words); - struct avd_decoded_buffer *d=&dst[!!mode];size_t off=AVD_AV1_SEG_OFFSET(0x800000,d->av1.color_size,d->av1.seg_size); - assert(fwrite((u8 *)d->base.vb.vb2_buf.cpu+off,1,d->av1.seg_size+ALIGN(AVD_CDFS_SIZE,AVD_ALIGN),out)==d->av1.seg_size+ALIGN(AVD_CDFS_SIZE,AVD_ALIGN));fclose(out); - if(mode==0){ - size_t cdf=AVD_AV1_CDFS_OFFSET(0x800000,dst[0].av1.color_size); - memset((u8 *)dst[0].base.vb.vb2_buf.cpu+cdf,0x57,AVD_CDFS_SIZE); - memset((u8 *)dst[0].base.vb.vb2_buf.cpu+cdf-dst[0].av1.seg_size,0x36,dst[0].av1.seg_size); - } - } - avd_av1_stop(&ctx);free(program.cpu); - for(unsigned int i=0;i<2;i++)free(dst[i].base.vb.vb2_buf.cpu); -} -''' - for name, body in [('legacy-before', old), ('legacy-after', av1)]: - shim = 'static bool avd_av1_ready(struct avd_ctx *ctx){(void)ctx;return false;}\n' if name == 'legacy-before' else '' - build_run(name, common + codec(body) + shim + post + legacy.replace('@NAME@', name)) - for mode in range(8): - assert (OUT / f'legacy-before-{mode}.bin').read_bytes() == (OUT / f'legacy-after-{mode}.bin').read_bytes(), mode - print('PASS: separate baseline/current executables emit byte-identical legacy AV1 programs and CDF/map tails for eight header cases') diff --git a/drivers/media/platform/apple/avd/tests/unicorn-av1-test.py b/drivers/media/platform/apple/avd/tests/unicorn-av1-test.py deleted file mode 100644 index 93f4651686776..0000000000000 --- a/drivers/media/platform/apple/avd/tests/unicorn-av1-test.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-only OR MIT -"""AV1 production programs: VP(N+1) starts after VP(N) DONE, during PP(N). - -Usage: unicorn-av1-test.py FIRMWARE_TREE v5-t2_IMAGE AV1_HOST_OUTPUT -CDF/map input snapshots are host fixtures. CM3 code, descriptors, program words, -PP commands and configure_stream writes are production. No pixels are decoded. -""" -import importlib.util -from pathlib import Path -import struct -import sys -from unicorn.arm_const import UC_ARM_REG_SP - -fw, image, programs = map(lambda p: Path(p).resolve(), sys.argv[1:4]) -spec = importlib.util.spec_from_file_location('transport', fw / 'tests/transport-test.py') -transport = importlib.util.module_from_spec(spec) -spec.loader.exec_module(transport) - - -class AV1(transport.Transport): - def __init__(self): - self.booting = False - self.expected = [] - self.events = [] - self.live = self.pp_owner = None - self.feed = self.word = 0 - super().__init__(image) - assert self.version == 5 and self.tier == 2 - self.booting = True - self.uc.reg_write(UC_ARM_REG_SP, self.u32(0)) - self.uc.emu_start(self.u32(4), 0, count=500000) - assert not self.booting - - def route(self, irq): - if irq < 14: - router = self.u32(0x50010010) & (1 << irq) - else: - off = irq - 14 - router = self.u32(0x50010014 + off // 32 * 4) & (1 << (off % 32)) - nvic = self.u32(0xe000e100 + irq // 32 * 4) & (1 << (irq % 32)) - assert router and nvic, f'IRQ {irq} masked' - self.call(f'irq{irq}') - - def write(self, uc, access, addr, size, value, user): - if self.booting and addr == 0x50010090 and value == 1: - self.booting = False - uc.emu_stop() - if addr == self.decode + self.fifo + 9 * 4: - idx, words = self.expected[self.feed] - assert value == words[self.word], (idx, self.word, value) - if not self.word: - assert self.live is None, 'successor issued before VP DONE' - self.live = idx - self.events.append(f'AV1 VP START {idx}; PP owner={self.pp_owner}') - self.word += 1 - if self.word == len(words): - self.feed += 1 - self.word = 0 - if addr == self.decode + self.pp: - idx = (value >> 4) & 15 - assert self.pp_owner is None, 'successor PP issued before predecessor PP DONE' - assert value == 0x2b00020f | idx << 4 - self.pp_owner = idx - self.events.append(f'PP START {idx}') - if addr == 0x50010060: - self.events.append(f'notification {value:#x}') - super().write(uc, access, addr, size, value, user) - - def submit_host(self, idx, start): - stem = f'av1-overlap-{idx}' - data = (programs / f'{stem}.bin').read_bytes() - words = list(struct.unpack('<' + 'I' * (len(data) // 4), data)) - desc = list(struct.unpack('<16I', (programs / f'{stem}-desc.bin').read_bytes())) - assert desc[:4] == [0x41565131, 0x10 | idx, 3, 9] - assert desc[4:6] == [0, 0] and desc[6] == len(words) - assert desc[8:16] == [1, 0, 0, 1, 0, 0, 0, 0] - assert words[0] == 0x2b000200 | idx << 4 - assert words[-1] == 0x2b000400 | idx << 4 - mmio = (programs / f'{stem}-mmio.bin').read_bytes() - for offset in (0x1d0, 0x20c, 0x248, 0x284, 0x2c0): - at = offset + idx * 4 - self.put(self.decode + at, struct.unpack_from(' Date: Sun, 1 Nov 2026 19:46:45 +1100 Subject: [PATCH 63/73] media: apple: avd: fix slice hold negotiation and validate unsupported syntax Port the conformance fixes onto the merged frame-queue rewrite. Advertise HOLD_CAPTURE_BUF before CREATE_BUFS probing and replace format features on S_FMT. Intermediate slices retain CAPTURE through the core HOLD API; the last slice still detaches both buffers without completing DMA ownership. Reject SP/SI, FMO, unsupported SPS controls and arbitrary slice order before building instructions. Check order before selecting a bank can discard a held picture. Drop rejected unsubmitted slices and the job table without releasing a submitted bank belonging to another picture. Map sparse DPB indexes to packed reference-header slots and count every VALID header, including inactive entries. Dense legacy programs retain their bytes; sparse reference operands and inactive-entry counts change intentionally. Fold the old host/queue cases into the existing program and dispatch tests, including exact baseline programs, every DPB occupancy map, CREATE_BUFS probing, held/final/error dispatch and production two-bank preparation. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-drv.c | 2 + .../platform/apple/avd/avd-h264-params.h | 44 +++ drivers/media/platform/apple/avd/avd-h264.c | 128 ++++-- drivers/media/platform/apple/avd/avd-hevc.c | 13 +- drivers/media/platform/apple/avd/avd-v4l2.c | 3 +- .../media/platform/apple/avd/tests/README.rst | 42 ++ .../platform/apple/avd/tests/dispatch-test.py | 101 ++++- .../platform/apple/avd/tests/program-test.py | 367 +++++++++++++++++- tools/asahi/avd-codec-tail-tests.py | 7 + 9 files changed, 669 insertions(+), 38 deletions(-) create mode 100644 drivers/media/platform/apple/avd/avd-h264-params.h diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index d6042c3fb1e92..57021503e6e10 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -151,6 +151,8 @@ static int avd_m2m_queue_init(void *priv, struct vb2_queue *src_vq, src_vq->dev = ctx->dev->v4l2_dev.dev; src_vq->supports_requests = true; src_vq->allow_cache_hints = true; + /* CREATE_BUFS may probe HOLD support before the first S_FMT. */ + src_vq->subsystem_flags = ctx->coded_fmt_desc->subsystem_flags; ret = vb2_queue_init(src_vq); if (ret) diff --git a/drivers/media/platform/apple/avd/avd-h264-params.h b/drivers/media/platform/apple/avd/avd-h264-params.h new file mode 100644 index 0000000000000..eaf6a60b71ff3 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-h264-params.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef AVD_H264_PARAMS_H +#define AVD_H264_PARAMS_H + +#include + +/* The reference headers contain only VALID entries, in DPB array order. */ +static inline unsigned int +avd_h264_ref_slot(const struct v4l2_ctrl_h264_decode_params *decode, + unsigned int index) +{ + unsigned int slot = 0; + + for (unsigned int i = 0; i < index; i++) + if (decode->dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_VALID) + slot++; + + return slot; +} + +static inline unsigned int +avd_h264_num_refs(const struct v4l2_ctrl_h264_decode_params *decode) +{ + return avd_h264_ref_slot(decode, V4L2_H264_NUM_DPB_ENTRIES); +} + +/* SP/SI require switching-picture state which the instruction builder lacks. */ +static inline bool +avd_h264_slice_type_supported(const struct v4l2_ctrl_h264_slice_params *sl) +{ + return sl->slice_type == V4L2_H264_SLICE_TYPE_I || + sl->slice_type == V4L2_H264_SLICE_TYPE_P || + sl->slice_type == V4L2_H264_SLICE_TYPE_B; +} + +/* Pictures are held until their final slice; ASO is not implemented. */ +static inline bool avd_h264_slice_order_supported(unsigned int held, + unsigned int last_mb, + unsigned int first_mb) +{ + return held ? first_mb > last_mb : first_mb == 0; +} + +#endif /* AVD_H264_PARAMS_H */ diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index 5ff0026ef3730..439fc2cf96bda 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -22,6 +22,7 @@ #include "avd.h" #include "avd-inst.h" +#include "avd-h264-params.h" #define H264_SCL_DIMS (0x1000000 | ((64 / 4) << 5) | (((16 / 4) << 5) - 1)) @@ -75,6 +76,7 @@ struct avd_bank { size_t alloc_slice_num; /* allocated size of the held slices */ size_t slice_bytes; + u32 last_mb; struct avd_h264_bufs { /* VP to PP instruction FIFO for pictures of this bank */ @@ -499,7 +501,8 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) push(AVD_OP_REF | AVD_OP_REF_LIST_IDX(0) | AVD_OP_REF_LOOP_IDX(i) | AVD_OP_REF_DBP_IDX( - sl->ref_pic_list0[i].index), + avd_h264_ref_slot(decode, + sl->ref_pic_list0[i].index)), "slc_6e8_cmd_ref_list_0"); if (sl->slice_type == V4L2_H264_SLICE_TYPE_B) { @@ -509,7 +512,8 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) push(AVD_OP_REF | AVD_OP_REF_LIST_IDX(1) | AVD_OP_REF_LOOP_IDX(i) | AVD_OP_REF_DBP_IDX( - sl->ref_pic_list1[i].index), + avd_h264_ref_slot(decode, + sl->ref_pic_list1[i].index)), "slc_6e8_cmd_ref_list_0"); } stream_weights(ctx, run); @@ -608,7 +612,21 @@ static int avd_h264_validate_pps(struct avd_ctx *ctx, const struct v4l2_ctrl_h264_pps *pps) { if (pps->num_slice_groups_minus1 != 0) { - dev_err(ctx->dev->dev, "pps->num_slice_groups_minus1 != 0"); + dev_err_ratelimited(ctx->dev->dev, + "H.264 slice groups (FMO) are unsupported: no slice-group map in the stateless API\n"); + return -EINVAL; + } + + return 0; +} + +static int avd_h264_validate_slice(struct avd_ctx *ctx, + const struct v4l2_ctrl_h264_slice_params *sl) +{ + if (!avd_h264_slice_type_supported(sl)) { + dev_err_ratelimited(ctx->dev->dev, + "H.264 slice type %u is unsupported (SP/SI switching pictures)\n", + sl->slice_type); return -EINVAL; } @@ -621,9 +639,13 @@ static int avd_h264_validate_sps(struct avd_ctx *ctx, if (sps->chroma_format_idc > 2) /* Only 4:0:0, 4:2:0 and 4:2:2 are supported */ return -EINVAL; - if (sps->bit_depth_luma_minus8 != sps->bit_depth_chroma_minus8) - /* Luma and chroma bit depth mismatch */ + if (sps->bit_depth_luma_minus8 != sps->bit_depth_chroma_minus8) { + dev_err_ratelimited(ctx->dev->dev, + "H.264 unequal luma/chroma bit depths (%u/%u) are unsupported\n", + sps->bit_depth_luma_minus8 + 8, + sps->bit_depth_chroma_minus8 + 8); return -EINVAL; + } if (sps->bit_depth_luma_minus8 != 0 && sps->bit_depth_luma_minus8 != 2) /* Only 8 and 10 bit have capture formats */ return -EINVAL; @@ -644,14 +666,7 @@ static void avd_h264_drop_bank_slices(struct avd_ctx *ctx, struct avd_bank *bank avd_buf_free(ctx->dev, &bank->slices[i]); bank->slice_num = 0; bank->slice_bytes = 0; -} - -/* Free the slices held for the picture being built. */ -static void avd_h264_drop_slices(struct avd_ctx *ctx) -{ - struct avd_h264_ctx *h264_ctx = ctx->priv; - - avd_h264_drop_bank_slices(ctx, h264_ctx->b); + bank->last_mb = 0; } static void avd_h264_stop(struct avd_ctx *ctx) @@ -808,8 +823,8 @@ static int avd_h264_realloc_slices(struct avd_ctx *ctx, size_t max_slices) /* * Check that every reference in the slice's ref lists points at a DPB entry * that is VALID. stream_refs() only pushes a reference header (and its RVRA - * addresses) for VALID entries and the slice command indexes those by DPB - * index, so a reference to any other entry makes the VP dereference an + * addresses) for VALID entries and the slice command indexes their packed + * header slots, so a reference to any other entry makes the VP dereference an * uninitialised reference slot: that ends in a DART translation fault, a * frame that never completes and a watchdog reset of the VP, which also * disturbs the other contexts. It happens whenever a client starts decoding @@ -916,21 +931,67 @@ static int avd_h264_pick_bank(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst) return 0; } +/* A rejected slice must never release the other picture's DMA-owned bank. */ +static void avd_h264_abort_picture(struct avd_ctx *ctx, + struct vb2_v4l2_buffer *dst) +{ + struct avd_h264_ctx *h264_ctx = ctx->priv; + struct avd_bank *bank = h264_ctx->b; + struct avd_decoded_buffer *buf = vb2_to_avd_decoded_buf(&dst->vb2_buf); + + if (!bank->busy || bank->submitted || + buf->bank != bank - h264_ctx->bank + 1) + return; + avd_h264_drop_bank_slices(ctx, bank); + bank->busy = false; + buf->bank = 0; +} + static int avd_h264_run(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; struct v4l2_h264_reflist_builder reflist_builder; struct avd_h264_run run; + size_t held = h264_ctx->b->busy && !h264_ctx->b->submitted ? + h264_ctx->b->slice_num : 0; int ret; avd_h264_run_preamble(ctx, &run); - if (run.slice_params && is_new_frame(run.slice_params)) { + if (!run.sps || !run.pps || !run.slice_params || !run.decode_params) { + ret = -EINVAL; + goto err_free_jobs; + } + + /* Recheck controls applied from this request before building instructions. */ + ret = avd_h264_validate_sps(ctx, run.sps); + if (!ret) + ret = avd_h264_validate_pps(ctx, run.pps); + if (!ret) + ret = avd_h264_validate_slice(ctx, run.slice_params); + if (ret) + goto err_free_jobs; + + /* Check before pick_bank() can discard an unfinished picture's slices. */ + if (!avd_h264_slice_order_supported(held, h264_ctx->b->last_mb, + run.slice_params->first_mb_in_slice)) { + dev_err_ratelimited(ctx->dev->dev, + "H.264 arbitrary slice order (ASO) is unsupported\n"); + ret = -EINVAL; + goto err_free_jobs; + } + + if (is_new_frame(run.slice_params)) { ret = avd_h264_pick_bank(ctx, run.base.bufs.dst); - if (ret) { - avd_run_postamble(ctx, &run.base); - return ret; - } + if (ret) + goto err_free_jobs; + } + + if (!h264_ctx->b->busy || h264_ctx->b->submitted || + vb2_to_avd_decoded_buf(&run.base.bufs.dst->vb2_buf)->bank != + h264_ctx->b - h264_ctx->bank + 1) { + ret = -EINVAL; + goto err_free_jobs; } struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); @@ -942,8 +1003,8 @@ static int avd_h264_run(struct avd_ctx *ctx) &run.slice_data_off)) { dev_dbg_ratelimited(ctx->dev->dev, "slice has no data after its header\n"); - avd_run_postamble(ctx, &run.base); - return -EINVAL; + ret = -EINVAL; + goto err_free_jobs; } if (!avd_h264_slice_fits(run.sps, h264_ctx->b->slice_num, @@ -951,8 +1012,6 @@ static int avd_h264_run(struct avd_ctx *ctx) ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].sizeimage)) { dev_err_ratelimited(ctx->dev->dev, "too many slices for one picture, dropping it\n"); - /* The unsubmitted job table points at the dropped slices. */ - avd_h264_drop_slices(ctx); ret = -EINVAL; goto err_free_jobs; } @@ -969,6 +1028,7 @@ static int avd_h264_run(struct avd_ctx *ctx) goto err_free_jobs; memcpy(h264_ctx->b->active_slice->cpu, data, payload_len); h264_ctx->b->slice_num++; + h264_ctx->b->last_mb = run.slice_params->first_mb_in_slice; h264_ctx->b->slice_bytes += PAGE_ALIGN((size_t)payload_len); h264_ctx->b->monochrome = run.sps->chroma_format_idc == 0; @@ -976,19 +1036,19 @@ static int avd_h264_run(struct avd_ctx *ctx) v4l2_h264_init_reflist_builder(&reflist_builder, run.decode_params, run.sps, run.decode_params->dpb); - run.num_valid = reflist_builder.num_valid; + /* stream_refs() emits VALID entries, including inactive references. */ + run.num_valid = avd_h264_num_refs(run.decode_params); run.cur_poc = reflist_builder.cur_pic_order_count; v4l2_h264_build_p_ref_list(&reflist_builder, h264_ctx->reflists.p); v4l2_h264_build_b_ref_lists(&reflist_builder, h264_ctx->reflists.b0, h264_ctx->reflists.b1); - avd_run_postamble(ctx, &run.base); - if (!avd_h264_refs_valid(&run)) { dev_dbg_ratelimited(ctx->dev->dev, "slice references an invalid DPB entry, dropping frame\n"); - return -EINVAL; + ret = -EINVAL; + goto err_free_jobs; } if (is_new_frame(run.slice_params)) { @@ -996,15 +1056,18 @@ static int avd_h264_run(struct avd_ctx *ctx) ret = avd_init_job(ctx, AVD_CODEC_H264, h264_ctx->b->alloc_slice_num + 1); if (ret) - return ret; + goto err_free_jobs; stream_hdr(ctx, &run); } - if (!ctx->job.segments) - return -EINVAL; + if (!ctx->job.segments) { + ret = -EINVAL; + goto err_free_jobs; + } ctx->job.num++; stream_slice(ctx, &run); + avd_run_postamble(ctx, &run.base); if (run.base.bufs.src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF) { avd_job_finish(ctx, VB2_BUF_STATE_DONE); @@ -1016,6 +1079,7 @@ static int avd_h264_run(struct avd_ctx *ctx) return avd_submit_job(ctx); err_free_jobs: + avd_h264_abort_picture(ctx, run.base.bufs.dst); kvfree(ctx->job.segments); ctx->job.segments = NULL; avd_run_postamble(ctx, &run.base); @@ -1092,6 +1156,8 @@ static int avd_h264_try_ctrl(struct avd_ctx *ctx, struct v4l2_ctrl *ctrl) return avd_h264_validate_sps(ctx, ctrl->p_new.p_h264_sps); if (ctrl->id == V4L2_CID_STATELESS_H264_PPS) return avd_h264_validate_pps(ctx, ctrl->p_new.p_h264_pps); + if (ctrl->id == V4L2_CID_STATELESS_H264_SLICE_PARAMS) + return avd_h264_validate_slice(ctx, ctrl->p_new.p_h264_slice_params); return 0; } diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index b2ce0f3711390..53652a01c0efd 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -1055,9 +1055,13 @@ static int avd_hevc_validate_sps(struct avd_ctx *ctx, if (sps->pic_width_in_luma_samples > ctx->coded_fmt.fmt.pix_mp.width || sps->pic_height_in_luma_samples > ctx->coded_fmt.fmt.pix_mp.height) return -EINVAL; - if (sps->bit_depth_luma_minus8 != sps->bit_depth_chroma_minus8) - /* Luma and chroma bit depth mismatch: the firmware faults (H0 error) */ + if (sps->bit_depth_luma_minus8 != sps->bit_depth_chroma_minus8) { + dev_err_ratelimited(ctx->dev->dev, + "HEVC unequal luma/chroma bit depths (%u/%u) are unsupported\n", + sps->bit_depth_luma_minus8 + 8, + sps->bit_depth_chroma_minus8 + 8); return -EINVAL; + } if (sps->bit_depth_luma_minus8 != 0 && sps->bit_depth_luma_minus8 != 2) /* Only 8 and 10 bit have capture formats */ return -EINVAL; @@ -1455,6 +1459,11 @@ static int avd_hevc_run(struct avd_ctx *ctx) if (ret) return ret; + /* Also validate controls applied from a queued request before submission. */ + ret = avd_hevc_validate_sps(ctx, run.sps); + if (ret) + goto complete; + avd_hevc_normalize_slices(&run, vb2_get_plane_payload(&run.base.bufs.src->vb2_buf, 0)); dst = vb2_to_avd_decoded_buf(&run.base.bufs.dst->vb2_buf); diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index 46f856eb1d314..d59d042c7a058 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -250,6 +250,7 @@ static const struct avd_ctrl_desc avd_h264_ctrl_descs[] = { }, { .cfg.id = V4L2_CID_STATELESS_H264_SLICE_PARAMS, + .cfg.ops = &avd_ctrl_ops, }, { .cfg.id = V4L2_CID_STATELESS_H264_DECODE_MODE, @@ -687,7 +688,7 @@ static int avd_s_output_fmt(struct file *file, void *priv, cap_fmt->fmt.pix_mp.quantization = f->fmt.pix_mp.quantization; /* Enable format specific queue features */ - vq->subsystem_flags |= desc->subsystem_flags; + vq->subsystem_flags = desc->subsystem_flags; return 0; } diff --git a/drivers/media/platform/apple/avd/tests/README.rst b/drivers/media/platform/apple/avd/tests/README.rst index d2b10320f2b75..62b040f82a4da 100644 --- a/drivers/media/platform/apple/avd/tests/README.rst +++ b/drivers/media/platform/apple/avd/tests/README.rst @@ -231,3 +231,45 @@ safe visibility/synchronization rule, then two scratch banks and a dependency on the actual primary reference (which need not be the preceding picture). No AV1 or VP9 pixel correctness or throughput claim follows from these host and emulator tests; device qualification remains necessary. + +H.264 conformance fixes +---------------------- + +The H.264 cases live in ``program-test.py`` and ``dispatch-test.py``. The +program test also compiles the emitters from the pinned merged rewrite, +``757fc07a6930ee12b04fe034138dc9880dc4ff3f``, with the same synthetic controls +and addresses. It retains ``before.json``, ``after.json`` and generated C. +Set ``AVD_H264_BASELINE_ROOT`` to an exported baseline driver directory when +running outside a Git checkout. ``AVD_H264_BASELINE_REF`` can select another +Git revision explicitly. + +The complete-program matrix covers zero through sixteen dense DPB entries, +a sparse DPB, I/P/B slices, CAVLC/CABAC, explicit/implicit weights, +first/continuation slices and both address encodings. Dense programs remain +byte-identical. Sparse programs may change only the low four-bit packed DPB +slot operand of reference-list commands. All 65536 DPB occupancy maps are +checked, including VALID entries without ACTIVE. The reference-header count +now counts all VALID entries, matching the headers actually emitted; an +inactive VALID entry can therefore change the count field in each header. + +The production prepare function is exercised with two banks: picture 0 stays +submitted while picture 1 holds slices, rejects syntax or malformed data, +and is restarted. Rejection drops only picture 1's unsubmitted slices and +job table. The saved CAPTURE bank selects PP retirement even when the other +bank is the current builder. SP/SI switching slices, FMO, non-increasing +macroblock positions and unequal luma/chroma bit depths are rejected. A repeated first slice before the previous picture +finishes is rejected before bank selection can discard its held slices. Both H.264 and HEVC applied SPS validation is tested. + +The dispatch test compiles the production queue initializer, probes +CREATE_BUFS capabilities before S_FMT, and checks format switches remove +stale HOLD support. Pictures with 1, 3, 4, 9, 10, 12 and 20 slices each reach +the transport exactly once. Intermediate slices use the core HOLD helper: +only OUTPUT completes, CAPTURE and its bank remain owned by the builder. +The final slice uses ``v4l2_m2m_job_finish_detach()``: both buffers leave the +scheduler atomically, neither completes early, and the bank remains owned +until tagged PP retirement. Advertising HOLD does not prevent that final +handoff. The legacy transport follows the same slice collection path with +one bank; the queued transport permits two banks. + +These are host control/program/ownership regressions. The lead's conformance +run must still establish decoded pixels and absence of hardware timeouts. diff --git a/drivers/media/platform/apple/avd/tests/dispatch-test.py b/drivers/media/platform/apple/avd/tests/dispatch-test.py index 055f0d544bcae..4a46a45e14100 100755 --- a/drivers/media/platform/apple/avd/tests/dispatch-test.py +++ b/drivers/media/platform/apple/avd/tests/dispatch-test.py @@ -209,6 +209,15 @@ assert(!running&&!src[5].done&&!dst[5].done);avd_retire(&d); assert(src[5].done&&dst[5].done); c[0].broken=false;prepare_result=0; + /* A rejected intermediate slice retains CAPTURE until its final/error slice. */ + struct vb2_v4l2_buffer rejected={.flags=V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF}, + rejected_dst={},rejected_last={}; + prepare_result=-EINVAL;m[0].src=&rejected;m[0].dst=&rejected_dst;submit(&c[0]); + assert(running==&c[0]&&!rejected.done&&!rejected_dst.done);avd_retire(&d); + assert(!running&&rejected.done&&!rejected_dst.done&&rejected_dst.is_held&&!d.inflight); + m[0].src=&rejected_last;submit(&c[0]); + assert(!running&&rejected_last.done&&rejected_dst.done&&!d.inflight); + c[0].broken=false;prepare_result=0; /* PM failure and stopped-stream dispatch finish the current job once. */ pm_error=-EIO;m[0].src=&src[6];m[0].dst=&dst[6];submit(&c[0]);assert(!running&&!d.inflight); pm_error=0;c[0].draining=true;m[0].src=&src[7];m[0].dst=&dst[7];submit(&c[0]); @@ -227,6 +236,23 @@ assert(d.inflight==2&&!running&&!src[9].done&&!dst[9].done&&!src[10].done&&!dst[10].done); complete(&d,&c[0],0);assert(src[9].done&&dst[9].done&&!src[10].done&&!dst[10].done); complete(&d,&c[0],0);assert(src[10].done&&dst[10].done&&!d.inflight&&!pm_refs&&!raw_finishes); + /* Each probed multislice picture submits exactly once after its last slice. */ + const unsigned int pictures[] = {1,3,4,9,10,12,20}; + for(unsigned int p=0;p +#include +#include +#include +#define VB2_MMAP 1 +#define VB2_DMABUF 2 +#define VB2_MAX_FRAME 32 +#define VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF 1 +#define max(a, b) ((a) > (b) ? (a) : (b)) +struct vb2_queue { + unsigned int type, io_modes, dma_attrs, buf_struct_size, timestamp_flags; + unsigned int subsystem_flags, max_num_buffers; + void *drv_priv, *lock, *dev; + const void *ops, *mem_ops; + bool supports_requests, allow_cache_hints, bidirectional; +}; +struct avd_dev { int vdev_lock; void *dev; struct { void *dev; } v4l2_dev; }; +struct avd_coded_fmt_desc { unsigned int subsystem_flags; }; +struct avd_ctx { struct avd_dev *dev; const struct avd_coded_fmt_desc *coded_fmt_desc; }; +struct v4l2_m2m_buffer { int unused; }; +struct avd_decoded_buffer { int unused; }; +static const int avd_queue_ops, vb2_dma_contig_memops; +static unsigned int flags_at_init; +static int vb2_queue_init(struct vb2_queue *q) +{ + if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) + flags_at_init = q->subsystem_flags; + return 0; +} + +''' +QUEUE_MAIN = r''' + +int main(void) +{ + const struct avd_coded_fmt_desc h264 = { 1 }, vp9 = { 0 }; + struct avd_dev dev = {0}; + struct avd_ctx ctx = { .dev = &dev, .coded_fmt_desc = &h264 }; + struct vb2_queue src = {0}, dst = {0}; + if (avd_m2m_queue_init(&ctx, &src, &dst)) return 1; + /* CREATE_BUFS count=0 returns these capabilities. FFmpeg caches them. */ + unsigned int cached_hold = src.subsystem_flags & 1; + select_features(&src, &h264); + assert(flags_at_init == 1 && cached_hold == 1); + select_features(&src, &vp9); + assert(src.subsystem_flags == 0); + select_features(&src, &h264); + assert(src.subsystem_flags == 1); + assert(flags_at_init == 1 && cached_hold == 1 && src.subsystem_flags == 1); + puts("PASS: CREATE_BUFS before S_FMT exposes HOLD; format switches clear stale flags"); + return 0; +} + +''' + +def queue_capabilities(out): + source = (ROOT / 'avd-v4l2.c').read_text() + import re + assignment = re.search(r'vq->subsystem_flags\s*\|?=\s*desc->subsystem_flags;', source)[0] + code = QUEUE_PRELUDE + function((ROOT / 'avd-drv.c').read_text(), 'avd_m2m_queue_init') + code += '\nstatic void select_features(struct vb2_queue *vq, const struct avd_coded_fmt_desc *desc) {\n' + code += assignment + '\n}\n' + QUEUE_MAIN + dest = out / 'queue-capabilities.c' + dest.write_text(code) + exe = dest.with_suffix('') + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-Wall', '-Wextra', '-Werror', + '-fsanitize=address,undefined', '-g', str(dest), '-o', str(exe)], check=True) + subprocess.run([str(exe)], check=True) + + def main(): out = Path(sys.argv[1]).resolve() out.mkdir(parents=True, exist_ok=True) - names = ['avd-queue.c', 'avd-queue-internal.h', 'avd-queue.h', 'avd-v4l2.c'] + queue_capabilities(out) + names = ['avd-drv.c', 'avd-queue.c', 'avd-queue-internal.h', 'avd-queue.h', 'avd-v4l2.c'] (out / 'core.sha256').write_text(hashlib.sha256(CORE.read_bytes()).hexdigest()+' v4l2-mem2mem.c\n') (out / 'inputs.sha256').write_text(''.join(hashlib.sha256((ROOT / n).read_bytes()).hexdigest()+' '+n+'\n' for n in names)) qh = (ROOT / 'avd-queue.h').read_text() diff --git a/drivers/media/platform/apple/avd/tests/program-test.py b/drivers/media/platform/apple/avd/tests/program-test.py index ad5d2ec009d95..6ee23e6370a10 100755 --- a/drivers/media/platform/apple/avd/tests/program-test.py +++ b/drivers/media/platform/apple/avd/tests/program-test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-only OR MIT +# SPDX-License-Identifier: GPL-2.0-only """Compile the production H.264 emitters and open-program builder on a host. Only allocation, controls, and buffer addresses are fixtures. No copied hardware @@ -8,6 +8,7 @@ pixel decoder. Run in a build slot: program-test.py OUTPUT_DIRECTORY. """ import os +import json import hashlib from pathlib import Path import re @@ -336,11 +337,370 @@ ''' + +def h264_prelude(): + # Keep the shared MMIO/allocation boundary stable for the VP9 suite. + pre = PRELUDE.replace('static struct avd_decoded_buffer reference;', + 'static struct avd_decoded_buffer references[16];\n#define reference references[0]') + pre = pre.replace('return &reference;', 'return &references[t % 16];') + return '#include \n' + pre + '#define dev_err(d,...) ((void)(d))\n' + + +RUN_FIXTURE = r''' +#define dev_dbg_ratelimited(d,...) ((void)(d)) +#define PAGE_SIZE 4096 +#define PAGE_ALIGN(n) (((n)+4095)&~4095UL) +struct v4l2_h264_reflist_builder { unsigned int num_valid; s32 cur_pic_order_count; }; +static void v4l2_h264_init_reflist_builder(struct v4l2_h264_reflist_builder *b, + const struct v4l2_ctrl_h264_decode_params *d,const struct v4l2_ctrl_h264_sps *s, + const struct v4l2_h264_dpb_entry *dpb) { + (void)d;(void)s;b->num_valid=0;b->cur_pic_order_count=40; + for(unsigned int i=0;i<16;i++) if(dpb[i].flags&V4L2_H264_DPB_ENTRY_FLAG_ACTIVE)b->num_valid++; +} +static void v4l2_h264_build_p_ref_list(void *b,void *r) { (void)b;(void)r; } +static void v4l2_h264_build_b_ref_lists(void *b,void *r,void *s) { (void)b;(void)r;(void)s; } +static struct avd_h264_run fixture_run; +static unsigned int postambles,held_finishes,submissions; +static bool valid_data=true; +static struct vb2_v4l2_buffer *v4l2_m2m_next_src_buf(void *m) { (void)m;return fixture_run.base.bufs.src; } +static size_t vb2_get_plane_payload(struct vb2_buffer *b,int p) { (void)b;(void)p;return valid_data?16:2; } +static void *vb2_plane_vaddr(struct vb2_buffer *b,int p) { (void)b;(void)p;static u8 data[16];return data; } +static void avd_h264_run_preamble(struct avd_ctx *ctx,struct avd_h264_run *r) { (void)ctx;*r=fixture_run; } +static void avd_run_postamble(struct avd_ctx *ctx,struct avd_run *r) { (void)ctx;(void)r;postambles++; } +static bool avd_h264_slice_fits(const struct v4l2_ctrl_h264_sps *s,size_t n,size_t b,u32 l,size_t z) +{ (void)s;(void)b;(void)l;(void)z;return n<20; } +static size_t avd_h264_max_slices(const struct v4l2_ctrl_h264_sps *s) { (void)s;return 20; } +static int avd_h264_realloc_slices(struct avd_ctx *c,size_t n) { (void)c;(void)n;abort(); } +static int avd_init_job(struct avd_ctx *c,unsigned int codec,size_t n) { + free(c->job.segments);c->job.codec=codec;c->job.num=0; + c->job.segments=calloc(n,sizeof(*c->job.segments));return c->job.segments?0:-ENOMEM; +} +static void avd_job_finish(struct avd_ctx *c,enum vb2_buffer_state s) { (void)c;assert(s==VB2_BUF_STATE_DONE);held_finishes++; } +static int avd_submit_job(struct avd_ctx *c) { submissions++;return avd_open_build_program(c); } +''' + +RUN_MAIN = r''' +int main(void) { + struct avd_variant variant={.revision=4,.quirks=AVD_QUIRK_LSR,.fifo_slots=15}; + struct avd_buf programs[2]={}; + struct avd_frame frames[2]={{.idx=0,.prog=&programs[0]},{.idx=1,.prog=&programs[1]}}; + struct avd_dev dev={.variant=&variant,.queued=true,.building=&frames[0]}; + static const struct avd_coded_fmt_ops ops={0};static const struct fmt_desc fmt={.ops=&ops}; + struct avd_h264_ctx hc={.nbanks=2};hc.b=&hc.bank[0]; + struct avd_ctx ctx={.dev=&dev,.priv=&hc,.coded_fmt_desc=&fmt}; + struct avd_decoded_buffer dst[2]={};struct vb2_v4l2_buffer src={0}; + struct v4l2_ctrl_h264_sps sps={.chroma_format_idc=1,.pic_width_in_mbs_minus1=119, + .pic_height_in_map_units_minus1=67,.flags=V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY}; + struct v4l2_ctrl_h264_pps pps={0};struct v4l2_ctrl_h264_slice_params sl={.slice_type=V4L2_H264_SLICE_TYPE_I,.header_bit_size=32}; + struct v4l2_ctrl_h264_decode_params decode={0};struct v4l2_ctrl_h264_pred_weights weights={0}; + fixture_run=(struct avd_h264_run){.sps=&sps,.pps=&pps,.slice_params=&sl,.decode_params=&decode, + .pred_weights=&weights,.base={.bufs={.src=&src,.dst=&dst[0].base.vb}, + .y_out=0x220000000,.uv_out=0x220100000,.comp_out=0x220200000},.addresses.mv_color=0x220300000}; + ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; + reference.base.vb.vb2_buf.dma=0x240000000; + for(unsigned int b=0;b<2;b++) { + assert(!avd_h264_alloc_bufs(&ctx,&hc.bank[b])); + hc.bank[b].alloc_slice_num=20;hc.bank[b].slices=calloc(20,sizeof(struct avd_buf)); + } + /* Applied SPS controls reject unequal depths for both codecs. */ + struct v4l2_ctrl_hevc_sps hevc={.chroma_format_idc=1, + .pic_width_in_luma_samples=1920,.pic_height_in_luma_samples=1088}; + ctx.coded_fmt.fmt.pix_mp.width=1920;ctx.coded_fmt.fmt.pix_mp.height=1088; + for(unsigned int luma=0;luma<5;luma++) for(unsigned int chroma=0;chroma<5;chroma++) { + sps.bit_depth_luma_minus8=hevc.bit_depth_luma_minus8=luma; + sps.bit_depth_chroma_minus8=hevc.bit_depth_chroma_minus8=chroma; + int expected=luma==chroma&&(luma==0||luma==2)?0:-EINVAL; + assert(avd_h264_validate_sps(&ctx,&sps)==expected); + assert(avd_hevc_validate_sps(&ctx,&hevc)==expected); + } + sps.bit_depth_luma_minus8=sps.bit_depth_chroma_minus8=0; + /* A VALID but inactive DPB entry must be included in the header's count. */ + decode.dpb[0].flags=V4L2_H264_DPB_ENTRY_FLAG_VALID|V4L2_H264_DPB_ENTRY_FLAG_ACTIVE; + decode.dpb[7].flags=V4L2_H264_DPB_ENTRY_FLAG_VALID; + decode.dpb[7].reference_ts=0; + for(unsigned int i=0;i<16;i++)references[i].base.vb.vb2_buf.dma=0x240000000+i*0x100000; + assert(!avd_h264_run(&ctx));assert(submissions==1&&hc.bank[0].submitted&&dst[0].bank==1); + /* Compare the complete header to one built with the emitted VALID count. */ + struct avd_segment expected={0};struct avd_job saved=ctx.job; + ctx.job=(struct avd_job){.segments=&expected};fixture_run.num_valid=2;fixture_run.cur_poc=40; + stream_hdr(&ctx,&fixture_run); + assert(!memcmp((u32 *)programs[0].cpu+1,expected.instructions,expected.num*sizeof(u32))); + ctx.job=saved; + /* Keep picture 0 DMA-owned while picture 1 holds and rejects a slice. */ + dev.building=&frames[1];fixture_run.base.bufs.dst=&dst[1].base.vb; + src.flags=V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;sl.first_mb_in_slice=0;dma_count=0; + assert(!avd_h264_run(&ctx));assert(held_finishes==1&&hc.bank[1].busy&&!hc.bank[1].submitted); + assert(hc.bank[0].busy&&hc.bank[0].submitted&&dst[0].bank==1); + sl.first_mb_in_slice=33;sl.slice_type=V4L2_H264_SLICE_TYPE_SP; + assert(avd_h264_run(&ctx)==-EINVAL);assert(!hc.bank[1].busy&&!hc.bank[1].slice_num&&!dst[1].bank&&!ctx.job.segments); + assert(hc.bank[0].busy&&hc.bank[0].submitted&&hc.bank[0].slice_num==1); + /* A missing-first continuation cannot borrow the submitted picture's bank. */ + sl.slice_type=V4L2_H264_SLICE_TYPE_I;assert(avd_h264_run(&ctx)==-EINVAL); + /* Repeated/decreasing macroblock positions, FMO, unequal depths and bad data abort only the builder. */ + for(unsigned int failure=0;failure<7;failure++) { + sl.first_mb_in_slice=0;dma_count=0;assert(!avd_h264_run(&ctx)); + sl.first_mb_in_slice=33;dma_count=0;assert(!avd_h264_run(&ctx)); + sl.first_mb_in_slice=failure==0?33:failure==1?16:failure==6?0:66; + if(failure==2)pps.num_slice_groups_minus1=1; + if(failure==3)sps.bit_depth_chroma_minus8=2; + if(failure==4)valid_data=false; + if(failure==5)sl.slice_type=V4L2_H264_SLICE_TYPE_SI; + assert(avd_h264_run(&ctx)==-EINVAL); + assert(!hc.bank[1].busy&&!hc.bank[1].slice_num&&!ctx.job.segments&&!dst[1].bank); + assert(hc.bank[0].busy&&hc.bank[0].submitted&&dst[0].bank==1); + pps.num_slice_groups_minus1=0;sps.bit_depth_chroma_minus8=0;valid_data=true;sl.slice_type=V4L2_H264_SLICE_TYPE_I; + } + /* Final slice submits once; retirement selects CAPTURE's bank, not hc.b. */ + sl.first_mb_in_slice=0;dma_count=0;assert(!avd_h264_run(&ctx)); + sl.first_mb_in_slice=33;src.flags=0;dma_count=0;assert(!avd_h264_run(&ctx)); + assert(submissions==2&&hc.bank[0].submitted&&hc.bank[1].submitted); + assert(avd_h264_pick_bank(&ctx,&dst[1].base.vb)==-EBUSY); + avd_h264_done(&ctx,&src,&dst[0].base.vb,VB2_BUF_STATE_DONE); + assert(!hc.bank[0].busy&&hc.bank[1].busy&&hc.b==&hc.bank[1]); + avd_h264_done(&ctx,&src,&dst[1].base.vb,VB2_BUF_STATE_DONE);assert(!hc.bank[1].busy); + assert(postambles==27&&held_finishes==16&&submissions==2); + /* The legacy transport collects the same held slices in its single bank. */ + dev.queued=false;hc.nbanks=1;dev.building=&frames[0];fixture_run.base.bufs.dst=&dst[0].base.vb; + src.flags=V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;sl.first_mb_in_slice=0;dma_count=0; + assert(!avd_h264_run(&ctx));assert(hc.b==&hc.bank[0]&&!hc.bank[0].submitted); + sl.first_mb_in_slice=33;dma_count=0;assert(!avd_h264_run(&ctx));assert(submissions==2); + sl.first_mb_in_slice=66;src.flags=0;dma_count=0;assert(!avd_h264_run(&ctx)); + assert(submissions==3&&frames[0].fifo_iova==hc.bank[0].bufs.inst.addr&&!ctx.fifo_idx); + avd_h264_done(&ctx,&src,&dst[0].base.vb,VB2_BUF_STATE_DONE);assert(!hc.bank[0].busy); + for(unsigned int b=0;b<2;b++) { + struct avd_h264_bufs *buf=&hc.bank[b].bufs; + free(buf->inst.cpu);free(buf->pipe_state.cpu);free(buf->above_info.cpu);free(buf->ip_above.cpu); + free(buf->lf_above.cpu);free(buf->lf_above_info.cpu);free(buf->mv_above_info.cpu);free(hc.bank[b].slices);free(programs[b].cpu); + } + puts("PASS: production H.264 prepare: VALID/inactive count, two-bank HOLD/rejection/final submission, PP retirement ownership"); +} +''' + +def prepare_regressions(out): + source = (ROOT / 'avd-h264.c').read_text() + h264 = re.sub(r'^#include.*\n', '', source.split('static int avd_h264_alloc_bufs(')[0], flags=re.M) + inst = re.sub(r'^#include.*\n', '', (ROOT / 'avd-inst.h').read_text(), flags=re.M) + helpers = re.sub(r'^#include.*\n', '', (ROOT / 'avd-h264-params.h').read_text(), flags=re.M) + pre = h264_prelude().replace('struct vb2_v4l2_buffer *dst; } bufs;', 'struct vb2_v4l2_buffer *src,*dst; } bufs;') + pre = pre.replace('struct { struct { struct { struct { u32 bytesperline; } plane_fmt[1]; } pix_mp; } fmt; } decoded_fmt;', + 'struct { struct { struct v4l2_pix_format_mplane pix_mp; } fmt; } decoded_fmt,coded_fmt;') + pre = pre.replace('struct vb2_buffer vb2_buf; };', 'struct vb2_buffer vb2_buf; unsigned int flags; };') + pre = pre.replace('unsigned int bank; };', 'unsigned int bank; size_t grey_chroma_offset; bool grey_chroma_10bit; };') + pre += 'enum vb2_buffer_state { VB2_BUF_STATE_DONE, VB2_BUF_STATE_ERROR };\n' + code = pre + helpers + inst + h264 + for name in ('avd_h264_alloc_bufs', 'avd_h264_validate_sps', 'avd_h264_validate_pps', + 'avd_h264_validate_slice', 'avd_h264_drop_bank_slices', 'avd_h264_pick_bank', + 'avd_h264_abort_picture', 'avd_h264_refs_valid', 'avd_h264_done'): + code += function(source, name) + code += function((ROOT / 'avd-backend-open.c').read_text(), 'avd_open_build_program') + code += function((ROOT / 'avd-hevc.c').read_text(), 'avd_hevc_validate_sps') + code += RUN_FIXTURE + function(source, 'avd_h264_run') + RUN_MAIN + dest = out / 'prepare.c' + dest.write_text(code) + exe = dest.with_suffix('') + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-Wall', '-Wextra', '-Werror', + '-Wno-sign-compare', '-Wno-unused-function', '-fsanitize=address,undefined', + '-g', str(dest), '-o', str(exe)], check=True) + subprocess.run([str(exe)], check=True) + + +CONFORMANCE_MAIN = r''' + +int main(void) +{ + struct avd_variant variant = { .revision = 4 }; + struct avd_dev dev = { .variant = &variant }; + struct avd_h264_ctx hc = {0}; + typeof(*((struct avd_ctx *)0)->fh.m2m_ctx) m2m = {0}; + struct avd_decoded_buffer destination = {0}; + struct avd_segment segments[2]; + struct avd_ctx ctx_storage = { .priv = &hc, .dev = &dev, + .fh.m2m_ctx = &m2m, .job.segments = segments, .decomp = true }; + struct avd_ctx *ctx = &ctx_storage; + struct v4l2_ctrl_h264_decode_params dec = {0}; + struct v4l2_ctrl_h264_sps sps = { .chroma_format_idc = 1, + .pic_width_in_mbs_minus1 = 11, .pic_height_in_map_units_minus1 = 8, + .flags = V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY }; + struct v4l2_ctrl_h264_pps pps = {0}; + struct v4l2_ctrl_h264_slice_params sl = {0}; + struct v4l2_ctrl_h264_scaling_matrix scaling = {0}; + struct v4l2_ctrl_h264_pred_weights weights = {0}; + struct avd_buf coded = { .addr = 0x123456780ULL, .size = 256 }; + struct avd_h264_run run = { .decode_params = &dec, .sps = &sps, + .pps = &pps, .slice_params = &sl, .scaling_matrix = &scaling, + .pred_weights = &weights, .cur_poc = 40, .slice_data_off = 7, + .base.bufs.dst = &destination.base.vb, + .base.y_out = 0x20000000, .base.uv_out = 0x20006c00, + .base.comp_out = 0x20010000, .addresses.mv_color = 0x20020000 }; + hc.nbanks = 2; hc.b = &hc.bank[0]; hc.b->active_slice = &coded; + hc.b->bufs.pipe_state.addr=0x10000000; + hc.b->bufs.above_info.addr=0x10010000; + hc.b->bufs.ip_above.addr=0x10020000; + hc.b->bufs.lf_above.addr=0x10030000; + hc.b->bufs.lf_above_info.addr=0x10040000; + hc.b->bufs.mv_above_info.addr=0x10050000; + ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline = 192; + for (unsigned int i = 0; i < 16; i++) { + references[i].base.vb.vb2_buf.dma = 0x30000000 + i * 0x100000; + references[i].base.vb.vb2_buf.planes[0].length = 0x30000; + references[i].comp.start_offset = 0x10000; + for (unsigned int j = 0; j < 4; j++) + references[i].comp.offsets[j] = ctx->comp.offsets[j] = j * 0x1000; + } + /* Dense DPBs cover every reference-count, slice type and weight mode. + * Sparse DPBs deliberately place references on either side of holes. */ + for (unsigned int mask_case = 0; mask_case < 18; mask_case++) { + unsigned int mask = mask_case <= 16 ? (1U << mask_case) - 1 : 0x8429; + unsigned int count = __builtin_popcount(mask), ids[16], n = 0; + for (unsigned int i = 0; i < 16; i++) { + dec.dpb[i].flags = mask & BIT(i) ? + V4L2_H264_DPB_ENTRY_FLAG_VALID | V4L2_H264_DPB_ENTRY_FLAG_ACTIVE : 0; + dec.dpb[i].reference_ts = i; + dec.dpb[i].top_field_order_cnt = 2 * i; + if (mask & BIT(i)) ids[n++] = i; + } + for (unsigned int rev = 0; rev < 4; rev++) { + variant.revision = rev ? 4 : 3; + variant.quirks = rev < 2 ? AVD_QUIRK_LSR | AVD_QUIRK_NO_PIPE_STATE : + rev == 2 ? AVD_QUIRK_LSR : 0; + for (unsigned int type = 0; type < 3; type++) { + if (type != V4L2_H264_SLICE_TYPE_I && !count) continue; + for (unsigned int mode = 0; mode < 4; mode++) { + for (unsigned int mb = 0; mb < 2; mb++) { + pps.flags = mode & 1 ? V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE : 0; + if (mode == 2) pps.flags |= V4L2_H264_PPS_FLAG_WEIGHTED_PRED; + pps.weighted_bipred_idc = mode == 3 ? 2 : 0; + sl.slice_type = type; + sl.first_mb_in_slice = mb ? 33 : 0; + sl.header_bit_size = 53; + sl.cabac_init_idc = mode % 3; + sl.slice_qp_delta = mode ? -2 : 3; + sl.flags = mode & 1 ? V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED : 0; + sl.num_ref_idx_l0_active_minus1 = count ? count - 1 : 0; + sl.num_ref_idx_l1_active_minus1 = count > 1 ? 1 : 0; + for (unsigned int i = 0; i < count; i++) { + sl.ref_pic_list0[i].index = ids[i]; + sl.ref_pic_list1[i].index = ids[count - 1 - i]; + for (unsigned int j = 0; j < 2; j++) { + weights.weight_factors[j].luma_weight[i] = i % 2 ? -1 : 1; + weights.weight_factors[j].luma_offset[i] = i % 3; + weights.weight_factors[j].chroma_weight[i][0] = 1; + weights.weight_factors[j].chroma_weight[i][1] = 1; + } + } + run.num_valid = count; + dec.flags = count ? 0 : V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC; + ctx->job.num = 0; dma_count = 0; + segments[0].num = segments[1].num = 0; + stream_hdr(ctx, &run); + ctx->job.num = 1; + stream_slice(ctx, &run); + printf("%u/%u/%u/%u/%u", mask_case, rev, type, mode, mb); + for (unsigned int seg = 0; seg < 2; seg++) + for (unsigned int i = 0; i < segments[seg].num; i++) + printf(" %08x", segments[seg].instructions[i]); + puts(""); + + } + } + } + } + } + for (unsigned int type = 0; type < 10; type++) { + sl.slice_type = type; + printf("guard/type/%u %d\n", type, VALIDATE_SLICE(ctx, &sl)); + } + for (unsigned int groups = 0; groups < 8; groups++) { + pps.num_slice_groups_minus1 = groups; + printf("guard/groups/%u %d\n", groups, avd_h264_validate_pps(ctx, &pps)); + } + if (!avd_h264_slice_order_supported(0, 0, 0) || + !avd_h264_slice_order_supported(1, 0, 33) || + !avd_h264_slice_order_supported(2, 33, 66) || + avd_h264_slice_order_supported(0, 0, 33) || + avd_h264_slice_order_supported(1, 33, 33) || + avd_h264_slice_order_supported(2, 66, 33) || + avd_h264_slice_order_supported(2, 66, 0)) return 4; + /* Every possible DPB occupancy must map to dense header slots. */ + for (unsigned int mask = 0; mask < 65536; mask++) { + unsigned int count = 0; + for (unsigned int i = 0; i < 16; i++) { + dec.dpb[i].flags = mask & BIT(i) ? V4L2_H264_DPB_ENTRY_FLAG_VALID : 0; + if (mask & BIT(i)) { + if (avd_h264_ref_slot(&dec, i) != count++) return 2; + } + } + if (avd_h264_num_refs(&dec) != count) return 3; + } + return 0; +} + +''' + +def conformance(out): + if os.environ.get('AVD_H264_BASELINE_ROOT'): + baseline = Path(os.environ['AVD_H264_BASELINE_ROOT']).resolve() + else: + baseline = out / 'baseline' + baseline.mkdir(exist_ok=True) + ref = os.environ.get('AVD_H264_BASELINE_REF', '757fc07a6930ee12b04fe034138dc9880dc4ff3f') + for name in ('avd-h264.c', 'avd-inst.h'): + data = subprocess.check_output(['git', '-C', str(ROOT), 'show', + f'{ref}:drivers/media/platform/apple/avd/{name}']) + (baseline / name).write_bytes(data) + (out / 'baseline-inputs.sha256').write_text(''.join( + hashlib.sha256((baseline / name).read_bytes()).hexdigest() + ' ' + name + '\n' + for name in ('avd-h264.c', 'avd-inst.h'))) + programs = {} + for label, root in (('before', baseline), ('after', ROOT)): + source = (root / 'avd-h264.c').read_text() + h264 = source.split('static int avd_h264_alloc_bufs(')[0] + h264 = re.sub(r'^#include.*\n', '', h264, flags=re.M) + inst = re.sub(r'^#include.*\n', '', (root / 'avd-inst.h').read_text(), flags=re.M) + # Helpers are exercised separately; the baseline emitters do not use them. + helpers = re.sub(r'^#include.*\n', '', (ROOT / 'avd-h264-params.h').read_text(), flags=re.M) + checks = function(source, 'avd_h264_validate_pps') + if 'static int avd_h264_validate_slice(' in source: + checks += function(source, 'avd_h264_validate_slice') + checks += '#define VALIDATE_SLICE avd_h264_validate_slice\n' + else: + checks += '#define VALIDATE_SLICE(ctx, sl) 0\n' + dest = out / ('conformance-' + label + '.c') + dest.write_text(h264_prelude() + helpers + inst + h264 + checks + CONFORMANCE_MAIN) + exe = dest.with_suffix('') + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-O1', '-Wall', '-Wextra', + '-Werror', '-Wno-unused-function', '-Wno-unused-parameter', '-Wno-sign-compare', + '-fsanitize=address,undefined', '-g', str(dest), '-o', str(exe)], check=True) + data = subprocess.check_output([str(exe)], text=True) + programs[label] = dict(line.split(' ', 1) for line in data.splitlines()) + (out / (label + '.json')).write_text(json.dumps(programs[label], indent=2) + '\n') + unchanged = sparse = 0 + for key, program in programs['after'].items(): + if key.startswith('guard/type/'): + assert program == ('0' if int(key.rsplit('/', 1)[1]) < 3 else '-22'), key + elif key.startswith('guard/groups/'): + assert program == ('0' if key.endswith('/0') else '-22'), key + elif not key.startswith('17/') or key.split('/')[2] == '2': + assert program == programs['before'][key], key + unchanged += 1 + else: + before, after = programs['before'][key].split(), program.split() + assert len(before) == len(after) and before != after, key + for old, new in zip(before, after): + if old != new: + assert int(old, 16) >> 20 == 0x2dc, (key, old, new) + assert int(old, 16) & ~15 == int(new, 16) & ~15, (key, old, new) + sparse += 1 + print(f'PASS: {unchanged} complete legacy programs identical; {sparse} sparse programs change only reference slots; all 65536 DPB maps; syntax guards', flush=True) + + def main(): out = Path(sys.argv[1]).resolve() out.mkdir(parents=True, exist_ok=True) - names = ['avd-h264.c', 'avd-inst.h', 'avd-backend-open.c', 'avd-backend-queue.c', 'avd-hw.c'] + names = ['avd-h264-params.h', 'avd-h264.c', 'avd-hevc.c', 'avd-inst.h', 'avd-backend-open.c', 'avd-backend-queue.c', 'avd-hw.c'] (out / 'inputs.sha256').write_text(''.join(hashlib.sha256((ROOT / n).read_bytes()).hexdigest()+' '+n+'\n' for n in names)) + prepare_regressions(out) + conformance(out) h264 = (ROOT / 'avd-h264.c').read_text().split('static int avd_h264_alloc_bufs(')[0] h264 = re.sub(r'^#include.*\n', '', h264, flags=re.M) inst = (ROOT / 'avd-inst.h').read_text() @@ -359,7 +719,8 @@ def main(): 'static inline void push_address(struct avd_ctx *ctx, dma_addr_t addr)\n{\n record_dma(ctx,addr,(ctx->dev->variant->quirks&AVD_QUIRK_LSR)?8:0,(ctx->dev->variant->quirks&AVD_QUIRK_LSR)?1:2);') inst = inst.replace('u32 offsets[4])\n{', 'u32 offsets[4])\n{\n assert(addr);') inst = inst.replace('push(ctx, (addr + offsets[i]) >> 7);', '{ record_dma(ctx,addr+offsets[i],7,1); push(ctx, (addr + offsets[i]) >> 7); }') - code.write_text(PRELUDE + inst + h264 + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_alloc_bufs') + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_drop_bank_slices') + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_pick_bank') + builder + wire + regs + config + + helpers = re.sub(r'^#include.*\n', '', (ROOT / 'avd-h264-params.h').read_text(), flags=re.M) + code.write_text(PRELUDE + helpers + inst + h264 + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_alloc_bufs') + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_drop_bank_slices') + function((ROOT / 'avd-h264.c').read_text(), 'avd_h264_pick_bank') + builder + wire + regs + config + function(qfw, 'avd_qfw_check') + function(qfw, 'avd_qfw_stream') + function(qfw, 'avd_qfw_submit') + function(qfw, 'avd_qfw_fail_all') + function(qfw, 'avd_qfw_irq') + MAIN) exe = out / 'program-test' diff --git a/tools/asahi/avd-codec-tail-tests.py b/tools/asahi/avd-codec-tail-tests.py index a7f2e039957f1..a1d57aa2e763c 100755 --- a/tools/asahi/avd-codec-tail-tests.py +++ b/tools/asahi/avd-codec-tail-tests.py @@ -178,6 +178,13 @@ def main(): "body_sha256": hashlib.sha256(body.encode()).hexdigest(), "defines_sha256": hashlib.sha256(defines.encode()).hexdigest(), "cases_per_variant": len(cases), "results": results, "passed": ok} + if ok and not args.ref and not args.mutations: + tests = ROOT / "drivers/media/platform/apple/avd/tests" + for script in ("program-test.py", "dispatch-test.py"): + subprocess.run([sys.executable, str(tests / script), + str(args.output / script.removesuffix(".py"))], + check=True) + receipt["h264_program_and_dispatch"] = "PASS" (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") print(json.dumps({"passed": ok, "cases": len(cases), "variants": [{k: x[k] for k in ("name", "failed_cases", "verdict")} From 630053fe50bba7f8d43f10267d5c3d4c01b6545e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 19:47:04 +1100 Subject: [PATCH 64/73] media: apple: avd: use an intra header for reference-free non-IDR I pictures Port the reference-free intra-header fix onto the banked H.264 builder. Frame-number gaps can evict every real reference, leaving a non-IDR I picture with an empty hardware DPB. Its inter header cannot encode the empty table because the reference count lives in the first entry. Use the existing IDR intra-only header for an I picture with no VALID references. This sets INTRA, clears NOT_IDR and omits the empty reference preamble. Pictures with references retain the existing header on both transports. Extend the rewrite's program test with 32 exact complete-program checks against the pinned legacy IDR path, preserving before/after evidence. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-h264.c | 24 +++++++++++++------ .../media/platform/apple/avd/tests/README.rst | 8 +++++++ .../platform/apple/avd/tests/program-test.py | 24 +++++++++++++++++-- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index 439fc2cf96bda..41d8fc12d8d4a 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -222,9 +222,21 @@ static void stream_scaling(struct avd_ctx *ctx, struct avd_h264_run *run) } } +static bool avd_h264_intra_header(const struct avd_h264_run *run) +{ + /* + * frame_num gaps can evict every real reference. A non-IDR I picture + * still decodes without references, but an inter header has no way to + * encode an empty reference table: its count lives in the first entry. + * Use the same reference-free header as an IDR picture in that case. + */ + return (run->decode_params->flags & V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC) || + (!run->num_valid && + run->slice_params->slice_type == V4L2_H264_SLICE_TYPE_I); +} + static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) { - const struct v4l2_ctrl_h264_decode_params *decode = run->decode_params; const struct v4l2_ctrl_h264_sps *sps = run->sps; const struct v4l2_ctrl_h264_pps *pps = run->pps; struct avd_dev *avd = ctx->dev; @@ -232,11 +244,10 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) u32 bytesperline; u32 width = (sps->pic_width_in_mbs_minus1 + 1) * 16; u32 height = (sps->pic_height_in_map_units_minus1 + 1) * 16; + bool intra_header = avd_h264_intra_header(run); push(AVD_OP_HDR | AVD_OP_HDR_FLAG_DECOMP(ctx->decomp) | - AVD_OP_HDR_FLAG_INTRA( - decode->flags & - V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC) | + AVD_OP_HDR_FLAG_INTRA(intra_header) | AVD_OP_HDR_CONST | AVD_OP_HDR_FLAG_PIPE_STATE( !(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)), @@ -269,8 +280,7 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) push(H264_FLAG_ENTROPY_CODING_MODE( pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE) | - H264_FLAG_NOT_IDR(!(decode->flags & - V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC)) | + H264_FLAG_NOT_IDR(!intra_header) | h264_FLAG_CI_PRED( pps->flags & V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED), @@ -321,7 +331,7 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) push(0, "cm3_mark_end_section"); push(((height - 1) << 16) | (width - 1), "hdr_54_height_width"); - if (!(decode->flags & V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC)) + if (!intra_header) stream_refs(ctx, run); if (pps->flags & V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT) diff --git a/drivers/media/platform/apple/avd/tests/README.rst b/drivers/media/platform/apple/avd/tests/README.rst index 62b040f82a4da..1d4f3917f63c4 100644 --- a/drivers/media/platform/apple/avd/tests/README.rst +++ b/drivers/media/platform/apple/avd/tests/README.rst @@ -273,3 +273,11 @@ one bank; the queued transport permits two banks. These are host control/program/ownership regressions. The lead's conformance run must still establish decoded pixels and absence of hardware timeouts. + +A non-IDR I picture with no VALID DPB entries now uses the existing +reference-free intra header. The intended changes are the header INTRA bit, +clearing the H.264 NOT_IDR flag, and omitting the empty reference-table +preamble. Across all four revision/address variants, weight modes and +first/continuation slices, its complete program must equal the baseline IDR +program exactly. Pictures with references retain their original header. +The baseline comparison retains all 32 before/after empty-DPB cases. diff --git a/drivers/media/platform/apple/avd/tests/program-test.py b/drivers/media/platform/apple/avd/tests/program-test.py index 6ee23e6370a10..1d6ea38f3a83e 100755 --- a/drivers/media/platform/apple/avd/tests/program-test.py +++ b/drivers/media/platform/apple/avd/tests/program-test.py @@ -600,6 +600,19 @@ def prepare_regressions(out): for (unsigned int i = 0; i < segments[seg].num; i++) printf(" %08x", segments[seg].instructions[i]); puts(""); + if (!count) { + dec.flags = 0; + ctx->job.num = 0; dma_count = 0; + segments[0].num = segments[1].num = 0; + stream_hdr(ctx, &run); + ctx->job.num = 1; + stream_slice(ctx, &run); + printf("empty-intra/%u/%u/%u", rev, mode, mb); + for (unsigned int seg = 0; seg < 2; seg++) + for (unsigned int i = 0; i < segments[seg].num; i++) + printf(" %08x", segments[seg].instructions[i]); + puts(""); + } } } @@ -674,12 +687,19 @@ def conformance(out): data = subprocess.check_output([str(exe)], text=True) programs[label] = dict(line.split(' ', 1) for line in data.splitlines()) (out / (label + '.json')).write_text(json.dumps(programs[label], indent=2) + '\n') - unchanged = sparse = 0 + unchanged = sparse = normalized = 0 for key, program in programs['after'].items(): if key.startswith('guard/type/'): assert program == ('0' if int(key.rsplit('/', 1)[1]) < 3 else '-22'), key elif key.startswith('guard/groups/'): assert program == ('0' if key.endswith('/0') else '-22'), key + elif key.startswith('empty-intra/'): + _, revision, mode, mb = key.split('/') + idr = f'0/{revision}/2/{mode}/{mb}' + assert program == programs['before'][idr], key + assert program == programs['after'][idr], key + assert program != programs['before'][key], key + normalized += 1 elif not key.startswith('17/') or key.split('/')[2] == '2': assert program == programs['before'][key], key unchanged += 1 @@ -691,7 +711,7 @@ def conformance(out): assert int(old, 16) >> 20 == 0x2dc, (key, old, new) assert int(old, 16) & ~15 == int(new, 16) & ~15, (key, old, new) sparse += 1 - print(f'PASS: {unchanged} complete legacy programs identical; {sparse} sparse programs change only reference slots; all 65536 DPB maps; syntax guards', flush=True) + print(f'PASS: {unchanged} complete legacy programs identical; {sparse} sparse programs change only reference slots; {normalized} empty-DPB non-IDR I programs equal legacy IDR; all 65536 DPB maps; syntax guards', flush=True) def main(): From 77998310962683a739a62eb988fda6eb461662d8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 21:28:56 +1100 Subject: [PATCH 65/73] Revert "drm/apple: Switch desktop 26.x external displays to VRR without a DRM mode set" This reverts commit 64e23b40571458f482244ae36e29dcfc92379dd5. Reselecting the mode on every VRR_ENABLED change blanks an external display for about 5 seconds each time. GNOME turns VRR on when a fullscreen client is scanned out directly and off again when it stops (a cursor move is enough), so on an MSI MPG 325CQRXF a fullscreen browser kept dropping out. Keeping adaptive sync installed and moving only the rate does not work on these processors either: with VRR_ENABLED clear, neither a minimum rate nor a media target rate of 280 Hz kept the display fixed, and an idle desktop ran variable and flickered. Go back to requiring a mode set for a VRR_ENABLED change on the external processors until the way the firmware holds a fixed rate with adaptive sync installed is known. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 -- drivers/gpu/drm/apple/dcp.c | 16 +++++++++--- drivers/gpu/drm/apple/iomfb.c | 25 ------------------- .../drivers/gpu/apple-dcp/lifecycle.py | 13 ++++++---- 4 files changed, 20 insertions(+), 36 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 105ec1d3b1d4f..57b70d4fbe03c 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -274,8 +274,6 @@ struct dcp_desktop_state { * the firmware was last given. Zero when not installed that way. */ u32 vrr_min_on, vrr_min_off, vrr_min_sent; - /* VRR_ENABLED as of the last mode set (external processors). */ - bool vrr_requested; /* The frame's colour matrix, sent after an adaptive sync update. */ struct iomfb_set_matrix_req matrix; /* diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 97e0ca217ad21..2cbd0771e971b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1510,11 +1510,19 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi return -EINVAL; /* - * On desktop 26.6 firmware VRR_ENABLED takes effect with the next - * frame, without a DRM mode set: the panel through a new minimum - * rate (dcp_vrr_update()), the external processors by reselecting - * their timing (dcp_flush()). + * On desktop 26.6 firmware the panel takes VRR_ENABLED with the next + * frame, without a mode set (see dcp_vrr_update()). The external + * processors ignore a new minimum rate on a live mode, so they still + * change adaptive sync with a mode set. */ + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && crtc_state->enable && + !dcp_vrr_live(dcp)) { + struct drm_crtc_state *old = + drm_atomic_get_old_crtc_state(commit, crtc); + + if (old && old->vrr_enabled != crtc_state->vrr_enabled) + crtc_state->mode_changed = true; + } /* DRM core only caps the blob size; the gamma update copies a full table. */ if (crtc_state->color_mgmt_changed && crtc_state->gamma_lut && diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 296b8b6b20387..63ef48f0e0e53 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -2642,8 +2642,6 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, dcp->output_edr = dcp->pq_requested && dcp_panel_edr_supported(dcp) && dcp_crtc_bt2020(dcp, crtc, commit); dcp->pq_requested &= dcp_hdr_output_supported(dcp); - if (dcp->desktop) - dcp->desktop->vrr_requested = crtc_state->vrr_enabled; dcp_hdr_metadata_forget(dcp); dcp_mode_begin(&dcp->mode_state); switch (dcp->fw_compat) { @@ -3187,29 +3185,6 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) dcp_crtc_atomic_modeset(crtc, commit); } } - - /* - * The external processors take adaptive sync only with a mode set; - * macOS too reselects the same timing to switch a display between - * fixed and variable refresh. Compositors flip VRR_ENABLED in plain - * page flips (mutter when a fullscreen client comes and goes), so a - * change on a live VRR timing reselects the mode here, once. - */ - if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && !dcp_vrr_live(dcp) && - READ_ONCE(dcp->mode_state.valid) && dcp->connector && - READ_ONCE(dcp->connector->connected)) { - struct drm_crtc_state *cs = drm_atomic_get_new_crtc_state(commit, crtc); - struct dcp_display_mode mode; - - if (cs && cs->active && !drm_atomic_crtc_needs_modeset(cs) && - cs->vrr_enabled != dcp->desktop->vrr_requested && - lookup_mode(dcp, &cs->mode, &mode) && mode.vrr) { - dev_dbg(dcp->dev, "VRR_ENABLED %u: reselecting the mode\n", - cs->vrr_enabled); - dcp_mode_invalidate(&dcp->mode_state); - dcp_crtc_atomic_modeset(crtc, commit); - } - } if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { iomfb_v14_7_flush(dcp, crtc, commit); return; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 89081523f183d..89dd1f31f954b 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -1730,6 +1730,9 @@ def vrr(root): static int dcp_crtc_check_hdr(struct apple_dcp *d, struct drm_crtc *c, struct drm_atomic_commit *s) { (void)d; (void)c; (void)s; return 0; } +/* The panel of a desktop 26.6 machine takes VRR_ENABLED live. */ +static bool dcp_vrr_live(struct apple_dcp *d) +{ return d->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && d->main_display; } static bool iomfb_is_m4(struct apple_dcp *d) { (void)d; return false; } static int iomfb_m4_error(struct apple_dcp *d) { (void)d; assert(false); return -ENODEV; } static int iomfb_m4_get_mode(struct apple_dcp *d,struct drm_display_mode *m) @@ -1788,10 +1791,10 @@ def vrr(root): return 0; } """ - # Desktop 26.6 takes VRR_ENABLED with the next frame on the panel and the - # external processors alike (the latter reselect their mode in flush): - # compositors toggle it in plain page flips, so a VRR_ENABLED change - # must never become a DRM mode set. + # Desktop 26.6 takes VRR_ENABLED live on the panel (main display), with + # the next frame: compositors toggle it in plain page flips, so there a + # VRR_ENABLED change must never become a mode set. The external + # processors ignore a live minimum rate and still apply it with a mode set. desktop = r""" dcp.fw_compat = DCP_FIRMWARE_V_26_6_DESKTOP; dcp.hw.iomfb_method_profile = 0; dcp.mode_state.valid = true; @@ -1809,7 +1812,7 @@ def vrr(root): state.no_old_crtc = old_vrr == 2; state.old_crtc.vrr_enabled = old_vrr == 1; assert(dcp_crtc_atomic_check(&crtc, &state) == 0); - bool expected = false; + bool expected = !panel && enable && old_vrr != 2 && (old_vrr == 1) != vrr; if (state.crtc.mode_changed != expected) { fprintf(stderr, "desktop panel=%u enable=%u old_vrr=%u vrr=%u: mode_changed=%d expected %d\n", panel, enable, old_vrr, vrr, state.crtc.mode_changed, expected); From 7e87ed301342bf0d9a4af84a073202aee005d7d7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 19:01:23 +1100 Subject: [PATCH 66/73] pmdomain: apple: pmc: provide cluster enable bits as power domains On T8140 the video encoder cluster (VENC-CLSTR-V) has an enable bit in a PMC control word, with a matching status word. The PMP only raises the encoder performance state while the cluster is enabled, and macOS sets the bit, under its PMC lock, whenever the encoder is powered. Describe such bits as child nodes with apple,cluster-bit, in an optional "clusters" register range (control word, then status), and provide each as a power domain: power on sets the bit and waits for its status, power off clears it. The read-modify-write runs under the PMC lock. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/power/apple,pmc.yaml | 42 ++++++- drivers/pmdomain/apple/pmc.c | 114 ++++++++++++++++++ 2 files changed, 152 insertions(+), 4 deletions(-) diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index 383b42d560e98..f4dae18decef7 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -30,23 +30,29 @@ properties: - apple,t8140-pmc reg: - maxItems: 2 + minItems: 2 + maxItems: 3 description: | The perf-state floor words of the voter agents ("voters"), and the per-agent words holding the floor the PMC honours for each agent with - the per-rail interface enable words ("rails"). + the per-rail interface enable words ("rails"). Optionally, the cluster + enable control word followed by its status word ("clusters"). reg-names: description: | The rails block may come first, so that the node's unit address does not collide with a node covering the voters block. oneOf: - - items: + - minItems: 2 + items: - const: voters - const: rails - - items: + - const: clusters + - minItems: 2 + items: - const: rails - const: voters + - const: clusters nonposted-mmio: true @@ -143,6 +149,34 @@ patternProperties: - apple,rail - apple,level + '^cluster-[a-z0-9-]+$': + type: object + description: | + A cluster enable bit in the "clusters" words, provided as a power + domain. The bit is set, and its status awaited, while the domain is + powered. The T8140 video encoder cluster must be enabled for the PMP to + raise the encoder's performance state. + additionalProperties: false + + properties: + label: + description: Name of the power domain. + + '#power-domain-cells': + const: 0 + + power-domains: + maxItems: 1 + + apple,cluster-bit: + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 31 + description: Bit of the cluster in the control and status words. + + required: + - '#power-domain-cells' + - apple,cluster-bit + '^performance-controller@[0-9a-f]+$': type: object description: The host performance controller's voter agent. diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 85b2578351af1..5d07b595d7cce 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include @@ -93,6 +94,8 @@ struct apple_pmc { const struct apple_pmc_hw *hw; void __iomem *voters; void __iomem *rails; + /* Optional cluster enable control and status words (T8140). */ + void __iomem *clusters; /* * Serializes read-modify-write cycles on enable and floor words, and * protects @enable and @dcs_state. @@ -122,6 +125,8 @@ struct apple_pmc { * @rail: with @floor, the rail * @level: with @floor, the performance state held while powered * @on: with @floor, whether the floor is held; protected by pmc->lock + * @cluster: the domain is a cluster enable bit in the "clusters" region + * @bit: with @cluster, the bit in the control and status words */ struct apple_pmc_domain { struct generic_pm_domain genpd; @@ -133,6 +138,8 @@ struct apple_pmc_domain { unsigned int rail; unsigned int level; bool on; + bool cluster; + unsigned int bit; bool provider; }; @@ -634,10 +641,39 @@ static int apple_pmc_floor_set(struct apple_pmc_domain *dom, bool on) return ret; } +/* The cluster control word is at +0 and its status at +4. */ +#define APPLE_PMC_CLUSTER_CTRL 0x0 +#define APPLE_PMC_CLUSTER_STATUS 0x4 +#define APPLE_PMC_CLUSTER_TIMEOUT_US 10000 + +static int apple_pmc_cluster_set(struct apple_pmc_domain *dom, bool on) +{ + struct apple_pmc *pmc = dom->pmc; + u32 mask = BIT(dom->bit), val; + unsigned long flags; + int ret; + + raw_spin_lock_irqsave(&pmc->lock, flags); + val = readl(pmc->clusters + APPLE_PMC_CLUSTER_CTRL); + writel(on ? val | mask : val & ~mask, + pmc->clusters + APPLE_PMC_CLUSTER_CTRL); + raw_spin_unlock_irqrestore(&pmc->lock, flags); + + ret = readl_poll_timeout_atomic(pmc->clusters + APPLE_PMC_CLUSTER_STATUS, + val, !!(val & mask) == on, 1, + APPLE_PMC_CLUSTER_TIMEOUT_US); + if (ret) + dev_err(pmc->dev, "%s: cluster bit %u did not turn %s (status %#x)\n", + dom->genpd.name, dom->bit, str_on_off(on), val); + return ret; +} + static int apple_pmc_domain_power_on(struct generic_pm_domain *genpd) { struct apple_pmc_domain *dom = genpd_to_apple_pmc_domain(genpd); + if (dom->cluster) + return apple_pmc_cluster_set(dom, true); if (dom->floor) return apple_pmc_floor_set(dom, true); return apple_pmc_agent_enable(dom->pmc, dom->agent, true); @@ -647,6 +683,8 @@ static int apple_pmc_domain_power_off(struct generic_pm_domain *genpd) { struct apple_pmc_domain *dom = genpd_to_apple_pmc_domain(genpd); + if (dom->cluster) + return apple_pmc_cluster_set(dom, false); if (dom->floor) return apple_pmc_floor_set(dom, false); return apple_pmc_agent_enable(dom->pmc, dom->agent, false); @@ -740,6 +778,75 @@ static int apple_pmc_domain_add(struct apple_pmc *pmc, struct device_node *np) return 0; } +/* + * A cluster enable bit, set while the domain is powered: the T8140 video + * encoder cluster must be enabled before its performance request takes + * effect. Starts off; the bootloader leaves the bits clear. + */ +static int apple_pmc_cluster_add(struct apple_pmc *pmc, struct device_node *np) +{ + struct device *dev = pmc->dev; + struct apple_pmc_domain *dom; + struct of_phandle_args parent; + const char *label; + u32 bit; + int i, ret; + + if (!pmc->clusters) + return dev_err_probe(dev, -EINVAL, "%pOF: no clusters region\n", np); + if (of_property_read_u32(np, "apple,cluster-bit", &bit) || bit >= 32) + return dev_err_probe(dev, -EINVAL, "%pOF: invalid cluster bit\n", np); + + dom = kzalloc_obj(*dom); + if (!dom) + return -ENOMEM; + + dom->pmc = pmc; + dom->np = of_node_get(np); + dom->cluster = true; + dom->bit = bit; + if (of_property_read_string(np, "label", &label)) + label = np->name; + dom->genpd.name = kasprintf(GFP_KERNEL, "pmc-%s", label); + if (!dom->genpd.name) { + of_node_put(dom->np); + kfree(dom); + return -ENOMEM; + } + dom->genpd.power_on = apple_pmc_domain_power_on; + dom->genpd.power_off = apple_pmc_domain_power_off; + + ret = pm_genpd_init(&dom->genpd, NULL, + !(readl(pmc->clusters + APPLE_PMC_CLUSTER_STATUS) & BIT(bit))); + if (ret) { + of_node_put(dom->np); + kfree(dom->genpd.name); + kfree(dom); + return dev_err_probe(dev, ret, "%pOF: genpd init failed\n", np); + } + list_add_tail(&dom->list, &pmc->domains); + + ret = of_genpd_add_provider_simple(np, &dom->genpd); + if (ret) + return dev_err_probe(dev, ret, "%pOF: provider failed\n", np); + dom->provider = true; + + for (i = 0; !of_parse_phandle_with_args(np, "power-domains", + "#power-domain-cells", i, + &parent); i++) { + struct of_phandle_args child = { .np = np }; + + ret = of_genpd_add_subdomain(&parent, &child); + of_node_put(parent.np); + if (ret) + return dev_err_probe(dev, ret, + "%pOF: failed to join parent domain\n", + np); + } + + return 0; +} + /* A rail floor held through the AP's agent while the domain is powered. */ static int apple_pmc_floor_add(struct apple_pmc *pmc, struct device_node *np) { @@ -905,6 +1012,11 @@ static int apple_pmc_probe(struct platform_device *pdev) pmc->rails = devm_platform_ioremap_resource_byname(pdev, "rails"); if (IS_ERR(pmc->rails)) return PTR_ERR(pmc->rails); + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "clusters")) { + pmc->clusters = devm_platform_ioremap_resource_byname(pdev, "clusters"); + if (IS_ERR(pmc->clusters)) + return PTR_ERR(pmc->clusters); + } /* An unpowered PMC reads all-ones; do not write to it. */ for (rail = 0; rail < pmc->hw->rails; rail++) { @@ -953,6 +1065,8 @@ static int apple_pmc_probe(struct platform_device *pdev) continue; if (of_property_present(child, "apple,rail")) ret = apple_pmc_floor_add(pmc, child); + else if (of_property_present(child, "apple,cluster-bit")) + ret = apple_pmc_cluster_add(pmc, child); else ret = apple_pmc_domain_add(pmc, child); if (ret) { From bc801bece9f80316a6bb2a3abeed63b17e7a8c38 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 19:01:23 +1100 Subject: [PATCH 67/73] pmdomain: apple: pmp-report: send a soc-device performance request with its power request A PMP soc-device can have a performance request beside its power request: on T8140 the encoder (soc-device 15, AVE0) asks for its VMAX state in SOC-DEV-DVFS entry 0x198, and the PMP then votes the AVE rail up itself, as it does under macOS. Let a report entry name that entry and level with apple,dvfs. After the PMP acknowledges the power request, write the level with the valid bit in one 64-bit store; before clearing the power request, write level 0. Never read a request entry back after storing it: that read retires the pending PTD update before the PMP handles it, and the request is lost. The T8140 offsets gain the update aperture and the SOC-DEV-DVFS range. The T6030 graph host test harness follows the entry struct. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../power/apple,t8140-pmp-report.yaml | 15 +++++- drivers/pmdomain/apple/pmp-report.c | 53 +++++++++++++++++-- tools/asahi/t6030-pmp-graph-tests.c | 13 ++++- tools/asahi/t6030-pmp-graph-tests.py | 3 +- 4 files changed, 77 insertions(+), 7 deletions(-) diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml index c3baafe00064c..9ca5d2c1777f3 100644 --- a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -115,7 +115,20 @@ patternProperties: const: 0 power-domains: - maxItems: 1 + minItems: 1 + maxItems: 2 + + apple,dvfs: + $ref: /schemas/types.yaml#/definitions/uint32-array + items: + - description: SOC-DEV-DVFS entry holding the device's request. + - description: Performance level requested while powered (0-31). + maximum: 31 + description: + The device also has a performance request. While the domain is + powered, after the PMP acknowledges the device, the host writes the + level (with the valid bit) to the entry once; before clearing the + device request it writes level 0. apple,always-on: type: boolean diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index f09b38aa310ca..e8b6fd640a1f3 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -70,6 +70,9 @@ struct apple_pmp_report_offsets { u32 status; u32 fast_die; u32 fast_die_lanes; + /* Update aperture of every entry (8 bytes each) and SOC-DEV-DVFS range. */ + u32 update; + u32 dvfs_first, dvfs_count; bool running_any; bool defer_requests; bool starts_pmp; @@ -1463,6 +1466,9 @@ static const struct apple_pmp_report_offsets apple_pmp_offsets_t6031 = { static const struct apple_pmp_report_offsets apple_pmp_offsets_t8140 = { .tgt_read = 0x1880, .tgt_write = 0x10c40, + .update = 0x10000, + .dvfs_first = 0x190, + .dvfs_count = 16, .actual = 0x18c0, .status = 0x10, .fast_die = 0x1c10, @@ -1526,6 +1532,13 @@ struct apple_pmp_report_entry { u32 id; /* The PMP does not acknowledge this request (a started PMP only). */ bool no_ack; + /* + * With @dvfs, the SOC-DEV-DVFS entry this soc-device's performance + * request lives in, and the request word written while powered. + */ + bool dvfs; + u32 dvfs_entry; + u64 dvfs_on; bool removed; }; @@ -1569,6 +1582,19 @@ static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id, 50000); } +/* SOC-DEV-DVFS request words carry a valid bit; level 0 is the floor. */ +#define PMP_DVFS_VALID BIT_ULL(61) + +/* + * One 64-bit store of the whole request word. Never read a request entry + * back after storing it: the read retires the pending PTD update before the + * PMP handles it, and the request is lost. + */ +static void apple_pmp_report_dvfs_write(struct apple_pmp_report *rep, u32 entry, u64 val) +{ + writeq(val, rep->base + rep->offsets->update + 8 * entry); +} + static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, bool enable) { struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); @@ -1580,10 +1606,15 @@ static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, boo rcu_read_lock(); list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { - if (rep->dev->of_node == ent->report_node) { - ret = apple_pmp_report_set_state(rep, ent->id, enable, ent->no_ack); - break; - } + if (rep->dev->of_node != ent->report_node) + continue; + /* The device request is admitted first and the level dropped first. */ + if (ent->dvfs && !enable) + apple_pmp_report_dvfs_write(rep, ent->dvfs_entry, PMP_DVFS_VALID); + ret = apple_pmp_report_set_state(rep, ent->id, enable, ent->no_ack); + if (ent->dvfs && enable && !ret) + apple_pmp_report_dvfs_write(rep, ent->dvfs_entry, ent->dvfs_on); + break; } rcu_read_unlock(); return ret; @@ -1661,6 +1692,20 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) } ent->no_ack = of_property_read_bool(node, "apple,no-ack"); } + if (of_property_present(node, "apple,dvfs")) { + u32 dvfs[2]; + + ret = of_property_read_u32_array(node, "apple,dvfs", dvfs, 2); + if (ret || !rep || !rep->offsets->update || dvfs[1] > 0x1f || + dvfs[0] < rep->offsets->dvfs_first || + dvfs[0] >= rep->offsets->dvfs_first + rep->offsets->dvfs_count) { + ret = dev_err_probe(dev, -EINVAL, "invalid apple,dvfs\n"); + goto err_free; + } + ent->dvfs = true; + ent->dvfs_entry = dvfs[0]; + ent->dvfs_on = PMP_DVFS_VALID | dvfs[1]; + } ret = of_property_read_string(node, "label", &name); if (ret < 0) { diff --git a/tools/asahi/t6030-pmp-graph-tests.c b/tools/asahi/t6030-pmp-graph-tests.c index 3d2493e57fb63..3746d2a3e3fdb 100644 --- a/tools/asahi/t6030-pmp-graph-tests.c +++ b/tools/asahi/t6030-pmp-graph-tests.c @@ -99,6 +99,7 @@ struct generic_pm_domain { }; struct apple_pmp_report_offsets { u32 tgt_read, tgt_write, actual, status, fast_die, fast_die_lanes; + u32 update, dvfs_first, dvfs_count; bool starts_pmp; const char *name, *fallback, *pwrstate; bool devices_ordered; @@ -195,8 +196,11 @@ struct apple_pmp_report_entry { struct generic_pm_domain genpd; struct device_node *node, *report_node; u32 id; - bool no_ack, removed; + bool no_ack, dvfs, removed; + u32 dvfs_entry; + u64 dvfs_on; }; +#define PMP_DVFS_VALID BIT_ULL(61) #define genpd_to_apple_pmp_report_entry(p) \ container_of(p, struct apple_pmp_report_entry, genpd) struct of_phandle_iterator { @@ -480,6 +484,13 @@ static int of_property_read_u32(const struct device_node *n, const char *p, *v = n->reg; return 0; } +/* No T6030 graph node carries an array property the probe reads. */ +static int of_property_read_u32_array(const struct device_node *n, + const char *p, u32 *v, size_t count) +{ + (void)n; (void)p; (void)v; (void)count; + return -EINVAL; +} static int of_property_count_u32_elems(const struct device_node *n, const char *p) { diff --git a/tools/asahi/t6030-pmp-graph-tests.py b/tools/asahi/t6030-pmp-graph-tests.py index bbe4dd99ea6c0..ea83b1c743925 100644 --- a/tools/asahi/t6030-pmp-graph-tests.py +++ b/tools/asahi/t6030-pmp-graph-tests.py @@ -41,7 +41,8 @@ def main(): 'static bool apple_pmp_start_stopping(', 'static void apple_pmp_start_work(', 'static void apple_pmp_start_drain(', 'static int apple_pmp_start_init(', 'static int apple_pmp_start_wait(', 'int apple_pmp_report_wait_ready(', - 'static int apple_pmp_report_set_state(', 'static int apple_pmp_report_entry_set_state(', + 'static int apple_pmp_report_set_state(', 'static void apple_pmp_report_dvfs_write(', + 'static int apple_pmp_report_entry_set_state(', 'static int apple_pmp_report_entry_power_on(', 'static int apple_pmp_report_entry_power_off(', 'static void apple_pmp_report_entry_free(', 'static int apple_pmp_report_entry_probe(', 'static int apple_pmp_report_probe(', 'static struct device_node *apple_pmp_report_t6030_entry(', From bd56bb8be467bda22d27e4b7059e6402166c57e9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 19:01:23 +1100 Subject: [PATCH 68/73] arm64: dts: apple: t8140: give the encoder its PMP performance request Add the VENC cluster enable bit to the PMC, a PMP report entry for soc-device 15 (AVE0) under VENC0_SYS and the cluster that requests VMAX in SOC-DEV-DVFS entry 0x198, and hand it to the encoder as its "perf" power domain, held while the encoder is powered. On a Neo one-shot (fleet14c, the same requests made by a test module) 1080p H.264 went from 110 to 165 fps (macOS 178), HEVC 101 to 153 and 4K H.264 30 to 47, with the PMP voting the AVE rail to state 2. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 32 ++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index cbdbf4ab8a36b..d25741151ea15 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -531,6 +531,20 @@ power-domains = <&ps_gfx>; apple,always-on; }; + + /* + * PMP soc-device 15 (AVE0): the encoder's power request + * and, in SOC-DEV-DVFS entry 0x198, its VMAX performance + * request. The PMP then votes the AVE rail up itself. + */ + pmp_report_ave0: report@f { + compatible = "apple,t8140-pmp-v2-report-entry"; + reg = <0xf>; + label = "pmp-ave0"; + #power-domain-cells = <0>; + power-domains = <&ps_venc0_sys>, <&pmc_venc_cluster>; + apple,dvfs = <0x198 2>; + }; }; pmp: pmp@300500000 { @@ -1043,8 +1057,10 @@ , ; power-domains = <&ps_venc0_dma>, <&ps_venc0_pipe4>, - <&ps_venc0_pipe5>, <&ps_venc0_me0>, <&ps_venc0_me1>; - power-domain-names = "dma", "pipe4", "pipe5", "me0", "me1"; + <&ps_venc0_pipe5>, <&ps_venc0_me0>, <&ps_venc0_me1>, + <&pmp_report_ave0>; + power-domain-names = "dma", "pipe4", "pipe5", "me0", "me1", + "perf"; iommus = <&ave_dart0 0>, <&ave_dart1 0>; status = "disabled"; }; @@ -1264,8 +1280,9 @@ pmc: power-management@302800000 { compatible = "apple,t8140-pmc"; reg = <0x3 0x02800000 0x0 0x10000>, - <0x3 0x028aa000 0x0 0x3000>; - reg-names = "voters", "rails"; + <0x3 0x028aa000 0x0 0x3000>, + <0x3 0x028a9c18 0x0 0x8>; + reg-names = "voters", "rails", "clusters"; power-domains = <&ps_pmc>; #address-cells = <1>; #size-cells = <0>; @@ -1274,6 +1291,13 @@ apple,static-voters = <0 1 3>; status = "disabled"; + /* VENC-CLSTR-V: the video encoder cluster, bit 3. */ + pmc_venc_cluster: cluster-venc { + label = "venc-cluster"; + #power-domain-cells = <0>; + apple,cluster-bit = <3>; + }; + clpc: performance-controller@1 { compatible = "apple,t8140-clpc"; reg = <1>; From a0b92325abf41664b0a357a71570e663c96855da Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 21:28:55 +1100 Subject: [PATCH 69/73] drm/apple: Return HDR outputs to SDR when the compositor exits With GNOME in HDR on the J616s panel, the shutdown splash came out far too bright. Connector properties outlive the client that set them, so Plymouth inherited HDR_OUTPUT_METADATA with the PQ EOTF and the BT.2020 colorspace, and its SDR frames were shown as PQ near the panel's peak. When a file that was DRM master closes and no other client has taken master, take every output it left in HDR back to SDR with the same commit an HDR-off from the compositor would make: no HDR metadata and the default colorspace. A compositor that only drops master for a VT switch keeps its state, since it does not send it again when it comes back. On the J616s panel at brightness 300, SDR white after a PQ client exits now draws 4.4 W (SDR white 6.0 W, the PQ client 14.9 W). Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 76 ++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 75e52d2454385..8991a85e5326a 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -149,6 +149,70 @@ static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, } #endif +/* + * HDR is asked for by the compositor that set it, but connector properties + * outlive their client. When it exits, the next client (Plymouth at + * shutdown, the fbdev console) would inherit HDR_OUTPUT_METADATA and BT.2020 + * and have its SDR frames shown as PQ, at the panel's peak. So when a + * former master closes and no client has taken over, take its outputs back + * to SDR the way an HDR-off commit from it would. A master that only drops + * master for a VT switch keeps its state: it comes back without resending + * it. + */ +static void apple_drm_reset_hdr(struct drm_device *dev) +{ + struct drm_modeset_acquire_ctx ctx; + struct drm_connector_list_iter iter; + struct drm_atomic_commit *state; + struct drm_connector *conn; + bool changed; + int ret; + + drm_modeset_acquire_init(&ctx, 0); + state = drm_atomic_commit_alloc(dev); + if (!state) + goto out_ctx; + state->acquire_ctx = &ctx; +retry: + changed = false; + ret = drm_modeset_lock(&dev->mode_config.connection_mutex, &ctx); + if (ret) + goto out; + drm_connector_list_iter_begin(dev, &iter); + drm_for_each_connector_iter(conn, &iter) { + struct drm_connector_state *cs = conn->state; + + if (!cs->crtc || (!cs->hdr_output_metadata && + cs->colorspace == DRM_MODE_COLORIMETRY_DEFAULT)) + continue; + cs = drm_atomic_get_connector_state(state, conn); + if (IS_ERR(cs)) { + ret = PTR_ERR(cs); + break; + } + drm_property_blob_put(cs->hdr_output_metadata); + cs->hdr_output_metadata = NULL; + cs->colorspace = DRM_MODE_COLORIMETRY_DEFAULT; + changed = true; + } + drm_connector_list_iter_end(&iter); + if (!ret && changed) + ret = drm_atomic_commit(state); +out: + if (ret == -EDEADLK) { + drm_atomic_commit_clear(state); + drm_modeset_backoff(&ctx); + goto retry; + } + if (ret) + drm_warn(dev, "HDR output state kept after the DRM master left (%d)\n", + ret); + drm_atomic_commit_put(state); +out_ctx: + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); +} + /* * A compositor or boot splash is closing the device. drm_release() has * already taken its file off the file list, so the Type-C fabric can tell @@ -156,8 +220,16 @@ static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, */ static void apple_drm_postclose(struct drm_device *drm, struct drm_file *file) { - if (file->was_master) - dcp_typec_reorder(); + bool unowned; + + if (!file->was_master) + return; + dcp_typec_reorder(); + mutex_lock(&drm->master_mutex); + unowned = !drm->master; + mutex_unlock(&drm->master_mutex); + if (unowned) + apple_drm_reset_hdr(drm); } static const struct drm_driver apple_drm_driver = { From 79fc99269ef4c2e1b84229826031bd4f343573db Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 18:38:15 +1100 Subject: [PATCH 70/73] drm/apple: Report the PQ reference white of desktop 26.x panels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A desktop 26.x panel shows PQ as EDR: PQ 100 cd/m² lands on the white the backlight sets, and brighter levels use the headroom above it, up to the panel's peak. The backlight stays in effect while it decodes PQ. macOS uses the panel the same way. Compositors can't tell this from the panel EDID or any existing connector property. They take PQ as absolute, with the reference white at 203 cd/m², and mutter also drives the backlight to its maximum in BT.2100 mode because it can't tell whether the backlight still works. On this panel that made the HDR desktop about twice as bright as SDR, or more. Attach an immutable PQ_REFERENCE_LUMINANCE range property, in cd/m², to the panel connector of a desktop 26.x DCP. Its presence says the backlight still applies in PQ, and its value is the PQ level that shows at the white the backlight sets. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 8991a85e5326a..98534a725e839 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -672,6 +672,29 @@ static const struct drm_connector_helper_funcs apple_connector_helper_funcs = { .atomic_check = apple_connector_atomic_check, }; +/* + * A desktop 26.x panel shows PQ as EDR (see dcp_edr_prepare()): PQ 100 cd/m² + * lands on the white the backlight sets, and brighter levels use the + * headroom above it, up to the panel's peak. macOS uses the panel the same + * way. PQ_REFERENCE_LUMINANCE tells a compositor where that white is in PQ, + * so it can put its reference white there and keep using the backlight as + * the brightness control in PQ. + */ +#define DCP_EDR_PQ_REFERENCE_NITS 100 + +static int apple_connector_attach_pq_reference(struct drm_connector *connector) +{ + struct drm_property *prop; + + prop = drm_property_create_range(connector->dev, DRM_MODE_PROP_IMMUTABLE, + "PQ_REFERENCE_LUMINANCE", 0, 10000); + if (!prop) + return -ENOMEM; + drm_object_attach_property(&connector->base, prop, + DCP_EDR_PQ_REFERENCE_NITS); + return 0; +} + /* * External displays can take a PQ BT.2020 signal where the firmware * offers one, and a desktop 26.x panel PQ content it shows as EDR; @@ -705,6 +728,8 @@ static int apple_connector_attach_hdr(struct drm_connector *connector, if (ret) return ret; drm_connector_attach_hdr_output_metadata_property(connector); + if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) + return apple_connector_attach_pq_reference(connector); return 0; } From 7ab63b181b562be90d7633fdb744915d7f0c90e2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 19:12:58 +1100 Subject: [PATCH 71/73] drm/apple: Stop the HDR InfoFrame before an external link leaves PQ When HDR ends on a live desktop 26.x external link, the driver only set the SDR colour mode and relied on the link to drop the Dynamic Range and Mastering InfoFrame. macOS first stops that InfoFrame with DCPAV selector 0x14 (stopInfoFrame, type 0x87) on dcpav-service-epic, requires the stop to succeed, and only then sets the SDR colour mode. It never sends an empty metadata block. Do the same. Track whether the firmware took a metadata block, so that the link carries the InfoFrame. When a mode set leaves PQ on such a link, send the 0x14 stop first. The stop message is a fixed 0x80 bytes: the DCPAV header, a 32-byte InfoFrame at 0x40 whose type byte (0x87) selects the slot to clear, the link source at 0x60 and the DCP status at 0x70. A failed stop is reported, and the mode set goes ahead. A link set up again (power off, new service) carries no InfoFrame, so nothing is stopped there. The host selftest checks the stop message byte for byte against an independent encoding, and checks its reply status offset. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-hdr-metadata.h | 49 ++++++++++++++++-- drivers/gpu/drm/apple/dcp-internal.h | 7 ++- drivers/gpu/drm/apple/dcp.c | 2 + drivers/gpu/drm/apple/iomfb.c | 50 +++++++++++++++++-- .../drivers/gpu/apple-dcp/display.py | 26 +++++++++- 5 files changed, 123 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-hdr-metadata.h b/drivers/gpu/drm/apple/dcp-hdr-metadata.h index f7590c32b13c8..15937ac47fe9a 100644 --- a/drivers/gpu/drm/apple/dcp-hdr-metadata.h +++ b/drivers/gpu/drm/apple/dcp-hdr-metadata.h @@ -10,6 +10,13 @@ * CTA-861.3 Type 1 body in the order of struct hdr_metadata_infoframe from * display_primaries to max_fall, little-endian. The EOTF is not in it: the * firmware takes it from the colour mode the link runs. + * + * Ending HDR on a live link, the host first clears that InfoFrame with + * selector 0x14 (stopInfoFrame), then sets the SDR colour mode; it never + * sends an empty metadata block. The stop message has a fixed 0x80 bytes: + * the usual header, a 32-byte InfoFrame at 0x40 whose type byte selects + * the slot to clear (0x87, Dynamic Range and Mastering), the link source + * at 0x60 (0) and the DCP status at 0x70. */ #ifndef __APPLE_DCP_HDR_METADATA_H__ #define __APPLE_DCP_HDR_METADATA_H__ @@ -20,6 +27,7 @@ #include #include +#define DCP_HDR_SELECTOR_STOP_INFOFRAME 0x14 #define DCP_HDR_SELECTOR_SET_STATIC 0x15 #define DCP_HDR_IPC_MAGIC 0x69706378 /* "xcpi" */ #define DCP_HDR_DATA_SIZE 24 @@ -49,6 +57,11 @@ #define DCP_HDR_MSG_STATUS 0x40 #define DCP_HDR_MSG_SIZE (DCP_HDR_MSG_HEADER + DCP_HDR_OBJECT_SIZE) +#define DCP_HDR_STOP_SIZE 0x80 +#define DCP_HDR_STOP_INFOFRAME 0x40 +#define DCP_HDR_STOP_STATUS 0x70 +#define DCP_HDR_INFOFRAME_DRM 0x87 + /* The 24-byte Type 1 body of @hdr; -EINVAL unless it is a Type 1 block. */ static inline int dcp_hdr_data(u8 data[DCP_HDR_DATA_SIZE], const struct hdr_output_metadata *hdr) @@ -97,17 +110,43 @@ static inline void dcp_hdr_message(u8 msg[DCP_HDR_MSG_SIZE], memcpy(obj + DCP_HDR_OFF_DATA, data, DCP_HDR_DATA_SIZE); } -/* The result of a selector 0x15 command: transport, AFK, then DCP status. */ -static inline int dcp_hdr_reply(int transport, u32 retcode, const u8 *reply, - size_t length) +/* The selector 0x14 message clearing the Dynamic Range and Mastering InfoFrame. */ +static inline void dcp_hdr_stop_message(u8 msg[DCP_HDR_STOP_SIZE]) +{ + memset(msg, 0, DCP_HDR_STOP_SIZE); + put_unaligned_le32(DCP_HDR_SELECTOR_STOP_INFOFRAME, msg + 0x04); + put_unaligned_le32(DCP_HDR_STOP_SIZE - 0x40, msg + 0x08); + put_unaligned_le32(DCP_HDR_IPC_MAGIC, msg + 0x0c); + msg[DCP_HDR_STOP_INFOFRAME] = DCP_HDR_INFOFRAME_DRM; +} + +/* The result of a DCPAV command: transport, AFK, then DCP status at @status. */ +static inline int dcp_hdr_result(int transport, u32 retcode, const u8 *reply, + size_t length, size_t status) { if (transport) return transport; if (retcode) return -EIO; - if (length < DCP_HDR_MSG_STATUS + 4) + if (length < status + 4) return -EPROTO; - return get_unaligned_le32(reply + DCP_HDR_MSG_STATUS) ? -EIO : 0; + return get_unaligned_le32(reply + status) ? -EIO : 0; +} + +/* The result of a selector 0x15 command. */ +static inline int dcp_hdr_reply(int transport, u32 retcode, const u8 *reply, + size_t length) +{ + return dcp_hdr_result(transport, retcode, reply, length, + DCP_HDR_MSG_STATUS); +} + +/* The result of a selector 0x14 command. */ +static inline int dcp_hdr_stop_reply(int transport, u32 retcode, + const u8 *reply, size_t length) +{ + return dcp_hdr_result(transport, retcode, reply, length, + DCP_HDR_STOP_STATUS); } #endif /* __APPLE_DCP_HDR_METADATA_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 105ec1d3b1d4f..38495cee3ed73 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -280,13 +280,16 @@ struct dcp_desktop_state { struct iomfb_set_matrix_req matrix; /* * HDR10 static metadata (external processors): the pinned - * dcpav-service-epic service, and the Type 1 body last sent on this - * link (taken or refused), if any. hdr_lock protects both. + * dcpav-service-epic service, the Type 1 body last sent on this link + * (taken or refused), if any, and whether the firmware took one, so + * that the link carries a Dynamic Range and Mastering InfoFrame. + * hdr_lock protects all of them. */ spinlock_t hdr_lock; struct apple_epic_service *av_service; u8 hdr_sent[24]; bool hdr_sent_valid; + bool hdr_infoframe; u32 present_id; bool present_pending; /* diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 97e0ca217ad21..72847642ab40a 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3315,6 +3315,7 @@ static void dcp_av_service_init(struct apple_epic_service *service, old = d->av_service; d->av_service = service; d->hdr_sent_valid = false; + d->hdr_infoframe = false; } if (old) afk_service_put(old); @@ -3330,6 +3331,7 @@ static void dcp_av_service_teardown(struct apple_epic_service *service) if (owned) { d->av_service = NULL; d->hdr_sent_valid = false; + d->hdr_infoframe = false; } } if (owned) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 296b8b6b20387..066b8ab182165 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -2392,8 +2392,48 @@ void dcp_hdr_metadata_forget(struct apple_dcp *dcp) if (!d) return; - scoped_guard(spinlock_irqsave, &d->hdr_lock) + scoped_guard(spinlock_irqsave, &d->hdr_lock) { d->hdr_sent_valid = false; + d->hdr_infoframe = false; + } +} + +/* + * Clear the Dynamic Range and Mastering InfoFrame of a live link before it + * leaves PQ, as macOS does: DCPAV selector 0x14 first, then the SDR colour + * mode. Setting the SDR colour mode alone would leave the sink an HDR + * InfoFrame for an SDR signal. + */ +static void dcp_hdr_infoframe_stop(struct apple_dcp *dcp) +{ + struct dcp_desktop_state *d = dcp->desktop; + struct apple_epic_service *service = NULL; + u8 msg[DCP_HDR_STOP_SIZE], reply[DCP_HDR_STOP_SIZE] = {}; + size_t received = 0; + u32 retcode = 0; + int ret; + + if (!d || READ_ONCE(dcp->crashed)) + return; + scoped_guard(spinlock_irqsave, &d->hdr_lock) + if (d->hdr_infoframe) + service = afk_service_get(d->av_service); + if (!service) + return; + + dcp_hdr_stop_message(msg); + ret = afk_send_command_with_reply_len(service, EPIC_SUBTYPE_STD_SERVICE, + msg, sizeof(msg), reply, sizeof(reply), + &retcode, &received); + ret = dcp_hdr_stop_reply(ret, retcode, reply, received); + if (ret) + dev_warn(dcp->dev, "HDR InfoFrame not stopped: %d\n", ret); + else + dev_dbg(dcp->dev, "HDR InfoFrame stopped\n"); + scoped_guard(spinlock_irqsave, &d->hdr_lock) + if (d->av_service == service) + d->hdr_infoframe = false; + afk_service_put(service); } /* Whether @pdev takes a change of HDR_OUTPUT_METADATA without a mode set. */ @@ -2410,8 +2450,8 @@ bool dcp_hdr_metadata_live(struct platform_device *pdev) * link carries PQ: after the mode set that selected a PQ colour mode, and * whenever the metadata changes on it. Never per frame, and never on an * SDR link (the firmware would build a DRM InfoFrame with the SDR EOTF). - * Going back to SDR is a mode set, and the link set up again carries no - * InfoFrame. + * Going back to SDR is a mode set that first stops the InfoFrame + * (dcp_hdr_infoframe_stop()). */ static void dcp_hdr_metadata_sync(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_commit *commit) @@ -2473,6 +2513,8 @@ static void dcp_hdr_metadata_sync(struct apple_dcp *dcp, struct drm_crtc *crtc, if (d->av_service == service) { memcpy(d->hdr_sent, data, sizeof(data)); d->hdr_sent_valid = true; + if (!ret) + d->hdr_infoframe = true; } } afk_service_put(service); @@ -2644,6 +2686,8 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, dcp->pq_requested &= dcp_hdr_output_supported(dcp); if (dcp->desktop) dcp->desktop->vrr_requested = crtc_state->vrr_enabled; + if (!dcp->pq_requested) + dcp_hdr_infoframe_stop(dcp); dcp_hdr_metadata_forget(dcp); dcp_mode_begin(&dcp->mode_state); switch (dcp->fw_compat) { diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/display.py b/tools/testing/selftests/drivers/gpu/apple-dcp/display.py index e8e355955052f..256c370e22f96 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/display.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/display.py @@ -409,6 +409,11 @@ def pad(raw): struct.pack_into(' #include @@ -437,6 +442,7 @@ def pad(raw): }; """ + header + r""" static const u8 expected[] = {""" + expected + r"""}; +static const u8 expected_stop[] = {""" + expected_stop + r"""}; int main(void) { struct hdr_output_metadata hdr = { .metadata_type = 0, .hdmi_metadata_type1 = { @@ -467,7 +473,25 @@ def pad(raw): assert(dcp_hdr_reply(0, 0, reply, 0x44) == 0); put_unaligned_le32(0xe00002c2, reply + 0x40); assert(dcp_hdr_reply(0, 0, reply, sizeof(reply)) == -EIO); - printf("PASS: HDR10 static metadata message (%zu bytes) and reply checks\n", sizeof(msg)); + /* Ending HDR: the selector 0x14 stop message, DCP status at 0x70. */ + { + u8 stop[DCP_HDR_STOP_SIZE], sreply[DCP_HDR_STOP_SIZE] = {0}; + + assert(sizeof(expected_stop) == DCP_HDR_STOP_SIZE); + memset(stop, 0xa5, sizeof(stop)); + dcp_hdr_stop_message(stop); + assert(!memcmp(stop, expected_stop, sizeof(stop))); + assert(dcp_hdr_stop_reply(-EIO, 0, sreply, sizeof(sreply)) == -EIO); + assert(dcp_hdr_stop_reply(0, 3, sreply, sizeof(sreply)) == -EIO); + assert(dcp_hdr_stop_reply(0, 0, sreply, 0x73) == -EPROTO); + assert(dcp_hdr_stop_reply(0, 0, sreply, 0x74) == 0); + /* the 0x15 status offset does not count for a stop */ + put_unaligned_le32(1, sreply + 0x40); + assert(dcp_hdr_stop_reply(0, 0, sreply, sizeof(sreply)) == 0); + put_unaligned_le32(1, sreply + 0x70); + assert(dcp_hdr_stop_reply(0, 0, sreply, sizeof(sreply)) == -EIO); + } + printf("PASS: HDR10 static metadata message (%zu bytes), InfoFrame stop and reply checks\n", sizeof(msg)); return 0; } """ From 041ba02beb1099c0abb00f53e6d7fa8cbb62a1c5 Mon Sep 17 00:00:00 2001 From: Ryan Murray <81541646+RyanTheTide@users.noreply.github.com> Date: Sun, 1 Nov 2026 18:58:36 +1100 Subject: [PATCH 72/73] media: apple: avd: dispatch queued pictures over firmware codec pools Port multi-VP selection onto the current queued transport without losing its SRAM word stores, IRQ/mailbox fixes, saved frame FIFO or tagged VP DONE. Negotiate an optional AVM2 codec pool record and rotate submissions through its VP masks. Old firmware retains the k4m selector and two picture banks. Size H.264 and HEVC banks for the advertised VPs plus ordered PP ownership, limited by the available FIFOs. Their VP work uses private slice state; firmware still starts PP in submission order. Keep VP9 probability producer barriers and AV1's single-picture path intact. Exercise negotiation, fallback, rotation/retry, production picture banks and routed single-stream programs, including reverse VP completion order. Document the inferred T6040 pool and the remaining silicon qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../platform/apple/avd/avd-backend-queue.c | 83 ++++++++++++++ drivers/media/platform/apple/avd/avd-h264.c | 12 +-- drivers/media/platform/apple/avd/avd-hevc.c | 4 +- .../media/platform/apple/avd/avd-protocol.rst | 49 +++++++++ drivers/media/platform/apple/avd/avd.h | 1 + .../platform/apple/avd/tests/program-test.py | 102 ++++++++++++++++-- .../apple/avd/tests/unicorn-inflight-test.py | 73 +++++++++++-- 7 files changed, 299 insertions(+), 25 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c index 7d21921d17229..5853d9d050a93 100644 --- a/drivers/media/platform/apple/avd/avd-backend-queue.c +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -32,6 +32,11 @@ #define AVDQ_FEAT_SRAM_RING BIT(0) #define AVDQ_FEAT_MULTI_KICK BIT(1) #define AVDQ_FEAT_VP_DONE BIT(2) +#define AVDQ_FEAT_MULTI_VP BIT(3) +#define AVDQ_POOL_OFFSET 0x7f80 +#define AVDQ_POOL_MAGIC 0x41564d32 /* AVM2 */ +#define AVDQ_POOL_VERSION 1 +#define AVDQ_MAX_VPS 12 #define AVDQ_FLAG_VP_DONE BIT(1) #define AVDQ_VP_NOTIFY 0x54540000 #define AVDQ_FLAG_SRAM BIT(0) @@ -64,10 +69,63 @@ struct avd_qfw { u32 wp; /* host write count, free-running */ u32 fifo_slots; u32 codec_mask; /* info+0x24, bit n = codec n */ + u32 vp_pool[4]; /* firmware masks of hardware VP indices */ + u32 next_vp[4]; /* per-codec rotation; queue_mutex owns submission */ + bool multi_vp; bool vp_done; bool usable; /* last boot found a matching ABI 2 image */ }; +static bool avd_qfw_pool_valid(const u32 *pool, u32 limit, u32 codecs) +{ + u32 used = 0; + unsigned int codec; + + if (pool[0] != AVDQ_POOL_MAGIC || pool[1] != AVDQ_POOL_VERSION || + !limit || limit > AVDQ_MAX_VPS || pool[7] || (pool[6] & ~codecs)) + return false; + for (codec = 0; codec < 4; codec++) { + u32 mask = pool[2 + codec]; + + if (!!mask != !!(codecs & BIT(codec)) || + mask >> limit || (mask & used)) + return false; + used |= mask; + } + return true; +} + +static unsigned int avd_qfw_pick_vp(struct avd_qfw *q, u32 codec, u32 fallback) +{ + unsigned int n; + + if (!q->multi_vp) + return fallback; + for (n = 0; n < AVDQ_MAX_VPS; n++) { + unsigned int vp = (q->next_vp[codec] + n) % AVDQ_MAX_VPS; + + if (q->vp_pool[codec] & BIT(vp)) { + q->next_vp[codec] = (vp + 1) % AVDQ_MAX_VPS; + return vp; + } + } + return fallback; +} + +static unsigned int avd_qfw_bank_limit(struct avd_qfw *q, unsigned int codec) +{ + unsigned int banks = 1; + u32 pool; + + /* k4m keeps two banks with old queue firmware. */ + if (!q->multi_vp || codec > AVD_CODEC_H264) + return 2; + /* One bank per VP plus one retained by PP. No SoC counts in the host. */ + for (pool = q->vp_pool[codec]; pool; pool &= pool - 1) + banks++; + return min_t(unsigned int, banks, q->fifo_slots); +} + static void __iomem *avd_qfw_desc(struct avd_qfw *q, unsigned int idx) { return q->sram + AVDQ_RING_OFFSET + idx * AVDQ_DESC_BYTES; @@ -127,6 +185,11 @@ static bool avd_qfw_check(struct avd_dev *avd) info[i] = readl(q->sram + AVDQ_INFO + 4 * i); q->usable = false; q->vp_done = false; + q->multi_vp = false; + for (i = 0; i < 4; i++) { + q->vp_pool[i] = 0; + q->next_vp[i] = 0; + } if (info[0] != AVDQ_MAGIC) return false; if (info[1] != AVDQ_VERSION || info[2] != AVDQ_COUNT || @@ -153,6 +216,19 @@ static bool avd_qfw_check(struct avd_dev *avd) q->ring_words = info[13]; q->fifo_slots = min_t(u32, info[10], avd->variant->fifo_slots ?: 1); q->codec_mask = info[9]; + if (info[11] & AVDQ_FEAT_MULTI_VP) { + u32 pool[8]; + + for (i = 0; i < ARRAY_SIZE(pool); i++) + pool[i] = readl(q->sram + AVDQ_POOL_OFFSET + 4 * i); + if (avd_qfw_pool_valid(pool, info[8], info[9])) { + for (i = 0; i < 4; i++) + q->vp_pool[i] = pool[2 + i]; + q->multi_vp = true; + } else { + dev_warn_once(avd->dev, "invalid firmware VP pool, using one VP per codec\n"); + } + } q->wp = 0; q->usable = true; dev_dbg(avd->dev, "queue ABI %u: aperture %#x ring %#x/%u words FIFO slots %u codecs %#x\n", @@ -258,6 +334,8 @@ static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) vp = avd->variant->vp_slot_base[codec]; if (extra > AVDQ_MAX_PP_EXTRA) return -E2BIG; + /* Advance only after all retry/error exits; prepared programs are VP-neutral. */ + vp = avd_qfw_pick_vp(q, codec, vp); mask = codec == AVD_CODEC_VP9 && avd->variant->revision == 3 ? AVD_OP_EXEC_REV3_VP9_MASK : 0; @@ -384,6 +462,11 @@ bool avd_qfw_has_vp_done(const struct avd_dev *avd) return avd_backend_is_qfw(avd) && avd->qfw->vp_done; } +unsigned int avd_qfw_picture_banks(const struct avd_dev *avd, unsigned int codec) +{ + return avd_backend_is_qfw(avd) ? avd_qfw_bank_limit(avd->qfw, codec) : 1; +} + /* * SRAM survives a CM3 reload, so a stale info block from an earlier ABI 2 * image could make a legacy image look queue capable. Clear the magic before diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index 41d8fc12d8d4a..fd1d4d53025ae 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -59,12 +59,12 @@ struct avd_h264_run { }; /* - * One bank per picture in flight. With the queued transport two pictures of - * a stream overlap (VP of one, PP of the other), so everything the hardware - * reads or writes while decoding a picture, and the CPU copies of its slices, - * exists once per bank. The legacy transport uses a single bank. + * One bank per picture in flight. The queued transport retains a bank for PP + * while the advertised VP pool processes later pictures. Everything the + * hardware reads or writes for a picture, and the CPU copies of its slices, + * exists once per bank. Old queue firmware keeps two; legacy uses one. */ -#define AVD_H264_BANKS 2 +#define AVD_H264_BANKS AVD_MAX_FRAMES struct avd_bank { bool monochrome; @@ -727,7 +727,7 @@ static int avd_h264_start(struct avd_ctx *ctx) ctx->priv = h264_ctx; /* Overlap pictures only where the firmware chains VP to PP itself. */ - h264_ctx->nbanks = avd_backend_is_qfw(ctx->dev) ? AVD_H264_BANKS : 1; + h264_ctx->nbanks = avd_qfw_picture_banks(ctx->dev, AVD_CODEC_H264); h264_ctx->b = &h264_ctx->bank[0]; for (unsigned int i = 0; i < h264_ctx->nbanks; i++) { diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index 53652a01c0efd..d444865379e92 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -80,7 +80,7 @@ struct avd_hevc_run { }; /* See avd-h264.c: one bank per picture in flight on the queued transport. */ -#define AVD_HEVC_BANKS 2 +#define AVD_HEVC_BANKS AVD_MAX_FRAMES struct avd_hevc_bank { bool busy; @@ -1129,7 +1129,7 @@ static int avd_hevc_start(struct avd_ctx *ctx) ctx->priv = hevc_ctx; /* Overlap pictures only where the firmware chains VP to PP itself. */ - hevc_ctx->nbanks = avd_backend_is_qfw(ctx->dev) ? AVD_HEVC_BANKS : 1; + hevc_ctx->nbanks = avd_qfw_picture_banks(ctx->dev, AVD_CODEC_HEVC); hevc_ctx->b = &hevc_ctx->bank[0]; for (i = 0; i < hevc_ctx->nbanks; i++) { ret = avd_hevc_alloc_bufs(ctx, &hevc_ctx->bank[i]); diff --git a/drivers/media/platform/apple/avd/avd-protocol.rst b/drivers/media/platform/apple/avd/avd-protocol.rst index ed75e333dd739..138b5d81cdafd 100644 --- a/drivers/media/platform/apple/avd/avd-protocol.rst +++ b/drivers/media/platform/apple/avd/avd-protocol.rst @@ -98,3 +98,52 @@ Licensed implementation references: eiln/avd (MIT), avd_emu.py, and https://github.com/aurora-silicon/aura/blob/main/proxyclient/hv/trace_avd.py (MIT). The Borage facts above come from the supplied clean SPEC sections 3-5; no binary or captured Apple command images are used. + +Firmware-advertised multi-VP pools +--------------------------------- + +The queued open transport uses the optional ABI 2 feature bit 3 and the AVM2 +pool record at SRAM +0x7f80. Its eight u32 words are magic 0x41564d32, version 1, +four codec VP masks (HEVC, H.264, VP9, AV1), an inferred-count codec mask, and a +zero reserved word. Masks must be disjoint, within the firmware VP register +limit and consistent with its codec mask. An absent or malformed extension +keeps the k4m single-VP selector and two H.264/HEVC banks. Reload clears all +cached masks and rotation state before negotiation, so an older image cannot +inherit a prior multi-VP capability. + +For a valid pool the host rotates successive submissions through its hardware +VP indices. The prepared program names its private FIFO, not a VP; retrying a +busy mailbox cannot consume a rotation turn or alter the saved program/FIFO. +Separate streams share that rotation, so they also use distinct engines. +Firmware retains exclusive VP picture ownership through DONE and feeds the +SRAM programs in order. PP starts strictly in descriptor submission order, +even when VP completion order differs. Within each context, the host preserves +submission and retirement order. + +H.264 and HEVC allocate one scratch bank per advertised VP plus a bank that PP +may still own, bounded by the physical FIFO count. The VP work of the next +picture reads its own bitstream and slice state; reference reconstruction is +consumed by the ordered PP stage. Buffers, instruction FIFOs and scratch banks +remain owned through PP completion, not merely VP DONE. No reference wait or +probability dependency is removed by choosing another engine. + +VP9 retains its two banks and producer-side backward-adaptation barrier from +the VP-DONE implementation. Its tagged VP callback remains separate from full +PP retirement. AV1 retains its single-picture CDF/segmentation dependency path. +The advertised pools may select different engines across independent contexts; +neither codec's dependency rules are relaxed. + +Pool counts live in the firmware image, not an additional host SoC table. +The avd-fw v5-t2 T6040 counts (HEVC 4, H.264 4, VP9 1, AV1 3) are inferred from +T8132 and marked as inferred by the image. The v2/v3/v4/v5-t0/v5-t1 pools use +the public codec variant counts. The T6020 v3-t2 image reuses v3-t1 tunables; +it and all multi-engine silicon behavior still require device qualification. + +The host program fixture checks negotiation, invalid/overlapping/absent pool +fallback, reset, rotation, bank limits, saved FIFO retry and unchanged legacy +programs. It emits both ordinary and multi-VP programs. Run the routed test +with ``--multi`` to verify the production H.264 submissions use all advertised +engines, retain each bank through PP, handle reversed VP DONE, and keep PP and +retirement in order. The firmware transport test covers every advertised codec +and engine and tagged VP notifications on every image. These event models do +not decode pixels or establish throughput. diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 2da75d833b75a..a1825a1029ec5 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -411,6 +411,7 @@ void avd_qfw_prepare_boot(struct avd_dev *avd); void avd_qfw_debug(struct avd_dev *avd, struct avd_frame *f, const char *where); bool avd_backend_is_qfw(const struct avd_dev *avd); bool avd_qfw_has_vp_done(const struct avd_dev *avd); +unsigned int avd_qfw_picture_banks(const struct avd_dev *avd, unsigned int codec); int avd_backend_select(struct avd_dev *avd); bool avd_mbox_pop(struct avd_dev *avd, u32 *msg); int avd_backend_boot(struct avd_dev *avd); diff --git a/drivers/media/platform/apple/avd/tests/program-test.py b/drivers/media/platform/apple/avd/tests/program-test.py index 1d6ea38f3a83e..74826bebd4256 100755 --- a/drivers/media/platform/apple/avd/tests/program-test.py +++ b/drivers/media/platform/apple/avd/tests/program-test.py @@ -47,6 +47,7 @@ #define VISIBLE_IF_KUNIT #define AVD_QUIRK_LSR 1 #define AVD_QUIRK_NO_PIPE_STATE 2 +#define AVD_CODEC_HEVC 0 #define AVD_CODEC_H264 1 #define AVD_CODEC_VP9 2 #define AVD_MAX_INST 512 @@ -172,7 +173,35 @@ u32 info[16]={AVDQ_MAGIC,2,16,64,0x1090000,0x8000,256,0,9,7,15,3,0,0x1800,0,0}; memcpy(q.sram+AVDQ_INFO,info,sizeof(info));assert(avd_qfw_check(&dev)&&!q.vp_done); info[11]=7;memcpy(q.sram+AVDQ_INFO,info,sizeof(info));assert(avd_qfw_check(&dev)&&q.vp_done); - info[1]=3;memcpy(q.sram+AVDQ_INFO,info,sizeof(info));assert(!avd_qfw_check(&dev)&&!q.vp_done); + + u32 pool[8]={AVDQ_POOL_MAGIC,AVDQ_POOL_VERSION,15,240,256,0,0,0}; + info[11]=15;memcpy(q.sram+AVDQ_INFO,info,sizeof(info)); + memcpy(q.sram+AVDQ_POOL_OFFSET,pool,sizeof(pool)); + assert(avd_qfw_check(&dev)&&q.multi_vp&&q.vp_done); + assert(avd_qfw_bank_limit(&q,AVD_CODEC_HEVC)==5&&avd_qfw_bank_limit(&q,AVD_CODEC_H264)==5); + for(unsigned int n=0;n<8;n++) { + assert(avd_qfw_pick_vp(&q,0,99)==n%4); + assert(avd_qfw_pick_vp(&q,1,99)==4+n%4); + assert(avd_qfw_pick_vp(&q,2,99)==8); + } + for(unsigned int n=0;n<8;n++) { + u32 bad[8];memcpy(bad,pool,sizeof(bad)); + switch(n) { + case 0:bad[0]=0;break;case 1:bad[1]++;break; + case 2:bad[2]=0;break;case 3:bad[3]|=1;break; + case 4:bad[3]|=1<<12;break;case 5:bad[5]=1<<9;break; + case 6:bad[6]=8;break;case 7:bad[7]=1;break; + } + memcpy(q.sram+AVDQ_POOL_OFFSET,bad,sizeof(bad)); + assert(avd_qfw_check(&dev)&&!q.multi_vp&&q.vp_done); + assert(avd_qfw_pick_vp(&q,1,4)==4&&avd_qfw_bank_limit(&q,1)==2); + } + memcpy(q.sram+AVDQ_POOL_OFFSET,pool,sizeof(pool)); + assert(avd_qfw_check(&dev)&&q.multi_vp); + info[11]=7;memcpy(q.sram+AVDQ_INFO,info,sizeof(info)); + assert(avd_qfw_check(&dev)&&!q.multi_vp&&q.vp_done); + assert(avd_qfw_pick_vp(&q,1,4)==4&&avd_qfw_bank_limit(&q,1)==2); + info[1]=3;memcpy(q.sram+AVDQ_INFO,info,sizeof(info));assert(!avd_qfw_check(&dev)&&!q.vp_done&&!q.multi_vp); memset(q.sram,0,0x10000); struct avd_buf slice = { .size=4096, .addr=0x200800000ULL }; struct v4l2_ctrl_h264_sps sps = { .pic_width_in_mbs_minus1=119, @@ -194,11 +223,14 @@ if (!strcmp(argv[2],"v5-t0")) { variant.fifo_slots=7;variant.vp_slot_base[1]=4; } } else if (!strcmp(argv[2],"v3-t0")) variant.quirks|=AVD_QUIRK_NO_PIPE_STATE; q.fifo_slots=variant.fifo_slots; + bool multi=getenv("AVD_MULTI_VP")!=NULL; + q.multi_vp=multi;q.vp_pool[0]=15;q.vp_pool[1]=!strcmp(argv[2],"v5-t0")?16:240; + unsigned int test_banks=multi?avd_qfw_bank_limit(&q,AVD_CODEC_H264):2; ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; run.base.bufs.dst=&dst.base.vb; for(unsigned int c=0;c<8;c++) { - ctx.priv=&h264[c];h264[c].nbanks=AVD_H264_BANKS;h264[c].b=&h264[c].bank[0]; - for(unsigned int b=0;bb==&hc->bank[idx%AVD_H264_BANKS]);hc->b->active_slice=&slice; + assert(hc->b==&hc->bank[idx%test_banks]);hc->b->active_slice=&slice; hc->b->submitted=true; ctx.job_fifo=hc->b->bufs.inst.addr; assert(ctx.job_fifo && hc->b->bufs.above_info.addr && hc->b->bufs.ip_above.addr && @@ -262,7 +294,10 @@ assert(readl(q.sram+0x7ffc)==frame.prog_words); assert(!readl(q.sram+0x8000+idx*64+D_PP_EXTRA)); assert(readl(dev.ctrl+lo+idx*4)==(v5?(u32)frame.fifo_iova:frame.fifo_iova>>8)); - assert(readl(dev.ctrl+(v5?0x16c:0x10c))==7&&readl(dev.ctrl+(v5?0x190:0x120))==5); + unsigned int selected=readl(q.sram+0x8000+idx*64+D_VP); + unsigned int expected=multi?(!strcmp(argv[2],"v5-t0")?4:4+idx%4):4; + assert(selected==expected); + assert(readl(dev.ctrl+(v5?0x15c:0xfc)+selected*4)==7&&readl(dev.ctrl+(v5?0x190:0x120))==5); snprintf(path,sizeof(path),"%s/mmio-%u.bin",argv[1],idx); out=fopen(path,"wb");assert(out);assert(fwrite(dev.ctrl,0x10000,1,out)==1);assert(!fclose(out)); snprintf(path,sizeof(path),"%s/descriptor-%u.bin",argv[1],idx); @@ -284,6 +319,11 @@ } assert(!fclose(out)); } + if (multi && dev.queued) { + ctx.priv=&h264[0]; + for(unsigned int b=0;b0;n--) + avd_hevc_done(&ctx,NULL,&dst[n-1].base.vb,VB2_BUF_STATE_ERROR); + for(unsigned int n=0;n> 4) & 15 == idx fifo = self.fifo_addr(idx) - self.events.append(f'VP4 START slot={idx} fifo={fifo:#x}') + self.events.append(f'VP{vp} START slot={idx} fifo={fifo:#x}') self.feed_word += 1 if self.feed_word == len(words): self.feed_idx += 1 @@ -95,11 +103,16 @@ def write(self, uc, access, addr, size, value, user): def submit_host(self, idx, start, tag): variant = image.stem.removeprefix('avd-fw-') - source = programs / variant + source = programs / (('multi-' if self.multi else '') + variant) data = (source / f'descriptor-{idx}.bin').read_bytes() desc = list(struct.unpack('<16I', data)) words = list(struct.unpack('<' + 'I' * desc[6], (source / f'h264-{idx}.bin').read_bytes())) - assert desc[:4] == [0x41565131, 0x10 | idx, 1, 4] + assert desc[:3] == [0x41565131, 0x10 | idx, 1] + vp = desc[3] + assert self.pool[1] & (1 << vp) + if not self.multi: + assert vp == 4 + self.engines[idx] = vp assert desc[4:6] == [0, 0] and desc[8:12] == [1, 0, 0, 1] assert desc[12:] == [0, 0, 0, 0] assert desc[7] == 0x2b000200 | idx << 4 | self.info[10] @@ -126,7 +139,7 @@ def submit_host(self, idx, start, tag): for off in offsets: at = off + idx * 4 self.put(self.decode + at, struct.unpack_from(' Date: Sun, 1 Nov 2026 18:58:56 +1100 Subject: [PATCH 73/73] media: apple: avd: match the const backend query in bank fixtures The production picture-bank query accepts a const device. Match that signature in the host boundary so its HEVC bank lifetime checks compile with warnings treated as errors. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/tests/program-test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/avd/tests/program-test.py b/drivers/media/platform/apple/avd/tests/program-test.py index 74826bebd4256..2778a416d3a93 100755 --- a/drivers/media/platform/apple/avd/tests/program-test.py +++ b/drivers/media/platform/apple/avd/tests/program-test.py @@ -120,7 +120,7 @@ assert(addr && (addr>>shift) && dma_countjob.num,ctx->job.segments[ctx->job.num].num,shift,width,addr}; } -static bool avd_backend_is_qfw(struct avd_dev *d) { return d->queued; } +static bool avd_backend_is_qfw(const struct avd_dev *d) { return d->queued; } static dma_addr_t next_iova=0x204000000ULL; static int avd_buf_alloc(struct avd_dev *d, struct avd_buf *b, size_t n) { (void)d; free(b->cpu); b->cpu = calloc(1,n); b->size = n;