diff --git a/drivers/media/platform/apple/avd/H264-FRAME-OPS.rst b/drivers/media/platform/apple/avd/H264-FRAME-OPS.rst new file mode 100644 index 0000000000000..f28e5decc2c5f --- /dev/null +++ b/drivers/media/platform/apple/avd/H264-FRAME-OPS.rst @@ -0,0 +1,48 @@ +H.264 picture preparation and ownership +====================================== + +H.264 exports CPU prepare/pack_cmd/retire operations. The queued open backend +accepts its complete instruction program; the older open backend selects the +legacy callbacks. Both paths collect slice descriptors and request-control +snapshots until the final request of a HOLD_CAPTURE_BUF picture. Only metadata +is copied. Each descriptor points at the OUTPUT DMA address plus data_offset +and the emulation-prevention-aware slice-header offset. + +The core removes intermediate OUTPUTs from the scheduler without completing +them. It transfers their ownership to the final frame and completes them after +CAPTURE retirement. Incomplete pictures release their OUTPUTs and codec state +on streamoff or preparation failure. Userspace must queue every slice of a +picture before waiting for its OUTPUT buffers; it cannot recycle a held slice. +The codec never operates mem2mem queues or MMIO in its frame callbacks. + +Each live picture owns independent scratch and a VP-to-PP instruction FIFO. +Sixteen owners are cached per context, with reuse only after retirement. Motion +vectors remain in their reference CAPTURE allocation. H.264 uses DEP_NONE. + +Apple layout boundary +--------------------- + +The public MIT eiln/avd avid/h264/fp.py and halv3.py describe Viola/V3. The +packer writes its SPS/PPS/scaling, DPB, output, weights and slice fields, with +600 separately bounded slice records. This public comparison does not establish +the later LilyD layout selected by T6000 firmware. Match data must select a +proved layout rather than infer one from the hardware register revision. + +Borage has an independent table with the command projection proved by SPEC 6.2. +The exact +0x3c/+0x44 packed-address rules, both work-count bounds, and the full +parser-compatible syntax layout remain unproved. These fields retain numeric +source offsets and return EOPNOTSUPP; this draft cannot enable Apple decoding. +T8112 and T8140 remain explicit per-SoC ledger TODO rows (t-3862). No later-class +layout falls back to V3. + +Verification +------------ + +On a Linux build host run tests/run-host-tests.py. It compiles the production +builder and parameter writer, compares 96 synthetic I/P/B 1/4-slice programs +against the licensed frozen Linux builder, and exercises the actual prepare and +retire functions across separate held requests and sixteen live pictures. +Additional tests compile the core source-ownership helpers and check final +completion, native/legacy partial abort, V3 field bounds and Borage projection. +These are host tests, not target decode or GStreamer/ffmpeg conformance results. +The encoded-stream comparison with eiln's decoder remains separate validation. diff --git a/drivers/media/platform/apple/avd/Kconfig b/drivers/media/platform/apple/avd/Kconfig index ed5ad44276c31..d3bb9763b6230 100644 --- a/drivers/media/platform/apple/avd/Kconfig +++ b/drivers/media/platform/apple/avd/Kconfig @@ -24,8 +24,7 @@ config VIDEO_APPLE_AVD_KUNIT_TEST default KUNIT_ALL_TESTS help Build KUnit tests for the Apple video decoder. They exercise the - checks that the H.264 path makes on the slices that userspace - queues, such as finding the slice data after the slice header, - at the limits and one past them. + H.264 slice bounds, frame-slot ownership, and runtime power management, + including suspend gating and resubmission after firmware resume. If unsure, say N. diff --git a/drivers/media/platform/apple/avd/Makefile b/drivers/media/platform/apple/avd/Makefile index 4d1877c414d41..46e143b648034 100644 --- a/drivers/media/platform/apple/avd/Makefile +++ b/drivers/media/platform/apple/avd/Makefile @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only -apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-h264.o avd-vp9.o avd-hevc.o \ - avd-av1.o avd-av1-entropymode.o -apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o +apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-pm.o avd-h264.o avd-h264-fp.o avd-vp9.o \ + avd-hevc.o avd-av1.o avd-av1-entropymode.o avd-queue.o avd-backend-open.o \ + avd-backend-apple.o avd-backend-queue.o +apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o avd-queue-test.o \ + avd-pm-test.o obj-$(CONFIG_VIDEO_APPLE_AVD) += apple-avd.o diff --git a/drivers/media/platform/apple/avd/avd-apple-protocol.h b/drivers/media/platform/apple/avd/avd-apple-protocol.h new file mode 100644 index 0000000000000..d0b90c5d48e4c --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-apple-protocol.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_APPLE_PROTOCOL_H_ +#define AVD_APPLE_PROTOCOL_H_ + +#include +#include + +#include "avd-queue.h" + +#define AVD_APPLE_INBOX_BASE 0x01482cf8 +#define AVD_APPLE_COMMAND_SIZE 0x60 +#define AVD_APPLE_RESULT_WORDS 20 +#define AVD_APPLE_SHADOW_SIZE 64 + +/* + * [C] Expected address notification, from the MIT V3 trace_avd.py precedent. + * Keep this rule in one place for correction by the Borage capture. + */ +#define AVD_APPLE_NOTIFY_BASE AVD_APPLE_INBOX_BASE + +/* SPEC 5.1: only these session opcodes have proved builders. */ +static inline int +avd_apple_session_command(u32 cmd[24], u32 client, u32 opcode, + u32 property, u32 value) +{ + unsigned int i; + + if (client >= 1024 || (opcode != 0 && opcode != 2 && opcode != 5 && opcode != 6)) + return -EINVAL; + for (i = 0; i < 24; i++) + cmd[i] = 0; + cmd[0] = opcode | (client << 16); + if (opcode == 6) { + cmd[1] = property; + cmd[2] = value; + } + return 0; +} + +static inline int avd_apple_decode_prefix(struct avd_frame *f, u32 client, + u32 selector, u32 work_sequence) +{ + unsigned int i; + + if (f->idx >= AVD_MAX_FRAMES || client >= 1024 || selector >= 16 || work_sequence >= 4) + return -EINVAL; + for (i = 0; i < 24; i++) + f->cmd[i] = 0; + f->cmd[0] = 1 | (selector << 10) | (client << 22); + f->cmd[1] = f->idx | (work_sequence << 16); + f->cmd[3] = f->frame_num; + return 0; +} + +struct avd_apple_reply_layout { + u8 type_word; + u8 type_shift; + u32 type_mask; + u8 client_word; + u8 client_shift; + u32 client_mask; + u8 frame_word; + u8 status_word; + u32 done_type; + bool verified; +}; + +struct avd_apple_reply { + u32 type; + u32 client; + u32 frame; + int status; +}; + +static inline int +avd_apple_parse_reply(const u32 *record, size_t words, + const struct avd_apple_reply_layout *layout, + struct avd_apple_reply *reply) +{ + if (words != AVD_APPLE_RESULT_WORDS) + return -EMSGSIZE; + if (!layout->verified) + return -EOPNOTSUPP; + if (layout->type_word >= words || layout->client_word >= words || + layout->frame_word >= words || layout->status_word >= words || + layout->type_shift >= 32 || layout->client_shift >= 32 || + !layout->type_mask || !layout->client_mask || layout->done_type == 7) + return -EINVAL; + reply->type = (record[layout->type_word] >> layout->type_shift) & layout->type_mask; + reply->client = (record[layout->client_word] >> layout->client_shift) & layout->client_mask; + reply->frame = record[layout->frame_word]; + /* SPEC 4.3: type 7 is parsing/VP done, never final buffer ownership. */ + if (reply->type == 7) + return -EINPROGRESS; + if (reply->type != layout->done_type) + return -ENOMSG; + reply->status = record[layout->status_word] ? -EIO : 0; + return 0; +} + +static inline u32 avd_apple_notification(unsigned int slot) +{ + return AVD_APPLE_NOTIFY_BASE + AVD_APPLE_COMMAND_SIZE * slot; +} + +struct avd_apple_hw { + const char *fw_name; + /* All offsets below are relative to a DT-owned "apple" resource. */ + u32 aperture_origin; + u32 imem; + u32 imem_size; + u32 version_offset; + u32 itcm_start; + u32 itcm_end; + u32 itcm_clear; + u32 itcm_clear_value; + u32 itcm_enable; + u32 itcm_enable_value; + u32 link_setup; + u32 link_setup_value; + u32 link_state; + u32 link_up; + u32 link_up_value; + u32 ready; + u32 notify; + u32 receive; + u32 receive_status; + u32 receive_pending; + u32 receive_ack; + u32 receive_ack_value; + u32 inbox_consumer; + u32 result_base; + struct avd_apple_reply_layout reply; + /* Match-data fact, never a module parameter or a userspace setting. */ + bool verified; +}; + +extern const struct avd_apple_hw avd_borage_hw; +extern const u8 avd_borage_config[64]; + +#endif /* AVD_APPLE_PROTOCOL_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index 60d69bab8a342..6bd5b1a9a6227 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -1515,6 +1515,13 @@ static enum avd_image_fmt avd_av1_get_image_fmt(struct avd_ctx *ctx, #undef BIT_DEPTH } +static unsigned int avd_av1_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_av1_ctx *priv = ctx->priv; + + return max_t(int, priv->submit_num, 1); +} + static void avd_av1_submit(struct avd_ctx *ctx) { struct avd_av1_ctx *av1_ctx = ctx->priv; @@ -1570,6 +1577,7 @@ const struct avd_coded_fmt_ops avd_av1_fmt_ops = { .stop = avd_av1_stop, .run = avd_av1_run, .submit = avd_av1_submit, + .pp_kicks = avd_av1_pp_kicks, .try_ctrl = avd_av1_try_ctrl, .get_image_fmt = avd_av1_get_image_fmt, }; diff --git a/drivers/media/platform/apple/avd/avd-backend-apple.c b/drivers/media/platform/apple/avd/avd-backend-apple.c new file mode 100644 index 0000000000000..8345f107e4c14 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-apple.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include + +#include "avd.h" +#include "avd-apple-protocol.h" +#include "avd-queue-internal.h" + +/* SPEC 3.2. Unknown field meanings are intentionally not named. */ +const u8 avd_borage_config[64] = { + [0x0a] = 0x55, [0x0b] = 0x01, /* +0x08: 0x01558000 */ + [0x09] = 0x80, + [0x11] = 0x8a, [0x12] = 0x55, [0x13] = 0x01, + [0x19] = 0x8e, [0x1a] = 0x55, [0x1b] = 0x01, + [0x28] = 0x0a, [0x2a] = 0x04, + [0x2c] = 0x04, + [0x38] = 0x08, [0x3a] = 0x10, [0x3b] = 0x01, +}; + +/* + * Source: clean SPEC 3-5, AppleAVD 993.1 / T6040 Borage. The SPEC does + * not supply the ITCM/version register offsets, link register operations or + * reply field layout. Do not substitute V3/open-firmware offsets for them. + * avd-protocol.rst records every unresolved field and its source/capture. + */ +const struct avd_apple_hw avd_borage_hw = { + .fw_name = "apple/avd-apple-t6040.bin", + .verified = false, +}; + +struct avd_apple { + void __iomem *regs; + const struct avd_apple_hw *hw; + const struct firmware *image; + u32 shadow[AVD_APPLE_SHADOW_SIZE]; + unsigned int shadow_read; + unsigned int shadow_write; + unsigned int producer; + u32 version; +}; + +static void avd_apple_release_image(void *data) +{ + struct avd_apple *apple = data; + + release_firmware(apple->image); +} + +static int avd_apple_image_present(struct avd_apple *apple) +{ + const struct avd_apple_hw *hw = apple->hw; + u32 start = readl(apple->regs + hw->itcm_start); + u32 end = readl(apple->regs + hw->itcm_end); + u32 version; + + if (end <= start || end - start < 4 || end - start > hw->imem_size || + hw->version_offset > end - start - 4) + return -ENOENT; + version = readl(apple->regs + hw->imem + hw->version_offset); + if (!version || version == U32_MAX) + return -ENOENT; + apple->version = version; + return 0; +} + +static int avd_apple_probe(struct avd_dev *avd) +{ + const struct avd_apple_hw *hw = avd->variant->apple; + struct avd_apple *apple; + int ret; + + if (!hw || !hw->verified || !hw->reply.verified) { + dev_info(avd->dev, "Apple transport awaits Borage register/reply facts; using open firmware\n"); + return -EOPNOTSUPP; + } + apple = devm_kzalloc(avd->dev, sizeof(*apple), GFP_KERNEL); + if (!apple) + return -ENOMEM; + apple->hw = hw; + apple->regs = devm_platform_ioremap_resource_byname(avd->pdev, "apple"); + if (IS_ERR(apple->regs)) + return PTR_ERR(apple->regs); + /* Firmware bytes remain in vendorfw; this driver never embeds an image. */ + ret = firmware_request_nowarn(&apple->image, hw->fw_name, avd->dev); + if (ret && ret != -ENOENT) + return ret; + ret = devm_add_action_or_reset(avd->dev, avd_apple_release_image, apple); + if (ret) + return ret; + avd->backend_priv = apple; + return 0; +} + +static int avd_apple_load(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + const struct firmware *fw = apple->image; + size_t i, rounded; + + if (!fw) + return -ENOENT; + if (!fw->size || fw->size > hw->imem_size || + hw->imem_size < 4 || hw->version_offset > hw->imem_size - 4) + return -EINVAL; + rounded = ALIGN(fw->size, 4); + if (rounded > hw->imem_size) + return -EFBIG; + writel(hw->itcm_clear_value, apple->regs + hw->itcm_clear); + writel(0, apple->regs + hw->itcm_start); + writel(rounded, apple->regs + hw->itcm_end); + /* Bounded word copies pad the last word without reading past fw->data. */ + for (i = 0; i < rounded; i += 4) { + u8 bytes[4] = {}; + + memcpy(bytes, fw->data + i, min_t(size_t, 4, fw->size - i)); + writel(get_unaligned_le32(bytes), apple->regs + hw->imem + i); + } + writel(hw->itcm_enable_value, apple->regs + hw->itcm_enable); + return avd_apple_image_present(apple); +} + +static int avd_apple_boot(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + u32 value; + int ret; + + ret = avd_apple_image_present(apple); + if (ret) { + ret = avd_apple_load(avd); + if (ret) + return ret; + } else { + /* Suspend stopped the MCPU without clearing the retained image. */ + writel(hw->itcm_enable_value, apple->regs + hw->itcm_enable); + } + dev_info(avd->dev, "Apple MCPU firmware %u.%u.%u (%08x)\n", + apple->version >> 8, (apple->version >> 4) & 0xf, + apple->version & 0xf, apple->version); + /* SPEC 3.2: decode connection, aperture 1, state 4, then LinkUp. */ + writel(hw->link_setup_value, apple->regs + hw->link_setup); + ret = readl_poll_timeout(apple->regs + hw->link_state, value, + value == 4, 10, 10000); + if (ret) + return ret; + writel(hw->link_up_value, apple->regs + hw->link_up); + ret = readl_poll_timeout(apple->regs + hw->ready, value, + value == 1, 10, 10000); + if (!ret) { + apple->producer = 0; + apple->shadow_read = 0; + apple->shadow_write = 0; + } + return ret; +} + +static void avd_apple_shutdown(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + + /* MCPU disable value must be proved together with enable in the ledger. */ + writel(0, apple->regs + apple->hw->itcm_enable); +} + +static unsigned int avd_apple_max_inflight(struct avd_dev *avd) +{ + /* + * Every frame in VP->PP flight owns one hardware instruction FIFO + * (EXEC bits 7:4), so the FIFO count bounds the frames in flight: + * 15 on v3+ tier 1 parts, 7 on T8103 and T8140. + */ + return min_t(unsigned int, AVD_MAX_FRAMES, avd->variant->fifo_slots ?: 1); +} + +static int avd_apple_send(struct avd_dev *avd, const u32 cmd[24]) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + unsigned int consumer = readl(apple->regs + hw->inbox_consumer) & 7; + u32 address = AVD_APPLE_INBOX_BASE + apple->producer * AVD_APPLE_COMMAND_SIZE; + unsigned int i; + + if (avd_inbox_full(apple->producer, consumer)) + return -EBUSY; + for (i = 0; i < 24; i++) + writel(cmd[i], apple->regs + address - hw->aperture_origin + i * 4); + /* MMIO commands and coherent fp payload precede the four-byte doorbell. */ + wmb(); + writel(avd_apple_notification(apple->producer), apple->regs + hw->notify); + apple->producer = (apple->producer + 1) & 7; + return 0; +} + +static int avd_apple_session_start(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 0, 0, 0)) + return -EINVAL; + return avd_apple_send(ctx->dev, cmd); +} + +static void avd_apple_session_stop(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 2, 0, 0)) + return; + if (avd_apple_send(ctx->dev, cmd)) + dev_warn(ctx->dev->dev, "session stop inbox full\n"); +} + +static void avd_apple_abort(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 5, 0, 0)) + return; + /* Core follows abort with reset before returning any DMA buffer. */ + avd_apple_send(ctx->dev, cmd); +} + +static int avd_apple_set_property(struct avd_ctx *ctx, u32 property, u32 value) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 6, property, value)) + return -EINVAL; + return avd_apple_send(ctx->dev, cmd); +} + +static int avd_apple_submit(struct avd_dev *avd, struct avd_frame *f) +{ + const struct avd_codec_frame_ops *ops = f->ctx->coded_fmt_desc->frame_ops; + u32 selector; + int ret; + + if (!ops || !ops->pack_cmd || !f->fp->cpu || f->ctx->client_id >= 1024) + return -EOPNOTSUPP; + /* SPEC 5.2 prefix precedes the codec-specific builder. */ + selector = get_unaligned_le32(f->fp->cpu + 4) & 0xf; + ret = avd_apple_decode_prefix(f, f->ctx->client_id, selector, f->seq & 3); + if (ret) + return ret; + ret = ops->pack_cmd(f->ctx, f); + if (ret) + return ret; + return avd_apple_send(avd, f->cmd); +} + +static irqreturn_t avd_apple_irq(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + unsigned int budget = AVD_APPLE_SHADOW_SIZE; + + while (budget-- && (readl(apple->regs + hw->receive_status) & hw->receive_pending)) { + unsigned int next = (apple->shadow_write + 1) & 63; + + if (next == apple->shadow_read) + break; + apple->shadow[apple->shadow_write] = readl(apple->regs + hw->receive); + apple->shadow_write = next; + writel(hw->receive_ack_value, apple->regs + hw->receive_ack); + } + while (apple->shadow_read != apple->shadow_write) { + u32 record[AVD_APPLE_RESULT_WORDS]; + struct avd_apple_reply reply; + unsigned int i; + int ret; + + /* Result notification indexing must be verified with the layout. */ + u32 index = apple->shadow[apple->shadow_read]; + + apple->shadow_read = (apple->shadow_read + 1) & 63; + if (index >= AVD_APPLE_SHADOW_SIZE) + continue; + for (i = 0; i < AVD_APPLE_RESULT_WORDS; i++) + record[i] = readl(apple->regs + hw->result_base + + index * sizeof(record) + i * 4); + ret = avd_apple_parse_reply(record, ARRAY_SIZE(record), &hw->reply, &reply); + if (ret) + continue; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + + if (f->state == AVD_FRAME_SUBMITTED && f->frame_num == reply.frame && + f->ctx->client_id == reply.client) { + dma_rmb(); + avd_frame_complete(avd, f->tag, reply.status); + break; + } + } + } + mod_delayed_work(system_wq, &avd->queue_work, 0); + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_apple_backend = { + .name = "apple-mcpu", + .probe = avd_apple_probe, + .boot = avd_apple_boot, + .shutdown = avd_apple_shutdown, + .max_inflight = avd_apple_max_inflight, + .session_start = avd_apple_session_start, + .session_stop = avd_apple_session_stop, + .submit = avd_apple_submit, + .abort = avd_apple_abort, + .set_property = avd_apple_set_property, + .irq = avd_apple_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, +}; diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c new file mode 100644 index 0000000000000..fdff791f422bc --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Legacy transport split from avd-drv.c; Copyright The Asahi Linux Contributors. */ +#include +#include +#include +#include +#include +#include + +#include "avd.h" +#include "avd-inst.h" +#include "avd-regs.h" + +/* + * Compatibility transport for firmware without the queued-program ABI. The + * codec's CPU builder emits a per-slot DMA-visible instruction stream. Only + * this backend touches the VP FIFO. Its untagged PP interrupt limits it to one + * frame; the queued avd-fw transport can replace these ops without changing + * buffer ownership in the core. + */ +int avd_open_build_program(struct avd_ctx *ctx) +{ + struct avd_dev *avd = ctx->dev; + struct avd_frame *f = avd->building; + struct avd_job *job = &ctx->job; + size_t count = 1, pos = 0, i, j; + u32 *words, mask, start; + int ret; + + if (!f || !job->segments) + return -EINVAL; + for (i = 0; i <= job->num; i++) { + if (job->segments[i].num > AVD_MAX_INST || + check_add_overflow(count, job->segments[i].num + !!i, &count)) + return -EOVERFLOW; + } + if (count > SIZE_MAX / sizeof(u32)) + return -EOVERFLOW; + if (f->prog->size < count * sizeof(u32)) { + ret = avd_buf_alloc(avd, f->prog, count * sizeof(u32)); + if (ret) + return ret; + } + mask = job->codec == AVD_CODEC_VP9 && avd->variant->revision == 3 ? + AVD_OP_EXEC_REV3_VP9_MASK : 0; + start = AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | + AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3); + /* The queued transport gives every frame in flight its own FIFO. */ + ctx->fifo_idx = avd_backend_is_qfw(avd) ? f->idx : 0; + f->fifo_iova = ctx->job_fifo ?: ctx->inst.addr; + ctx->job_fifo = 0; + words = f->prog->cpu; + words[pos++] = AVD_OP_EXEC | mask | start | AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx); + for (i = 0; i <= job->num; i++) { + for (j = 0; j < job->segments[i].num; j++) + words[pos++] = job->segments[i].instructions[j]; + if (i) + words[pos++] = AVD_OP_EXEC | mask | AVD_OP_EXEC_FLAG_END(i == job->num); + } + f->prog_words = pos; + f->cmd[0] = job->codec; + kvfree(job->segments); + job->segments = NULL; + return 0; +} + +static unsigned int avd_open_max_inflight(struct avd_dev *avd) +{ + return 1; +} + +static int avd_open_submit(struct avd_dev *avd, struct avd_frame *f) +{ + struct avd_ctx *ctx = f->ctx; + u32 codec = f->cmd[0], vp = 0; + u32 *words = f->prog->cpu; + unsigned int i; + void __iomem *reg; + + if (avd->compat_active) + return -EBUSY; + /* Native frame_ops require firmware-side PP chaining and tagged replies. */ + if (avd_codec_frame_ops(ctx)) + return -EOPNOTSUPP; + if (codec > AVD_CODEC_AV1 || !words || !f->prog_words || + f->prog_words > f->prog->size / sizeof(u32)) + return -EINVAL; + if (!(avd->capabilities & BIT(codec))) + return -EOPNOTSUPP; + for (i = 0; i < codec; i++) + vp += avd->variant->vp_slots[i]; + if (avd->variant->vp_slot_base[codec]) + vp = avd->variant->vp_slot_base[codec]; + avd->variant->configure_stream(avd, f->fifo_iova ?: ctx->inst.addr, ctx->fifo_idx, vp); + reg = avd->ctrl + avd->variant->vp_slot_offset + vp * 4; + avd->compat_active = f; + avd->compat_ctx = ctx; + for (i = 0; i < f->prog_words; i++) { + /* Preserve the old firmware's segment FIFO pacing in this shim. */ + if (i && (words[i] & GENMASK(31, 24)) == AVD_OP_EXEC && + (!(avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) || + codec == AVD_CODEC_HEVC)) { + u32 max = readl(avd->ctrl + avd->variant->submit_queue_max_offset + vp * 4); + u32 cur = readl(avd->ctrl + + avd->variant->submit_queue_status_offset + vp * 4); + + if (cur >= max) { + /* Already emitted words: only reset may release this slot. */ + f->deadline = jiffies; + return 0; + } + if (cur >= max / 2) { + /* + * Wait for the VP to drain below half full, but + * only as long as it takes. The old fixed sleep + * (100-150 us on v5 pacing parts, 500-650 us + * elsewhere) stays the upper bound. + */ + u32 wait_us = avd->variant->quirks & + AVD_QUIRK_V5_QUEUE_PACING ? 150 : 650; + + readl_relaxed_poll_timeout(avd->ctrl + + avd->variant->submit_queue_status_offset + vp * 4, + cur, cur < max / 2, 10, wait_us); + } + } + writel(words[i], reg); + } + return 0; +} + +static irqreturn_t avd_open_irq(struct avd_dev *avd) +{ + struct avd_frame *f = avd->compat_active; + u32 status; + + if (!avd_mbox_pop(avd, &status)) + return IRQ_HANDLED; + if (!f) { + /* + * The old driver kicked PP for every VP DONE. A VP DONE with no + * frame in flight (its frame already completed on a surplus PP + * DONE of a multi-kick picture) still needs that kick, or the + * next picture's PP is never started and it times out. + */ + if ((status & 0x11000) == 0 && (status & 0x100) && avd->compat_ctx && + avd->compat_ctx->coded_fmt_desc->ops->submit) + avd->compat_ctx->coded_fmt_desc->ops->submit(avd->compat_ctx); + return IRQ_HANDLED; + } + if (status & 0x1000) { + avd->compat_active = NULL; + avd_frame_complete(avd, f->tag, 0); + } else if (status & 0x100) { + /* Old firmware only: no request tag and no firmware-side PP kick. */ + if (f->ctx->coded_fmt_desc->ops->submit) + f->ctx->coded_fmt_desc->ops->submit(f->ctx); + } else { + dev_err_ratelimited(avd->dev, "open firmware error %#x\n", status); + f->deadline = jiffies; + mod_delayed_work(system_wq, &avd->queue_work, 0); + } + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_open_backend = { + .name = "open-compat", + .boot = avd_boot, + .shutdown = avd_shutdown, + .max_inflight = avd_open_max_inflight, + .submit = avd_open_submit, + .irq = avd_open_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, +}; + +static void avd_open_capabilities(struct avd_dev *avd) +{ + static const struct { + const char *name; + u32 bit; + } codecs[] = { + { "h265", AVD_CAPABILITY_HEVC }, + { "h264", AVD_CAPABILITY_H264 }, + { "vp9", AVD_CAPABILITY_VP9 }, + { "av1", AVD_CAPABILITY_AV1 }, + }; + const char *info; + unsigned int i; + + /* Without a CODECS trailer, keep codecs that need a newer firmware hidden. */ + avd->capabilities = avd->variant->capabilities & + ~avd->variant->fw_caps_required; + avd->fw_capabilities = avd->capabilities; + if (!avd->fw || avd->fw->size < 0x40) + return; + info = (const char *)avd->fw->data + avd->fw->size - 0x40; + if (memcmp(info, "CODECS:", 7)) + return; /* fw_capabilities already set above */ + avd->capabilities = 0; + for (i = 0; i < ARRAY_SIZE(codecs); i++) + if (strnstr(info, codecs[i].name, 0x40)) + avd->capabilities |= avd->variant->capabilities & codecs[i].bit; + avd->fw_capabilities = avd->capabilities; +} + +static int avd_open_request_firmware(struct avd_dev *avd) +{ + int ret; + + if (!avd->variant->fw_name_fallback) + return request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); + ret = firmware_request_nowarn(&avd->fw, avd->variant->fw_name, avd->dev); + if (ret != -ENOENT) + return ret; + dev_info(avd->dev, "%s not installed, using %s\n", + avd->variant->fw_name, avd->variant->fw_name_fallback); + return request_firmware(&avd->fw, avd->variant->fw_name_fallback, avd->dev); +} + +int avd_backend_select(struct avd_dev *avd) +{ + int ret; + + /* Apple probe returns -ENOENT only when no usable image is present. */ + if (avd->variant->apple) { + ret = avd_apple_backend.probe(avd); + if (!ret) { + avd->backend = &avd_apple_backend; + avd->capabilities = avd->variant->capabilities; + return 0; + } + if (ret != -ENOENT && ret != -EOPNOTSUPP) + return ret; + } + avd->backend = &avd_open_backend; + ret = avd_open_request_firmware(avd); + if (!ret) + avd_open_capabilities(avd); + if (!ret) + ret = avd_qfw_probe(avd); + return ret; +} + +int avd_backend_boot(struct avd_dev *avd) +{ + int ret; + + if (!avd_backend_is_apple(avd)) { + avd_pm_prepare_piodma(avd); + avd_qfw_prepare_boot(avd); + } + ret = avd->backend->boot(avd); + /* Never switch command ABIs after frames have used this backend. */ + if (ret == -ENOENT && !avd->fw_generation && avd_backend_is_apple(avd)) { + ret = avd_open_request_firmware(avd); + if (!ret) + ret = avd_qfw_probe(avd); + if (ret) + return ret; + avd->backend = &avd_open_backend; + avd_open_capabilities(avd); + avd_pm_prepare_piodma(avd); + avd_qfw_prepare_boot(avd); + ret = avd->backend->boot(avd); + } + if (!ret) + avd_qfw_select(avd); + if (!ret) + avd->fw_generation++; + return ret; +} diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c new file mode 100644 index 0000000000000..0a73e703de604 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Queued transport for the open AVD firmware (avd-fw queue ABI 2). + * + * The codec's CPU builder still produces the per-frame instruction program. + * Instead of writing it word by word into the VP FIFO register and kicking + * the PP from the VP-done interrupt, the host streams the program into a word + * ring in CM3 SRAM and posts one descriptor per frame. The firmware feeds the + * VP FIFO from SRAM, starts PP itself when the VP is done, and posts a tagged + * completion. No PIODMA (and so no second DART stream) is involved. + */ +#include +#include +#include +#include +#include + +#include "avd.h" +#include "avd-inst.h" +#include "avd-regs.h" + +#define AVDQ_MAGIC 0x41565131 /* "AVQ1" */ +#define AVDQ_VERSION 2 +#define AVDQ_COUNT 16 +#define AVDQ_DESC_BYTES 64 +#define AVDQ_RING_OFFSET 0x8000 +#define AVDQ_INFO 0x7fc0 +/* Firmware private state starts here; the SRAM ring must end before it. */ +#define AVDQ_RING_LIMIT 0x6000 +#define AVDQ_INFO_RP (AVDQ_INFO + 0x38) +#define AVDQ_INFO_WP (AVDQ_INFO + 0x3c) +#define AVDQ_FEAT_SRAM_RING BIT(0) +#define AVDQ_FEAT_MULTI_KICK BIT(1) +#define AVDQ_FLAG_SRAM BIT(0) +#define AVDQ_MAX_PP_EXTRA 255 +#define AVDQ_STATE_SUBMITTED 1 +#define AVDQ_STATE_COMPLETE 2 +#define AVDQ_NOTIFY_MASK 0xffff0000 +#define AVDQ_NOTIFY 0x54530000 +/* SRAM addresses as the CM3 doorbell sees them: 0x01000000 + offset from AVD base. */ +#define AVDQ_CM3_APERTURE(off) (0x01000000 + (off)) +#define AVD_REG_MBOX0_STATUS 0x50 +#define AVD_REG_MBOX0_SEND 0x54 +#define AVDQ_STREAM_TIMEOUT_US (1000 * USEC_PER_MSEC) + +/* Descriptor word offsets (bytes), avd-fw docs/queue.md. */ +enum { + D_MAGIC = 0x00, D_TAG = 0x04, D_CODEC = 0x08, D_VP = 0x0c, + D_PROG_LO = 0x10, D_PROG_HI = 0x14, D_WORDS = 0x18, D_KICK = 0x1c, + D_STATE = 0x20, D_STATUS = 0x24, D_CTAG = 0x28, D_FLAGS = 0x2c, + D_PP_EXTRA = 0x30, D_PP_CONT = 0x34, D_RSVD0 = 0x38, D_RSVD1 = 0x3c, +}; + +struct avd_qfw { + void __iomem *sram; + resource_size_t sram_size; + u32 aperture; /* expected CM3 address of SRAM byte 0 */ + u32 ring_off; + u32 ring_words; + u32 wp; /* host write count, free-running */ + u32 fifo_slots; + u32 codec_mask; /* info+0x24, bit n = codec n */ + bool usable; /* last boot found a matching ABI 2 image */ +}; + +static void __iomem *avd_qfw_desc(struct avd_qfw *q, unsigned int idx) +{ + return q->sram + AVDQ_RING_OFFSET + idx * AVDQ_DESC_BYTES; +} + +int avd_qfw_probe(struct avd_dev *avd) +{ + struct resource *code, *sram; + struct avd_qfw *q; + + code = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "code"); + sram = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "sram"); + if (!code || !sram || sram->start < code->start || + resource_size(sram) < AVDQ_RING_OFFSET + AVDQ_COUNT * AVDQ_DESC_BYTES) + return 0; /* no queue mode on this node: legacy transport only */ + q = devm_kzalloc(avd->dev, sizeof(*q), GFP_KERNEL); + if (!q) + return -ENOMEM; + q->sram = devm_ioremap_resource(avd->dev, sram); + if (IS_ERR(q->sram)) + return PTR_ERR(q->sram); + q->sram_size = resource_size(sram); + /* The code window sits at +0x80000 from the decoder base on every SoC. */ + q->aperture = AVDQ_CM3_APERTURE(0x80000 + (u32)(sram->start - code->start)); + avd->qfw = q; + return 0; +} + +/* Read the firmware's info block after boot; true when ABI 2 queue mode fits. */ +static bool avd_qfw_check(struct avd_dev *avd) +{ + struct avd_qfw *q = avd->qfw; + u32 info[16]; + unsigned int i; + + if (!q) + return false; + for (i = 0; i < ARRAY_SIZE(info); i++) + info[i] = readl(q->sram + AVDQ_INFO + 4 * i); + q->usable = false; + if (info[0] != AVDQ_MAGIC) + return false; + if (info[1] != AVDQ_VERSION || info[2] != AVDQ_COUNT || + info[3] != AVDQ_DESC_BYTES || info[5] != AVDQ_RING_OFFSET || + !(info[11] & AVDQ_FEAT_SRAM_RING) || + !(info[11] & AVDQ_FEAT_MULTI_KICK)) { + dev_info_once(avd->dev, "open firmware queue ABI %u not supported, using legacy transport\n", + info[1]); + return false; + } + if (info[4] != q->aperture) { + dev_warn_once(avd->dev, "firmware SRAM aperture %#x, node needs %#x: wrong image tier, using legacy transport\n", + info[4], q->aperture); + return false; + } + if (info[13] < 1024 || info[12] % 4 || + (u64)info[12] + 4ULL * info[13] > AVDQ_RING_LIMIT || + !info[10] || info[10] > AVDQ_COUNT || info[14] || info[15]) { + dev_warn_once(avd->dev, "firmware SRAM ring geometry invalid, using legacy transport\n"); + return false; + } + q->ring_off = info[12]; + q->ring_words = info[13]; + q->fifo_slots = min_t(u32, info[10], avd->variant->fifo_slots ?: 1); + q->codec_mask = info[9]; + q->wp = 0; + q->usable = true; + return true; +} + +static int avd_qfw_boot(struct avd_dev *avd) +{ + int ret = avd_boot(avd); + + if (ret) + return ret; + /* AP -> CM3 doorbells use mailbox 0; completions stay on mailbox 1. */ + writel_relaxed(AVD_MBOX_ENABLE, avd->mbox + AVD_REG_MBOX0_STATUS); + return 0; +} + +static unsigned int avd_qfw_max_inflight(struct avd_dev *avd) +{ + /* Each frame in flight owns one hardware instruction FIFO (EXEC 7:4). */ + return min_t(unsigned int, AVD_MAX_FRAMES, avd->qfw->fifo_slots); +} + +static void avd_qfw_fail_all(struct avd_dev *avd) +{ + unsigned int i; + + /* ABI: any fatal word or I/O status needs a whole-queue reset. */ + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (avd->frames[i].state == AVD_FRAME_SUBMITTED) + avd->frames[i].deadline = jiffies; + mod_delayed_work(system_wq, &avd->queue_work, 0); +} + +static int avd_qfw_stream(struct avd_dev *avd, const u32 *words, u32 count) +{ + struct avd_qfw *q = avd->qfw; + unsigned long timeout = jiffies + usecs_to_jiffies(AVDQ_STREAM_TIMEOUT_US); + u32 done = 0; + + while (done < count) { + u32 rp = readl(q->sram + AVDQ_INFO_RP); + u32 used = q->wp - rp, space, pos, chunk; + + if (used > q->ring_words) + return -EIO; + space = q->ring_words - used; + if (!space) { + if (time_after(jiffies, timeout)) + return -ETIMEDOUT; + usleep_range(20, 50); + continue; + } + pos = q->wp % q->ring_words; + chunk = min3(space, count - done, q->ring_words - pos); + memcpy_toio(q->sram + q->ring_off + 4 * pos, words + done, 4 * chunk); + q->wp += chunk; + done += chunk; + /* Words before the count that publishes them. */ + wmb(); + writel(q->wp, q->sram + AVDQ_INFO_WP); + } + return 0; +} + +static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) +{ + struct avd_qfw *q = avd->qfw; + struct avd_ctx *ctx = f->ctx; + const struct avd_coded_fmt_ops *ops = ctx->coded_fmt_desc->ops; + u32 codec = f->cmd[0], vp = 0, kick, cont, mask, extra = 0; + u32 *words = f->prog->cpu; + void __iomem *d = avd_qfw_desc(q, f->idx); + unsigned int i; + int ret; + + /* H.264 native programs use the same SRAM ring descriptor. */ + if (avd_codec_frame_ops(ctx) && codec != AVD_CODEC_H264) + return -EOPNOTSUPP; + if (codec > AVD_CODEC_AV1 || !words || !f->prog_words || + f->prog_words > f->prog->size / sizeof(u32)) + return -EINVAL; + if (!(avd->capabilities & BIT(codec))) + return -EOPNOTSUPP; + /* The program's EXEC words name FIFO f->idx; it must be a real FIFO. */ + if (f->idx >= q->fifo_slots || + (!avd_codec_frame_ops(ctx) && ctx->fifo_idx != f->idx)) + return -EINVAL; + for (i = 0; i < codec; i++) + vp += avd->variant->vp_slots[i]; + if (avd->variant->vp_slot_base[codec]) + vp = avd->variant->vp_slot_base[codec]; + if (ops->pp_kicks) + extra = ops->pp_kicks(ctx) - 1; + if (extra > AVDQ_MAX_PP_EXTRA) + return -E2BIG; + + mask = codec == AVD_CODEC_VP9 && avd->variant->revision == 3 ? + AVD_OP_EXEC_REV3_VP9_MASK : 0; + cont = AVD_OP_EXEC | mask | AVD_OP_EXEC_FIFO_IDX(f->idx) | + AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots); + kick = cont | AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | + AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3 && + codec != AVD_CODEC_AV1); + + /* FIFO registers of other live frames are never touched. */ + avd->variant->configure_stream(avd, f->fifo_iova ?: ctx->inst.addr, f->idx, vp); + + writel(AVDQ_MAGIC, d + D_MAGIC); + writel(f->tag, d + D_TAG); + writel(codec, d + D_CODEC); + writel(vp, d + D_VP); + writel(q->wp, d + D_PROG_LO); + writel(0, d + D_PROG_HI); + writel(f->prog_words, d + D_WORDS); + writel(kick, d + D_KICK); + writel(0, d + D_STATUS); + writel(0, d + D_CTAG); + writel(AVDQ_FLAG_SRAM, d + D_FLAGS); + writel(extra, d + D_PP_EXTRA); + writel(extra ? cont : 0, d + D_PP_CONT); + writel(0, d + D_RSVD0); + writel(0, d + D_RSVD1); + writel(AVDQ_STATE_SUBMITTED, d + D_STATE); + /* Descriptor before the doorbell that hands it to the firmware. */ + wmb(); + (void)readl(d + D_STATE); /* flush posted aperture writes */ + writel(q->aperture + AVDQ_RING_OFFSET + f->idx * AVDQ_DESC_BYTES, + avd->mbox + AVD_REG_MBOX0_SEND); + + /* The firmware may start feeding while the rest of the program streams in. */ + ret = avd_qfw_stream(avd, words, f->prog_words); + if (ret) { + dev_err_ratelimited(avd->dev, "queue: program stream failed: %d\n", ret); + /* The descriptor is already posted: only a reset releases it. */ + f->deadline = jiffies; + } + return 0; +} + +static irqreturn_t avd_qfw_irq(struct avd_dev *avd) +{ + struct avd_qfw *q = avd->qfw; + struct avd_frame *f; + void __iomem *d; + u32 word, state, status, ctag; + unsigned int idx; + + if (!avd_mbox_pop(avd, &word)) + return IRQ_HANDLED; + if ((word & AVDQ_NOTIFY_MASK) != AVDQ_NOTIFY) { + dev_err_ratelimited(avd->dev, "queue: firmware reported %#x, resetting\n", word); + avd_qfw_fail_all(avd); + return IRQ_HANDLED; + } + idx = word & 15; + d = avd_qfw_desc(q, idx); + state = readl(d + D_STATE); + status = readl(d + D_STATUS); + ctag = readl(d + D_CTAG); + f = &avd->frames[idx]; + if (state != AVDQ_STATE_COMPLETE || ((ctag ^ word) & 0xfff0) || + f->state != AVD_FRAME_SUBMITTED || ctag != f->tag) { + dev_warn_ratelimited(avd->dev, "queue: stale completion %#x (state %u tag %#x)\n", + word, state, ctag); + return IRQ_HANDLED; + } + writel(0, d + D_STATE); + avd_frame_complete(avd, ctag, status == 0 ? 0 : status == 1 ? -EINVAL : -EIO); + if (status > 1) + avd_qfw_fail_all(avd); + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_qfw_backend = { + .name = "open-queue", + .boot = avd_qfw_boot, + .shutdown = avd_shutdown, + .max_inflight = avd_qfw_max_inflight, + .submit = avd_qfw_submit, + .irq = avd_qfw_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, +}; + +bool avd_backend_is_qfw(const struct avd_dev *avd) +{ + return avd->backend == &avd_qfw_backend; +} + +/* + * SRAM survives a CM3 reload, so a stale info block from an earlier ABI 2 + * image could make a legacy image look queue capable. Clear the magic before + * every open-firmware boot; only an image that publishes it again qualifies. + */ +void avd_qfw_prepare_boot(struct avd_dev *avd) +{ + if (avd->qfw) + writel(0, avd->qfw->sram + AVDQ_INFO); +} + +/* + * Called after every successful open-firmware boot (probe, resume, reset), + * with no frame owned by the firmware. Picks the queued transport when the + * loaded image speaks ABI 2 for this node, else the legacy one. + */ +void avd_qfw_select(struct avd_dev *avd) +{ + const struct avd_backend_ops *old = avd->backend; + + if (old != &avd_open_backend && old != &avd_qfw_backend) + return; + if (avd_qfw_check(avd)) { + /* Legacy boot left mailbox 0 disabled. */ + writel_relaxed(AVD_MBOX_ENABLE, avd->mbox + AVD_REG_MBOX0_STATUS); + avd->backend = &avd_qfw_backend; + /* + * Codecs held back for a newer firmware (T6040 HEVC) are + * enabled by a queue image that lists them in its codec mask. + */ + avd->capabilities = avd->fw_capabilities | + (avd->variant->capabilities & avd->qfw->codec_mask); + } else { + avd->backend = &avd_open_backend; + avd->capabilities = avd->fw_capabilities; + } + if (avd->backend != old) + dev_info(avd->dev, "using %s transport (%u frames in flight)\n", + avd->backend->name, avd->backend->max_inflight(avd)); +} diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index bf5bf96f4c62a..fc23c0935f36f 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -22,8 +22,7 @@ #include "avd.h" #include "avd-inst.h" #include "avd-regs.h" - -#define AVD_CODEC_INFO_MAX_SIZE 0x40 +#include "avd-apple-protocol.h" static void calc_tile_meta(u32 w, u32 h, u32 bpb, u32 tile_dim, u32 meta_hdr_bytes, u32 *tile, u32 *meta) @@ -73,31 +72,6 @@ void fill_comp(struct avd_comp *comp, enum avd_image_fmt image_fmt, u32 width, comp->size = y_meta + y + uv_meta + uv; } -int avd_buf_alloc(struct avd_dev *avd, struct avd_buf *buf, size_t size) -{ - /* Keep size equal to the payload length used by the H.264 slice path. */ - avd_buf_free(avd, buf); - - if (size <= 0) - return -ENOMEM; - - buf->size = size; - buf->cpu = - dma_alloc_coherent(avd->dev, buf->size, &buf->addr, GFP_KERNEL); - if (!buf->cpu) { - memset(buf, 0, sizeof(*buf)); - return -ENOMEM; - } - return 0; -} - -void avd_buf_free(struct avd_dev *avd, struct avd_buf *buf) -{ - if (buf->cpu) - dma_free_coherent(avd->dev, buf->size, buf->cpu, buf->addr); - memset(buf, 0, sizeof(*buf)); -} - struct avd_decoded_buffer * avd_get_ref_buf(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst, u64 timestamp) { @@ -116,28 +90,6 @@ avd_get_ref_buf(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst, u64 timestamp) return vb2_to_avd_decoded_buf(buf); } -static int avd_wait_submission_queue(struct avd_ctx *ctx, int vp) -{ - struct avd_dev *avd = ctx->dev; - u32 max = readl_relaxed(avd->ctrl + - avd->variant->submit_queue_max_offset + vp * 4); - u32 cur = readl_relaxed( - avd->ctrl + avd->variant->submit_queue_status_offset + vp * 4); - - if (cur == max) { - dev_err(avd->dev, "instruction que full! %d/%d", cur, max); - return 1; - } - - if (cur >= max / 2) { - if (avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) - usleep_range(100, 150); - else - usleep_range(500, 650); - } - return 0; -} - int avd_init_job(struct avd_ctx *ctx, enum avd_codec codec, size_t segments) { int ret = 0; @@ -155,287 +107,20 @@ int avd_init_job(struct avd_ctx *ctx, enum avd_codec codec, size_t segments) int avd_submit_job(struct avd_ctx *ctx) { - struct avd_dev *avd = ctx->dev; - struct avd_job *sub = &ctx->job; - struct avd_segment *segments, *seg; - unsigned long flags; - size_t num; - int i, idx = 0, vp = 0; - void __iomem *reg; - u32 exec_mask = sub->codec == AVD_CODEC_VP9 && - avd->variant->revision == 3 ? - AVD_OP_EXEC_REV3_VP9_MASK : - 0; - u32 exec_rev_flag = - AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | - AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3); - - ctx->fifo_idx = 0; - for (i = 0; i < sub->codec; i++) - vp += avd->variant->vp_slots[i]; - if (avd->variant->vp_slot_base[sub->codec]) - vp = avd->variant->vp_slot_base[sub->codec]; - - /* - * avd_device_run() keeps this job from being finished (and the next one - * from starting) until it returns, and arms the watchdog itself once the - * job belongs to the hardware. Still walk a local copy of the table. - */ - segments = sub->segments; - num = sub->num; - if (WARN_ON_ONCE(!segments)) - return -EINVAL; - - avd->variant->configure_stream(avd, ctx->inst.addr, ctx->fifo_idx, vp); - reg = avd->ctrl + avd->variant->vp_slot_offset + vp * 4; - - /* the first segment is always the header (needs special handling) */ - - writel(AVD_OP_EXEC | exec_mask | exec_rev_flag | - AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx), - reg); - seg = &segments[idx++]; - for (i = 0; i < seg->num; i++) - writel(seg->instructions[i], reg); - for (; idx <= num; idx++) { - seg = &segments[idx]; - for (i = 0; i < seg->num; i++) - writel(seg->instructions[i], reg); - if ((!(avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) || - sub->codec == AVD_CODEC_HEVC) && - avd_wait_submission_queue(ctx, vp)) - break; - if (idx == num) { - /* A completion from here on belongs to this job. */ - spin_lock_irqsave(&avd->job_lock, flags); - avd->job_end_sent = true; - spin_unlock_irqrestore(&avd->job_lock, flags); - } - writel(AVD_OP_EXEC | exec_mask | - AVD_OP_EXEC_FLAG_END(idx == num), - reg); - } - - if (sub->segments == segments) - sub->segments = NULL; - kvfree(segments); - return 0; -} - -static int avd_reset(struct avd_dev *avd) -{ - int ret = 0; - - ret = pm_runtime_resume_and_get(avd->dev); - if (ret < 0) - return ret; - - ret = reset_control_reset(avd->rstc); - if (ret) - dev_err(avd->dev, "reset: failed: %d", ret); - - if (avd->empty_domain) { - iommu_attach_device(avd->empty_domain, avd->dev); - iommu_detach_device(avd->empty_domain, avd->dev); - } - - ret = avd_boot(avd); - if (ret) - dev_err(avd->dev, "reset: failed to boot"); - - pm_runtime_put_autosuspend(avd->dev); - - return ret; -} - -static void avd_watchdog_func(struct work_struct *work) -{ - struct avd_dev *avd; - struct avd_ctx *ctx; - unsigned long flags; - bool claimed; - int ret; - ctx = container_of(to_delayed_work(work), struct avd_ctx, - watchdog_work); - if (!ctx) - return; - - avd = ctx->dev; - - /* - * Time out only the job this watchdog was armed for, and only while the - * hardware still owns it. Otherwise that job already finished, and - * resetting the hardware would break whichever job is running now. - */ - spin_lock_irqsave(&avd->job_lock, flags); - claimed = avd->job_ctx == ctx && avd->job_seq == ctx->wd_seq && - avd->job_state == AVD_JOB_SUBMITTED; - if (claimed) - avd->job_state = AVD_JOB_FINISHING; - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: watchdog seq %llu: %s\n", ctx->wd_seq, - claimed ? "timeout" : "stale, ignored"); - if (!claimed) - return; - - dev_err(avd->dev, "Frame processing timed out!"); - - writel(0, avd->mbox + AVD_REG_MBOX_IRQ_ENABLE); - ret = avd_reset(avd); - if (ret) - dev_err(avd->dev, "failed to reset: %d", ret); - - avd_job_finish(ctx, VB2_BUF_STATE_ERROR); + return avd_open_build_program(ctx); } static irqreturn_t avd_irq_handler(int irq, void *data) { struct avd_dev *avd = data; - struct avd_ctx *ctx = v4l2_m2m_get_curr_priv(avd->m2m_dev); - - enum vb2_buffer_state state; - unsigned long flags; - bool finish = false, stale = false; - u32 status; - if (!ctx) - return IRQ_HANDLED; - - status = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); - - writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); - - if (status & 0x10000) { /* dbg */ - dev_warn(avd->dev, "no handler for IRQ: %3d", status &~0x10000); - writel_relaxed(0, avd->mbox + AVD_REG_MBOX_IRQ_ENABLE); - return IRQ_HANDLED; - } - - if (status & 0x1000) { - /* pp is done ! we are done */ - state = VB2_BUF_STATE_DONE; - } else if (status & 0x100) { - /* a vp is done, kick the pp and hope for the best */ - if(ctx->coded_fmt_desc->ops->submit) - ctx->coded_fmt_desc->ops->submit(ctx); - goto done; - } else { - dev_err(avd->dev, "H%d error", status); - /* - * avd-fw err(n): VP n raised its error interrupt and stopped, - * and the PP only starts on a VP done, so the job cannot - * complete any more. Run the watchdog now instead of after its - * timeout: it resets the AVD and fails only this frame, and it - * ignores a job the hardware does not own yet. - */ - mod_delayed_work(system_wq, &ctx->watchdog_work, 0); - goto done; - } - - /* - * Finish only a job the hardware owns. While avd_device_run() is still - * setting up or submitting, a completion is left over from an older job - * unless this job's END command already went out; then remember it and - * let avd_device_run() finish the job when it returns. - */ - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_ctx != ctx) { - stale = true; - } else if (avd->job_state == AVD_JOB_SUBMITTED) { - avd->job_state = AVD_JOB_FINISHING; - finish = true; - } else if (avd->job_state == AVD_JOB_RUNNING && avd->job_end_sent) { - avd->job_pending = true; - avd->job_pending_result = state; - } else { - stale = true; - } - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: irq %#x: %s\n", status, - finish ? "finish" : stale ? "stale, ignored" : "pending"); - if (stale) - dev_warn_ratelimited(avd->dev, "ignoring stale completion"); - if (finish) { - cancel_delayed_work(&ctx->watchdog_work); - avd_job_finish(ctx, state); - } + /* An interrupt raised before a reset or power-off belongs to that instance. */ + unsigned int gen = READ_ONCE(avd->pm.irq_gen); + irqreturn_t ret; -done: - return IRQ_HANDLED; -} - -static void avd_device_run(void *priv) -{ - struct avd_ctx *ctx = priv; - struct avd_dev *avd = ctx->dev; - const struct avd_coded_fmt_desc *desc = ctx->coded_fmt_desc; - enum vb2_buffer_state result = VB2_BUF_STATE_ERROR; - const char *outcome = "already finished"; - unsigned long flags; - bool finish = false; - u64 seq; - int ret; - - if (WARN_ON(!desc)) - return; - - /* - * IRQ completion cannot wait for an already running watchdog. Drain it - * here, before reusing wd_seq, so an old callback cannot time out this - * context's next job. device_run is sleepable; hold no job_lock here. - */ - cancel_delayed_work_sync(&ctx->watchdog_work); - - ret = pm_runtime_resume_and_get(avd->dev); - if (ret < 0) { - avd_job_finish_no_pm(ctx, VB2_BUF_STATE_ERROR); - return; - } - - /* - * While the job is RUNNING neither completion IRQs nor the watchdog may - * finish it: v4l2-mem2mem would start the next job on a kworker while - * this one still builds and submits ctx->job. - */ - spin_lock_irqsave(&avd->job_lock, flags); - seq = ++avd->job_seq; - avd->job_ctx = ctx; - avd->job_state = AVD_JOB_RUNNING; - avd->job_end_sent = false; - avd->job_pending = false; - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: run seq %llu\n", seq); - - ret = desc->ops->run(ctx); - if (ret) { - dev_dbg(avd->dev, "avd-job: run seq %llu failed: %d\n", seq, ret); - avd_job_finish(ctx, VB2_BUF_STATE_ERROR); - return; - } - - /* - * The run either finished the job itself (held capture buffer) or handed - * it to the hardware: arm the watchdog, or finish right away if the - * completion already arrived. - */ - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_seq == seq && avd->job_state == AVD_JOB_RUNNING) { - if (avd->job_pending) { - avd->job_state = AVD_JOB_FINISHING; - result = avd->job_pending_result; - finish = true; - outcome = "completed during submission"; - } else { - avd->job_state = AVD_JOB_SUBMITTED; - ctx->wd_seq = seq; - schedule_delayed_work(&ctx->watchdog_work, msecs_to_jiffies(2000)); - outcome = "submitted"; - } - } - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: run seq %llu: %s\n", seq, outcome); - - if (finish) - avd_job_finish(ctx, result); + mutex_lock(&avd->queue_mutex); + ret = avd_pm_irq(avd, gen); + mutex_unlock(&avd->queue_mutex); + return ret; } /* @@ -444,12 +129,7 @@ static void avd_device_run(void *priv) * userspace decode ahead to ride out bitrate spikes the hardware can't * decode in real time (e.g. 150+ Mbit/s 4K60 H.264). */ -static unsigned int max_capture_buffers = 128; -module_param(max_capture_buffers, uint, 0444); -MODULE_PARM_DESC(max_capture_buffers, - "Maximum number of capture buffers per context (default 128)"); - -static int avd_queue_init(void *priv, struct vb2_queue *src_vq, +static int avd_m2m_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq) { struct avd_ctx *ctx = priv; @@ -486,7 +166,7 @@ static int avd_queue_init(void *priv, struct vb2_queue *src_vq, dst_vq->lock = &ctx->dev->vdev_lock; dst_vq->dev = ctx->dev->v4l2_dev.dev; dst_vq->allow_cache_hints = true; - dst_vq->max_num_buffers = max(max_capture_buffers, VB2_MAX_FRAME); + dst_vq->max_num_buffers = max(128U, VB2_MAX_FRAME); return vb2_queue_init(dst_vq); } @@ -507,14 +187,15 @@ static int avd_open(struct file *filp) if (ret) goto err_free_ctx; - INIT_DELAYED_WORK(&ctx->watchdog_work, avd_watchdog_func); + INIT_LIST_HEAD(&ctx->queue_node); + ctx->next_dep = AVD_DEP_PREV_DONE; avd_reset_coded_fmt(ctx); avd_reset_decoded_fmt(ctx); v4l2_fh_init(&ctx->fh, video_devdata(filp)); - ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(avd->m2m_dev, ctx, avd_queue_init); + ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(avd->m2m_dev, ctx, avd_m2m_queue_init); if (IS_ERR(ctx->fh.m2m_ctx)) { ret = PTR_ERR(ctx->fh.m2m_ctx); goto err_free_ctx; @@ -524,6 +205,7 @@ static int avd_open(struct file *filp) if (ret) goto err_cleanup_m2m_ctx; + avd_queue_add_ctx(ctx); v4l2_fh_add(&ctx->fh, filp); return 0; @@ -542,9 +224,9 @@ static int avd_release(struct file *filp) struct avd_ctx *ctx = file_to_ctx(filp); v4l2_fh_del(&ctx->fh, filp); + avd_queue_del_ctx(ctx); v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); - /* The job has finished; its watchdog must not run on the freed ctx. */ - cancel_delayed_work_sync(&ctx->watchdog_work); + cancel_work_sync(&ctx->submit_work); v4l2_ctrl_handler_free(&ctx->ctrl_hdl); v4l2_fh_exit(&ctx->fh); avd_buf_free(ctx->dev, &ctx->inst); @@ -564,7 +246,9 @@ static const struct v4l2_file_operations avd_fops = { }; static const struct v4l2_m2m_ops avd_m2m_ops = { - .device_run = avd_device_run, + .device_run = avd_queue_run, + .job_ready = avd_queue_ready, + .job_abort = avd_queue_abort, }; static const struct media_device_ops avd_media_ops = { @@ -807,10 +491,12 @@ static const struct avd_variant avd_t8132_variant = { * T6040 (M4 Pro): the T8132 decoder. Its first HEVC job stalled the VP when * it was sent the pre-v5 header (no DMA configuration words, no reserved * scratch slot); T6040 runs the same v5 firmware generation as T8140, so it - * gets the v5 header. HEVC stays hidden from V4L2 clients until that is - * verified on the hardware. + * gets the v5 header. HEVC is advertised only when the loaded firmware + * names h265 in its CODECS trailer. */ static const struct avd_variant avd_t6040_variant = { + .apple = &avd_borage_hw, + .fp_size = 0x1130d0, .vp_slots = { [AVD_CODEC_HEVC] = 4, [AVD_CODEC_H264] = 4, @@ -819,10 +505,15 @@ static const struct avd_variant avd_t6040_variant = { }, .fifo_slots = 15, .configure_stream = t8122_configure_stream, - .capabilities = AVD_CAPABILITY_H264 | + .capabilities = AVD_CAPABILITY_HEVC | + AVD_CAPABILITY_H264 | AVD_CAPABILITY_VP9 | AVD_CAPABILITY_AV1, - .fw_name = "apple/avd-fw-v5-t1.bin", + /* HEVC only with firmware whose CODECS trailer lists h265. */ + .fw_caps_required = AVD_CAPABILITY_HEVC, + /* v5-t2 matches T6040's SRAM aperture, which the queued transport needs. */ + .fw_name = "apple/avd-fw-v5-t2.bin", + .fw_name_fallback = "apple/avd-fw-v5-t1.bin", .revision = 4, .vp_slot_offset = 0xc, .submit_offset = 0x40, @@ -870,11 +561,17 @@ static const struct of_device_id avd_of_match[] = { MODULE_DEVICE_TABLE(of, avd_of_match); +static void avd_release_open_firmware(void *data) +{ + struct avd_dev *avd = data; + + release_firmware(avd->fw); +} + static int avd_probe(struct platform_device *pdev) { struct avd_dev *avd; const struct of_device_id *match; - const char *codec_info = NULL; int ret, irq; avd = devm_kzalloc(&pdev->dev, sizeof(*avd), GFP_KERNEL); @@ -886,12 +583,17 @@ static int avd_probe(struct platform_device *pdev) avd->pdev = pdev; mutex_init(&avd->vdev_lock); - spin_lock_init(&avd->job_lock); + avd_pm_init(avd, NULL, &avd_pm_queue_ops); + ret = avd_queue_init(avd); + if (ret) + return ret; match = of_match_node(avd_of_match, pdev->dev.of_node); avd->variant = match->data; avd->rstc = devm_reset_control_get_exclusive(avd->dev, NULL); + if (IS_ERR(avd->rstc)) + return PTR_ERR(avd->rstc); avd->code = devm_platform_ioremap_resource_byname(pdev, "code"); if (IS_ERR(avd->code)) @@ -902,6 +604,9 @@ static int avd_probe(struct platform_device *pdev) avd->ctrl = devm_platform_ioremap_resource_byname(pdev, "ctrl"); if (IS_ERR(avd->ctrl)) return PTR_ERR(avd->ctrl); + ret = avd_pm_map_piodma(avd); + if (ret) + return ret; avd->domain = iommu_get_domain_for_dev(avd->dev); if (avd->domain) { @@ -912,39 +617,16 @@ static int avd_probe(struct platform_device *pdev) } } - ret = request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); - - if (ret) { - dev_err(avd->dev, "failed to load firmware: %d", ret); + ret = avd_backend_select(avd); + if (ret) + return ret; + ret = devm_add_action_or_reset(avd->dev, avd_release_open_firmware, avd); + if (ret) return ret; - } - - /* Determine firmware and HW variant capabilities */ - if (avd->fw->size >= AVD_CODEC_INFO_MAX_SIZE) - codec_info = avd->fw->data + avd->fw->size - AVD_CODEC_INFO_MAX_SIZE; - if (codec_info && strncmp(codec_info, "CODECS:", 7) == 0) { - unsigned int caps = 0; - unsigned int variant_caps = avd->variant->capabilities; - - if (strnstr(codec_info, "av1", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_AV1; - if (strnstr(codec_info, "h264", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_H264; - if (strnstr(codec_info, "h265", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_HEVC; - if (strnstr(codec_info, "vp9", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_VP9; - - dev_info(avd->dev, "Firmware codecs: %.27s\n", codec_info + 7); - - avd->capabilities = caps; - } else { - /* Backward compatibility to old firmware, do not upstream. */ - avd->capabilities = avd->variant->capabilities; - } ret = dma_set_mask_and_coherent(avd->dev, - DMA_BIT_MASK((avd->variant->quirks & AVD_QUIRK_LSR) ? 38 : 64)); + DMA_BIT_MASK(avd_backend_is_apple(avd) ? 42 : + (avd->variant->quirks & AVD_QUIRK_LSR) ? 38 : 64)); if (ret) { dev_err(avd->dev, "Failed to set DMA mask"); @@ -954,17 +636,27 @@ static int avd_probe(struct platform_device *pdev) irq = platform_get_irq(pdev, 1); if (irq < 0) return irq; + avd->pm.irq = irq; ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, - avd_irq_handler, IRQF_ONESHOT, + avd_irq_handler, IRQF_ONESHOT | IRQF_NO_AUTOEN, dev_name(&pdev->dev), avd); if (ret) { dev_err(avd->dev, "Could not request IRQ 1"); return ret; } - pm_runtime_set_autosuspend_delay(avd->dev, 100); - pm_runtime_use_autosuspend(avd->dev); - pm_runtime_enable(avd->dev); + avd_pm_enable(avd); + + /* + * Boot the firmware once now: the transport and the codec list depend + * on what the loaded image reports, and clients enumerate formats and + * size their contexts before the first decode would power it up. + */ + ret = pm_runtime_resume_and_get(avd->dev); + if (ret < 0) + dev_warn(avd->dev, "firmware probe boot failed: %d\n", ret); + else + pm_runtime_put_autosuspend(avd->dev); ret = avd_v4l2_init(avd); if (ret) @@ -981,42 +673,19 @@ static int avd_probe(struct platform_device *pdev) static void avd_remove(struct platform_device *pdev) { struct avd_dev *avd = platform_get_drvdata(pdev); + int ret; - release_firmware(avd->fw); - + v4l2_m2m_suspend(avd->m2m_dev); + ret = avd->pm.queue->drain(avd); + if (ret) + dev_warn(avd->dev, "queue drain failed, resetting before teardown: %d\n", ret); + avd_queue_cleanup(avd); + avd_pm_disable(avd); avd_v4l2_cleanup(avd); - if (avd->empty_domain) iommu_domain_free(avd->empty_domain); - - pm_runtime_disable(avd->dev); - pm_runtime_dont_use_autosuspend(avd->dev); } -static __maybe_unused int avd_runtime_resume(struct device *dev) -{ - int ret; - struct avd_dev *avd = platform_get_drvdata(to_platform_device(dev)); - - ret = avd_boot(avd); - if (ret) - dev_err(dev, "failed to boot"); - return ret; -} - -static __maybe_unused int avd_runtime_suspend(struct device *dev) -{ - struct avd_dev *avd = platform_get_drvdata(to_platform_device(dev)); - - avd_shutdown(avd); - return 0; -} - -static const struct dev_pm_ops avd_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS( - pm_runtime_force_suspend, - pm_runtime_force_resume) SET_RUNTIME_PM_OPS(avd_runtime_suspend, - avd_runtime_resume, NULL) }; - static struct platform_driver avd_driver = { .probe = avd_probe, .remove = avd_remove, diff --git a/drivers/media/platform/apple/avd/avd-h264-fp.c b/drivers/media/platform/apple/avd/avd-h264-fp.c new file mode 100644 index 0000000000000..6c221e74b9b78 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-h264-fp.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* H.264 frame-parameter facts. No firmware or captured frame images. */ +#include +#include +#include +#include + +#include "avd.h" +#include "avd-queue.h" +#include "avd-h264-fp.h" + +/* + * Each generation has an independent provenance. Register revisions or firmware + * tiers do not imply a frame layout. Select this enum from OF match data only. + * + * V3: https://github.com/eiln/avd/blob/main/avid/h264/fp.py (MIT), with + * avid/h264/halv3.py for the address shifts and instruction fields. + * Borage: clean-room protocol SPEC (2026-10-31), sections 4.1, 6.1, 6.2. + * T6000 selects LilyD: V3 is comparison evidence, not its selected Apple ABI. + * TODO rows: Daybreak per-SoC ledger task t-3862; no V3/Borage fallback. + */ +struct avd_h264_fp_layout_info { + enum avd_h264_fp_layout layout; + size_t size; + const char *source; +}; + +static const struct avd_h264_fp_layout_info avd_h264_fp_layouts[] = { + { AVD_H264_FP_V3, 0x8b4e0, "MIT eiln/avd avid/h264/fp.py: Viola/V3 comparison" }, + { AVD_H264_FP_BORAGE, 0x1130d0, "SPEC 6.2: T6040/Borage" }, + { AVD_H264_FP_T8112_TODO, 0, "TODO t-3862: T8112 per-SoC ledger" }, + { AVD_H264_FP_T8140_TODO, 0, "TODO t-3862: T8140 per-SoC ledger" }, +}; + +/* Offsets and array capacities from the public V3 layout, not Borage. */ +static const struct { + u16 offset; + u16 count; +} v3_fields[AVD_H264_FP_NONE] = { + [AVD_H264_FP_MV_ABOVE_INFO] = { 0xb8, 1 }, + [AVD_H264_FP_MV_COLOR] = { 0xbc, 1 }, + [AVD_H264_FP_HDR_D0_REF_HDR] = { 0xd0, 16 }, + [AVD_H264_FP_HDR_4C_PIC_SCALING_LIST_DIMS] = { 0x4c, 1 }, + [AVD_H264_FP_SCL_46C_PIC_SCALING_MATRIX_4X4] = { 0x46c, 24 }, + [AVD_H264_FP_SCL_4CC_PIC_SCALING_MATRIX_8X8] = { 0x4cc, 96 }, + [AVD_H264_FP_HDR_34_START_HDR] = { 0x34, 1 }, + [AVD_H264_FP_HDR_38_MODE] = { 0x38, 1 }, + [AVD_H264_FP_HDR_3C_HEIGHT_WIDTH] = { 0x3c, 1 }, + [AVD_H264_FP_HDR_40_ZERO] = { 0x40, 1 }, + [AVD_H264_FP_HDR_28_HEIGHT_WIDTH_SHIFT3] = { 0x28, 1 }, + [AVD_H264_FP_HDR_2C_SPS_PARAM] = { 0x2c, 1 }, + [AVD_H264_FP_HDR_44_FLAGS] = { 0x44, 1 }, + [AVD_H264_FP_HDR_48_CHROMA_QP_INDEX_OFFSET] = { 0x48, 1 }, + [AVD_H264_FP_HDR_58_CONST_3A] = { 0x58, 1 }, + [AVD_H264_FP_HDR_9C_PPS_TILE_ADDR_LSB8] = { 0x9c, 1 }, + [AVD_H264_FP_IP_ABOVE] = { 0xa0, 1 }, + [AVD_H264_FP_LF_ABOVE] = { 0xa4, 1 }, + [AVD_H264_FP_LF_ABOVE_INFO] = { 0xa8, 1 }, + [AVD_H264_FP_HDR_210_Y_ADDR_LSB8] = { 0x210, 1 }, + [AVD_H264_FP_HDR_218_WIDTH_ALIGN] = { 0x218, 1 }, + [AVD_H264_FP_HDR_214_UV_ADDR_LSB8] = { 0x214, 1 }, + [AVD_H264_FP_HDR_21C_WIDTH_ALIGN] = { 0x21c, 1 }, + [AVD_H264_FP_HDR_54_HEIGHT_WIDTH] = { 0x54, 1 }, + [AVD_H264_FP_SLC_76C_CMD_WEIGHTS_DENOM] = { 0x76c, 1 }, + [AVD_H264_FP_SLC_LUMA_WEIGHTS] = { 0x770, 96 }, + [AVD_H264_FP_SLC_LUMA_OFFSETS] = { 0x8f0, 96 }, + [AVD_H264_FP_SLC_CHROMA_WEIGHTS_0_] = { 0x770, 96 }, + [AVD_H264_FP_SLC_CHROMA_OFFSETS_0_] = { 0x8f0, 96 }, + [AVD_H264_FP_SLC_CHROMA_WEIGHTS_1_] = { 0x770, 96 }, + [AVD_H264_FP_SLC_CHROMA_OFFSETS_1_] = { 0x8f0, 96 }, + [AVD_H264_FP_SLC_A7C_CMD_SET_CODED_SLICE] = { 0xa7c, 1 }, + [AVD_H264_FP_SLC_A84_SLICE_ADDR_LOW] = { 0xa84, 1 }, + [AVD_H264_FP_SLC_A88_SLICE_HDR_SIZE] = { 0xa88, 1 }, + [AVD_H264_FP_SLC_A70_CMD_QUANT_PARAM] = { 0xa70, 1 }, + [AVD_H264_FP_SLC_A74_CMD_DEBLOCKING_FILTER] = { 0xa74, 1 }, + [AVD_H264_FP_SLC_6E8_CMD_REF_LIST_0] = { 0x6e8, 32 }, + [AVD_H264_FP_SLC_6E4_CMD_REF_TYPE] = { 0x6e4, 1 }, + [AVD_H264_FP_SLC_A78_SPS_TILE_ADDR2_LSB8] = { 0xa78, 1 }, +}; + +static int fp_word(struct avd_buf *buf, size_t offset, u32 word) +{ + if (!buf || !buf->cpu || buf->size < sizeof(u32) || + offset > buf->size - sizeof(u32)) + return -ENOSPC; + put_unaligned_le32(word, (u8 *)buf->cpu + offset); + return 0; +} + +int avd_h264_fp_begin(struct avd_h264_fp_cursor *cursor, struct avd_buf *buf, + enum avd_h264_fp_layout layout) +{ + unsigned int i; + size_t size = 0; + + memset(cursor, 0, sizeof(*cursor)); + for (i = 0; i < ARRAY_SIZE(avd_h264_fp_layouts); i++) + if (avd_h264_fp_layouts[i].layout == layout) + size = avd_h264_fp_layouts[i].size; + if (layout != AVD_H264_FP_V3 || !size) + return -EOPNOTSUPP; + if (!buf || !buf->cpu || buf->size < size) + return -ENOSPC; + memset(buf->cpu, 0, size); + cursor->buf = buf; + /* V3 piodma words are transport-owned; do not invent their encodings. */ + fp_word(buf, 0x04, 1); + fp_word(buf, 0x14, 0xdeadcafe); + fp_word(buf, 0x18, 0x101); + fp_word(buf, 0x1c, 1); + fp_word(buf, 0x20, 0x8b4c0); + /* Public DFW metadata in AVDH264V3DFWScalingList. */ + fp_word(buf, 0x284, 0x1de28b5); + fp_word(buf, 0x468, 0x1de28b5); + fp_word(buf, 0x64c, 0x764099); + fp_word(buf, 0x6c8, 0x124111); + return 0; +} + +/* MIT fp.py: 600 slice records, 0x3b4 bytes each, before input +0x8b4c0. */ +int avd_h264_fp_slice(struct avd_h264_fp_cursor *cursor, unsigned int index, + unsigned int count) +{ + unsigned int i; + + if (!cursor->buf || !count || count > 600 || index >= count) + return -EINVAL; + cursor->slice = index; + cursor->weights = 0; + cursor->offsets = 0; + for (i = AVD_H264_FP_SLC_76C_CMD_WEIGHTS_DENOM; i < AVD_H264_FP_NONE; i++) + cursor->count[i] = 0; + return fp_word(cursor->buf, 0x1c, count); +} + +int avd_h264_fp_emit(struct avd_h264_fp_cursor *cursor, + enum avd_h264_fp_field field, u32 word) +{ + unsigned int index; + + if (field == AVD_H264_FP_NONE) + return 0; + if ((unsigned int)field >= AVD_H264_FP_NONE) + return -EINVAL; + /* Luma and both chroma components share one ordered weights array. */ + if (v3_fields[field].offset == 0x770) + index = cursor->weights++; + else if (v3_fields[field].offset == 0x8f0) + index = cursor->offsets++; + else + index = cursor->count[field]++; + if (index >= v3_fields[field].count) + return -E2BIG; + return fp_word(cursor->buf, v3_fields[field].offset + index * 4 + + (v3_fields[field].offset >= 0x6e0 ? cursor->slice * 0x3b4 : 0), + word); +} + +int avd_h264_fp_address(struct avd_h264_fp_cursor *cursor, + enum avd_h264_fp_field field, unsigned int index, + dma_addr_t addr) +{ + if (field == AVD_H264_FP_NONE) + return 0; + /* Builder scratch annotations already identify distinct fields. */ + if (field != AVD_H264_FP_HDR_9C_PPS_TILE_ADDR_LSB8) + index = 0; + if ((unsigned int)field >= AVD_H264_FP_NONE || index > 7 || + (addr & 0xff) || (addr >> 8) > U32_MAX) + return -ERANGE; + return fp_word(cursor->buf, v3_fields[field].offset + index * 4 + + (v3_fields[field].offset >= 0x6e0 ? cursor->slice * 0x3b4 : 0), + addr >> 8); +} + +int avd_h264_fp_ref(struct avd_h264_fp_cursor *cursor, bool reference, + unsigned int index, unsigned int plane, dma_addr_t addr) +{ + if (plane >= 4 || index >= 16 || (addr & 0x7f) || (addr >> 7) > U32_MAX) + return -ERANGE; + return fp_word(cursor->buf, + reference ? 0x110 + plane * 0x40 + index * 4 : 0xc0 + plane * 4, + addr >> 7); +} + +/* + * [C] Borage fp+0x3d8 bits 0:6 -> cmd+0x00 bits 2:8 (SPEC 6.2). + * [C] Borage fp+0x18/+0x19 -> cmd+0x00 bits 0/1 (SPEC 6.1). + * [C] cmd+0x3c/+0x44 combine fp+0x3dc/+0x5c0/+0x7a4 and + * MMIO 0x01482220; the shift/mask rules are absent from SPEC 6.2. + * The two +0x1c work counts' exact bounds are also not specified there. + * Keep these fields unnamed; no guessed command may reach firmware. + */ +int avd_h264_borage_cmd_fields(struct avd_frame *f) +{ + const u8 *fp; + u32 work, slot, flags; + u16 count; + + if (!f || !f->fp || !f->fp->cpu || f->fp->size < 0x1130d0 || + f->idx >= AVD_MAX_FRAMES) + return -EINVAL; + fp = f->fp->cpu; + count = get_unaligned_le16(fp + 0x1a); + /* validateAvcFields' independently established slice-count bound. */ + if (count >= 0x385) + return -EINVAL; + work = (f->cmd[1] >> 16) & 0xffff; + if (work >= 4) + return -EINVAL; + work = 0x01483120 + 0x1814 * work; + slot = 0x1130d0 * f->idx; + flags = (fp[0x18] & 1) | ((fp[0x19] & 1) << 1) | + ((get_unaligned_le32(fp + 0x3d8) & 0x7f) << 2) | ((u32)count << 16); + /* Preserve the common prefix outside the codec's documented masks. */ + f->cmd[0] = (f->cmd[0] & ~0xffff01ffU) | flags; + f->cmd[1] = (f->cmd[1] & 0xffff0000U) | get_unaligned_le16(fp + 0x1c); + f->cmd[3] = work; + f->cmd[4] = slot + 0x3dc; + f->cmd[5] = work + 0x0c; + f->cmd[6] = slot + 0x5c0; + f->cmd[7] = slot + 0x8a0; + f->cmd[10] = work + 0x34; + f->cmd[11] = work + 0x350; + f->cmd[12] = work + 0x3ac; + /* Projection is reviewable/testable, but not a publishable descriptor. */ + return -EOPNOTSUPP; +} + +int avd_h264_fp_pack_cmd(enum avd_h264_fp_layout layout, struct avd_frame *f) +{ + /* V3 command transport is not the Borage 0x60 command (SPEC 6.2). */ + if (layout != AVD_H264_FP_BORAGE) + return -EOPNOTSUPP; + return avd_h264_borage_cmd_fields(f); +} diff --git a/drivers/media/platform/apple/avd/avd-h264-fp.h b/drivers/media/platform/apple/avd/avd-h264-fp.h new file mode 100644 index 0000000000000..d750ba3f5a36a --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-h264-fp.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_H264_FP_H_ +#define AVD_H264_FP_H_ + +#include + +struct avd_buf; +struct avd_frame; + +enum avd_h264_fp_layout { + AVD_H264_FP_UNKNOWN, + AVD_H264_FP_V3, + AVD_H264_FP_BORAGE, + AVD_H264_FP_T8112_TODO, + AVD_H264_FP_T8140_TODO, +}; + +enum avd_h264_fp_field { + AVD_H264_FP_MV_ABOVE_INFO, + AVD_H264_FP_MV_COLOR, + AVD_H264_FP_HDR_D0_REF_HDR, + AVD_H264_FP_HDR_4C_PIC_SCALING_LIST_DIMS, + AVD_H264_FP_SCL_46C_PIC_SCALING_MATRIX_4X4, + AVD_H264_FP_SCL_4CC_PIC_SCALING_MATRIX_8X8, + AVD_H264_FP_HDR_34_START_HDR, + AVD_H264_FP_HDR_38_MODE, + AVD_H264_FP_HDR_3C_HEIGHT_WIDTH, + AVD_H264_FP_HDR_40_ZERO, + AVD_H264_FP_HDR_28_HEIGHT_WIDTH_SHIFT3, + AVD_H264_FP_HDR_2C_SPS_PARAM, + AVD_H264_FP_HDR_44_FLAGS, + AVD_H264_FP_HDR_48_CHROMA_QP_INDEX_OFFSET, + AVD_H264_FP_HDR_58_CONST_3A, + AVD_H264_FP_HDR_9C_PPS_TILE_ADDR_LSB8, + AVD_H264_FP_IP_ABOVE, + AVD_H264_FP_LF_ABOVE, + AVD_H264_FP_LF_ABOVE_INFO, + AVD_H264_FP_HDR_210_Y_ADDR_LSB8, + AVD_H264_FP_HDR_218_WIDTH_ALIGN, + AVD_H264_FP_HDR_214_UV_ADDR_LSB8, + AVD_H264_FP_HDR_21C_WIDTH_ALIGN, + AVD_H264_FP_HDR_54_HEIGHT_WIDTH, + AVD_H264_FP_SLC_76C_CMD_WEIGHTS_DENOM, + AVD_H264_FP_SLC_LUMA_WEIGHTS, + AVD_H264_FP_SLC_LUMA_OFFSETS, + AVD_H264_FP_SLC_CHROMA_WEIGHTS_0_, + AVD_H264_FP_SLC_CHROMA_OFFSETS_0_, + AVD_H264_FP_SLC_CHROMA_WEIGHTS_1_, + AVD_H264_FP_SLC_CHROMA_OFFSETS_1_, + AVD_H264_FP_SLC_A7C_CMD_SET_CODED_SLICE, + AVD_H264_FP_SLC_A84_SLICE_ADDR_LOW, + AVD_H264_FP_SLC_A88_SLICE_HDR_SIZE, + AVD_H264_FP_SLC_A70_CMD_QUANT_PARAM, + AVD_H264_FP_SLC_A74_CMD_DEBLOCKING_FILTER, + AVD_H264_FP_SLC_6E8_CMD_REF_LIST_0, + AVD_H264_FP_SLC_6E4_CMD_REF_TYPE, + AVD_H264_FP_SLC_A78_SPS_TILE_ADDR2_LSB8, + AVD_H264_FP_NONE, +}; + +struct avd_h264_fp_cursor { + struct avd_buf *buf; + u16 count[AVD_H264_FP_NONE]; + u16 weights; + u16 offsets; + u16 slice; +}; + +int avd_h264_fp_begin(struct avd_h264_fp_cursor *cursor, struct avd_buf *buf, + enum avd_h264_fp_layout layout); +int avd_h264_fp_slice(struct avd_h264_fp_cursor *cursor, unsigned int index, + unsigned int count); +int avd_h264_fp_emit(struct avd_h264_fp_cursor *cursor, + enum avd_h264_fp_field field, u32 word); +int avd_h264_fp_address(struct avd_h264_fp_cursor *cursor, + enum avd_h264_fp_field field, unsigned int index, + dma_addr_t addr); +int avd_h264_fp_ref(struct avd_h264_fp_cursor *cursor, bool reference, + unsigned int index, unsigned int plane, dma_addr_t addr); +int avd_h264_fp_pack_cmd(enum avd_h264_fp_layout layout, struct avd_frame *f); +/* Known Borage projection only; returns -EOPNOTSUPP for unresolved fields. */ +int avd_h264_borage_cmd_fields(struct avd_frame *f); + +#endif /* AVD_H264_FP_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index e8bc8d0066ead..f18fdcab83950 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -16,12 +16,15 @@ #include #include #include +#include #include #include #include "avd.h" #include "avd-inst.h" +#include "avd-queue.h" +#include "avd-h264-fp.h" #define H264_SCL_DIMS (0x1000000 | ((64 / 4) << 5) | (((16 / 4) << 5) - 1)) @@ -31,13 +34,25 @@ #define H264_TRANSFORM_8X8_MODE(v) FIELD_PREP(BIT(7), !!(v)) -/* - * Most slices held for one picture. This bounds the slice table, the job - * table and the memory that a client can pin. - */ +/* not a hardware constraint */ #define MAX_SLICES 4096 +struct avd_h264_ctx; + +/* Only metadata is copied. The core owns each OUTPUT until picture retire. */ +struct avd_h264_slice { + struct avd_buf data; + struct v4l2_ctrl_h264_slice_params params; + struct v4l2_ctrl_h264_pred_weights weights; + u32 data_offset; +}; + struct avd_h264_run { + struct avd_frame *frame; + struct avd_h264_ctx *picture; + struct avd_h264_fp_cursor fp_cursor; + unsigned int ref_index; + int error; struct avd_run base; const struct v4l2_ctrl_h264_decode_params *decode_params; @@ -66,12 +81,21 @@ struct avd_h264_ctx { struct v4l2_h264_reference b1[V4L2_H264_REF_LIST_LEN]; } reflists; - struct avd_buf *slices; + /* CPU view and DMA address of OUTPUT; never owns coded-data memory. */ + struct avd_buf slice; + struct avd_h264_ctx *slots[AVD_MAX_FRAMES]; + struct avd_h264_ctx *pending; + bool busy; struct avd_buf *active_slice; size_t slice_num; - size_t alloc_slice_num; - /* allocated size of the held slices */ size_t slice_bytes; + size_t alloc_slices; + struct avd_h264_slice *slices; + struct vb2_v4l2_buffer *dst; + struct v4l2_ctrl_h264_sps sps; + struct v4l2_ctrl_h264_pps pps; + struct v4l2_ctrl_h264_decode_params decode; + struct v4l2_ctrl_h264_scaling_matrix scaling; struct avd_h264_bufs { struct avd_buf inst; @@ -107,24 +131,90 @@ static inline u32 mv_color_size(u32 w, u32 h) /* sorry for the formatting */ +static void avd_h264_emit(struct avd_ctx *ctx, struct avd_h264_run *run, + u32 word, enum avd_h264_fp_field field) +{ + struct avd_frame *f = run->frame; + + if (!f) { + (push)(ctx, word); + return; + } + if (run->error) + return; + if (f->prog && f->prog->cpu) { + if (f->prog_words >= f->prog->size / sizeof(__le32)) { + run->error = -ENOSPC; + return; + } + ((__le32 *)f->prog->cpu)[f->prog_words++] = cpu_to_le32(word); + } else if (!run->fp_cursor.buf) { + run->error = -EINVAL; + return; + } + if (run->fp_cursor.buf) + run->error = avd_h264_fp_emit(&run->fp_cursor, field, word); +} + +static void avd_h264_emit_addr(struct avd_ctx *ctx, struct avd_h264_run *run, + dma_addr_t addr, enum avd_h264_fp_field field, + unsigned int index) +{ + /* Open instruction addresses retain the existing per-variant encoding. */ + if (ctx->dev->variant->quirks & AVD_QUIRK_LSR) { + avd_h264_emit(ctx, run, addr >> 8, AVD_H264_FP_NONE); + } else { + avd_h264_emit(ctx, run, lower_32_bits(addr), AVD_H264_FP_NONE); + avd_h264_emit(ctx, run, upper_32_bits(addr), AVD_H264_FP_NONE); + } + if (run->fp_cursor.buf && !run->error) + /* V3 stores the same shifted word as the LSR instruction. */ + run->error = avd_h264_fp_address(&run->fp_cursor, field, index, + addr & ~(dma_addr_t)0xff); +} + +static void avd_h264_emit_comp(struct avd_ctx *ctx, struct avd_h264_run *run, + dma_addr_t addr, bool reference) +{ + unsigned int i; + + for (i = 0; i < 4; i++) { + dma_addr_t a = addr + ctx->comp.offsets[i]; + + if (ctx->dev->variant->quirks & AVD_QUIRK_LSR) + avd_h264_emit(ctx, run, a >> 7, AVD_H264_FP_NONE); + else + avd_h264_emit_addr(ctx, run, a, AVD_H264_FP_NONE, 0); + if (run->fp_cursor.buf && !run->error) + run->error = avd_h264_fp_ref(&run->fp_cursor, reference, + run->ref_index, i, a); + } +} + +#undef push +#undef pusha +#define push(word, field) avd_h264_emit(ctx, run, (word), (field)) +#define pusha(addr, field, index) \ + avd_h264_emit_addr(ctx, run, (addr), (field), (index)) + static void stream_refs(struct avd_ctx *ctx, struct avd_h264_run *run) { const struct v4l2_ctrl_h264_decode_params *decode = run->decode_params; const struct v4l2_h264_dpb_entry *dpb = decode->dpb; - struct avd_h264_ctx *h264_ctx = ctx->priv; + struct avd_h264_ctx *h264_ctx = run->picture; struct avd_decoded_buffer *dst, *ref; dma_addr_t addr; dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); - push(0, ""); - pusha(h264_ctx->bufs.mv_above_info.addr, "mv_above_info", 7); - pusha(run->addresses.mv_color, "mv_color", 0); + push(0, AVD_H264_FP_NONE); + pusha(h264_ctx->bufs.mv_above_info.addr, AVD_H264_FP_MV_ABOVE_INFO, 7); + pusha(run->addresses.mv_color, AVD_H264_FP_MV_COLOR, 0); - push(0, ""); - push(0, ""); - push(0, ""); - push(0, ""); + push(0, AVD_H264_FP_NONE); + push(0, AVD_H264_FP_NONE); + push(0, AVD_H264_FP_NONE); + push(0, AVD_H264_FP_NONE); for (int i = 0; i < ARRAY_SIZE(decode->dpb); i++) { if (!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_VALID)) @@ -141,9 +231,10 @@ static void stream_refs(struct avd_ctx *ctx, struct avd_h264_run *run) V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM) | AVD_REF_DELTA_POC(run->cur_poc - dpb[i].top_field_order_cnt), - "hdr_d0_ref_hdr"); + AVD_H264_FP_HDR_D0_REF_HDR); - push_comp(ctx, addr, ctx->comp.offsets); + avd_h264_emit_comp(ctx, run, addr, true); + run->ref_index++; } } @@ -153,7 +244,7 @@ static void stream_scaling(struct avd_ctx *ctx, struct avd_h264_run *run) const struct v4l2_ctrl_h264_scaling_matrix *scaling = run->scaling_matrix; - push(H264_SCL_DIMS, "hdr_4c_pic_scaling_list_dims"); + push(H264_SCL_DIMS, AVD_H264_FP_HDR_4C_PIC_SCALING_LIST_DIMS); for (int i = 0; i < 6; i++) for (int j = 0; j < 16; j += 4) @@ -165,7 +256,7 @@ static void stream_scaling(struct avd_ctx *ctx, struct avd_h264_run *run) scaling->scaling_list_4x4[i][j + 2]) | AVD_SCALING_I0( scaling->scaling_list_4x4[i][j + 3]), - "scl_46c_pic_scaling_matrix_4x4"); + AVD_H264_FP_SCL_46C_PIC_SCALING_MATRIX_4X4); /* Instead of 8x8 raster scan order avd expects 4 4x4 subblocks */ static const u8 map[16] = { @@ -190,15 +281,15 @@ static void stream_scaling(struct avd_ctx *ctx, struct avd_h264_run *run) AVD_SCALING_I0( scaling->scaling_list_8x8 [i][map[j] + 3]), - "scl_4cc_pic_scaling_matrix_8x8"); + AVD_H264_FP_SCL_4CC_PIC_SCALING_MATRIX_8X8); } else { for (int i = 0; i < ARRAY_SIZE(default_8x8_intra); i++) push(default_8x8_intra[i], - "scl_4cc_pic_scaling_matrix_8x8"); + AVD_H264_FP_SCL_4CC_PIC_SCALING_MATRIX_8X8); for (int i = 0; i < ARRAY_SIZE(default_8x8_inter); i++) push(default_8x8_inter[i], - "scl_4cc_pic_scaling_matrix_8x8"); + AVD_H264_FP_SCL_4CC_PIC_SCALING_MATRIX_8X8); } } @@ -208,7 +299,7 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) const struct v4l2_ctrl_h264_sps *sps = run->sps; const struct v4l2_ctrl_h264_pps *pps = run->pps; struct avd_dev *avd = ctx->dev; - struct avd_h264_ctx *h264_ctx = ctx->priv; + struct avd_h264_ctx *h264_ctx = run->picture; u32 bytesperline; u32 width = (sps->pic_width_in_mbs_minus1 + 1) * 16; u32 height = (sps->pic_height_in_map_units_minus1 + 1) * 16; @@ -220,18 +311,18 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) AVD_OP_HDR_CONST | AVD_OP_HDR_FLAG_PIPE_STATE( !(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)), - "hdr_34_start_hdr"); + AVD_H264_FP_HDR_34_START_HDR); - push(AVD_HDR_CODEC_MODE(AVD_CODEC_H264), "hdr_38_mode"); + push(AVD_HDR_CODEC_MODE(AVD_CODEC_H264), AVD_H264_FP_HDR_38_MODE); push(AVD_HDR_HEIGHT(height - 1) | AVD_HDR_WIDTH(width - 1), - "hdr_3c_height_width"); + AVD_H264_FP_HDR_3C_HEIGHT_WIDTH); - push(0, "hdr_40_zero"); + push(0, AVD_H264_FP_HDR_40_ZERO); push(AVD_HDR_HEIGHT((height - 1) >> 3) | AVD_HDR_WIDTH((width - 1) >> 3), - "hdr_28_height_width_shift3"); + AVD_H264_FP_HDR_28_HEIGHT_WIDTH_SHIFT3); /* TODO: did i mix up luma/chroma in hevc or here? */ push(AVD_HDR_COMMON_CHROMA_FORMAT(sps->chroma_format_idc) | @@ -245,7 +336,7 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) AVD_HDR_COMMON_FLAG0( sps->flags & V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE), - "hdr_2c_sps_param"); + AVD_H264_FP_HDR_2C_SPS_PARAM); push(H264_FLAG_ENTROPY_CODING_MODE( pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE) | @@ -254,52 +345,52 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) h264_FLAG_CI_PRED( pps->flags & V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED), - "hdr_44_flags"); + AVD_H264_FP_HDR_44_FLAGS); push(AVD_HDR_H26X_QP_OFFSET_CB(pps->chroma_qp_index_offset) | AVD_HDR_H26X_QP_OFFSET_CR( pps->second_chroma_qp_index_offset), - "hdr_48_chroma_qp_index_offset"); + AVD_H264_FP_HDR_48_CHROMA_QP_INDEX_OFFSET); push(AVD_HDR_FEAT_H26X | AVD_HDR_FEAT_COMMON | AVD_HDR_FEAT_H264 | AVD_HDR_FEAT_PIPE_STATE_EN( !(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)), - "hdr_58_const_3a"); + AVD_H264_FP_HDR_58_CONST_3A); - push(0, ""); - push(0, ""); + push(0, AVD_H264_FP_NONE); + push(0, AVD_H264_FP_NONE); if (avd->variant->revision == 3) - push(0, "zero"); + push(0, AVD_H264_FP_NONE); - pusha(h264_ctx->bufs.above_info.addr, "hdr_9c_pps_tile_addr_lsb8", 0); + pusha(h264_ctx->bufs.above_info.addr, AVD_H264_FP_HDR_9C_PPS_TILE_ADDR_LSB8, 0); - push(0, ""); - push(0, ""); + push(0, AVD_H264_FP_NONE); + push(0, AVD_H264_FP_NONE); if (avd->variant->revision == 3) - push(0, "zero"); + push(0, AVD_H264_FP_NONE); else if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) - pusha(h264_ctx->bufs.pipe_state.addr, "pipe_state", 0); + pusha(h264_ctx->bufs.pipe_state.addr, AVD_H264_FP_NONE, 0); - pusha(h264_ctx->bufs.ip_above.addr, "ip_above", 1); - pusha(h264_ctx->bufs.lf_above.addr, "lf_above", 2); - pusha(h264_ctx->bufs.lf_above_info.addr, "lf_above_info", 3); - push(0, ""); + pusha(h264_ctx->bufs.ip_above.addr, AVD_H264_FP_IP_ABOVE, 1); + pusha(h264_ctx->bufs.lf_above.addr, AVD_H264_FP_LF_ABOVE, 2); + pusha(h264_ctx->bufs.lf_above_info.addr, AVD_H264_FP_LF_ABOVE_INFO, 3); + push(0, AVD_H264_FP_NONE); - push_comp(ctx, run->base.comp_out, ctx->comp.offsets); + avd_h264_emit_comp(ctx, run, run->base.comp_out, false); bytesperline = ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline; if (avd->variant->quirks & AVD_QUIRK_LSR) bytesperline = bytesperline >> 4; - pusha(run->base.y_out, "hdr_210_y_addr_lsb8", 0); - push(bytesperline, "hdr_218_width_align"); - pusha(run->base.uv_out, "hdr_214_uv_addr_lsb8", 0); - push(bytesperline, "hdr_21c_width_align"); + pusha(run->base.y_out, AVD_H264_FP_HDR_210_Y_ADDR_LSB8, 0); + push(bytesperline, AVD_H264_FP_HDR_218_WIDTH_ALIGN); + pusha(run->base.uv_out, AVD_H264_FP_HDR_214_UV_ADDR_LSB8, 0); + push(bytesperline, AVD_H264_FP_HDR_21C_WIDTH_ALIGN); - push(0, "cm3_mark_end_section"); - push(((height - 1) << 16) | (width - 1), "hdr_54_height_width"); + push(0, AVD_H264_FP_NONE); + push(((height - 1) << 16) | (width - 1), AVD_H264_FP_HDR_54_HEIGHT_WIDTH); if (!(decode->flags & V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC)) stream_refs(ctx, run); @@ -307,7 +398,7 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) if (pps->flags & V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT) stream_scaling(ctx, run); else - push(0, "cm3_mark_end_section_scl"); + push(0, AVD_H264_FP_NONE); } #define DEFAULT_WEIGHT_DENOM \ @@ -336,7 +427,7 @@ static void stream_weights(struct avd_ctx *ctx, struct avd_h264_run *run) weights->chroma_log2_weight_denom : 0) | (default_weights ? DEFAULT_WEIGHT_DENOM : 0), - "slc_76c_cmd_weights_denom"); + AVD_H264_FP_SLC_76C_CMD_WEIGHTS_DENOM); if (!pred_weight_req) return; @@ -361,11 +452,11 @@ static void stream_weights(struct avd_ctx *ctx, struct avd_h264_run *run) AVD_OP_WEIGHTS_INDEX(i) | AVD_OP_WEIGHTS_WEIGHT( factors.luma_weight[i]), - "slc_luma_weights"); + AVD_H264_FP_SLC_LUMA_WEIGHTS); push(AVD_OP_OFFSETS | AVD_OP_OFFSETS_OFFSET( factors.luma_offset[i]), - "slc_luma_offsets"); + AVD_H264_FP_SLC_LUMA_OFFSETS); } if ((factors.chroma_weight[i][0] != chroma_denom) || @@ -377,21 +468,21 @@ static void stream_weights(struct avd_ctx *ctx, struct avd_h264_run *run) AVD_OP_WEIGHTS_INDEX(i) | AVD_OP_WEIGHTS_WEIGHT( factors.chroma_weight[i][0]), - "slc_chroma_weights[0]"); + AVD_H264_FP_SLC_CHROMA_WEIGHTS_0_); push(AVD_OP_OFFSETS | AVD_OP_OFFSETS_OFFSET( factors.chroma_offset[i][0]), - "slc_chroma_offsets[0]"); + AVD_H264_FP_SLC_CHROMA_OFFSETS_0_); push(AVD_OP_WEIGHTS | AVD_OP_WEIGHTS_IDENT(3) | AVD_OP_WEIGHTS_LIST_IDX(y) | AVD_OP_WEIGHTS_INDEX(i) | AVD_OP_WEIGHTS_WEIGHT( factors.chroma_weight[i][1]), - "slc_chroma_weights[1]"); + AVD_H264_FP_SLC_CHROMA_WEIGHTS_1_); push(AVD_OP_OFFSETS | AVD_OP_OFFSETS_OFFSET( factors.chroma_offset[i][1]), - "slc_chroma_offsets[1]"); + AVD_H264_FP_SLC_CHROMA_OFFSETS_1_); } } } @@ -436,7 +527,7 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) const struct v4l2_ctrl_h264_pps *pps = run->pps; const struct v4l2_ctrl_h264_sps *sps = run->sps; const struct v4l2_ctrl_h264_slice_params *sl = run->slice_params; - struct avd_h264_ctx *h264_ctx = ctx->priv; + struct avd_h264_ctx *h264_ctx = run->picture; u32 payload_len = h264_ctx->active_slice->size; bool en_mode = (pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE) == 0; @@ -448,22 +539,22 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) AVD_OP_CODED_DATA_BIT_OFF( en_mode ? (sl->header_bit_size % 8) : 0) | AVD_OP_CODED_DATA_ADDR(slc_a84 >> 32), - "slc_a7c_cmd_set_coded_slice"); - push((u32)(slc_a84 & 0xffffffff), "slc_a84_slice_addr_low"); + AVD_H264_FP_SLC_A7C_CMD_SET_CODED_SLICE); + push((u32)(slc_a84 & 0xffffffff), AVD_H264_FP_SLC_A84_SLICE_ADDR_LOW); /* should not include trailing 0? */ - push(payload_len - off, "slc_a88_slice_hdr_size"); + push(payload_len - off, AVD_H264_FP_SLC_A88_SLICE_HDR_SIZE); push(AVD_OP_SL_LOC | AVD_OP_SL_LOC_Y(sl->first_mb_in_slice / (sps->pic_width_in_mbs_minus1 + 1)) | AVD_OP_SL_LOC_X(sl->first_mb_in_slice % (sps->pic_width_in_mbs_minus1 + 1)), - "cm3_cmd_exec_mb_vp"); + AVD_H264_FP_NONE); push(AVD_OP_QP | AVD_OP_QP_VAL(26 + pps->pic_init_qp_minus26 + sl->slice_qp_delta), - "slc_a70_cmd_quant_param"); + AVD_H264_FP_SLC_A70_CMD_QUANT_PARAM); push(AVD_OP_DBLK | AVD_OP_DBLK_FLAG_FULL_EN( @@ -472,7 +563,7 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) 1) | AVD_OP_DBLK_OFF1(sl->slice_beta_offset_div2) | AVD_OP_DBLK_OFF0(sl->slice_alpha_c0_offset_div2), - "slc_a74_cmd_deblocking_filter"); + AVD_H264_FP_SLC_A74_CMD_DEBLOCKING_FILTER); if (sl->slice_type == V4L2_H264_SLICE_TYPE_P || sl->slice_type == V4L2_H264_SLICE_TYPE_B) { @@ -482,7 +573,7 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) AVD_OP_REF_LOOP_IDX(i) | AVD_OP_REF_DBP_IDX( sl->ref_pic_list0[i].index), - "slc_6e8_cmd_ref_list_0"); + AVD_H264_FP_SLC_6E8_CMD_REF_LIST_0); if (sl->slice_type == V4L2_H264_SLICE_TYPE_B) { u32 num_ref_idx_active = @@ -492,16 +583,16 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) AVD_OP_REF_LOOP_IDX(i) | AVD_OP_REF_DBP_IDX( sl->ref_pic_list1[i].index), - "slc_6e8_cmd_ref_list_0"); + AVD_H264_FP_SLC_6E8_CMD_REF_LIST_0); } stream_weights(ctx, run); } if (sl->first_mb_in_slice == 0) { - push(AVD_OP_SL_DIM_START, "cm3_cmd_set_mb_dims"); + push(AVD_OP_SL_DIM_START, AVD_H264_FP_NONE); push(AVD_SL_DIM_END_Y(sps->pic_height_in_map_units_minus1) | AVD_SL_DIM_END_X(sps->pic_width_in_mbs_minus1), - "cm3_set_mb_dims"); + AVD_H264_FP_NONE); } push(AVD_OP_SL_REF | AVD_OP_SL_REF_FLAG_CABAC(sl->cabac_init_idc == 1) | @@ -519,13 +610,14 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) V4L2_H264_SLICE_TYPE_B) , - "slc_6e4_cmd_ref_type"); + AVD_H264_FP_SLC_6E4_CMD_REF_TYPE); if (sl->slice_type == V4L2_H264_SLICE_TYPE_B) { /* bidirectional reference of previous mv */ - struct vb2_buffer *vb = vb2_find_buffer( - &ctx->fh.m2m_ctx->cap_q_ctx.q, + struct avd_decoded_buffer *ref = avd_get_ref_buf( + ctx, run->base.bufs.dst, decode->dpb[sl->ref_pic_list1[0].index].reference_ts); + struct vb2_buffer *vb = &ref->base.vb.vb2_buf; dma_addr_t mv_color_addr = vb ? vb2_dma_contig_plane_dma_addr(vb, 0) + @@ -534,14 +626,14 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) fmt_height(ctx))) : run->addresses.mv_color; - pusha(mv_color_addr, "slc_a78_sps_tile_addr2_lsb8", 0); + pusha(mv_color_addr, AVD_H264_FP_SLC_A78_SPS_TILE_ADDR2_LSB8, 0); } } -static int avd_h264_alloc_bufs(struct avd_ctx *ctx) +static int avd_h264_alloc_bufs(struct avd_ctx *ctx, + struct avd_h264_ctx *h264_ctx) { struct avd_dev *dev = ctx->dev; - struct avd_h264_ctx *h264_ctx = ctx->priv; int ret, w, bit_depth, mb; w = fmt_width(ctx); bit_depth = (ctx->image_fmt == AVD_IMG_FMT_420_10BIT || @@ -549,11 +641,6 @@ static int avd_h264_alloc_bufs(struct avd_ctx *ctx) 10 : 8; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.inst, fifo_size()); - if (ret) { - return ret; - } - if (!(dev->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) { ret = avd_buf_alloc(dev, &h264_ctx->bufs.pipe_state, 0x200); if (ret) { @@ -561,6 +648,10 @@ static int avd_h264_alloc_bufs(struct avd_ctx *ctx) } } + ret = avd_buf_alloc(dev, &h264_ctx->bufs.inst, fifo_size()); + if (ret) + return ret; + mb = DIV_ROUND_UP(w, 16); ret = avd_buf_alloc(dev, &h264_ctx->bufs.above_info, mb * 20); @@ -620,37 +711,35 @@ static int avd_h264_validate_sps(struct avd_ctx *ctx, return 0; } -/* Free the slices held for the current picture. */ -static void avd_h264_drop_slices(struct avd_ctx *ctx) -{ - struct avd_h264_ctx *h264_ctx = ctx->priv; - - for (size_t i = 0; i < h264_ctx->slice_num; i++) - avd_buf_free(ctx->dev, &h264_ctx->slices[i]); - h264_ctx->slice_num = 0; - h264_ctx->slice_bytes = 0; -} - -static void avd_h264_stop(struct avd_ctx *ctx) +static void avd_h264_free_picture(struct avd_ctx *ctx, + struct avd_h264_ctx *h264_ctx) { - struct avd_h264_ctx *h264_ctx = ctx->priv; struct avd_dev *dev = ctx->dev; if (!h264_ctx) return; - - avd_buf_free(dev, &h264_ctx->bufs.pipe_state); + kvfree(h264_ctx->slices); avd_buf_free(dev, &h264_ctx->bufs.inst); + avd_buf_free(dev, &h264_ctx->bufs.pipe_state); avd_buf_free(dev, &h264_ctx->bufs.above_info); avd_buf_free(dev, &h264_ctx->bufs.lf_above_info); avd_buf_free(dev, &h264_ctx->bufs.lf_above); avd_buf_free(dev, &h264_ctx->bufs.ip_above); avd_buf_free(dev, &h264_ctx->bufs.mv_above_info); + kfree(h264_ctx); +} - avd_h264_drop_slices(ctx); - kfree(h264_ctx->slices); +static void avd_h264_stop(struct avd_ctx *ctx) +{ + struct avd_h264_ctx *h264_ctx = ctx->priv; + unsigned int i; - kfree(h264_ctx); + if (!h264_ctx) + return; + /* Core drains every frame before stop; no slot can still be DMA-owned. */ + for (i = 0; i < AVD_MAX_FRAMES; i++) + avd_h264_free_picture(ctx, h264_ctx->slots[i]); + avd_h264_free_picture(ctx, h264_ctx); ctx->priv = NULL; } @@ -674,31 +763,17 @@ static int avd_h264_start(struct avd_ctx *ctx) ctx->priv = h264_ctx; - /* assume one slice per row */ - h264_ctx->alloc_slice_num = - ctrl->p_new.p_h264_sps->pic_height_in_map_units_minus1 + 1; - h264_ctx->slices = kcalloc(h264_ctx->alloc_slice_num, - sizeof(*h264_ctx->slices), GFP_KERNEL); - if (!h264_ctx->slices) { - ret = -ENOMEM; - goto err_free_ctx; - } - - ret = avd_h264_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; - + /* Scratch is allocated lazily and stays owned until picture retire. */ + ctx->max_frames = AVD_MAX_FRAMES; return 0; - -err_free_ctx: - avd_h264_stop(ctx); - return ret; } static void avd_h264_run_preamble(struct avd_ctx *ctx, struct avd_h264_run *run) { struct v4l2_ctrl *ctrl; - u32 dst_len, mv_color_len; + + avd_run_preamble(ctx, &run->base); + run->picture = ctx->priv; ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_H264_DECODE_PARAMS); @@ -716,57 +791,6 @@ static void avd_h264_run_preamble(struct avd_ctx *ctx, struct avd_h264_run *run) ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_H264_PRED_WEIGHTS); run->pred_weights = ctrl ? ctrl->p_cur.p : NULL; - - avd_run_preamble(ctx, &run->base); - - dst_len = run->base.bufs.dst->vb2_buf.planes[0].length; - - mv_color_len = mv_color_size(fmt_width(ctx), fmt_height(ctx)); - - run->addresses.mv_color = run->base.y_out + (dst_len - mv_color_len); -} - -static int avd_h264_realloc_slices(struct avd_ctx *ctx, size_t max_slices) -{ - struct avd_h264_ctx *h264_ctx = ctx->priv; - void *tmp; - size_t alloc_slice_num; - - tmp = h264_ctx->slices; - /* grow by half, and by at least one for a one-row picture */ - alloc_slice_num = h264_ctx->alloc_slice_num + - max_t(size_t, h264_ctx->alloc_slice_num / 2, 1); - alloc_slice_num = min(alloc_slice_num, max_slices); - h264_ctx->slices = kcalloc(alloc_slice_num, sizeof(*h264_ctx->slices), - GFP_KERNEL); - if (!h264_ctx->slices) { - /* make deallocating a little easier */ - h264_ctx->slices = tmp; - return -ENOMEM; - } - - h264_ctx->alloc_slice_num = alloc_slice_num; - memcpy(h264_ctx->slices, tmp, - sizeof(*h264_ctx->slices) * h264_ctx->slice_num); - kfree(tmp); - - /* a slice that arrives before the frame's header has no job table yet */ - if (!ctx->job.segments) - return 0; - - tmp = ctx->job.segments; - ctx->job.segments = kvcalloc(alloc_slice_num + 1, - sizeof(*ctx->job.segments), GFP_KERNEL); - if (!ctx->job.segments) { - ctx->job.segments = tmp; - return -ENOMEM; - } - - memcpy(ctx->job.segments, tmp, - sizeof(*ctx->job.segments) * (ctx->job.num + 1)); - kvfree(tmp); - - return 0; } /* @@ -849,103 +873,236 @@ VISIBLE_IF_KUNIT bool avd_h264_slice_fits(const struct v4l2_ctrl_h264_sps *sps, u32 len, size_t decoded_size) { return held < avd_h264_max_slices(sps) && - held_bytes + PAGE_ALIGN((size_t)len) <= - avd_h264_slice_budget(decoded_size); + PAGE_ALIGN((size_t)len) <= avd_h264_slice_budget(decoded_size) && + held_bytes <= avd_h264_slice_budget(decoded_size) - PAGE_ALIGN((size_t)len); } -static int avd_h264_run(struct avd_ctx *ctx) +/* Reset metadata only: OUTPUT memory belongs to the queue, never the codec. */ +static void avd_h264_release_picture(struct avd_h264_ctx *h264, + struct avd_h264_ctx *picture) { - struct avd_h264_ctx *h264_ctx = ctx->priv; - struct v4l2_h264_reflist_builder reflist_builder; - struct avd_h264_run run; - int ret; - - avd_h264_run_preamble(ctx, &run); + if (h264->pending == picture) + h264->pending = NULL; + if (picture->dst) + vb2_to_avd_decoded_buf(&picture->dst->vb2_buf)->bank = 0; + picture->slice_num = 0; + picture->slice_bytes = 0; + picture->dst = NULL; + picture->busy = false; +} - struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); - u32 payload_len = vb2_get_plane_payload(&src->vb2_buf, 0); - const u8 *data = vb2_plane_vaddr(&src->vb2_buf, 0); +static int avd_h264_collect(struct avd_ctx *ctx, struct avd_h264_run *run) +{ + struct avd_h264_ctx *h264 = ctx->priv, *picture = h264->pending; + struct vb2_v4l2_buffer *src = run->base.bufs.src; + struct vb2_buffer *vb = &src->vb2_buf; + const struct v4l2_ctrl_h264_slice_params *sl = run->slice_params; + struct avd_h264_slice *slice, *slices; + const u8 *data = vb2_plane_vaddr(vb, 0); + size_t payload = vb2_get_plane_payload(vb, 0); + size_t offset = vb->planes[0].data_offset; + size_t capacity, total_mbs; + unsigned int i; + u32 data_offset; + int ret; - if (!data || !avd_h264_find_slice_data(data, payload_len, - avd_h264_header_len(&run), - &run.slice_data_off)) { - dev_dbg_ratelimited(ctx->dev->dev, - "slice has no data after its header\n"); - avd_run_postamble(ctx, &run.base); + /* Attach existing state even on validation failure, for error retirement. */ + run->picture = picture; + if (!run->sps || !run->pps || !run->decode_params || !sl || + !run->scaling_matrix || !run->pred_weights) + return -EINVAL; + ret = avd_h264_validate_sps(ctx, run->sps); + if (!ret) + ret = avd_h264_validate_pps(ctx, run->pps); + if (ret || !avd_h264_refs_valid(run)) + return ret ?: -EINVAL; + total_mbs = (size_t)(run->sps->pic_width_in_mbs_minus1 + 1) * + (run->sps->pic_height_in_map_units_minus1 + 1); + if (sl->first_mb_in_slice >= total_mbs || + sl->slice_type > V4L2_H264_SLICE_TYPE_I || + run->pred_weights->luma_log2_weight_denom > 7 || + run->pred_weights->chroma_log2_weight_denom > 7 || + (run->sps->pic_width_in_mbs_minus1 + 1) * 16 != ALIGN(fmt_width(ctx), 16) || + (run->sps->pic_height_in_map_units_minus1 + 1) * 16 != ALIGN(fmt_height(ctx), 16)) + return -EINVAL; + if (!data || offset > payload || payload > vb2_plane_size(vb, 0) || + payload - offset > U32_MAX || payload - offset < 3 || + sl->header_bit_size > (payload - offset) * 8 || + !avd_h264_find_slice_data(data + offset, payload - offset, + avd_h264_header_len(run), &data_offset)) + return -EINVAL; + if (!sl->first_mb_in_slice) { + /* A new picture cannot silently reuse a partially collected picture. */ + if (picture) + return -EINVAL; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + picture = h264->slots[i]; + if (!picture) { + picture = kzalloc(sizeof(*picture), GFP_KERNEL); + if (!picture) + return -ENOMEM; + ret = avd_h264_alloc_bufs(ctx, picture); + if (ret) { + avd_h264_free_picture(ctx, picture); + return ret; + } + h264->slots[i] = picture; + } + if (!picture->busy) + break; + } + if (i == AVD_MAX_FRAMES) + return -EBUSY; + picture->busy = true; + picture->dst = run->base.bufs.dst; + picture->sps = *run->sps; + picture->pps = *run->pps; + picture->decode = *run->decode_params; + picture->scaling = *run->scaling_matrix; + picture->monochrome = run->sps->chroma_format_idc == 0; + h264->pending = picture; + run->picture = picture; + vb2_to_avd_decoded_buf(&picture->dst->vb2_buf)->bank = i + 1; + } else if (!picture || picture->dst != run->base.bufs.dst || + !picture->slice_num || + run->decode_params->frame_num != picture->decode.frame_num || + run->decode_params->top_field_order_cnt != picture->decode.top_field_order_cnt || + run->decode_params->bottom_field_order_cnt != + picture->decode.bottom_field_order_cnt || + memcmp(run->decode_params->dpb, picture->decode.dpb, + sizeof(picture->decode.dpb)) || + memcmp(run->sps, &picture->sps, sizeof(picture->sps)) || + memcmp(run->pps, &picture->pps, sizeof(picture->pps)) || + memcmp(run->scaling_matrix, &picture->scaling, sizeof(picture->scaling))) { return -EINVAL; } - - if (!avd_h264_slice_fits(run.sps, h264_ctx->slice_num, - h264_ctx->slice_bytes, payload_len, - ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].sizeimage)) { - dev_err_ratelimited(ctx->dev->dev, - "too many slices for one picture, dropping it\n"); - /* The unsubmitted job table points at the dropped slices. */ - avd_h264_drop_slices(ctx); - ret = -EINVAL; - goto err_free_jobs; - } - - if (h264_ctx->slice_num >= h264_ctx->alloc_slice_num) { - ret = avd_h264_realloc_slices(ctx, avd_h264_max_slices(run.sps)); - if (ret) - goto err_free_jobs; + if (!avd_h264_slice_fits(run->sps, picture->slice_num, + picture->slice_bytes, payload - offset, + ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].sizeimage)) + return -E2BIG; + if (picture->slice_num == picture->alloc_slices) { + capacity = min_t(size_t, avd_h264_max_slices(run->sps), + max_t(size_t, 4, picture->alloc_slices * 2)); + slices = kvmalloc_array(capacity, sizeof(*slices), GFP_KERNEL); + if (!slices) + return -ENOMEM; + if (picture->slice_num) + memcpy(slices, picture->slices, + picture->slice_num * sizeof(*slices)); + kvfree(picture->slices); + picture->slices = slices; + picture->alloc_slices = capacity; } + slice = &picture->slices[picture->slice_num++]; + slice->data = (struct avd_buf) { + .addr = vb2_dma_contig_plane_dma_addr(vb, 0) + offset, + .size = payload - offset, + }; + slice->params = *sl; + slice->weights = *run->pred_weights; + slice->data_offset = data_offset; + picture->slice_bytes += PAGE_ALIGN(payload - offset); + return 0; +} - h264_ctx->active_slice = &h264_ctx->slices[h264_ctx->slice_num]; - ret = avd_buf_alloc(ctx->dev, h264_ctx->active_slice, payload_len); - if (ret) - goto err_free_jobs; - memcpy(h264_ctx->active_slice->cpu, data, payload_len); - h264_ctx->slice_num++; - h264_ctx->slice_bytes += PAGE_ALIGN((size_t)payload_len); - h264_ctx->monochrome = run.sps->chroma_format_idc == 0; - - /* Build the P/B{0,1} ref lists. */ - v4l2_h264_init_reflist_builder(&reflist_builder, run.decode_params, - run.sps, run.decode_params->dpb); - - run.num_valid = reflist_builder.num_valid; - run.cur_poc = reflist_builder.cur_pic_order_count; - - v4l2_h264_build_p_ref_list(&reflist_builder, h264_ctx->reflists.p); - v4l2_h264_build_b_ref_lists(&reflist_builder, h264_ctx->reflists.b0, - h264_ctx->reflists.b1); - - avd_run_postamble(ctx, &run.base); +static int avd_h264_build_picture(struct avd_ctx *ctx, struct avd_h264_run *run) +{ + struct avd_h264_ctx *picture = run->picture; + struct avd_frame *f = run->frame; + struct vb2_buffer *dst = &picture->dst->vb2_buf; + struct v4l2_h264_reflist_builder builder; + size_t i, mv_len = mv_color_size(fmt_width(ctx), fmt_height(ctx)); + int ret; - if (!avd_h264_refs_valid(&run)) { - dev_dbg_ratelimited(ctx->dev->dev, - "slice references an invalid DPB entry, dropping frame\n"); + if (mv_len > vb2_plane_size(dst, 0)) return -EINVAL; - } - - if (is_new_frame(run.slice_params)) { - /* the header segment plus one per slice */ - ret = avd_init_job(ctx, AVD_CODEC_H264, - h264_ctx->alloc_slice_num + 1); + run->sps = &picture->sps; + run->pps = &picture->pps; + run->decode_params = &picture->decode; + run->scaling_matrix = &picture->scaling; + run->base.y_out = vb2_dma_contig_plane_dma_addr(dst, 0); + run->base.uv_out = run->base.y_out + + (size_t)ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline * + ctx->decoded_fmt.fmt.pix_mp.height; + run->base.comp_out = run->base.y_out + ctx->comp.start_offset; + run->addresses.mv_color = run->base.y_out + vb2_plane_size(dst, 0) - mv_len; + vb2_to_avd_decoded_buf(dst)->comp = ctx->comp; + v4l2_h264_init_reflist_builder(&builder, run->decode_params, run->sps, + run->decode_params->dpb); + run->num_valid = builder.num_valid; + run->cur_poc = builder.cur_pic_order_count; + if (f) { + f->frame_num = picture->decode.frame_num; + f->fifo_iova = picture->bufs.inst.addr; + if (run->fp_cursor.buf) { + ret = avd_h264_fp_slice(&run->fp_cursor, 0, picture->slice_num); + if (ret) + return ret; + } + avd_h264_emit(ctx, run, AVD_OP_EXEC | + AVD_OP_EXEC_FLAG_START_REV4(ctx->dev->variant->revision == 4) | + AVD_OP_EXEC_FLAG_START_REV3(ctx->dev->variant->revision == 3) | + AVD_OP_EXEC_FIFO_IDX(f->idx), AVD_H264_FP_NONE); + } else { + ret = avd_init_job(ctx, AVD_CODEC_H264, picture->slice_num + 1); if (ret) return ret; - stream_hdr(ctx, &run); } + ctx->decomp = true; + stream_hdr(ctx, run); + for (i = 0; i < picture->slice_num; i++) { + struct avd_h264_slice *slice = &picture->slices[i]; + + picture->active_slice = &slice->data; + run->slice_params = &slice->params; + run->pred_weights = &slice->weights; + run->slice_data_off = slice->data_offset; + if (f && run->fp_cursor.buf) { + ret = avd_h264_fp_slice(&run->fp_cursor, i, picture->slice_num); + if (ret) + return ret; + } + if (!f) + ctx->job.num++; + stream_slice(ctx, run); + if (f) + avd_h264_emit(ctx, run, AVD_OP_EXEC | + AVD_OP_EXEC_FLAG_END(i == picture->slice_num - 1), + AVD_H264_FP_NONE); + if (run->error) + return run->error; + } + return 0; +} - if (!ctx->job.segments) - return -EINVAL; - - ctx->job.num++; - stream_slice(ctx, &run); +static int avd_h264_run(struct avd_ctx *ctx) +{ + struct avd_h264_ctx *h264 = ctx->priv; + struct avd_h264_run run = {}; + int ret; + avd_h264_run_preamble(ctx, &run); + ret = avd_h264_collect(ctx, &run); + avd_run_postamble(ctx, &run.base); + if (ret) + goto fail; if (run.base.bufs.src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF) { avd_job_finish(ctx, VB2_BUF_STATE_DONE); return 0; } - - return avd_submit_job(ctx); - -err_free_jobs: + h264->pending = NULL; + ret = avd_h264_build_picture(ctx, &run); + if (ret) + goto fail; + ctx->job_fifo = run.picture->bufs.inst.addr; + ret = avd_submit_job(ctx); + if (!ret) + return 0; + fail: kvfree(ctx->job.segments); ctx->job.segments = NULL; - avd_run_postamble(ctx, &run.base); + if (run.picture) + avd_h264_release_picture(h264, run.picture); return ret; } @@ -953,24 +1110,26 @@ static void avd_h264_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, struct vb2_v4l2_buffer *dst_buf, enum vb2_buffer_state result) { - struct avd_h264_ctx *h264_ctx = ctx->priv; - - if (!(src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF)) { - avd_h264_drop_slices(ctx); + struct avd_h264_ctx *h264 = ctx->priv, *picture; + struct avd_decoded_buffer *dst; - /* 4:0:0 has no chroma to decode; the picture is grey */ - if (h264_ctx->monochrome && dst_buf && result == VB2_BUF_STATE_DONE) { - const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; - struct avd_decoded_buffer *dst = - vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + if (!dst_buf || ((src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF) && + result == VB2_BUF_STATE_DONE)) + return; + dst = vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + if (!dst->bank || dst->bank > AVD_MAX_FRAMES) + return; + picture = h264->slots[dst->bank - 1]; + if (!picture || picture->dst != dst_buf) + return; + if (picture->monochrome && result == VB2_BUF_STATE_DONE) { + const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; - /* vb2 must return cache ownership before the CPU writes. */ - dst->grey_chroma_offset = - (size_t)pix->plane_fmt[0].bytesperline * pix->height; - dst->grey_chroma_10bit = - ctx->image_fmt == AVD_IMG_FMT_420_10BIT; - } + dst->grey_chroma_offset = (size_t)pix->plane_fmt[0].bytesperline * pix->height; + dst->grey_chroma_10bit = ctx->image_fmt == AVD_IMG_FMT_420_10BIT; } + dst->bank = 0; + avd_h264_release_picture(h264, picture); } static enum avd_image_fmt avd_h264_get_image_fmt(struct avd_ctx *ctx, @@ -1026,6 +1185,86 @@ static void avd_h264_submit(struct avd_ctx *ctx) ctx->dev->ctrl + ctx->dev->variant->submit_offset); } +static void avd_h264_frame_controls(struct avd_ctx *ctx, struct avd_h264_run *run) +{ + struct v4l2_ctrl *ctrl; + +#define H264_CTRL(member, id) do { \ + ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, id); \ + run->member = ctrl ? ctrl->p_cur.p : NULL; \ +} while (0) + H264_CTRL(decode_params, V4L2_CID_STATELESS_H264_DECODE_PARAMS); + H264_CTRL(sps, V4L2_CID_STATELESS_H264_SPS); + H264_CTRL(pps, V4L2_CID_STATELESS_H264_PPS); + H264_CTRL(slice_params, V4L2_CID_STATELESS_H264_SLICE_PARAMS); + H264_CTRL(scaling_matrix, V4L2_CID_STATELESS_H264_SCALING_MATRIX); + H264_CTRL(pred_weights, V4L2_CID_STATELESS_H264_PRED_WEIGHTS); +#undef H264_CTRL +} + +static int avd_h264_prepare(struct avd_ctx *ctx, struct avd_frame *f) +{ + struct avd_h264_ctx *h264 = ctx->priv; + struct avd_h264_run run = { .frame = f }; + bool apple = avd_backend_is_apple(ctx->dev); + int ret; + + f->dep = AVD_DEP_NONE; + f->prog_words = 0; + if (!h264 || f->idx >= AVD_MAX_FRAMES || !f->src || !f->dst || + (!apple && (!f->prog || !f->prog->cpu))) + return -EINVAL; + f->codec_priv = h264->pending; + avd_h264_frame_controls(ctx, &run); + run.base.bufs.src = f->src; + run.base.bufs.dst = f->dst; + ret = avd_h264_collect(ctx, &run); + f->codec_priv = run.picture; + if (ret) + return ret; + if (f->src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF) + return 0; + h264->pending = NULL; + if (apple) { + ret = avd_h264_fp_begin(&run.fp_cursor, f->fp, + ctx->dev->variant->h264_fp_layout); + if (ret) + return ret; + } else { + /* Native queued-open descriptor; Apple prefix remains core-owned. */ + f->cmd[0] = AVD_CODEC_H264; + } + return avd_h264_build_picture(ctx, &run); +} + +static int avd_h264_pack_cmd(struct avd_ctx *ctx, struct avd_frame *f) +{ + return avd_h264_fp_pack_cmd(ctx->dev->variant->h264_fp_layout, f); +} + +static void avd_h264_retire(struct avd_ctx *ctx, struct avd_frame *f, + enum vb2_buffer_state state) +{ + struct avd_h264_ctx *h264 = ctx->priv, *picture = f->codec_priv; + + if (!picture) + return; + if ((f->src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF) && + state == VB2_BUF_STATE_DONE) + return; + if (picture->dst == f->dst) + avd_h264_done(ctx, f->src, f->dst, state); + else + avd_h264_release_picture(h264, picture); + f->codec_priv = NULL; +} + +const struct avd_codec_frame_ops avd_h264_frame_ops = { + .prepare = avd_h264_prepare, + .pack_cmd = avd_h264_pack_cmd, + .retire = avd_h264_retire, +}; + const struct avd_coded_fmt_ops avd_h264_fmt_ops = { .adjust_decoded_fmt = avd_h264_adjust_decoded_fmt, .start = avd_h264_start, diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index 9dfa20ab71913..b2ce0f3711390 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -79,10 +79,14 @@ struct avd_hevc_run { struct avd_hevc_tile_info tile_info; }; -struct avd_hevc_ctx { - struct v4l2_ctrl_hevc_scaling_matrix scaling_matrix_cache; +/* See avd-h264.c: one bank per picture in flight on the queued transport. */ +#define AVD_HEVC_BANKS 2 +struct avd_hevc_bank { + bool busy; struct avd_h264_bufs { + /* VP to PP instruction FIFO for pictures of this bank */ + struct avd_buf inst; struct avd_buf mv_above_info; struct avd_buf az_above; struct avd_buf ip_above; @@ -91,9 +95,17 @@ struct avd_hevc_ctx { struct avd_buf lf_left; struct avd_buf lf_left_info; struct avd_buf sw_left; - struct avd_buf inst; struct avd_buf pipe_state; } bufs; +}; + +struct avd_hevc_ctx { + struct v4l2_ctrl_hevc_scaling_matrix scaling_matrix_cache; + + unsigned int nbanks; + struct avd_hevc_bank bank[AVD_HEVC_BANKS]; + /* bank of the picture being built */ + struct avd_hevc_bank *b; int submit_num; }; @@ -110,7 +122,7 @@ static void stream_refs(struct avd_ctx *ctx, struct avd_hevc_run *run) push(ctx->dev->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER ? HEVC_DMA_CONFIG : 0, "reference_dma_config"); - pusha(hevc_ctx->bufs.mv_above_info.addr, "mv_above_info", 7); + pusha(hevc_ctx->b->bufs.mv_above_info.addr, "mv_above_info", 7); pusha(run->addresses.mv_color, "mv_color", 0); push(0, ""); @@ -357,30 +369,30 @@ static void set_header(struct avd_ctx *ctx, struct avd_hevc_run *run) if (avd->variant->revision == 3) push(0, "zero"); else if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) - pusha(hevc_ctx->bufs.pipe_state.addr, "pipe_state", 0); + pusha(hevc_ctx->b->bufs.pipe_state.addr, "pipe_state", 0); - pusha(hevc_ctx->bufs.ip_above.addr, "scratch_addr", 0); - pusha(hevc_ctx->bufs.lf_above.addr, "scratch_addr", 1); - pusha(hevc_ctx->bufs.lf_above_info.addr, "scratch_addr", 2); + pusha(hevc_ctx->b->bufs.ip_above.addr, "scratch_addr", 0); + pusha(hevc_ctx->b->bufs.lf_above.addr, "scratch_addr", 1); + pusha(hevc_ctx->b->bufs.lf_above_info.addr, "scratch_addr", 2); if ((pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) || !(avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER)) { - pusha(hevc_ctx->bufs.lf_left.addr, + pusha(hevc_ctx->b->bufs.lf_left.addr, "scratch_addr", 3); - pusha(hevc_ctx->bufs.lf_left_info.addr, + pusha(hevc_ctx->b->bufs.lf_left_info.addr, "scratch_addr", 4); if (avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER) pusha(0, "reserved_scratch", 0); - pusha(hevc_ctx->bufs.az_above.addr, + pusha(hevc_ctx->b->bufs.az_above.addr, "scratch_addr", 8); - pusha(hevc_ctx->bufs.sw_left.addr, + pusha(hevc_ctx->b->bufs.sw_left.addr, "scratch_addr", 9); } else { pusha(0, "", 3); pusha(0, "", 4); if (avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER) pusha(0, "reserved_scratch", 0); - pusha(hevc_ctx->bufs.az_above.addr, + pusha(hevc_ctx->b->bufs.az_above.addr, "scratch_addr", 8); pusha(0, "", 9); } @@ -1056,17 +1068,16 @@ static int avd_hevc_validate_sps(struct avd_ctx *ctx, return 0; } -static int avd_hevc_alloc_bufs(struct avd_ctx *ctx) +static int avd_hevc_alloc_bufs(struct avd_ctx *ctx, struct avd_hevc_bank *bank) { struct avd_dev *avd = ctx->dev; - struct avd_hevc_ctx *hevc_ctx = ctx->priv; int ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.inst, fifo_size()); + ret = avd_buf_alloc(avd, &bank->bufs.inst, fifo_size()); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.pipe_state, 0x200); + ret = avd_buf_alloc(avd, &bank->bufs.pipe_state, 0x200); if (ret) return ret; @@ -1077,20 +1088,26 @@ static void avd_hevc_stop(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; struct avd_dev *avd = ctx->dev; + unsigned int i; + ctx->max_frames = 1; if (!hevc_ctx) return; - avd_buf_free(avd, &hevc_ctx->bufs.pipe_state); - avd_buf_free(avd, &hevc_ctx->bufs.inst); - avd_buf_free(avd, &hevc_ctx->bufs.mv_above_info); - avd_buf_free(avd, &hevc_ctx->bufs.az_above); - avd_buf_free(avd, &hevc_ctx->bufs.ip_above); - avd_buf_free(avd, &hevc_ctx->bufs.lf_above); - avd_buf_free(avd, &hevc_ctx->bufs.lf_above_info); - avd_buf_free(avd, &hevc_ctx->bufs.lf_left); - avd_buf_free(avd, &hevc_ctx->bufs.lf_left_info); - avd_buf_free(avd, &hevc_ctx->bufs.sw_left); + for (i = 0; i < AVD_HEVC_BANKS; i++) { + struct avd_h264_bufs *bufs = &hevc_ctx->bank[i].bufs; + + avd_buf_free(avd, &bufs->pipe_state); + avd_buf_free(avd, &bufs->inst); + avd_buf_free(avd, &bufs->mv_above_info); + avd_buf_free(avd, &bufs->az_above); + avd_buf_free(avd, &bufs->ip_above); + avd_buf_free(avd, &bufs->lf_above); + avd_buf_free(avd, &bufs->lf_above_info); + avd_buf_free(avd, &bufs->lf_left); + avd_buf_free(avd, &bufs->lf_left_info); + avd_buf_free(avd, &bufs->sw_left); + } kfree(hevc_ctx); ctx->priv = NULL; @@ -1099,6 +1116,7 @@ static void avd_hevc_stop(struct avd_ctx *ctx) static int avd_hevc_start(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx; + unsigned int i; int ret; hevc_ctx = kzalloc(sizeof(*hevc_ctx), GFP_KERNEL); @@ -1106,9 +1124,15 @@ static int avd_hevc_start(struct avd_ctx *ctx) return -ENOMEM; ctx->priv = hevc_ctx; - ret = avd_hevc_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; + /* Overlap pictures only where the firmware chains VP to PP itself. */ + hevc_ctx->nbanks = avd_backend_is_qfw(ctx->dev) ? AVD_HEVC_BANKS : 1; + hevc_ctx->b = &hevc_ctx->bank[0]; + for (i = 0; i < hevc_ctx->nbanks; i++) { + ret = avd_hevc_alloc_bufs(ctx, &hevc_ctx->bank[i]); + if (ret) + goto err_free_ctx; + } + ctx->max_frames = hevc_ctx->nbanks; return 0; @@ -1117,6 +1141,38 @@ static int avd_hevc_start(struct avd_ctx *ctx) return ret; } +/* Take a bank no unretired picture uses; record it in the CAPTURE buffer. */ +static int avd_hevc_pick_bank(struct avd_ctx *ctx, struct avd_decoded_buffer *dst) +{ + struct avd_hevc_ctx *hevc_ctx = ctx->priv; + unsigned int i; + + for (i = 0; i < hevc_ctx->nbanks; i++) { + if (hevc_ctx->bank[i].busy) + continue; + hevc_ctx->bank[i].busy = true; + hevc_ctx->b = &hevc_ctx->bank[i]; + dst->bank = i + 1; + return 0; + } + return -EBUSY; +} + +static void avd_hevc_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, + struct vb2_v4l2_buffer *dst_buf, + enum vb2_buffer_state result) +{ + struct avd_hevc_ctx *hevc_ctx = ctx->priv; + struct avd_decoded_buffer *buf; + + if (!dst_buf || !hevc_ctx) + return; + buf = vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + if (buf->bank && buf->bank <= hevc_ctx->nbanks) + hevc_ctx->bank[buf->bank - 1].busy = false; + buf->bank = 0; +} + static int avd_hevc_alloc_scratch(struct avd_ctx *ctx, struct avd_hevc_run *run) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; @@ -1152,51 +1208,51 @@ static int avd_hevc_alloc_scratch(struct avd_ctx *ctx, struct avd_hevc_run *run) tile_info->row_height[i] : max_row; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.ip_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.ip_above, ((max_col * max_cu_width) / 4) * bit_depth); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.mv_above_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.mv_above_info, ((max_col * max_cu_width) / 16) * 20); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_above, DIV_ROUND_UP(w, 16) * bit_depth * (10 + 6) + (cols - 1) * 256); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_above_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_above_info, DIV_ROUND_UP(w + 7, 16) * 36 + cols * 128); if (ret) return ret; if (pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) { - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_left, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_left, ((max_row * max_cu_width) / 4) * 36 + 144 /* why? */); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_left_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_left_info, ((max_row * max_cu_width) / 4) * 9); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.az_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.az_above, /* not sure if its cols or rows */ DIV_ROUND_UP(w, 64) * 144 + (cols - 1) * 128); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.sw_left, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.sw_left, ((max_row * max_cu_width) / 4) * 216); if (ret) return ret; } else { - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.az_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.az_above, DIV_ROUND_UP(w + 7, 16) * 4 * bit_depth); if (ret) return ret; @@ -1282,6 +1338,51 @@ static int avd_hevc_compute_tiles(struct avd_ctx *ctx, struct avd_hevc_run *run) return 0; } +/* + * The slice builder below expects the GStreamer layout: the slices tile the + * OUTPUT buffer as whole NALs, bit_size covers the whole NAL and + * data_byte_offset is the slice data's offset inside it. FFmpeg's request + * API backend instead gives each slice's data_byte_offset from the start of + * the buffer and a bit_size of the slice data only, so every slice was given + * a wrong address and size and the VP raised its error interrupt on every + * frame. Detect the absolute layout (complete and unambiguous) and rewrite + * the run's private copy into the equivalent NAL-relative description. + */ +static void avd_hevc_normalize_slices(struct avd_hevc_run *run, u32 payload) +{ + struct v4l2_ctrl_hevc_slice_params *sl = + (struct v4l2_ctrl_hevc_slice_params *)run->sl; + u64 nal_end = 0, data_end = 0, prev_end = 0; + bool nal = true, absolute = true; + int i; + + if (!run->num_slices || !payload) + return; + for (i = 0; i < run->num_slices; i++) { + u64 bytes = DIV_ROUND_UP((u64)sl[i].bit_size, 8); + u64 start = sl[i].data_byte_offset; + + if (!bytes || start >= bytes || nal_end + bytes > payload) + nal = false; + nal_end += bytes; + if (!bytes || start < data_end || start + bytes > payload) + absolute = false; + data_end = start + bytes; + } + nal &= nal_end == payload; + absolute &= data_end == payload; + if (!absolute || nal) + return; + for (i = 0; i < run->num_slices; i++) { + u64 bytes = DIV_ROUND_UP((u64)sl[i].bit_size, 8); + u64 start = sl[i].data_byte_offset; + + sl[i].data_byte_offset = start - prev_end; + sl[i].bit_size = (start - prev_end + bytes) * 8; + prev_end = start + bytes; + } +} + static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) { struct v4l2_ctrl *ctrl, *sl, *ep; @@ -1345,6 +1446,7 @@ static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) static int avd_hevc_run(struct avd_ctx *ctx) { + struct avd_hevc_ctx *hevc_ctx = ctx->priv; struct avd_hevc_run run = {}; struct avd_decoded_buffer *dst; int ret; @@ -1353,9 +1455,15 @@ static int avd_hevc_run(struct avd_ctx *ctx) if (ret) return ret; + avd_hevc_normalize_slices(&run, + vb2_get_plane_payload(&run.base.bufs.src->vb2_buf, 0)); dst = vb2_to_avd_decoded_buf(&run.base.bufs.dst->vb2_buf); update_dec_buf_info(dst, &run.sl[0]); + ret = avd_hevc_pick_bank(ctx, dst); + if (ret) + goto complete; + ret = avd_hevc_compute_tiles(ctx, &run); if (ret) goto complete; @@ -1372,8 +1480,10 @@ static int avd_hevc_run(struct avd_ctx *ctx) set_header(ctx, &run); ret = stream_slices(ctx, &run); avd_run_postamble(ctx, &run.base); - if (!ret) + if (!ret) { + ctx->job_fifo = hevc_ctx->b->bufs.inst.addr; ret = avd_submit_job(ctx); + } goto done; complete: @@ -1395,6 +1505,13 @@ static int avd_hevc_try_ctrl(struct avd_ctx *ctx, struct v4l2_ctrl *ctrl) return 0; } +static unsigned int avd_hevc_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_hevc_ctx *priv = ctx->priv; + + return max_t(int, priv->submit_num, 1); +} + static void avd_hevc_submit(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; @@ -1420,7 +1537,9 @@ const struct avd_coded_fmt_ops avd_hevc_fmt_ops = { .start = avd_hevc_start, .stop = avd_hevc_stop, .run = avd_hevc_run, + .done = avd_hevc_done, .submit = avd_hevc_submit, + .pp_kicks = avd_hevc_pp_kicks, .try_ctrl = avd_hevc_try_ctrl, .get_image_fmt = avd_hevc_get_image_fmt, }; diff --git a/drivers/media/platform/apple/avd/avd-hw.c b/drivers/media/platform/apple/avd/avd-hw.c index a3987ac4510de..b64c0bf4d78ea 100644 --- a/drivers/media/platform/apple/avd/avd-hw.c +++ b/drivers/media/platform/apple/avd/avd-hw.c @@ -42,9 +42,12 @@ int avd_boot(struct avd_dev *avd) { + struct resource *code = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "code"); u32 val; int ret; + if (!avd->fw || !avd->fw->size || !code || avd->fw->size > resource_size(code)) + return -EINVAL; if (avd->variant->revision != 3) dev_info_once(avd->dev, "booting hw version: %04x", readl_relaxed(avd->ctrl)); @@ -64,6 +67,28 @@ int avd_boot(struct avd_dev *avd) return 0; } +/* + * Pop one firmware message. Reading RETRIEVE on an empty mailbox returns the + * previous message again, and a repeated PP DONE completes the next frame + * before its PP has run. An interrupt can also come just before its message + * is readable, so allow 10 us; after that treat the interrupt as spurious. + * Always acknowledges the interrupt. Returns false when there is no message. + */ +bool avd_mbox_pop(struct avd_dev *avd, u32 *msg) +{ + u32 st; + int ret; + + ret = readl_poll_timeout_atomic(avd->mbox + AVD_REG_MBOX1_STATUS, st, + !(st & AVD_MBOX1_STATUS_EMPTY), 1, 10); + if (!ret) + *msg = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); + writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); + if (ret) + dev_dbg_ratelimited(avd->dev, "spurious interrupt, mailbox empty\n"); + return !ret; +} + void avd_shutdown(struct avd_dev *avd) { writel_relaxed(AVD_RUN_CTRL_UNK_STOP, avd->mbox + AVD_REG_RUN_CTRL); diff --git a/drivers/media/platform/apple/avd/avd-inst.h b/drivers/media/platform/apple/avd/avd-inst.h index e9b9ef908b8d5..dcdb205d79315 100644 --- a/drivers/media/platform/apple/avd/avd-inst.h +++ b/drivers/media/platform/apple/avd/avd-inst.h @@ -3,6 +3,7 @@ #ifndef AVD_INST_H_ #define AVD_INST_H_ /* instruction stream things */ +#include #include #include "avd.h" diff --git a/drivers/media/platform/apple/avd/avd-pm-test.c b/drivers/media/platform/apple/avd/avd-pm-test.c new file mode 100644 index 0000000000000..6756d036b3f76 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm-test.c @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include + +#include "avd.h" +#include "avd-regs.h" + +struct avd_pm_test { + struct avd_dev avd; + bool queue_initialized; + struct avd_deferred_unmap unmap; + unsigned int draining; + unsigned int step; + unsigned int unmap_step; + unsigned int boot_step; + unsigned int resubmit_step; + unsigned int shutdowns; + unsigned int submissions; + int boot_error; + int suspend_error; + bool unmapped_active; +}; + +static struct avd_pm_test *avd_pm_fixture(struct avd_dev *avd) +{ + return container_of(avd, struct avd_pm_test, avd); +} + +static bool avd_pm_test_idle(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + return avd_pm_can_suspend(fixture->avd.frames, fixture->draining); +} + +static int avd_pm_test_drain(struct avd_dev *avd) +{ + return avd_pm_test_idle(avd) ? 0 : -EBUSY; +} + +static int avd_pm_test_boot(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->boot_step = ++fixture->step; + return fixture->boot_error; +} + +static void avd_pm_test_shutdown(struct avd_dev *avd) +{ + avd_pm_fixture(avd)->shutdowns++; +} + +static int avd_pm_test_suspend(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + if (fixture->suspend_error) + return fixture->suspend_error; + return avd_pm_backend_suspend(avd); +} + +static int avd_pm_test_submit(struct avd_dev *avd, struct avd_frame *frame) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->resubmit_step = ++fixture->step; + fixture->submissions++; + frame->state = AVD_FRAME_SUBMITTED; + return 0; +} + +static void avd_pm_test_resubmit(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + unsigned int i; + + fixture->resubmit_step = ++fixture->step; + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (fixture->avd.frames[i].state == AVD_FRAME_RESERVED) + avd->backend->submit(avd, &fixture->avd.frames[i]); +} + +static void avd_pm_test_unmap(struct avd_dev *avd, + struct avd_deferred_unmap *unmap) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->unmap_step = ++fixture->step; + fixture->unmapped_active = avd->pm.dart_active; +} + +static const struct avd_queue_pm_ops avd_pm_test_queue = { + .idle = avd_pm_test_idle, + .drain = avd_pm_test_drain, + .resubmit = avd_pm_test_resubmit, +}; + +static const struct avd_backend_ops avd_pm_test_backend = { + .name = "test", + .boot = avd_pm_test_boot, + .shutdown = avd_pm_test_shutdown, + .submit = avd_pm_test_submit, + .suspend = avd_pm_test_suspend, + .resume = avd_pm_backend_resume, +}; + +static int avd_pm_test_init(struct kunit *test) +{ + struct avd_pm_test *fixture; + + fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + if (!fixture) + return -ENOMEM; + test->priv = fixture; + avd_pm_init(&fixture->avd, &avd_pm_test_backend, &avd_pm_test_queue); + INIT_LIST_HEAD(&fixture->unmap.node); + fixture->unmap.unmap = avd_pm_test_unmap; + return 0; +} + +static void avd_pm_suspend_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + unsigned int i, state; + + KUNIT_EXPECT_TRUE(test, avd_pm_test_idle(&fixture->avd)); + fixture->draining = 1; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), -EBUSY); + fixture->draining = 0; + + /* An occupied slot anywhere vetoes idle, even after completion IRQ. */ + for (i = 0; i < AVD_MAX_FRAMES; i++) { + for (state = AVD_FRAME_RESERVED; state <= AVD_FRAME_DONE; state++) { + fixture->avd.frames[i].state = state; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), -EBUSY); + } + fixture->avd.frames[i].state = AVD_FRAME_FREE; + } + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), 0); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); +} + +static void avd_pm_resume_order_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_EQ(test, fixture->unmap_step, 0U); + fixture->avd.frames[0].state = AVD_FRAME_RESERVED; + fixture->avd.frames[1].state = AVD_FRAME_SUBMITTED; + fixture->avd.frames[2].state = AVD_FRAME_DONE; + fixture->avd.frames[15].state = AVD_FRAME_RESERVED; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_TRUE(test, fixture->unmapped_active); + KUNIT_EXPECT_LT(test, fixture->unmap_step, fixture->boot_step); + KUNIT_EXPECT_LT(test, fixture->boot_step, fixture->resubmit_step); + KUNIT_EXPECT_EQ(test, fixture->submissions, 2U); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[0].state, AVD_FRAME_SUBMITTED); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[2].state, AVD_FRAME_DONE); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); +} + +static void avd_pm_resume_failure_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + fixture->avd.frames[0].state = AVD_FRAME_RESERVED; + fixture->boot_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_power_on(avd), -ETIMEDOUT); + KUNIT_EXPECT_FALSE(test, avd->pm.dart_active); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[0].state, AVD_FRAME_RESERVED); + + /* A later successful resume retries the still-unposted frame once. */ + fixture->boot_error = 0; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_EQ(test, fixture->submissions, 1U); +} + +static void avd_pm_suspend_failure_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + fixture->suspend_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, avd->pm.dart_active); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); +} + +static void avd_pm_deferred_unmap_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + unsigned int step; + + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_TRUE(test, fixture->unmapped_active); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); + KUNIT_ASSERT_EQ(test, avd_pm_power_off(avd), 0); + step = fixture->unmap_step; + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_EQ(test, fixture->unmap_step, step); + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_GT(test, fixture->unmap_step, step); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); +} + +static void avd_pm_real_queue_resume_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct avd_ctx *ctx; + unsigned int i; + + avd->dev = kunit_device_register(test, "avd-pm-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, avd->dev); + KUNIT_ASSERT_EQ(test, avd_queue_init(avd), 0); + fixture->queue_initialized = true; + avd->backend = &avd_pm_test_backend; + avd->pm.queue = &avd_pm_queue_ops; + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->dev = avd; + for (i = 0; i < 3; i++) { + avd->frames[i].ctx = ctx; + avd->frames[i].seq = i + 1; + avd->frames[i].deadline = jiffies + msecs_to_jiffies(60000); + avd->frames[i].state = AVD_FRAME_RESERVED; + } + /* This one was posted before the power transition; never replay it. */ + avd->frames[1].state = AVD_FRAME_SUBMITTED; + avd->inflight = 3; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + flush_delayed_work(&avd->queue_work); + cancel_delayed_work_sync(&avd->queue_work); + KUNIT_EXPECT_EQ(test, fixture->submissions, 2U); + KUNIT_EXPECT_LT(test, fixture->boot_step, fixture->resubmit_step); + KUNIT_EXPECT_EQ(test, avd->frames[0].state, AVD_FRAME_SUBMITTED); + KUNIT_EXPECT_EQ(test, avd->frames[2].state, AVD_FRAME_SUBMITTED); +} + +static void avd_pm_real_queue_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + spin_lock_init(&avd->queue_lock); + avd->pm.queue = &avd_pm_queue_ops; + avd_pm_drain_begin(avd); + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -EBUSY); + avd_pm_drain_end(avd); + avd->frames[15].state = AVD_FRAME_DONE; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -EBUSY); + avd->frames[15].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), 0); +} + +static void avd_pm_open_boot_replay_test(struct kunit *test) +{ + static const struct avd_variant variant = { .revision = 3 }; + static const u8 image[] = { 0x12, 0x34, 0x56, 0x78 }; + static const struct firmware fw = { .data = image, .size = sizeof(image) }; + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct resource code = { + .name = "code", .flags = IORESOURCE_MEM, + .start = 0x1000, .end = 0x10ff, + }; + struct platform_device *pdev; + u8 observed[sizeof(image)]; + unsigned int i; + + pdev = kunit_kzalloc(test, sizeof(*pdev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pdev); + pdev->resource = &code; + pdev->num_resources = 1; + avd->pdev = pdev; + avd->variant = &variant; + avd->fw = &fw; + avd->backend = &avd_open_backend; + /* Ordinary test RAM, never a mapped target aperture. */ + avd->code = (__force void __iomem *)kunit_kzalloc(test, 0x100, GFP_KERNEL); + avd->mbox = (__force void __iomem *)kunit_kzalloc(test, 0x100, GFP_KERNEL); + avd->pm.piodma = (__force void __iomem *)kunit_kzalloc(test, 0x28, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, avd->code); + KUNIT_ASSERT_NOT_NULL(test, avd->mbox); + KUNIT_ASSERT_NOT_NULL(test, avd->pm.piodma); + avd->pm.sram_phys = 0x123456780ULL; + for (i = 0; i < 2; i++) { + /* Model loss of both ITCM and PIODMA configuration between boots. */ + memset_io(avd->code, 0, sizeof(image)); + writel(0, avd->pm.piodma + 0x24); + /* The fixture supplies firmware readiness; no firmware is executed. */ + writel(1, avd->mbox + AVD_REG_FLAG0_SET); + KUNIT_ASSERT_EQ(test, avd_backend_boot(avd), 0); + KUNIT_EXPECT_EQ(test, readl(avd->pm.piodma + 0x24), 0x12345678U); + memcpy_fromio(observed, avd->code, sizeof(observed)); + KUNIT_EXPECT_MEMEQ(test, observed, image, sizeof(image)); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_RUN_CTRL), + (u32)AVD_RUN_CTRL_UNK_RUN); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX1_STATUS), + (u32)AVD_MBOX_ENABLE); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX_IRQ_ENABLE), + (u32)AVD_MBOX1_NOT_EMPTY); + avd_shutdown(avd); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_RUN_CTRL), + (u32)(AVD_RUN_CTRL_UNK_STOP)); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_FLAG0_CLR), 1U); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX_IRQ_ENABLE), 0U); + } +} + +static void avd_pm_recovery_dart_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct iommu_domain domain = {}; + + /* A stopped device cannot be reset through powered-off registers. */ + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); + avd->pm.dart_active = true; + avd->domain = &domain; + /* No empty domain means no valid DART flush: never reboot anyway. */ + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -ENODEV); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); + KUNIT_EXPECT_EQ(test, fixture->boot_step, 0U); +} + +static void avd_pm_recovery_boot_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + avd->pm.dart_active = true; + fixture->boot_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 2U); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); + fixture->boot_error = 0; + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), 0); + KUNIT_EXPECT_EQ(test, avd->fw_generation, 1ULL); + /* Recovery must fail old slots before core reopens the host IRQ. */ + KUNIT_EXPECT_FALSE(test, avd->pm.irq_enabled); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); +} + +static void avd_pm_test_exit(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + + if (fixture->queue_initialized) + cancel_delayed_work_sync(&fixture->avd.queue_work); +} + +static struct kunit_case avd_pm_test_cases[] = { + KUNIT_CASE(avd_pm_suspend_gate_test), + KUNIT_CASE(avd_pm_resume_order_test), + KUNIT_CASE(avd_pm_resume_failure_test), + KUNIT_CASE(avd_pm_suspend_failure_test), + KUNIT_CASE(avd_pm_deferred_unmap_test), + KUNIT_CASE(avd_pm_real_queue_resume_test), + KUNIT_CASE(avd_pm_real_queue_gate_test), + KUNIT_CASE(avd_pm_open_boot_replay_test), + KUNIT_CASE(avd_pm_recovery_dart_gate_test), + KUNIT_CASE(avd_pm_recovery_boot_test), + {} +}; + +static struct kunit_suite avd_pm_test_suite = { + .name = "apple-avd-pm", + .init = avd_pm_test_init, + .exit = avd_pm_test_exit, + .test_cases = avd_pm_test_cases, +}; + +kunit_test_suite(avd_pm_test_suite); diff --git a/drivers/media/platform/apple/avd/avd-pm.c b/drivers/media/platform/apple/avd/avd-pm.c new file mode 100644 index 0000000000000..3876aec76f2ee --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.c @@ -0,0 +1,450 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include +#include + +#include "avd.h" + +/* [C] Keep Linux's existing default until the idle-timer capture is available. */ +#define AVD_AUTOSUSPEND_MS 100 +#define AVD_DRAIN_TIMEOUT_MS 5000 +#define AVD_PIODMA_SRAM_BASE 0x24 + +void avd_pm_enable_irq(struct avd_dev *avd) +{ + unsigned long flags; + + if (avd->pm.irq >= 0 && !avd->pm.irq_enabled) { + spin_lock_irqsave(&avd->pm.irq_lock, flags); + avd->pm.hw_on = true; + spin_unlock_irqrestore(&avd->pm.irq_lock, flags); + enable_irq(avd->pm.irq); + avd->pm.irq_enabled = true; + } +} + +/* + * Callers may hold queue_mutex, which the threaded handler takes first, so + * never wait for the handler here (disable_irq() would deadlock). Instead + * close hardware access under irq_lock: a handler already touching the + * decoder finishes before this returns, and any later one sees hw_on clear + * or a new irq_gen and leaves the hardware alone. + */ +static void avd_pm_disable_irq(struct avd_dev *avd) +{ + unsigned long flags; + + if (avd->pm.irq >= 0 && avd->pm.irq_enabled) { + spin_lock_irqsave(&avd->pm.irq_lock, flags); + avd->pm.hw_on = false; + avd->pm.irq_gen++; + spin_unlock_irqrestore(&avd->pm.irq_lock, flags); + disable_irq_nosync(avd->pm.irq); + avd->pm.irq_enabled = false; + } +} + +/* Run the backend's interrupt handler only against a live firmware instance. */ +irqreturn_t avd_pm_irq(struct avd_dev *avd, unsigned int gen) +{ + irqreturn_t ret = IRQ_HANDLED; + unsigned long flags; + + spin_lock_irqsave(&avd->pm.irq_lock, flags); + if (avd->pm.hw_on && gen == avd->pm.irq_gen) + ret = avd->backend->irq(avd); + spin_unlock_irqrestore(&avd->pm.irq_lock, flags); + return ret; +} + +struct avd_dma_release { + struct avd_deferred_unmap unmap; + void *cpu; + dma_addr_t addr; + size_t size; +}; + +static void avd_dma_unmap(struct avd_dev *avd, + struct avd_deferred_unmap *unmap) +{ + struct avd_dma_release *release; + + release = container_of(unmap, struct avd_dma_release, unmap); + dma_free_coherent(avd->dev, release->size, release->cpu, release->addr); + kfree(release); +} + +int avd_buf_alloc(struct avd_dev *avd, struct avd_buf *buf, size_t size) +{ + struct avd_dma_release *release; + + avd_buf_free(avd, buf); + if (!size) + return -ENOMEM; + + /* Allocate the release record now: freeing must never need to allocate. */ + release = kzalloc_obj(*release); + if (!release) + return -ENOMEM; + INIT_LIST_HEAD(&release->unmap.node); + release->unmap.unmap = avd_dma_unmap; + release->size = size; + release->cpu = dma_alloc_coherent(avd->dev, size, &release->addr, GFP_KERNEL); + if (!release->cpu) { + kfree(release); + return -ENOMEM; + } + + /* Payload length, not an allocation rounding, is used by codec packers. */ + buf->cpu = release->cpu; + buf->addr = release->addr; + buf->size = size; + buf->release = release; + return 0; +} + +void avd_buf_free(struct avd_dev *avd, struct avd_buf *buf) +{ + if (buf->release) + avd_pm_unmap(avd, &buf->release->unmap); + /* The record outlives the context when unmapping has to be deferred. */ + memset(buf, 0, sizeof(*buf)); +} + +bool avd_pm_can_suspend(const struct avd_frame *frames, unsigned int draining) +{ + unsigned int i; + + if (draining) + return false; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (frames[i].state != AVD_FRAME_FREE) + return false; + + return true; +} + +static bool avd_pm_queue_idle(struct avd_dev *avd) +{ + unsigned long flags; + bool idle; + + spin_lock_irqsave(&avd->queue_lock, flags); + idle = avd_pm_can_suspend(avd->frames, avd->pm.draining); + spin_unlock_irqrestore(&avd->queue_lock, flags); + return idle; +} + +void avd_pm_queue_changed(struct avd_dev *avd) +{ + wake_up_all(&avd->pm.idle_wait); +} + +void avd_pm_drain_begin(struct avd_dev *avd) +{ + unsigned long flags; + + spin_lock_irqsave(&avd->queue_lock, flags); + avd->pm.draining++; + spin_unlock_irqrestore(&avd->queue_lock, flags); +} + +void avd_pm_drain_end(struct avd_dev *avd) +{ + unsigned long flags; + + spin_lock_irqsave(&avd->queue_lock, flags); + avd->pm.draining--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); +} + +static void avd_pm_resubmit(struct avd_dev *avd) +{ + /* queue_mutex may be held by the caller of pm_runtime_resume_and_get(). */ + mod_delayed_work(system_wq, &avd->queue_work, 0); +} + +static int avd_pm_queue_drain(struct avd_dev *avd) +{ + avd_pm_resubmit(avd); + if (!wait_event_timeout(avd->pm.idle_wait, avd_pm_queue_idle(avd), + msecs_to_jiffies(AVD_DRAIN_TIMEOUT_MS))) + return -EBUSY; + return 0; +} + +const struct avd_queue_pm_ops avd_pm_queue_ops = { + .idle = avd_pm_queue_idle, + .drain = avd_pm_queue_drain, + .resubmit = avd_pm_resubmit, +}; + +void avd_pm_init(struct avd_dev *avd, const struct avd_backend_ops *backend, + const struct avd_queue_pm_ops *queue) +{ + mutex_init(&avd->pm.lock); + INIT_LIST_HEAD(&avd->pm.deferred_unmaps); + init_waitqueue_head(&avd->pm.idle_wait); + avd->pm.dart_active = false; + avd->pm.irq = -1; + avd->pm.irq_enabled = false; + spin_lock_init(&avd->pm.irq_lock); + avd->pm.hw_on = false; + avd->pm.irq_gen = 0; + avd->backend = backend; + avd->pm.queue = queue; +} + +void avd_pm_enable(struct avd_dev *avd) +{ + pm_runtime_set_autosuspend_delay(avd->dev, AVD_AUTOSUSPEND_MS); + pm_runtime_use_autosuspend(avd->dev); + pm_runtime_enable(avd->dev); +} + +int avd_pm_map_piodma(struct avd_dev *avd) +{ + struct resource *piodma, *sram; + + piodma = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "piodma"); + /* Old DTs support only the legacy transport; never invent a resource. */ + if (!piodma) + return 0; + sram = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "sram"); + if (!sram || !IS_ALIGNED(sram->start, 16) || + (sram->start >> 4) > U32_MAX || + resource_size(piodma) < AVD_PIODMA_SRAM_BASE + sizeof(u32)) + return -EINVAL; + avd->pm.piodma = devm_ioremap_resource(avd->dev, piodma); + if (IS_ERR(avd->pm.piodma)) + return PTR_ERR(avd->pm.piodma); + avd->pm.sram_phys = sram->start; + if (of_count_phandle_with_args(avd->dev->of_node, "iommus", "#iommu-cells") < 2) + dev_warn(avd->dev, "PIODMA transport requires decode and DMA IOMMU streams\n"); + return 0; +} + +void avd_pm_prepare_piodma(struct avd_dev *avd) +{ + /* Public CM3 boot ABI: replay the SRAM base after every power loss/reset. */ + if (avd->pm.piodma) + writel((u32)(avd->pm.sram_phys >> 4), + avd->pm.piodma + AVD_PIODMA_SRAM_BASE); +} + +void avd_pm_disable(struct avd_dev *avd) +{ + int ret; + + /* Power suppliers before flushing frees queued by cleanup while idle. */ + ret = pm_runtime_resume_and_get(avd->dev); + if (ret >= 0) + ret = pm_runtime_put_sync_suspend(avd->dev); + if (ret < 0) + dev_warn(avd->dev, "final power transition failed: %d\n", ret); + pm_runtime_disable(avd->dev); + /* Also close the host IRQ if the final hardware transition failed. */ + avd_pm_disable_irq(avd); + /* No queue_mutex is held here: let a woken handler thread finish. */ + if (avd->pm.irq >= 0) + synchronize_irq(avd->pm.irq); + pm_runtime_dont_use_autosuspend(avd->dev); + /* Resume may have kicked the already-stopped queue. */ + cancel_delayed_work_sync(&avd->queue_work); + WARN_ON(!list_empty(&avd->pm.deferred_unmaps)); +} + +/* pm.lock held; release records own their storage and may free themselves. */ +static void avd_pm_drain_unmaps(struct avd_dev *avd) +{ + struct avd_deferred_unmap *unmap, *next; + + list_for_each_entry_safe(unmap, next, &avd->pm.deferred_unmaps, node) { + list_del_init(&unmap->node); + unmap->unmap(avd, unmap); + } +} + +void avd_pm_unmap(struct avd_dev *avd, struct avd_deferred_unmap *unmap) +{ + mutex_lock(&avd->pm.lock); + if (avd->pm.dart_active) + unmap->unmap(avd, unmap); + else + list_add_tail(&unmap->node, &avd->pm.deferred_unmaps); + mutex_unlock(&avd->pm.lock); +} + +int avd_pm_backend_suspend(struct avd_dev *avd) +{ + avd->backend->shutdown(avd); + return 0; +} + +int avd_pm_backend_resume(struct avd_dev *avd) +{ + /* Includes transport setup, ITCM verification/load and a new generation. */ + return avd_backend_boot(avd); +} + +int avd_pm_recover(struct avd_dev *avd) +{ + bool suspended; + int ret; + + /* + * Recovery is rare and touches the decoder, its reset and its DART: + * leave a step-by-step trail in the log before each step, so a failure + * that takes the machine down still shows how far recovery got. + */ + if (!avd->pm.dart_active) { + dev_err(avd->dev, "recover: decoder DART inactive, not touching it\n"); + return -EHOSTDOWN; + } + /* The PM KUnit fixture has no struct device. */ + suspended = avd->dev && pm_runtime_suspended(avd->dev); + dev_warn(avd->dev, "recover: start (runtime %s, irq %s)\n", + suspended ? "suspended" : "active", + avd->pm.irq_enabled ? "on" : "off"); + /* Every caller holds a runtime PM reference; anything else is a bug. */ + if (WARN_ONCE(suspended, + "avd: recovery with the decoder powered off\n")) + return -EHOSTDOWN; + avd_pm_disable_irq(avd); + dev_warn(avd->dev, "recover: stop firmware\n"); + avd->backend->shutdown(avd); + dev_warn(avd->dev, "recover: reset\n"); + ret = reset_control_reset(avd->rstc); + if (ret) { + dev_err(avd->dev, "recover: reset failed: %d\n", ret); + return ret; + } + if (avd->domain) { + if (!avd->empty_domain) + return -ENODEV; + dev_warn(avd->dev, "recover: DART flush\n"); + ret = iommu_attach_device(avd->empty_domain, avd->dev); + if (ret) { + dev_err(avd->dev, "recover: DART empty attach failed: %d\n", ret); + return ret; + } + iommu_detach_device(avd->empty_domain, avd->dev); + /* Do not reboot against an empty/blocked domain after a failed restore. */ + if (iommu_get_domain_for_dev(avd->dev) != avd->domain) { + dev_err(avd->dev, "recover: DART domain not restored\n"); + return -EIO; + } + } + dev_warn(avd->dev, "recover: boot firmware\n"); + ret = avd_pm_backend_resume(avd); + if (ret) { + dev_err(avd->dev, "recover: boot failed: %d\n", ret); + avd->backend->shutdown(avd); + return ret; + } + dev_warn(avd->dev, "recover: done\n"); + return 0; +} + +int avd_pm_power_on(struct avd_dev *avd) +{ + const struct avd_backend_ops *backend = avd->backend; + int ret; + + /* genpd and the IOMMU suppliers resume before this runtime callback. */ + mutex_lock(&avd->pm.lock); + avd->pm.dart_active = true; + avd_pm_drain_unmaps(avd); + mutex_unlock(&avd->pm.lock); + + /* Do not hold pm.lock: boot may allocate or release DMA buffers. */ + ret = backend->resume ? backend->resume(avd) : avd_pm_backend_resume(avd); + if (ret) { + avd->backend->shutdown(avd); + mutex_lock(&avd->pm.lock); + avd->pm.dart_active = false; + mutex_unlock(&avd->pm.lock); + return ret; + } + avd_pm_enable_irq(avd); + /* A clean boot after power-up ends a failed recovery's fault state. */ + WRITE_ONCE(avd->faulted, false); + + if (avd->pm.queue->resubmit) + avd->pm.queue->resubmit(avd); + + return 0; +} + +int avd_pm_power_off(struct avd_dev *avd) +{ + const struct avd_backend_ops *backend = avd->backend; + int ret; + + /* PM references cover reservation through retirement, including drain. */ + if (!avd->pm.queue->idle(avd)) + return -EBUSY; + + if (backend->suspend) { + ret = backend->suspend(avd); + if (ret) + return ret; + } else { + backend->shutdown(avd); + } + avd_pm_disable_irq(avd); + + /* No more firmware DMA; suppliers stay on until this callback returns. */ + mutex_lock(&avd->pm.lock); + avd_pm_drain_unmaps(avd); + avd->pm.dart_active = false; + mutex_unlock(&avd->pm.lock); + return 0; +} + +static int avd_runtime_resume(struct device *dev) +{ + return avd_pm_power_on(dev_get_drvdata(dev)); +} + +static int avd_runtime_suspend(struct device *dev) +{ + return avd_pm_power_off(dev_get_drvdata(dev)); +} + +static int avd_system_suspend(struct device *dev) +{ + struct avd_dev *avd = dev_get_drvdata(dev); + int ret; + + v4l2_m2m_suspend(avd->m2m_dev); + ret = avd->pm.queue->drain(avd); + if (!ret) + ret = pm_runtime_force_suspend(dev); + if (ret) + v4l2_m2m_resume(avd->m2m_dev); + return ret; +} + +static int avd_system_resume(struct device *dev) +{ + struct avd_dev *avd = dev_get_drvdata(dev); + int ret; + + ret = pm_runtime_force_resume(dev); + if (ret) + return ret; + v4l2_m2m_resume(avd->m2m_dev); + return 0; +} + +const struct dev_pm_ops avd_pm_ops = { + SYSTEM_SLEEP_PM_OPS(avd_system_suspend, avd_system_resume) + RUNTIME_PM_OPS(avd_runtime_suspend, avd_runtime_resume, NULL) +}; diff --git a/drivers/media/platform/apple/avd/avd-pm.h b/drivers/media/platform/apple/avd/avd-pm.h new file mode 100644 index 0000000000000..5fe55f595beb2 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_PM_H_ +#define AVD_PM_H_ + +#include +#include +#include + +#include "avd-queue.h" + +struct avd_dma_release; + +struct avd_queue_pm_ops { + /* Snapshot under the queue lock: all FREE, no context draining. */ + bool (*idle)(struct avd_dev *avd); + /* Scheduling is paused; finish/retire all accepted frames. May sleep. */ + int (*drain)(struct avd_dev *avd); + /* Kick prepared RESERVED entries only, respecting dependencies. */ + void (*resubmit)(struct avd_dev *avd); +}; + +struct avd_deferred_unmap { + struct list_head node; + void (*unmap)(struct avd_dev *avd, struct avd_deferred_unmap *unmap); +}; + +struct avd_pm { + /* Serializes DMA releases against DART activation and firmware boot. */ + struct mutex lock; + struct list_head deferred_unmaps; + bool dart_active; + int irq; + bool irq_enabled; + /* + * irq_lock guards hw_on and irq_gen. The threaded handler touches the + * decoder only under irq_lock with hw_on set and an unchanged irq_gen, + * so turning the IRQ off never has to wait for the handler thread + * (which may itself be waiting for queue_mutex). + */ + spinlock_t irq_lock; + bool hw_on; + unsigned int irq_gen; + void __iomem *piodma; + phys_addr_t sram_phys; + wait_queue_head_t idle_wait; + /* Protected by queue_lock; counts drain operations, not stopped streams. */ + unsigned int draining; + const struct avd_queue_pm_ops *queue; +}; + +/* The caller holds the queue lock, including the draining-context count. */ +bool avd_pm_can_suspend(const struct avd_frame *frames, unsigned int draining); + +void avd_pm_init(struct avd_dev *avd, const struct avd_backend_ops *backend, + const struct avd_queue_pm_ops *queue); +void avd_pm_enable(struct avd_dev *avd); +int avd_pm_map_piodma(struct avd_dev *avd); +void avd_pm_prepare_piodma(struct avd_dev *avd); +/* After queue cleanup: release deferred DMA mappings before the final off. */ +void avd_pm_disable(struct avd_dev *avd); +/* Called in process context, after the last frame has retired its mapping. */ +void avd_pm_unmap(struct avd_dev *avd, struct avd_deferred_unmap *unmap); +int avd_pm_power_on(struct avd_dev *avd); +int avd_pm_power_off(struct avd_dev *avd); +/* Caller holds a PM reference and queue_mutex; leaves the host IRQ closed. */ +int avd_pm_recover(struct avd_dev *avd); +/* After recovery has invalidated every previously firmware-owned slot. */ +void avd_pm_enable_irq(struct avd_dev *avd); +irqreturn_t avd_pm_irq(struct avd_dev *avd, unsigned int gen); +void avd_pm_drain_begin(struct avd_dev *avd); +void avd_pm_drain_end(struct avd_dev *avd); +void avd_pm_queue_changed(struct avd_dev *avd); +extern const struct avd_queue_pm_ops avd_pm_queue_ops; + +/* Backends may share these when suspend loses all firmware context. */ +int avd_pm_backend_suspend(struct avd_dev *avd); +int avd_pm_backend_resume(struct avd_dev *avd); +extern const struct dev_pm_ops avd_pm_ops; + +#endif /* AVD_PM_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-pm.kunitconfig b/drivers/media/platform/apple/avd/avd-pm.kunitconfig new file mode 100644 index 0000000000000..4d9b37f71ff41 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.kunitconfig @@ -0,0 +1,14 @@ +CONFIG_KUNIT=y +CONFIG_COMPILE_TEST=y +CONFIG_OF=y +CONFIG_PM=y +CONFIG_SUSPEND=y +CONFIG_PM_SLEEP=y +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_VIDEO_DEV=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_VIDEO_APPLE_AVD=y +CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST=y +CONFIG_WERROR=y diff --git a/drivers/media/platform/apple/avd/avd-protocol.rst b/drivers/media/platform/apple/avd/avd-protocol.rst new file mode 100644 index 0000000000000..ce9eaf0bf2720 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-protocol.rst @@ -0,0 +1,97 @@ +AVD queued transport implementation status +========================================== + +The pool owns sixteen independently tagged slots. A source/destination pair +and a runtime-PM reference remain owned by the slot until in-order retirement +within its context. A duplicate or obsolete tag cannot complete a reused +slot. Inbox-full leaves a prepared slot reserved and retries without losing +its buffers. Timeout or streamoff resets the whole engine before failing all +slots affected by that reset. Slot deadlines survive inbox retries. + +The first commit adding avd-queue.h is the shared interface. Later additions +only extend it. Codec callbacks own codec-private state and must release that +state from retire(), including when prepare() failed part way through. + +Open firmware compatibility +-------------------------- + +The deployed, untagged open-firmware protocol has a one-frame limit. Existing +codec run() callbacks build a DMA-visible per-slot program; only the backend +writes those instructions to the VP FIFO. That compatibility backend retains +the old host-side PP kick. A native frame_ops program must use the queued +avd-fw transport before it can replace the compatibility path; this backend +returns EOPNOTSUPP for native programs rather than guessing their wire format. + +The V4L2 core currently warns unconditionally when job_finish() is used on a +queue advertising HOLD_CAPTURE_BUF. Such queues retain the scheduler until +retirement and use buf_done_and_job_finish(), including held slice requests. +Other queues release the scheduler as soon as their buffers belong to a slot. +Removing this restriction requires a V4L2 core API change outside this driver's +scope. It is not safe to toggle subsystem_flags around job_finish(). + +Apple Borage transport +---------------------- + +Only T6040 has Borage match data. The Apple backend contains the 64-byte +little-endian connection configuration, session command builders, decode +prefix, eight-entry inbox publishing, a 64-entry completion shadow and bounded +20-word result parsing. Type 7 is VP/parsing completion and never releases a +buffer. The notification address rule has one definition, +AVD_APPLE_NOTIFY_BASE, so a capture can correct it. + +Boot checks a nonzero ITCM span and a nonzero/non-all-ones version word before +accepting a preloaded image. A separately installed firmware image can restore +ITCM after power loss. The last image word is zero padded without reading +past the firmware object. No Apple image is embedded in this source tree. + +Activation remains disabled in match data: the supplied clean specification +has no ITCM register offsets, link programming sequence, result-area address, +result field layout or final-completion value. It would be unsafe to invent +these or reuse the different open-firmware mailbox. The closest executable +path is the open compatibility backend. The Apple register description and +reply layout must be completed together from clean facts; setting verified +alone is insufficient. The DT must then describe the complete Apple aperture +as an owned named resource before probe can map it. + +Unresolved field ledger +----------------------- + +This is the single ledger for fields marked [C] in the common transport. No +semantic names have been assigned to the unknown bits. Codec-specific fields +remain in their codec packers' ledgers. + +============================== ================================================ +Field Source and required evidence +============================== ================================================ +Config +00/+08/+10/+18/+20/ SPEC 3.2: exact values, meanings unproved; +28/+2c/+30/+38 capture connection construction and LinkUp +ITCM start/end/clear/enable SPEC 3.1: operations, offsets/values absent; + clean per-SoC register ledger required +IMEM version-word offset SPEC 3.1: display format proved, offset absent +Mailbox link register writes SPEC 3.2: aperture 1, state 4, LinkUp proved; + offsets and bit operations absent +Mailbox ready register SPEC 3.1: status 1, poll 10 us; offset absent +Mailbox notification word SPEC 4.2; expected inbox-entry address from MIT + aura trace_avd.py V3 precedent; capture Borage +Inbox consumption/full rule SPEC 4.2: host positions modulo eight; + hardware consumer register/doorbell absent +Result-area base/indexing SPEC 4.3: 64 entries, 20 words; base and mailbox + notification interpretation absent +Reply type/client/frame/status SPEC 4.3: correlation proved, word layout absent; + synthetic KUnit layout is not a hardware fact +Final reply type/status values SPEC 4.3: only VP done = 7 proved; capture final + completion and error replies before activation +Decode +00 bit 9 SPEC 5.2: boolean argument, meaning unproved; + left zero until the producer is established +Decode +50..+5f SPEC 5: Borage extension, nonzero use unproved; + initialized to zero before codec packing +Commands 3/4 and other values SPEC 5.1: unproved; never sent +PIODMA mapping stream binding SPEC 7: three mapper roles proved, Linux binding + and per-domain fp/program DVA ownership pending +Retention and clock floors SPEC 8: per-SoC capture required; no guessed knobs +============================== ================================================ + +Licensed implementation references: eiln/avd (MIT), avd_emu.py, and +https://github.com/aurora-silicon/aura/blob/main/proxyclient/hv/trace_avd.py +(MIT). The Borage facts above come from the supplied clean SPEC sections 3-5; +no binary or captured Apple command images are used. diff --git a/drivers/media/platform/apple/avd/avd-queue-internal.h b/drivers/media/platform/apple/avd/avd-queue-internal.h new file mode 100644 index 0000000000000..0e2a668d5a62f --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue-internal.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_QUEUE_INTERNAL_H_ +#define AVD_QUEUE_INTERNAL_H_ + +#include +#include +#include + +#include "avd-queue.h" + +/* Caller serializes the pool; tags are never recycled during device lifetime. */ +static inline int avd_slot_reserve(struct avd_frame *f, struct avd_ctx *ctx, + u64 seq) +{ + if (f->state != AVD_FRAME_FREE) + return -EBUSY; + if (!seq || seq > (U32_MAX >> 4)) + return -EOVERFLOW; + f->seq = seq; + f->tag = (seq << 4) | f->idx; + f->ctx = ctx; + f->state = AVD_FRAME_RESERVED; + f->dep = AVD_DEP_NONE; + f->status = 0; + f->prog_words = 0; + f->codec_priv = NULL; + return 0; +} + +/* A full inbox owns nothing: keep the prepared slot, tag and deadline. */ +static inline void avd_slot_submit_result(struct avd_frame *f, int status) +{ + if (status == -EBUSY) { + f->state = AVD_FRAME_RESERVED; + } else if (status) { + f->status = status; + f->state = AVD_FRAME_DONE; + } +} + +static inline bool avd_slot_complete(struct avd_frame *f, u32 tag, int status) +{ + if (f->state != AVD_FRAME_SUBMITTED || f->tag != tag) + return false; + f->status = status; + f->state = AVD_FRAME_DONE; + return true; +} + +static inline bool avd_slot_first(const struct avd_frame *pool, + const struct avd_frame *f) +{ + unsigned int i; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (pool[i].state != AVD_FRAME_FREE && pool[i].ctx == f->ctx && + pool[i].seq < f->seq) + return false; + return true; +} + +static inline bool avd_slot_expired(const struct avd_frame *f, unsigned long now) +{ + return (f->state == AVD_FRAME_SUBMITTED || + f->state == AVD_FRAME_RESERVED) && time_after_eq(now, f->deadline); +} + +static inline bool avd_inbox_full(unsigned int producer, unsigned int consumer) +{ + return ((producer + 1) & 7) == consumer; +} + +#endif /* AVD_QUEUE_INTERNAL_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-queue-test.c b/drivers/media/platform/apple/avd/avd-queue-test.c new file mode 100644 index 0000000000000..1efae38789b57 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue-test.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "avd.h" +#include "avd-apple-protocol.h" +#include "avd-queue-internal.h" + +static void avd_slots_and_tags(struct kunit *test) +{ + struct avd_frame *pool; + struct avd_ctx *ctx = (void *)test; + unsigned int i; + + pool = kunit_kcalloc(test, AVD_MAX_FRAMES, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + pool[i].idx = i; + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[i], ctx, i + 1), 0); + KUNIT_EXPECT_EQ(test, pool[i].tag, ((i + 1) << 4) | i); + KUNIT_EXPECT_EQ(test, pool[i].state, AVD_FRAME_RESERVED); + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[i], ctx, 100), -EBUSY); + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag, 0)); + pool[i].state = AVD_FRAME_SUBMITTED; + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag + 16, 0)); + KUNIT_EXPECT_TRUE(test, avd_slot_complete(&pool[i], pool[i].tag, 0)); + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag, -EIO)); + } + pool[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[0], ctx, 0), -EOVERFLOW); + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[0], ctx, (u64)(U32_MAX >> 4) + 1), + -EOVERFLOW); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[0], ctx, 100), 0); + pool[0].state = AVD_FRAME_SUBMITTED; + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[0], 16, 0)); +} + +static void avd_retirement_order(struct kunit *test) +{ + struct avd_frame *pool; + struct avd_ctx *a = (void *)test; + struct avd_ctx *b = (void *)&pool; + + pool = kunit_kcalloc(test, AVD_MAX_FRAMES, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[0], a, 1), 0); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[1], b, 2), 0); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[2], a, 3), 0); + pool[0].state = AVD_FRAME_SUBMITTED; + pool[1].state = AVD_FRAME_SUBMITTED; + pool[2].state = AVD_FRAME_SUBMITTED; + KUNIT_ASSERT_TRUE(test, avd_slot_complete(&pool[2], pool[2].tag, 0)); + KUNIT_EXPECT_FALSE(test, avd_slot_first(pool, &pool[2])); + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[1])); + KUNIT_ASSERT_TRUE(test, avd_slot_complete(&pool[0], pool[0].tag, -ETIMEDOUT)); + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[0])); + pool[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[2])); + KUNIT_EXPECT_EQ(test, pool[0].status, -ETIMEDOUT); +} + +static void avd_watchdog_and_retry(struct kunit *test) +{ + struct avd_frame f = { .state = AVD_FRAME_SUBMITTED, .deadline = ULONG_MAX - 2 }; + unsigned int producer, consumer; + u32 tag = 0x124; + unsigned long deadline = f.deadline; + + f.tag = tag; + avd_slot_submit_result(&f, -EBUSY); + KUNIT_EXPECT_EQ(test, f.state, AVD_FRAME_RESERVED); + KUNIT_EXPECT_EQ(test, f.tag, tag); + KUNIT_EXPECT_EQ(test, f.deadline, deadline); + f.state = AVD_FRAME_SUBMITTED; + avd_slot_submit_result(&f, 0); + KUNIT_EXPECT_EQ(test, f.state, AVD_FRAME_SUBMITTED); + + KUNIT_EXPECT_FALSE(test, avd_slot_expired(&f, ULONG_MAX - 3)); + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, ULONG_MAX - 2)); + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, 1)); + f.state = AVD_FRAME_DONE; + KUNIT_EXPECT_FALSE(test, avd_slot_expired(&f, 1)); + f.state = AVD_FRAME_RESERVED; + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, 1)); + for (producer = 0; producer < 8; producer++) { + for (consumer = 0; consumer < 8; consumer++) + KUNIT_EXPECT_EQ(test, avd_inbox_full(producer, consumer), + ((producer + 1) & 7) == consumer); + consumer = (producer + 1) & 7; + KUNIT_EXPECT_TRUE(test, avd_inbox_full(producer, consumer)); + /* A consumer advance reopens the same pending inbox slot. */ + consumer = (consumer + 1) & 7; + KUNIT_EXPECT_FALSE(test, avd_inbox_full(producer, consumer)); + KUNIT_EXPECT_EQ(test, avd_apple_notification(producer), + 0x01482cf8U + 0x60 * producer); + } +} + +static void avd_completion_parser(struct kunit *test) +{ + /* Synthetic layout for parser testing; never a claimed Borage wire ABI. */ + struct avd_apple_reply_layout layout = { + .type_word = 0, .type_mask = 0x1f, + .client_word = 1, .client_shift = 16, .client_mask = 0x3ff, + .frame_word = 3, .status_word = 4, .done_type = 8, .verified = true, + }; + struct avd_apple_reply reply; + u32 record[AVD_APPLE_RESULT_WORDS] = { 7, 5 << 16, 0, 0x12340 }; + + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 19, &layout, &reply), -EMSGSIZE); + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EINPROGRESS); + record[0] = 8; + KUNIT_ASSERT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), 0); + KUNIT_EXPECT_EQ(test, reply.client, 5U); + KUNIT_EXPECT_EQ(test, reply.frame, 0x12340U); + KUNIT_EXPECT_EQ(test, reply.status, 0); + record[4] = 1; + KUNIT_ASSERT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), 0); + KUNIT_EXPECT_EQ(test, reply.status, -EIO); + record[0] = 3; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -ENOMSG); + layout.frame_word = 20; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EINVAL); + layout.verified = false; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EOPNOTSUPP); +} + +static void avd_command_builders(struct kunit *test) +{ + struct avd_frame f = { .idx = 15, .frame_num = 0x1234 }; + u32 cmd[24]; + const u64 config[] = { + 0, 0x01558000, 0x01558a00, 0x01558e00, 0, + 0x000000040004000aULL, 0, 0x01100008, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(config); i++) + KUNIT_EXPECT_EQ(test, get_unaligned_le64(avd_borage_config + i * 8), config[i]); + KUNIT_ASSERT_EQ(test, avd_apple_decode_prefix(&f, 1023, 15, 3), 0); + KUNIT_EXPECT_EQ(test, f.cmd[0], 1U | (15U << 10) | (1023U << 22)); + KUNIT_EXPECT_EQ(test, f.cmd[1], 15U | (3U << 16)); + KUNIT_EXPECT_EQ(test, f.cmd[3], 0x1234U); + KUNIT_EXPECT_EQ(test, f.cmd[23], 0U); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 1024, 0, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 0, 16, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 0, 0, 4), -EINVAL); + KUNIT_ASSERT_EQ(test, avd_apple_session_command(cmd, 2, 6, 17, 42), 0); + KUNIT_EXPECT_EQ(test, cmd[0], 6U | (2U << 16)); + KUNIT_EXPECT_EQ(test, cmd[1], 17U); + KUNIT_EXPECT_EQ(test, cmd[2], 42U); + KUNIT_EXPECT_EQ(test, cmd[23], 0U); + KUNIT_EXPECT_EQ(test, avd_apple_session_command(cmd, 2, 3, 0, 0), -EINVAL); +} + +static unsigned int avd_test_max_inflight(struct avd_dev *avd) +{ + return AVD_MAX_FRAMES; +} + +static void avd_dispatch_reservation(struct kunit *test) +{ + static const struct avd_backend_ops backend = { + .max_inflight = avd_test_max_inflight, + }; + struct avd_dev *avd = kunit_kzalloc(test, sizeof(*avd), GFP_KERNEL); + struct avd_ctx *ctx = kunit_kcalloc(test, 2, sizeof(*ctx), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, avd); + KUNIT_ASSERT_NOT_NULL(test, ctx); + spin_lock_init(&avd->queue_lock); + avd->backend = &backend; + ctx[0].dev = avd; + ctx[1].dev = avd; + /* Every context with work is ready; capacity is checked at dispatch. */ + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[0]), 1); + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); + avd->inflight = AVD_MAX_FRAMES; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); + avd->inflight = 1; + avd->frames[0].state = AVD_FRAME_SUBMITTED; + avd->frames[0].ctx = &ctx[1]; + ctx[1].next_dep = AVD_DEP_PREV_DONE; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 0); + avd->frames[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); +} + +static struct kunit_case avd_queue_cases[] = { + KUNIT_CASE(avd_slots_and_tags), + KUNIT_CASE(avd_retirement_order), + KUNIT_CASE(avd_watchdog_and_retry), + KUNIT_CASE(avd_completion_parser), + KUNIT_CASE(avd_command_builders), + KUNIT_CASE(avd_dispatch_reservation), + {} +}; + +static struct kunit_suite avd_queue_suite = { + .name = "apple-avd-queue", + .test_cases = avd_queue_cases, +}; + +kunit_test_suite(avd_queue_suite); diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c new file mode 100644 index 0000000000000..915514fddbf05 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -0,0 +1,656 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include + +#include "avd.h" +#include "avd-queue-internal.h" + +#define AVD_FRAME_TIMEOUT msecs_to_jiffies(2000) +/* A stream that fails this many frames in a row in hardware is stopped. */ +#define AVD_CTX_MAX_HW_ERRORS 3 +/* Never reset the decoder more often than this. */ +#define AVD_RECOVER_GAP_MS 20 +/* More than AVD_STORM_MAX recoveries within AVD_STORM_WINDOW: cool down. */ +#define AVD_STORM_MAX 8 +#define AVD_STORM_WINDOW msecs_to_jiffies(10000) +#define AVD_COOLDOWN msecs_to_jiffies(10000) +#define AVD_RETRY_DELAY msecs_to_jiffies(1) +#define AVD_PROGRAM_SIZE SZ_2M + +bool avd_backend_is_apple(const struct avd_dev *avd) +{ + return avd->backend == &avd_apple_backend; +} + +/* Reuse the m2m list node after removing an OUTPUT from its ready queue. */ +static void avd_sources_done(struct avd_ctx *ctx, struct list_head *sources, + enum vb2_buffer_state state) +{ + struct v4l2_m2m_buffer *buf, *next; + + list_for_each_entry_safe(buf, next, sources, list) { + list_del_init(&buf->list); + if (avd_codec_frame_ops(ctx) && buf->vb.vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(buf->vb.vb2_buf.req_obj.req, + &ctx->ctrl_hdl); + v4l2_m2m_buf_done(&buf->vb, state); + } +} + +static void avd_partial_abort(struct avd_ctx *ctx) +{ + const struct avd_codec_frame_ops *ops = avd_codec_frame_ops(ctx); + + if (list_empty(&ctx->slice_sources)) + return; + if (ops && ops->retire) + ops->retire(ctx, &ctx->partial, VB2_BUF_STATE_ERROR); + else if (!ops && ctx->coded_fmt_desc->ops->done) + ctx->coded_fmt_desc->ops->done(ctx, ctx->partial.src, + ctx->partial.dst, VB2_BUF_STATE_ERROR); + if (ctx->partial.dst) + ctx->partial.dst->is_held = false; + avd_sources_done(ctx, &ctx->slice_sources, VB2_BUF_STATE_ERROR); + memset(&ctx->partial, 0, sizeof(ctx->partial)); +} + +static void avd_source_hold(struct avd_ctx *ctx, struct avd_frame *f) +{ + struct v4l2_m2m_buffer *source; + + /* Returning this OUTPUT now would let userspace overwrite DMA data. */ + v4l2_m2m_buf_copy_metadata(f->src, f->dst); + v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); + source = container_of(f->src, struct v4l2_m2m_buffer, vb); + list_add_tail(&source->list, &ctx->slice_sources); + ctx->partial = *f; + /* Core retains CAPTURE on the ready queue. Avoid m2m timestamp-based + * auto-completion of a partially collected picture. + */ + f->dst->is_held = false; + f->codec_priv = NULL; +} + +/* Transfer partial-picture source ownership even on prepare failure. */ +static void avd_sources_attach(struct avd_ctx *ctx, struct avd_frame *f) +{ + list_splice_tail_init(&ctx->slice_sources, &ctx->dev->frame_sources[f->idx]); + memset(&ctx->partial, 0, sizeof(ctx->partial)); +} + +/* queue_mutex excludes callbacks and context destruction from retirement. */ +static void avd_retire(struct avd_dev *avd) +{ + struct avd_frame *f; + unsigned long flags; + unsigned int i; + bool progress; + + do { + progress = false; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_ctx *ctx; + enum vb2_buffer_state state; + + f = &avd->frames[i]; + spin_lock_irqsave(&avd->queue_lock, flags); + if (f->state != AVD_FRAME_DONE || + !avd_slot_first(avd->frames, f)) { + spin_unlock_irqrestore(&avd->queue_lock, flags); + continue; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + ctx = f->ctx; + state = f->status ? VB2_BUF_STATE_ERROR : VB2_BUF_STATE_DONE; + if (!f->status) { + ctx->hw_errors = 0; + } else if (++ctx->hw_errors >= AVD_CTX_MAX_HW_ERRORS && !ctx->broken) { + ctx->broken = true; + dev_err(avd->dev, "stream stopped after %u failed frames; restart streaming to retry\n", + ctx->hw_errors); + } + if (avd_codec_frame_ops(ctx)) { + if (avd_codec_frame_ops(ctx)->retire) + avd_codec_frame_ops(ctx)->retire(ctx, f, state); + } else if (ctx->coded_fmt_desc->ops->done) { + ctx->coded_fmt_desc->ops->done(ctx, f->src, f->dst, state); + } + if (avd_codec_frame_ops(ctx) && f->src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(f->src->vb2_buf.req_obj.req, + &ctx->ctrl_hdl); + /* CAPTURE first: OUTPUT completion signals the media request. */ + if (f->scheduler_held) { + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, + ctx->fh.m2m_ctx, state); + } else { + v4l2_m2m_buf_done(f->dst, state); + avd_sources_done(ctx, &avd->frame_sources[f->idx], state); + v4l2_m2m_buf_done(f->src, state); + } + spin_lock_irqsave(&avd->queue_lock, flags); + f->ctx = NULL; + f->src = NULL; + f->dst = NULL; + f->state = AVD_FRAME_FREE; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + progress = true; + } + } while (progress); +} + +/* A reset invalidates every firmware-owned slot, including other clients. */ +static void avd_reset_queue(struct avd_dev *avd, int status) +{ + struct avd_ctx *ctx; + unsigned long flags; + unsigned int i; + int ret; + + /* + * A stream whose every frame faults (an H0 error storm) used to reset + * the decoder back to back, about once per millisecond, until the + * machine hung. Space recoveries out, and after a storm stop taking + * new frames for a while instead of resetting again. + */ + if (avd->last_recover && + time_before(jiffies, avd->last_recover + msecs_to_jiffies(AVD_RECOVER_GAP_MS))) + msleep(AVD_RECOVER_GAP_MS); + if (!avd->storm_count || time_after(jiffies, avd->storm_start + AVD_STORM_WINDOW)) { + avd->storm_start = jiffies; + avd->storm_count = 0; + } + if (++avd->storm_count > AVD_STORM_MAX) { + dev_err(avd->dev, "%u decoder recoveries in %u s: failing new frames for %u s\n", + avd->storm_count, jiffies_to_msecs(AVD_STORM_WINDOW) / 1000, + jiffies_to_msecs(AVD_COOLDOWN) / 1000); + WRITE_ONCE(avd->cooldown_until, jiffies + AVD_COOLDOWN); + avd->storm_count = 0; + } + ret = avd_pm_recover(avd); + avd->last_recover = jiffies ?: 1; + WRITE_ONCE(avd->faulted, !!ret); + if (ret) + dev_err(avd->dev, "queue reset failed: %d\n", ret); + avd->compat_active = NULL; + list_for_each_entry(ctx, &avd->contexts, queue_node) + ctx->session_started = false; + spin_lock_irqsave(&avd->queue_lock, flags); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state == AVD_FRAME_FREE) + continue; + avd->frames[i].status = status; + avd->frames[i].state = AVD_FRAME_DONE; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (!ret) + avd_pm_enable_irq(avd); +} + +static int avd_send_frame(struct avd_dev *avd, struct avd_frame *f) +{ + unsigned long flags; + int ret; + + if (f->dep == AVD_DEP_PREV_DONE && !avd_slot_first(avd->frames, f)) + return -EBUSY; + if ((!f->ctx->session_started || + f->ctx->session_generation != avd->fw_generation) && + avd->backend->session_start) { + ret = avd->backend->session_start(f->ctx); + if (ret) + return ret; + f->ctx->session_started = true; + f->ctx->session_generation = avd->fw_generation; + } + /* Publish ownership before the doorbell; IRQs may arrive in submit(). */ + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_SUBMITTED; + spin_unlock_irqrestore(&avd->queue_lock, flags); + dma_wmb(); + ret = avd->backend->submit(avd, f); + spin_lock_irqsave(&avd->queue_lock, flags); + avd_slot_submit_result(f, ret); + spin_unlock_irqrestore(&avd->queue_lock, flags); + return ret; +} + +static void avd_queue_worker(struct work_struct *work) +{ + struct avd_dev *avd = container_of(to_delayed_work(work), + struct avd_dev, queue_work); + struct avd_ctx *ctx; + unsigned long flags; + unsigned int i; + bool pending; + unsigned long delay = AVD_FRAME_TIMEOUT; + + mutex_lock(&avd->queue_mutex); + if (avd->stopping) + goto unlock; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + bool expired; + + spin_lock_irqsave(&avd->queue_lock, flags); + expired = avd_slot_expired(&avd->frames[i], jiffies); + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (expired) { + struct avd_frame *t = &avd->frames[i]; + + dev_err(avd->dev, "frame tag %#x timed out (%p4cc, state %d, %u in flight, %s)\n", + t->tag, t->ctx ? &t->ctx->coded_fmt_desc->fourcc : &(u32){0}, + t->state, avd->inflight, avd->backend->name); + avd_reset_queue(avd, -ETIMEDOUT); + break; + } + } + avd_retire(avd); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + int ret; + + if (f->state != AVD_FRAME_RESERVED) + continue; + ret = avd_send_frame(avd, f); + if (ret && ret != -EBUSY) { + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + } + } + avd_retire(avd); + /* Restart a job that found no free slot. */ + spin_lock_irqsave(&avd->queue_lock, flags); + ctx = avd->inflight < avd->backend->max_inflight(avd) ? avd->deferred_ctx : NULL; + if (ctx) + avd->deferred_ctx = NULL; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (ctx) + schedule_work(&ctx->submit_work); + /* Freed slots/dependencies must wake contexts whose job_ready was false. */ + list_for_each_entry(ctx, &avd->contexts, queue_node) + v4l2_m2m_try_schedule(ctx->fh.m2m_ctx); + spin_lock_irqsave(&avd->queue_lock, flags); + pending = avd->inflight != 0; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + unsigned long remaining; + + if (f->state == AVD_FRAME_FREE) + continue; + remaining = time_after(f->deadline, jiffies) ? f->deadline - jiffies : 0; + if (f->state == AVD_FRAME_RESERVED) + remaining = min(remaining, AVD_RETRY_DELAY); + delay = min(delay, remaining); + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (pending) + mod_delayed_work(system_wq, &avd->queue_work, delay); + unlock: + mutex_unlock(&avd->queue_mutex); +} + +void avd_frame_complete(struct avd_dev *avd, u32 tag, int status) +{ + struct avd_frame *f = &avd->frames[tag & (AVD_MAX_FRAMES - 1)]; + unsigned long flags; + bool accepted; + + spin_lock_irqsave(&avd->queue_lock, flags); + accepted = avd_slot_complete(f, tag, status); + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (accepted) + mod_delayed_work(system_wq, &avd->queue_work, 0); +} + +int avd_queue_ready(void *priv) +{ + struct avd_ctx *ctx = priv; + struct avd_dev *avd = ctx->dev; + unsigned long flags; + unsigned int i; + bool ready; + + spin_lock_irqsave(&avd->queue_lock, flags); + /* + * A faulted or cooling-down decoder still schedules: submission fails + * those frames at once, so clients see errors instead of waiting. + */ + /* + * Slot capacity is not a readiness condition: every context with work + * enters the m2m job queue, which runs them in order, and a job that + * finds no free slot is deferred until one retires. Gating readiness on + * one reserved context let the first context in the list win every + * time and starved the others (multi-stream decodes timing out). + */ + ready = !READ_ONCE(ctx->draining) && !READ_ONCE(avd->stopping); + if (ready && ctx->next_dep == AVD_DEP_PREV_DONE) { + /* Legacy codecs may hold several pictures in separate banks. */ + unsigned int limit = ctx->coded_fmt_desc && !avd_codec_frame_ops(ctx) ? + max(ctx->max_frames, 1U) : 1, held = 0; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (avd->frames[i].state != AVD_FRAME_FREE && + avd->frames[i].ctx == ctx) + held++; + ready = held < limit; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + return ready; +} + +static void avd_queue_submit_work(struct work_struct *work) +{ + struct avd_ctx *ctx = container_of(work, struct avd_ctx, submit_work); + struct avd_dev *avd = ctx->dev; + const struct avd_codec_frame_ops *ops = avd_codec_frame_ops(ctx); + struct avd_frame *f = NULL; + unsigned long flags; + unsigned int i; + int ret; + + mutex_lock(&avd->queue_mutex); + spin_lock_irqsave(&avd->queue_lock, flags); + if (avd->deferred_ctx == ctx) + avd->deferred_ctx = NULL; + if (avd->stopping || ctx->draining || avd->faulted || ctx->broken || + time_before(jiffies, READ_ONCE(avd->cooldown_until))) { + struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_partial_abort(ctx); + /* Complete the request's controls too, or its fd never signals. */ + if (src && src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + goto unlock; + } + if (avd->inflight < avd->backend->max_inflight(avd)) { + for (i = 0; i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state == AVD_FRAME_FREE) { + f = &avd->frames[i]; + break; + } + } + } + if (!f) { + /* Keep the m2m job; the worker restarts it when a slot frees. */ + avd->deferred_ctx = ctx; + spin_unlock_irqrestore(&avd->queue_lock, flags); + mod_delayed_work(system_wq, &avd->queue_work, 0); + goto unlock; + } + ret = avd_slot_reserve(f, ctx, ++avd->frame_seq); + if (!ret) + avd->inflight++; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (WARN_ON(ret)) { + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + goto unlock; + } + ret = pm_runtime_resume_and_get(avd->dev); + if (ret < 0) + goto fail_no_pm; + f->src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + f->dst = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); + /* + * Only a slice that holds its CAPTURE buffer needs the m2m scheduler + * until retirement. The last slice of a picture releases both buffers + * to the slot, so the next picture can be prepared while it decodes. + */ + f->scheduler_held = (ctx->fh.m2m_ctx->out_q_ctx.q.subsystem_flags & + VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF) && + (f->src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF); + f->frame_num = f->tag; + f->deadline = jiffies + AVD_FRAME_TIMEOUT; + memset(f->cmd, 0, sizeof(f->cmd)); + if (ops) { + if (!f->prog->cpu) { + ret = avd_buf_alloc(avd, f->prog, AVD_PROGRAM_SIZE); + if (ret) + goto fail; + } + /* NULL fp is also the codec-facing indication of an open program. */ + f->fp = NULL; + if (avd_backend_is_apple(avd)) { + struct avd_buf *fp = avd->frame_fp[f->idx]; + + if (!fp) { + fp = devm_kzalloc(avd->dev, sizeof(*fp), GFP_KERNEL); + avd->frame_fp[f->idx] = fp; + } + f->fp = fp; + if (!f->fp) { + ret = -ENOMEM; + goto fail; + } + if (!f->fp->cpu) { + ret = avd_buf_alloc(avd, f->fp, avd->variant->fp_size); + if (ret) + goto fail; + } + } + if (f->src->vb2_buf.req_obj.req) { + ret = v4l2_ctrl_request_setup(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + if (ret) + goto fail; + } + ret = ops->prepare(ctx, f); + } else { + avd->building = f; + avd->compat_held = false; + ret = ctx->coded_fmt_desc->ops->run(ctx); + avd->building = NULL; + } + if (ret) + goto fail; + ctx->next_dep = ops ? f->dep : AVD_DEP_PREV_DONE; + if ((ops && (f->src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF)) || + (!ops && avd->compat_held)) { + avd_source_hold(ctx, f); + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_FREE; + f->ctx = NULL; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + v4l2_m2m_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx); + goto unlock; + } + avd_sources_attach(ctx, f); + v4l2_m2m_buf_copy_metadata(f->src, f->dst); + if (!f->scheduler_held) { + v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); + v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + } + f->dst->is_held = false; + ret = avd_send_frame(avd, f); + if (ret && ret != -EBUSY) { + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + } + /* Buffers now belong to the slot, not to the m2m scheduler. */ + if (!f->scheduler_held) + v4l2_m2m_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx); + mod_delayed_work(system_wq, &avd->queue_work, 0); + goto unlock; + fail: + avd_sources_attach(ctx, f); + /* Failure ends a held picture as well as its final request. */ + f->scheduler_held = false; + if (f->dst) + f->dst->is_held = false; + /* Detach before retirement, including partially prepared codec state. */ + if (!f->scheduler_held) { + v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); + v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + } + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (!f->scheduler_held) + v4l2_m2m_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx); + mod_delayed_work(system_wq, &avd->queue_work, 0); + goto unlock; + fail_no_pm: + avd_partial_abort(ctx); + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_FREE; + f->ctx = NULL; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + unlock: + mutex_unlock(&avd->queue_mutex); +} + +void avd_queue_run(void *priv) +{ + struct avd_ctx *ctx = priv; + + /* try_schedule() may invoke device_run synchronously under queue_mutex. */ + schedule_work(&ctx->submit_work); +} + +void avd_queue_drain(struct avd_ctx *ctx) +{ + struct avd_dev *avd = ctx->dev; + unsigned long flags; + unsigned int i; + bool finish_deferred = false; + int ret; + + /* Session stop/reset accesses registers even when there are no frames. */ + ret = pm_runtime_resume_and_get(avd->dev); + avd_pm_drain_begin(avd); + mutex_lock(&avd->queue_mutex); + spin_lock_irqsave(&avd->queue_lock, flags); + WRITE_ONCE(ctx->draining, true); + if (avd->deferred_ctx == ctx) { + avd->deferred_ctx = NULL; + finish_deferred = true; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + /* A deferred job is still m2m's running job: end it so abort can wait. */ + if (finish_deferred) + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + for (i = 0; ret >= 0 && i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state != AVD_FRAME_FREE && avd->frames[i].ctx == ctx) { + if (avd->backend->abort) + avd->backend->abort(ctx); + avd_reset_queue(avd, -ECANCELED); + break; + } + } + avd_retire(avd); + avd_partial_abort(ctx); + if (ret >= 0 && ctx->session_started && avd->backend->session_stop && + ctx->session_generation == avd->fw_generation) + avd->backend->session_stop(ctx); + ctx->session_started = false; + if (!avd->stopping) + mod_delayed_work(system_wq, &avd->queue_work, 0); + mutex_unlock(&avd->queue_mutex); + avd_pm_drain_end(avd); + if (ret >= 0) { + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + } +} + +void avd_queue_abort(void *priv) +{ + avd_queue_drain(priv); +} + +void avd_queue_add_ctx(struct avd_ctx *ctx) +{ + struct avd_ctx *other; + u32 id; + + INIT_WORK(&ctx->submit_work, avd_queue_submit_work); + INIT_LIST_HEAD(&ctx->slice_sources); + mutex_lock(&ctx->dev->queue_mutex); + /* Borage decode prefix has ten client bits. */ + for (id = 0; id < 1024; id++) { + bool used = false; + + list_for_each_entry(other, &ctx->dev->contexts, queue_node) + used |= other->client_id == id; + if (!used) + break; + } + ctx->client_id = id; + list_add_tail(&ctx->queue_node, &ctx->dev->contexts); + mutex_unlock(&ctx->dev->queue_mutex); +} + +void avd_queue_del_ctx(struct avd_ctx *ctx) +{ + avd_queue_drain(ctx); + mutex_lock(&ctx->dev->queue_mutex); + if (ctx->dev->compat_ctx == ctx) + ctx->dev->compat_ctx = NULL; + list_del_init(&ctx->queue_node); + mutex_unlock(&ctx->dev->queue_mutex); +} + +int avd_queue_init(struct avd_dev *avd) +{ + unsigned int i; + + spin_lock_init(&avd->queue_lock); + mutex_init(&avd->queue_mutex); + INIT_LIST_HEAD(&avd->contexts); + INIT_DELAYED_WORK(&avd->queue_work, avd_queue_worker); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + + f->idx = i; + INIT_LIST_HEAD(&avd->frame_sources[i]); + f->prog = devm_kzalloc(avd->dev, sizeof(*f->prog), GFP_KERNEL); + if (!f->prog) + return -ENOMEM; + } + return 0; +} + +void avd_queue_cleanup(struct avd_dev *avd) +{ + struct avd_ctx *ctx; + unsigned int i; + + mutex_lock(&avd->queue_mutex); + WRITE_ONCE(avd->stopping, true); + if (avd->inflight) { + avd_reset_queue(avd, -ESHUTDOWN); + avd_retire(avd); + } + list_for_each_entry(ctx, &avd->contexts, queue_node) + avd_partial_abort(ctx); + mutex_unlock(&avd->queue_mutex); + cancel_delayed_work_sync(&avd->queue_work); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + avd_buf_free(avd, avd->frames[i].prog); + if (avd->frame_fp[i]) + avd_buf_free(avd, avd->frame_fp[i]); + } +} diff --git a/drivers/media/platform/apple/avd/avd-queue.h b/drivers/media/platform/apple/avd/avd-queue.h new file mode 100644 index 0000000000000..cd912259c636a --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_QUEUE_H_ +#define AVD_QUEUE_H_ + +#include +#include +#include + +struct avd_dev; +struct avd_ctx; +struct avd_buf; /* avd.h: { void *cpu; dma_addr_t addr; size_t size; } */ + +#define AVD_MAX_FRAMES 16 /* SPEC 4.1: 16 frame-param slots */ + +enum avd_frame_state { + AVD_FRAME_FREE = 0, /* SPEC: slot state 0 */ + AVD_FRAME_RESERVED, /* 1: codec is filling it on the CPU */ + AVD_FRAME_SUBMITTED, /* 3: owned by firmware */ + AVD_FRAME_DONE, /* completion seen, waiting for in-order retire */ +}; + +/* Codec says what must finish before this frame may be submitted. */ +enum avd_frame_dep { + AVD_DEP_NONE = 0, /* independent: may overlap the previous frame */ + AVD_DEP_PREV_DONE, /* previous frame of this ctx must be DONE first + * (VP9 backward adaptation, AV1 CDF update, ...) */ +}; + +struct avd_frame { + unsigned int idx; /* 0..AVD_MAX_FRAMES-1, also the fp slot */ + u32 tag; /* (seq << 4) | idx, echoed by completion */ + u64 seq; /* global submission order */ + enum avd_frame_state state; + enum avd_frame_dep dep; + int status; /* 0 or -EIO/-ETIMEDOUT from the backend */ + + struct avd_ctx *ctx; + struct vb2_v4l2_buffer *src; /* OUTPUT, held until retire */ + struct vb2_v4l2_buffer *dst; /* CAPTURE, held until retire */ + u32 frame_num; /* SPEC 5.2 +0x0c */ + + /* open backend: instruction program, DMA-visible, built by the codec */ + struct avd_buf *prog; /* per-slot, allocated by core */ + size_t prog_words; + + /* apple backend: frame-params slot (SPEC 0x1130d0 on Borage) */ + struct avd_buf *fp; /* per-slot, allocated by core, size from match data */ + u32 cmd[24]; /* 0x60 Borage command; core fills the 5.2 prefix, + * codec pack_cmd() fills the codec part */ + + void *codec_priv; /* per-frame codec state, owned by the codec */ + unsigned long deadline; /* jiffies, per-frame watchdog */ + /* Legacy m2m HOLD_CAPTURE API retains scheduler ownership until retire. */ + bool scheduler_held; + /* VP to PP instruction FIFO for this frame (open backends). */ + dma_addr_t fifo_iova; +}; + +struct avd_backend_ops { + const char *name; + int (*probe)(struct avd_dev *avd); /* apertures, fw source, version */ + int (*boot)(struct avd_dev *avd); /* power-up: fw present/loaded, link up, ready */ + void (*shutdown)(struct avd_dev *avd); + unsigned int (*max_inflight)(struct avd_dev *avd); + int (*session_start)(struct avd_ctx *ctx); /* apple: cmd 0; open: may be NULL */ + void (*session_stop)(struct avd_ctx *ctx); /* apple: cmd 2 */ + /* Non-blocking. -EBUSY means inbox full: core retries on next completion. */ + int (*submit)(struct avd_dev *avd, struct avd_frame *f); + void (*abort)(struct avd_ctx *ctx); /* apple: cmd 5 */ + irqreturn_t (*irq)(struct avd_dev *avd); /* calls avd_frame_complete() */ + int (*suspend)(struct avd_dev *avd); + int (*resume)(struct avd_dev *avd); + int (*set_property)(struct avd_ctx *ctx, u32 property, u32 value); +}; + +/* Codec half. Replaces avd_coded_fmt_ops.run/submit/done. CPU only, no MMIO. */ +struct avd_codec_frame_ops { + /* Read the request's controls, set f->dep, fill f->prog (open backend) + * or f->fp (apple backend), allocate f->codec_priv if needed. */ + int (*prepare)(struct avd_ctx *ctx, struct avd_frame *f); + /* Apple backend: fill the codec part of f->cmd (SPEC 6.x). */ + int (*pack_cmd)(struct avd_ctx *ctx, struct avd_frame *f); + /* Called in submission order per ctx, after completion or failure. */ + void (*retire)(struct avd_ctx *ctx, struct avd_frame *f, + enum vb2_buffer_state state); +}; + +/* Core API used by backends and codecs. */ +void avd_frame_complete(struct avd_dev *avd, u32 tag, int status); /* any ctx, IRQ-safe */ +bool avd_backend_is_apple(const struct avd_dev *avd); + +#endif /* AVD_QUEUE_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-regs.h b/drivers/media/platform/apple/avd/avd-regs.h index 388f3e64895f5..ace0e4c48f40a 100644 --- a/drivers/media/platform/apple/avd/avd-regs.h +++ b/drivers/media/platform/apple/avd/avd-regs.h @@ -13,6 +13,8 @@ #define AVD_MBOX1_NOT_EMPTY BIT(3) #define AVD_REG_MBOX1_STATUS 0x5c +/* Read side: bits 8-11 write pointer, bits 12-15 read pointer, bit 17 empty. */ +#define AVD_MBOX1_STATUS_EMPTY BIT(17) #define AVD_REG_MBOX1_RETRIEVE 0x64 #define AVD_MBOX_ENABLE BIT(0) diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index 4622e92055486..472aa6f9f6628 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -416,6 +416,7 @@ static const struct avd_coded_fmt_desc avd_coded_fmts[] = { }, .ctrls = &avd_h264_ctrls, .ops = &avd_h264_fmt_ops, + .frame_ops = &avd_h264_frame_ops, .num_decoded_fmts = ARRAY_SIZE(avd_h264_decoded_fmts), .decoded_fmts = avd_h264_decoded_fmts, .subsystem_flags = VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF, @@ -893,8 +894,12 @@ static int avd_start_streaming(struct vb2_queue *q, unsigned int count) struct vb2_v4l2_buffer *vbuf; int ret; + WRITE_ONCE(ctx->draining, false); if (V4L2_TYPE_IS_CAPTURE(q->type)) return 0; + /* A new stream gets a fresh hardware error budget. */ + ctx->hw_errors = 0; + ctx->broken = false; desc = ctx->coded_fmt_desc; if (WARN_ON(!desc)) { @@ -917,7 +922,7 @@ static int avd_start_streaming(struct vb2_queue *q, unsigned int count) return ret; } -static void avd_queue_cleanup(struct vb2_queue *vq, u32 state) +static void avd_m2m_queue_cleanup(struct vb2_queue *vq, u32 state) { struct avd_ctx *ctx = vb2_get_drv_priv(vq); @@ -942,6 +947,7 @@ static void avd_stop_streaming(struct vb2_queue *q) { struct avd_ctx *ctx = vb2_get_drv_priv(q); + avd_queue_drain(ctx); if (V4L2_TYPE_IS_OUTPUT(q->type)) { const struct avd_coded_fmt_desc *desc = ctx->coded_fmt_desc; @@ -956,7 +962,7 @@ static void avd_stop_streaming(struct vb2_queue *q) ctx->job.segments = NULL; } - avd_queue_cleanup(q, VB2_BUF_STATE_ERROR); + avd_m2m_queue_cleanup(q, VB2_BUF_STATE_ERROR); } const struct vb2_ops avd_queue_ops = { @@ -970,36 +976,17 @@ const struct vb2_ops avd_queue_ops = { .stop_streaming = avd_stop_streaming, }; +/* Legacy multi-slice codecs finish held OUTPUT requests during prepare. */ void avd_job_finish_no_pm(struct avd_ctx *ctx, enum vb2_buffer_state result) { - struct avd_dev *avd = ctx->dev; - unsigned long flags; - - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_ctx == ctx) { - avd->job_state = AVD_JOB_IDLE; - avd->job_ctx = NULL; - avd->job_pending = false; - } - spin_unlock_irqrestore(&avd->job_lock, flags); - - if (ctx->coded_fmt_desc->ops->done) { - struct vb2_v4l2_buffer *src_buf, *dst_buf; - - src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); - dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); - ctx->coded_fmt_desc->ops->done(ctx, src_buf, dst_buf, result); - } - - v4l2_m2m_buf_done_and_job_finish(ctx->dev->m2m_dev, ctx->fh.m2m_ctx, - result); + if (WARN_ON(!ctx->dev->building)) + return; + ctx->dev->compat_held = true; + ctx->dev->building->status = result == VB2_BUF_STATE_DONE ? 0 : -EIO; } void avd_job_finish(struct avd_ctx *ctx, enum vb2_buffer_state result) { - struct avd_dev *avd = ctx->dev; - - pm_runtime_put_autosuspend(avd->dev); avd_job_finish_no_pm(ctx, result); } diff --git a/drivers/media/platform/apple/avd/avd-vp9.c b/drivers/media/platform/apple/avd/avd-vp9.c index 4cdcb6594c4d2..37374b73e8a86 100644 --- a/drivers/media/platform/apple/avd/avd-vp9.c +++ b/drivers/media/platform/apple/avd/avd-vp9.c @@ -1049,6 +1049,13 @@ static enum avd_image_fmt avd_vp9_get_image_fmt(struct avd_ctx *ctx, #undef BIT_DEPTH } +static unsigned int avd_vp9_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_vp9_ctx *priv = ctx->priv; + + return max_t(int, priv->submit_num, 1); +} + static void avd_vp9_submit(struct avd_ctx *ctx) { struct avd_vp9_ctx *vp9_ctx = ctx->priv; @@ -1078,5 +1085,6 @@ const struct avd_coded_fmt_ops avd_vp9_fmt_ops = { .run = avd_vp9_run, .done = avd_vp9_done, .submit = avd_vp9_submit, + .pp_kicks = avd_vp9_pp_kicks, .get_image_fmt = avd_vp9_get_image_fmt, }; diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index ad7accb6e9615..6257939664216 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -19,6 +19,12 @@ #include #include +#include "avd-pm.h" +#include + +#include "avd-queue.h" +#include "avd-h264-fp.h" + #include #include #include @@ -123,6 +129,8 @@ struct avd_decoded_buffer { /* Pending CPU fill, retained with the buffer until dequeue. */ size_t grey_chroma_offset; bool grey_chroma_10bit; + /* H.264/HEVC: scratch bank + 1 of the picture decoded into this buffer */ + u8 bank; union { struct avd_vp9_decoded_buffer_info vp9; @@ -144,6 +152,8 @@ struct avd_coded_fmt_ops { void (*adjust_decoded_fmt)(struct avd_ctx *ctx, struct v4l2_pix_format_mplane *pix_mp); void (*submit)(struct avd_ctx *ctx); + /* PP EXEC commands the last built frame needs (submit's count), >= 1. */ + unsigned int (*pp_kicks)(struct avd_ctx *ctx); int (*start)(struct avd_ctx *ctx); void (*stop)(struct avd_ctx *ctx); int (*run)(struct avd_ctx *ctx); @@ -173,20 +183,30 @@ struct avd_coded_fmt_desc { struct v4l2_frmsize_stepwise frmsize; const struct avd_ctrls *ctrls; const struct avd_coded_fmt_ops *ops; + const struct avd_codec_frame_ops *frame_ops; unsigned int num_decoded_fmts; const struct avd_decoded_fmt_desc *decoded_fmts; u32 subsystem_flags; unsigned int capability; }; +struct avd_apple_hw; + struct avd_variant { + const struct avd_apple_hw *apple; + enum avd_h264_fp_layout h264_fp_layout; + size_t fp_size; unsigned int vp_slots[4]; unsigned int vp_slot_base[4]; unsigned int fifo_slots; unsigned int capabilities; + /* Codecs advertised only when the firmware CODECS trailer lists them. */ + unsigned int fw_caps_required; void (*configure_stream)(struct avd_dev *avd, dma_addr_t addr, u8 fifo_idx, u32 vp_slot); const char* fw_name; + /* Older image tried when fw_name is not installed (legacy transport). */ + const char *fw_name_fallback; unsigned char revision; /* the same as the device tree */ /* just for convenience */ u32 vp_slot_offset; @@ -196,23 +216,16 @@ struct avd_variant { unsigned int quirks; }; -/* Lifecycle of the one m2m job the device runs at a time. */ -enum avd_job_state { - AVD_JOB_IDLE, - AVD_JOB_RUNNING, /* inside avd_device_run() */ - AVD_JOB_SUBMITTED, /* handed to the hardware, watchdog armed */ - AVD_JOB_FINISHING, /* claimed by the IRQ handler or the watchdog */ -}; - struct avd_dev { struct device *dev; + struct avd_pm pm; struct v4l2_device v4l2_dev; struct media_device mdev; struct video_device vdev; struct v4l2_m2m_dev *m2m_dev; struct platform_device *pdev; - const struct firmware *fw; /* fw is lost on suspend */ + const struct firmware *fw; /* separately installed open firmware */ void __iomem *code; void __iomem *mbox; @@ -223,19 +236,50 @@ struct avd_dev { struct mutex vdev_lock; - /* The current job; see avd_device_run(). Protected by job_lock. */ - spinlock_t job_lock; - enum avd_job_state job_state; - struct avd_ctx *job_ctx; - u64 job_seq; - bool job_end_sent; - bool job_pending; - enum vb2_buffer_state job_pending_result; + /* queue_lock protects slot state/tag and job_ready observations. */ + spinlock_t queue_lock; + /* Serializes CPU preparation, submission, retirement and reset. */ + struct mutex queue_mutex; + struct delayed_work queue_work; + struct list_head contexts; + /* One m2m dispatch reservation; never identifies firmware-owned work. */ + /* + * m2m runs one device_run at a time until job_finish; a job that finds + * no free slot waits here and is restarted when a frame retires. + */ + struct avd_ctx *deferred_ctx; + struct avd_frame frames[AVD_MAX_FRAMES]; + struct avd_buf *frame_fp[AVD_MAX_FRAMES]; + /* Earlier OUTPUTs in each submitted multi-slice picture. */ + struct list_head frame_sources[AVD_MAX_FRAMES]; + u64 frame_seq; + unsigned int inflight; + bool stopping; + bool faulted; + /* Recovery pacing: when the last one ran, and a sliding storm count. */ + unsigned long last_recover; + unsigned long storm_start; + unsigned int storm_count; + /* After a recovery storm, fail new frames until this time. */ + unsigned long cooldown_until; + u64 fw_generation; + const struct avd_backend_ops *backend; + void *backend_priv; + /* Only the open-firmware compatibility path uses these. */ + struct avd_frame *building; + /* Open firmware queue ABI 2 state, NULL when the node has no SRAM. */ + struct avd_qfw *qfw; + struct avd_frame *compat_active; + /* Context of the last legacy submission, for VP DONE without a frame. */ + struct avd_ctx *compat_ctx; + bool compat_held; struct reset_control *rstc; const struct avd_variant *variant; unsigned int capabilities; + /* What the loaded image's CODECS trailer (or its absence) allows. */ + unsigned int fw_capabilities; }; struct avd_segment { @@ -253,6 +297,7 @@ struct avd_buf { void *cpu; dma_addr_t addr; size_t size; + struct avd_dma_release *release; }; struct avd_ctx { @@ -267,14 +312,29 @@ struct avd_ctx { enum avd_image_fmt image_fmt; bool decomp; - struct delayed_work watchdog_work; - /* job_seq of the job watchdog_work was armed for */ - u64 wd_seq; + struct list_head queue_node; + struct work_struct submit_work; + /* Unsubmitted picture: OUTPUTs stay ACTIVE through final completion. */ + struct list_head slice_sources; + struct avd_frame partial; + bool draining; + /* Consecutive frames that failed in hardware; reset by a good frame. */ + unsigned int hw_errors; + /* Too many hardware errors: fail every frame until streaming restarts. */ + bool broken; + bool session_started; + u64 session_generation; + enum avd_frame_dep next_dep; + u32 client_id; void *priv; struct avd_comp comp; int fifo_idx; + /* Pictures of this context the queue may hold at once (0 or 1: one). */ + unsigned int max_frames; + /* VP to PP FIFO for the job being built; 0 means ctx->inst. */ + dma_addr_t job_fifo; struct avd_job job; struct avd_buf inst; }; @@ -302,6 +362,7 @@ bool avd_h264_slice_fits(const struct v4l2_ctrl_h264_sps *sps, size_t held, size_t held_bytes, u32 len, size_t decoded_size); #endif +extern const struct avd_codec_frame_ops avd_h264_frame_ops; extern const struct avd_coded_fmt_ops avd_h264_fmt_ops; extern const struct avd_coded_fmt_ops avd_hevc_fmt_ops; extern const struct avd_coded_fmt_ops avd_vp9_fmt_ops; @@ -330,6 +391,35 @@ static inline struct avd_ctx *file_to_ctx(struct file *filp) /* hw stuff */ int avd_boot(struct avd_dev *avd); +int avd_queue_init(struct avd_dev *avd); +void avd_queue_cleanup(struct avd_dev *avd); +void avd_queue_run(void *priv); +int avd_queue_ready(void *priv); +void avd_queue_abort(void *priv); +void avd_queue_drain(struct avd_ctx *ctx); +void avd_queue_add_ctx(struct avd_ctx *ctx); +void avd_queue_del_ctx(struct avd_ctx *ctx); +extern const struct avd_backend_ops avd_open_backend; + +/* Old open firmware needs the legacy callbacks; native queued transport does not. */ +static inline const struct avd_codec_frame_ops *avd_codec_frame_ops(struct avd_ctx *ctx) +{ + if (ctx->dev->backend == &avd_open_backend) + return NULL; + return ctx->coded_fmt_desc->frame_ops; +} + +int avd_open_build_program(struct avd_ctx *ctx); +extern const struct avd_backend_ops avd_apple_backend; +extern const struct avd_backend_ops avd_qfw_backend; +struct avd_qfw; +int avd_qfw_probe(struct avd_dev *avd); +void avd_qfw_select(struct avd_dev *avd); +void avd_qfw_prepare_boot(struct avd_dev *avd); +bool avd_backend_is_qfw(const struct avd_dev *avd); +int avd_backend_select(struct avd_dev *avd); +bool avd_mbox_pop(struct avd_dev *avd, u32 *msg); +int avd_backend_boot(struct avd_dev *avd); void avd_shutdown(struct avd_dev *avd); void avd_status(struct avd_dev *avd, u32 vp); diff --git a/drivers/media/platform/apple/avd/tests/.kunitconfig b/drivers/media/platform/apple/avd/tests/.kunitconfig new file mode 100644 index 0000000000000..c079bc44af531 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/.kunitconfig @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +CONFIG_KUNIT=y +CONFIG_COMPILE_TEST=y +CONFIG_MEDIA_SUPPORT=y +# CONFIG_MEDIA_SUPPORT_FILTER is not set +CONFIG_VIDEO_DEV=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_OF=y +CONFIG_VIDEO_APPLE_AVD=y +CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST=y +CONFIG_WERROR=y diff --git a/drivers/media/platform/apple/avd/tests/builder-main.c b/drivers/media/platform/apple/avd/tests/builder-main.c new file mode 100644 index 0000000000000..1456ee52b48ff --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/builder-main.c @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +int main(void) +{ + static const u8 data[] = { 0x65, 0, 0, 3, 0x80, 0x42, 0, 0, 3, 4, 5, 6, 7, 8 }; + for (int quirks = 0; quirks < 2; quirks++) + for (int revision = 3; revision <= 4; revision++) + for (int type = 0; type < 3; type++) + for (int slices = 1; slices <= 4; slices += 3) + for (int custom = 0; custom < 2; custom++) + for (int cabac = 0; cabac < 2; cabac++) { + struct avd_variant variant = { .quirks = quirks, .revision = revision }; + struct avd_dev dev = { .variant = &variant }; + struct avd_h264_ctx *picture = calloc(1, sizeof(*picture)); + struct avd_segment segments[5] = {}; + struct avd_ctx ctx = { .dev = &dev, .priv = picture, .decomp = true, + .comp = { .offsets = { 0, 0x100, 0x200, 0x300 } }, + .job = { .segments = segments }, + }; + struct avd_h264_run r = {}; + struct v4l2_ctrl_h264_sps sps = { .pic_width_in_mbs_minus1 = 3, + .pic_height_in_map_units_minus1 = 3, .chroma_format_idc = 1, + .flags = V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY | V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE }; + struct v4l2_ctrl_h264_pps pps = { .flags = V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT | + (custom ? V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE : 0), + .weighted_bipred_idc = custom ? 1 : 2 }; + struct v4l2_ctrl_h264_slice_params sl = { .slice_type = type, + .header_bit_size = 33, .num_ref_idx_l0_active_minus1 = 1, + .num_ref_idx_l1_active_minus1 = 1 }; + struct v4l2_ctrl_h264_decode_params decode = { .flags = type == 2 ? V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC : 0 }; + struct v4l2_ctrl_h264_scaling_matrix scaling; + struct v4l2_ctrl_h264_pred_weights weights = { .luma_log2_weight_denom = 2, .chroma_log2_weight_denom = 2 }; + struct vb2_v4l2_buffer dst = {}; + u32 output[AVD_MAX_INST * 5]; size_t words = 0; +#ifdef TEST_FRAME + struct avd_buf program = { .cpu = output, .size = sizeof(output) }; + struct avd_frame frame = { .prog = &program }; + struct avd_buf fp = { .cpu = calloc(1, 0x8b4e0), .size = 0x8b4e0 }; + if (slices == 1 && quirks) assert(!avd_h264_fp_begin(&r.fp_cursor, &fp, AVD_H264_FP_V3)); + r.frame = &frame; r.picture = picture; +#endif + if (cabac) pps.flags |= V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE; + if (custom) pps.flags |= V4L2_H264_PPS_FLAG_WEIGHTED_PRED; + memset(&scaling, 16, sizeof(scaling)); + for (int l = 0; l < 2; l++) for (int i = 0; i < 32; i++) { + weights.weight_factors[l].luma_weight[i] = custom ? i+1 : 4; + weights.weight_factors[l].luma_offset[i] = custom ? -i : 0; + for (int c = 0; c < 2; c++) { + weights.weight_factors[l].chroma_weight[i][c] = custom ? i+c+1 : 4; + weights.weight_factors[l].chroma_offset[i][c] = custom ? -i-c : 0; + } + } + for (int i = 0; i < 2; i++) { + decode.dpb[i].flags = V4L2_H264_DPB_ENTRY_FLAG_VALID; + decode.dpb[i].reference_ts = i; + decode.dpb[i].top_field_order_cnt = i*2; + sl.ref_pic_list0[i].index = i; sl.ref_pic_list1[i].index = i; + refs[i].base.vb.vb2_buf.dma = 0x600000 + i*0x10000; + refs[i].base.vb.vb2_buf.planes[0].length = 0x10000; + refs[i].comp.start_offset = 0x1000; + } + ctx.decoded_fmt.fmt.pix_mp.width = ctx.decoded_fmt.fmt.pix_mp.height = 64; + ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline = 64; + picture->slice_num = slices; +#ifdef TEST_FRAME + picture->slice = (struct avd_buf) { .cpu = (void *)data, .addr = 0x123450000ULL, .size = sizeof(data) }; + picture->active_slice = &picture->slice; +#else + picture->slices = calloc(4, sizeof(*picture->slices)); + picture->slices[0] = (struct avd_buf) { .cpu = (void *)data, .addr = 0x123450000ULL, .size = sizeof(data) }; + picture->active_slice = &picture->slices[0]; +#endif + picture->bufs.above_info.addr = 0x100000; + picture->bufs.ip_above.addr = 0x110000; + picture->bufs.lf_above.addr = 0x120000; + picture->bufs.lf_above_info.addr = 0x130000; + picture->bufs.mv_above_info.addr = 0x140000; + picture->bufs.pipe_state.addr = 0x150000; + r.base.bufs.dst = &dst; + r.base.y_out = 0x200000; r.base.uv_out = 0x210000; r.base.comp_out = 0x220000; + r.addresses.mv_color = 0x300000; + r.sps = &sps; r.pps = &pps; r.slice_params = &sl; r.decode_params = &decode; + r.scaling_matrix = &scaling; r.pred_weights = &weights; + r.num_valid = 2; r.cur_poc = 6; + assert(avd_h264_find_slice_data(data, sizeof(data), avd_h264_header_len(&r), &r.slice_data_off)); + stream_hdr(&ctx, &r); + for (int i = 0; i < slices; i++) { + ctx.job.num = i+1; sl.first_mb_in_slice = i*4; + stream_slice(&ctx, &r); + } +#ifdef TEST_FRAME + assert(!r.error); words = frame.prog_words; + if (r.fp_cursor.buf) { + const u8 *f = fp.cpu; + assert(get_unaligned_le32(f+0x3c) == 0x003f003f); + assert(get_unaligned_le32(f+0x210) == (r.base.y_out >> 8)); + assert(get_unaligned_le32(f+0xc0) == (r.base.comp_out >> 7)); + assert(get_unaligned_le32(f+0xa84) == (u32)(picture->slice.addr + r.slice_data_off)); + assert(get_unaligned_le32(f+0xa88) == sizeof(data)-r.slice_data_off); + assert(get_unaligned_le32(f+0x4cc) == (custom ? 0x10101010 : 0x060a0d10)); + if (type != 2) assert(get_unaligned_le32(f+0x110) == ((refs[0].base.vb.vb2_buf.dma + 0x1000) >> 7)); + } + free(fp.cpu); +#else + for (int i = 0; i <= slices; i++) { + memcpy(output + words, segments[i].instructions, segments[i].num*4); + words += segments[i].num; + } +#endif + fwrite(&words, sizeof(words), 1, stdout); fwrite(output, 4, words, stdout); +#ifdef TEST_FRAME + r.fp_cursor.buf = NULL; r.error = 0; frame.prog_words = 0; + program.size = sizeof(u32); output[1] = 0xdeadbeef; + stream_hdr(&ctx, &r); + assert(r.error == -ENOSPC && frame.prog_words == 1); + assert(output[1] == 0xdeadbeef); +#endif +#ifndef TEST_FRAME + free(picture->slices); +#endif + free(picture); + } + return 0; +} diff --git a/drivers/media/platform/apple/avd/tests/fp-main.c b/drivers/media/platform/apple/avd/tests/fp-main.c new file mode 100644 index 0000000000000..4bfe03e02db07 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/fp-main.c @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +int main(void) +{ + struct avd_buf fp = { .cpu = calloc(1, 0x1130d0), .size = 0x1130d0 }; + struct avd_h264_fp_cursor cursor; + struct avd_frame frame = { .idx = 15, .fp = &fp }; + u8 *p = fp.cpu; + assert(avd_h264_fp_begin(&cursor, &fp, AVD_H264_FP_T8112_TODO) == -EOPNOTSUPP); + assert(avd_h264_fp_begin(&cursor, &fp, AVD_H264_FP_T8140_TODO) == -EOPNOTSUPP); + assert(avd_h264_fp_begin(&cursor, &fp, AVD_H264_FP_BORAGE) == -EOPNOTSUPP); + fp.size = 0xa94; + assert(avd_h264_fp_begin(&cursor, &fp, AVD_H264_FP_V3) == -ENOSPC); + fp.size = 0x1130d0; + assert(!avd_h264_fp_begin(&cursor, &fp, AVD_H264_FP_V3)); + assert(get_unaligned_le32(p+0x14) == 0xdeadcafe); + assert(get_unaligned_le32(p+0x1c) == 1); + assert(get_unaligned_le32(p+0x20) == 0x8b4c0); + assert(!avd_h264_fp_emit(&cursor, AVD_H264_FP_HDR_3C_HEIGHT_WIDTH, 0x0437077f)); + assert(get_unaligned_le32(p+0x3c) == 0x0437077f); + assert(avd_h264_fp_emit(&cursor, AVD_H264_FP_HDR_3C_HEIGHT_WIDTH, 0) == -E2BIG); + assert(!avd_h264_fp_address(&cursor, AVD_H264_FP_HDR_210_Y_ADDR_LSB8, 0, 0x123450000ULL)); + assert(get_unaligned_le32(p+0x210) == 0x1234500); + assert(avd_h264_fp_address(&cursor, AVD_H264_FP_HDR_210_Y_ADDR_LSB8, 0, 0x100001) == -ERANGE); + assert(!avd_h264_fp_ref(&cursor, true, 15, 3, 0x400000)); + assert(get_unaligned_le32(p+0x20c) == 0x8000); + assert(avd_h264_fp_ref(&cursor, true, 16, 0, 0) == -ERANGE); + assert(avd_h264_fp_ref(&cursor, false, 0, 4, 0) == -ERANGE); + assert(!avd_h264_fp_emit(&cursor, AVD_H264_FP_SLC_LUMA_WEIGHTS, 0x2de04001)); + assert(!avd_h264_fp_emit(&cursor, AVD_H264_FP_SLC_CHROMA_WEIGHTS_0_, 0x2de08002)); + assert(get_unaligned_le32(p+0x770) == 0x2de04001); + assert(get_unaligned_le32(p+0x774) == 0x2de08002); + assert(fp_word(&fp, fp.size-3, 0) == -ENOSPC); + for (int i = 2; i < 96; i++) assert(!avd_h264_fp_emit(&cursor, AVD_H264_FP_SLC_LUMA_WEIGHTS, i)); + assert(avd_h264_fp_emit(&cursor, AVD_H264_FP_SLC_LUMA_WEIGHTS, 0) == -E2BIG); + assert(!avd_h264_fp_slice(&cursor, 599, 600)); + assert(!avd_h264_fp_emit(&cursor, AVD_H264_FP_SLC_A84_SLICE_ADDR_LOW, 0x12345678)); + assert(get_unaligned_le32(p+0xa84+599*0x3b4) == 0x12345678); + assert(get_unaligned_le32(p+0x1c) == 600); + assert(avd_h264_fp_slice(&cursor, 600, 601) == -EINVAL); + assert(avd_h264_fp_slice(&cursor, 600, 600) == -EINVAL); + assert(avd_h264_fp_slice(&cursor, 0, 0) == -EINVAL); + memset(p, 0, fp.size); + p[0x18] = 1; p[0x19] = 1; + put_unaligned_le32(0x7f, p+0x3d8); + p[0x1a] = 4; p[0x1c] = 2; + frame.cmd[0] = 0xa5000200; + frame.cmd[1] = 3U << 16; + frame.cmd[18] = 0x12345678; + assert(avd_h264_borage_cmd_fields(&frame) == -EOPNOTSUPP); + assert(frame.cmd[0] == 0x000403ff); + assert(frame.cmd[1] == 0x30002); + assert(frame.cmd[3] == 0x01483120 + 3*0x1814); + assert(frame.cmd[4] == 15*0x1130d0 + 0x3dc); + assert(frame.cmd[5] == frame.cmd[3] + 0xc); + assert(frame.cmd[6] == 15*0x1130d0 + 0x5c0); + assert(frame.cmd[7] == 15*0x1130d0 + 0x8a0); + assert(frame.cmd[10] == 0x01483154 + 3*0x1814); + assert(frame.cmd[11] == 0x01483470 + 3*0x1814); + assert(frame.cmd[12] == frame.cmd[3] + 0x3ac); + assert(frame.cmd[18] == 0x12345678); + p[0x1a] = 0x85; p[0x1b] = 3; + assert(avd_h264_borage_cmd_fields(&frame) == -EINVAL); + p[0x1a] = 0x84; + assert(avd_h264_borage_cmd_fields(&frame) == -EOPNOTSUPP); + frame.cmd[1] = 4U << 16; + assert(avd_h264_borage_cmd_fields(&frame) == -EINVAL); + assert(avd_h264_fp_pack_cmd(AVD_H264_FP_V3, &frame) == -EOPNOTSUPP); + free(fp.cpu); + puts("PASS: V3 field/address/array bounds and Borage known-command projection; unknown layouts/commands rejected"); + return 0; +} diff --git a/drivers/media/platform/apple/avd/tests/h264-builder-baseline.c b/drivers/media/platform/apple/avd/tests/h264-builder-baseline.c new file mode 100644 index 0000000000000..21266e766d626 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/h264-builder-baseline.c @@ -0,0 +1,534 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Apple AVD VPU codec driver + * + * Copyright The Asahi Linux Contributors + * Copyright 2023 Eileen Yoon + * + * Copyright (c) 2014 Rockchip Electronics Co., Ltd. + * Hertz Wong + * Herman Chen + * + * Copyright (C) 2014 Google, Inc. + * Tomasz Figa + */ + +/* Frozen Linux builder, copyright Asahi Linux Contributors and Eileen Yoon. + * Baseline aurora-wip fbc4e79, before frame_ops; synthetic regression only. */ +#define H264_SCL_DIMS (0x1000000 | ((64 / 4) << 5) | (((16 / 4) << 5) - 1)) + +#define h264_FLAG_CI_PRED(v) FIELD_PREP(BIT(19), !!(v)) +#define H264_FLAG_ENTROPY_CODING_MODE(v) FIELD_PREP(BIT(20), !!(v)) +#define H264_FLAG_NOT_IDR(v) FIELD_PREP(BIT(21), !!(v)) + +#define H264_TRANSFORM_8X8_MODE(v) FIELD_PREP(BIT(7), !!(v)) + +/* + * Most slices held for one picture. This bounds the slice table, the job + * table and the memory that a client can pin. + */ +#define MAX_SLICES 4096 + +struct avd_h264_run { + struct avd_run base; + + const struct v4l2_ctrl_h264_decode_params *decode_params; + const struct v4l2_ctrl_h264_sps *sps; + const struct v4l2_ctrl_h264_pps *pps; + const struct v4l2_ctrl_h264_slice_params *slice_params; + + const struct v4l2_ctrl_h264_scaling_matrix *scaling_matrix; + const struct v4l2_ctrl_h264_pred_weights *pred_weights; + + struct run_addr { + dma_addr_t mv_color; + } addresses; + + s32 cur_poc; + u8 num_valid; + u32 slice_data_off; +}; + +/* state */ +struct avd_h264_ctx { + bool monochrome; + struct avd_h264_reflists { + struct v4l2_h264_reference p[V4L2_H264_REF_LIST_LEN]; + struct v4l2_h264_reference b0[V4L2_H264_REF_LIST_LEN]; + struct v4l2_h264_reference b1[V4L2_H264_REF_LIST_LEN]; + } reflists; + + struct avd_buf *slices; + struct avd_buf *active_slice; + size_t slice_num; + size_t alloc_slice_num; + /* allocated size of the held slices */ + size_t slice_bytes; + + struct avd_h264_bufs { + struct avd_buf inst; + struct avd_buf pipe_state; + struct avd_buf above_info; + struct avd_buf lf_above_info; + struct avd_buf lf_above; + struct avd_buf ip_above; + struct avd_buf mv_above_info; + } bufs; +}; + +/* ffmpeg submits wrong timestamp, use first_mb_in_slice as a workaround */ +#define is_new_frame(sl) \ + (sl->first_mb_in_slice == 0) /* (ctx->fh.m2m_ctx->new_frame) */ + +/* scaling matrix */ +static const u32 default_8x8_intra[] = { + 0x060a0d10, 0x0a0b1012, 0x0d101217, 0x10121719, 0x1217191b, 0x17191b1d, + 0x191b1d1f, 0x1b1d1f21, 0x1217191b, 0x17191b1d, 0x191b1d1f, 0x1b1d1f21, + 0x1d1f2124, 0x1f212426, 0x21242628, 0x2426282a, +}; +static const u32 default_8x8_inter[] = { + 0x090d0f11, 0x0d0d1113, 0x0f111315, 0x11131516, 0x13151618, 0x15161819, + 0x1618191b, 0x18191b1c, 0x13151618, 0x15161819, 0x1618191b, 0x18191b1c, + 0x191b1c1e, 0x1b1c1e20, 0x1c1e2021, 0x1e202123, +}; + +static inline u32 mv_color_size(u32 w, u32 h) +{ + return (DIV_ROUND_UP(w, 16) + 1) * (DIV_ROUND_UP(h, 16) + 1) * 64; +} + +/* sorry for the formatting */ + +static void stream_refs(struct avd_ctx *ctx, struct avd_h264_run *run) +{ + const struct v4l2_ctrl_h264_decode_params *decode = run->decode_params; + const struct v4l2_h264_dpb_entry *dpb = decode->dpb; + struct avd_h264_ctx *h264_ctx = ctx->priv; + struct avd_decoded_buffer *dst, *ref; + dma_addr_t addr; + + dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); + + push(0, ""); + pusha(h264_ctx->bufs.mv_above_info.addr, "mv_above_info", 7); + pusha(run->addresses.mv_color, "mv_color", 0); + + push(0, ""); + push(0, ""); + push(0, ""); + push(0, ""); + + for (int i = 0; i < ARRAY_SIZE(decode->dpb); i++) { + if (!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_VALID)) + continue; + + ref = avd_get_ref_buf(ctx, &dst->base.vb, dpb[i].reference_ts); + + addr = vb2_dma_contig_plane_dma_addr(&ref->base.vb.vb2_buf, 0) + + ref->comp.start_offset; + + push(AVD_REF_NUM(run->num_valid - 1) | AVD_REF_FLAG_CONST | + AVD_REF_FLAG_LONG( + dpb[i].flags & + V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM) | + AVD_REF_DELTA_POC(run->cur_poc - + dpb[i].top_field_order_cnt), + "hdr_d0_ref_hdr"); + + push_comp(ctx, addr, ctx->comp.offsets); + } +} + +static void stream_scaling(struct avd_ctx *ctx, struct avd_h264_run *run) +{ + const struct v4l2_ctrl_h264_pps *pps = run->pps; + const struct v4l2_ctrl_h264_scaling_matrix *scaling = + run->scaling_matrix; + + push(H264_SCL_DIMS, "hdr_4c_pic_scaling_list_dims"); + + for (int i = 0; i < 6; i++) + for (int j = 0; j < 16; j += 4) + push(AVD_SCALING_I3( + scaling->scaling_list_4x4[i][j + 0]) | + AVD_SCALING_I2( + scaling->scaling_list_4x4[i][j + 1]) | + AVD_SCALING_I1( + scaling->scaling_list_4x4[i][j + 2]) | + AVD_SCALING_I0( + scaling->scaling_list_4x4[i][j + 3]), + "scl_46c_pic_scaling_matrix_4x4"); + + /* Instead of 8x8 raster scan order avd expects 4 4x4 subblocks */ + static const u8 map[16] = { + 0, 8, 16, 24, /* top left */ + 4, 12, 20, 28, /* top right */ + 32, 40, 48, 56, /* bottom left */ + 36, 44, 52, 60, /* bottom right */ + }; + + /* 7.3.2.2, only matrix 0 and 1 are used if chroma_format_idc < 3 */ + if (pps->flags & V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE) { + for (int i = 0; i < 2; i++) + for (int j = 0; j < 16; j++) + push(AVD_SCALING_I3(scaling->scaling_list_8x8 + [i][map[j] + 0]) | + AVD_SCALING_I2( + scaling->scaling_list_8x8 + [i][map[j] + 1]) | + AVD_SCALING_I1( + scaling->scaling_list_8x8 + [i][map[j] + 2]) | + AVD_SCALING_I0( + scaling->scaling_list_8x8 + [i][map[j] + 3]), + "scl_4cc_pic_scaling_matrix_8x8"); + + } else { + for (int i = 0; i < ARRAY_SIZE(default_8x8_intra); i++) + push(default_8x8_intra[i], + "scl_4cc_pic_scaling_matrix_8x8"); + for (int i = 0; i < ARRAY_SIZE(default_8x8_inter); i++) + push(default_8x8_inter[i], + "scl_4cc_pic_scaling_matrix_8x8"); + } +} + +static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) +{ + const struct v4l2_ctrl_h264_decode_params *decode = run->decode_params; + const struct v4l2_ctrl_h264_sps *sps = run->sps; + const struct v4l2_ctrl_h264_pps *pps = run->pps; + struct avd_dev *avd = ctx->dev; + struct avd_h264_ctx *h264_ctx = ctx->priv; + u32 bytesperline; + u32 width = (sps->pic_width_in_mbs_minus1 + 1) * 16; + u32 height = (sps->pic_height_in_map_units_minus1 + 1) * 16; + + push(AVD_OP_HDR | AVD_OP_HDR_FLAG_DECOMP(ctx->decomp) | + AVD_OP_HDR_FLAG_INTRA( + decode->flags & + V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC) | + AVD_OP_HDR_CONST | + AVD_OP_HDR_FLAG_PIPE_STATE( + !(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)), + "hdr_34_start_hdr"); + + push(AVD_HDR_CODEC_MODE(AVD_CODEC_H264), "hdr_38_mode"); + + push(AVD_HDR_HEIGHT(height - 1) | AVD_HDR_WIDTH(width - 1), + "hdr_3c_height_width"); + + push(0, "hdr_40_zero"); + + push(AVD_HDR_HEIGHT((height - 1) >> 3) | + AVD_HDR_WIDTH((width - 1) >> 3), + "hdr_28_height_width_shift3"); + + /* TODO: did i mix up luma/chroma in hevc or here? */ + push(AVD_HDR_COMMON_CHROMA_FORMAT(sps->chroma_format_idc) | + AVD_HDR_COMMON_BIT_DEPTH_L(sps->bit_depth_luma_minus8) | + AVD_HDR_COMMON_BIT_DEPTH_C(sps->bit_depth_chroma_minus8) | + AVD_HDR_COMMON_MIN_LUMA_CBS(1) | + AVD_HDR_COMMON_LUMA_CBS(1) | + H264_TRANSFORM_8X8_MODE( + pps->flags & + V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE) | + AVD_HDR_COMMON_FLAG0( + sps->flags & + V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE), + "hdr_2c_sps_param"); + + push(H264_FLAG_ENTROPY_CODING_MODE( + pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE) | + H264_FLAG_NOT_IDR(!(decode->flags & + V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC)) | + h264_FLAG_CI_PRED( + pps->flags & + V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED), + "hdr_44_flags"); + + push(AVD_HDR_H26X_QP_OFFSET_CB(pps->chroma_qp_index_offset) | + AVD_HDR_H26X_QP_OFFSET_CR( + pps->second_chroma_qp_index_offset), + "hdr_48_chroma_qp_index_offset"); + + push(AVD_HDR_FEAT_H26X | AVD_HDR_FEAT_COMMON | AVD_HDR_FEAT_H264 | + AVD_HDR_FEAT_PIPE_STATE_EN( + !(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)), + "hdr_58_const_3a"); + + push(0, ""); + push(0, ""); + + if (avd->variant->revision == 3) + push(0, "zero"); + + pusha(h264_ctx->bufs.above_info.addr, "hdr_9c_pps_tile_addr_lsb8", 0); + + push(0, ""); + push(0, ""); + + if (avd->variant->revision == 3) + push(0, "zero"); + else if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) + pusha(h264_ctx->bufs.pipe_state.addr, "pipe_state", 0); + + pusha(h264_ctx->bufs.ip_above.addr, "ip_above", 1); + pusha(h264_ctx->bufs.lf_above.addr, "lf_above", 2); + pusha(h264_ctx->bufs.lf_above_info.addr, "lf_above_info", 3); + push(0, ""); + + push_comp(ctx, run->base.comp_out, ctx->comp.offsets); + + bytesperline = ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline; + if (avd->variant->quirks & AVD_QUIRK_LSR) + bytesperline = bytesperline >> 4; + + pusha(run->base.y_out, "hdr_210_y_addr_lsb8", 0); + push(bytesperline, "hdr_218_width_align"); + pusha(run->base.uv_out, "hdr_214_uv_addr_lsb8", 0); + push(bytesperline, "hdr_21c_width_align"); + + push(0, "cm3_mark_end_section"); + push(((height - 1) << 16) | (width - 1), "hdr_54_height_width"); + + if (!(decode->flags & V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC)) + stream_refs(ctx, run); + + if (pps->flags & V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT) + stream_scaling(ctx, run); + else + push(0, "cm3_mark_end_section_scl"); +} + +#define DEFAULT_WEIGHT_DENOM \ + (AVD_OP_WEIGHTS_HDR_LUMA(5) | AVD_OP_WEIGHTS_HDR_CHROMA(5)) + +static void stream_weights(struct avd_ctx *ctx, struct avd_h264_run *run) +{ + int luma_denom, chroma_denom; + struct v4l2_h264_weight_factors factors; + const struct v4l2_ctrl_h264_pred_weights *weights = run->pred_weights; + const struct v4l2_ctrl_h264_pps *pps = run->pps; + const struct v4l2_ctrl_h264_slice_params *sl = run->slice_params; + + bool pred_weight_req = V4L2_H264_CTRL_PRED_WEIGHTS_REQUIRED(pps, sl); + bool default_weights = pps->weighted_bipred_idc == 2 && + !pred_weight_req; + + push(AVD_OP_WEIGHTS_HDR | AVD_OP_WEIGHTS_HDR_FLAG1(default_weights) | + AVD_OP_WEIGHTS_HDR_FLAG0(pred_weight_req) | + AVD_OP_WEIGHTS_HDR_LUMA( + !default_weights ? + weights->luma_log2_weight_denom : + 0) | + AVD_OP_WEIGHTS_HDR_CHROMA( + !default_weights ? + weights->chroma_log2_weight_denom : + 0) | + (default_weights ? DEFAULT_WEIGHT_DENOM : 0), + "slc_76c_cmd_weights_denom"); + + if (!pred_weight_req) + return; + + luma_denom = 1 << weights->luma_log2_weight_denom; + chroma_denom = 1 << weights->chroma_log2_weight_denom; + + for (int y = 0; y < 2; y++) { + if (y == 1 && sl->slice_type != V4L2_H264_SLICE_TYPE_B) + break; + + factors = weights->weight_factors[y]; + int to = y == 0 ? sl->num_ref_idx_l0_active_minus1 : + sl->num_ref_idx_l1_active_minus1; + for (int i = 0; i < to + 1; i++) { + /* Avd only expects offsets/weights if they are not the default + * ones, otherwise we get artifacts */ + if ((factors.luma_weight[i] != luma_denom) || + (factors.luma_offset[i] != 0)) { + push(AVD_OP_WEIGHTS | AVD_OP_WEIGHTS_IDENT(1) | + AVD_OP_WEIGHTS_LIST_IDX(y) | + AVD_OP_WEIGHTS_INDEX(i) | + AVD_OP_WEIGHTS_WEIGHT( + factors.luma_weight[i]), + "slc_luma_weights"); + push(AVD_OP_OFFSETS | + AVD_OP_OFFSETS_OFFSET( + factors.luma_offset[i]), + "slc_luma_offsets"); + } + + if ((factors.chroma_weight[i][0] != chroma_denom) || + (factors.chroma_offset[i][0] != 0) || + (factors.chroma_weight[i][1] != chroma_denom) || + (factors.chroma_offset[i][1] != 0)) { + push(AVD_OP_WEIGHTS | AVD_OP_WEIGHTS_IDENT(2) | + AVD_OP_WEIGHTS_LIST_IDX(y) | + AVD_OP_WEIGHTS_INDEX(i) | + AVD_OP_WEIGHTS_WEIGHT( + factors.chroma_weight[i][0]), + "slc_chroma_weights[0]"); + push(AVD_OP_OFFSETS | + AVD_OP_OFFSETS_OFFSET( + factors.chroma_offset[i][0]), + "slc_chroma_offsets[0]"); + push(AVD_OP_WEIGHTS | AVD_OP_WEIGHTS_IDENT(3) | + AVD_OP_WEIGHTS_LIST_IDX(y) | + AVD_OP_WEIGHTS_INDEX(i) | + AVD_OP_WEIGHTS_WEIGHT( + factors.chroma_weight[i][1]), + "slc_chroma_weights[1]"); + push(AVD_OP_OFFSETS | + AVD_OP_OFFSETS_OFFSET( + factors.chroma_offset[i][1]), + "slc_chroma_offsets[1]"); + } + } + } +} + +/* + * Find the slice data in a slice: skip the two bytes in front of the slice + * header, then the header_len bytes of the header itself, which do not count + * its emulation prevention bytes. At least one byte of slice data must follow. + */ +VISIBLE_IF_KUNIT bool avd_h264_find_slice_data(const u8 *data, u32 len, + u32 header_len, u32 *off) +{ + u32 pos = 2, bytes_read = 2; + + while (bytes_read < header_len && pos < len) { + if (data[pos - 2] != 0x00 || data[pos - 1] != 0x00 || + data[pos] != 0x03) + bytes_read++; + pos++; + } + + if (pos >= len) + return false; + + *off = pos; + return true; +} + +/* Bytes of slice header in front of the slice data. */ +static u32 avd_h264_header_len(const struct avd_h264_run *run) +{ + /* CABAC slice data starts on a byte boundary. */ + bool cabac = run->pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE; + + return (run->slice_params->header_bit_size + (cabac ? 7 : 0)) / 8; +} + +static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) +{ + const struct v4l2_ctrl_h264_decode_params *decode = run->decode_params; + const struct v4l2_ctrl_h264_pps *pps = run->pps; + const struct v4l2_ctrl_h264_sps *sps = run->sps; + const struct v4l2_ctrl_h264_slice_params *sl = run->slice_params; + struct avd_h264_ctx *h264_ctx = ctx->priv; + u32 payload_len = h264_ctx->active_slice->size; + bool en_mode = (pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE) == + 0; + u32 off = run->slice_data_off; + + dma_addr_t slc_a84 = h264_ctx->active_slice->addr + off; + + push(AVD_OP_CODED_DATA | + AVD_OP_CODED_DATA_BIT_OFF( + en_mode ? (sl->header_bit_size % 8) : 0) | + AVD_OP_CODED_DATA_ADDR(slc_a84 >> 32), + "slc_a7c_cmd_set_coded_slice"); + push((u32)(slc_a84 & 0xffffffff), "slc_a84_slice_addr_low"); + + /* should not include trailing 0? */ + push(payload_len - off, "slc_a88_slice_hdr_size"); + + push(AVD_OP_SL_LOC | + AVD_OP_SL_LOC_Y(sl->first_mb_in_slice / + (sps->pic_width_in_mbs_minus1 + 1)) | + AVD_OP_SL_LOC_X(sl->first_mb_in_slice % + (sps->pic_width_in_mbs_minus1 + 1)), + "cm3_cmd_exec_mb_vp"); + + push(AVD_OP_QP | AVD_OP_QP_VAL(26 + pps->pic_init_qp_minus26 + + sl->slice_qp_delta), + "slc_a70_cmd_quant_param"); + + push(AVD_OP_DBLK | + AVD_OP_DBLK_FLAG_FULL_EN( + sl->disable_deblocking_filter_idc == 0) | + AVD_OP_DBLK_FLAG_EN(sl->disable_deblocking_filter_idc != + 1) | + AVD_OP_DBLK_OFF1(sl->slice_beta_offset_div2) | + AVD_OP_DBLK_OFF0(sl->slice_alpha_c0_offset_div2), + "slc_a74_cmd_deblocking_filter"); + + if (sl->slice_type == V4L2_H264_SLICE_TYPE_P || + sl->slice_type == V4L2_H264_SLICE_TYPE_B) { + u32 num_ref_idx_active = sl->num_ref_idx_l0_active_minus1 + 1; + for (u32 i = 0; i < num_ref_idx_active; i++) + push(AVD_OP_REF | AVD_OP_REF_LIST_IDX(0) | + AVD_OP_REF_LOOP_IDX(i) | + AVD_OP_REF_DBP_IDX( + sl->ref_pic_list0[i].index), + "slc_6e8_cmd_ref_list_0"); + + if (sl->slice_type == V4L2_H264_SLICE_TYPE_B) { + u32 num_ref_idx_active = + sl->num_ref_idx_l1_active_minus1 + 1; + for (u32 i = 0; i < num_ref_idx_active; i++) + push(AVD_OP_REF | AVD_OP_REF_LIST_IDX(1) | + AVD_OP_REF_LOOP_IDX(i) | + AVD_OP_REF_DBP_IDX( + sl->ref_pic_list1[i].index), + "slc_6e8_cmd_ref_list_0"); + } + stream_weights(ctx, run); + } + + if (sl->first_mb_in_slice == 0) { + push(AVD_OP_SL_DIM_START, "cm3_cmd_set_mb_dims"); + push(AVD_SL_DIM_END_Y(sps->pic_height_in_map_units_minus1) | + AVD_SL_DIM_END_X(sps->pic_width_in_mbs_minus1), + "cm3_set_mb_dims"); + } + + push(AVD_OP_SL_REF | AVD_OP_SL_REF_FLAG_CABAC(sl->cabac_init_idc == 1) | + AVD_OP_SL_REF_FLAG1(sl->cabac_init_idc == 2) | + AVD_OP_SL_REF_FLAG2( + !(sl->flags & + V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED)) | + AVD_OP_SL_REF_NUM_L0(sl->num_ref_idx_l0_active_minus1) | + AVD_OP_SL_REF_NUM_L1(sl->num_ref_idx_l1_active_minus1) | + AVD_OP_SL_REF_SLICE_P(sl->slice_type == + V4L2_H264_SLICE_TYPE_P) | + AVD_OP_SL_REF_SLICE_I(sl->slice_type == + V4L2_H264_SLICE_TYPE_I) | + AVD_OP_SL_REF_SLICE_B(sl->slice_type == + V4L2_H264_SLICE_TYPE_B) + + , + "slc_6e4_cmd_ref_type"); + + if (sl->slice_type == V4L2_H264_SLICE_TYPE_B) { + /* bidirectional reference of previous mv */ + struct vb2_buffer *vb = vb2_find_buffer( + &ctx->fh.m2m_ctx->cap_q_ctx.q, + decode->dpb[sl->ref_pic_list1[0].index].reference_ts); + + dma_addr_t mv_color_addr = + vb ? vb2_dma_contig_plane_dma_addr(vb, 0) + + (vb->planes[0].length - + mv_color_size(fmt_width(ctx), + fmt_height(ctx))) : + run->addresses.mv_color; + + pusha(mv_color_addr, "slc_a78_sps_tile_addr2_lsb8", 0); + } +} + + +static int avd_h264_alloc_bufs; /* extraction end marker */ diff --git a/drivers/media/platform/apple/avd/tests/host-compat.h b/drivers/media/platform/apple/avd/tests/host-compat.h new file mode 100644 index 0000000000000..18bae54ee9149 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/host-compat.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef int32_t s32; +typedef uint64_t u64; +typedef uint32_t __le32; +typedef uint64_t dma_addr_t; +#define BIT(n) (1U << (n)) +#define GENMASK(h,l) ((UINT32_MAX >> (31-(h))) & (UINT32_MAX << (l))) +#define FIELD_PREP(mask,v) (((u32)(v) << __builtin_ctz(mask)) & (mask)) +#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) +#define DIV_ROUND_UP(n,d) (((n)+(d)-1)/(d)) +#define cpu_to_le32(x) (x) +#define lower_32_bits(x) ((u32)(x)) +#define upper_32_bits(x) ((u32)((x)>>32)) +#define U32_MAX UINT32_MAX +#define AVD_QUIRK_LSR BIT(0) +#define AVD_QUIRK_NO_PIPE_STATE BIT(1) +#define AVD_MAX_INST 512 +#define AVD_MAX_FRAMES 16 +#define AVD_H_ 1 +#define AVD_QUEUE_H_ 1 +#define VISIBLE_IF_KUNIT +#define dev_info(...) ((void)0) +struct avd_buf { void *cpu; dma_addr_t addr; size_t size; }; +struct avd_variant { unsigned int quirks; unsigned char revision; unsigned int h264_fp_layout; }; +struct avd_dev { const struct avd_variant *variant; }; +struct vb2_buffer { struct { size_t length, data_offset; } planes[1]; dma_addr_t dma; void *data; size_t payload; }; +struct vb2_v4l2_buffer { struct vb2_buffer vb2_buf; u32 flags; }; +struct avd_comp { u32 size; size_t start_offset; u32 offsets[4]; }; +struct avd_decoded_buffer { struct { struct vb2_v4l2_buffer vb; } base; struct avd_comp comp; size_t grey_chroma_offset; bool grey_chroma_10bit; u8 bank; }; +struct avd_segment { size_t num; u32 instructions[AVD_MAX_INST]; }; +struct avd_job { size_t num; struct avd_segment *segments; }; +struct v4l2_ctrl { u32 id; struct { void *p; } p_cur; }; +struct v4l2_ctrl_handler { struct v4l2_ctrl ctrls[6]; }; +struct avd_ctx { + struct v4l2_ctrl_handler ctrl_hdl; unsigned int max_frames, image_fmt; struct avd_dev *dev; void *priv; bool decomp; struct avd_comp comp; struct avd_job job; + struct { struct { struct { int q; } cap_q_ctx; } *m2m_ctx; } fh; + struct v4l2_format decoded_fmt; +}; +struct avd_run { + struct { struct vb2_v4l2_buffer *src, *dst; } bufs; + dma_addr_t coded_in, y_out, uv_out, comp_out; +}; +struct avd_frame { + unsigned int idx; struct avd_buf *prog, *fp; size_t prog_words; u32 cmd[24]; + struct vb2_v4l2_buffer *src, *dst; void *codec_priv; u32 frame_num; int dep; dma_addr_t fifo_iova; +}; +enum { AVD_CODEC_H264 = 1 }; +static struct avd_decoded_buffer refs[16]; +#define vb2_to_avd_decoded_buf(vb) ((struct avd_decoded_buffer *)(vb)) +#define vb2_dma_contig_plane_dma_addr(vb, p) ((vb)->dma) +#define avd_get_ref_buf(ctx,dst,ts) ((void)(ctx), (void)(dst), &refs[(ts)]) +#define vb2_find_buffer(q,ts) (&refs[(ts)].base.vb.vb2_buf) +#define fmt_width(ctx) ((ctx)->decoded_fmt.fmt.pix_mp.width) +#define fmt_height(ctx) ((ctx)->decoded_fmt.fmt.pix_mp.height) +static inline void put_unaligned_le32(u32 x, void *p) { memcpy(p, &x, 4); } +static inline u32 get_unaligned_le32(const void *p) { u32 x; memcpy(&x,p,4);return x; } +static inline u16 get_unaligned_le16(const void *p) { u16 x; memcpy(&x,p,2);return x; } + +#define min_t(t,a,b) ((t)(a) < (t)(b) ? (t)(a) : (t)(b)) +#define max_t(t,a,b) ((t)(a) > (t)(b) ? (t)(a) : (t)(b)) +#define ALIGN(x,a) (((x)+(a)-1)&~((a)-1)) +#define PAGE_SIZE 4096 +#define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE) +#define GFP_KERNEL 0 +#define kzalloc(s,g) calloc(1,s) +#define kfree(p) free(p) +#define kvfree(p) free(p) +#define kvmalloc_array(n,s,g) malloc((n)*(s)) +#define dev_err(...) ((void)0) +#define vb2_plane_size(vb,p) ((vb)->planes[p].length) +#define vb2_plane_vaddr(vb,p) ((vb)->data) +#define vb2_get_plane_payload(vb,p) ((vb)->payload) +#define AVD_DEP_NONE 0 +#define AVD_IMG_FMT_420_10BIT 2 +#define AVD_IMG_FMT_422_10BIT 4 +enum vb2_buffer_state { VB2_BUF_STATE_DONE, VB2_BUF_STATE_ERROR }; +struct v4l2_h264_reflist_builder { unsigned int num_valid; s32 cur_pic_order_count; }; +static inline void v4l2_h264_init_reflist_builder(struct v4l2_h264_reflist_builder *b, + const struct v4l2_ctrl_h264_decode_params *d, const struct v4l2_ctrl_h264_sps *s, + const struct v4l2_h264_dpb_entry *dpb) +{ + (void)s; b->num_valid=0; b->cur_pic_order_count=d->top_field_order_cnt; + for (int i=0;i<16;i++) b->num_valid += !!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_VALID); +} +static inline struct v4l2_ctrl *v4l2_ctrl_find(struct v4l2_ctrl_handler *h, u32 id) +{ + for (int i=0;i<6;i++) if (h->ctrls[i].id==id) return &h->ctrls[i]; + return NULL; +} +static bool apple_backend; +static inline bool avd_backend_is_apple(struct avd_dev *d) { (void)d; return apple_backend; } +static inline int avd_init_job(struct avd_ctx *ctx, int codec, size_t segments) +{ + (void)codec; free(ctx->job.segments); ctx->job.num=0; + ctx->job.segments=calloc(segments,sizeof(*ctx->job.segments)); + return ctx->job.segments ? 0 : -ENOMEM; +} diff --git a/drivers/media/platform/apple/avd/tests/include/linux/errno.h b/drivers/media/platform/apple/avd/tests/include/linux/errno.h new file mode 100644 index 0000000000000..32a2df3ba8d20 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/errno.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include_next diff --git a/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h b/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h new file mode 100644 index 0000000000000..ccda48a6de34b --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_INTERRUPT_H_ +#define AVD_HOST_INTERRUPT_H_ +#define irqreturn_t int +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h b/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h new file mode 100644 index 0000000000000..a73bb63db2c8e --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_JIFFIES_H_ +#define AVD_HOST_JIFFIES_H_ +#include +#define time_after_eq(a, b) ((long)((a) - (b)) >= 0) +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/linux/limits.h b/drivers/media/platform/apple/avd/tests/include/linux/limits.h new file mode 100644 index 0000000000000..304aeeeff0fbd --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/limits.h @@ -0,0 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include +#include +#define U32_MAX UINT32_MAX diff --git a/drivers/media/platform/apple/avd/tests/include/linux/types.h b/drivers/media/platform/apple/avd/tests/include/linux/types.h new file mode 100644 index 0000000000000..8b9638abb9a7f --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/types.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_TYPES_H_ +#define AVD_HOST_TYPES_H_ +#include +#include +#include_next +typedef __u8 u8; +typedef __u32 u32; +typedef __u64 u64; +typedef __u64 dma_addr_t; +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h b/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h new file mode 100644 index 0000000000000..6cd054cc198d3 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_VB2_H_ +#define AVD_HOST_VB2_H_ +enum vb2_buffer_state { VB2_BUF_STATE_DONE, VB2_BUF_STATE_ERROR }; +#endif diff --git a/drivers/media/platform/apple/avd/tests/lifecycle-main.c b/drivers/media/platform/apple/avd/tests/lifecycle-main.c new file mode 100644 index 0000000000000..08022bdc43145 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/lifecycle-main.c @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +int main(void) +{ + struct avd_variant variant = { .quirks = AVD_QUIRK_LSR, .revision = 3, + .h264_fp_layout = AVD_H264_FP_V3 }; + struct avd_dev dev = { .variant = &variant }; + struct avd_h264_ctx h264 = {}; + struct avd_ctx ctx = { .dev = &dev, .priv = &h264 }; + struct v4l2_ctrl_h264_sps sps = { .chroma_format_idc = 1, + .pic_width_in_mbs_minus1 = 3, .pic_height_in_map_units_minus1 = 6, + .flags = V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY }; + struct v4l2_ctrl_h264_pps pps = {}; + struct v4l2_ctrl_h264_decode_params decode = { .flags = V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC }; + struct v4l2_ctrl_h264_slice_params sl = { .slice_type = V4L2_H264_SLICE_TYPE_I, .header_bit_size = 33 }; + struct v4l2_ctrl_h264_scaling_matrix scaling; + struct v4l2_ctrl_h264_pred_weights weights = {}; + struct avd_decoded_buffer dst[17] = {}; + struct vb2_v4l2_buffer src[17][4] = {}; + struct avd_buf program[17] = {}, fp = { .cpu = calloc(1, 0x8b4e0), .size = 0x8b4e0 }; + struct avd_frame f[17] = {}; + u8 data[40] = { 0, 0, 0, 0x65, 0, 0, 3, 0x80, 0x42 }; + unsigned int ids[] = { V4L2_CID_STATELESS_H264_SPS, V4L2_CID_STATELESS_H264_PPS, + V4L2_CID_STATELESS_H264_DECODE_PARAMS, V4L2_CID_STATELESS_H264_SLICE_PARAMS, + V4L2_CID_STATELESS_H264_SCALING_MATRIX, V4L2_CID_STATELESS_H264_PRED_WEIGHTS }; + void *controls[] = { &sps, &pps, &decode, &sl, &scaling, &weights }; + memset(&scaling, 16, sizeof(scaling)); + ctx.decoded_fmt.fmt.pix_mp.width = 64; + ctx.decoded_fmt.fmt.pix_mp.height = 112; + ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline = 64; + ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].sizeimage = 0x10000; + for (int i = 0; i < 6; i++) ctx.ctrl_hdl.ctrls[i] = (struct v4l2_ctrl) { .id = ids[i], .p_cur.p = controls[i] }; + for (int n = 0; n < 17; n++) { + program[n] = (struct avd_buf) { .cpu = calloc(1024, 4), .size = 4096 }; + dst[n].base.vb.vb2_buf.dma = 0x200000 + n * 0x10000; + dst[n].base.vb.vb2_buf.planes[0].length = 0x10000; + f[n] = (struct avd_frame) { .idx = n % 16, .dst = &dst[n].base.vb, .prog = &program[n] }; + for (int i = 0; i < 4; i++) { + src[n][i].vb2_buf = (struct vb2_buffer) { .dma = 0x10000000 + n * 0x10000 + i * 0x1000, + .data = data, .payload = sizeof(data), .planes = { { .length = sizeof(data), .data_offset = 3 } } }; + } + } + for (int i=0; i<2; i++) { + decode.dpb[i].flags=V4L2_H264_DPB_ENTRY_FLAG_VALID; + decode.dpb[i].reference_ts=i; + refs[i].base.vb.vb2_buf.dma=0x600000+i*0x10000; + refs[i].base.vb.vb2_buf.planes[0].length=0x10000; + } + sl.ref_pic_list1[0].index=1; + for (int n = 0; n < 16; n++) { + sl.slice_type=n%3; + decode.flags=sl.slice_type==V4L2_H264_SLICE_TYPE_I ? V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC : 0; + int count = n & 1 ? 1 : 4; + decode.frame_num = n; + for (int i = 0; i < count; i++) { + sl.first_mb_in_slice = !n && i == 1 ? 8 : !n && i == 2 ? 4 : i * 4; + f[n].src = &src[n][i]; + src[n][i].flags = i + 1 < count ? V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF : 0; + /* Intermediate slots and programs may be reused by unrelated contexts. */ + f[n].idx = i + 1 < count ? 15 : n; + assert(!avd_h264_prepare(&ctx, &f[n])); + assert(f[n].dep == AVD_DEP_NONE); + if (i + 1 < count) { + assert(!f[n].prog_words && h264.pending); + avd_h264_retire(&ctx, &f[n], VB2_BUF_STATE_DONE); + memset(program[n].cpu, 0xcd, program[n].size); + } + } + assert(!h264.pending && f[n].codec_priv && f[n].prog_words); + u32 *program_words=program[n].cpu; + assert(program_words[0] == (AVD_OP_EXEC | AVD_OP_EXEC_FLAG_START_REV3(1) | AVD_OP_EXEC_FIFO_IDX(n))); + assert(program_words[f[n].prog_words-1] == (AVD_OP_EXEC | AVD_OP_EXEC_FLAG_END(1))); + unsigned int exec_count=0; + for (size_t i=0;ibusy && picture->slice_num == (size_t)count); + for (int i = 0; i < count; i++) { + assert(!picture->slices[i].data.cpu); + assert(picture->slices[i].data.addr == src[n][i].vb2_buf.dma + 3); + bool found = false; + u32 *words = program[n].cpu; + for (size_t j = 0; j + 2 < f[n].prog_words; j++) + if ((words[j] & 0xfff00000) == AVD_OP_CODED_DATA && + words[j+1] == (u32)(picture->slices[i].data.addr + picture->slices[i].data_offset)) found = true; + assert(found); + } + for (int i = 0; i < n; i++) { + assert(f[i].codec_priv != f[n].codec_priv); + assert(f[i].fifo_iova != f[n].fifo_iova); + } + } + sl.first_mb_in_slice = 0; f[16].src = &src[16][0]; + assert(avd_h264_prepare(&ctx, &f[16]) == -EBUSY); + for (int n = 15; n >= 0; n--) avd_h264_retire(&ctx, &f[n], n & 1 ? VB2_BUF_STATE_DONE : VB2_BUF_STATE_ERROR); + for (int n = 0; n < 16; n++) assert(!h264.slots[n]->busy); + /* Corrupt continuation releases the partial picture, including a held request. */ + f[0].src = &src[0][0]; src[0][0].flags = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF; + assert(!avd_h264_prepare(&ctx, &f[0])); + sl.first_mb_in_slice = 0; + assert(avd_h264_prepare(&ctx, &f[0]) == -EINVAL); + avd_h264_retire(&ctx, &f[0], VB2_BUF_STATE_ERROR); + assert(!h264.pending && !h264.slots[0]->busy); + /* Streamoff retires the last held request with ERROR. */ + assert(!avd_h264_prepare(&ctx, &f[0])); + avd_h264_retire(&ctx, &f[0], VB2_BUF_STATE_ERROR); + assert(!h264.pending && !h264.slots[0]->busy); + /* Capacity failure is bounded and fully retireable. */ + src[0][0].flags = 0; program[0].size = 4; + ((u32 *)program[0].cpu)[1] = 0xdeadbeef; + assert(avd_h264_prepare(&ctx, &f[0]) == -ENOSPC); + assert(((u32 *)program[0].cpu)[1] == 0xdeadbeef); + avd_h264_retire(&ctx, &f[0], VB2_BUF_STATE_ERROR); + program[0].size = 4096; + /* Apple V3 uses an independent record for every accumulated slice. */ + apple_backend = true; f[0].fp = &fp; + for (int i = 0; i < 4; i++) { + sl.first_mb_in_slice = i * 4; + f[0].src = &src[0][i]; src[0][i].flags = i < 3 ? V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF : 0; + assert(!avd_h264_prepare(&ctx, &f[0])); + } + assert(get_unaligned_le32((u8 *)fp.cpu + 0x1c) == 4); + for (int i = 0; i < 4; i++) { + struct avd_h264_ctx *picture = f[0].codec_priv; + assert(get_unaligned_le32((u8 *)fp.cpu + 0xa84 + 0x3b4*i) == + (u32)(src[0][i].vb2_buf.dma + 3 + picture->slices[i].data_offset)); + } + avd_h264_retire(&ctx, &f[0], VB2_BUF_STATE_DONE); + assert(!avd_h264_slice_fits(&sps, 0, SIZE_MAX, 1, 0x10000)); + for (int n = 0; n < 17; n++) free(program[n].cpu); + for (int n = 0; n < 16; n++) avd_h264_free_picture(&ctx, h264.slots[n]); + free(fp.cpu); + puts("PASS: actual prepare/retire, held 1/4-slice pictures, 16 independent scratch owners, OUTPUT DMA offsets, V3 records, abort and capacity errors"); + return 0; +} diff --git a/drivers/media/platform/apple/avd/tests/queue-host.c b/drivers/media/platform/apple/avd/tests/queue-host.c new file mode 100644 index 0000000000000..5983d7ad5d6a3 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/queue-host.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Exercise the production helpers with deterministic shuffled completions. */ +#include +#include +#include + +#include "../avd-apple-protocol.h" +#include "../avd-queue-internal.h" + +int main(void) +{ + struct avd_frame pool[AVD_MAX_FRAMES] = {}; + int clients[3]; + u64 last[3] = {}, seq = 0; + u32 rng = 0x3851, old_tag[AVD_MAX_FRAMES] = {}; + unsigned int i, step, completed = 0; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + pool[i].idx = i; + for (step = 0; step < 100000; step++) { + struct avd_frame *f; + + rng = rng * 1664525 + 1013904223; + f = &pool[(rng >> 16) & 15]; + if (f->state == AVD_FRAME_FREE) { + struct avd_ctx *ctx = (void *)&clients[(rng >> 8) % 3]; + + assert(!avd_slot_reserve(f, ctx, ++seq)); + assert(!avd_slot_complete(f, f->tag, 0)); + f->deadline = step + 100; + f->state = AVD_FRAME_SUBMITTED; + assert(!avd_slot_complete(f, old_tag[f->idx], 0)); + old_tag[f->idx] = f->tag; + } else if (f->state == AVD_FRAME_SUBMITTED) { + assert(avd_slot_complete(f, f->tag, (rng & 1) ? -EIO : 0)); + assert(!avd_slot_complete(f, f->tag, 0)); + } + for (i = 0; i < AVD_MAX_FRAMES; i++) { + unsigned int c; + + f = &pool[i]; + if (f->state != AVD_FRAME_DONE || !avd_slot_first(pool, f)) + continue; + for (c = 0; c < 3; c++) + if (f->ctx == (void *)&clients[c]) + break; + assert(c < 3 && f->seq > last[c]); + last[c] = f->seq; + f->state = AVD_FRAME_FREE; + completed++; + } + } + for (i = 0; i < 8; i++) { + assert(avd_inbox_full(i, (i + 1) & 7)); + assert(!avd_inbox_full(i, (i + 2) & 7)); + assert(avd_apple_notification(i) == 0x01482cf8U + 0x60 * i); + } + { + struct avd_apple_reply_layout layout = { + .type_word = 0, .type_mask = 31, .client_word = 1, + .client_mask = 1023, .frame_word = 3, .status_word = 4, + .done_type = 8, .verified = true, + }; + struct avd_apple_reply reply; + u32 record[20] = { 7, 9, 0, 1234 }; + + assert(avd_apple_parse_reply(record, 20, &layout, &reply) == -EINPROGRESS); + record[0] = 8; + assert(!avd_apple_parse_reply(record, 20, &layout, &reply)); + assert(reply.frame == 1234 && reply.client == 9 && !reply.status); + record[4] = 1; + assert(!avd_apple_parse_reply(record, 20, &layout, &reply)); + assert(reply.status == -EIO); + assert(avd_apple_parse_reply(record, 19, &layout, &reply) == -EMSGSIZE); + } + assert(completed > 1000); + printf("PASS: 100000 shuffled operations, %u ordered retirements\n", completed); + return 0; +} diff --git a/drivers/media/platform/apple/avd/tests/run-host-tests.py b/drivers/media/platform/apple/avd/tests/run-host-tests.py new file mode 100644 index 0000000000000..fffb1b8d20f71 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/run-host-tests.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile actual instruction builders against synthetic V4L2 controls. + +The frozen baseline is Linux's licensed pre-frame_ops H.264 builder; no Apple +images or captured parameters are used. The shim replaces only kernel plumbing. +Run on a Linux host: python3 tests/run-host-tests.py --baseline-file FILE +""" +from pathlib import Path +import argparse, os, subprocess, tempfile +ap=argparse.ArgumentParser(); ap.add_argument('--baseline-file', type=Path, default=Path(__file__).parent/"h264-builder-baseline.c") +args=ap.parse_args() +root=Path(__file__).resolve().parent.parent +tests=root/'tests' +def strip_includes(s): + return '\n'.join(l for l in s.splitlines() if not l.startswith('#include'))+'\n' +def builder(s): + return s[s.index('#define H264_SCL_DIMS'):s.index('static int avd_h264_alloc_bufs')] +with tempfile.TemporaryDirectory(prefix='avd-h264-host-') as td: + td=Path(td) + # Keep the inline instruction encoding functions used by the original builder. + inst=strip_includes((root/'avd-inst.h').read_text()) + fp_h=strip_includes((root/'avd-h264-fp.h').read_text()) + fp_c=strip_includes((root/'avd-h264-fp.c').read_text()) + common=(tests/'host-compat.h').read_text()+'\n'+inst+'\n'+fp_h+'\n'+fp_c + outputs=[] + for name, src in [('baseline',args.baseline_file.read_text()),('frame',(root/'avd-h264.c').read_text())]: + path=td/(name+'.c'); exe=td/name + path.write_text(common+'\n'+builder(src)+'\n'+(tests/'builder-main.c').read_text()) + subprocess.run([os.environ.get('HOSTCC','cc'),'-std=gnu11','-Wall','-Wextra','-Werror','-Wno-unused-function','-Wno-unused-variable','-Wno-unused-parameter','-Wno-sign-compare',*(['-DTEST_FRAME'] if name=='frame' else []),str(path),'-o',str(exe)],check=True) + outputs.append(subprocess.check_output([str(exe)])) + assert outputs[0]==outputs[1], 'frame program differs from baseline' + print('PASS: 96 synthetic I/P/B, 1/4-slice, scaling/weights, address-format cases match baseline byte for byte') + path=td/'fp.c'; exe=td/'fp' + path.write_text(common+'\n'+(tests/'fp-main.c').read_text()) + subprocess.run([os.environ.get('HOSTCC','cc'),'-std=gnu11','-Wall','-Wextra','-Werror','-Wno-unused-function','-Wno-unused-variable','-Wno-unused-parameter',str(path),'-o',str(exe)],check=True) + subprocess.run([str(exe)],check=True) + +# Exercise actual prepare/retire and picture collection across separate requests. +src=(root/'avd-h264.c').read_text() +def section(first, last): + return src[src.index(first):src.index(last, src.index(first))] +lifecycle=(section('static int avd_h264_validate_pps(', 'static void avd_h264_free_picture(')+ + section('static bool avd_h264_refs_valid(', 'static int avd_h264_run(struct avd_ctx *ctx)')+ + section('static void avd_h264_done(', 'static enum avd_image_fmt avd_h264_get_image_fmt')+ + section('static void avd_h264_frame_controls(', 'static int avd_h264_pack_cmd(')+ + section('static void avd_h264_retire(', 'const struct avd_codec_frame_ops avd_h264_frame_ops')) +plumbing = r""" +static unsigned int scratch_serial; +static int avd_h264_alloc_bufs(struct avd_ctx *ctx, struct avd_h264_ctx *picture) +{ + (void)ctx; picture->bufs.inst.addr=0x40000000ULL + ++scratch_serial * 0x100000; + return 0; +} +static void avd_h264_free_picture(struct avd_ctx *ctx, struct avd_h264_ctx *picture) +{ (void)ctx; free(picture->slices); free(picture); } +""" +with tempfile.TemporaryDirectory(prefix='avd-h264-lifecycle-') as td: + td=Path(td) + path=td/'lifecycle.c'; exe=td/'lifecycle' + path.write_text(common+'\n'+builder(src)+'\n'+plumbing+'\n'+lifecycle+'\n'+(tests/'lifecycle-main.c').read_text()) + flags=[os.environ.get('HOSTCC','cc'),'-std=gnu11','-Wall','-Wextra','-Werror','-Wno-unused-function','-Wno-unused-variable','-Wno-unused-parameter','-Wno-sign-compare'] + subprocess.run(flags+[str(path),'-o',str(exe)],check=True) + subprocess.run([str(exe)],check=True) + queue=(root/'avd-queue.c').read_text() + helpers=queue[queue.index('static void avd_sources_done('):queue.index('/* queue_mutex excludes')] + path=td/'sources.c'; exe=td/'sources' + path.write_text((tests/'sources-main.c').read_text().replace('/* PRODUCTION_HELPERS */',helpers)) + subprocess.run(flags+[str(path),'-o',str(exe)],check=True) + subprocess.run([str(exe)],check=True) diff --git a/drivers/media/platform/apple/avd/tests/run-host.sh b/drivers/media/platform/apple/avd/tests/run-host.sh new file mode 100755 index 0000000000000..70e287f14732d --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/run-host.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only OR MIT +set -eu +avd_test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +avd_test_out=${1:?usage: run-host.sh OUTPUT_BINARY} +${CC:-cc} -std=gnu11 -Wall -Wextra -Werror -O2 \ + -I"$avd_test_dir/include" "$avd_test_dir/queue-host.c" -o "$avd_test_out" +"$avd_test_out" diff --git a/drivers/media/platform/apple/avd/tests/sources-main.c b/drivers/media/platform/apple/avd/tests/sources-main.c new file mode 100644 index 0000000000000..bd31c0a8e873f --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/sources-main.c @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include +#include +#include +#include +#include +struct list_head { struct list_head *next, *prev; }; +#define container_of(p,t,m) ((t *)((char *)(p) - offsetof(t,m))) +#define INIT_LIST_HEAD(h) ((h)->next = (h)->prev = (h)) +static bool list_empty(struct list_head *h) { return h->next == h; } +static void list_add_tail(struct list_head *n, struct list_head *h) +{ n->prev=h->prev; n->next=h; h->prev->next=n; h->prev=n; } +static void list_del_init(struct list_head *n) +{ n->prev->next=n->next; n->next->prev=n->prev; INIT_LIST_HEAD(n); } +static void list_splice_tail_init(struct list_head *s, struct list_head *d) +{ + if (list_empty(s)) return; + s->next->prev=d->prev; d->prev->next=s->next; s->prev->next=d; d->prev=s->prev; + INIT_LIST_HEAD(s); +} +#define list_for_each_entry_safe(p,n,h,m) \ + for (p=container_of((h)->next,__typeof__(*p),m), n=container_of(p->m.next,__typeof__(*p),m); \ + &p->m != (h); p=n,n=container_of(n->m.next,__typeof__(*n),m)) +enum vb2_buffer_state { VB2_BUF_STATE_DONE, VB2_BUF_STATE_ERROR }; +struct request { unsigned int complete; }; +struct vb2_v4l2_buffer { + struct { struct { struct request *req; } req_obj; } vb2_buf; + bool is_held, done; enum vb2_buffer_state state; +}; +struct v4l2_m2m_buffer { struct vb2_v4l2_buffer vb; struct list_head list; }; +struct avd_frame { unsigned int idx; struct vb2_v4l2_buffer *src, *dst; void *codec_priv; }; +struct avd_ctx; +struct avd_codec_frame_ops { void (*retire)(struct avd_ctx *,struct avd_frame *,enum vb2_buffer_state); }; +struct legacy_ops { void (*done)(struct avd_ctx *,struct vb2_v4l2_buffer *,struct vb2_v4l2_buffer *,enum vb2_buffer_state); }; +struct desc { const struct legacy_ops *ops; }; +struct avd_dev { struct list_head frame_sources[16]; }; +struct avd_ctx { + struct avd_dev *dev; struct list_head slice_sources; struct avd_frame partial; + struct desc *coded_fmt_desc; int ctrl_hdl; + struct { void *m2m_ctx; } fh; const struct avd_codec_frame_ops *ops; +}; +static unsigned int removed, retired, legacy_retired; +static const struct avd_codec_frame_ops *avd_codec_frame_ops(struct avd_ctx *ctx) { return ctx->ops; } +static void v4l2_ctrl_request_complete(struct request *req, int *hdl) { (void)hdl; req->complete++; } +static void v4l2_m2m_buf_done(struct vb2_v4l2_buffer *buf, enum vb2_buffer_state state) +{ assert(!buf->done); buf->done=true; buf->state=state; } +static void v4l2_m2m_buf_copy_metadata(struct vb2_v4l2_buffer *src, struct vb2_v4l2_buffer *dst) +{ (void)src; (void)dst; } +static void v4l2_m2m_src_buf_remove(void *m2m) { (void)m2m; removed++; } +/* PRODUCTION_HELPERS */ +static void retire(struct avd_ctx *ctx, struct avd_frame *f, enum vb2_buffer_state state) +{ (void)ctx; assert(f->codec_priv && state==VB2_BUF_STATE_ERROR); retired++; } +static void legacy_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src, + struct vb2_v4l2_buffer *dst, enum vb2_buffer_state state) +{ (void)ctx; assert(src && dst && state==VB2_BUF_STATE_ERROR); legacy_retired++; } +int main(void) +{ + struct avd_dev dev; struct avd_codec_frame_ops ops={ .retire=retire }; + struct legacy_ops old={ .done=legacy_done }; struct desc desc={ .ops=&old }; + struct avd_ctx ctx={ .dev=&dev, .ops=&ops, .coded_fmt_desc=&desc }; + struct v4l2_m2m_buffer src[16][3]={}; struct request req[16][3]={}; + struct vb2_v4l2_buffer dst[16]={}; + INIT_LIST_HEAD(&ctx.slice_sources); + for (int n=0;n<16;n++) { + INIT_LIST_HEAD(&dev.frame_sources[n]); + for (int i=0;i<3;i++) { + struct avd_frame f={ .idx=n, .src=&src[n][i].vb, .dst=&dst[n], .codec_priv=&dst[n] }; + INIT_LIST_HEAD(&src[n][i].list); src[n][i].vb.vb2_buf.req_obj.req=&req[n][i]; + avd_source_hold(&ctx,&f); + assert(!src[n][i].vb.done && !req[n][i].complete && !dst[n].is_held); + assert(!f.codec_priv && ctx.partial.codec_priv==&dst[n]); + } + struct avd_frame f={ .idx=n }; + avd_sources_attach(&ctx,&f); + assert(list_empty(&ctx.slice_sources) && !ctx.partial.codec_priv); + } + assert(removed==48); + for (int n=15;n>=0;n--) { + avd_sources_done(&ctx,&dev.frame_sources[n],n&1 ? VB2_BUF_STATE_DONE : VB2_BUF_STATE_ERROR); + for (int i=0;i<3;i++) assert(src[n][i].vb.done && req[n][i].complete==1); + assert(list_empty(&dev.frame_sources[n])); + } + for (int legacy=0;legacy<2;legacy++) { + ctx.ops=legacy ? NULL : &ops; + for (int i=0;i<3;i++) { + struct avd_frame f={ .src=&src[0][i].vb, .dst=&dst[0], .codec_priv=&dst[0] }; + src[0][i].vb.done=false; req[0][i].complete=0; + avd_source_hold(&ctx,&f); + } + avd_partial_abort(&ctx); + assert(list_empty(&ctx.slice_sources) && !ctx.partial.codec_priv); + for (int i=0;i<3;i++) assert(src[0][i].vb.done && src[0][i].vb.state==VB2_BUF_STATE_ERROR); + } + assert(retired==1 && legacy_retired==1); + puts("PASS: production source ownership helpers retain 48 OUTPUTs across 16 pictures; final completion and native/legacy partial abort release once"); +}