From a78baf2f16f9081f138a054c0c971e4cbe0096f8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 22:37:37 +1100 Subject: [PATCH 1/5] media: apple: stage AV1 frame operations with isolated state Move AV1 instruction and scratch ownership into frames and retain inherited CDF and segmentation state through reference-counted DMA allocations. Wire the frame operations and preserve the proved Borage offsets while refusing incomplete Apple payloads. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/Makefile | 2 +- drivers/media/platform/apple/avd/avd-av1-fp.c | 110 ++ drivers/media/platform/apple/avd/avd-av1-fp.h | 39 + .../platform/apple/avd/avd-av1-frame-ops.rst | 71 ++ drivers/media/platform/apple/avd/avd-av1.c | 688 ++++++----- drivers/media/platform/apple/avd/avd-drv.c | 4 +- drivers/media/platform/apple/avd/avd-queue.h | 87 ++ drivers/media/platform/apple/avd/avd-v4l2.c | 1 + drivers/media/platform/apple/avd/avd.h | 6 + .../platform/apple/avd/tests/avd-av1-legacy.c | 1060 +++++++++++++++++ .../platform/apple/avd/tests/host-driver.h | 34 + .../platform/apple/avd/tests/host-shim.h | 58 + .../platform/apple/avd/tests/host-test.c | 243 ++++ .../platform/apple/avd/tests/legacy-emitter.h | 23 + .../apple/avd/tests/run-host-tests.py | 53 + 15 files changed, 2210 insertions(+), 269 deletions(-) create mode 100644 drivers/media/platform/apple/avd/avd-av1-fp.c create mode 100644 drivers/media/platform/apple/avd/avd-av1-fp.h create mode 100644 drivers/media/platform/apple/avd/avd-av1-frame-ops.rst create mode 100644 drivers/media/platform/apple/avd/avd-queue.h create mode 100644 drivers/media/platform/apple/avd/tests/avd-av1-legacy.c create mode 100644 drivers/media/platform/apple/avd/tests/host-driver.h create mode 100644 drivers/media/platform/apple/avd/tests/host-shim.h create mode 100644 drivers/media/platform/apple/avd/tests/host-test.c create mode 100644 drivers/media/platform/apple/avd/tests/legacy-emitter.h create mode 100755 drivers/media/platform/apple/avd/tests/run-host-tests.py diff --git a/drivers/media/platform/apple/avd/Makefile b/drivers/media/platform/apple/avd/Makefile index 4d1877c414d41..2c8bc9ca5534a 100644 --- a/drivers/media/platform/apple/avd/Makefile +++ b/drivers/media/platform/apple/avd/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-h264.o avd-vp9.o avd-hevc.o \ - avd-av1.o avd-av1-entropymode.o + avd-av1.o avd-av1-fp.o avd-av1-entropymode.o apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o obj-$(CONFIG_VIDEO_APPLE_AVD) += apple-avd.o diff --git a/drivers/media/platform/apple/avd/avd-av1-fp.c b/drivers/media/platform/apple/avd/avd-av1-fp.c new file mode 100644 index 0000000000000..85dca37478ac9 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-av1-fp.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * AV1 Apple frame parameters. Source: Apple AVD protocol SPEC (2026-10-31), + * sections 5, 6.1, 6.5, 6.6, 10. No AV1 layout exists in the public V3 code. + * Keep unproved fields unnamed and refuse submission until they are settled. + */ +#include +#include "avd.h" +#include "avd-av1-fp.h" + +#define BORAGE_FP_SIZE 0x1130d0 +#define BORAGE_WORK_CAP 0x1002 + +/* + * [C] table: source destination / unresolved rule + * SPEC 6.1 fp+0x18,+0x19 cmd+0x00 bits 0,1; meaning + * SPEC 6.5 fp+0x4f4 cmd+0x00 bits 2:6,18:19; + * selected source bits unspecified + * SPEC 6.5 fp+0x4f4 cmd+0x00 bits 7:11,13:17; + * source shifts unspecified + * SPEC 6.5 work array at fp+0x1c cmd+0x1c; array encoding + * SPEC 6.5 fp+0x4fc,+0x5e0,0x01481780 cmd+0x3c,+0x44; packed addresses + * SPEC 6.5 fp+0x4f8 cmd+0x4c; source meaning + * SPEC 6.6 sequence/frame/tile/grain syntax/DVA offsets still open + * SPEC 10 T8140 TODO: identify selected packer; + * Daybreak per-SoC ledger task + * t-3862-daybreak-avd-per-soc-ledger- + * t8140-t6000-t8112-pe + */ + +int avd_av1_validate_tiles(const struct v4l2_ctrl_av1_frame *frame, + const struct v4l2_ctrl_av1_tile_group_entry *tiles, + unsigned int count, size_t payload) +{ + const struct v4l2_av1_tile_info *info = &frame->tile_info; + unsigned int i, n; + + if (!info->tile_cols || !info->tile_rows || + info->tile_cols > V4L2_AV1_MAX_TILE_COLS || + info->tile_rows > V4L2_AV1_MAX_TILE_ROWS) + return -EINVAL; + n = info->tile_cols * info->tile_rows; + if (n > V4L2_AV1_MAX_TILE_COUNT || n > BORAGE_WORK_CAP || + count != n || info->context_update_tile_id >= n) + return -EINVAL; + for (i = 0; i < n; i++) { + if (tiles[i].tile_row != i / info->tile_cols || + tiles[i].tile_col != i % info->tile_cols || + !tiles[i].tile_size || tiles[i].tile_offset > payload || + tiles[i].tile_size > payload - tiles[i].tile_offset) + return -EINVAL; + if (i && tiles[i].tile_offset < + (u64)tiles[i - 1].tile_offset + tiles[i - 1].tile_size) + return -EINVAL; + } + return 0; +} + +int avd_av1_pack_fp(struct avd_ctx *ctx, struct avd_frame *f, + const struct v4l2_ctrl_av1_sequence *seq, + const struct v4l2_ctrl_av1_frame *frame, + const struct v4l2_ctrl_av1_tile_group_entry *tiles, + const struct v4l2_ctrl_av1_film_grain *grain) +{ + u8 *fp; + + if (ctx->dev->variant->av1_fp_layout != AVD_AV1_FP_BORAGE) + return -EOPNOTSUPP; + if (!f->fp || !f->fp->cpu || f->fp->size < BORAGE_FP_SIZE) + return -ENOSPC; + fp = f->fp->cpu; + memset(fp, 0, f->fp->size); + put_unaligned_le32(f->frame_num, fp + 0x10); /* SPEC 5.2 */ + put_unaligned_le16(frame->tile_info.tile_cols, fp + 0x3e4); + put_unaligned_le16(frame->tile_info.tile_rows, fp + 0x3e6); + /* SPEC 6.6 does not locate seq/frame/tiles/grain syntax or surface DVAs. + * A native V4L2 struct or Linux instruction stream is not an Apple fp. */ + return -EOPNOTSUPP; +} + +int avd_av1_pack_cmd(struct avd_ctx *ctx, struct avd_frame *f) +{ + u8 *cmd = (u8 *)f->cmd; + const u8 *fp; + u32 work, base; + + if (ctx->dev->variant->av1_fp_layout != AVD_AV1_FP_BORAGE) + return -EOPNOTSUPP; + if (!f->fp || !f->fp->cpu || f->fp->size < BORAGE_FP_SIZE) + return -ENOSPC; + fp = f->fp->cpu; + /* Work index is the modulo-four sequence from the core prefix. */ + work = get_unaligned_le16(cmd + 6); + if (work >= 4 || !get_unaligned_le16(fp + 0x1a) || + get_unaligned_le16(fp + 0x1a) > BORAGE_WORK_CAP) + return -EINVAL; + base = 0x01483118 + 0x1814 * work; + put_unaligned_le16(get_unaligned_le16(fp + 0x1a), cmd + 0x04); + put_unaligned_le32(base, cmd + 0x0c); + put_unaligned_le32(base + 8, cmd + 0x14); + put_unaligned_le32(0x014832c0 + 0x1814 * work, cmd + 0x24); + put_unaligned_le32(0x014834dc + 0x1814 * work, cmd + 0x28); + put_unaligned_le32(base + 0x4d0, cmd + 0x2c); + /* Slot-relative offsets; their packed DVA encoding is still [C]. */ + put_unaligned_le32(0x5f8, cmd + 0x30); + put_unaligned_le32(0x4fc, cmd + 0x34); + put_unaligned_le32(0x5e0, cmd + 0x38); + put_unaligned_le32(get_unaligned_le32(fp + 0x4f8), cmd + 0x4c); + return -EOPNOTSUPP; +} diff --git a/drivers/media/platform/apple/avd/avd-av1-fp.h b/drivers/media/platform/apple/avd/avd-av1-fp.h new file mode 100644 index 0000000000000..8e292f6402dac --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-av1-fp.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_AV1_FP_H_ +#define AVD_AV1_FP_H_ + +#include + +struct avd_ctx; +struct avd_frame; +struct v4l2_ctrl_av1_sequence; +struct v4l2_ctrl_av1_frame; +struct v4l2_ctrl_av1_tile_group_entry; +struct v4l2_ctrl_av1_film_grain; + +enum avd_av1_fp_layout { + AVD_AV1_FP_UNKNOWN, + AVD_AV1_FP_BORAGE, + AVD_AV1_FP_T8140_TODO, +}; + +/* Incoming edge: the primary reference selects the previous HW CDF output. */ +static inline bool avd_av1_cdf_dep(bool uses_primary, u64 primary_timestamp, + bool have_prev, u64 prev_timestamp, + bool prev_updates_cdf) +{ + return uses_primary && have_prev && prev_updates_cdf && + primary_timestamp == prev_timestamp; +} + +int avd_av1_validate_tiles(const struct v4l2_ctrl_av1_frame *frame, + const struct v4l2_ctrl_av1_tile_group_entry *tiles, + unsigned int count, size_t payload); +int avd_av1_pack_fp(struct avd_ctx *ctx, struct avd_frame *f, + const struct v4l2_ctrl_av1_sequence *seq, + const struct v4l2_ctrl_av1_frame *frame, + const struct v4l2_ctrl_av1_tile_group_entry *tiles, + const struct v4l2_ctrl_av1_film_grain *grain); +int avd_av1_pack_cmd(struct avd_ctx *ctx, struct avd_frame *f); + +#endif diff --git a/drivers/media/platform/apple/avd/avd-av1-frame-ops.rst b/drivers/media/platform/apple/avd/avd-av1-frame-ops.rst new file mode 100644 index 0000000000000..727331197bde9 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-av1-frame-ops.rst @@ -0,0 +1,71 @@ +AV1 frame operations +==================== + +This codec series targets avd_codec_frame_ops. It requires the accompanying +16-slot core and queued open-firmware backend before decode is usable. The +legacy core rejects a codec without a run callback instead of dereferencing it. +The queue header is the shared contract, unchanged. + +prepare consumes controls already applied by the core and the slot's src/dst. +It never accesses an m2m queue, completes a request, or touches MMIO. The open +program is a little-endian stream: header EXEC, the existing header words, then +one tile payload and its EXEC per tile (END on the last tile). prog_words counts +all words, including EXEC records. The backend owns VP selection, FIFO assignment, +submission and VP/PP chaining. The core supplies fp only for an Apple submission; +NULL fp selects preparation of the open program. + +Each live frame owns its pipe state and every above/left/MV scratch allocation. +CDFs and segmentation maps are separate coherent DMA allocations. Capture-index +records retain their output state; in-flight frames retain input and output +state independently. A disabled CDF/map update inherits an immutable allocation, +including its ownership reference, rather than copying a potentially unfinished +capture tail. Reusing the original capture index therefore cannot free inherited +state. Color/MV reference data already stored in capture buffers remains subject +to the stateless reference-buffer lifetime contract. + +The incoming dependency is PREV_DONE exactly when the current frame uses the +previous frame as primary reference and that previous frame enables frame-end +CDF updates. Defaults (intra, error resilient or PRIMARY_REF_NONE), a different +primary reference, and disabled frame-end updates produce NONE. prepare does +not read hardware CDF/map output on the CPU. It refuses an unfinished older CDF +or segmentation producer with EAGAIN when the specified dependency rule cannot +express the dependency. The integrating core must retry preparation after an +earlier completion; the present contract does not define that retry behavior. +The core must also propagate failed producers to already prepared dependants, +serialize prepare calls per context, and drain/retire all slots before stop. + +Apple layout evidence and remaining work +---------------------------------------- + +The source is the clean-room Apple AVD protocol SPEC dated 2026-10-31, sections +5, 6.1, 6.5, 6.6 and 10. Public V3 layouts in eiln/avd cannot supply AV1 fields: +those generations have no AV1 hardware. avd-av1-fp.c retains an unnamed [C] +table with the missing fields and their sources. It writes the proved Borage +tile columns/rows at 0x3e4/0x3e6 and stages the proved command destinations, +work-count bound 0x1002 and fp+0x4f8 -> cmd+0x4c copy. Both Apple pack operations +return EOPNOTSUPP: the SPEC omits the syntax/DVA layout, source flag shifts and +packed-address encoding needed for a valid command. No guessed payload is sent. +T6040 selects Borage; T8140 explicitly selects a TODO row pending the Daybreak +per-SoC ledger task t-3862. Other Apple layouts stay unsupported. Existing AV1 +hardware capability bits are retained; T6000/T8112 have neither AV1 capability +nor an AV1 VP. Film grain synthesis is unsupported by the existing open builder +and an APPLY_GRAIN request is rejected explicitly. + +Verification +------------ + +Run ``python3 drivers/media/platform/apple/avd/tests/run-host-tests.py`` from a +Git checkout, including a sparse checkout. A host C compiler with ASan and UBSan +is required. Tests compile the production CPU code with small DMA/V4L2 doubles +and the checkout's real AV1 UAPI definitions. They cover live slot isolation, +CDF inheritance, dependency cases, allocation failure cleanup, payload bounds, +program capacity and Apple-layout refusal. The frozen pre-conversion packing +oracle comes from aurora-wip fbc4e7934ca7325f3483b72a51d24c74aa8536a4, with +its original license notice. Program comparison uses the same resolved DMA +addresses in both builders; allocation ownership intentionally changes addresses. + +These are synthetic control vectors, not SVT-AV1 bitstreams or hardware output +validation. Film grain is a rejection test. AV1 show-existing has no decode-frame +control in the stateless UAPI: userspace redisplays a reference without submitting +a new decode. Queued firmware execution, Apple payload completion, retry semantics +and end-to-end SVT-AV1 decode remain integration/qualification work. diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index 60d69bab8a342..c988c19597bd3 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -16,11 +16,20 @@ * Tomasz Figa */ +#include +#include #include #include "avd.h" #include "avd-av1-entropymode.h" #include "avd-inst.h" +#include "avd-av1-fp.h" + +/* The AV1 builder writes only the current frame's DMA program. */ +#undef push +#undef pusha +#define push(inst, name) avd_av1_push(run, (inst)) +#define pusha(addr, name, i) avd_av1_push_address(ctx, run, (addr)) /* * This comes from @@ -208,44 +217,154 @@ #define AV1_PRIMARY_REF_NONE 7 +/* A capture reference and every in-flight reader hold their own references. */ +struct avd_av1_aux { + refcount_t refs; + struct avd_buf buf; + u64 producer; + bool done; + int status; +}; + +struct avd_av1_ref { + struct list_head node; + unsigned int index; + u64 timestamp; + u16 width, height; + struct avd_av1_aux *cdf; + struct avd_av1_aux *seg; +}; + +struct avd_av1_scratch { + struct avd_buf pipe_state; + struct avd_buf above_info; + struct avd_buf rf_above_info; + struct avd_buf ip_above; + struct avd_buf lf_above; + struct avd_buf lf_above_info; + struct avd_buf lf_left; + struct avd_buf lf_left_info; + struct avd_buf sr_left; + struct avd_buf rf_left; + struct avd_buf rf_left_info; + struct avd_buf mv_above_info; +}; + struct avd_av1_run { struct avd_run base; + struct avd_frame *slot; + struct avd_av1_scratch bufs; + struct avd_av1_aux *cdf_in, *cdf_out; + struct avd_av1_aux *seg_in, *seg_out; + bool cdf_written, seg_written; + int error; struct { dma_addr_t probs_out; + dma_addr_t probs_in; dma_addr_t color; dma_addr_t seg; dma_addr_t prev_seg; } addresses; + /* Used only while prepare() owns the request controls. */ const struct v4l2_ctrl_av1_sequence *seq; const struct v4l2_ctrl_av1_frame *frame; const struct v4l2_ctrl_av1_tile_group_entry *tile_group; const struct v4l2_ctrl_av1_film_grain *grain; + unsigned int num_tiles; }; struct avd_av1_ctx { - u8 submit_num; - struct { - struct avd_buf inst; - struct avd_buf pipe_state; - struct avd_buf probs; - struct avd_buf no_prev_seg; - struct avd_buf above_info; - struct avd_buf rf_above_info; - struct avd_buf az_above; - struct avd_buf ip_above; - struct avd_buf lf_above; - struct avd_buf lf_above_info; - struct avd_buf lf_left; - struct avd_buf lf_left_info; - struct avd_buf sr_left; - struct avd_buf rf_left; - struct avd_buf rf_left_info; - struct avd_buf mv_above_info; - } bufs; + struct list_head refs; + u64 prev_seq; + u64 prev_timestamp; + bool prev_updates_cdf; + bool have_prev; + unsigned long active_slots; }; +static void avd_av1_push(struct avd_av1_run *run, u32 inst) +{ + struct avd_frame *f = run->slot; + + if (f->prog_words >= f->prog->size / sizeof(__le32)) { + run->error = -ENOSPC; + return; + } + ((__le32 *)f->prog->cpu)[f->prog_words++] = cpu_to_le32(inst); +} + +static void avd_av1_push_address(struct avd_ctx *ctx, struct avd_av1_run *run, + dma_addr_t addr) +{ + if (ctx->dev->variant->quirks & AVD_QUIRK_LSR) { + avd_av1_push(run, addr >> 8); + } else { + avd_av1_push(run, lower_32_bits(addr)); + avd_av1_push(run, upper_32_bits(addr)); + } +} + +static void avd_av1_push_comp(struct avd_ctx *ctx, struct avd_av1_run *run, + dma_addr_t addr, const u32 offsets[4]) +{ + unsigned int i; + + for (i = 0; i < 4; i++) { + if (ctx->dev->variant->quirks & AVD_QUIRK_LSR) + avd_av1_push(run, (addr + offsets[i]) >> 7); + else + avd_av1_push_address(ctx, run, addr + offsets[i]); + } +} + +static struct avd_av1_aux *avd_av1_aux_get(struct avd_av1_aux *aux) +{ + if (aux) + refcount_inc(&aux->refs); + return aux; +} + +static void avd_av1_aux_put(struct avd_ctx *ctx, struct avd_av1_aux *aux) +{ + if (aux && refcount_dec_and_test(&aux->refs)) { + avd_buf_free(ctx->dev, &aux->buf); + kfree(aux); + } +} + +static struct avd_av1_aux *avd_av1_aux_alloc(struct avd_ctx *ctx, size_t size, + u64 producer, bool done) +{ + struct avd_av1_aux *aux; + + aux = kzalloc(sizeof(*aux), GFP_KERNEL); + if (!aux) + return NULL; + if (avd_buf_alloc(ctx->dev, &aux->buf, size)) { + kfree(aux); + return NULL; + } + refcount_set(&aux->refs, 1); + aux->producer = producer; + aux->done = done; + memset(aux->buf.cpu, 0, size); + return aux; +} + +static struct avd_av1_ref *avd_av1_find_ref(struct avd_ctx *ctx, + unsigned int index, u64 timestamp) +{ + struct avd_av1_ctx *av1 = ctx->priv; + struct avd_av1_ref *ref; + + list_for_each_entry(ref, &av1->refs, node) + if (ref->index == index && ref->timestamp == timestamp) + return ref; + return NULL; +} + /* * from verisilicon/rockchip_vpu981_hw_av1_dec.c */ @@ -304,7 +423,7 @@ static short avd_av1_dec_resolve_divisor_32(u32 d, short *shift) } static void -avd_av1_dec_get_shear_params(const u32 *params, s64 *alpha, +avd_av1_dec_get_shear_params(const s32 *params, s64 *alpha, s64 *beta, s64 *gamma, s64 *delta) { const int *mat = params; @@ -341,7 +460,6 @@ avd_av1_dec_get_shear_params(const u32 *params, s64 *alpha, static void set_refs(struct avd_ctx *ctx, struct avd_av1_run *run) { - struct avd_av1_ctx *av1_ctx = ctx->priv; const struct v4l2_ctrl_av1_frame *frame = run->frame; const struct v4l2_av1_global_motion *gm = &frame->global_motion; int i, ref_idx; @@ -355,7 +473,7 @@ static void set_refs(struct avd_ctx *ctx, struct avd_av1_run *run) dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); push(0, "ref_cnst0"); - pusha(av1_ctx->bufs.mv_above_info.addr, "mv_above_info", 0); + pusha(run->bufs.mv_above_info.addr, "mv_above_info", 0); for (i = 0; i < 4; i++) push(0, "ref_cnst1"); @@ -427,7 +545,7 @@ static void set_refs(struct avd_ctx *ctx, struct avd_av1_run *run) push(AV1_REF_SCALE_X(x_scale) | AV1_REF_SCALE_Y(y_scale), "ref_scale"); - push_comp(ctx, addr, ref->comp.offsets); + avd_av1_push_comp(ctx, run, addr, ref->comp.offsets); } } @@ -634,7 +752,6 @@ static void calc_upscale(int frame_width, int upscaled_width, int sub_x, static void set_header(struct avd_ctx *ctx, struct avd_av1_run *run) { - struct avd_av1_ctx *av1_ctx = ctx->priv; const struct v4l2_ctrl_av1_frame *frame = run->frame; const struct v4l2_ctrl_av1_sequence *seq = run->seq; const struct v4l2_av1_loop_restoration *lr = &frame->loop_restoration; @@ -657,7 +774,7 @@ static void set_header(struct avd_ctx *ctx, struct avd_av1_run *run) dma_addr_t ref_addr; dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); - push(AVD_OP_HDR | AVD_OP_HDR_FLAG_DECOMP(ctx->decomp) | + push(AVD_OP_HDR | AVD_OP_HDR_FLAG_DECOMP(1) | AVD_OP_HDR_FLAG_INTRA(intra_only && !intrabc) | AVD_OP_HDR_CONST | AVD_OP_HDR_FLAG_PIPE_STATE(1), "op_hdr"); @@ -817,8 +934,8 @@ static void set_header(struct avd_ctx *ctx, struct avd_av1_run *run) push(0, ""); pusha(run->addresses.probs_out, "probs_out", 0); - pusha(av1_ctx->bufs.probs.addr, "probs", 1); - pusha(av1_ctx->bufs.above_info.addr, "col", 0); + pusha(run->addresses.probs_in, "probs", 1); + pusha(run->bufs.above_info.addr, "col", 0); pusha(run->addresses.seg, "seg", 0); pusha(run->addresses.prev_seg, "prev_seg", 1); pusha(run->addresses.color, "w_color", 0); @@ -954,22 +1071,22 @@ static void set_header(struct avd_ctx *ctx, struct avd_av1_run *run) push(0, ""); push(0, ""); - pusha(av1_ctx->bufs.pipe_state.addr, "pipe_state", 0); - pusha(av1_ctx->bufs.ip_above.addr, "ip_above", 0); - pusha(av1_ctx->bufs.lf_above.addr, "lf_above", 2); - pusha(av1_ctx->bufs.lf_above_info.addr, "lf_above_info", 3); - pusha(av1_ctx->bufs.lf_left.addr, "lf_left", 0); - pusha(av1_ctx->bufs.lf_left_info.addr, "lf_left_info", 1); + pusha(run->bufs.pipe_state.addr, "pipe_state", 0); + pusha(run->bufs.ip_above.addr, "ip_above", 0); + pusha(run->bufs.lf_above.addr, "lf_above", 2); + pusha(run->bufs.lf_above_info.addr, "lf_above_info", 3); + pusha(run->bufs.lf_left.addr, "lf_left", 0); + pusha(run->bufs.lf_left_info.addr, "lf_left_info", 1); /* make it obvious if its used */ pusha((dma_addr_t)0xa0a0a0a0a0a0, "", 0); - pusha(av1_ctx->bufs.rf_above_info.addr, "rf_above_info", 0); - pusha(av1_ctx->bufs.sr_left.addr, "sr_left", 2); - pusha(av1_ctx->bufs.rf_left_info.addr, "rf_left_info", 1); - pusha(av1_ctx->bufs.rf_left.addr, "rf_left", 3); + pusha(run->bufs.rf_above_info.addr, "rf_above_info", 0); + pusha(run->bufs.sr_left.addr, "sr_left", 2); + pusha(run->bufs.rf_left_info.addr, "rf_left_info", 1); + pusha(run->bufs.rf_left.addr, "rf_left", 3); push(0, "mark_section"); - push_comp(ctx, run->base.comp_out, ctx->comp.offsets); + avd_av1_push_comp(ctx, run, run->base.comp_out, ctx->comp.offsets); pusha((u64)0, "packed_fmt_scratch", 0); @@ -991,7 +1108,6 @@ static void set_header(struct avd_ctx *ctx, struct avd_av1_run *run) static void set_tiles(struct avd_ctx *ctx, struct avd_av1_run *run) { - struct avd_av1_ctx *av1_ctx = ctx->priv; const struct v4l2_ctrl_av1_frame *frame = run->frame; const struct v4l2_ctrl_av1_sequence *seq = run->seq; const struct v4l2_ctrl_av1_tile_group_entry *tile_group; @@ -1001,11 +1117,9 @@ static void set_tiles(struct avd_ctx *ctx, struct avd_av1_run *run) seq->flags & V4L2_AV1_SEQUENCE_FLAG_USE_128X128_SUPERBLOCK ? 5 : 4; - av1_ctx->submit_num = tile_info->tile_cols * tile_info->tile_rows; for (row = 0; row < tile_info->tile_rows; row++) { for (col = 0; col < tile_info->tile_cols; col++) { - ctx->job.num++; tile_id = row * tile_info->tile_cols + col; tile_group = &run->tile_group[tile_id]; @@ -1049,105 +1163,95 @@ static void set_tiles(struct avd_ctx *ctx, struct avd_av1_run *run) tile_info->width_in_sbs_minus_1 [tile_group->tile_col]), "tile_op_end"); -#ifdef DEBUG_INST - pr_info("\n"); -#endif + push(AVD_OP_EXEC | AVD_OP_EXEC_FLAG_END( + tile_id + 1 == run->num_tiles), "tile_exec"); } } } -static void avd_av1_set_prob(struct avd_ctx *ctx, struct avd_av1_run *run) +/* No CPU reads from hardware-owned capture tails during prepare(). */ +static int avd_av1_set_aux(struct avd_ctx *ctx, struct avd_av1_run *run) { - struct avd_av1_ctx *av1_ctx = ctx->priv; - bool error_resilient_mode = !!( - run->frame->flags & V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE); - bool frame_is_intra = - ((run->frame->frame_type == V4L2_AV1_KEY_FRAME) || - (run->frame->frame_type == V4L2_AV1_INTRA_ONLY_FRAME)); - struct avd_decoded_buffer *dst, *ref; - dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); - - if (error_resilient_mode || frame_is_intra || - run->frame->primary_ref_frame == AV1_PRIMARY_REF_NONE) { - avd_av1_default_coeff_probs(run->frame->quantization.base_q_idx, - av1_ctx->bufs.probs.cpu); - avd_av1_set_default_cdfs(av1_ctx->bufs.probs.cpu); - } else { - ref = avd_get_ref_buf( - ctx, &dst->base.vb, - run->frame->reference_frame_ts - [run->frame->ref_frame_idx - [run->frame->primary_ref_frame]]); - memcpy(av1_ctx->bufs.probs.cpu, - vb2_plane_vaddr(&ref->base.vb.vb2_buf, 0) + - AVD_AV1_CDFS_OFFSET( - ref->base.vb.vb2_buf.planes[0].length, - ref->av1.color_size), - sizeof(struct avd_av1_cdfs)); - } - - /* - * AVD writes nothing when DISABLE_FRAME_END_UPDATE_CDF is - * enabled - */ - memcpy(vb2_plane_vaddr(&dst->base.vb.vb2_buf, 0) + - AVD_AV1_CDFS_OFFSET(dst->base.vb.vb2_buf.planes[0].length, - dst->av1.color_size), - av1_ctx->bufs.probs.cpu, sizeof(struct avd_av1_cdfs)); -} - -/* - * The segment ids the frame reads as the previous ones (load_previous_segment_ids()): - * the primary reference frame's map if that frame has the same size, else zeroes. - * The AVD reads them whenever segmentation is on, and writes the frame's map only - * when update_map is set; otherwise fill in the map the spec saves with the frame. - */ -static void avd_av1_set_seg(struct avd_ctx *ctx, struct avd_av1_run *run) -{ - struct avd_av1_ctx *av1_ctx = ctx->priv; + struct avd_av1_ctx *av1 = ctx->priv; const struct v4l2_ctrl_av1_frame *frame = run->frame; - u8 flags = frame->segmentation.flags; - struct avd_decoded_buffer *dst, *ref = NULL; - void *seg, *prev_seg = NULL; + struct avd_av1_ref *ref = NULL; + struct avd_decoded_buffer *dst, *primary; + bool defaults = frame->primary_ref_frame == AV1_PRIMARY_REF_NONE || + frame->frame_type == V4L2_AV1_KEY_FRAME || + frame->frame_type == V4L2_AV1_INTRA_ONLY_FRAME || + (frame->flags & V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE); + bool seg_enabled = frame->segmentation.flags & + V4L2_AV1_SEGMENTATION_FLAG_ENABLED; + u64 timestamp = 0; dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); - seg = vb2_plane_vaddr(&dst->base.vb.vb2_buf, 0) + - AVD_AV1_SEG_OFFSET(dst->base.vb.vb2_buf.planes[0].length, - dst->av1.color_size, dst->av1.seg_size); - - if (frame->primary_ref_frame != AV1_PRIMARY_REF_NONE) { - ref = avd_get_ref_buf( - ctx, &dst->base.vb, - frame->reference_frame_ts - [frame->ref_frame_idx[frame->primary_ref_frame]]); - /* avd_get_ref_buf() returns dst for a missing reference */ - /* same MiCols and MiRows, as libdav1d compares them */ - if (ref == dst || - DIV_ROUND_UP(ref->av1.width + 1, 8) != - DIV_ROUND_UP(frame->frame_width_minus_1 + 1, 8) || - DIV_ROUND_UP(ref->av1.height + 1, 8) != - DIV_ROUND_UP(frame->frame_height_minus_1 + 1, 8)) - ref = NULL; + if (!defaults) { + timestamp = frame->reference_frame_ts[ + frame->ref_frame_idx[frame->primary_ref_frame]]; + primary = avd_get_ref_buf(ctx, &dst->base.vb, timestamp); + if (primary == dst) + return -EINVAL; + ref = avd_av1_find_ref(ctx, primary->base.vb.vb2_buf.index, + timestamp); + if (!ref || READ_ONCE(ref->cdf->status)) + return -EINVAL; + run->cdf_in = avd_av1_aux_get(ref->cdf); + } else { + run->cdf_in = avd_av1_aux_alloc(ctx, AVD_CDFS_SIZE, 0, true); + if (!run->cdf_in) + return -ENOMEM; + avd_av1_default_coeff_probs(frame->quantization.base_q_idx, + run->cdf_in->buf.cpu); + avd_av1_set_default_cdfs(run->cdf_in->buf.cpu); } - if (ref) { - size_t off = AVD_AV1_SEG_OFFSET( - ref->base.vb.vb2_buf.planes[0].length, - ref->av1.color_size, ref->av1.seg_size); + run->slot->dep = avd_av1_cdf_dep(!defaults, timestamp, + av1->have_prev, av1->prev_timestamp, av1->prev_updates_cdf) ? + AVD_DEP_PREV_DONE : AVD_DEP_NONE; + /* PREV_DONE drains earlier frames too, but do not invent a dependency + * for a different, unfinished CDF producer. The queue must retry it. */ + if (!READ_ONCE(run->cdf_in->done) && run->slot->dep == AVD_DEP_NONE) + return -EAGAIN; + + run->cdf_written = !(frame->flags & + V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF); + if (run->cdf_written) + run->cdf_out = avd_av1_aux_alloc(ctx, AVD_CDFS_SIZE, + run->slot->seq, false); + else + run->cdf_out = avd_av1_aux_get(run->cdf_in); + if (!run->cdf_out) + return -ENOMEM; - run->addresses.prev_seg = - vb2_dma_contig_plane_dma_addr(&ref->base.vb.vb2_buf, 0) + - off; - prev_seg = vb2_plane_vaddr(&ref->base.vb.vb2_buf, 0) + off; + if (ref && seg_enabled && ref->width == frame->frame_width_minus_1 && + ref->height == frame->frame_height_minus_1) { + if (READ_ONCE(ref->seg->status)) + return -EINVAL; + run->seg_in = avd_av1_aux_get(ref->seg); + /* The two segmentation addresses let hardware read the previous + * map directly. No speculative CPU copy of an unfinished map. */ + if (!READ_ONCE(run->seg_in->done) && run->slot->dep == AVD_DEP_NONE) + return -EAGAIN; } else { - run->addresses.prev_seg = av1_ctx->bufs.no_prev_seg.addr; + run->seg_in = avd_av1_aux_alloc(ctx, dst->av1.seg_size, 0, true); + if (!run->seg_in) + return -ENOMEM; } + run->seg_written = seg_enabled && (frame->segmentation.flags & + V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP); + if (run->seg_written) + run->seg_out = avd_av1_aux_alloc(ctx, dst->av1.seg_size, + run->slot->seq, false); + else + run->seg_out = avd_av1_aux_get(run->seg_in); + if (!run->seg_out) + return -ENOMEM; - if (!(flags & V4L2_AV1_SEGMENTATION_FLAG_ENABLED) || - (!(flags & V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP) && !prev_seg)) - memset(seg, 0, dst->av1.seg_size); - else if (!(flags & V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP)) - memcpy(seg, prev_seg, dst->av1.seg_size); + run->addresses.probs_in = run->cdf_in->buf.addr; + run->addresses.probs_out = run->cdf_out->buf.addr; + run->addresses.prev_seg = run->seg_in->buf.addr; + run->addresses.seg = run->seg_out->buf.addr; + return 0; } static int avd_av1_seg_size(int w, int h) @@ -1189,13 +1293,11 @@ static void update_dec_buf_info(struct avd_decoded_buffer *buf, buf->av1.ref_frame_idx[i] = frame->ref_frame_idx[i]; } -static int avd_av1_run_preamble(struct avd_ctx *ctx, struct avd_av1_run *run) +static int avd_av1_controls(struct avd_ctx *ctx, struct avd_av1_run *run) { struct v4l2_ctrl *ctrl; int dst_len, tlb_len; - avd_run_preamble(ctx, &run->base); - ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_AV1_SEQUENCE); if (WARN_ON(!ctrl)) return -EINVAL; @@ -1229,6 +1331,7 @@ static int avd_av1_run_preamble(struct avd_ctx *ctx, struct avd_av1_run *run) if (WARN_ON(!ctrl)) return -EINVAL; run->tile_group = ctrl->p_cur.p; + run->num_tiles = ctrl->elems; ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_AV1_FILM_GRAIN); @@ -1257,7 +1360,6 @@ static int avd_av1_run_preamble(struct avd_ctx *ctx, struct avd_av1_run *run) static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) { struct avd_dev *avd = ctx->dev; - struct avd_av1_ctx *av1_ctx = ctx->priv; const struct v4l2_ctrl_av1_frame *frame = run->frame; const struct v4l2_ctrl_av1_sequence *seq = run->seq; const struct v4l2_av1_tile_info *tile_info = &frame->tile_info; @@ -1267,7 +1369,6 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) seq->flags & V4L2_AV1_SEQUENCE_FLAG_USE_128X128_SUPERBLOCK ? 1 : 0; int w = frame->frame_width_minus_1 + 1; - int h = frame->frame_height_minus_1 + 1; int bit_depth = seq->bit_depth; for (i = 0; i < tile_info->tile_cols; i++) { @@ -1280,43 +1381,35 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) max_sb_col <<= sb_shift; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.mv_above_info, max_sb_col * 80); + ret = avd_buf_alloc(avd, &run->bufs.mv_above_info, max_sb_col * 80); if (ret) return ret; /* first frame this is always much smaller */ - ret = avd_buf_alloc(avd, &av1_ctx->bufs.above_info, max_sb_col * 400); + ret = avd_buf_alloc(avd, &run->bufs.above_info, max_sb_col * 400); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.lf_above_info, sb_cols1); + ret = avd_buf_alloc(avd, &run->bufs.lf_above_info, sb_cols1); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.ip_above, + ret = avd_buf_alloc(avd, &run->bufs.ip_above, max_sb_col * bit_depth * 22); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.lf_above, + ret = avd_buf_alloc(avd, &run->bufs.lf_above, ALIGN(ALIGN(w, 8) * bit_depth * 2 + tile_info->tile_cols * bit_depth * 16, 128)); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.rf_above_info, sb_cols2); + ret = avd_buf_alloc(avd, &run->bufs.rf_above_info, sb_cols2); if (ret) return ret; - /* zeroes, read as the previous segment ids when no reference has them */ - if (av1_ctx->bufs.no_prev_seg.size != avd_av1_seg_size(w, h)) { - ret = avd_buf_alloc(avd, &av1_ctx->bufs.no_prev_seg, - avd_av1_seg_size(w, h)); - if (ret) - return ret; - } - /* this is not a mistake, but im not sure why */ if (tile_info->tile_cols > 1) { for (i = 0; i < tile_info->tile_rows; i++) { @@ -1326,7 +1419,7 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) max_sb_row = max_sb_row << sb_shift; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.lf_left_info, + ret = avd_buf_alloc(avd, &run->bufs.lf_left_info, max_sb_row * 72); if (ret) return ret; @@ -1341,24 +1434,24 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) * I counted blocks as luma or chroma with padding until it * changes to the other */ - ret = avd_buf_alloc(avd, &av1_ctx->bufs.lf_left, + ret = avd_buf_alloc(avd, &run->bufs.lf_left, (max_sb_row * 72 + 1) * 2 * bit_depth); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.sr_left, + ret = avd_buf_alloc(avd, &run->bufs.sr_left, max_sb_row * bit_depth * (53 + 52)); if (ret) return ret; ret = avd_buf_alloc( - avd, &av1_ctx->bufs.rf_left, + avd, &run->bufs.rf_left, max_sb_row * bit_depth * (192 + 96 + (bit_depth > 8 ? 12 : 0))); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.rf_left_info, + ret = avd_buf_alloc(avd, &run->bufs.rf_left_info, max_sb_row * 24); if (ret) return ret; @@ -1367,129 +1460,204 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) return 0; } -static int avd_av1_run(struct avd_ctx *ctx) +static void avd_av1_release_run(struct avd_ctx *ctx, struct avd_av1_run *run) { - struct avd_av1_run run; - struct avd_decoded_buffer *dst, *ref; - int ret; - - ret = avd_av1_run_preamble(ctx, &run); - if (ret) { - avd_run_postamble(ctx, &run.base); - return ret; - } - - /* The CDF and segmentation map copies need the CPU mappings. */ - dst = vb2_to_avd_decoded_buf(&run.base.bufs.dst->vb2_buf); - ref = dst; - if (run.frame->primary_ref_frame != AV1_PRIMARY_REF_NONE) - ref = avd_get_ref_buf(ctx, &dst->base.vb, - run.frame->reference_frame_ts - [run.frame->ref_frame_idx - [run.frame->primary_ref_frame]]); - if (!vb2_plane_vaddr(&dst->base.vb.vb2_buf, 0) || - !vb2_plane_vaddr(&ref->base.vb.vb2_buf, 0)) { - avd_run_postamble(ctx, &run.base); - return -ENOMEM; - } + avd_buf_free(ctx->dev, &run->bufs.pipe_state); + avd_buf_free(ctx->dev, &run->bufs.above_info); + avd_buf_free(ctx->dev, &run->bufs.rf_above_info); + avd_buf_free(ctx->dev, &run->bufs.ip_above); + avd_buf_free(ctx->dev, &run->bufs.lf_above); + avd_buf_free(ctx->dev, &run->bufs.lf_above_info); + avd_buf_free(ctx->dev, &run->bufs.lf_left); + avd_buf_free(ctx->dev, &run->bufs.lf_left_info); + avd_buf_free(ctx->dev, &run->bufs.sr_left); + avd_buf_free(ctx->dev, &run->bufs.rf_left); + avd_buf_free(ctx->dev, &run->bufs.rf_left_info); + avd_buf_free(ctx->dev, &run->bufs.mv_above_info); + avd_av1_aux_put(ctx, run->cdf_in); + avd_av1_aux_put(ctx, run->cdf_out); + avd_av1_aux_put(ctx, run->seg_in); + avd_av1_aux_put(ctx, run->seg_out); + kfree(run); +} - ret = avd_init_job(ctx, AVD_CODEC_AV1, - run.frame->tile_info.tile_cols * - run.frame->tile_info.tile_rows + - 1); - if (ret) { - avd_run_postamble(ctx, &run.base); - return ret; +static int avd_av1_publish_ref(struct avd_ctx *ctx, struct avd_av1_run *run) +{ + struct avd_av1_ctx *av1 = ctx->priv; + struct avd_av1_ref *ref, *old = NULL; + unsigned int index = run->base.bufs.dst->vb2_buf.index; + + list_for_each_entry(ref, &av1->refs, node) { + if (ref->index == index) { + old = ref; + break; + } } - - ret = avd_av1_alloc_work_bufs(ctx, &run); - if (ret) { - avd_run_postamble(ctx, &run.base); - return ret; + ref = old; + if (!ref) { + ref = kzalloc(sizeof(*ref), GFP_KERNEL); + if (!ref) + return -ENOMEM; + ref->index = index; + list_add_tail(&ref->node, &av1->refs); } - - update_dec_buf_info(dst, run.seq, run.frame); - - avd_av1_set_prob(ctx, &run); - avd_av1_set_seg(ctx, &run); - - set_header(ctx, &run); - set_tiles(ctx, &run); - - avd_run_postamble(ctx, &run.base); - - return avd_submit_job(ctx); + avd_av1_aux_put(ctx, ref->cdf); + avd_av1_aux_put(ctx, ref->seg); + ref->cdf = avd_av1_aux_get(run->cdf_out); + ref->seg = avd_av1_aux_get(run->seg_out); + ref->timestamp = run->base.bufs.dst->vb2_buf.timestamp; + ref->width = run->frame->frame_width_minus_1; + ref->height = run->frame->frame_height_minus_1; + av1->prev_timestamp = ref->timestamp; + av1->prev_seq = run->slot->seq; + av1->prev_updates_cdf = run->cdf_written; + av1->have_prev = true; + return 0; } -static int avd_av1_alloc_bufs(struct avd_ctx *ctx) +static int avd_av1_prepare(struct avd_ctx *ctx, struct avd_frame *f) { - struct avd_dev *avd = ctx->dev; - struct avd_av1_ctx *av1_ctx = ctx->priv; + struct avd_av1_ctx *av1 = ctx->priv; + struct avd_av1_run *run; + struct avd_decoded_buffer *dst; + unsigned int i; int ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.inst, fifo_size()); + if (!f->src || !f->dst || !f->prog || !f->prog->cpu || + f->idx >= AVD_MAX_FRAMES || f->codec_priv) + return -EINVAL; + if (test_and_set_bit(f->idx, &av1->active_slots)) + return -EBUSY; + run = kzalloc(sizeof(*run), GFP_KERNEL); + if (!run) { + clear_bit(f->idx, &av1->active_slots); + return -ENOMEM; + } + run->slot = f; + run->base.bufs.src = f->src; + run->base.bufs.dst = f->dst; + run->base.coded_in = vb2_dma_contig_plane_dma_addr(&f->src->vb2_buf, 0); + run->base.y_out = vb2_dma_contig_plane_dma_addr(&f->dst->vb2_buf, 0); + run->base.uv_out = run->base.y_out + + ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline * + ctx->decoded_fmt.fmt.pix_mp.height; + run->base.comp_out = run->base.y_out + ctx->comp.start_offset; + f->prog_words = 0; + f->dep = AVD_DEP_NONE; + ret = avd_av1_controls(ctx, run); if (ret) - return ret; - - /* no m1 to worry about :) */ - ret = avd_buf_alloc(avd, &av1_ctx->bufs.pipe_state, 0x200); + goto fail; + ret = avd_av1_validate_tiles(run->frame, run->tile_group, + run->num_tiles, + vb2_get_plane_payload(&f->src->vb2_buf, 0)); if (ret) - return ret; - - ret = avd_buf_alloc(avd, &av1_ctx->bufs.probs, - sizeof(struct avd_av1_cdfs)); + goto fail; + /* The old builder has no film grain synthesis program. */ + if (run->grain->flags & V4L2_AV1_FILM_GRAIN_FLAG_APPLY_GRAIN) { + ret = -EOPNOTSUPP; + goto fail; + } + for (i = 0; i < V4L2_AV1_REFS_PER_FRAME; i++) { + if (run->frame->ref_frame_idx[i] < 0 || + run->frame->ref_frame_idx[i] >= V4L2_AV1_TOTAL_REFS_PER_FRAME) { + ret = -EINVAL; + goto fail; + } + } + /* Borage syntax/DVA mappings remain [C]; never submit invented bytes. */ + if (f->fp) { + ret = avd_av1_pack_fp(ctx, f, run->seq, run->frame, + run->tile_group, run->grain); + if (ret) + goto fail; + } + ret = avd_av1_alloc_work_bufs(ctx, run); if (ret) - return ret; - + goto fail; + ret = avd_buf_alloc(ctx->dev, &run->bufs.pipe_state, 0x200); + if (ret) + goto fail; + dst = vb2_to_avd_decoded_buf(&f->dst->vb2_buf); + update_dec_buf_info(dst, run->seq, run->frame); + dst->comp = ctx->comp; + ret = avd_av1_set_aux(ctx, run); + if (ret) + goto fail; + avd_av1_push(run, AVD_OP_EXEC | + AVD_OP_EXEC_FLAG_START_REV4(ctx->dev->variant->revision == 4)); + set_header(ctx, run); + set_tiles(ctx, run); + if (run->error) { + ret = run->error; + goto fail; + } + ret = avd_av1_publish_ref(ctx, run); + if (ret) + goto fail; + /* The request controls can change as soon as prepare returns. */ + run->seq = NULL; + run->frame = NULL; + run->tile_group = NULL; + run->grain = NULL; + f->codec_priv = run; return 0; +fail: + avd_av1_release_run(ctx, run); + clear_bit(f->idx, &av1->active_slots); + f->prog_words = 0; + return ret; +} + +static void avd_av1_retire(struct avd_ctx *ctx, struct avd_frame *f, + enum vb2_buffer_state state) +{ + struct avd_av1_ctx *av1 = ctx->priv; + struct avd_av1_run *run = f->codec_priv; + int status = state == VB2_BUF_STATE_DONE ? 0 : -EIO; + + if (!run) + return; + if (run->cdf_written) { + WRITE_ONCE(run->cdf_out->status, status); + WRITE_ONCE(run->cdf_out->done, true); + } + if (run->seg_written) { + WRITE_ONCE(run->seg_out->status, status); + WRITE_ONCE(run->seg_out->done, true); + } + avd_av1_release_run(ctx, run); + f->codec_priv = NULL; + clear_bit(f->idx, &av1->active_slots); } static void avd_av1_stop(struct avd_ctx *ctx) { - struct avd_av1_ctx *av1_ctx = ctx->priv; - struct avd_dev *avd = ctx->dev; + struct avd_av1_ctx *av1 = ctx->priv; + struct avd_av1_ref *ref, *tmp; - avd_buf_free(avd, &av1_ctx->bufs.above_info); - avd_buf_free(avd, &av1_ctx->bufs.rf_above_info); - avd_buf_free(avd, &av1_ctx->bufs.az_above); - avd_buf_free(avd, &av1_ctx->bufs.ip_above); - avd_buf_free(avd, &av1_ctx->bufs.lf_above); - avd_buf_free(avd, &av1_ctx->bufs.lf_above_info); - avd_buf_free(avd, &av1_ctx->bufs.lf_left); - avd_buf_free(avd, &av1_ctx->bufs.lf_left_info); - avd_buf_free(avd, &av1_ctx->bufs.sr_left); - avd_buf_free(avd, &av1_ctx->bufs.rf_left); - avd_buf_free(avd, &av1_ctx->bufs.rf_left_info); - avd_buf_free(avd, &av1_ctx->bufs.no_prev_seg); - avd_buf_free(avd, &av1_ctx->bufs.mv_above_info); - - avd_buf_free(avd, &av1_ctx->bufs.pipe_state); - avd_buf_free(avd, &av1_ctx->bufs.inst); - avd_buf_free(avd, &av1_ctx->bufs.probs); - - kfree(av1_ctx); + /* Core drains and retires every slot before stopping the codec. */ + if (WARN_ON(av1->active_slots)) + return; + list_for_each_entry_safe(ref, tmp, &av1->refs, node) { + avd_av1_aux_put(ctx, ref->cdf); + avd_av1_aux_put(ctx, ref->seg); + list_del(&ref->node); + kfree(ref); + } + kfree(av1); ctx->priv = NULL; } static int avd_av1_start(struct avd_ctx *ctx) { - struct avd_av1_ctx *av1_ctx; - int ret; + struct avd_av1_ctx *av1; - av1_ctx = kzalloc(sizeof(*av1_ctx), GFP_KERNEL); - if (!av1_ctx) + av1 = kzalloc(sizeof(*av1), GFP_KERNEL); + if (!av1) return -ENOMEM; - - ctx->priv = av1_ctx; - ret = avd_av1_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; - + INIT_LIST_HEAD(&av1->refs); + ctx->priv = av1; return 0; - -err_free_ctx: - avd_av1_stop(ctx); - return ret; } static enum avd_image_fmt avd_av1_get_image_fmt(struct avd_ctx *ctx, @@ -1515,24 +1683,6 @@ static enum avd_image_fmt avd_av1_get_image_fmt(struct avd_ctx *ctx, #undef BIT_DEPTH } -static void avd_av1_submit(struct avd_ctx *ctx) -{ - struct avd_av1_ctx *av1_ctx = ctx->priv; - struct avd_dev *avd = ctx->dev; - - writel(AVD_OP_EXEC | - AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == - 4) | - AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx) | - AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots), - avd->ctrl + avd->variant->submit_offset); - for (int i = 0; i < av1_ctx->submit_num - 1; i++) { - writel(AVD_OP_EXEC | AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx) | - AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots), - avd->ctrl + avd->variant->submit_offset); - } -} - static void avd_av1_adjust_decoded_fmt(struct avd_ctx *ctx, struct v4l2_pix_format_mplane *pix_mp) { @@ -1568,8 +1718,12 @@ const struct avd_coded_fmt_ops avd_av1_fmt_ops = { .adjust_decoded_fmt = avd_av1_adjust_decoded_fmt, .start = avd_av1_start, .stop = avd_av1_stop, - .run = avd_av1_run, - .submit = avd_av1_submit, .try_ctrl = avd_av1_try_ctrl, .get_image_fmt = avd_av1_get_image_fmt, }; + +const struct avd_codec_frame_ops avd_av1_frame_ops = { + .prepare = avd_av1_prepare, + .pack_cmd = avd_av1_pack_cmd, + .retire = avd_av1_retire, +}; diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index bf5bf96f4c62a..6b9428ed13f3b 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -405,7 +405,7 @@ static void avd_device_run(void *priv) spin_unlock_irqrestore(&avd->job_lock, flags); dev_dbg(avd->dev, "avd-job: run seq %llu\n", seq); - ret = desc->ops->run(ctx); + ret = desc->ops->run ? desc->ops->run(ctx) : -EOPNOTSUPP; if (ret) { dev_dbg(avd->dev, "avd-job: run seq %llu failed: %d\n", seq, ret); avd_job_finish(ctx, VB2_BUF_STATE_ERROR); @@ -754,6 +754,7 @@ static const struct avd_variant avd_t8122_variant = { }; static const struct avd_variant avd_t8140_variant = { + .av1_fp_layout = AVD_AV1_FP_T8140_TODO, .vp_slots = { [AVD_CODEC_HEVC] = 2, [AVD_CODEC_H264] = 1, @@ -811,6 +812,7 @@ static const struct avd_variant avd_t8132_variant = { * verified on the hardware. */ static const struct avd_variant avd_t6040_variant = { + .av1_fp_layout = AVD_AV1_FP_BORAGE, .vp_slots = { [AVD_CODEC_HEVC] = 4, [AVD_CODEC_H264] = 4, diff --git a/drivers/media/platform/apple/avd/avd-queue.h b/drivers/media/platform/apple/avd/avd-queue.h new file mode 100644 index 0000000000000..ca04f908ef359 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_QUEUE_H_ +#define AVD_QUEUE_H_ + +#include +#include +#include + +struct avd_dev; +struct avd_ctx; +struct avd_buf; /* avd.h: { void *cpu; dma_addr_t addr; size_t size; } */ + +#define AVD_MAX_FRAMES 16 /* SPEC 4.1: 16 frame-param slots */ + +enum avd_frame_state { + AVD_FRAME_FREE = 0, /* SPEC: slot state 0 */ + AVD_FRAME_RESERVED, /* 1: codec is filling it on the CPU */ + AVD_FRAME_SUBMITTED, /* 3: owned by firmware */ + AVD_FRAME_DONE, /* completion seen, waiting for in-order retire */ +}; + +/* Codec says what must finish before this frame may be submitted. */ +enum avd_frame_dep { + AVD_DEP_NONE = 0, /* independent: may overlap the previous frame */ + AVD_DEP_PREV_DONE, /* previous frame of this ctx must be DONE first + * (VP9 backward adaptation, AV1 CDF update, ...) */ +}; + +struct avd_frame { + unsigned int idx; /* 0..AVD_MAX_FRAMES-1, also the fp slot */ + u32 tag; /* (seq << 4) | idx, echoed by completion */ + u64 seq; /* global submission order */ + enum avd_frame_state state; + enum avd_frame_dep dep; + int status; /* 0 or -EIO/-ETIMEDOUT from the backend */ + + struct avd_ctx *ctx; + struct vb2_v4l2_buffer *src; /* OUTPUT, held until retire */ + struct vb2_v4l2_buffer *dst; /* CAPTURE, held until retire */ + u32 frame_num; /* SPEC 5.2 +0x0c */ + + /* open backend: instruction program, DMA-visible, built by the codec */ + struct avd_buf *prog; /* per-slot, allocated by core */ + size_t prog_words; + + /* apple backend: frame-params slot (SPEC 0x1130d0 on Borage) */ + struct avd_buf *fp; /* per-slot, allocated by core, size from match data */ + u32 cmd[24]; /* 0x60 Borage command; core fills the 5.2 prefix, + * codec pack_cmd() fills the codec part */ + + void *codec_priv; /* per-frame codec state, owned by the codec */ + unsigned long deadline; /* jiffies, per-frame watchdog */ +}; + +struct avd_backend_ops { + const char *name; + int (*probe)(struct avd_dev *avd); /* apertures, fw source, version */ + int (*boot)(struct avd_dev *avd); /* power-up: fw present/loaded, link up, ready */ + void (*shutdown)(struct avd_dev *avd); + unsigned int (*max_inflight)(struct avd_dev *avd); + int (*session_start)(struct avd_ctx *ctx); /* apple: cmd 0; open: may be NULL */ + void (*session_stop)(struct avd_ctx *ctx); /* apple: cmd 2 */ + /* Non-blocking. -EBUSY means inbox full: core retries on next completion. */ + int (*submit)(struct avd_dev *avd, struct avd_frame *f); + void (*abort)(struct avd_ctx *ctx); /* apple: cmd 5 */ + irqreturn_t (*irq)(struct avd_dev *avd); /* calls avd_frame_complete() */ + int (*suspend)(struct avd_dev *avd); + int (*resume)(struct avd_dev *avd); +}; + +/* Codec half. Replaces avd_coded_fmt_ops.run/submit/done. CPU only, no MMIO. */ +struct avd_codec_frame_ops { + /* Read the request's controls, set f->dep, fill f->prog (open backend) + * or f->fp (apple backend), allocate f->codec_priv if needed. */ + int (*prepare)(struct avd_ctx *ctx, struct avd_frame *f); + /* Apple backend: fill the codec part of f->cmd (SPEC 6.x). */ + int (*pack_cmd)(struct avd_ctx *ctx, struct avd_frame *f); + /* Called in submission order per ctx, after completion or failure. */ + void (*retire)(struct avd_ctx *ctx, struct avd_frame *f, + enum vb2_buffer_state state); +}; + +/* Core API used by backends and codecs. */ +void avd_frame_complete(struct avd_dev *avd, u32 tag, int status); /* any ctx, IRQ-safe */ +bool avd_backend_is_apple(const struct avd_dev *avd); + +#endif /* AVD_QUEUE_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index 4622e92055486..1ff19368b6971 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -449,6 +449,7 @@ static const struct avd_coded_fmt_desc avd_coded_fmts[] = { }, .ctrls = &avd_av1_ctrls, .ops = &avd_av1_fmt_ops, + .frame_ops = &avd_av1_frame_ops, .num_decoded_fmts = ARRAY_SIZE(avd_av1_decoded_fmts), .decoded_fmts = avd_av1_decoded_fmts, .capability = AVD_CAPABILITY_AV1, diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index ad7accb6e9615..f38339fdf6e2c 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -14,6 +14,9 @@ #ifndef AVD_H_ #define AVD_H_ +#include "avd-queue.h" +#include "avd-av1-fp.h" + #include #include #include @@ -173,6 +176,7 @@ struct avd_coded_fmt_desc { struct v4l2_frmsize_stepwise frmsize; const struct avd_ctrls *ctrls; const struct avd_coded_fmt_ops *ops; + const struct avd_codec_frame_ops *frame_ops; unsigned int num_decoded_fmts; const struct avd_decoded_fmt_desc *decoded_fmts; u32 subsystem_flags; @@ -180,6 +184,7 @@ struct avd_coded_fmt_desc { }; struct avd_variant { + enum avd_av1_fp_layout av1_fp_layout; unsigned int vp_slots[4]; unsigned int vp_slot_base[4]; unsigned int fifo_slots; @@ -306,6 +311,7 @@ extern const struct avd_coded_fmt_ops avd_h264_fmt_ops; extern const struct avd_coded_fmt_ops avd_hevc_fmt_ops; extern const struct avd_coded_fmt_ops avd_vp9_fmt_ops; extern const struct avd_coded_fmt_ops avd_av1_fmt_ops; +extern const struct avd_codec_frame_ops avd_av1_frame_ops; extern const struct v4l2_ctrl_ops avd_ctrl_ops; extern const struct v4l2_ioctl_ops avd_ioctl_ops; diff --git a/drivers/media/platform/apple/avd/tests/avd-av1-legacy.c b/drivers/media/platform/apple/avd/tests/avd-av1-legacy.c new file mode 100644 index 0000000000000..206bca6db5e4b --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/avd-av1-legacy.c @@ -0,0 +1,1060 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Apple AVD VPU codec driver + * + * Copyright The Asahi Linux Contributors + * Copyright 2023 Eileen Yoon + * + * Copyright (c) 2023, Collabora + * Benjamin Gaignard + * + * Copyright (c) 2014 Rockchip Electronics Co., Ltd. + * Hertz Wong + * Herman Chen + * + * Copyright (C) 2014 Google, Inc. + * Tomasz Figa + */ + +#include + +#include "avd.h" +#include "avd-av1-entropymode.h" +#include "avd-inst.h" + +/* + * This comes from + * verisilicon/rockchip_vpu981_hw_av1_dec.c + * + * These 3 values aren't defined enum v4l2_av1_segment_feature because + * they are not part of the specification + */ +#define V4L2_AV1_SEG_LVL_ALT_LF_Y_H 2 +#define V4L2_AV1_SEG_LVL_ALT_LF_U 3 +#define V4L2_AV1_SEG_LVL_ALT_LF_V 4 + +#define DIV_LUT_BITS 8 +#define DIV_LUT_PREC_BITS 14 +#define WARPEDMODEL_PREC_BITS 16 +#define WARP_PARAM_REDUCE_BITS 6 + +#define DIV_LUT_NUM BIT(DIV_LUT_BITS) + +#define AV1_DIV_ROUND_UP_POW2(value, n) \ +({ \ + typeof(n) _n = n; \ + typeof(value) _value = value; \ + (_value + (BIT(_n) >> 1)) >> _n; \ +}) + +#define AV1_DIV_ROUND_UP_POW2_SIGNED(value, n) \ +({ \ + typeof(n) _n_ = n; \ + typeof(value) _value_ = value; \ + (((_value_) < 0) ? -AV1_DIV_ROUND_UP_POW2(-(_value_), (_n_)) \ + : AV1_DIV_ROUND_UP_POW2((_value_), (_n_))); \ +}) + +#define AV1_REF_SCALE_SHIFT 14 +#define SUPERRES_SCALE_BITS 14 +#define SUPERRES_EXTRA_BITS 8 +#define SUPERRES_SCALE_MASK ((1 << 14) - 1) + +#define AV1_CODEC_MODE_INTRABC(v) FIELD_PREP(BIT(28), !!(v)) + +#define AV1_HDR_JNT_COMP(v) FIELD_PREP(BIT(0), !!(v)) +#define AV1_HDR_ORDER_HINT_BITS(v) FIELD_PREP(GENMASK(3, 1), v) +#define AV1_HDR_ORDER_HINT(v) FIELD_PREP(BIT(4), !!(v)) +#define AV1_HDR_DUAL_FILTER(v) FIELD_PREP(BIT(5), !!(v)) +#define AV1_HDR_MASKED_COMPOUND(v) FIELD_PREP(BIT(6), !!(v)) +#define AV1_HDR_INTERINTRA_COMPOUND(v) FIELD_PREP(BIT(7), !!(v)) +#define AV1_HDR_INTRA_EDGE_FILTER(v) FIELD_PREP(BIT(8), !!(v)) +#define AV1_HDR_FILTER_INTRA(v) FIELD_PREP(BIT(9), !!(v)) +#define AV1_HDR_128X128_SUPERBLOCK(v) FIELD_PREP(BIT(10), !!(v)) +#define AV1_HDR_SCREEN_CONTENT_TOOLS(v) FIELD_PREP(BIT(11), !!(v)) + +#define AV1_FLAGS_TX_MODE_LARGEST(v) FIELD_PREP(BIT(0), !!(v)) +#define AV1_FLAGS_TX_MODE_SELECT(v) FIELD_PREP(BIT(1), !!(v)) +#define AV1_FLAGS_TX_REDUCED_SET(v) FIELD_PREP(BIT(2), !!(v)) +#define AV1_FLAGS_ALLOW_WARPED_MOTION(v) FIELD_PREP(BIT(3), !!(v)) +#define AV1_FLAGS_SKIP_MODE(v) FIELD_PREP(BIT(4), !!(v)) +#define AV1_FLAGS_MOTION_IS_SWITCHABLE(v) FIELD_PREP(BIT(5), !!(v)) +#define AV1_FLAGS_INTRABC(v) FIELD_PREP(BIT(6), !!(v)) +#define AV1_FLAGS_INTEGER_MV(v) FIELD_PREP(BIT(7), !!(v)) +#define AV1_FLAGS_DISABLE_CDF_UPDATE(v) FIELD_PREP(BIT(8), !!(v)) +#define AV1_FLAGS_TX_MODE_ONLY_4X4(v) FIELD_PREP(BIT(9), !!(v)) +#define AV1_FLAGS_HIGH_PRECISION_MV(v) FIELD_PREP(BIT(10), !!(v)) +#define AV1_FLAGS_REF_FRAME_MVS(v) FIELD_PREP(BIT(12), !!(v)) +#define AV1_FLAGS_SEG_TEMPORAL_UPDATE(v) FIELD_PREP(BIT(13), !!(v)) +#define AV1_FLAGS_SEG_UPDATE_MAP(v) FIELD_PREP(BIT(14), !!(v)) +#define AV1_FLAGS_SEG_ENABLED(v) FIELD_PREP(BIT(15), !!(v)) +#define AV1_FLAGS_SEG_UPDATE_DATA(v) FIELD_PREP(BIT(16), !!(v)) +#define AV1_FLAGS_INTRA(v) FIELD_PREP(BIT(31), !!(v)) +#define AV1_FLAGS_SKIP_MODE1(v) FIELD_PREP(GENMASK(19, 17), v) +#define AV1_FLAGS_SKIP_MODE0(v) FIELD_PREP(GENMASK(22, 20), v) +#define AV1_FLAGS_ORDER_HINT(v) FIELD_PREP(GENMASK(30, 23), v) + +#define AV1_FLAGS_REFERENCE_SELECT(v) FIELD_PREP(BIT(28), !!(v)) +#define AV1_FLAGS_INTERPOLATION_FILTER(v) FIELD_PREP(GENMASK(31, 29), v) + +#define AV1_SIGN_BIAS(v, i) (!!(v) << (i)) +#define AV1_REF_SLOT(v, i) (((v) & 7) << (7 + ((i) * 3))) + +#define AV1_SEG_ALT_Q(v) FIELD_PREP(GENMASK(8, 0), v) +#define AV1_SEG_ALT_Q_EN(v) FIELD_PREP(BIT(9), !!(v)) +#define AV1_SEG_REF_FRAME(v) FIELD_PREP(GENMASK(12, 10), v) +#define AV1_SEG_REF_FRAME_EN(v) FIELD_PREP(BIT(13), !!(v)) +#define AV1_SEG_REF_SKIP_EN(v) FIELD_PREP(BIT(14), !!(v)) +#define AV1_SEG_REF_GMV_EN(v) FIELD_PREP(BIT(15), !!(v)) + +#define AV1_SEG_LF_V(v) FIELD_PREP(GENMASK(6, 0), v) +#define AV1_SEG_LF_V_EN(v) FIELD_PREP(BIT(7), !!(v)) +#define AV1_SEG_LF_U(v) FIELD_PREP(GENMASK(14, 8), v) +#define AV1_SEG_LF_U_EN(v) FIELD_PREP(BIT(15), !!(v)) +#define AV1_SEG_LF_Y_H(v) FIELD_PREP(GENMASK(22, 16), v) +#define AV1_SEG_LF_Y_H_EN(v) FIELD_PREP(BIT(23), !!(v)) +#define AV1_SEG_LF_Y_V(v) FIELD_PREP(GENMASK(30, 24), v) +#define AV1_SEG_LF_Y_V_EN(v) FIELD_PREP(BIT(31), !!(v)) + +#define AV1_SEG_UNK0(v) FIELD_PREP(BIT(16), !!(v)) + +#define AV1_QP_U_AC(v) FIELD_PREP(GENMASK(6, 0), v) +#define AV1_QP_U_DC(v) FIELD_PREP(GENMASK(13, 7), v) +#define AV1_QP_Y_DC(v) FIELD_PREP(GENMASK(20, 14), v) +#define AV1_QP_BASE_IDX(v) FIELD_PREP(GENMASK(28, 21), v) +#define AV1_QP_DELTA_RES(v) FIELD_PREP(GENMASK(30, 29), v) +#define AV1_QP_PRESENT(v) FIELD_PREP(BIT(31), !!(v)) +#define AV1_QP_V_DC(v) FIELD_PREP(GENMASK(26, 20), v) +#define AV1_QP_V_AC(v) FIELD_PREP(GENMASK(19, 13), v) +#define AV1_QP_QMATRIX(v) FIELD_PREP(BIT(12), !!(v)) +#define AV1_QP_QM_Y(v) FIELD_PREP(GENMASK(11, 8), v) +#define AV1_QP_QM_U(v) FIELD_PREP(GENMASK(7, 4), v) +#define AV1_QP_QM_V(v) FIELD_PREP(GENMASK(3, 0), v) + +#define AV1_GM_VALID(v) FIELD_PREP(BIT(30), !!(v)) +#define AV1_GM_TYPE(v) FIELD_PREP(GENMASK(31, 30), v) +#define AV1_GM_PARAM0(v) FIELD_PREP(GENMASK(29, 15), v) +#define AV1_GM_PARAM1(v) FIELD_PREP(GENMASK(14, 0), v) + +#define AV1_GM_PARAM_SHEAR0(v) FIELD_PREP(GENMASK(15, 0), v) +#define AV1_GM_PARAM_SHEAR1(v) FIELD_PREP(GENMASK(31, 16), v) + +#define AV1_ORDER_HINT0(v) FIELD_PREP(GENMASK(7, 0), v) +#define AV1_ORDER_HINT1(v) FIELD_PREP(GENMASK(15, 8), v) +#define AV1_ORDER_HINT2(v) FIELD_PREP(GENMASK(23, 16), v) +#define AV1_ORDER_HINT3(v) FIELD_PREP(GENMASK(31, 24), v) +#define AV1_ORDER_HINT_REF(v) FIELD_PREP(GENMASK(31, 29), v) + +#define AV1_LF_DELTA_ENABLED(v) FIELD_PREP(BIT(27), !!(v)) +#define AV1_LF_DELTA_MULTI(v) FIELD_PREP(BIT(28), !!(v)) +#define AV1_LF_DELTA_PRESENT(v) FIELD_PREP(BIT(31), !!(v)) +#define AV1_LF_DELTA_RES(v) FIELD_PREP(GENMASK(30, 29), v) +#define AV1_LF_LV3(v) FIELD_PREP(GENMASK(5, 0), v) +#define AV1_LF_LV2(v) FIELD_PREP(GENMASK(11, 6), v) +#define AV1_LF_LV1(v) FIELD_PREP(GENMASK(17, 12), v) +#define AV1_LF_LV0(v) FIELD_PREP(GENMASK(23, 18), v) + +#define AV1_LF_SHARPNESS(v) FIELD_PREP(GENMASK(31, 28), v) +#define AV1_LF_REF0(v) FIELD_PREP(GENMASK(27, 21), v) +#define AV1_LF_REF1(v) FIELD_PREP(GENMASK(20, 14), v) +#define AV1_LF_REF2(v) FIELD_PREP(GENMASK(13, 7), v) +#define AV1_LF_REF3(v) FIELD_PREP(GENMASK(6, 0), v) + +#define AV1_LF_MODE0(v) FIELD_PREP(GENMASK(13, 7), v) +#define AV1_LF_MODE1(v) FIELD_PREP(GENMASK(6, 0), v) + +#define AV1_CDEF_EN(v) FIELD_PREP(BIT(28), !!(v)) +#define AV1_CDEF_BITS(v) FIELD_PREP(GENMASK(25, 24), v) +#define AV1_CDEF_DAMPING(v) FIELD_PREP(GENMASK(27, 26), v) + +#define AV1_CDEF_Y_PRI(v) FIELD_PREP(GENMASK(11, 8), v) +#define AV1_CDEF_Y_SEC(v) FIELD_PREP(GENMASK(7, 6), v) +#define AV1_CDEF_UV_PRI(v) FIELD_PREP(GENMASK(5, 2), v) +#define AV1_CDEF_UV_SEC(v) FIELD_PREP(GENMASK(1, 0), v) + +#define AV1_CDEF_LO(v) FIELD_PREP(GENMASK(11, 0), v) +#define AV1_CDEF_HI(v) FIELD_PREP(GENMASK(23, 12), v) + +#define AV1_LR_TYPE0(v) FIELD_PREP(GENMASK(11, 10), v) +#define AV1_LR_TYPE1(v) FIELD_PREP(GENMASK(9, 8), v) +#define AV1_LR_TYPE2(v) FIELD_PREP(GENMASK(7, 6), v) + +#define AV1_LR_UNIT2(v) FIELD_PREP(GENMASK(1, 0), v) +#define AV1_LR_UNIT1(v) FIELD_PREP(GENMASK(3, 2), v) +#define AV1_LR_UNIT0(v) FIELD_PREP(GENMASK(5, 4), v) + +#define AV1_REF_ORDER_HINT(v) FIELD_PREP(GENMASK(23, 0), v) +#define AV1_REF_SCALE_X(v) FIELD_PREP(GENMASK(31, 16), v) +#define AV1_REF_SCALE_Y(v) FIELD_PREP(GENMASK(15, 0), v) + +#define AV1_SUPERRES_USE(v) FIELD_PREP(BIT(31), !!(v)) +#define AV1_SUPERRES_DENOM(v) FIELD_PREP(GENMASK(30, 28), v) +/* im gonna asume 16 like the others */ +#define AV1_SUPERRES_UPSCALE(v) FIELD_PREP(GENMASK(16, 0), v) + +#define AV1_UPSCALE_STEPX(v) FIELD_PREP(GENMASK(13, 0), v) +#define AV1_UPSCALE_UPSCX(v) FIELD_PREP(GENMASK(31, 14), v) + +#define AVD_CDFS_SIZE sizeof(struct avd_av1_cdfs) + +/* Imported planes can have an unaligned length; AVD aligns both tail addresses. */ +#define AVD_AV1_COLOR_OFFSET(dst, cl) \ + (ALIGN_DOWN((dst), AVD_ALIGN) - ALIGN(cl, AVD_ALIGN)) +#define AVD_AV1_CDFS_OFFSET(dst, cl) \ + (AVD_AV1_COLOR_OFFSET(dst, cl) - ALIGN(AVD_CDFS_SIZE, AVD_ALIGN)) +/* The frame's segmentation map, saved with it like the CDFs (AV1 7.20) */ +#define AVD_AV1_SEG_OFFSET(dst, cl, sl) \ + (AVD_AV1_CDFS_OFFSET(dst, cl) - (sl)) + +#define AV1_PRIMARY_REF_NONE 7 + +struct avd_av1_run { + struct avd_run base; + + struct { + dma_addr_t probs_out; + dma_addr_t color; + dma_addr_t seg; + dma_addr_t prev_seg; + } addresses; + + const struct v4l2_ctrl_av1_sequence *seq; + const struct v4l2_ctrl_av1_frame *frame; + const struct v4l2_ctrl_av1_tile_group_entry *tile_group; + const struct v4l2_ctrl_av1_film_grain *grain; +}; + +struct avd_av1_ctx { + u8 submit_num; + struct { + struct avd_buf inst; + struct avd_buf pipe_state; + struct avd_buf probs; + struct avd_buf no_prev_seg; + struct avd_buf above_info; + struct avd_buf rf_above_info; + struct avd_buf az_above; + struct avd_buf ip_above; + struct avd_buf lf_above; + struct avd_buf lf_above_info; + struct avd_buf lf_left; + struct avd_buf lf_left_info; + struct avd_buf sr_left; + struct avd_buf rf_left; + struct avd_buf rf_left_info; + struct avd_buf mv_above_info; + } bufs; +}; + +/* + * from verisilicon/rockchip_vpu981_hw_av1_dec.c + */ +static const short div_lut[DIV_LUT_NUM + 1] = { + 16384, 16320, 16257, 16194, 16132, 16070, 16009, 15948, 15888, 15828, 15768, + 15709, 15650, 15592, 15534, 15477, 15420, 15364, 15308, 15252, 15197, 15142, + 15087, 15033, 14980, 14926, 14873, 14821, 14769, 14717, 14665, 14614, 14564, + 14513, 14463, 14413, 14364, 14315, 14266, 14218, 14170, 14122, 14075, 14028, + 13981, 13935, 13888, 13843, 13797, 13752, 13707, 13662, 13618, 13574, 13530, + 13487, 13443, 13400, 13358, 13315, 13273, 13231, 13190, 13148, 13107, 13066, + 13026, 12985, 12945, 12906, 12866, 12827, 12788, 12749, 12710, 12672, 12633, + 12596, 12558, 12520, 12483, 12446, 12409, 12373, 12336, 12300, 12264, 12228, + 12193, 12157, 12122, 12087, 12053, 12018, 11984, 11950, 11916, 11882, 11848, + 11815, 11782, 11749, 11716, 11683, 11651, 11619, 11586, 11555, 11523, 11491, + 11460, 11429, 11398, 11367, 11336, 11305, 11275, 11245, 11215, 11185, 11155, + 11125, 11096, 11067, 11038, 11009, 10980, 10951, 10923, 10894, 10866, 10838, + 10810, 10782, 10755, 10727, 10700, 10673, 10645, 10618, 10592, 10565, 10538, + 10512, 10486, 10460, 10434, 10408, 10382, 10356, 10331, 10305, 10280, 10255, + 10230, 10205, 10180, 10156, 10131, 10107, 10082, 10058, 10034, 10010, 9986, + 9963, 9939, 9916, 9892, 9869, 9846, 9823, 9800, 9777, 9754, 9732, + 9709, 9687, 9664, 9642, 9620, 9598, 9576, 9554, 9533, 9511, 9489, + 9468, 9447, 9425, 9404, 9383, 9362, 9341, 9321, 9300, 9279, 9259, + 9239, 9218, 9198, 9178, 9158, 9138, 9118, 9098, 9079, 9059, 9039, + 9020, 9001, 8981, 8962, 8943, 8924, 8905, 8886, 8867, 8849, 8830, + 8812, 8793, 8775, 8756, 8738, 8720, 8702, 8684, 8666, 8648, 8630, + 8613, 8595, 8577, 8560, 8542, 8525, 8508, 8490, 8473, 8456, 8439, + 8422, 8405, 8389, 8372, 8355, 8339, 8322, 8306, 8289, 8273, 8257, + 8240, 8224, 8208, 8192, +}; + +static inline int avd_av1_dec_get_msb(u32 n) +{ + if (n == 0) + return 0; + return 31 ^ __builtin_clz(n); +} + +static short avd_av1_dec_resolve_divisor_32(u32 d, short *shift) +{ + int f; + u64 e; + + *shift = avd_av1_dec_get_msb(d); + /* e is obtained from D after resetting the most significant 1 bit. */ + e = d - ((u32)1 << *shift); + /* Get the most significant DIV_LUT_BITS (8) bits of e into f */ + if (*shift > DIV_LUT_BITS) + f = AV1_DIV_ROUND_UP_POW2(e, *shift - DIV_LUT_BITS); + else + f = e << (DIV_LUT_BITS - *shift); + if (f > DIV_LUT_NUM) + return -1; + *shift += DIV_LUT_PREC_BITS; + /* Use f as lookup into the precomputed table of multipliers */ + return div_lut[f]; +} + +static void +avd_av1_dec_get_shear_params(const u32 *params, s64 *alpha, + s64 *beta, s64 *gamma, s64 *delta) +{ + const int *mat = params; + short shift; + short y; + long long gv, dv; + + if (mat[2] <= 0) + return; + + *alpha = clamp_val(mat[2] - (1 << WARPEDMODEL_PREC_BITS), S16_MIN, S16_MAX); + *beta = clamp_val(mat[3], S16_MIN, S16_MAX); + + y = avd_av1_dec_resolve_divisor_32(abs(mat[2]), &shift) * (mat[2] < 0 ? -1 : 1); + + gv = ((long long)mat[4] * (1 << WARPEDMODEL_PREC_BITS)) * y; + + *gamma = clamp_val((int)AV1_DIV_ROUND_UP_POW2_SIGNED(gv, shift), S16_MIN, S16_MAX); + + dv = ((long long)mat[3] * mat[4]) * y; + *delta = clamp_val(mat[5] - + (int)AV1_DIV_ROUND_UP_POW2_SIGNED(dv, shift) - (1 << WARPEDMODEL_PREC_BITS), + S16_MIN, S16_MAX); + + *alpha = AV1_DIV_ROUND_UP_POW2_SIGNED(*alpha, WARP_PARAM_REDUCE_BITS) + * (1 << WARP_PARAM_REDUCE_BITS); + *beta = AV1_DIV_ROUND_UP_POW2_SIGNED(*beta, WARP_PARAM_REDUCE_BITS) + * (1 << WARP_PARAM_REDUCE_BITS); + *gamma = AV1_DIV_ROUND_UP_POW2_SIGNED(*gamma, WARP_PARAM_REDUCE_BITS) + * (1 << WARP_PARAM_REDUCE_BITS); + *delta = AV1_DIV_ROUND_UP_POW2_SIGNED(*delta, WARP_PARAM_REDUCE_BITS) + * (1 << WARP_PARAM_REDUCE_BITS); +} + +static void set_refs(struct avd_ctx *ctx, struct avd_av1_run *run) +{ + struct avd_av1_ctx *av1_ctx = ctx->priv; + const struct v4l2_ctrl_av1_frame *frame = run->frame; + const struct v4l2_av1_global_motion *gm = &frame->global_motion; + int i, ref_idx; + struct avd_decoded_buffer *dst, *ref; + bool intrabc = !!(frame->flags & V4L2_AV1_FRAME_FLAG_ALLOW_INTRABC); + int height, width, x_scale, y_scale; + height = frame->frame_height_minus_1 + 1; + width = frame->frame_width_minus_1 + 1; + dma_addr_t addr; + s64 alpha = 0, beta = 0, gamma = 0, delta = 0; + dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); + + push(0, "ref_cnst0"); + pusha(av1_ctx->bufs.mv_above_info.addr, "mv_above_info", 0); + + for (i = 0; i < 4; i++) + push(0, "ref_cnst1"); + + for (i = 0; i < (intrabc ? 1 : V4L2_AV1_REFS_PER_FRAME); i++) { + if (intrabc) { + ref = dst; + } else { + ref_idx = i + V4L2_AV1_REF_LAST_FRAME; + ref = avd_get_ref_buf( + ctx, &dst->base.vb, + frame->reference_frame_ts + [frame->ref_frame_idx[i]]); + } + addr = vb2_dma_contig_plane_dma_addr(&ref->base.vb.vb2_buf, 0) + + ref->comp.start_offset; + + if (!intrabc) { + int shift = + (gm->flags[ref_idx] & + V4L2_AV1_GLOBAL_MOTION_FLAG_IS_TRANSLATION) ? + (WARPEDMODEL_PREC_BITS - + 2) - (frame->flags & V4L2_AV1_FRAME_FLAG_ALLOW_HIGH_PRECISION_MV ? + 1 : + 0) : + 10; + push(AV1_GM_TYPE(gm->type[ref_idx]) | + AV1_GM_PARAM0(gm->params[ref_idx][0] >> + shift) | + AV1_GM_PARAM1(gm->params[ref_idx][1] >> + shift), + "ref_gm_mv"); + push(AV1_GM_VALID(!(V4L2_AV1_GLOBAL_MOTION_IS_INVALID( + ref_idx) & + gm->invalid)) | + AV1_GM_PARAM0(AV1_DIV_ROUND_UP_POW2_SIGNED( + gm->params[ref_idx][2], 1)) | + AV1_GM_PARAM1(AV1_DIV_ROUND_UP_POW2_SIGNED( + gm->params[ref_idx][3], 1)), + "ref_gm_param"); + push(AV1_GM_PARAM1(gm->params[ref_idx][5] >> 1) | + AV1_GM_PARAM0(gm->params[ref_idx][4] >> 1), + "ref_gm_unk2"); + avd_av1_dec_get_shear_params(&gm->params[ref_idx][0], + &alpha, &beta, &gamma, + &delta); + push(AV1_GM_PARAM_SHEAR0(beta) | + AV1_GM_PARAM_SHEAR1(alpha), + "ref_gm_beta_alpha"); + + push(AV1_GM_PARAM_SHEAR0(delta) | + AV1_GM_PARAM_SHEAR1(gamma), + "ref_gm_delta_gamma"); + } + push(AVD_REF_FLAG_CONST | + AV1_REF_ORDER_HINT( + intrabc ? 0 : frame->order_hints[ref_idx]), + "ref_flag"); + push(AVD_HDR_HEIGHT(ref->av1.height) | + AVD_HDR_WIDTH(ref->av1.width), + "ref_height_width"); + + x_scale = ((ref->av1.upscaled_width << AV1_REF_SCALE_SHIFT) + + (width / 2)) / + width; + y_scale = (((ref->av1.height + 1) << AV1_REF_SCALE_SHIFT) + + (height / 2)) / + height; + push(AV1_REF_SCALE_X(x_scale) | AV1_REF_SCALE_Y(y_scale), + "ref_scale"); + + push_comp(ctx, addr, ref->comp.offsets); + } +} + +static int avd_av1_get_dist(struct avd_av1_run *run, int a, int b) +{ + const struct v4l2_ctrl_av1_sequence *seq = run->seq; + int bits = seq->order_hint_bits - 1; + int diff, m; + + if (!seq->order_hint_bits) + return 0; + + diff = a - b; + m = 1 << bits; + diff = (diff & (m - 1)) - (diff & m); + + return diff; +} + +static bool is_valid_ref_frame(struct avd_ctx *ctx, struct avd_av1_run *run, + struct avd_decoded_buffer *dst, int mi_cols, + int mi_rows, + enum v4l2_av1_reference_frame ref_frame) +{ + const struct v4l2_ctrl_av1_frame *frame = run->frame; + struct avd_decoded_buffer *ref = avd_get_ref_buf( + ctx, &dst->base.vb, + frame->reference_frame_ts + [frame->ref_frame_idx[ref_frame - + V4L2_AV1_REF_LAST_FRAME]]); + int ref_mi_cols = DIV_ROUND_UP(ref->av1.width + 1, 8); + int ref_mi_rows = DIV_ROUND_UP(ref->av1.height + 1, 8); + bool intra_only = ((ref->av1.frame_type == V4L2_AV1_KEY_FRAME) || + (ref->av1.frame_type == V4L2_AV1_INTRA_ONLY_FRAME)); + + return ref != dst && ref_mi_cols == mi_cols && ref_mi_rows == mi_rows && + !intra_only; +} + +static int avd_select_refs(struct avd_ctx *ctx, struct avd_av1_run *run, + u8 *selected_refs) +{ + const struct v4l2_ctrl_av1_frame *frame = run->frame; + const struct v4l2_ctrl_av1_sequence *seq = run->seq; + int ref_stamp = 2; + struct avd_decoded_buffer *dst, *ref; + bool use_last; + int ref_num = 0; + int order_hint = frame->order_hint; + int mi_cols = DIV_ROUND_UP(frame->frame_width_minus_1 + 1, 8); + int mi_rows = DIV_ROUND_UP(frame->frame_height_minus_1 + 1, 8); + dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); + + int last_ref_idx = frame->ref_frame_idx[V4L2_AV1_REF_INTRA_FRAME]; + if (seq->flags & (V4L2_AV1_SEQUENCE_FLAG_ENABLE_ORDER_HINT | + V4L2_AV1_SEQUENCE_FLAG_ENABLE_REF_FRAME_MVS)) { + ref = avd_get_ref_buf(ctx, &dst->base.vb, + frame->reference_frame_ts[last_ref_idx]); + if (dst != ref) { + use_last = + (ref->av1.order_hints[V4L2_AV1_REF_ALTREF_FRAME] != + frame->order_hints[V4L2_AV1_REF_GOLDEN_FRAME]); + if (use_last && + is_valid_ref_frame(ctx, run, dst, mi_cols, mi_rows, + V4L2_AV1_REF_LAST_FRAME)) + selected_refs[ref_num++] = + V4L2_AV1_REF_LAST_FRAME; + ref_stamp--; + } + + if (avd_av1_get_dist( + run, frame->order_hints[V4L2_AV1_REF_BWDREF_FRAME], + order_hint) > 0 && + is_valid_ref_frame(ctx, run, dst, mi_cols, mi_rows, + V4L2_AV1_REF_BWDREF_FRAME)) { + selected_refs[ref_num++] = V4L2_AV1_REF_BWDREF_FRAME; + ref_stamp--; + } + + if (avd_av1_get_dist( + run, frame->order_hints[V4L2_AV1_REF_ALTREF2_FRAME], + order_hint) > 0 && + is_valid_ref_frame(ctx, run, dst, mi_cols, mi_rows, + V4L2_AV1_REF_ALTREF2_FRAME)) { + selected_refs[ref_num++] = V4L2_AV1_REF_ALTREF2_FRAME; + ref_stamp--; + } + + if (ref_stamp >= 0 && + avd_av1_get_dist( + run, frame->order_hints[V4L2_AV1_REF_ALTREF_FRAME], + order_hint) > 0 && + is_valid_ref_frame(ctx, run, dst, mi_cols, mi_rows, + V4L2_AV1_REF_ALTREF_FRAME)) { + selected_refs[ref_num++] = V4L2_AV1_REF_ALTREF_FRAME; + ref_stamp--; + } + + if (ref_stamp >= 0 && + is_valid_ref_frame(ctx, run, dst, mi_cols, mi_rows, + V4L2_AV1_REF_LAST2_FRAME)) { + selected_refs[ref_num++] = V4L2_AV1_REF_LAST2_FRAME; + ref_stamp--; + } + } + + return ref_num; +} + +static void set_ref_hints(struct avd_ctx *ctx, struct avd_av1_run *run, + u8 *selected_refs) +{ + const struct v4l2_ctrl_av1_frame *frame = run->frame; + int i, ref_idx; + struct avd_decoded_buffer *dst, *ref; + dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); + + for (i = 0; i < 3; i++) { + if (selected_refs[i] < V4L2_AV1_REF_LAST_FRAME) { + push(0, ""); + push(0, ""); + } else { + ref_idx = frame->ref_frame_idx[selected_refs[i] - + V4L2_AV1_REF_LAST_FRAME]; + ref = avd_get_ref_buf( + ctx, &dst->base.vb, + frame->reference_frame_ts[ref_idx]); + + push(AV1_ORDER_HINT0(ref->av1.order_hints[3]) | + AV1_ORDER_HINT1(ref->av1.order_hints[2]) | + AV1_ORDER_HINT2(ref->av1.order_hints[1]) | + AV1_ORDER_HINT_REF( + (frame->flags & + V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS) ? + selected_refs[i] : + 0), + "order_hints0"); + + push(AV1_ORDER_HINT0(ref->av1.order_hints[7]) | + AV1_ORDER_HINT1(ref->av1.order_hints[6]) | + AV1_ORDER_HINT2(ref->av1.order_hints[5]) | + AV1_ORDER_HINT3(ref->av1.order_hints[4]), + "order_hints1"); + } + } +} + +static void set_ref_hdr(struct avd_ctx *ctx, struct avd_av1_run *run) +{ + const struct v4l2_ctrl_av1_frame *frame = run->frame; + u8 ref_slots[7] = { 1, 2, 3, 4, 5, 6, 7 }; + int sign_bias = 0; + int ref_slot = 0; + int i, j; + /* point to the first index where the frames equal */ + for (i = 0; i < 6; i++) { + for (j = i + 1; j < 7; j++) { + if (frame->reference_frame_ts[frame->ref_frame_idx[i]] == + frame->reference_frame_ts[frame->ref_frame_idx[j]]) { + ref_slots[j] = ref_slots[i]; + break; + } + } + } + for (i = 0; i < 7; i++) { + sign_bias |= AV1_SIGN_BIAS( + avd_av1_get_dist( + run, + frame->order_hints[i + V4L2_AV1_REF_LAST_FRAME], + frame->order_hint) > 0, + 6 - i); + + ref_slot |= AV1_REF_SLOT(ref_slots[i], 6 - i); + } + push(ref_slot | sign_bias | + AV1_FLAGS_REFERENCE_SELECT( + frame->flags & + V4L2_AV1_FRAME_FLAG_REFERENCE_SELECT) | + AV1_FLAGS_INTERPOLATION_FILTER( + frame->interpolation_filter), + "reference_select"); +} + +/* 7.16. Upscaling process */ +static void calc_upscale(int frame_width, int upscaled_width, int sub_x, + int *step_x, int *initial_subpel_x) +{ + int downscaled_plane_w, upscaled_plane_w, err; + + downscaled_plane_w = AV1_DIV_ROUND_UP_POW2(frame_width, sub_x); + upscaled_plane_w = AV1_DIV_ROUND_UP_POW2(upscaled_width, sub_x); + *step_x = ((downscaled_plane_w << SUPERRES_SCALE_BITS) + + (upscaled_plane_w / 2)) / + upscaled_plane_w; + err = (upscaled_plane_w * *step_x) - + (downscaled_plane_w << SUPERRES_SCALE_BITS); + *initial_subpel_x = (-((upscaled_plane_w - downscaled_plane_w) + << (SUPERRES_SCALE_BITS - 1)) + + upscaled_plane_w / 2) / + upscaled_plane_w + + (1 << (SUPERRES_EXTRA_BITS - 1)) - err / 2; + *initial_subpel_x &= SUPERRES_SCALE_MASK; +} + +static void set_header(struct avd_ctx *ctx, struct avd_av1_run *run) +{ + struct avd_av1_ctx *av1_ctx = ctx->priv; + const struct v4l2_ctrl_av1_frame *frame = run->frame; + const struct v4l2_ctrl_av1_sequence *seq = run->seq; + const struct v4l2_av1_loop_restoration *lr = &frame->loop_restoration; + const struct v4l2_av1_cdef *cdef = &frame->cdef; + const struct v4l2_av1_segmentation *seg = &frame->segmentation; + u32 bytesperline; + int i, segid, segval, ref_idx; + int step_x, initial_subpel_x; + u8 selected_refs[3] = { 0, 0, 0 }; + bool coded_lossless = frame->tx_mode == V4L2_AV1_TX_MODE_ONLY_4X4; + bool intrabc = !!(frame->flags & V4L2_AV1_FRAME_FLAG_ALLOW_INTRABC); + bool intra_only = !!((frame->frame_type == V4L2_AV1_KEY_FRAME || + frame->frame_type == V4L2_AV1_INTRA_ONLY_FRAME)); + bool cdef_enabled = + !(!(seq->flags & V4L2_AV1_SEQUENCE_FLAG_ENABLE_CDEF) || + frame->flags & V4L2_AV1_FRAME_FLAG_ALLOW_INTRABC || + coded_lossless); + + struct avd_decoded_buffer *dst, *ref; + dma_addr_t ref_addr; + dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); + + push(AVD_OP_HDR | AVD_OP_HDR_FLAG_DECOMP(ctx->decomp) | + AVD_OP_HDR_FLAG_INTRA(intra_only && !intrabc) | + AVD_OP_HDR_CONST | AVD_OP_HDR_FLAG_PIPE_STATE(1), + "op_hdr"); + + push(AVD_HDR_CODEC_MODE(AVD_CODEC_AV1) | + AV1_CODEC_MODE_INTRABC(intrabc), + "codec"); + + /* + * TODO: height_width_* are kinda weird + * on most samples this is true + */ + push(AVD_HDR_HEIGHT(frame->frame_height_minus_1) | + AVD_HDR_WIDTH(frame->frame_width_minus_1), + "height_width_1"); + push(0, ""); + push(AVD_HDR_HEIGHT(frame->frame_height_minus_1) | + AVD_HDR_WIDTH(frame->frame_width_minus_1), + "height_width_2"); + + push(AVD_HDR_COMMON_CHROMA_FORMAT(1) | + AVD_HDR_COMMON_BIT_DEPTH_C(seq->bit_depth - 8) | + AVD_HDR_COMMON_BIT_DEPTH_L(seq->bit_depth - 8) | + AV1_HDR_JNT_COMP(seq->flags & + V4L2_AV1_SEQUENCE_FLAG_ENABLE_JNT_COMP) | + AV1_HDR_DUAL_FILTER( + seq->flags & + V4L2_AV1_SEQUENCE_FLAG_ENABLE_DUAL_FILTER) | + AV1_HDR_ORDER_HINT_BITS((seq->order_hint_bits - 1)) | + AV1_HDR_ORDER_HINT( + seq->flags & + V4L2_AV1_SEQUENCE_FLAG_ENABLE_ORDER_HINT) | + AV1_HDR_MASKED_COMPOUND( + seq->flags & + V4L2_AV1_SEQUENCE_FLAG_ENABLE_MASKED_COMPOUND) | + AV1_HDR_INTERINTRA_COMPOUND( + seq->flags & + V4L2_AV1_SEQUENCE_FLAG_ENABLE_INTERINTRA_COMPOUND) | + AV1_HDR_INTRA_EDGE_FILTER( + seq->flags & + V4L2_AV1_SEQUENCE_FLAG_ENABLE_INTRA_EDGE_FILTER) | + AV1_HDR_FILTER_INTRA( + seq->flags & + V4L2_AV1_SEQUENCE_FLAG_ENABLE_FILTER_INTRA) | + AV1_HDR_128X128_SUPERBLOCK( + seq->flags & + V4L2_AV1_SEQUENCE_FLAG_USE_128X128_SUPERBLOCK) | + AV1_HDR_SCREEN_CONTENT_TOOLS( + frame->flags & + V4L2_AV1_FRAME_FLAG_ALLOW_SCREEN_CONTENT_TOOLS), + "hdr_common"); + + push(AV1_FLAGS_TX_MODE_LARGEST(frame->tx_mode == + V4L2_AV1_TX_MODE_LARGEST) | + AV1_FLAGS_TX_MODE_SELECT(frame->tx_mode == + V4L2_AV1_TX_MODE_SELECT) | + AV1_FLAGS_TX_REDUCED_SET( + frame->flags & + V4L2_AV1_FRAME_FLAG_REDUCED_TX_SET) | + AV1_FLAGS_ALLOW_WARPED_MOTION( + frame->flags & + V4L2_AV1_FRAME_FLAG_ALLOW_WARPED_MOTION) | + AV1_FLAGS_SKIP_MODE( + frame->flags & + V4L2_AV1_FRAME_FLAG_SKIP_MODE_PRESENT) | + AV1_FLAGS_MOTION_IS_SWITCHABLE( + frame->flags & + V4L2_AV1_FRAME_FLAG_IS_MOTION_MODE_SWITCHABLE) | + AV1_FLAGS_INTRABC(frame->flags & + V4L2_AV1_FRAME_FLAG_ALLOW_INTRABC) | + AV1_FLAGS_INTEGER_MV( + frame->flags & + V4L2_AV1_FRAME_FLAG_FORCE_INTEGER_MV) | + AV1_FLAGS_DISABLE_CDF_UPDATE( + frame->flags & + V4L2_AV1_FRAME_FLAG_DISABLE_CDF_UPDATE) | + AV1_FLAGS_TX_MODE_ONLY_4X4(frame->tx_mode == + V4L2_AV1_TX_MODE_ONLY_4X4) | + AV1_FLAGS_HIGH_PRECISION_MV( + frame->flags & + V4L2_AV1_FRAME_FLAG_ALLOW_HIGH_PRECISION_MV) | + AV1_FLAGS_REF_FRAME_MVS( + frame->flags & + V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS) | + AV1_FLAGS_SEG_TEMPORAL_UPDATE( + frame->segmentation.flags & + V4L2_AV1_SEGMENTATION_FLAG_TEMPORAL_UPDATE) | + AV1_FLAGS_SEG_UPDATE_MAP( + frame->segmentation.flags & + V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP) | + AV1_FLAGS_SEG_ENABLED(frame->segmentation.flags & + V4L2_AV1_SEGMENTATION_FLAG_ENABLED) | + /* im not sure this is correct */ + AV1_FLAGS_SEG_UPDATE_DATA( + frame->segmentation.flags & + V4L2_AV1_SEGMENTATION_FLAG_UPDATE_DATA) | + AV1_FLAGS_SKIP_MODE1( + frame->flags & V4L2_AV1_FRAME_FLAG_SKIP_MODE_ALLOWED ? + frame->skip_mode_frame[1] - 1 : + 7) | + AV1_FLAGS_SKIP_MODE0( + frame->flags & V4L2_AV1_FRAME_FLAG_SKIP_MODE_ALLOWED ? + frame->skip_mode_frame[0] - 1 : + 7) | + AV1_FLAGS_ORDER_HINT(frame->order_hint) | + AV1_FLAGS_INTRA(intra_only), + "av1_flags0"); + + set_ref_hdr(ctx, run); + + for (segid = 0; segid < V4L2_AV1_MAX_SEGMENTS; segid++) { +#define SEG_FEAT_EN(feat) \ + (seg->feature_enabled[segid] & V4L2_AV1_SEGMENT_FEATURE_ENABLED(feat)) + /* what? why? */ + segval = AV1_SEG_UNK0(frame->tx_mode == + V4L2_AV1_TX_MODE_ONLY_4X4); + + segval |= AV1_SEG_ALT_Q( + seg->feature_data[segid][V4L2_AV1_SEG_LVL_ALT_Q]); + segval |= AV1_SEG_ALT_Q_EN(SEG_FEAT_EN(V4L2_AV1_SEG_LVL_ALT_Q)); + segval |= AV1_SEG_REF_FRAME( + seg->feature_data[segid][V4L2_AV1_SEG_LVL_REF_FRAME]); + segval |= AV1_SEG_REF_FRAME_EN( + SEG_FEAT_EN(V4L2_AV1_SEG_LVL_REF_FRAME)); + segval |= AV1_SEG_REF_SKIP_EN( + SEG_FEAT_EN(V4L2_AV1_SEG_LVL_REF_SKIP)); + segval |= AV1_SEG_REF_GMV_EN( + SEG_FEAT_EN(V4L2_AV1_SEG_LVL_REF_GLOBALMV)); + push(segval, "seg_0"); + segval = 0; + segval |= AV1_SEG_LF_V( + seg->feature_data[segid][V4L2_AV1_SEG_LVL_ALT_LF_V]); + segval |= + AV1_SEG_LF_V_EN(SEG_FEAT_EN(V4L2_AV1_SEG_LVL_ALT_LF_V)); + segval |= AV1_SEG_LF_U( + seg->feature_data[segid][V4L2_AV1_SEG_LVL_ALT_LF_U]); + segval |= + AV1_SEG_LF_U_EN(SEG_FEAT_EN(V4L2_AV1_SEG_LVL_ALT_LF_U)); + segval |= AV1_SEG_LF_Y_H( + seg->feature_data[segid][V4L2_AV1_SEG_LVL_ALT_LF_Y_H]); + segval |= AV1_SEG_LF_Y_H_EN( + SEG_FEAT_EN(V4L2_AV1_SEG_LVL_ALT_LF_Y_H)); + segval |= AV1_SEG_LF_Y_V( + seg->feature_data[segid][V4L2_AV1_SEG_LVL_ALT_LF_Y_V]); + segval |= AV1_SEG_LF_Y_V_EN( + SEG_FEAT_EN(V4L2_AV1_SEG_LVL_ALT_LF_Y_V)); + push(segval, "seg_1"); +#undef SEG_FEAT_EN + } + + push(AVD_HDR_FEAT_VP9, "feat_en"); + + avd_select_refs(ctx, run, selected_refs); + set_ref_hints(ctx, run, selected_refs); + + push(0, ""); + push(0, ""); + + pusha(run->addresses.probs_out, "probs_out", 0); + pusha(av1_ctx->bufs.probs.addr, "probs", 1); + pusha(av1_ctx->bufs.above_info.addr, "col", 0); + pusha(run->addresses.seg, "seg", 0); + pusha(run->addresses.prev_seg, "prev_seg", 1); + pusha(run->addresses.color, "w_color", 0); + + for (i = 0; i < 3; i++) { + if (selected_refs[i] < V4L2_AV1_REF_LAST_FRAME) { + pusha((dma_addr_t)0, "", 0); + } else { + ref_idx = frame->ref_frame_idx[selected_refs[i] - + V4L2_AV1_REF_LAST_FRAME]; + ref = avd_get_ref_buf( + ctx, &dst->base.vb, + frame->reference_frame_ts[ref_idx]); + ref_addr = + vb2_dma_contig_plane_dma_addr( + &ref->base.vb.vb2_buf, 0) + + AVD_AV1_COLOR_OFFSET( + ref->base.vb.vb2_buf.planes[0].length, + ref->av1.color_size); + + pusha(ref_addr, "r_color", ref_idx); + } + } + + push(AV1_QP_PRESENT(frame->quantization.flags & + V4L2_AV1_QUANTIZATION_FLAG_DELTA_Q_PRESENT) | + AV1_QP_DELTA_RES(frame->quantization.delta_q_res) | + AV1_QP_BASE_IDX(frame->quantization.base_q_idx) | + AV1_QP_Y_DC(frame->quantization.delta_q_y_dc) | + AV1_QP_U_DC(frame->quantization.delta_q_u_dc) | + AV1_QP_U_AC(frame->quantization.delta_q_u_ac), + "qp_base_q_idx"); + + push(AV1_QP_V_DC(frame->quantization.delta_q_v_dc) | + AV1_QP_V_AC(frame->quantization.delta_q_v_ac) | + AV1_QP_QMATRIX(frame->quantization.flags & + V4L2_AV1_QUANTIZATION_FLAG_USING_QMATRIX) | + AV1_QP_QM_Y(frame->quantization.qm_y) | + AV1_QP_QM_U(frame->quantization.qm_u) | + AV1_QP_QM_V(frame->quantization.qm_v), + "qp_qm"); + + push(AV1_LF_DELTA_ENABLED(frame->loop_filter.flags & + V4L2_AV1_LOOP_FILTER_FLAG_DELTA_ENABLED) | + AV1_LF_DELTA_PRESENT( + frame->loop_filter.flags & + V4L2_AV1_LOOP_FILTER_FLAG_DELTA_LF_PRESENT) | + AV1_LF_DELTA_MULTI( + frame->loop_filter.flags & + V4L2_AV1_LOOP_FILTER_FLAG_DELTA_LF_MULTI) | + AV1_LF_DELTA_RES(frame->loop_filter.delta_lf_res) | + AV1_LF_SHARPNESS(frame->loop_filter.sharpness) | + AV1_LF_LV0(frame->loop_filter.level[0]) | + AV1_LF_LV1(frame->loop_filter.level[1]) | + AV1_LF_LV2(frame->loop_filter.level[2]) | + AV1_LF_LV3(frame->loop_filter.level[3]), + "lf_update"); + + push(AV1_LF_REF0(frame->loop_filter.ref_deltas[0]) | + AV1_LF_REF1(frame->loop_filter.ref_deltas[1]) | + AV1_LF_REF2(frame->loop_filter.ref_deltas[2]) | + AV1_LF_REF3(frame->loop_filter.ref_deltas[3]), + "lf_sharp_ref_deltas"); + + push(AV1_LF_REF0(frame->loop_filter.ref_deltas[4]) | + AV1_LF_REF1(frame->loop_filter.ref_deltas[5]) | + AV1_LF_REF2(frame->loop_filter.ref_deltas[6]) | + AV1_LF_REF3(frame->loop_filter.ref_deltas[7]), + "lf_ref_deltas"); + + push(AV1_LF_MODE0(frame->loop_filter.mode_deltas[0]) | + AV1_LF_MODE1(frame->loop_filter.mode_deltas[1]), + "lf_mode"); + +#define AVD_AV1_SEC_FIXUP(i) ((i) == 4 ? 3 : i) + +#define AV1_CDEF_PACK_ONE(i) \ + (AV1_CDEF_Y_PRI(cdef->y_pri_strength[i]) | \ + AV1_CDEF_Y_SEC(AVD_AV1_SEC_FIXUP(cdef->y_sec_strength[i])) | \ + AV1_CDEF_UV_PRI(cdef->uv_pri_strength[i]) | \ + AV1_CDEF_UV_SEC(AVD_AV1_SEC_FIXUP(cdef->uv_sec_strength[i]))) + +#define AV1_CDEF_PACK(i) \ + AV1_CDEF_HI(AV1_CDEF_PACK_ONE(((i) * 2))) | \ + AV1_CDEF_LO(AV1_CDEF_PACK_ONE(((i) * 2) + 1)) + push(AV1_CDEF_EN(cdef_enabled) | AV1_CDEF_BITS(frame->cdef.bits) | + AV1_CDEF_DAMPING(frame->cdef.damping_minus_3) | + AV1_CDEF_PACK(0), + "cdef0"); + push(AV1_CDEF_PACK(1), "cdef1"); + push(AV1_CDEF_PACK(2), "cdef2"); + push(AV1_CDEF_PACK(3), "cdef3"); +#undef AV1_CDEF_PACK_ONE +#undef AV1_CDEF_PACK + + push(AV1_SUPERRES_USE(frame->flags & V4L2_AV1_FRAME_FLAG_USE_SUPERRES) | + AV1_SUPERRES_DENOM( + frame->flags & V4L2_AV1_FRAME_FLAG_USE_SUPERRES ? + (frame->superres_denom - 1) : + 0) | + AV1_SUPERRES_UPSCALE(frame->upscaled_width - 1), + "superres"); + + calc_upscale(frame->frame_width_minus_1 + 1, frame->upscaled_width, 0, + &step_x, &initial_subpel_x); + push(AV1_UPSCALE_UPSCX(initial_subpel_x) | AV1_UPSCALE_STEPX(step_x), + "upscale_p0"); + calc_upscale(frame->frame_width_minus_1 + 1, frame->upscaled_width, + !!(seq->flags & V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_X), + &step_x, &initial_subpel_x); + push(AV1_UPSCALE_UPSCX(initial_subpel_x) | AV1_UPSCALE_STEPX(step_x), + "upscale_p1"); + + u8 restoration_unit_size[V4L2_AV1_NUM_PLANES_MAX] = { 3, 3, 3 }; + + if (lr->flags & V4L2_AV1_LOOP_RESTORATION_FLAG_USES_LR) { + restoration_unit_size[0] = 1 + lr->lr_unit_shift; + restoration_unit_size[1] = + 1 + lr->lr_unit_shift - lr->lr_uv_shift; + restoration_unit_size[2] = + 1 + lr->lr_unit_shift - lr->lr_uv_shift; + } + + push(AV1_LR_TYPE0(frame->loop_restoration.frame_restoration_type[0]) | + AV1_LR_TYPE1( + frame->loop_restoration.frame_restoration_type[1]) | + AV1_LR_TYPE2( + frame->loop_restoration.frame_restoration_type[2]) | + AV1_LR_UNIT0(restoration_unit_size[0]) | + AV1_LR_UNIT1(restoration_unit_size[1]) | + AV1_LR_UNIT2(restoration_unit_size[2]), + "lr"); + + push(0, ""); + push(0, ""); + pusha(av1_ctx->bufs.pipe_state.addr, "pipe_state", 0); + pusha(av1_ctx->bufs.ip_above.addr, "ip_above", 0); + pusha(av1_ctx->bufs.lf_above.addr, "lf_above", 2); + pusha(av1_ctx->bufs.lf_above_info.addr, "lf_above_info", 3); + pusha(av1_ctx->bufs.lf_left.addr, "lf_left", 0); + pusha(av1_ctx->bufs.lf_left_info.addr, "lf_left_info", 1); + /* make it obvious if its used */ + pusha((dma_addr_t)0xa0a0a0a0a0a0, "", 0); + pusha(av1_ctx->bufs.rf_above_info.addr, "rf_above_info", 0); + pusha(av1_ctx->bufs.sr_left.addr, "sr_left", 2); + pusha(av1_ctx->bufs.rf_left_info.addr, "rf_left_info", 1); + pusha(av1_ctx->bufs.rf_left.addr, "rf_left", 3); + + push(0, "mark_section"); + + push_comp(ctx, run->base.comp_out, ctx->comp.offsets); + + pusha((u64)0, "packed_fmt_scratch", 0); + + /* ignored if decomp is disabled */ + bytesperline = ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline; + pusha(run->base.y_out, "y", 0); + push(bytesperline, "bytesperline_y"); + pusha(run->base.uv_out, "uv", 0); + push(bytesperline, "bytesperline_uv"); + + push(0, "mark_section"); + + push(AVD_HDR_HEIGHT(frame->render_height_minus_1 + 1) | + AVD_HDR_WIDTH(frame->render_width_minus_1 + 1), + "height_width_3"); + if (!intra_only || intrabc) + set_refs(ctx, run); +} + +static void set_tiles(struct avd_ctx *ctx, struct avd_av1_run *run) +{ + struct avd_av1_ctx *av1_ctx = ctx->priv; + const struct v4l2_ctrl_av1_frame *frame = run->frame; + const struct v4l2_ctrl_av1_sequence *seq = run->seq; + const struct v4l2_ctrl_av1_tile_group_entry *tile_group; + const struct v4l2_av1_tile_info *tile_info = &frame->tile_info; + int row, col, sb_row, sb_col, tile_id; + int sb_shift = + seq->flags & V4L2_AV1_SEQUENCE_FLAG_USE_128X128_SUPERBLOCK ? 5 : + 4; + + av1_ctx->submit_num = tile_info->tile_cols * tile_info->tile_rows; + + for (row = 0; row < tile_info->tile_rows; row++) { + for (col = 0; col < tile_info->tile_cols; col++) { + ctx->job.num++; + tile_id = row * tile_info->tile_cols + col; + tile_group = &run->tile_group[tile_id]; + + push(AVD_OP_CODED_DATA | + AVD_OP_CODED_DATA_ADDR( + (run->base.coded_in + + tile_group->tile_offset) >> + 32), + "tile_start"); + push((u32)((run->base.coded_in + + tile_group->tile_offset) & + 0xffffffff), + "tile_addr"); + push(tile_group->tile_size, "tile_size"); + + sb_row = + tile_info->mi_row_starts[tile_group->tile_row] >> + sb_shift; + sb_col = + tile_info->mi_col_starts[tile_group->tile_col] >> + sb_shift; + push(AVD_OP_SL_DIM_START | + AVD_OP_SL_DIM_START_Y(sb_row) | + AVD_OP_SL_DIM_START_X(sb_col), + "tile_op_start"); + + push(AVD_SL_DIM_END_ROW( + !(frame->flags & + V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF) && + tile_id == + tile_info->context_update_tile_id ? + 8 : + 0) | + AVD_SL_DIM_END_COL(col) | + AVD_SL_DIM_END_Y( + sb_row + + tile_info->height_in_sbs_minus_1 + [tile_group->tile_row]) | + AVD_SL_DIM_END_X( + sb_col + + tile_info->width_in_sbs_minus_1 + [tile_group->tile_col]), + "tile_op_end"); +#ifdef DEBUG_INST + pr_info("\n"); +#endif + } + } +} + + +/* End of packing oracle, aurora-wip fbc4e7934ca7325f3483b72a51d24c74aa8536a4. */ diff --git a/drivers/media/platform/apple/avd/tests/host-driver.h b/drivers/media/platform/apple/avd/tests/host-driver.h new file mode 100644 index 0000000000000..6548a9fb8124e --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/host-driver.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#define AVD_ALIGN 256 +#define AVD_MAX_INST 512 +#define AVD_QUIRK_LSR BIT(0) +enum avd_codec {AVD_CODEC_HEVC,AVD_CODEC_H264,AVD_CODEC_VP9,AVD_CODEC_AV1}; +enum avd_image_fmt {AVD_IMG_FMT_ANY,AVD_IMG_FMT_420_8BIT,AVD_IMG_FMT_420_10BIT,AVD_IMG_FMT_422_8BIT,AVD_IMG_FMT_422_10BIT}; +enum vb2_buffer_state {VB2_BUF_STATE_DONE, VB2_BUF_STATE_ERROR}; +struct vb2_buffer {unsigned int index; u64 timestamp; struct {size_t length;} planes[1];dma_addr_t dma;size_t payload;}; +struct vb2_v4l2_buffer {struct vb2_buffer vb2_buf;}; +struct avd_comp {u32 size;size_t start_offset;u32 offsets[4];}; +struct avd_decoded_buffer {struct {struct vb2_v4l2_buffer vb;} base;struct avd_comp comp;struct {u16 width,height,upscaled_width;u8 bit_depth;enum v4l2_av1_frame_type frame_type;u32 order_hints[8];u8 ref_frame_idx[7];bool intrabc;size_t color_size,seg_size;} av1;}; +struct avd_run {struct {struct vb2_v4l2_buffer *src,*dst;} bufs;dma_addr_t coded_in,y_out,uv_out,comp_out;}; +struct avd_buf {void *cpu;dma_addr_t addr;size_t size;}; +struct avd_segment {size_t num;u32 instructions[AVD_MAX_INST];}; +struct avd_job {size_t num;struct avd_segment *segments;}; +struct v4l2_pix_format_mplane {u32 width,height;struct {u32 bytesperline,sizeimage;} plane_fmt[1];}; +struct v4l2_ctrl {u32 id;unsigned int elems;union {void *p;} p_cur;union {struct v4l2_ctrl_av1_sequence *p_av1_sequence;} p_new;}; +struct avd_variant {unsigned int quirks,revision;enum avd_av1_fp_layout av1_fp_layout;}; +struct avd_dev {struct avd_variant *variant;}; +struct avd_ctx {struct avd_dev *dev;void *priv;struct {struct {struct v4l2_pix_format_mplane pix_mp;} fmt;} decoded_fmt;struct avd_comp comp;bool decomp;struct v4l2_ctrl ctrl_hdl[4];struct avd_job job;}; +struct avd_coded_fmt_ops {void (*adjust_decoded_fmt)(struct avd_ctx *,struct v4l2_pix_format_mplane *);int (*start)(struct avd_ctx *);void (*stop)(struct avd_ctx *);int (*try_ctrl)(struct avd_ctx *,struct v4l2_ctrl *);enum avd_image_fmt (*get_image_fmt)(struct avd_ctx *,struct v4l2_ctrl *);}; +static int allocations, fail_after=-1; +static dma_addr_t next_dma=0x10000000; +static int avd_buf_alloc(struct avd_dev *d,struct avd_buf *b,size_t n) {if(fail_after==0)return -ENOMEM;if(fail_after>0)fail_after--;b->cpu=calloc(1,n);if(!b->cpu)return -ENOMEM;b->size=n;b->addr=next_dma;next_dma+=ALIGN(n,256);allocations++;return 0;} +static void avd_buf_free(struct avd_dev *d,struct avd_buf *b) {if(b->cpu){free(b->cpu);allocations--;}memset(b,0,sizeof(*b));} +static struct avd_decoded_buffer buffers[64]; +static struct avd_decoded_buffer *vb2_to_avd_decoded_buf(struct vb2_buffer *b) {return container_of(b,struct avd_decoded_buffer,base.vb.vb2_buf);} +static struct avd_decoded_buffer *avd_get_ref_buf(struct avd_ctx *c,struct vb2_v4l2_buffer *dst,u64 ts) {for(int i=0;i<64;i++)if(buffers[i].base.vb.vb2_buf.timestamp==ts)return &buffers[i];return vb2_to_avd_decoded_buf(&dst->vb2_buf);} +static dma_addr_t vb2_dma_contig_plane_dma_addr(struct vb2_buffer *b,int n) {return b->dma;} +static size_t vb2_get_plane_payload(struct vb2_buffer *b,int n) {return b->payload;} +static struct v4l2_ctrl *v4l2_ctrl_find(void *h,u32 id) {struct v4l2_ctrl *p=h;for(int i=0;i<4;i++)if(p[i].id==id)return p+i;return NULL;} +struct avd_av1_cdfs {u8 data[16384];}; +static void avd_av1_default_coeff_probs(u32 q,void *p) {memset(p,q,sizeof(struct avd_av1_cdfs));} +static void avd_av1_set_default_cdfs(void *p) {} diff --git a/drivers/media/platform/apple/avd/tests/host-shim.h b/drivers/media/platform/apple/avd/tests/host-shim.h new file mode 100644 index 0000000000000..b4bede6afad91 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/host-shim.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Minimal CPU/DMA test doubles; no device IO or Linux kernel build required. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8, __u8; +typedef int8_t s8, __s8; +typedef uint16_t u16, __u16; +typedef int16_t s16, __s16; +typedef uint32_t u32, __u32, __le32; +typedef int32_t s32, __s32; +typedef uint64_t u64, __u64, dma_addr_t; +typedef int64_t s64, __s64; +typedef int irqreturn_t; +#define BIT(x) (1UL << (x)) +#define GENMASK(h,l) ((UINT32_MAX << (l)) & (UINT32_MAX >> (31-(h)))) +#define FIELD_PREP(m,v) (((u32)(v) << __builtin_ctz(m)) & (m)) +#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) +#define ALIGN(v,a) (((v)+(a)-1)&~((typeof(v))(a)-1)) +#define ALIGN_DOWN(v,a) ((v)&~((typeof(v))(a)-1)) +#define DIV_ROUND_UP(v,d) (((v)+(d)-1)/(d)) +#define clamp_val(v,l,h) ((v)<(l)?(l):((v)>(h)?(h):(v))) +#define S16_MIN INT16_MIN +#define S16_MAX INT16_MAX +#define lower_32_bits(x) ((u32)(x)) +#define upper_32_bits(x) ((u32)((u64)(x)>>32)) +#define cpu_to_le32(x) (x) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define GFP_KERNEL 0 +#define WARN_ON(x) (!!(x)) +#define kzalloc(n,f) calloc(1,n) +#define kfree(p) free(p) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +static u16 get_unaligned_le16(const void *p) { const u8 *b=p; return b[0]|b[1]<<8; } +static u32 get_unaligned_le32(const void *p) { const u8 *b=p; return b[0]|b[1]<<8|b[2]<<16|(u32)b[3]<<24; } +static void put_unaligned_le16(u16 x, void *p) { u8 *b=p; b[0]=x; b[1]=x>>8; } +static void put_unaligned_le32(u32 x, void *p) { u8 *b=p; for(int i=0;i<4;i++) b[i]=x>>(i*8); } +typedef struct { unsigned int n; } refcount_t; +static void refcount_set(refcount_t *r,unsigned int n) {r->n=n;} +static void refcount_inc(refcount_t *r) {assert(r->n);r->n++;} +static bool refcount_dec_and_test(refcount_t *r) {assert(r->n);return !--r->n;} +static int test_and_set_bit(unsigned int n,unsigned long *p) {int b=!!(*p&BIT(n));*p|=BIT(n);return b;} +static void clear_bit(unsigned int n,unsigned long *p) {*p&=~BIT(n);} +struct list_head {struct list_head *next,*prev;}; +#define INIT_LIST_HEAD(h) ((h)->next=(h)->prev=(h)) +static void list_add_tail(struct list_head *n,struct list_head *h) {n->prev=h->prev;n->next=h;h->prev->next=n;h->prev=n;} +static void list_del(struct list_head *n) {n->prev->next=n->next;n->next->prev=n->prev;} +#define list_entry(p,t,m) container_of(p,t,m) +#define list_for_each_entry(p,h,m) for(p=list_entry((h)->next,typeof(*p),m); &p->m!=(h);p=list_entry(p->m.next,typeof(*p),m)) +#define list_for_each_entry_safe(p,q,h,m) for(p=list_entry((h)->next,typeof(*p),m),q=list_entry(p->m.next,typeof(*p),m);&p->m!=(h);p=q,q=list_entry(q->m.next,typeof(*q),m)) +#define V4L2_CID_CODEC_STATELESS_BASE 0 diff --git a/drivers/media/platform/apple/avd/tests/host-test.c b/drivers/media/platform/apple/avd/tests/host-test.c new file mode 100644 index 0000000000000..816e46acab54c --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/host-test.c @@ -0,0 +1,243 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +static struct v4l2_ctrl_av1_sequence sequence; +static struct v4l2_ctrl_av1_frame params; +static struct v4l2_ctrl_av1_tile_group_entry tiles[512]; +static struct v4l2_ctrl_av1_film_grain grain; +static struct avd_variant variant = {.revision=4}; +static struct avd_dev device = {.variant=&variant}; + +static void init_ctx(struct avd_ctx *ctx) +{ + memset(ctx,0,sizeof(*ctx)); + ctx->dev=&device; + ctx->decoded_fmt.fmt.pix_mp.width=256; + ctx->decoded_fmt.fmt.pix_mp.height=256; + ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=256; + ctx->comp.start_offset=65536; + ctx->comp.offsets[0]=256; + ctx->comp.offsets[1]=512; + ctx->comp.offsets[2]=768; + ctx->comp.offsets[3]=1024; + ctx->decomp=true; + ctx->ctrl_hdl[0]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_SEQUENCE,.p_cur.p=&sequence}; + ctx->ctrl_hdl[1]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_FRAME,.p_cur.p=¶ms}; + ctx->ctrl_hdl[2]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY,.p_cur.p=tiles,.elems=1}; + ctx->ctrl_hdl[3]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_FILM_GRAIN,.p_cur.p=&grain}; + assert(avd_av1_start(ctx)==0); +} + +static void init_params(void) +{ + memset(&sequence,0,sizeof(sequence)); + memset(¶ms,0,sizeof(params)); + memset(&grain,0,sizeof(grain)); + memset(tiles,0,sizeof(tiles)); + sequence.bit_depth=8; + sequence.order_hint_bits=8; + sequence.flags=V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_X|V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_Y; + params.frame_type=V4L2_AV1_KEY_FRAME; + params.primary_ref_frame=7; + params.frame_width_minus_1=127; + params.frame_height_minus_1=127; + params.upscaled_width=128; + params.superres_denom=8; + params.tx_mode=V4L2_AV1_TX_MODE_LARGEST; + params.tile_info.tile_cols=1; + params.tile_info.tile_rows=1; + params.tile_info.width_in_sbs_minus_1[0]=1; + params.tile_info.height_in_sbs_minus_1[0]=1; + tiles[0].tile_size=128; + for(int i=0;i<64;i++) { + buffers[i].base.vb.vb2_buf=(struct vb2_buffer){.index=i,.timestamp=1000+i,.planes={{.length=1048576}},.dma=0x200000000ULL+i*0x200000,.payload=4096}; + buffers[i].av1.width=127; + buffers[i].av1.height=127; + buffers[i].av1.upscaled_width=128; + } +} + +static struct avd_frame make_frame(unsigned int idx,unsigned int dst,struct avd_buf *prog) +{ + return (struct avd_frame){.idx=idx,.seq=dst+1,.frame_num=dst+1, + .src=&buffers[63].base.vb,.dst=&buffers[dst].base.vb,.prog=prog}; +} + +static void dependency_table(void) +{ + for(int uses=0;uses<2;uses++)for(int prev=0;prev<2;prev++) + for(int update=0;update<2;update++)for(int same=0;same<2;same++) + assert(avd_av1_cdf_dep(uses,123,prev,same?123:456,update)== + (uses&&prev&&update&&same)); +} + +static void sixteen_slots(void) +{ + struct avd_ctx ctx; + struct avd_frame f[17]; + struct avd_buf prog[17]={0}; + u8 snapshots[16][4096]; + init_params();init_ctx(&ctx); + for(int i=0;i<17;i++) { + assert(!avd_buf_alloc(&device,&prog[i],4096)); + f[i]=make_frame(i,i,&prog[i]); + } + for(int i=0;i<16;i++) { + assert(!avd_av1_prepare(&ctx,&f[i])); + assert(f[i].dep==AVD_DEP_NONE); + assert(f[i].prog_words>100); + memcpy(snapshots[i],prog[i].cpu,4096); + struct avd_av1_run *r=f[i].codec_priv; + assert(!r->seq&&!r->frame&&!r->tile_group&&!r->grain); + for(int j=0;jbufs.above_info.cpu!=s->bufs.above_info.cpu); + assert(r->bufs.pipe_state.cpu!=s->bufs.pipe_state.cpu); + assert(r->cdf_in!=s->cdf_in&&r->cdf_out!=s->cdf_out); + assert(r->seg_in!=s->seg_in); + assert(!memcmp(snapshots[j],prog[j].cpu,4096)); + } + } + assert(avd_av1_prepare(&ctx,&f[16])==-EINVAL); + for(int i=0;i<16;i++)avd_av1_retire(&ctx,&f[i],VB2_BUF_STATE_DONE); + avd_av1_stop(&ctx); + for(int i=0;i<17;i++)avd_buf_free(&device,&prog[i]); + assert(!allocations); +} + +static void cdf_chain(void) +{ + struct avd_ctx ctx; + struct avd_frame f[3]; + struct avd_buf prog[3]={0}; + init_params();init_ctx(&ctx); + for(int i=0;i<3;i++) {assert(!avd_buf_alloc(&device,&prog[i],4096));f[i]=make_frame(i,i,&prog[i]);} + assert(!avd_av1_prepare(&ctx,&f[0])); + params.frame_type=V4L2_AV1_INTER_FRAME; + params.primary_ref_frame=0; + params.reference_frame_ts[0]=1000; + params.flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; + assert(!avd_av1_prepare(&ctx,&f[1])); + assert(f[1].dep==AVD_DEP_PREV_DONE); + struct avd_av1_run *a=f[0].codec_priv,*b=f[1].codec_priv; + assert(b->cdf_in==a->cdf_out&&b->cdf_out==a->cdf_out); + params.reference_frame_ts[0]=1001; + int saved=allocations; + assert(avd_av1_prepare(&ctx,&f[2])==-EAGAIN); + assert(f[2].dep==AVD_DEP_NONE&&saved==allocations); + avd_av1_retire(&ctx,&f[0],VB2_BUF_STATE_DONE); + assert(!avd_av1_prepare(&ctx,&f[2])); + assert(f[2].dep==AVD_DEP_NONE); + assert(((struct avd_av1_run *)f[2].codec_priv)->cdf_in==b->cdf_out); + avd_av1_retire(&ctx,&f[1],VB2_BUF_STATE_DONE); + avd_av1_retire(&ctx,&f[2],VB2_BUF_STATE_ERROR); + avd_av1_stop(&ctx); + for(int i=0;i<3;i++)avd_buf_free(&device,&prog[i]); + assert(!allocations); +} + +static void allocation_failures(void) +{ + for(int fail=0;fail<16;fail++) { + struct avd_ctx ctx;struct avd_buf prog={0}; + init_params();init_ctx(&ctx); + assert(!avd_buf_alloc(&device,&prog,4096)); + struct avd_frame f=make_frame(0,0,&prog); + fail_after=fail; + int rc=avd_av1_prepare(&ctx,&f); + fail_after=-1; + if(!rc)avd_av1_retire(&ctx,&f,VB2_BUF_STATE_ERROR); + else {assert(rc==-ENOMEM);assert(!f.codec_priv&&!f.prog_words);} + avd_av1_stop(&ctx);avd_buf_free(&device,&prog); + assert(!allocations); + } +} + +static void bounds_and_fp(void) +{ + struct avd_ctx ctx;struct avd_buf fp={0},prog={0}; + init_params();init_ctx(&ctx); + assert(!avd_av1_validate_tiles(¶ms,tiles,1,128)); + assert(avd_av1_validate_tiles(¶ms,tiles,0,128)==-EINVAL); + tiles[0].tile_offset=UINT32_MAX; + assert(avd_av1_validate_tiles(¶ms,tiles,1,128)==-EINVAL); + tiles[0].tile_offset=0; + assert(!avd_buf_alloc(&device,&fp,0x1130d0)); + assert(!avd_buf_alloc(&device,&prog,4096)); + struct avd_frame f=make_frame(0,0,&prog);f.fp=&fp; + variant.av1_fp_layout=AVD_AV1_FP_BORAGE; + assert(avd_av1_prepare(&ctx,&f)==-EOPNOTSUPP); + assert(get_unaligned_le16((u8 *)fp.cpu+0x3e4)==1); + assert(get_unaligned_le16((u8 *)fp.cpu+0x3e6)==1); + put_unaligned_le16(0x1002,(u8 *)fp.cpu+0x1a); + put_unaligned_le32(0xdeadbeef,(u8 *)fp.cpu+0x4f8); + put_unaligned_le16(3,(u8 *)f.cmd+6); + assert(avd_av1_pack_cmd(&ctx,&f)==-EOPNOTSUPP); + assert(get_unaligned_le32((u8 *)f.cmd+0x4c)==0xdeadbeef); + assert(get_unaligned_le32((u8 *)f.cmd+0xc)==0x01483118+0x1814*3); + put_unaligned_le16(0x1003,(u8 *)fp.cpu+0x1a); + assert(avd_av1_pack_cmd(&ctx,&f)==-EINVAL); + variant.av1_fp_layout=AVD_AV1_FP_T8140_TODO; + assert(avd_av1_pack_cmd(&ctx,&f)==-EOPNOTSUPP); + variant.av1_fp_layout=AVD_AV1_FP_UNKNOWN; + f.fp=NULL;grain.flags=V4L2_AV1_FILM_GRAIN_FLAG_APPLY_GRAIN; + assert(avd_av1_prepare(&ctx,&f)==-EOPNOTSUPP); + grain.flags=0;prog.size=4; + assert(avd_av1_prepare(&ctx,&f)==-ENOSPC); + avd_av1_stop(&ctx);avd_buf_free(&device,&fp);avd_buf_free(&device,&prog); + assert(!allocations); +} + +static void program_parity(void) +{ + for(int test=0;test<12;test++) { + struct avd_ctx ctx;struct avd_buf prog={0}; + init_params();init_ctx(&ctx); + if(test&1)sequence.bit_depth=10; + if(test&2) { + sequence.flags|=V4L2_AV1_SEQUENCE_FLAG_ENABLE_CDEF; + params.cdef.bits=2;params.cdef.damping_minus_3=1; + for(int i=0;i<4;i++){params.cdef.y_pri_strength[i]=i;params.cdef.uv_pri_strength[i]=3-i;} + } + if(test&4) { + params.upscaled_width=160;params.superres_denom=10; + params.loop_restoration.frame_restoration_type[0]=V4L2_AV1_FRAME_RESTORE_WIENER; + params.loop_restoration.loop_restoration_size[0]=64; + } + if(test&8) { + params.tile_info.tile_cols=2;params.tile_info.tile_rows=2; + for(int i=0;i<2;i++) { + params.tile_info.width_in_sbs_minus_1[i]=0; + params.tile_info.height_in_sbs_minus_1[i]=0; + params.tile_info.mi_col_starts[i]=i*16; + params.tile_info.mi_row_starts[i]=i*16; + } + for(int i=0;i<4;i++)tiles[i]=(struct v4l2_ctrl_av1_tile_group_entry){.tile_offset=i*128,.tile_size=128,.tile_col=i%2,.tile_row=i/2}; + ctx.ctrl_hdl[2].elems=4; + } + assert(!avd_buf_alloc(&device,&prog,8192)); + struct avd_frame f=make_frame(0,0,&prog); + assert(!avd_av1_prepare(&ctx,&f)); + struct avd_av1_run *r=f.codec_priv; + r->seq=&sequence;r->frame=¶ms;r->tile_group=tiles;r->grain=&grain; + ctx.job.segments=calloc(r->num_tiles+1,sizeof(struct avd_segment)); + legacy_set_header(&ctx,r);legacy_set_tiles(&ctx,r); + u32 expected[2048];unsigned int n=0; + expected[n++]=AVD_OP_EXEC|AVD_OP_EXEC_FLAG_START_REV4(1); + for(unsigned int i=0;i<=r->num_tiles;i++) { + struct avd_segment *seg=&ctx.job.segments[i]; + for(unsigned int j=0;jnum;j++)expected[n++]=seg->instructions[j]; + if(i)expected[n++]=AVD_OP_EXEC|AVD_OP_EXEC_FLAG_END(i==r->num_tiles); + } + assert(n==f.prog_words&&!memcmp(expected,prog.cpu,n*sizeof(u32))); + free(ctx.job.segments); + avd_av1_retire(&ctx,&f,VB2_BUF_STATE_DONE);avd_av1_stop(&ctx); + avd_buf_free(&device,&prog);assert(!allocations); + } + puts("AV1 legacy program parity: 12 synthetic control vectors: PASS"); +} + +int main(void) +{ + program_parity();dependency_table();sixteen_slots();cdf_chain();allocation_failures();bounds_and_fp(); + puts("AV1 host tests: dependency table, 16 slots, CDF inheritance, allocation failures, bounds, Apple refusal: PASS"); + return 0; +} diff --git a/drivers/media/platform/apple/avd/tests/legacy-emitter.h b/drivers/media/platform/apple/avd/tests/legacy-emitter.h new file mode 100644 index 0000000000000..ea3e36f10005d --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/legacy-emitter.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#undef push +#undef pusha +static void test_legacy_push(struct avd_ctx *ctx,u32 word) +{ + struct avd_segment *s=&ctx->job.segments[ctx->job.num]; + assert(s->numinstructions[s->num++]=word; +} +static void test_legacy_address(struct avd_ctx *ctx,dma_addr_t addr) +{ + if(ctx->dev->variant->quirks&AVD_QUIRK_LSR)test_legacy_push(ctx,addr>>8); + else {test_legacy_push(ctx,lower_32_bits(addr));test_legacy_push(ctx,upper_32_bits(addr));} +} +static void test_legacy_comp(struct avd_ctx *ctx,dma_addr_t addr,u32 offsets[4]) +{ + for(int i=0;i<4;i++) { + if(ctx->dev->variant->quirks&AVD_QUIRK_LSR)test_legacy_push(ctx,(addr+offsets[i])>>7); + else test_legacy_address(ctx,addr+offsets[i]); + } +} +#define push(inst,name) test_legacy_push(ctx,(inst)) +#define pusha(addr,name,i) test_legacy_address(ctx,(addr)) diff --git a/drivers/media/platform/apple/avd/tests/run-host-tests.py b/drivers/media/platform/apple/avd/tests/run-host-tests.py new file mode 100755 index 0000000000000..2bb0352698219 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/run-host-tests.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile production CPU code with DMA/V4L2 test doubles, then run with ASan/UBSan. +The UAPI definitions come from this checkout; git show also supports sparse trees. +""" +import os +from pathlib import Path +import re +import subprocess +import tempfile + +HERE = Path(__file__).resolve().parent +DRIVER = HERE.parent + +def without_includes(text): + return re.sub(r'^#include .*\n', '', text, flags=re.M) + +uapi = subprocess.check_output(['git', '-C', str(DRIVER), 'show', + 'HEAD:include/uapi/linux/v4l2-controls.h'], text=True) +uapi = uapi.split('/* Stateless AV1 controls */')[1].split('/* MPEG-compression')[0] +with tempfile.TemporaryDirectory(prefix='avd-av1-test-') as tmp: + out = Path(tmp) + source = (HERE / 'host-shim.h').read_text() + uapi + source += without_includes((DRIVER / 'avd-av1-fp.h').read_text()) + source += (HERE / 'host-driver.h').read_text() + source += without_includes((DRIVER / 'avd-queue.h').read_text()) + inst = without_includes((DRIVER / 'avd-inst.h').read_text()) + inst = inst[:inst.index('static inline void push(')] + '\n#endif\n' + source += inst + source += without_includes((DRIVER / 'avd-av1-fp.c').read_text()) + source += without_includes((DRIVER / 'avd-av1.c').read_text()) + # Freeze the pre-conversion arithmetic/packing as an independent oracle. + legacy = (HERE / 'avd-av1-legacy.c').read_text() + legacy = legacy[legacy.index('static const short div_lut'):legacy.index('/* End of packing oracle,')] + names = ['div_lut', 'avd_av1_dec_get_msb', 'avd_av1_dec_resolve_divisor_32', + 'avd_av1_dec_get_shear_params', 'set_refs', 'avd_av1_get_dist', + 'is_valid_ref_frame', 'avd_select_refs', 'set_ref_hints', 'set_ref_hdr', + 'calc_upscale', 'set_header', 'set_tiles'] + for name in names: + legacy = re.sub(r'\b' + name + r'\b', 'legacy_' + name, legacy) + legacy = legacy.replace('struct avd_av1_ctx *av1_ctx = ctx->priv;', '') + legacy = legacy.replace('av1_ctx->bufs.', 'run->bufs.') + legacy = legacy.replace('run->bufs.probs.addr', 'run->addresses.probs_in') + legacy = re.sub(r'av1_ctx->submit_num = .*?;', '', legacy) + legacy = legacy.replace('push_comp(ctx, ', 'test_legacy_comp(ctx, ') + source += (HERE / 'legacy-emitter.h').read_text() + legacy + source += (HERE / 'host-test.c').read_text() + (out / 'test.c').write_text(source) + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-g', '-O1', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', + '-Werror=implicit-function-declaration', str(out / 'test.c'), + '-o', str(out / 'test')], check=True) + subprocess.run([str(out / 'test')], check=True) From c23c41544415ff0835db4548acb9f2fee1f0f96e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 22:38:59 +1100 Subject: [PATCH 2/5] media: apple: order AV1 auxiliary completion state and test inheritance Publish auxiliary producer status before completion and acquire it before accepting a finished reference. Extend the host oracle to inter frames and exercise capture reuse with inherited CDF and segmentation ownership. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-av1.c | 26 +++++++---- .../platform/apple/avd/tests/host-shim.h | 2 + .../platform/apple/avd/tests/host-test.c | 45 +++++++++++++++++-- .../apple/avd/tests/run-host-tests.py | 4 +- 4 files changed, 64 insertions(+), 13 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index c988c19597bd3..757b99049df38 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -353,6 +353,13 @@ static struct avd_av1_aux *avd_av1_aux_alloc(struct avd_ctx *ctx, size_t size, return aux; } +static int avd_av1_aux_ready(struct avd_av1_aux *aux, enum avd_frame_dep dep) +{ + if (smp_load_acquire(&aux->done)) + return READ_ONCE(aux->status); + return dep == AVD_DEP_PREV_DONE ? 0 : -EAGAIN; +} + static struct avd_av1_ref *avd_av1_find_ref(struct avd_ctx *ctx, unsigned int index, u64 timestamp) { @@ -1183,6 +1190,7 @@ static int avd_av1_set_aux(struct avd_ctx *ctx, struct avd_av1_run *run) bool seg_enabled = frame->segmentation.flags & V4L2_AV1_SEGMENTATION_FLAG_ENABLED; u64 timestamp = 0; + int ret; dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); if (!defaults) { @@ -1193,7 +1201,7 @@ static int avd_av1_set_aux(struct avd_ctx *ctx, struct avd_av1_run *run) return -EINVAL; ref = avd_av1_find_ref(ctx, primary->base.vb.vb2_buf.index, timestamp); - if (!ref || READ_ONCE(ref->cdf->status)) + if (!ref) return -EINVAL; run->cdf_in = avd_av1_aux_get(ref->cdf); } else { @@ -1210,8 +1218,9 @@ static int avd_av1_set_aux(struct avd_ctx *ctx, struct avd_av1_run *run) AVD_DEP_PREV_DONE : AVD_DEP_NONE; /* PREV_DONE drains earlier frames too, but do not invent a dependency * for a different, unfinished CDF producer. The queue must retry it. */ - if (!READ_ONCE(run->cdf_in->done) && run->slot->dep == AVD_DEP_NONE) - return -EAGAIN; + ret = avd_av1_aux_ready(run->cdf_in, run->slot->dep); + if (ret) + return ret; run->cdf_written = !(frame->flags & V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF); @@ -1225,13 +1234,12 @@ static int avd_av1_set_aux(struct avd_ctx *ctx, struct avd_av1_run *run) if (ref && seg_enabled && ref->width == frame->frame_width_minus_1 && ref->height == frame->frame_height_minus_1) { - if (READ_ONCE(ref->seg->status)) - return -EINVAL; run->seg_in = avd_av1_aux_get(ref->seg); /* The two segmentation addresses let hardware read the previous * map directly. No speculative CPU copy of an unfinished map. */ - if (!READ_ONCE(run->seg_in->done) && run->slot->dep == AVD_DEP_NONE) - return -EAGAIN; + ret = avd_av1_aux_ready(run->seg_in, run->slot->dep); + if (ret) + return ret; } else { run->seg_in = avd_av1_aux_alloc(ctx, dst->av1.seg_size, 0, true); if (!run->seg_in) @@ -1619,11 +1627,11 @@ static void avd_av1_retire(struct avd_ctx *ctx, struct avd_frame *f, return; if (run->cdf_written) { WRITE_ONCE(run->cdf_out->status, status); - WRITE_ONCE(run->cdf_out->done, true); + smp_store_release(&run->cdf_out->done, true); } if (run->seg_written) { WRITE_ONCE(run->seg_out->status, status); - WRITE_ONCE(run->seg_out->done, true); + smp_store_release(&run->seg_out->done, true); } avd_av1_release_run(ctx, run); f->codec_priv = NULL; diff --git a/drivers/media/platform/apple/avd/tests/host-shim.h b/drivers/media/platform/apple/avd/tests/host-shim.h index b4bede6afad91..43713bb58e870 100644 --- a/drivers/media/platform/apple/avd/tests/host-shim.h +++ b/drivers/media/platform/apple/avd/tests/host-shim.h @@ -31,6 +31,8 @@ typedef int irqreturn_t; #define lower_32_bits(x) ((u32)(x)) #define upper_32_bits(x) ((u32)((u64)(x)>>32)) #define cpu_to_le32(x) (x) +#define smp_load_acquire(p) (*(p)) +#define smp_store_release(p,v) (*(p)=(v)) #define READ_ONCE(x) (x) #define WRITE_ONCE(x,v) ((x)=(v)) #define GFP_KERNEL 0 diff --git a/drivers/media/platform/apple/avd/tests/host-test.c b/drivers/media/platform/apple/avd/tests/host-test.c index 816e46acab54c..8a48eea771062 100644 --- a/drivers/media/platform/apple/avd/tests/host-test.c +++ b/drivers/media/platform/apple/avd/tests/host-test.c @@ -134,6 +134,38 @@ static void cdf_chain(void) assert(!allocations); } +static void reference_reuse_and_segmentation(void) +{ + struct avd_ctx ctx;struct avd_buf prog[3]={0};struct avd_frame f[3]; + init_params();init_ctx(&ctx); + for(int i=0;i<3;i++){assert(!avd_buf_alloc(&device,&prog[i],4096));f[i]=make_frame(i,i,&prog[i]);} + params.segmentation.flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED|V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP; + assert(!avd_av1_prepare(&ctx,&f[0])); + struct avd_av1_run *a=f[0].codec_priv; + struct avd_av1_aux *cdf=a->cdf_out,*seg=a->seg_out; + params.frame_type=V4L2_AV1_INTER_FRAME;params.primary_ref_frame=0; + params.reference_frame_ts[0]=1000; + params.segmentation.flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED; + params.flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; + assert(!avd_av1_prepare(&ctx,&f[1])); + struct avd_av1_run *b=f[1].codec_priv; + assert(b->cdf_out==cdf&&b->seg_out==seg); + avd_av1_retire(&ctx,&f[0],VB2_BUF_STATE_DONE); + /* Reuse the original capture: inherited state belongs to buffer 1 too. */ + params.primary_ref_frame=7;params.frame_type=V4L2_AV1_KEY_FRAME; + params.flags=0;params.segmentation.flags=0; + buffers[0].base.vb.vb2_buf.timestamp=2000; + f[2].dst=&buffers[0].base.vb; + assert(!avd_av1_prepare(&ctx,&f[2])); + assert(cdf->refs.n>=2&&seg->refs.n>=2); + assert(b->cdf_in==cdf&&b->seg_in==seg&&cdf->done&&seg->done); + avd_av1_retire(&ctx,&f[1],VB2_BUF_STATE_DONE); + avd_av1_retire(&ctx,&f[2],VB2_BUF_STATE_ERROR); + avd_av1_stop(&ctx); + for(int i=0;i<3;i++)avd_buf_free(&device,&prog[i]); + assert(!allocations); +} + static void allocation_failures(void) { for(int fail=0;fail<16;fail++) { @@ -188,10 +220,17 @@ static void bounds_and_fp(void) static void program_parity(void) { - for(int test=0;test<12;test++) { + for(int test=0;test<24;test++) { struct avd_ctx ctx;struct avd_buf prog={0}; init_params();init_ctx(&ctx); if(test&1)sequence.bit_depth=10; + if(test&16) { + params.frame_type=V4L2_AV1_INTER_FRAME; + sequence.flags|=V4L2_AV1_SEQUENCE_FLAG_ENABLE_ORDER_HINT|V4L2_AV1_SEQUENCE_FLAG_ENABLE_REF_FRAME_MVS; + params.flags|=V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS; + params.order_hint=4; + for(int i=0;i<7;i++){params.ref_frame_idx[i]=i;params.reference_frame_ts[i]=1001+i;} + } if(test&2) { sequence.flags|=V4L2_AV1_SEQUENCE_FLAG_ENABLE_CDEF; params.cdef.bits=2;params.cdef.damping_minus_3=1; @@ -232,12 +271,12 @@ static void program_parity(void) avd_av1_retire(&ctx,&f,VB2_BUF_STATE_DONE);avd_av1_stop(&ctx); avd_buf_free(&device,&prog);assert(!allocations); } - puts("AV1 legacy program parity: 12 synthetic control vectors: PASS"); + puts("AV1 legacy program parity: 24 synthetic control vectors: PASS"); } int main(void) { - program_parity();dependency_table();sixteen_slots();cdf_chain();allocation_failures();bounds_and_fp(); + program_parity();dependency_table();sixteen_slots();cdf_chain();reference_reuse_and_segmentation();allocation_failures();bounds_and_fp(); puts("AV1 host tests: dependency table, 16 slots, CDF inheritance, allocation failures, bounds, Apple refusal: PASS"); return 0; } diff --git a/drivers/media/platform/apple/avd/tests/run-host-tests.py b/drivers/media/platform/apple/avd/tests/run-host-tests.py index 2bb0352698219..c7c61f8050d75 100755 --- a/drivers/media/platform/apple/avd/tests/run-host-tests.py +++ b/drivers/media/platform/apple/avd/tests/run-host-tests.py @@ -43,11 +43,13 @@ def without_includes(text): legacy = legacy.replace('run->bufs.probs.addr', 'run->addresses.probs_in') legacy = re.sub(r'av1_ctx->submit_num = .*?;', '', legacy) legacy = legacy.replace('push_comp(ctx, ', 'test_legacy_comp(ctx, ') + source += '#pragma GCC diagnostic push\n#pragma GCC diagnostic ignored \"-Wpointer-sign\"\n' source += (HERE / 'legacy-emitter.h').read_text() + legacy + source += '#pragma GCC diagnostic pop\n' source += (HERE / 'host-test.c').read_text() (out / 'test.c').write_text(source) subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-g', '-O1', '-fsanitize=address,undefined', '-fno-omit-frame-pointer', - '-Werror=implicit-function-declaration', str(out / 'test.c'), + '-Werror', str(out / 'test.c'), '-o', str(out / 'test')], check=True) subprocess.run([str(out / 'test')], check=True) From 834614796c878e183f37b630f48523f0299baf8f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 22:39:33 +1100 Subject: [PATCH 3/5] media: apple: validate AV1 work against fixed control capacity The tile-group control carries a fixed 512-entry array. Validate that capacity covers the tile grid, and use the grid product for program work rather than treating unused array entries as tiles. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/avd-av1-fp.c | 3 ++- drivers/media/platform/apple/avd/avd-av1.c | 3 +++ drivers/media/platform/apple/avd/tests/host-test.c | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-av1-fp.c b/drivers/media/platform/apple/avd/avd-av1-fp.c index 85dca37478ac9..7861b53358a1b 100644 --- a/drivers/media/platform/apple/avd/avd-av1-fp.c +++ b/drivers/media/platform/apple/avd/avd-av1-fp.c @@ -41,7 +41,8 @@ int avd_av1_validate_tiles(const struct v4l2_ctrl_av1_frame *frame, return -EINVAL; n = info->tile_cols * info->tile_rows; if (n > V4L2_AV1_MAX_TILE_COUNT || n > BORAGE_WORK_CAP || - count != n || info->context_update_tile_id >= n) + count < n || count > V4L2_AV1_MAX_TILE_COUNT || + info->context_update_tile_id >= n) return -EINVAL; for (i = 0; i < n; i++) { if (tiles[i].tile_row != i / info->tile_cols || diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index 757b99049df38..6cd4823559b4b 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -1560,6 +1560,9 @@ static int avd_av1_prepare(struct avd_ctx *ctx, struct avd_frame *f) vb2_get_plane_payload(&f->src->vb2_buf, 0)); if (ret) goto fail; + /* The control is a fixed-capacity array; only the tile grid is work. */ + run->num_tiles = run->frame->tile_info.tile_cols * + run->frame->tile_info.tile_rows; /* The old builder has no film grain synthesis program. */ if (run->grain->flags & V4L2_AV1_FILM_GRAIN_FLAG_APPLY_GRAIN) { ret = -EOPNOTSUPP; diff --git a/drivers/media/platform/apple/avd/tests/host-test.c b/drivers/media/platform/apple/avd/tests/host-test.c index 8a48eea771062..a08956e81177c 100644 --- a/drivers/media/platform/apple/avd/tests/host-test.c +++ b/drivers/media/platform/apple/avd/tests/host-test.c @@ -21,7 +21,7 @@ static void init_ctx(struct avd_ctx *ctx) ctx->decomp=true; ctx->ctrl_hdl[0]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_SEQUENCE,.p_cur.p=&sequence}; ctx->ctrl_hdl[1]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_FRAME,.p_cur.p=¶ms}; - ctx->ctrl_hdl[2]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY,.p_cur.p=tiles,.elems=1}; + ctx->ctrl_hdl[2]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY,.p_cur.p=tiles,.elems=V4L2_AV1_MAX_TILE_COUNT}; ctx->ctrl_hdl[3]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_FILM_GRAIN,.p_cur.p=&grain}; assert(avd_av1_start(ctx)==0); } @@ -188,6 +188,7 @@ static void bounds_and_fp(void) struct avd_ctx ctx;struct avd_buf fp={0},prog={0}; init_params();init_ctx(&ctx); assert(!avd_av1_validate_tiles(¶ms,tiles,1,128)); + assert(!avd_av1_validate_tiles(¶ms,tiles,V4L2_AV1_MAX_TILE_COUNT,128)); assert(avd_av1_validate_tiles(¶ms,tiles,0,128)==-EINVAL); tiles[0].tile_offset=UINT32_MAX; assert(avd_av1_validate_tiles(¶ms,tiles,1,128)==-EINVAL); From 2e6cb0421ce693fdb9fbd04d1326cd236b38a1cd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 04:12:37 +1100 Subject: [PATCH 4/5] media: apple: integrate AV1 with the shared AVD queue Resolve the AV1 conversion against avd/rewrite-integ at 01de6ce703a1. Keep the core queue header verbatim, retain its transports and power management, and add the AV1 layout selectors to the integrated match data. Preserve the published AV1 ancestry with this ordinary integration snapshot so updating the draft needs neither a merge nor a force push. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/avd/Kconfig | 5 +- drivers/media/platform/apple/avd/Makefile | 8 +- .../platform/apple/avd/avd-apple-protocol.h | 141 +++++ .../platform/apple/avd/avd-backend-apple.c | 317 ++++++++++ .../platform/apple/avd/avd-backend-open.c | 272 ++++++++ .../platform/apple/avd/avd-backend-queue.c | 357 +++++++++++ drivers/media/platform/apple/avd/avd-drv.c | 481 +++----------- drivers/media/platform/apple/avd/avd-h264.c | 253 +++++--- drivers/media/platform/apple/avd/avd-hevc.c | 199 ++++-- drivers/media/platform/apple/avd/avd-hw.c | 25 + drivers/media/platform/apple/avd/avd-inst.h | 1 + .../media/platform/apple/avd/avd-pm-test.c | 386 ++++++++++++ drivers/media/platform/apple/avd/avd-pm.c | 450 +++++++++++++ drivers/media/platform/apple/avd/avd-pm.h | 80 +++ .../platform/apple/avd/avd-pm.kunitconfig | 14 + .../media/platform/apple/avd/avd-protocol.rst | 97 +++ .../platform/apple/avd/avd-queue-internal.h | 73 +++ .../media/platform/apple/avd/avd-queue-test.c | 204 ++++++ drivers/media/platform/apple/avd/avd-queue.c | 591 ++++++++++++++++++ drivers/media/platform/apple/avd/avd-queue.h | 5 + drivers/media/platform/apple/avd/avd-regs.h | 2 + drivers/media/platform/apple/avd/avd-v4l2.c | 38 +- drivers/media/platform/apple/avd/avd-vp9.c | 8 + drivers/media/platform/apple/avd/avd.h | 116 +++- .../platform/apple/avd/tests/.kunitconfig | 13 + .../apple/avd/tests/include/linux/errno.h | 2 + .../apple/avd/tests/include/linux/interrupt.h | 5 + .../apple/avd/tests/include/linux/jiffies.h | 6 + .../apple/avd/tests/include/linux/limits.h | 4 + .../apple/avd/tests/include/linux/types.h | 11 + .../avd/tests/include/media/videobuf2-v4l2.h | 5 + .../platform/apple/avd/tests/queue-host.c | 79 +++ .../platform/apple/avd/tests/run-host.sh | 8 + 33 files changed, 3670 insertions(+), 586 deletions(-) create mode 100644 drivers/media/platform/apple/avd/avd-apple-protocol.h create mode 100644 drivers/media/platform/apple/avd/avd-backend-apple.c create mode 100644 drivers/media/platform/apple/avd/avd-backend-open.c create mode 100644 drivers/media/platform/apple/avd/avd-backend-queue.c create mode 100644 drivers/media/platform/apple/avd/avd-pm-test.c create mode 100644 drivers/media/platform/apple/avd/avd-pm.c create mode 100644 drivers/media/platform/apple/avd/avd-pm.h create mode 100644 drivers/media/platform/apple/avd/avd-pm.kunitconfig create mode 100644 drivers/media/platform/apple/avd/avd-protocol.rst create mode 100644 drivers/media/platform/apple/avd/avd-queue-internal.h create mode 100644 drivers/media/platform/apple/avd/avd-queue-test.c create mode 100644 drivers/media/platform/apple/avd/avd-queue.c create mode 100644 drivers/media/platform/apple/avd/tests/.kunitconfig create mode 100644 drivers/media/platform/apple/avd/tests/include/linux/errno.h create mode 100644 drivers/media/platform/apple/avd/tests/include/linux/interrupt.h create mode 100644 drivers/media/platform/apple/avd/tests/include/linux/jiffies.h create mode 100644 drivers/media/platform/apple/avd/tests/include/linux/limits.h create mode 100644 drivers/media/platform/apple/avd/tests/include/linux/types.h create mode 100644 drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h create mode 100644 drivers/media/platform/apple/avd/tests/queue-host.c create mode 100755 drivers/media/platform/apple/avd/tests/run-host.sh diff --git a/drivers/media/platform/apple/avd/Kconfig b/drivers/media/platform/apple/avd/Kconfig index ed5ad44276c31..d3bb9763b6230 100644 --- a/drivers/media/platform/apple/avd/Kconfig +++ b/drivers/media/platform/apple/avd/Kconfig @@ -24,8 +24,7 @@ config VIDEO_APPLE_AVD_KUNIT_TEST default KUNIT_ALL_TESTS help Build KUnit tests for the Apple video decoder. They exercise the - checks that the H.264 path makes on the slices that userspace - queues, such as finding the slice data after the slice header, - at the limits and one past them. + H.264 slice bounds, frame-slot ownership, and runtime power management, + including suspend gating and resubmission after firmware resume. If unsure, say N. diff --git a/drivers/media/platform/apple/avd/Makefile b/drivers/media/platform/apple/avd/Makefile index 2c8bc9ca5534a..f603787990fe8 100644 --- a/drivers/media/platform/apple/avd/Makefile +++ b/drivers/media/platform/apple/avd/Makefile @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only -apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-h264.o avd-vp9.o avd-hevc.o \ - avd-av1.o avd-av1-fp.o avd-av1-entropymode.o -apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o +apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-pm.o avd-h264.o avd-vp9.o \ + avd-hevc.o avd-av1.o avd-av1-fp.o avd-av1-entropymode.o avd-queue.o avd-backend-open.o \ + avd-backend-apple.o avd-backend-queue.o +apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o avd-queue-test.o \ + avd-pm-test.o obj-$(CONFIG_VIDEO_APPLE_AVD) += apple-avd.o diff --git a/drivers/media/platform/apple/avd/avd-apple-protocol.h b/drivers/media/platform/apple/avd/avd-apple-protocol.h new file mode 100644 index 0000000000000..d0b90c5d48e4c --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-apple-protocol.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_APPLE_PROTOCOL_H_ +#define AVD_APPLE_PROTOCOL_H_ + +#include +#include + +#include "avd-queue.h" + +#define AVD_APPLE_INBOX_BASE 0x01482cf8 +#define AVD_APPLE_COMMAND_SIZE 0x60 +#define AVD_APPLE_RESULT_WORDS 20 +#define AVD_APPLE_SHADOW_SIZE 64 + +/* + * [C] Expected address notification, from the MIT V3 trace_avd.py precedent. + * Keep this rule in one place for correction by the Borage capture. + */ +#define AVD_APPLE_NOTIFY_BASE AVD_APPLE_INBOX_BASE + +/* SPEC 5.1: only these session opcodes have proved builders. */ +static inline int +avd_apple_session_command(u32 cmd[24], u32 client, u32 opcode, + u32 property, u32 value) +{ + unsigned int i; + + if (client >= 1024 || (opcode != 0 && opcode != 2 && opcode != 5 && opcode != 6)) + return -EINVAL; + for (i = 0; i < 24; i++) + cmd[i] = 0; + cmd[0] = opcode | (client << 16); + if (opcode == 6) { + cmd[1] = property; + cmd[2] = value; + } + return 0; +} + +static inline int avd_apple_decode_prefix(struct avd_frame *f, u32 client, + u32 selector, u32 work_sequence) +{ + unsigned int i; + + if (f->idx >= AVD_MAX_FRAMES || client >= 1024 || selector >= 16 || work_sequence >= 4) + return -EINVAL; + for (i = 0; i < 24; i++) + f->cmd[i] = 0; + f->cmd[0] = 1 | (selector << 10) | (client << 22); + f->cmd[1] = f->idx | (work_sequence << 16); + f->cmd[3] = f->frame_num; + return 0; +} + +struct avd_apple_reply_layout { + u8 type_word; + u8 type_shift; + u32 type_mask; + u8 client_word; + u8 client_shift; + u32 client_mask; + u8 frame_word; + u8 status_word; + u32 done_type; + bool verified; +}; + +struct avd_apple_reply { + u32 type; + u32 client; + u32 frame; + int status; +}; + +static inline int +avd_apple_parse_reply(const u32 *record, size_t words, + const struct avd_apple_reply_layout *layout, + struct avd_apple_reply *reply) +{ + if (words != AVD_APPLE_RESULT_WORDS) + return -EMSGSIZE; + if (!layout->verified) + return -EOPNOTSUPP; + if (layout->type_word >= words || layout->client_word >= words || + layout->frame_word >= words || layout->status_word >= words || + layout->type_shift >= 32 || layout->client_shift >= 32 || + !layout->type_mask || !layout->client_mask || layout->done_type == 7) + return -EINVAL; + reply->type = (record[layout->type_word] >> layout->type_shift) & layout->type_mask; + reply->client = (record[layout->client_word] >> layout->client_shift) & layout->client_mask; + reply->frame = record[layout->frame_word]; + /* SPEC 4.3: type 7 is parsing/VP done, never final buffer ownership. */ + if (reply->type == 7) + return -EINPROGRESS; + if (reply->type != layout->done_type) + return -ENOMSG; + reply->status = record[layout->status_word] ? -EIO : 0; + return 0; +} + +static inline u32 avd_apple_notification(unsigned int slot) +{ + return AVD_APPLE_NOTIFY_BASE + AVD_APPLE_COMMAND_SIZE * slot; +} + +struct avd_apple_hw { + const char *fw_name; + /* All offsets below are relative to a DT-owned "apple" resource. */ + u32 aperture_origin; + u32 imem; + u32 imem_size; + u32 version_offset; + u32 itcm_start; + u32 itcm_end; + u32 itcm_clear; + u32 itcm_clear_value; + u32 itcm_enable; + u32 itcm_enable_value; + u32 link_setup; + u32 link_setup_value; + u32 link_state; + u32 link_up; + u32 link_up_value; + u32 ready; + u32 notify; + u32 receive; + u32 receive_status; + u32 receive_pending; + u32 receive_ack; + u32 receive_ack_value; + u32 inbox_consumer; + u32 result_base; + struct avd_apple_reply_layout reply; + /* Match-data fact, never a module parameter or a userspace setting. */ + bool verified; +}; + +extern const struct avd_apple_hw avd_borage_hw; +extern const u8 avd_borage_config[64]; + +#endif /* AVD_APPLE_PROTOCOL_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-backend-apple.c b/drivers/media/platform/apple/avd/avd-backend-apple.c new file mode 100644 index 0000000000000..8345f107e4c14 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-apple.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include + +#include "avd.h" +#include "avd-apple-protocol.h" +#include "avd-queue-internal.h" + +/* SPEC 3.2. Unknown field meanings are intentionally not named. */ +const u8 avd_borage_config[64] = { + [0x0a] = 0x55, [0x0b] = 0x01, /* +0x08: 0x01558000 */ + [0x09] = 0x80, + [0x11] = 0x8a, [0x12] = 0x55, [0x13] = 0x01, + [0x19] = 0x8e, [0x1a] = 0x55, [0x1b] = 0x01, + [0x28] = 0x0a, [0x2a] = 0x04, + [0x2c] = 0x04, + [0x38] = 0x08, [0x3a] = 0x10, [0x3b] = 0x01, +}; + +/* + * Source: clean SPEC 3-5, AppleAVD 993.1 / T6040 Borage. The SPEC does + * not supply the ITCM/version register offsets, link register operations or + * reply field layout. Do not substitute V3/open-firmware offsets for them. + * avd-protocol.rst records every unresolved field and its source/capture. + */ +const struct avd_apple_hw avd_borage_hw = { + .fw_name = "apple/avd-apple-t6040.bin", + .verified = false, +}; + +struct avd_apple { + void __iomem *regs; + const struct avd_apple_hw *hw; + const struct firmware *image; + u32 shadow[AVD_APPLE_SHADOW_SIZE]; + unsigned int shadow_read; + unsigned int shadow_write; + unsigned int producer; + u32 version; +}; + +static void avd_apple_release_image(void *data) +{ + struct avd_apple *apple = data; + + release_firmware(apple->image); +} + +static int avd_apple_image_present(struct avd_apple *apple) +{ + const struct avd_apple_hw *hw = apple->hw; + u32 start = readl(apple->regs + hw->itcm_start); + u32 end = readl(apple->regs + hw->itcm_end); + u32 version; + + if (end <= start || end - start < 4 || end - start > hw->imem_size || + hw->version_offset > end - start - 4) + return -ENOENT; + version = readl(apple->regs + hw->imem + hw->version_offset); + if (!version || version == U32_MAX) + return -ENOENT; + apple->version = version; + return 0; +} + +static int avd_apple_probe(struct avd_dev *avd) +{ + const struct avd_apple_hw *hw = avd->variant->apple; + struct avd_apple *apple; + int ret; + + if (!hw || !hw->verified || !hw->reply.verified) { + dev_info(avd->dev, "Apple transport awaits Borage register/reply facts; using open firmware\n"); + return -EOPNOTSUPP; + } + apple = devm_kzalloc(avd->dev, sizeof(*apple), GFP_KERNEL); + if (!apple) + return -ENOMEM; + apple->hw = hw; + apple->regs = devm_platform_ioremap_resource_byname(avd->pdev, "apple"); + if (IS_ERR(apple->regs)) + return PTR_ERR(apple->regs); + /* Firmware bytes remain in vendorfw; this driver never embeds an image. */ + ret = firmware_request_nowarn(&apple->image, hw->fw_name, avd->dev); + if (ret && ret != -ENOENT) + return ret; + ret = devm_add_action_or_reset(avd->dev, avd_apple_release_image, apple); + if (ret) + return ret; + avd->backend_priv = apple; + return 0; +} + +static int avd_apple_load(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + const struct firmware *fw = apple->image; + size_t i, rounded; + + if (!fw) + return -ENOENT; + if (!fw->size || fw->size > hw->imem_size || + hw->imem_size < 4 || hw->version_offset > hw->imem_size - 4) + return -EINVAL; + rounded = ALIGN(fw->size, 4); + if (rounded > hw->imem_size) + return -EFBIG; + writel(hw->itcm_clear_value, apple->regs + hw->itcm_clear); + writel(0, apple->regs + hw->itcm_start); + writel(rounded, apple->regs + hw->itcm_end); + /* Bounded word copies pad the last word without reading past fw->data. */ + for (i = 0; i < rounded; i += 4) { + u8 bytes[4] = {}; + + memcpy(bytes, fw->data + i, min_t(size_t, 4, fw->size - i)); + writel(get_unaligned_le32(bytes), apple->regs + hw->imem + i); + } + writel(hw->itcm_enable_value, apple->regs + hw->itcm_enable); + return avd_apple_image_present(apple); +} + +static int avd_apple_boot(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + u32 value; + int ret; + + ret = avd_apple_image_present(apple); + if (ret) { + ret = avd_apple_load(avd); + if (ret) + return ret; + } else { + /* Suspend stopped the MCPU without clearing the retained image. */ + writel(hw->itcm_enable_value, apple->regs + hw->itcm_enable); + } + dev_info(avd->dev, "Apple MCPU firmware %u.%u.%u (%08x)\n", + apple->version >> 8, (apple->version >> 4) & 0xf, + apple->version & 0xf, apple->version); + /* SPEC 3.2: decode connection, aperture 1, state 4, then LinkUp. */ + writel(hw->link_setup_value, apple->regs + hw->link_setup); + ret = readl_poll_timeout(apple->regs + hw->link_state, value, + value == 4, 10, 10000); + if (ret) + return ret; + writel(hw->link_up_value, apple->regs + hw->link_up); + ret = readl_poll_timeout(apple->regs + hw->ready, value, + value == 1, 10, 10000); + if (!ret) { + apple->producer = 0; + apple->shadow_read = 0; + apple->shadow_write = 0; + } + return ret; +} + +static void avd_apple_shutdown(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + + /* MCPU disable value must be proved together with enable in the ledger. */ + writel(0, apple->regs + apple->hw->itcm_enable); +} + +static unsigned int avd_apple_max_inflight(struct avd_dev *avd) +{ + /* + * Every frame in VP->PP flight owns one hardware instruction FIFO + * (EXEC bits 7:4), so the FIFO count bounds the frames in flight: + * 15 on v3+ tier 1 parts, 7 on T8103 and T8140. + */ + return min_t(unsigned int, AVD_MAX_FRAMES, avd->variant->fifo_slots ?: 1); +} + +static int avd_apple_send(struct avd_dev *avd, const u32 cmd[24]) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + unsigned int consumer = readl(apple->regs + hw->inbox_consumer) & 7; + u32 address = AVD_APPLE_INBOX_BASE + apple->producer * AVD_APPLE_COMMAND_SIZE; + unsigned int i; + + if (avd_inbox_full(apple->producer, consumer)) + return -EBUSY; + for (i = 0; i < 24; i++) + writel(cmd[i], apple->regs + address - hw->aperture_origin + i * 4); + /* MMIO commands and coherent fp payload precede the four-byte doorbell. */ + wmb(); + writel(avd_apple_notification(apple->producer), apple->regs + hw->notify); + apple->producer = (apple->producer + 1) & 7; + return 0; +} + +static int avd_apple_session_start(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 0, 0, 0)) + return -EINVAL; + return avd_apple_send(ctx->dev, cmd); +} + +static void avd_apple_session_stop(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 2, 0, 0)) + return; + if (avd_apple_send(ctx->dev, cmd)) + dev_warn(ctx->dev->dev, "session stop inbox full\n"); +} + +static void avd_apple_abort(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 5, 0, 0)) + return; + /* Core follows abort with reset before returning any DMA buffer. */ + avd_apple_send(ctx->dev, cmd); +} + +static int avd_apple_set_property(struct avd_ctx *ctx, u32 property, u32 value) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 6, property, value)) + return -EINVAL; + return avd_apple_send(ctx->dev, cmd); +} + +static int avd_apple_submit(struct avd_dev *avd, struct avd_frame *f) +{ + const struct avd_codec_frame_ops *ops = f->ctx->coded_fmt_desc->frame_ops; + u32 selector; + int ret; + + if (!ops || !ops->pack_cmd || !f->fp->cpu || f->ctx->client_id >= 1024) + return -EOPNOTSUPP; + /* SPEC 5.2 prefix precedes the codec-specific builder. */ + selector = get_unaligned_le32(f->fp->cpu + 4) & 0xf; + ret = avd_apple_decode_prefix(f, f->ctx->client_id, selector, f->seq & 3); + if (ret) + return ret; + ret = ops->pack_cmd(f->ctx, f); + if (ret) + return ret; + return avd_apple_send(avd, f->cmd); +} + +static irqreturn_t avd_apple_irq(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + unsigned int budget = AVD_APPLE_SHADOW_SIZE; + + while (budget-- && (readl(apple->regs + hw->receive_status) & hw->receive_pending)) { + unsigned int next = (apple->shadow_write + 1) & 63; + + if (next == apple->shadow_read) + break; + apple->shadow[apple->shadow_write] = readl(apple->regs + hw->receive); + apple->shadow_write = next; + writel(hw->receive_ack_value, apple->regs + hw->receive_ack); + } + while (apple->shadow_read != apple->shadow_write) { + u32 record[AVD_APPLE_RESULT_WORDS]; + struct avd_apple_reply reply; + unsigned int i; + int ret; + + /* Result notification indexing must be verified with the layout. */ + u32 index = apple->shadow[apple->shadow_read]; + + apple->shadow_read = (apple->shadow_read + 1) & 63; + if (index >= AVD_APPLE_SHADOW_SIZE) + continue; + for (i = 0; i < AVD_APPLE_RESULT_WORDS; i++) + record[i] = readl(apple->regs + hw->result_base + + index * sizeof(record) + i * 4); + ret = avd_apple_parse_reply(record, ARRAY_SIZE(record), &hw->reply, &reply); + if (ret) + continue; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + + if (f->state == AVD_FRAME_SUBMITTED && f->frame_num == reply.frame && + f->ctx->client_id == reply.client) { + dma_rmb(); + avd_frame_complete(avd, f->tag, reply.status); + break; + } + } + } + mod_delayed_work(system_wq, &avd->queue_work, 0); + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_apple_backend = { + .name = "apple-mcpu", + .probe = avd_apple_probe, + .boot = avd_apple_boot, + .shutdown = avd_apple_shutdown, + .max_inflight = avd_apple_max_inflight, + .session_start = avd_apple_session_start, + .session_stop = avd_apple_session_stop, + .submit = avd_apple_submit, + .abort = avd_apple_abort, + .set_property = avd_apple_set_property, + .irq = avd_apple_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, +}; diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c new file mode 100644 index 0000000000000..9680dc4ef64e1 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Legacy transport split from avd-drv.c; Copyright The Asahi Linux Contributors. */ +#include +#include +#include +#include +#include +#include + +#include "avd.h" +#include "avd-inst.h" +#include "avd-regs.h" + +/* + * Compatibility transport for firmware without the queued-program ABI. The + * codec's CPU builder emits a per-slot DMA-visible instruction stream. Only + * this backend touches the VP FIFO. Its untagged PP interrupt limits it to one + * frame; the queued avd-fw transport can replace these ops without changing + * buffer ownership in the core. + */ +int avd_open_build_program(struct avd_ctx *ctx) +{ + struct avd_dev *avd = ctx->dev; + struct avd_frame *f = avd->building; + struct avd_job *job = &ctx->job; + size_t count = 1, pos = 0, i, j; + u32 *words, mask, start; + int ret; + + if (!f || !job->segments) + return -EINVAL; + for (i = 0; i <= job->num; i++) { + if (job->segments[i].num > AVD_MAX_INST || + check_add_overflow(count, job->segments[i].num + !!i, &count)) + return -EOVERFLOW; + } + if (count > SIZE_MAX / sizeof(u32)) + return -EOVERFLOW; + if (f->prog->size < count * sizeof(u32)) { + ret = avd_buf_alloc(avd, f->prog, count * sizeof(u32)); + if (ret) + return ret; + } + mask = job->codec == AVD_CODEC_VP9 && avd->variant->revision == 3 ? + AVD_OP_EXEC_REV3_VP9_MASK : 0; + start = AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | + AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3); + /* The queued transport gives every frame in flight its own FIFO. */ + ctx->fifo_idx = avd_backend_is_qfw(avd) ? f->idx : 0; + f->fifo_iova = ctx->job_fifo ?: ctx->inst.addr; + ctx->job_fifo = 0; + words = f->prog->cpu; + words[pos++] = AVD_OP_EXEC | mask | start | AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx); + for (i = 0; i <= job->num; i++) { + for (j = 0; j < job->segments[i].num; j++) + words[pos++] = job->segments[i].instructions[j]; + if (i) + words[pos++] = AVD_OP_EXEC | mask | AVD_OP_EXEC_FLAG_END(i == job->num); + } + f->prog_words = pos; + f->cmd[0] = job->codec; + kvfree(job->segments); + job->segments = NULL; + return 0; +} + +static unsigned int avd_open_max_inflight(struct avd_dev *avd) +{ + return 1; +} + +static int avd_open_submit(struct avd_dev *avd, struct avd_frame *f) +{ + struct avd_ctx *ctx = f->ctx; + u32 codec = f->cmd[0], vp = 0; + u32 *words = f->prog->cpu; + unsigned int i; + void __iomem *reg; + + if (avd->compat_active) + return -EBUSY; + /* Native frame_ops require firmware-side PP chaining and tagged replies. */ + if (ctx->coded_fmt_desc->frame_ops) + return -EOPNOTSUPP; + if (codec > AVD_CODEC_AV1 || !words || !f->prog_words || + f->prog_words > f->prog->size / sizeof(u32)) + return -EINVAL; + if (!(avd->capabilities & BIT(codec))) + return -EOPNOTSUPP; + for (i = 0; i < codec; i++) + vp += avd->variant->vp_slots[i]; + if (avd->variant->vp_slot_base[codec]) + vp = avd->variant->vp_slot_base[codec]; + avd->variant->configure_stream(avd, ctx->inst.addr, ctx->fifo_idx, vp); + reg = avd->ctrl + avd->variant->vp_slot_offset + vp * 4; + avd->compat_active = f; + avd->compat_ctx = ctx; + for (i = 0; i < f->prog_words; i++) { + /* Preserve the old firmware's segment FIFO pacing in this shim. */ + if (i && (words[i] & GENMASK(31, 24)) == AVD_OP_EXEC && + (!(avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) || + codec == AVD_CODEC_HEVC)) { + u32 max = readl(avd->ctrl + avd->variant->submit_queue_max_offset + vp * 4); + u32 cur = readl(avd->ctrl + + avd->variant->submit_queue_status_offset + vp * 4); + + if (cur >= max) { + /* Already emitted words: only reset may release this slot. */ + f->deadline = jiffies; + return 0; + } + if (cur >= max / 2) { + /* + * Wait for the VP to drain below half full, but + * only as long as it takes. The old fixed sleep + * (100-150 us on v5 pacing parts, 500-650 us + * elsewhere) stays the upper bound. + */ + u32 wait_us = avd->variant->quirks & + AVD_QUIRK_V5_QUEUE_PACING ? 150 : 650; + + readl_relaxed_poll_timeout(avd->ctrl + + avd->variant->submit_queue_status_offset + vp * 4, + cur, cur < max / 2, 10, wait_us); + } + } + writel(words[i], reg); + } + return 0; +} + +static irqreturn_t avd_open_irq(struct avd_dev *avd) +{ + struct avd_frame *f = avd->compat_active; + u32 status; + + if (!avd_mbox_pop(avd, &status)) + return IRQ_HANDLED; + if (!f) { + /* + * The old driver kicked PP for every VP DONE. A VP DONE with no + * frame in flight (its frame already completed on a surplus PP + * DONE of a multi-kick picture) still needs that kick, or the + * next picture's PP is never started and it times out. + */ + if ((status & 0x11000) == 0 && (status & 0x100) && avd->compat_ctx && + avd->compat_ctx->coded_fmt_desc->ops->submit) + avd->compat_ctx->coded_fmt_desc->ops->submit(avd->compat_ctx); + return IRQ_HANDLED; + } + if (status & 0x1000) { + avd->compat_active = NULL; + avd_frame_complete(avd, f->tag, 0); + } else if (status & 0x100) { + /* Old firmware only: no request tag and no firmware-side PP kick. */ + if (f->ctx->coded_fmt_desc->ops->submit) + f->ctx->coded_fmt_desc->ops->submit(f->ctx); + } else { + dev_err_ratelimited(avd->dev, "open firmware error %#x\n", status); + f->deadline = jiffies; + mod_delayed_work(system_wq, &avd->queue_work, 0); + } + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_open_backend = { + .name = "open-compat", + .boot = avd_boot, + .shutdown = avd_shutdown, + .max_inflight = avd_open_max_inflight, + .submit = avd_open_submit, + .irq = avd_open_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, +}; + +static void avd_open_capabilities(struct avd_dev *avd) +{ + static const struct { + const char *name; + u32 bit; + } codecs[] = { + { "h265", AVD_CAPABILITY_HEVC }, + { "h264", AVD_CAPABILITY_H264 }, + { "vp9", AVD_CAPABILITY_VP9 }, + { "av1", AVD_CAPABILITY_AV1 }, + }; + const char *info; + unsigned int i; + + /* Without a CODECS trailer, keep codecs that need a newer firmware hidden. */ + avd->capabilities = avd->variant->capabilities & + ~avd->variant->fw_caps_required; + avd->fw_capabilities = avd->capabilities; + if (!avd->fw || avd->fw->size < 0x40) + return; + info = (const char *)avd->fw->data + avd->fw->size - 0x40; + if (memcmp(info, "CODECS:", 7)) + return; /* fw_capabilities already set above */ + avd->capabilities = 0; + for (i = 0; i < ARRAY_SIZE(codecs); i++) + if (strnstr(info, codecs[i].name, 0x40)) + avd->capabilities |= avd->variant->capabilities & codecs[i].bit; + avd->fw_capabilities = avd->capabilities; +} + +static int avd_open_request_firmware(struct avd_dev *avd) +{ + int ret; + + if (!avd->variant->fw_name_fallback) + return request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); + ret = firmware_request_nowarn(&avd->fw, avd->variant->fw_name, avd->dev); + if (ret != -ENOENT) + return ret; + dev_info(avd->dev, "%s not installed, using %s\n", + avd->variant->fw_name, avd->variant->fw_name_fallback); + return request_firmware(&avd->fw, avd->variant->fw_name_fallback, avd->dev); +} + +int avd_backend_select(struct avd_dev *avd) +{ + int ret; + + /* Apple probe returns -ENOENT only when no usable image is present. */ + if (avd->variant->apple) { + ret = avd_apple_backend.probe(avd); + if (!ret) { + avd->backend = &avd_apple_backend; + avd->capabilities = avd->variant->capabilities; + return 0; + } + if (ret != -ENOENT && ret != -EOPNOTSUPP) + return ret; + } + avd->backend = &avd_open_backend; + ret = avd_open_request_firmware(avd); + if (!ret) + avd_open_capabilities(avd); + if (!ret) + ret = avd_qfw_probe(avd); + return ret; +} + +int avd_backend_boot(struct avd_dev *avd) +{ + int ret; + + if (!avd_backend_is_apple(avd)) { + avd_pm_prepare_piodma(avd); + avd_qfw_prepare_boot(avd); + } + ret = avd->backend->boot(avd); + /* Never switch command ABIs after frames have used this backend. */ + if (ret == -ENOENT && !avd->fw_generation && avd_backend_is_apple(avd)) { + ret = avd_open_request_firmware(avd); + if (!ret) + ret = avd_qfw_probe(avd); + if (ret) + return ret; + avd->backend = &avd_open_backend; + avd_open_capabilities(avd); + avd_pm_prepare_piodma(avd); + avd_qfw_prepare_boot(avd); + ret = avd->backend->boot(avd); + } + if (!ret) + avd_qfw_select(avd); + if (!ret) + avd->fw_generation++; + return ret; +} diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c new file mode 100644 index 0000000000000..a706602386c17 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Queued transport for the open AVD firmware (avd-fw queue ABI 2). + * + * The codec's CPU builder still produces the per-frame instruction program. + * Instead of writing it word by word into the VP FIFO register and kicking + * the PP from the VP-done interrupt, the host streams the program into a word + * ring in CM3 SRAM and posts one descriptor per frame. The firmware feeds the + * VP FIFO from SRAM, starts PP itself when the VP is done, and posts a tagged + * completion. No PIODMA (and so no second DART stream) is involved. + */ +#include +#include +#include +#include +#include + +#include "avd.h" +#include "avd-inst.h" +#include "avd-regs.h" + +#define AVDQ_MAGIC 0x41565131 /* "AVQ1" */ +#define AVDQ_VERSION 2 +#define AVDQ_COUNT 16 +#define AVDQ_DESC_BYTES 64 +#define AVDQ_RING_OFFSET 0x8000 +#define AVDQ_INFO 0x7fc0 +/* Firmware private state starts here; the SRAM ring must end before it. */ +#define AVDQ_RING_LIMIT 0x6000 +#define AVDQ_INFO_RP (AVDQ_INFO + 0x38) +#define AVDQ_INFO_WP (AVDQ_INFO + 0x3c) +#define AVDQ_FEAT_SRAM_RING BIT(0) +#define AVDQ_FEAT_MULTI_KICK BIT(1) +#define AVDQ_FLAG_SRAM BIT(0) +#define AVDQ_MAX_PP_EXTRA 255 +#define AVDQ_STATE_SUBMITTED 1 +#define AVDQ_STATE_COMPLETE 2 +#define AVDQ_NOTIFY_MASK 0xffff0000 +#define AVDQ_NOTIFY 0x54530000 +/* SRAM addresses as the CM3 doorbell sees them: 0x01000000 + offset from AVD base. */ +#define AVDQ_CM3_APERTURE(off) (0x01000000 + (off)) +#define AVD_REG_MBOX0_STATUS 0x50 +#define AVD_REG_MBOX0_SEND 0x54 +#define AVDQ_STREAM_TIMEOUT_US (1000 * USEC_PER_MSEC) + +/* Descriptor word offsets (bytes), avd-fw docs/queue.md. */ +enum { + D_MAGIC = 0x00, D_TAG = 0x04, D_CODEC = 0x08, D_VP = 0x0c, + D_PROG_LO = 0x10, D_PROG_HI = 0x14, D_WORDS = 0x18, D_KICK = 0x1c, + D_STATE = 0x20, D_STATUS = 0x24, D_CTAG = 0x28, D_FLAGS = 0x2c, + D_PP_EXTRA = 0x30, D_PP_CONT = 0x34, D_RSVD0 = 0x38, D_RSVD1 = 0x3c, +}; + +struct avd_qfw { + void __iomem *sram; + resource_size_t sram_size; + u32 aperture; /* expected CM3 address of SRAM byte 0 */ + u32 ring_off; + u32 ring_words; + u32 wp; /* host write count, free-running */ + u32 fifo_slots; + u32 codec_mask; /* info+0x24, bit n = codec n */ + bool usable; /* last boot found a matching ABI 2 image */ +}; + +static void __iomem *avd_qfw_desc(struct avd_qfw *q, unsigned int idx) +{ + return q->sram + AVDQ_RING_OFFSET + idx * AVDQ_DESC_BYTES; +} + +int avd_qfw_probe(struct avd_dev *avd) +{ + struct resource *code, *sram; + struct avd_qfw *q; + + code = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "code"); + sram = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "sram"); + if (!code || !sram || sram->start < code->start || + resource_size(sram) < AVDQ_RING_OFFSET + AVDQ_COUNT * AVDQ_DESC_BYTES) + return 0; /* no queue mode on this node: legacy transport only */ + q = devm_kzalloc(avd->dev, sizeof(*q), GFP_KERNEL); + if (!q) + return -ENOMEM; + q->sram = devm_ioremap_resource(avd->dev, sram); + if (IS_ERR(q->sram)) + return PTR_ERR(q->sram); + q->sram_size = resource_size(sram); + /* The code window sits at +0x80000 from the decoder base on every SoC. */ + q->aperture = AVDQ_CM3_APERTURE(0x80000 + (u32)(sram->start - code->start)); + avd->qfw = q; + return 0; +} + +/* Read the firmware's info block after boot; true when ABI 2 queue mode fits. */ +static bool avd_qfw_check(struct avd_dev *avd) +{ + struct avd_qfw *q = avd->qfw; + u32 info[16]; + unsigned int i; + + if (!q) + return false; + for (i = 0; i < ARRAY_SIZE(info); i++) + info[i] = readl(q->sram + AVDQ_INFO + 4 * i); + q->usable = false; + if (info[0] != AVDQ_MAGIC) + return false; + if (info[1] != AVDQ_VERSION || info[2] != AVDQ_COUNT || + info[3] != AVDQ_DESC_BYTES || info[5] != AVDQ_RING_OFFSET || + !(info[11] & AVDQ_FEAT_SRAM_RING) || + !(info[11] & AVDQ_FEAT_MULTI_KICK)) { + dev_info_once(avd->dev, "open firmware queue ABI %u not supported, using legacy transport\n", + info[1]); + return false; + } + if (info[4] != q->aperture) { + dev_warn_once(avd->dev, "firmware SRAM aperture %#x, node needs %#x: wrong image tier, using legacy transport\n", + info[4], q->aperture); + return false; + } + if (info[13] < 1024 || info[12] % 4 || + (u64)info[12] + 4ULL * info[13] > AVDQ_RING_LIMIT || + !info[10] || info[10] > AVDQ_COUNT || info[14] || info[15]) { + dev_warn_once(avd->dev, "firmware SRAM ring geometry invalid, using legacy transport\n"); + return false; + } + q->ring_off = info[12]; + q->ring_words = info[13]; + q->fifo_slots = min_t(u32, info[10], avd->variant->fifo_slots ?: 1); + q->codec_mask = info[9]; + q->wp = 0; + q->usable = true; + return true; +} + +static int avd_qfw_boot(struct avd_dev *avd) +{ + int ret = avd_boot(avd); + + if (ret) + return ret; + /* AP -> CM3 doorbells use mailbox 0; completions stay on mailbox 1. */ + writel_relaxed(AVD_MBOX_ENABLE, avd->mbox + AVD_REG_MBOX0_STATUS); + return 0; +} + +static unsigned int avd_qfw_max_inflight(struct avd_dev *avd) +{ + /* Each frame in flight owns one hardware instruction FIFO (EXEC 7:4). */ + return min_t(unsigned int, AVD_MAX_FRAMES, avd->qfw->fifo_slots); +} + +static void avd_qfw_fail_all(struct avd_dev *avd) +{ + unsigned int i; + + /* ABI: any fatal word or I/O status needs a whole-queue reset. */ + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (avd->frames[i].state == AVD_FRAME_SUBMITTED) + avd->frames[i].deadline = jiffies; + mod_delayed_work(system_wq, &avd->queue_work, 0); +} + +static int avd_qfw_stream(struct avd_dev *avd, const u32 *words, u32 count) +{ + struct avd_qfw *q = avd->qfw; + unsigned long timeout = jiffies + usecs_to_jiffies(AVDQ_STREAM_TIMEOUT_US); + u32 done = 0; + + while (done < count) { + u32 rp = readl(q->sram + AVDQ_INFO_RP); + u32 used = q->wp - rp, space, pos, chunk; + + if (used > q->ring_words) + return -EIO; + space = q->ring_words - used; + if (!space) { + if (time_after(jiffies, timeout)) + return -ETIMEDOUT; + usleep_range(20, 50); + continue; + } + pos = q->wp % q->ring_words; + chunk = min3(space, count - done, q->ring_words - pos); + memcpy_toio(q->sram + q->ring_off + 4 * pos, words + done, 4 * chunk); + q->wp += chunk; + done += chunk; + /* Words before the count that publishes them. */ + wmb(); + writel(q->wp, q->sram + AVDQ_INFO_WP); + } + return 0; +} + +static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) +{ + struct avd_qfw *q = avd->qfw; + struct avd_ctx *ctx = f->ctx; + const struct avd_coded_fmt_ops *ops = ctx->coded_fmt_desc->ops; + u32 codec = f->cmd[0], vp = 0, kick, cont, mask, extra = 0; + u32 *words = f->prog->cpu; + void __iomem *d = avd_qfw_desc(q, f->idx); + unsigned int i; + int ret; + + if (ctx->coded_fmt_desc->frame_ops) + return -EOPNOTSUPP; + if (codec > AVD_CODEC_AV1 || !words || !f->prog_words || + f->prog_words > f->prog->size / sizeof(u32)) + return -EINVAL; + if (!(avd->capabilities & BIT(codec))) + return -EOPNOTSUPP; + /* The program's EXEC words name FIFO f->idx; it must be a real FIFO. */ + if (f->idx >= q->fifo_slots || ctx->fifo_idx != f->idx) + return -EINVAL; + for (i = 0; i < codec; i++) + vp += avd->variant->vp_slots[i]; + if (avd->variant->vp_slot_base[codec]) + vp = avd->variant->vp_slot_base[codec]; + if (ops->pp_kicks) + extra = ops->pp_kicks(ctx) - 1; + if (extra > AVDQ_MAX_PP_EXTRA) + return -E2BIG; + + mask = codec == AVD_CODEC_VP9 && avd->variant->revision == 3 ? + AVD_OP_EXEC_REV3_VP9_MASK : 0; + cont = AVD_OP_EXEC | mask | AVD_OP_EXEC_FIFO_IDX(f->idx) | + AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots); + kick = cont | AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | + AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3 && + codec != AVD_CODEC_AV1); + + /* FIFO registers of other live frames are never touched. */ + avd->variant->configure_stream(avd, f->fifo_iova ?: ctx->inst.addr, f->idx, vp); + + writel(AVDQ_MAGIC, d + D_MAGIC); + writel(f->tag, d + D_TAG); + writel(codec, d + D_CODEC); + writel(vp, d + D_VP); + writel(q->wp, d + D_PROG_LO); + writel(0, d + D_PROG_HI); + writel(f->prog_words, d + D_WORDS); + writel(kick, d + D_KICK); + writel(0, d + D_STATUS); + writel(0, d + D_CTAG); + writel(AVDQ_FLAG_SRAM, d + D_FLAGS); + writel(extra, d + D_PP_EXTRA); + writel(extra ? cont : 0, d + D_PP_CONT); + writel(0, d + D_RSVD0); + writel(0, d + D_RSVD1); + writel(AVDQ_STATE_SUBMITTED, d + D_STATE); + /* Descriptor before the doorbell that hands it to the firmware. */ + wmb(); + (void)readl(d + D_STATE); /* flush posted aperture writes */ + writel(q->aperture + AVDQ_RING_OFFSET + f->idx * AVDQ_DESC_BYTES, + avd->mbox + AVD_REG_MBOX0_SEND); + + /* The firmware may start feeding while the rest of the program streams in. */ + ret = avd_qfw_stream(avd, words, f->prog_words); + if (ret) { + dev_err_ratelimited(avd->dev, "queue: program stream failed: %d\n", ret); + /* The descriptor is already posted: only a reset releases it. */ + f->deadline = jiffies; + } + return 0; +} + +static irqreturn_t avd_qfw_irq(struct avd_dev *avd) +{ + struct avd_qfw *q = avd->qfw; + struct avd_frame *f; + void __iomem *d; + u32 word, state, status, ctag; + unsigned int idx; + + if (!avd_mbox_pop(avd, &word)) + return IRQ_HANDLED; + if ((word & AVDQ_NOTIFY_MASK) != AVDQ_NOTIFY) { + dev_err_ratelimited(avd->dev, "queue: firmware reported %#x, resetting\n", word); + avd_qfw_fail_all(avd); + return IRQ_HANDLED; + } + idx = word & 15; + d = avd_qfw_desc(q, idx); + state = readl(d + D_STATE); + status = readl(d + D_STATUS); + ctag = readl(d + D_CTAG); + f = &avd->frames[idx]; + if (state != AVDQ_STATE_COMPLETE || ((ctag ^ word) & 0xfff0) || + f->state != AVD_FRAME_SUBMITTED || ctag != f->tag) { + dev_warn_ratelimited(avd->dev, "queue: stale completion %#x (state %u tag %#x)\n", + word, state, ctag); + return IRQ_HANDLED; + } + writel(0, d + D_STATE); + avd_frame_complete(avd, ctag, status == 0 ? 0 : status == 1 ? -EINVAL : -EIO); + if (status > 1) + avd_qfw_fail_all(avd); + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_qfw_backend = { + .name = "open-queue", + .boot = avd_qfw_boot, + .shutdown = avd_shutdown, + .max_inflight = avd_qfw_max_inflight, + .submit = avd_qfw_submit, + .irq = avd_qfw_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, +}; + +bool avd_backend_is_qfw(const struct avd_dev *avd) +{ + return avd->backend == &avd_qfw_backend; +} + +/* + * SRAM survives a CM3 reload, so a stale info block from an earlier ABI 2 + * image could make a legacy image look queue capable. Clear the magic before + * every open-firmware boot; only an image that publishes it again qualifies. + */ +void avd_qfw_prepare_boot(struct avd_dev *avd) +{ + if (avd->qfw) + writel(0, avd->qfw->sram + AVDQ_INFO); +} + +/* + * Called after every successful open-firmware boot (probe, resume, reset), + * with no frame owned by the firmware. Picks the queued transport when the + * loaded image speaks ABI 2 for this node, else the legacy one. + */ +void avd_qfw_select(struct avd_dev *avd) +{ + const struct avd_backend_ops *old = avd->backend; + + if (old != &avd_open_backend && old != &avd_qfw_backend) + return; + if (avd_qfw_check(avd)) { + /* Legacy boot left mailbox 0 disabled. */ + writel_relaxed(AVD_MBOX_ENABLE, avd->mbox + AVD_REG_MBOX0_STATUS); + avd->backend = &avd_qfw_backend; + /* + * Codecs held back for a newer firmware (T6040 HEVC) are + * enabled by a queue image that lists them in its codec mask. + */ + avd->capabilities = avd->fw_capabilities | + (avd->variant->capabilities & avd->qfw->codec_mask); + } else { + avd->backend = &avd_open_backend; + avd->capabilities = avd->fw_capabilities; + } + if (avd->backend != old) + dev_info(avd->dev, "using %s transport (%u frames in flight)\n", + avd->backend->name, avd->backend->max_inflight(avd)); +} diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 6b9428ed13f3b..1cb58c1b2f7c6 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -22,8 +22,7 @@ #include "avd.h" #include "avd-inst.h" #include "avd-regs.h" - -#define AVD_CODEC_INFO_MAX_SIZE 0x40 +#include "avd-apple-protocol.h" static void calc_tile_meta(u32 w, u32 h, u32 bpb, u32 tile_dim, u32 meta_hdr_bytes, u32 *tile, u32 *meta) @@ -73,31 +72,6 @@ void fill_comp(struct avd_comp *comp, enum avd_image_fmt image_fmt, u32 width, comp->size = y_meta + y + uv_meta + uv; } -int avd_buf_alloc(struct avd_dev *avd, struct avd_buf *buf, size_t size) -{ - /* Keep size equal to the payload length used by the H.264 slice path. */ - avd_buf_free(avd, buf); - - if (size <= 0) - return -ENOMEM; - - buf->size = size; - buf->cpu = - dma_alloc_coherent(avd->dev, buf->size, &buf->addr, GFP_KERNEL); - if (!buf->cpu) { - memset(buf, 0, sizeof(*buf)); - return -ENOMEM; - } - return 0; -} - -void avd_buf_free(struct avd_dev *avd, struct avd_buf *buf) -{ - if (buf->cpu) - dma_free_coherent(avd->dev, buf->size, buf->cpu, buf->addr); - memset(buf, 0, sizeof(*buf)); -} - struct avd_decoded_buffer * avd_get_ref_buf(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst, u64 timestamp) { @@ -116,28 +90,6 @@ avd_get_ref_buf(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst, u64 timestamp) return vb2_to_avd_decoded_buf(buf); } -static int avd_wait_submission_queue(struct avd_ctx *ctx, int vp) -{ - struct avd_dev *avd = ctx->dev; - u32 max = readl_relaxed(avd->ctrl + - avd->variant->submit_queue_max_offset + vp * 4); - u32 cur = readl_relaxed( - avd->ctrl + avd->variant->submit_queue_status_offset + vp * 4); - - if (cur == max) { - dev_err(avd->dev, "instruction que full! %d/%d", cur, max); - return 1; - } - - if (cur >= max / 2) { - if (avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) - usleep_range(100, 150); - else - usleep_range(500, 650); - } - return 0; -} - int avd_init_job(struct avd_ctx *ctx, enum avd_codec codec, size_t segments) { int ret = 0; @@ -155,287 +107,20 @@ int avd_init_job(struct avd_ctx *ctx, enum avd_codec codec, size_t segments) int avd_submit_job(struct avd_ctx *ctx) { - struct avd_dev *avd = ctx->dev; - struct avd_job *sub = &ctx->job; - struct avd_segment *segments, *seg; - unsigned long flags; - size_t num; - int i, idx = 0, vp = 0; - void __iomem *reg; - u32 exec_mask = sub->codec == AVD_CODEC_VP9 && - avd->variant->revision == 3 ? - AVD_OP_EXEC_REV3_VP9_MASK : - 0; - u32 exec_rev_flag = - AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | - AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3); - - ctx->fifo_idx = 0; - for (i = 0; i < sub->codec; i++) - vp += avd->variant->vp_slots[i]; - if (avd->variant->vp_slot_base[sub->codec]) - vp = avd->variant->vp_slot_base[sub->codec]; - - /* - * avd_device_run() keeps this job from being finished (and the next one - * from starting) until it returns, and arms the watchdog itself once the - * job belongs to the hardware. Still walk a local copy of the table. - */ - segments = sub->segments; - num = sub->num; - if (WARN_ON_ONCE(!segments)) - return -EINVAL; - - avd->variant->configure_stream(avd, ctx->inst.addr, ctx->fifo_idx, vp); - reg = avd->ctrl + avd->variant->vp_slot_offset + vp * 4; - - /* the first segment is always the header (needs special handling) */ - - writel(AVD_OP_EXEC | exec_mask | exec_rev_flag | - AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx), - reg); - seg = &segments[idx++]; - for (i = 0; i < seg->num; i++) - writel(seg->instructions[i], reg); - for (; idx <= num; idx++) { - seg = &segments[idx]; - for (i = 0; i < seg->num; i++) - writel(seg->instructions[i], reg); - if ((!(avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) || - sub->codec == AVD_CODEC_HEVC) && - avd_wait_submission_queue(ctx, vp)) - break; - if (idx == num) { - /* A completion from here on belongs to this job. */ - spin_lock_irqsave(&avd->job_lock, flags); - avd->job_end_sent = true; - spin_unlock_irqrestore(&avd->job_lock, flags); - } - writel(AVD_OP_EXEC | exec_mask | - AVD_OP_EXEC_FLAG_END(idx == num), - reg); - } - - if (sub->segments == segments) - sub->segments = NULL; - kvfree(segments); - return 0; -} - -static int avd_reset(struct avd_dev *avd) -{ - int ret = 0; - - ret = pm_runtime_resume_and_get(avd->dev); - if (ret < 0) - return ret; - - ret = reset_control_reset(avd->rstc); - if (ret) - dev_err(avd->dev, "reset: failed: %d", ret); - - if (avd->empty_domain) { - iommu_attach_device(avd->empty_domain, avd->dev); - iommu_detach_device(avd->empty_domain, avd->dev); - } - - ret = avd_boot(avd); - if (ret) - dev_err(avd->dev, "reset: failed to boot"); - - pm_runtime_put_autosuspend(avd->dev); - - return ret; -} - -static void avd_watchdog_func(struct work_struct *work) -{ - struct avd_dev *avd; - struct avd_ctx *ctx; - unsigned long flags; - bool claimed; - int ret; - ctx = container_of(to_delayed_work(work), struct avd_ctx, - watchdog_work); - if (!ctx) - return; - - avd = ctx->dev; - - /* - * Time out only the job this watchdog was armed for, and only while the - * hardware still owns it. Otherwise that job already finished, and - * resetting the hardware would break whichever job is running now. - */ - spin_lock_irqsave(&avd->job_lock, flags); - claimed = avd->job_ctx == ctx && avd->job_seq == ctx->wd_seq && - avd->job_state == AVD_JOB_SUBMITTED; - if (claimed) - avd->job_state = AVD_JOB_FINISHING; - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: watchdog seq %llu: %s\n", ctx->wd_seq, - claimed ? "timeout" : "stale, ignored"); - if (!claimed) - return; - - dev_err(avd->dev, "Frame processing timed out!"); - - writel(0, avd->mbox + AVD_REG_MBOX_IRQ_ENABLE); - ret = avd_reset(avd); - if (ret) - dev_err(avd->dev, "failed to reset: %d", ret); - - avd_job_finish(ctx, VB2_BUF_STATE_ERROR); + return avd_open_build_program(ctx); } static irqreturn_t avd_irq_handler(int irq, void *data) { struct avd_dev *avd = data; - struct avd_ctx *ctx = v4l2_m2m_get_curr_priv(avd->m2m_dev); - - enum vb2_buffer_state state; - unsigned long flags; - bool finish = false, stale = false; - u32 status; - if (!ctx) - return IRQ_HANDLED; - - status = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); - - writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); - - if (status & 0x10000) { /* dbg */ - dev_warn(avd->dev, "no handler for IRQ: %3d", status &~0x10000); - writel_relaxed(0, avd->mbox + AVD_REG_MBOX_IRQ_ENABLE); - return IRQ_HANDLED; - } - - if (status & 0x1000) { - /* pp is done ! we are done */ - state = VB2_BUF_STATE_DONE; - } else if (status & 0x100) { - /* a vp is done, kick the pp and hope for the best */ - if(ctx->coded_fmt_desc->ops->submit) - ctx->coded_fmt_desc->ops->submit(ctx); - goto done; - } else { - dev_err(avd->dev, "H%d error", status); - /* - * avd-fw err(n): VP n raised its error interrupt and stopped, - * and the PP only starts on a VP done, so the job cannot - * complete any more. Run the watchdog now instead of after its - * timeout: it resets the AVD and fails only this frame, and it - * ignores a job the hardware does not own yet. - */ - mod_delayed_work(system_wq, &ctx->watchdog_work, 0); - goto done; - } - - /* - * Finish only a job the hardware owns. While avd_device_run() is still - * setting up or submitting, a completion is left over from an older job - * unless this job's END command already went out; then remember it and - * let avd_device_run() finish the job when it returns. - */ - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_ctx != ctx) { - stale = true; - } else if (avd->job_state == AVD_JOB_SUBMITTED) { - avd->job_state = AVD_JOB_FINISHING; - finish = true; - } else if (avd->job_state == AVD_JOB_RUNNING && avd->job_end_sent) { - avd->job_pending = true; - avd->job_pending_result = state; - } else { - stale = true; - } - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: irq %#x: %s\n", status, - finish ? "finish" : stale ? "stale, ignored" : "pending"); - if (stale) - dev_warn_ratelimited(avd->dev, "ignoring stale completion"); - if (finish) { - cancel_delayed_work(&ctx->watchdog_work); - avd_job_finish(ctx, state); - } + /* An interrupt raised before a reset or power-off belongs to that instance. */ + unsigned int gen = READ_ONCE(avd->pm.irq_gen); + irqreturn_t ret; -done: - return IRQ_HANDLED; -} - -static void avd_device_run(void *priv) -{ - struct avd_ctx *ctx = priv; - struct avd_dev *avd = ctx->dev; - const struct avd_coded_fmt_desc *desc = ctx->coded_fmt_desc; - enum vb2_buffer_state result = VB2_BUF_STATE_ERROR; - const char *outcome = "already finished"; - unsigned long flags; - bool finish = false; - u64 seq; - int ret; - - if (WARN_ON(!desc)) - return; - - /* - * IRQ completion cannot wait for an already running watchdog. Drain it - * here, before reusing wd_seq, so an old callback cannot time out this - * context's next job. device_run is sleepable; hold no job_lock here. - */ - cancel_delayed_work_sync(&ctx->watchdog_work); - - ret = pm_runtime_resume_and_get(avd->dev); - if (ret < 0) { - avd_job_finish_no_pm(ctx, VB2_BUF_STATE_ERROR); - return; - } - - /* - * While the job is RUNNING neither completion IRQs nor the watchdog may - * finish it: v4l2-mem2mem would start the next job on a kworker while - * this one still builds and submits ctx->job. - */ - spin_lock_irqsave(&avd->job_lock, flags); - seq = ++avd->job_seq; - avd->job_ctx = ctx; - avd->job_state = AVD_JOB_RUNNING; - avd->job_end_sent = false; - avd->job_pending = false; - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: run seq %llu\n", seq); - - ret = desc->ops->run ? desc->ops->run(ctx) : -EOPNOTSUPP; - if (ret) { - dev_dbg(avd->dev, "avd-job: run seq %llu failed: %d\n", seq, ret); - avd_job_finish(ctx, VB2_BUF_STATE_ERROR); - return; - } - - /* - * The run either finished the job itself (held capture buffer) or handed - * it to the hardware: arm the watchdog, or finish right away if the - * completion already arrived. - */ - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_seq == seq && avd->job_state == AVD_JOB_RUNNING) { - if (avd->job_pending) { - avd->job_state = AVD_JOB_FINISHING; - result = avd->job_pending_result; - finish = true; - outcome = "completed during submission"; - } else { - avd->job_state = AVD_JOB_SUBMITTED; - ctx->wd_seq = seq; - schedule_delayed_work(&ctx->watchdog_work, msecs_to_jiffies(2000)); - outcome = "submitted"; - } - } - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: run seq %llu: %s\n", seq, outcome); - - if (finish) - avd_job_finish(ctx, result); + mutex_lock(&avd->queue_mutex); + ret = avd_pm_irq(avd, gen); + mutex_unlock(&avd->queue_mutex); + return ret; } /* @@ -444,12 +129,7 @@ static void avd_device_run(void *priv) * userspace decode ahead to ride out bitrate spikes the hardware can't * decode in real time (e.g. 150+ Mbit/s 4K60 H.264). */ -static unsigned int max_capture_buffers = 128; -module_param(max_capture_buffers, uint, 0444); -MODULE_PARM_DESC(max_capture_buffers, - "Maximum number of capture buffers per context (default 128)"); - -static int avd_queue_init(void *priv, struct vb2_queue *src_vq, +static int avd_m2m_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq) { struct avd_ctx *ctx = priv; @@ -486,7 +166,7 @@ static int avd_queue_init(void *priv, struct vb2_queue *src_vq, dst_vq->lock = &ctx->dev->vdev_lock; dst_vq->dev = ctx->dev->v4l2_dev.dev; dst_vq->allow_cache_hints = true; - dst_vq->max_num_buffers = max(max_capture_buffers, VB2_MAX_FRAME); + dst_vq->max_num_buffers = max(128U, VB2_MAX_FRAME); return vb2_queue_init(dst_vq); } @@ -507,14 +187,15 @@ static int avd_open(struct file *filp) if (ret) goto err_free_ctx; - INIT_DELAYED_WORK(&ctx->watchdog_work, avd_watchdog_func); + INIT_LIST_HEAD(&ctx->queue_node); + ctx->next_dep = AVD_DEP_PREV_DONE; avd_reset_coded_fmt(ctx); avd_reset_decoded_fmt(ctx); v4l2_fh_init(&ctx->fh, video_devdata(filp)); - ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(avd->m2m_dev, ctx, avd_queue_init); + ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(avd->m2m_dev, ctx, avd_m2m_queue_init); if (IS_ERR(ctx->fh.m2m_ctx)) { ret = PTR_ERR(ctx->fh.m2m_ctx); goto err_free_ctx; @@ -524,6 +205,7 @@ static int avd_open(struct file *filp) if (ret) goto err_cleanup_m2m_ctx; + avd_queue_add_ctx(ctx); v4l2_fh_add(&ctx->fh, filp); return 0; @@ -542,9 +224,9 @@ static int avd_release(struct file *filp) struct avd_ctx *ctx = file_to_ctx(filp); v4l2_fh_del(&ctx->fh, filp); + avd_queue_del_ctx(ctx); v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); - /* The job has finished; its watchdog must not run on the freed ctx. */ - cancel_delayed_work_sync(&ctx->watchdog_work); + cancel_work_sync(&ctx->submit_work); v4l2_ctrl_handler_free(&ctx->ctrl_hdl); v4l2_fh_exit(&ctx->fh); avd_buf_free(ctx->dev, &ctx->inst); @@ -564,7 +246,9 @@ static const struct v4l2_file_operations avd_fops = { }; static const struct v4l2_m2m_ops avd_m2m_ops = { - .device_run = avd_device_run, + .device_run = avd_queue_run, + .job_ready = avd_queue_ready, + .job_abort = avd_queue_abort, }; static const struct media_device_ops avd_media_ops = { @@ -808,11 +492,13 @@ static const struct avd_variant avd_t8132_variant = { * T6040 (M4 Pro): the T8132 decoder. Its first HEVC job stalled the VP when * it was sent the pre-v5 header (no DMA configuration words, no reserved * scratch slot); T6040 runs the same v5 firmware generation as T8140, so it - * gets the v5 header. HEVC stays hidden from V4L2 clients until that is - * verified on the hardware. + * gets the v5 header. HEVC is advertised only when the loaded firmware + * names h265 in its CODECS trailer. */ static const struct avd_variant avd_t6040_variant = { .av1_fp_layout = AVD_AV1_FP_BORAGE, + .apple = &avd_borage_hw, + .fp_size = 0x1130d0, .vp_slots = { [AVD_CODEC_HEVC] = 4, [AVD_CODEC_H264] = 4, @@ -821,10 +507,15 @@ static const struct avd_variant avd_t6040_variant = { }, .fifo_slots = 15, .configure_stream = t8122_configure_stream, - .capabilities = AVD_CAPABILITY_H264 | + .capabilities = AVD_CAPABILITY_HEVC | + AVD_CAPABILITY_H264 | AVD_CAPABILITY_VP9 | AVD_CAPABILITY_AV1, - .fw_name = "apple/avd-fw-v5-t1.bin", + /* HEVC only with firmware whose CODECS trailer lists h265. */ + .fw_caps_required = AVD_CAPABILITY_HEVC, + /* v5-t2 matches T6040's SRAM aperture, which the queued transport needs. */ + .fw_name = "apple/avd-fw-v5-t2.bin", + .fw_name_fallback = "apple/avd-fw-v5-t1.bin", .revision = 4, .vp_slot_offset = 0xc, .submit_offset = 0x40, @@ -872,11 +563,17 @@ static const struct of_device_id avd_of_match[] = { MODULE_DEVICE_TABLE(of, avd_of_match); +static void avd_release_open_firmware(void *data) +{ + struct avd_dev *avd = data; + + release_firmware(avd->fw); +} + static int avd_probe(struct platform_device *pdev) { struct avd_dev *avd; const struct of_device_id *match; - const char *codec_info = NULL; int ret, irq; avd = devm_kzalloc(&pdev->dev, sizeof(*avd), GFP_KERNEL); @@ -888,12 +585,17 @@ static int avd_probe(struct platform_device *pdev) avd->pdev = pdev; mutex_init(&avd->vdev_lock); - spin_lock_init(&avd->job_lock); + avd_pm_init(avd, NULL, &avd_pm_queue_ops); + ret = avd_queue_init(avd); + if (ret) + return ret; match = of_match_node(avd_of_match, pdev->dev.of_node); avd->variant = match->data; avd->rstc = devm_reset_control_get_exclusive(avd->dev, NULL); + if (IS_ERR(avd->rstc)) + return PTR_ERR(avd->rstc); avd->code = devm_platform_ioremap_resource_byname(pdev, "code"); if (IS_ERR(avd->code)) @@ -904,6 +606,9 @@ static int avd_probe(struct platform_device *pdev) avd->ctrl = devm_platform_ioremap_resource_byname(pdev, "ctrl"); if (IS_ERR(avd->ctrl)) return PTR_ERR(avd->ctrl); + ret = avd_pm_map_piodma(avd); + if (ret) + return ret; avd->domain = iommu_get_domain_for_dev(avd->dev); if (avd->domain) { @@ -914,39 +619,16 @@ static int avd_probe(struct platform_device *pdev) } } - ret = request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); - - if (ret) { - dev_err(avd->dev, "failed to load firmware: %d", ret); + ret = avd_backend_select(avd); + if (ret) + return ret; + ret = devm_add_action_or_reset(avd->dev, avd_release_open_firmware, avd); + if (ret) return ret; - } - - /* Determine firmware and HW variant capabilities */ - if (avd->fw->size >= AVD_CODEC_INFO_MAX_SIZE) - codec_info = avd->fw->data + avd->fw->size - AVD_CODEC_INFO_MAX_SIZE; - if (codec_info && strncmp(codec_info, "CODECS:", 7) == 0) { - unsigned int caps = 0; - unsigned int variant_caps = avd->variant->capabilities; - - if (strnstr(codec_info, "av1", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_AV1; - if (strnstr(codec_info, "h264", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_H264; - if (strnstr(codec_info, "h265", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_HEVC; - if (strnstr(codec_info, "vp9", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_VP9; - - dev_info(avd->dev, "Firmware codecs: %.27s\n", codec_info + 7); - - avd->capabilities = caps; - } else { - /* Backward compatibility to old firmware, do not upstream. */ - avd->capabilities = avd->variant->capabilities; - } ret = dma_set_mask_and_coherent(avd->dev, - DMA_BIT_MASK((avd->variant->quirks & AVD_QUIRK_LSR) ? 38 : 64)); + DMA_BIT_MASK(avd_backend_is_apple(avd) ? 42 : + (avd->variant->quirks & AVD_QUIRK_LSR) ? 38 : 64)); if (ret) { dev_err(avd->dev, "Failed to set DMA mask"); @@ -956,17 +638,27 @@ static int avd_probe(struct platform_device *pdev) irq = platform_get_irq(pdev, 1); if (irq < 0) return irq; + avd->pm.irq = irq; ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, - avd_irq_handler, IRQF_ONESHOT, + avd_irq_handler, IRQF_ONESHOT | IRQF_NO_AUTOEN, dev_name(&pdev->dev), avd); if (ret) { dev_err(avd->dev, "Could not request IRQ 1"); return ret; } - pm_runtime_set_autosuspend_delay(avd->dev, 100); - pm_runtime_use_autosuspend(avd->dev); - pm_runtime_enable(avd->dev); + avd_pm_enable(avd); + + /* + * Boot the firmware once now: the transport and the codec list depend + * on what the loaded image reports, and clients enumerate formats and + * size their contexts before the first decode would power it up. + */ + ret = pm_runtime_resume_and_get(avd->dev); + if (ret < 0) + dev_warn(avd->dev, "firmware probe boot failed: %d\n", ret); + else + pm_runtime_put_autosuspend(avd->dev); ret = avd_v4l2_init(avd); if (ret) @@ -983,42 +675,19 @@ static int avd_probe(struct platform_device *pdev) static void avd_remove(struct platform_device *pdev) { struct avd_dev *avd = platform_get_drvdata(pdev); + int ret; - release_firmware(avd->fw); - + v4l2_m2m_suspend(avd->m2m_dev); + ret = avd->pm.queue->drain(avd); + if (ret) + dev_warn(avd->dev, "queue drain failed, resetting before teardown: %d\n", ret); + avd_queue_cleanup(avd); + avd_pm_disable(avd); avd_v4l2_cleanup(avd); - if (avd->empty_domain) iommu_domain_free(avd->empty_domain); - - pm_runtime_disable(avd->dev); - pm_runtime_dont_use_autosuspend(avd->dev); } -static __maybe_unused int avd_runtime_resume(struct device *dev) -{ - int ret; - struct avd_dev *avd = platform_get_drvdata(to_platform_device(dev)); - - ret = avd_boot(avd); - if (ret) - dev_err(dev, "failed to boot"); - return ret; -} - -static __maybe_unused int avd_runtime_suspend(struct device *dev) -{ - struct avd_dev *avd = platform_get_drvdata(to_platform_device(dev)); - - avd_shutdown(avd); - return 0; -} - -static const struct dev_pm_ops avd_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS( - pm_runtime_force_suspend, - pm_runtime_force_resume) SET_RUNTIME_PM_OPS(avd_runtime_suspend, - avd_runtime_resume, NULL) }; - static struct platform_driver avd_driver = { .probe = avd_probe, .remove = avd_remove, diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index e8bc8d0066ead..5ff0026ef3730 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -57,15 +57,18 @@ struct avd_h264_run { u32 slice_data_off; }; -/* state */ -struct avd_h264_ctx { - bool monochrome; - struct avd_h264_reflists { - struct v4l2_h264_reference p[V4L2_H264_REF_LIST_LEN]; - struct v4l2_h264_reference b0[V4L2_H264_REF_LIST_LEN]; - struct v4l2_h264_reference b1[V4L2_H264_REF_LIST_LEN]; - } reflists; +/* + * One bank per picture in flight. With the queued transport two pictures of + * a stream overlap (VP of one, PP of the other), so everything the hardware + * reads or writes while decoding a picture, and the CPU copies of its slices, + * exists once per bank. The legacy transport uses a single bank. + */ +#define AVD_H264_BANKS 2 +struct avd_bank { + bool monochrome; + bool busy; /* a picture of this bank is not retired yet */ + bool submitted; /* ... and its last slice went to the hardware */ struct avd_buf *slices; struct avd_buf *active_slice; size_t slice_num; @@ -74,6 +77,7 @@ struct avd_h264_ctx { size_t slice_bytes; struct avd_h264_bufs { + /* VP to PP instruction FIFO for pictures of this bank */ struct avd_buf inst; struct avd_buf pipe_state; struct avd_buf above_info; @@ -84,6 +88,20 @@ struct avd_h264_ctx { } bufs; }; +/* state */ +struct avd_h264_ctx { + struct avd_h264_reflists { + struct v4l2_h264_reference p[V4L2_H264_REF_LIST_LEN]; + struct v4l2_h264_reference b0[V4L2_H264_REF_LIST_LEN]; + struct v4l2_h264_reference b1[V4L2_H264_REF_LIST_LEN]; + } reflists; + + unsigned int nbanks; + struct avd_bank bank[AVD_H264_BANKS]; + /* bank of the picture being built */ + struct avd_bank *b; +}; + /* ffmpeg submits wrong timestamp, use first_mb_in_slice as a workaround */ #define is_new_frame(sl) \ (sl->first_mb_in_slice == 0) /* (ctx->fh.m2m_ctx->new_frame) */ @@ -118,7 +136,7 @@ static void stream_refs(struct avd_ctx *ctx, struct avd_h264_run *run) dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); push(0, ""); - pusha(h264_ctx->bufs.mv_above_info.addr, "mv_above_info", 7); + pusha(h264_ctx->b->bufs.mv_above_info.addr, "mv_above_info", 7); pusha(run->addresses.mv_color, "mv_color", 0); push(0, ""); @@ -272,7 +290,7 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) if (avd->variant->revision == 3) push(0, "zero"); - pusha(h264_ctx->bufs.above_info.addr, "hdr_9c_pps_tile_addr_lsb8", 0); + pusha(h264_ctx->b->bufs.above_info.addr, "hdr_9c_pps_tile_addr_lsb8", 0); push(0, ""); push(0, ""); @@ -280,11 +298,11 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) if (avd->variant->revision == 3) push(0, "zero"); else if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) - pusha(h264_ctx->bufs.pipe_state.addr, "pipe_state", 0); + pusha(h264_ctx->b->bufs.pipe_state.addr, "pipe_state", 0); - pusha(h264_ctx->bufs.ip_above.addr, "ip_above", 1); - pusha(h264_ctx->bufs.lf_above.addr, "lf_above", 2); - pusha(h264_ctx->bufs.lf_above_info.addr, "lf_above_info", 3); + pusha(h264_ctx->b->bufs.ip_above.addr, "ip_above", 1); + pusha(h264_ctx->b->bufs.lf_above.addr, "lf_above", 2); + pusha(h264_ctx->b->bufs.lf_above_info.addr, "lf_above_info", 3); push(0, ""); push_comp(ctx, run->base.comp_out, ctx->comp.offsets); @@ -437,12 +455,12 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) const struct v4l2_ctrl_h264_sps *sps = run->sps; const struct v4l2_ctrl_h264_slice_params *sl = run->slice_params; struct avd_h264_ctx *h264_ctx = ctx->priv; - u32 payload_len = h264_ctx->active_slice->size; + u32 payload_len = h264_ctx->b->active_slice->size; bool en_mode = (pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE) == 0; u32 off = run->slice_data_off; - dma_addr_t slc_a84 = h264_ctx->active_slice->addr + off; + dma_addr_t slc_a84 = h264_ctx->b->active_slice->addr + off; push(AVD_OP_CODED_DATA | AVD_OP_CODED_DATA_BIT_OFF( @@ -538,10 +556,9 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) } } -static int avd_h264_alloc_bufs(struct avd_ctx *ctx) +static int avd_h264_alloc_bufs(struct avd_ctx *ctx, struct avd_bank *bank) { struct avd_dev *dev = ctx->dev; - struct avd_h264_ctx *h264_ctx = ctx->priv; int ret, w, bit_depth, mb; w = fmt_width(ctx); bit_depth = (ctx->image_fmt == AVD_IMG_FMT_420_10BIT || @@ -549,13 +566,13 @@ static int avd_h264_alloc_bufs(struct avd_ctx *ctx) 10 : 8; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.inst, fifo_size()); + ret = avd_buf_alloc(dev, &bank->bufs.inst, fifo_size()); if (ret) { return ret; } if (!(dev->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) { - ret = avd_buf_alloc(dev, &h264_ctx->bufs.pipe_state, 0x200); + ret = avd_buf_alloc(dev, &bank->bufs.pipe_state, 0x200); if (ret) { return ret; } @@ -563,24 +580,24 @@ static int avd_h264_alloc_bufs(struct avd_ctx *ctx) mb = DIV_ROUND_UP(w, 16); - ret = avd_buf_alloc(dev, &h264_ctx->bufs.above_info, mb * 20); + ret = avd_buf_alloc(dev, &bank->bufs.above_info, mb * 20); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.ip_above, bit_depth * 4 * mb); + ret = avd_buf_alloc(dev, &bank->bufs.ip_above, bit_depth * 4 * mb); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.lf_above, + ret = avd_buf_alloc(dev, &bank->bufs.lf_above, bit_depth * 4 * 4 * mb); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.lf_above_info, 32 * mb); + ret = avd_buf_alloc(dev, &bank->bufs.lf_above_info, 32 * mb); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.mv_above_info, 32 * mb); + ret = avd_buf_alloc(dev, &bank->bufs.mv_above_info, 32 * mb); if (ret) return ret; @@ -620,35 +637,46 @@ static int avd_h264_validate_sps(struct avd_ctx *ctx, return 0; } -/* Free the slices held for the current picture. */ +/* Free the slices held for one bank's picture. */ +static void avd_h264_drop_bank_slices(struct avd_ctx *ctx, struct avd_bank *bank) +{ + for (size_t i = 0; i < bank->slice_num; i++) + avd_buf_free(ctx->dev, &bank->slices[i]); + bank->slice_num = 0; + bank->slice_bytes = 0; +} + +/* Free the slices held for the picture being built. */ static void avd_h264_drop_slices(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; - for (size_t i = 0; i < h264_ctx->slice_num; i++) - avd_buf_free(ctx->dev, &h264_ctx->slices[i]); - h264_ctx->slice_num = 0; - h264_ctx->slice_bytes = 0; + avd_h264_drop_bank_slices(ctx, h264_ctx->b); } static void avd_h264_stop(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; struct avd_dev *dev = ctx->dev; + unsigned int i; + ctx->max_frames = 1; if (!h264_ctx) return; - avd_buf_free(dev, &h264_ctx->bufs.pipe_state); - avd_buf_free(dev, &h264_ctx->bufs.inst); - avd_buf_free(dev, &h264_ctx->bufs.above_info); - avd_buf_free(dev, &h264_ctx->bufs.lf_above_info); - avd_buf_free(dev, &h264_ctx->bufs.lf_above); - avd_buf_free(dev, &h264_ctx->bufs.ip_above); - avd_buf_free(dev, &h264_ctx->bufs.mv_above_info); - - avd_h264_drop_slices(ctx); - kfree(h264_ctx->slices); + for (i = 0; i < AVD_H264_BANKS; i++) { + struct avd_bank *bank = &h264_ctx->bank[i]; + + avd_buf_free(dev, &bank->bufs.pipe_state); + avd_buf_free(dev, &bank->bufs.inst); + avd_buf_free(dev, &bank->bufs.above_info); + avd_buf_free(dev, &bank->bufs.lf_above_info); + avd_buf_free(dev, &bank->bufs.lf_above); + avd_buf_free(dev, &bank->bufs.ip_above); + avd_buf_free(dev, &bank->bufs.mv_above_info); + avd_h264_drop_bank_slices(ctx, bank); + kfree(bank->slices); + } kfree(h264_ctx); ctx->priv = NULL; @@ -673,20 +701,28 @@ static int avd_h264_start(struct avd_ctx *ctx) return -ENOMEM; ctx->priv = h264_ctx; + /* Overlap pictures only where the firmware chains VP to PP itself. */ + h264_ctx->nbanks = avd_backend_is_qfw(ctx->dev) ? AVD_H264_BANKS : 1; + h264_ctx->b = &h264_ctx->bank[0]; + + for (unsigned int i = 0; i < h264_ctx->nbanks; i++) { + struct avd_bank *bank = &h264_ctx->bank[i]; + + /* assume one slice per row */ + bank->alloc_slice_num = + ctrl->p_new.p_h264_sps->pic_height_in_map_units_minus1 + 1; + bank->slices = kcalloc(bank->alloc_slice_num, + sizeof(*bank->slices), GFP_KERNEL); + if (!bank->slices) { + ret = -ENOMEM; + goto err_free_ctx; + } - /* assume one slice per row */ - h264_ctx->alloc_slice_num = - ctrl->p_new.p_h264_sps->pic_height_in_map_units_minus1 + 1; - h264_ctx->slices = kcalloc(h264_ctx->alloc_slice_num, - sizeof(*h264_ctx->slices), GFP_KERNEL); - if (!h264_ctx->slices) { - ret = -ENOMEM; - goto err_free_ctx; + ret = avd_h264_alloc_bufs(ctx, bank); + if (ret) + goto err_free_ctx; } - - ret = avd_h264_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; + ctx->max_frames = h264_ctx->nbanks; return 0; @@ -732,22 +768,22 @@ static int avd_h264_realloc_slices(struct avd_ctx *ctx, size_t max_slices) void *tmp; size_t alloc_slice_num; - tmp = h264_ctx->slices; + tmp = h264_ctx->b->slices; /* grow by half, and by at least one for a one-row picture */ - alloc_slice_num = h264_ctx->alloc_slice_num + - max_t(size_t, h264_ctx->alloc_slice_num / 2, 1); + alloc_slice_num = h264_ctx->b->alloc_slice_num + + max_t(size_t, h264_ctx->b->alloc_slice_num / 2, 1); alloc_slice_num = min(alloc_slice_num, max_slices); - h264_ctx->slices = kcalloc(alloc_slice_num, sizeof(*h264_ctx->slices), + h264_ctx->b->slices = kcalloc(alloc_slice_num, sizeof(*h264_ctx->b->slices), GFP_KERNEL); - if (!h264_ctx->slices) { + if (!h264_ctx->b->slices) { /* make deallocating a little easier */ - h264_ctx->slices = tmp; + h264_ctx->b->slices = tmp; return -ENOMEM; } - h264_ctx->alloc_slice_num = alloc_slice_num; - memcpy(h264_ctx->slices, tmp, - sizeof(*h264_ctx->slices) * h264_ctx->slice_num); + h264_ctx->b->alloc_slice_num = alloc_slice_num; + memcpy(h264_ctx->b->slices, tmp, + sizeof(*h264_ctx->b->slices) * h264_ctx->b->slice_num); kfree(tmp); /* a slice that arrives before the frame's header has no job table yet */ @@ -853,6 +889,33 @@ VISIBLE_IF_KUNIT bool avd_h264_slice_fits(const struct v4l2_ctrl_h264_sps *sps, avd_h264_slice_budget(decoded_size); } +/* + * Start a new picture in a bank no unretired picture uses, and remember the + * bank in its CAPTURE buffer so retirement frees the right one. + */ +static int avd_h264_pick_bank(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst) +{ + struct avd_h264_ctx *h264_ctx = ctx->priv; + struct avd_bank *bank = NULL; + unsigned int i; + + /* A picture whose last slice never came: start over in its bank. */ + if (h264_ctx->b->busy && !h264_ctx->b->submitted) + bank = h264_ctx->b; + for (i = 0; !bank && i < h264_ctx->nbanks; i++) + if (!h264_ctx->bank[i].busy) + bank = &h264_ctx->bank[i]; + if (!bank) + return -EBUSY; + avd_h264_drop_bank_slices(ctx, bank); + bank->busy = true; + bank->submitted = false; + h264_ctx->b = bank; + /* 0 means no bank, so a stale or missing value never frees one. */ + vb2_to_avd_decoded_buf(&dst->vb2_buf)->bank = bank - h264_ctx->bank + 1; + return 0; +} + static int avd_h264_run(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; @@ -862,6 +925,14 @@ static int avd_h264_run(struct avd_ctx *ctx) avd_h264_run_preamble(ctx, &run); + if (run.slice_params && is_new_frame(run.slice_params)) { + ret = avd_h264_pick_bank(ctx, run.base.bufs.dst); + if (ret) { + avd_run_postamble(ctx, &run.base); + return ret; + } + } + struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); u32 payload_len = vb2_get_plane_payload(&src->vb2_buf, 0); const u8 *data = vb2_plane_vaddr(&src->vb2_buf, 0); @@ -875,8 +946,8 @@ static int avd_h264_run(struct avd_ctx *ctx) return -EINVAL; } - if (!avd_h264_slice_fits(run.sps, h264_ctx->slice_num, - h264_ctx->slice_bytes, payload_len, + if (!avd_h264_slice_fits(run.sps, h264_ctx->b->slice_num, + h264_ctx->b->slice_bytes, payload_len, ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].sizeimage)) { dev_err_ratelimited(ctx->dev->dev, "too many slices for one picture, dropping it\n"); @@ -886,20 +957,20 @@ static int avd_h264_run(struct avd_ctx *ctx) goto err_free_jobs; } - if (h264_ctx->slice_num >= h264_ctx->alloc_slice_num) { + if (h264_ctx->b->slice_num >= h264_ctx->b->alloc_slice_num) { ret = avd_h264_realloc_slices(ctx, avd_h264_max_slices(run.sps)); if (ret) goto err_free_jobs; } - h264_ctx->active_slice = &h264_ctx->slices[h264_ctx->slice_num]; - ret = avd_buf_alloc(ctx->dev, h264_ctx->active_slice, payload_len); + h264_ctx->b->active_slice = &h264_ctx->b->slices[h264_ctx->b->slice_num]; + ret = avd_buf_alloc(ctx->dev, h264_ctx->b->active_slice, payload_len); if (ret) goto err_free_jobs; - memcpy(h264_ctx->active_slice->cpu, data, payload_len); - h264_ctx->slice_num++; - h264_ctx->slice_bytes += PAGE_ALIGN((size_t)payload_len); - h264_ctx->monochrome = run.sps->chroma_format_idc == 0; + memcpy(h264_ctx->b->active_slice->cpu, data, payload_len); + h264_ctx->b->slice_num++; + h264_ctx->b->slice_bytes += PAGE_ALIGN((size_t)payload_len); + h264_ctx->b->monochrome = run.sps->chroma_format_idc == 0; /* Build the P/B{0,1} ref lists. */ v4l2_h264_init_reflist_builder(&reflist_builder, run.decode_params, @@ -923,7 +994,7 @@ static int avd_h264_run(struct avd_ctx *ctx) if (is_new_frame(run.slice_params)) { /* the header segment plus one per slice */ ret = avd_init_job(ctx, AVD_CODEC_H264, - h264_ctx->alloc_slice_num + 1); + h264_ctx->b->alloc_slice_num + 1); if (ret) return ret; stream_hdr(ctx, &run); @@ -940,6 +1011,8 @@ static int avd_h264_run(struct avd_ctx *ctx) return 0; } + ctx->job_fifo = h264_ctx->b->bufs.inst.addr; + h264_ctx->b->submitted = true; return avd_submit_job(ctx); err_free_jobs: @@ -954,22 +1027,32 @@ static void avd_h264_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, enum vb2_buffer_state result) { struct avd_h264_ctx *h264_ctx = ctx->priv; + struct avd_decoded_buffer *buf; + struct avd_bank *bank; - if (!(src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF)) { - avd_h264_drop_slices(ctx); + if (src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF || !dst_buf) + return; - /* 4:0:0 has no chroma to decode; the picture is grey */ - if (h264_ctx->monochrome && dst_buf && result == VB2_BUF_STATE_DONE) { - const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; - struct avd_decoded_buffer *dst = - vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); - - /* vb2 must return cache ownership before the CPU writes. */ - dst->grey_chroma_offset = - (size_t)pix->plane_fmt[0].bytesperline * pix->height; - dst->grey_chroma_10bit = - ctx->image_fmt == AVD_IMG_FMT_420_10BIT; - } + buf = vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + if (!buf->bank || buf->bank > h264_ctx->nbanks) + return; + bank = &h264_ctx->bank[buf->bank - 1]; + buf->bank = 0; + avd_h264_drop_bank_slices(ctx, bank); + bank->busy = false; + bank->submitted = false; + + /* 4:0:0 has no chroma to decode; the picture is grey */ + if (bank->monochrome && result == VB2_BUF_STATE_DONE) { + const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; + struct avd_decoded_buffer *dst = + vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + + /* vb2 must return cache ownership before the CPU writes. */ + dst->grey_chroma_offset = + (size_t)pix->plane_fmt[0].bytesperline * pix->height; + dst->grey_chroma_10bit = + ctx->image_fmt == AVD_IMG_FMT_420_10BIT; } } diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index 9dfa20ab71913..b2ce0f3711390 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -79,10 +79,14 @@ struct avd_hevc_run { struct avd_hevc_tile_info tile_info; }; -struct avd_hevc_ctx { - struct v4l2_ctrl_hevc_scaling_matrix scaling_matrix_cache; +/* See avd-h264.c: one bank per picture in flight on the queued transport. */ +#define AVD_HEVC_BANKS 2 +struct avd_hevc_bank { + bool busy; struct avd_h264_bufs { + /* VP to PP instruction FIFO for pictures of this bank */ + struct avd_buf inst; struct avd_buf mv_above_info; struct avd_buf az_above; struct avd_buf ip_above; @@ -91,9 +95,17 @@ struct avd_hevc_ctx { struct avd_buf lf_left; struct avd_buf lf_left_info; struct avd_buf sw_left; - struct avd_buf inst; struct avd_buf pipe_state; } bufs; +}; + +struct avd_hevc_ctx { + struct v4l2_ctrl_hevc_scaling_matrix scaling_matrix_cache; + + unsigned int nbanks; + struct avd_hevc_bank bank[AVD_HEVC_BANKS]; + /* bank of the picture being built */ + struct avd_hevc_bank *b; int submit_num; }; @@ -110,7 +122,7 @@ static void stream_refs(struct avd_ctx *ctx, struct avd_hevc_run *run) push(ctx->dev->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER ? HEVC_DMA_CONFIG : 0, "reference_dma_config"); - pusha(hevc_ctx->bufs.mv_above_info.addr, "mv_above_info", 7); + pusha(hevc_ctx->b->bufs.mv_above_info.addr, "mv_above_info", 7); pusha(run->addresses.mv_color, "mv_color", 0); push(0, ""); @@ -357,30 +369,30 @@ static void set_header(struct avd_ctx *ctx, struct avd_hevc_run *run) if (avd->variant->revision == 3) push(0, "zero"); else if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) - pusha(hevc_ctx->bufs.pipe_state.addr, "pipe_state", 0); + pusha(hevc_ctx->b->bufs.pipe_state.addr, "pipe_state", 0); - pusha(hevc_ctx->bufs.ip_above.addr, "scratch_addr", 0); - pusha(hevc_ctx->bufs.lf_above.addr, "scratch_addr", 1); - pusha(hevc_ctx->bufs.lf_above_info.addr, "scratch_addr", 2); + pusha(hevc_ctx->b->bufs.ip_above.addr, "scratch_addr", 0); + pusha(hevc_ctx->b->bufs.lf_above.addr, "scratch_addr", 1); + pusha(hevc_ctx->b->bufs.lf_above_info.addr, "scratch_addr", 2); if ((pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) || !(avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER)) { - pusha(hevc_ctx->bufs.lf_left.addr, + pusha(hevc_ctx->b->bufs.lf_left.addr, "scratch_addr", 3); - pusha(hevc_ctx->bufs.lf_left_info.addr, + pusha(hevc_ctx->b->bufs.lf_left_info.addr, "scratch_addr", 4); if (avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER) pusha(0, "reserved_scratch", 0); - pusha(hevc_ctx->bufs.az_above.addr, + pusha(hevc_ctx->b->bufs.az_above.addr, "scratch_addr", 8); - pusha(hevc_ctx->bufs.sw_left.addr, + pusha(hevc_ctx->b->bufs.sw_left.addr, "scratch_addr", 9); } else { pusha(0, "", 3); pusha(0, "", 4); if (avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER) pusha(0, "reserved_scratch", 0); - pusha(hevc_ctx->bufs.az_above.addr, + pusha(hevc_ctx->b->bufs.az_above.addr, "scratch_addr", 8); pusha(0, "", 9); } @@ -1056,17 +1068,16 @@ static int avd_hevc_validate_sps(struct avd_ctx *ctx, return 0; } -static int avd_hevc_alloc_bufs(struct avd_ctx *ctx) +static int avd_hevc_alloc_bufs(struct avd_ctx *ctx, struct avd_hevc_bank *bank) { struct avd_dev *avd = ctx->dev; - struct avd_hevc_ctx *hevc_ctx = ctx->priv; int ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.inst, fifo_size()); + ret = avd_buf_alloc(avd, &bank->bufs.inst, fifo_size()); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.pipe_state, 0x200); + ret = avd_buf_alloc(avd, &bank->bufs.pipe_state, 0x200); if (ret) return ret; @@ -1077,20 +1088,26 @@ static void avd_hevc_stop(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; struct avd_dev *avd = ctx->dev; + unsigned int i; + ctx->max_frames = 1; if (!hevc_ctx) return; - avd_buf_free(avd, &hevc_ctx->bufs.pipe_state); - avd_buf_free(avd, &hevc_ctx->bufs.inst); - avd_buf_free(avd, &hevc_ctx->bufs.mv_above_info); - avd_buf_free(avd, &hevc_ctx->bufs.az_above); - avd_buf_free(avd, &hevc_ctx->bufs.ip_above); - avd_buf_free(avd, &hevc_ctx->bufs.lf_above); - avd_buf_free(avd, &hevc_ctx->bufs.lf_above_info); - avd_buf_free(avd, &hevc_ctx->bufs.lf_left); - avd_buf_free(avd, &hevc_ctx->bufs.lf_left_info); - avd_buf_free(avd, &hevc_ctx->bufs.sw_left); + for (i = 0; i < AVD_HEVC_BANKS; i++) { + struct avd_h264_bufs *bufs = &hevc_ctx->bank[i].bufs; + + avd_buf_free(avd, &bufs->pipe_state); + avd_buf_free(avd, &bufs->inst); + avd_buf_free(avd, &bufs->mv_above_info); + avd_buf_free(avd, &bufs->az_above); + avd_buf_free(avd, &bufs->ip_above); + avd_buf_free(avd, &bufs->lf_above); + avd_buf_free(avd, &bufs->lf_above_info); + avd_buf_free(avd, &bufs->lf_left); + avd_buf_free(avd, &bufs->lf_left_info); + avd_buf_free(avd, &bufs->sw_left); + } kfree(hevc_ctx); ctx->priv = NULL; @@ -1099,6 +1116,7 @@ static void avd_hevc_stop(struct avd_ctx *ctx) static int avd_hevc_start(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx; + unsigned int i; int ret; hevc_ctx = kzalloc(sizeof(*hevc_ctx), GFP_KERNEL); @@ -1106,9 +1124,15 @@ static int avd_hevc_start(struct avd_ctx *ctx) return -ENOMEM; ctx->priv = hevc_ctx; - ret = avd_hevc_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; + /* Overlap pictures only where the firmware chains VP to PP itself. */ + hevc_ctx->nbanks = avd_backend_is_qfw(ctx->dev) ? AVD_HEVC_BANKS : 1; + hevc_ctx->b = &hevc_ctx->bank[0]; + for (i = 0; i < hevc_ctx->nbanks; i++) { + ret = avd_hevc_alloc_bufs(ctx, &hevc_ctx->bank[i]); + if (ret) + goto err_free_ctx; + } + ctx->max_frames = hevc_ctx->nbanks; return 0; @@ -1117,6 +1141,38 @@ static int avd_hevc_start(struct avd_ctx *ctx) return ret; } +/* Take a bank no unretired picture uses; record it in the CAPTURE buffer. */ +static int avd_hevc_pick_bank(struct avd_ctx *ctx, struct avd_decoded_buffer *dst) +{ + struct avd_hevc_ctx *hevc_ctx = ctx->priv; + unsigned int i; + + for (i = 0; i < hevc_ctx->nbanks; i++) { + if (hevc_ctx->bank[i].busy) + continue; + hevc_ctx->bank[i].busy = true; + hevc_ctx->b = &hevc_ctx->bank[i]; + dst->bank = i + 1; + return 0; + } + return -EBUSY; +} + +static void avd_hevc_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, + struct vb2_v4l2_buffer *dst_buf, + enum vb2_buffer_state result) +{ + struct avd_hevc_ctx *hevc_ctx = ctx->priv; + struct avd_decoded_buffer *buf; + + if (!dst_buf || !hevc_ctx) + return; + buf = vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + if (buf->bank && buf->bank <= hevc_ctx->nbanks) + hevc_ctx->bank[buf->bank - 1].busy = false; + buf->bank = 0; +} + static int avd_hevc_alloc_scratch(struct avd_ctx *ctx, struct avd_hevc_run *run) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; @@ -1152,51 +1208,51 @@ static int avd_hevc_alloc_scratch(struct avd_ctx *ctx, struct avd_hevc_run *run) tile_info->row_height[i] : max_row; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.ip_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.ip_above, ((max_col * max_cu_width) / 4) * bit_depth); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.mv_above_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.mv_above_info, ((max_col * max_cu_width) / 16) * 20); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_above, DIV_ROUND_UP(w, 16) * bit_depth * (10 + 6) + (cols - 1) * 256); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_above_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_above_info, DIV_ROUND_UP(w + 7, 16) * 36 + cols * 128); if (ret) return ret; if (pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) { - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_left, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_left, ((max_row * max_cu_width) / 4) * 36 + 144 /* why? */); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_left_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_left_info, ((max_row * max_cu_width) / 4) * 9); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.az_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.az_above, /* not sure if its cols or rows */ DIV_ROUND_UP(w, 64) * 144 + (cols - 1) * 128); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.sw_left, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.sw_left, ((max_row * max_cu_width) / 4) * 216); if (ret) return ret; } else { - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.az_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.az_above, DIV_ROUND_UP(w + 7, 16) * 4 * bit_depth); if (ret) return ret; @@ -1282,6 +1338,51 @@ static int avd_hevc_compute_tiles(struct avd_ctx *ctx, struct avd_hevc_run *run) return 0; } +/* + * The slice builder below expects the GStreamer layout: the slices tile the + * OUTPUT buffer as whole NALs, bit_size covers the whole NAL and + * data_byte_offset is the slice data's offset inside it. FFmpeg's request + * API backend instead gives each slice's data_byte_offset from the start of + * the buffer and a bit_size of the slice data only, so every slice was given + * a wrong address and size and the VP raised its error interrupt on every + * frame. Detect the absolute layout (complete and unambiguous) and rewrite + * the run's private copy into the equivalent NAL-relative description. + */ +static void avd_hevc_normalize_slices(struct avd_hevc_run *run, u32 payload) +{ + struct v4l2_ctrl_hevc_slice_params *sl = + (struct v4l2_ctrl_hevc_slice_params *)run->sl; + u64 nal_end = 0, data_end = 0, prev_end = 0; + bool nal = true, absolute = true; + int i; + + if (!run->num_slices || !payload) + return; + for (i = 0; i < run->num_slices; i++) { + u64 bytes = DIV_ROUND_UP((u64)sl[i].bit_size, 8); + u64 start = sl[i].data_byte_offset; + + if (!bytes || start >= bytes || nal_end + bytes > payload) + nal = false; + nal_end += bytes; + if (!bytes || start < data_end || start + bytes > payload) + absolute = false; + data_end = start + bytes; + } + nal &= nal_end == payload; + absolute &= data_end == payload; + if (!absolute || nal) + return; + for (i = 0; i < run->num_slices; i++) { + u64 bytes = DIV_ROUND_UP((u64)sl[i].bit_size, 8); + u64 start = sl[i].data_byte_offset; + + sl[i].data_byte_offset = start - prev_end; + sl[i].bit_size = (start - prev_end + bytes) * 8; + prev_end = start + bytes; + } +} + static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) { struct v4l2_ctrl *ctrl, *sl, *ep; @@ -1345,6 +1446,7 @@ static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) static int avd_hevc_run(struct avd_ctx *ctx) { + struct avd_hevc_ctx *hevc_ctx = ctx->priv; struct avd_hevc_run run = {}; struct avd_decoded_buffer *dst; int ret; @@ -1353,9 +1455,15 @@ static int avd_hevc_run(struct avd_ctx *ctx) if (ret) return ret; + avd_hevc_normalize_slices(&run, + vb2_get_plane_payload(&run.base.bufs.src->vb2_buf, 0)); dst = vb2_to_avd_decoded_buf(&run.base.bufs.dst->vb2_buf); update_dec_buf_info(dst, &run.sl[0]); + ret = avd_hevc_pick_bank(ctx, dst); + if (ret) + goto complete; + ret = avd_hevc_compute_tiles(ctx, &run); if (ret) goto complete; @@ -1372,8 +1480,10 @@ static int avd_hevc_run(struct avd_ctx *ctx) set_header(ctx, &run); ret = stream_slices(ctx, &run); avd_run_postamble(ctx, &run.base); - if (!ret) + if (!ret) { + ctx->job_fifo = hevc_ctx->b->bufs.inst.addr; ret = avd_submit_job(ctx); + } goto done; complete: @@ -1395,6 +1505,13 @@ static int avd_hevc_try_ctrl(struct avd_ctx *ctx, struct v4l2_ctrl *ctrl) return 0; } +static unsigned int avd_hevc_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_hevc_ctx *priv = ctx->priv; + + return max_t(int, priv->submit_num, 1); +} + static void avd_hevc_submit(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; @@ -1420,7 +1537,9 @@ const struct avd_coded_fmt_ops avd_hevc_fmt_ops = { .start = avd_hevc_start, .stop = avd_hevc_stop, .run = avd_hevc_run, + .done = avd_hevc_done, .submit = avd_hevc_submit, + .pp_kicks = avd_hevc_pp_kicks, .try_ctrl = avd_hevc_try_ctrl, .get_image_fmt = avd_hevc_get_image_fmt, }; diff --git a/drivers/media/platform/apple/avd/avd-hw.c b/drivers/media/platform/apple/avd/avd-hw.c index a3987ac4510de..b64c0bf4d78ea 100644 --- a/drivers/media/platform/apple/avd/avd-hw.c +++ b/drivers/media/platform/apple/avd/avd-hw.c @@ -42,9 +42,12 @@ int avd_boot(struct avd_dev *avd) { + struct resource *code = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "code"); u32 val; int ret; + if (!avd->fw || !avd->fw->size || !code || avd->fw->size > resource_size(code)) + return -EINVAL; if (avd->variant->revision != 3) dev_info_once(avd->dev, "booting hw version: %04x", readl_relaxed(avd->ctrl)); @@ -64,6 +67,28 @@ int avd_boot(struct avd_dev *avd) return 0; } +/* + * Pop one firmware message. Reading RETRIEVE on an empty mailbox returns the + * previous message again, and a repeated PP DONE completes the next frame + * before its PP has run. An interrupt can also come just before its message + * is readable, so allow 10 us; after that treat the interrupt as spurious. + * Always acknowledges the interrupt. Returns false when there is no message. + */ +bool avd_mbox_pop(struct avd_dev *avd, u32 *msg) +{ + u32 st; + int ret; + + ret = readl_poll_timeout_atomic(avd->mbox + AVD_REG_MBOX1_STATUS, st, + !(st & AVD_MBOX1_STATUS_EMPTY), 1, 10); + if (!ret) + *msg = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); + writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); + if (ret) + dev_dbg_ratelimited(avd->dev, "spurious interrupt, mailbox empty\n"); + return !ret; +} + void avd_shutdown(struct avd_dev *avd) { writel_relaxed(AVD_RUN_CTRL_UNK_STOP, avd->mbox + AVD_REG_RUN_CTRL); diff --git a/drivers/media/platform/apple/avd/avd-inst.h b/drivers/media/platform/apple/avd/avd-inst.h index e9b9ef908b8d5..dcdb205d79315 100644 --- a/drivers/media/platform/apple/avd/avd-inst.h +++ b/drivers/media/platform/apple/avd/avd-inst.h @@ -3,6 +3,7 @@ #ifndef AVD_INST_H_ #define AVD_INST_H_ /* instruction stream things */ +#include #include #include "avd.h" diff --git a/drivers/media/platform/apple/avd/avd-pm-test.c b/drivers/media/platform/apple/avd/avd-pm-test.c new file mode 100644 index 0000000000000..6756d036b3f76 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm-test.c @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include + +#include "avd.h" +#include "avd-regs.h" + +struct avd_pm_test { + struct avd_dev avd; + bool queue_initialized; + struct avd_deferred_unmap unmap; + unsigned int draining; + unsigned int step; + unsigned int unmap_step; + unsigned int boot_step; + unsigned int resubmit_step; + unsigned int shutdowns; + unsigned int submissions; + int boot_error; + int suspend_error; + bool unmapped_active; +}; + +static struct avd_pm_test *avd_pm_fixture(struct avd_dev *avd) +{ + return container_of(avd, struct avd_pm_test, avd); +} + +static bool avd_pm_test_idle(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + return avd_pm_can_suspend(fixture->avd.frames, fixture->draining); +} + +static int avd_pm_test_drain(struct avd_dev *avd) +{ + return avd_pm_test_idle(avd) ? 0 : -EBUSY; +} + +static int avd_pm_test_boot(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->boot_step = ++fixture->step; + return fixture->boot_error; +} + +static void avd_pm_test_shutdown(struct avd_dev *avd) +{ + avd_pm_fixture(avd)->shutdowns++; +} + +static int avd_pm_test_suspend(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + if (fixture->suspend_error) + return fixture->suspend_error; + return avd_pm_backend_suspend(avd); +} + +static int avd_pm_test_submit(struct avd_dev *avd, struct avd_frame *frame) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->resubmit_step = ++fixture->step; + fixture->submissions++; + frame->state = AVD_FRAME_SUBMITTED; + return 0; +} + +static void avd_pm_test_resubmit(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + unsigned int i; + + fixture->resubmit_step = ++fixture->step; + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (fixture->avd.frames[i].state == AVD_FRAME_RESERVED) + avd->backend->submit(avd, &fixture->avd.frames[i]); +} + +static void avd_pm_test_unmap(struct avd_dev *avd, + struct avd_deferred_unmap *unmap) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->unmap_step = ++fixture->step; + fixture->unmapped_active = avd->pm.dart_active; +} + +static const struct avd_queue_pm_ops avd_pm_test_queue = { + .idle = avd_pm_test_idle, + .drain = avd_pm_test_drain, + .resubmit = avd_pm_test_resubmit, +}; + +static const struct avd_backend_ops avd_pm_test_backend = { + .name = "test", + .boot = avd_pm_test_boot, + .shutdown = avd_pm_test_shutdown, + .submit = avd_pm_test_submit, + .suspend = avd_pm_test_suspend, + .resume = avd_pm_backend_resume, +}; + +static int avd_pm_test_init(struct kunit *test) +{ + struct avd_pm_test *fixture; + + fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + if (!fixture) + return -ENOMEM; + test->priv = fixture; + avd_pm_init(&fixture->avd, &avd_pm_test_backend, &avd_pm_test_queue); + INIT_LIST_HEAD(&fixture->unmap.node); + fixture->unmap.unmap = avd_pm_test_unmap; + return 0; +} + +static void avd_pm_suspend_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + unsigned int i, state; + + KUNIT_EXPECT_TRUE(test, avd_pm_test_idle(&fixture->avd)); + fixture->draining = 1; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), -EBUSY); + fixture->draining = 0; + + /* An occupied slot anywhere vetoes idle, even after completion IRQ. */ + for (i = 0; i < AVD_MAX_FRAMES; i++) { + for (state = AVD_FRAME_RESERVED; state <= AVD_FRAME_DONE; state++) { + fixture->avd.frames[i].state = state; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), -EBUSY); + } + fixture->avd.frames[i].state = AVD_FRAME_FREE; + } + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), 0); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); +} + +static void avd_pm_resume_order_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_EQ(test, fixture->unmap_step, 0U); + fixture->avd.frames[0].state = AVD_FRAME_RESERVED; + fixture->avd.frames[1].state = AVD_FRAME_SUBMITTED; + fixture->avd.frames[2].state = AVD_FRAME_DONE; + fixture->avd.frames[15].state = AVD_FRAME_RESERVED; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_TRUE(test, fixture->unmapped_active); + KUNIT_EXPECT_LT(test, fixture->unmap_step, fixture->boot_step); + KUNIT_EXPECT_LT(test, fixture->boot_step, fixture->resubmit_step); + KUNIT_EXPECT_EQ(test, fixture->submissions, 2U); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[0].state, AVD_FRAME_SUBMITTED); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[2].state, AVD_FRAME_DONE); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); +} + +static void avd_pm_resume_failure_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + fixture->avd.frames[0].state = AVD_FRAME_RESERVED; + fixture->boot_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_power_on(avd), -ETIMEDOUT); + KUNIT_EXPECT_FALSE(test, avd->pm.dart_active); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[0].state, AVD_FRAME_RESERVED); + + /* A later successful resume retries the still-unposted frame once. */ + fixture->boot_error = 0; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_EQ(test, fixture->submissions, 1U); +} + +static void avd_pm_suspend_failure_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + fixture->suspend_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, avd->pm.dart_active); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); +} + +static void avd_pm_deferred_unmap_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + unsigned int step; + + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_TRUE(test, fixture->unmapped_active); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); + KUNIT_ASSERT_EQ(test, avd_pm_power_off(avd), 0); + step = fixture->unmap_step; + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_EQ(test, fixture->unmap_step, step); + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_GT(test, fixture->unmap_step, step); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); +} + +static void avd_pm_real_queue_resume_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct avd_ctx *ctx; + unsigned int i; + + avd->dev = kunit_device_register(test, "avd-pm-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, avd->dev); + KUNIT_ASSERT_EQ(test, avd_queue_init(avd), 0); + fixture->queue_initialized = true; + avd->backend = &avd_pm_test_backend; + avd->pm.queue = &avd_pm_queue_ops; + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->dev = avd; + for (i = 0; i < 3; i++) { + avd->frames[i].ctx = ctx; + avd->frames[i].seq = i + 1; + avd->frames[i].deadline = jiffies + msecs_to_jiffies(60000); + avd->frames[i].state = AVD_FRAME_RESERVED; + } + /* This one was posted before the power transition; never replay it. */ + avd->frames[1].state = AVD_FRAME_SUBMITTED; + avd->inflight = 3; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + flush_delayed_work(&avd->queue_work); + cancel_delayed_work_sync(&avd->queue_work); + KUNIT_EXPECT_EQ(test, fixture->submissions, 2U); + KUNIT_EXPECT_LT(test, fixture->boot_step, fixture->resubmit_step); + KUNIT_EXPECT_EQ(test, avd->frames[0].state, AVD_FRAME_SUBMITTED); + KUNIT_EXPECT_EQ(test, avd->frames[2].state, AVD_FRAME_SUBMITTED); +} + +static void avd_pm_real_queue_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + spin_lock_init(&avd->queue_lock); + avd->pm.queue = &avd_pm_queue_ops; + avd_pm_drain_begin(avd); + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -EBUSY); + avd_pm_drain_end(avd); + avd->frames[15].state = AVD_FRAME_DONE; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -EBUSY); + avd->frames[15].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), 0); +} + +static void avd_pm_open_boot_replay_test(struct kunit *test) +{ + static const struct avd_variant variant = { .revision = 3 }; + static const u8 image[] = { 0x12, 0x34, 0x56, 0x78 }; + static const struct firmware fw = { .data = image, .size = sizeof(image) }; + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct resource code = { + .name = "code", .flags = IORESOURCE_MEM, + .start = 0x1000, .end = 0x10ff, + }; + struct platform_device *pdev; + u8 observed[sizeof(image)]; + unsigned int i; + + pdev = kunit_kzalloc(test, sizeof(*pdev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pdev); + pdev->resource = &code; + pdev->num_resources = 1; + avd->pdev = pdev; + avd->variant = &variant; + avd->fw = &fw; + avd->backend = &avd_open_backend; + /* Ordinary test RAM, never a mapped target aperture. */ + avd->code = (__force void __iomem *)kunit_kzalloc(test, 0x100, GFP_KERNEL); + avd->mbox = (__force void __iomem *)kunit_kzalloc(test, 0x100, GFP_KERNEL); + avd->pm.piodma = (__force void __iomem *)kunit_kzalloc(test, 0x28, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, avd->code); + KUNIT_ASSERT_NOT_NULL(test, avd->mbox); + KUNIT_ASSERT_NOT_NULL(test, avd->pm.piodma); + avd->pm.sram_phys = 0x123456780ULL; + for (i = 0; i < 2; i++) { + /* Model loss of both ITCM and PIODMA configuration between boots. */ + memset_io(avd->code, 0, sizeof(image)); + writel(0, avd->pm.piodma + 0x24); + /* The fixture supplies firmware readiness; no firmware is executed. */ + writel(1, avd->mbox + AVD_REG_FLAG0_SET); + KUNIT_ASSERT_EQ(test, avd_backend_boot(avd), 0); + KUNIT_EXPECT_EQ(test, readl(avd->pm.piodma + 0x24), 0x12345678U); + memcpy_fromio(observed, avd->code, sizeof(observed)); + KUNIT_EXPECT_MEMEQ(test, observed, image, sizeof(image)); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_RUN_CTRL), + (u32)AVD_RUN_CTRL_UNK_RUN); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX1_STATUS), + (u32)AVD_MBOX_ENABLE); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX_IRQ_ENABLE), + (u32)AVD_MBOX1_NOT_EMPTY); + avd_shutdown(avd); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_RUN_CTRL), + (u32)(AVD_RUN_CTRL_UNK_STOP)); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_FLAG0_CLR), 1U); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX_IRQ_ENABLE), 0U); + } +} + +static void avd_pm_recovery_dart_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct iommu_domain domain = {}; + + /* A stopped device cannot be reset through powered-off registers. */ + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); + avd->pm.dart_active = true; + avd->domain = &domain; + /* No empty domain means no valid DART flush: never reboot anyway. */ + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -ENODEV); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); + KUNIT_EXPECT_EQ(test, fixture->boot_step, 0U); +} + +static void avd_pm_recovery_boot_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + avd->pm.dart_active = true; + fixture->boot_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 2U); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); + fixture->boot_error = 0; + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), 0); + KUNIT_EXPECT_EQ(test, avd->fw_generation, 1ULL); + /* Recovery must fail old slots before core reopens the host IRQ. */ + KUNIT_EXPECT_FALSE(test, avd->pm.irq_enabled); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); +} + +static void avd_pm_test_exit(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + + if (fixture->queue_initialized) + cancel_delayed_work_sync(&fixture->avd.queue_work); +} + +static struct kunit_case avd_pm_test_cases[] = { + KUNIT_CASE(avd_pm_suspend_gate_test), + KUNIT_CASE(avd_pm_resume_order_test), + KUNIT_CASE(avd_pm_resume_failure_test), + KUNIT_CASE(avd_pm_suspend_failure_test), + KUNIT_CASE(avd_pm_deferred_unmap_test), + KUNIT_CASE(avd_pm_real_queue_resume_test), + KUNIT_CASE(avd_pm_real_queue_gate_test), + KUNIT_CASE(avd_pm_open_boot_replay_test), + KUNIT_CASE(avd_pm_recovery_dart_gate_test), + KUNIT_CASE(avd_pm_recovery_boot_test), + {} +}; + +static struct kunit_suite avd_pm_test_suite = { + .name = "apple-avd-pm", + .init = avd_pm_test_init, + .exit = avd_pm_test_exit, + .test_cases = avd_pm_test_cases, +}; + +kunit_test_suite(avd_pm_test_suite); diff --git a/drivers/media/platform/apple/avd/avd-pm.c b/drivers/media/platform/apple/avd/avd-pm.c new file mode 100644 index 0000000000000..3876aec76f2ee --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.c @@ -0,0 +1,450 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include +#include + +#include "avd.h" + +/* [C] Keep Linux's existing default until the idle-timer capture is available. */ +#define AVD_AUTOSUSPEND_MS 100 +#define AVD_DRAIN_TIMEOUT_MS 5000 +#define AVD_PIODMA_SRAM_BASE 0x24 + +void avd_pm_enable_irq(struct avd_dev *avd) +{ + unsigned long flags; + + if (avd->pm.irq >= 0 && !avd->pm.irq_enabled) { + spin_lock_irqsave(&avd->pm.irq_lock, flags); + avd->pm.hw_on = true; + spin_unlock_irqrestore(&avd->pm.irq_lock, flags); + enable_irq(avd->pm.irq); + avd->pm.irq_enabled = true; + } +} + +/* + * Callers may hold queue_mutex, which the threaded handler takes first, so + * never wait for the handler here (disable_irq() would deadlock). Instead + * close hardware access under irq_lock: a handler already touching the + * decoder finishes before this returns, and any later one sees hw_on clear + * or a new irq_gen and leaves the hardware alone. + */ +static void avd_pm_disable_irq(struct avd_dev *avd) +{ + unsigned long flags; + + if (avd->pm.irq >= 0 && avd->pm.irq_enabled) { + spin_lock_irqsave(&avd->pm.irq_lock, flags); + avd->pm.hw_on = false; + avd->pm.irq_gen++; + spin_unlock_irqrestore(&avd->pm.irq_lock, flags); + disable_irq_nosync(avd->pm.irq); + avd->pm.irq_enabled = false; + } +} + +/* Run the backend's interrupt handler only against a live firmware instance. */ +irqreturn_t avd_pm_irq(struct avd_dev *avd, unsigned int gen) +{ + irqreturn_t ret = IRQ_HANDLED; + unsigned long flags; + + spin_lock_irqsave(&avd->pm.irq_lock, flags); + if (avd->pm.hw_on && gen == avd->pm.irq_gen) + ret = avd->backend->irq(avd); + spin_unlock_irqrestore(&avd->pm.irq_lock, flags); + return ret; +} + +struct avd_dma_release { + struct avd_deferred_unmap unmap; + void *cpu; + dma_addr_t addr; + size_t size; +}; + +static void avd_dma_unmap(struct avd_dev *avd, + struct avd_deferred_unmap *unmap) +{ + struct avd_dma_release *release; + + release = container_of(unmap, struct avd_dma_release, unmap); + dma_free_coherent(avd->dev, release->size, release->cpu, release->addr); + kfree(release); +} + +int avd_buf_alloc(struct avd_dev *avd, struct avd_buf *buf, size_t size) +{ + struct avd_dma_release *release; + + avd_buf_free(avd, buf); + if (!size) + return -ENOMEM; + + /* Allocate the release record now: freeing must never need to allocate. */ + release = kzalloc_obj(*release); + if (!release) + return -ENOMEM; + INIT_LIST_HEAD(&release->unmap.node); + release->unmap.unmap = avd_dma_unmap; + release->size = size; + release->cpu = dma_alloc_coherent(avd->dev, size, &release->addr, GFP_KERNEL); + if (!release->cpu) { + kfree(release); + return -ENOMEM; + } + + /* Payload length, not an allocation rounding, is used by codec packers. */ + buf->cpu = release->cpu; + buf->addr = release->addr; + buf->size = size; + buf->release = release; + return 0; +} + +void avd_buf_free(struct avd_dev *avd, struct avd_buf *buf) +{ + if (buf->release) + avd_pm_unmap(avd, &buf->release->unmap); + /* The record outlives the context when unmapping has to be deferred. */ + memset(buf, 0, sizeof(*buf)); +} + +bool avd_pm_can_suspend(const struct avd_frame *frames, unsigned int draining) +{ + unsigned int i; + + if (draining) + return false; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (frames[i].state != AVD_FRAME_FREE) + return false; + + return true; +} + +static bool avd_pm_queue_idle(struct avd_dev *avd) +{ + unsigned long flags; + bool idle; + + spin_lock_irqsave(&avd->queue_lock, flags); + idle = avd_pm_can_suspend(avd->frames, avd->pm.draining); + spin_unlock_irqrestore(&avd->queue_lock, flags); + return idle; +} + +void avd_pm_queue_changed(struct avd_dev *avd) +{ + wake_up_all(&avd->pm.idle_wait); +} + +void avd_pm_drain_begin(struct avd_dev *avd) +{ + unsigned long flags; + + spin_lock_irqsave(&avd->queue_lock, flags); + avd->pm.draining++; + spin_unlock_irqrestore(&avd->queue_lock, flags); +} + +void avd_pm_drain_end(struct avd_dev *avd) +{ + unsigned long flags; + + spin_lock_irqsave(&avd->queue_lock, flags); + avd->pm.draining--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); +} + +static void avd_pm_resubmit(struct avd_dev *avd) +{ + /* queue_mutex may be held by the caller of pm_runtime_resume_and_get(). */ + mod_delayed_work(system_wq, &avd->queue_work, 0); +} + +static int avd_pm_queue_drain(struct avd_dev *avd) +{ + avd_pm_resubmit(avd); + if (!wait_event_timeout(avd->pm.idle_wait, avd_pm_queue_idle(avd), + msecs_to_jiffies(AVD_DRAIN_TIMEOUT_MS))) + return -EBUSY; + return 0; +} + +const struct avd_queue_pm_ops avd_pm_queue_ops = { + .idle = avd_pm_queue_idle, + .drain = avd_pm_queue_drain, + .resubmit = avd_pm_resubmit, +}; + +void avd_pm_init(struct avd_dev *avd, const struct avd_backend_ops *backend, + const struct avd_queue_pm_ops *queue) +{ + mutex_init(&avd->pm.lock); + INIT_LIST_HEAD(&avd->pm.deferred_unmaps); + init_waitqueue_head(&avd->pm.idle_wait); + avd->pm.dart_active = false; + avd->pm.irq = -1; + avd->pm.irq_enabled = false; + spin_lock_init(&avd->pm.irq_lock); + avd->pm.hw_on = false; + avd->pm.irq_gen = 0; + avd->backend = backend; + avd->pm.queue = queue; +} + +void avd_pm_enable(struct avd_dev *avd) +{ + pm_runtime_set_autosuspend_delay(avd->dev, AVD_AUTOSUSPEND_MS); + pm_runtime_use_autosuspend(avd->dev); + pm_runtime_enable(avd->dev); +} + +int avd_pm_map_piodma(struct avd_dev *avd) +{ + struct resource *piodma, *sram; + + piodma = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "piodma"); + /* Old DTs support only the legacy transport; never invent a resource. */ + if (!piodma) + return 0; + sram = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "sram"); + if (!sram || !IS_ALIGNED(sram->start, 16) || + (sram->start >> 4) > U32_MAX || + resource_size(piodma) < AVD_PIODMA_SRAM_BASE + sizeof(u32)) + return -EINVAL; + avd->pm.piodma = devm_ioremap_resource(avd->dev, piodma); + if (IS_ERR(avd->pm.piodma)) + return PTR_ERR(avd->pm.piodma); + avd->pm.sram_phys = sram->start; + if (of_count_phandle_with_args(avd->dev->of_node, "iommus", "#iommu-cells") < 2) + dev_warn(avd->dev, "PIODMA transport requires decode and DMA IOMMU streams\n"); + return 0; +} + +void avd_pm_prepare_piodma(struct avd_dev *avd) +{ + /* Public CM3 boot ABI: replay the SRAM base after every power loss/reset. */ + if (avd->pm.piodma) + writel((u32)(avd->pm.sram_phys >> 4), + avd->pm.piodma + AVD_PIODMA_SRAM_BASE); +} + +void avd_pm_disable(struct avd_dev *avd) +{ + int ret; + + /* Power suppliers before flushing frees queued by cleanup while idle. */ + ret = pm_runtime_resume_and_get(avd->dev); + if (ret >= 0) + ret = pm_runtime_put_sync_suspend(avd->dev); + if (ret < 0) + dev_warn(avd->dev, "final power transition failed: %d\n", ret); + pm_runtime_disable(avd->dev); + /* Also close the host IRQ if the final hardware transition failed. */ + avd_pm_disable_irq(avd); + /* No queue_mutex is held here: let a woken handler thread finish. */ + if (avd->pm.irq >= 0) + synchronize_irq(avd->pm.irq); + pm_runtime_dont_use_autosuspend(avd->dev); + /* Resume may have kicked the already-stopped queue. */ + cancel_delayed_work_sync(&avd->queue_work); + WARN_ON(!list_empty(&avd->pm.deferred_unmaps)); +} + +/* pm.lock held; release records own their storage and may free themselves. */ +static void avd_pm_drain_unmaps(struct avd_dev *avd) +{ + struct avd_deferred_unmap *unmap, *next; + + list_for_each_entry_safe(unmap, next, &avd->pm.deferred_unmaps, node) { + list_del_init(&unmap->node); + unmap->unmap(avd, unmap); + } +} + +void avd_pm_unmap(struct avd_dev *avd, struct avd_deferred_unmap *unmap) +{ + mutex_lock(&avd->pm.lock); + if (avd->pm.dart_active) + unmap->unmap(avd, unmap); + else + list_add_tail(&unmap->node, &avd->pm.deferred_unmaps); + mutex_unlock(&avd->pm.lock); +} + +int avd_pm_backend_suspend(struct avd_dev *avd) +{ + avd->backend->shutdown(avd); + return 0; +} + +int avd_pm_backend_resume(struct avd_dev *avd) +{ + /* Includes transport setup, ITCM verification/load and a new generation. */ + return avd_backend_boot(avd); +} + +int avd_pm_recover(struct avd_dev *avd) +{ + bool suspended; + int ret; + + /* + * Recovery is rare and touches the decoder, its reset and its DART: + * leave a step-by-step trail in the log before each step, so a failure + * that takes the machine down still shows how far recovery got. + */ + if (!avd->pm.dart_active) { + dev_err(avd->dev, "recover: decoder DART inactive, not touching it\n"); + return -EHOSTDOWN; + } + /* The PM KUnit fixture has no struct device. */ + suspended = avd->dev && pm_runtime_suspended(avd->dev); + dev_warn(avd->dev, "recover: start (runtime %s, irq %s)\n", + suspended ? "suspended" : "active", + avd->pm.irq_enabled ? "on" : "off"); + /* Every caller holds a runtime PM reference; anything else is a bug. */ + if (WARN_ONCE(suspended, + "avd: recovery with the decoder powered off\n")) + return -EHOSTDOWN; + avd_pm_disable_irq(avd); + dev_warn(avd->dev, "recover: stop firmware\n"); + avd->backend->shutdown(avd); + dev_warn(avd->dev, "recover: reset\n"); + ret = reset_control_reset(avd->rstc); + if (ret) { + dev_err(avd->dev, "recover: reset failed: %d\n", ret); + return ret; + } + if (avd->domain) { + if (!avd->empty_domain) + return -ENODEV; + dev_warn(avd->dev, "recover: DART flush\n"); + ret = iommu_attach_device(avd->empty_domain, avd->dev); + if (ret) { + dev_err(avd->dev, "recover: DART empty attach failed: %d\n", ret); + return ret; + } + iommu_detach_device(avd->empty_domain, avd->dev); + /* Do not reboot against an empty/blocked domain after a failed restore. */ + if (iommu_get_domain_for_dev(avd->dev) != avd->domain) { + dev_err(avd->dev, "recover: DART domain not restored\n"); + return -EIO; + } + } + dev_warn(avd->dev, "recover: boot firmware\n"); + ret = avd_pm_backend_resume(avd); + if (ret) { + dev_err(avd->dev, "recover: boot failed: %d\n", ret); + avd->backend->shutdown(avd); + return ret; + } + dev_warn(avd->dev, "recover: done\n"); + return 0; +} + +int avd_pm_power_on(struct avd_dev *avd) +{ + const struct avd_backend_ops *backend = avd->backend; + int ret; + + /* genpd and the IOMMU suppliers resume before this runtime callback. */ + mutex_lock(&avd->pm.lock); + avd->pm.dart_active = true; + avd_pm_drain_unmaps(avd); + mutex_unlock(&avd->pm.lock); + + /* Do not hold pm.lock: boot may allocate or release DMA buffers. */ + ret = backend->resume ? backend->resume(avd) : avd_pm_backend_resume(avd); + if (ret) { + avd->backend->shutdown(avd); + mutex_lock(&avd->pm.lock); + avd->pm.dart_active = false; + mutex_unlock(&avd->pm.lock); + return ret; + } + avd_pm_enable_irq(avd); + /* A clean boot after power-up ends a failed recovery's fault state. */ + WRITE_ONCE(avd->faulted, false); + + if (avd->pm.queue->resubmit) + avd->pm.queue->resubmit(avd); + + return 0; +} + +int avd_pm_power_off(struct avd_dev *avd) +{ + const struct avd_backend_ops *backend = avd->backend; + int ret; + + /* PM references cover reservation through retirement, including drain. */ + if (!avd->pm.queue->idle(avd)) + return -EBUSY; + + if (backend->suspend) { + ret = backend->suspend(avd); + if (ret) + return ret; + } else { + backend->shutdown(avd); + } + avd_pm_disable_irq(avd); + + /* No more firmware DMA; suppliers stay on until this callback returns. */ + mutex_lock(&avd->pm.lock); + avd_pm_drain_unmaps(avd); + avd->pm.dart_active = false; + mutex_unlock(&avd->pm.lock); + return 0; +} + +static int avd_runtime_resume(struct device *dev) +{ + return avd_pm_power_on(dev_get_drvdata(dev)); +} + +static int avd_runtime_suspend(struct device *dev) +{ + return avd_pm_power_off(dev_get_drvdata(dev)); +} + +static int avd_system_suspend(struct device *dev) +{ + struct avd_dev *avd = dev_get_drvdata(dev); + int ret; + + v4l2_m2m_suspend(avd->m2m_dev); + ret = avd->pm.queue->drain(avd); + if (!ret) + ret = pm_runtime_force_suspend(dev); + if (ret) + v4l2_m2m_resume(avd->m2m_dev); + return ret; +} + +static int avd_system_resume(struct device *dev) +{ + struct avd_dev *avd = dev_get_drvdata(dev); + int ret; + + ret = pm_runtime_force_resume(dev); + if (ret) + return ret; + v4l2_m2m_resume(avd->m2m_dev); + return 0; +} + +const struct dev_pm_ops avd_pm_ops = { + SYSTEM_SLEEP_PM_OPS(avd_system_suspend, avd_system_resume) + RUNTIME_PM_OPS(avd_runtime_suspend, avd_runtime_resume, NULL) +}; diff --git a/drivers/media/platform/apple/avd/avd-pm.h b/drivers/media/platform/apple/avd/avd-pm.h new file mode 100644 index 0000000000000..5fe55f595beb2 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_PM_H_ +#define AVD_PM_H_ + +#include +#include +#include + +#include "avd-queue.h" + +struct avd_dma_release; + +struct avd_queue_pm_ops { + /* Snapshot under the queue lock: all FREE, no context draining. */ + bool (*idle)(struct avd_dev *avd); + /* Scheduling is paused; finish/retire all accepted frames. May sleep. */ + int (*drain)(struct avd_dev *avd); + /* Kick prepared RESERVED entries only, respecting dependencies. */ + void (*resubmit)(struct avd_dev *avd); +}; + +struct avd_deferred_unmap { + struct list_head node; + void (*unmap)(struct avd_dev *avd, struct avd_deferred_unmap *unmap); +}; + +struct avd_pm { + /* Serializes DMA releases against DART activation and firmware boot. */ + struct mutex lock; + struct list_head deferred_unmaps; + bool dart_active; + int irq; + bool irq_enabled; + /* + * irq_lock guards hw_on and irq_gen. The threaded handler touches the + * decoder only under irq_lock with hw_on set and an unchanged irq_gen, + * so turning the IRQ off never has to wait for the handler thread + * (which may itself be waiting for queue_mutex). + */ + spinlock_t irq_lock; + bool hw_on; + unsigned int irq_gen; + void __iomem *piodma; + phys_addr_t sram_phys; + wait_queue_head_t idle_wait; + /* Protected by queue_lock; counts drain operations, not stopped streams. */ + unsigned int draining; + const struct avd_queue_pm_ops *queue; +}; + +/* The caller holds the queue lock, including the draining-context count. */ +bool avd_pm_can_suspend(const struct avd_frame *frames, unsigned int draining); + +void avd_pm_init(struct avd_dev *avd, const struct avd_backend_ops *backend, + const struct avd_queue_pm_ops *queue); +void avd_pm_enable(struct avd_dev *avd); +int avd_pm_map_piodma(struct avd_dev *avd); +void avd_pm_prepare_piodma(struct avd_dev *avd); +/* After queue cleanup: release deferred DMA mappings before the final off. */ +void avd_pm_disable(struct avd_dev *avd); +/* Called in process context, after the last frame has retired its mapping. */ +void avd_pm_unmap(struct avd_dev *avd, struct avd_deferred_unmap *unmap); +int avd_pm_power_on(struct avd_dev *avd); +int avd_pm_power_off(struct avd_dev *avd); +/* Caller holds a PM reference and queue_mutex; leaves the host IRQ closed. */ +int avd_pm_recover(struct avd_dev *avd); +/* After recovery has invalidated every previously firmware-owned slot. */ +void avd_pm_enable_irq(struct avd_dev *avd); +irqreturn_t avd_pm_irq(struct avd_dev *avd, unsigned int gen); +void avd_pm_drain_begin(struct avd_dev *avd); +void avd_pm_drain_end(struct avd_dev *avd); +void avd_pm_queue_changed(struct avd_dev *avd); +extern const struct avd_queue_pm_ops avd_pm_queue_ops; + +/* Backends may share these when suspend loses all firmware context. */ +int avd_pm_backend_suspend(struct avd_dev *avd); +int avd_pm_backend_resume(struct avd_dev *avd); +extern const struct dev_pm_ops avd_pm_ops; + +#endif /* AVD_PM_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-pm.kunitconfig b/drivers/media/platform/apple/avd/avd-pm.kunitconfig new file mode 100644 index 0000000000000..4d9b37f71ff41 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.kunitconfig @@ -0,0 +1,14 @@ +CONFIG_KUNIT=y +CONFIG_COMPILE_TEST=y +CONFIG_OF=y +CONFIG_PM=y +CONFIG_SUSPEND=y +CONFIG_PM_SLEEP=y +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_VIDEO_DEV=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_VIDEO_APPLE_AVD=y +CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST=y +CONFIG_WERROR=y diff --git a/drivers/media/platform/apple/avd/avd-protocol.rst b/drivers/media/platform/apple/avd/avd-protocol.rst new file mode 100644 index 0000000000000..ce9eaf0bf2720 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-protocol.rst @@ -0,0 +1,97 @@ +AVD queued transport implementation status +========================================== + +The pool owns sixteen independently tagged slots. A source/destination pair +and a runtime-PM reference remain owned by the slot until in-order retirement +within its context. A duplicate or obsolete tag cannot complete a reused +slot. Inbox-full leaves a prepared slot reserved and retries without losing +its buffers. Timeout or streamoff resets the whole engine before failing all +slots affected by that reset. Slot deadlines survive inbox retries. + +The first commit adding avd-queue.h is the shared interface. Later additions +only extend it. Codec callbacks own codec-private state and must release that +state from retire(), including when prepare() failed part way through. + +Open firmware compatibility +-------------------------- + +The deployed, untagged open-firmware protocol has a one-frame limit. Existing +codec run() callbacks build a DMA-visible per-slot program; only the backend +writes those instructions to the VP FIFO. That compatibility backend retains +the old host-side PP kick. A native frame_ops program must use the queued +avd-fw transport before it can replace the compatibility path; this backend +returns EOPNOTSUPP for native programs rather than guessing their wire format. + +The V4L2 core currently warns unconditionally when job_finish() is used on a +queue advertising HOLD_CAPTURE_BUF. Such queues retain the scheduler until +retirement and use buf_done_and_job_finish(), including held slice requests. +Other queues release the scheduler as soon as their buffers belong to a slot. +Removing this restriction requires a V4L2 core API change outside this driver's +scope. It is not safe to toggle subsystem_flags around job_finish(). + +Apple Borage transport +---------------------- + +Only T6040 has Borage match data. The Apple backend contains the 64-byte +little-endian connection configuration, session command builders, decode +prefix, eight-entry inbox publishing, a 64-entry completion shadow and bounded +20-word result parsing. Type 7 is VP/parsing completion and never releases a +buffer. The notification address rule has one definition, +AVD_APPLE_NOTIFY_BASE, so a capture can correct it. + +Boot checks a nonzero ITCM span and a nonzero/non-all-ones version word before +accepting a preloaded image. A separately installed firmware image can restore +ITCM after power loss. The last image word is zero padded without reading +past the firmware object. No Apple image is embedded in this source tree. + +Activation remains disabled in match data: the supplied clean specification +has no ITCM register offsets, link programming sequence, result-area address, +result field layout or final-completion value. It would be unsafe to invent +these or reuse the different open-firmware mailbox. The closest executable +path is the open compatibility backend. The Apple register description and +reply layout must be completed together from clean facts; setting verified +alone is insufficient. The DT must then describe the complete Apple aperture +as an owned named resource before probe can map it. + +Unresolved field ledger +----------------------- + +This is the single ledger for fields marked [C] in the common transport. No +semantic names have been assigned to the unknown bits. Codec-specific fields +remain in their codec packers' ledgers. + +============================== ================================================ +Field Source and required evidence +============================== ================================================ +Config +00/+08/+10/+18/+20/ SPEC 3.2: exact values, meanings unproved; +28/+2c/+30/+38 capture connection construction and LinkUp +ITCM start/end/clear/enable SPEC 3.1: operations, offsets/values absent; + clean per-SoC register ledger required +IMEM version-word offset SPEC 3.1: display format proved, offset absent +Mailbox link register writes SPEC 3.2: aperture 1, state 4, LinkUp proved; + offsets and bit operations absent +Mailbox ready register SPEC 3.1: status 1, poll 10 us; offset absent +Mailbox notification word SPEC 4.2; expected inbox-entry address from MIT + aura trace_avd.py V3 precedent; capture Borage +Inbox consumption/full rule SPEC 4.2: host positions modulo eight; + hardware consumer register/doorbell absent +Result-area base/indexing SPEC 4.3: 64 entries, 20 words; base and mailbox + notification interpretation absent +Reply type/client/frame/status SPEC 4.3: correlation proved, word layout absent; + synthetic KUnit layout is not a hardware fact +Final reply type/status values SPEC 4.3: only VP done = 7 proved; capture final + completion and error replies before activation +Decode +00 bit 9 SPEC 5.2: boolean argument, meaning unproved; + left zero until the producer is established +Decode +50..+5f SPEC 5: Borage extension, nonzero use unproved; + initialized to zero before codec packing +Commands 3/4 and other values SPEC 5.1: unproved; never sent +PIODMA mapping stream binding SPEC 7: three mapper roles proved, Linux binding + and per-domain fp/program DVA ownership pending +Retention and clock floors SPEC 8: per-SoC capture required; no guessed knobs +============================== ================================================ + +Licensed implementation references: eiln/avd (MIT), avd_emu.py, and +https://github.com/aurora-silicon/aura/blob/main/proxyclient/hv/trace_avd.py +(MIT). The Borage facts above come from the supplied clean SPEC sections 3-5; +no binary or captured Apple command images are used. diff --git a/drivers/media/platform/apple/avd/avd-queue-internal.h b/drivers/media/platform/apple/avd/avd-queue-internal.h new file mode 100644 index 0000000000000..0e2a668d5a62f --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue-internal.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_QUEUE_INTERNAL_H_ +#define AVD_QUEUE_INTERNAL_H_ + +#include +#include +#include + +#include "avd-queue.h" + +/* Caller serializes the pool; tags are never recycled during device lifetime. */ +static inline int avd_slot_reserve(struct avd_frame *f, struct avd_ctx *ctx, + u64 seq) +{ + if (f->state != AVD_FRAME_FREE) + return -EBUSY; + if (!seq || seq > (U32_MAX >> 4)) + return -EOVERFLOW; + f->seq = seq; + f->tag = (seq << 4) | f->idx; + f->ctx = ctx; + f->state = AVD_FRAME_RESERVED; + f->dep = AVD_DEP_NONE; + f->status = 0; + f->prog_words = 0; + f->codec_priv = NULL; + return 0; +} + +/* A full inbox owns nothing: keep the prepared slot, tag and deadline. */ +static inline void avd_slot_submit_result(struct avd_frame *f, int status) +{ + if (status == -EBUSY) { + f->state = AVD_FRAME_RESERVED; + } else if (status) { + f->status = status; + f->state = AVD_FRAME_DONE; + } +} + +static inline bool avd_slot_complete(struct avd_frame *f, u32 tag, int status) +{ + if (f->state != AVD_FRAME_SUBMITTED || f->tag != tag) + return false; + f->status = status; + f->state = AVD_FRAME_DONE; + return true; +} + +static inline bool avd_slot_first(const struct avd_frame *pool, + const struct avd_frame *f) +{ + unsigned int i; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (pool[i].state != AVD_FRAME_FREE && pool[i].ctx == f->ctx && + pool[i].seq < f->seq) + return false; + return true; +} + +static inline bool avd_slot_expired(const struct avd_frame *f, unsigned long now) +{ + return (f->state == AVD_FRAME_SUBMITTED || + f->state == AVD_FRAME_RESERVED) && time_after_eq(now, f->deadline); +} + +static inline bool avd_inbox_full(unsigned int producer, unsigned int consumer) +{ + return ((producer + 1) & 7) == consumer; +} + +#endif /* AVD_QUEUE_INTERNAL_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-queue-test.c b/drivers/media/platform/apple/avd/avd-queue-test.c new file mode 100644 index 0000000000000..1efae38789b57 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue-test.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "avd.h" +#include "avd-apple-protocol.h" +#include "avd-queue-internal.h" + +static void avd_slots_and_tags(struct kunit *test) +{ + struct avd_frame *pool; + struct avd_ctx *ctx = (void *)test; + unsigned int i; + + pool = kunit_kcalloc(test, AVD_MAX_FRAMES, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + pool[i].idx = i; + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[i], ctx, i + 1), 0); + KUNIT_EXPECT_EQ(test, pool[i].tag, ((i + 1) << 4) | i); + KUNIT_EXPECT_EQ(test, pool[i].state, AVD_FRAME_RESERVED); + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[i], ctx, 100), -EBUSY); + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag, 0)); + pool[i].state = AVD_FRAME_SUBMITTED; + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag + 16, 0)); + KUNIT_EXPECT_TRUE(test, avd_slot_complete(&pool[i], pool[i].tag, 0)); + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag, -EIO)); + } + pool[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[0], ctx, 0), -EOVERFLOW); + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[0], ctx, (u64)(U32_MAX >> 4) + 1), + -EOVERFLOW); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[0], ctx, 100), 0); + pool[0].state = AVD_FRAME_SUBMITTED; + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[0], 16, 0)); +} + +static void avd_retirement_order(struct kunit *test) +{ + struct avd_frame *pool; + struct avd_ctx *a = (void *)test; + struct avd_ctx *b = (void *)&pool; + + pool = kunit_kcalloc(test, AVD_MAX_FRAMES, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[0], a, 1), 0); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[1], b, 2), 0); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[2], a, 3), 0); + pool[0].state = AVD_FRAME_SUBMITTED; + pool[1].state = AVD_FRAME_SUBMITTED; + pool[2].state = AVD_FRAME_SUBMITTED; + KUNIT_ASSERT_TRUE(test, avd_slot_complete(&pool[2], pool[2].tag, 0)); + KUNIT_EXPECT_FALSE(test, avd_slot_first(pool, &pool[2])); + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[1])); + KUNIT_ASSERT_TRUE(test, avd_slot_complete(&pool[0], pool[0].tag, -ETIMEDOUT)); + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[0])); + pool[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[2])); + KUNIT_EXPECT_EQ(test, pool[0].status, -ETIMEDOUT); +} + +static void avd_watchdog_and_retry(struct kunit *test) +{ + struct avd_frame f = { .state = AVD_FRAME_SUBMITTED, .deadline = ULONG_MAX - 2 }; + unsigned int producer, consumer; + u32 tag = 0x124; + unsigned long deadline = f.deadline; + + f.tag = tag; + avd_slot_submit_result(&f, -EBUSY); + KUNIT_EXPECT_EQ(test, f.state, AVD_FRAME_RESERVED); + KUNIT_EXPECT_EQ(test, f.tag, tag); + KUNIT_EXPECT_EQ(test, f.deadline, deadline); + f.state = AVD_FRAME_SUBMITTED; + avd_slot_submit_result(&f, 0); + KUNIT_EXPECT_EQ(test, f.state, AVD_FRAME_SUBMITTED); + + KUNIT_EXPECT_FALSE(test, avd_slot_expired(&f, ULONG_MAX - 3)); + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, ULONG_MAX - 2)); + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, 1)); + f.state = AVD_FRAME_DONE; + KUNIT_EXPECT_FALSE(test, avd_slot_expired(&f, 1)); + f.state = AVD_FRAME_RESERVED; + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, 1)); + for (producer = 0; producer < 8; producer++) { + for (consumer = 0; consumer < 8; consumer++) + KUNIT_EXPECT_EQ(test, avd_inbox_full(producer, consumer), + ((producer + 1) & 7) == consumer); + consumer = (producer + 1) & 7; + KUNIT_EXPECT_TRUE(test, avd_inbox_full(producer, consumer)); + /* A consumer advance reopens the same pending inbox slot. */ + consumer = (consumer + 1) & 7; + KUNIT_EXPECT_FALSE(test, avd_inbox_full(producer, consumer)); + KUNIT_EXPECT_EQ(test, avd_apple_notification(producer), + 0x01482cf8U + 0x60 * producer); + } +} + +static void avd_completion_parser(struct kunit *test) +{ + /* Synthetic layout for parser testing; never a claimed Borage wire ABI. */ + struct avd_apple_reply_layout layout = { + .type_word = 0, .type_mask = 0x1f, + .client_word = 1, .client_shift = 16, .client_mask = 0x3ff, + .frame_word = 3, .status_word = 4, .done_type = 8, .verified = true, + }; + struct avd_apple_reply reply; + u32 record[AVD_APPLE_RESULT_WORDS] = { 7, 5 << 16, 0, 0x12340 }; + + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 19, &layout, &reply), -EMSGSIZE); + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EINPROGRESS); + record[0] = 8; + KUNIT_ASSERT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), 0); + KUNIT_EXPECT_EQ(test, reply.client, 5U); + KUNIT_EXPECT_EQ(test, reply.frame, 0x12340U); + KUNIT_EXPECT_EQ(test, reply.status, 0); + record[4] = 1; + KUNIT_ASSERT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), 0); + KUNIT_EXPECT_EQ(test, reply.status, -EIO); + record[0] = 3; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -ENOMSG); + layout.frame_word = 20; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EINVAL); + layout.verified = false; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EOPNOTSUPP); +} + +static void avd_command_builders(struct kunit *test) +{ + struct avd_frame f = { .idx = 15, .frame_num = 0x1234 }; + u32 cmd[24]; + const u64 config[] = { + 0, 0x01558000, 0x01558a00, 0x01558e00, 0, + 0x000000040004000aULL, 0, 0x01100008, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(config); i++) + KUNIT_EXPECT_EQ(test, get_unaligned_le64(avd_borage_config + i * 8), config[i]); + KUNIT_ASSERT_EQ(test, avd_apple_decode_prefix(&f, 1023, 15, 3), 0); + KUNIT_EXPECT_EQ(test, f.cmd[0], 1U | (15U << 10) | (1023U << 22)); + KUNIT_EXPECT_EQ(test, f.cmd[1], 15U | (3U << 16)); + KUNIT_EXPECT_EQ(test, f.cmd[3], 0x1234U); + KUNIT_EXPECT_EQ(test, f.cmd[23], 0U); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 1024, 0, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 0, 16, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 0, 0, 4), -EINVAL); + KUNIT_ASSERT_EQ(test, avd_apple_session_command(cmd, 2, 6, 17, 42), 0); + KUNIT_EXPECT_EQ(test, cmd[0], 6U | (2U << 16)); + KUNIT_EXPECT_EQ(test, cmd[1], 17U); + KUNIT_EXPECT_EQ(test, cmd[2], 42U); + KUNIT_EXPECT_EQ(test, cmd[23], 0U); + KUNIT_EXPECT_EQ(test, avd_apple_session_command(cmd, 2, 3, 0, 0), -EINVAL); +} + +static unsigned int avd_test_max_inflight(struct avd_dev *avd) +{ + return AVD_MAX_FRAMES; +} + +static void avd_dispatch_reservation(struct kunit *test) +{ + static const struct avd_backend_ops backend = { + .max_inflight = avd_test_max_inflight, + }; + struct avd_dev *avd = kunit_kzalloc(test, sizeof(*avd), GFP_KERNEL); + struct avd_ctx *ctx = kunit_kcalloc(test, 2, sizeof(*ctx), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, avd); + KUNIT_ASSERT_NOT_NULL(test, ctx); + spin_lock_init(&avd->queue_lock); + avd->backend = &backend; + ctx[0].dev = avd; + ctx[1].dev = avd; + /* Every context with work is ready; capacity is checked at dispatch. */ + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[0]), 1); + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); + avd->inflight = AVD_MAX_FRAMES; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); + avd->inflight = 1; + avd->frames[0].state = AVD_FRAME_SUBMITTED; + avd->frames[0].ctx = &ctx[1]; + ctx[1].next_dep = AVD_DEP_PREV_DONE; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 0); + avd->frames[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); +} + +static struct kunit_case avd_queue_cases[] = { + KUNIT_CASE(avd_slots_and_tags), + KUNIT_CASE(avd_retirement_order), + KUNIT_CASE(avd_watchdog_and_retry), + KUNIT_CASE(avd_completion_parser), + KUNIT_CASE(avd_command_builders), + KUNIT_CASE(avd_dispatch_reservation), + {} +}; + +static struct kunit_suite avd_queue_suite = { + .name = "apple-avd-queue", + .test_cases = avd_queue_cases, +}; + +kunit_test_suite(avd_queue_suite); diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c new file mode 100644 index 0000000000000..226de0e4d7574 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -0,0 +1,591 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include + +#include "avd.h" +#include "avd-queue-internal.h" + +#define AVD_FRAME_TIMEOUT msecs_to_jiffies(2000) +/* A stream that fails this many frames in a row in hardware is stopped. */ +#define AVD_CTX_MAX_HW_ERRORS 3 +/* Never reset the decoder more often than this. */ +#define AVD_RECOVER_GAP_MS 20 +/* More than AVD_STORM_MAX recoveries within AVD_STORM_WINDOW: cool down. */ +#define AVD_STORM_MAX 8 +#define AVD_STORM_WINDOW msecs_to_jiffies(10000) +#define AVD_COOLDOWN msecs_to_jiffies(10000) +#define AVD_RETRY_DELAY msecs_to_jiffies(1) +#define AVD_PROGRAM_SIZE SZ_2M + +bool avd_backend_is_apple(const struct avd_dev *avd) +{ + return avd->backend == &avd_apple_backend; +} + +/* queue_mutex excludes callbacks and context destruction from retirement. */ +static void avd_retire(struct avd_dev *avd) +{ + struct avd_frame *f; + unsigned long flags; + unsigned int i; + bool progress; + + do { + progress = false; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_ctx *ctx; + enum vb2_buffer_state state; + + f = &avd->frames[i]; + spin_lock_irqsave(&avd->queue_lock, flags); + if (f->state != AVD_FRAME_DONE || + !avd_slot_first(avd->frames, f)) { + spin_unlock_irqrestore(&avd->queue_lock, flags); + continue; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + ctx = f->ctx; + state = f->status ? VB2_BUF_STATE_ERROR : VB2_BUF_STATE_DONE; + if (!f->status) { + ctx->hw_errors = 0; + } else if (++ctx->hw_errors >= AVD_CTX_MAX_HW_ERRORS && !ctx->broken) { + ctx->broken = true; + dev_err(avd->dev, "stream stopped after %u failed frames; restart streaming to retry\n", + ctx->hw_errors); + } + if (ctx->coded_fmt_desc->frame_ops) { + if (ctx->coded_fmt_desc->frame_ops->retire) + ctx->coded_fmt_desc->frame_ops->retire(ctx, f, state); + } else if (ctx->coded_fmt_desc->ops->done) { + ctx->coded_fmt_desc->ops->done(ctx, f->src, f->dst, state); + } + if (ctx->coded_fmt_desc->frame_ops && f->src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(f->src->vb2_buf.req_obj.req, + &ctx->ctrl_hdl); + /* CAPTURE first: OUTPUT completion signals the media request. */ + if (f->scheduler_held) { + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, + ctx->fh.m2m_ctx, state); + } else { + v4l2_m2m_buf_done(f->dst, state); + v4l2_m2m_buf_done(f->src, state); + } + spin_lock_irqsave(&avd->queue_lock, flags); + f->ctx = NULL; + f->src = NULL; + f->dst = NULL; + f->state = AVD_FRAME_FREE; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + progress = true; + } + } while (progress); +} + +/* A reset invalidates every firmware-owned slot, including other clients. */ +static void avd_reset_queue(struct avd_dev *avd, int status) +{ + struct avd_ctx *ctx; + unsigned long flags; + unsigned int i; + int ret; + + /* + * A stream whose every frame faults (an H0 error storm) used to reset + * the decoder back to back, about once per millisecond, until the + * machine hung. Space recoveries out, and after a storm stop taking + * new frames for a while instead of resetting again. + */ + if (avd->last_recover && + time_before(jiffies, avd->last_recover + msecs_to_jiffies(AVD_RECOVER_GAP_MS))) + msleep(AVD_RECOVER_GAP_MS); + if (!avd->storm_count || time_after(jiffies, avd->storm_start + AVD_STORM_WINDOW)) { + avd->storm_start = jiffies; + avd->storm_count = 0; + } + if (++avd->storm_count > AVD_STORM_MAX) { + dev_err(avd->dev, "%u decoder recoveries in %u s: failing new frames for %u s\n", + avd->storm_count, jiffies_to_msecs(AVD_STORM_WINDOW) / 1000, + jiffies_to_msecs(AVD_COOLDOWN) / 1000); + WRITE_ONCE(avd->cooldown_until, jiffies + AVD_COOLDOWN); + avd->storm_count = 0; + } + ret = avd_pm_recover(avd); + avd->last_recover = jiffies ?: 1; + WRITE_ONCE(avd->faulted, !!ret); + if (ret) + dev_err(avd->dev, "queue reset failed: %d\n", ret); + avd->compat_active = NULL; + list_for_each_entry(ctx, &avd->contexts, queue_node) + ctx->session_started = false; + spin_lock_irqsave(&avd->queue_lock, flags); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state == AVD_FRAME_FREE) + continue; + avd->frames[i].status = status; + avd->frames[i].state = AVD_FRAME_DONE; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (!ret) + avd_pm_enable_irq(avd); +} + +static int avd_send_frame(struct avd_dev *avd, struct avd_frame *f) +{ + unsigned long flags; + int ret; + + if (f->dep == AVD_DEP_PREV_DONE && !avd_slot_first(avd->frames, f)) + return -EBUSY; + if ((!f->ctx->session_started || + f->ctx->session_generation != avd->fw_generation) && + avd->backend->session_start) { + ret = avd->backend->session_start(f->ctx); + if (ret) + return ret; + f->ctx->session_started = true; + f->ctx->session_generation = avd->fw_generation; + } + /* Publish ownership before the doorbell; IRQs may arrive in submit(). */ + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_SUBMITTED; + spin_unlock_irqrestore(&avd->queue_lock, flags); + dma_wmb(); + ret = avd->backend->submit(avd, f); + spin_lock_irqsave(&avd->queue_lock, flags); + avd_slot_submit_result(f, ret); + spin_unlock_irqrestore(&avd->queue_lock, flags); + return ret; +} + +static void avd_queue_worker(struct work_struct *work) +{ + struct avd_dev *avd = container_of(to_delayed_work(work), + struct avd_dev, queue_work); + struct avd_ctx *ctx; + unsigned long flags; + unsigned int i; + bool pending; + unsigned long delay = AVD_FRAME_TIMEOUT; + + mutex_lock(&avd->queue_mutex); + if (avd->stopping) + goto unlock; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + bool expired; + + spin_lock_irqsave(&avd->queue_lock, flags); + expired = avd_slot_expired(&avd->frames[i], jiffies); + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (expired) { + struct avd_frame *t = &avd->frames[i]; + + dev_err(avd->dev, "frame tag %#x timed out (%p4cc, state %d, %u in flight, %s)\n", + t->tag, t->ctx ? &t->ctx->coded_fmt_desc->fourcc : &(u32){0}, + t->state, avd->inflight, avd->backend->name); + avd_reset_queue(avd, -ETIMEDOUT); + break; + } + } + avd_retire(avd); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + int ret; + + if (f->state != AVD_FRAME_RESERVED) + continue; + ret = avd_send_frame(avd, f); + if (ret && ret != -EBUSY) { + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + } + } + avd_retire(avd); + /* Restart a job that found no free slot. */ + spin_lock_irqsave(&avd->queue_lock, flags); + ctx = avd->inflight < avd->backend->max_inflight(avd) ? avd->deferred_ctx : NULL; + if (ctx) + avd->deferred_ctx = NULL; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (ctx) + schedule_work(&ctx->submit_work); + /* Freed slots/dependencies must wake contexts whose job_ready was false. */ + list_for_each_entry(ctx, &avd->contexts, queue_node) + v4l2_m2m_try_schedule(ctx->fh.m2m_ctx); + spin_lock_irqsave(&avd->queue_lock, flags); + pending = avd->inflight != 0; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + unsigned long remaining; + + if (f->state == AVD_FRAME_FREE) + continue; + remaining = time_after(f->deadline, jiffies) ? f->deadline - jiffies : 0; + if (f->state == AVD_FRAME_RESERVED) + remaining = min(remaining, AVD_RETRY_DELAY); + delay = min(delay, remaining); + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (pending) + mod_delayed_work(system_wq, &avd->queue_work, delay); + unlock: + mutex_unlock(&avd->queue_mutex); +} + +void avd_frame_complete(struct avd_dev *avd, u32 tag, int status) +{ + struct avd_frame *f = &avd->frames[tag & (AVD_MAX_FRAMES - 1)]; + unsigned long flags; + bool accepted; + + spin_lock_irqsave(&avd->queue_lock, flags); + accepted = avd_slot_complete(f, tag, status); + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (accepted) + mod_delayed_work(system_wq, &avd->queue_work, 0); +} + +int avd_queue_ready(void *priv) +{ + struct avd_ctx *ctx = priv; + struct avd_dev *avd = ctx->dev; + unsigned long flags; + unsigned int i; + bool ready; + + spin_lock_irqsave(&avd->queue_lock, flags); + /* + * A faulted or cooling-down decoder still schedules: submission fails + * those frames at once, so clients see errors instead of waiting. + */ + /* + * Slot capacity is not a readiness condition: every context with work + * enters the m2m job queue, which runs them in order, and a job that + * finds no free slot is deferred until one retires. Gating readiness on + * one reserved context let the first context in the list win every + * time and starved the others (multi-stream decodes timing out). + */ + ready = !READ_ONCE(ctx->draining) && !READ_ONCE(avd->stopping); + if (ready && ctx->next_dep == AVD_DEP_PREV_DONE) { + /* Legacy codecs may hold several pictures in separate banks. */ + unsigned int limit = ctx->coded_fmt_desc && !ctx->coded_fmt_desc->frame_ops ? + max(ctx->max_frames, 1U) : 1, held = 0; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (avd->frames[i].state != AVD_FRAME_FREE && + avd->frames[i].ctx == ctx) + held++; + ready = held < limit; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + return ready; +} + +static void avd_queue_submit_work(struct work_struct *work) +{ + struct avd_ctx *ctx = container_of(work, struct avd_ctx, submit_work); + struct avd_dev *avd = ctx->dev; + const struct avd_codec_frame_ops *ops = ctx->coded_fmt_desc->frame_ops; + struct avd_frame *f = NULL; + unsigned long flags; + unsigned int i; + int ret; + + mutex_lock(&avd->queue_mutex); + spin_lock_irqsave(&avd->queue_lock, flags); + if (avd->deferred_ctx == ctx) + avd->deferred_ctx = NULL; + if (avd->stopping || ctx->draining || avd->faulted || ctx->broken || + time_before(jiffies, READ_ONCE(avd->cooldown_until))) { + struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + + spin_unlock_irqrestore(&avd->queue_lock, flags); + /* Complete the request's controls too, or its fd never signals. */ + if (src && src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + goto unlock; + } + if (avd->inflight < avd->backend->max_inflight(avd)) { + for (i = 0; i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state == AVD_FRAME_FREE) { + f = &avd->frames[i]; + break; + } + } + } + if (!f) { + /* Keep the m2m job; the worker restarts it when a slot frees. */ + avd->deferred_ctx = ctx; + spin_unlock_irqrestore(&avd->queue_lock, flags); + mod_delayed_work(system_wq, &avd->queue_work, 0); + goto unlock; + } + ret = avd_slot_reserve(f, ctx, ++avd->frame_seq); + if (!ret) + avd->inflight++; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (WARN_ON(ret)) { + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + goto unlock; + } + ret = pm_runtime_resume_and_get(avd->dev); + if (ret < 0) + goto fail_no_pm; + f->src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + f->dst = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); + /* + * Only a slice that holds its CAPTURE buffer needs the m2m scheduler + * until retirement. The last slice of a picture releases both buffers + * to the slot, so the next picture can be prepared while it decodes. + */ + f->scheduler_held = (ctx->fh.m2m_ctx->out_q_ctx.q.subsystem_flags & + VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF) && + (f->src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF); + f->frame_num = f->tag; + f->deadline = jiffies + AVD_FRAME_TIMEOUT; + memset(f->cmd, 0, sizeof(f->cmd)); + if (ops) { + if (!f->prog->cpu) { + ret = avd_buf_alloc(avd, f->prog, AVD_PROGRAM_SIZE); + if (ret) + goto fail; + } + /* NULL fp is also the codec-facing indication of an open program. */ + f->fp = NULL; + if (avd_backend_is_apple(avd)) { + struct avd_buf *fp = avd->frame_fp[f->idx]; + + if (!fp) { + fp = devm_kzalloc(avd->dev, sizeof(*fp), GFP_KERNEL); + avd->frame_fp[f->idx] = fp; + } + f->fp = fp; + if (!f->fp) { + ret = -ENOMEM; + goto fail; + } + if (!f->fp->cpu) { + ret = avd_buf_alloc(avd, f->fp, avd->variant->fp_size); + if (ret) + goto fail; + } + } + if (f->src->vb2_buf.req_obj.req) { + ret = v4l2_ctrl_request_setup(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + if (ret) + goto fail; + } + ret = ops->prepare(ctx, f); + } else { + avd->building = f; + avd->compat_held = false; + ret = ctx->coded_fmt_desc->ops->run(ctx); + avd->building = NULL; + } + if (ret) + goto fail; + ctx->next_dep = ops ? f->dep : AVD_DEP_PREV_DONE; + if ((ops && (f->src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF)) || + (!ops && avd->compat_held)) { + if (ops && ops->retire) + ops->retire(ctx, f, VB2_BUF_STATE_DONE); + else if (!ops && ctx->coded_fmt_desc->ops->done) + ctx->coded_fmt_desc->ops->done(ctx, f->src, f->dst, VB2_BUF_STATE_DONE); + if (ops && f->src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_FREE; + f->ctx = NULL; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_DONE); + goto unlock; + } + v4l2_m2m_buf_copy_metadata(f->src, f->dst); + if (!f->scheduler_held) { + v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); + v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + } + f->dst->is_held = false; + ret = avd_send_frame(avd, f); + if (ret && ret != -EBUSY) { + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + } + /* Buffers now belong to the slot, not to the m2m scheduler. */ + if (!f->scheduler_held) + v4l2_m2m_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx); + mod_delayed_work(system_wq, &avd->queue_work, 0); + goto unlock; + fail: + /* Detach before retirement, including partially prepared codec state. */ + if (!f->scheduler_held) { + v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); + v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + } + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (!f->scheduler_held) + v4l2_m2m_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx); + mod_delayed_work(system_wq, &avd->queue_work, 0); + goto unlock; + fail_no_pm: + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_FREE; + f->ctx = NULL; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + unlock: + mutex_unlock(&avd->queue_mutex); +} + +void avd_queue_run(void *priv) +{ + struct avd_ctx *ctx = priv; + + /* try_schedule() may invoke device_run synchronously under queue_mutex. */ + schedule_work(&ctx->submit_work); +} + +void avd_queue_drain(struct avd_ctx *ctx) +{ + struct avd_dev *avd = ctx->dev; + unsigned long flags; + unsigned int i; + bool finish_deferred = false; + int ret; + + /* Session stop/reset accesses registers even when there are no frames. */ + ret = pm_runtime_resume_and_get(avd->dev); + avd_pm_drain_begin(avd); + mutex_lock(&avd->queue_mutex); + spin_lock_irqsave(&avd->queue_lock, flags); + WRITE_ONCE(ctx->draining, true); + if (avd->deferred_ctx == ctx) { + avd->deferred_ctx = NULL; + finish_deferred = true; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + /* A deferred job is still m2m's running job: end it so abort can wait. */ + if (finish_deferred) + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + for (i = 0; ret >= 0 && i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state != AVD_FRAME_FREE && avd->frames[i].ctx == ctx) { + if (avd->backend->abort) + avd->backend->abort(ctx); + avd_reset_queue(avd, -ECANCELED); + break; + } + } + avd_retire(avd); + if (ret >= 0 && ctx->session_started && avd->backend->session_stop && + ctx->session_generation == avd->fw_generation) + avd->backend->session_stop(ctx); + ctx->session_started = false; + if (!avd->stopping) + mod_delayed_work(system_wq, &avd->queue_work, 0); + mutex_unlock(&avd->queue_mutex); + avd_pm_drain_end(avd); + if (ret >= 0) { + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + } +} + +void avd_queue_abort(void *priv) +{ + avd_queue_drain(priv); +} + +void avd_queue_add_ctx(struct avd_ctx *ctx) +{ + struct avd_ctx *other; + u32 id; + + INIT_WORK(&ctx->submit_work, avd_queue_submit_work); + mutex_lock(&ctx->dev->queue_mutex); + /* Borage decode prefix has ten client bits. */ + for (id = 0; id < 1024; id++) { + bool used = false; + + list_for_each_entry(other, &ctx->dev->contexts, queue_node) + used |= other->client_id == id; + if (!used) + break; + } + ctx->client_id = id; + list_add_tail(&ctx->queue_node, &ctx->dev->contexts); + mutex_unlock(&ctx->dev->queue_mutex); +} + +void avd_queue_del_ctx(struct avd_ctx *ctx) +{ + avd_queue_drain(ctx); + mutex_lock(&ctx->dev->queue_mutex); + if (ctx->dev->compat_ctx == ctx) + ctx->dev->compat_ctx = NULL; + list_del_init(&ctx->queue_node); + mutex_unlock(&ctx->dev->queue_mutex); +} + +int avd_queue_init(struct avd_dev *avd) +{ + unsigned int i; + + spin_lock_init(&avd->queue_lock); + mutex_init(&avd->queue_mutex); + INIT_LIST_HEAD(&avd->contexts); + INIT_DELAYED_WORK(&avd->queue_work, avd_queue_worker); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + + f->idx = i; + f->prog = devm_kzalloc(avd->dev, sizeof(*f->prog), GFP_KERNEL); + if (!f->prog) + return -ENOMEM; + } + return 0; +} + +void avd_queue_cleanup(struct avd_dev *avd) +{ + unsigned int i; + + mutex_lock(&avd->queue_mutex); + WRITE_ONCE(avd->stopping, true); + if (avd->inflight) { + avd_reset_queue(avd, -ESHUTDOWN); + avd_retire(avd); + } + mutex_unlock(&avd->queue_mutex); + cancel_delayed_work_sync(&avd->queue_work); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + avd_buf_free(avd, avd->frames[i].prog); + if (avd->frame_fp[i]) + avd_buf_free(avd, avd->frame_fp[i]); + } +} diff --git a/drivers/media/platform/apple/avd/avd-queue.h b/drivers/media/platform/apple/avd/avd-queue.h index ca04f908ef359..cd912259c636a 100644 --- a/drivers/media/platform/apple/avd/avd-queue.h +++ b/drivers/media/platform/apple/avd/avd-queue.h @@ -50,6 +50,10 @@ struct avd_frame { void *codec_priv; /* per-frame codec state, owned by the codec */ unsigned long deadline; /* jiffies, per-frame watchdog */ + /* Legacy m2m HOLD_CAPTURE API retains scheduler ownership until retire. */ + bool scheduler_held; + /* VP to PP instruction FIFO for this frame (open backends). */ + dma_addr_t fifo_iova; }; struct avd_backend_ops { @@ -66,6 +70,7 @@ struct avd_backend_ops { irqreturn_t (*irq)(struct avd_dev *avd); /* calls avd_frame_complete() */ int (*suspend)(struct avd_dev *avd); int (*resume)(struct avd_dev *avd); + int (*set_property)(struct avd_ctx *ctx, u32 property, u32 value); }; /* Codec half. Replaces avd_coded_fmt_ops.run/submit/done. CPU only, no MMIO. */ diff --git a/drivers/media/platform/apple/avd/avd-regs.h b/drivers/media/platform/apple/avd/avd-regs.h index 388f3e64895f5..ace0e4c48f40a 100644 --- a/drivers/media/platform/apple/avd/avd-regs.h +++ b/drivers/media/platform/apple/avd/avd-regs.h @@ -13,6 +13,8 @@ #define AVD_MBOX1_NOT_EMPTY BIT(3) #define AVD_REG_MBOX1_STATUS 0x5c +/* Read side: bits 8-11 write pointer, bits 12-15 read pointer, bit 17 empty. */ +#define AVD_MBOX1_STATUS_EMPTY BIT(17) #define AVD_REG_MBOX1_RETRIEVE 0x64 #define AVD_MBOX_ENABLE BIT(0) diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index 1ff19368b6971..56f9f7e541df0 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -894,8 +894,12 @@ static int avd_start_streaming(struct vb2_queue *q, unsigned int count) struct vb2_v4l2_buffer *vbuf; int ret; + WRITE_ONCE(ctx->draining, false); if (V4L2_TYPE_IS_CAPTURE(q->type)) return 0; + /* A new stream gets a fresh hardware error budget. */ + ctx->hw_errors = 0; + ctx->broken = false; desc = ctx->coded_fmt_desc; if (WARN_ON(!desc)) { @@ -918,7 +922,7 @@ static int avd_start_streaming(struct vb2_queue *q, unsigned int count) return ret; } -static void avd_queue_cleanup(struct vb2_queue *vq, u32 state) +static void avd_m2m_queue_cleanup(struct vb2_queue *vq, u32 state) { struct avd_ctx *ctx = vb2_get_drv_priv(vq); @@ -943,6 +947,7 @@ static void avd_stop_streaming(struct vb2_queue *q) { struct avd_ctx *ctx = vb2_get_drv_priv(q); + avd_queue_drain(ctx); if (V4L2_TYPE_IS_OUTPUT(q->type)) { const struct avd_coded_fmt_desc *desc = ctx->coded_fmt_desc; @@ -957,7 +962,7 @@ static void avd_stop_streaming(struct vb2_queue *q) ctx->job.segments = NULL; } - avd_queue_cleanup(q, VB2_BUF_STATE_ERROR); + avd_m2m_queue_cleanup(q, VB2_BUF_STATE_ERROR); } const struct vb2_ops avd_queue_ops = { @@ -971,36 +976,17 @@ const struct vb2_ops avd_queue_ops = { .stop_streaming = avd_stop_streaming, }; +/* Legacy multi-slice codecs finish held OUTPUT requests during prepare. */ void avd_job_finish_no_pm(struct avd_ctx *ctx, enum vb2_buffer_state result) { - struct avd_dev *avd = ctx->dev; - unsigned long flags; - - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_ctx == ctx) { - avd->job_state = AVD_JOB_IDLE; - avd->job_ctx = NULL; - avd->job_pending = false; - } - spin_unlock_irqrestore(&avd->job_lock, flags); - - if (ctx->coded_fmt_desc->ops->done) { - struct vb2_v4l2_buffer *src_buf, *dst_buf; - - src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); - dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); - ctx->coded_fmt_desc->ops->done(ctx, src_buf, dst_buf, result); - } - - v4l2_m2m_buf_done_and_job_finish(ctx->dev->m2m_dev, ctx->fh.m2m_ctx, - result); + if (WARN_ON(!ctx->dev->building)) + return; + ctx->dev->compat_held = true; + ctx->dev->building->status = result == VB2_BUF_STATE_DONE ? 0 : -EIO; } void avd_job_finish(struct avd_ctx *ctx, enum vb2_buffer_state result) { - struct avd_dev *avd = ctx->dev; - - pm_runtime_put_autosuspend(avd->dev); avd_job_finish_no_pm(ctx, result); } diff --git a/drivers/media/platform/apple/avd/avd-vp9.c b/drivers/media/platform/apple/avd/avd-vp9.c index 4cdcb6594c4d2..37374b73e8a86 100644 --- a/drivers/media/platform/apple/avd/avd-vp9.c +++ b/drivers/media/platform/apple/avd/avd-vp9.c @@ -1049,6 +1049,13 @@ static enum avd_image_fmt avd_vp9_get_image_fmt(struct avd_ctx *ctx, #undef BIT_DEPTH } +static unsigned int avd_vp9_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_vp9_ctx *priv = ctx->priv; + + return max_t(int, priv->submit_num, 1); +} + static void avd_vp9_submit(struct avd_ctx *ctx) { struct avd_vp9_ctx *vp9_ctx = ctx->priv; @@ -1078,5 +1085,6 @@ const struct avd_coded_fmt_ops avd_vp9_fmt_ops = { .run = avd_vp9_run, .done = avd_vp9_done, .submit = avd_vp9_submit, + .pp_kicks = avd_vp9_pp_kicks, .get_image_fmt = avd_vp9_get_image_fmt, }; diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index f38339fdf6e2c..ef6084ed566ba 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -14,14 +14,17 @@ #ifndef AVD_H_ #define AVD_H_ -#include "avd-queue.h" -#include "avd-av1-fp.h" - #include #include #include #include +#include "avd-pm.h" +#include + +#include "avd-queue.h" +#include "avd-av1-fp.h" + #include #include #include @@ -126,6 +129,8 @@ struct avd_decoded_buffer { /* Pending CPU fill, retained with the buffer until dequeue. */ size_t grey_chroma_offset; bool grey_chroma_10bit; + /* H.264/HEVC: scratch bank + 1 of the picture decoded into this buffer */ + u8 bank; union { struct avd_vp9_decoded_buffer_info vp9; @@ -147,6 +152,8 @@ struct avd_coded_fmt_ops { void (*adjust_decoded_fmt)(struct avd_ctx *ctx, struct v4l2_pix_format_mplane *pix_mp); void (*submit)(struct avd_ctx *ctx); + /* PP EXEC commands the last built frame needs (submit's count), >= 1. */ + unsigned int (*pp_kicks)(struct avd_ctx *ctx); int (*start)(struct avd_ctx *ctx); void (*stop)(struct avd_ctx *ctx); int (*run)(struct avd_ctx *ctx); @@ -183,15 +190,23 @@ struct avd_coded_fmt_desc { unsigned int capability; }; +struct avd_apple_hw; + struct avd_variant { enum avd_av1_fp_layout av1_fp_layout; + const struct avd_apple_hw *apple; + size_t fp_size; unsigned int vp_slots[4]; unsigned int vp_slot_base[4]; unsigned int fifo_slots; unsigned int capabilities; + /* Codecs advertised only when the firmware CODECS trailer lists them. */ + unsigned int fw_caps_required; void (*configure_stream)(struct avd_dev *avd, dma_addr_t addr, u8 fifo_idx, u32 vp_slot); const char* fw_name; + /* Older image tried when fw_name is not installed (legacy transport). */ + const char *fw_name_fallback; unsigned char revision; /* the same as the device tree */ /* just for convenience */ u32 vp_slot_offset; @@ -201,23 +216,16 @@ struct avd_variant { unsigned int quirks; }; -/* Lifecycle of the one m2m job the device runs at a time. */ -enum avd_job_state { - AVD_JOB_IDLE, - AVD_JOB_RUNNING, /* inside avd_device_run() */ - AVD_JOB_SUBMITTED, /* handed to the hardware, watchdog armed */ - AVD_JOB_FINISHING, /* claimed by the IRQ handler or the watchdog */ -}; - struct avd_dev { struct device *dev; + struct avd_pm pm; struct v4l2_device v4l2_dev; struct media_device mdev; struct video_device vdev; struct v4l2_m2m_dev *m2m_dev; struct platform_device *pdev; - const struct firmware *fw; /* fw is lost on suspend */ + const struct firmware *fw; /* separately installed open firmware */ void __iomem *code; void __iomem *mbox; @@ -228,19 +236,48 @@ struct avd_dev { struct mutex vdev_lock; - /* The current job; see avd_device_run(). Protected by job_lock. */ - spinlock_t job_lock; - enum avd_job_state job_state; - struct avd_ctx *job_ctx; - u64 job_seq; - bool job_end_sent; - bool job_pending; - enum vb2_buffer_state job_pending_result; + /* queue_lock protects slot state/tag and job_ready observations. */ + spinlock_t queue_lock; + /* Serializes CPU preparation, submission, retirement and reset. */ + struct mutex queue_mutex; + struct delayed_work queue_work; + struct list_head contexts; + /* One m2m dispatch reservation; never identifies firmware-owned work. */ + /* + * m2m runs one device_run at a time until job_finish; a job that finds + * no free slot waits here and is restarted when a frame retires. + */ + struct avd_ctx *deferred_ctx; + struct avd_frame frames[AVD_MAX_FRAMES]; + struct avd_buf *frame_fp[AVD_MAX_FRAMES]; + u64 frame_seq; + unsigned int inflight; + bool stopping; + bool faulted; + /* Recovery pacing: when the last one ran, and a sliding storm count. */ + unsigned long last_recover; + unsigned long storm_start; + unsigned int storm_count; + /* After a recovery storm, fail new frames until this time. */ + unsigned long cooldown_until; + u64 fw_generation; + const struct avd_backend_ops *backend; + void *backend_priv; + /* Only the open-firmware compatibility path uses these. */ + struct avd_frame *building; + /* Open firmware queue ABI 2 state, NULL when the node has no SRAM. */ + struct avd_qfw *qfw; + struct avd_frame *compat_active; + /* Context of the last legacy submission, for VP DONE without a frame. */ + struct avd_ctx *compat_ctx; + bool compat_held; struct reset_control *rstc; const struct avd_variant *variant; unsigned int capabilities; + /* What the loaded image's CODECS trailer (or its absence) allows. */ + unsigned int fw_capabilities; }; struct avd_segment { @@ -258,6 +295,7 @@ struct avd_buf { void *cpu; dma_addr_t addr; size_t size; + struct avd_dma_release *release; }; struct avd_ctx { @@ -272,14 +310,26 @@ struct avd_ctx { enum avd_image_fmt image_fmt; bool decomp; - struct delayed_work watchdog_work; - /* job_seq of the job watchdog_work was armed for */ - u64 wd_seq; + struct list_head queue_node; + struct work_struct submit_work; + bool draining; + /* Consecutive frames that failed in hardware; reset by a good frame. */ + unsigned int hw_errors; + /* Too many hardware errors: fail every frame until streaming restarts. */ + bool broken; + bool session_started; + u64 session_generation; + enum avd_frame_dep next_dep; + u32 client_id; void *priv; struct avd_comp comp; int fifo_idx; + /* Pictures of this context the queue may hold at once (0 or 1: one). */ + unsigned int max_frames; + /* VP to PP FIFO for the job being built; 0 means ctx->inst. */ + dma_addr_t job_fifo; struct avd_job job; struct avd_buf inst; }; @@ -336,6 +386,26 @@ static inline struct avd_ctx *file_to_ctx(struct file *filp) /* hw stuff */ int avd_boot(struct avd_dev *avd); +int avd_queue_init(struct avd_dev *avd); +void avd_queue_cleanup(struct avd_dev *avd); +void avd_queue_run(void *priv); +int avd_queue_ready(void *priv); +void avd_queue_abort(void *priv); +void avd_queue_drain(struct avd_ctx *ctx); +void avd_queue_add_ctx(struct avd_ctx *ctx); +void avd_queue_del_ctx(struct avd_ctx *ctx); +int avd_open_build_program(struct avd_ctx *ctx); +extern const struct avd_backend_ops avd_open_backend; +extern const struct avd_backend_ops avd_apple_backend; +extern const struct avd_backend_ops avd_qfw_backend; +struct avd_qfw; +int avd_qfw_probe(struct avd_dev *avd); +void avd_qfw_select(struct avd_dev *avd); +void avd_qfw_prepare_boot(struct avd_dev *avd); +bool avd_backend_is_qfw(const struct avd_dev *avd); +int avd_backend_select(struct avd_dev *avd); +bool avd_mbox_pop(struct avd_dev *avd, u32 *msg); +int avd_backend_boot(struct avd_dev *avd); void avd_shutdown(struct avd_dev *avd); void avd_status(struct avd_dev *avd, u32 vp); diff --git a/drivers/media/platform/apple/avd/tests/.kunitconfig b/drivers/media/platform/apple/avd/tests/.kunitconfig new file mode 100644 index 0000000000000..c079bc44af531 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/.kunitconfig @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +CONFIG_KUNIT=y +CONFIG_COMPILE_TEST=y +CONFIG_MEDIA_SUPPORT=y +# CONFIG_MEDIA_SUPPORT_FILTER is not set +CONFIG_VIDEO_DEV=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_OF=y +CONFIG_VIDEO_APPLE_AVD=y +CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST=y +CONFIG_WERROR=y diff --git a/drivers/media/platform/apple/avd/tests/include/linux/errno.h b/drivers/media/platform/apple/avd/tests/include/linux/errno.h new file mode 100644 index 0000000000000..32a2df3ba8d20 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/errno.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include_next diff --git a/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h b/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h new file mode 100644 index 0000000000000..ccda48a6de34b --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_INTERRUPT_H_ +#define AVD_HOST_INTERRUPT_H_ +#define irqreturn_t int +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h b/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h new file mode 100644 index 0000000000000..a73bb63db2c8e --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_JIFFIES_H_ +#define AVD_HOST_JIFFIES_H_ +#include +#define time_after_eq(a, b) ((long)((a) - (b)) >= 0) +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/linux/limits.h b/drivers/media/platform/apple/avd/tests/include/linux/limits.h new file mode 100644 index 0000000000000..304aeeeff0fbd --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/limits.h @@ -0,0 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include +#include +#define U32_MAX UINT32_MAX diff --git a/drivers/media/platform/apple/avd/tests/include/linux/types.h b/drivers/media/platform/apple/avd/tests/include/linux/types.h new file mode 100644 index 0000000000000..8b9638abb9a7f --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/types.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_TYPES_H_ +#define AVD_HOST_TYPES_H_ +#include +#include +#include_next +typedef __u8 u8; +typedef __u32 u32; +typedef __u64 u64; +typedef __u64 dma_addr_t; +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h b/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h new file mode 100644 index 0000000000000..6cd054cc198d3 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_VB2_H_ +#define AVD_HOST_VB2_H_ +enum vb2_buffer_state { VB2_BUF_STATE_DONE, VB2_BUF_STATE_ERROR }; +#endif diff --git a/drivers/media/platform/apple/avd/tests/queue-host.c b/drivers/media/platform/apple/avd/tests/queue-host.c new file mode 100644 index 0000000000000..5983d7ad5d6a3 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/queue-host.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Exercise the production helpers with deterministic shuffled completions. */ +#include +#include +#include + +#include "../avd-apple-protocol.h" +#include "../avd-queue-internal.h" + +int main(void) +{ + struct avd_frame pool[AVD_MAX_FRAMES] = {}; + int clients[3]; + u64 last[3] = {}, seq = 0; + u32 rng = 0x3851, old_tag[AVD_MAX_FRAMES] = {}; + unsigned int i, step, completed = 0; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + pool[i].idx = i; + for (step = 0; step < 100000; step++) { + struct avd_frame *f; + + rng = rng * 1664525 + 1013904223; + f = &pool[(rng >> 16) & 15]; + if (f->state == AVD_FRAME_FREE) { + struct avd_ctx *ctx = (void *)&clients[(rng >> 8) % 3]; + + assert(!avd_slot_reserve(f, ctx, ++seq)); + assert(!avd_slot_complete(f, f->tag, 0)); + f->deadline = step + 100; + f->state = AVD_FRAME_SUBMITTED; + assert(!avd_slot_complete(f, old_tag[f->idx], 0)); + old_tag[f->idx] = f->tag; + } else if (f->state == AVD_FRAME_SUBMITTED) { + assert(avd_slot_complete(f, f->tag, (rng & 1) ? -EIO : 0)); + assert(!avd_slot_complete(f, f->tag, 0)); + } + for (i = 0; i < AVD_MAX_FRAMES; i++) { + unsigned int c; + + f = &pool[i]; + if (f->state != AVD_FRAME_DONE || !avd_slot_first(pool, f)) + continue; + for (c = 0; c < 3; c++) + if (f->ctx == (void *)&clients[c]) + break; + assert(c < 3 && f->seq > last[c]); + last[c] = f->seq; + f->state = AVD_FRAME_FREE; + completed++; + } + } + for (i = 0; i < 8; i++) { + assert(avd_inbox_full(i, (i + 1) & 7)); + assert(!avd_inbox_full(i, (i + 2) & 7)); + assert(avd_apple_notification(i) == 0x01482cf8U + 0x60 * i); + } + { + struct avd_apple_reply_layout layout = { + .type_word = 0, .type_mask = 31, .client_word = 1, + .client_mask = 1023, .frame_word = 3, .status_word = 4, + .done_type = 8, .verified = true, + }; + struct avd_apple_reply reply; + u32 record[20] = { 7, 9, 0, 1234 }; + + assert(avd_apple_parse_reply(record, 20, &layout, &reply) == -EINPROGRESS); + record[0] = 8; + assert(!avd_apple_parse_reply(record, 20, &layout, &reply)); + assert(reply.frame == 1234 && reply.client == 9 && !reply.status); + record[4] = 1; + assert(!avd_apple_parse_reply(record, 20, &layout, &reply)); + assert(reply.status == -EIO); + assert(avd_apple_parse_reply(record, 19, &layout, &reply) == -EMSGSIZE); + } + assert(completed > 1000); + printf("PASS: 100000 shuffled operations, %u ordered retirements\n", completed); + return 0; +} diff --git a/drivers/media/platform/apple/avd/tests/run-host.sh b/drivers/media/platform/apple/avd/tests/run-host.sh new file mode 100755 index 0000000000000..70e287f14732d --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/run-host.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only OR MIT +set -eu +avd_test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +avd_test_out=${1:?usage: run-host.sh OUTPUT_BINARY} +${CC:-cc} -std=gnu11 -Wall -Wextra -Werror -O2 \ + -I"$avd_test_dir/include" "$avd_test_dir/queue-host.c" -o "$avd_test_out" +"$avd_test_out" From 0128fe92a11263d827cd262cb2f2cfa2956b73c6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 04:33:32 +1100 Subject: [PATCH 5/5] media: apple: preserve AV1 open decode during frame ops rollout Retain legacy run, completion and PP kick callbacks while open transports still reject native frame operations. Adapt the shared CPU builder using explicit segment boundaries and keep its scratch buffers alive until completion. Select the compatibility path after boot and bind retirement to that selection across backend resets. Preserve grain-less decode for film-grain controls. Publish AV1 reference state with the source timestamp before core copies CAPTURE metadata, and retain segmentation maps for matching MiCols/MiRows. Cover backend selection, legacy submission and failure cleanup, CDF publication, segmentation sizing, 64 packing vectors and sixteen live slots with host tests. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../platform/apple/avd/avd-av1-frame-ops.rst | 115 +++++++------- drivers/media/platform/apple/avd/avd-av1.c | 111 +++++++++++++- .../platform/apple/avd/avd-backend-open.c | 16 +- .../platform/apple/avd/avd-backend-queue.c | 2 +- drivers/media/platform/apple/avd/avd-queue.c | 16 +- drivers/media/platform/apple/avd/avd.h | 13 ++ .../platform/apple/avd/tests/host-driver.h | 25 +++- .../platform/apple/avd/tests/host-shim.h | 1 + .../platform/apple/avd/tests/host-test.c | 141 +++++++++++++++++- .../apple/avd/tests/run-host-tests.py | 5 + 10 files changed, 369 insertions(+), 76 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-av1-frame-ops.rst b/drivers/media/platform/apple/avd/avd-av1-frame-ops.rst index 727331197bde9..bb96d18dcc083 100644 --- a/drivers/media/platform/apple/avd/avd-av1-frame-ops.rst +++ b/drivers/media/platform/apple/avd/avd-av1-frame-ops.rst @@ -1,71 +1,82 @@ AV1 frame operations ==================== -This codec series targets avd_codec_frame_ops. It requires the accompanying -16-slot core and queued open-firmware backend before decode is usable. The -legacy core rejects a codec without a run callback instead of dereferencing it. -The queue header is the shared contract, unchanged. +AV1 implements avd_codec_frame_ops and retains the legacy run/done/submit and +pp_kicks callbacks. The current open transports still require legacy segments +and codec-specific PP kicks. avd_get_frame_ops selects that compatibility path +when a legacy run callback exists, and selects frame_ops on Apple. Retirement +uses the operations selected at preparation, including across a backend reset. +The queue header is core's unmodified shared contract. -prepare consumes controls already applied by the core and the slot's src/dst. -It never accesses an m2m queue, completes a request, or touches MMIO. The open -program is a little-endian stream: header EXEC, the existing header words, then -one tile payload and its EXEC per tile (END on the last tile). prog_words counts -all words, including EXEC records. The backend owns VP selection, FIFO assignment, -submission and VP/PP chaining. The core supplies fp only for an Apple submission; -NULL fp selects preparation of the open program. +The compatibility adapter applies the request controls, invokes the same CPU +prepare implementation, and copies the recorded header/tile ranges into the +legacy job. It never finds boundaries by treating arbitrary instruction data +as opcodes. Scratch and auxiliary DMA allocations stay alive until done calls +retire. AV1 uses one legacy picture per context until the open transports accept +native frame_ops; the queued transport gets the exact number of PP kicks. +Open decode retains aurora-wip's grain-less fallback for APPLY_GRAIN requests. +Film-grain synthesis is not implemented by the existing instruction builder. -Each live frame owns its pipe state and every above/left/MV scratch allocation. -CDFs and segmentation maps are separate coherent DMA allocations. Capture-index -records retain their output state; in-flight frames retain input and output -state independently. A disabled CDF/map update inherits an immutable allocation, -including its ownership reference, rather than copying a potentially unfinished -capture tail. Reusing the original capture index therefore cannot free inherited -state. Color/MV reference data already stored in capture buffers remains subject -to the stateless reference-buffer lifetime contract. +prepare consumes applied controls and the slot's src/dst. It never accesses an +m2m queue, completes a request, or touches MMIO. The little-endian open program +contains header EXEC, the existing header words, and one tile payload plus EXEC +per tile (END on the last tile). prog_words includes EXEC records. Only the +compatibility adapter accesses the legacy job; PP register writes live in the +transport. Core supplies fp only for an Apple submission. -The incoming dependency is PREV_DONE exactly when the current frame uses the +Each live frame owns its pipe state and all above/left/MV scratch allocations. +CDFs and segmentation maps have reference-counted coherent DMA allocations. +Capture-index records retain output state; live frames retain their inputs and +outputs independently. Disabled updates inherit an immutable allocation rather +than copying unfinished hardware output. Reusing the original capture index +cannot free inherited state. Color/MV data in capture buffers follows stateless +reference-buffer lifetime rules. Segmentation inherits a map only when MiCols +and MiRows match, retaining the existing builder's size comparison. Reference +publication uses OUTPUT's timestamp: +core copies that metadata to CAPTURE only after prepare has returned. + +The incoming dependency is PREV_DONE exactly when the current frame selects the previous frame as primary reference and that previous frame enables frame-end -CDF updates. Defaults (intra, error resilient or PRIMARY_REF_NONE), a different -primary reference, and disabled frame-end updates produce NONE. prepare does -not read hardware CDF/map output on the CPU. It refuses an unfinished older CDF -or segmentation producer with EAGAIN when the specified dependency rule cannot -express the dependency. The integrating core must retry preparation after an -earlier completion; the present contract does not define that retry behavior. -The core must also propagate failed producers to already prepared dependants, -serialize prepare calls per context, and drain/retire all slots before stop. +CDF updates. Intra/error-resilient/PRIMARY_REF_NONE defaults, another primary +reference, and disabled frame-end updates produce NONE. prepare does not read +hardware CDF/map output on the CPU. An unfinished older producer which that +rule cannot express returns EAGAIN. Native integration still needs preparation +retry and propagation of failed producers to already prepared dependants. +Legacy decode is serial and retires its producer before the next preparation. Apple layout evidence and remaining work ---------------------------------------- The source is the clean-room Apple AVD protocol SPEC dated 2026-10-31, sections 5, 6.1, 6.5, 6.6 and 10. Public V3 layouts in eiln/avd cannot supply AV1 fields: -those generations have no AV1 hardware. avd-av1-fp.c retains an unnamed [C] -table with the missing fields and their sources. It writes the proved Borage -tile columns/rows at 0x3e4/0x3e6 and stages the proved command destinations, -work-count bound 0x1002 and fp+0x4f8 -> cmd+0x4c copy. Both Apple pack operations -return EOPNOTSUPP: the SPEC omits the syntax/DVA layout, source flag shifts and -packed-address encoding needed for a valid command. No guessed payload is sent. -T6040 selects Borage; T8140 explicitly selects a TODO row pending the Daybreak -per-SoC ledger task t-3862. Other Apple layouts stay unsupported. Existing AV1 -hardware capability bits are retained; T6000/T8112 have neither AV1 capability -nor an AV1 VP. Film grain synthesis is unsupported by the existing open builder -and an APPLY_GRAIN request is rejected explicitly. +those generations have no AV1 hardware. avd-av1-fp.c keeps an unnamed [C] table +with sources for the missing fields. It stages proved Borage tile columns/rows +at 0x3e4/0x3e6, command destinations, work-count cap 0x1002 and the +fp+0x4f8 -> cmd+0x4c copy. Both Apple pack operations return EOPNOTSUPP: +the SPEC omits syntax/DVA mappings, source flag shifts and packed-address +encoding. These facts do not yet describe a usable Apple AV1 payload. + +T6040 selects Borage. T8140 has a distinct TODO row tied to Daybreak's per-SoC +ledger task t-3862. Other Apple layouts stay unsupported. Hardware capability +masks still govern format exposure; T6000/T8112 have no AV1 capability or VP. Verification ------------ Run ``python3 drivers/media/platform/apple/avd/tests/run-host-tests.py`` from a -Git checkout, including a sparse checkout. A host C compiler with ASan and UBSan -is required. Tests compile the production CPU code with small DMA/V4L2 doubles -and the checkout's real AV1 UAPI definitions. They cover live slot isolation, -CDF inheritance, dependency cases, allocation failure cleanup, payload bounds, -program capacity and Apple-layout refusal. The frozen pre-conversion packing -oracle comes from aurora-wip fbc4e7934ca7325f3483b72a51d24c74aa8536a4, with -its original license notice. Program comparison uses the same resolved DMA -addresses in both builders; allocation ownership intentionally changes addresses. +Git checkout (sparse checkouts work). A host C compiler with ASan and UBSan is +required. Tests compile production CPU code and the production backend selector +with DMA/V4L2 doubles and the checkout's AV1 UAPI definitions. The licensed +packing oracle is frozen from aurora-wip +fbc4e7934ca7325f3483b72a51d24c74aa8536a4. Program comparison resolves the same +DMA addresses for both builders; changed ownership intentionally changes them. -These are synthetic control vectors, not SVT-AV1 bitstreams or hardware output -validation. Film grain is a rejection test. AV1 show-existing has no decode-frame -control in the stateless UAPI: userspace redisplays a reference without submitting -a new decode. Queued firmware execution, Apple payload completion, retry semantics -and end-to-end SVT-AV1 decode remain integration/qualification work. +Coverage includes 64 parity vectors (tiles, CDEF, restoration, superres, 8/10-bit, +inter frames and grain-less fallback), legacy multi-tile submission/PP counts, +submission failure cleanup, source-timestamp publication, producer errors, the +dependency table, sixteen live slots, inherited state and capture reuse, +allocation failure cleanup, payload/program bounds and Apple-layout refusal. +These are synthetic controls, not SVT-AV1 bitstreams or pixel validation. +Show-existing is a userspace reference redisplay without a new stateless decode +control. Native open execution, the missing Apple payload facts, retry/failure +integration and end-to-end stream qualification remain outstanding. diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index 6cd4823559b4b..6e64b55747180 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -273,6 +273,8 @@ struct avd_av1_run { const struct v4l2_ctrl_av1_tile_group_entry *tile_group; const struct v4l2_ctrl_av1_film_grain *grain; unsigned int num_tiles; + /* Ends exclude EXEC words; used only by the legacy transport adapter. */ + size_t segment_end[V4L2_AV1_MAX_TILE_COUNT + 1]; }; struct avd_av1_ctx { @@ -282,6 +284,8 @@ struct avd_av1_ctx { bool prev_updates_cdf; bool have_prev; unsigned long active_slots; + struct avd_frame legacy_frame; + struct avd_buf legacy_prog; }; static void avd_av1_push(struct avd_av1_run *run, u32 inst) @@ -1170,6 +1174,7 @@ static void set_tiles(struct avd_ctx *ctx, struct avd_av1_run *run) tile_info->width_in_sbs_minus_1 [tile_group->tile_col]), "tile_op_end"); + run->segment_end[tile_id + 1] = run->slot->prog_words; push(AVD_OP_EXEC | AVD_OP_EXEC_FLAG_END( tile_id + 1 == run->num_tiles), "tile_exec"); } @@ -1232,8 +1237,11 @@ static int avd_av1_set_aux(struct avd_ctx *ctx, struct avd_av1_run *run) if (!run->cdf_out) return -ENOMEM; - if (ref && seg_enabled && ref->width == frame->frame_width_minus_1 && - ref->height == frame->frame_height_minus_1) { + if (ref && seg_enabled && + DIV_ROUND_UP(ref->width + 1, 8) == + DIV_ROUND_UP(frame->frame_width_minus_1 + 1, 8) && + DIV_ROUND_UP(ref->height + 1, 8) == + DIV_ROUND_UP(frame->frame_height_minus_1 + 1, 8)) { run->seg_in = avd_av1_aux_get(ref->seg); /* The two segmentation addresses let hardware read the previous * map directly. No speculative CPU copy of an unfinished map. */ @@ -1513,7 +1521,8 @@ static int avd_av1_publish_ref(struct avd_ctx *ctx, struct avd_av1_run *run) avd_av1_aux_put(ctx, ref->seg); ref->cdf = avd_av1_aux_get(run->cdf_out); ref->seg = avd_av1_aux_get(run->seg_out); - ref->timestamp = run->base.bufs.dst->vb2_buf.timestamp; + /* Core copies source metadata to CAPTURE after prepare returns. */ + ref->timestamp = run->base.bufs.src->vb2_buf.timestamp; ref->width = run->frame->frame_width_minus_1; ref->height = run->frame->frame_height_minus_1; av1->prev_timestamp = ref->timestamp; @@ -1563,11 +1572,7 @@ static int avd_av1_prepare(struct avd_ctx *ctx, struct avd_frame *f) /* The control is a fixed-capacity array; only the tile grid is work. */ run->num_tiles = run->frame->tile_info.tile_cols * run->frame->tile_info.tile_rows; - /* The old builder has no film grain synthesis program. */ - if (run->grain->flags & V4L2_AV1_FILM_GRAIN_FLAG_APPLY_GRAIN) { - ret = -EOPNOTSUPP; - goto fail; - } + /* Preserve the existing open decoder's grain-less output fallback. */ for (i = 0; i < V4L2_AV1_REFS_PER_FRAME; i++) { if (run->frame->ref_frame_idx[i] < 0 || run->frame->ref_frame_idx[i] >= V4L2_AV1_TOTAL_REFS_PER_FRAME) { @@ -1597,6 +1602,7 @@ static int avd_av1_prepare(struct avd_ctx *ctx, struct avd_frame *f) avd_av1_push(run, AVD_OP_EXEC | AVD_OP_EXEC_FLAG_START_REV4(ctx->dev->variant->revision == 4)); set_header(ctx, run); + run->segment_end[0] = f->prog_words; set_tiles(ctx, run); if (run->error) { ret = run->error; @@ -1641,6 +1647,88 @@ static void avd_av1_retire(struct avd_ctx *ctx, struct avd_frame *f, clear_bit(f->idx, &av1->active_slots); } +/* + * The open transports still consume legacy segments and PP kick counts. Keep + * that interface until they accept native frame_ops. All packing and auxiliary + * ownership stay in prepare/retire; only this adapter accesses the legacy job. + */ +static int avd_av1_run(struct avd_ctx *ctx) +{ + struct avd_av1_ctx *av1 = ctx->priv; + struct avd_frame *f = &av1->legacy_frame; + struct avd_av1_run *run; + struct avd_run base; + const __le32 *words; + size_t pos = 1, end, i, j; + int ret; + + avd_run_preamble(ctx, &base); + if (!av1->legacy_prog.cpu) { + ret = avd_buf_alloc(ctx->dev, &av1->legacy_prog, + (AVD_MAX_INST + 1) * (V4L2_AV1_MAX_TILE_COUNT + 1) * sizeof(u32)); + if (ret) + goto complete_controls; + } + f->src = base.bufs.src; + f->dst = base.bufs.dst; + f->prog = &av1->legacy_prog; + f->seq++; + ret = avd_av1_prepare(ctx, f); + if (ret) + goto complete_controls; + run = f->codec_priv; + ret = avd_init_job(ctx, AVD_CODEC_AV1, run->num_tiles + 1); + if (ret) + goto retire; + words = f->prog->cpu; + for (i = 0; i <= run->num_tiles; i++) { + struct avd_segment *seg = &ctx->job.segments[i]; + + end = run->segment_end[i]; + if (end < pos || end - pos > AVD_MAX_INST) { + ret = -EOVERFLOW; + goto retire; + } + seg->num = end - pos; + for (j = 0; j < seg->num; j++) + seg->instructions[j] = le32_to_cpu(words[pos + j]); + pos = end + !!i; + } + ctx->job.num = run->num_tiles; + avd_run_postamble(ctx, &base); + ret = avd_submit_job(ctx); + if (!ret) + return 0; + avd_av1_retire(ctx, f, VB2_BUF_STATE_ERROR); + return ret; +retire: + avd_av1_retire(ctx, f, VB2_BUF_STATE_ERROR); +complete_controls: + avd_run_postamble(ctx, &base); + return ret; +} + +static void avd_av1_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src, + struct vb2_v4l2_buffer *dst, enum vb2_buffer_state state) +{ + struct avd_av1_ctx *av1 = ctx->priv; + + avd_av1_retire(ctx, &av1->legacy_frame, state); +} + +static unsigned int avd_av1_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_av1_ctx *av1 = ctx->priv; + struct avd_av1_run *run = av1->legacy_frame.codec_priv; + + return run ? run->num_tiles : 1; +} + +static void avd_av1_submit(struct avd_ctx *ctx) +{ + avd_open_av1_submit(ctx, avd_av1_pp_kicks(ctx)); +} + static void avd_av1_stop(struct avd_ctx *ctx) { struct avd_av1_ctx *av1 = ctx->priv; @@ -1655,6 +1743,7 @@ static void avd_av1_stop(struct avd_ctx *ctx) list_del(&ref->node); kfree(ref); } + avd_buf_free(ctx->dev, &av1->legacy_prog); kfree(av1); ctx->priv = NULL; } @@ -1667,6 +1756,8 @@ static int avd_av1_start(struct avd_ctx *ctx) if (!av1) return -ENOMEM; INIT_LIST_HEAD(&av1->refs); + /* The embedded legacy frame must retire before this ctx runs again. */ + ctx->max_frames = 1; ctx->priv = av1; return 0; } @@ -1729,6 +1820,10 @@ const struct avd_coded_fmt_ops avd_av1_fmt_ops = { .adjust_decoded_fmt = avd_av1_adjust_decoded_fmt, .start = avd_av1_start, .stop = avd_av1_stop, + .run = avd_av1_run, + .done = avd_av1_done, + .submit = avd_av1_submit, + .pp_kicks = avd_av1_pp_kicks, .try_ctrl = avd_av1_try_ctrl, .get_image_fmt = avd_av1_get_image_fmt, }; diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c index 9680dc4ef64e1..3ef5b7642b147 100644 --- a/drivers/media/platform/apple/avd/avd-backend-open.c +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -80,7 +80,7 @@ static int avd_open_submit(struct avd_dev *avd, struct avd_frame *f) if (avd->compat_active) return -EBUSY; /* Native frame_ops require firmware-side PP chaining and tagged replies. */ - if (ctx->coded_fmt_desc->frame_ops) + if (avd_get_frame_ops(ctx)) return -EOPNOTSUPP; if (codec > AVD_CODEC_AV1 || !words || !f->prog_words || f->prog_words > f->prog->size / sizeof(u32)) @@ -129,6 +129,20 @@ static int avd_open_submit(struct avd_dev *avd, struct avd_frame *f) return 0; } +/* Legacy AV1: one PP kick for every tile, matching the original transport. */ +void avd_open_av1_submit(struct avd_ctx *ctx, unsigned int kicks) +{ + struct avd_dev *avd = ctx->dev; + u32 cmd = AVD_OP_EXEC | AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx) | + AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots); + unsigned int i; + + writel(cmd | AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4), + avd->ctrl + avd->variant->submit_offset); + for (i = 1; i < kicks; i++) + writel(cmd, avd->ctrl + avd->variant->submit_offset); +} + static irqreturn_t avd_open_irq(struct avd_dev *avd) { struct avd_frame *f = avd->compat_active; diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c index a706602386c17..3902d75cbbd1f 100644 --- a/drivers/media/platform/apple/avd/avd-backend-queue.c +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -203,7 +203,7 @@ static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) unsigned int i; int ret; - if (ctx->coded_fmt_desc->frame_ops) + if (avd_get_frame_ops(ctx)) return -EOPNOTSUPP; if (codec > AVD_CODEC_AV1 || !words || !f->prog_words || f->prog_words > f->prog->size / sizeof(u32)) diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c index 226de0e4d7574..ecbe21d5444b2 100644 --- a/drivers/media/platform/apple/avd/avd-queue.c +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -38,6 +38,7 @@ static void avd_retire(struct avd_dev *avd) progress = false; for (i = 0; i < AVD_MAX_FRAMES; i++) { struct avd_ctx *ctx; + const struct avd_codec_frame_ops *ops; enum vb2_buffer_state state; f = &avd->frames[i]; @@ -49,6 +50,7 @@ static void avd_retire(struct avd_dev *avd) } spin_unlock_irqrestore(&avd->queue_lock, flags); ctx = f->ctx; + ops = avd->frame_codec_ops[i]; state = f->status ? VB2_BUF_STATE_ERROR : VB2_BUF_STATE_DONE; if (!f->status) { ctx->hw_errors = 0; @@ -57,13 +59,13 @@ static void avd_retire(struct avd_dev *avd) dev_err(avd->dev, "stream stopped after %u failed frames; restart streaming to retry\n", ctx->hw_errors); } - if (ctx->coded_fmt_desc->frame_ops) { - if (ctx->coded_fmt_desc->frame_ops->retire) - ctx->coded_fmt_desc->frame_ops->retire(ctx, f, state); + if (ops) { + if (ops->retire) + ops->retire(ctx, f, state); } else if (ctx->coded_fmt_desc->ops->done) { ctx->coded_fmt_desc->ops->done(ctx, f->src, f->dst, state); } - if (ctx->coded_fmt_desc->frame_ops && f->src->vb2_buf.req_obj.req) + if (ops && f->src->vb2_buf.req_obj.req) v4l2_ctrl_request_complete(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); /* CAPTURE first: OUTPUT completion signals the media request. */ @@ -277,7 +279,7 @@ int avd_queue_ready(void *priv) ready = !READ_ONCE(ctx->draining) && !READ_ONCE(avd->stopping); if (ready && ctx->next_dep == AVD_DEP_PREV_DONE) { /* Legacy codecs may hold several pictures in separate banks. */ - unsigned int limit = ctx->coded_fmt_desc && !ctx->coded_fmt_desc->frame_ops ? + unsigned int limit = ctx->coded_fmt_desc && !avd_get_frame_ops(ctx) ? max(ctx->max_frames, 1U) : 1, held = 0; for (i = 0; i < AVD_MAX_FRAMES; i++) @@ -294,7 +296,7 @@ static void avd_queue_submit_work(struct work_struct *work) { struct avd_ctx *ctx = container_of(work, struct avd_ctx, submit_work); struct avd_dev *avd = ctx->dev; - const struct avd_codec_frame_ops *ops = ctx->coded_fmt_desc->frame_ops; + const struct avd_codec_frame_ops *ops; struct avd_frame *f = NULL; unsigned long flags; unsigned int i; @@ -343,6 +345,8 @@ static void avd_queue_submit_work(struct work_struct *work) ret = pm_runtime_resume_and_get(avd->dev); if (ret < 0) goto fail_no_pm; + ops = avd_get_frame_ops(ctx); + avd->frame_codec_ops[f->idx] = ops; f->src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); f->dst = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); /* diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index ef6084ed566ba..b946a320acc58 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -249,6 +249,7 @@ struct avd_dev { */ struct avd_ctx *deferred_ctx; struct avd_frame frames[AVD_MAX_FRAMES]; + const struct avd_codec_frame_ops *frame_codec_ops[AVD_MAX_FRAMES]; struct avd_buf *frame_fp[AVD_MAX_FRAMES]; u64 frame_seq; unsigned int inflight; @@ -403,6 +404,18 @@ int avd_qfw_probe(struct avd_dev *avd); void avd_qfw_select(struct avd_dev *avd); void avd_qfw_prepare_boot(struct avd_dev *avd); bool avd_backend_is_qfw(const struct avd_dev *avd); + +/* Open transports still need legacy segments and codec-specific PP kicks. */ +static inline const struct avd_codec_frame_ops * +avd_get_frame_ops(const struct avd_ctx *ctx) +{ + if (ctx->coded_fmt_desc->ops->run && !avd_backend_is_apple(ctx->dev)) + return NULL; + return ctx->coded_fmt_desc->frame_ops; +} + +void avd_open_av1_submit(struct avd_ctx *ctx, unsigned int kicks); + int avd_backend_select(struct avd_dev *avd); bool avd_mbox_pop(struct avd_dev *avd, u32 *msg); int avd_backend_boot(struct avd_dev *avd); diff --git a/drivers/media/platform/apple/avd/tests/host-driver.h b/drivers/media/platform/apple/avd/tests/host-driver.h index 6548a9fb8124e..8cc8aa9b64f96 100644 --- a/drivers/media/platform/apple/avd/tests/host-driver.h +++ b/drivers/media/platform/apple/avd/tests/host-driver.h @@ -16,14 +16,15 @@ struct avd_job {size_t num;struct avd_segment *segments;}; struct v4l2_pix_format_mplane {u32 width,height;struct {u32 bytesperline,sizeimage;} plane_fmt[1];}; struct v4l2_ctrl {u32 id;unsigned int elems;union {void *p;} p_cur;union {struct v4l2_ctrl_av1_sequence *p_av1_sequence;} p_new;}; struct avd_variant {unsigned int quirks,revision;enum avd_av1_fp_layout av1_fp_layout;}; -struct avd_dev {struct avd_variant *variant;}; -struct avd_ctx {struct avd_dev *dev;void *priv;struct {struct {struct v4l2_pix_format_mplane pix_mp;} fmt;} decoded_fmt;struct avd_comp comp;bool decomp;struct v4l2_ctrl ctrl_hdl[4];struct avd_job job;}; -struct avd_coded_fmt_ops {void (*adjust_decoded_fmt)(struct avd_ctx *,struct v4l2_pix_format_mplane *);int (*start)(struct avd_ctx *);void (*stop)(struct avd_ctx *);int (*try_ctrl)(struct avd_ctx *,struct v4l2_ctrl *);enum avd_image_fmt (*get_image_fmt)(struct avd_ctx *,struct v4l2_ctrl *);}; +struct avd_dev {struct avd_variant *variant;bool apple;}; +struct avd_ctx {struct avd_dev *dev;const struct avd_coded_fmt_desc *coded_fmt_desc;void *priv;struct {struct {struct v4l2_pix_format_mplane pix_mp;} fmt;} decoded_fmt;struct avd_comp comp;bool decomp;unsigned int max_frames;struct v4l2_ctrl ctrl_hdl[4];struct avd_job job;struct vb2_v4l2_buffer *next_src,*next_dst;}; +struct avd_coded_fmt_ops {int (*run)(struct avd_ctx *);void (*done)(struct avd_ctx *,struct vb2_v4l2_buffer *,struct vb2_v4l2_buffer *,enum vb2_buffer_state);void (*submit)(struct avd_ctx *);unsigned int (*pp_kicks)(struct avd_ctx *);void (*adjust_decoded_fmt)(struct avd_ctx *,struct v4l2_pix_format_mplane *);int (*start)(struct avd_ctx *);void (*stop)(struct avd_ctx *);int (*try_ctrl)(struct avd_ctx *,struct v4l2_ctrl *);enum avd_image_fmt (*get_image_fmt)(struct avd_ctx *,struct v4l2_ctrl *);}; static int allocations, fail_after=-1; static dma_addr_t next_dma=0x10000000; static int avd_buf_alloc(struct avd_dev *d,struct avd_buf *b,size_t n) {if(fail_after==0)return -ENOMEM;if(fail_after>0)fail_after--;b->cpu=calloc(1,n);if(!b->cpu)return -ENOMEM;b->size=n;b->addr=next_dma;next_dma+=ALIGN(n,256);allocations++;return 0;} static void avd_buf_free(struct avd_dev *d,struct avd_buf *b) {if(b->cpu){free(b->cpu);allocations--;}memset(b,0,sizeof(*b));} static struct avd_decoded_buffer buffers[64]; +static struct vb2_v4l2_buffer sources[64]; static struct avd_decoded_buffer *vb2_to_avd_decoded_buf(struct vb2_buffer *b) {return container_of(b,struct avd_decoded_buffer,base.vb.vb2_buf);} static struct avd_decoded_buffer *avd_get_ref_buf(struct avd_ctx *c,struct vb2_v4l2_buffer *dst,u64 ts) {for(int i=0;i<64;i++)if(buffers[i].base.vb.vb2_buf.timestamp==ts)return &buffers[i];return vb2_to_avd_decoded_buf(&dst->vb2_buf);} static dma_addr_t vb2_dma_contig_plane_dma_addr(struct vb2_buffer *b,int n) {return b->dma;} @@ -32,3 +33,21 @@ static struct v4l2_ctrl *v4l2_ctrl_find(void *h,u32 id) {struct v4l2_ctrl *p=h;f struct avd_av1_cdfs {u8 data[16384];}; static void avd_av1_default_coeff_probs(u32 q,void *p) {memset(p,q,sizeof(struct avd_av1_cdfs));} static void avd_av1_set_default_cdfs(void *p) {} + +static int legacy_submit_error, legacy_controls_setup, legacy_controls_complete; +static unsigned int legacy_kicks; +static u32 legacy_sent[32768]; +static size_t legacy_sent_words; +static void avd_run_preamble(struct avd_ctx *c,struct avd_run *r) { + memset(r,0,sizeof(*r));r->bufs.src=c->next_src;r->bufs.dst=c->next_dst;c->decomp=true;legacy_controls_setup++; +} +static void avd_run_postamble(struct avd_ctx *c,struct avd_run *r) {legacy_controls_complete++;} +static int avd_init_job(struct avd_ctx *c,enum avd_codec codec,unsigned int n) { + free(c->job.segments);c->job.segments=calloc(n,sizeof(*c->job.segments));c->job.num=0; + return c->job.segments?0:-ENOMEM; +} +static int avd_submit_job(struct avd_ctx *c); +static void avd_open_av1_submit(struct avd_ctx *c,unsigned int count) {legacy_kicks=count;} + +struct avd_coded_fmt_desc {const struct avd_coded_fmt_ops *ops;const struct avd_codec_frame_ops *frame_ops;}; +bool avd_backend_is_apple(const struct avd_dev *d) {return d->apple;} diff --git a/drivers/media/platform/apple/avd/tests/host-shim.h b/drivers/media/platform/apple/avd/tests/host-shim.h index 43713bb58e870..f3d0ce84f12f2 100644 --- a/drivers/media/platform/apple/avd/tests/host-shim.h +++ b/drivers/media/platform/apple/avd/tests/host-shim.h @@ -31,6 +31,7 @@ typedef int irqreturn_t; #define lower_32_bits(x) ((u32)(x)) #define upper_32_bits(x) ((u32)((u64)(x)>>32)) #define cpu_to_le32(x) (x) +#define le32_to_cpu(x) (x) #define smp_load_acquire(p) (*(p)) #define smp_store_release(p,v) (*(p)=(v)) #define READ_ONCE(x) (x) diff --git a/drivers/media/platform/apple/avd/tests/host-test.c b/drivers/media/platform/apple/avd/tests/host-test.c index a08956e81177c..eb02f4633f4b1 100644 --- a/drivers/media/platform/apple/avd/tests/host-test.c +++ b/drivers/media/platform/apple/avd/tests/host-test.c @@ -1,4 +1,16 @@ /* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +static int avd_submit_job(struct avd_ctx *c) +{ + legacy_sent_words=0; + legacy_sent[legacy_sent_words++]=AVD_OP_EXEC|AVD_OP_EXEC_FLAG_START_REV4(c->dev->variant->revision==4); + for(size_t i=0;i<=c->job.num;i++) { + for(size_t j=0;jjob.segments[i].num;j++)legacy_sent[legacy_sent_words++]=c->job.segments[i].instructions[j]; + if(i)legacy_sent[legacy_sent_words++]=AVD_OP_EXEC|AVD_OP_EXEC_FLAG_END(i==c->job.num); + } + free(c->job.segments);c->job.segments=NULL; + return legacy_submit_error; +} + static struct v4l2_ctrl_av1_sequence sequence; static struct v4l2_ctrl_av1_frame params; static struct v4l2_ctrl_av1_tile_group_entry tiles[512]; @@ -24,6 +36,7 @@ static void init_ctx(struct avd_ctx *ctx) ctx->ctrl_hdl[2]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY,.p_cur.p=tiles,.elems=V4L2_AV1_MAX_TILE_COUNT}; ctx->ctrl_hdl[3]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_FILM_GRAIN,.p_cur.p=&grain}; assert(avd_av1_start(ctx)==0); + assert(ctx->max_frames==1); } static void init_params(void) @@ -49,6 +62,7 @@ static void init_params(void) tiles[0].tile_size=128; for(int i=0;i<64;i++) { buffers[i].base.vb.vb2_buf=(struct vb2_buffer){.index=i,.timestamp=1000+i,.planes={{.length=1048576}},.dma=0x200000000ULL+i*0x200000,.payload=4096}; + sources[i].vb2_buf=(struct vb2_buffer){.index=i,.timestamp=1000+i,.planes={{.length=4096}},.dma=0x10000000ULL+i*0x200000,.payload=4096}; buffers[i].av1.width=127; buffers[i].av1.height=127; buffers[i].av1.upscaled_width=128; @@ -58,7 +72,7 @@ static void init_params(void) static struct avd_frame make_frame(unsigned int idx,unsigned int dst,struct avd_buf *prog) { return (struct avd_frame){.idx=idx,.seq=dst+1,.frame_num=dst+1, - .src=&buffers[63].base.vb,.dst=&buffers[dst].base.vb,.prog=prog}; + .src=&sources[dst],.dst=&buffers[dst].base.vb,.prog=prog}; } static void dependency_table(void) @@ -156,6 +170,7 @@ static void reference_reuse_and_segmentation(void) params.flags=0;params.segmentation.flags=0; buffers[0].base.vb.vb2_buf.timestamp=2000; f[2].dst=&buffers[0].base.vb; + f[2].src->vb2_buf.timestamp=2000; assert(!avd_av1_prepare(&ctx,&f[2])); assert(cdf->refs.n>=2&&seg->refs.n>=2); assert(b->cdf_in==cdf&&b->seg_in==seg&&cdf->done&&seg->done); @@ -212,7 +227,8 @@ static void bounds_and_fp(void) assert(avd_av1_pack_cmd(&ctx,&f)==-EOPNOTSUPP); variant.av1_fp_layout=AVD_AV1_FP_UNKNOWN; f.fp=NULL;grain.flags=V4L2_AV1_FILM_GRAIN_FLAG_APPLY_GRAIN; - assert(avd_av1_prepare(&ctx,&f)==-EOPNOTSUPP); + assert(!avd_av1_prepare(&ctx,&f)); + avd_av1_retire(&ctx,&f,VB2_BUF_STATE_DONE); grain.flags=0;prog.size=4; assert(avd_av1_prepare(&ctx,&f)==-ENOSPC); avd_av1_stop(&ctx);avd_buf_free(&device,&fp);avd_buf_free(&device,&prog); @@ -221,10 +237,11 @@ static void bounds_and_fp(void) static void program_parity(void) { - for(int test=0;test<24;test++) { + for(int test=0;test<64;test++) { struct avd_ctx ctx;struct avd_buf prog={0}; init_params();init_ctx(&ctx); if(test&1)sequence.bit_depth=10; + if(test&32)grain.flags=V4L2_AV1_FILM_GRAIN_FLAG_APPLY_GRAIN; if(test&16) { params.frame_type=V4L2_AV1_INTER_FRAME; sequence.flags|=V4L2_AV1_SEQUENCE_FLAG_ENABLE_ORDER_HINT|V4L2_AV1_SEQUENCE_FLAG_ENABLE_REF_FRAME_MVS; @@ -272,12 +289,126 @@ static void program_parity(void) avd_av1_retire(&ctx,&f,VB2_BUF_STATE_DONE);avd_av1_stop(&ctx); avd_buf_free(&device,&prog);assert(!allocations); } - puts("AV1 legacy program parity: 24 synthetic control vectors: PASS"); + puts("AV1 legacy program parity: 64 synthetic control vectors (including grain-less fallback): PASS"); +} + +static void legacy_transport(void) +{ + for(int error=0;error<2;error++) { + struct avd_ctx ctx; + init_params();init_ctx(&ctx); + params.tile_info.tile_cols=2;params.tile_info.tile_rows=2; + for(int i=0;i<2;i++) { + params.tile_info.width_in_sbs_minus_1[i]=0; + params.tile_info.height_in_sbs_minus_1[i]=0; + } + for(int i=0;i<4;i++)tiles[i]=(struct v4l2_ctrl_av1_tile_group_entry){.tile_offset=i*128,.tile_size=128,.tile_col=i%2,.tile_row=i/2}; + grain.flags=V4L2_AV1_FILM_GRAIN_FLAG_APPLY_GRAIN; + ctx.next_src=&sources[0];ctx.next_dst=&buffers[0].base.vb; + legacy_controls_setup=legacy_controls_complete=0; + legacy_submit_error=error?-EIO:0; + assert(avd_av1_fmt_ops.run&&avd_av1_fmt_ops.done&&avd_av1_fmt_ops.submit); + assert(avd_av1_fmt_ops.run(&ctx)==legacy_submit_error); + assert(legacy_controls_setup==1&&legacy_controls_complete==1); + struct avd_av1_ctx *av1=ctx.priv; + assert(legacy_sent_words==av1->legacy_frame.prog_words); + assert(!memcmp(legacy_sent,av1->legacy_prog.cpu,legacy_sent_words*sizeof(u32))); + if(!error) { + assert(av1->legacy_frame.codec_priv&&av1->active_slots==1); + assert(avd_av1_fmt_ops.pp_kicks(&ctx)==4); + avd_av1_fmt_ops.submit(&ctx);assert(legacy_kicks==4); + } + avd_av1_fmt_ops.done(&ctx,ctx.next_src,ctx.next_dst,error?VB2_BUF_STATE_ERROR:VB2_BUF_STATE_DONE); + assert(!av1->legacy_frame.codec_priv&&!av1->active_slots); + /* Serial decode keeps reference auxiliary state through completion. */ + if(!error) { + params.frame_type=V4L2_AV1_INTER_FRAME;params.primary_ref_frame=0; + params.reference_frame_ts[0]=1000;ctx.next_src=&sources[1];ctx.next_dst=&buffers[1].base.vb; + assert(!avd_av1_fmt_ops.run(&ctx)); + assert(av1->legacy_frame.dep==AVD_DEP_PREV_DONE); + avd_av1_fmt_ops.done(&ctx,ctx.next_src,ctx.next_dst,VB2_BUF_STATE_DONE); + } + avd_av1_stop(&ctx);assert(!allocations); + } + legacy_submit_error=0; + puts("AV1 legacy transport: multi-tile grain-less decode, PP kicks, CDF reuse, submit failure cleanup: PASS"); +} + +static void source_timestamp_and_failed_producer(void) +{ + for(int failed=0;failed<2;failed++) { + struct avd_ctx ctx;struct avd_frame f[2];struct avd_buf prog[2]={0}; + init_params();init_ctx(&ctx); + for(int i=0;i<2;i++){assert(!avd_buf_alloc(&device,&prog[i],4096));f[i]=make_frame(i,i,&prog[i]);} + buffers[0].base.vb.vb2_buf.timestamp=0; + assert(!avd_av1_prepare(&ctx,&f[0])); + struct avd_av1_ctx *av1=ctx.priv; + assert(av1->prev_timestamp==1000); + assert(avd_av1_find_ref(&ctx,0,1000)); + /* Model core's metadata copy after prepare, before the next request. */ + buffers[0].base.vb.vb2_buf.timestamp=1000; + params.frame_type=V4L2_AV1_INTER_FRAME;params.primary_ref_frame=0; + params.reference_frame_ts[0]=1000; + if(failed)avd_av1_retire(&ctx,&f[0],VB2_BUF_STATE_ERROR); + int rc=avd_av1_prepare(&ctx,&f[1]); + assert(rc==(failed?-EIO:0)); + if(!failed) { + assert(f[1].dep==AVD_DEP_PREV_DONE); + avd_av1_retire(&ctx,&f[0],VB2_BUF_STATE_DONE); + avd_av1_retire(&ctx,&f[1],VB2_BUF_STATE_DONE); + } + avd_av1_stop(&ctx); + for(int i=0;i<2;i++)avd_buf_free(&device,&prog[i]); + assert(!allocations); + } + puts("AV1 reference publication: source timestamp and failed producer: PASS"); +} + +static void backend_selection(void) +{ + struct avd_ctx ctx={.dev=&device}; + struct avd_coded_fmt_ops ops=avd_av1_fmt_ops; + struct avd_coded_fmt_desc desc={.ops=&ops,.frame_ops=&avd_av1_frame_ops}; + ctx.coded_fmt_desc=&desc; + for(int apple=0;apple<2;apple++)for(int legacy=0;legacy<2;legacy++) { + device.apple=apple;ops.run=legacy?avd_av1_run:NULL; + assert(avd_get_frame_ops(&ctx)==((apple||!legacy)?&avd_av1_frame_ops:NULL)); + assert(desc.frame_ops==&avd_av1_frame_ops); + } + device.apple=false; + puts("AV1 backend selection: Apple frame_ops and open legacy fallback: PASS"); +} + +static void segmentation_mi_size(void) +{ + const unsigned int widths[]={126,120,119}; + for(unsigned int w=0;w<3;w++) { + struct avd_ctx ctx;struct avd_frame f[2];struct avd_buf prog[2]={0}; + init_params();init_ctx(&ctx); + for(int i=0;i<2;i++){assert(!avd_buf_alloc(&device,&prog[i],4096));f[i]=make_frame(i,i,&prog[i]);} + params.segmentation.flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED|V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP; + assert(!avd_av1_prepare(&ctx,&f[0])); + struct avd_av1_run *a=f[0].codec_priv; + params.frame_type=V4L2_AV1_INTER_FRAME;params.primary_ref_frame=0; + params.reference_frame_ts[0]=1000; + params.flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; + params.segmentation.flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED; + params.frame_width_minus_1=widths[w];params.upscaled_width=widths[w]+1; + assert(!avd_av1_prepare(&ctx,&f[1])); + struct avd_av1_run *b=f[1].codec_priv; + assert((b->seg_in==a->seg_out)==(widths[w]>=120)); + avd_av1_retire(&ctx,&f[0],VB2_BUF_STATE_DONE); + avd_av1_retire(&ctx,&f[1],VB2_BUF_STATE_DONE); + avd_av1_stop(&ctx); + for(int i=0;i<2;i++)avd_buf_free(&device,&prog[i]); + assert(!allocations); + } + puts("AV1 segmentation: inherit only for equal MiCols/MiRows: PASS"); } int main(void) { - program_parity();dependency_table();sixteen_slots();cdf_chain();reference_reuse_and_segmentation();allocation_failures();bounds_and_fp(); + segmentation_mi_size();backend_selection();source_timestamp_and_failed_producer();legacy_transport();program_parity();dependency_table();sixteen_slots();cdf_chain();reference_reuse_and_segmentation();allocation_failures();bounds_and_fp(); puts("AV1 host tests: dependency table, 16 slots, CDF inheritance, allocation failures, bounds, Apple refusal: PASS"); return 0; } diff --git a/drivers/media/platform/apple/avd/tests/run-host-tests.py b/drivers/media/platform/apple/avd/tests/run-host-tests.py index c7c61f8050d75..616093e125c66 100755 --- a/drivers/media/platform/apple/avd/tests/run-host-tests.py +++ b/drivers/media/platform/apple/avd/tests/run-host-tests.py @@ -24,6 +24,11 @@ def without_includes(text): source += without_includes((DRIVER / 'avd-av1-fp.h').read_text()) source += (HERE / 'host-driver.h').read_text() source += without_includes((DRIVER / 'avd-queue.h').read_text()) + # Compile the production transport selector, not a duplicate test rule. + header = (DRIVER / 'avd.h').read_text() + source += re.search(r'static inline const struct avd_codec_frame_ops \*\n' + r'avd_get_frame_ops.*?\n}', header, re.S).group(0) + '\n' + inst = without_includes((DRIVER / 'avd-inst.h').read_text()) inst = inst[:inst.index('static inline void push(')] + '\n#endif\n' source += inst