diff --git a/drivers/media/platform/apple/avd/Kconfig b/drivers/media/platform/apple/avd/Kconfig index ed5ad44276c31..d3bb9763b6230 100644 --- a/drivers/media/platform/apple/avd/Kconfig +++ b/drivers/media/platform/apple/avd/Kconfig @@ -24,8 +24,7 @@ config VIDEO_APPLE_AVD_KUNIT_TEST default KUNIT_ALL_TESTS help Build KUnit tests for the Apple video decoder. They exercise the - checks that the H.264 path makes on the slices that userspace - queues, such as finding the slice data after the slice header, - at the limits and one past them. + H.264 slice bounds, frame-slot ownership, and runtime power management, + including suspend gating and resubmission after firmware resume. If unsure, say N. diff --git a/drivers/media/platform/apple/avd/Makefile b/drivers/media/platform/apple/avd/Makefile index 4d1877c414d41..f603787990fe8 100644 --- a/drivers/media/platform/apple/avd/Makefile +++ b/drivers/media/platform/apple/avd/Makefile @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only -apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-h264.o avd-vp9.o avd-hevc.o \ - avd-av1.o avd-av1-entropymode.o -apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o +apple-avd-y := avd-drv.o avd-v4l2.o avd-hw.o avd-pm.o avd-h264.o avd-vp9.o \ + avd-hevc.o avd-av1.o avd-av1-fp.o avd-av1-entropymode.o avd-queue.o avd-backend-open.o \ + avd-backend-apple.o avd-backend-queue.o +apple-avd-$(CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST) += avd-h264-test.o avd-queue-test.o \ + avd-pm-test.o obj-$(CONFIG_VIDEO_APPLE_AVD) += apple-avd.o diff --git a/drivers/media/platform/apple/avd/avd-apple-protocol.h b/drivers/media/platform/apple/avd/avd-apple-protocol.h new file mode 100644 index 0000000000000..d0b90c5d48e4c --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-apple-protocol.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_APPLE_PROTOCOL_H_ +#define AVD_APPLE_PROTOCOL_H_ + +#include +#include + +#include "avd-queue.h" + +#define AVD_APPLE_INBOX_BASE 0x01482cf8 +#define AVD_APPLE_COMMAND_SIZE 0x60 +#define AVD_APPLE_RESULT_WORDS 20 +#define AVD_APPLE_SHADOW_SIZE 64 + +/* + * [C] Expected address notification, from the MIT V3 trace_avd.py precedent. + * Keep this rule in one place for correction by the Borage capture. + */ +#define AVD_APPLE_NOTIFY_BASE AVD_APPLE_INBOX_BASE + +/* SPEC 5.1: only these session opcodes have proved builders. */ +static inline int +avd_apple_session_command(u32 cmd[24], u32 client, u32 opcode, + u32 property, u32 value) +{ + unsigned int i; + + if (client >= 1024 || (opcode != 0 && opcode != 2 && opcode != 5 && opcode != 6)) + return -EINVAL; + for (i = 0; i < 24; i++) + cmd[i] = 0; + cmd[0] = opcode | (client << 16); + if (opcode == 6) { + cmd[1] = property; + cmd[2] = value; + } + return 0; +} + +static inline int avd_apple_decode_prefix(struct avd_frame *f, u32 client, + u32 selector, u32 work_sequence) +{ + unsigned int i; + + if (f->idx >= AVD_MAX_FRAMES || client >= 1024 || selector >= 16 || work_sequence >= 4) + return -EINVAL; + for (i = 0; i < 24; i++) + f->cmd[i] = 0; + f->cmd[0] = 1 | (selector << 10) | (client << 22); + f->cmd[1] = f->idx | (work_sequence << 16); + f->cmd[3] = f->frame_num; + return 0; +} + +struct avd_apple_reply_layout { + u8 type_word; + u8 type_shift; + u32 type_mask; + u8 client_word; + u8 client_shift; + u32 client_mask; + u8 frame_word; + u8 status_word; + u32 done_type; + bool verified; +}; + +struct avd_apple_reply { + u32 type; + u32 client; + u32 frame; + int status; +}; + +static inline int +avd_apple_parse_reply(const u32 *record, size_t words, + const struct avd_apple_reply_layout *layout, + struct avd_apple_reply *reply) +{ + if (words != AVD_APPLE_RESULT_WORDS) + return -EMSGSIZE; + if (!layout->verified) + return -EOPNOTSUPP; + if (layout->type_word >= words || layout->client_word >= words || + layout->frame_word >= words || layout->status_word >= words || + layout->type_shift >= 32 || layout->client_shift >= 32 || + !layout->type_mask || !layout->client_mask || layout->done_type == 7) + return -EINVAL; + reply->type = (record[layout->type_word] >> layout->type_shift) & layout->type_mask; + reply->client = (record[layout->client_word] >> layout->client_shift) & layout->client_mask; + reply->frame = record[layout->frame_word]; + /* SPEC 4.3: type 7 is parsing/VP done, never final buffer ownership. */ + if (reply->type == 7) + return -EINPROGRESS; + if (reply->type != layout->done_type) + return -ENOMSG; + reply->status = record[layout->status_word] ? -EIO : 0; + return 0; +} + +static inline u32 avd_apple_notification(unsigned int slot) +{ + return AVD_APPLE_NOTIFY_BASE + AVD_APPLE_COMMAND_SIZE * slot; +} + +struct avd_apple_hw { + const char *fw_name; + /* All offsets below are relative to a DT-owned "apple" resource. */ + u32 aperture_origin; + u32 imem; + u32 imem_size; + u32 version_offset; + u32 itcm_start; + u32 itcm_end; + u32 itcm_clear; + u32 itcm_clear_value; + u32 itcm_enable; + u32 itcm_enable_value; + u32 link_setup; + u32 link_setup_value; + u32 link_state; + u32 link_up; + u32 link_up_value; + u32 ready; + u32 notify; + u32 receive; + u32 receive_status; + u32 receive_pending; + u32 receive_ack; + u32 receive_ack_value; + u32 inbox_consumer; + u32 result_base; + struct avd_apple_reply_layout reply; + /* Match-data fact, never a module parameter or a userspace setting. */ + bool verified; +}; + +extern const struct avd_apple_hw avd_borage_hw; +extern const u8 avd_borage_config[64]; + +#endif /* AVD_APPLE_PROTOCOL_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-av1-fp.c b/drivers/media/platform/apple/avd/avd-av1-fp.c new file mode 100644 index 0000000000000..7861b53358a1b --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-av1-fp.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * AV1 Apple frame parameters. Source: Apple AVD protocol SPEC (2026-10-31), + * sections 5, 6.1, 6.5, 6.6, 10. No AV1 layout exists in the public V3 code. + * Keep unproved fields unnamed and refuse submission until they are settled. + */ +#include +#include "avd.h" +#include "avd-av1-fp.h" + +#define BORAGE_FP_SIZE 0x1130d0 +#define BORAGE_WORK_CAP 0x1002 + +/* + * [C] table: source destination / unresolved rule + * SPEC 6.1 fp+0x18,+0x19 cmd+0x00 bits 0,1; meaning + * SPEC 6.5 fp+0x4f4 cmd+0x00 bits 2:6,18:19; + * selected source bits unspecified + * SPEC 6.5 fp+0x4f4 cmd+0x00 bits 7:11,13:17; + * source shifts unspecified + * SPEC 6.5 work array at fp+0x1c cmd+0x1c; array encoding + * SPEC 6.5 fp+0x4fc,+0x5e0,0x01481780 cmd+0x3c,+0x44; packed addresses + * SPEC 6.5 fp+0x4f8 cmd+0x4c; source meaning + * SPEC 6.6 sequence/frame/tile/grain syntax/DVA offsets still open + * SPEC 10 T8140 TODO: identify selected packer; + * Daybreak per-SoC ledger task + * t-3862-daybreak-avd-per-soc-ledger- + * t8140-t6000-t8112-pe + */ + +int avd_av1_validate_tiles(const struct v4l2_ctrl_av1_frame *frame, + const struct v4l2_ctrl_av1_tile_group_entry *tiles, + unsigned int count, size_t payload) +{ + const struct v4l2_av1_tile_info *info = &frame->tile_info; + unsigned int i, n; + + if (!info->tile_cols || !info->tile_rows || + info->tile_cols > V4L2_AV1_MAX_TILE_COLS || + info->tile_rows > V4L2_AV1_MAX_TILE_ROWS) + return -EINVAL; + n = info->tile_cols * info->tile_rows; + if (n > V4L2_AV1_MAX_TILE_COUNT || n > BORAGE_WORK_CAP || + count < n || count > V4L2_AV1_MAX_TILE_COUNT || + info->context_update_tile_id >= n) + return -EINVAL; + for (i = 0; i < n; i++) { + if (tiles[i].tile_row != i / info->tile_cols || + tiles[i].tile_col != i % info->tile_cols || + !tiles[i].tile_size || tiles[i].tile_offset > payload || + tiles[i].tile_size > payload - tiles[i].tile_offset) + return -EINVAL; + if (i && tiles[i].tile_offset < + (u64)tiles[i - 1].tile_offset + tiles[i - 1].tile_size) + return -EINVAL; + } + return 0; +} + +int avd_av1_pack_fp(struct avd_ctx *ctx, struct avd_frame *f, + const struct v4l2_ctrl_av1_sequence *seq, + const struct v4l2_ctrl_av1_frame *frame, + const struct v4l2_ctrl_av1_tile_group_entry *tiles, + const struct v4l2_ctrl_av1_film_grain *grain) +{ + u8 *fp; + + if (ctx->dev->variant->av1_fp_layout != AVD_AV1_FP_BORAGE) + return -EOPNOTSUPP; + if (!f->fp || !f->fp->cpu || f->fp->size < BORAGE_FP_SIZE) + return -ENOSPC; + fp = f->fp->cpu; + memset(fp, 0, f->fp->size); + put_unaligned_le32(f->frame_num, fp + 0x10); /* SPEC 5.2 */ + put_unaligned_le16(frame->tile_info.tile_cols, fp + 0x3e4); + put_unaligned_le16(frame->tile_info.tile_rows, fp + 0x3e6); + /* SPEC 6.6 does not locate seq/frame/tiles/grain syntax or surface DVAs. + * A native V4L2 struct or Linux instruction stream is not an Apple fp. */ + return -EOPNOTSUPP; +} + +int avd_av1_pack_cmd(struct avd_ctx *ctx, struct avd_frame *f) +{ + u8 *cmd = (u8 *)f->cmd; + const u8 *fp; + u32 work, base; + + if (ctx->dev->variant->av1_fp_layout != AVD_AV1_FP_BORAGE) + return -EOPNOTSUPP; + if (!f->fp || !f->fp->cpu || f->fp->size < BORAGE_FP_SIZE) + return -ENOSPC; + fp = f->fp->cpu; + /* Work index is the modulo-four sequence from the core prefix. */ + work = get_unaligned_le16(cmd + 6); + if (work >= 4 || !get_unaligned_le16(fp + 0x1a) || + get_unaligned_le16(fp + 0x1a) > BORAGE_WORK_CAP) + return -EINVAL; + base = 0x01483118 + 0x1814 * work; + put_unaligned_le16(get_unaligned_le16(fp + 0x1a), cmd + 0x04); + put_unaligned_le32(base, cmd + 0x0c); + put_unaligned_le32(base + 8, cmd + 0x14); + put_unaligned_le32(0x014832c0 + 0x1814 * work, cmd + 0x24); + put_unaligned_le32(0x014834dc + 0x1814 * work, cmd + 0x28); + put_unaligned_le32(base + 0x4d0, cmd + 0x2c); + /* Slot-relative offsets; their packed DVA encoding is still [C]. */ + put_unaligned_le32(0x5f8, cmd + 0x30); + put_unaligned_le32(0x4fc, cmd + 0x34); + put_unaligned_le32(0x5e0, cmd + 0x38); + put_unaligned_le32(get_unaligned_le32(fp + 0x4f8), cmd + 0x4c); + return -EOPNOTSUPP; +} diff --git a/drivers/media/platform/apple/avd/avd-av1-fp.h b/drivers/media/platform/apple/avd/avd-av1-fp.h new file mode 100644 index 0000000000000..8e292f6402dac --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-av1-fp.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_AV1_FP_H_ +#define AVD_AV1_FP_H_ + +#include + +struct avd_ctx; +struct avd_frame; +struct v4l2_ctrl_av1_sequence; +struct v4l2_ctrl_av1_frame; +struct v4l2_ctrl_av1_tile_group_entry; +struct v4l2_ctrl_av1_film_grain; + +enum avd_av1_fp_layout { + AVD_AV1_FP_UNKNOWN, + AVD_AV1_FP_BORAGE, + AVD_AV1_FP_T8140_TODO, +}; + +/* Incoming edge: the primary reference selects the previous HW CDF output. */ +static inline bool avd_av1_cdf_dep(bool uses_primary, u64 primary_timestamp, + bool have_prev, u64 prev_timestamp, + bool prev_updates_cdf) +{ + return uses_primary && have_prev && prev_updates_cdf && + primary_timestamp == prev_timestamp; +} + +int avd_av1_validate_tiles(const struct v4l2_ctrl_av1_frame *frame, + const struct v4l2_ctrl_av1_tile_group_entry *tiles, + unsigned int count, size_t payload); +int avd_av1_pack_fp(struct avd_ctx *ctx, struct avd_frame *f, + const struct v4l2_ctrl_av1_sequence *seq, + const struct v4l2_ctrl_av1_frame *frame, + const struct v4l2_ctrl_av1_tile_group_entry *tiles, + const struct v4l2_ctrl_av1_film_grain *grain); +int avd_av1_pack_cmd(struct avd_ctx *ctx, struct avd_frame *f); + +#endif diff --git a/drivers/media/platform/apple/avd/avd-av1-frame-ops.rst b/drivers/media/platform/apple/avd/avd-av1-frame-ops.rst new file mode 100644 index 0000000000000..bb96d18dcc083 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-av1-frame-ops.rst @@ -0,0 +1,82 @@ +AV1 frame operations +==================== + +AV1 implements avd_codec_frame_ops and retains the legacy run/done/submit and +pp_kicks callbacks. The current open transports still require legacy segments +and codec-specific PP kicks. avd_get_frame_ops selects that compatibility path +when a legacy run callback exists, and selects frame_ops on Apple. Retirement +uses the operations selected at preparation, including across a backend reset. +The queue header is core's unmodified shared contract. + +The compatibility adapter applies the request controls, invokes the same CPU +prepare implementation, and copies the recorded header/tile ranges into the +legacy job. It never finds boundaries by treating arbitrary instruction data +as opcodes. Scratch and auxiliary DMA allocations stay alive until done calls +retire. AV1 uses one legacy picture per context until the open transports accept +native frame_ops; the queued transport gets the exact number of PP kicks. +Open decode retains aurora-wip's grain-less fallback for APPLY_GRAIN requests. +Film-grain synthesis is not implemented by the existing instruction builder. + +prepare consumes applied controls and the slot's src/dst. It never accesses an +m2m queue, completes a request, or touches MMIO. The little-endian open program +contains header EXEC, the existing header words, and one tile payload plus EXEC +per tile (END on the last tile). prog_words includes EXEC records. Only the +compatibility adapter accesses the legacy job; PP register writes live in the +transport. Core supplies fp only for an Apple submission. + +Each live frame owns its pipe state and all above/left/MV scratch allocations. +CDFs and segmentation maps have reference-counted coherent DMA allocations. +Capture-index records retain output state; live frames retain their inputs and +outputs independently. Disabled updates inherit an immutable allocation rather +than copying unfinished hardware output. Reusing the original capture index +cannot free inherited state. Color/MV data in capture buffers follows stateless +reference-buffer lifetime rules. Segmentation inherits a map only when MiCols +and MiRows match, retaining the existing builder's size comparison. Reference +publication uses OUTPUT's timestamp: +core copies that metadata to CAPTURE only after prepare has returned. + +The incoming dependency is PREV_DONE exactly when the current frame selects the +previous frame as primary reference and that previous frame enables frame-end +CDF updates. Intra/error-resilient/PRIMARY_REF_NONE defaults, another primary +reference, and disabled frame-end updates produce NONE. prepare does not read +hardware CDF/map output on the CPU. An unfinished older producer which that +rule cannot express returns EAGAIN. Native integration still needs preparation +retry and propagation of failed producers to already prepared dependants. +Legacy decode is serial and retires its producer before the next preparation. + +Apple layout evidence and remaining work +---------------------------------------- + +The source is the clean-room Apple AVD protocol SPEC dated 2026-10-31, sections +5, 6.1, 6.5, 6.6 and 10. Public V3 layouts in eiln/avd cannot supply AV1 fields: +those generations have no AV1 hardware. avd-av1-fp.c keeps an unnamed [C] table +with sources for the missing fields. It stages proved Borage tile columns/rows +at 0x3e4/0x3e6, command destinations, work-count cap 0x1002 and the +fp+0x4f8 -> cmd+0x4c copy. Both Apple pack operations return EOPNOTSUPP: +the SPEC omits syntax/DVA mappings, source flag shifts and packed-address +encoding. These facts do not yet describe a usable Apple AV1 payload. + +T6040 selects Borage. T8140 has a distinct TODO row tied to Daybreak's per-SoC +ledger task t-3862. Other Apple layouts stay unsupported. Hardware capability +masks still govern format exposure; T6000/T8112 have no AV1 capability or VP. + +Verification +------------ + +Run ``python3 drivers/media/platform/apple/avd/tests/run-host-tests.py`` from a +Git checkout (sparse checkouts work). A host C compiler with ASan and UBSan is +required. Tests compile production CPU code and the production backend selector +with DMA/V4L2 doubles and the checkout's AV1 UAPI definitions. The licensed +packing oracle is frozen from aurora-wip +fbc4e7934ca7325f3483b72a51d24c74aa8536a4. Program comparison resolves the same +DMA addresses for both builders; changed ownership intentionally changes them. + +Coverage includes 64 parity vectors (tiles, CDEF, restoration, superres, 8/10-bit, +inter frames and grain-less fallback), legacy multi-tile submission/PP counts, +submission failure cleanup, source-timestamp publication, producer errors, the +dependency table, sixteen live slots, inherited state and capture reuse, +allocation failure cleanup, payload/program bounds and Apple-layout refusal. +These are synthetic controls, not SVT-AV1 bitstreams or pixel validation. +Show-existing is a userspace reference redisplay without a new stateless decode +control. Native open execution, the missing Apple payload facts, retry/failure +integration and end-to-end stream qualification remain outstanding. diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index 60d69bab8a342..6e64b55747180 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -16,11 +16,20 @@ * Tomasz Figa */ +#include +#include #include #include "avd.h" #include "avd-av1-entropymode.h" #include "avd-inst.h" +#include "avd-av1-fp.h" + +/* The AV1 builder writes only the current frame's DMA program. */ +#undef push +#undef pusha +#define push(inst, name) avd_av1_push(run, (inst)) +#define pusha(addr, name, i) avd_av1_push_address(ctx, run, (addr)) /* * This comes from @@ -208,44 +217,165 @@ #define AV1_PRIMARY_REF_NONE 7 +/* A capture reference and every in-flight reader hold their own references. */ +struct avd_av1_aux { + refcount_t refs; + struct avd_buf buf; + u64 producer; + bool done; + int status; +}; + +struct avd_av1_ref { + struct list_head node; + unsigned int index; + u64 timestamp; + u16 width, height; + struct avd_av1_aux *cdf; + struct avd_av1_aux *seg; +}; + +struct avd_av1_scratch { + struct avd_buf pipe_state; + struct avd_buf above_info; + struct avd_buf rf_above_info; + struct avd_buf ip_above; + struct avd_buf lf_above; + struct avd_buf lf_above_info; + struct avd_buf lf_left; + struct avd_buf lf_left_info; + struct avd_buf sr_left; + struct avd_buf rf_left; + struct avd_buf rf_left_info; + struct avd_buf mv_above_info; +}; + struct avd_av1_run { struct avd_run base; + struct avd_frame *slot; + struct avd_av1_scratch bufs; + struct avd_av1_aux *cdf_in, *cdf_out; + struct avd_av1_aux *seg_in, *seg_out; + bool cdf_written, seg_written; + int error; struct { dma_addr_t probs_out; + dma_addr_t probs_in; dma_addr_t color; dma_addr_t seg; dma_addr_t prev_seg; } addresses; + /* Used only while prepare() owns the request controls. */ const struct v4l2_ctrl_av1_sequence *seq; const struct v4l2_ctrl_av1_frame *frame; const struct v4l2_ctrl_av1_tile_group_entry *tile_group; const struct v4l2_ctrl_av1_film_grain *grain; + unsigned int num_tiles; + /* Ends exclude EXEC words; used only by the legacy transport adapter. */ + size_t segment_end[V4L2_AV1_MAX_TILE_COUNT + 1]; }; struct avd_av1_ctx { - u8 submit_num; - struct { - struct avd_buf inst; - struct avd_buf pipe_state; - struct avd_buf probs; - struct avd_buf no_prev_seg; - struct avd_buf above_info; - struct avd_buf rf_above_info; - struct avd_buf az_above; - struct avd_buf ip_above; - struct avd_buf lf_above; - struct avd_buf lf_above_info; - struct avd_buf lf_left; - struct avd_buf lf_left_info; - struct avd_buf sr_left; - struct avd_buf rf_left; - struct avd_buf rf_left_info; - struct avd_buf mv_above_info; - } bufs; + struct list_head refs; + u64 prev_seq; + u64 prev_timestamp; + bool prev_updates_cdf; + bool have_prev; + unsigned long active_slots; + struct avd_frame legacy_frame; + struct avd_buf legacy_prog; }; +static void avd_av1_push(struct avd_av1_run *run, u32 inst) +{ + struct avd_frame *f = run->slot; + + if (f->prog_words >= f->prog->size / sizeof(__le32)) { + run->error = -ENOSPC; + return; + } + ((__le32 *)f->prog->cpu)[f->prog_words++] = cpu_to_le32(inst); +} + +static void avd_av1_push_address(struct avd_ctx *ctx, struct avd_av1_run *run, + dma_addr_t addr) +{ + if (ctx->dev->variant->quirks & AVD_QUIRK_LSR) { + avd_av1_push(run, addr >> 8); + } else { + avd_av1_push(run, lower_32_bits(addr)); + avd_av1_push(run, upper_32_bits(addr)); + } +} + +static void avd_av1_push_comp(struct avd_ctx *ctx, struct avd_av1_run *run, + dma_addr_t addr, const u32 offsets[4]) +{ + unsigned int i; + + for (i = 0; i < 4; i++) { + if (ctx->dev->variant->quirks & AVD_QUIRK_LSR) + avd_av1_push(run, (addr + offsets[i]) >> 7); + else + avd_av1_push_address(ctx, run, addr + offsets[i]); + } +} + +static struct avd_av1_aux *avd_av1_aux_get(struct avd_av1_aux *aux) +{ + if (aux) + refcount_inc(&aux->refs); + return aux; +} + +static void avd_av1_aux_put(struct avd_ctx *ctx, struct avd_av1_aux *aux) +{ + if (aux && refcount_dec_and_test(&aux->refs)) { + avd_buf_free(ctx->dev, &aux->buf); + kfree(aux); + } +} + +static struct avd_av1_aux *avd_av1_aux_alloc(struct avd_ctx *ctx, size_t size, + u64 producer, bool done) +{ + struct avd_av1_aux *aux; + + aux = kzalloc(sizeof(*aux), GFP_KERNEL); + if (!aux) + return NULL; + if (avd_buf_alloc(ctx->dev, &aux->buf, size)) { + kfree(aux); + return NULL; + } + refcount_set(&aux->refs, 1); + aux->producer = producer; + aux->done = done; + memset(aux->buf.cpu, 0, size); + return aux; +} + +static int avd_av1_aux_ready(struct avd_av1_aux *aux, enum avd_frame_dep dep) +{ + if (smp_load_acquire(&aux->done)) + return READ_ONCE(aux->status); + return dep == AVD_DEP_PREV_DONE ? 0 : -EAGAIN; +} + +static struct avd_av1_ref *avd_av1_find_ref(struct avd_ctx *ctx, + unsigned int index, u64 timestamp) +{ + struct avd_av1_ctx *av1 = ctx->priv; + struct avd_av1_ref *ref; + + list_for_each_entry(ref, &av1->refs, node) + if (ref->index == index && ref->timestamp == timestamp) + return ref; + return NULL; +} + /* * from verisilicon/rockchip_vpu981_hw_av1_dec.c */ @@ -304,7 +434,7 @@ static short avd_av1_dec_resolve_divisor_32(u32 d, short *shift) } static void -avd_av1_dec_get_shear_params(const u32 *params, s64 *alpha, +avd_av1_dec_get_shear_params(const s32 *params, s64 *alpha, s64 *beta, s64 *gamma, s64 *delta) { const int *mat = params; @@ -341,7 +471,6 @@ avd_av1_dec_get_shear_params(const u32 *params, s64 *alpha, static void set_refs(struct avd_ctx *ctx, struct avd_av1_run *run) { - struct avd_av1_ctx *av1_ctx = ctx->priv; const struct v4l2_ctrl_av1_frame *frame = run->frame; const struct v4l2_av1_global_motion *gm = &frame->global_motion; int i, ref_idx; @@ -355,7 +484,7 @@ static void set_refs(struct avd_ctx *ctx, struct avd_av1_run *run) dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); push(0, "ref_cnst0"); - pusha(av1_ctx->bufs.mv_above_info.addr, "mv_above_info", 0); + pusha(run->bufs.mv_above_info.addr, "mv_above_info", 0); for (i = 0; i < 4; i++) push(0, "ref_cnst1"); @@ -427,7 +556,7 @@ static void set_refs(struct avd_ctx *ctx, struct avd_av1_run *run) push(AV1_REF_SCALE_X(x_scale) | AV1_REF_SCALE_Y(y_scale), "ref_scale"); - push_comp(ctx, addr, ref->comp.offsets); + avd_av1_push_comp(ctx, run, addr, ref->comp.offsets); } } @@ -634,7 +763,6 @@ static void calc_upscale(int frame_width, int upscaled_width, int sub_x, static void set_header(struct avd_ctx *ctx, struct avd_av1_run *run) { - struct avd_av1_ctx *av1_ctx = ctx->priv; const struct v4l2_ctrl_av1_frame *frame = run->frame; const struct v4l2_ctrl_av1_sequence *seq = run->seq; const struct v4l2_av1_loop_restoration *lr = &frame->loop_restoration; @@ -657,7 +785,7 @@ static void set_header(struct avd_ctx *ctx, struct avd_av1_run *run) dma_addr_t ref_addr; dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); - push(AVD_OP_HDR | AVD_OP_HDR_FLAG_DECOMP(ctx->decomp) | + push(AVD_OP_HDR | AVD_OP_HDR_FLAG_DECOMP(1) | AVD_OP_HDR_FLAG_INTRA(intra_only && !intrabc) | AVD_OP_HDR_CONST | AVD_OP_HDR_FLAG_PIPE_STATE(1), "op_hdr"); @@ -817,8 +945,8 @@ static void set_header(struct avd_ctx *ctx, struct avd_av1_run *run) push(0, ""); pusha(run->addresses.probs_out, "probs_out", 0); - pusha(av1_ctx->bufs.probs.addr, "probs", 1); - pusha(av1_ctx->bufs.above_info.addr, "col", 0); + pusha(run->addresses.probs_in, "probs", 1); + pusha(run->bufs.above_info.addr, "col", 0); pusha(run->addresses.seg, "seg", 0); pusha(run->addresses.prev_seg, "prev_seg", 1); pusha(run->addresses.color, "w_color", 0); @@ -954,22 +1082,22 @@ static void set_header(struct avd_ctx *ctx, struct avd_av1_run *run) push(0, ""); push(0, ""); - pusha(av1_ctx->bufs.pipe_state.addr, "pipe_state", 0); - pusha(av1_ctx->bufs.ip_above.addr, "ip_above", 0); - pusha(av1_ctx->bufs.lf_above.addr, "lf_above", 2); - pusha(av1_ctx->bufs.lf_above_info.addr, "lf_above_info", 3); - pusha(av1_ctx->bufs.lf_left.addr, "lf_left", 0); - pusha(av1_ctx->bufs.lf_left_info.addr, "lf_left_info", 1); + pusha(run->bufs.pipe_state.addr, "pipe_state", 0); + pusha(run->bufs.ip_above.addr, "ip_above", 0); + pusha(run->bufs.lf_above.addr, "lf_above", 2); + pusha(run->bufs.lf_above_info.addr, "lf_above_info", 3); + pusha(run->bufs.lf_left.addr, "lf_left", 0); + pusha(run->bufs.lf_left_info.addr, "lf_left_info", 1); /* make it obvious if its used */ pusha((dma_addr_t)0xa0a0a0a0a0a0, "", 0); - pusha(av1_ctx->bufs.rf_above_info.addr, "rf_above_info", 0); - pusha(av1_ctx->bufs.sr_left.addr, "sr_left", 2); - pusha(av1_ctx->bufs.rf_left_info.addr, "rf_left_info", 1); - pusha(av1_ctx->bufs.rf_left.addr, "rf_left", 3); + pusha(run->bufs.rf_above_info.addr, "rf_above_info", 0); + pusha(run->bufs.sr_left.addr, "sr_left", 2); + pusha(run->bufs.rf_left_info.addr, "rf_left_info", 1); + pusha(run->bufs.rf_left.addr, "rf_left", 3); push(0, "mark_section"); - push_comp(ctx, run->base.comp_out, ctx->comp.offsets); + avd_av1_push_comp(ctx, run, run->base.comp_out, ctx->comp.offsets); pusha((u64)0, "packed_fmt_scratch", 0); @@ -991,7 +1119,6 @@ static void set_header(struct avd_ctx *ctx, struct avd_av1_run *run) static void set_tiles(struct avd_ctx *ctx, struct avd_av1_run *run) { - struct avd_av1_ctx *av1_ctx = ctx->priv; const struct v4l2_ctrl_av1_frame *frame = run->frame; const struct v4l2_ctrl_av1_sequence *seq = run->seq; const struct v4l2_ctrl_av1_tile_group_entry *tile_group; @@ -1001,11 +1128,9 @@ static void set_tiles(struct avd_ctx *ctx, struct avd_av1_run *run) seq->flags & V4L2_AV1_SEQUENCE_FLAG_USE_128X128_SUPERBLOCK ? 5 : 4; - av1_ctx->submit_num = tile_info->tile_cols * tile_info->tile_rows; for (row = 0; row < tile_info->tile_rows; row++) { for (col = 0; col < tile_info->tile_cols; col++) { - ctx->job.num++; tile_id = row * tile_info->tile_cols + col; tile_group = &run->tile_group[tile_id]; @@ -1049,105 +1174,100 @@ static void set_tiles(struct avd_ctx *ctx, struct avd_av1_run *run) tile_info->width_in_sbs_minus_1 [tile_group->tile_col]), "tile_op_end"); -#ifdef DEBUG_INST - pr_info("\n"); -#endif + run->segment_end[tile_id + 1] = run->slot->prog_words; + push(AVD_OP_EXEC | AVD_OP_EXEC_FLAG_END( + tile_id + 1 == run->num_tiles), "tile_exec"); } } } -static void avd_av1_set_prob(struct avd_ctx *ctx, struct avd_av1_run *run) -{ - struct avd_av1_ctx *av1_ctx = ctx->priv; - bool error_resilient_mode = !!( - run->frame->flags & V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE); - bool frame_is_intra = - ((run->frame->frame_type == V4L2_AV1_KEY_FRAME) || - (run->frame->frame_type == V4L2_AV1_INTRA_ONLY_FRAME)); - struct avd_decoded_buffer *dst, *ref; - dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); - - if (error_resilient_mode || frame_is_intra || - run->frame->primary_ref_frame == AV1_PRIMARY_REF_NONE) { - avd_av1_default_coeff_probs(run->frame->quantization.base_q_idx, - av1_ctx->bufs.probs.cpu); - avd_av1_set_default_cdfs(av1_ctx->bufs.probs.cpu); - } else { - ref = avd_get_ref_buf( - ctx, &dst->base.vb, - run->frame->reference_frame_ts - [run->frame->ref_frame_idx - [run->frame->primary_ref_frame]]); - memcpy(av1_ctx->bufs.probs.cpu, - vb2_plane_vaddr(&ref->base.vb.vb2_buf, 0) + - AVD_AV1_CDFS_OFFSET( - ref->base.vb.vb2_buf.planes[0].length, - ref->av1.color_size), - sizeof(struct avd_av1_cdfs)); - } - - /* - * AVD writes nothing when DISABLE_FRAME_END_UPDATE_CDF is - * enabled - */ - memcpy(vb2_plane_vaddr(&dst->base.vb.vb2_buf, 0) + - AVD_AV1_CDFS_OFFSET(dst->base.vb.vb2_buf.planes[0].length, - dst->av1.color_size), - av1_ctx->bufs.probs.cpu, sizeof(struct avd_av1_cdfs)); -} - -/* - * The segment ids the frame reads as the previous ones (load_previous_segment_ids()): - * the primary reference frame's map if that frame has the same size, else zeroes. - * The AVD reads them whenever segmentation is on, and writes the frame's map only - * when update_map is set; otherwise fill in the map the spec saves with the frame. - */ -static void avd_av1_set_seg(struct avd_ctx *ctx, struct avd_av1_run *run) +/* No CPU reads from hardware-owned capture tails during prepare(). */ +static int avd_av1_set_aux(struct avd_ctx *ctx, struct avd_av1_run *run) { - struct avd_av1_ctx *av1_ctx = ctx->priv; + struct avd_av1_ctx *av1 = ctx->priv; const struct v4l2_ctrl_av1_frame *frame = run->frame; - u8 flags = frame->segmentation.flags; - struct avd_decoded_buffer *dst, *ref = NULL; - void *seg, *prev_seg = NULL; + struct avd_av1_ref *ref = NULL; + struct avd_decoded_buffer *dst, *primary; + bool defaults = frame->primary_ref_frame == AV1_PRIMARY_REF_NONE || + frame->frame_type == V4L2_AV1_KEY_FRAME || + frame->frame_type == V4L2_AV1_INTRA_ONLY_FRAME || + (frame->flags & V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE); + bool seg_enabled = frame->segmentation.flags & + V4L2_AV1_SEGMENTATION_FLAG_ENABLED; + u64 timestamp = 0; + int ret; dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); - seg = vb2_plane_vaddr(&dst->base.vb.vb2_buf, 0) + - AVD_AV1_SEG_OFFSET(dst->base.vb.vb2_buf.planes[0].length, - dst->av1.color_size, dst->av1.seg_size); - - if (frame->primary_ref_frame != AV1_PRIMARY_REF_NONE) { - ref = avd_get_ref_buf( - ctx, &dst->base.vb, - frame->reference_frame_ts - [frame->ref_frame_idx[frame->primary_ref_frame]]); - /* avd_get_ref_buf() returns dst for a missing reference */ - /* same MiCols and MiRows, as libdav1d compares them */ - if (ref == dst || - DIV_ROUND_UP(ref->av1.width + 1, 8) != - DIV_ROUND_UP(frame->frame_width_minus_1 + 1, 8) || - DIV_ROUND_UP(ref->av1.height + 1, 8) != - DIV_ROUND_UP(frame->frame_height_minus_1 + 1, 8)) - ref = NULL; + if (!defaults) { + timestamp = frame->reference_frame_ts[ + frame->ref_frame_idx[frame->primary_ref_frame]]; + primary = avd_get_ref_buf(ctx, &dst->base.vb, timestamp); + if (primary == dst) + return -EINVAL; + ref = avd_av1_find_ref(ctx, primary->base.vb.vb2_buf.index, + timestamp); + if (!ref) + return -EINVAL; + run->cdf_in = avd_av1_aux_get(ref->cdf); + } else { + run->cdf_in = avd_av1_aux_alloc(ctx, AVD_CDFS_SIZE, 0, true); + if (!run->cdf_in) + return -ENOMEM; + avd_av1_default_coeff_probs(frame->quantization.base_q_idx, + run->cdf_in->buf.cpu); + avd_av1_set_default_cdfs(run->cdf_in->buf.cpu); } - if (ref) { - size_t off = AVD_AV1_SEG_OFFSET( - ref->base.vb.vb2_buf.planes[0].length, - ref->av1.color_size, ref->av1.seg_size); + run->slot->dep = avd_av1_cdf_dep(!defaults, timestamp, + av1->have_prev, av1->prev_timestamp, av1->prev_updates_cdf) ? + AVD_DEP_PREV_DONE : AVD_DEP_NONE; + /* PREV_DONE drains earlier frames too, but do not invent a dependency + * for a different, unfinished CDF producer. The queue must retry it. */ + ret = avd_av1_aux_ready(run->cdf_in, run->slot->dep); + if (ret) + return ret; - run->addresses.prev_seg = - vb2_dma_contig_plane_dma_addr(&ref->base.vb.vb2_buf, 0) + - off; - prev_seg = vb2_plane_vaddr(&ref->base.vb.vb2_buf, 0) + off; + run->cdf_written = !(frame->flags & + V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF); + if (run->cdf_written) + run->cdf_out = avd_av1_aux_alloc(ctx, AVD_CDFS_SIZE, + run->slot->seq, false); + else + run->cdf_out = avd_av1_aux_get(run->cdf_in); + if (!run->cdf_out) + return -ENOMEM; + + if (ref && seg_enabled && + DIV_ROUND_UP(ref->width + 1, 8) == + DIV_ROUND_UP(frame->frame_width_minus_1 + 1, 8) && + DIV_ROUND_UP(ref->height + 1, 8) == + DIV_ROUND_UP(frame->frame_height_minus_1 + 1, 8)) { + run->seg_in = avd_av1_aux_get(ref->seg); + /* The two segmentation addresses let hardware read the previous + * map directly. No speculative CPU copy of an unfinished map. */ + ret = avd_av1_aux_ready(run->seg_in, run->slot->dep); + if (ret) + return ret; } else { - run->addresses.prev_seg = av1_ctx->bufs.no_prev_seg.addr; + run->seg_in = avd_av1_aux_alloc(ctx, dst->av1.seg_size, 0, true); + if (!run->seg_in) + return -ENOMEM; } + run->seg_written = seg_enabled && (frame->segmentation.flags & + V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP); + if (run->seg_written) + run->seg_out = avd_av1_aux_alloc(ctx, dst->av1.seg_size, + run->slot->seq, false); + else + run->seg_out = avd_av1_aux_get(run->seg_in); + if (!run->seg_out) + return -ENOMEM; - if (!(flags & V4L2_AV1_SEGMENTATION_FLAG_ENABLED) || - (!(flags & V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP) && !prev_seg)) - memset(seg, 0, dst->av1.seg_size); - else if (!(flags & V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP)) - memcpy(seg, prev_seg, dst->av1.seg_size); + run->addresses.probs_in = run->cdf_in->buf.addr; + run->addresses.probs_out = run->cdf_out->buf.addr; + run->addresses.prev_seg = run->seg_in->buf.addr; + run->addresses.seg = run->seg_out->buf.addr; + return 0; } static int avd_av1_seg_size(int w, int h) @@ -1189,13 +1309,11 @@ static void update_dec_buf_info(struct avd_decoded_buffer *buf, buf->av1.ref_frame_idx[i] = frame->ref_frame_idx[i]; } -static int avd_av1_run_preamble(struct avd_ctx *ctx, struct avd_av1_run *run) +static int avd_av1_controls(struct avd_ctx *ctx, struct avd_av1_run *run) { struct v4l2_ctrl *ctrl; int dst_len, tlb_len; - avd_run_preamble(ctx, &run->base); - ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_AV1_SEQUENCE); if (WARN_ON(!ctrl)) return -EINVAL; @@ -1229,6 +1347,7 @@ static int avd_av1_run_preamble(struct avd_ctx *ctx, struct avd_av1_run *run) if (WARN_ON(!ctrl)) return -EINVAL; run->tile_group = ctrl->p_cur.p; + run->num_tiles = ctrl->elems; ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl, V4L2_CID_STATELESS_AV1_FILM_GRAIN); @@ -1257,7 +1376,6 @@ static int avd_av1_run_preamble(struct avd_ctx *ctx, struct avd_av1_run *run) static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) { struct avd_dev *avd = ctx->dev; - struct avd_av1_ctx *av1_ctx = ctx->priv; const struct v4l2_ctrl_av1_frame *frame = run->frame; const struct v4l2_ctrl_av1_sequence *seq = run->seq; const struct v4l2_av1_tile_info *tile_info = &frame->tile_info; @@ -1267,7 +1385,6 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) seq->flags & V4L2_AV1_SEQUENCE_FLAG_USE_128X128_SUPERBLOCK ? 1 : 0; int w = frame->frame_width_minus_1 + 1; - int h = frame->frame_height_minus_1 + 1; int bit_depth = seq->bit_depth; for (i = 0; i < tile_info->tile_cols; i++) { @@ -1280,43 +1397,35 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) max_sb_col <<= sb_shift; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.mv_above_info, max_sb_col * 80); + ret = avd_buf_alloc(avd, &run->bufs.mv_above_info, max_sb_col * 80); if (ret) return ret; /* first frame this is always much smaller */ - ret = avd_buf_alloc(avd, &av1_ctx->bufs.above_info, max_sb_col * 400); + ret = avd_buf_alloc(avd, &run->bufs.above_info, max_sb_col * 400); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.lf_above_info, sb_cols1); + ret = avd_buf_alloc(avd, &run->bufs.lf_above_info, sb_cols1); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.ip_above, + ret = avd_buf_alloc(avd, &run->bufs.ip_above, max_sb_col * bit_depth * 22); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.lf_above, + ret = avd_buf_alloc(avd, &run->bufs.lf_above, ALIGN(ALIGN(w, 8) * bit_depth * 2 + tile_info->tile_cols * bit_depth * 16, 128)); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.rf_above_info, sb_cols2); + ret = avd_buf_alloc(avd, &run->bufs.rf_above_info, sb_cols2); if (ret) return ret; - /* zeroes, read as the previous segment ids when no reference has them */ - if (av1_ctx->bufs.no_prev_seg.size != avd_av1_seg_size(w, h)) { - ret = avd_buf_alloc(avd, &av1_ctx->bufs.no_prev_seg, - avd_av1_seg_size(w, h)); - if (ret) - return ret; - } - /* this is not a mistake, but im not sure why */ if (tile_info->tile_cols > 1) { for (i = 0; i < tile_info->tile_rows; i++) { @@ -1326,7 +1435,7 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) max_sb_row = max_sb_row << sb_shift; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.lf_left_info, + ret = avd_buf_alloc(avd, &run->bufs.lf_left_info, max_sb_row * 72); if (ret) return ret; @@ -1341,24 +1450,24 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) * I counted blocks as luma or chroma with padding until it * changes to the other */ - ret = avd_buf_alloc(avd, &av1_ctx->bufs.lf_left, + ret = avd_buf_alloc(avd, &run->bufs.lf_left, (max_sb_row * 72 + 1) * 2 * bit_depth); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.sr_left, + ret = avd_buf_alloc(avd, &run->bufs.sr_left, max_sb_row * bit_depth * (53 + 52)); if (ret) return ret; ret = avd_buf_alloc( - avd, &av1_ctx->bufs.rf_left, + avd, &run->bufs.rf_left, max_sb_row * bit_depth * (192 + 96 + (bit_depth > 8 ? 12 : 0))); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.rf_left_info, + ret = avd_buf_alloc(avd, &run->bufs.rf_left_info, max_sb_row * 24); if (ret) return ret; @@ -1367,129 +1476,290 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) return 0; } -static int avd_av1_run(struct avd_ctx *ctx) +static void avd_av1_release_run(struct avd_ctx *ctx, struct avd_av1_run *run) { - struct avd_av1_run run; - struct avd_decoded_buffer *dst, *ref; - int ret; + avd_buf_free(ctx->dev, &run->bufs.pipe_state); + avd_buf_free(ctx->dev, &run->bufs.above_info); + avd_buf_free(ctx->dev, &run->bufs.rf_above_info); + avd_buf_free(ctx->dev, &run->bufs.ip_above); + avd_buf_free(ctx->dev, &run->bufs.lf_above); + avd_buf_free(ctx->dev, &run->bufs.lf_above_info); + avd_buf_free(ctx->dev, &run->bufs.lf_left); + avd_buf_free(ctx->dev, &run->bufs.lf_left_info); + avd_buf_free(ctx->dev, &run->bufs.sr_left); + avd_buf_free(ctx->dev, &run->bufs.rf_left); + avd_buf_free(ctx->dev, &run->bufs.rf_left_info); + avd_buf_free(ctx->dev, &run->bufs.mv_above_info); + avd_av1_aux_put(ctx, run->cdf_in); + avd_av1_aux_put(ctx, run->cdf_out); + avd_av1_aux_put(ctx, run->seg_in); + avd_av1_aux_put(ctx, run->seg_out); + kfree(run); +} - ret = avd_av1_run_preamble(ctx, &run); - if (ret) { - avd_run_postamble(ctx, &run.base); - return ret; +static int avd_av1_publish_ref(struct avd_ctx *ctx, struct avd_av1_run *run) +{ + struct avd_av1_ctx *av1 = ctx->priv; + struct avd_av1_ref *ref, *old = NULL; + unsigned int index = run->base.bufs.dst->vb2_buf.index; + + list_for_each_entry(ref, &av1->refs, node) { + if (ref->index == index) { + old = ref; + break; + } + } + ref = old; + if (!ref) { + ref = kzalloc(sizeof(*ref), GFP_KERNEL); + if (!ref) + return -ENOMEM; + ref->index = index; + list_add_tail(&ref->node, &av1->refs); } + avd_av1_aux_put(ctx, ref->cdf); + avd_av1_aux_put(ctx, ref->seg); + ref->cdf = avd_av1_aux_get(run->cdf_out); + ref->seg = avd_av1_aux_get(run->seg_out); + /* Core copies source metadata to CAPTURE after prepare returns. */ + ref->timestamp = run->base.bufs.src->vb2_buf.timestamp; + ref->width = run->frame->frame_width_minus_1; + ref->height = run->frame->frame_height_minus_1; + av1->prev_timestamp = ref->timestamp; + av1->prev_seq = run->slot->seq; + av1->prev_updates_cdf = run->cdf_written; + av1->have_prev = true; + return 0; +} - /* The CDF and segmentation map copies need the CPU mappings. */ - dst = vb2_to_avd_decoded_buf(&run.base.bufs.dst->vb2_buf); - ref = dst; - if (run.frame->primary_ref_frame != AV1_PRIMARY_REF_NONE) - ref = avd_get_ref_buf(ctx, &dst->base.vb, - run.frame->reference_frame_ts - [run.frame->ref_frame_idx - [run.frame->primary_ref_frame]]); - if (!vb2_plane_vaddr(&dst->base.vb.vb2_buf, 0) || - !vb2_plane_vaddr(&ref->base.vb.vb2_buf, 0)) { - avd_run_postamble(ctx, &run.base); +static int avd_av1_prepare(struct avd_ctx *ctx, struct avd_frame *f) +{ + struct avd_av1_ctx *av1 = ctx->priv; + struct avd_av1_run *run; + struct avd_decoded_buffer *dst; + unsigned int i; + int ret; + + if (!f->src || !f->dst || !f->prog || !f->prog->cpu || + f->idx >= AVD_MAX_FRAMES || f->codec_priv) + return -EINVAL; + if (test_and_set_bit(f->idx, &av1->active_slots)) + return -EBUSY; + run = kzalloc(sizeof(*run), GFP_KERNEL); + if (!run) { + clear_bit(f->idx, &av1->active_slots); return -ENOMEM; } - - ret = avd_init_job(ctx, AVD_CODEC_AV1, - run.frame->tile_info.tile_cols * - run.frame->tile_info.tile_rows + - 1); - if (ret) { - avd_run_postamble(ctx, &run.base); - return ret; + run->slot = f; + run->base.bufs.src = f->src; + run->base.bufs.dst = f->dst; + run->base.coded_in = vb2_dma_contig_plane_dma_addr(&f->src->vb2_buf, 0); + run->base.y_out = vb2_dma_contig_plane_dma_addr(&f->dst->vb2_buf, 0); + run->base.uv_out = run->base.y_out + + ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline * + ctx->decoded_fmt.fmt.pix_mp.height; + run->base.comp_out = run->base.y_out + ctx->comp.start_offset; + f->prog_words = 0; + f->dep = AVD_DEP_NONE; + ret = avd_av1_controls(ctx, run); + if (ret) + goto fail; + ret = avd_av1_validate_tiles(run->frame, run->tile_group, + run->num_tiles, + vb2_get_plane_payload(&f->src->vb2_buf, 0)); + if (ret) + goto fail; + /* The control is a fixed-capacity array; only the tile grid is work. */ + run->num_tiles = run->frame->tile_info.tile_cols * + run->frame->tile_info.tile_rows; + /* Preserve the existing open decoder's grain-less output fallback. */ + for (i = 0; i < V4L2_AV1_REFS_PER_FRAME; i++) { + if (run->frame->ref_frame_idx[i] < 0 || + run->frame->ref_frame_idx[i] >= V4L2_AV1_TOTAL_REFS_PER_FRAME) { + ret = -EINVAL; + goto fail; + } } - - ret = avd_av1_alloc_work_bufs(ctx, &run); - if (ret) { - avd_run_postamble(ctx, &run.base); - return ret; + /* Borage syntax/DVA mappings remain [C]; never submit invented bytes. */ + if (f->fp) { + ret = avd_av1_pack_fp(ctx, f, run->seq, run->frame, + run->tile_group, run->grain); + if (ret) + goto fail; } + ret = avd_av1_alloc_work_bufs(ctx, run); + if (ret) + goto fail; + ret = avd_buf_alloc(ctx->dev, &run->bufs.pipe_state, 0x200); + if (ret) + goto fail; + dst = vb2_to_avd_decoded_buf(&f->dst->vb2_buf); + update_dec_buf_info(dst, run->seq, run->frame); + dst->comp = ctx->comp; + ret = avd_av1_set_aux(ctx, run); + if (ret) + goto fail; + avd_av1_push(run, AVD_OP_EXEC | + AVD_OP_EXEC_FLAG_START_REV4(ctx->dev->variant->revision == 4)); + set_header(ctx, run); + run->segment_end[0] = f->prog_words; + set_tiles(ctx, run); + if (run->error) { + ret = run->error; + goto fail; + } + ret = avd_av1_publish_ref(ctx, run); + if (ret) + goto fail; + /* The request controls can change as soon as prepare returns. */ + run->seq = NULL; + run->frame = NULL; + run->tile_group = NULL; + run->grain = NULL; + f->codec_priv = run; + return 0; +fail: + avd_av1_release_run(ctx, run); + clear_bit(f->idx, &av1->active_slots); + f->prog_words = 0; + return ret; +} - update_dec_buf_info(dst, run.seq, run.frame); - - avd_av1_set_prob(ctx, &run); - avd_av1_set_seg(ctx, &run); - - set_header(ctx, &run); - set_tiles(ctx, &run); - - avd_run_postamble(ctx, &run.base); +static void avd_av1_retire(struct avd_ctx *ctx, struct avd_frame *f, + enum vb2_buffer_state state) +{ + struct avd_av1_ctx *av1 = ctx->priv; + struct avd_av1_run *run = f->codec_priv; + int status = state == VB2_BUF_STATE_DONE ? 0 : -EIO; - return avd_submit_job(ctx); + if (!run) + return; + if (run->cdf_written) { + WRITE_ONCE(run->cdf_out->status, status); + smp_store_release(&run->cdf_out->done, true); + } + if (run->seg_written) { + WRITE_ONCE(run->seg_out->status, status); + smp_store_release(&run->seg_out->done, true); + } + avd_av1_release_run(ctx, run); + f->codec_priv = NULL; + clear_bit(f->idx, &av1->active_slots); } -static int avd_av1_alloc_bufs(struct avd_ctx *ctx) +/* + * The open transports still consume legacy segments and PP kick counts. Keep + * that interface until they accept native frame_ops. All packing and auxiliary + * ownership stay in prepare/retire; only this adapter accesses the legacy job. + */ +static int avd_av1_run(struct avd_ctx *ctx) { - struct avd_dev *avd = ctx->dev; - struct avd_av1_ctx *av1_ctx = ctx->priv; + struct avd_av1_ctx *av1 = ctx->priv; + struct avd_frame *f = &av1->legacy_frame; + struct avd_av1_run *run; + struct avd_run base; + const __le32 *words; + size_t pos = 1, end, i, j; int ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.inst, fifo_size()); + avd_run_preamble(ctx, &base); + if (!av1->legacy_prog.cpu) { + ret = avd_buf_alloc(ctx->dev, &av1->legacy_prog, + (AVD_MAX_INST + 1) * (V4L2_AV1_MAX_TILE_COUNT + 1) * sizeof(u32)); + if (ret) + goto complete_controls; + } + f->src = base.bufs.src; + f->dst = base.bufs.dst; + f->prog = &av1->legacy_prog; + f->seq++; + ret = avd_av1_prepare(ctx, f); if (ret) - return ret; - - /* no m1 to worry about :) */ - ret = avd_buf_alloc(avd, &av1_ctx->bufs.pipe_state, 0x200); + goto complete_controls; + run = f->codec_priv; + ret = avd_init_job(ctx, AVD_CODEC_AV1, run->num_tiles + 1); if (ret) - return ret; + goto retire; + words = f->prog->cpu; + for (i = 0; i <= run->num_tiles; i++) { + struct avd_segment *seg = &ctx->job.segments[i]; + + end = run->segment_end[i]; + if (end < pos || end - pos > AVD_MAX_INST) { + ret = -EOVERFLOW; + goto retire; + } + seg->num = end - pos; + for (j = 0; j < seg->num; j++) + seg->instructions[j] = le32_to_cpu(words[pos + j]); + pos = end + !!i; + } + ctx->job.num = run->num_tiles; + avd_run_postamble(ctx, &base); + ret = avd_submit_job(ctx); + if (!ret) + return 0; + avd_av1_retire(ctx, f, VB2_BUF_STATE_ERROR); + return ret; +retire: + avd_av1_retire(ctx, f, VB2_BUF_STATE_ERROR); +complete_controls: + avd_run_postamble(ctx, &base); + return ret; +} - ret = avd_buf_alloc(avd, &av1_ctx->bufs.probs, - sizeof(struct avd_av1_cdfs)); - if (ret) - return ret; +static void avd_av1_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src, + struct vb2_v4l2_buffer *dst, enum vb2_buffer_state state) +{ + struct avd_av1_ctx *av1 = ctx->priv; - return 0; + avd_av1_retire(ctx, &av1->legacy_frame, state); +} + +static unsigned int avd_av1_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_av1_ctx *av1 = ctx->priv; + struct avd_av1_run *run = av1->legacy_frame.codec_priv; + + return run ? run->num_tiles : 1; +} + +static void avd_av1_submit(struct avd_ctx *ctx) +{ + avd_open_av1_submit(ctx, avd_av1_pp_kicks(ctx)); } static void avd_av1_stop(struct avd_ctx *ctx) { - struct avd_av1_ctx *av1_ctx = ctx->priv; - struct avd_dev *avd = ctx->dev; + struct avd_av1_ctx *av1 = ctx->priv; + struct avd_av1_ref *ref, *tmp; - avd_buf_free(avd, &av1_ctx->bufs.above_info); - avd_buf_free(avd, &av1_ctx->bufs.rf_above_info); - avd_buf_free(avd, &av1_ctx->bufs.az_above); - avd_buf_free(avd, &av1_ctx->bufs.ip_above); - avd_buf_free(avd, &av1_ctx->bufs.lf_above); - avd_buf_free(avd, &av1_ctx->bufs.lf_above_info); - avd_buf_free(avd, &av1_ctx->bufs.lf_left); - avd_buf_free(avd, &av1_ctx->bufs.lf_left_info); - avd_buf_free(avd, &av1_ctx->bufs.sr_left); - avd_buf_free(avd, &av1_ctx->bufs.rf_left); - avd_buf_free(avd, &av1_ctx->bufs.rf_left_info); - avd_buf_free(avd, &av1_ctx->bufs.no_prev_seg); - avd_buf_free(avd, &av1_ctx->bufs.mv_above_info); - - avd_buf_free(avd, &av1_ctx->bufs.pipe_state); - avd_buf_free(avd, &av1_ctx->bufs.inst); - avd_buf_free(avd, &av1_ctx->bufs.probs); - - kfree(av1_ctx); + /* Core drains and retires every slot before stopping the codec. */ + if (WARN_ON(av1->active_slots)) + return; + list_for_each_entry_safe(ref, tmp, &av1->refs, node) { + avd_av1_aux_put(ctx, ref->cdf); + avd_av1_aux_put(ctx, ref->seg); + list_del(&ref->node); + kfree(ref); + } + avd_buf_free(ctx->dev, &av1->legacy_prog); + kfree(av1); ctx->priv = NULL; } static int avd_av1_start(struct avd_ctx *ctx) { - struct avd_av1_ctx *av1_ctx; - int ret; + struct avd_av1_ctx *av1; - av1_ctx = kzalloc(sizeof(*av1_ctx), GFP_KERNEL); - if (!av1_ctx) + av1 = kzalloc(sizeof(*av1), GFP_KERNEL); + if (!av1) return -ENOMEM; - - ctx->priv = av1_ctx; - ret = avd_av1_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; - + INIT_LIST_HEAD(&av1->refs); + /* The embedded legacy frame must retire before this ctx runs again. */ + ctx->max_frames = 1; + ctx->priv = av1; return 0; - -err_free_ctx: - avd_av1_stop(ctx); - return ret; } static enum avd_image_fmt avd_av1_get_image_fmt(struct avd_ctx *ctx, @@ -1515,24 +1785,6 @@ static enum avd_image_fmt avd_av1_get_image_fmt(struct avd_ctx *ctx, #undef BIT_DEPTH } -static void avd_av1_submit(struct avd_ctx *ctx) -{ - struct avd_av1_ctx *av1_ctx = ctx->priv; - struct avd_dev *avd = ctx->dev; - - writel(AVD_OP_EXEC | - AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == - 4) | - AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx) | - AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots), - avd->ctrl + avd->variant->submit_offset); - for (int i = 0; i < av1_ctx->submit_num - 1; i++) { - writel(AVD_OP_EXEC | AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx) | - AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots), - avd->ctrl + avd->variant->submit_offset); - } -} - static void avd_av1_adjust_decoded_fmt(struct avd_ctx *ctx, struct v4l2_pix_format_mplane *pix_mp) { @@ -1569,7 +1821,15 @@ const struct avd_coded_fmt_ops avd_av1_fmt_ops = { .start = avd_av1_start, .stop = avd_av1_stop, .run = avd_av1_run, + .done = avd_av1_done, .submit = avd_av1_submit, + .pp_kicks = avd_av1_pp_kicks, .try_ctrl = avd_av1_try_ctrl, .get_image_fmt = avd_av1_get_image_fmt, }; + +const struct avd_codec_frame_ops avd_av1_frame_ops = { + .prepare = avd_av1_prepare, + .pack_cmd = avd_av1_pack_cmd, + .retire = avd_av1_retire, +}; diff --git a/drivers/media/platform/apple/avd/avd-backend-apple.c b/drivers/media/platform/apple/avd/avd-backend-apple.c new file mode 100644 index 0000000000000..8345f107e4c14 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-apple.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include + +#include "avd.h" +#include "avd-apple-protocol.h" +#include "avd-queue-internal.h" + +/* SPEC 3.2. Unknown field meanings are intentionally not named. */ +const u8 avd_borage_config[64] = { + [0x0a] = 0x55, [0x0b] = 0x01, /* +0x08: 0x01558000 */ + [0x09] = 0x80, + [0x11] = 0x8a, [0x12] = 0x55, [0x13] = 0x01, + [0x19] = 0x8e, [0x1a] = 0x55, [0x1b] = 0x01, + [0x28] = 0x0a, [0x2a] = 0x04, + [0x2c] = 0x04, + [0x38] = 0x08, [0x3a] = 0x10, [0x3b] = 0x01, +}; + +/* + * Source: clean SPEC 3-5, AppleAVD 993.1 / T6040 Borage. The SPEC does + * not supply the ITCM/version register offsets, link register operations or + * reply field layout. Do not substitute V3/open-firmware offsets for them. + * avd-protocol.rst records every unresolved field and its source/capture. + */ +const struct avd_apple_hw avd_borage_hw = { + .fw_name = "apple/avd-apple-t6040.bin", + .verified = false, +}; + +struct avd_apple { + void __iomem *regs; + const struct avd_apple_hw *hw; + const struct firmware *image; + u32 shadow[AVD_APPLE_SHADOW_SIZE]; + unsigned int shadow_read; + unsigned int shadow_write; + unsigned int producer; + u32 version; +}; + +static void avd_apple_release_image(void *data) +{ + struct avd_apple *apple = data; + + release_firmware(apple->image); +} + +static int avd_apple_image_present(struct avd_apple *apple) +{ + const struct avd_apple_hw *hw = apple->hw; + u32 start = readl(apple->regs + hw->itcm_start); + u32 end = readl(apple->regs + hw->itcm_end); + u32 version; + + if (end <= start || end - start < 4 || end - start > hw->imem_size || + hw->version_offset > end - start - 4) + return -ENOENT; + version = readl(apple->regs + hw->imem + hw->version_offset); + if (!version || version == U32_MAX) + return -ENOENT; + apple->version = version; + return 0; +} + +static int avd_apple_probe(struct avd_dev *avd) +{ + const struct avd_apple_hw *hw = avd->variant->apple; + struct avd_apple *apple; + int ret; + + if (!hw || !hw->verified || !hw->reply.verified) { + dev_info(avd->dev, "Apple transport awaits Borage register/reply facts; using open firmware\n"); + return -EOPNOTSUPP; + } + apple = devm_kzalloc(avd->dev, sizeof(*apple), GFP_KERNEL); + if (!apple) + return -ENOMEM; + apple->hw = hw; + apple->regs = devm_platform_ioremap_resource_byname(avd->pdev, "apple"); + if (IS_ERR(apple->regs)) + return PTR_ERR(apple->regs); + /* Firmware bytes remain in vendorfw; this driver never embeds an image. */ + ret = firmware_request_nowarn(&apple->image, hw->fw_name, avd->dev); + if (ret && ret != -ENOENT) + return ret; + ret = devm_add_action_or_reset(avd->dev, avd_apple_release_image, apple); + if (ret) + return ret; + avd->backend_priv = apple; + return 0; +} + +static int avd_apple_load(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + const struct firmware *fw = apple->image; + size_t i, rounded; + + if (!fw) + return -ENOENT; + if (!fw->size || fw->size > hw->imem_size || + hw->imem_size < 4 || hw->version_offset > hw->imem_size - 4) + return -EINVAL; + rounded = ALIGN(fw->size, 4); + if (rounded > hw->imem_size) + return -EFBIG; + writel(hw->itcm_clear_value, apple->regs + hw->itcm_clear); + writel(0, apple->regs + hw->itcm_start); + writel(rounded, apple->regs + hw->itcm_end); + /* Bounded word copies pad the last word without reading past fw->data. */ + for (i = 0; i < rounded; i += 4) { + u8 bytes[4] = {}; + + memcpy(bytes, fw->data + i, min_t(size_t, 4, fw->size - i)); + writel(get_unaligned_le32(bytes), apple->regs + hw->imem + i); + } + writel(hw->itcm_enable_value, apple->regs + hw->itcm_enable); + return avd_apple_image_present(apple); +} + +static int avd_apple_boot(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + u32 value; + int ret; + + ret = avd_apple_image_present(apple); + if (ret) { + ret = avd_apple_load(avd); + if (ret) + return ret; + } else { + /* Suspend stopped the MCPU without clearing the retained image. */ + writel(hw->itcm_enable_value, apple->regs + hw->itcm_enable); + } + dev_info(avd->dev, "Apple MCPU firmware %u.%u.%u (%08x)\n", + apple->version >> 8, (apple->version >> 4) & 0xf, + apple->version & 0xf, apple->version); + /* SPEC 3.2: decode connection, aperture 1, state 4, then LinkUp. */ + writel(hw->link_setup_value, apple->regs + hw->link_setup); + ret = readl_poll_timeout(apple->regs + hw->link_state, value, + value == 4, 10, 10000); + if (ret) + return ret; + writel(hw->link_up_value, apple->regs + hw->link_up); + ret = readl_poll_timeout(apple->regs + hw->ready, value, + value == 1, 10, 10000); + if (!ret) { + apple->producer = 0; + apple->shadow_read = 0; + apple->shadow_write = 0; + } + return ret; +} + +static void avd_apple_shutdown(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + + /* MCPU disable value must be proved together with enable in the ledger. */ + writel(0, apple->regs + apple->hw->itcm_enable); +} + +static unsigned int avd_apple_max_inflight(struct avd_dev *avd) +{ + /* + * Every frame in VP->PP flight owns one hardware instruction FIFO + * (EXEC bits 7:4), so the FIFO count bounds the frames in flight: + * 15 on v3+ tier 1 parts, 7 on T8103 and T8140. + */ + return min_t(unsigned int, AVD_MAX_FRAMES, avd->variant->fifo_slots ?: 1); +} + +static int avd_apple_send(struct avd_dev *avd, const u32 cmd[24]) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + unsigned int consumer = readl(apple->regs + hw->inbox_consumer) & 7; + u32 address = AVD_APPLE_INBOX_BASE + apple->producer * AVD_APPLE_COMMAND_SIZE; + unsigned int i; + + if (avd_inbox_full(apple->producer, consumer)) + return -EBUSY; + for (i = 0; i < 24; i++) + writel(cmd[i], apple->regs + address - hw->aperture_origin + i * 4); + /* MMIO commands and coherent fp payload precede the four-byte doorbell. */ + wmb(); + writel(avd_apple_notification(apple->producer), apple->regs + hw->notify); + apple->producer = (apple->producer + 1) & 7; + return 0; +} + +static int avd_apple_session_start(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 0, 0, 0)) + return -EINVAL; + return avd_apple_send(ctx->dev, cmd); +} + +static void avd_apple_session_stop(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 2, 0, 0)) + return; + if (avd_apple_send(ctx->dev, cmd)) + dev_warn(ctx->dev->dev, "session stop inbox full\n"); +} + +static void avd_apple_abort(struct avd_ctx *ctx) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 5, 0, 0)) + return; + /* Core follows abort with reset before returning any DMA buffer. */ + avd_apple_send(ctx->dev, cmd); +} + +static int avd_apple_set_property(struct avd_ctx *ctx, u32 property, u32 value) +{ + u32 cmd[24] = {}; + + if (avd_apple_session_command(cmd, ctx->client_id, 6, property, value)) + return -EINVAL; + return avd_apple_send(ctx->dev, cmd); +} + +static int avd_apple_submit(struct avd_dev *avd, struct avd_frame *f) +{ + const struct avd_codec_frame_ops *ops = f->ctx->coded_fmt_desc->frame_ops; + u32 selector; + int ret; + + if (!ops || !ops->pack_cmd || !f->fp->cpu || f->ctx->client_id >= 1024) + return -EOPNOTSUPP; + /* SPEC 5.2 prefix precedes the codec-specific builder. */ + selector = get_unaligned_le32(f->fp->cpu + 4) & 0xf; + ret = avd_apple_decode_prefix(f, f->ctx->client_id, selector, f->seq & 3); + if (ret) + return ret; + ret = ops->pack_cmd(f->ctx, f); + if (ret) + return ret; + return avd_apple_send(avd, f->cmd); +} + +static irqreturn_t avd_apple_irq(struct avd_dev *avd) +{ + struct avd_apple *apple = avd->backend_priv; + const struct avd_apple_hw *hw = apple->hw; + unsigned int budget = AVD_APPLE_SHADOW_SIZE; + + while (budget-- && (readl(apple->regs + hw->receive_status) & hw->receive_pending)) { + unsigned int next = (apple->shadow_write + 1) & 63; + + if (next == apple->shadow_read) + break; + apple->shadow[apple->shadow_write] = readl(apple->regs + hw->receive); + apple->shadow_write = next; + writel(hw->receive_ack_value, apple->regs + hw->receive_ack); + } + while (apple->shadow_read != apple->shadow_write) { + u32 record[AVD_APPLE_RESULT_WORDS]; + struct avd_apple_reply reply; + unsigned int i; + int ret; + + /* Result notification indexing must be verified with the layout. */ + u32 index = apple->shadow[apple->shadow_read]; + + apple->shadow_read = (apple->shadow_read + 1) & 63; + if (index >= AVD_APPLE_SHADOW_SIZE) + continue; + for (i = 0; i < AVD_APPLE_RESULT_WORDS; i++) + record[i] = readl(apple->regs + hw->result_base + + index * sizeof(record) + i * 4); + ret = avd_apple_parse_reply(record, ARRAY_SIZE(record), &hw->reply, &reply); + if (ret) + continue; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + + if (f->state == AVD_FRAME_SUBMITTED && f->frame_num == reply.frame && + f->ctx->client_id == reply.client) { + dma_rmb(); + avd_frame_complete(avd, f->tag, reply.status); + break; + } + } + } + mod_delayed_work(system_wq, &avd->queue_work, 0); + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_apple_backend = { + .name = "apple-mcpu", + .probe = avd_apple_probe, + .boot = avd_apple_boot, + .shutdown = avd_apple_shutdown, + .max_inflight = avd_apple_max_inflight, + .session_start = avd_apple_session_start, + .session_stop = avd_apple_session_stop, + .submit = avd_apple_submit, + .abort = avd_apple_abort, + .set_property = avd_apple_set_property, + .irq = avd_apple_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, +}; diff --git a/drivers/media/platform/apple/avd/avd-backend-open.c b/drivers/media/platform/apple/avd/avd-backend-open.c new file mode 100644 index 0000000000000..3ef5b7642b147 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-open.c @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Legacy transport split from avd-drv.c; Copyright The Asahi Linux Contributors. */ +#include +#include +#include +#include +#include +#include + +#include "avd.h" +#include "avd-inst.h" +#include "avd-regs.h" + +/* + * Compatibility transport for firmware without the queued-program ABI. The + * codec's CPU builder emits a per-slot DMA-visible instruction stream. Only + * this backend touches the VP FIFO. Its untagged PP interrupt limits it to one + * frame; the queued avd-fw transport can replace these ops without changing + * buffer ownership in the core. + */ +int avd_open_build_program(struct avd_ctx *ctx) +{ + struct avd_dev *avd = ctx->dev; + struct avd_frame *f = avd->building; + struct avd_job *job = &ctx->job; + size_t count = 1, pos = 0, i, j; + u32 *words, mask, start; + int ret; + + if (!f || !job->segments) + return -EINVAL; + for (i = 0; i <= job->num; i++) { + if (job->segments[i].num > AVD_MAX_INST || + check_add_overflow(count, job->segments[i].num + !!i, &count)) + return -EOVERFLOW; + } + if (count > SIZE_MAX / sizeof(u32)) + return -EOVERFLOW; + if (f->prog->size < count * sizeof(u32)) { + ret = avd_buf_alloc(avd, f->prog, count * sizeof(u32)); + if (ret) + return ret; + } + mask = job->codec == AVD_CODEC_VP9 && avd->variant->revision == 3 ? + AVD_OP_EXEC_REV3_VP9_MASK : 0; + start = AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | + AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3); + /* The queued transport gives every frame in flight its own FIFO. */ + ctx->fifo_idx = avd_backend_is_qfw(avd) ? f->idx : 0; + f->fifo_iova = ctx->job_fifo ?: ctx->inst.addr; + ctx->job_fifo = 0; + words = f->prog->cpu; + words[pos++] = AVD_OP_EXEC | mask | start | AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx); + for (i = 0; i <= job->num; i++) { + for (j = 0; j < job->segments[i].num; j++) + words[pos++] = job->segments[i].instructions[j]; + if (i) + words[pos++] = AVD_OP_EXEC | mask | AVD_OP_EXEC_FLAG_END(i == job->num); + } + f->prog_words = pos; + f->cmd[0] = job->codec; + kvfree(job->segments); + job->segments = NULL; + return 0; +} + +static unsigned int avd_open_max_inflight(struct avd_dev *avd) +{ + return 1; +} + +static int avd_open_submit(struct avd_dev *avd, struct avd_frame *f) +{ + struct avd_ctx *ctx = f->ctx; + u32 codec = f->cmd[0], vp = 0; + u32 *words = f->prog->cpu; + unsigned int i; + void __iomem *reg; + + if (avd->compat_active) + return -EBUSY; + /* Native frame_ops require firmware-side PP chaining and tagged replies. */ + if (avd_get_frame_ops(ctx)) + return -EOPNOTSUPP; + if (codec > AVD_CODEC_AV1 || !words || !f->prog_words || + f->prog_words > f->prog->size / sizeof(u32)) + return -EINVAL; + if (!(avd->capabilities & BIT(codec))) + return -EOPNOTSUPP; + for (i = 0; i < codec; i++) + vp += avd->variant->vp_slots[i]; + if (avd->variant->vp_slot_base[codec]) + vp = avd->variant->vp_slot_base[codec]; + avd->variant->configure_stream(avd, ctx->inst.addr, ctx->fifo_idx, vp); + reg = avd->ctrl + avd->variant->vp_slot_offset + vp * 4; + avd->compat_active = f; + avd->compat_ctx = ctx; + for (i = 0; i < f->prog_words; i++) { + /* Preserve the old firmware's segment FIFO pacing in this shim. */ + if (i && (words[i] & GENMASK(31, 24)) == AVD_OP_EXEC && + (!(avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) || + codec == AVD_CODEC_HEVC)) { + u32 max = readl(avd->ctrl + avd->variant->submit_queue_max_offset + vp * 4); + u32 cur = readl(avd->ctrl + + avd->variant->submit_queue_status_offset + vp * 4); + + if (cur >= max) { + /* Already emitted words: only reset may release this slot. */ + f->deadline = jiffies; + return 0; + } + if (cur >= max / 2) { + /* + * Wait for the VP to drain below half full, but + * only as long as it takes. The old fixed sleep + * (100-150 us on v5 pacing parts, 500-650 us + * elsewhere) stays the upper bound. + */ + u32 wait_us = avd->variant->quirks & + AVD_QUIRK_V5_QUEUE_PACING ? 150 : 650; + + readl_relaxed_poll_timeout(avd->ctrl + + avd->variant->submit_queue_status_offset + vp * 4, + cur, cur < max / 2, 10, wait_us); + } + } + writel(words[i], reg); + } + return 0; +} + +/* Legacy AV1: one PP kick for every tile, matching the original transport. */ +void avd_open_av1_submit(struct avd_ctx *ctx, unsigned int kicks) +{ + struct avd_dev *avd = ctx->dev; + u32 cmd = AVD_OP_EXEC | AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx) | + AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots); + unsigned int i; + + writel(cmd | AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4), + avd->ctrl + avd->variant->submit_offset); + for (i = 1; i < kicks; i++) + writel(cmd, avd->ctrl + avd->variant->submit_offset); +} + +static irqreturn_t avd_open_irq(struct avd_dev *avd) +{ + struct avd_frame *f = avd->compat_active; + u32 status; + + if (!avd_mbox_pop(avd, &status)) + return IRQ_HANDLED; + if (!f) { + /* + * The old driver kicked PP for every VP DONE. A VP DONE with no + * frame in flight (its frame already completed on a surplus PP + * DONE of a multi-kick picture) still needs that kick, or the + * next picture's PP is never started and it times out. + */ + if ((status & 0x11000) == 0 && (status & 0x100) && avd->compat_ctx && + avd->compat_ctx->coded_fmt_desc->ops->submit) + avd->compat_ctx->coded_fmt_desc->ops->submit(avd->compat_ctx); + return IRQ_HANDLED; + } + if (status & 0x1000) { + avd->compat_active = NULL; + avd_frame_complete(avd, f->tag, 0); + } else if (status & 0x100) { + /* Old firmware only: no request tag and no firmware-side PP kick. */ + if (f->ctx->coded_fmt_desc->ops->submit) + f->ctx->coded_fmt_desc->ops->submit(f->ctx); + } else { + dev_err_ratelimited(avd->dev, "open firmware error %#x\n", status); + f->deadline = jiffies; + mod_delayed_work(system_wq, &avd->queue_work, 0); + } + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_open_backend = { + .name = "open-compat", + .boot = avd_boot, + .shutdown = avd_shutdown, + .max_inflight = avd_open_max_inflight, + .submit = avd_open_submit, + .irq = avd_open_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, +}; + +static void avd_open_capabilities(struct avd_dev *avd) +{ + static const struct { + const char *name; + u32 bit; + } codecs[] = { + { "h265", AVD_CAPABILITY_HEVC }, + { "h264", AVD_CAPABILITY_H264 }, + { "vp9", AVD_CAPABILITY_VP9 }, + { "av1", AVD_CAPABILITY_AV1 }, + }; + const char *info; + unsigned int i; + + /* Without a CODECS trailer, keep codecs that need a newer firmware hidden. */ + avd->capabilities = avd->variant->capabilities & + ~avd->variant->fw_caps_required; + avd->fw_capabilities = avd->capabilities; + if (!avd->fw || avd->fw->size < 0x40) + return; + info = (const char *)avd->fw->data + avd->fw->size - 0x40; + if (memcmp(info, "CODECS:", 7)) + return; /* fw_capabilities already set above */ + avd->capabilities = 0; + for (i = 0; i < ARRAY_SIZE(codecs); i++) + if (strnstr(info, codecs[i].name, 0x40)) + avd->capabilities |= avd->variant->capabilities & codecs[i].bit; + avd->fw_capabilities = avd->capabilities; +} + +static int avd_open_request_firmware(struct avd_dev *avd) +{ + int ret; + + if (!avd->variant->fw_name_fallback) + return request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); + ret = firmware_request_nowarn(&avd->fw, avd->variant->fw_name, avd->dev); + if (ret != -ENOENT) + return ret; + dev_info(avd->dev, "%s not installed, using %s\n", + avd->variant->fw_name, avd->variant->fw_name_fallback); + return request_firmware(&avd->fw, avd->variant->fw_name_fallback, avd->dev); +} + +int avd_backend_select(struct avd_dev *avd) +{ + int ret; + + /* Apple probe returns -ENOENT only when no usable image is present. */ + if (avd->variant->apple) { + ret = avd_apple_backend.probe(avd); + if (!ret) { + avd->backend = &avd_apple_backend; + avd->capabilities = avd->variant->capabilities; + return 0; + } + if (ret != -ENOENT && ret != -EOPNOTSUPP) + return ret; + } + avd->backend = &avd_open_backend; + ret = avd_open_request_firmware(avd); + if (!ret) + avd_open_capabilities(avd); + if (!ret) + ret = avd_qfw_probe(avd); + return ret; +} + +int avd_backend_boot(struct avd_dev *avd) +{ + int ret; + + if (!avd_backend_is_apple(avd)) { + avd_pm_prepare_piodma(avd); + avd_qfw_prepare_boot(avd); + } + ret = avd->backend->boot(avd); + /* Never switch command ABIs after frames have used this backend. */ + if (ret == -ENOENT && !avd->fw_generation && avd_backend_is_apple(avd)) { + ret = avd_open_request_firmware(avd); + if (!ret) + ret = avd_qfw_probe(avd); + if (ret) + return ret; + avd->backend = &avd_open_backend; + avd_open_capabilities(avd); + avd_pm_prepare_piodma(avd); + avd_qfw_prepare_boot(avd); + ret = avd->backend->boot(avd); + } + if (!ret) + avd_qfw_select(avd); + if (!ret) + avd->fw_generation++; + return ret; +} diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c new file mode 100644 index 0000000000000..3902d75cbbd1f --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Queued transport for the open AVD firmware (avd-fw queue ABI 2). + * + * The codec's CPU builder still produces the per-frame instruction program. + * Instead of writing it word by word into the VP FIFO register and kicking + * the PP from the VP-done interrupt, the host streams the program into a word + * ring in CM3 SRAM and posts one descriptor per frame. The firmware feeds the + * VP FIFO from SRAM, starts PP itself when the VP is done, and posts a tagged + * completion. No PIODMA (and so no second DART stream) is involved. + */ +#include +#include +#include +#include +#include + +#include "avd.h" +#include "avd-inst.h" +#include "avd-regs.h" + +#define AVDQ_MAGIC 0x41565131 /* "AVQ1" */ +#define AVDQ_VERSION 2 +#define AVDQ_COUNT 16 +#define AVDQ_DESC_BYTES 64 +#define AVDQ_RING_OFFSET 0x8000 +#define AVDQ_INFO 0x7fc0 +/* Firmware private state starts here; the SRAM ring must end before it. */ +#define AVDQ_RING_LIMIT 0x6000 +#define AVDQ_INFO_RP (AVDQ_INFO + 0x38) +#define AVDQ_INFO_WP (AVDQ_INFO + 0x3c) +#define AVDQ_FEAT_SRAM_RING BIT(0) +#define AVDQ_FEAT_MULTI_KICK BIT(1) +#define AVDQ_FLAG_SRAM BIT(0) +#define AVDQ_MAX_PP_EXTRA 255 +#define AVDQ_STATE_SUBMITTED 1 +#define AVDQ_STATE_COMPLETE 2 +#define AVDQ_NOTIFY_MASK 0xffff0000 +#define AVDQ_NOTIFY 0x54530000 +/* SRAM addresses as the CM3 doorbell sees them: 0x01000000 + offset from AVD base. */ +#define AVDQ_CM3_APERTURE(off) (0x01000000 + (off)) +#define AVD_REG_MBOX0_STATUS 0x50 +#define AVD_REG_MBOX0_SEND 0x54 +#define AVDQ_STREAM_TIMEOUT_US (1000 * USEC_PER_MSEC) + +/* Descriptor word offsets (bytes), avd-fw docs/queue.md. */ +enum { + D_MAGIC = 0x00, D_TAG = 0x04, D_CODEC = 0x08, D_VP = 0x0c, + D_PROG_LO = 0x10, D_PROG_HI = 0x14, D_WORDS = 0x18, D_KICK = 0x1c, + D_STATE = 0x20, D_STATUS = 0x24, D_CTAG = 0x28, D_FLAGS = 0x2c, + D_PP_EXTRA = 0x30, D_PP_CONT = 0x34, D_RSVD0 = 0x38, D_RSVD1 = 0x3c, +}; + +struct avd_qfw { + void __iomem *sram; + resource_size_t sram_size; + u32 aperture; /* expected CM3 address of SRAM byte 0 */ + u32 ring_off; + u32 ring_words; + u32 wp; /* host write count, free-running */ + u32 fifo_slots; + u32 codec_mask; /* info+0x24, bit n = codec n */ + bool usable; /* last boot found a matching ABI 2 image */ +}; + +static void __iomem *avd_qfw_desc(struct avd_qfw *q, unsigned int idx) +{ + return q->sram + AVDQ_RING_OFFSET + idx * AVDQ_DESC_BYTES; +} + +int avd_qfw_probe(struct avd_dev *avd) +{ + struct resource *code, *sram; + struct avd_qfw *q; + + code = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "code"); + sram = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "sram"); + if (!code || !sram || sram->start < code->start || + resource_size(sram) < AVDQ_RING_OFFSET + AVDQ_COUNT * AVDQ_DESC_BYTES) + return 0; /* no queue mode on this node: legacy transport only */ + q = devm_kzalloc(avd->dev, sizeof(*q), GFP_KERNEL); + if (!q) + return -ENOMEM; + q->sram = devm_ioremap_resource(avd->dev, sram); + if (IS_ERR(q->sram)) + return PTR_ERR(q->sram); + q->sram_size = resource_size(sram); + /* The code window sits at +0x80000 from the decoder base on every SoC. */ + q->aperture = AVDQ_CM3_APERTURE(0x80000 + (u32)(sram->start - code->start)); + avd->qfw = q; + return 0; +} + +/* Read the firmware's info block after boot; true when ABI 2 queue mode fits. */ +static bool avd_qfw_check(struct avd_dev *avd) +{ + struct avd_qfw *q = avd->qfw; + u32 info[16]; + unsigned int i; + + if (!q) + return false; + for (i = 0; i < ARRAY_SIZE(info); i++) + info[i] = readl(q->sram + AVDQ_INFO + 4 * i); + q->usable = false; + if (info[0] != AVDQ_MAGIC) + return false; + if (info[1] != AVDQ_VERSION || info[2] != AVDQ_COUNT || + info[3] != AVDQ_DESC_BYTES || info[5] != AVDQ_RING_OFFSET || + !(info[11] & AVDQ_FEAT_SRAM_RING) || + !(info[11] & AVDQ_FEAT_MULTI_KICK)) { + dev_info_once(avd->dev, "open firmware queue ABI %u not supported, using legacy transport\n", + info[1]); + return false; + } + if (info[4] != q->aperture) { + dev_warn_once(avd->dev, "firmware SRAM aperture %#x, node needs %#x: wrong image tier, using legacy transport\n", + info[4], q->aperture); + return false; + } + if (info[13] < 1024 || info[12] % 4 || + (u64)info[12] + 4ULL * info[13] > AVDQ_RING_LIMIT || + !info[10] || info[10] > AVDQ_COUNT || info[14] || info[15]) { + dev_warn_once(avd->dev, "firmware SRAM ring geometry invalid, using legacy transport\n"); + return false; + } + q->ring_off = info[12]; + q->ring_words = info[13]; + q->fifo_slots = min_t(u32, info[10], avd->variant->fifo_slots ?: 1); + q->codec_mask = info[9]; + q->wp = 0; + q->usable = true; + return true; +} + +static int avd_qfw_boot(struct avd_dev *avd) +{ + int ret = avd_boot(avd); + + if (ret) + return ret; + /* AP -> CM3 doorbells use mailbox 0; completions stay on mailbox 1. */ + writel_relaxed(AVD_MBOX_ENABLE, avd->mbox + AVD_REG_MBOX0_STATUS); + return 0; +} + +static unsigned int avd_qfw_max_inflight(struct avd_dev *avd) +{ + /* Each frame in flight owns one hardware instruction FIFO (EXEC 7:4). */ + return min_t(unsigned int, AVD_MAX_FRAMES, avd->qfw->fifo_slots); +} + +static void avd_qfw_fail_all(struct avd_dev *avd) +{ + unsigned int i; + + /* ABI: any fatal word or I/O status needs a whole-queue reset. */ + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (avd->frames[i].state == AVD_FRAME_SUBMITTED) + avd->frames[i].deadline = jiffies; + mod_delayed_work(system_wq, &avd->queue_work, 0); +} + +static int avd_qfw_stream(struct avd_dev *avd, const u32 *words, u32 count) +{ + struct avd_qfw *q = avd->qfw; + unsigned long timeout = jiffies + usecs_to_jiffies(AVDQ_STREAM_TIMEOUT_US); + u32 done = 0; + + while (done < count) { + u32 rp = readl(q->sram + AVDQ_INFO_RP); + u32 used = q->wp - rp, space, pos, chunk; + + if (used > q->ring_words) + return -EIO; + space = q->ring_words - used; + if (!space) { + if (time_after(jiffies, timeout)) + return -ETIMEDOUT; + usleep_range(20, 50); + continue; + } + pos = q->wp % q->ring_words; + chunk = min3(space, count - done, q->ring_words - pos); + memcpy_toio(q->sram + q->ring_off + 4 * pos, words + done, 4 * chunk); + q->wp += chunk; + done += chunk; + /* Words before the count that publishes them. */ + wmb(); + writel(q->wp, q->sram + AVDQ_INFO_WP); + } + return 0; +} + +static int avd_qfw_submit(struct avd_dev *avd, struct avd_frame *f) +{ + struct avd_qfw *q = avd->qfw; + struct avd_ctx *ctx = f->ctx; + const struct avd_coded_fmt_ops *ops = ctx->coded_fmt_desc->ops; + u32 codec = f->cmd[0], vp = 0, kick, cont, mask, extra = 0; + u32 *words = f->prog->cpu; + void __iomem *d = avd_qfw_desc(q, f->idx); + unsigned int i; + int ret; + + if (avd_get_frame_ops(ctx)) + return -EOPNOTSUPP; + if (codec > AVD_CODEC_AV1 || !words || !f->prog_words || + f->prog_words > f->prog->size / sizeof(u32)) + return -EINVAL; + if (!(avd->capabilities & BIT(codec))) + return -EOPNOTSUPP; + /* The program's EXEC words name FIFO f->idx; it must be a real FIFO. */ + if (f->idx >= q->fifo_slots || ctx->fifo_idx != f->idx) + return -EINVAL; + for (i = 0; i < codec; i++) + vp += avd->variant->vp_slots[i]; + if (avd->variant->vp_slot_base[codec]) + vp = avd->variant->vp_slot_base[codec]; + if (ops->pp_kicks) + extra = ops->pp_kicks(ctx) - 1; + if (extra > AVDQ_MAX_PP_EXTRA) + return -E2BIG; + + mask = codec == AVD_CODEC_VP9 && avd->variant->revision == 3 ? + AVD_OP_EXEC_REV3_VP9_MASK : 0; + cont = AVD_OP_EXEC | mask | AVD_OP_EXEC_FIFO_IDX(f->idx) | + AVD_OP_EXEC_FIFO_MASK(avd->variant->fifo_slots); + kick = cont | AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | + AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3 && + codec != AVD_CODEC_AV1); + + /* FIFO registers of other live frames are never touched. */ + avd->variant->configure_stream(avd, f->fifo_iova ?: ctx->inst.addr, f->idx, vp); + + writel(AVDQ_MAGIC, d + D_MAGIC); + writel(f->tag, d + D_TAG); + writel(codec, d + D_CODEC); + writel(vp, d + D_VP); + writel(q->wp, d + D_PROG_LO); + writel(0, d + D_PROG_HI); + writel(f->prog_words, d + D_WORDS); + writel(kick, d + D_KICK); + writel(0, d + D_STATUS); + writel(0, d + D_CTAG); + writel(AVDQ_FLAG_SRAM, d + D_FLAGS); + writel(extra, d + D_PP_EXTRA); + writel(extra ? cont : 0, d + D_PP_CONT); + writel(0, d + D_RSVD0); + writel(0, d + D_RSVD1); + writel(AVDQ_STATE_SUBMITTED, d + D_STATE); + /* Descriptor before the doorbell that hands it to the firmware. */ + wmb(); + (void)readl(d + D_STATE); /* flush posted aperture writes */ + writel(q->aperture + AVDQ_RING_OFFSET + f->idx * AVDQ_DESC_BYTES, + avd->mbox + AVD_REG_MBOX0_SEND); + + /* The firmware may start feeding while the rest of the program streams in. */ + ret = avd_qfw_stream(avd, words, f->prog_words); + if (ret) { + dev_err_ratelimited(avd->dev, "queue: program stream failed: %d\n", ret); + /* The descriptor is already posted: only a reset releases it. */ + f->deadline = jiffies; + } + return 0; +} + +static irqreturn_t avd_qfw_irq(struct avd_dev *avd) +{ + struct avd_qfw *q = avd->qfw; + struct avd_frame *f; + void __iomem *d; + u32 word, state, status, ctag; + unsigned int idx; + + if (!avd_mbox_pop(avd, &word)) + return IRQ_HANDLED; + if ((word & AVDQ_NOTIFY_MASK) != AVDQ_NOTIFY) { + dev_err_ratelimited(avd->dev, "queue: firmware reported %#x, resetting\n", word); + avd_qfw_fail_all(avd); + return IRQ_HANDLED; + } + idx = word & 15; + d = avd_qfw_desc(q, idx); + state = readl(d + D_STATE); + status = readl(d + D_STATUS); + ctag = readl(d + D_CTAG); + f = &avd->frames[idx]; + if (state != AVDQ_STATE_COMPLETE || ((ctag ^ word) & 0xfff0) || + f->state != AVD_FRAME_SUBMITTED || ctag != f->tag) { + dev_warn_ratelimited(avd->dev, "queue: stale completion %#x (state %u tag %#x)\n", + word, state, ctag); + return IRQ_HANDLED; + } + writel(0, d + D_STATE); + avd_frame_complete(avd, ctag, status == 0 ? 0 : status == 1 ? -EINVAL : -EIO); + if (status > 1) + avd_qfw_fail_all(avd); + return IRQ_HANDLED; +} + +const struct avd_backend_ops avd_qfw_backend = { + .name = "open-queue", + .boot = avd_qfw_boot, + .shutdown = avd_shutdown, + .max_inflight = avd_qfw_max_inflight, + .submit = avd_qfw_submit, + .irq = avd_qfw_irq, + .suspend = avd_pm_backend_suspend, + .resume = avd_pm_backend_resume, +}; + +bool avd_backend_is_qfw(const struct avd_dev *avd) +{ + return avd->backend == &avd_qfw_backend; +} + +/* + * SRAM survives a CM3 reload, so a stale info block from an earlier ABI 2 + * image could make a legacy image look queue capable. Clear the magic before + * every open-firmware boot; only an image that publishes it again qualifies. + */ +void avd_qfw_prepare_boot(struct avd_dev *avd) +{ + if (avd->qfw) + writel(0, avd->qfw->sram + AVDQ_INFO); +} + +/* + * Called after every successful open-firmware boot (probe, resume, reset), + * with no frame owned by the firmware. Picks the queued transport when the + * loaded image speaks ABI 2 for this node, else the legacy one. + */ +void avd_qfw_select(struct avd_dev *avd) +{ + const struct avd_backend_ops *old = avd->backend; + + if (old != &avd_open_backend && old != &avd_qfw_backend) + return; + if (avd_qfw_check(avd)) { + /* Legacy boot left mailbox 0 disabled. */ + writel_relaxed(AVD_MBOX_ENABLE, avd->mbox + AVD_REG_MBOX0_STATUS); + avd->backend = &avd_qfw_backend; + /* + * Codecs held back for a newer firmware (T6040 HEVC) are + * enabled by a queue image that lists them in its codec mask. + */ + avd->capabilities = avd->fw_capabilities | + (avd->variant->capabilities & avd->qfw->codec_mask); + } else { + avd->backend = &avd_open_backend; + avd->capabilities = avd->fw_capabilities; + } + if (avd->backend != old) + dev_info(avd->dev, "using %s transport (%u frames in flight)\n", + avd->backend->name, avd->backend->max_inflight(avd)); +} diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index bf5bf96f4c62a..1cb58c1b2f7c6 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -22,8 +22,7 @@ #include "avd.h" #include "avd-inst.h" #include "avd-regs.h" - -#define AVD_CODEC_INFO_MAX_SIZE 0x40 +#include "avd-apple-protocol.h" static void calc_tile_meta(u32 w, u32 h, u32 bpb, u32 tile_dim, u32 meta_hdr_bytes, u32 *tile, u32 *meta) @@ -73,31 +72,6 @@ void fill_comp(struct avd_comp *comp, enum avd_image_fmt image_fmt, u32 width, comp->size = y_meta + y + uv_meta + uv; } -int avd_buf_alloc(struct avd_dev *avd, struct avd_buf *buf, size_t size) -{ - /* Keep size equal to the payload length used by the H.264 slice path. */ - avd_buf_free(avd, buf); - - if (size <= 0) - return -ENOMEM; - - buf->size = size; - buf->cpu = - dma_alloc_coherent(avd->dev, buf->size, &buf->addr, GFP_KERNEL); - if (!buf->cpu) { - memset(buf, 0, sizeof(*buf)); - return -ENOMEM; - } - return 0; -} - -void avd_buf_free(struct avd_dev *avd, struct avd_buf *buf) -{ - if (buf->cpu) - dma_free_coherent(avd->dev, buf->size, buf->cpu, buf->addr); - memset(buf, 0, sizeof(*buf)); -} - struct avd_decoded_buffer * avd_get_ref_buf(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst, u64 timestamp) { @@ -116,28 +90,6 @@ avd_get_ref_buf(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst, u64 timestamp) return vb2_to_avd_decoded_buf(buf); } -static int avd_wait_submission_queue(struct avd_ctx *ctx, int vp) -{ - struct avd_dev *avd = ctx->dev; - u32 max = readl_relaxed(avd->ctrl + - avd->variant->submit_queue_max_offset + vp * 4); - u32 cur = readl_relaxed( - avd->ctrl + avd->variant->submit_queue_status_offset + vp * 4); - - if (cur == max) { - dev_err(avd->dev, "instruction que full! %d/%d", cur, max); - return 1; - } - - if (cur >= max / 2) { - if (avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) - usleep_range(100, 150); - else - usleep_range(500, 650); - } - return 0; -} - int avd_init_job(struct avd_ctx *ctx, enum avd_codec codec, size_t segments) { int ret = 0; @@ -155,287 +107,20 @@ int avd_init_job(struct avd_ctx *ctx, enum avd_codec codec, size_t segments) int avd_submit_job(struct avd_ctx *ctx) { - struct avd_dev *avd = ctx->dev; - struct avd_job *sub = &ctx->job; - struct avd_segment *segments, *seg; - unsigned long flags; - size_t num; - int i, idx = 0, vp = 0; - void __iomem *reg; - u32 exec_mask = sub->codec == AVD_CODEC_VP9 && - avd->variant->revision == 3 ? - AVD_OP_EXEC_REV3_VP9_MASK : - 0; - u32 exec_rev_flag = - AVD_OP_EXEC_FLAG_START_REV4(avd->variant->revision == 4) | - AVD_OP_EXEC_FLAG_START_REV3(avd->variant->revision == 3); - - ctx->fifo_idx = 0; - for (i = 0; i < sub->codec; i++) - vp += avd->variant->vp_slots[i]; - if (avd->variant->vp_slot_base[sub->codec]) - vp = avd->variant->vp_slot_base[sub->codec]; - - /* - * avd_device_run() keeps this job from being finished (and the next one - * from starting) until it returns, and arms the watchdog itself once the - * job belongs to the hardware. Still walk a local copy of the table. - */ - segments = sub->segments; - num = sub->num; - if (WARN_ON_ONCE(!segments)) - return -EINVAL; - - avd->variant->configure_stream(avd, ctx->inst.addr, ctx->fifo_idx, vp); - reg = avd->ctrl + avd->variant->vp_slot_offset + vp * 4; - - /* the first segment is always the header (needs special handling) */ - - writel(AVD_OP_EXEC | exec_mask | exec_rev_flag | - AVD_OP_EXEC_FIFO_IDX(ctx->fifo_idx), - reg); - seg = &segments[idx++]; - for (i = 0; i < seg->num; i++) - writel(seg->instructions[i], reg); - for (; idx <= num; idx++) { - seg = &segments[idx]; - for (i = 0; i < seg->num; i++) - writel(seg->instructions[i], reg); - if ((!(avd->variant->quirks & AVD_QUIRK_V5_QUEUE_PACING) || - sub->codec == AVD_CODEC_HEVC) && - avd_wait_submission_queue(ctx, vp)) - break; - if (idx == num) { - /* A completion from here on belongs to this job. */ - spin_lock_irqsave(&avd->job_lock, flags); - avd->job_end_sent = true; - spin_unlock_irqrestore(&avd->job_lock, flags); - } - writel(AVD_OP_EXEC | exec_mask | - AVD_OP_EXEC_FLAG_END(idx == num), - reg); - } - - if (sub->segments == segments) - sub->segments = NULL; - kvfree(segments); - return 0; -} - -static int avd_reset(struct avd_dev *avd) -{ - int ret = 0; - - ret = pm_runtime_resume_and_get(avd->dev); - if (ret < 0) - return ret; - - ret = reset_control_reset(avd->rstc); - if (ret) - dev_err(avd->dev, "reset: failed: %d", ret); - - if (avd->empty_domain) { - iommu_attach_device(avd->empty_domain, avd->dev); - iommu_detach_device(avd->empty_domain, avd->dev); - } - - ret = avd_boot(avd); - if (ret) - dev_err(avd->dev, "reset: failed to boot"); - - pm_runtime_put_autosuspend(avd->dev); - - return ret; -} - -static void avd_watchdog_func(struct work_struct *work) -{ - struct avd_dev *avd; - struct avd_ctx *ctx; - unsigned long flags; - bool claimed; - int ret; - ctx = container_of(to_delayed_work(work), struct avd_ctx, - watchdog_work); - if (!ctx) - return; - - avd = ctx->dev; - - /* - * Time out only the job this watchdog was armed for, and only while the - * hardware still owns it. Otherwise that job already finished, and - * resetting the hardware would break whichever job is running now. - */ - spin_lock_irqsave(&avd->job_lock, flags); - claimed = avd->job_ctx == ctx && avd->job_seq == ctx->wd_seq && - avd->job_state == AVD_JOB_SUBMITTED; - if (claimed) - avd->job_state = AVD_JOB_FINISHING; - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: watchdog seq %llu: %s\n", ctx->wd_seq, - claimed ? "timeout" : "stale, ignored"); - if (!claimed) - return; - - dev_err(avd->dev, "Frame processing timed out!"); - - writel(0, avd->mbox + AVD_REG_MBOX_IRQ_ENABLE); - ret = avd_reset(avd); - if (ret) - dev_err(avd->dev, "failed to reset: %d", ret); - - avd_job_finish(ctx, VB2_BUF_STATE_ERROR); + return avd_open_build_program(ctx); } static irqreturn_t avd_irq_handler(int irq, void *data) { struct avd_dev *avd = data; - struct avd_ctx *ctx = v4l2_m2m_get_curr_priv(avd->m2m_dev); - - enum vb2_buffer_state state; - unsigned long flags; - bool finish = false, stale = false; - u32 status; - if (!ctx) - return IRQ_HANDLED; - - status = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); - - writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); - - if (status & 0x10000) { /* dbg */ - dev_warn(avd->dev, "no handler for IRQ: %3d", status &~0x10000); - writel_relaxed(0, avd->mbox + AVD_REG_MBOX_IRQ_ENABLE); - return IRQ_HANDLED; - } - - if (status & 0x1000) { - /* pp is done ! we are done */ - state = VB2_BUF_STATE_DONE; - } else if (status & 0x100) { - /* a vp is done, kick the pp and hope for the best */ - if(ctx->coded_fmt_desc->ops->submit) - ctx->coded_fmt_desc->ops->submit(ctx); - goto done; - } else { - dev_err(avd->dev, "H%d error", status); - /* - * avd-fw err(n): VP n raised its error interrupt and stopped, - * and the PP only starts on a VP done, so the job cannot - * complete any more. Run the watchdog now instead of after its - * timeout: it resets the AVD and fails only this frame, and it - * ignores a job the hardware does not own yet. - */ - mod_delayed_work(system_wq, &ctx->watchdog_work, 0); - goto done; - } - - /* - * Finish only a job the hardware owns. While avd_device_run() is still - * setting up or submitting, a completion is left over from an older job - * unless this job's END command already went out; then remember it and - * let avd_device_run() finish the job when it returns. - */ - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_ctx != ctx) { - stale = true; - } else if (avd->job_state == AVD_JOB_SUBMITTED) { - avd->job_state = AVD_JOB_FINISHING; - finish = true; - } else if (avd->job_state == AVD_JOB_RUNNING && avd->job_end_sent) { - avd->job_pending = true; - avd->job_pending_result = state; - } else { - stale = true; - } - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: irq %#x: %s\n", status, - finish ? "finish" : stale ? "stale, ignored" : "pending"); - if (stale) - dev_warn_ratelimited(avd->dev, "ignoring stale completion"); - if (finish) { - cancel_delayed_work(&ctx->watchdog_work); - avd_job_finish(ctx, state); - } + /* An interrupt raised before a reset or power-off belongs to that instance. */ + unsigned int gen = READ_ONCE(avd->pm.irq_gen); + irqreturn_t ret; -done: - return IRQ_HANDLED; -} - -static void avd_device_run(void *priv) -{ - struct avd_ctx *ctx = priv; - struct avd_dev *avd = ctx->dev; - const struct avd_coded_fmt_desc *desc = ctx->coded_fmt_desc; - enum vb2_buffer_state result = VB2_BUF_STATE_ERROR; - const char *outcome = "already finished"; - unsigned long flags; - bool finish = false; - u64 seq; - int ret; - - if (WARN_ON(!desc)) - return; - - /* - * IRQ completion cannot wait for an already running watchdog. Drain it - * here, before reusing wd_seq, so an old callback cannot time out this - * context's next job. device_run is sleepable; hold no job_lock here. - */ - cancel_delayed_work_sync(&ctx->watchdog_work); - - ret = pm_runtime_resume_and_get(avd->dev); - if (ret < 0) { - avd_job_finish_no_pm(ctx, VB2_BUF_STATE_ERROR); - return; - } - - /* - * While the job is RUNNING neither completion IRQs nor the watchdog may - * finish it: v4l2-mem2mem would start the next job on a kworker while - * this one still builds and submits ctx->job. - */ - spin_lock_irqsave(&avd->job_lock, flags); - seq = ++avd->job_seq; - avd->job_ctx = ctx; - avd->job_state = AVD_JOB_RUNNING; - avd->job_end_sent = false; - avd->job_pending = false; - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: run seq %llu\n", seq); - - ret = desc->ops->run(ctx); - if (ret) { - dev_dbg(avd->dev, "avd-job: run seq %llu failed: %d\n", seq, ret); - avd_job_finish(ctx, VB2_BUF_STATE_ERROR); - return; - } - - /* - * The run either finished the job itself (held capture buffer) or handed - * it to the hardware: arm the watchdog, or finish right away if the - * completion already arrived. - */ - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_seq == seq && avd->job_state == AVD_JOB_RUNNING) { - if (avd->job_pending) { - avd->job_state = AVD_JOB_FINISHING; - result = avd->job_pending_result; - finish = true; - outcome = "completed during submission"; - } else { - avd->job_state = AVD_JOB_SUBMITTED; - ctx->wd_seq = seq; - schedule_delayed_work(&ctx->watchdog_work, msecs_to_jiffies(2000)); - outcome = "submitted"; - } - } - spin_unlock_irqrestore(&avd->job_lock, flags); - dev_dbg(avd->dev, "avd-job: run seq %llu: %s\n", seq, outcome); - - if (finish) - avd_job_finish(ctx, result); + mutex_lock(&avd->queue_mutex); + ret = avd_pm_irq(avd, gen); + mutex_unlock(&avd->queue_mutex); + return ret; } /* @@ -444,12 +129,7 @@ static void avd_device_run(void *priv) * userspace decode ahead to ride out bitrate spikes the hardware can't * decode in real time (e.g. 150+ Mbit/s 4K60 H.264). */ -static unsigned int max_capture_buffers = 128; -module_param(max_capture_buffers, uint, 0444); -MODULE_PARM_DESC(max_capture_buffers, - "Maximum number of capture buffers per context (default 128)"); - -static int avd_queue_init(void *priv, struct vb2_queue *src_vq, +static int avd_m2m_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq) { struct avd_ctx *ctx = priv; @@ -486,7 +166,7 @@ static int avd_queue_init(void *priv, struct vb2_queue *src_vq, dst_vq->lock = &ctx->dev->vdev_lock; dst_vq->dev = ctx->dev->v4l2_dev.dev; dst_vq->allow_cache_hints = true; - dst_vq->max_num_buffers = max(max_capture_buffers, VB2_MAX_FRAME); + dst_vq->max_num_buffers = max(128U, VB2_MAX_FRAME); return vb2_queue_init(dst_vq); } @@ -507,14 +187,15 @@ static int avd_open(struct file *filp) if (ret) goto err_free_ctx; - INIT_DELAYED_WORK(&ctx->watchdog_work, avd_watchdog_func); + INIT_LIST_HEAD(&ctx->queue_node); + ctx->next_dep = AVD_DEP_PREV_DONE; avd_reset_coded_fmt(ctx); avd_reset_decoded_fmt(ctx); v4l2_fh_init(&ctx->fh, video_devdata(filp)); - ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(avd->m2m_dev, ctx, avd_queue_init); + ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(avd->m2m_dev, ctx, avd_m2m_queue_init); if (IS_ERR(ctx->fh.m2m_ctx)) { ret = PTR_ERR(ctx->fh.m2m_ctx); goto err_free_ctx; @@ -524,6 +205,7 @@ static int avd_open(struct file *filp) if (ret) goto err_cleanup_m2m_ctx; + avd_queue_add_ctx(ctx); v4l2_fh_add(&ctx->fh, filp); return 0; @@ -542,9 +224,9 @@ static int avd_release(struct file *filp) struct avd_ctx *ctx = file_to_ctx(filp); v4l2_fh_del(&ctx->fh, filp); + avd_queue_del_ctx(ctx); v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); - /* The job has finished; its watchdog must not run on the freed ctx. */ - cancel_delayed_work_sync(&ctx->watchdog_work); + cancel_work_sync(&ctx->submit_work); v4l2_ctrl_handler_free(&ctx->ctrl_hdl); v4l2_fh_exit(&ctx->fh); avd_buf_free(ctx->dev, &ctx->inst); @@ -564,7 +246,9 @@ static const struct v4l2_file_operations avd_fops = { }; static const struct v4l2_m2m_ops avd_m2m_ops = { - .device_run = avd_device_run, + .device_run = avd_queue_run, + .job_ready = avd_queue_ready, + .job_abort = avd_queue_abort, }; static const struct media_device_ops avd_media_ops = { @@ -754,6 +438,7 @@ static const struct avd_variant avd_t8122_variant = { }; static const struct avd_variant avd_t8140_variant = { + .av1_fp_layout = AVD_AV1_FP_T8140_TODO, .vp_slots = { [AVD_CODEC_HEVC] = 2, [AVD_CODEC_H264] = 1, @@ -807,10 +492,13 @@ static const struct avd_variant avd_t8132_variant = { * T6040 (M4 Pro): the T8132 decoder. Its first HEVC job stalled the VP when * it was sent the pre-v5 header (no DMA configuration words, no reserved * scratch slot); T6040 runs the same v5 firmware generation as T8140, so it - * gets the v5 header. HEVC stays hidden from V4L2 clients until that is - * verified on the hardware. + * gets the v5 header. HEVC is advertised only when the loaded firmware + * names h265 in its CODECS trailer. */ static const struct avd_variant avd_t6040_variant = { + .av1_fp_layout = AVD_AV1_FP_BORAGE, + .apple = &avd_borage_hw, + .fp_size = 0x1130d0, .vp_slots = { [AVD_CODEC_HEVC] = 4, [AVD_CODEC_H264] = 4, @@ -819,10 +507,15 @@ static const struct avd_variant avd_t6040_variant = { }, .fifo_slots = 15, .configure_stream = t8122_configure_stream, - .capabilities = AVD_CAPABILITY_H264 | + .capabilities = AVD_CAPABILITY_HEVC | + AVD_CAPABILITY_H264 | AVD_CAPABILITY_VP9 | AVD_CAPABILITY_AV1, - .fw_name = "apple/avd-fw-v5-t1.bin", + /* HEVC only with firmware whose CODECS trailer lists h265. */ + .fw_caps_required = AVD_CAPABILITY_HEVC, + /* v5-t2 matches T6040's SRAM aperture, which the queued transport needs. */ + .fw_name = "apple/avd-fw-v5-t2.bin", + .fw_name_fallback = "apple/avd-fw-v5-t1.bin", .revision = 4, .vp_slot_offset = 0xc, .submit_offset = 0x40, @@ -870,11 +563,17 @@ static const struct of_device_id avd_of_match[] = { MODULE_DEVICE_TABLE(of, avd_of_match); +static void avd_release_open_firmware(void *data) +{ + struct avd_dev *avd = data; + + release_firmware(avd->fw); +} + static int avd_probe(struct platform_device *pdev) { struct avd_dev *avd; const struct of_device_id *match; - const char *codec_info = NULL; int ret, irq; avd = devm_kzalloc(&pdev->dev, sizeof(*avd), GFP_KERNEL); @@ -886,12 +585,17 @@ static int avd_probe(struct platform_device *pdev) avd->pdev = pdev; mutex_init(&avd->vdev_lock); - spin_lock_init(&avd->job_lock); + avd_pm_init(avd, NULL, &avd_pm_queue_ops); + ret = avd_queue_init(avd); + if (ret) + return ret; match = of_match_node(avd_of_match, pdev->dev.of_node); avd->variant = match->data; avd->rstc = devm_reset_control_get_exclusive(avd->dev, NULL); + if (IS_ERR(avd->rstc)) + return PTR_ERR(avd->rstc); avd->code = devm_platform_ioremap_resource_byname(pdev, "code"); if (IS_ERR(avd->code)) @@ -902,6 +606,9 @@ static int avd_probe(struct platform_device *pdev) avd->ctrl = devm_platform_ioremap_resource_byname(pdev, "ctrl"); if (IS_ERR(avd->ctrl)) return PTR_ERR(avd->ctrl); + ret = avd_pm_map_piodma(avd); + if (ret) + return ret; avd->domain = iommu_get_domain_for_dev(avd->dev); if (avd->domain) { @@ -912,39 +619,16 @@ static int avd_probe(struct platform_device *pdev) } } - ret = request_firmware(&avd->fw, avd->variant->fw_name, avd->dev); - - if (ret) { - dev_err(avd->dev, "failed to load firmware: %d", ret); + ret = avd_backend_select(avd); + if (ret) + return ret; + ret = devm_add_action_or_reset(avd->dev, avd_release_open_firmware, avd); + if (ret) return ret; - } - - /* Determine firmware and HW variant capabilities */ - if (avd->fw->size >= AVD_CODEC_INFO_MAX_SIZE) - codec_info = avd->fw->data + avd->fw->size - AVD_CODEC_INFO_MAX_SIZE; - if (codec_info && strncmp(codec_info, "CODECS:", 7) == 0) { - unsigned int caps = 0; - unsigned int variant_caps = avd->variant->capabilities; - - if (strnstr(codec_info, "av1", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_AV1; - if (strnstr(codec_info, "h264", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_H264; - if (strnstr(codec_info, "h265", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_HEVC; - if (strnstr(codec_info, "vp9", AVD_CODEC_INFO_MAX_SIZE)) - caps |= variant_caps & AVD_CAPABILITY_VP9; - - dev_info(avd->dev, "Firmware codecs: %.27s\n", codec_info + 7); - - avd->capabilities = caps; - } else { - /* Backward compatibility to old firmware, do not upstream. */ - avd->capabilities = avd->variant->capabilities; - } ret = dma_set_mask_and_coherent(avd->dev, - DMA_BIT_MASK((avd->variant->quirks & AVD_QUIRK_LSR) ? 38 : 64)); + DMA_BIT_MASK(avd_backend_is_apple(avd) ? 42 : + (avd->variant->quirks & AVD_QUIRK_LSR) ? 38 : 64)); if (ret) { dev_err(avd->dev, "Failed to set DMA mask"); @@ -954,17 +638,27 @@ static int avd_probe(struct platform_device *pdev) irq = platform_get_irq(pdev, 1); if (irq < 0) return irq; + avd->pm.irq = irq; ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, - avd_irq_handler, IRQF_ONESHOT, + avd_irq_handler, IRQF_ONESHOT | IRQF_NO_AUTOEN, dev_name(&pdev->dev), avd); if (ret) { dev_err(avd->dev, "Could not request IRQ 1"); return ret; } - pm_runtime_set_autosuspend_delay(avd->dev, 100); - pm_runtime_use_autosuspend(avd->dev); - pm_runtime_enable(avd->dev); + avd_pm_enable(avd); + + /* + * Boot the firmware once now: the transport and the codec list depend + * on what the loaded image reports, and clients enumerate formats and + * size their contexts before the first decode would power it up. + */ + ret = pm_runtime_resume_and_get(avd->dev); + if (ret < 0) + dev_warn(avd->dev, "firmware probe boot failed: %d\n", ret); + else + pm_runtime_put_autosuspend(avd->dev); ret = avd_v4l2_init(avd); if (ret) @@ -981,42 +675,19 @@ static int avd_probe(struct platform_device *pdev) static void avd_remove(struct platform_device *pdev) { struct avd_dev *avd = platform_get_drvdata(pdev); + int ret; - release_firmware(avd->fw); - + v4l2_m2m_suspend(avd->m2m_dev); + ret = avd->pm.queue->drain(avd); + if (ret) + dev_warn(avd->dev, "queue drain failed, resetting before teardown: %d\n", ret); + avd_queue_cleanup(avd); + avd_pm_disable(avd); avd_v4l2_cleanup(avd); - if (avd->empty_domain) iommu_domain_free(avd->empty_domain); - - pm_runtime_disable(avd->dev); - pm_runtime_dont_use_autosuspend(avd->dev); } -static __maybe_unused int avd_runtime_resume(struct device *dev) -{ - int ret; - struct avd_dev *avd = platform_get_drvdata(to_platform_device(dev)); - - ret = avd_boot(avd); - if (ret) - dev_err(dev, "failed to boot"); - return ret; -} - -static __maybe_unused int avd_runtime_suspend(struct device *dev) -{ - struct avd_dev *avd = platform_get_drvdata(to_platform_device(dev)); - - avd_shutdown(avd); - return 0; -} - -static const struct dev_pm_ops avd_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS( - pm_runtime_force_suspend, - pm_runtime_force_resume) SET_RUNTIME_PM_OPS(avd_runtime_suspend, - avd_runtime_resume, NULL) }; - static struct platform_driver avd_driver = { .probe = avd_probe, .remove = avd_remove, diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index e8bc8d0066ead..5ff0026ef3730 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -57,15 +57,18 @@ struct avd_h264_run { u32 slice_data_off; }; -/* state */ -struct avd_h264_ctx { - bool monochrome; - struct avd_h264_reflists { - struct v4l2_h264_reference p[V4L2_H264_REF_LIST_LEN]; - struct v4l2_h264_reference b0[V4L2_H264_REF_LIST_LEN]; - struct v4l2_h264_reference b1[V4L2_H264_REF_LIST_LEN]; - } reflists; +/* + * One bank per picture in flight. With the queued transport two pictures of + * a stream overlap (VP of one, PP of the other), so everything the hardware + * reads or writes while decoding a picture, and the CPU copies of its slices, + * exists once per bank. The legacy transport uses a single bank. + */ +#define AVD_H264_BANKS 2 +struct avd_bank { + bool monochrome; + bool busy; /* a picture of this bank is not retired yet */ + bool submitted; /* ... and its last slice went to the hardware */ struct avd_buf *slices; struct avd_buf *active_slice; size_t slice_num; @@ -74,6 +77,7 @@ struct avd_h264_ctx { size_t slice_bytes; struct avd_h264_bufs { + /* VP to PP instruction FIFO for pictures of this bank */ struct avd_buf inst; struct avd_buf pipe_state; struct avd_buf above_info; @@ -84,6 +88,20 @@ struct avd_h264_ctx { } bufs; }; +/* state */ +struct avd_h264_ctx { + struct avd_h264_reflists { + struct v4l2_h264_reference p[V4L2_H264_REF_LIST_LEN]; + struct v4l2_h264_reference b0[V4L2_H264_REF_LIST_LEN]; + struct v4l2_h264_reference b1[V4L2_H264_REF_LIST_LEN]; + } reflists; + + unsigned int nbanks; + struct avd_bank bank[AVD_H264_BANKS]; + /* bank of the picture being built */ + struct avd_bank *b; +}; + /* ffmpeg submits wrong timestamp, use first_mb_in_slice as a workaround */ #define is_new_frame(sl) \ (sl->first_mb_in_slice == 0) /* (ctx->fh.m2m_ctx->new_frame) */ @@ -118,7 +136,7 @@ static void stream_refs(struct avd_ctx *ctx, struct avd_h264_run *run) dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); push(0, ""); - pusha(h264_ctx->bufs.mv_above_info.addr, "mv_above_info", 7); + pusha(h264_ctx->b->bufs.mv_above_info.addr, "mv_above_info", 7); pusha(run->addresses.mv_color, "mv_color", 0); push(0, ""); @@ -272,7 +290,7 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) if (avd->variant->revision == 3) push(0, "zero"); - pusha(h264_ctx->bufs.above_info.addr, "hdr_9c_pps_tile_addr_lsb8", 0); + pusha(h264_ctx->b->bufs.above_info.addr, "hdr_9c_pps_tile_addr_lsb8", 0); push(0, ""); push(0, ""); @@ -280,11 +298,11 @@ static void stream_hdr(struct avd_ctx *ctx, struct avd_h264_run *run) if (avd->variant->revision == 3) push(0, "zero"); else if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) - pusha(h264_ctx->bufs.pipe_state.addr, "pipe_state", 0); + pusha(h264_ctx->b->bufs.pipe_state.addr, "pipe_state", 0); - pusha(h264_ctx->bufs.ip_above.addr, "ip_above", 1); - pusha(h264_ctx->bufs.lf_above.addr, "lf_above", 2); - pusha(h264_ctx->bufs.lf_above_info.addr, "lf_above_info", 3); + pusha(h264_ctx->b->bufs.ip_above.addr, "ip_above", 1); + pusha(h264_ctx->b->bufs.lf_above.addr, "lf_above", 2); + pusha(h264_ctx->b->bufs.lf_above_info.addr, "lf_above_info", 3); push(0, ""); push_comp(ctx, run->base.comp_out, ctx->comp.offsets); @@ -437,12 +455,12 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) const struct v4l2_ctrl_h264_sps *sps = run->sps; const struct v4l2_ctrl_h264_slice_params *sl = run->slice_params; struct avd_h264_ctx *h264_ctx = ctx->priv; - u32 payload_len = h264_ctx->active_slice->size; + u32 payload_len = h264_ctx->b->active_slice->size; bool en_mode = (pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE) == 0; u32 off = run->slice_data_off; - dma_addr_t slc_a84 = h264_ctx->active_slice->addr + off; + dma_addr_t slc_a84 = h264_ctx->b->active_slice->addr + off; push(AVD_OP_CODED_DATA | AVD_OP_CODED_DATA_BIT_OFF( @@ -538,10 +556,9 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) } } -static int avd_h264_alloc_bufs(struct avd_ctx *ctx) +static int avd_h264_alloc_bufs(struct avd_ctx *ctx, struct avd_bank *bank) { struct avd_dev *dev = ctx->dev; - struct avd_h264_ctx *h264_ctx = ctx->priv; int ret, w, bit_depth, mb; w = fmt_width(ctx); bit_depth = (ctx->image_fmt == AVD_IMG_FMT_420_10BIT || @@ -549,13 +566,13 @@ static int avd_h264_alloc_bufs(struct avd_ctx *ctx) 10 : 8; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.inst, fifo_size()); + ret = avd_buf_alloc(dev, &bank->bufs.inst, fifo_size()); if (ret) { return ret; } if (!(dev->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) { - ret = avd_buf_alloc(dev, &h264_ctx->bufs.pipe_state, 0x200); + ret = avd_buf_alloc(dev, &bank->bufs.pipe_state, 0x200); if (ret) { return ret; } @@ -563,24 +580,24 @@ static int avd_h264_alloc_bufs(struct avd_ctx *ctx) mb = DIV_ROUND_UP(w, 16); - ret = avd_buf_alloc(dev, &h264_ctx->bufs.above_info, mb * 20); + ret = avd_buf_alloc(dev, &bank->bufs.above_info, mb * 20); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.ip_above, bit_depth * 4 * mb); + ret = avd_buf_alloc(dev, &bank->bufs.ip_above, bit_depth * 4 * mb); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.lf_above, + ret = avd_buf_alloc(dev, &bank->bufs.lf_above, bit_depth * 4 * 4 * mb); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.lf_above_info, 32 * mb); + ret = avd_buf_alloc(dev, &bank->bufs.lf_above_info, 32 * mb); if (ret) return ret; - ret = avd_buf_alloc(dev, &h264_ctx->bufs.mv_above_info, 32 * mb); + ret = avd_buf_alloc(dev, &bank->bufs.mv_above_info, 32 * mb); if (ret) return ret; @@ -620,35 +637,46 @@ static int avd_h264_validate_sps(struct avd_ctx *ctx, return 0; } -/* Free the slices held for the current picture. */ +/* Free the slices held for one bank's picture. */ +static void avd_h264_drop_bank_slices(struct avd_ctx *ctx, struct avd_bank *bank) +{ + for (size_t i = 0; i < bank->slice_num; i++) + avd_buf_free(ctx->dev, &bank->slices[i]); + bank->slice_num = 0; + bank->slice_bytes = 0; +} + +/* Free the slices held for the picture being built. */ static void avd_h264_drop_slices(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; - for (size_t i = 0; i < h264_ctx->slice_num; i++) - avd_buf_free(ctx->dev, &h264_ctx->slices[i]); - h264_ctx->slice_num = 0; - h264_ctx->slice_bytes = 0; + avd_h264_drop_bank_slices(ctx, h264_ctx->b); } static void avd_h264_stop(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; struct avd_dev *dev = ctx->dev; + unsigned int i; + ctx->max_frames = 1; if (!h264_ctx) return; - avd_buf_free(dev, &h264_ctx->bufs.pipe_state); - avd_buf_free(dev, &h264_ctx->bufs.inst); - avd_buf_free(dev, &h264_ctx->bufs.above_info); - avd_buf_free(dev, &h264_ctx->bufs.lf_above_info); - avd_buf_free(dev, &h264_ctx->bufs.lf_above); - avd_buf_free(dev, &h264_ctx->bufs.ip_above); - avd_buf_free(dev, &h264_ctx->bufs.mv_above_info); - - avd_h264_drop_slices(ctx); - kfree(h264_ctx->slices); + for (i = 0; i < AVD_H264_BANKS; i++) { + struct avd_bank *bank = &h264_ctx->bank[i]; + + avd_buf_free(dev, &bank->bufs.pipe_state); + avd_buf_free(dev, &bank->bufs.inst); + avd_buf_free(dev, &bank->bufs.above_info); + avd_buf_free(dev, &bank->bufs.lf_above_info); + avd_buf_free(dev, &bank->bufs.lf_above); + avd_buf_free(dev, &bank->bufs.ip_above); + avd_buf_free(dev, &bank->bufs.mv_above_info); + avd_h264_drop_bank_slices(ctx, bank); + kfree(bank->slices); + } kfree(h264_ctx); ctx->priv = NULL; @@ -673,20 +701,28 @@ static int avd_h264_start(struct avd_ctx *ctx) return -ENOMEM; ctx->priv = h264_ctx; + /* Overlap pictures only where the firmware chains VP to PP itself. */ + h264_ctx->nbanks = avd_backend_is_qfw(ctx->dev) ? AVD_H264_BANKS : 1; + h264_ctx->b = &h264_ctx->bank[0]; + + for (unsigned int i = 0; i < h264_ctx->nbanks; i++) { + struct avd_bank *bank = &h264_ctx->bank[i]; + + /* assume one slice per row */ + bank->alloc_slice_num = + ctrl->p_new.p_h264_sps->pic_height_in_map_units_minus1 + 1; + bank->slices = kcalloc(bank->alloc_slice_num, + sizeof(*bank->slices), GFP_KERNEL); + if (!bank->slices) { + ret = -ENOMEM; + goto err_free_ctx; + } - /* assume one slice per row */ - h264_ctx->alloc_slice_num = - ctrl->p_new.p_h264_sps->pic_height_in_map_units_minus1 + 1; - h264_ctx->slices = kcalloc(h264_ctx->alloc_slice_num, - sizeof(*h264_ctx->slices), GFP_KERNEL); - if (!h264_ctx->slices) { - ret = -ENOMEM; - goto err_free_ctx; + ret = avd_h264_alloc_bufs(ctx, bank); + if (ret) + goto err_free_ctx; } - - ret = avd_h264_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; + ctx->max_frames = h264_ctx->nbanks; return 0; @@ -732,22 +768,22 @@ static int avd_h264_realloc_slices(struct avd_ctx *ctx, size_t max_slices) void *tmp; size_t alloc_slice_num; - tmp = h264_ctx->slices; + tmp = h264_ctx->b->slices; /* grow by half, and by at least one for a one-row picture */ - alloc_slice_num = h264_ctx->alloc_slice_num + - max_t(size_t, h264_ctx->alloc_slice_num / 2, 1); + alloc_slice_num = h264_ctx->b->alloc_slice_num + + max_t(size_t, h264_ctx->b->alloc_slice_num / 2, 1); alloc_slice_num = min(alloc_slice_num, max_slices); - h264_ctx->slices = kcalloc(alloc_slice_num, sizeof(*h264_ctx->slices), + h264_ctx->b->slices = kcalloc(alloc_slice_num, sizeof(*h264_ctx->b->slices), GFP_KERNEL); - if (!h264_ctx->slices) { + if (!h264_ctx->b->slices) { /* make deallocating a little easier */ - h264_ctx->slices = tmp; + h264_ctx->b->slices = tmp; return -ENOMEM; } - h264_ctx->alloc_slice_num = alloc_slice_num; - memcpy(h264_ctx->slices, tmp, - sizeof(*h264_ctx->slices) * h264_ctx->slice_num); + h264_ctx->b->alloc_slice_num = alloc_slice_num; + memcpy(h264_ctx->b->slices, tmp, + sizeof(*h264_ctx->b->slices) * h264_ctx->b->slice_num); kfree(tmp); /* a slice that arrives before the frame's header has no job table yet */ @@ -853,6 +889,33 @@ VISIBLE_IF_KUNIT bool avd_h264_slice_fits(const struct v4l2_ctrl_h264_sps *sps, avd_h264_slice_budget(decoded_size); } +/* + * Start a new picture in a bank no unretired picture uses, and remember the + * bank in its CAPTURE buffer so retirement frees the right one. + */ +static int avd_h264_pick_bank(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst) +{ + struct avd_h264_ctx *h264_ctx = ctx->priv; + struct avd_bank *bank = NULL; + unsigned int i; + + /* A picture whose last slice never came: start over in its bank. */ + if (h264_ctx->b->busy && !h264_ctx->b->submitted) + bank = h264_ctx->b; + for (i = 0; !bank && i < h264_ctx->nbanks; i++) + if (!h264_ctx->bank[i].busy) + bank = &h264_ctx->bank[i]; + if (!bank) + return -EBUSY; + avd_h264_drop_bank_slices(ctx, bank); + bank->busy = true; + bank->submitted = false; + h264_ctx->b = bank; + /* 0 means no bank, so a stale or missing value never frees one. */ + vb2_to_avd_decoded_buf(&dst->vb2_buf)->bank = bank - h264_ctx->bank + 1; + return 0; +} + static int avd_h264_run(struct avd_ctx *ctx) { struct avd_h264_ctx *h264_ctx = ctx->priv; @@ -862,6 +925,14 @@ static int avd_h264_run(struct avd_ctx *ctx) avd_h264_run_preamble(ctx, &run); + if (run.slice_params && is_new_frame(run.slice_params)) { + ret = avd_h264_pick_bank(ctx, run.base.bufs.dst); + if (ret) { + avd_run_postamble(ctx, &run.base); + return ret; + } + } + struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); u32 payload_len = vb2_get_plane_payload(&src->vb2_buf, 0); const u8 *data = vb2_plane_vaddr(&src->vb2_buf, 0); @@ -875,8 +946,8 @@ static int avd_h264_run(struct avd_ctx *ctx) return -EINVAL; } - if (!avd_h264_slice_fits(run.sps, h264_ctx->slice_num, - h264_ctx->slice_bytes, payload_len, + if (!avd_h264_slice_fits(run.sps, h264_ctx->b->slice_num, + h264_ctx->b->slice_bytes, payload_len, ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].sizeimage)) { dev_err_ratelimited(ctx->dev->dev, "too many slices for one picture, dropping it\n"); @@ -886,20 +957,20 @@ static int avd_h264_run(struct avd_ctx *ctx) goto err_free_jobs; } - if (h264_ctx->slice_num >= h264_ctx->alloc_slice_num) { + if (h264_ctx->b->slice_num >= h264_ctx->b->alloc_slice_num) { ret = avd_h264_realloc_slices(ctx, avd_h264_max_slices(run.sps)); if (ret) goto err_free_jobs; } - h264_ctx->active_slice = &h264_ctx->slices[h264_ctx->slice_num]; - ret = avd_buf_alloc(ctx->dev, h264_ctx->active_slice, payload_len); + h264_ctx->b->active_slice = &h264_ctx->b->slices[h264_ctx->b->slice_num]; + ret = avd_buf_alloc(ctx->dev, h264_ctx->b->active_slice, payload_len); if (ret) goto err_free_jobs; - memcpy(h264_ctx->active_slice->cpu, data, payload_len); - h264_ctx->slice_num++; - h264_ctx->slice_bytes += PAGE_ALIGN((size_t)payload_len); - h264_ctx->monochrome = run.sps->chroma_format_idc == 0; + memcpy(h264_ctx->b->active_slice->cpu, data, payload_len); + h264_ctx->b->slice_num++; + h264_ctx->b->slice_bytes += PAGE_ALIGN((size_t)payload_len); + h264_ctx->b->monochrome = run.sps->chroma_format_idc == 0; /* Build the P/B{0,1} ref lists. */ v4l2_h264_init_reflist_builder(&reflist_builder, run.decode_params, @@ -923,7 +994,7 @@ static int avd_h264_run(struct avd_ctx *ctx) if (is_new_frame(run.slice_params)) { /* the header segment plus one per slice */ ret = avd_init_job(ctx, AVD_CODEC_H264, - h264_ctx->alloc_slice_num + 1); + h264_ctx->b->alloc_slice_num + 1); if (ret) return ret; stream_hdr(ctx, &run); @@ -940,6 +1011,8 @@ static int avd_h264_run(struct avd_ctx *ctx) return 0; } + ctx->job_fifo = h264_ctx->b->bufs.inst.addr; + h264_ctx->b->submitted = true; return avd_submit_job(ctx); err_free_jobs: @@ -954,22 +1027,32 @@ static void avd_h264_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, enum vb2_buffer_state result) { struct avd_h264_ctx *h264_ctx = ctx->priv; + struct avd_decoded_buffer *buf; + struct avd_bank *bank; - if (!(src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF)) { - avd_h264_drop_slices(ctx); + if (src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF || !dst_buf) + return; - /* 4:0:0 has no chroma to decode; the picture is grey */ - if (h264_ctx->monochrome && dst_buf && result == VB2_BUF_STATE_DONE) { - const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; - struct avd_decoded_buffer *dst = - vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); - - /* vb2 must return cache ownership before the CPU writes. */ - dst->grey_chroma_offset = - (size_t)pix->plane_fmt[0].bytesperline * pix->height; - dst->grey_chroma_10bit = - ctx->image_fmt == AVD_IMG_FMT_420_10BIT; - } + buf = vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + if (!buf->bank || buf->bank > h264_ctx->nbanks) + return; + bank = &h264_ctx->bank[buf->bank - 1]; + buf->bank = 0; + avd_h264_drop_bank_slices(ctx, bank); + bank->busy = false; + bank->submitted = false; + + /* 4:0:0 has no chroma to decode; the picture is grey */ + if (bank->monochrome && result == VB2_BUF_STATE_DONE) { + const struct v4l2_pix_format_mplane *pix = &ctx->decoded_fmt.fmt.pix_mp; + struct avd_decoded_buffer *dst = + vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + + /* vb2 must return cache ownership before the CPU writes. */ + dst->grey_chroma_offset = + (size_t)pix->plane_fmt[0].bytesperline * pix->height; + dst->grey_chroma_10bit = + ctx->image_fmt == AVD_IMG_FMT_420_10BIT; } } diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index 9dfa20ab71913..b2ce0f3711390 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -79,10 +79,14 @@ struct avd_hevc_run { struct avd_hevc_tile_info tile_info; }; -struct avd_hevc_ctx { - struct v4l2_ctrl_hevc_scaling_matrix scaling_matrix_cache; +/* See avd-h264.c: one bank per picture in flight on the queued transport. */ +#define AVD_HEVC_BANKS 2 +struct avd_hevc_bank { + bool busy; struct avd_h264_bufs { + /* VP to PP instruction FIFO for pictures of this bank */ + struct avd_buf inst; struct avd_buf mv_above_info; struct avd_buf az_above; struct avd_buf ip_above; @@ -91,9 +95,17 @@ struct avd_hevc_ctx { struct avd_buf lf_left; struct avd_buf lf_left_info; struct avd_buf sw_left; - struct avd_buf inst; struct avd_buf pipe_state; } bufs; +}; + +struct avd_hevc_ctx { + struct v4l2_ctrl_hevc_scaling_matrix scaling_matrix_cache; + + unsigned int nbanks; + struct avd_hevc_bank bank[AVD_HEVC_BANKS]; + /* bank of the picture being built */ + struct avd_hevc_bank *b; int submit_num; }; @@ -110,7 +122,7 @@ static void stream_refs(struct avd_ctx *ctx, struct avd_hevc_run *run) push(ctx->dev->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER ? HEVC_DMA_CONFIG : 0, "reference_dma_config"); - pusha(hevc_ctx->bufs.mv_above_info.addr, "mv_above_info", 7); + pusha(hevc_ctx->b->bufs.mv_above_info.addr, "mv_above_info", 7); pusha(run->addresses.mv_color, "mv_color", 0); push(0, ""); @@ -357,30 +369,30 @@ static void set_header(struct avd_ctx *ctx, struct avd_hevc_run *run) if (avd->variant->revision == 3) push(0, "zero"); else if (!(avd->variant->quirks & AVD_QUIRK_NO_PIPE_STATE)) - pusha(hevc_ctx->bufs.pipe_state.addr, "pipe_state", 0); + pusha(hevc_ctx->b->bufs.pipe_state.addr, "pipe_state", 0); - pusha(hevc_ctx->bufs.ip_above.addr, "scratch_addr", 0); - pusha(hevc_ctx->bufs.lf_above.addr, "scratch_addr", 1); - pusha(hevc_ctx->bufs.lf_above_info.addr, "scratch_addr", 2); + pusha(hevc_ctx->b->bufs.ip_above.addr, "scratch_addr", 0); + pusha(hevc_ctx->b->bufs.lf_above.addr, "scratch_addr", 1); + pusha(hevc_ctx->b->bufs.lf_above_info.addr, "scratch_addr", 2); if ((pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) || !(avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER)) { - pusha(hevc_ctx->bufs.lf_left.addr, + pusha(hevc_ctx->b->bufs.lf_left.addr, "scratch_addr", 3); - pusha(hevc_ctx->bufs.lf_left_info.addr, + pusha(hevc_ctx->b->bufs.lf_left_info.addr, "scratch_addr", 4); if (avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER) pusha(0, "reserved_scratch", 0); - pusha(hevc_ctx->bufs.az_above.addr, + pusha(hevc_ctx->b->bufs.az_above.addr, "scratch_addr", 8); - pusha(hevc_ctx->bufs.sw_left.addr, + pusha(hevc_ctx->b->bufs.sw_left.addr, "scratch_addr", 9); } else { pusha(0, "", 3); pusha(0, "", 4); if (avd->variant->quirks & AVD_QUIRK_HEVC_V5_HEADER) pusha(0, "reserved_scratch", 0); - pusha(hevc_ctx->bufs.az_above.addr, + pusha(hevc_ctx->b->bufs.az_above.addr, "scratch_addr", 8); pusha(0, "", 9); } @@ -1056,17 +1068,16 @@ static int avd_hevc_validate_sps(struct avd_ctx *ctx, return 0; } -static int avd_hevc_alloc_bufs(struct avd_ctx *ctx) +static int avd_hevc_alloc_bufs(struct avd_ctx *ctx, struct avd_hevc_bank *bank) { struct avd_dev *avd = ctx->dev; - struct avd_hevc_ctx *hevc_ctx = ctx->priv; int ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.inst, fifo_size()); + ret = avd_buf_alloc(avd, &bank->bufs.inst, fifo_size()); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.pipe_state, 0x200); + ret = avd_buf_alloc(avd, &bank->bufs.pipe_state, 0x200); if (ret) return ret; @@ -1077,20 +1088,26 @@ static void avd_hevc_stop(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; struct avd_dev *avd = ctx->dev; + unsigned int i; + ctx->max_frames = 1; if (!hevc_ctx) return; - avd_buf_free(avd, &hevc_ctx->bufs.pipe_state); - avd_buf_free(avd, &hevc_ctx->bufs.inst); - avd_buf_free(avd, &hevc_ctx->bufs.mv_above_info); - avd_buf_free(avd, &hevc_ctx->bufs.az_above); - avd_buf_free(avd, &hevc_ctx->bufs.ip_above); - avd_buf_free(avd, &hevc_ctx->bufs.lf_above); - avd_buf_free(avd, &hevc_ctx->bufs.lf_above_info); - avd_buf_free(avd, &hevc_ctx->bufs.lf_left); - avd_buf_free(avd, &hevc_ctx->bufs.lf_left_info); - avd_buf_free(avd, &hevc_ctx->bufs.sw_left); + for (i = 0; i < AVD_HEVC_BANKS; i++) { + struct avd_h264_bufs *bufs = &hevc_ctx->bank[i].bufs; + + avd_buf_free(avd, &bufs->pipe_state); + avd_buf_free(avd, &bufs->inst); + avd_buf_free(avd, &bufs->mv_above_info); + avd_buf_free(avd, &bufs->az_above); + avd_buf_free(avd, &bufs->ip_above); + avd_buf_free(avd, &bufs->lf_above); + avd_buf_free(avd, &bufs->lf_above_info); + avd_buf_free(avd, &bufs->lf_left); + avd_buf_free(avd, &bufs->lf_left_info); + avd_buf_free(avd, &bufs->sw_left); + } kfree(hevc_ctx); ctx->priv = NULL; @@ -1099,6 +1116,7 @@ static void avd_hevc_stop(struct avd_ctx *ctx) static int avd_hevc_start(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx; + unsigned int i; int ret; hevc_ctx = kzalloc(sizeof(*hevc_ctx), GFP_KERNEL); @@ -1106,9 +1124,15 @@ static int avd_hevc_start(struct avd_ctx *ctx) return -ENOMEM; ctx->priv = hevc_ctx; - ret = avd_hevc_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; + /* Overlap pictures only where the firmware chains VP to PP itself. */ + hevc_ctx->nbanks = avd_backend_is_qfw(ctx->dev) ? AVD_HEVC_BANKS : 1; + hevc_ctx->b = &hevc_ctx->bank[0]; + for (i = 0; i < hevc_ctx->nbanks; i++) { + ret = avd_hevc_alloc_bufs(ctx, &hevc_ctx->bank[i]); + if (ret) + goto err_free_ctx; + } + ctx->max_frames = hevc_ctx->nbanks; return 0; @@ -1117,6 +1141,38 @@ static int avd_hevc_start(struct avd_ctx *ctx) return ret; } +/* Take a bank no unretired picture uses; record it in the CAPTURE buffer. */ +static int avd_hevc_pick_bank(struct avd_ctx *ctx, struct avd_decoded_buffer *dst) +{ + struct avd_hevc_ctx *hevc_ctx = ctx->priv; + unsigned int i; + + for (i = 0; i < hevc_ctx->nbanks; i++) { + if (hevc_ctx->bank[i].busy) + continue; + hevc_ctx->bank[i].busy = true; + hevc_ctx->b = &hevc_ctx->bank[i]; + dst->bank = i + 1; + return 0; + } + return -EBUSY; +} + +static void avd_hevc_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src_buf, + struct vb2_v4l2_buffer *dst_buf, + enum vb2_buffer_state result) +{ + struct avd_hevc_ctx *hevc_ctx = ctx->priv; + struct avd_decoded_buffer *buf; + + if (!dst_buf || !hevc_ctx) + return; + buf = vb2_to_avd_decoded_buf(&dst_buf->vb2_buf); + if (buf->bank && buf->bank <= hevc_ctx->nbanks) + hevc_ctx->bank[buf->bank - 1].busy = false; + buf->bank = 0; +} + static int avd_hevc_alloc_scratch(struct avd_ctx *ctx, struct avd_hevc_run *run) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; @@ -1152,51 +1208,51 @@ static int avd_hevc_alloc_scratch(struct avd_ctx *ctx, struct avd_hevc_run *run) tile_info->row_height[i] : max_row; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.ip_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.ip_above, ((max_col * max_cu_width) / 4) * bit_depth); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.mv_above_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.mv_above_info, ((max_col * max_cu_width) / 16) * 20); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_above, DIV_ROUND_UP(w, 16) * bit_depth * (10 + 6) + (cols - 1) * 256); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_above_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_above_info, DIV_ROUND_UP(w + 7, 16) * 36 + cols * 128); if (ret) return ret; if (pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) { - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_left, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_left, ((max_row * max_cu_width) / 4) * 36 + 144 /* why? */); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.lf_left_info, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.lf_left_info, ((max_row * max_cu_width) / 4) * 9); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.az_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.az_above, /* not sure if its cols or rows */ DIV_ROUND_UP(w, 64) * 144 + (cols - 1) * 128); if (ret) return ret; - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.sw_left, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.sw_left, ((max_row * max_cu_width) / 4) * 216); if (ret) return ret; } else { - ret = avd_buf_alloc(avd, &hevc_ctx->bufs.az_above, + ret = avd_buf_alloc(avd, &hevc_ctx->b->bufs.az_above, DIV_ROUND_UP(w + 7, 16) * 4 * bit_depth); if (ret) return ret; @@ -1282,6 +1338,51 @@ static int avd_hevc_compute_tiles(struct avd_ctx *ctx, struct avd_hevc_run *run) return 0; } +/* + * The slice builder below expects the GStreamer layout: the slices tile the + * OUTPUT buffer as whole NALs, bit_size covers the whole NAL and + * data_byte_offset is the slice data's offset inside it. FFmpeg's request + * API backend instead gives each slice's data_byte_offset from the start of + * the buffer and a bit_size of the slice data only, so every slice was given + * a wrong address and size and the VP raised its error interrupt on every + * frame. Detect the absolute layout (complete and unambiguous) and rewrite + * the run's private copy into the equivalent NAL-relative description. + */ +static void avd_hevc_normalize_slices(struct avd_hevc_run *run, u32 payload) +{ + struct v4l2_ctrl_hevc_slice_params *sl = + (struct v4l2_ctrl_hevc_slice_params *)run->sl; + u64 nal_end = 0, data_end = 0, prev_end = 0; + bool nal = true, absolute = true; + int i; + + if (!run->num_slices || !payload) + return; + for (i = 0; i < run->num_slices; i++) { + u64 bytes = DIV_ROUND_UP((u64)sl[i].bit_size, 8); + u64 start = sl[i].data_byte_offset; + + if (!bytes || start >= bytes || nal_end + bytes > payload) + nal = false; + nal_end += bytes; + if (!bytes || start < data_end || start + bytes > payload) + absolute = false; + data_end = start + bytes; + } + nal &= nal_end == payload; + absolute &= data_end == payload; + if (!absolute || nal) + return; + for (i = 0; i < run->num_slices; i++) { + u64 bytes = DIV_ROUND_UP((u64)sl[i].bit_size, 8); + u64 start = sl[i].data_byte_offset; + + sl[i].data_byte_offset = start - prev_end; + sl[i].bit_size = (start - prev_end + bytes) * 8; + prev_end = start + bytes; + } +} + static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) { struct v4l2_ctrl *ctrl, *sl, *ep; @@ -1345,6 +1446,7 @@ static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) static int avd_hevc_run(struct avd_ctx *ctx) { + struct avd_hevc_ctx *hevc_ctx = ctx->priv; struct avd_hevc_run run = {}; struct avd_decoded_buffer *dst; int ret; @@ -1353,9 +1455,15 @@ static int avd_hevc_run(struct avd_ctx *ctx) if (ret) return ret; + avd_hevc_normalize_slices(&run, + vb2_get_plane_payload(&run.base.bufs.src->vb2_buf, 0)); dst = vb2_to_avd_decoded_buf(&run.base.bufs.dst->vb2_buf); update_dec_buf_info(dst, &run.sl[0]); + ret = avd_hevc_pick_bank(ctx, dst); + if (ret) + goto complete; + ret = avd_hevc_compute_tiles(ctx, &run); if (ret) goto complete; @@ -1372,8 +1480,10 @@ static int avd_hevc_run(struct avd_ctx *ctx) set_header(ctx, &run); ret = stream_slices(ctx, &run); avd_run_postamble(ctx, &run.base); - if (!ret) + if (!ret) { + ctx->job_fifo = hevc_ctx->b->bufs.inst.addr; ret = avd_submit_job(ctx); + } goto done; complete: @@ -1395,6 +1505,13 @@ static int avd_hevc_try_ctrl(struct avd_ctx *ctx, struct v4l2_ctrl *ctrl) return 0; } +static unsigned int avd_hevc_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_hevc_ctx *priv = ctx->priv; + + return max_t(int, priv->submit_num, 1); +} + static void avd_hevc_submit(struct avd_ctx *ctx) { struct avd_hevc_ctx *hevc_ctx = ctx->priv; @@ -1420,7 +1537,9 @@ const struct avd_coded_fmt_ops avd_hevc_fmt_ops = { .start = avd_hevc_start, .stop = avd_hevc_stop, .run = avd_hevc_run, + .done = avd_hevc_done, .submit = avd_hevc_submit, + .pp_kicks = avd_hevc_pp_kicks, .try_ctrl = avd_hevc_try_ctrl, .get_image_fmt = avd_hevc_get_image_fmt, }; diff --git a/drivers/media/platform/apple/avd/avd-hw.c b/drivers/media/platform/apple/avd/avd-hw.c index a3987ac4510de..b64c0bf4d78ea 100644 --- a/drivers/media/platform/apple/avd/avd-hw.c +++ b/drivers/media/platform/apple/avd/avd-hw.c @@ -42,9 +42,12 @@ int avd_boot(struct avd_dev *avd) { + struct resource *code = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "code"); u32 val; int ret; + if (!avd->fw || !avd->fw->size || !code || avd->fw->size > resource_size(code)) + return -EINVAL; if (avd->variant->revision != 3) dev_info_once(avd->dev, "booting hw version: %04x", readl_relaxed(avd->ctrl)); @@ -64,6 +67,28 @@ int avd_boot(struct avd_dev *avd) return 0; } +/* + * Pop one firmware message. Reading RETRIEVE on an empty mailbox returns the + * previous message again, and a repeated PP DONE completes the next frame + * before its PP has run. An interrupt can also come just before its message + * is readable, so allow 10 us; after that treat the interrupt as spurious. + * Always acknowledges the interrupt. Returns false when there is no message. + */ +bool avd_mbox_pop(struct avd_dev *avd, u32 *msg) +{ + u32 st; + int ret; + + ret = readl_poll_timeout_atomic(avd->mbox + AVD_REG_MBOX1_STATUS, st, + !(st & AVD_MBOX1_STATUS_EMPTY), 1, 10); + if (!ret) + *msg = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); + writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); + if (ret) + dev_dbg_ratelimited(avd->dev, "spurious interrupt, mailbox empty\n"); + return !ret; +} + void avd_shutdown(struct avd_dev *avd) { writel_relaxed(AVD_RUN_CTRL_UNK_STOP, avd->mbox + AVD_REG_RUN_CTRL); diff --git a/drivers/media/platform/apple/avd/avd-inst.h b/drivers/media/platform/apple/avd/avd-inst.h index e9b9ef908b8d5..dcdb205d79315 100644 --- a/drivers/media/platform/apple/avd/avd-inst.h +++ b/drivers/media/platform/apple/avd/avd-inst.h @@ -3,6 +3,7 @@ #ifndef AVD_INST_H_ #define AVD_INST_H_ /* instruction stream things */ +#include #include #include "avd.h" diff --git a/drivers/media/platform/apple/avd/avd-pm-test.c b/drivers/media/platform/apple/avd/avd-pm-test.c new file mode 100644 index 0000000000000..6756d036b3f76 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm-test.c @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include + +#include "avd.h" +#include "avd-regs.h" + +struct avd_pm_test { + struct avd_dev avd; + bool queue_initialized; + struct avd_deferred_unmap unmap; + unsigned int draining; + unsigned int step; + unsigned int unmap_step; + unsigned int boot_step; + unsigned int resubmit_step; + unsigned int shutdowns; + unsigned int submissions; + int boot_error; + int suspend_error; + bool unmapped_active; +}; + +static struct avd_pm_test *avd_pm_fixture(struct avd_dev *avd) +{ + return container_of(avd, struct avd_pm_test, avd); +} + +static bool avd_pm_test_idle(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + return avd_pm_can_suspend(fixture->avd.frames, fixture->draining); +} + +static int avd_pm_test_drain(struct avd_dev *avd) +{ + return avd_pm_test_idle(avd) ? 0 : -EBUSY; +} + +static int avd_pm_test_boot(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->boot_step = ++fixture->step; + return fixture->boot_error; +} + +static void avd_pm_test_shutdown(struct avd_dev *avd) +{ + avd_pm_fixture(avd)->shutdowns++; +} + +static int avd_pm_test_suspend(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + if (fixture->suspend_error) + return fixture->suspend_error; + return avd_pm_backend_suspend(avd); +} + +static int avd_pm_test_submit(struct avd_dev *avd, struct avd_frame *frame) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->resubmit_step = ++fixture->step; + fixture->submissions++; + frame->state = AVD_FRAME_SUBMITTED; + return 0; +} + +static void avd_pm_test_resubmit(struct avd_dev *avd) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + unsigned int i; + + fixture->resubmit_step = ++fixture->step; + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (fixture->avd.frames[i].state == AVD_FRAME_RESERVED) + avd->backend->submit(avd, &fixture->avd.frames[i]); +} + +static void avd_pm_test_unmap(struct avd_dev *avd, + struct avd_deferred_unmap *unmap) +{ + struct avd_pm_test *fixture = avd_pm_fixture(avd); + + fixture->unmap_step = ++fixture->step; + fixture->unmapped_active = avd->pm.dart_active; +} + +static const struct avd_queue_pm_ops avd_pm_test_queue = { + .idle = avd_pm_test_idle, + .drain = avd_pm_test_drain, + .resubmit = avd_pm_test_resubmit, +}; + +static const struct avd_backend_ops avd_pm_test_backend = { + .name = "test", + .boot = avd_pm_test_boot, + .shutdown = avd_pm_test_shutdown, + .submit = avd_pm_test_submit, + .suspend = avd_pm_test_suspend, + .resume = avd_pm_backend_resume, +}; + +static int avd_pm_test_init(struct kunit *test) +{ + struct avd_pm_test *fixture; + + fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + if (!fixture) + return -ENOMEM; + test->priv = fixture; + avd_pm_init(&fixture->avd, &avd_pm_test_backend, &avd_pm_test_queue); + INIT_LIST_HEAD(&fixture->unmap.node); + fixture->unmap.unmap = avd_pm_test_unmap; + return 0; +} + +static void avd_pm_suspend_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + unsigned int i, state; + + KUNIT_EXPECT_TRUE(test, avd_pm_test_idle(&fixture->avd)); + fixture->draining = 1; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), -EBUSY); + fixture->draining = 0; + + /* An occupied slot anywhere vetoes idle, even after completion IRQ. */ + for (i = 0; i < AVD_MAX_FRAMES; i++) { + for (state = AVD_FRAME_RESERVED; state <= AVD_FRAME_DONE; state++) { + fixture->avd.frames[i].state = state; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), -EBUSY); + } + fixture->avd.frames[i].state = AVD_FRAME_FREE; + } + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); + KUNIT_EXPECT_EQ(test, avd_pm_power_off(&fixture->avd), 0); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); +} + +static void avd_pm_resume_order_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_EQ(test, fixture->unmap_step, 0U); + fixture->avd.frames[0].state = AVD_FRAME_RESERVED; + fixture->avd.frames[1].state = AVD_FRAME_SUBMITTED; + fixture->avd.frames[2].state = AVD_FRAME_DONE; + fixture->avd.frames[15].state = AVD_FRAME_RESERVED; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_TRUE(test, fixture->unmapped_active); + KUNIT_EXPECT_LT(test, fixture->unmap_step, fixture->boot_step); + KUNIT_EXPECT_LT(test, fixture->boot_step, fixture->resubmit_step); + KUNIT_EXPECT_EQ(test, fixture->submissions, 2U); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[0].state, AVD_FRAME_SUBMITTED); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[2].state, AVD_FRAME_DONE); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); +} + +static void avd_pm_resume_failure_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + fixture->avd.frames[0].state = AVD_FRAME_RESERVED; + fixture->boot_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_power_on(avd), -ETIMEDOUT); + KUNIT_EXPECT_FALSE(test, avd->pm.dart_active); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); + KUNIT_EXPECT_EQ(test, fixture->avd.frames[0].state, AVD_FRAME_RESERVED); + + /* A later successful resume retries the still-unposted frame once. */ + fixture->boot_error = 0; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_EQ(test, fixture->submissions, 1U); +} + +static void avd_pm_suspend_failure_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + fixture->suspend_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, avd->pm.dart_active); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); +} + +static void avd_pm_deferred_unmap_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + unsigned int step; + + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_TRUE(test, fixture->unmapped_active); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); + KUNIT_ASSERT_EQ(test, avd_pm_power_off(avd), 0); + step = fixture->unmap_step; + avd_pm_unmap(avd, &fixture->unmap); + KUNIT_EXPECT_EQ(test, fixture->unmap_step, step); + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + KUNIT_EXPECT_GT(test, fixture->unmap_step, step); + KUNIT_EXPECT_TRUE(test, list_empty(&avd->pm.deferred_unmaps)); +} + +static void avd_pm_real_queue_resume_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct avd_ctx *ctx; + unsigned int i; + + avd->dev = kunit_device_register(test, "avd-pm-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, avd->dev); + KUNIT_ASSERT_EQ(test, avd_queue_init(avd), 0); + fixture->queue_initialized = true; + avd->backend = &avd_pm_test_backend; + avd->pm.queue = &avd_pm_queue_ops; + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->dev = avd; + for (i = 0; i < 3; i++) { + avd->frames[i].ctx = ctx; + avd->frames[i].seq = i + 1; + avd->frames[i].deadline = jiffies + msecs_to_jiffies(60000); + avd->frames[i].state = AVD_FRAME_RESERVED; + } + /* This one was posted before the power transition; never replay it. */ + avd->frames[1].state = AVD_FRAME_SUBMITTED; + avd->inflight = 3; + KUNIT_ASSERT_EQ(test, avd_pm_power_on(avd), 0); + flush_delayed_work(&avd->queue_work); + cancel_delayed_work_sync(&avd->queue_work); + KUNIT_EXPECT_EQ(test, fixture->submissions, 2U); + KUNIT_EXPECT_LT(test, fixture->boot_step, fixture->resubmit_step); + KUNIT_EXPECT_EQ(test, avd->frames[0].state, AVD_FRAME_SUBMITTED); + KUNIT_EXPECT_EQ(test, avd->frames[2].state, AVD_FRAME_SUBMITTED); +} + +static void avd_pm_real_queue_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + spin_lock_init(&avd->queue_lock); + avd->pm.queue = &avd_pm_queue_ops; + avd_pm_drain_begin(avd); + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -EBUSY); + avd_pm_drain_end(avd); + avd->frames[15].state = AVD_FRAME_DONE; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), -EBUSY); + avd->frames[15].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_pm_power_off(avd), 0); +} + +static void avd_pm_open_boot_replay_test(struct kunit *test) +{ + static const struct avd_variant variant = { .revision = 3 }; + static const u8 image[] = { 0x12, 0x34, 0x56, 0x78 }; + static const struct firmware fw = { .data = image, .size = sizeof(image) }; + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct resource code = { + .name = "code", .flags = IORESOURCE_MEM, + .start = 0x1000, .end = 0x10ff, + }; + struct platform_device *pdev; + u8 observed[sizeof(image)]; + unsigned int i; + + pdev = kunit_kzalloc(test, sizeof(*pdev), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pdev); + pdev->resource = &code; + pdev->num_resources = 1; + avd->pdev = pdev; + avd->variant = &variant; + avd->fw = &fw; + avd->backend = &avd_open_backend; + /* Ordinary test RAM, never a mapped target aperture. */ + avd->code = (__force void __iomem *)kunit_kzalloc(test, 0x100, GFP_KERNEL); + avd->mbox = (__force void __iomem *)kunit_kzalloc(test, 0x100, GFP_KERNEL); + avd->pm.piodma = (__force void __iomem *)kunit_kzalloc(test, 0x28, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, avd->code); + KUNIT_ASSERT_NOT_NULL(test, avd->mbox); + KUNIT_ASSERT_NOT_NULL(test, avd->pm.piodma); + avd->pm.sram_phys = 0x123456780ULL; + for (i = 0; i < 2; i++) { + /* Model loss of both ITCM and PIODMA configuration between boots. */ + memset_io(avd->code, 0, sizeof(image)); + writel(0, avd->pm.piodma + 0x24); + /* The fixture supplies firmware readiness; no firmware is executed. */ + writel(1, avd->mbox + AVD_REG_FLAG0_SET); + KUNIT_ASSERT_EQ(test, avd_backend_boot(avd), 0); + KUNIT_EXPECT_EQ(test, readl(avd->pm.piodma + 0x24), 0x12345678U); + memcpy_fromio(observed, avd->code, sizeof(observed)); + KUNIT_EXPECT_MEMEQ(test, observed, image, sizeof(image)); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_RUN_CTRL), + (u32)AVD_RUN_CTRL_UNK_RUN); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX1_STATUS), + (u32)AVD_MBOX_ENABLE); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX_IRQ_ENABLE), + (u32)AVD_MBOX1_NOT_EMPTY); + avd_shutdown(avd); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_RUN_CTRL), + (u32)(AVD_RUN_CTRL_UNK_STOP)); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_FLAG0_CLR), 1U); + KUNIT_EXPECT_EQ(test, readl(avd->mbox + AVD_REG_MBOX_IRQ_ENABLE), 0U); + } +} + +static void avd_pm_recovery_dart_gate_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + struct iommu_domain domain = {}; + + /* A stopped device cannot be reset through powered-off registers. */ + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 0U); + avd->pm.dart_active = true; + avd->domain = &domain; + /* No empty domain means no valid DART flush: never reboot anyway. */ + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -ENODEV); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 1U); + KUNIT_EXPECT_EQ(test, fixture->boot_step, 0U); +} + +static void avd_pm_recovery_boot_test(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + struct avd_dev *avd = &fixture->avd; + + avd->pm.dart_active = true; + fixture->boot_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, fixture->shutdowns, 2U); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); + fixture->boot_error = 0; + KUNIT_EXPECT_EQ(test, avd_pm_recover(avd), 0); + KUNIT_EXPECT_EQ(test, avd->fw_generation, 1ULL); + /* Recovery must fail old slots before core reopens the host IRQ. */ + KUNIT_EXPECT_FALSE(test, avd->pm.irq_enabled); + KUNIT_EXPECT_EQ(test, fixture->resubmit_step, 0U); +} + +static void avd_pm_test_exit(struct kunit *test) +{ + struct avd_pm_test *fixture = test->priv; + + if (fixture->queue_initialized) + cancel_delayed_work_sync(&fixture->avd.queue_work); +} + +static struct kunit_case avd_pm_test_cases[] = { + KUNIT_CASE(avd_pm_suspend_gate_test), + KUNIT_CASE(avd_pm_resume_order_test), + KUNIT_CASE(avd_pm_resume_failure_test), + KUNIT_CASE(avd_pm_suspend_failure_test), + KUNIT_CASE(avd_pm_deferred_unmap_test), + KUNIT_CASE(avd_pm_real_queue_resume_test), + KUNIT_CASE(avd_pm_real_queue_gate_test), + KUNIT_CASE(avd_pm_open_boot_replay_test), + KUNIT_CASE(avd_pm_recovery_dart_gate_test), + KUNIT_CASE(avd_pm_recovery_boot_test), + {} +}; + +static struct kunit_suite avd_pm_test_suite = { + .name = "apple-avd-pm", + .init = avd_pm_test_init, + .exit = avd_pm_test_exit, + .test_cases = avd_pm_test_cases, +}; + +kunit_test_suite(avd_pm_test_suite); diff --git a/drivers/media/platform/apple/avd/avd-pm.c b/drivers/media/platform/apple/avd/avd-pm.c new file mode 100644 index 0000000000000..3876aec76f2ee --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.c @@ -0,0 +1,450 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include +#include + +#include "avd.h" + +/* [C] Keep Linux's existing default until the idle-timer capture is available. */ +#define AVD_AUTOSUSPEND_MS 100 +#define AVD_DRAIN_TIMEOUT_MS 5000 +#define AVD_PIODMA_SRAM_BASE 0x24 + +void avd_pm_enable_irq(struct avd_dev *avd) +{ + unsigned long flags; + + if (avd->pm.irq >= 0 && !avd->pm.irq_enabled) { + spin_lock_irqsave(&avd->pm.irq_lock, flags); + avd->pm.hw_on = true; + spin_unlock_irqrestore(&avd->pm.irq_lock, flags); + enable_irq(avd->pm.irq); + avd->pm.irq_enabled = true; + } +} + +/* + * Callers may hold queue_mutex, which the threaded handler takes first, so + * never wait for the handler here (disable_irq() would deadlock). Instead + * close hardware access under irq_lock: a handler already touching the + * decoder finishes before this returns, and any later one sees hw_on clear + * or a new irq_gen and leaves the hardware alone. + */ +static void avd_pm_disable_irq(struct avd_dev *avd) +{ + unsigned long flags; + + if (avd->pm.irq >= 0 && avd->pm.irq_enabled) { + spin_lock_irqsave(&avd->pm.irq_lock, flags); + avd->pm.hw_on = false; + avd->pm.irq_gen++; + spin_unlock_irqrestore(&avd->pm.irq_lock, flags); + disable_irq_nosync(avd->pm.irq); + avd->pm.irq_enabled = false; + } +} + +/* Run the backend's interrupt handler only against a live firmware instance. */ +irqreturn_t avd_pm_irq(struct avd_dev *avd, unsigned int gen) +{ + irqreturn_t ret = IRQ_HANDLED; + unsigned long flags; + + spin_lock_irqsave(&avd->pm.irq_lock, flags); + if (avd->pm.hw_on && gen == avd->pm.irq_gen) + ret = avd->backend->irq(avd); + spin_unlock_irqrestore(&avd->pm.irq_lock, flags); + return ret; +} + +struct avd_dma_release { + struct avd_deferred_unmap unmap; + void *cpu; + dma_addr_t addr; + size_t size; +}; + +static void avd_dma_unmap(struct avd_dev *avd, + struct avd_deferred_unmap *unmap) +{ + struct avd_dma_release *release; + + release = container_of(unmap, struct avd_dma_release, unmap); + dma_free_coherent(avd->dev, release->size, release->cpu, release->addr); + kfree(release); +} + +int avd_buf_alloc(struct avd_dev *avd, struct avd_buf *buf, size_t size) +{ + struct avd_dma_release *release; + + avd_buf_free(avd, buf); + if (!size) + return -ENOMEM; + + /* Allocate the release record now: freeing must never need to allocate. */ + release = kzalloc_obj(*release); + if (!release) + return -ENOMEM; + INIT_LIST_HEAD(&release->unmap.node); + release->unmap.unmap = avd_dma_unmap; + release->size = size; + release->cpu = dma_alloc_coherent(avd->dev, size, &release->addr, GFP_KERNEL); + if (!release->cpu) { + kfree(release); + return -ENOMEM; + } + + /* Payload length, not an allocation rounding, is used by codec packers. */ + buf->cpu = release->cpu; + buf->addr = release->addr; + buf->size = size; + buf->release = release; + return 0; +} + +void avd_buf_free(struct avd_dev *avd, struct avd_buf *buf) +{ + if (buf->release) + avd_pm_unmap(avd, &buf->release->unmap); + /* The record outlives the context when unmapping has to be deferred. */ + memset(buf, 0, sizeof(*buf)); +} + +bool avd_pm_can_suspend(const struct avd_frame *frames, unsigned int draining) +{ + unsigned int i; + + if (draining) + return false; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (frames[i].state != AVD_FRAME_FREE) + return false; + + return true; +} + +static bool avd_pm_queue_idle(struct avd_dev *avd) +{ + unsigned long flags; + bool idle; + + spin_lock_irqsave(&avd->queue_lock, flags); + idle = avd_pm_can_suspend(avd->frames, avd->pm.draining); + spin_unlock_irqrestore(&avd->queue_lock, flags); + return idle; +} + +void avd_pm_queue_changed(struct avd_dev *avd) +{ + wake_up_all(&avd->pm.idle_wait); +} + +void avd_pm_drain_begin(struct avd_dev *avd) +{ + unsigned long flags; + + spin_lock_irqsave(&avd->queue_lock, flags); + avd->pm.draining++; + spin_unlock_irqrestore(&avd->queue_lock, flags); +} + +void avd_pm_drain_end(struct avd_dev *avd) +{ + unsigned long flags; + + spin_lock_irqsave(&avd->queue_lock, flags); + avd->pm.draining--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); +} + +static void avd_pm_resubmit(struct avd_dev *avd) +{ + /* queue_mutex may be held by the caller of pm_runtime_resume_and_get(). */ + mod_delayed_work(system_wq, &avd->queue_work, 0); +} + +static int avd_pm_queue_drain(struct avd_dev *avd) +{ + avd_pm_resubmit(avd); + if (!wait_event_timeout(avd->pm.idle_wait, avd_pm_queue_idle(avd), + msecs_to_jiffies(AVD_DRAIN_TIMEOUT_MS))) + return -EBUSY; + return 0; +} + +const struct avd_queue_pm_ops avd_pm_queue_ops = { + .idle = avd_pm_queue_idle, + .drain = avd_pm_queue_drain, + .resubmit = avd_pm_resubmit, +}; + +void avd_pm_init(struct avd_dev *avd, const struct avd_backend_ops *backend, + const struct avd_queue_pm_ops *queue) +{ + mutex_init(&avd->pm.lock); + INIT_LIST_HEAD(&avd->pm.deferred_unmaps); + init_waitqueue_head(&avd->pm.idle_wait); + avd->pm.dart_active = false; + avd->pm.irq = -1; + avd->pm.irq_enabled = false; + spin_lock_init(&avd->pm.irq_lock); + avd->pm.hw_on = false; + avd->pm.irq_gen = 0; + avd->backend = backend; + avd->pm.queue = queue; +} + +void avd_pm_enable(struct avd_dev *avd) +{ + pm_runtime_set_autosuspend_delay(avd->dev, AVD_AUTOSUSPEND_MS); + pm_runtime_use_autosuspend(avd->dev); + pm_runtime_enable(avd->dev); +} + +int avd_pm_map_piodma(struct avd_dev *avd) +{ + struct resource *piodma, *sram; + + piodma = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "piodma"); + /* Old DTs support only the legacy transport; never invent a resource. */ + if (!piodma) + return 0; + sram = platform_get_resource_byname(avd->pdev, IORESOURCE_MEM, "sram"); + if (!sram || !IS_ALIGNED(sram->start, 16) || + (sram->start >> 4) > U32_MAX || + resource_size(piodma) < AVD_PIODMA_SRAM_BASE + sizeof(u32)) + return -EINVAL; + avd->pm.piodma = devm_ioremap_resource(avd->dev, piodma); + if (IS_ERR(avd->pm.piodma)) + return PTR_ERR(avd->pm.piodma); + avd->pm.sram_phys = sram->start; + if (of_count_phandle_with_args(avd->dev->of_node, "iommus", "#iommu-cells") < 2) + dev_warn(avd->dev, "PIODMA transport requires decode and DMA IOMMU streams\n"); + return 0; +} + +void avd_pm_prepare_piodma(struct avd_dev *avd) +{ + /* Public CM3 boot ABI: replay the SRAM base after every power loss/reset. */ + if (avd->pm.piodma) + writel((u32)(avd->pm.sram_phys >> 4), + avd->pm.piodma + AVD_PIODMA_SRAM_BASE); +} + +void avd_pm_disable(struct avd_dev *avd) +{ + int ret; + + /* Power suppliers before flushing frees queued by cleanup while idle. */ + ret = pm_runtime_resume_and_get(avd->dev); + if (ret >= 0) + ret = pm_runtime_put_sync_suspend(avd->dev); + if (ret < 0) + dev_warn(avd->dev, "final power transition failed: %d\n", ret); + pm_runtime_disable(avd->dev); + /* Also close the host IRQ if the final hardware transition failed. */ + avd_pm_disable_irq(avd); + /* No queue_mutex is held here: let a woken handler thread finish. */ + if (avd->pm.irq >= 0) + synchronize_irq(avd->pm.irq); + pm_runtime_dont_use_autosuspend(avd->dev); + /* Resume may have kicked the already-stopped queue. */ + cancel_delayed_work_sync(&avd->queue_work); + WARN_ON(!list_empty(&avd->pm.deferred_unmaps)); +} + +/* pm.lock held; release records own their storage and may free themselves. */ +static void avd_pm_drain_unmaps(struct avd_dev *avd) +{ + struct avd_deferred_unmap *unmap, *next; + + list_for_each_entry_safe(unmap, next, &avd->pm.deferred_unmaps, node) { + list_del_init(&unmap->node); + unmap->unmap(avd, unmap); + } +} + +void avd_pm_unmap(struct avd_dev *avd, struct avd_deferred_unmap *unmap) +{ + mutex_lock(&avd->pm.lock); + if (avd->pm.dart_active) + unmap->unmap(avd, unmap); + else + list_add_tail(&unmap->node, &avd->pm.deferred_unmaps); + mutex_unlock(&avd->pm.lock); +} + +int avd_pm_backend_suspend(struct avd_dev *avd) +{ + avd->backend->shutdown(avd); + return 0; +} + +int avd_pm_backend_resume(struct avd_dev *avd) +{ + /* Includes transport setup, ITCM verification/load and a new generation. */ + return avd_backend_boot(avd); +} + +int avd_pm_recover(struct avd_dev *avd) +{ + bool suspended; + int ret; + + /* + * Recovery is rare and touches the decoder, its reset and its DART: + * leave a step-by-step trail in the log before each step, so a failure + * that takes the machine down still shows how far recovery got. + */ + if (!avd->pm.dart_active) { + dev_err(avd->dev, "recover: decoder DART inactive, not touching it\n"); + return -EHOSTDOWN; + } + /* The PM KUnit fixture has no struct device. */ + suspended = avd->dev && pm_runtime_suspended(avd->dev); + dev_warn(avd->dev, "recover: start (runtime %s, irq %s)\n", + suspended ? "suspended" : "active", + avd->pm.irq_enabled ? "on" : "off"); + /* Every caller holds a runtime PM reference; anything else is a bug. */ + if (WARN_ONCE(suspended, + "avd: recovery with the decoder powered off\n")) + return -EHOSTDOWN; + avd_pm_disable_irq(avd); + dev_warn(avd->dev, "recover: stop firmware\n"); + avd->backend->shutdown(avd); + dev_warn(avd->dev, "recover: reset\n"); + ret = reset_control_reset(avd->rstc); + if (ret) { + dev_err(avd->dev, "recover: reset failed: %d\n", ret); + return ret; + } + if (avd->domain) { + if (!avd->empty_domain) + return -ENODEV; + dev_warn(avd->dev, "recover: DART flush\n"); + ret = iommu_attach_device(avd->empty_domain, avd->dev); + if (ret) { + dev_err(avd->dev, "recover: DART empty attach failed: %d\n", ret); + return ret; + } + iommu_detach_device(avd->empty_domain, avd->dev); + /* Do not reboot against an empty/blocked domain after a failed restore. */ + if (iommu_get_domain_for_dev(avd->dev) != avd->domain) { + dev_err(avd->dev, "recover: DART domain not restored\n"); + return -EIO; + } + } + dev_warn(avd->dev, "recover: boot firmware\n"); + ret = avd_pm_backend_resume(avd); + if (ret) { + dev_err(avd->dev, "recover: boot failed: %d\n", ret); + avd->backend->shutdown(avd); + return ret; + } + dev_warn(avd->dev, "recover: done\n"); + return 0; +} + +int avd_pm_power_on(struct avd_dev *avd) +{ + const struct avd_backend_ops *backend = avd->backend; + int ret; + + /* genpd and the IOMMU suppliers resume before this runtime callback. */ + mutex_lock(&avd->pm.lock); + avd->pm.dart_active = true; + avd_pm_drain_unmaps(avd); + mutex_unlock(&avd->pm.lock); + + /* Do not hold pm.lock: boot may allocate or release DMA buffers. */ + ret = backend->resume ? backend->resume(avd) : avd_pm_backend_resume(avd); + if (ret) { + avd->backend->shutdown(avd); + mutex_lock(&avd->pm.lock); + avd->pm.dart_active = false; + mutex_unlock(&avd->pm.lock); + return ret; + } + avd_pm_enable_irq(avd); + /* A clean boot after power-up ends a failed recovery's fault state. */ + WRITE_ONCE(avd->faulted, false); + + if (avd->pm.queue->resubmit) + avd->pm.queue->resubmit(avd); + + return 0; +} + +int avd_pm_power_off(struct avd_dev *avd) +{ + const struct avd_backend_ops *backend = avd->backend; + int ret; + + /* PM references cover reservation through retirement, including drain. */ + if (!avd->pm.queue->idle(avd)) + return -EBUSY; + + if (backend->suspend) { + ret = backend->suspend(avd); + if (ret) + return ret; + } else { + backend->shutdown(avd); + } + avd_pm_disable_irq(avd); + + /* No more firmware DMA; suppliers stay on until this callback returns. */ + mutex_lock(&avd->pm.lock); + avd_pm_drain_unmaps(avd); + avd->pm.dart_active = false; + mutex_unlock(&avd->pm.lock); + return 0; +} + +static int avd_runtime_resume(struct device *dev) +{ + return avd_pm_power_on(dev_get_drvdata(dev)); +} + +static int avd_runtime_suspend(struct device *dev) +{ + return avd_pm_power_off(dev_get_drvdata(dev)); +} + +static int avd_system_suspend(struct device *dev) +{ + struct avd_dev *avd = dev_get_drvdata(dev); + int ret; + + v4l2_m2m_suspend(avd->m2m_dev); + ret = avd->pm.queue->drain(avd); + if (!ret) + ret = pm_runtime_force_suspend(dev); + if (ret) + v4l2_m2m_resume(avd->m2m_dev); + return ret; +} + +static int avd_system_resume(struct device *dev) +{ + struct avd_dev *avd = dev_get_drvdata(dev); + int ret; + + ret = pm_runtime_force_resume(dev); + if (ret) + return ret; + v4l2_m2m_resume(avd->m2m_dev); + return 0; +} + +const struct dev_pm_ops avd_pm_ops = { + SYSTEM_SLEEP_PM_OPS(avd_system_suspend, avd_system_resume) + RUNTIME_PM_OPS(avd_runtime_suspend, avd_runtime_resume, NULL) +}; diff --git a/drivers/media/platform/apple/avd/avd-pm.h b/drivers/media/platform/apple/avd/avd-pm.h new file mode 100644 index 0000000000000..5fe55f595beb2 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_PM_H_ +#define AVD_PM_H_ + +#include +#include +#include + +#include "avd-queue.h" + +struct avd_dma_release; + +struct avd_queue_pm_ops { + /* Snapshot under the queue lock: all FREE, no context draining. */ + bool (*idle)(struct avd_dev *avd); + /* Scheduling is paused; finish/retire all accepted frames. May sleep. */ + int (*drain)(struct avd_dev *avd); + /* Kick prepared RESERVED entries only, respecting dependencies. */ + void (*resubmit)(struct avd_dev *avd); +}; + +struct avd_deferred_unmap { + struct list_head node; + void (*unmap)(struct avd_dev *avd, struct avd_deferred_unmap *unmap); +}; + +struct avd_pm { + /* Serializes DMA releases against DART activation and firmware boot. */ + struct mutex lock; + struct list_head deferred_unmaps; + bool dart_active; + int irq; + bool irq_enabled; + /* + * irq_lock guards hw_on and irq_gen. The threaded handler touches the + * decoder only under irq_lock with hw_on set and an unchanged irq_gen, + * so turning the IRQ off never has to wait for the handler thread + * (which may itself be waiting for queue_mutex). + */ + spinlock_t irq_lock; + bool hw_on; + unsigned int irq_gen; + void __iomem *piodma; + phys_addr_t sram_phys; + wait_queue_head_t idle_wait; + /* Protected by queue_lock; counts drain operations, not stopped streams. */ + unsigned int draining; + const struct avd_queue_pm_ops *queue; +}; + +/* The caller holds the queue lock, including the draining-context count. */ +bool avd_pm_can_suspend(const struct avd_frame *frames, unsigned int draining); + +void avd_pm_init(struct avd_dev *avd, const struct avd_backend_ops *backend, + const struct avd_queue_pm_ops *queue); +void avd_pm_enable(struct avd_dev *avd); +int avd_pm_map_piodma(struct avd_dev *avd); +void avd_pm_prepare_piodma(struct avd_dev *avd); +/* After queue cleanup: release deferred DMA mappings before the final off. */ +void avd_pm_disable(struct avd_dev *avd); +/* Called in process context, after the last frame has retired its mapping. */ +void avd_pm_unmap(struct avd_dev *avd, struct avd_deferred_unmap *unmap); +int avd_pm_power_on(struct avd_dev *avd); +int avd_pm_power_off(struct avd_dev *avd); +/* Caller holds a PM reference and queue_mutex; leaves the host IRQ closed. */ +int avd_pm_recover(struct avd_dev *avd); +/* After recovery has invalidated every previously firmware-owned slot. */ +void avd_pm_enable_irq(struct avd_dev *avd); +irqreturn_t avd_pm_irq(struct avd_dev *avd, unsigned int gen); +void avd_pm_drain_begin(struct avd_dev *avd); +void avd_pm_drain_end(struct avd_dev *avd); +void avd_pm_queue_changed(struct avd_dev *avd); +extern const struct avd_queue_pm_ops avd_pm_queue_ops; + +/* Backends may share these when suspend loses all firmware context. */ +int avd_pm_backend_suspend(struct avd_dev *avd); +int avd_pm_backend_resume(struct avd_dev *avd); +extern const struct dev_pm_ops avd_pm_ops; + +#endif /* AVD_PM_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-pm.kunitconfig b/drivers/media/platform/apple/avd/avd-pm.kunitconfig new file mode 100644 index 0000000000000..4d9b37f71ff41 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-pm.kunitconfig @@ -0,0 +1,14 @@ +CONFIG_KUNIT=y +CONFIG_COMPILE_TEST=y +CONFIG_OF=y +CONFIG_PM=y +CONFIG_SUSPEND=y +CONFIG_PM_SLEEP=y +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_VIDEO_DEV=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_VIDEO_APPLE_AVD=y +CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST=y +CONFIG_WERROR=y diff --git a/drivers/media/platform/apple/avd/avd-protocol.rst b/drivers/media/platform/apple/avd/avd-protocol.rst new file mode 100644 index 0000000000000..ce9eaf0bf2720 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-protocol.rst @@ -0,0 +1,97 @@ +AVD queued transport implementation status +========================================== + +The pool owns sixteen independently tagged slots. A source/destination pair +and a runtime-PM reference remain owned by the slot until in-order retirement +within its context. A duplicate or obsolete tag cannot complete a reused +slot. Inbox-full leaves a prepared slot reserved and retries without losing +its buffers. Timeout or streamoff resets the whole engine before failing all +slots affected by that reset. Slot deadlines survive inbox retries. + +The first commit adding avd-queue.h is the shared interface. Later additions +only extend it. Codec callbacks own codec-private state and must release that +state from retire(), including when prepare() failed part way through. + +Open firmware compatibility +-------------------------- + +The deployed, untagged open-firmware protocol has a one-frame limit. Existing +codec run() callbacks build a DMA-visible per-slot program; only the backend +writes those instructions to the VP FIFO. That compatibility backend retains +the old host-side PP kick. A native frame_ops program must use the queued +avd-fw transport before it can replace the compatibility path; this backend +returns EOPNOTSUPP for native programs rather than guessing their wire format. + +The V4L2 core currently warns unconditionally when job_finish() is used on a +queue advertising HOLD_CAPTURE_BUF. Such queues retain the scheduler until +retirement and use buf_done_and_job_finish(), including held slice requests. +Other queues release the scheduler as soon as their buffers belong to a slot. +Removing this restriction requires a V4L2 core API change outside this driver's +scope. It is not safe to toggle subsystem_flags around job_finish(). + +Apple Borage transport +---------------------- + +Only T6040 has Borage match data. The Apple backend contains the 64-byte +little-endian connection configuration, session command builders, decode +prefix, eight-entry inbox publishing, a 64-entry completion shadow and bounded +20-word result parsing. Type 7 is VP/parsing completion and never releases a +buffer. The notification address rule has one definition, +AVD_APPLE_NOTIFY_BASE, so a capture can correct it. + +Boot checks a nonzero ITCM span and a nonzero/non-all-ones version word before +accepting a preloaded image. A separately installed firmware image can restore +ITCM after power loss. The last image word is zero padded without reading +past the firmware object. No Apple image is embedded in this source tree. + +Activation remains disabled in match data: the supplied clean specification +has no ITCM register offsets, link programming sequence, result-area address, +result field layout or final-completion value. It would be unsafe to invent +these or reuse the different open-firmware mailbox. The closest executable +path is the open compatibility backend. The Apple register description and +reply layout must be completed together from clean facts; setting verified +alone is insufficient. The DT must then describe the complete Apple aperture +as an owned named resource before probe can map it. + +Unresolved field ledger +----------------------- + +This is the single ledger for fields marked [C] in the common transport. No +semantic names have been assigned to the unknown bits. Codec-specific fields +remain in their codec packers' ledgers. + +============================== ================================================ +Field Source and required evidence +============================== ================================================ +Config +00/+08/+10/+18/+20/ SPEC 3.2: exact values, meanings unproved; +28/+2c/+30/+38 capture connection construction and LinkUp +ITCM start/end/clear/enable SPEC 3.1: operations, offsets/values absent; + clean per-SoC register ledger required +IMEM version-word offset SPEC 3.1: display format proved, offset absent +Mailbox link register writes SPEC 3.2: aperture 1, state 4, LinkUp proved; + offsets and bit operations absent +Mailbox ready register SPEC 3.1: status 1, poll 10 us; offset absent +Mailbox notification word SPEC 4.2; expected inbox-entry address from MIT + aura trace_avd.py V3 precedent; capture Borage +Inbox consumption/full rule SPEC 4.2: host positions modulo eight; + hardware consumer register/doorbell absent +Result-area base/indexing SPEC 4.3: 64 entries, 20 words; base and mailbox + notification interpretation absent +Reply type/client/frame/status SPEC 4.3: correlation proved, word layout absent; + synthetic KUnit layout is not a hardware fact +Final reply type/status values SPEC 4.3: only VP done = 7 proved; capture final + completion and error replies before activation +Decode +00 bit 9 SPEC 5.2: boolean argument, meaning unproved; + left zero until the producer is established +Decode +50..+5f SPEC 5: Borage extension, nonzero use unproved; + initialized to zero before codec packing +Commands 3/4 and other values SPEC 5.1: unproved; never sent +PIODMA mapping stream binding SPEC 7: three mapper roles proved, Linux binding + and per-domain fp/program DVA ownership pending +Retention and clock floors SPEC 8: per-SoC capture required; no guessed knobs +============================== ================================================ + +Licensed implementation references: eiln/avd (MIT), avd_emu.py, and +https://github.com/aurora-silicon/aura/blob/main/proxyclient/hv/trace_avd.py +(MIT). The Borage facts above come from the supplied clean SPEC sections 3-5; +no binary or captured Apple command images are used. diff --git a/drivers/media/platform/apple/avd/avd-queue-internal.h b/drivers/media/platform/apple/avd/avd-queue-internal.h new file mode 100644 index 0000000000000..0e2a668d5a62f --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue-internal.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_QUEUE_INTERNAL_H_ +#define AVD_QUEUE_INTERNAL_H_ + +#include +#include +#include + +#include "avd-queue.h" + +/* Caller serializes the pool; tags are never recycled during device lifetime. */ +static inline int avd_slot_reserve(struct avd_frame *f, struct avd_ctx *ctx, + u64 seq) +{ + if (f->state != AVD_FRAME_FREE) + return -EBUSY; + if (!seq || seq > (U32_MAX >> 4)) + return -EOVERFLOW; + f->seq = seq; + f->tag = (seq << 4) | f->idx; + f->ctx = ctx; + f->state = AVD_FRAME_RESERVED; + f->dep = AVD_DEP_NONE; + f->status = 0; + f->prog_words = 0; + f->codec_priv = NULL; + return 0; +} + +/* A full inbox owns nothing: keep the prepared slot, tag and deadline. */ +static inline void avd_slot_submit_result(struct avd_frame *f, int status) +{ + if (status == -EBUSY) { + f->state = AVD_FRAME_RESERVED; + } else if (status) { + f->status = status; + f->state = AVD_FRAME_DONE; + } +} + +static inline bool avd_slot_complete(struct avd_frame *f, u32 tag, int status) +{ + if (f->state != AVD_FRAME_SUBMITTED || f->tag != tag) + return false; + f->status = status; + f->state = AVD_FRAME_DONE; + return true; +} + +static inline bool avd_slot_first(const struct avd_frame *pool, + const struct avd_frame *f) +{ + unsigned int i; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (pool[i].state != AVD_FRAME_FREE && pool[i].ctx == f->ctx && + pool[i].seq < f->seq) + return false; + return true; +} + +static inline bool avd_slot_expired(const struct avd_frame *f, unsigned long now) +{ + return (f->state == AVD_FRAME_SUBMITTED || + f->state == AVD_FRAME_RESERVED) && time_after_eq(now, f->deadline); +} + +static inline bool avd_inbox_full(unsigned int producer, unsigned int consumer) +{ + return ((producer + 1) & 7) == consumer; +} + +#endif /* AVD_QUEUE_INTERNAL_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-queue-test.c b/drivers/media/platform/apple/avd/avd-queue-test.c new file mode 100644 index 0000000000000..1efae38789b57 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue-test.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "avd.h" +#include "avd-apple-protocol.h" +#include "avd-queue-internal.h" + +static void avd_slots_and_tags(struct kunit *test) +{ + struct avd_frame *pool; + struct avd_ctx *ctx = (void *)test; + unsigned int i; + + pool = kunit_kcalloc(test, AVD_MAX_FRAMES, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + pool[i].idx = i; + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[i], ctx, i + 1), 0); + KUNIT_EXPECT_EQ(test, pool[i].tag, ((i + 1) << 4) | i); + KUNIT_EXPECT_EQ(test, pool[i].state, AVD_FRAME_RESERVED); + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[i], ctx, 100), -EBUSY); + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag, 0)); + pool[i].state = AVD_FRAME_SUBMITTED; + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag + 16, 0)); + KUNIT_EXPECT_TRUE(test, avd_slot_complete(&pool[i], pool[i].tag, 0)); + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[i], pool[i].tag, -EIO)); + } + pool[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[0], ctx, 0), -EOVERFLOW); + KUNIT_EXPECT_EQ(test, avd_slot_reserve(&pool[0], ctx, (u64)(U32_MAX >> 4) + 1), + -EOVERFLOW); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[0], ctx, 100), 0); + pool[0].state = AVD_FRAME_SUBMITTED; + KUNIT_EXPECT_FALSE(test, avd_slot_complete(&pool[0], 16, 0)); +} + +static void avd_retirement_order(struct kunit *test) +{ + struct avd_frame *pool; + struct avd_ctx *a = (void *)test; + struct avd_ctx *b = (void *)&pool; + + pool = kunit_kcalloc(test, AVD_MAX_FRAMES, sizeof(*pool), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pool); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[0], a, 1), 0); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[1], b, 2), 0); + KUNIT_ASSERT_EQ(test, avd_slot_reserve(&pool[2], a, 3), 0); + pool[0].state = AVD_FRAME_SUBMITTED; + pool[1].state = AVD_FRAME_SUBMITTED; + pool[2].state = AVD_FRAME_SUBMITTED; + KUNIT_ASSERT_TRUE(test, avd_slot_complete(&pool[2], pool[2].tag, 0)); + KUNIT_EXPECT_FALSE(test, avd_slot_first(pool, &pool[2])); + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[1])); + KUNIT_ASSERT_TRUE(test, avd_slot_complete(&pool[0], pool[0].tag, -ETIMEDOUT)); + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[0])); + pool[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_TRUE(test, avd_slot_first(pool, &pool[2])); + KUNIT_EXPECT_EQ(test, pool[0].status, -ETIMEDOUT); +} + +static void avd_watchdog_and_retry(struct kunit *test) +{ + struct avd_frame f = { .state = AVD_FRAME_SUBMITTED, .deadline = ULONG_MAX - 2 }; + unsigned int producer, consumer; + u32 tag = 0x124; + unsigned long deadline = f.deadline; + + f.tag = tag; + avd_slot_submit_result(&f, -EBUSY); + KUNIT_EXPECT_EQ(test, f.state, AVD_FRAME_RESERVED); + KUNIT_EXPECT_EQ(test, f.tag, tag); + KUNIT_EXPECT_EQ(test, f.deadline, deadline); + f.state = AVD_FRAME_SUBMITTED; + avd_slot_submit_result(&f, 0); + KUNIT_EXPECT_EQ(test, f.state, AVD_FRAME_SUBMITTED); + + KUNIT_EXPECT_FALSE(test, avd_slot_expired(&f, ULONG_MAX - 3)); + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, ULONG_MAX - 2)); + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, 1)); + f.state = AVD_FRAME_DONE; + KUNIT_EXPECT_FALSE(test, avd_slot_expired(&f, 1)); + f.state = AVD_FRAME_RESERVED; + KUNIT_EXPECT_TRUE(test, avd_slot_expired(&f, 1)); + for (producer = 0; producer < 8; producer++) { + for (consumer = 0; consumer < 8; consumer++) + KUNIT_EXPECT_EQ(test, avd_inbox_full(producer, consumer), + ((producer + 1) & 7) == consumer); + consumer = (producer + 1) & 7; + KUNIT_EXPECT_TRUE(test, avd_inbox_full(producer, consumer)); + /* A consumer advance reopens the same pending inbox slot. */ + consumer = (consumer + 1) & 7; + KUNIT_EXPECT_FALSE(test, avd_inbox_full(producer, consumer)); + KUNIT_EXPECT_EQ(test, avd_apple_notification(producer), + 0x01482cf8U + 0x60 * producer); + } +} + +static void avd_completion_parser(struct kunit *test) +{ + /* Synthetic layout for parser testing; never a claimed Borage wire ABI. */ + struct avd_apple_reply_layout layout = { + .type_word = 0, .type_mask = 0x1f, + .client_word = 1, .client_shift = 16, .client_mask = 0x3ff, + .frame_word = 3, .status_word = 4, .done_type = 8, .verified = true, + }; + struct avd_apple_reply reply; + u32 record[AVD_APPLE_RESULT_WORDS] = { 7, 5 << 16, 0, 0x12340 }; + + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 19, &layout, &reply), -EMSGSIZE); + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EINPROGRESS); + record[0] = 8; + KUNIT_ASSERT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), 0); + KUNIT_EXPECT_EQ(test, reply.client, 5U); + KUNIT_EXPECT_EQ(test, reply.frame, 0x12340U); + KUNIT_EXPECT_EQ(test, reply.status, 0); + record[4] = 1; + KUNIT_ASSERT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), 0); + KUNIT_EXPECT_EQ(test, reply.status, -EIO); + record[0] = 3; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -ENOMSG); + layout.frame_word = 20; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EINVAL); + layout.verified = false; + KUNIT_EXPECT_EQ(test, avd_apple_parse_reply(record, 20, &layout, &reply), -EOPNOTSUPP); +} + +static void avd_command_builders(struct kunit *test) +{ + struct avd_frame f = { .idx = 15, .frame_num = 0x1234 }; + u32 cmd[24]; + const u64 config[] = { + 0, 0x01558000, 0x01558a00, 0x01558e00, 0, + 0x000000040004000aULL, 0, 0x01100008, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(config); i++) + KUNIT_EXPECT_EQ(test, get_unaligned_le64(avd_borage_config + i * 8), config[i]); + KUNIT_ASSERT_EQ(test, avd_apple_decode_prefix(&f, 1023, 15, 3), 0); + KUNIT_EXPECT_EQ(test, f.cmd[0], 1U | (15U << 10) | (1023U << 22)); + KUNIT_EXPECT_EQ(test, f.cmd[1], 15U | (3U << 16)); + KUNIT_EXPECT_EQ(test, f.cmd[3], 0x1234U); + KUNIT_EXPECT_EQ(test, f.cmd[23], 0U); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 1024, 0, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 0, 16, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, avd_apple_decode_prefix(&f, 0, 0, 4), -EINVAL); + KUNIT_ASSERT_EQ(test, avd_apple_session_command(cmd, 2, 6, 17, 42), 0); + KUNIT_EXPECT_EQ(test, cmd[0], 6U | (2U << 16)); + KUNIT_EXPECT_EQ(test, cmd[1], 17U); + KUNIT_EXPECT_EQ(test, cmd[2], 42U); + KUNIT_EXPECT_EQ(test, cmd[23], 0U); + KUNIT_EXPECT_EQ(test, avd_apple_session_command(cmd, 2, 3, 0, 0), -EINVAL); +} + +static unsigned int avd_test_max_inflight(struct avd_dev *avd) +{ + return AVD_MAX_FRAMES; +} + +static void avd_dispatch_reservation(struct kunit *test) +{ + static const struct avd_backend_ops backend = { + .max_inflight = avd_test_max_inflight, + }; + struct avd_dev *avd = kunit_kzalloc(test, sizeof(*avd), GFP_KERNEL); + struct avd_ctx *ctx = kunit_kcalloc(test, 2, sizeof(*ctx), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, avd); + KUNIT_ASSERT_NOT_NULL(test, ctx); + spin_lock_init(&avd->queue_lock); + avd->backend = &backend; + ctx[0].dev = avd; + ctx[1].dev = avd; + /* Every context with work is ready; capacity is checked at dispatch. */ + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[0]), 1); + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); + avd->inflight = AVD_MAX_FRAMES; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); + avd->inflight = 1; + avd->frames[0].state = AVD_FRAME_SUBMITTED; + avd->frames[0].ctx = &ctx[1]; + ctx[1].next_dep = AVD_DEP_PREV_DONE; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 0); + avd->frames[0].state = AVD_FRAME_FREE; + KUNIT_EXPECT_EQ(test, avd_queue_ready(&ctx[1]), 1); +} + +static struct kunit_case avd_queue_cases[] = { + KUNIT_CASE(avd_slots_and_tags), + KUNIT_CASE(avd_retirement_order), + KUNIT_CASE(avd_watchdog_and_retry), + KUNIT_CASE(avd_completion_parser), + KUNIT_CASE(avd_command_builders), + KUNIT_CASE(avd_dispatch_reservation), + {} +}; + +static struct kunit_suite avd_queue_suite = { + .name = "apple-avd-queue", + .test_cases = avd_queue_cases, +}; + +kunit_test_suite(avd_queue_suite); diff --git a/drivers/media/platform/apple/avd/avd-queue.c b/drivers/media/platform/apple/avd/avd-queue.c new file mode 100644 index 0000000000000..ecbe21d5444b2 --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue.c @@ -0,0 +1,595 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include + +#include "avd.h" +#include "avd-queue-internal.h" + +#define AVD_FRAME_TIMEOUT msecs_to_jiffies(2000) +/* A stream that fails this many frames in a row in hardware is stopped. */ +#define AVD_CTX_MAX_HW_ERRORS 3 +/* Never reset the decoder more often than this. */ +#define AVD_RECOVER_GAP_MS 20 +/* More than AVD_STORM_MAX recoveries within AVD_STORM_WINDOW: cool down. */ +#define AVD_STORM_MAX 8 +#define AVD_STORM_WINDOW msecs_to_jiffies(10000) +#define AVD_COOLDOWN msecs_to_jiffies(10000) +#define AVD_RETRY_DELAY msecs_to_jiffies(1) +#define AVD_PROGRAM_SIZE SZ_2M + +bool avd_backend_is_apple(const struct avd_dev *avd) +{ + return avd->backend == &avd_apple_backend; +} + +/* queue_mutex excludes callbacks and context destruction from retirement. */ +static void avd_retire(struct avd_dev *avd) +{ + struct avd_frame *f; + unsigned long flags; + unsigned int i; + bool progress; + + do { + progress = false; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_ctx *ctx; + const struct avd_codec_frame_ops *ops; + enum vb2_buffer_state state; + + f = &avd->frames[i]; + spin_lock_irqsave(&avd->queue_lock, flags); + if (f->state != AVD_FRAME_DONE || + !avd_slot_first(avd->frames, f)) { + spin_unlock_irqrestore(&avd->queue_lock, flags); + continue; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + ctx = f->ctx; + ops = avd->frame_codec_ops[i]; + state = f->status ? VB2_BUF_STATE_ERROR : VB2_BUF_STATE_DONE; + if (!f->status) { + ctx->hw_errors = 0; + } else if (++ctx->hw_errors >= AVD_CTX_MAX_HW_ERRORS && !ctx->broken) { + ctx->broken = true; + dev_err(avd->dev, "stream stopped after %u failed frames; restart streaming to retry\n", + ctx->hw_errors); + } + if (ops) { + if (ops->retire) + ops->retire(ctx, f, state); + } else if (ctx->coded_fmt_desc->ops->done) { + ctx->coded_fmt_desc->ops->done(ctx, f->src, f->dst, state); + } + if (ops && f->src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(f->src->vb2_buf.req_obj.req, + &ctx->ctrl_hdl); + /* CAPTURE first: OUTPUT completion signals the media request. */ + if (f->scheduler_held) { + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, + ctx->fh.m2m_ctx, state); + } else { + v4l2_m2m_buf_done(f->dst, state); + v4l2_m2m_buf_done(f->src, state); + } + spin_lock_irqsave(&avd->queue_lock, flags); + f->ctx = NULL; + f->src = NULL; + f->dst = NULL; + f->state = AVD_FRAME_FREE; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + progress = true; + } + } while (progress); +} + +/* A reset invalidates every firmware-owned slot, including other clients. */ +static void avd_reset_queue(struct avd_dev *avd, int status) +{ + struct avd_ctx *ctx; + unsigned long flags; + unsigned int i; + int ret; + + /* + * A stream whose every frame faults (an H0 error storm) used to reset + * the decoder back to back, about once per millisecond, until the + * machine hung. Space recoveries out, and after a storm stop taking + * new frames for a while instead of resetting again. + */ + if (avd->last_recover && + time_before(jiffies, avd->last_recover + msecs_to_jiffies(AVD_RECOVER_GAP_MS))) + msleep(AVD_RECOVER_GAP_MS); + if (!avd->storm_count || time_after(jiffies, avd->storm_start + AVD_STORM_WINDOW)) { + avd->storm_start = jiffies; + avd->storm_count = 0; + } + if (++avd->storm_count > AVD_STORM_MAX) { + dev_err(avd->dev, "%u decoder recoveries in %u s: failing new frames for %u s\n", + avd->storm_count, jiffies_to_msecs(AVD_STORM_WINDOW) / 1000, + jiffies_to_msecs(AVD_COOLDOWN) / 1000); + WRITE_ONCE(avd->cooldown_until, jiffies + AVD_COOLDOWN); + avd->storm_count = 0; + } + ret = avd_pm_recover(avd); + avd->last_recover = jiffies ?: 1; + WRITE_ONCE(avd->faulted, !!ret); + if (ret) + dev_err(avd->dev, "queue reset failed: %d\n", ret); + avd->compat_active = NULL; + list_for_each_entry(ctx, &avd->contexts, queue_node) + ctx->session_started = false; + spin_lock_irqsave(&avd->queue_lock, flags); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state == AVD_FRAME_FREE) + continue; + avd->frames[i].status = status; + avd->frames[i].state = AVD_FRAME_DONE; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (!ret) + avd_pm_enable_irq(avd); +} + +static int avd_send_frame(struct avd_dev *avd, struct avd_frame *f) +{ + unsigned long flags; + int ret; + + if (f->dep == AVD_DEP_PREV_DONE && !avd_slot_first(avd->frames, f)) + return -EBUSY; + if ((!f->ctx->session_started || + f->ctx->session_generation != avd->fw_generation) && + avd->backend->session_start) { + ret = avd->backend->session_start(f->ctx); + if (ret) + return ret; + f->ctx->session_started = true; + f->ctx->session_generation = avd->fw_generation; + } + /* Publish ownership before the doorbell; IRQs may arrive in submit(). */ + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_SUBMITTED; + spin_unlock_irqrestore(&avd->queue_lock, flags); + dma_wmb(); + ret = avd->backend->submit(avd, f); + spin_lock_irqsave(&avd->queue_lock, flags); + avd_slot_submit_result(f, ret); + spin_unlock_irqrestore(&avd->queue_lock, flags); + return ret; +} + +static void avd_queue_worker(struct work_struct *work) +{ + struct avd_dev *avd = container_of(to_delayed_work(work), + struct avd_dev, queue_work); + struct avd_ctx *ctx; + unsigned long flags; + unsigned int i; + bool pending; + unsigned long delay = AVD_FRAME_TIMEOUT; + + mutex_lock(&avd->queue_mutex); + if (avd->stopping) + goto unlock; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + bool expired; + + spin_lock_irqsave(&avd->queue_lock, flags); + expired = avd_slot_expired(&avd->frames[i], jiffies); + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (expired) { + struct avd_frame *t = &avd->frames[i]; + + dev_err(avd->dev, "frame tag %#x timed out (%p4cc, state %d, %u in flight, %s)\n", + t->tag, t->ctx ? &t->ctx->coded_fmt_desc->fourcc : &(u32){0}, + t->state, avd->inflight, avd->backend->name); + avd_reset_queue(avd, -ETIMEDOUT); + break; + } + } + avd_retire(avd); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + int ret; + + if (f->state != AVD_FRAME_RESERVED) + continue; + ret = avd_send_frame(avd, f); + if (ret && ret != -EBUSY) { + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + } + } + avd_retire(avd); + /* Restart a job that found no free slot. */ + spin_lock_irqsave(&avd->queue_lock, flags); + ctx = avd->inflight < avd->backend->max_inflight(avd) ? avd->deferred_ctx : NULL; + if (ctx) + avd->deferred_ctx = NULL; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (ctx) + schedule_work(&ctx->submit_work); + /* Freed slots/dependencies must wake contexts whose job_ready was false. */ + list_for_each_entry(ctx, &avd->contexts, queue_node) + v4l2_m2m_try_schedule(ctx->fh.m2m_ctx); + spin_lock_irqsave(&avd->queue_lock, flags); + pending = avd->inflight != 0; + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + unsigned long remaining; + + if (f->state == AVD_FRAME_FREE) + continue; + remaining = time_after(f->deadline, jiffies) ? f->deadline - jiffies : 0; + if (f->state == AVD_FRAME_RESERVED) + remaining = min(remaining, AVD_RETRY_DELAY); + delay = min(delay, remaining); + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (pending) + mod_delayed_work(system_wq, &avd->queue_work, delay); + unlock: + mutex_unlock(&avd->queue_mutex); +} + +void avd_frame_complete(struct avd_dev *avd, u32 tag, int status) +{ + struct avd_frame *f = &avd->frames[tag & (AVD_MAX_FRAMES - 1)]; + unsigned long flags; + bool accepted; + + spin_lock_irqsave(&avd->queue_lock, flags); + accepted = avd_slot_complete(f, tag, status); + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (accepted) + mod_delayed_work(system_wq, &avd->queue_work, 0); +} + +int avd_queue_ready(void *priv) +{ + struct avd_ctx *ctx = priv; + struct avd_dev *avd = ctx->dev; + unsigned long flags; + unsigned int i; + bool ready; + + spin_lock_irqsave(&avd->queue_lock, flags); + /* + * A faulted or cooling-down decoder still schedules: submission fails + * those frames at once, so clients see errors instead of waiting. + */ + /* + * Slot capacity is not a readiness condition: every context with work + * enters the m2m job queue, which runs them in order, and a job that + * finds no free slot is deferred until one retires. Gating readiness on + * one reserved context let the first context in the list win every + * time and starved the others (multi-stream decodes timing out). + */ + ready = !READ_ONCE(ctx->draining) && !READ_ONCE(avd->stopping); + if (ready && ctx->next_dep == AVD_DEP_PREV_DONE) { + /* Legacy codecs may hold several pictures in separate banks. */ + unsigned int limit = ctx->coded_fmt_desc && !avd_get_frame_ops(ctx) ? + max(ctx->max_frames, 1U) : 1, held = 0; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + if (avd->frames[i].state != AVD_FRAME_FREE && + avd->frames[i].ctx == ctx) + held++; + ready = held < limit; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + return ready; +} + +static void avd_queue_submit_work(struct work_struct *work) +{ + struct avd_ctx *ctx = container_of(work, struct avd_ctx, submit_work); + struct avd_dev *avd = ctx->dev; + const struct avd_codec_frame_ops *ops; + struct avd_frame *f = NULL; + unsigned long flags; + unsigned int i; + int ret; + + mutex_lock(&avd->queue_mutex); + spin_lock_irqsave(&avd->queue_lock, flags); + if (avd->deferred_ctx == ctx) + avd->deferred_ctx = NULL; + if (avd->stopping || ctx->draining || avd->faulted || ctx->broken || + time_before(jiffies, READ_ONCE(avd->cooldown_until))) { + struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + + spin_unlock_irqrestore(&avd->queue_lock, flags); + /* Complete the request's controls too, or its fd never signals. */ + if (src && src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + goto unlock; + } + if (avd->inflight < avd->backend->max_inflight(avd)) { + for (i = 0; i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state == AVD_FRAME_FREE) { + f = &avd->frames[i]; + break; + } + } + } + if (!f) { + /* Keep the m2m job; the worker restarts it when a slot frees. */ + avd->deferred_ctx = ctx; + spin_unlock_irqrestore(&avd->queue_lock, flags); + mod_delayed_work(system_wq, &avd->queue_work, 0); + goto unlock; + } + ret = avd_slot_reserve(f, ctx, ++avd->frame_seq); + if (!ret) + avd->inflight++; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (WARN_ON(ret)) { + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + goto unlock; + } + ret = pm_runtime_resume_and_get(avd->dev); + if (ret < 0) + goto fail_no_pm; + ops = avd_get_frame_ops(ctx); + avd->frame_codec_ops[f->idx] = ops; + f->src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + f->dst = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); + /* + * Only a slice that holds its CAPTURE buffer needs the m2m scheduler + * until retirement. The last slice of a picture releases both buffers + * to the slot, so the next picture can be prepared while it decodes. + */ + f->scheduler_held = (ctx->fh.m2m_ctx->out_q_ctx.q.subsystem_flags & + VB2_V4L2_FL_SUPPORTS_M2M_HOLD_CAPTURE_BUF) && + (f->src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF); + f->frame_num = f->tag; + f->deadline = jiffies + AVD_FRAME_TIMEOUT; + memset(f->cmd, 0, sizeof(f->cmd)); + if (ops) { + if (!f->prog->cpu) { + ret = avd_buf_alloc(avd, f->prog, AVD_PROGRAM_SIZE); + if (ret) + goto fail; + } + /* NULL fp is also the codec-facing indication of an open program. */ + f->fp = NULL; + if (avd_backend_is_apple(avd)) { + struct avd_buf *fp = avd->frame_fp[f->idx]; + + if (!fp) { + fp = devm_kzalloc(avd->dev, sizeof(*fp), GFP_KERNEL); + avd->frame_fp[f->idx] = fp; + } + f->fp = fp; + if (!f->fp) { + ret = -ENOMEM; + goto fail; + } + if (!f->fp->cpu) { + ret = avd_buf_alloc(avd, f->fp, avd->variant->fp_size); + if (ret) + goto fail; + } + } + if (f->src->vb2_buf.req_obj.req) { + ret = v4l2_ctrl_request_setup(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + if (ret) + goto fail; + } + ret = ops->prepare(ctx, f); + } else { + avd->building = f; + avd->compat_held = false; + ret = ctx->coded_fmt_desc->ops->run(ctx); + avd->building = NULL; + } + if (ret) + goto fail; + ctx->next_dep = ops ? f->dep : AVD_DEP_PREV_DONE; + if ((ops && (f->src->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF)) || + (!ops && avd->compat_held)) { + if (ops && ops->retire) + ops->retire(ctx, f, VB2_BUF_STATE_DONE); + else if (!ops && ctx->coded_fmt_desc->ops->done) + ctx->coded_fmt_desc->ops->done(ctx, f->src, f->dst, VB2_BUF_STATE_DONE); + if (ops && f->src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(f->src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_FREE; + f->ctx = NULL; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_DONE); + goto unlock; + } + v4l2_m2m_buf_copy_metadata(f->src, f->dst); + if (!f->scheduler_held) { + v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); + v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + } + f->dst->is_held = false; + ret = avd_send_frame(avd, f); + if (ret && ret != -EBUSY) { + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + } + /* Buffers now belong to the slot, not to the m2m scheduler. */ + if (!f->scheduler_held) + v4l2_m2m_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx); + mod_delayed_work(system_wq, &avd->queue_work, 0); + goto unlock; + fail: + /* Detach before retirement, including partially prepared codec state. */ + if (!f->scheduler_held) { + v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); + v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + } + spin_lock_irqsave(&avd->queue_lock, flags); + f->status = ret; + f->state = AVD_FRAME_DONE; + spin_unlock_irqrestore(&avd->queue_lock, flags); + if (!f->scheduler_held) + v4l2_m2m_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx); + mod_delayed_work(system_wq, &avd->queue_work, 0); + goto unlock; + fail_no_pm: + spin_lock_irqsave(&avd->queue_lock, flags); + f->state = AVD_FRAME_FREE; + f->ctx = NULL; + avd->inflight--; + spin_unlock_irqrestore(&avd->queue_lock, flags); + avd_pm_queue_changed(avd); + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + unlock: + mutex_unlock(&avd->queue_mutex); +} + +void avd_queue_run(void *priv) +{ + struct avd_ctx *ctx = priv; + + /* try_schedule() may invoke device_run synchronously under queue_mutex. */ + schedule_work(&ctx->submit_work); +} + +void avd_queue_drain(struct avd_ctx *ctx) +{ + struct avd_dev *avd = ctx->dev; + unsigned long flags; + unsigned int i; + bool finish_deferred = false; + int ret; + + /* Session stop/reset accesses registers even when there are no frames. */ + ret = pm_runtime_resume_and_get(avd->dev); + avd_pm_drain_begin(avd); + mutex_lock(&avd->queue_mutex); + spin_lock_irqsave(&avd->queue_lock, flags); + WRITE_ONCE(ctx->draining, true); + if (avd->deferred_ctx == ctx) { + avd->deferred_ctx = NULL; + finish_deferred = true; + } + spin_unlock_irqrestore(&avd->queue_lock, flags); + /* A deferred job is still m2m's running job: end it so abort can wait. */ + if (finish_deferred) + v4l2_m2m_buf_done_and_job_finish(avd->m2m_dev, ctx->fh.m2m_ctx, + VB2_BUF_STATE_ERROR); + for (i = 0; ret >= 0 && i < AVD_MAX_FRAMES; i++) { + if (avd->frames[i].state != AVD_FRAME_FREE && avd->frames[i].ctx == ctx) { + if (avd->backend->abort) + avd->backend->abort(ctx); + avd_reset_queue(avd, -ECANCELED); + break; + } + } + avd_retire(avd); + if (ret >= 0 && ctx->session_started && avd->backend->session_stop && + ctx->session_generation == avd->fw_generation) + avd->backend->session_stop(ctx); + ctx->session_started = false; + if (!avd->stopping) + mod_delayed_work(system_wq, &avd->queue_work, 0); + mutex_unlock(&avd->queue_mutex); + avd_pm_drain_end(avd); + if (ret >= 0) { + pm_runtime_mark_last_busy(avd->dev); + pm_runtime_put_autosuspend(avd->dev); + } +} + +void avd_queue_abort(void *priv) +{ + avd_queue_drain(priv); +} + +void avd_queue_add_ctx(struct avd_ctx *ctx) +{ + struct avd_ctx *other; + u32 id; + + INIT_WORK(&ctx->submit_work, avd_queue_submit_work); + mutex_lock(&ctx->dev->queue_mutex); + /* Borage decode prefix has ten client bits. */ + for (id = 0; id < 1024; id++) { + bool used = false; + + list_for_each_entry(other, &ctx->dev->contexts, queue_node) + used |= other->client_id == id; + if (!used) + break; + } + ctx->client_id = id; + list_add_tail(&ctx->queue_node, &ctx->dev->contexts); + mutex_unlock(&ctx->dev->queue_mutex); +} + +void avd_queue_del_ctx(struct avd_ctx *ctx) +{ + avd_queue_drain(ctx); + mutex_lock(&ctx->dev->queue_mutex); + if (ctx->dev->compat_ctx == ctx) + ctx->dev->compat_ctx = NULL; + list_del_init(&ctx->queue_node); + mutex_unlock(&ctx->dev->queue_mutex); +} + +int avd_queue_init(struct avd_dev *avd) +{ + unsigned int i; + + spin_lock_init(&avd->queue_lock); + mutex_init(&avd->queue_mutex); + INIT_LIST_HEAD(&avd->contexts); + INIT_DELAYED_WORK(&avd->queue_work, avd_queue_worker); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + struct avd_frame *f = &avd->frames[i]; + + f->idx = i; + f->prog = devm_kzalloc(avd->dev, sizeof(*f->prog), GFP_KERNEL); + if (!f->prog) + return -ENOMEM; + } + return 0; +} + +void avd_queue_cleanup(struct avd_dev *avd) +{ + unsigned int i; + + mutex_lock(&avd->queue_mutex); + WRITE_ONCE(avd->stopping, true); + if (avd->inflight) { + avd_reset_queue(avd, -ESHUTDOWN); + avd_retire(avd); + } + mutex_unlock(&avd->queue_mutex); + cancel_delayed_work_sync(&avd->queue_work); + for (i = 0; i < AVD_MAX_FRAMES; i++) { + avd_buf_free(avd, avd->frames[i].prog); + if (avd->frame_fp[i]) + avd_buf_free(avd, avd->frame_fp[i]); + } +} diff --git a/drivers/media/platform/apple/avd/avd-queue.h b/drivers/media/platform/apple/avd/avd-queue.h new file mode 100644 index 0000000000000..cd912259c636a --- /dev/null +++ b/drivers/media/platform/apple/avd/avd-queue.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_QUEUE_H_ +#define AVD_QUEUE_H_ + +#include +#include +#include + +struct avd_dev; +struct avd_ctx; +struct avd_buf; /* avd.h: { void *cpu; dma_addr_t addr; size_t size; } */ + +#define AVD_MAX_FRAMES 16 /* SPEC 4.1: 16 frame-param slots */ + +enum avd_frame_state { + AVD_FRAME_FREE = 0, /* SPEC: slot state 0 */ + AVD_FRAME_RESERVED, /* 1: codec is filling it on the CPU */ + AVD_FRAME_SUBMITTED, /* 3: owned by firmware */ + AVD_FRAME_DONE, /* completion seen, waiting for in-order retire */ +}; + +/* Codec says what must finish before this frame may be submitted. */ +enum avd_frame_dep { + AVD_DEP_NONE = 0, /* independent: may overlap the previous frame */ + AVD_DEP_PREV_DONE, /* previous frame of this ctx must be DONE first + * (VP9 backward adaptation, AV1 CDF update, ...) */ +}; + +struct avd_frame { + unsigned int idx; /* 0..AVD_MAX_FRAMES-1, also the fp slot */ + u32 tag; /* (seq << 4) | idx, echoed by completion */ + u64 seq; /* global submission order */ + enum avd_frame_state state; + enum avd_frame_dep dep; + int status; /* 0 or -EIO/-ETIMEDOUT from the backend */ + + struct avd_ctx *ctx; + struct vb2_v4l2_buffer *src; /* OUTPUT, held until retire */ + struct vb2_v4l2_buffer *dst; /* CAPTURE, held until retire */ + u32 frame_num; /* SPEC 5.2 +0x0c */ + + /* open backend: instruction program, DMA-visible, built by the codec */ + struct avd_buf *prog; /* per-slot, allocated by core */ + size_t prog_words; + + /* apple backend: frame-params slot (SPEC 0x1130d0 on Borage) */ + struct avd_buf *fp; /* per-slot, allocated by core, size from match data */ + u32 cmd[24]; /* 0x60 Borage command; core fills the 5.2 prefix, + * codec pack_cmd() fills the codec part */ + + void *codec_priv; /* per-frame codec state, owned by the codec */ + unsigned long deadline; /* jiffies, per-frame watchdog */ + /* Legacy m2m HOLD_CAPTURE API retains scheduler ownership until retire. */ + bool scheduler_held; + /* VP to PP instruction FIFO for this frame (open backends). */ + dma_addr_t fifo_iova; +}; + +struct avd_backend_ops { + const char *name; + int (*probe)(struct avd_dev *avd); /* apertures, fw source, version */ + int (*boot)(struct avd_dev *avd); /* power-up: fw present/loaded, link up, ready */ + void (*shutdown)(struct avd_dev *avd); + unsigned int (*max_inflight)(struct avd_dev *avd); + int (*session_start)(struct avd_ctx *ctx); /* apple: cmd 0; open: may be NULL */ + void (*session_stop)(struct avd_ctx *ctx); /* apple: cmd 2 */ + /* Non-blocking. -EBUSY means inbox full: core retries on next completion. */ + int (*submit)(struct avd_dev *avd, struct avd_frame *f); + void (*abort)(struct avd_ctx *ctx); /* apple: cmd 5 */ + irqreturn_t (*irq)(struct avd_dev *avd); /* calls avd_frame_complete() */ + int (*suspend)(struct avd_dev *avd); + int (*resume)(struct avd_dev *avd); + int (*set_property)(struct avd_ctx *ctx, u32 property, u32 value); +}; + +/* Codec half. Replaces avd_coded_fmt_ops.run/submit/done. CPU only, no MMIO. */ +struct avd_codec_frame_ops { + /* Read the request's controls, set f->dep, fill f->prog (open backend) + * or f->fp (apple backend), allocate f->codec_priv if needed. */ + int (*prepare)(struct avd_ctx *ctx, struct avd_frame *f); + /* Apple backend: fill the codec part of f->cmd (SPEC 6.x). */ + int (*pack_cmd)(struct avd_ctx *ctx, struct avd_frame *f); + /* Called in submission order per ctx, after completion or failure. */ + void (*retire)(struct avd_ctx *ctx, struct avd_frame *f, + enum vb2_buffer_state state); +}; + +/* Core API used by backends and codecs. */ +void avd_frame_complete(struct avd_dev *avd, u32 tag, int status); /* any ctx, IRQ-safe */ +bool avd_backend_is_apple(const struct avd_dev *avd); + +#endif /* AVD_QUEUE_H_ */ diff --git a/drivers/media/platform/apple/avd/avd-regs.h b/drivers/media/platform/apple/avd/avd-regs.h index 388f3e64895f5..ace0e4c48f40a 100644 --- a/drivers/media/platform/apple/avd/avd-regs.h +++ b/drivers/media/platform/apple/avd/avd-regs.h @@ -13,6 +13,8 @@ #define AVD_MBOX1_NOT_EMPTY BIT(3) #define AVD_REG_MBOX1_STATUS 0x5c +/* Read side: bits 8-11 write pointer, bits 12-15 read pointer, bit 17 empty. */ +#define AVD_MBOX1_STATUS_EMPTY BIT(17) #define AVD_REG_MBOX1_RETRIEVE 0x64 #define AVD_MBOX_ENABLE BIT(0) diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index 4622e92055486..56f9f7e541df0 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -449,6 +449,7 @@ static const struct avd_coded_fmt_desc avd_coded_fmts[] = { }, .ctrls = &avd_av1_ctrls, .ops = &avd_av1_fmt_ops, + .frame_ops = &avd_av1_frame_ops, .num_decoded_fmts = ARRAY_SIZE(avd_av1_decoded_fmts), .decoded_fmts = avd_av1_decoded_fmts, .capability = AVD_CAPABILITY_AV1, @@ -893,8 +894,12 @@ static int avd_start_streaming(struct vb2_queue *q, unsigned int count) struct vb2_v4l2_buffer *vbuf; int ret; + WRITE_ONCE(ctx->draining, false); if (V4L2_TYPE_IS_CAPTURE(q->type)) return 0; + /* A new stream gets a fresh hardware error budget. */ + ctx->hw_errors = 0; + ctx->broken = false; desc = ctx->coded_fmt_desc; if (WARN_ON(!desc)) { @@ -917,7 +922,7 @@ static int avd_start_streaming(struct vb2_queue *q, unsigned int count) return ret; } -static void avd_queue_cleanup(struct vb2_queue *vq, u32 state) +static void avd_m2m_queue_cleanup(struct vb2_queue *vq, u32 state) { struct avd_ctx *ctx = vb2_get_drv_priv(vq); @@ -942,6 +947,7 @@ static void avd_stop_streaming(struct vb2_queue *q) { struct avd_ctx *ctx = vb2_get_drv_priv(q); + avd_queue_drain(ctx); if (V4L2_TYPE_IS_OUTPUT(q->type)) { const struct avd_coded_fmt_desc *desc = ctx->coded_fmt_desc; @@ -956,7 +962,7 @@ static void avd_stop_streaming(struct vb2_queue *q) ctx->job.segments = NULL; } - avd_queue_cleanup(q, VB2_BUF_STATE_ERROR); + avd_m2m_queue_cleanup(q, VB2_BUF_STATE_ERROR); } const struct vb2_ops avd_queue_ops = { @@ -970,36 +976,17 @@ const struct vb2_ops avd_queue_ops = { .stop_streaming = avd_stop_streaming, }; +/* Legacy multi-slice codecs finish held OUTPUT requests during prepare. */ void avd_job_finish_no_pm(struct avd_ctx *ctx, enum vb2_buffer_state result) { - struct avd_dev *avd = ctx->dev; - unsigned long flags; - - spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_ctx == ctx) { - avd->job_state = AVD_JOB_IDLE; - avd->job_ctx = NULL; - avd->job_pending = false; - } - spin_unlock_irqrestore(&avd->job_lock, flags); - - if (ctx->coded_fmt_desc->ops->done) { - struct vb2_v4l2_buffer *src_buf, *dst_buf; - - src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); - dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); - ctx->coded_fmt_desc->ops->done(ctx, src_buf, dst_buf, result); - } - - v4l2_m2m_buf_done_and_job_finish(ctx->dev->m2m_dev, ctx->fh.m2m_ctx, - result); + if (WARN_ON(!ctx->dev->building)) + return; + ctx->dev->compat_held = true; + ctx->dev->building->status = result == VB2_BUF_STATE_DONE ? 0 : -EIO; } void avd_job_finish(struct avd_ctx *ctx, enum vb2_buffer_state result) { - struct avd_dev *avd = ctx->dev; - - pm_runtime_put_autosuspend(avd->dev); avd_job_finish_no_pm(ctx, result); } diff --git a/drivers/media/platform/apple/avd/avd-vp9.c b/drivers/media/platform/apple/avd/avd-vp9.c index 4cdcb6594c4d2..37374b73e8a86 100644 --- a/drivers/media/platform/apple/avd/avd-vp9.c +++ b/drivers/media/platform/apple/avd/avd-vp9.c @@ -1049,6 +1049,13 @@ static enum avd_image_fmt avd_vp9_get_image_fmt(struct avd_ctx *ctx, #undef BIT_DEPTH } +static unsigned int avd_vp9_pp_kicks(struct avd_ctx *ctx) +{ + struct avd_vp9_ctx *priv = ctx->priv; + + return max_t(int, priv->submit_num, 1); +} + static void avd_vp9_submit(struct avd_ctx *ctx) { struct avd_vp9_ctx *vp9_ctx = ctx->priv; @@ -1078,5 +1085,6 @@ const struct avd_coded_fmt_ops avd_vp9_fmt_ops = { .run = avd_vp9_run, .done = avd_vp9_done, .submit = avd_vp9_submit, + .pp_kicks = avd_vp9_pp_kicks, .get_image_fmt = avd_vp9_get_image_fmt, }; diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index ad7accb6e9615..b946a320acc58 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -19,6 +19,12 @@ #include #include +#include "avd-pm.h" +#include + +#include "avd-queue.h" +#include "avd-av1-fp.h" + #include #include #include @@ -123,6 +129,8 @@ struct avd_decoded_buffer { /* Pending CPU fill, retained with the buffer until dequeue. */ size_t grey_chroma_offset; bool grey_chroma_10bit; + /* H.264/HEVC: scratch bank + 1 of the picture decoded into this buffer */ + u8 bank; union { struct avd_vp9_decoded_buffer_info vp9; @@ -144,6 +152,8 @@ struct avd_coded_fmt_ops { void (*adjust_decoded_fmt)(struct avd_ctx *ctx, struct v4l2_pix_format_mplane *pix_mp); void (*submit)(struct avd_ctx *ctx); + /* PP EXEC commands the last built frame needs (submit's count), >= 1. */ + unsigned int (*pp_kicks)(struct avd_ctx *ctx); int (*start)(struct avd_ctx *ctx); void (*stop)(struct avd_ctx *ctx); int (*run)(struct avd_ctx *ctx); @@ -173,20 +183,30 @@ struct avd_coded_fmt_desc { struct v4l2_frmsize_stepwise frmsize; const struct avd_ctrls *ctrls; const struct avd_coded_fmt_ops *ops; + const struct avd_codec_frame_ops *frame_ops; unsigned int num_decoded_fmts; const struct avd_decoded_fmt_desc *decoded_fmts; u32 subsystem_flags; unsigned int capability; }; +struct avd_apple_hw; + struct avd_variant { + enum avd_av1_fp_layout av1_fp_layout; + const struct avd_apple_hw *apple; + size_t fp_size; unsigned int vp_slots[4]; unsigned int vp_slot_base[4]; unsigned int fifo_slots; unsigned int capabilities; + /* Codecs advertised only when the firmware CODECS trailer lists them. */ + unsigned int fw_caps_required; void (*configure_stream)(struct avd_dev *avd, dma_addr_t addr, u8 fifo_idx, u32 vp_slot); const char* fw_name; + /* Older image tried when fw_name is not installed (legacy transport). */ + const char *fw_name_fallback; unsigned char revision; /* the same as the device tree */ /* just for convenience */ u32 vp_slot_offset; @@ -196,23 +216,16 @@ struct avd_variant { unsigned int quirks; }; -/* Lifecycle of the one m2m job the device runs at a time. */ -enum avd_job_state { - AVD_JOB_IDLE, - AVD_JOB_RUNNING, /* inside avd_device_run() */ - AVD_JOB_SUBMITTED, /* handed to the hardware, watchdog armed */ - AVD_JOB_FINISHING, /* claimed by the IRQ handler or the watchdog */ -}; - struct avd_dev { struct device *dev; + struct avd_pm pm; struct v4l2_device v4l2_dev; struct media_device mdev; struct video_device vdev; struct v4l2_m2m_dev *m2m_dev; struct platform_device *pdev; - const struct firmware *fw; /* fw is lost on suspend */ + const struct firmware *fw; /* separately installed open firmware */ void __iomem *code; void __iomem *mbox; @@ -223,19 +236,49 @@ struct avd_dev { struct mutex vdev_lock; - /* The current job; see avd_device_run(). Protected by job_lock. */ - spinlock_t job_lock; - enum avd_job_state job_state; - struct avd_ctx *job_ctx; - u64 job_seq; - bool job_end_sent; - bool job_pending; - enum vb2_buffer_state job_pending_result; + /* queue_lock protects slot state/tag and job_ready observations. */ + spinlock_t queue_lock; + /* Serializes CPU preparation, submission, retirement and reset. */ + struct mutex queue_mutex; + struct delayed_work queue_work; + struct list_head contexts; + /* One m2m dispatch reservation; never identifies firmware-owned work. */ + /* + * m2m runs one device_run at a time until job_finish; a job that finds + * no free slot waits here and is restarted when a frame retires. + */ + struct avd_ctx *deferred_ctx; + struct avd_frame frames[AVD_MAX_FRAMES]; + const struct avd_codec_frame_ops *frame_codec_ops[AVD_MAX_FRAMES]; + struct avd_buf *frame_fp[AVD_MAX_FRAMES]; + u64 frame_seq; + unsigned int inflight; + bool stopping; + bool faulted; + /* Recovery pacing: when the last one ran, and a sliding storm count. */ + unsigned long last_recover; + unsigned long storm_start; + unsigned int storm_count; + /* After a recovery storm, fail new frames until this time. */ + unsigned long cooldown_until; + u64 fw_generation; + const struct avd_backend_ops *backend; + void *backend_priv; + /* Only the open-firmware compatibility path uses these. */ + struct avd_frame *building; + /* Open firmware queue ABI 2 state, NULL when the node has no SRAM. */ + struct avd_qfw *qfw; + struct avd_frame *compat_active; + /* Context of the last legacy submission, for VP DONE without a frame. */ + struct avd_ctx *compat_ctx; + bool compat_held; struct reset_control *rstc; const struct avd_variant *variant; unsigned int capabilities; + /* What the loaded image's CODECS trailer (or its absence) allows. */ + unsigned int fw_capabilities; }; struct avd_segment { @@ -253,6 +296,7 @@ struct avd_buf { void *cpu; dma_addr_t addr; size_t size; + struct avd_dma_release *release; }; struct avd_ctx { @@ -267,14 +311,26 @@ struct avd_ctx { enum avd_image_fmt image_fmt; bool decomp; - struct delayed_work watchdog_work; - /* job_seq of the job watchdog_work was armed for */ - u64 wd_seq; + struct list_head queue_node; + struct work_struct submit_work; + bool draining; + /* Consecutive frames that failed in hardware; reset by a good frame. */ + unsigned int hw_errors; + /* Too many hardware errors: fail every frame until streaming restarts. */ + bool broken; + bool session_started; + u64 session_generation; + enum avd_frame_dep next_dep; + u32 client_id; void *priv; struct avd_comp comp; int fifo_idx; + /* Pictures of this context the queue may hold at once (0 or 1: one). */ + unsigned int max_frames; + /* VP to PP FIFO for the job being built; 0 means ctx->inst. */ + dma_addr_t job_fifo; struct avd_job job; struct avd_buf inst; }; @@ -306,6 +362,7 @@ extern const struct avd_coded_fmt_ops avd_h264_fmt_ops; extern const struct avd_coded_fmt_ops avd_hevc_fmt_ops; extern const struct avd_coded_fmt_ops avd_vp9_fmt_ops; extern const struct avd_coded_fmt_ops avd_av1_fmt_ops; +extern const struct avd_codec_frame_ops avd_av1_frame_ops; extern const struct v4l2_ctrl_ops avd_ctrl_ops; extern const struct v4l2_ioctl_ops avd_ioctl_ops; @@ -330,6 +387,38 @@ static inline struct avd_ctx *file_to_ctx(struct file *filp) /* hw stuff */ int avd_boot(struct avd_dev *avd); +int avd_queue_init(struct avd_dev *avd); +void avd_queue_cleanup(struct avd_dev *avd); +void avd_queue_run(void *priv); +int avd_queue_ready(void *priv); +void avd_queue_abort(void *priv); +void avd_queue_drain(struct avd_ctx *ctx); +void avd_queue_add_ctx(struct avd_ctx *ctx); +void avd_queue_del_ctx(struct avd_ctx *ctx); +int avd_open_build_program(struct avd_ctx *ctx); +extern const struct avd_backend_ops avd_open_backend; +extern const struct avd_backend_ops avd_apple_backend; +extern const struct avd_backend_ops avd_qfw_backend; +struct avd_qfw; +int avd_qfw_probe(struct avd_dev *avd); +void avd_qfw_select(struct avd_dev *avd); +void avd_qfw_prepare_boot(struct avd_dev *avd); +bool avd_backend_is_qfw(const struct avd_dev *avd); + +/* Open transports still need legacy segments and codec-specific PP kicks. */ +static inline const struct avd_codec_frame_ops * +avd_get_frame_ops(const struct avd_ctx *ctx) +{ + if (ctx->coded_fmt_desc->ops->run && !avd_backend_is_apple(ctx->dev)) + return NULL; + return ctx->coded_fmt_desc->frame_ops; +} + +void avd_open_av1_submit(struct avd_ctx *ctx, unsigned int kicks); + +int avd_backend_select(struct avd_dev *avd); +bool avd_mbox_pop(struct avd_dev *avd, u32 *msg); +int avd_backend_boot(struct avd_dev *avd); void avd_shutdown(struct avd_dev *avd); void avd_status(struct avd_dev *avd, u32 vp); diff --git a/drivers/media/platform/apple/avd/tests/.kunitconfig b/drivers/media/platform/apple/avd/tests/.kunitconfig new file mode 100644 index 0000000000000..c079bc44af531 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/.kunitconfig @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +CONFIG_KUNIT=y +CONFIG_COMPILE_TEST=y +CONFIG_MEDIA_SUPPORT=y +# CONFIG_MEDIA_SUPPORT_FILTER is not set +CONFIG_VIDEO_DEV=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_OF=y +CONFIG_VIDEO_APPLE_AVD=y +CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST=y +CONFIG_WERROR=y diff --git a/drivers/media/platform/apple/avd/tests/avd-av1-legacy.c b/drivers/media/platform/apple/avd/tests/avd-av1-legacy.c new file mode 100644 index 0000000000000..206bca6db5e4b --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/avd-av1-legacy.c @@ -0,0 +1,1060 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Apple AVD VPU codec driver + * + * Copyright The Asahi Linux Contributors + * Copyright 2023 Eileen Yoon + * + * Copyright (c) 2023, Collabora + * Benjamin Gaignard + * + * Copyright (c) 2014 Rockchip Electronics Co., Ltd. + * Hertz Wong + * Herman Chen + * + * Copyright (C) 2014 Google, Inc. + * Tomasz Figa + */ + +#include + +#include "avd.h" +#include "avd-av1-entropymode.h" +#include "avd-inst.h" + +/* + * This comes from + * verisilicon/rockchip_vpu981_hw_av1_dec.c + * + * These 3 values aren't defined enum v4l2_av1_segment_feature because + * they are not part of the specification + */ +#define V4L2_AV1_SEG_LVL_ALT_LF_Y_H 2 +#define V4L2_AV1_SEG_LVL_ALT_LF_U 3 +#define V4L2_AV1_SEG_LVL_ALT_LF_V 4 + +#define DIV_LUT_BITS 8 +#define DIV_LUT_PREC_BITS 14 +#define WARPEDMODEL_PREC_BITS 16 +#define WARP_PARAM_REDUCE_BITS 6 + +#define DIV_LUT_NUM BIT(DIV_LUT_BITS) + +#define AV1_DIV_ROUND_UP_POW2(value, n) \ +({ \ + typeof(n) _n = n; \ + typeof(value) _value = value; \ + (_value + (BIT(_n) >> 1)) >> _n; \ +}) + +#define AV1_DIV_ROUND_UP_POW2_SIGNED(value, n) \ +({ \ + typeof(n) _n_ = n; \ + typeof(value) _value_ = value; \ + (((_value_) < 0) ? -AV1_DIV_ROUND_UP_POW2(-(_value_), (_n_)) \ + : AV1_DIV_ROUND_UP_POW2((_value_), (_n_))); \ +}) + +#define AV1_REF_SCALE_SHIFT 14 +#define SUPERRES_SCALE_BITS 14 +#define SUPERRES_EXTRA_BITS 8 +#define SUPERRES_SCALE_MASK ((1 << 14) - 1) + +#define AV1_CODEC_MODE_INTRABC(v) FIELD_PREP(BIT(28), !!(v)) + +#define AV1_HDR_JNT_COMP(v) FIELD_PREP(BIT(0), !!(v)) +#define AV1_HDR_ORDER_HINT_BITS(v) FIELD_PREP(GENMASK(3, 1), v) +#define AV1_HDR_ORDER_HINT(v) FIELD_PREP(BIT(4), !!(v)) +#define AV1_HDR_DUAL_FILTER(v) FIELD_PREP(BIT(5), !!(v)) +#define AV1_HDR_MASKED_COMPOUND(v) FIELD_PREP(BIT(6), !!(v)) +#define AV1_HDR_INTERINTRA_COMPOUND(v) FIELD_PREP(BIT(7), !!(v)) +#define AV1_HDR_INTRA_EDGE_FILTER(v) FIELD_PREP(BIT(8), !!(v)) +#define AV1_HDR_FILTER_INTRA(v) FIELD_PREP(BIT(9), !!(v)) +#define AV1_HDR_128X128_SUPERBLOCK(v) FIELD_PREP(BIT(10), !!(v)) +#define AV1_HDR_SCREEN_CONTENT_TOOLS(v) FIELD_PREP(BIT(11), !!(v)) + +#define AV1_FLAGS_TX_MODE_LARGEST(v) FIELD_PREP(BIT(0), !!(v)) +#define AV1_FLAGS_TX_MODE_SELECT(v) FIELD_PREP(BIT(1), !!(v)) +#define AV1_FLAGS_TX_REDUCED_SET(v) FIELD_PREP(BIT(2), !!(v)) +#define AV1_FLAGS_ALLOW_WARPED_MOTION(v) FIELD_PREP(BIT(3), !!(v)) +#define AV1_FLAGS_SKIP_MODE(v) FIELD_PREP(BIT(4), !!(v)) +#define AV1_FLAGS_MOTION_IS_SWITCHABLE(v) FIELD_PREP(BIT(5), !!(v)) +#define AV1_FLAGS_INTRABC(v) FIELD_PREP(BIT(6), !!(v)) +#define AV1_FLAGS_INTEGER_MV(v) FIELD_PREP(BIT(7), !!(v)) +#define AV1_FLAGS_DISABLE_CDF_UPDATE(v) FIELD_PREP(BIT(8), !!(v)) +#define AV1_FLAGS_TX_MODE_ONLY_4X4(v) FIELD_PREP(BIT(9), !!(v)) +#define AV1_FLAGS_HIGH_PRECISION_MV(v) FIELD_PREP(BIT(10), !!(v)) +#define AV1_FLAGS_REF_FRAME_MVS(v) FIELD_PREP(BIT(12), !!(v)) +#define AV1_FLAGS_SEG_TEMPORAL_UPDATE(v) FIELD_PREP(BIT(13), !!(v)) +#define AV1_FLAGS_SEG_UPDATE_MAP(v) FIELD_PREP(BIT(14), !!(v)) +#define AV1_FLAGS_SEG_ENABLED(v) FIELD_PREP(BIT(15), !!(v)) +#define AV1_FLAGS_SEG_UPDATE_DATA(v) FIELD_PREP(BIT(16), !!(v)) +#define AV1_FLAGS_INTRA(v) FIELD_PREP(BIT(31), !!(v)) +#define AV1_FLAGS_SKIP_MODE1(v) FIELD_PREP(GENMASK(19, 17), v) +#define AV1_FLAGS_SKIP_MODE0(v) FIELD_PREP(GENMASK(22, 20), v) +#define AV1_FLAGS_ORDER_HINT(v) FIELD_PREP(GENMASK(30, 23), v) + +#define AV1_FLAGS_REFERENCE_SELECT(v) FIELD_PREP(BIT(28), !!(v)) +#define AV1_FLAGS_INTERPOLATION_FILTER(v) FIELD_PREP(GENMASK(31, 29), v) + +#define AV1_SIGN_BIAS(v, i) (!!(v) << (i)) +#define AV1_REF_SLOT(v, i) (((v) & 7) << (7 + ((i) * 3))) + +#define AV1_SEG_ALT_Q(v) FIELD_PREP(GENMASK(8, 0), v) +#define AV1_SEG_ALT_Q_EN(v) FIELD_PREP(BIT(9), !!(v)) +#define AV1_SEG_REF_FRAME(v) FIELD_PREP(GENMASK(12, 10), v) +#define AV1_SEG_REF_FRAME_EN(v) FIELD_PREP(BIT(13), !!(v)) +#define AV1_SEG_REF_SKIP_EN(v) FIELD_PREP(BIT(14), !!(v)) +#define AV1_SEG_REF_GMV_EN(v) FIELD_PREP(BIT(15), !!(v)) + +#define AV1_SEG_LF_V(v) FIELD_PREP(GENMASK(6, 0), v) +#define AV1_SEG_LF_V_EN(v) FIELD_PREP(BIT(7), !!(v)) +#define AV1_SEG_LF_U(v) FIELD_PREP(GENMASK(14, 8), v) +#define AV1_SEG_LF_U_EN(v) FIELD_PREP(BIT(15), !!(v)) +#define AV1_SEG_LF_Y_H(v) FIELD_PREP(GENMASK(22, 16), v) +#define AV1_SEG_LF_Y_H_EN(v) FIELD_PREP(BIT(23), !!(v)) +#define AV1_SEG_LF_Y_V(v) FIELD_PREP(GENMASK(30, 24), v) +#define AV1_SEG_LF_Y_V_EN(v) FIELD_PREP(BIT(31), !!(v)) + +#define AV1_SEG_UNK0(v) FIELD_PREP(BIT(16), !!(v)) + +#define AV1_QP_U_AC(v) FIELD_PREP(GENMASK(6, 0), v) +#define AV1_QP_U_DC(v) FIELD_PREP(GENMASK(13, 7), v) +#define AV1_QP_Y_DC(v) FIELD_PREP(GENMASK(20, 14), v) +#define AV1_QP_BASE_IDX(v) FIELD_PREP(GENMASK(28, 21), v) +#define AV1_QP_DELTA_RES(v) FIELD_PREP(GENMASK(30, 29), v) +#define AV1_QP_PRESENT(v) FIELD_PREP(BIT(31), !!(v)) +#define AV1_QP_V_DC(v) FIELD_PREP(GENMASK(26, 20), v) +#define AV1_QP_V_AC(v) FIELD_PREP(GENMASK(19, 13), v) +#define AV1_QP_QMATRIX(v) FIELD_PREP(BIT(12), !!(v)) +#define AV1_QP_QM_Y(v) FIELD_PREP(GENMASK(11, 8), v) +#define AV1_QP_QM_U(v) FIELD_PREP(GENMASK(7, 4), v) +#define AV1_QP_QM_V(v) FIELD_PREP(GENMASK(3, 0), v) + +#define AV1_GM_VALID(v) FIELD_PREP(BIT(30), !!(v)) +#define AV1_GM_TYPE(v) FIELD_PREP(GENMASK(31, 30), v) +#define AV1_GM_PARAM0(v) FIELD_PREP(GENMASK(29, 15), v) +#define AV1_GM_PARAM1(v) FIELD_PREP(GENMASK(14, 0), v) + +#define AV1_GM_PARAM_SHEAR0(v) FIELD_PREP(GENMASK(15, 0), v) +#define AV1_GM_PARAM_SHEAR1(v) FIELD_PREP(GENMASK(31, 16), v) + +#define AV1_ORDER_HINT0(v) FIELD_PREP(GENMASK(7, 0), v) +#define AV1_ORDER_HINT1(v) FIELD_PREP(GENMASK(15, 8), v) +#define AV1_ORDER_HINT2(v) FIELD_PREP(GENMASK(23, 16), v) +#define AV1_ORDER_HINT3(v) FIELD_PREP(GENMASK(31, 24), v) +#define AV1_ORDER_HINT_REF(v) FIELD_PREP(GENMASK(31, 29), v) + +#define AV1_LF_DELTA_ENABLED(v) FIELD_PREP(BIT(27), !!(v)) +#define AV1_LF_DELTA_MULTI(v) FIELD_PREP(BIT(28), !!(v)) +#define AV1_LF_DELTA_PRESENT(v) FIELD_PREP(BIT(31), !!(v)) +#define AV1_LF_DELTA_RES(v) FIELD_PREP(GENMASK(30, 29), v) +#define AV1_LF_LV3(v) FIELD_PREP(GENMASK(5, 0), v) +#define AV1_LF_LV2(v) FIELD_PREP(GENMASK(11, 6), v) +#define AV1_LF_LV1(v) FIELD_PREP(GENMASK(17, 12), v) +#define AV1_LF_LV0(v) FIELD_PREP(GENMASK(23, 18), v) + +#define AV1_LF_SHARPNESS(v) FIELD_PREP(GENMASK(31, 28), v) +#define AV1_LF_REF0(v) FIELD_PREP(GENMASK(27, 21), v) +#define AV1_LF_REF1(v) FIELD_PREP(GENMASK(20, 14), v) +#define AV1_LF_REF2(v) FIELD_PREP(GENMASK(13, 7), v) +#define AV1_LF_REF3(v) FIELD_PREP(GENMASK(6, 0), v) + +#define AV1_LF_MODE0(v) FIELD_PREP(GENMASK(13, 7), v) +#define AV1_LF_MODE1(v) FIELD_PREP(GENMASK(6, 0), v) + +#define AV1_CDEF_EN(v) FIELD_PREP(BIT(28), !!(v)) +#define AV1_CDEF_BITS(v) FIELD_PREP(GENMASK(25, 24), v) +#define AV1_CDEF_DAMPING(v) FIELD_PREP(GENMASK(27, 26), v) + +#define AV1_CDEF_Y_PRI(v) FIELD_PREP(GENMASK(11, 8), v) +#define AV1_CDEF_Y_SEC(v) FIELD_PREP(GENMASK(7, 6), v) +#define AV1_CDEF_UV_PRI(v) FIELD_PREP(GENMASK(5, 2), v) +#define AV1_CDEF_UV_SEC(v) FIELD_PREP(GENMASK(1, 0), v) + +#define AV1_CDEF_LO(v) FIELD_PREP(GENMASK(11, 0), v) +#define AV1_CDEF_HI(v) FIELD_PREP(GENMASK(23, 12), v) + +#define AV1_LR_TYPE0(v) FIELD_PREP(GENMASK(11, 10), v) +#define AV1_LR_TYPE1(v) FIELD_PREP(GENMASK(9, 8), v) +#define AV1_LR_TYPE2(v) FIELD_PREP(GENMASK(7, 6), v) + +#define AV1_LR_UNIT2(v) FIELD_PREP(GENMASK(1, 0), v) +#define AV1_LR_UNIT1(v) FIELD_PREP(GENMASK(3, 2), v) +#define AV1_LR_UNIT0(v) FIELD_PREP(GENMASK(5, 4), v) + +#define AV1_REF_ORDER_HINT(v) FIELD_PREP(GENMASK(23, 0), v) +#define AV1_REF_SCALE_X(v) FIELD_PREP(GENMASK(31, 16), v) +#define AV1_REF_SCALE_Y(v) FIELD_PREP(GENMASK(15, 0), v) + +#define AV1_SUPERRES_USE(v) FIELD_PREP(BIT(31), !!(v)) +#define AV1_SUPERRES_DENOM(v) FIELD_PREP(GENMASK(30, 28), v) +/* im gonna asume 16 like the others */ +#define AV1_SUPERRES_UPSCALE(v) FIELD_PREP(GENMASK(16, 0), v) + +#define AV1_UPSCALE_STEPX(v) FIELD_PREP(GENMASK(13, 0), v) +#define AV1_UPSCALE_UPSCX(v) FIELD_PREP(GENMASK(31, 14), v) + +#define AVD_CDFS_SIZE sizeof(struct avd_av1_cdfs) + +/* Imported planes can have an unaligned length; AVD aligns both tail addresses. */ +#define AVD_AV1_COLOR_OFFSET(dst, cl) \ + (ALIGN_DOWN((dst), AVD_ALIGN) - ALIGN(cl, AVD_ALIGN)) +#define AVD_AV1_CDFS_OFFSET(dst, cl) \ + (AVD_AV1_COLOR_OFFSET(dst, cl) - ALIGN(AVD_CDFS_SIZE, AVD_ALIGN)) +/* The frame's segmentation map, saved with it like the CDFs (AV1 7.20) */ +#define AVD_AV1_SEG_OFFSET(dst, cl, sl) \ + (AVD_AV1_CDFS_OFFSET(dst, cl) - (sl)) + +#define AV1_PRIMARY_REF_NONE 7 + +struct avd_av1_run { + struct avd_run base; + + struct { + dma_addr_t probs_out; + dma_addr_t color; + dma_addr_t seg; + dma_addr_t prev_seg; + } addresses; + + const struct v4l2_ctrl_av1_sequence *seq; + const struct v4l2_ctrl_av1_frame *frame; + const struct v4l2_ctrl_av1_tile_group_entry *tile_group; + const struct v4l2_ctrl_av1_film_grain *grain; +}; + +struct avd_av1_ctx { + u8 submit_num; + struct { + struct avd_buf inst; + struct avd_buf pipe_state; + struct avd_buf probs; + struct avd_buf no_prev_seg; + struct avd_buf above_info; + struct avd_buf rf_above_info; + struct avd_buf az_above; + struct avd_buf ip_above; + struct avd_buf lf_above; + struct avd_buf lf_above_info; + struct avd_buf lf_left; + struct avd_buf lf_left_info; + struct avd_buf sr_left; + struct avd_buf rf_left; + struct avd_buf rf_left_info; + struct avd_buf mv_above_info; + } bufs; +}; + +/* + * from verisilicon/rockchip_vpu981_hw_av1_dec.c + */ +static const short div_lut[DIV_LUT_NUM + 1] = { + 16384, 16320, 16257, 16194, 16132, 16070, 16009, 15948, 15888, 15828, 15768, + 15709, 15650, 15592, 15534, 15477, 15420, 15364, 15308, 15252, 15197, 15142, + 15087, 15033, 14980, 14926, 14873, 14821, 14769, 14717, 14665, 14614, 14564, + 14513, 14463, 14413, 14364, 14315, 14266, 14218, 14170, 14122, 14075, 14028, + 13981, 13935, 13888, 13843, 13797, 13752, 13707, 13662, 13618, 13574, 13530, + 13487, 13443, 13400, 13358, 13315, 13273, 13231, 13190, 13148, 13107, 13066, + 13026, 12985, 12945, 12906, 12866, 12827, 12788, 12749, 12710, 12672, 12633, + 12596, 12558, 12520, 12483, 12446, 12409, 12373, 12336, 12300, 12264, 12228, + 12193, 12157, 12122, 12087, 12053, 12018, 11984, 11950, 11916, 11882, 11848, + 11815, 11782, 11749, 11716, 11683, 11651, 11619, 11586, 11555, 11523, 11491, + 11460, 11429, 11398, 11367, 11336, 11305, 11275, 11245, 11215, 11185, 11155, + 11125, 11096, 11067, 11038, 11009, 10980, 10951, 10923, 10894, 10866, 10838, + 10810, 10782, 10755, 10727, 10700, 10673, 10645, 10618, 10592, 10565, 10538, + 10512, 10486, 10460, 10434, 10408, 10382, 10356, 10331, 10305, 10280, 10255, + 10230, 10205, 10180, 10156, 10131, 10107, 10082, 10058, 10034, 10010, 9986, + 9963, 9939, 9916, 9892, 9869, 9846, 9823, 9800, 9777, 9754, 9732, + 9709, 9687, 9664, 9642, 9620, 9598, 9576, 9554, 9533, 9511, 9489, + 9468, 9447, 9425, 9404, 9383, 9362, 9341, 9321, 9300, 9279, 9259, + 9239, 9218, 9198, 9178, 9158, 9138, 9118, 9098, 9079, 9059, 9039, + 9020, 9001, 8981, 8962, 8943, 8924, 8905, 8886, 8867, 8849, 8830, + 8812, 8793, 8775, 8756, 8738, 8720, 8702, 8684, 8666, 8648, 8630, + 8613, 8595, 8577, 8560, 8542, 8525, 8508, 8490, 8473, 8456, 8439, + 8422, 8405, 8389, 8372, 8355, 8339, 8322, 8306, 8289, 8273, 8257, + 8240, 8224, 8208, 8192, +}; + +static inline int avd_av1_dec_get_msb(u32 n) +{ + if (n == 0) + return 0; + return 31 ^ __builtin_clz(n); +} + +static short avd_av1_dec_resolve_divisor_32(u32 d, short *shift) +{ + int f; + u64 e; + + *shift = avd_av1_dec_get_msb(d); + /* e is obtained from D after resetting the most significant 1 bit. */ + e = d - ((u32)1 << *shift); + /* Get the most significant DIV_LUT_BITS (8) bits of e into f */ + if (*shift > DIV_LUT_BITS) + f = AV1_DIV_ROUND_UP_POW2(e, *shift - DIV_LUT_BITS); + else + f = e << (DIV_LUT_BITS - *shift); + if (f > DIV_LUT_NUM) + return -1; + *shift += DIV_LUT_PREC_BITS; + /* Use f as lookup into the precomputed table of multipliers */ + return div_lut[f]; +} + +static void +avd_av1_dec_get_shear_params(const u32 *params, s64 *alpha, + s64 *beta, s64 *gamma, s64 *delta) +{ + const int *mat = params; + short shift; + short y; + long long gv, dv; + + if (mat[2] <= 0) + return; + + *alpha = clamp_val(mat[2] - (1 << WARPEDMODEL_PREC_BITS), S16_MIN, S16_MAX); + *beta = clamp_val(mat[3], S16_MIN, S16_MAX); + + y = avd_av1_dec_resolve_divisor_32(abs(mat[2]), &shift) * (mat[2] < 0 ? -1 : 1); + + gv = ((long long)mat[4] * (1 << WARPEDMODEL_PREC_BITS)) * y; + + *gamma = clamp_val((int)AV1_DIV_ROUND_UP_POW2_SIGNED(gv, shift), S16_MIN, S16_MAX); + + dv = ((long long)mat[3] * mat[4]) * y; + *delta = clamp_val(mat[5] - + (int)AV1_DIV_ROUND_UP_POW2_SIGNED(dv, shift) - (1 << WARPEDMODEL_PREC_BITS), + S16_MIN, S16_MAX); + + *alpha = AV1_DIV_ROUND_UP_POW2_SIGNED(*alpha, WARP_PARAM_REDUCE_BITS) + * (1 << WARP_PARAM_REDUCE_BITS); + *beta = AV1_DIV_ROUND_UP_POW2_SIGNED(*beta, WARP_PARAM_REDUCE_BITS) + * (1 << WARP_PARAM_REDUCE_BITS); + *gamma = AV1_DIV_ROUND_UP_POW2_SIGNED(*gamma, WARP_PARAM_REDUCE_BITS) + * (1 << WARP_PARAM_REDUCE_BITS); + *delta = AV1_DIV_ROUND_UP_POW2_SIGNED(*delta, WARP_PARAM_REDUCE_BITS) + * (1 << WARP_PARAM_REDUCE_BITS); +} + +static void set_refs(struct avd_ctx *ctx, struct avd_av1_run *run) +{ + struct avd_av1_ctx *av1_ctx = ctx->priv; + const struct v4l2_ctrl_av1_frame *frame = run->frame; + const struct v4l2_av1_global_motion *gm = &frame->global_motion; + int i, ref_idx; + struct avd_decoded_buffer *dst, *ref; + bool intrabc = !!(frame->flags & V4L2_AV1_FRAME_FLAG_ALLOW_INTRABC); + int height, width, x_scale, y_scale; + height = frame->frame_height_minus_1 + 1; + width = frame->frame_width_minus_1 + 1; + dma_addr_t addr; + s64 alpha = 0, beta = 0, gamma = 0, delta = 0; + dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); + + push(0, "ref_cnst0"); + pusha(av1_ctx->bufs.mv_above_info.addr, "mv_above_info", 0); + + for (i = 0; i < 4; i++) + push(0, "ref_cnst1"); + + for (i = 0; i < (intrabc ? 1 : V4L2_AV1_REFS_PER_FRAME); i++) { + if (intrabc) { + ref = dst; + } else { + ref_idx = i + V4L2_AV1_REF_LAST_FRAME; + ref = avd_get_ref_buf( + ctx, &dst->base.vb, + frame->reference_frame_ts + [frame->ref_frame_idx[i]]); + } + addr = vb2_dma_contig_plane_dma_addr(&ref->base.vb.vb2_buf, 0) + + ref->comp.start_offset; + + if (!intrabc) { + int shift = + (gm->flags[ref_idx] & + V4L2_AV1_GLOBAL_MOTION_FLAG_IS_TRANSLATION) ? + (WARPEDMODEL_PREC_BITS - + 2) - (frame->flags & V4L2_AV1_FRAME_FLAG_ALLOW_HIGH_PRECISION_MV ? + 1 : + 0) : + 10; + push(AV1_GM_TYPE(gm->type[ref_idx]) | + AV1_GM_PARAM0(gm->params[ref_idx][0] >> + shift) | + AV1_GM_PARAM1(gm->params[ref_idx][1] >> + shift), + "ref_gm_mv"); + push(AV1_GM_VALID(!(V4L2_AV1_GLOBAL_MOTION_IS_INVALID( + ref_idx) & + gm->invalid)) | + AV1_GM_PARAM0(AV1_DIV_ROUND_UP_POW2_SIGNED( + gm->params[ref_idx][2], 1)) | + AV1_GM_PARAM1(AV1_DIV_ROUND_UP_POW2_SIGNED( + gm->params[ref_idx][3], 1)), + "ref_gm_param"); + push(AV1_GM_PARAM1(gm->params[ref_idx][5] >> 1) | + AV1_GM_PARAM0(gm->params[ref_idx][4] >> 1), + "ref_gm_unk2"); + avd_av1_dec_get_shear_params(&gm->params[ref_idx][0], + &alpha, &beta, &gamma, + &delta); + push(AV1_GM_PARAM_SHEAR0(beta) | + AV1_GM_PARAM_SHEAR1(alpha), + "ref_gm_beta_alpha"); + + push(AV1_GM_PARAM_SHEAR0(delta) | + AV1_GM_PARAM_SHEAR1(gamma), + "ref_gm_delta_gamma"); + } + push(AVD_REF_FLAG_CONST | + AV1_REF_ORDER_HINT( + intrabc ? 0 : frame->order_hints[ref_idx]), + "ref_flag"); + push(AVD_HDR_HEIGHT(ref->av1.height) | + AVD_HDR_WIDTH(ref->av1.width), + "ref_height_width"); + + x_scale = ((ref->av1.upscaled_width << AV1_REF_SCALE_SHIFT) + + (width / 2)) / + width; + y_scale = (((ref->av1.height + 1) << AV1_REF_SCALE_SHIFT) + + (height / 2)) / + height; + push(AV1_REF_SCALE_X(x_scale) | AV1_REF_SCALE_Y(y_scale), + "ref_scale"); + + push_comp(ctx, addr, ref->comp.offsets); + } +} + +static int avd_av1_get_dist(struct avd_av1_run *run, int a, int b) +{ + const struct v4l2_ctrl_av1_sequence *seq = run->seq; + int bits = seq->order_hint_bits - 1; + int diff, m; + + if (!seq->order_hint_bits) + return 0; + + diff = a - b; + m = 1 << bits; + diff = (diff & (m - 1)) - (diff & m); + + return diff; +} + +static bool is_valid_ref_frame(struct avd_ctx *ctx, struct avd_av1_run *run, + struct avd_decoded_buffer *dst, int mi_cols, + int mi_rows, + enum v4l2_av1_reference_frame ref_frame) +{ + const struct v4l2_ctrl_av1_frame *frame = run->frame; + struct avd_decoded_buffer *ref = avd_get_ref_buf( + ctx, &dst->base.vb, + frame->reference_frame_ts + [frame->ref_frame_idx[ref_frame - + V4L2_AV1_REF_LAST_FRAME]]); + int ref_mi_cols = DIV_ROUND_UP(ref->av1.width + 1, 8); + int ref_mi_rows = DIV_ROUND_UP(ref->av1.height + 1, 8); + bool intra_only = ((ref->av1.frame_type == V4L2_AV1_KEY_FRAME) || + (ref->av1.frame_type == V4L2_AV1_INTRA_ONLY_FRAME)); + + return ref != dst && ref_mi_cols == mi_cols && ref_mi_rows == mi_rows && + !intra_only; +} + +static int avd_select_refs(struct avd_ctx *ctx, struct avd_av1_run *run, + u8 *selected_refs) +{ + const struct v4l2_ctrl_av1_frame *frame = run->frame; + const struct v4l2_ctrl_av1_sequence *seq = run->seq; + int ref_stamp = 2; + struct avd_decoded_buffer *dst, *ref; + bool use_last; + int ref_num = 0; + int order_hint = frame->order_hint; + int mi_cols = DIV_ROUND_UP(frame->frame_width_minus_1 + 1, 8); + int mi_rows = DIV_ROUND_UP(frame->frame_height_minus_1 + 1, 8); + dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); + + int last_ref_idx = frame->ref_frame_idx[V4L2_AV1_REF_INTRA_FRAME]; + if (seq->flags & (V4L2_AV1_SEQUENCE_FLAG_ENABLE_ORDER_HINT | + V4L2_AV1_SEQUENCE_FLAG_ENABLE_REF_FRAME_MVS)) { + ref = avd_get_ref_buf(ctx, &dst->base.vb, + frame->reference_frame_ts[last_ref_idx]); + if (dst != ref) { + use_last = + (ref->av1.order_hints[V4L2_AV1_REF_ALTREF_FRAME] != + frame->order_hints[V4L2_AV1_REF_GOLDEN_FRAME]); + if (use_last && + is_valid_ref_frame(ctx, run, dst, mi_cols, mi_rows, + V4L2_AV1_REF_LAST_FRAME)) + selected_refs[ref_num++] = + V4L2_AV1_REF_LAST_FRAME; + ref_stamp--; + } + + if (avd_av1_get_dist( + run, frame->order_hints[V4L2_AV1_REF_BWDREF_FRAME], + order_hint) > 0 && + is_valid_ref_frame(ctx, run, dst, mi_cols, mi_rows, + V4L2_AV1_REF_BWDREF_FRAME)) { + selected_refs[ref_num++] = V4L2_AV1_REF_BWDREF_FRAME; + ref_stamp--; + } + + if (avd_av1_get_dist( + run, frame->order_hints[V4L2_AV1_REF_ALTREF2_FRAME], + order_hint) > 0 && + is_valid_ref_frame(ctx, run, dst, mi_cols, mi_rows, + V4L2_AV1_REF_ALTREF2_FRAME)) { + selected_refs[ref_num++] = V4L2_AV1_REF_ALTREF2_FRAME; + ref_stamp--; + } + + if (ref_stamp >= 0 && + avd_av1_get_dist( + run, frame->order_hints[V4L2_AV1_REF_ALTREF_FRAME], + order_hint) > 0 && + is_valid_ref_frame(ctx, run, dst, mi_cols, mi_rows, + V4L2_AV1_REF_ALTREF_FRAME)) { + selected_refs[ref_num++] = V4L2_AV1_REF_ALTREF_FRAME; + ref_stamp--; + } + + if (ref_stamp >= 0 && + is_valid_ref_frame(ctx, run, dst, mi_cols, mi_rows, + V4L2_AV1_REF_LAST2_FRAME)) { + selected_refs[ref_num++] = V4L2_AV1_REF_LAST2_FRAME; + ref_stamp--; + } + } + + return ref_num; +} + +static void set_ref_hints(struct avd_ctx *ctx, struct avd_av1_run *run, + u8 *selected_refs) +{ + const struct v4l2_ctrl_av1_frame *frame = run->frame; + int i, ref_idx; + struct avd_decoded_buffer *dst, *ref; + dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); + + for (i = 0; i < 3; i++) { + if (selected_refs[i] < V4L2_AV1_REF_LAST_FRAME) { + push(0, ""); + push(0, ""); + } else { + ref_idx = frame->ref_frame_idx[selected_refs[i] - + V4L2_AV1_REF_LAST_FRAME]; + ref = avd_get_ref_buf( + ctx, &dst->base.vb, + frame->reference_frame_ts[ref_idx]); + + push(AV1_ORDER_HINT0(ref->av1.order_hints[3]) | + AV1_ORDER_HINT1(ref->av1.order_hints[2]) | + AV1_ORDER_HINT2(ref->av1.order_hints[1]) | + AV1_ORDER_HINT_REF( + (frame->flags & + V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS) ? + selected_refs[i] : + 0), + "order_hints0"); + + push(AV1_ORDER_HINT0(ref->av1.order_hints[7]) | + AV1_ORDER_HINT1(ref->av1.order_hints[6]) | + AV1_ORDER_HINT2(ref->av1.order_hints[5]) | + AV1_ORDER_HINT3(ref->av1.order_hints[4]), + "order_hints1"); + } + } +} + +static void set_ref_hdr(struct avd_ctx *ctx, struct avd_av1_run *run) +{ + const struct v4l2_ctrl_av1_frame *frame = run->frame; + u8 ref_slots[7] = { 1, 2, 3, 4, 5, 6, 7 }; + int sign_bias = 0; + int ref_slot = 0; + int i, j; + /* point to the first index where the frames equal */ + for (i = 0; i < 6; i++) { + for (j = i + 1; j < 7; j++) { + if (frame->reference_frame_ts[frame->ref_frame_idx[i]] == + frame->reference_frame_ts[frame->ref_frame_idx[j]]) { + ref_slots[j] = ref_slots[i]; + break; + } + } + } + for (i = 0; i < 7; i++) { + sign_bias |= AV1_SIGN_BIAS( + avd_av1_get_dist( + run, + frame->order_hints[i + V4L2_AV1_REF_LAST_FRAME], + frame->order_hint) > 0, + 6 - i); + + ref_slot |= AV1_REF_SLOT(ref_slots[i], 6 - i); + } + push(ref_slot | sign_bias | + AV1_FLAGS_REFERENCE_SELECT( + frame->flags & + V4L2_AV1_FRAME_FLAG_REFERENCE_SELECT) | + AV1_FLAGS_INTERPOLATION_FILTER( + frame->interpolation_filter), + "reference_select"); +} + +/* 7.16. Upscaling process */ +static void calc_upscale(int frame_width, int upscaled_width, int sub_x, + int *step_x, int *initial_subpel_x) +{ + int downscaled_plane_w, upscaled_plane_w, err; + + downscaled_plane_w = AV1_DIV_ROUND_UP_POW2(frame_width, sub_x); + upscaled_plane_w = AV1_DIV_ROUND_UP_POW2(upscaled_width, sub_x); + *step_x = ((downscaled_plane_w << SUPERRES_SCALE_BITS) + + (upscaled_plane_w / 2)) / + upscaled_plane_w; + err = (upscaled_plane_w * *step_x) - + (downscaled_plane_w << SUPERRES_SCALE_BITS); + *initial_subpel_x = (-((upscaled_plane_w - downscaled_plane_w) + << (SUPERRES_SCALE_BITS - 1)) + + upscaled_plane_w / 2) / + upscaled_plane_w + + (1 << (SUPERRES_EXTRA_BITS - 1)) - err / 2; + *initial_subpel_x &= SUPERRES_SCALE_MASK; +} + +static void set_header(struct avd_ctx *ctx, struct avd_av1_run *run) +{ + struct avd_av1_ctx *av1_ctx = ctx->priv; + const struct v4l2_ctrl_av1_frame *frame = run->frame; + const struct v4l2_ctrl_av1_sequence *seq = run->seq; + const struct v4l2_av1_loop_restoration *lr = &frame->loop_restoration; + const struct v4l2_av1_cdef *cdef = &frame->cdef; + const struct v4l2_av1_segmentation *seg = &frame->segmentation; + u32 bytesperline; + int i, segid, segval, ref_idx; + int step_x, initial_subpel_x; + u8 selected_refs[3] = { 0, 0, 0 }; + bool coded_lossless = frame->tx_mode == V4L2_AV1_TX_MODE_ONLY_4X4; + bool intrabc = !!(frame->flags & V4L2_AV1_FRAME_FLAG_ALLOW_INTRABC); + bool intra_only = !!((frame->frame_type == V4L2_AV1_KEY_FRAME || + frame->frame_type == V4L2_AV1_INTRA_ONLY_FRAME)); + bool cdef_enabled = + !(!(seq->flags & V4L2_AV1_SEQUENCE_FLAG_ENABLE_CDEF) || + frame->flags & V4L2_AV1_FRAME_FLAG_ALLOW_INTRABC || + coded_lossless); + + struct avd_decoded_buffer *dst, *ref; + dma_addr_t ref_addr; + dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); + + push(AVD_OP_HDR | AVD_OP_HDR_FLAG_DECOMP(ctx->decomp) | + AVD_OP_HDR_FLAG_INTRA(intra_only && !intrabc) | + AVD_OP_HDR_CONST | AVD_OP_HDR_FLAG_PIPE_STATE(1), + "op_hdr"); + + push(AVD_HDR_CODEC_MODE(AVD_CODEC_AV1) | + AV1_CODEC_MODE_INTRABC(intrabc), + "codec"); + + /* + * TODO: height_width_* are kinda weird + * on most samples this is true + */ + push(AVD_HDR_HEIGHT(frame->frame_height_minus_1) | + AVD_HDR_WIDTH(frame->frame_width_minus_1), + "height_width_1"); + push(0, ""); + push(AVD_HDR_HEIGHT(frame->frame_height_minus_1) | + AVD_HDR_WIDTH(frame->frame_width_minus_1), + "height_width_2"); + + push(AVD_HDR_COMMON_CHROMA_FORMAT(1) | + AVD_HDR_COMMON_BIT_DEPTH_C(seq->bit_depth - 8) | + AVD_HDR_COMMON_BIT_DEPTH_L(seq->bit_depth - 8) | + AV1_HDR_JNT_COMP(seq->flags & + V4L2_AV1_SEQUENCE_FLAG_ENABLE_JNT_COMP) | + AV1_HDR_DUAL_FILTER( + seq->flags & + V4L2_AV1_SEQUENCE_FLAG_ENABLE_DUAL_FILTER) | + AV1_HDR_ORDER_HINT_BITS((seq->order_hint_bits - 1)) | + AV1_HDR_ORDER_HINT( + seq->flags & + V4L2_AV1_SEQUENCE_FLAG_ENABLE_ORDER_HINT) | + AV1_HDR_MASKED_COMPOUND( + seq->flags & + V4L2_AV1_SEQUENCE_FLAG_ENABLE_MASKED_COMPOUND) | + AV1_HDR_INTERINTRA_COMPOUND( + seq->flags & + V4L2_AV1_SEQUENCE_FLAG_ENABLE_INTERINTRA_COMPOUND) | + AV1_HDR_INTRA_EDGE_FILTER( + seq->flags & + V4L2_AV1_SEQUENCE_FLAG_ENABLE_INTRA_EDGE_FILTER) | + AV1_HDR_FILTER_INTRA( + seq->flags & + V4L2_AV1_SEQUENCE_FLAG_ENABLE_FILTER_INTRA) | + AV1_HDR_128X128_SUPERBLOCK( + seq->flags & + V4L2_AV1_SEQUENCE_FLAG_USE_128X128_SUPERBLOCK) | + AV1_HDR_SCREEN_CONTENT_TOOLS( + frame->flags & + V4L2_AV1_FRAME_FLAG_ALLOW_SCREEN_CONTENT_TOOLS), + "hdr_common"); + + push(AV1_FLAGS_TX_MODE_LARGEST(frame->tx_mode == + V4L2_AV1_TX_MODE_LARGEST) | + AV1_FLAGS_TX_MODE_SELECT(frame->tx_mode == + V4L2_AV1_TX_MODE_SELECT) | + AV1_FLAGS_TX_REDUCED_SET( + frame->flags & + V4L2_AV1_FRAME_FLAG_REDUCED_TX_SET) | + AV1_FLAGS_ALLOW_WARPED_MOTION( + frame->flags & + V4L2_AV1_FRAME_FLAG_ALLOW_WARPED_MOTION) | + AV1_FLAGS_SKIP_MODE( + frame->flags & + V4L2_AV1_FRAME_FLAG_SKIP_MODE_PRESENT) | + AV1_FLAGS_MOTION_IS_SWITCHABLE( + frame->flags & + V4L2_AV1_FRAME_FLAG_IS_MOTION_MODE_SWITCHABLE) | + AV1_FLAGS_INTRABC(frame->flags & + V4L2_AV1_FRAME_FLAG_ALLOW_INTRABC) | + AV1_FLAGS_INTEGER_MV( + frame->flags & + V4L2_AV1_FRAME_FLAG_FORCE_INTEGER_MV) | + AV1_FLAGS_DISABLE_CDF_UPDATE( + frame->flags & + V4L2_AV1_FRAME_FLAG_DISABLE_CDF_UPDATE) | + AV1_FLAGS_TX_MODE_ONLY_4X4(frame->tx_mode == + V4L2_AV1_TX_MODE_ONLY_4X4) | + AV1_FLAGS_HIGH_PRECISION_MV( + frame->flags & + V4L2_AV1_FRAME_FLAG_ALLOW_HIGH_PRECISION_MV) | + AV1_FLAGS_REF_FRAME_MVS( + frame->flags & + V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS) | + AV1_FLAGS_SEG_TEMPORAL_UPDATE( + frame->segmentation.flags & + V4L2_AV1_SEGMENTATION_FLAG_TEMPORAL_UPDATE) | + AV1_FLAGS_SEG_UPDATE_MAP( + frame->segmentation.flags & + V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP) | + AV1_FLAGS_SEG_ENABLED(frame->segmentation.flags & + V4L2_AV1_SEGMENTATION_FLAG_ENABLED) | + /* im not sure this is correct */ + AV1_FLAGS_SEG_UPDATE_DATA( + frame->segmentation.flags & + V4L2_AV1_SEGMENTATION_FLAG_UPDATE_DATA) | + AV1_FLAGS_SKIP_MODE1( + frame->flags & V4L2_AV1_FRAME_FLAG_SKIP_MODE_ALLOWED ? + frame->skip_mode_frame[1] - 1 : + 7) | + AV1_FLAGS_SKIP_MODE0( + frame->flags & V4L2_AV1_FRAME_FLAG_SKIP_MODE_ALLOWED ? + frame->skip_mode_frame[0] - 1 : + 7) | + AV1_FLAGS_ORDER_HINT(frame->order_hint) | + AV1_FLAGS_INTRA(intra_only), + "av1_flags0"); + + set_ref_hdr(ctx, run); + + for (segid = 0; segid < V4L2_AV1_MAX_SEGMENTS; segid++) { +#define SEG_FEAT_EN(feat) \ + (seg->feature_enabled[segid] & V4L2_AV1_SEGMENT_FEATURE_ENABLED(feat)) + /* what? why? */ + segval = AV1_SEG_UNK0(frame->tx_mode == + V4L2_AV1_TX_MODE_ONLY_4X4); + + segval |= AV1_SEG_ALT_Q( + seg->feature_data[segid][V4L2_AV1_SEG_LVL_ALT_Q]); + segval |= AV1_SEG_ALT_Q_EN(SEG_FEAT_EN(V4L2_AV1_SEG_LVL_ALT_Q)); + segval |= AV1_SEG_REF_FRAME( + seg->feature_data[segid][V4L2_AV1_SEG_LVL_REF_FRAME]); + segval |= AV1_SEG_REF_FRAME_EN( + SEG_FEAT_EN(V4L2_AV1_SEG_LVL_REF_FRAME)); + segval |= AV1_SEG_REF_SKIP_EN( + SEG_FEAT_EN(V4L2_AV1_SEG_LVL_REF_SKIP)); + segval |= AV1_SEG_REF_GMV_EN( + SEG_FEAT_EN(V4L2_AV1_SEG_LVL_REF_GLOBALMV)); + push(segval, "seg_0"); + segval = 0; + segval |= AV1_SEG_LF_V( + seg->feature_data[segid][V4L2_AV1_SEG_LVL_ALT_LF_V]); + segval |= + AV1_SEG_LF_V_EN(SEG_FEAT_EN(V4L2_AV1_SEG_LVL_ALT_LF_V)); + segval |= AV1_SEG_LF_U( + seg->feature_data[segid][V4L2_AV1_SEG_LVL_ALT_LF_U]); + segval |= + AV1_SEG_LF_U_EN(SEG_FEAT_EN(V4L2_AV1_SEG_LVL_ALT_LF_U)); + segval |= AV1_SEG_LF_Y_H( + seg->feature_data[segid][V4L2_AV1_SEG_LVL_ALT_LF_Y_H]); + segval |= AV1_SEG_LF_Y_H_EN( + SEG_FEAT_EN(V4L2_AV1_SEG_LVL_ALT_LF_Y_H)); + segval |= AV1_SEG_LF_Y_V( + seg->feature_data[segid][V4L2_AV1_SEG_LVL_ALT_LF_Y_V]); + segval |= AV1_SEG_LF_Y_V_EN( + SEG_FEAT_EN(V4L2_AV1_SEG_LVL_ALT_LF_Y_V)); + push(segval, "seg_1"); +#undef SEG_FEAT_EN + } + + push(AVD_HDR_FEAT_VP9, "feat_en"); + + avd_select_refs(ctx, run, selected_refs); + set_ref_hints(ctx, run, selected_refs); + + push(0, ""); + push(0, ""); + + pusha(run->addresses.probs_out, "probs_out", 0); + pusha(av1_ctx->bufs.probs.addr, "probs", 1); + pusha(av1_ctx->bufs.above_info.addr, "col", 0); + pusha(run->addresses.seg, "seg", 0); + pusha(run->addresses.prev_seg, "prev_seg", 1); + pusha(run->addresses.color, "w_color", 0); + + for (i = 0; i < 3; i++) { + if (selected_refs[i] < V4L2_AV1_REF_LAST_FRAME) { + pusha((dma_addr_t)0, "", 0); + } else { + ref_idx = frame->ref_frame_idx[selected_refs[i] - + V4L2_AV1_REF_LAST_FRAME]; + ref = avd_get_ref_buf( + ctx, &dst->base.vb, + frame->reference_frame_ts[ref_idx]); + ref_addr = + vb2_dma_contig_plane_dma_addr( + &ref->base.vb.vb2_buf, 0) + + AVD_AV1_COLOR_OFFSET( + ref->base.vb.vb2_buf.planes[0].length, + ref->av1.color_size); + + pusha(ref_addr, "r_color", ref_idx); + } + } + + push(AV1_QP_PRESENT(frame->quantization.flags & + V4L2_AV1_QUANTIZATION_FLAG_DELTA_Q_PRESENT) | + AV1_QP_DELTA_RES(frame->quantization.delta_q_res) | + AV1_QP_BASE_IDX(frame->quantization.base_q_idx) | + AV1_QP_Y_DC(frame->quantization.delta_q_y_dc) | + AV1_QP_U_DC(frame->quantization.delta_q_u_dc) | + AV1_QP_U_AC(frame->quantization.delta_q_u_ac), + "qp_base_q_idx"); + + push(AV1_QP_V_DC(frame->quantization.delta_q_v_dc) | + AV1_QP_V_AC(frame->quantization.delta_q_v_ac) | + AV1_QP_QMATRIX(frame->quantization.flags & + V4L2_AV1_QUANTIZATION_FLAG_USING_QMATRIX) | + AV1_QP_QM_Y(frame->quantization.qm_y) | + AV1_QP_QM_U(frame->quantization.qm_u) | + AV1_QP_QM_V(frame->quantization.qm_v), + "qp_qm"); + + push(AV1_LF_DELTA_ENABLED(frame->loop_filter.flags & + V4L2_AV1_LOOP_FILTER_FLAG_DELTA_ENABLED) | + AV1_LF_DELTA_PRESENT( + frame->loop_filter.flags & + V4L2_AV1_LOOP_FILTER_FLAG_DELTA_LF_PRESENT) | + AV1_LF_DELTA_MULTI( + frame->loop_filter.flags & + V4L2_AV1_LOOP_FILTER_FLAG_DELTA_LF_MULTI) | + AV1_LF_DELTA_RES(frame->loop_filter.delta_lf_res) | + AV1_LF_SHARPNESS(frame->loop_filter.sharpness) | + AV1_LF_LV0(frame->loop_filter.level[0]) | + AV1_LF_LV1(frame->loop_filter.level[1]) | + AV1_LF_LV2(frame->loop_filter.level[2]) | + AV1_LF_LV3(frame->loop_filter.level[3]), + "lf_update"); + + push(AV1_LF_REF0(frame->loop_filter.ref_deltas[0]) | + AV1_LF_REF1(frame->loop_filter.ref_deltas[1]) | + AV1_LF_REF2(frame->loop_filter.ref_deltas[2]) | + AV1_LF_REF3(frame->loop_filter.ref_deltas[3]), + "lf_sharp_ref_deltas"); + + push(AV1_LF_REF0(frame->loop_filter.ref_deltas[4]) | + AV1_LF_REF1(frame->loop_filter.ref_deltas[5]) | + AV1_LF_REF2(frame->loop_filter.ref_deltas[6]) | + AV1_LF_REF3(frame->loop_filter.ref_deltas[7]), + "lf_ref_deltas"); + + push(AV1_LF_MODE0(frame->loop_filter.mode_deltas[0]) | + AV1_LF_MODE1(frame->loop_filter.mode_deltas[1]), + "lf_mode"); + +#define AVD_AV1_SEC_FIXUP(i) ((i) == 4 ? 3 : i) + +#define AV1_CDEF_PACK_ONE(i) \ + (AV1_CDEF_Y_PRI(cdef->y_pri_strength[i]) | \ + AV1_CDEF_Y_SEC(AVD_AV1_SEC_FIXUP(cdef->y_sec_strength[i])) | \ + AV1_CDEF_UV_PRI(cdef->uv_pri_strength[i]) | \ + AV1_CDEF_UV_SEC(AVD_AV1_SEC_FIXUP(cdef->uv_sec_strength[i]))) + +#define AV1_CDEF_PACK(i) \ + AV1_CDEF_HI(AV1_CDEF_PACK_ONE(((i) * 2))) | \ + AV1_CDEF_LO(AV1_CDEF_PACK_ONE(((i) * 2) + 1)) + push(AV1_CDEF_EN(cdef_enabled) | AV1_CDEF_BITS(frame->cdef.bits) | + AV1_CDEF_DAMPING(frame->cdef.damping_minus_3) | + AV1_CDEF_PACK(0), + "cdef0"); + push(AV1_CDEF_PACK(1), "cdef1"); + push(AV1_CDEF_PACK(2), "cdef2"); + push(AV1_CDEF_PACK(3), "cdef3"); +#undef AV1_CDEF_PACK_ONE +#undef AV1_CDEF_PACK + + push(AV1_SUPERRES_USE(frame->flags & V4L2_AV1_FRAME_FLAG_USE_SUPERRES) | + AV1_SUPERRES_DENOM( + frame->flags & V4L2_AV1_FRAME_FLAG_USE_SUPERRES ? + (frame->superres_denom - 1) : + 0) | + AV1_SUPERRES_UPSCALE(frame->upscaled_width - 1), + "superres"); + + calc_upscale(frame->frame_width_minus_1 + 1, frame->upscaled_width, 0, + &step_x, &initial_subpel_x); + push(AV1_UPSCALE_UPSCX(initial_subpel_x) | AV1_UPSCALE_STEPX(step_x), + "upscale_p0"); + calc_upscale(frame->frame_width_minus_1 + 1, frame->upscaled_width, + !!(seq->flags & V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_X), + &step_x, &initial_subpel_x); + push(AV1_UPSCALE_UPSCX(initial_subpel_x) | AV1_UPSCALE_STEPX(step_x), + "upscale_p1"); + + u8 restoration_unit_size[V4L2_AV1_NUM_PLANES_MAX] = { 3, 3, 3 }; + + if (lr->flags & V4L2_AV1_LOOP_RESTORATION_FLAG_USES_LR) { + restoration_unit_size[0] = 1 + lr->lr_unit_shift; + restoration_unit_size[1] = + 1 + lr->lr_unit_shift - lr->lr_uv_shift; + restoration_unit_size[2] = + 1 + lr->lr_unit_shift - lr->lr_uv_shift; + } + + push(AV1_LR_TYPE0(frame->loop_restoration.frame_restoration_type[0]) | + AV1_LR_TYPE1( + frame->loop_restoration.frame_restoration_type[1]) | + AV1_LR_TYPE2( + frame->loop_restoration.frame_restoration_type[2]) | + AV1_LR_UNIT0(restoration_unit_size[0]) | + AV1_LR_UNIT1(restoration_unit_size[1]) | + AV1_LR_UNIT2(restoration_unit_size[2]), + "lr"); + + push(0, ""); + push(0, ""); + pusha(av1_ctx->bufs.pipe_state.addr, "pipe_state", 0); + pusha(av1_ctx->bufs.ip_above.addr, "ip_above", 0); + pusha(av1_ctx->bufs.lf_above.addr, "lf_above", 2); + pusha(av1_ctx->bufs.lf_above_info.addr, "lf_above_info", 3); + pusha(av1_ctx->bufs.lf_left.addr, "lf_left", 0); + pusha(av1_ctx->bufs.lf_left_info.addr, "lf_left_info", 1); + /* make it obvious if its used */ + pusha((dma_addr_t)0xa0a0a0a0a0a0, "", 0); + pusha(av1_ctx->bufs.rf_above_info.addr, "rf_above_info", 0); + pusha(av1_ctx->bufs.sr_left.addr, "sr_left", 2); + pusha(av1_ctx->bufs.rf_left_info.addr, "rf_left_info", 1); + pusha(av1_ctx->bufs.rf_left.addr, "rf_left", 3); + + push(0, "mark_section"); + + push_comp(ctx, run->base.comp_out, ctx->comp.offsets); + + pusha((u64)0, "packed_fmt_scratch", 0); + + /* ignored if decomp is disabled */ + bytesperline = ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline; + pusha(run->base.y_out, "y", 0); + push(bytesperline, "bytesperline_y"); + pusha(run->base.uv_out, "uv", 0); + push(bytesperline, "bytesperline_uv"); + + push(0, "mark_section"); + + push(AVD_HDR_HEIGHT(frame->render_height_minus_1 + 1) | + AVD_HDR_WIDTH(frame->render_width_minus_1 + 1), + "height_width_3"); + if (!intra_only || intrabc) + set_refs(ctx, run); +} + +static void set_tiles(struct avd_ctx *ctx, struct avd_av1_run *run) +{ + struct avd_av1_ctx *av1_ctx = ctx->priv; + const struct v4l2_ctrl_av1_frame *frame = run->frame; + const struct v4l2_ctrl_av1_sequence *seq = run->seq; + const struct v4l2_ctrl_av1_tile_group_entry *tile_group; + const struct v4l2_av1_tile_info *tile_info = &frame->tile_info; + int row, col, sb_row, sb_col, tile_id; + int sb_shift = + seq->flags & V4L2_AV1_SEQUENCE_FLAG_USE_128X128_SUPERBLOCK ? 5 : + 4; + + av1_ctx->submit_num = tile_info->tile_cols * tile_info->tile_rows; + + for (row = 0; row < tile_info->tile_rows; row++) { + for (col = 0; col < tile_info->tile_cols; col++) { + ctx->job.num++; + tile_id = row * tile_info->tile_cols + col; + tile_group = &run->tile_group[tile_id]; + + push(AVD_OP_CODED_DATA | + AVD_OP_CODED_DATA_ADDR( + (run->base.coded_in + + tile_group->tile_offset) >> + 32), + "tile_start"); + push((u32)((run->base.coded_in + + tile_group->tile_offset) & + 0xffffffff), + "tile_addr"); + push(tile_group->tile_size, "tile_size"); + + sb_row = + tile_info->mi_row_starts[tile_group->tile_row] >> + sb_shift; + sb_col = + tile_info->mi_col_starts[tile_group->tile_col] >> + sb_shift; + push(AVD_OP_SL_DIM_START | + AVD_OP_SL_DIM_START_Y(sb_row) | + AVD_OP_SL_DIM_START_X(sb_col), + "tile_op_start"); + + push(AVD_SL_DIM_END_ROW( + !(frame->flags & + V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF) && + tile_id == + tile_info->context_update_tile_id ? + 8 : + 0) | + AVD_SL_DIM_END_COL(col) | + AVD_SL_DIM_END_Y( + sb_row + + tile_info->height_in_sbs_minus_1 + [tile_group->tile_row]) | + AVD_SL_DIM_END_X( + sb_col + + tile_info->width_in_sbs_minus_1 + [tile_group->tile_col]), + "tile_op_end"); +#ifdef DEBUG_INST + pr_info("\n"); +#endif + } + } +} + + +/* End of packing oracle, aurora-wip fbc4e7934ca7325f3483b72a51d24c74aa8536a4. */ diff --git a/drivers/media/platform/apple/avd/tests/host-driver.h b/drivers/media/platform/apple/avd/tests/host-driver.h new file mode 100644 index 0000000000000..8cc8aa9b64f96 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/host-driver.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#define AVD_ALIGN 256 +#define AVD_MAX_INST 512 +#define AVD_QUIRK_LSR BIT(0) +enum avd_codec {AVD_CODEC_HEVC,AVD_CODEC_H264,AVD_CODEC_VP9,AVD_CODEC_AV1}; +enum avd_image_fmt {AVD_IMG_FMT_ANY,AVD_IMG_FMT_420_8BIT,AVD_IMG_FMT_420_10BIT,AVD_IMG_FMT_422_8BIT,AVD_IMG_FMT_422_10BIT}; +enum vb2_buffer_state {VB2_BUF_STATE_DONE, VB2_BUF_STATE_ERROR}; +struct vb2_buffer {unsigned int index; u64 timestamp; struct {size_t length;} planes[1];dma_addr_t dma;size_t payload;}; +struct vb2_v4l2_buffer {struct vb2_buffer vb2_buf;}; +struct avd_comp {u32 size;size_t start_offset;u32 offsets[4];}; +struct avd_decoded_buffer {struct {struct vb2_v4l2_buffer vb;} base;struct avd_comp comp;struct {u16 width,height,upscaled_width;u8 bit_depth;enum v4l2_av1_frame_type frame_type;u32 order_hints[8];u8 ref_frame_idx[7];bool intrabc;size_t color_size,seg_size;} av1;}; +struct avd_run {struct {struct vb2_v4l2_buffer *src,*dst;} bufs;dma_addr_t coded_in,y_out,uv_out,comp_out;}; +struct avd_buf {void *cpu;dma_addr_t addr;size_t size;}; +struct avd_segment {size_t num;u32 instructions[AVD_MAX_INST];}; +struct avd_job {size_t num;struct avd_segment *segments;}; +struct v4l2_pix_format_mplane {u32 width,height;struct {u32 bytesperline,sizeimage;} plane_fmt[1];}; +struct v4l2_ctrl {u32 id;unsigned int elems;union {void *p;} p_cur;union {struct v4l2_ctrl_av1_sequence *p_av1_sequence;} p_new;}; +struct avd_variant {unsigned int quirks,revision;enum avd_av1_fp_layout av1_fp_layout;}; +struct avd_dev {struct avd_variant *variant;bool apple;}; +struct avd_ctx {struct avd_dev *dev;const struct avd_coded_fmt_desc *coded_fmt_desc;void *priv;struct {struct {struct v4l2_pix_format_mplane pix_mp;} fmt;} decoded_fmt;struct avd_comp comp;bool decomp;unsigned int max_frames;struct v4l2_ctrl ctrl_hdl[4];struct avd_job job;struct vb2_v4l2_buffer *next_src,*next_dst;}; +struct avd_coded_fmt_ops {int (*run)(struct avd_ctx *);void (*done)(struct avd_ctx *,struct vb2_v4l2_buffer *,struct vb2_v4l2_buffer *,enum vb2_buffer_state);void (*submit)(struct avd_ctx *);unsigned int (*pp_kicks)(struct avd_ctx *);void (*adjust_decoded_fmt)(struct avd_ctx *,struct v4l2_pix_format_mplane *);int (*start)(struct avd_ctx *);void (*stop)(struct avd_ctx *);int (*try_ctrl)(struct avd_ctx *,struct v4l2_ctrl *);enum avd_image_fmt (*get_image_fmt)(struct avd_ctx *,struct v4l2_ctrl *);}; +static int allocations, fail_after=-1; +static dma_addr_t next_dma=0x10000000; +static int avd_buf_alloc(struct avd_dev *d,struct avd_buf *b,size_t n) {if(fail_after==0)return -ENOMEM;if(fail_after>0)fail_after--;b->cpu=calloc(1,n);if(!b->cpu)return -ENOMEM;b->size=n;b->addr=next_dma;next_dma+=ALIGN(n,256);allocations++;return 0;} +static void avd_buf_free(struct avd_dev *d,struct avd_buf *b) {if(b->cpu){free(b->cpu);allocations--;}memset(b,0,sizeof(*b));} +static struct avd_decoded_buffer buffers[64]; +static struct vb2_v4l2_buffer sources[64]; +static struct avd_decoded_buffer *vb2_to_avd_decoded_buf(struct vb2_buffer *b) {return container_of(b,struct avd_decoded_buffer,base.vb.vb2_buf);} +static struct avd_decoded_buffer *avd_get_ref_buf(struct avd_ctx *c,struct vb2_v4l2_buffer *dst,u64 ts) {for(int i=0;i<64;i++)if(buffers[i].base.vb.vb2_buf.timestamp==ts)return &buffers[i];return vb2_to_avd_decoded_buf(&dst->vb2_buf);} +static dma_addr_t vb2_dma_contig_plane_dma_addr(struct vb2_buffer *b,int n) {return b->dma;} +static size_t vb2_get_plane_payload(struct vb2_buffer *b,int n) {return b->payload;} +static struct v4l2_ctrl *v4l2_ctrl_find(void *h,u32 id) {struct v4l2_ctrl *p=h;for(int i=0;i<4;i++)if(p[i].id==id)return p+i;return NULL;} +struct avd_av1_cdfs {u8 data[16384];}; +static void avd_av1_default_coeff_probs(u32 q,void *p) {memset(p,q,sizeof(struct avd_av1_cdfs));} +static void avd_av1_set_default_cdfs(void *p) {} + +static int legacy_submit_error, legacy_controls_setup, legacy_controls_complete; +static unsigned int legacy_kicks; +static u32 legacy_sent[32768]; +static size_t legacy_sent_words; +static void avd_run_preamble(struct avd_ctx *c,struct avd_run *r) { + memset(r,0,sizeof(*r));r->bufs.src=c->next_src;r->bufs.dst=c->next_dst;c->decomp=true;legacy_controls_setup++; +} +static void avd_run_postamble(struct avd_ctx *c,struct avd_run *r) {legacy_controls_complete++;} +static int avd_init_job(struct avd_ctx *c,enum avd_codec codec,unsigned int n) { + free(c->job.segments);c->job.segments=calloc(n,sizeof(*c->job.segments));c->job.num=0; + return c->job.segments?0:-ENOMEM; +} +static int avd_submit_job(struct avd_ctx *c); +static void avd_open_av1_submit(struct avd_ctx *c,unsigned int count) {legacy_kicks=count;} + +struct avd_coded_fmt_desc {const struct avd_coded_fmt_ops *ops;const struct avd_codec_frame_ops *frame_ops;}; +bool avd_backend_is_apple(const struct avd_dev *d) {return d->apple;} diff --git a/drivers/media/platform/apple/avd/tests/host-shim.h b/drivers/media/platform/apple/avd/tests/host-shim.h new file mode 100644 index 0000000000000..f3d0ce84f12f2 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/host-shim.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Minimal CPU/DMA test doubles; no device IO or Linux kernel build required. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8, __u8; +typedef int8_t s8, __s8; +typedef uint16_t u16, __u16; +typedef int16_t s16, __s16; +typedef uint32_t u32, __u32, __le32; +typedef int32_t s32, __s32; +typedef uint64_t u64, __u64, dma_addr_t; +typedef int64_t s64, __s64; +typedef int irqreturn_t; +#define BIT(x) (1UL << (x)) +#define GENMASK(h,l) ((UINT32_MAX << (l)) & (UINT32_MAX >> (31-(h)))) +#define FIELD_PREP(m,v) (((u32)(v) << __builtin_ctz(m)) & (m)) +#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) +#define ALIGN(v,a) (((v)+(a)-1)&~((typeof(v))(a)-1)) +#define ALIGN_DOWN(v,a) ((v)&~((typeof(v))(a)-1)) +#define DIV_ROUND_UP(v,d) (((v)+(d)-1)/(d)) +#define clamp_val(v,l,h) ((v)<(l)?(l):((v)>(h)?(h):(v))) +#define S16_MIN INT16_MIN +#define S16_MAX INT16_MAX +#define lower_32_bits(x) ((u32)(x)) +#define upper_32_bits(x) ((u32)((u64)(x)>>32)) +#define cpu_to_le32(x) (x) +#define le32_to_cpu(x) (x) +#define smp_load_acquire(p) (*(p)) +#define smp_store_release(p,v) (*(p)=(v)) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define GFP_KERNEL 0 +#define WARN_ON(x) (!!(x)) +#define kzalloc(n,f) calloc(1,n) +#define kfree(p) free(p) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +static u16 get_unaligned_le16(const void *p) { const u8 *b=p; return b[0]|b[1]<<8; } +static u32 get_unaligned_le32(const void *p) { const u8 *b=p; return b[0]|b[1]<<8|b[2]<<16|(u32)b[3]<<24; } +static void put_unaligned_le16(u16 x, void *p) { u8 *b=p; b[0]=x; b[1]=x>>8; } +static void put_unaligned_le32(u32 x, void *p) { u8 *b=p; for(int i=0;i<4;i++) b[i]=x>>(i*8); } +typedef struct { unsigned int n; } refcount_t; +static void refcount_set(refcount_t *r,unsigned int n) {r->n=n;} +static void refcount_inc(refcount_t *r) {assert(r->n);r->n++;} +static bool refcount_dec_and_test(refcount_t *r) {assert(r->n);return !--r->n;} +static int test_and_set_bit(unsigned int n,unsigned long *p) {int b=!!(*p&BIT(n));*p|=BIT(n);return b;} +static void clear_bit(unsigned int n,unsigned long *p) {*p&=~BIT(n);} +struct list_head {struct list_head *next,*prev;}; +#define INIT_LIST_HEAD(h) ((h)->next=(h)->prev=(h)) +static void list_add_tail(struct list_head *n,struct list_head *h) {n->prev=h->prev;n->next=h;h->prev->next=n;h->prev=n;} +static void list_del(struct list_head *n) {n->prev->next=n->next;n->next->prev=n->prev;} +#define list_entry(p,t,m) container_of(p,t,m) +#define list_for_each_entry(p,h,m) for(p=list_entry((h)->next,typeof(*p),m); &p->m!=(h);p=list_entry(p->m.next,typeof(*p),m)) +#define list_for_each_entry_safe(p,q,h,m) for(p=list_entry((h)->next,typeof(*p),m),q=list_entry(p->m.next,typeof(*p),m);&p->m!=(h);p=q,q=list_entry(q->m.next,typeof(*q),m)) +#define V4L2_CID_CODEC_STATELESS_BASE 0 diff --git a/drivers/media/platform/apple/avd/tests/host-test.c b/drivers/media/platform/apple/avd/tests/host-test.c new file mode 100644 index 0000000000000..eb02f4633f4b1 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/host-test.c @@ -0,0 +1,414 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +static int avd_submit_job(struct avd_ctx *c) +{ + legacy_sent_words=0; + legacy_sent[legacy_sent_words++]=AVD_OP_EXEC|AVD_OP_EXEC_FLAG_START_REV4(c->dev->variant->revision==4); + for(size_t i=0;i<=c->job.num;i++) { + for(size_t j=0;jjob.segments[i].num;j++)legacy_sent[legacy_sent_words++]=c->job.segments[i].instructions[j]; + if(i)legacy_sent[legacy_sent_words++]=AVD_OP_EXEC|AVD_OP_EXEC_FLAG_END(i==c->job.num); + } + free(c->job.segments);c->job.segments=NULL; + return legacy_submit_error; +} + +static struct v4l2_ctrl_av1_sequence sequence; +static struct v4l2_ctrl_av1_frame params; +static struct v4l2_ctrl_av1_tile_group_entry tiles[512]; +static struct v4l2_ctrl_av1_film_grain grain; +static struct avd_variant variant = {.revision=4}; +static struct avd_dev device = {.variant=&variant}; + +static void init_ctx(struct avd_ctx *ctx) +{ + memset(ctx,0,sizeof(*ctx)); + ctx->dev=&device; + ctx->decoded_fmt.fmt.pix_mp.width=256; + ctx->decoded_fmt.fmt.pix_mp.height=256; + ctx->decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=256; + ctx->comp.start_offset=65536; + ctx->comp.offsets[0]=256; + ctx->comp.offsets[1]=512; + ctx->comp.offsets[2]=768; + ctx->comp.offsets[3]=1024; + ctx->decomp=true; + ctx->ctrl_hdl[0]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_SEQUENCE,.p_cur.p=&sequence}; + ctx->ctrl_hdl[1]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_FRAME,.p_cur.p=¶ms}; + ctx->ctrl_hdl[2]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY,.p_cur.p=tiles,.elems=V4L2_AV1_MAX_TILE_COUNT}; + ctx->ctrl_hdl[3]=(struct v4l2_ctrl){.id=V4L2_CID_STATELESS_AV1_FILM_GRAIN,.p_cur.p=&grain}; + assert(avd_av1_start(ctx)==0); + assert(ctx->max_frames==1); +} + +static void init_params(void) +{ + memset(&sequence,0,sizeof(sequence)); + memset(¶ms,0,sizeof(params)); + memset(&grain,0,sizeof(grain)); + memset(tiles,0,sizeof(tiles)); + sequence.bit_depth=8; + sequence.order_hint_bits=8; + sequence.flags=V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_X|V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_Y; + params.frame_type=V4L2_AV1_KEY_FRAME; + params.primary_ref_frame=7; + params.frame_width_minus_1=127; + params.frame_height_minus_1=127; + params.upscaled_width=128; + params.superres_denom=8; + params.tx_mode=V4L2_AV1_TX_MODE_LARGEST; + params.tile_info.tile_cols=1; + params.tile_info.tile_rows=1; + params.tile_info.width_in_sbs_minus_1[0]=1; + params.tile_info.height_in_sbs_minus_1[0]=1; + tiles[0].tile_size=128; + for(int i=0;i<64;i++) { + buffers[i].base.vb.vb2_buf=(struct vb2_buffer){.index=i,.timestamp=1000+i,.planes={{.length=1048576}},.dma=0x200000000ULL+i*0x200000,.payload=4096}; + sources[i].vb2_buf=(struct vb2_buffer){.index=i,.timestamp=1000+i,.planes={{.length=4096}},.dma=0x10000000ULL+i*0x200000,.payload=4096}; + buffers[i].av1.width=127; + buffers[i].av1.height=127; + buffers[i].av1.upscaled_width=128; + } +} + +static struct avd_frame make_frame(unsigned int idx,unsigned int dst,struct avd_buf *prog) +{ + return (struct avd_frame){.idx=idx,.seq=dst+1,.frame_num=dst+1, + .src=&sources[dst],.dst=&buffers[dst].base.vb,.prog=prog}; +} + +static void dependency_table(void) +{ + for(int uses=0;uses<2;uses++)for(int prev=0;prev<2;prev++) + for(int update=0;update<2;update++)for(int same=0;same<2;same++) + assert(avd_av1_cdf_dep(uses,123,prev,same?123:456,update)== + (uses&&prev&&update&&same)); +} + +static void sixteen_slots(void) +{ + struct avd_ctx ctx; + struct avd_frame f[17]; + struct avd_buf prog[17]={0}; + u8 snapshots[16][4096]; + init_params();init_ctx(&ctx); + for(int i=0;i<17;i++) { + assert(!avd_buf_alloc(&device,&prog[i],4096)); + f[i]=make_frame(i,i,&prog[i]); + } + for(int i=0;i<16;i++) { + assert(!avd_av1_prepare(&ctx,&f[i])); + assert(f[i].dep==AVD_DEP_NONE); + assert(f[i].prog_words>100); + memcpy(snapshots[i],prog[i].cpu,4096); + struct avd_av1_run *r=f[i].codec_priv; + assert(!r->seq&&!r->frame&&!r->tile_group&&!r->grain); + for(int j=0;jbufs.above_info.cpu!=s->bufs.above_info.cpu); + assert(r->bufs.pipe_state.cpu!=s->bufs.pipe_state.cpu); + assert(r->cdf_in!=s->cdf_in&&r->cdf_out!=s->cdf_out); + assert(r->seg_in!=s->seg_in); + assert(!memcmp(snapshots[j],prog[j].cpu,4096)); + } + } + assert(avd_av1_prepare(&ctx,&f[16])==-EINVAL); + for(int i=0;i<16;i++)avd_av1_retire(&ctx,&f[i],VB2_BUF_STATE_DONE); + avd_av1_stop(&ctx); + for(int i=0;i<17;i++)avd_buf_free(&device,&prog[i]); + assert(!allocations); +} + +static void cdf_chain(void) +{ + struct avd_ctx ctx; + struct avd_frame f[3]; + struct avd_buf prog[3]={0}; + init_params();init_ctx(&ctx); + for(int i=0;i<3;i++) {assert(!avd_buf_alloc(&device,&prog[i],4096));f[i]=make_frame(i,i,&prog[i]);} + assert(!avd_av1_prepare(&ctx,&f[0])); + params.frame_type=V4L2_AV1_INTER_FRAME; + params.primary_ref_frame=0; + params.reference_frame_ts[0]=1000; + params.flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; + assert(!avd_av1_prepare(&ctx,&f[1])); + assert(f[1].dep==AVD_DEP_PREV_DONE); + struct avd_av1_run *a=f[0].codec_priv,*b=f[1].codec_priv; + assert(b->cdf_in==a->cdf_out&&b->cdf_out==a->cdf_out); + params.reference_frame_ts[0]=1001; + int saved=allocations; + assert(avd_av1_prepare(&ctx,&f[2])==-EAGAIN); + assert(f[2].dep==AVD_DEP_NONE&&saved==allocations); + avd_av1_retire(&ctx,&f[0],VB2_BUF_STATE_DONE); + assert(!avd_av1_prepare(&ctx,&f[2])); + assert(f[2].dep==AVD_DEP_NONE); + assert(((struct avd_av1_run *)f[2].codec_priv)->cdf_in==b->cdf_out); + avd_av1_retire(&ctx,&f[1],VB2_BUF_STATE_DONE); + avd_av1_retire(&ctx,&f[2],VB2_BUF_STATE_ERROR); + avd_av1_stop(&ctx); + for(int i=0;i<3;i++)avd_buf_free(&device,&prog[i]); + assert(!allocations); +} + +static void reference_reuse_and_segmentation(void) +{ + struct avd_ctx ctx;struct avd_buf prog[3]={0};struct avd_frame f[3]; + init_params();init_ctx(&ctx); + for(int i=0;i<3;i++){assert(!avd_buf_alloc(&device,&prog[i],4096));f[i]=make_frame(i,i,&prog[i]);} + params.segmentation.flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED|V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP; + assert(!avd_av1_prepare(&ctx,&f[0])); + struct avd_av1_run *a=f[0].codec_priv; + struct avd_av1_aux *cdf=a->cdf_out,*seg=a->seg_out; + params.frame_type=V4L2_AV1_INTER_FRAME;params.primary_ref_frame=0; + params.reference_frame_ts[0]=1000; + params.segmentation.flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED; + params.flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; + assert(!avd_av1_prepare(&ctx,&f[1])); + struct avd_av1_run *b=f[1].codec_priv; + assert(b->cdf_out==cdf&&b->seg_out==seg); + avd_av1_retire(&ctx,&f[0],VB2_BUF_STATE_DONE); + /* Reuse the original capture: inherited state belongs to buffer 1 too. */ + params.primary_ref_frame=7;params.frame_type=V4L2_AV1_KEY_FRAME; + params.flags=0;params.segmentation.flags=0; + buffers[0].base.vb.vb2_buf.timestamp=2000; + f[2].dst=&buffers[0].base.vb; + f[2].src->vb2_buf.timestamp=2000; + assert(!avd_av1_prepare(&ctx,&f[2])); + assert(cdf->refs.n>=2&&seg->refs.n>=2); + assert(b->cdf_in==cdf&&b->seg_in==seg&&cdf->done&&seg->done); + avd_av1_retire(&ctx,&f[1],VB2_BUF_STATE_DONE); + avd_av1_retire(&ctx,&f[2],VB2_BUF_STATE_ERROR); + avd_av1_stop(&ctx); + for(int i=0;i<3;i++)avd_buf_free(&device,&prog[i]); + assert(!allocations); +} + +static void allocation_failures(void) +{ + for(int fail=0;fail<16;fail++) { + struct avd_ctx ctx;struct avd_buf prog={0}; + init_params();init_ctx(&ctx); + assert(!avd_buf_alloc(&device,&prog,4096)); + struct avd_frame f=make_frame(0,0,&prog); + fail_after=fail; + int rc=avd_av1_prepare(&ctx,&f); + fail_after=-1; + if(!rc)avd_av1_retire(&ctx,&f,VB2_BUF_STATE_ERROR); + else {assert(rc==-ENOMEM);assert(!f.codec_priv&&!f.prog_words);} + avd_av1_stop(&ctx);avd_buf_free(&device,&prog); + assert(!allocations); + } +} + +static void bounds_and_fp(void) +{ + struct avd_ctx ctx;struct avd_buf fp={0},prog={0}; + init_params();init_ctx(&ctx); + assert(!avd_av1_validate_tiles(¶ms,tiles,1,128)); + assert(!avd_av1_validate_tiles(¶ms,tiles,V4L2_AV1_MAX_TILE_COUNT,128)); + assert(avd_av1_validate_tiles(¶ms,tiles,0,128)==-EINVAL); + tiles[0].tile_offset=UINT32_MAX; + assert(avd_av1_validate_tiles(¶ms,tiles,1,128)==-EINVAL); + tiles[0].tile_offset=0; + assert(!avd_buf_alloc(&device,&fp,0x1130d0)); + assert(!avd_buf_alloc(&device,&prog,4096)); + struct avd_frame f=make_frame(0,0,&prog);f.fp=&fp; + variant.av1_fp_layout=AVD_AV1_FP_BORAGE; + assert(avd_av1_prepare(&ctx,&f)==-EOPNOTSUPP); + assert(get_unaligned_le16((u8 *)fp.cpu+0x3e4)==1); + assert(get_unaligned_le16((u8 *)fp.cpu+0x3e6)==1); + put_unaligned_le16(0x1002,(u8 *)fp.cpu+0x1a); + put_unaligned_le32(0xdeadbeef,(u8 *)fp.cpu+0x4f8); + put_unaligned_le16(3,(u8 *)f.cmd+6); + assert(avd_av1_pack_cmd(&ctx,&f)==-EOPNOTSUPP); + assert(get_unaligned_le32((u8 *)f.cmd+0x4c)==0xdeadbeef); + assert(get_unaligned_le32((u8 *)f.cmd+0xc)==0x01483118+0x1814*3); + put_unaligned_le16(0x1003,(u8 *)fp.cpu+0x1a); + assert(avd_av1_pack_cmd(&ctx,&f)==-EINVAL); + variant.av1_fp_layout=AVD_AV1_FP_T8140_TODO; + assert(avd_av1_pack_cmd(&ctx,&f)==-EOPNOTSUPP); + variant.av1_fp_layout=AVD_AV1_FP_UNKNOWN; + f.fp=NULL;grain.flags=V4L2_AV1_FILM_GRAIN_FLAG_APPLY_GRAIN; + assert(!avd_av1_prepare(&ctx,&f)); + avd_av1_retire(&ctx,&f,VB2_BUF_STATE_DONE); + grain.flags=0;prog.size=4; + assert(avd_av1_prepare(&ctx,&f)==-ENOSPC); + avd_av1_stop(&ctx);avd_buf_free(&device,&fp);avd_buf_free(&device,&prog); + assert(!allocations); +} + +static void program_parity(void) +{ + for(int test=0;test<64;test++) { + struct avd_ctx ctx;struct avd_buf prog={0}; + init_params();init_ctx(&ctx); + if(test&1)sequence.bit_depth=10; + if(test&32)grain.flags=V4L2_AV1_FILM_GRAIN_FLAG_APPLY_GRAIN; + if(test&16) { + params.frame_type=V4L2_AV1_INTER_FRAME; + sequence.flags|=V4L2_AV1_SEQUENCE_FLAG_ENABLE_ORDER_HINT|V4L2_AV1_SEQUENCE_FLAG_ENABLE_REF_FRAME_MVS; + params.flags|=V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS; + params.order_hint=4; + for(int i=0;i<7;i++){params.ref_frame_idx[i]=i;params.reference_frame_ts[i]=1001+i;} + } + if(test&2) { + sequence.flags|=V4L2_AV1_SEQUENCE_FLAG_ENABLE_CDEF; + params.cdef.bits=2;params.cdef.damping_minus_3=1; + for(int i=0;i<4;i++){params.cdef.y_pri_strength[i]=i;params.cdef.uv_pri_strength[i]=3-i;} + } + if(test&4) { + params.upscaled_width=160;params.superres_denom=10; + params.loop_restoration.frame_restoration_type[0]=V4L2_AV1_FRAME_RESTORE_WIENER; + params.loop_restoration.loop_restoration_size[0]=64; + } + if(test&8) { + params.tile_info.tile_cols=2;params.tile_info.tile_rows=2; + for(int i=0;i<2;i++) { + params.tile_info.width_in_sbs_minus_1[i]=0; + params.tile_info.height_in_sbs_minus_1[i]=0; + params.tile_info.mi_col_starts[i]=i*16; + params.tile_info.mi_row_starts[i]=i*16; + } + for(int i=0;i<4;i++)tiles[i]=(struct v4l2_ctrl_av1_tile_group_entry){.tile_offset=i*128,.tile_size=128,.tile_col=i%2,.tile_row=i/2}; + ctx.ctrl_hdl[2].elems=4; + } + assert(!avd_buf_alloc(&device,&prog,8192)); + struct avd_frame f=make_frame(0,0,&prog); + assert(!avd_av1_prepare(&ctx,&f)); + struct avd_av1_run *r=f.codec_priv; + r->seq=&sequence;r->frame=¶ms;r->tile_group=tiles;r->grain=&grain; + ctx.job.segments=calloc(r->num_tiles+1,sizeof(struct avd_segment)); + legacy_set_header(&ctx,r);legacy_set_tiles(&ctx,r); + u32 expected[2048];unsigned int n=0; + expected[n++]=AVD_OP_EXEC|AVD_OP_EXEC_FLAG_START_REV4(1); + for(unsigned int i=0;i<=r->num_tiles;i++) { + struct avd_segment *seg=&ctx.job.segments[i]; + for(unsigned int j=0;jnum;j++)expected[n++]=seg->instructions[j]; + if(i)expected[n++]=AVD_OP_EXEC|AVD_OP_EXEC_FLAG_END(i==r->num_tiles); + } + assert(n==f.prog_words&&!memcmp(expected,prog.cpu,n*sizeof(u32))); + free(ctx.job.segments); + avd_av1_retire(&ctx,&f,VB2_BUF_STATE_DONE);avd_av1_stop(&ctx); + avd_buf_free(&device,&prog);assert(!allocations); + } + puts("AV1 legacy program parity: 64 synthetic control vectors (including grain-less fallback): PASS"); +} + +static void legacy_transport(void) +{ + for(int error=0;error<2;error++) { + struct avd_ctx ctx; + init_params();init_ctx(&ctx); + params.tile_info.tile_cols=2;params.tile_info.tile_rows=2; + for(int i=0;i<2;i++) { + params.tile_info.width_in_sbs_minus_1[i]=0; + params.tile_info.height_in_sbs_minus_1[i]=0; + } + for(int i=0;i<4;i++)tiles[i]=(struct v4l2_ctrl_av1_tile_group_entry){.tile_offset=i*128,.tile_size=128,.tile_col=i%2,.tile_row=i/2}; + grain.flags=V4L2_AV1_FILM_GRAIN_FLAG_APPLY_GRAIN; + ctx.next_src=&sources[0];ctx.next_dst=&buffers[0].base.vb; + legacy_controls_setup=legacy_controls_complete=0; + legacy_submit_error=error?-EIO:0; + assert(avd_av1_fmt_ops.run&&avd_av1_fmt_ops.done&&avd_av1_fmt_ops.submit); + assert(avd_av1_fmt_ops.run(&ctx)==legacy_submit_error); + assert(legacy_controls_setup==1&&legacy_controls_complete==1); + struct avd_av1_ctx *av1=ctx.priv; + assert(legacy_sent_words==av1->legacy_frame.prog_words); + assert(!memcmp(legacy_sent,av1->legacy_prog.cpu,legacy_sent_words*sizeof(u32))); + if(!error) { + assert(av1->legacy_frame.codec_priv&&av1->active_slots==1); + assert(avd_av1_fmt_ops.pp_kicks(&ctx)==4); + avd_av1_fmt_ops.submit(&ctx);assert(legacy_kicks==4); + } + avd_av1_fmt_ops.done(&ctx,ctx.next_src,ctx.next_dst,error?VB2_BUF_STATE_ERROR:VB2_BUF_STATE_DONE); + assert(!av1->legacy_frame.codec_priv&&!av1->active_slots); + /* Serial decode keeps reference auxiliary state through completion. */ + if(!error) { + params.frame_type=V4L2_AV1_INTER_FRAME;params.primary_ref_frame=0; + params.reference_frame_ts[0]=1000;ctx.next_src=&sources[1];ctx.next_dst=&buffers[1].base.vb; + assert(!avd_av1_fmt_ops.run(&ctx)); + assert(av1->legacy_frame.dep==AVD_DEP_PREV_DONE); + avd_av1_fmt_ops.done(&ctx,ctx.next_src,ctx.next_dst,VB2_BUF_STATE_DONE); + } + avd_av1_stop(&ctx);assert(!allocations); + } + legacy_submit_error=0; + puts("AV1 legacy transport: multi-tile grain-less decode, PP kicks, CDF reuse, submit failure cleanup: PASS"); +} + +static void source_timestamp_and_failed_producer(void) +{ + for(int failed=0;failed<2;failed++) { + struct avd_ctx ctx;struct avd_frame f[2];struct avd_buf prog[2]={0}; + init_params();init_ctx(&ctx); + for(int i=0;i<2;i++){assert(!avd_buf_alloc(&device,&prog[i],4096));f[i]=make_frame(i,i,&prog[i]);} + buffers[0].base.vb.vb2_buf.timestamp=0; + assert(!avd_av1_prepare(&ctx,&f[0])); + struct avd_av1_ctx *av1=ctx.priv; + assert(av1->prev_timestamp==1000); + assert(avd_av1_find_ref(&ctx,0,1000)); + /* Model core's metadata copy after prepare, before the next request. */ + buffers[0].base.vb.vb2_buf.timestamp=1000; + params.frame_type=V4L2_AV1_INTER_FRAME;params.primary_ref_frame=0; + params.reference_frame_ts[0]=1000; + if(failed)avd_av1_retire(&ctx,&f[0],VB2_BUF_STATE_ERROR); + int rc=avd_av1_prepare(&ctx,&f[1]); + assert(rc==(failed?-EIO:0)); + if(!failed) { + assert(f[1].dep==AVD_DEP_PREV_DONE); + avd_av1_retire(&ctx,&f[0],VB2_BUF_STATE_DONE); + avd_av1_retire(&ctx,&f[1],VB2_BUF_STATE_DONE); + } + avd_av1_stop(&ctx); + for(int i=0;i<2;i++)avd_buf_free(&device,&prog[i]); + assert(!allocations); + } + puts("AV1 reference publication: source timestamp and failed producer: PASS"); +} + +static void backend_selection(void) +{ + struct avd_ctx ctx={.dev=&device}; + struct avd_coded_fmt_ops ops=avd_av1_fmt_ops; + struct avd_coded_fmt_desc desc={.ops=&ops,.frame_ops=&avd_av1_frame_ops}; + ctx.coded_fmt_desc=&desc; + for(int apple=0;apple<2;apple++)for(int legacy=0;legacy<2;legacy++) { + device.apple=apple;ops.run=legacy?avd_av1_run:NULL; + assert(avd_get_frame_ops(&ctx)==((apple||!legacy)?&avd_av1_frame_ops:NULL)); + assert(desc.frame_ops==&avd_av1_frame_ops); + } + device.apple=false; + puts("AV1 backend selection: Apple frame_ops and open legacy fallback: PASS"); +} + +static void segmentation_mi_size(void) +{ + const unsigned int widths[]={126,120,119}; + for(unsigned int w=0;w<3;w++) { + struct avd_ctx ctx;struct avd_frame f[2];struct avd_buf prog[2]={0}; + init_params();init_ctx(&ctx); + for(int i=0;i<2;i++){assert(!avd_buf_alloc(&device,&prog[i],4096));f[i]=make_frame(i,i,&prog[i]);} + params.segmentation.flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED|V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP; + assert(!avd_av1_prepare(&ctx,&f[0])); + struct avd_av1_run *a=f[0].codec_priv; + params.frame_type=V4L2_AV1_INTER_FRAME;params.primary_ref_frame=0; + params.reference_frame_ts[0]=1000; + params.flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; + params.segmentation.flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED; + params.frame_width_minus_1=widths[w];params.upscaled_width=widths[w]+1; + assert(!avd_av1_prepare(&ctx,&f[1])); + struct avd_av1_run *b=f[1].codec_priv; + assert((b->seg_in==a->seg_out)==(widths[w]>=120)); + avd_av1_retire(&ctx,&f[0],VB2_BUF_STATE_DONE); + avd_av1_retire(&ctx,&f[1],VB2_BUF_STATE_DONE); + avd_av1_stop(&ctx); + for(int i=0;i<2;i++)avd_buf_free(&device,&prog[i]); + assert(!allocations); + } + puts("AV1 segmentation: inherit only for equal MiCols/MiRows: PASS"); +} + +int main(void) +{ + segmentation_mi_size();backend_selection();source_timestamp_and_failed_producer();legacy_transport();program_parity();dependency_table();sixteen_slots();cdf_chain();reference_reuse_and_segmentation();allocation_failures();bounds_and_fp(); + puts("AV1 host tests: dependency table, 16 slots, CDF inheritance, allocation failures, bounds, Apple refusal: PASS"); + return 0; +} diff --git a/drivers/media/platform/apple/avd/tests/include/linux/errno.h b/drivers/media/platform/apple/avd/tests/include/linux/errno.h new file mode 100644 index 0000000000000..32a2df3ba8d20 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/errno.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include_next diff --git a/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h b/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h new file mode 100644 index 0000000000000..ccda48a6de34b --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/interrupt.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_INTERRUPT_H_ +#define AVD_HOST_INTERRUPT_H_ +#define irqreturn_t int +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h b/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h new file mode 100644 index 0000000000000..a73bb63db2c8e --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/jiffies.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_JIFFIES_H_ +#define AVD_HOST_JIFFIES_H_ +#include +#define time_after_eq(a, b) ((long)((a) - (b)) >= 0) +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/linux/limits.h b/drivers/media/platform/apple/avd/tests/include/linux/limits.h new file mode 100644 index 0000000000000..304aeeeff0fbd --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/limits.h @@ -0,0 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include +#include +#define U32_MAX UINT32_MAX diff --git a/drivers/media/platform/apple/avd/tests/include/linux/types.h b/drivers/media/platform/apple/avd/tests/include/linux/types.h new file mode 100644 index 0000000000000..8b9638abb9a7f --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/linux/types.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_TYPES_H_ +#define AVD_HOST_TYPES_H_ +#include +#include +#include_next +typedef __u8 u8; +typedef __u32 u32; +typedef __u64 u64; +typedef __u64 dma_addr_t; +#endif diff --git a/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h b/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h new file mode 100644 index 0000000000000..6cd054cc198d3 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/include/media/videobuf2-v4l2.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef AVD_HOST_VB2_H_ +#define AVD_HOST_VB2_H_ +enum vb2_buffer_state { VB2_BUF_STATE_DONE, VB2_BUF_STATE_ERROR }; +#endif diff --git a/drivers/media/platform/apple/avd/tests/legacy-emitter.h b/drivers/media/platform/apple/avd/tests/legacy-emitter.h new file mode 100644 index 0000000000000..ea3e36f10005d --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/legacy-emitter.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#undef push +#undef pusha +static void test_legacy_push(struct avd_ctx *ctx,u32 word) +{ + struct avd_segment *s=&ctx->job.segments[ctx->job.num]; + assert(s->numinstructions[s->num++]=word; +} +static void test_legacy_address(struct avd_ctx *ctx,dma_addr_t addr) +{ + if(ctx->dev->variant->quirks&AVD_QUIRK_LSR)test_legacy_push(ctx,addr>>8); + else {test_legacy_push(ctx,lower_32_bits(addr));test_legacy_push(ctx,upper_32_bits(addr));} +} +static void test_legacy_comp(struct avd_ctx *ctx,dma_addr_t addr,u32 offsets[4]) +{ + for(int i=0;i<4;i++) { + if(ctx->dev->variant->quirks&AVD_QUIRK_LSR)test_legacy_push(ctx,(addr+offsets[i])>>7); + else test_legacy_address(ctx,addr+offsets[i]); + } +} +#define push(inst,name) test_legacy_push(ctx,(inst)) +#define pusha(addr,name,i) test_legacy_address(ctx,(addr)) diff --git a/drivers/media/platform/apple/avd/tests/queue-host.c b/drivers/media/platform/apple/avd/tests/queue-host.c new file mode 100644 index 0000000000000..5983d7ad5d6a3 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/queue-host.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Exercise the production helpers with deterministic shuffled completions. */ +#include +#include +#include + +#include "../avd-apple-protocol.h" +#include "../avd-queue-internal.h" + +int main(void) +{ + struct avd_frame pool[AVD_MAX_FRAMES] = {}; + int clients[3]; + u64 last[3] = {}, seq = 0; + u32 rng = 0x3851, old_tag[AVD_MAX_FRAMES] = {}; + unsigned int i, step, completed = 0; + + for (i = 0; i < AVD_MAX_FRAMES; i++) + pool[i].idx = i; + for (step = 0; step < 100000; step++) { + struct avd_frame *f; + + rng = rng * 1664525 + 1013904223; + f = &pool[(rng >> 16) & 15]; + if (f->state == AVD_FRAME_FREE) { + struct avd_ctx *ctx = (void *)&clients[(rng >> 8) % 3]; + + assert(!avd_slot_reserve(f, ctx, ++seq)); + assert(!avd_slot_complete(f, f->tag, 0)); + f->deadline = step + 100; + f->state = AVD_FRAME_SUBMITTED; + assert(!avd_slot_complete(f, old_tag[f->idx], 0)); + old_tag[f->idx] = f->tag; + } else if (f->state == AVD_FRAME_SUBMITTED) { + assert(avd_slot_complete(f, f->tag, (rng & 1) ? -EIO : 0)); + assert(!avd_slot_complete(f, f->tag, 0)); + } + for (i = 0; i < AVD_MAX_FRAMES; i++) { + unsigned int c; + + f = &pool[i]; + if (f->state != AVD_FRAME_DONE || !avd_slot_first(pool, f)) + continue; + for (c = 0; c < 3; c++) + if (f->ctx == (void *)&clients[c]) + break; + assert(c < 3 && f->seq > last[c]); + last[c] = f->seq; + f->state = AVD_FRAME_FREE; + completed++; + } + } + for (i = 0; i < 8; i++) { + assert(avd_inbox_full(i, (i + 1) & 7)); + assert(!avd_inbox_full(i, (i + 2) & 7)); + assert(avd_apple_notification(i) == 0x01482cf8U + 0x60 * i); + } + { + struct avd_apple_reply_layout layout = { + .type_word = 0, .type_mask = 31, .client_word = 1, + .client_mask = 1023, .frame_word = 3, .status_word = 4, + .done_type = 8, .verified = true, + }; + struct avd_apple_reply reply; + u32 record[20] = { 7, 9, 0, 1234 }; + + assert(avd_apple_parse_reply(record, 20, &layout, &reply) == -EINPROGRESS); + record[0] = 8; + assert(!avd_apple_parse_reply(record, 20, &layout, &reply)); + assert(reply.frame == 1234 && reply.client == 9 && !reply.status); + record[4] = 1; + assert(!avd_apple_parse_reply(record, 20, &layout, &reply)); + assert(reply.status == -EIO); + assert(avd_apple_parse_reply(record, 19, &layout, &reply) == -EMSGSIZE); + } + assert(completed > 1000); + printf("PASS: 100000 shuffled operations, %u ordered retirements\n", completed); + return 0; +} diff --git a/drivers/media/platform/apple/avd/tests/run-host-tests.py b/drivers/media/platform/apple/avd/tests/run-host-tests.py new file mode 100755 index 0000000000000..616093e125c66 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/run-host-tests.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile production CPU code with DMA/V4L2 test doubles, then run with ASan/UBSan. +The UAPI definitions come from this checkout; git show also supports sparse trees. +""" +import os +from pathlib import Path +import re +import subprocess +import tempfile + +HERE = Path(__file__).resolve().parent +DRIVER = HERE.parent + +def without_includes(text): + return re.sub(r'^#include .*\n', '', text, flags=re.M) + +uapi = subprocess.check_output(['git', '-C', str(DRIVER), 'show', + 'HEAD:include/uapi/linux/v4l2-controls.h'], text=True) +uapi = uapi.split('/* Stateless AV1 controls */')[1].split('/* MPEG-compression')[0] +with tempfile.TemporaryDirectory(prefix='avd-av1-test-') as tmp: + out = Path(tmp) + source = (HERE / 'host-shim.h').read_text() + uapi + source += without_includes((DRIVER / 'avd-av1-fp.h').read_text()) + source += (HERE / 'host-driver.h').read_text() + source += without_includes((DRIVER / 'avd-queue.h').read_text()) + # Compile the production transport selector, not a duplicate test rule. + header = (DRIVER / 'avd.h').read_text() + source += re.search(r'static inline const struct avd_codec_frame_ops \*\n' + r'avd_get_frame_ops.*?\n}', header, re.S).group(0) + '\n' + + inst = without_includes((DRIVER / 'avd-inst.h').read_text()) + inst = inst[:inst.index('static inline void push(')] + '\n#endif\n' + source += inst + source += without_includes((DRIVER / 'avd-av1-fp.c').read_text()) + source += without_includes((DRIVER / 'avd-av1.c').read_text()) + # Freeze the pre-conversion arithmetic/packing as an independent oracle. + legacy = (HERE / 'avd-av1-legacy.c').read_text() + legacy = legacy[legacy.index('static const short div_lut'):legacy.index('/* End of packing oracle,')] + names = ['div_lut', 'avd_av1_dec_get_msb', 'avd_av1_dec_resolve_divisor_32', + 'avd_av1_dec_get_shear_params', 'set_refs', 'avd_av1_get_dist', + 'is_valid_ref_frame', 'avd_select_refs', 'set_ref_hints', 'set_ref_hdr', + 'calc_upscale', 'set_header', 'set_tiles'] + for name in names: + legacy = re.sub(r'\b' + name + r'\b', 'legacy_' + name, legacy) + legacy = legacy.replace('struct avd_av1_ctx *av1_ctx = ctx->priv;', '') + legacy = legacy.replace('av1_ctx->bufs.', 'run->bufs.') + legacy = legacy.replace('run->bufs.probs.addr', 'run->addresses.probs_in') + legacy = re.sub(r'av1_ctx->submit_num = .*?;', '', legacy) + legacy = legacy.replace('push_comp(ctx, ', 'test_legacy_comp(ctx, ') + source += '#pragma GCC diagnostic push\n#pragma GCC diagnostic ignored \"-Wpointer-sign\"\n' + source += (HERE / 'legacy-emitter.h').read_text() + legacy + source += '#pragma GCC diagnostic pop\n' + source += (HERE / 'host-test.c').read_text() + (out / 'test.c').write_text(source) + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-g', '-O1', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', + '-Werror', str(out / 'test.c'), + '-o', str(out / 'test')], check=True) + subprocess.run([str(out / 'test')], check=True) diff --git a/drivers/media/platform/apple/avd/tests/run-host.sh b/drivers/media/platform/apple/avd/tests/run-host.sh new file mode 100755 index 0000000000000..70e287f14732d --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/run-host.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only OR MIT +set -eu +avd_test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +avd_test_out=${1:?usage: run-host.sh OUTPUT_BINARY} +${CC:-cc} -std=gnu11 -Wall -Wextra -Werror -O2 \ + -I"$avd_test_dir/include" "$avd_test_dir/queue-host.c" -o "$avd_test_out" +"$avd_test_out"