diff --git a/Documentation/devicetree/bindings/nvme/apple,nvme-apif.yaml b/Documentation/devicetree/bindings/nvme/apple,nvme-apif.yaml new file mode 100644 index 0000000000000..9d5b357a56379 --- /dev/null +++ b/Documentation/devicetree/bindings/nvme/apple,nvme-apif.yaml @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/nvme/apple,nvme-apif.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Resident Apple ANS controller using the Aurora platform interface + +maintainers: + - Dj + +description: | + A boot-time guest description for the native table-6 ABI with 64-entry + linear queues. The producer stops all NVMe DMA and quiesces both sides + of RTKit before handoff. Native queue registrations survive controller + disable, so the consumer must reuse the four registered queue pages. + This binding describes an ABI, not a promise of support for every SoC. + The controller uses identity DMA addresses and coherent shared memory. + +properties: + compatible: + const: apple,nvme-apif + reg: + items: + - description: NVMe controller registers + - description: ANS coprocessor control registers + reg-names: + items: + - const: nvme + - const: ans + interrupts: + maxItems: 1 + mboxes: + maxItems: 1 + apple,sart: + $ref: /schemas/types.yaml#/definitions/phandle + description: SART provider for the resident controller + aurora,apif: + $ref: /schemas/types.yaml#/definitions/phandle + description: Version-2 native APIF transport + dma-coherent: true + asahi,rtkit-quiesced: + type: boolean + description: Both RTKit peers were stopped before their buffers were released + memory-region: + minItems: 4 + maxItems: 4 + memory-region-names: + items: + - const: admin-sq + - const: admin-cq + - const: io-sq + - const: io-cq + power-domains: + minItems: 2 + maxItems: 3 + power-domain-names: + minItems: 2 + items: + - const: ans + - const: apcie0 + - enum: [apcie1, apcie_phy] + resets: + maxItems: 1 + +required: + - compatible + - reg + - reg-names + - interrupts + - mboxes + - apple,sart + - aurora,apif + - dma-coherent + - asahi,rtkit-quiesced + - memory-region + - memory-region-names + +additionalProperties: false diff --git a/Documentation/driver-api/firmware/aurora-apif-nvme.rst b/Documentation/driver-api/firmware/aurora-apif-nvme.rst new file mode 100644 index 0000000000000..ab4e9b7088bad --- /dev/null +++ b/Documentation/driver-api/firmware/aurora-apif-nvme.rst @@ -0,0 +1,39 @@ +Resident ANS over APIF +====================== + +``apple,nvme-apif`` selects the native table-6 queue ABI. It keeps the +existing Apple NVMe queue depth, batching, crypto fields, recovery ownership +and synchronous post-M4 flush policy. Linux builds and translates the TCB +and physical page list; Aura forwards selectors and arguments unchanged. +No payload copying or monitor-side NVMe policy is required. + +The boot producer must quiesce RTKit and stop controller DMA before Linux +starts. It supplies ``asahi,rtkit-quiesced`` and four distinct reserved-memory +pages named ``admin-sq``, ``admin-cq``, ``io-sq`` and ``io-cq``. Each is coherent, +16 KiB aligned, 16 KiB sized and ``no-map``. Their guest addresses must map to +the native physical pages already registered with the firmware. Linux clears +and reuses those pages; it does not substitute new queue addresses. They stay +reserved after unbind because the native queue registrations survive it. + +This profile requires an identity DMA aperture. It checks each request's +physical translation against the DMA address in its command and PRP chain. +An offset aperture is rejected before native request registration. Native +metadata addresses are independently translated, including the scratch TCB +and page list. Queue-control and MAP endpoints return void; UNMAP must return +one before a request's backing can be reused. + +A lost MAP response retains that request and prevents new registration until +recovery has stopped the controller and explicitly invalidated every retained +tag. Failed invalidation quarantines the controller and retains its DMA, +requests and APIF client until platform reset. SART unmap failures also retain +Linux-owned backing. These are ownership failures, not successful I/O. + +Live RTKit adoption is deliberately excluded from this new profile: controller +CC.EN/RDY alone cannot prove a live mailbox session or authorize inherited +buffers outside guest RAM. Existing raw-controller profiles keep their existing +validated adoption path. The guest producer must provide the stopped handoff +above; a version-1 live-session DT is not a drop-in replacement. + +Build and host fault tests do not qualify this path on hardware. Boot, +read/write/flush durability, controller reset and teardown must be tested with +the matching Aura version-2 monitor and generated handoff description. diff --git a/Documentation/driver-api/firmware/index.rst b/Documentation/driver-api/firmware/index.rst index bd0811b39446f..f50972fec7047 100644 --- a/Documentation/driver-api/firmware/index.rst +++ b/Documentation/driver-api/firmware/index.rst @@ -5,6 +5,7 @@ Linux Firmware API .. toctree:: aurora-apif + aurora-apif-nvme introduction core efi/index diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig index de0fa14096286..139d0d691e6b5 100644 --- a/drivers/nvme/host/Kconfig +++ b/drivers/nvme/host/Kconfig @@ -128,6 +128,7 @@ config NVME_APPLE depends on APPLE_RTKIT && APPLE_SART depends on ARCH_APPLE || COMPILE_TEST select BLK_INLINE_ENCRYPTION + select AURORA_APIF_MMIO if ARM64_16K_PAGES select NVME_CORE help This provides support for the NVMe controller embedded in Apple SoCs diff --git a/drivers/nvme/host/apple-apif.h b/drivers/nvme/host/apple-apif.h new file mode 100644 index 0000000000000..53d29b5c3b8c2 --- /dev/null +++ b/drivers/nvme/host/apple-apif.h @@ -0,0 +1,343 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Resident ANS uses native table 6; Linux owns queue and DMA lifetimes. */ +#define APPLE_NVME_APIF_PAGES 257 +#define APPLE_NVME_APIF_LIST_SIZE SZ_4K + +static inline void **apple_nvme_iod_list(struct request *req); +static inline struct apple_nvme * +queue_to_apple_nvme(struct apple_nvme_queue *q); + +static int apple_nvme_apif_translate(struct apple_nvme *anv, u64 address, + size_t size, u64 *pa) +{ + struct aurora_apif_op op = { + .selector = AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, + AURORA_APIF_TRANSLATE), + .arg = { address, size }, + }; + int ret = aurora_apif_submit(anv->apif, &op, 1, NULL); + + if (!ret) + *pa = op.ret; + return ret; +} + +static int apple_nvme_apif_handoff(struct apple_nvme *anv) +{ + static const char *const names[] = { + "admin-sq", + "admin-cq", + "io-sq", + "io-cq", + }; + struct device_node *np = anv->dev->of_node; + phys_addr_t addresses[ARRAY_SIZE(names)]; + int i, j; + + /* Only admit an explicit stopped mailbox and four pinned queue pages. + * CC.EN/RDY describe controller state, not ownership of the RTKit session. + */ + if (!IS_ENABLED(CONFIG_AURORA_APIF_MMIO) || + !of_property_read_bool(np, "asahi,rtkit-quiesced") || + !dev_is_dma_coherent(anv->dev) || + of_count_phandle_with_args(np, "memory-region", NULL) != + ARRAY_SIZE(names) || + of_property_count_strings(np, "memory-region-names") != + ARRAY_SIZE(names)) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(names); i++) { + struct device_node *node; + struct reserved_mem *rmem; + int index = of_property_match_string(np, "memory-region-names", + names[i]); + + if (index < 0) + return index; + node = of_parse_phandle(np, "memory-region", index); + if (!node) + return -EINVAL; + rmem = of_reserved_mem_lookup(node); + if (!rmem || !of_property_read_bool(node, "no-map") || + !IS_ALIGNED(rmem->base, SZ_16K) || rmem->size != SZ_16K) { + of_node_put(node); + return -EINVAL; + } + addresses[i] = rmem->base; + of_node_put(node); + for (j = 0; j < i; j++) + if (addresses[j] == addresses[i]) + return -EINVAL; + } + return 0; +} + +static void apple_nvme_apif_free(void *data) +{ + struct apple_nvme *anv = data; + + if (READ_ONCE(anv->quarantined)) + return; + if (anv->apif_scratch) + dma_free_coherent(anv->dev, SZ_16K, anv->apif_scratch, + anv->apif_scratch_dma); + kfree(anv->apif_ops); + aurora_apif_put(anv->apif); +} + +static int apple_nvme_apif_init(struct apple_nvme *anv) +{ + int ret; + + anv->apif = aurora_apif_get(anv->dev); + if (IS_ERR(anv->apif)) + return PTR_ERR(anv->apif); + ret = devm_add_action_or_reset(anv->dev, apple_nvme_apif_free, anv); + if (ret) + return ret; + anv->apif_ops = kcalloc(APPLE_NVME_APIF_PAGES, sizeof(*anv->apif_ops), + GFP_KERNEL); + anv->apif_scratch = dma_alloc_coherent( + anv->dev, SZ_16K, &anv->apif_scratch_dma, GFP_KERNEL); + return anv->apif_ops && anv->apif_scratch ? 0 : -ENOMEM; +} + +static void apple_nvme_apif_queue_free(struct apple_nvme *anv, + struct apple_nvme_queue *q) +{ + if ((q->apif_grants & BIT(0)) && + apple_sart_remove_allowed_region(anv->sart, q->sq_dma_addr, SZ_16K)) + dev_err(anv->dev, + "retaining pinned SQ grant after failed SART unmap\n"); + if ((q->apif_grants & BIT(1)) && + apple_sart_remove_allowed_region(anv->sart, q->cq_dma_addr, SZ_16K)) + dev_err(anv->dev, + "retaining pinned CQ grant after failed SART unmap\n"); + if (q->tcbs && apple_sart_remove_allowed_region( + anv->sart, q->tcb_dma_addr, SZ_16K)) { + dev_err(anv->dev, + "retaining TCB page after failed SART unmap\n"); + return; + } + if (q->tcbs) + dma_free_coherent(anv->dev, SZ_16K, q->tcbs, q->tcb_dma_addr); + /* SQ/CQ pages remain reserved for the entire machine boot. */ +} + +static int apple_nvme_apif_queue_alloc(struct apple_nvme *anv, + struct apple_nvme_queue *q) +{ + struct device_node *np = anv->dev->of_node; + const char *names[] = { q->is_adminq ? "admin-sq" : "io-sq", + q->is_adminq ? "admin-cq" : "io-cq" }; + int i, ret; + + for (i = 0; i < ARRAY_SIZE(names); i++) { + struct device_node *node; + struct reserved_mem *rmem; + void *buffer; + int index = of_property_match_string(np, "memory-region-names", + names[i]); + + if (index < 0) + return index; + node = of_parse_phandle(np, "memory-region", index); + rmem = of_reserved_mem_lookup(node); + of_node_put(node); + if (!rmem) + return -EINVAL; + buffer = devm_memremap(anv->dev, rmem->base, SZ_16K, + MEMREMAP_WB); + if (IS_ERR(buffer)) + return PTR_ERR(buffer); + ret = apple_sart_add_allowed_region( + anv->sart, phys_to_dma(anv->dev, rmem->base), SZ_16K); + if (ret) + return ret; + q->apif_grants |= BIT(i); + memset(buffer, 0, SZ_16K); + if (!i) { + q->sqes = buffer; + q->sq_dma_addr = phys_to_dma(anv->dev, rmem->base); + } else { + q->cqes = buffer; + q->cq_dma_addr = phys_to_dma(anv->dev, rmem->base); + } + } + q->tcbs = dma_alloc_coherent(anv->dev, SZ_16K, &q->tcb_dma_addr, + GFP_KERNEL); + if (!q->tcbs) + return -ENOMEM; + ret = apple_sart_add_allowed_region(anv->sart, q->tcb_dma_addr, SZ_16K); + if (ret) { + dma_free_coherent(anv->dev, SZ_16K, q->tcbs, q->tcb_dma_addr); + q->tcbs = NULL; + return ret; + } + q->cq_phase = 1; + return 0; +} + +static int apple_nvme_apif_admin(struct apple_nvme *anv) +{ + struct aurora_apif_op ops[] = { + { .selector = AURORA_APIF_SELECTOR(6, 3), .arg = { 253, 2 } }, + { .selector = AURORA_APIF_SELECTOR(6, 0) }, + { .selector = AURORA_APIF_SELECTOR(6, 4), + .arg = { 0, APPLE_NVME_AQ_DEPTH - 1, 0, + APPLE_NVME_AQ_DEPTH - 1 } }, + { .selector = AURORA_APIF_SELECTOR(6, 5), + .arg = { anv->hw->max_queue_depth - 1, + anv->hw->max_queue_depth - 1 } }, + { .selector = AURORA_APIF_SELECTOR(6, 7) }, + { .selector = AURORA_APIF_SELECTOR(6, 6) }, + }; + int ret; + + ret = apple_nvme_apif_translate( + anv, dma_to_phys(anv->dev, anv->adminq.sq_dma_addr), SZ_16K, + &ops[2].arg[0]); + if (ret) + return ret; + ret = apple_nvme_apif_translate( + anv, dma_to_phys(anv->dev, anv->adminq.cq_dma_addr), SZ_16K, + &ops[2].arg[2]); + if (ret) + return ret; + ret = apple_nvme_apif_translate( + anv, dma_to_phys(anv->dev, anv->ioq.cq_dma_addr), SZ_16K, + &ops[4].arg[0]); + if (ret) + return ret; + ret = apple_nvme_apif_translate( + anv, dma_to_phys(anv->dev, anv->ioq.sq_dma_addr), SZ_16K, + &ops[5].arg[0]); + if (ret) + return ret; + /* These queue control endpoints have void native results. */ + return aurora_apif_submit(anv->apif, ops, ARRAY_SIZE(ops), NULL); +} + +/* The shared scratch page is protected by anv->lock, alongside CQ processing. */ +static int apple_nvme_apif_request(struct apple_nvme_queue *q, + struct request *req, bool map) +{ + struct apple_nvme *anv = queue_to_apple_nvme(q); + struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); + u32 tag = nvme_tag_from_cid(iod->cmd.common.command_id); + struct apple_nvmmu_tcb *copy = + anv->apif_scratch + APPLE_NVME_APIF_LIST_SIZE; + u64 *list = anv->apif_scratch; + struct aurora_apif_op op; + u64 first, second, scratch_pa; + u32 count, i, done; + int ret; + + lockdep_assert_held(&anv->lock); + if (tag >= anv->hw->max_queue_depth) + return -EINVAL; + if (!map) { + if (!iod->apif_registered) + return 0; + op = (struct aurora_apif_op){ + .selector = AURORA_APIF_SELECTOR(6, 2), + .arg = { q->is_adminq ? 0 : 1, tag, 0 }, + }; + ret = aurora_apif_submit(anv->apif, &op, 1, NULL); + if (ret || op.ret != 1) + return ret ?: -EIO; + iod->apif_registered = false; + return 0; + } + if (anv->apif_uncertain) + return -EIO; + first = le64_to_cpu(iod->cmd.common.dptr.prp1); + second = le64_to_cpu(iod->cmd.common.dptr.prp2); + count = blk_rq_payload_bytes(req) ? + DIV_ROUND_UP((first & (SZ_4K - 1)) + + blk_rq_payload_bytes(req), + SZ_4K) : + !!first; + if (count > APPLE_NVME_APIF_PAGES || + (count == 1 && !IS_ALIGNED(first, SZ_4K))) + return -EINVAL; + if (count) { + list[0] = first; + if (count == 2) + list[1] = second; + else if (count > 2) { + __le64 *prps = apple_nvme_iod_list(req)[0]; + + for (i = 1; i < count; i++) + list[i] = le64_to_cpu(prps[i - 1]); + } + } + for (i = 0; i < count; i++) { + if (i && !IS_ALIGNED(list[i], SZ_4K)) + return -EINVAL; + anv->apif_ops[i] = (struct aurora_apif_op){ + .selector = AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, + AURORA_APIF_TRANSLATE), + .arg = { dma_to_phys(anv->dev, list[i] & ~(SZ_4K - 1)), + SZ_4K }, + }; + } + if (count) { + ret = aurora_apif_submit(anv->apif, anv->apif_ops, count, NULL); + if (ret) + return ret; + /* Commands and PRP chains carry DMA addresses directly. Until ANS + * supports another aperture, require the identity handoff contract. + */ + for (i = 0; i < count; i++) + if (anv->apif_ops[i].ret != (list[i] & ~(SZ_4K - 1))) + return -EINVAL; + } + ret = apple_nvme_apif_translate( + anv, dma_to_phys(anv->dev, anv->apif_scratch_dma), SZ_16K, + &scratch_pa); + if (ret) + return ret; + memcpy(copy, &q->tcbs[tag], sizeof(*copy)); + copy->length = cpu_to_le16( + count ? count - ((first & (SZ_4K - 1)) ? 2 : 1) : 0); + op = (struct aurora_apif_op){ + .selector = AURORA_APIF_SELECTOR(6, 1), + .arg = { q->is_adminq ? 0 : 1, tag, + scratch_pa + APPLE_NVME_APIF_LIST_SIZE, scratch_pa, + count }, + }; + dma_wmb(); + /* Mark before dispatch: a lost response cannot prove MAP did not run. */ + iod->apif_registered = true; + ret = aurora_apif_submit(anv->apif, &op, 1, &done); + if (ret) + anv->apif_uncertain = true; + return ret; +} + +struct apple_nvme_apif_drain { + int error; +}; + +static bool apple_nvme_apif_drain_request(struct request *req, void *data) +{ + struct apple_nvme_apif_drain *drain = data; + struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); + + if (iod->apif_registered && apple_nvme_apif_request(iod->q, req, false)) + drain->error = -EIO; + return true; +} + +static int apple_nvme_apif_drain(struct apple_nvme *anv) +{ + struct apple_nvme_apif_drain drain = {}; + + blk_mq_tagset_busy_iter(&anv->tagset, apple_nvme_apif_drain_request, + &drain); + blk_mq_tagset_busy_iter(&anv->admin_tagset, + apple_nvme_apif_drain_request, &drain); + if (!drain.error) + anv->apif_uncertain = false; + return drain.error; +} diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index f6c18221ded3e..fe6d751c096e9 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -31,12 +32,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include @@ -139,6 +142,7 @@ struct apple_nvmmu_tcb { * a MMIO register. */ struct apple_nvme_queue { + u8 apif_grants; struct nvme_command *sqes; struct nvme_completion *cqes; struct apple_nvmmu_tcb *tcbs; @@ -185,6 +189,7 @@ struct apple_nvme_iod { struct nvme_request req; struct nvme_command cmd; struct apple_nvme_queue *q; + bool apif_registered; int npages; /* In the PRP list. 0 means small pool in use */ int nents; /* Used in scatterlist */ dma_addr_t first_dma; @@ -197,6 +202,7 @@ struct apple_nvme_hw { bool has_queue_count; bool has_separate_nvmmu; bool needs_ioq_registers; + bool apif; u32 max_queue_depth; unsigned long quirks; }; @@ -232,6 +238,11 @@ struct apple_nvme { struct apple_sart *sart; struct apple_rtkit *rtk; struct reset_control *reset; + struct aurora_apif *apif; + struct aurora_apif_op *apif_ops; + void *apif_scratch; + dma_addr_t apif_scratch_dma; + bool apif_uncertain; bool owns_rtkit; bool inherited_rtkit; @@ -325,10 +336,16 @@ static void apple_nvme_put_resource_owner(void *data) kref_put(&anv->ref, apple_nvme_release); } +static int apple_nvme_apif_request(struct apple_nvme_queue *q, + struct request *req, bool map); +#include "apple-apif.h" + static bool apple_nvme_can_adopt_rtkit(struct apple_nvme *anv) { u32 cc, csts; + if (anv->hw->apif && of_property_read_bool(anv->dev->of_node, "asahi,rtkit-quiesced")) + return false; if (!anv->hw->needs_ioq_registers || !(readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & APPLE_ANS_COPROC_CPU_CONTROL_RUN) || @@ -354,6 +371,8 @@ static int apple_nvme_read_handoff(struct apple_nvme *anv) const char *handoff; int ret; + if (anv->hw->apif) + return apple_nvme_apif_handoff(anv); ret = of_property_read_string(anv->dev->of_node, "apple,rtkit-handoff", &handoff); if (ret == -EINVAL) @@ -511,6 +530,8 @@ static int apple_nvme_sart_dma_setup(void *cookie, return ret; } + if (anv->hw->apif) + bfr->size = ALIGN(bfr->size, SZ_16K); bfr->buffer = dma_alloc_coherent(anv->dev, bfr->size, &bfr->iova, GFP_KERNEL); if (!bfr->buffer) @@ -539,7 +560,10 @@ static void apple_nvme_sart_dma_destroy(void *cookie, kfree(mapping); return; } - apple_sart_remove_allowed_region(anv->sart, bfr->iova, bfr->size); + if (apple_sart_remove_allowed_region(anv->sart, bfr->iova, bfr->size)) { + dev_err(anv->dev, "retaining RTKit buffer after failed SART unmap\n"); + return; + } dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova); } @@ -632,6 +656,12 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, * and the final CQ update. */ spin_lock_irq(&anv->lock); + if (anv->hw->apif && apple_nvme_apif_request(q, req, true)) { + spin_unlock_irq(&anv->lock); + /* Timeout recovery stops DMA before cancellation releases this tag. */ + return; + } + dma_wmb(); writel(tag, q->sq_db); spin_unlock_irq(&anv->lock); } @@ -915,7 +945,7 @@ apple_nvme_queue_tagset(struct apple_nvme *anv, struct apple_nvme_queue *q) return anv->tagset.tags[0]; } -static inline void apple_nvme_handle_cqe(struct apple_nvme_queue *q, +static inline bool apple_nvme_handle_cqe(struct apple_nvme_queue *q, struct io_comp_batch *iob, u16 idx) { struct apple_nvme *anv = queue_to_apple_nvme(q); @@ -923,20 +953,23 @@ static inline void apple_nvme_handle_cqe(struct apple_nvme_queue *q, __u16 command_id = READ_ONCE(cqe->command_id); struct request *req; - if (anv->hw->has_lsq_nvmmu) + if (anv->hw->has_lsq_nvmmu && !anv->hw->apif) apple_nvmmu_inval(q, command_id); req = nvme_find_rq(apple_nvme_queue_tagset(anv, q), command_id); if (unlikely(!req)) { dev_warn(anv->dev, "invalid id %d completed", command_id); - return; + return true; } + if (anv->hw->apif && apple_nvme_apif_request(q, req, false)) + return false; if (!nvme_try_complete_req(req, cqe->status, cqe->result) && !blk_mq_add_to_batch(req, iob, nvme_req(req)->status != NVME_SC_SUCCESS, apple_nvme_complete_batch)) apple_nvme_complete_rq(req); + return true; } static inline void apple_nvme_update_cq_head(struct apple_nvme_queue *q) @@ -964,7 +997,8 @@ static bool apple_nvme_poll_cq(struct apple_nvme_queue *q, * the cqe requires a full read memory barrier */ dma_rmb(); - apple_nvme_handle_cqe(q, iob, q->cq_head); + if (!apple_nvme_handle_cqe(q, iob, q->cq_head)) + break; apple_nvme_update_cq_head(q); } @@ -1174,6 +1208,7 @@ static int apple_nvme_init_request(struct blk_mq_tag_set *set, struct nvme_request *nreq = nvme_req(req); iod->q = q; + iod->apif_registered = false; nreq->ctrl = &anv->ctrl; nreq->cmd = &iod->cmd; @@ -1321,6 +1356,16 @@ static int apple_nvme_disable_locked(struct apple_nvme *anv, bool shutdown) return ret; } + if (anv->hw->apif) { + spin_lock_irqsave(&anv->lock, flags); + ret = apple_nvme_apif_drain(anv); + spin_unlock_irqrestore(&anv->lock, flags); + if (ret) { + apple_nvme_quarantine(anv, ret); + return ret; + } + } + spin_lock_irqsave(&anv->lock, flags); apple_nvme_handle_cq(&anv->ioq, true); apple_nvme_handle_cq(&anv->adminq, true); @@ -1732,6 +1777,12 @@ static void apple_nvme_reset_work(struct work_struct *work) dma_set_max_seg_size(anv->dev, 0xffffffff); + if (anv->hw->apif) { + ret = apple_nvme_apif_admin(anv); + if (ret) + goto out; + goto queues_programmed; + } if (anv->hw->has_lsq_nvmmu) { /* * Enable NVMMU and linear submission queues which is required @@ -1793,6 +1844,7 @@ static void apple_nvme_reset_work(struct work_struct *work) anv->mmio_nvmmu + APPLE_NVMMU_IOSQ_TCB_BASE); } +queues_programmed: anv->ctrl.sqsize = anv->hw->max_queue_depth - 1; /* 0's based queue depth */ anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP); @@ -1844,7 +1896,7 @@ static void apple_nvme_reset_work(struct work_struct *work) } } apple_nvme_init_queue(&anv->ioq); - if (anv->hw->needs_ioq_registers) { + if (anv->hw->needs_ioq_registers && !anv->hw->apif) { /* * Post-M4 firmware also takes the I/O queue addresses through * dedicated registers. Without them it crashes and its crash log @@ -2075,6 +2127,10 @@ static void apple_nvme_free_queue(void *data) if (READ_ONCE(anv->quarantined)) return; + if (anv->hw->apif) { + apple_nvme_apif_queue_free(anv, q); + return; + } if (q->tcbs) dma_free_coherent(anv->dev, anv->hw->max_queue_depth * sizeof(struct apple_nvmmu_tcb), @@ -2096,6 +2152,8 @@ static int apple_nvme_queue_alloc(struct apple_nvme *anv, ret = devm_add_action_or_reset(anv->dev, apple_nvme_free_queue, q); if (ret) return ret; + if (anv->hw->apif) + return apple_nvme_apif_queue_alloc(anv, q); q->cqes = dma_alloc_coherent(anv->dev, depth * sizeof(struct nvme_completion), &q->cq_dma_addr, GFP_KERNEL); @@ -2342,7 +2400,9 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) goto out_detach_genpd; } - anv->reset = devm_reset_control_array_get_exclusive(anv->dev); + anv->reset = anv->hw->apif ? + devm_reset_control_array_get_optional_exclusive(anv->dev) : + devm_reset_control_array_get_exclusive(anv->dev); if (IS_ERR(anv->reset)) { ret = dev_err_probe(dev, PTR_ERR(anv->reset), "Failed to get reset control"); @@ -2354,6 +2414,11 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) INIT_WORK(&anv->recovery_work, apple_nvme_recovery_work); spin_lock_init(&anv->lock); + if (anv->hw->apif) { + ret = apple_nvme_apif_init(anv); + if (ret) + goto out_detach_genpd; + } ret = apple_nvme_queue_alloc(anv, &anv->adminq); if (ret) goto out_detach_genpd; @@ -2675,7 +2740,16 @@ static const struct apple_nvme_hw apple_nvme_t8140_hw = { .quirks = NVME_QUIRK_BROKEN_FUA, }; +static const struct apple_nvme_hw apple_nvme_apif_hw = { + .has_lsq_nvmmu = true, + .needs_ioq_registers = true, + .apif = true, + .max_queue_depth = 64, + .quirks = NVME_QUIRK_BROKEN_FUA, +}; + static const struct of_device_id apple_nvme_of_match[] = { + { .compatible = "apple,nvme-apif", .data = &apple_nvme_apif_hw }, { .compatible = "apple,t8015-nvme-ans2", .data = &apple_nvme_t8015_hw }, { .compatible = "apple,t8103-nvme-ans2", .data = &apple_nvme_t8103_hw }, { .compatible = "apple,t8132-nvme-ans2", .data = &apple_nvme_t8132_hw }, diff --git a/include/linux/soc/aurora/apif-mmio.h b/include/linux/soc/aurora/apif-mmio.h index d85ceb7a1cd40..fbf865f3a24c6 100644 --- a/include/linux/soc/aurora/apif-mmio.h +++ b/include/linux/soc/aurora/apif-mmio.h @@ -3,6 +3,8 @@ #define _LINUX_SOC_AURORA_APIF_MMIO_H #include +#include +#include struct device; struct aurora_apif; @@ -34,6 +36,7 @@ struct aurora_apif_boot_iommu { /* Each client owns a buffer. Submission is synchronous and atomic-context * safe. Unbind drains the client and subsequent calls return -ENODEV. */ +#if IS_REACHABLE(CONFIG_AURORA_APIF_MMIO) struct aurora_apif *aurora_apif_get(struct device *dev); void aurora_apif_put(struct aurora_apif *apif); /* @done counts executed operations. The transport does not interpret native @@ -42,4 +45,22 @@ void aurora_apif_put(struct aurora_apif *apif); int aurora_apif_submit(struct aurora_apif *apif, struct aurora_apif_op *ops, u32 count, u32 *done); +#else +static inline struct aurora_apif *aurora_apif_get(struct device *dev) +{ + return ERR_PTR(-ENODEV); +} +static inline void aurora_apif_put(struct aurora_apif *apif) +{ +} +static inline int aurora_apif_submit(struct aurora_apif *apif, + struct aurora_apif_op *ops, u32 count, + u32 *done) +{ + if (done) + *done = 0; + return -ENODEV; +} +#endif + #endif diff --git a/tools/testing/selftests/aurora-apif/apif_mmio.py b/tools/testing/selftests/aurora-apif/apif_mmio.py index 9d6e0be1ac7e3..7e2cdc0c63232 100755 --- a/tools/testing/selftests/aurora-apif/apif_mmio.py +++ b/tools/testing/selftests/aurora-apif/apif_mmio.py @@ -53,6 +53,7 @@ def function(path, name): #define IOMMU_NOEXEC 8 #define IOMMU_MMIO 16 #define GENMASK_ULL(h,l) ((~0ULL << (l)) & (~0ULL >> (63-(h)))) +#define IS_REACHABLE(x) 1 #define GFP_ATOMIC 0 #define kmalloc(n,flags) malloc(n) #define virt_to_phys(p) ((u64)(uintptr_t)(p)) @@ -268,12 +269,83 @@ def sart_test(): ''' +def nvme_test(): + helpers = "\n".join(function("drivers/nvme/host/apple-apif.h", name) for name in + ["apple_nvme_apif_translate", "apple_nvme_apif_request"]) + return HEADER + r''' +#define SZ_4K 4096UL +#define APPLE_NVME_APIF_PAGES 257 +#define APPLE_NVME_APIF_LIST_SIZE SZ_4K +#define DIV_ROUND_UP(n,d) (((n)+(d)-1)/(d)) +#define IS_ALIGNED(n,a) (!((n)&((a)-1))) +#define lockdep_assert_held(p) ((void)(p)) +#define dma_wmb() ((void)0) +#define cpu_to_le16(v) (v) +#define le64_to_cpu(v) (v) +typedef uint64_t __le64; +struct command { struct { struct {u64 prp1, prp2;} dptr; u32 command_id; } common; }; +struct apple_nvmmu_tcb { u32 pad; unsigned short length; char rest[122]; }; +struct apple_nvme_iod { struct command cmd; bool apif_registered; __le64 *prps; }; +struct request { struct apple_nvme_iod iod; u32 bytes; }; +struct apple_nvme_hw { u32 max_queue_depth; }; +struct apple_nvme { void *dev; int lock; struct aurora_apif *apif; struct apple_nvme_hw *hw; + struct aurora_apif_op *apif_ops; void *apif_scratch; u64 apif_scratch_dma; bool apif_uncertain; }; +struct apple_nvme_queue { struct apple_nvme *anv; bool is_adminq; struct apple_nvmmu_tcb tcbs[64]; }; +static struct apple_nvme *queue_to_apple_nvme(struct apple_nvme_queue *q) {return q->anv;} +static struct apple_nvme_iod *blk_mq_rq_to_pdu(struct request *r) {return &r->iod;} +static u32 nvme_tag_from_cid(u32 tag) {return tag;} +static u32 blk_rq_payload_bytes(struct request *r) {return r->bytes;} +static void **apple_nvme_iod_list(struct request *r) {return (void **)&r->iod.prps;} +static u64 dma_to_phys(void *dev, u64 p) {(void)dev; return p;} +static int map_calls, translate_calls, mode; +static struct aurora_apif_op mapped; +int aurora_apif_submit(struct aurora_apif *apif, struct aurora_apif_op *ops, u32 n, u32 *done) { + (void)apif; if(done) *done=n; + for(u32 i=0;ilength==255); + assert(translate_calls>=257); + return 0; +} +''' + + def main(): compiler = shlex.split(os.environ.get("HOSTCC", "cc")) - print("TAP version 13\n1..3", flush=True) + print("TAP version 13\n1..4", flush=True) with tempfile.TemporaryDirectory(prefix="apif-selftest-") as directory: for i, (name, test) in enumerate([( "transport", transport_test), - ("dart", dart_test), ("sart", sart_test)], 1): + ("dart", dart_test), ("sart", sart_test), ("nvme", nvme_test)], 1): path = Path(directory) / name path.with_suffix(".c").write_text(test()) subprocess.run(compiler + ["-std=gnu11", "-Wall", "-Wextra", "-Werror",