diff --git a/.mailmap b/.mailmap index 0b9298a55d2de..70080097bc1a3 100644 --- a/.mailmap +++ b/.mailmap @@ -243,6 +243,7 @@ Dengcheng Zhu Dikshita Agarwal Dikshita Agarwal +Dj <43033058+DjDeveloperr@users.noreply.github.com> Dmitry Baryshkov Dmitry Baryshkov <[dbaryshkov@gmail.com]> Dmitry Baryshkov diff --git a/Documentation/devicetree/bindings/firmware/aurora,apif-mmio.yaml b/Documentation/devicetree/bindings/firmware/aurora,apif-mmio.yaml new file mode 100644 index 0000000000000..c154786a0593c --- /dev/null +++ b/Documentation/devicetree/bindings/firmware/aurora,apif-mmio.yaml @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/firmware/aurora,apif-mmio.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Aurora Platform Interface over Memory-Mapped I/O + +maintainers: + - Dj + +description: | + Synthetic MMIO transport supplied by Aura's EL2-resident monitor. Version 2 + forwards native SPTM selectors and arguments and provides memory-ownership + services. Peripheral policy belongs to consumers, which reference this node + through the aurora,apif phandle. The register page must be absent from the + guest stage-2 mappings so accesses reach Aura. + +properties: + compatible: + const: aurora,apif-mmio + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + apif: firmware@61ff0000 { + compatible = "aurora,apif-mmio"; + reg = <0x61ff0000 0x4000>; + }; diff --git a/Documentation/devicetree/bindings/iommu/apple,dart-apif.yaml b/Documentation/devicetree/bindings/iommu/apple,dart-apif.yaml new file mode 100644 index 0000000000000..f41c38f4d492a --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/apple,dart-apif.yaml @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iommu/apple,dart-apif.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple DART mediated through Aura APIF + +maintainers: + - Dj + +description: | + A DART instance accessed through native SPTM calls rather than guest MMIO. + Linux owns stream grouping, table setup, mapping rollback and DMA apertures. + Firmware-owned streams are adopted without replacing their roots or replaying + retained mappings. Unit identifiers are firmware metadata, not chip IDs. + +properties: + compatible: + const: apple,dart-apif + '#iommu-cells': + const: 1 + description: Stream identifier. + aurora,apif: + $ref: /schemas/types.yaml#/definitions/phandle + description: APIF version-2 transport provider. + apple,apif-dart-id: + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 255 + description: Native SPTM DART unit identifier. + apple,apif-num-streams: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 256 + default: 256 + description: Number of streams supported by this unit. + apple,apif-retained-sids: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 256 + uniqueItems: true + items: + maximum: 255 + description: Firmware-owned stream IDs which Aura adopted at bootstrap. + apple,apif-iova-force: + $ref: /schemas/types.yaml#/definitions/uint64 + enum: [0, 1099511627776] + description: | + Optional native bit-40 force mask for a low-address device DMA view. + Must not overlap apple,dma-range. Omit for a native DVA aperture. + apple,apif-reserve-last-page: + type: boolean + description: Native aperture forbids allocating its final 16 KiB granule. + apple,dma-range: + $ref: /schemas/types.yaml#/definitions/uint64-array + minItems: 2 + maxItems: 2 + description: | + Device-visible DMA aperture base and size, aligned to 16 KiB and within + 42 bits. Defaults to the low 4 GiB. This is not a CPU mapping. + +required: + - compatible + - '#iommu-cells' + - aurora,apif + - apple,apif-dart-id + +additionalProperties: false + +examples: + - | + iommu { + compatible = "apple,dart-apif"; + #iommu-cells = <1>; + aurora,apif = <&apif>; + apple,apif-dart-id = <5>; + apple,apif-num-streams = <16>; + }; diff --git a/Documentation/devicetree/bindings/soc/apple/apple,sart-apif.yaml b/Documentation/devicetree/bindings/soc/apple/apple,sart-apif.yaml new file mode 100644 index 0000000000000..4e5ece8bebaf5 --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,sart-apif.yaml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,sart-apif.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SART mediated through Aura APIF + +maintainers: + - Dj + +description: | + SART provider using native SPTM table-5 calls. Existing consumers reference + this provider through their apple,sart phandle. Linux tracks only its own + regions; unknown firmware grants are not reported as inherited regions. + +properties: + compatible: + const: apple,sart-apif + aurora,apif: + $ref: /schemas/types.yaml#/definitions/phandle + description: APIF version-2 transport provider. + +required: + - compatible + - aurora,apif + +additionalProperties: false + +examples: + - | + sart { + compatible = "apple,sart-apif"; + aurora,apif = <&apif>; + }; diff --git a/Documentation/driver-api/firmware/aurora-apif.rst b/Documentation/driver-api/firmware/aurora-apif.rst new file mode 100644 index 0000000000000..f13cf3ab3f03a --- /dev/null +++ b/Documentation/driver-api/firmware/aurora-apif.rst @@ -0,0 +1,100 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +=============================================== +Aurora Platform Interface over Memory-Mapped I/O +=============================================== + +Aura is Aurora's EL2-resident monitor for Apple Silicon, providing SPTM +mediation and nested virtualization. APIF is a synchronous, trusted-kernel +transport. Linux supplies native selectors and endpoint arguments; Aura +forwards them unchanged. Invalid native calls can stop the machine. This +interface does not isolate an untrusted kernel. + +Wire contract +============= + +The version-2 synthetic register page occupies 16 KiB at guest IPA 0x61ff0000. +Registers are little-endian 64-bit fields: magic at 0x000 is +0x4f494d4d46495041 (APIFMMIO), version at 0x008 is 2, maximum operations at +0x010, and feature flags at 0x018 (bit 0 native, bit 1 memory, bit 2 I/O). +A 64-bit write at 0x100 supplies a 16-byte-aligned guest IPA for the batch. +The entire buffer must fit within one guest RAM bank. The register page is +unmapped at guest stage 2; Aura handles its data aborts. + +A batch has a 16-byte header (u32 count, u32 flags, s32 status, u32 done), +then count 64-byte operations (u64 selector, u64 arg[6], u64 ret). Flags are +zero. Selector upper/lower halves give native table/endpoint (x16); arguments +are native x0-x5 and ret is native x0, unchanged. Transport status does not +interpret native results. Void and boolean endpoints need different caller +handling. A failed Aura service stops the batch; done is the completed prefix. + +Each client owns a physically contiguous buffer and serializes its submissions +with IRQs masked. The transport orders memory around the trapping store and +preserves the Linux stack in x9 across the synchronous redispatch contract. +Device links order supplier lifetime; client revocation drains submissions and +subsequent calls return ENODEV. It is safe to call from atomic context. Provider +and consumers must be described together in the guest DT; this series does not +rewrite existing board DTs or enable the backend on an existing boot. + +Ownership services +================== + +Table 0xffffffff supplies Aura services (other tables are native selectors): + +* 1 TRANSLATE(guest IPA, size): native PA for a contiguous guest range. +* 2 ALLOC_FRAME(owner, address): zeroed 16 KiB native frame, CPU alias removed. +* 3 FREE_FRAME(PA): restore Aura's alias after detachment and retype to type 11. +* 4 MAP_IO(guest IPA, PA, size, readonly): map bounded page-aligned device I/O. +* 5 GUEST_ROOT(): guest stage-2 root PA. +* 6 BOOT_IOMMU(kind, unit, SID, index, output IPA, size): bootstrap snapshot. +* 7 BOOT_TABLE(kind, unit, SID, address, level): existing bootstrap table PA. +* 8 FIND_FRAME(owner, address): registered lent frame PA or zero. +* 9 TAG_FRAME(PA, owner, address): register a unique opaque key after attachment. + +Unused arguments are zero. Frame keys are bookkeeping, not an ownership +transition. The pool is capped at 1024 frames. Never free attached or uncertain +frames. MAP_IO does not roll back a partially successful mapping; a failed +mapping requires recovery before its address range can be reused. Initialize +batch status to EINPROGRESS so a missing completion cannot be mistaken for +success. Lost table-operation completion stops further DART transactions. + +BOOT_IOMMU kind 0 means DART. Index U64_MAX returns root level, run count and +DVA aperture; another index returns a retained mapping. The 64-byte output is +u64 level, count, base, size, pa, ipa, attrs, reserved. An unavailable IPA is +U64_MAX. BOOT_TABLE levels 0-2 return existing bootstrap tables, while U64_MAX +returns the bootstrap initialized flag. Bootstrap queries never replay maps. + +DART and SART providers +======================= + +APPLE_DART_APIF supplies an IOMMU provider for apple,dart-apif. It preserves +stream groups, shared IOVA allocation, retained mappings, bounded batches and +partial-prefix rollback. Linux builds native physical page lists, complements +IOMMU allow bits into native deny bits, and sets up typed tables. Both DMA +backing and list-buffer pointers are translated before native calls. Retained +streams remain reserved and are excluded from ordinary power-down. + +APPLE_SART_APIF supplies apple,sart-apif through the existing apple_sart API. +The raw MMIO provider continues using its current operations. The APIF provider +tracks Linux-owned regions and respects endpoint-specific return semantics; +unknown firmware grants are not advertised as inherited. Neither provider +prints routine successful calls or runs probe-time mapping self-tests. + +The existing arm64 TSO task interface discovers guest ACTLR support through +its toggle/readback probe; APIF does not add a second TSO protocol. The Apple +WFI preservation quirk also covers captured T6050 parts 0x064/0x065 and uses +the same CPU predicate to avoid unsafe timed waits in arbitrary delay callers. +Existing CPU ranges are preserved. This does not claim future Ultra support. + +Validation +========== + +Build the providers and run the production-function host tests with:: + + make ARCH=arm64 LLVM=1 Image + make -C tools/testing/selftests/aurora-apif run_tests + +Host tests exercise framing, prefix completion, ownership ordering, list +translation and endpoint return handling. Hardware validation of this cleaned +version-2 integration is pending. GPU backend changes are outside this series; +legacy version-1 Aura aliases remain available to existing clients. diff --git a/Documentation/driver-api/firmware/index.rst b/Documentation/driver-api/firmware/index.rst index 86a3dd4bc3f8a..bd0811b39446f 100644 --- a/Documentation/driver-api/firmware/index.rst +++ b/Documentation/driver-api/firmware/index.rst @@ -4,6 +4,7 @@ Linux Firmware API .. toctree:: + aurora-apif introduction core efi/index diff --git a/MAINTAINERS b/MAINTAINERS index 99987f9eca95b..0e00f210893f1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4359,6 +4359,21 @@ F: rust/helpers/auxiliary.c F: rust/kernel/auxiliary.rs F: samples/rust/rust_driver_auxiliary.rs +AURORA PLATFORM MMIO INTERFACE +M: Dj +S: Maintained +F: Documentation/devicetree/bindings/firmware/aurora,apif-mmio.yaml +F: Documentation/devicetree/bindings/iommu/apple,dart-apif.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,sart-apif.yaml +F: Documentation/driver-api/firmware/aurora-apif.rst +F: drivers/firmware/aurora-apif-mmio.c +F: drivers/iommu/apple-dart-apif.c +F: drivers/soc/apple/apple-sart-apif.c +F: drivers/soc/apple/sart-backend.h +F: include/linux/apple-dart-apif.h +F: include/linux/soc/aurora/apif-mmio.h +F: tools/testing/selftests/aurora-apif/ + AUXILIARY DISPLAY DRIVERS M: Andy Shevchenko R: Geert Uytterhoeven diff --git a/arch/arm64/include/asm/apple-idle.h b/arch/arm64/include/asm/apple-idle.h index 56f5d32e3d41b..bec7a77b70c64 100644 --- a/arch/arm64/include/asm/apple-idle.h +++ b/arch/arm64/include/asm/apple-idle.h @@ -2,7 +2,22 @@ #ifndef __ASM_APPLE_IDLE_H #define __ASM_APPLE_IDLE_H -/* WFI that preserves x18-x30 on affected Apple cores (T8140 and T8152). */ +#include + +#ifdef CONFIG_ARCH_APPLE +extern struct static_key_false apple_wfi_loses_regs; +static __always_inline bool apple_wfi_needs_save(void) +{ + return static_branch_unlikely(&apple_wfi_loses_regs); +} +#else +static __always_inline bool apple_wfi_needs_save(void) +{ + return false; +} +#endif + +/* WFI that preserves x18-x30 on cores selected by the idle quirk. */ void apple_cpu_wfi(void); #endif diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 809e39962fe23..acbee4d749533 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -157,6 +157,8 @@ #define APPLE_CPU_PART_TAHITI_P 0x061 #define APPLE_CPU_PART_T8142_E 0x062 #define APPLE_CPU_PART_T8142_P 0x063 +#define APPLE_CPU_PART_T6050_064 0x064 +#define APPLE_CPU_PART_T6050_065 0x065 #define APPLE_CPU_PART_T8152_E 0x072 #define APPLE_CPU_PART_T8152_P 0x073 @@ -256,6 +258,8 @@ #define MIDR_APPLE_TAHITI_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_TAHITI_P) #define MIDR_APPLE_T8142_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8142_E) #define MIDR_APPLE_T8142_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8142_P) +#define MIDR_APPLE_T6050_064 MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T6050_064) +#define MIDR_APPLE_T6050_065 MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T6050_065) #define MIDR_APPLE_T8152_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8152_E) #define MIDR_APPLE_T8152_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8152_P) #define MIDR_AMPERE1 MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1) diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c index 14cde8592c86e..86c3099494aa6 100644 --- a/arch/arm64/kernel/idle.c +++ b/arch/arm64/kernel/idle.c @@ -41,18 +41,20 @@ early_param("idle", setup_idle); #ifdef CONFIG_ARCH_APPLE /* - * Apple T8140 (A18 Pro), T8142 (M5), and T8152 (M6) CPUs can return from + * Apple T8140 (A18 Pro), T8142 (M5), T6050, and T8152 (M6) CPUs can return from * WFI with general-purpose registers cleared. T8140 also loses FP/SIMD - * state; conservatively preserve it on all three parts. The key is enabled once + * state; conservatively preserve it on these parts. The key is enabled once * at boot on those parts; everywhere else the check is a patched-out branch. */ -static DEFINE_STATIC_KEY_FALSE(apple_wfi_loses_regs); +DEFINE_STATIC_KEY_FALSE(apple_wfi_loses_regs); static const struct midr_range apple_wfi_loses_regs_cpus[] __initconst = { MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_E), MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_P), MIDR_ALL_VERSIONS(MIDR_APPLE_T8142_E), MIDR_ALL_VERSIONS(MIDR_APPLE_T8142_P), + MIDR_ALL_VERSIONS(MIDR_APPLE_T6050_064), + MIDR_ALL_VERSIONS(MIDR_APPLE_T6050_065), MIDR_ALL_VERSIONS(MIDR_APPLE_T8152_E), MIDR_ALL_VERSIONS(MIDR_APPLE_T8152_P), {} @@ -67,16 +69,6 @@ static int __init apple_wfi_quirk_init(void) return 0; } early_initcall(apple_wfi_quirk_init); - -static __always_inline bool apple_wfi_needs_save(void) -{ - return static_branch_unlikely(&apple_wfi_loses_regs); -} -#else -static __always_inline bool apple_wfi_needs_save(void) -{ - return false; -} #endif /* diff --git a/arch/arm64/lib/delay.c b/arch/arm64/lib/delay.c index e0c5e615b5e5d..2dd5d8394e494 100644 --- a/arch/arm64/lib/delay.c +++ b/arch/arm64/lib/delay.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -43,17 +44,9 @@ void __delay(unsigned long cycles) { cycles_t start = __delay_cycles(); - /* - * T8142 can clear registers on WFIT just as it does on WFI. A delay - * may run with live kernel SIMD state, so the idle task's save/flush - * path is not applicable here. Keep the core awake for short delays. - */ - if (IS_ENABLED(CONFIG_ARCH_APPLE)) { - u32 midr = read_cpuid_id() & MIDR_CPU_MODEL_MASK; - - if (midr == MIDR_APPLE_T8142_E || midr == MIDR_APPLE_T8142_P) - goto counter_delay; - } + /* The idle task's SIMD save path does not apply to arbitrary delay callers. */ + if (apple_wfi_needs_save()) + goto counter_delay; if (alternative_has_cap_unlikely(ARM64_HAS_WFXT)) { u64 end = start + cycles; diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index bbd2155d84838..7fa13654a037d 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -6,6 +6,16 @@ menu "Firmware Drivers" +config AURORA_APIF_MMIO + tristate "Aurora platform MMIO interface" + depends on ARM64 && OF + help + Submit synchronous, batched platform operations to an Aura monitor. + The interface forwards native SPTM selectors, arguments and results + without interpreting peripheral policy. Clients own their buffers. + Submission is synchronous and supports atomic-context consumers. + + source "drivers/firmware/arm_scmi/Kconfig" config ARM_SCPI_PROTOCOL diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile index 4ddec2820c96f..73deaed370eab 100644 --- a/drivers/firmware/Makefile +++ b/drivers/firmware/Makefile @@ -38,3 +38,5 @@ obj-y += samsung/ obj-y += smccc/ obj-y += tegra/ obj-y += xilinx/ + +obj-$(CONFIG_AURORA_APIF_MMIO) += aurora-apif-mmio.o diff --git a/drivers/firmware/aurora-apif-mmio.c b/drivers/firmware/aurora-apif-mmio.c new file mode 100644 index 0000000000000..aa6ec65c6248c --- /dev/null +++ b/drivers/firmware/aurora-apif-mmio.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Aurora Platform Interface over Memory-Mapped I/O. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define APIF_MAGIC 0x4f494d4d46495041ULL +#define APIF_VERSION 2 +#define APIF_REG_MAGIC 0x000 +#define APIF_REG_VERSION 0x008 +#define APIF_REG_MAX_OPS 0x010 +#define APIF_REG_FEATURES 0x018 +#define APIF_REG_DOORBELL 0x100 +#define APIF_FEAT_NATIVE BIT_ULL(0) +#define APIF_BATCH_BYTES SZ_16K + +struct apif_batch { + u32 count, flags; + s32 status; + u32 done; + struct aurora_apif_op ops[]; +}; + +#define APIF_BATCH_MAX \ + ((APIF_BATCH_BYTES - sizeof(struct apif_batch)) / \ + sizeof(struct aurora_apif_op)) + +struct apif_provider { + void __iomem *regs; + u32 max_ops; +}; + +struct aurora_apif { + struct kref refs; + raw_spinlock_t lock; + bool live; + void __iomem *doorbell; + struct apif_batch *batch; + u32 max_ops; +}; + +static struct platform_driver apif_driver; +static_assert(sizeof(struct apif_batch) == 16); +static_assert(sizeof(struct aurora_apif_op) == 64); +static_assert(sizeof(struct aurora_apif_boot_iommu) == 64); + +/* SPTM redispatch restores SP_EL1 from its exception-stack contract. Keep the + * Linux stack in a preserved GPR across the trapping store. The monitor must + * preserve all GPRs, including x9, and return synchronously after the batch. + */ +static void apif_ring(struct aurora_apif *apif) +{ + u64 pa = virt_to_phys(apif->batch); + + /* Publish the request before its synchronous doorbell. */ + wmb(); + asm volatile("mov x9, sp\n" + "str %x[pa], [%[db]]\n" + "mov sp, x9\n" + : + : [pa] "r"(pa), [db] "r"(apif->doorbell) + : "x9", "memory"); + /* Observe completion and returned fields after redispatch. */ + rmb(); +} + +int aurora_apif_submit(struct aurora_apif *apif, struct aurora_apif_op *ops, + u32 count, u32 *done) +{ + unsigned long flags; + u32 total = 0; + int ret = 0; + + if (done) + *done = 0; + if (!apif || (count && !ops)) + return -EINVAL; + raw_spin_lock_irqsave(&apif->lock, flags); + if (!apif->live) { + ret = -ENODEV; + goto out; + } + while (total < count) { + u32 n = min(count - total, apif->max_ops); + u32 reported; + + apif->batch->count = n; + apif->batch->flags = 0; + apif->batch->status = -EINPROGRESS; + apif->batch->done = 0; + memcpy(apif->batch->ops, ops + total, n * sizeof(*ops)); + apif_ring(apif); + reported = READ_ONCE(apif->batch->done); + if (reported > n) { + ret = -EIO; + break; + } + memcpy(ops + total, apif->batch->ops, n * sizeof(*ops)); + total += reported; + ret = READ_ONCE(apif->batch->status); + if (ret || reported != n) { + if (ret >= 0 || ret == -EINPROGRESS) + ret = -EIO; + break; + } + } +out: + raw_spin_unlock_irqrestore(&apif->lock, flags); + if (done) + *done = total; + return ret; +} +EXPORT_SYMBOL_GPL(aurora_apif_submit); + +static void apif_free(struct kref *refs) +{ + kfree(container_of(refs, struct aurora_apif, refs)); +} + +void aurora_apif_put(struct aurora_apif *apif) +{ + kref_put(&apif->refs, apif_free); +} +EXPORT_SYMBOL_GPL(aurora_apif_put); + +static void apif_revoke(void *data) +{ + struct aurora_apif *apif = data; + unsigned long flags; + + raw_spin_lock_irqsave(&apif->lock, flags); + apif->live = false; + apif->doorbell = NULL; + apif->batch = NULL; + raw_spin_unlock_irqrestore(&apif->lock, flags); + aurora_apif_put(apif); +} + +struct aurora_apif *aurora_apif_get(struct device *dev) +{ + struct device_node *node; + struct platform_device *supplier; + struct apif_provider *provider; + struct aurora_apif *apif; + struct device_link *link; + int ret; + + node = of_parse_phandle(dev->of_node, "aurora,apif", 0); + if (!node) + return ERR_PTR(-ENODEV); + supplier = of_find_device_by_node(node); + of_node_put(node); + if (!supplier) + return ERR_PTR(-EPROBE_DEFER); + if (supplier->dev.driver != &apif_driver.driver) { + ret = -EPROBE_DEFER; + goto put_device; + } + link = device_link_add(dev, &supplier->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + if (!link) { + ret = -ENOMEM; + goto put_device; + } + provider = platform_get_drvdata(supplier); + if (!provider) { + ret = -EPROBE_DEFER; + goto remove_link; + } + apif = kzalloc(sizeof(*apif), GFP_KERNEL); + if (!apif) { + ret = -ENOMEM; + goto remove_link; + } + kref_init(&apif->refs); + raw_spin_lock_init(&apif->lock); + apif->batch = devm_kzalloc(dev, APIF_BATCH_BYTES, GFP_KERNEL); + if (!apif->batch) { + ret = -ENOMEM; + aurora_apif_put(apif); + goto remove_link; + } + apif->max_ops = provider->max_ops; + apif->doorbell = provider->regs + APIF_REG_DOORBELL; + apif->live = true; + kref_get(&apif->refs); + ret = devm_add_action_or_reset(dev, apif_revoke, apif); + if (ret) { + aurora_apif_put(apif); + goto remove_link; + } + put_device(&supplier->dev); + return apif; +remove_link: + device_link_del(link); +put_device: + put_device(&supplier->dev); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(aurora_apif_get); + +static int apif_probe(struct platform_device *pdev) +{ + struct apif_provider *provider; + struct resource *resource; + u64 max_ops; + + resource = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!resource || + resource_size(resource) < APIF_REG_DOORBELL + sizeof(u64)) + return -EINVAL; + provider = devm_kzalloc(&pdev->dev, sizeof(*provider), GFP_KERNEL); + if (!provider) + return -ENOMEM; + provider->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(provider->regs)) + return PTR_ERR(provider->regs); + if (readq(provider->regs + APIF_REG_MAGIC) != APIF_MAGIC || + readq(provider->regs + APIF_REG_VERSION) != APIF_VERSION || + !(readq(provider->regs + APIF_REG_FEATURES) & APIF_FEAT_NATIVE)) + return dev_err_probe(&pdev->dev, -ENODEV, + "unsupported APIF interface\n"); + max_ops = readq(provider->regs + APIF_REG_MAX_OPS); + if (!max_ops) + return -EINVAL; + provider->max_ops = min_t(u64, max_ops, APIF_BATCH_MAX); + platform_set_drvdata(pdev, provider); + return 0; +} + +static const struct of_device_id apif_of_match[] = { + { .compatible = "aurora,apif-mmio" }, + {} +}; +MODULE_DEVICE_TABLE(of, apif_of_match); + +static struct platform_driver apif_driver = { + .driver = { + .name = "aurora-apif-mmio", + .of_match_table = apif_of_match, + }, + .probe = apif_probe, +}; +module_platform_driver(apif_driver); + +MODULE_DESCRIPTION("Aurora Platform Interface over Memory-Mapped I/O"); +MODULE_LICENSE("GPL"); diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 43c03b46f43d2..a90a6e22dca9c 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -316,6 +316,19 @@ config APPLE_DART Say Y here if you are using an Apple SoC. + +config APPLE_DART_APIF + bool "Apple DART over the Aurora platform interface" + depends on ARM64_16K_PAGES && OF + depends on ARCH_APPLE || COMPILE_TEST + select AURORA_APIF_MMIO + select IOMMU_API + help + Apple DART backend for a trusted Linux guest under Aura and SPTM. + Native mapping and retyping policy lives in Linux; the monitor + provides call transport, frame ownership and bootstrap metadata. + Existing raw-MMIO DART drivers continue to serve other systems. + config APPLE_DART_KUNIT_TEST bool "KUnit tests for Apple DART locked domains" if !KUNIT_ALL_TESTS depends on APPLE_DART=y && KUNIT=y diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 0275821f4ef98..b5d8f669e1990 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -37,3 +37,5 @@ obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o obj-$(CONFIG_APPLE_DART) += apple-dart.o obj-$(CONFIG_IOMMU_DEBUG_PAGEALLOC) += iommu-debug-pagealloc.o + +obj-$(CONFIG_APPLE_DART_APIF) += apple-dart-apif.o diff --git a/drivers/iommu/apple-dart-apif.c b/drivers/iommu/apple-dart-apif.c new file mode 100644 index 0000000000000..cf2f43bafc9c1 --- /dev/null +++ b/drivers/iommu/apple-dart-apif.c @@ -0,0 +1,1904 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Apple DART IOMMU backend using Aura's native SPTM call transport. + * Linux owns stream topology, translation-table setup, DMA apertures and + * transactional mapping bookkeeping. Aura forwards native selectors 1:1. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dma-iommu.h" + +/* Internal operations used by the IOMMU bookkeeping; not the APIF wire ABI. */ +#define APIF_DART_INIT 1 /* () INIT + POWERUP */ +#define APIF_DART_ATTACH 2 /* (sid, dva, level) */ +#define APIF_DART_MAP 3 /* (sid, dva, ipa, size, prot) */ +#define APIF_DART_UNMAP 4 /* (sid, dva, size) */ +#define APIF_DART_ENABLE 5 /* (sid) */ +#define APIF_DART_DISABLE 6 /* (sid) */ + +#define APIF_DART_RETAINED_INFO 9 +#define APIF_DART_RETAINED_RUN 10 + +struct apif_op { + u32 op; + u32 flags; + u64 arg[6]; + u64 ret; +}; + +struct apif_batch { + u32 count; + u32 flags; + s32 status; + u32 done; + struct apif_op op[]; +}; + +#define DART_APIF_PAGE_SHIFT 14 +#define DART_APIF_PAGE_SIZE (1UL << DART_APIF_PAGE_SHIFT) /* 16 KiB */ +/* apif's DART MAP forwards one contiguous run within a single 32 MiB leaf-table + * span, so every MAP op must stay inside one aligned block. + */ +#define DART_APIF_BLOCK_SHIFT 25 +#define DART_APIF_BLOCK_SIZE (1UL << DART_APIF_BLOCK_SHIFT) /* 32 MiB */ + +#define DART_APIF_MAX_STREAMS 256 +#define DART_APIF_APERTURE_END (SZ_4G - 1) +#define DART_APIF_NATIVE_APERTURE_END GENMASK_ULL(41, 0) + +/* Preserve batched per-block operations and the existing rollback accounting. */ +#define APIF_BATCH_MAX_OPS \ + ((PAGE_SIZE - sizeof(struct apif_batch)) / sizeof(struct apif_op)) + +static const struct iommu_ops apple_dart_apif_ops; + +struct apif_retained_run { + u64 dva, ipa, size; +}; +struct apif_retained_sid { + unsigned int root_level, count; + struct apif_retained_run *runs; +}; + +struct apple_dart_apif { + struct device *dev; + struct iommu_device iommu; + struct aurora_apif *apif; + struct aurora_apif_op *native_ops; + struct aurora_apif_boot_iommu *boot_info; + u64 *page_lists[APIF_BATCH_MAX_OPS]; + u64 page_list_pa[APIF_BATCH_MAX_OPS]; + u32 dart_id; + u32 num_streams; + u64 iova_force; /* Native address force mask, hidden from DMA API/device addresses. */ + u64 dma_min, dma_max; + bool reserve_last_page; + bool inited; + bool ownership_uncertain; /* A table operation needs a fresh boot after lost completion. */ + struct apif_retained_sid *retained[DART_APIF_MAX_STREAMS]; + struct mutex init_lock; /* serialises one-time init */ + /* Protected by apple_dart_apif_groups_lock; group-owned lifetime. */ + struct iommu_group *sid2group[DART_APIF_MAX_STREAMS]; + + struct apif_batch *batch; /* internal mapping-operation buffer */ + spinlock_t batch_lock; /* protects operation buffers and stream updates */ +}; + +/* Match the native DART driver's capacity for shared display DMA domains. */ +#define APIF_MAX_DARTS 8 + +struct apif_stream_map { + struct apple_dart_apif *dart; + DECLARE_BITMAP(sids, DART_APIF_MAX_STREAMS); +}; + +struct apple_dart_apif_master { + unsigned int num_darts; + struct apif_stream_map streams[APIF_MAX_DARTS]; +}; + +struct apple_dart_apif_domain { + struct iommu_domain domain; + unsigned int num_darts; + struct apif_stream_map streams[APIF_MAX_DARTS]; + struct mutex lock; /* serialises domain attachments */ + /* Mapping calls may run concurrently on distinct ranges. An attachment + * must replay a stable mapping set before publishing its new streams. + */ + rwlock_t streams_lock; + /* Key: dart-id:8, sid:8, block:40. SPTM tables cannot be attached twice. */ + struct xarray l2_attached; + bool has_retained; + /* One canonical mapping set and IOVA allocator for every DART. */ + struct xarray mappings; +}; + +static struct apif_stream_map * +apif_find_stream(struct apple_dart_apif_domain *dom, + struct apple_dart_apif *dart) +{ + if (dom) + for (unsigned int i = 0; i < dom->num_darts; i++) + if (dom->streams[i].dart == dart) + return &dom->streams[i]; + return NULL; +} + +/* Domain topology is fixed before the first attachment. Later devices can + * add SIDs on these DARTs, but cannot change an active allocator's aperture. + */ +static int apif_domain_configure(struct apple_dart_apif_domain *dom, + const struct apple_dart_apif_master *master) +{ + u64 start = 0, end = DART_APIF_NATIVE_APERTURE_END; + + if (!master || !master->num_darts) + return -ENODEV; + if (dom->num_darts) { + for (unsigned int i = 0; i < master->num_darts; i++) + if (!apif_find_stream(dom, master->streams[i].dart)) + return -EINVAL; + return 0; + } + for (unsigned int i = 0; i < master->num_darts; i++) { + start = max(start, master->streams[i].dart->dma_min); + end = min(end, master->streams[i].dart->dma_max); + } + if (start > end) + return -EINVAL; + for (unsigned int i = 0; i < master->num_darts; i++) + dom->streams[i].dart = master->streams[i].dart; + dom->domain.geometry.aperture_start = start; + dom->domain.geometry.aperture_end = end; + dom->num_darts = master->num_darts; + return 0; +} + +static struct apple_dart_apif_domain *to_apif_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct apple_dart_apif_domain, domain); +} + +/* The native DMA address space strips the force bits before returning a device DMA address + * and restores it for DART operations. Translate only the bounded transport + * batch; all live-map bookkeeping and the DMA aperture remain device-visible. + * The force bits are disjoint from the validated low-4-GiB aperture. + */ +static void apif_force_iovas(struct apple_dart_apif *dart, bool restore) +{ + if (!dart->iova_force) + return; + for (u32 i = 0; i < dart->batch->count; i++) { + struct apif_op *op = &dart->batch->op[i]; + + switch (op->op) { + case APIF_DART_ATTACH: + case APIF_DART_MAP: + case APIF_DART_UNMAP: + if (restore) + op->arg[1] &= ~dart->iova_force; + else + op->arg[1] |= dart->iova_force; + break; + } + } +} + +static void apif_put_transport(void *data) +{ + aurora_apif_put(data); +} + +static void apif_free_page_lists(void *data) +{ + struct apple_dart_apif *dart = data; + + for (unsigned int i = 0; i < APIF_BATCH_MAX_OPS; i++) + kfree(dart->page_lists[i]); +} + +static int apif_one(struct apple_dart_apif *dart, struct aurora_apif_op *op) +{ + return aurora_apif_submit(dart->apif, op, 1, NULL); +} + +static int apif_prepare_table(struct apple_dart_apif *dart, u64 sid, u64 dva, + u64 level) +{ + u64 base = level ? round_down(dva, 1ULL << (level == 1 ? 36 : 25)) : 0; + u64 owner = 0x4441525400000000ULL | ((u64)dart->dart_id << 16) | + (sid << 8) | level; + struct aurora_apif_op op = { + .selector = AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, + AURORA_APIF_BOOT_TABLE), + .arg = { AURORA_APIF_BOOT_DART, dart->dart_id, sid, dva, + level }, + }; + u64 pa; + int ret; + + if (sid >= dart->num_streams || level > 2) + return -EINVAL; + if (level == 2 && dart->retained[sid] && + !dart->retained[sid]->root_level) { + ret = apif_prepare_table(dart, sid, dva, 1); + if (ret) + return ret; + } + ret = apif_one(dart, &op); + if (ret || op.ret) + return ret; + op = (struct aurora_apif_op){ + .selector = AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, + AURORA_APIF_FIND_FRAME), + .arg = { owner, base }, + }; + ret = apif_one(dart, &op); + if (ret || op.ret) + return ret; + op = (struct aurora_apif_op){ + .selector = AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, + AURORA_APIF_ALLOC_FRAME), + }; + ret = apif_one(dart, &op); + if (ret) + return ret; + pa = op.ret; + op = (struct aurora_apif_op){ + .selector = AURORA_APIF_SELECTOR(0, 1), /* RETYPE */ + .arg = { pa, 11, 24, + (((level << 16) | (sid << 8) | dart->dart_id) << 32) | + 5 }, + }; + ret = apif_one(dart, &op); + if (ret) { + dart->ownership_uncertain = true; + return ret; /* Retain the frame until a fresh boot. */ + } + if (!op.ret) { + op = (struct aurora_apif_op){ + .selector = AURORA_APIF_SELECTOR( + AURORA_APIF_SERVICE, AURORA_APIF_FREE_FRAME), + .arg = { pa }, + }; + apif_one(dart, &op); + return -EIO; + } + op = (struct aurora_apif_op){ + .selector = AURORA_APIF_SELECTOR(3, 0), /* DART MAP_TABLE */ + .arg = { dart->dart_id, sid, dva, level, pa }, + }; + ret = apif_one(dart, &op); + if (ret) { + dart->ownership_uncertain = true; + return ret; + } + /* Publish the opaque ownership key only after the table is attached. */ + op = (struct aurora_apif_op){ + .selector = AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, + AURORA_APIF_TAG_FRAME), + .arg = { pa, owner, base }, + }; + ret = apif_one(dart, &op); + if (ret) + dart->ownership_uncertain = true; + return ret; +} + +static int apif_boot_query(struct apple_dart_apif *dart, + struct apif_op *request) +{ + struct aurora_apif_boot_iommu *info = dart->boot_info; + struct aurora_apif_op op = { + .selector = AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, + AURORA_APIF_BOOT_IOMMU), + .arg = { AURORA_APIF_BOOT_DART, dart->dart_id, request->arg[0], + request->op == APIF_DART_RETAINED_INFO ? + U64_MAX : + request->arg[1], + virt_to_phys(info), sizeof(*info) }, + }; + int ret = apif_one(dart, &op); + + if (ret) + return ret; + if (request->op == APIF_DART_RETAINED_INFO) { + request->arg[1] = info->level; + request->arg[2] = info->count; + request->arg[3] = info->base; + request->arg[4] = info->size; + request->arg[5] = 1; + } else { + request->arg[2] = info->base; + request->arg[3] = info->ipa; + request->arg[4] = info->size; + request->arg[5] = info->attrs; + } + return 0; +} + +/* Build native calls in Linux, including address lists and permission flags. + * Address translations form a separate batch, so failure cannot send a MAP + * with an untranslated pointer. Mapping batches keep the original capacity + * and prefix-completion accounting used by rollback. Caller holds batch_lock. + */ +static int apif_ring(struct apple_dart_apif *dart) +{ + struct aurora_apif_op *native = dart->native_ops; + u32 count = dart->batch->count, n = 0, translated = 0, done = 0; + u32 ends[APIF_BATCH_MAX_OPS]; + int ret = 0; + + dart->batch->done = 0; + if (dart->ownership_uncertain) { + dart->batch->status = -EIO; + return -EIO; + } + if (count > APIF_BATCH_MAX_OPS) + return -EINVAL; + apif_force_iovas(dart, false); + for (u32 i = 0; i < count; i++) { + struct apif_op *op = &dart->batch->op[i]; + + if (op->flags) { + ret = -EINVAL; + goto out; + } + if (op->op == APIF_DART_MAP) { + if (op->arg[4] & + ~(IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE | + IOMMU_NOEXEC | IOMMU_MMIO)) { + ret = -EINVAL; + goto out; + } + if (!dart->page_lists[i]) + dart->page_lists[i] = + kmalloc(SZ_16K, GFP_ATOMIC); + if (!dart->page_lists[i]) { + ret = -ENOMEM; + goto out; + } + native[translated++] = (struct aurora_apif_op){ + .selector = AURORA_APIF_SELECTOR( + AURORA_APIF_SERVICE, + AURORA_APIF_TRANSLATE), + .arg = { op->arg[2], op->arg[3] }, + }; + native[translated++] = (struct aurora_apif_op){ + .selector = AURORA_APIF_SELECTOR( + AURORA_APIF_SERVICE, + AURORA_APIF_TRANSLATE), + .arg = { virt_to_phys(dart->page_lists[i]), + SZ_16K }, + }; + } + } + if (translated) { + ret = aurora_apif_submit(dart->apif, native, translated, NULL); + if (ret) + goto out; + translated = 0; + for (u32 i = 0; i < count; i++) + if (dart->batch->op[i].op == APIF_DART_MAP) { + dart->batch->op[i].arg[5] = + native[translated++].ret; + dart->page_list_pa[i] = + native[translated++].ret; + } + } + for (u32 i = 0; i < count; i++) { + struct apif_op *op = &dart->batch->op[i]; + u64 *a = op->arg; + struct aurora_apif_op call = {}; + + call.arg[0] = dart->dart_id; + call.arg[1] = a[0]; + op->ret = 0; + switch (op->op) { + case APIF_DART_INIT: { + struct aurora_apif_op query = { + .selector = AURORA_APIF_SELECTOR( + AURORA_APIF_SERVICE, + AURORA_APIF_BOOT_TABLE), + .arg = { AURORA_APIF_BOOT_DART, dart->dart_id, + 0, 0, U64_MAX }, + }; + + ret = apif_one(dart, &query); + if (ret) + goto out; + if (!query.ret) { + call.selector = + AURORA_APIF_SELECTOR(3, 6); /* INIT */ + native[n++] = call; + call.selector = AURORA_APIF_SELECTOR( + 3, 5); /* POWERUP */ + native[n++] = call; + } + break; + } + case APIF_DART_ATTACH: + ret = apif_prepare_table(dart, a[0], a[1], a[2]); + if (ret) + goto out; + break; + case APIF_DART_RETAINED_INFO: + case APIF_DART_RETAINED_RUN: + ret = apif_boot_query(dart, op); + if (ret) + goto out; + break; + case APIF_DART_MAP: { + u64 pa = a[5], size = a[3]; + + if (!size || size > DART_APIF_BLOCK_SIZE || + ((a[1] | pa | size) & (DART_APIF_PAGE_SIZE - 1)) || + (a[1] >> 25) != ((a[1] + size - 1) >> 25)) { + ret = -EINVAL; + goto out; + } + for (u64 p = 0; p < size / DART_APIF_PAGE_SIZE; p++) + dart->page_lists[i][p] = + pa + p * DART_APIF_PAGE_SIZE; + call.selector = AURORA_APIF_SELECTOR(3, 2); /* MAP */ + call.arg[2] = a[1]; + call.arg[3] = dart->page_list_pa[i]; + call.arg[4] = size; + call.arg[5] = (~a[4]) & 7; + native[n++] = call; + break; + } + case APIF_DART_UNMAP: + call.selector = AURORA_APIF_SELECTOR(3, 3); + call.arg[2] = a[1]; + call.arg[3] = a[2]; + call.arg[4] = 2; + native[n++] = call; + break; + case APIF_DART_ENABLE: + case APIF_DART_DISABLE: + call.selector = AURORA_APIF_SELECTOR( + 3, op->op == APIF_DART_ENABLE ? 8 : 7); + native[n++] = call; + break; + case 7: /* POWERUP */ + case 8: /* POWERDOWN */ + for (u32 sid = 0; sid < dart->num_streams; sid++) + if (dart->retained[sid]) { + if (op->op == 8) { + ret = -EBUSY; + goto out; + } + goto prepared; + } + call.selector = + AURORA_APIF_SELECTOR(3, op->op == 7 ? 5 : 4); + native[n++] = call; + break; + default: + ret = -EINVAL; + goto out; + } +prepared: + ends[i] = n; + } + ret = aurora_apif_submit(dart->apif, native, n, &done); + for (u32 i = 0; i < count && ends[i] <= done; i++) + dart->batch->done = i + 1; +out: + apif_force_iovas(dart, true); + dart->batch->status = ret; + return ret; +} + +/* Setup-only host queries. Firmware-owned mappings are reserved, never replayed + * through MAP, and never inserted in the Linux-owned mappings xarray. + */ +static int apif_load_retained(struct apple_dart_apif *dart) +{ + struct device_node *np = dart->dev->of_node; + int count = of_property_count_u32_elems(np, "apple,apif-retained-sids"); + int ret; + + if (!of_find_property(np, "apple,apif-retained-sids", NULL)) + return 0; + if (count < 1 || count > dart->num_streams || dart->iova_force) + return -EINVAL; + for (int i = 0; i < count; i++) { + struct apif_retained_sid *r; + u32 sid; + u64 previous = 0; + struct apif_op *op = &dart->batch->op[0]; + + if (of_property_read_u32_index(np, "apple,apif-retained-sids", + i, &sid) || + sid >= dart->num_streams || dart->retained[sid]) + return -EINVAL; + dart->batch->count = 1; + *op = (struct apif_op){ .op = APIF_DART_RETAINED_INFO, + .arg = { sid } }; + ret = apif_ring(dart); + if (ret) + return ret; + if (op->arg[1] > 1 || op->arg[2] > 131072 || op->arg[5] != 1 || + op->arg[3] != dart->dma_min || + op->arg[4] != dart->dma_max - dart->dma_min + 1) + return -EINVAL; + r = devm_kzalloc(dart->dev, sizeof(*r), GFP_KERNEL); + if (!r) + return -ENOMEM; + r->root_level = op->arg[1]; + r->count = op->arg[2]; + r->runs = devm_kcalloc(dart->dev, r->count, sizeof(*r->runs), + GFP_KERNEL); + if (r->count && !r->runs) + return -ENOMEM; + for (u64 j = 0; j < r->count; j++) { + struct apif_retained_run *run = &r->runs[j]; + + *op = (struct apif_op){ .op = APIF_DART_RETAINED_RUN, + .arg = { sid, j } }; + ret = apif_ring(dart); + if (ret) + return ret; + run->dva = op->arg[2]; + run->ipa = op->arg[3]; + run->size = op->arg[4]; + if (!run->size || + (run->dva | run->size) & + (DART_APIF_PAGE_SIZE - 1) || + run->dva < dart->dma_min || + run->dva > dart->dma_max || + run->size - 1 > dart->dma_max - run->dva || + (j && run->dva < previous) || + (run->ipa != U64_MAX && + ((run->ipa & (DART_APIF_PAGE_SIZE - 1)) || + run->size > U64_MAX - run->ipa))) + return -EINVAL; + previous = run->dva + run->size; + } + dart->retained[sid] = r; + } + return 0; +} + +static const struct apif_retained_run * +apif_retained_lookup(const struct apif_retained_sid *r, u64 dva, u64 size) +{ + unsigned int low = 0, high = r->count; + + while (low < high) { + unsigned int mid = low + (high - low) / 2; + const struct apif_retained_run *run = &r->runs[mid]; + + if (run->dva + run->size <= dva) + low = mid + 1; + else + high = mid; + } + return low < r->count && r->runs[low].dva < dva + size ? &r->runs[low] : + NULL; +} + +static bool apif_retained_overlap(struct apple_dart_apif_domain *dom, u64 dva, + u64 size) +{ + unsigned int sid; + + if (!dom->has_retained) + return false; + for (unsigned int i = 0; i < dom->num_darts; i++) { + struct apif_stream_map *stream = &dom->streams[i]; + + for_each_set_bit(sid, stream->sids, stream->dart->num_streams) { + struct apif_retained_sid *r = + stream->dart->retained[sid]; + + if (r && apif_retained_lookup(r, dva, size)) + return true; + } + } + return false; +} + +static phys_addr_t apif_retained_translate(struct apple_dart_apif_domain *dom, + u64 dva) +{ + unsigned int sid; + u64 result = 0; + + if (!dom->has_retained) + return 0; + /* Firmware may retain different mappings on different display engines. + * Only a unanimous translation is meaningful for their shared domain. + */ + for (unsigned int i = 0; i < dom->num_darts; i++) { + struct apif_stream_map *stream = &dom->streams[i]; + + for_each_set_bit(sid, stream->sids, stream->dart->num_streams) { + struct apif_retained_sid *r = + stream->dart->retained[sid]; + const struct apif_retained_run *run = + r ? apif_retained_lookup(r, dva, 1) : NULL; + u64 pa; + + if (!run || dva < run->dva || run->ipa == U64_MAX) + return 0; + pa = run->ipa + dva - run->dva; + if (result && result != pa) + return 0; + result = pa; + } + } + return result; +} + +/* Bring the DART up in SPTM exactly once, before any stream is enabled. */ +static int apple_dart_apif_ensure_init(struct apple_dart_apif *dart) +{ + int ret = 0; + + mutex_lock(&dart->init_lock); + if (!dart->inited) { + unsigned long flags; + + spin_lock_irqsave(&dart->batch_lock, flags); + dart->batch->count = 1; + dart->batch->op[0] = (struct apif_op){ .op = APIF_DART_INIT }; + ret = apif_ring(dart); + spin_unlock_irqrestore(&dart->batch_lock, flags); + if (ret) + ret = -EIO; + else + dart->inited = true; + } + mutex_unlock(&dart->init_lock); + return ret; +} + +/* Live mappings are indexed by IOVA page. Normal map/unmap only touches the + * requested range; a complete walk is needed solely when adding a new SID. + * Zero tagged values reserve XArray storage before any hardware mapping. + */ +#define APIF_MAP_PROT_BITS 8 + +static unsigned long apif_mapping_value(phys_addr_t pa, int prot) +{ + return ((pa >> DART_APIF_PAGE_SHIFT) << APIF_MAP_PROT_BITS) | prot; +} + +static int apif_reserve_mappings(struct apple_dart_apif_domain *dom, + unsigned long iova, size_t size, gfp_t gfp) +{ + unsigned long first = iova >> DART_APIF_PAGE_SHIFT; + unsigned long pages = size >> DART_APIF_PAGE_SHIFT; + unsigned long page; + int ret; + + for (page = 0; page < pages; page++) { + ret = xa_insert(&dom->mappings, first + page, xa_mk_value(0), + gfp); + if (ret) + goto undo; + } + return 0; +undo: + while (page) + xa_erase(&dom->mappings, first + --page); + return ret; +} + +static void apif_release_reservations(struct apple_dart_apif_domain *dom, + unsigned long iova, size_t size) +{ + unsigned long first = iova >> DART_APIF_PAGE_SHIFT; + unsigned long pages = size >> DART_APIF_PAGE_SHIFT; + + for (unsigned long page = 0; page < pages; page++) + if (xa_load(&dom->mappings, first + page) == xa_mk_value(0)) + xa_erase(&dom->mappings, first + page); +} + +static void apif_record_mapping(struct apple_dart_apif_domain *dom, u64 iova, + u64 pa, size_t size, int prot) +{ + for (size_t off = 0; off < size; off += DART_APIF_PAGE_SIZE) { + unsigned long index = (iova + off) >> DART_APIF_PAGE_SHIFT; + void *entry = xa_mk_value(apif_mapping_value(pa + off, prot)); + + /* Reserved entries retain their nodes, so replacing a reservation + * requires no allocation, even in this IRQ-disabled section. + */ + WARN_ON(xa_err( + xa_store(&dom->mappings, index, entry, GFP_NOWAIT))); + } +} + +static void apif_forget_mapping(struct apple_dart_apif_domain *dom, u64 iova, + size_t size) +{ + for (size_t off = 0; off < size; off += DART_APIF_PAGE_SIZE) + xa_erase(&dom->mappings, (iova + off) >> DART_APIF_PAGE_SHIFT); +} + +static int apif_prepare_leaf(struct apple_dart_apif_domain *dom, + struct apif_stream_map *stream, unsigned int sid, + u64 iova) +{ + struct apple_dart_apif *dart = stream->dart; + unsigned long key = ((unsigned long)dart->dart_id << 48) | + ((unsigned long)sid << 40) | + (iova >> DART_APIF_BLOCK_SHIFT); + int ret; + + if (xa_load(&dom->l2_attached, key)) + return 0; + ret = xa_insert(&dom->l2_attached, key, xa_mk_value(0), GFP_ATOMIC); + if (ret) + return ret; + dart->batch->count = 1; + dart->batch->op[0] = (struct apif_op){ + .op = APIF_DART_ATTACH, + .arg = { sid, iova, 2 }, + }; + ret = apif_ring(dart); + if (ret) { + xa_erase(&dom->l2_attached, key); + return ret; + } + WARN_ON(xa_err( + xa_store(&dom->l2_attached, key, xa_mk_value(1), GFP_NOWAIT))); + return 0; +} + +/* Return the next physically contiguous, same-protection run, stopping at a + * leaf-table boundary. Reservations are not yet live and are skipped. + */ +static bool apif_next_mapping(struct apple_dart_apif_domain *dom, + unsigned long *cursor, struct apif_op *op, + unsigned int sid) +{ + unsigned long index = *cursor, value, first; + void *entry; + u64 iova, pa; + size_t size; + int prot; + + for (;;) { + entry = xa_find(&dom->mappings, &index, ULONG_MAX, XA_PRESENT); + if (!entry) + return false; + value = xa_to_value(entry); + if (value) + break; + index++; + } + first = index; + prot = value & ((1UL << APIF_MAP_PROT_BITS) - 1); + pa = (value >> APIF_MAP_PROT_BITS) << DART_APIF_PAGE_SHIFT; + iova = (u64)first << DART_APIF_PAGE_SHIFT; + size = DART_APIF_PAGE_SIZE; + while ((iova + size) & (DART_APIF_BLOCK_SIZE - 1)) { + entry = xa_load(&dom->mappings, index + 1); + if (entry != xa_mk_value(apif_mapping_value(pa + size, prot))) + break; + index++; + size += DART_APIF_PAGE_SIZE; + } + *cursor = index + 1; + *op = (struct apif_op){ + .op = APIF_DART_MAP, + .arg = { sid, iova, pa, size, prot }, + }; + return true; +} + +static int apif_undo_replay(struct apple_dart_apif_domain *dom, + struct apif_stream_map *stream, unsigned int sid, + u64 mapped_end) +{ + struct apple_dart_apif *dart = stream->dart; + unsigned long cursor = 0; + struct apif_op op; + + while (apif_next_mapping(dom, &cursor, &op, sid) && + op.arg[1] < mapped_end) { + dart->batch->count = 1; + dart->batch->op[0] = (struct apif_op){ + .op = APIF_DART_UNMAP, + .arg = { sid, op.arg[1], + min_t(u64, op.arg[3], + mapped_end - op.arg[1]) }, + }; + if (apif_ring(dart)) { + dev_err(dart->dev, "failed to undo SID %u replay\n", + sid); + return -EIO; + } + } + return 0; +} + +/* New SID is disabled throughout replay. On failure, keep it disabled and + * remove the successful replay prefix. Existing SIDs/maps stay untouched. + */ +static int apif_replay_sid(struct apple_dart_apif_domain *dom, + struct apif_stream_map *stream, unsigned int sid) +{ + struct apple_dart_apif *dart = stream->dart; + unsigned long cursor = 0; + struct apif_op op; + u64 mapped_end = 0; + u32 n = 0; + int ret; + + while (apif_next_mapping(dom, &cursor, &op, sid)) { + if (dart->retained[sid] && + apif_retained_lookup(dart->retained[sid], op.arg[1], + op.arg[3])) + return -EBUSY; + ret = apif_prepare_leaf(dom, stream, sid, op.arg[1]); + if (ret) + return ret; + } + cursor = 0; + for (;;) { + bool have = apif_next_mapping(dom, &cursor, &op, sid); + + if (have) + dart->batch->op[n++] = op; + if (n == APIF_BATCH_MAX_OPS || (!have && n)) { + dart->batch->count = n; + ret = apif_ring(dart); + for (u32 i = 0; i < min(dart->batch->done, n); i++) + mapped_end = dart->batch->op[i].arg[1] + + dart->batch->op[i].arg[3]; + if (ret) + goto undo; + n = 0; + } + if (!have) + return 0; + } +undo: + apif_undo_replay(dom, stream, sid, mapped_end); + return ret; +} + +static int apple_dart_apif_attach_dev(struct iommu_domain *domain, + struct device *dev, + struct iommu_domain *old) +{ + struct apple_dart_apif_master *master = dev_iommu_priv_get(dev); + struct apple_dart_apif_domain *dom = to_apif_domain(domain); + struct apple_dart_apif_domain *prior = NULL; + unsigned long added[APIF_MAX_DARTS] + [BITS_TO_LONGS(DART_APIF_MAX_STREAMS)] = {}; + unsigned long moved[APIF_MAX_DARTS] + [BITS_TO_LONGS(DART_APIF_MAX_STREAMS)] = {}; + unsigned long flags; + unsigned int sid; + int ret; + + if (!master || !master->num_darts) + return -ENODEV; + for (unsigned int i = 0; i < master->num_darts; i++) { + ret = apple_dart_apif_ensure_init(master->streams[i].dart); + if (ret) + return ret; + } + if (old && old != domain && old->owner == &apple_dart_apif_ops && + old->ops == apple_dart_apif_ops.default_domain_ops) + prior = to_apif_domain(old); + if (prior && (unsigned long)prior < (unsigned long)dom) { + mutex_lock(&prior->lock); + mutex_lock_nested(&dom->lock, SINGLE_DEPTH_NESTING); + } else { + mutex_lock(&dom->lock); + if (prior) + mutex_lock_nested(&prior->lock, SINGLE_DEPTH_NESTING); + } + ret = apif_domain_configure(dom, master); + if (ret) + goto unlock_mutex; + + local_irq_save(flags); + if (prior && (unsigned long)prior < (unsigned long)dom) { + write_lock(&prior->streams_lock); + write_lock_nested(&dom->streams_lock, SINGLE_DEPTH_NESTING); + } else { + write_lock(&dom->streams_lock); + if (prior) + write_lock_nested(&prior->streams_lock, + SINGLE_DEPTH_NESTING); + } + for (unsigned int i = 0; i < master->num_darts && !ret; i++) { + struct apple_dart_apif *dart = master->streams[i].dart; + struct apif_stream_map *stream = apif_find_stream(dom, dart); + struct apif_stream_map *before = apif_find_stream(prior, dart); + + spin_lock(&dart->batch_lock); + for_each_set_bit(sid, master->streams[i].sids, + dart->num_streams) { + if (test_bit(sid, stream->sids)) + continue; + if (dart->retained[sid] && before && + test_bit(sid, before->sids)) { + set_bit(sid, moved[i]); + ret = apif_undo_replay(prior, before, sid, + U64_MAX); + if (ret) + break; + } + dart->batch->count = dart->retained[sid] ? 1 : 2; + dart->batch->op[0] = (struct apif_op){ + .op = dart->retained[sid] ? + APIF_DART_RETAINED_INFO : + APIF_DART_DISABLE, + .arg = { sid }, + }; + if (!dart->retained[sid]) + dart->batch->op[1] = (struct apif_op){ + .op = APIF_DART_ATTACH, + .arg = { sid, dart->dma_min, 1 }, + }; + ret = apif_ring(dart); + if (ret) + break; + ret = apif_replay_sid(dom, stream, sid); + if (ret) + break; + /* Include this SID in unwind even when ENABLE's completion fails. */ + set_bit(sid, added[i]); + dart->batch->count = 1; + dart->batch->op[0] = (struct apif_op){ + .op = dart->retained[sid] ? + APIF_DART_RETAINED_INFO : + APIF_DART_ENABLE, + .arg = { sid }, + }; + ret = apif_ring(dart); + if (ret) + break; + dom->has_retained |= !!dart->retained[sid]; + set_bit(sid, stream->sids); + } + spin_unlock(&dart->batch_lock); + } + + /* Attach is one transaction across all of the master's DARTs. */ + for (unsigned int i = 0; i < master->num_darts; i++) { + struct apple_dart_apif *dart = master->streams[i].dart; + struct apif_stream_map *stream = apif_find_stream(dom, dart); + struct apif_stream_map *before = apif_find_stream(prior, dart); + + spin_lock(&dart->batch_lock); + if (ret) { + for_each_set_bit(sid, added[i], dart->num_streams) { + dart->batch->count = 1; + dart->batch->op[0] = (struct apif_op){ + .op = dart->retained[sid] ? + APIF_DART_RETAINED_INFO : + APIF_DART_DISABLE, + .arg = { sid }, + }; + if (apif_ring(dart)) + dev_err(dart->dev, + "failed to disable SID %u during attach unwind\n", + sid); + apif_undo_replay(dom, stream, sid, U64_MAX); + clear_bit(sid, stream->sids); + } + } + for_each_set_bit(sid, moved[i], dart->num_streams) { + if (ret) { + if (apif_replay_sid(prior, before, sid)) + dev_err(dart->dev, + "failed to restore previous domain SID %u\n", + sid); + } else { + clear_bit(sid, before->sids); + } + } + spin_unlock(&dart->batch_lock); + } + if (prior) + write_unlock(&prior->streams_lock); + write_unlock(&dom->streams_lock); + local_irq_restore(flags); +unlock_mutex: + if (prior) + mutex_unlock(&prior->lock); + mutex_unlock(&dom->lock); + return ret; +} + +struct apif_map_progress { + size_t mapped; + unsigned int streams; + struct apif_op partial; +}; + +/* Track only fully installed stream cohorts. The batch may split one cohort + * without reducing batching; keep its progress across doorbells. + */ +static int apif_flush_maps(struct apple_dart_apif_domain *dom, + struct apif_stream_map *stream, + struct apif_map_progress *progress) +{ + struct apple_dart_apif *dart = stream->dart; + unsigned int total_streams = + bitmap_weight(stream->sids, dart->num_streams); + int ret = apif_ring(dart); + + for (u32 i = 0; i < min(dart->batch->done, dart->batch->count); i++) { + struct apif_op *op = &dart->batch->op[i]; + + progress->partial = *op; + if (++progress->streams == total_streams) { + progress->mapped += op->arg[3]; + progress->streams = 0; + } + } + if (ret && progress->streams) { + unsigned int sid; + u32 n = 0; + + for_each_set_bit(sid, stream->sids, dart->num_streams) { + if (n == progress->streams) + break; + dart->batch->op[n++] = (struct apif_op){ + .op = APIF_DART_UNMAP, + .arg = { sid, progress->partial.arg[1], + progress->partial.arg[3] }, + }; + } + dart->batch->count = n; + if (apif_ring(dart)) + dev_err(dart->dev, "failed to undo partial DART map\n"); + } + return ret; +} + +static int apif_map_streams(struct apple_dart_apif_domain *dom, + struct apif_stream_map *stream, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int prot, gfp_t gfp, size_t *mapped) +{ + struct apple_dart_apif *dart = stream->dart; + struct apif_map_progress progress = {}; + size_t total; + unsigned long flags; + u32 n = 0; + int ret; + + *mapped = 0; + if (!dart || bitmap_empty(stream->sids, dart->num_streams)) + return -ENODEV; + if (pgsize != DART_APIF_PAGE_SIZE || !pgcount || + check_mul_overflow(pgsize, pgcount, &total) || + (iova | paddr) & (DART_APIF_PAGE_SIZE - 1) || + prot & ~(IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE | IOMMU_NOEXEC | + IOMMU_MMIO) || + !paddr || iova < dart->dma_min || iova > dart->dma_max || + total - 1 > dart->dma_max - iova || + iova + total > DART_APIF_NATIVE_APERTURE_END) + return -EINVAL; + spin_lock_irqsave(&dart->batch_lock, flags); + /* Install all leaf tables before accumulating the mapping batch. */ + for (size_t pos = 0; pos < total;) { + u64 cur = iova + pos; + size_t chunk = + min_t(size_t, total - pos, + DART_APIF_BLOCK_SIZE - + (cur & (DART_APIF_BLOCK_SIZE - 1))); + unsigned int sid; + + for_each_set_bit(sid, stream->sids, dart->num_streams) { + ret = apif_prepare_leaf(dom, stream, sid, cur); + if (ret) + goto out; + } + pos += chunk; + } + for (size_t pos = 0; pos < total;) { + u64 cur = iova + pos; + size_t chunk = + min_t(size_t, total - pos, + DART_APIF_BLOCK_SIZE - + (cur & (DART_APIF_BLOCK_SIZE - 1))); + unsigned int sid; + + for_each_set_bit(sid, stream->sids, dart->num_streams) { + dart->batch->op[n++] = (struct apif_op){ + .op = APIF_DART_MAP, + .arg = { sid, cur, paddr + pos, chunk, prot }, + }; + if (n == APIF_BATCH_MAX_OPS) { + dart->batch->count = n; + ret = apif_flush_maps(dom, stream, &progress); + if (ret) + goto out; + n = 0; + } + } + pos += chunk; + } + if (n) { + dart->batch->count = n; + ret = apif_flush_maps(dom, stream, &progress); + } +out: + spin_unlock_irqrestore(&dart->batch_lock, flags); + *mapped = progress.mapped; + return ret ? -EIO : 0; +} + +static int apif_restore_partial_unmap(struct apple_dart_apif_domain *dom, + struct apif_stream_map *stream, + const struct apif_op *partial, + unsigned int streams) +{ + struct apple_dart_apif *dart = stream->dart; + unsigned long cursor = partial->arg[1] >> DART_APIF_PAGE_SHIFT; + u64 end = partial->arg[1] + partial->arg[2]; + struct apif_op op; + u32 n = 0; + + while (apif_next_mapping(dom, &cursor, &op, 0) && op.arg[1] < end) { + unsigned int sid, count = 0; + + op.arg[3] = min_t(u64, op.arg[3], end - op.arg[1]); + for_each_set_bit(sid, stream->sids, dart->num_streams) { + if (count++ == streams) + break; + op.arg[0] = sid; + dart->batch->op[n++] = op; + if (n == APIF_BATCH_MAX_OPS) { + dart->batch->count = n; + if (apif_ring(dart)) + return -EIO; + n = 0; + } + } + } + if (!n) + return 0; + dart->batch->count = n; + return apif_ring(dart); +} + +static int apif_flush_unmaps(struct apple_dart_apif_domain *dom, + struct apif_stream_map *stream, + struct apif_map_progress *progress) +{ + struct apple_dart_apif *dart = stream->dart; + unsigned int total_streams = + bitmap_weight(stream->sids, dart->num_streams); + int ret = apif_ring(dart); + + for (u32 i = 0; i < min(dart->batch->done, dart->batch->count); i++) { + struct apif_op *op = &dart->batch->op[i]; + + progress->partial = *op; + if (++progress->streams == total_streams) { + progress->mapped += op->arg[2]; + progress->streams = 0; + } + } + if (ret && progress->streams && + apif_restore_partial_unmap(dom, stream, &progress->partial, + progress->streams)) + dev_err(dart->dev, "failed to restore partial DART unmap\n"); + return ret; +} + +static size_t apif_unmap_streams(struct apple_dart_apif_domain *dom, + struct apif_stream_map *stream, + unsigned long iova, size_t pgsize, + size_t pgcount, + struct iommu_iotlb_gather *gather) +{ + struct apple_dart_apif *dart = stream->dart; + struct apif_map_progress progress = {}; + size_t total; + unsigned long flags; + u32 n = 0; + + if (!dart || bitmap_empty(stream->sids, dart->num_streams) || + pgsize != DART_APIF_PAGE_SIZE || !pgcount || + check_mul_overflow(pgsize, pgcount, &total) || + (iova & (DART_APIF_PAGE_SIZE - 1)) || iova < dart->dma_min || + iova > dart->dma_max || total - 1 > dart->dma_max - iova) + return 0; + spin_lock_irqsave(&dart->batch_lock, flags); + for (size_t pos = 0; pos < total;) { + u64 cur = iova + pos; + size_t chunk = + min_t(size_t, total - pos, + DART_APIF_BLOCK_SIZE - + (cur & (DART_APIF_BLOCK_SIZE - 1))); + unsigned int sid; + + for_each_set_bit(sid, stream->sids, dart->num_streams) { + dart->batch->op[n++] = (struct apif_op){ + .op = APIF_DART_UNMAP, + .arg = { sid, cur, chunk }, + }; + if (n == APIF_BATCH_MAX_OPS) { + dart->batch->count = n; + if (apif_flush_unmaps(dom, stream, &progress)) + goto out; + n = 0; + } + } + pos += chunk; + } + if (n) { + dart->batch->count = n; + apif_flush_unmaps(dom, stream, &progress); + } +out: + spin_unlock_irqrestore(&dart->batch_lock, flags); + return progress.mapped; +} + +/* A buffer has one IOVA and one software mapping, regardless of how many + * display engines can scan it out. Forward whole batches to each DART; do + * not split them into per-page or per-stream transactions. + */ +static int apple_dart_apif_map_pages(struct iommu_domain *domain, + unsigned long iova, phys_addr_t paddr, + size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct apple_dart_apif_domain *dom = to_apif_domain(domain); + size_t done[APIF_MAX_DARTS] = {}, total; + unsigned long flags; + unsigned int active = 0; + int ret; + + *mapped = 0; + if (!dom->num_darts) + return -ENODEV; + if (pgsize != DART_APIF_PAGE_SIZE || !pgcount || + check_mul_overflow(pgsize, pgcount, &total) || + (iova | paddr) & (DART_APIF_PAGE_SIZE - 1) || !paddr || + total - 1 > U64_MAX - paddr || + prot & ~(IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE | IOMMU_NOEXEC | + IOMMU_MMIO) || + iova < domain->geometry.aperture_start || + iova > domain->geometry.aperture_end || + total - 1 > domain->geometry.aperture_end - iova || + iova + total > DART_APIF_NATIVE_APERTURE_END) + return -EINVAL; + ret = apif_reserve_mappings(dom, iova, total, gfp); + if (ret) + return ret; + read_lock_irqsave(&dom->streams_lock, flags); + if (apif_retained_overlap(dom, iova, total)) { + ret = -EBUSY; + goto out; + } + for (unsigned int i = 0; i < dom->num_darts; i++) { + struct apif_stream_map *stream = &dom->streams[i]; + + if (bitmap_empty(stream->sids, stream->dart->num_streams)) + continue; + active++; + ret = apif_map_streams(dom, stream, iova, paddr, pgsize, + pgcount, prot, gfp, &done[i]); + if (ret) + break; + } + if (!active) + ret = -ENODEV; + if (!ret) { + apif_record_mapping(dom, iova, paddr, total, prot); + *mapped = total; + } else { + /* A later DART failed. Undo the successful prefixes on every + * engine before letting the DMA layer reuse this IOVA. + */ + for (unsigned int i = 0; i < dom->num_darts; i++) + if (done[i] && + apif_unmap_streams(dom, &dom->streams[i], iova, + pgsize, done[i] / pgsize, + NULL) != done[i]) + dev_err(dom->streams[i].dart->dev, + "failed to unwind multi-DART mapping\n"); + } +out: + read_unlock_irqrestore(&dom->streams_lock, flags); + if (ret) + apif_release_reservations(dom, iova, total); + return ret; +} + +static size_t apple_dart_apif_unmap_pages(struct iommu_domain *domain, + unsigned long iova, size_t pgsize, + size_t pgcount, + struct iommu_iotlb_gather *gather) +{ + struct apple_dart_apif_domain *dom = to_apif_domain(domain); + size_t done[APIF_MAX_DARTS] = {}, total, common; + unsigned long flags; + unsigned int active = 0; + + if (!dom->num_darts || pgsize != DART_APIF_PAGE_SIZE || !pgcount || + check_mul_overflow(pgsize, pgcount, &total) || + (iova & (DART_APIF_PAGE_SIZE - 1)) || + iova < domain->geometry.aperture_start || + iova > domain->geometry.aperture_end || + total - 1 > domain->geometry.aperture_end - iova) + return 0; + read_lock_irqsave(&dom->streams_lock, flags); + if (apif_retained_overlap(dom, iova, total)) { + common = 0; + goto out; + } + for (unsigned int i = 0; i < dom->num_darts; i++) { + struct apif_stream_map *stream = &dom->streams[i]; + + if (bitmap_empty(stream->sids, stream->dart->num_streams)) + continue; + active++; + done[i] = apif_unmap_streams(dom, stream, iova, pgsize, pgcount, + gather); + if (done[i] != total) + break; + } + common = active ? total : 0; + for (unsigned int i = 0; i < dom->num_darts; i++) + if (!bitmap_empty(dom->streams[i].sids, + dom->streams[i].dart->num_streams)) + common = min(common, done[i]); + /* Preserve the canonical mapping until all engines have agreed. On + * failure restore the suffix removed from earlier engines, retaining + * their existing leaf tables and full-sized transport batches. + */ + for (unsigned int i = 0; i < dom->num_darts; i++) { + struct apif_stream_map *stream = &dom->streams[i]; + struct apple_dart_apif *dart = stream->dart; + struct apif_op suffix = { + .arg = { 0, iova + common, done[i] - common }, + }; + + if (done[i] <= common) + continue; + spin_lock(&dart->batch_lock); + if (apif_restore_partial_unmap( + dom, stream, &suffix, + bitmap_weight(stream->sids, dart->num_streams))) + dev_err(dart->dev, + "failed to restore multi-DART unmap suffix\n"); + spin_unlock(&dart->batch_lock); + } + apif_forget_mapping(dom, iova, common); +out: + read_unlock_irqrestore(&dom->streams_lock, flags); + return common; +} + +static void apple_dart_apif_iotlb_sync(struct iommu_domain *domain, + struct iommu_iotlb_gather *gather) +{ + /* apif invalidates inside every MAP/UNMAP it forwards to SPTM, so a + * separate flush is a no-op. + */ +} + +static phys_addr_t apple_dart_apif_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova) +{ + struct apple_dart_apif_domain *dom = to_apif_domain(domain); + void *entry = xa_load(&dom->mappings, iova >> DART_APIF_PAGE_SHIFT); + unsigned long value = entry ? xa_to_value(entry) : 0; + + if (!value) + return apif_retained_translate(dom, iova); + return ((value >> APIF_MAP_PROT_BITS) << DART_APIF_PAGE_SHIFT) | + (iova & (DART_APIF_PAGE_SIZE - 1)); +} + +static void apple_dart_apif_domain_free(struct iommu_domain *domain) +{ + struct apple_dart_apif_domain *dom = to_apif_domain(domain); + + if (dom->has_retained) { + for (unsigned int i = 0; i < dom->num_darts; i++) { + struct apif_stream_map *stream = &dom->streams[i]; + unsigned int sid; + unsigned long flags; + + spin_lock_irqsave(&stream->dart->batch_lock, flags); + for_each_set_bit(sid, stream->sids, + stream->dart->num_streams) + apif_undo_replay(dom, stream, sid, U64_MAX); + spin_unlock_irqrestore(&stream->dart->batch_lock, + flags); + } + } + xa_destroy(&dom->l2_attached); + xa_destroy(&dom->mappings); + kfree(dom); +} + +static struct iommu_domain * +apple_dart_apif_domain_alloc_paging(struct device *dev) +{ + struct apple_dart_apif_domain *dom; + int ret; + + dom = kzalloc_obj(*dom); + if (!dom) + return NULL; + mutex_init(&dom->lock); + rwlock_init(&dom->streams_lock); + xa_init(&dom->l2_attached); + xa_init(&dom->mappings); + dom->domain.pgsize_bitmap = DART_APIF_PAGE_SIZE; + dom->domain.geometry.aperture_end = DART_APIF_APERTURE_END; + dom->domain.geometry.force_aperture = true; + if (dev && dev_iommu_priv_get(dev)) { + ret = apif_domain_configure(dom, dev_iommu_priv_get(dev)); + if (ret) { + kfree(dom); + return ERR_PTR(ret); + } + } + return &dom->domain; +} + +static struct iommu_device *apple_dart_apif_probe_device(struct device *dev) +{ + struct apple_dart_apif_master *master = dev_iommu_priv_get(dev); + + if (!dev_iommu_fwspec_get(dev) || !master || !master->num_darts) + return ERR_PTR(-ENODEV); + for (unsigned int i = 0; i < master->num_darts; i++) + device_link_add(dev, master->streams[i].dart->dev, + DL_FLAG_PM_RUNTIME | + DL_FLAG_AUTOREMOVE_SUPPLIER); + return &master->streams[0].dart->iommu; +} + +static void apple_dart_apif_release_device(struct device *dev) +{ + kfree(dev_iommu_priv_get(dev)); +} + +static int apple_dart_apif_of_xlate(struct device *dev, + const struct of_phandle_args *args) +{ + struct apple_dart_apif_master *master = dev_iommu_priv_get(dev); + struct platform_device *iommu_pdev; + struct apple_dart_apif *dart; + u32 sid; + + if (args->args_count != 1) + return -EINVAL; + sid = args->args[0]; + if (sid >= DART_APIF_MAX_STREAMS) + return -EINVAL; + + iommu_pdev = of_find_device_by_node(args->np); + if (!iommu_pdev) + return -ENODEV; + dart = platform_get_drvdata(iommu_pdev); + put_device(&iommu_pdev->dev); + if (!dart) + return -ENODEV; + + if (sid >= dart->num_streams) + return -EINVAL; + if (!master) { + master = kzalloc_obj(*master); + if (!master) + return -ENOMEM; + dev_iommu_priv_set(dev, master); + } + for (unsigned int i = 0; i < master->num_darts; i++) { + if (master->streams[i].dart == dart) { + set_bit(sid, master->streams[i].sids); + return 0; + } + } + if (master->num_darts == APIF_MAX_DARTS) + return -EINVAL; + master->streams[master->num_darts].dart = dart; + set_bit(sid, master->streams[master->num_darts].sids); + master->num_darts++; + return 0; +} + +/* Devices sharing a stream cannot have independent IOVA allocators. PCI + * multifunction/topology grouping is also required even with distinct SIDs. + * Mirrors the native Apple DART lifetime model: the group owns its SID union, + * and its release callback clears the reverse lookup before freeing it. + */ +static DEFINE_MUTEX(apple_dart_apif_groups_lock); + +static void apple_dart_apif_release_group(void *data) +{ + struct apple_dart_apif_master *group_master = data; + unsigned int sid; + + mutex_lock(&apple_dart_apif_groups_lock); + for (unsigned int i = 0; i < group_master->num_darts; i++) { + struct apif_stream_map *stream = &group_master->streams[i]; + + for_each_set_bit(sid, stream->sids, stream->dart->num_streams) + stream->dart->sid2group[sid] = NULL; + } + mutex_unlock(&apple_dart_apif_groups_lock); + kfree(group_master); +} + +static int apif_merge_master(struct apple_dart_apif_master *dst, + const struct apple_dart_apif_master *src) +{ + struct apple_dart_apif_master merged = *dst; + + for (unsigned int i = 0; i < src->num_darts; i++) { + const struct apif_stream_map *from = &src->streams[i]; + unsigned int j; + + for (j = 0; j < merged.num_darts; j++) + if (merged.streams[j].dart == from->dart) + break; + if (j == merged.num_darts) { + if (j == APIF_MAX_DARTS) + return -EINVAL; + merged.streams[j] = *from; + merged.num_darts++; + } else { + bitmap_or(merged.streams[j].sids, + merged.streams[j].sids, from->sids, + from->dart->num_streams); + } + } + *dst = merged; + return 0; +} + +static struct iommu_group *apple_dart_apif_device_group(struct device *dev) +{ + struct apple_dart_apif_master *master = dev_iommu_priv_get(dev); + struct apple_dart_apif_master *group_master; + struct iommu_group *group = NULL, *put_group = NULL; + struct iommu_group *result = ERR_PTR(-EINVAL); + unsigned int sid; + + if (!master || !master->num_darts) + return result; + mutex_lock(&apple_dart_apif_groups_lock); + for (unsigned int i = 0; i < master->num_darts; i++) { + struct apif_stream_map *stream = &master->streams[i]; + + for_each_set_bit(sid, stream->sids, stream->dart->num_streams) { + struct iommu_group *existing = + stream->dart->sid2group[sid]; + + if (!existing) + continue; + if (group && existing != group) + goto out; + group = existing; + } + } + if (group) { + group = iommu_group_ref_get(group); + } else { +#ifdef CONFIG_PCI + if (dev_is_pci(dev)) + group = pci_device_group(dev); + else +#endif + group = generic_device_group(dev); + } + if (IS_ERR(group)) { + result = group; + goto out; + } + if (!group) { + result = ERR_PTR(-ENOMEM); + goto out; + } + group_master = iommu_group_get_iommudata(group); + if (!group_master) { + group_master = kmemdup(master, sizeof(*master), GFP_KERNEL); + if (!group_master) { + result = ERR_PTR(-ENOMEM); + put_group = group; + goto out; + } + iommu_group_set_iommudata(group, group_master, + apple_dart_apif_release_group); + } else if (apif_merge_master(group_master, master)) { + put_group = group; + goto out; + } + for (unsigned int i = 0; i < master->num_darts; i++) { + struct apif_stream_map *stream = &master->streams[i]; + + for_each_set_bit(sid, stream->sids, stream->dart->num_streams) + stream->dart->sid2group[sid] = group; + } + result = group; +out: + mutex_unlock(&apple_dart_apif_groups_lock); + /* Dropping the last reference can invoke release_group: do it unlocked. */ + if (put_group) + iommu_group_put(put_group); + return result; +} + +static bool apple_dart_apif_capable(struct device *dev, enum iommu_cap cap) +{ + return cap == IOMMU_CAP_CACHE_COHERENCY; +} + +/* PCIe captures the MSI doorbell before DART translation. Advertise the + * identity MSI page to the DMA layer so it never tries to map this non-RAM + * address through the SPTM transport. Other devices retain normal reservations. + * This follows apple-dart.c's hardware MSI reservation (Asahi contributors). + */ +static void apple_dart_apif_get_resv_regions(struct device *dev, + struct list_head *head) +{ + struct apple_dart_apif_master *master = dev_iommu_priv_get(dev); + unsigned int sid; + + for (unsigned int i = 0; master && i < master->num_darts; i++) { + struct apif_stream_map *stream = &master->streams[i]; + + /* Reserve the exclusive-end granule when required by the aperture. + * Narrower native apertures declare that restriction in DT. + */ + if (stream->dart->dma_max == DART_APIF_NATIVE_APERTURE_END || + stream->dart->reserve_last_page) { + struct iommu_resv_region *region = + iommu_alloc_resv_region( + stream->dart->dma_max + 1 - + DART_APIF_PAGE_SIZE, + DART_APIF_PAGE_SIZE, 0, + IOMMU_RESV_RESERVED, GFP_KERNEL); + + if (region) + list_add_tail(®ion->list, head); + } + for_each_set_bit(sid, stream->sids, stream->dart->num_streams) { + struct apif_retained_sid *r = + stream->dart->retained[sid]; + if (!r) + continue; + for (unsigned int i = 0; i < r->count; i++) { + struct iommu_resv_region *region = + iommu_alloc_resv_region( + r->runs[i].dva, r->runs[i].size, + 0, IOMMU_RESV_RESERVED, + GFP_KERNEL); + if (region) + list_add_tail(®ion->list, head); + } + } + } + if (dev_is_pci(dev)) { + struct pci_host_bridge *bridge = + pci_find_host_bridge(to_pci_dev(dev)->bus); + struct device_node *np = bridge->dev.parent->of_node; + struct iommu_resv_region *region; + u64 address; + + if (of_device_is_compatible(np, "apple,apif-pcie") && + !of_property_read_u64(np, "apple,msi-address", &address)) { + region = iommu_alloc_resv_region( + address & PAGE_MASK, PAGE_SIZE, + IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO, + IOMMU_RESV_MSI, GFP_KERNEL); + if (region) + list_add_tail(®ion->list, head); + } + } + iommu_dma_get_resv_regions(dev, head); +} + +static const struct iommu_ops apple_dart_apif_ops = { + .get_resv_regions = apple_dart_apif_get_resv_regions, + .capable = apple_dart_apif_capable, + .domain_alloc_paging = apple_dart_apif_domain_alloc_paging, + .probe_device = apple_dart_apif_probe_device, + .release_device = apple_dart_apif_release_device, + .device_group = apple_dart_apif_device_group, + .of_xlate = apple_dart_apif_of_xlate, + .owner = THIS_MODULE, + .default_domain_ops = + &(const struct iommu_domain_ops){ + .attach_dev = apple_dart_apif_attach_dev, + .map_pages = apple_dart_apif_map_pages, + .unmap_pages = apple_dart_apif_unmap_pages, + .iotlb_sync = apple_dart_apif_iotlb_sync, + .iova_to_phys = apple_dart_apif_iova_to_phys, + .free = apple_dart_apif_domain_free, + }, +}; + +/* Used only after the PCI host has stopped every endpoint on this DART. + * Native POWERDOWN/POWERUP retains the SPTM-owned tables and live mappings. + */ +int apple_dart_apif_set_power(struct device *dev, bool on) +{ + struct apple_dart_apif_master *master; + struct apple_dart_apif *dart; + unsigned long flags; + int ret; + + if (!dev->iommu || !dev->iommu->iommu_dev || + dev->iommu->iommu_dev->ops != &apple_dart_apif_ops) + return -ENODEV; + master = dev_iommu_priv_get(dev); + if (!master || master->num_darts != 1) + return -ENODEV; + dart = master->streams[0].dart; + mutex_lock(&dart->init_lock); + if (!dart->inited) { + ret = -EINVAL; + goto out; + } + spin_lock_irqsave(&dart->batch_lock, flags); + dart->batch->count = 1; + dart->batch->op[0] = (struct apif_op){ .op = on ? 7 : 8 }; + ret = apif_ring(dart); + spin_unlock_irqrestore(&dart->batch_lock, flags); +out: + mutex_unlock(&dart->init_lock); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_apif_set_power); + +static int apple_dart_apif_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_dart_apif *dart; + int ret; + + dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL); + if (!dart) + return -ENOMEM; + + dart->dev = dev; + mutex_init(&dart->init_lock); + spin_lock_init(&dart->batch_lock); + + dart->apif = aurora_apif_get(dev); + if (IS_ERR(dart->apif)) + return dev_err_probe(dev, PTR_ERR(dart->apif), + "APIF transport unavailable\n"); + ret = devm_add_action_or_reset(dev, apif_put_transport, dart->apif); + if (ret) + return ret; + dart->native_ops = devm_kcalloc(dev, 2 * APIF_BATCH_MAX_OPS, + sizeof(*dart->native_ops), GFP_KERNEL); + if (!dart->native_ops) + return -ENOMEM; + dart->boot_info = + devm_kzalloc(dev, sizeof(*dart->boot_info), GFP_KERNEL); + if (!dart->boot_info) + return -ENOMEM; + if (of_property_read_u32(dev->of_node, "apple,apif-dart-id", + &dart->dart_id)) + return dev_err_probe(dev, -EINVAL, + "missing apple,apif-dart-id\n"); + if (dart->dart_id > 0xff) + return dev_err_probe(dev, -EINVAL, "dart-id out of range\n"); + + dart->dma_max = DART_APIF_APERTURE_END; + if (of_find_property(dev->of_node, "apple,dma-range", NULL)) { + u64 range[2], end; + + if (of_property_read_u64_array(dev->of_node, "apple,dma-range", + range, 2) || + !range[1] || + (range[0] | range[1]) & (DART_APIF_PAGE_SIZE - 1) || + check_add_overflow(range[0], range[1] - 1, &end) || + end > DART_APIF_NATIVE_APERTURE_END) + return dev_err_probe(dev, -EINVAL, "bad DMA range\n"); + dart->dma_min = range[0]; + dart->dma_max = end; + } + if (of_find_property(dev->of_node, "apple,apif-iova-force", NULL)) { + if (of_property_read_u64(dev->of_node, "apple,apif-iova-force", + &dart->iova_force) || + (dart->iova_force && dart->iova_force != BIT_ULL(40))) + return dev_err_probe(dev, -EINVAL, + "unsupported IOVA force mask\n"); + /* A native DVA aperture already contains its high address bits. + * Hidden force bits are only valid for the low-address device view. + */ + if (dart->iova_force && dart->dma_max >= dart->iova_force) + return dev_err_probe( + dev, -EINVAL, + "IOVA force overlaps DMA aperture\n"); + } + + dart->reserve_last_page = of_property_read_bool( + dev->of_node, "apple,apif-reserve-last-page"); + dart->num_streams = DART_APIF_MAX_STREAMS; + of_property_read_u32(dev->of_node, "apple,apif-num-streams", + &dart->num_streams); + if (!dart->num_streams || dart->num_streams > DART_APIF_MAX_STREAMS) + return dev_err_probe(dev, -EINVAL, "bad stream count\n"); + + dart->batch = devm_kzalloc(dev, PAGE_SIZE, GFP_KERNEL); + if (!dart->batch) + return -ENOMEM; + ret = apif_load_retained(dart); + if (ret) + return dev_err_probe(dev, ret, + "retained DART discovery failed\n"); + + ret = devm_add_action_or_reset(dev, apif_free_page_lists, dart); + if (ret) + return ret; + platform_set_drvdata(pdev, dart); + + ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, + "apple-dart-apif.%s", dev_name(dev)); + if (ret) + return ret; + + ret = iommu_device_register(&dart->iommu, &apple_dart_apif_ops, dev); + if (ret) + goto err_sysfs; + + return 0; + +err_sysfs: + iommu_device_sysfs_remove(&dart->iommu); + return ret; +} + +static void apple_dart_apif_remove(struct platform_device *pdev) +{ + struct apple_dart_apif *dart = platform_get_drvdata(pdev); + + iommu_device_unregister(&dart->iommu); + iommu_device_sysfs_remove(&dart->iommu); +} + +static const struct of_device_id apple_dart_apif_of_match[] = { + { .compatible = "apple,dart-apif" }, + {}, +}; +MODULE_DEVICE_TABLE(of, apple_dart_apif_of_match); + +static struct platform_driver apple_dart_apif_driver = { + .driver = { + .name = "apple-dart-apif", + .suppress_bind_attrs = true, + .of_match_table = apple_dart_apif_of_match, + }, + .probe = apple_dart_apif_probe, + .remove = apple_dart_apif_remove, +}; +module_platform_driver(apple_dart_apif_driver); + +MODULE_DESCRIPTION("Apple DART IOMMU over the Aurora platform interface"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 49152c8b5e1f0..183e81ae2b3c7 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -80,6 +80,16 @@ config APPLE_SART Say 'y' here if you have an Apple SoC. +config APPLE_SART_APIF + bool "Apple SART over the Aurora platform interface" + depends on ARM64 && OF && APPLE_SART=y + select AURORA_APIF_MMIO + help + SART provider for a trusted Linux guest under Aura and SPTM. + Consumers retain the existing Apple SART API. Native call selection, + region tracking and completion interpretation stay in this backend. + Existing raw-MMIO SART providers continue to serve other systems. + config APPLE_TUNABLE tristate depends on ARCH_APPLE || COMPILE_TEST diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index e6cb2e917bc0a..79d961c003304 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -35,3 +35,5 @@ obj-$(CONFIG_APPLE_MCC) += apple-mcc.o apple-mcc-y := mcc.o obj-$(CONFIG_APPLE_CLPC) += clpc.o + +obj-$(CONFIG_APPLE_SART_APIF) += apple-sart-apif.o diff --git a/drivers/soc/apple/apple-sart-apif.c b/drivers/soc/apple/apple-sart-apif.c new file mode 100644 index 0000000000000..3bbdaea969840 --- /dev/null +++ b/drivers/soc/apple/apple-sart-apif.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* SART policy stays in Linux; native table-5 calls use the common APIF client. */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sart-backend.h" + +#define SART_APIF_MAX_REGIONS 16 + +struct sart_apif { + struct device *dev; + struct aurora_apif *apif; + struct mutex lock; + bool active; + struct { + phys_addr_t address; + size_t size; + } regions[SART_APIF_MAX_REGIONS]; +}; + +static int sart_apif_region(struct apple_sart *sart, phys_addr_t address, + size_t size, bool map) +{ + struct sart_apif *state = apple_sart_backend_data(sart); + struct aurora_apif_op op; + u64 pa; + int i, ret; + + if (!size || ((address | size) & (SZ_16K - 1)) || + address > PHYS_ADDR_MAX - (size - 1)) + return -EINVAL; + mutex_lock(&state->lock); + for (i = 0; i < SART_APIF_MAX_REGIONS; i++) + if (map ? !state->regions[i].size : + state->regions[i].address == address && + state->regions[i].size == size) + break; + if (i == SART_APIF_MAX_REGIONS) { + ret = map ? -EBUSY : -ENOENT; + goto out; + } + op = (struct aurora_apif_op){ + .selector = AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, + AURORA_APIF_TRANSLATE), + .arg = { dma_to_phys(state->dev, address), size }, + }; + ret = aurora_apif_submit(state->apif, &op, 1, NULL); + if (ret) + goto out; + pa = op.ret; + if (!state->active) { + op = (struct aurora_apif_op){ + .selector = AURORA_APIF_SELECTOR( + 5, 0), /* SET_STATE, void result */ + .arg = { 1 }, + }; + ret = aurora_apif_submit(state->apif, &op, 1, NULL); + if (ret) + goto out; + state->active = true; + } + op = (struct aurora_apif_op){ + .selector = AURORA_APIF_SELECTOR(5, map ? 1 : 2), + .arg = { pa, size, map, map }, + }; + ret = aurora_apif_submit(state->apif, &op, 1, NULL); + /* MAP is void; UNMAP returns status. A refused unmap keeps the region + * live so the consumer cannot treat its DMA backing as released. + */ + if (!ret && !map && op.ret) + ret = -EIO; + if (!ret) { + state->regions[i].address = map ? address : 0; + state->regions[i].size = map ? size : 0; + } +out: + mutex_unlock(&state->lock); + return ret; +} + +static int sart_apif_add(struct apple_sart *sart, phys_addr_t address, + size_t size) +{ + return sart_apif_region(sart, address, size, true); +} + +static int sart_apif_remove(struct apple_sart *sart, phys_addr_t address, + size_t size) +{ + return sart_apif_region(sart, address, size, false); +} + +static const struct apple_sart_backend sart_apif_backend = { + .add = sart_apif_add, + .remove = sart_apif_remove, +}; + +static void sart_apif_put(void *data) +{ + aurora_apif_put(data); +} + +static int sart_apif_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct sart_apif *state; + struct apple_sart *sart; + int ret; + + state = devm_kzalloc(dev, sizeof(*state), GFP_KERNEL); + if (!state) + return -ENOMEM; + state->dev = dev; + mutex_init(&state->lock); + state->apif = aurora_apif_get(dev); + if (IS_ERR(state->apif)) + return dev_err_probe(dev, PTR_ERR(state->apif), + "APIF transport unavailable\n"); + ret = devm_add_action_or_reset(dev, sart_apif_put, state->apif); + if (ret) + return ret; + sart = apple_sart_create_backend(dev, &sart_apif_backend, state); + if (IS_ERR(sart)) + return PTR_ERR(sart); + platform_set_drvdata(pdev, sart); + return 0; +} + +static void sart_apif_shutdown(struct platform_device *pdev) +{ + struct apple_sart *sart = platform_get_drvdata(pdev); + struct sart_apif *state = apple_sart_backend_data(sart); + + /* Supplier ordering stops consumers before their Linux-owned grants. */ + for (unsigned int i = 0; i < SART_APIF_MAX_REGIONS; i++) + if (state->regions[i].size && + sart_apif_remove(sart, state->regions[i].address, + state->regions[i].size)) + dev_err(state->dev, + "failed to release DMA region at shutdown\n"); +} + +static const struct of_device_id sart_apif_of_match[] = { + { .compatible = "apple,sart-apif" }, + {} +}; +MODULE_DEVICE_TABLE(of, sart_apif_of_match); + +static struct platform_driver sart_apif_driver = { + .driver = { + .name = "apple-sart-apif", + .of_match_table = sart_apif_of_match, + .suppress_bind_attrs = true, + }, + .probe = sart_apif_probe, + .shutdown = sart_apif_shutdown, +}; +module_platform_driver(sart_apif_driver); + +MODULE_DESCRIPTION("Apple SART over the Aurora platform interface"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/apple/sart-backend.h b/drivers/soc/apple/sart-backend.h new file mode 100644 index 0000000000000..8a130fa48ccba --- /dev/null +++ b/drivers/soc/apple/sart-backend.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef APPLE_SART_BACKEND_H +#define APPLE_SART_BACKEND_H + +#include + +/* Alternate providers retain the existing consumer API. */ +struct apple_sart_backend { + int (*add)(struct apple_sart *sart, phys_addr_t address, size_t size); + int (*remove)(struct apple_sart *sart, phys_addr_t address, + size_t size); + bool (*inherited)(struct apple_sart *sart, phys_addr_t address, + size_t size); +}; + +struct apple_sart * +apple_sart_create_backend(struct device *dev, + const struct apple_sart_backend *backend, void *data); +void *apple_sart_backend_data(struct apple_sart *sart); + +#endif diff --git a/drivers/soc/apple/sart.c b/drivers/soc/apple/sart.c index b0b208c92741d..6abdfdd8bf522 100644 --- a/drivers/soc/apple/sart.c +++ b/drivers/soc/apple/sart.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT +// SPDX-License-Identifier: (GPL-2.0-only OR MIT) AND GPL-2.0-only /* * Apple SART device driver * Copyright (C) The Asahi Linux Contributors @@ -23,6 +23,7 @@ #include #include #include +#include "sart-backend.h" #define APPLE_SART_MAX_ENTRIES 16 @@ -82,8 +83,33 @@ struct apple_sart { unsigned long protected_entries; unsigned long used_entries; + const struct apple_sart_backend *backend; + void *backend_data; }; +struct apple_sart *apple_sart_create_backend(struct device *dev, + const struct apple_sart_backend *backend, void *data) +{ + struct apple_sart *sart; + + if (!backend || !backend->add || !backend->remove) + return ERR_PTR(-EINVAL); + sart = devm_kzalloc(dev, sizeof(*sart), GFP_KERNEL); + if (!sart) + return ERR_PTR(-ENOMEM); + sart->dev = dev; + sart->backend = backend; + sart->backend_data = data; + return sart; +} +EXPORT_SYMBOL_GPL(apple_sart_create_backend); + +void *apple_sart_backend_data(struct apple_sart *sart) +{ + return sart->backend_data; +} +EXPORT_SYMBOL_GPL(apple_sart_backend_data); + static void sart0_get_entry(struct apple_sart *sart, int index, u8 *flags, phys_addr_t *paddr, size_t *size) { @@ -251,6 +277,8 @@ bool apple_sart_is_inherited_region(struct apple_sart *sart, phys_addr_t paddr, { int i; + if (sart->backend) + return sart->backend->inherited && sart->backend->inherited(sart, paddr, size); if (!size || paddr > PHYS_ADDR_MAX - (size - 1)) return false; for (i = 0; i < APPLE_SART_MAX_ENTRIES; i++) { @@ -293,6 +321,8 @@ int apple_sart_add_allowed_region(struct apple_sart *sart, phys_addr_t paddr, { int i, ret; + if (sart->backend) + return sart->backend->add(sart, paddr, size); for (i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) { if (test_bit(i, &sart->protected_entries)) continue; @@ -327,6 +357,8 @@ int apple_sart_remove_allowed_region(struct apple_sart *sart, phys_addr_t paddr, { int i; + if (sart->backend) + return sart->backend->remove(sart, paddr, size); dev_dbg(sart->dev, "will remove [paddr: %pa, size: 0x%zx] from allowed regions\n", &paddr, size); @@ -398,6 +430,6 @@ static struct platform_driver apple_sart_driver = { }; module_platform_driver(apple_sart_driver); -MODULE_LICENSE("Dual MIT/GPL"); +MODULE_LICENSE("GPL"); MODULE_AUTHOR("Sven Peter "); MODULE_DESCRIPTION("Apple SART driver"); diff --git a/include/linux/apple-dart-apif.h b/include/linux/apple-dart-apif.h new file mode 100644 index 0000000000000..814b2a91aba94 --- /dev/null +++ b/include/linux/apple-dart-apif.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_APPLE_DART_APIF_H +#define _LINUX_APPLE_DART_APIF_H +#include +#include +#include +#if IS_REACHABLE(CONFIG_APPLE_DART_APIF) +/* Caller must quiesce every device on this DART before powering it down. + * Tables and mappings survive the native SPTM power transition. + */ +int apple_dart_apif_set_power(struct device *dev, bool on); +#else +static inline int apple_dart_apif_set_power(struct device *dev, bool on) +{ + return -EOPNOTSUPP; +} +#endif +#endif diff --git a/include/linux/soc/aurora/apif-mmio.h b/include/linux/soc/aurora/apif-mmio.h new file mode 100644 index 0000000000000..d85ceb7a1cd40 --- /dev/null +++ b/include/linux/soc/aurora/apif-mmio.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_SOC_AURORA_APIF_MMIO_H +#define _LINUX_SOC_AURORA_APIF_MMIO_H + +#include + +struct device; +struct aurora_apif; + +#define AURORA_APIF_SELECTOR(table, endpoint) \ + (((u64)(table) << 32) | (endpoint)) +#define AURORA_APIF_SERVICE 0xffffffffU +#define AURORA_APIF_TRANSLATE 1 +#define AURORA_APIF_ALLOC_FRAME 2 +#define AURORA_APIF_FREE_FRAME 3 +#define AURORA_APIF_MAP_IO 4 +#define AURORA_APIF_GUEST_ROOT 5 +#define AURORA_APIF_BOOT_IOMMU 6 +#define AURORA_APIF_BOOT_TABLE 7 +#define AURORA_APIF_FIND_FRAME 8 +#define AURORA_APIF_TAG_FRAME 9 +#define AURORA_APIF_BOOT_DART 0 + +struct aurora_apif_op { + u64 selector; + u64 arg[6]; + u64 ret; +}; + +struct aurora_apif_boot_iommu { + u64 level, count, base, size, pa, ipa, attrs, reserved; +}; + +/* Each client owns a buffer. Submission is synchronous and atomic-context + * safe. Unbind drains the client and subsequent calls return -ENODEV. + */ +struct aurora_apif *aurora_apif_get(struct device *dev); +void aurora_apif_put(struct aurora_apif *apif); +/* @done counts executed operations. The transport does not interpret native + * results; endpoint-specific status and ownership belong to the caller. + */ +int aurora_apif_submit(struct aurora_apif *apif, struct aurora_apif_op *ops, + u32 count, u32 *done); + +#endif diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index ac5717ad1078c..99709312cf09e 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -3,6 +3,7 @@ TARGETS += acct TARGETS += alsa TARGETS += amd-pstate TARGETS += arm64 +TARGETS += aurora-apif TARGETS += bpf TARGETS += breakpoints TARGETS += cachestat diff --git a/tools/testing/selftests/aurora-apif/Makefile b/tools/testing/selftests/aurora-apif/Makefile new file mode 100644 index 0000000000000..49020bdc3dfee --- /dev/null +++ b/tools/testing/selftests/aurora-apif/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +TEST_PROGS := apif_mmio.py +include ../lib.mk diff --git a/tools/testing/selftests/aurora-apif/apif_mmio.py b/tools/testing/selftests/aurora-apif/apif_mmio.py new file mode 100755 index 0000000000000..9d6e0be1ac7e3 --- /dev/null +++ b/tools/testing/selftests/aurora-apif/apif_mmio.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Host fault injection against production APIF functions; no firmware calls.""" +import os +from pathlib import Path +import re +import shlex +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[4] + + +def function(path, name): + source = (ROOT / path).read_text() + match = re.search(r"^.*\b" + name + r"\([^;]*?\)\n\{", source, re.M) + if not match: + raise RuntimeError(f"missing production function {name}") + start = match.start() + pos = match.end() - 1 + depth = 0 + for end in range(pos, len(source)): + if source[end] == "{": + depth += 1 + elif source[end] == "}": + depth -= 1 + if not depth: + return source[start:end + 1] + raise RuntimeError(f"unterminated function {name}") + + +HEADER = r''' +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef int32_t s32; +typedef uint64_t u64; +typedef uintptr_t phys_addr_t; +#define __packed __attribute__((packed)) +#define U64_MAX UINT64_MAX +#define PHYS_ADDR_MAX UINTPTR_MAX +#define SZ_16K 16384UL +#define SZ_4G (1ULL << 32) +#define PAGE_SIZE SZ_16K +#define IOMMU_READ 1 +#define IOMMU_WRITE 2 +#define IOMMU_CACHE 4 +#define IOMMU_NOEXEC 8 +#define IOMMU_MMIO 16 +#define GENMASK_ULL(h,l) ((~0ULL << (l)) & (~0ULL >> (63-(h)))) +#define GFP_ATOMIC 0 +#define kmalloc(n,flags) malloc(n) +#define virt_to_phys(p) ((u64)(uintptr_t)(p)) +#define round_down(v,a) ((v) & ~((a)-1)) +#define min(a,b) ((a)<(b)?(a):(b)) +#define READ_ONCE(v) (v) +#define raw_spin_lock_irqsave(lock,flags) ((void)(lock), (flags)=0) +#define raw_spin_unlock_irqrestore(lock,flags) ((void)(lock), (void)(flags)) +#define mutex_lock(lock) ((void)(lock)) +#define mutex_unlock(lock) ((void)(lock)) +''' +api = (ROOT / "include/linux/soc/aurora/apif-mmio.h").read_text() +api = "\n".join(line for line in api.splitlines() if not line.startswith("#include")) +HEADER += api + "\n" + + +def transport_test(): + submit = function("drivers/firmware/aurora-apif-mmio.c", "aurora_apif_submit") + return HEADER + r''' +struct apif_batch { u32 count, flags; s32 status; u32 done; struct aurora_apif_op ops[3]; }; +struct aurora_apif { int lock; bool live; struct apif_batch *batch; u32 max_ops; }; +static unsigned rings; +static int mode; +static void apif_ring(struct aurora_apif *apif) +{ + rings++; + apif->batch->done = apif->batch->count; + apif->batch->status = 0; + for (u32 i = 0; i < apif->batch->count; i++) apif->batch->ops[i].ret = U64_MAX; + if (mode == 1) { apif->batch->done = 1; apif->batch->status = -EINVAL; } + if (mode == 2) apif->batch->done++; + if (mode == 3) { apif->batch->done = 0; apif->batch->status = -EINPROGRESS; } + if (mode == 4) { apif->batch->done = 0; apif->batch->status = 1; } + if (mode == 5) { apif->batch->done = 1; } +} +''' + submit + r''' +int main(void) +{ + struct apif_batch batch = {}; + struct aurora_apif client = { .live = true, .batch = &batch, .max_ops = 3 }; + struct aurora_apif_op ops[8] = {}; + u32 done; + assert(!aurora_apif_submit(&client, ops, 8, &done)); + assert(done == 8 && rings == 3 && ops[7].ret == U64_MAX); + assert(!aurora_apif_submit(&client, NULL, 0, &done) && !done && rings == 3); + assert(aurora_apif_submit(&client, NULL, 1, &done) == -EINVAL && !done); + mode = 1; + assert(aurora_apif_submit(&client, ops, 8, &done) == -EINVAL && done == 1); + mode = 2; + assert(aurora_apif_submit(&client, ops, 8, &done) == -EIO && !done); + mode = 3; + assert(aurora_apif_submit(&client, ops, 8, &done) == -EIO && !done); + mode = 4; + assert(aurora_apif_submit(&client, ops, 8, &done) == -EIO && !done); + mode = 5; + assert(aurora_apif_submit(&client, ops, 8, &done) == -EIO && done == 1); + client.live = false; + assert(aurora_apif_submit(&client, ops, 1, &done) == -ENODEV && !done); + return 0; +} +''' + + +def dart_test(): + path = "drivers/iommu/apple-dart-apif.c" + source = (ROOT / path).read_text() + constants = source[source.index("#define APIF_DART_INIT"):source.index("static const struct iommu_ops")] + funcs = "\n".join(function(path, name) for name in + ["apif_force_iovas", "apif_one", "apif_prepare_table", "apif_boot_query", "apif_ring"]) + return HEADER + constants + r''' +struct apif_retained_sid { unsigned root_level; }; +struct apple_dart_apif { + struct aurora_apif *apif; + struct aurora_apif_op native_ops[2 * APIF_BATCH_MAX_OPS]; + struct aurora_apif_boot_iommu *boot_info; + u64 *page_lists[APIF_BATCH_MAX_OPS], page_list_pa[APIF_BATCH_MAX_OPS]; + u32 dart_id, num_streams; + u64 iova_force; + bool ownership_uncertain; + struct apif_retained_sid *retained[DART_APIF_MAX_STREAMS]; + struct apif_batch *batch; +}; +static struct aurora_apif_op calls[2048]; +static unsigned ncalls; +static u64 boot_table, found_table, retype_result = 1, fail_selector; +static int fail_status; +static int native_done = -1; +int aurora_apif_submit(struct aurora_apif *client, struct aurora_apif_op *ops, u32 count, u32 *done) +{ + (void)client; + if (done) *done = 0; + for (u32 i = 0; i < count; i++) { + struct aurora_apif_op *op = &ops[i]; + calls[ncalls++] = *op; + if (fail_selector && op->selector == fail_selector) return fail_status; + if (native_done >= 0 && (op->selector >> 32) != AURORA_APIF_SERVICE && i >= (u32)native_done) + return -EIO; + op->ret = 0; + if (op->selector == AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, AURORA_APIF_TRANSLATE)) + op->ret = op->arg[0] + SZ_4G; + if (op->selector == AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, AURORA_APIF_BOOT_TABLE)) + op->ret = boot_table; + if (op->selector == AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, AURORA_APIF_FIND_FRAME)) + op->ret = found_table; + if (op->selector == AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, AURORA_APIF_ALLOC_FRAME)) + op->ret = 0x90000000; + if (op->selector == AURORA_APIF_SELECTOR(0, 1)) op->ret = retype_result; + /* A void endpoint is allowed to return nonzero. */ + if (op->selector == AURORA_APIF_SELECTOR(3, 0)) op->ret = U64_MAX; + if (done) *done = i + 1; + } + return 0; +} +''' + funcs + r''' +int main(void) +{ + struct apple_dart_apif dart = { .dart_id = 5, .num_streams = 16 }; + struct aurora_apif_boot_iommu info = {}; + dart.boot_info = &info; + dart.batch = calloc(1, PAGE_SIZE); + assert(dart.batch); + boot_table = 0x80000000; + assert(!apif_prepare_table(&dart, 3, 0x2000000, 2) && ncalls == 1); + boot_table = 0; found_table = 0x80000000; ncalls = 0; + assert(!apif_prepare_table(&dart, 3, 0x2000000, 2) && ncalls == 2); + found_table = 0; ncalls = 0; + assert(!apif_prepare_table(&dart, 3, 0x2000000, 2) && ncalls == 6); + assert(calls[3].selector == AURORA_APIF_SELECTOR(0, 1)); + assert(calls[3].arg[0] == 0x90000000 && calls[3].arg[1] == 11 && calls[3].arg[2] == 24); + assert(calls[4].selector == AURORA_APIF_SELECTOR(3, 0)); + assert(calls[4].arg[0] == 5 && calls[4].arg[1] == 3 && calls[4].arg[4] == 0x90000000); + assert(calls[5].selector == AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, AURORA_APIF_TAG_FRAME)); + retype_result = 0; ncalls = 0; + assert(apif_prepare_table(&dart, 3, 0, 2) == -EIO && ncalls == 5); + assert(calls[4].selector == AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, AURORA_APIF_FREE_FRAME)); + retype_result = 1; ncalls = 0; + fail_selector = AURORA_APIF_SELECTOR(3, 0); fail_status = -ETIMEDOUT; + assert(apif_prepare_table(&dart, 3, 0, 2) == -ETIMEDOUT && dart.ownership_uncertain); + assert(apif_ring(&dart) == -EIO && !dart.batch->done); + dart.ownership_uncertain = false; fail_selector = 0; ncalls = 0; + dart.iova_force = 1ULL << 40; + dart.batch->count = 2; + for (unsigned i = 0; i < 2; i++) { + dart.batch->op[i] = (struct apif_op){ .op = APIF_DART_MAP, + .arg = { 3, i * 0x4000, 0x800000 + i * 0x4000, 0x4000, 3 | IOMMU_NOEXEC } }; + } + assert(!apif_ring(&dart) && dart.batch->done == 2 && ncalls == 6); + assert(calls[4].selector == AURORA_APIF_SELECTOR(3, 2)); + assert(calls[4].arg[2] == (1ULL << 40)); + assert(calls[4].arg[3] == virt_to_phys(dart.page_lists[0]) + SZ_4G); + assert(dart.page_lists[0][0] == 0x100800000ULL); + assert(calls[4].arg[5] == 4 && dart.batch->op[0].arg[1] == 0); + ncalls = 0; native_done = 1; + assert(apif_ring(&dart) == -EIO && dart.batch->done == 1); + assert(dart.batch->op[0].arg[1] == 0 && dart.batch->op[1].arg[1] == 0x4000); + native_done = -1; ncalls = 0; + dart.batch->op[0].arg[4] = 256; + assert(apif_ring(&dart) == -EINVAL && !ncalls && !dart.batch->done); + for (unsigned i = 0; i < APIF_BATCH_MAX_OPS; i++) free(dart.page_lists[i]); + free(dart.batch); + return 0; +} +''' + + +def sart_test(): + return HEADER + r''' +struct device {}; +struct mutex { int unused; }; +struct sart_apif { + struct device *dev; + struct aurora_apif *apif; + struct mutex lock; + bool active; + struct { phys_addr_t address; size_t size; } regions[16]; +}; +struct apple_sart { struct sart_apif *state; }; +#define SART_APIF_MAX_REGIONS 16 +#define apple_sart_backend_data(sart) ((sart)->state) +#define dma_to_phys(dev,address) (address) +static struct aurora_apif_op calls[8]; +static unsigned ncalls; +static u64 unmap_result; +int aurora_apif_submit(struct aurora_apif *client, struct aurora_apif_op *ops, u32 count, u32 *done) +{ + (void)client; (void)done; + assert(count == 1); + calls[ncalls++] = *ops; + ops->ret = U64_MAX; + if (ops->selector == AURORA_APIF_SELECTOR(AURORA_APIF_SERVICE, AURORA_APIF_TRANSLATE)) + ops->ret = ops->arg[0] + SZ_4G; + if (ops->selector == AURORA_APIF_SELECTOR(5, 2)) ops->ret = unmap_result; + return 0; +} +''' + function("drivers/soc/apple/apple-sart-apif.c", "sart_apif_region") + r''' +int main(void) +{ + struct sart_apif state = {}; + struct apple_sart sart = { &state }; + assert(sart_apif_region(&sart, 0, 0, true) == -EINVAL); + assert(sart_apif_region(&sart, 1, SZ_16K, true) == -EINVAL); + assert(!sart_apif_region(&sart, 0x800000, SZ_16K, true)); + assert(ncalls == 3 && state.active && state.regions[0].size == SZ_16K); + assert(calls[2].selector == AURORA_APIF_SELECTOR(5, 1) && calls[2].arg[0] == 0x100800000ULL); + ncalls = 0; unmap_result = 1; + assert(sart_apif_region(&sart, 0x800000, SZ_16K, false) == -EIO); + assert(state.regions[0].size == SZ_16K); + ncalls = 0; unmap_result = 0; + assert(!sart_apif_region(&sart, 0x800000, SZ_16K, false) && !state.regions[0].size); + assert(sart_apif_region(&sart, 0x800000, SZ_16K, false) == -ENOENT); + return 0; +} +''' + + +def main(): + compiler = shlex.split(os.environ.get("HOSTCC", "cc")) + print("TAP version 13\n1..3", flush=True) + with tempfile.TemporaryDirectory(prefix="apif-selftest-") as directory: + for i, (name, test) in enumerate([( "transport", transport_test), + ("dart", dart_test), ("sart", sart_test)], 1): + path = Path(directory) / name + path.with_suffix(".c").write_text(test()) + subprocess.run(compiler + ["-std=gnu11", "-Wall", "-Wextra", "-Werror", + str(path.with_suffix(".c")), "-o", str(path)], check=True) + subprocess.run([str(path)], check=True) + print(f"ok {i} - {name} production function", flush=True) + + +if __name__ == "__main__": + main()