diff --git a/.github/workflows/sep-source.yml b/.github/workflows/sep-source.yml index 11ff022654dbd..e77ffd8ad7340 100644 --- a/.github/workflows/sep-source.yml +++ b/.github/workflows/sep-source.yml @@ -8,9 +8,41 @@ on: - 'tools/aurora-bindgen/**' - 'drivers/soc/apple/**' - 'include/uapi/linux/apple-sep.h' + - 'tools/aurora-sep/**' - '.github/workflows/sep-source.yml' jobs: + source: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + filter: blob:none + sparse-checkout-cone-mode: false + sparse-checkout: | + drivers/soc/apple/** + include/uapi/linux/apple-sep.h + tools/aurora-sep/** + - name: Check biometric ioctl ABI sizes + run: | + cat > /tmp/sep-abi.c <<'EOF' + #include + _Static_assert(sizeof(struct sep_bio_enrol_start) == 144, "enrol ABI"); + _Static_assert(sizeof(struct sep_bio_enclave_list) == 520, "list ABI"); + _Static_assert(sizeof(struct sep_bio_delete) == 16, "delete ABI"); + int main(void) { return 0; } + EOF + cc -Wall -Werror -Iinclude/uapi /tmp/sep-abi.c -o /tmp/sep-abi + /tmp/sep-abi + - name: Test recovery utility and package sources + run: | + python3 -m unittest tools/aurora-sep/tests/test_sep_bio_identities.py -v + python3 -m unittest tools/aurora-sep/tests/test_orphan_delete.py -v + python3 -m unittest tools/aurora-sep/tests/test_enrol_admission.py -v + python3 -m unittest tools/aurora-sep/tests/test_owner_preparation.py -v + python3 -m unittest discover -s tools/aurora-sep/tests -p test_packaging.py -v + rust-build: runs-on: ubuntu-24.04 timeout-minutes: 45 diff --git a/Documentation/ABI/testing/sysfs-driver-ane-performance b/Documentation/ABI/testing/sysfs-driver-ane-performance new file mode 100644 index 0000000000000..0f28173a24e39 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-ane-performance @@ -0,0 +1,35 @@ +What: /sys/class/accel/accel*/device/performance_level +Date: October 2026 +Contact: asahi@lists.linux.dev +Description: + Requested ANE performance level, 0 through 5, or "unsupported" + when the device tree does not describe a supported controller. + This is a firmware request, not a measured clock frequency. + Work raises the level from the SoC match data. It drops to 0 + after 32 ms without a submission. A stream amortizes the clock + transition cost; the first job after idle includes that cost. + The runtime PM reference covers the entire request lifetime. + +What: /sys/class/accel/accel*/device/memory_vote +Date: October 2026 +Contact: asahi@lists.linux.dev +Description: + Requested average and peak memory bandwidth, in decimal kB/s, + separated by a space. Both are zero after idle release. The + requests are aggregated with other interconnect consumers. + The active values come from the high-level OPP's opp-avg-kBps + and opp-peak-kBps. "unsupported" means no interconnect path is + described. It does not mean a successful zero-bandwidth vote. + Current T600x trees have no host DCS interconnect provider; + describing a path requires a verified shared voting transport. + +What: /sys/class/accel/accel*/device/performance_error +Date: October 2026 +Contact: asahi@lists.linux.dev +Description: + Last performance transition result: zero on success or a + negative errno. Failed busy requests abort before enqueueing. + Failed idle releases retain the runtime PM reference, retry + up to three times, and remain visible here if still failing. + A subsequent workload retries release before requesting high. + An unknown DMA state retains the votes until recovery. diff --git a/Documentation/ABI/testing/sysfs-driver-aop-als b/Documentation/ABI/testing/sysfs-driver-aop-als new file mode 100644 index 0000000000000..c5b6acd796242 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-aop-als @@ -0,0 +1,13 @@ +What: /sys/bus/iio/devices/iio:deviceX/in_cct_input +Date: October 2026 +Contact: linux-iio@vger.kernel.org +Description: + Firmware-computed correlated colour temperature in Kelvin on + aop-sensors-als devices with a Redbird (VD6286) sensor. + Fractional Kelvin are reported with micro resolution. There + is no raw photodiode-to-CCT conversion in the host driver. + Only the documented 122-byte fake-HID report is decoded. + Before a valid positive finite CCT arrives, reading returns + ENODATA. A report in another layout or without a valid CCT + leaves the last valid reading in place, independently of lux. + CT720 and CT817 devices do not expose this attribute. diff --git a/Documentation/ABI/testing/sysfs-driver-apple-pmp-dram b/Documentation/ABI/testing/sysfs-driver-apple-pmp-dram new file mode 100644 index 0000000000000..2e7cfaba58445 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-apple-pmp-dram @@ -0,0 +1,49 @@ +What: /sys/bus/platform/devices/.../dram/type +Date: October 2026 +Contact: Ryan Murray +Description: + Read-only DRAM family from DT, currently "lpddr5". This family + includes LPDDR5X; the attribute does not assert the X suffix. + The dram group is present on an apple-pmp-report device only + when DT supplies a complete, consistent memory description. + Static attributes remain valid when firmware is stopped. + +What: /sys/bus/platform/devices/.../dram/max_data_rate +Date: October 2026 +Contact: Ryan Murray +Description: + Read-only maximum interface transfers per second per data pin + (divide by 1000000 for MT/s), from the highest DT OPP. This is + an inferred interface class, not a measured current rate or + a literal LPDDR5 command clock/WCK. The maximum state ID is + state_count minus one, independent of DT child order. + +What: /sys/bus/platform/devices/.../dram/peak_bandwidth +Date: October 2026 +Contact: Ryan Murray +Description: + Read-only inferred physical maximum in decimal bytes/second, + max_data_rate * data_bus_width / 8. This excludes ECC/check bits + and does not measure sustainable application bandwidth. + +What: /sys/bus/platform/devices/.../dram/bandwidth_class +Date: October 2026 +Contact: Ryan Murray +Description: + Read-only published maximum bandwidth class in decimal + bytes/second. It may be rounded relative to peak_bandwidth. + +What: /sys/bus/platform/devices/.../dram/data_bus_width +Date: October 2026 +Contact: Ryan Murray +Description: + Read-only inferred usable data width in bits, excluding ECC. + +What: /sys/bus/platform/devices/.../dram/state_count +Date: October 2026 +Contact: Ryan Murray +Description: + Read-only number of firmware DCS states, from DT. State IDs + are not frequencies. No current-state, current-rate, residency, + or transition-statistics attribute is provided without an + independently specified achieved-state transport. diff --git a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats index 11a964e6f17d8..10cba6c35673f 100644 --- a/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats +++ b/Documentation/ABI/testing/sysfs-driver-asahi-agx-stats @@ -64,5 +64,58 @@ Description: `#[repr(C)]` on the Rust side and its offsets are asserted at build time in `sysfs.c`; readers must parse `key value` lines and must not assume binary layout of this file. - G17 firmware has no supported decoder for this snapshot format; - its file reports unsupported rather than fabricated zero counters. + + G16S (T6040/V25G83) and G17P (T8140/V26A428) use a separate, + qualified snapshot. The G13/G14 output above is unchanged. + The following keys replace the legacy keys on these firmwares: + + utilisation_status, pstate_status + Each is valid, stale, or unavailable. Numeric metrics + are printed only while their capability is valid; + otherwise their value is the string unsupported. + + util_tiler_percent, util_renderer_percent, util_device_percent + Unsigned integer percent (0..100), from tag 0x13 + words 3, 4 and 6 respectively (zero-based). Device + utilisation is independent of the overlapping engines; + it is never their sum. Each complete valid packet from + either role contributes one sample to a shared integer + mean, rounded down. Reads do not consume samples. + A packet with any value above 100 invalidates the whole + utilisation capability. Zero is a valid sample. + + pstate Logical selector 0..10 from tag 0x10 word 6. Words 1 + and 2 form a little-endian u64 firmware timestamp. + The first timestamp establishes a baseline. Invalid + selectors and backwards timestamps remove pstate + validity; timestamps are never extended across epochs. + P0 is a logical state, not a core-power measurement. + + requested_frequency_hz, achieved_frequency_hz, busy_ns, + power_mw, temperature_raw + Always unsupported on these firmwares. The statistics + packet does not identify performance table A or B; + a selector is neither a host request nor a measured + frequency. No qualified busy-time duration, power unit + or temperature field is available. + + The host uses a 16 ms age/averaging-window limit for V25G83 and + 32 ms for V26A428 (two nominal periods). These are conservative + host policies, not firmware guarantees about packet duration. + The oldest contributing utilisation sample bounds its window. + A logical pstate has no qualified periodic-report cadence and + remains valid until a replacement or an epoch boundary; its + host receive time is retained in diagnostics, not used to invent + a periodic-report deadline. + Suspend/resume, restart, statistics-ring quarantine, observed + core gating and teardown end validity. After an epoch boundary, + each ring's already queued records are discarded before new + baselines are accepted. A quarantined statistics ring disables + both roles' exports until reprobe; a trace-ring error does not. + + Reads take the device mutex and do not wake the GPU. The + firmware_reports debugfs view retains per-role raw packets, + counts and host monotonic receive times, plus capability status, + policy, timestamp and diagnostic tick deltas. Tick deltas are + never accumulated as GPU busy time. Firmware role streams are + merged in drain order without deduplication. diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 7d342d1c1b4d1..53948d60375dc 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -1,40 +1,141 @@ .. SPDX-License-Identifier: GPL-2.0-only -T8140/J700 experimental radio PCIe bootstrap -============================================ - -The J700 path trains port 0, validates a bounded PIODMA bootstrap request, -and enables native ECAM for the MediaTek Wi-Fi and Bluetooth functions. -The root bridge forwards their BARs and DART translates endpoint DMA. - -This is a bring-up implementation, not general T8140 PCIe support. -Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``. The board device tree selects the -bootstrap by pointing the host's ``apple,piodma`` property at the engine, and -the kernel then runs it during the host probe. Passing -``pcie_apple_piodma_diag.enumerate=0`` keeps the root-only diagnostic path, -which rejects downstream configuration access. A kernel built without the -option always uses that root-only path. -Build the host and its DART in as well: the supplier retains published DMA -and has no removal contract. Keeping the supplier and host built in avoids -a module dependency on a provider whose lifetime cannot end at unload. - -Suspend-to-idle suspend retains the arena after checking the completed request, -idle FIFO and command state. Resume restores the existing configuration -banks without issuing a second bootstrap. Hibernation, deeper suspend and -kexec are refused while admission or retained DMA ownership is active. The -runtime kexec guard permits the kernel to support kexec on other machines. -Host removal, sysfs reset, hotplug, VFIO assignment and native AER/DPC -recovery are withheld on this retained hierarchy. Sleep is refused until -radio admission completes; the existing suspend-to-idle path remains available -after successful admission. Failed or uncertain bootstrap state -requires a full external hardware reset; there is no command retry. - -The PIODMA arena remains allocated until external reset. Controller removal, -teardown, memory reuse and arbitrary downstream devices are unqualified. -Fresh hardware qualification must check both radio functions across sleep. -Do not remove the controller or its IOMMU while this experiment is active. -These lifecycle contracts require further work before production support. +T8140 radio PCIe bootstrap +========================= + +The J700 radio link needs one validated PIODMA config read of function 0's +vendor/device word before native ECAM works. This is a bounded PCIe bootstrap, +not a generic DMA channel provider. Linux owns the SID17 DMA mapping. After +completion, the host validates byte, word and dword ECAM reads for MT7932 +Wi-Fi and Bluetooth, bridge forwarding, assigned BARs and cold bus-master +state before publishing either function to drivers. Link power states remain +disabled for this qualified pair. Other downstream devices are not admitted. + +Selection and compatibility +--------------------------- + +Build ``CONFIG_PCIE_APPLE_T8140_BOOTSTRAP=y`` with the host, DART and ASPM. +The host's ``apple,piodma`` phandle selects the engine; the engine's +``apple,pcie-host`` must point back to that same T8140 host. No machine name +or module parameter selects this path. Without the host property, downstream +config access stays closed and the host remains root-port-only. Malformed +links fail rather than silently selecting the root-only path. + +The production compatible is ``apple,t8140-pcie-bootstrap``. Existing Stage 2 +DTBs using only ``apple,t8140-piodma-diagnostic`` remain supported. New DTBs +list the production compatible followed by the legacy compatible, so older +kernels still recognize them. J700 receives its live DT from Stage 2; updating +a kernel's packaged DTB alone does not update the boot-time DT. + +Ownership and failure +--------------------- + +The state machine distinguishes CPU-owned allocation, lifecycle admission, +consumed attempt, irreversible engine ownership, completed read and admitted +radios. PM and kexec vetoes are acquired before PCI children exist. Contention +on the system transition or kexec lock defers admission without consuming the +attempt. The bounded retry and delayed host attachment retry only this +software admission. They never retry an engine command. + +A failed probe frees an unpublished coherent allocation and drops its power +reference. Host unwind releases temporary vetoes only after all children have +been removed and only before engine publication. A preflight failure consumes +the attempt, but does not pin lifecycle vetoes after host unwind. The first +engine write marks ownership irreversible, even if initialization or transfer +validation subsequently fails. The supplier, arena, IRQ and power reference +then remain retained until a full external hardware reset. Both successful +and failed retained states reject another command or a release transition. + +Completion IRQ/tag checks, a cleared command bit, FIFO capacity and stable +canaries establish admission observations. None establishes that the engine +has stopped retaining descriptor/completion addresses or can no longer issue +DMA. There is no qualified stop, drain, pointer invalidation or engine reset +sequence in the public implementation. A PCIe link reset affects the link; +it does not prove that SID17 engine ownership ended. Therefore the arena +cannot be freed after admission, and a failed bootstrap cannot safely be +retried merely by resetting the link. + +For the same reason, replaying the bootstrap in a kexec kernel or after image +restore is insufficient. A new kernel may reuse the old DMA storage and DART +mappings before it could establish any new engine state. Hibernation can +restore memory over retained ownership. Hibernation, restore, deeper suspend, +kexec, host removal, user resets, hotplug, VFIO assignment and native AER/DPC +recovery remain refused. Supporting these operations requires a proven engine +teardown/reset and endpoint recovery contract first. Renaming the provider +does not qualify those operations. + +Suspend-to-idle is allowed only after radio admission, with no IRQ fault, +idle command/FIFO checks and intact arena contents. Resume restores the +configuration banks after genpd power restoration without submitting another +command. A failed or incomplete admission still refuses suspend-to-idle. +Do not remove the supplier or its IOMMU while retained ownership exists. + +History and guard audit +----------------------- + +This implementation is derived from the GPL-2.0-only public Linux source and +its history. The original file refers to an unnamed accompanying provenance +ledger that is not distributed in this tree. That reference is not evidence +of teardown or reset semantics. No private implementation is used to extend +the hardware contract here. + +The following public commits explain the guards retained by this provider +and its host. Commit IDs refer to the history containing base +``4db3a5a22c43`` and are usable with ``git show``. + +* ``55039de66e40``: one fixed request, exact MMIO/ECAM ranges, 16 KiB pages, + SID17 translated low-36-bit DART aperture, prefixed engine pointers, + descriptor/data bounds, pristine engine state, IRQ budget/status/tag checks, + expected response and canaries. These restrict the unqualified engine to + the observed address and request contract; all remain enforced. +* ``55039de66e40`` and ``4429c97c6c7a``: typed radio identities, root forwarding, + translated endpoint DMA and cold bus-master checks protect publication. + They remain prerequisites, including admission in each radio driver's probe. +* ``f41094962726`` and ``bb2f7f1f075c``: unscan failed unpublished children and + require completed admission, so a later rescan cannot publish failed radios. +* ``dbe56bcf475c``, ``17f1649e4eee``: pre-child PM/kexec admission, nonblocking + lock order, shutdown/consumed-attempt checks and ordered state publication + prevent transition/probe deadlocks and exposure before guards exist. +* ``69a531f01634``, ``17f1649e4eee`` and ``0886eb8081df``: bounded contention + retry and delayed attachment on system_dfl_wq preserve progress without a + hardware retry. Permanent host references and non-joining cancellation avoid + waiting for a worker behind the host device lock. +* ``67eec79324c6``: device prepare veto covers snapshot ioctls whose notifier + ran before supplier probe. It complements the hibernation/restore notifier. +* ``bda690fe1fb5`` and DjDeveloperr's ``e91592e904f5``: retained suspend-to-idle + checks and configuration-only resume remain. The suspend decision stays in + device prepare, because the notifier runs before the target state is set. +* ``de84c23e4bed``, ``8edd500f3f6d``, ``90e8abce3609`` and ``c4badcb98927``: + retained hierarchy removal, reset, assignment, hotplug and AER/DPC refusal + remain. Permanent refusals return EOPNOTSUPP; admission contention defers. +* ``3a0f71240ae3`` and ``d9efe5417547``: the builtin kexec interlock and its + unload exceptions remain; an attempted transition cannot reclaim the arena. +* ``f45a057bda82``, ``dab09d9a6c9b``, ``b5144ea3ee99``, ``ecb830163924`` and + ``e23e829f3319``: the shared PCIe-C IRQ gate, interrupt ordering and callback + lifetime fixes remain untouched. A bootstrap cleanup must not undo them. + +Retired selection and presentation guards: + +* ``4a78a9f29455`` and ``1238f8c09656`` kept an immutable enumerate parameter. + ``3ce7b58f735d`` removed it: DT omission already selects root-only operation. +* ``b5773247555f`` restricted the provider to J700/T8140 machine names. The + reciprocal DT links, exact controller resources, DART/SID, root and radio + identity checks now carry that restriction without a machine-name switch. +* Diagnostic file, Kconfig and driver names are replaced by production names. + The old compatible alone remains an ABI. Raw packet and IRQ traces are debug + messages; the single success info line follows completed host admission. + +Validation +---------- + +``CONFIG_PCIE_APPLE_T8140_BOOTSTRAP_KUNIT_TEST`` tests ownership transitions, +pre-publication unwind, retained failure, sleep admission, arena bounds and +IRQ validation without hardware. The test does not qualify engine teardown. +Owner qualification must check both radio functions, Wi-Fi association and +Bluetooth power before and after three RTC-woken suspend-to-idle cycles. +Preserve error/warning logs against the baseline and keep the previous fleet +kernel as the rollback entry. Retained failure requires full hardware reset. J700's ``wifi0`` alias and PCI endpoint node allow the public bootloader to -pass the unit's own Wi-Fi address through ``local-mac-address``. The zero -placeholder in the static device tree is not a usable station address. +pass the unit's own Wi-Fi address through ``local-mac-address``. The static +zero placeholder is not a usable station address. diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst new file mode 100644 index 0000000000000..c644b0286078d --- /dev/null +++ b/Documentation/accel/apple-ane.rst @@ -0,0 +1,355 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Apple ANE firmware preflight +=========================== + +The shared ANE firmware module recognizes the H16 (``apple,t6041-ane``, +physical T6040/M4 Pro) and H17 (``apple,t8140-ane``) generations. Recognition +is not an inference support claim: both remain unavailable for execution. +They never enter the H13 task-manager or H14 disk-firmware boot paths. + +H17 ownership +------------- + +Linux has no ANE exclave ownership handoff. The module refuses registration +with ``-EOPNOTSUPP`` and one informational message when an H17 node is +present, including a disabled node. This check occurs before platform core +DMA configuration and automatic power-domain attachment. A probe-time +check alone would be too late. Removing ``apple,exclave-assigned`` does not +establish ownership and does not bypass refusal. + +H16 preload handoff +------------------ + +The bootloader must reserve the personalized TEXT and DATA preload segments +as separate, non-reusable ``no-map`` reserved-memory nodes, attach them in +that order through ``memory-region`` and name them ``firmware-text`` and +``firmware-data``. It must also supply ``apple,firmware-iovas`` as two 64-bit +values: ``0x10000000000`` and ``0x100000c0000``. Their sizes must be +``0xc0000`` and ``0x2ac000``. Physical placement is boot-dependent; board DT +sources must not contain captured physical addresses. The reservations must +remain intact until reboot, including after driver failure. + +H16 preflight checks resource geometry, firmware identity and these ranges +without MMIO or power access. No disk firmware is requested. Missing +reservations fail with ``-ENODATA``. Malformed reservations fail with +``-EINVAL``. Even a valid handoff currently fails with ``-EOPNOTSUPP``: +the strict preload validator, original-container HWX ABI, firmware process +resource decoder and cold boot backend are incomplete. The section-only +load ABI is explicitly refused for H16. No accel node or program-loading interface is published. + +The H16 DT nodes stay disabled. The five real PMGR providers already encode +the system/CPU/MPM/task/base dependency graph. The PMGR binding does not +represent the virtual no-state gates, so no fictitious register-backed +domains are created for them. SID 0 and SID 11 on all three DARTs use the +shared DMA domain; no untested stream remapping is added. + +Implemented SPEC-r2 components +------------------------------ + +The shared transport has an eight-descriptor dynamic channel parser with +unique names, bounded counts, owned-surface bounds and overlap checks. The +runtime uses copied descriptor geometry, verifies the descriptor snapshot, +and checks the complete IO reply envelope. Zero acknowledgment is success; +a nonzero acknowledgment is a firmware refusal, never a retry or success. +Malformed replies, guard changes and timeouts quarantine the instance. A +quarantined queue cannot allocate another BO, and no reset is attempted. + +The version-12 RTKit management state machine validates HELLO, endpoint maps, +system endpoint start, IOP/AP power acknowledgments and application endpoint +start ordering. It does not invent responses for unknown message types or +unimplemented system endpoint buffer requests. Its caller must provide one +absolute session deadline and a bounded 64-bit mailbox transport. + +H16 task streams begin with little-endian ``u32 1`` and twelve reserved zero +bytes. This marker precedes the first aligned task; it is not a task count. +The initial H16 gate rejects a zero marker, other values and nonzero reserved +bytes. The public H16 encoder fixes this prologue at commit +``927013f5480768b07c8e62a866e1514c41aba778`` of +``joshuaswarren/mil-hwx-compiler``, in +``plugins/H16G/Encoding/H16GALUEncoder.mm``. + +H16 validation helpers walk nine fixed task-header words, the documented +conditional extra word, dense/scatter records and the nine allowed register +blocks. A separate helper checks the eight-byte Mach-O relocation entries, +unique aligned offsets, the actual kernel-section ordinal and addend bounds. +Original-container, tensor, program and aperture validation is provided +separately as described below. + +The device-free selftest in ``tools/testing/selftests/drivers/ane`` generates +the SPEC-r2 fp16 add-128 inputs and CPU oracle and verifies all three pinned +buffer hashes. It can also check a supplied HWX against the pinned size/hash. +It never opens an accel device or claims a Linux inference result. + +Remaining implementation gates +------------------------------ + +SPEC-r3 supplies the custom descriptor/aperture layouts and a normalized +immutable preload digest with 42 ordered patchbay records. The complete +preload validator, original-container ABI, process-resource decoder and +physical boot backend still need implementation. Off-list preload mutations +remain refused. + +The 2026-10-31 H16 runtime observation did not capture cold boot, scratch +negotiation, RTKit management, CONFIG_GET or RESOURCE_INFO_GET. The version-12 +and 24 MHz predicates remain conservative proposed startup gates; runtime +traffic does not prove them. Resource and allocation ceilings stay unchanged. +H17 additionally requires a Linux ownership backend before direct access. +There are no module parameters to bypass these gates. + +H16 macOS 27.0 runtime contract +------------------------------ + +J616s has one ANE interrupt, 1054, used as the shared-memory channel +doorbell/completion line for all firmware-to-host replies and events. +It has no ANE exclave assignment. This identifies the interrupt role, not +its cold-start masking/acknowledgment sequence. H17's two IRQ roles remain +unknown. ANE runtime uses its own channel transport; it does not establish +an A7IOP/RTBuddy boot sequence. + +Host command bytes 0..3 are zero, the little-endian u16 ID is at +4 and +the u16 acknowledgment at +6. All observed successful replies had zero +acknowledgment. Nonzero meanings remain unknown and are refused. + +================== ====== ======= ================== +Command ID Channel Bytes +================== ====== ======= ================== +LOAD_PROGRAM 0x0200 1 0x1c0 +UNLOAD_PROGRAM 0x0201 1 0xc +CREATE_PROCESS 0x0202 1 0x258 +TERMINATE_PROCESS 0x0203 1 0x10 +EXECUTE 0x020c 7 0xca8/0xcc0/0xcd8 +POWER_GATE 0x002d 7 0xc +EVENT_REPORT 0x0300 6 0x60 (firmware to host) +================== ====== ======= ================== + +These sizes bound the observed version, not future firmware. POWER_GATE has +a u32 Boolean at +8. The execute-size increment of 0x18 counts custom BAR +records. The channel numbers are observed table indices; +their mapping to channel names remains unproved. The parser retains each +descriptor's original index rather than assuming name order is wire order. + +The transport prepares the H16 envelope and rejects unrecognized runtime +IDs, wrong sizes, channels and power-gate values before publication. +H14's 0x0204 procedure packet and short CREATE_PROCESS body remain H14-only; +H16's serializer and UUID completion table are connected to the shared +submission and event paths, subject to validated process resources. +The H16 runtime command table does not authorize startup commands. + +H16 execute and event fields +--------------------------- + +The macOS 27.0 field contract has a fixed 0xca8-byte execute body followed +by zero, one or two observed 0x18-byte custom BAR records. The 64-slot +ordinary buffer array starts at +0x60 with a 0x30-byte stride; the valid +count is at +0x28. A descriptor carries flags at +0, resource index at +4, +numeric class at +8, the fixed value 3 at +0x10, a u64 DVA at +0x18 and a +u64 byte span at +0x20. Other descriptor bytes are zero. The UUID at +execute +0x20 is independent of program/process/procedure IDs at +8/+0xc/+0x10. +The observed priority is 5 at +0x18; request flags and reserved bytes are zero. + +The custom BAR count is at +0xca0 and records start at +0xca8. Each carries +u32 configuration/value-type/BAR selectors, u8 bit offset/width, u16 flags +and a u64 value. ``ane_h16_execute_encode()`` accepts kernel-owned resource +metadata, checks nonempty counts up to 64 and overflow-free mapping spans, +and zeroes unused slots and reserved space. The caller must first match +indices, numeric classes, permissions, selectors and values to immutable +validated process metadata and keep mappings pinned. The helper does not +establish that authority. Its two-BAR ceiling, 64-bit field bound and +same-configuration/BAR overlap refusal are conservative draft restrictions, +not established firmware-wide limits. Unknown class and selector semantics +are not guessed. There is no userspace interface for these structures. + +Channel 7 acknowledges submission only. Channel 6 carries 0x60-byte 0x0300 +events, with UUID at +8, program/process/procedure at +0x10/+0x14/+0x18 and +state at +0x1c. State 0 is start, 3 hardware-start, 1 successful terminal +completion and 2 terminal failure. Failure was established from field use; +it was not present in the runtime sample. The first word selects report +slots or rolls independently; 0xfe00..0xfe03 is not a universal prefix or a +completion key. CPU-load and timestamp fields are diagnostic. + +``ane_h16_event_decode()`` validates the channel, exact size, command and +zero u16 acknowledgment before returning fields. The result helper requires +all four identity fields to match an outstanding request. Destination 1 is +routed away from the general scheduler; unknown states/destinations refuse. +A caller must register a unique pending UUID before publication, consume it +atomically once on a terminal report, synchronize DMA, and apply quarantine +rules before considering any mapping release. A classifier result alone +cannot prove safe release or reject duplicate events without pending state. + +The shared H16 submission path allocates a non-reused 64-bit UUID, installs +it in a bounded pending table before publishing channel 7, and waits for +both the transport reply and its own terminal event. Events may arrive +before the acknowledgment. States 0 and 3 leave the UUID pending; states +1 and 2 retire it once after matching every identifier. Unknown, duplicate +and late UUIDs cannot retire another request. An identifier mismatch for a pending UUID or a malformed report +quarantines the queue. ISP-destination reports and short +CPU-load notifications cannot complete a general-scheduler request. + +One absolute deadline covers acknowledgment and terminal completion; a +report at or beyond that deadline cannot succeed. Failure state 2, +transport refusal, timeout or corruption prevents output copy-back and +quarantines the queue without retry or reset. Firmware-visible process +mappings remain resident, including after file close. Closing a file +revokes its process capabilities before freeing its staging BO handles. + +Only private coherent DMA mappings may appear in H16 execute descriptors. +Userspace BOs serve as staging buffers: inputs are snapshotted before +publication and outputs are copied back after validated terminal success. +Write barriers order input and command stores before publication; DMA read +barriers order output reads after completion. Streaming DMA sync APIs must +not be applied to these coherent allocations. There is no userspace alias +to code, scratch or live firmware buffers. + +The resource map must come from the original HWX and a decoded firmware +process reply. A userspace index, numeric class or program/process ID cannot +create that map or confer ownership. Only established input/output classes +are admitted; custom BAR requests require a validated selector map and +remain unavailable. The legacy 0x28-byte completion predicate remains +H14-only. Channel-5 allocation/control/free operations require a validated +H16 reply provider and remain unavailable; H14 allocation replies are never +reused for H16. + +No original-container load ABI or H16 process-resource decoder is supplied +by this change. Consequently no H16 process can yet be installed, inference +remains refused and no accel node is registered. The submission integration +is preparatory and does not open the boot, preload, ownership, firmware +version or resource gates. + +KUnit covers byte-exact synthetic execute fields, all 1..64 buffer counts, +zero/one/two custom BAR lengths, wide DVAs/spans, reserved-zero output, +boundary and overflow refusal, all event states, interleaved identities, +ISP routing and malformed reports. No test packet is copied from a capture. + +Original HWX validation +----------------------- + +The validator consumes one kernel-owned immutable snapshot of the complete +HWX. It requires subtype 7 and bounded, exactly consumed load commands; +rejects unknown command kinds, duplicate descriptors, overlapping segments, +sections and file regions; and validates symbols against the complete +section-ordinal table. Tensor names, directions, normalized fp16 dimensions, +overflow-checked strides and full batched spans must agree with their +surfaces and buffer references. There must be at least one input and one +output, with the single output last. + +The linear and scratch program descriptors have exact fixed fields and +zero reserved bytes. All five 16-byte aperture records must equal whole +surface segment bases in order, followed by the optional kernel base; +reserved halves and unused records are zero. Interior, duplicate, reordered +and scratch/text aperture addresses are refused. Task counts and word counts +must agree with the independent stream walk. Relocations use the actual +kernel section ordinal, unique bounded offsets and valid addends, and may +patch only packed-record payload words. Headers, register selectors and +padding remain immutable. + +SHA-256 covers the entire original snapshot, including load commands and +padding. The snapshot digest is rechecked before execution. Validation +provides metadata, not allocation authority or permission to load a program; +the original-container ABI and firmware resource map remain required. + +The hardware-free KUnit suites cover pending UUID lifecycle, acknowledgment +versus completion, interleaving, progress, failures, malformed and mismatched +reports, routing, duplicates, timeouts, capacity and UUID exhaustion. HWX +tests generate synthetic linear/scratch forms with optional kernel data, +then exercise command, geometry, tensor, aperture, symbol, relocation and +digest refusal paths. No hardware inference result is implied. + +H13 timeout and engine recovery +============================== + +The host task-manager path (``ane.ko``) gives each submission one second, +including DART fault-drain retries. An expired submission returns +``-ETIMEDOUT``; recovery never silently retries a program whose output may be +partial. The task's GEM lookup references are released when the ioctl returns. + +Recovery holds the engine and PM-entry locks, drops the current consumer +runtime-PM reference, synchronously suspends its power-domain suppliers, and +checks both the PMGR ACTUAL fields and every attached genpd for power-off. +The SET window omits the shared SYS_CPU parent, whose DART consumers can +suspend asynchronously; it must also turn off before the consumer resumes. +Recovery then restores the consumer +reference, checks power-on before engine MMIO, and runs the probe task-manager +enable sequence. BO handles and mappings remain valid for resubmission. +The system-sleep force helpers are unsuitable here: a forced, in-use supplier +can retain its active runtime status and keep its parent domains powered. + +A power-off acknowledgment is required before reinitializing the engine. +Only after power-on and task-manager readiness are also verified does recovery +release the exceptional module pin and reclaim preserved BOs. If any step +fails, mappings, backing pages and the pin remain held; new submissions fail. +This also covers a partially resumed engine whose retained queue state is +uncertain. The ``reset`` sysfs write retries recovery. Module reload cannot +recover a device while that pin is held. Direct PMGR SET writes and speculative +writes to undocumented task-error registers are not part of this reset path. + +The ``ane-pm-recovery`` KUnit suite exercises the shared PM transaction against +real runtime-PM device links, including repeated cycles, single-domain devices, +consumer/supplier suspend errors, a missing power-off acknowledgment and a +failed resume. It checks that every exit restores the caller's PM reference. +The minimal config is +``tools/testing/selftests/drivers/ane/pm.kunitconfig``. + +Destructive recovery selftest +---------------------------- + +``tools/testing/selftests/drivers/ane/recovery.py`` is an explicitly invoked +hardware test, excluded from the default device-free selftests. It requires +exclusive ownership of an M1-family ANE, Python 3, the packaged H13 add fixture, +and a deliberately invalid convolution program. Run it through the hardware +controller with a watchdog and kernel-log capture:: + + python3 recovery.py /usr/share/omarchy-ane/fixtures/h13-anec/add/program-0.anec hang.anec + +The test keeps both clients open, submits the invalid program three times, +requires a bounded ``ETIMEDOUT`` and ``wedged=0`` each time, and runs twenty +bit-exact add jobs on the surviving client's original fd and mappings after +each reset. Every call changes its inputs and poisons the valid output lanes, +so stale output cannot pass. Its oracle uses exact fp16 quarter values and checks the complete +output, including padding. Closing both clients must return the module +reference count to its initial value; the boot identity must remain unchanged. +An operator should also verify three power-off acknowledgments in the kernel +log and a normal module unload/reload after all clients close. A failed reset +is a test failure, not permission to force unload the module. + +Generate ``hang.anec`` on a build host with ``make-hang-fixture.py`` and a clean +checkout of the MIT-licensed public compiler at commit +``ef2dbceefedff38ecce4f3dc593e148c5e7a6d54`` from +https://github.com/aurora-silicon/mil-hwx-compiler/pull/2:: + + python3 make-hang-fixture.py /path/to/mil-hwx-compiler hang.anec --cxx clang++ + +This first checks that the normal encoder refuses the shape. It bypasses only +that refusal in a temporary translation unit to generate a deterministic +stride-2 1x1 128-to-256 convolution at 56-to-28 spatial size, the refused family +reported to hang the M1 Pro. The compiler checkout is unchanged. No Apple +binary or private program capture is used. The generator writes a hash receipt; +the hardware test must fail if this particular instance does not actually +produce a timeout. The loader pins both fixture hashes and uses ABI 1 ioctls directly, so no +shared libane library is needed. ``--check-fixtures`` validates the two containers without +opening a device. + +H14 recovery lifecycle +---------------------- + +The firmware path (``ane_t6021.ko``) uses one bounded deadline for the command +acknowledgment and its finish event. H14 engine reset is not enabled by this +change. Reusing only the H13 task-manager re-arm would leave the firmware and +its program cache invalid. The corresponding recovery implementation needs to: + +* Serialize firmware commands and stop the RX worker before releasing the + consumer's runtime-PM reference and synchronously suspending its suppliers. +* Verify power-off for the CPU and engine partitions through their providers + before dropping boot pins or reclaiming firmware-visible allocations. +* Retain metadata for every firmware-owned BO and boot allocation, including + objects whose last userspace reference disappeared. The current release + path discards some of this metadata while retaining the backing memory. +* Restore the validated firmware image and run the existing fenced probe boot + sequence, including channel-table validation, host acknowledgment and + CONFIG_GET. A failed boot remains quarantined with its DMA backing held. +* Invalidate the cached program/process IDs across the restart and prevent a + stale userspace ID from aliasing a newly allocated firmware ID. Clients must + reload their programs before resubmitting. + +Those ownership and identifier changes require their own H14 reset test. +Quarantine must not be cleared solely because a power transition returned zero. diff --git a/Documentation/accel/index.rst b/Documentation/accel/index.rst index cbc7d4c3876a1..ba0f4f8dc6cef 100644 --- a/Documentation/accel/index.rst +++ b/Documentation/accel/index.rst @@ -8,6 +8,7 @@ Compute Accelerators :maxdepth: 1 introduction + apple-ane amdxdna/index qaic/index rocket/index diff --git a/Documentation/admin-guide/media/apple-ave.rst b/Documentation/admin-guide/media/apple-ave.rst new file mode 100644 index 0000000000000..d43e759481cd6 --- /dev/null +++ b/Documentation/admin-guide/media/apple-ave.rst @@ -0,0 +1,150 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Apple AVE video encoder +======================= + +The AVE block in Apple T8140 (A18 Pro) SoCs, as used in the MacBook Neo, +encodes H.264 and HEVC. ``CONFIG_VIDEO_APPLE_AVE`` builds the ``apple-ave`` +driver, a V4L2 memory-to-memory stateful encoder following +Documentation/userspace-api/media/v4l/dev-encoder.rst. The OUTPUT queue +takes NV12 frames, the CAPTURE queue returns H.264 (Baseline, Main, High) or +HEVC (Main) as Annex B. + +Usage +----- + +The encoder shows up as a ``/dev/videoN`` node named "Apple AVE video +encoder". GStreamer's ``v4l2h264enc`` and ``v4l2h265enc`` and ffmpeg's +``h264_v4l2m2m`` and ``hevc_v4l2m2m`` work with it, for example:: + + gst-launch-1.0 -e videotestsrc num-buffers=300 ! \ + video/x-raw,format=NV12,width=1920,height=1080,framerate=30/1 ! \ + v4l2h264enc extra-controls="controls,video_bitrate=8000000" ! \ + h264parse ! filesink location=out.h264 + + ffmpeg -i in.mkv -pix_fmt nv12 -c:v h264_v4l2m2m -b:v 8M -g 30 out.h264 + +Up to two streams can be encoded at the same time. + +Firmware +-------- + +The encoder runs firmware on its own coprocessor. The bootloader loads it into +two reserved, DART-mapped segments (TEXT and DATA) but does not start it. The +Aurora Stage 2 loader passes both regions in the device's ``memory-region`` +property and records their CRC32 under ``/chosen`` as +``aurora,ave-text-crc32`` and ``aurora,ave-data-crc32``. The driver only binds +while DATA still matches the recorded checksum, and keeps a copy of DATA that +it restores before every firmware boot. + +No firmware files are needed. The driver builds every command itself, +including the two large session commands (Start and Process), from the +session settings and a set of constant fields known for one firmware build: +AppleAVE2FW 9003.78.0, TEXT checksum ``69c54454``. T8140 rejects an unrecognized firmware at probe. Probe-only devices may +bind with another loader-verified build, but starting a stream fails with +``ENOTSUPP`` and a message naming the checksum until its command layout and +hardware boot sequence have been qualified. + +T6000 (M1 Pro) and T8112 (M2) have probe-only support in this driver. Their +device tree nodes and both encoder DARTs remain disabled. The firmware +handoff must reserve their preloaded TEXT (0xd0000 bytes) and DATA +(0x128000 bytes), preserve the ADT remap addresses and export both CRCs. +An enabled node with a valid handoff may bind, but encoding is refused with +an informational message until per-SoC command constants, MCC DSID and the +firmware CRCs have been qualified. The T8140 command constants are never a +fallback for these devices. Enabling a node alone does not enable encoding. + +The shared Start layout uses a smaller LFS reference surface on both +pre-M4 devices: at 1920x1080 its exact descriptor size is 0x85200 for AVC +and 0x1fe00 for HEVC, before the DMA allocation is page rounded. The +``apple-ave-sizes`` KUnit suite checks these sizes and alignment boundaries. + +T6041 (M4 Pro, including J616s reporting chip ID T6040) also has probe-only +support. Its TEXT and DATA reservations must be 0x14c000 and 0x12c000 bytes. +The existing AVE node and both DARTs remain disabled; the loader must publish +and preserve the live reservations and both pre-start CRCs before enabling +those nodes. A probe with a valid handoff reports the verified CRCs without +starting the coprocessor. An unknown layout refuses stream start before +allocating session buffers or acquiring runtime power. + +Command sizes, offsets, descriptor groups, constants, Process subtype and +buffer sizes are selected by the verified TEXT CRC, independently of the +SoC match data. Only ``69c54454`` currently selects an encoding layout. +There is no fallback layout and zero is not a wildcard CRC. Hardware boot +qualification is a separate requirement; presenting the J700 CRC on T6041 +does not authorize the J700 boot sequence. + +The driver describes the command layout observed on J616s with macOS 27.0 +26A428, including counted Start groups, packed AVC fields and sized Process +descriptors. It is deliberately absent from the CRC lookup: its TEXT CRC +and its relationship to the firmware supplied to Linux are still unknown. +The boot-config words, coprocessor start registers, DMA aperture and output +metadata/slice-header extraction also need qualification before encoding +can be enabled. The 26A428 capture and the J700 capture used different OS +versions; a SoC compatible cannot identify a command ABI. + +The public API continues to offer 8-bit NV12 and VBR/CBR. CQ and Main10 +layout facts are exercised by host tests without exposing new V4L2 modes. +The 26A428 layout has no proven fields for runtime bit-rate updates, forced +IDR or HEVC low delay. Such updates are refused rather than written at J700 +offsets. Frame rates other than 30 fps and unobserved no-reordering modes +are also refused for that layout. + +Host validation needs a C compiler with AddressSanitizer and UBSan and Python:: + + python3 drivers/media/platform/apple/ave/tests/run.py + +The tests compile the actual builder, allocation and reply functions with a +fake IPC/DMA boundary. They require no encoder, firmware or captured command +blocks. To compare all bytes of 288 synthetic J700 sessions against an older +checkout, add ``--compare-baseline /path/to/old/drivers/media/platform/apple/ave``. +Private capture comparisons must keep captured images outside the source tree +and mask only addresses, session tokens, clocks and host bookkeeping fields. + +With debugfs mounted, ``apple-ave/last_start_h264``, +``last_start_hevc``, ``last_process0_h264`` and ``last_process0_hevc`` hold the +last Start command built and the first Process command of the last session, +for comparison with reference captures. + +Sessions and power management +----------------------------- + +The firmware boots when the first stream starts and is halted when the last +one ends, as macOS does; the next stream starts it from reset. Runtime PM +powers the encoder down two seconds after the last stream. + +Up to five frames per stream are inside the firmware at once (three without +B-frames). The firmware reorders B-frames itself, so CAPTURE buffers come +back in bitstream order, each with the timestamp of the frame it codes. + +Across system sleep, open streams are drained, closed and the firmware is +halted. On resume each stream reopens on its next frame, which starts with +an IDR and fresh parameter sets; clients see no error. + +Every wait for the firmware is bounded. A command or frame that gets no +answer within 5 seconds (Start: 10 seconds) fails the stream with ``EIO``; +a firmware that stops answering keeps the encoder powered and is not used +again until reboot. The driver polls the firmware rings with sleeping waits. + +Limits +------ + +- Picture sizes from 320x240 to 2560x1600 in steps of two, up to 240 frames + per second. +- The encoder reads NV12 with a 64-byte aligned line stride, which + ``VIDIOC_S_FMT`` reports. Clients that fill buffers with a smaller line + size (ffmpeg does, for widths that are not a multiple of 64) are detected + from the payload size and their frames copied to the encoder's layout. +- Rate control is VBR, with ``BITRATE_PEAK`` as a one-second data-rate cap + (twice the average when left at 0), or CBR. In CBR mode, frames are padded + with filler data once more than half a second of bit rate has gone unused, + as macOS does; the driver adds the filler itself if the firmware does not. + Constant quality is not offered. ``BITRATE`` may change while streaming and applies from the next + frame; ``FORCE_KEY_FRAME`` makes the next frame an IDR. +- Either three B-frames or none (``B_FRAMES`` 3 or 0). H.264 Baseline and a + GOP size of 1 never use them. +- The level control is a minimum: the driver raises it to what the size, + frame rate and bit rate need. +- Buffers are mapped through the encoder's 256 MiB DMA window, so both queues + are limited to what fits next to the encoder's working memory (fewer + buffers at larger sizes). diff --git a/Documentation/admin-guide/media/apple-jpeg.rst b/Documentation/admin-guide/media/apple-jpeg.rst new file mode 100644 index 0000000000000..35cca8b11d9de --- /dev/null +++ b/Documentation/admin-guide/media/apple-jpeg.rst @@ -0,0 +1,54 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +:orphan: + +Apple JPEG scaffold +=================== + +This driver provides a common platform and V4L2 memory-to-memory scaffold +for the Apple T6041 and T8140 JPEG engines. Each DT platform device owns one +independent m2m scheduler, with formats and VB2 queues allocated per open file. +T6041's nodes reside in ``t6040.dtsi`` in this source tree. All JPEG engine +nodes and the new T8140 DART remain disabled. SoC DART defaults are disabled; +existing board overrides are preserved. In particular, J700 already enables +its first JPEG DART independently of the disabled codec engines. + +No target HWREV is qualified. Runtime resume reads HWREV with the power domain +active and returns ``-ENODEV`` for an unlisted revision, including zero and +all-ones readbacks. Both per-compatible allow-lists are deliberately empty. +The driver does not infer a revision from a compatible, an instance number, +an ADT hw-type, or a revision observed on another SoC. + +Identification precedes reset: the public reset sequence has not been qualified +on these targets, so issuing it before admission would undermine the refusal +boundary. There are no MMIO writes, interrupt requests, reset sequences or DMA +jobs in this scaffold. Probe failure balances runtime PM and never registers +a video node. The power domain and the IOMMU are managed by their normal kernel +providers; the scaffold requires a SID-0 IOMMU attachment and the binding's +exact high DMA window. This is a topology check, not proof of pointer aliasing. + +The future video node has JPEG/NV12M format negotiation for encode and decode, +buffer allocation/import, and queue teardown. STREAMON returns ``-EOPNOTSUPP`` +and returns queued buffers according to the VB2 failure contract. The scheduler +also refuses jobs. The advertised allocation dimensions and formats are draft +API scaffolding, not a tested codec capability. Open video files retain the +m2m/device allocation through final close after platform removal; they cannot +access removed MMIO resources. + +Before enabling nodes or adding a target revision, qualification must establish +the revision, bounded reset/idle sequence, interrupt acknowledgment/deassertion, +high DVA alias, timeout/recovery and supported geometry on both instances of +each SoC. Filling an HWREV list alone does not implement streaming. Reset and +job code must validate revisions again on every runtime resume, preserve the +shared power-domain accounting, and return DMA buffers only after quiescing +hardware. Encode/decode controls, JPEG header validation, media-controller +topology and actual job programming belong to the subsequent codec work. + +The HWREV register offset comes from the MIT-licensed m1n1 register +description at commit ``f3ac615eb1376d3bbba24cd84c414b9e41892adb``: + +https://github.com/aurora-silicon/m1n1/blob/f3ac615eb1376d3bbba24cd84c414b9e41892adb/proxyclient/m1n1/hw/jpeg.py + +The scaffold deliberately does not translate the experiment's unqualified +reset or interrupt writes. ``VIDEO_APPLE_JPEG_KUNIT_TEST`` tests empty target +allow-lists, invalid readbacks and exact matching with synthetic revisions. diff --git a/Documentation/admin-guide/media/index.rst b/Documentation/admin-guide/media/index.rst index b11737ae6c046..65aea88dac7ed 100644 --- a/Documentation/admin-guide/media/index.rst +++ b/Documentation/admin-guide/media/index.rst @@ -43,6 +43,7 @@ Documentation/process/debugging/media_specific_debugging_guide.rst v4l-drivers dvb-drivers + apple-ave **Copyright** |copy| 1999-2020 : LinuxTV Developers diff --git a/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml b/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml index a7339054136a8..383286892b97e 100644 --- a/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml +++ b/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml @@ -21,6 +21,9 @@ properties: - apple,t8132-display-subsystem - const: apple,t8132-display-diagnostics deprecated: true + - items: + - const: apple,t8140-display-subsystem + - const: apple,display-subsystem - items: - const: apple,t8132-display-subsystem - const: apple,t8132-display-diagnostics diff --git a/Documentation/devicetree/bindings/display/simple-framebuffer.yaml b/Documentation/devicetree/bindings/display/simple-framebuffer.yaml index 45ffdebc9d86b..66e7347805bdd 100644 --- a/Documentation/devicetree/bindings/display/simple-framebuffer.yaml +++ b/Documentation/devicetree/bindings/display/simple-framebuffer.yaml @@ -118,6 +118,8 @@ properties: - x8r8g8b8 - x8b8g8r8 + post-init-providers: true + display: $ref: /schemas/types.yaml#/definitions/phandle description: Primary display hardware node diff --git a/Documentation/devicetree/bindings/dma/apple,t8140-piodma-diagnostic.yaml b/Documentation/devicetree/bindings/dma/apple,t8140-pcie-bootstrap.yaml similarity index 53% rename from Documentation/devicetree/bindings/dma/apple,t8140-piodma-diagnostic.yaml rename to Documentation/devicetree/bindings/dma/apple,t8140-pcie-bootstrap.yaml index a6a8d7d3af77a..2db00ecb45a21 100644 --- a/Documentation/devicetree/bindings/dma/apple,t8140-piodma-diagnostic.yaml +++ b/Documentation/devicetree/bindings/dma/apple,t8140-pcie-bootstrap.yaml @@ -1,23 +1,31 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- -$id: http://devicetree.org/schemas/dma/apple,t8140-piodma-diagnostic.yaml# +$id: http://devicetree.org/schemas/dma/apple,t8140-pcie-bootstrap.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple T8140 experimental PCIe PIODMA bootstrap engine +title: Apple T8140 PCIe bootstrap engine maintainers: - Ace description: | - The J700 diagnostic driver uses this engine for a single retained request - before enabling native downstream ECAM access. It does not expose generic - DMA channels. This experimental binding describes that bounded bootstrap; - it does not establish an engine reset, removal or memory reuse contract. + This engine performs a single retained config read before native downstream + ECAM access is enabled for the T8140 radio PCIe host. It does not expose + generic DMA channels or establish an engine reset or memory reuse contract. + The host selects this engine with apple,piodma; apple,pcie-host must point + back to that host. The legacy compatible is supported for existing Stage 2 + device trees. properties: compatible: - const: apple,t8140-piodma-diagnostic + oneOf: + - const: apple,t8140-pcie-bootstrap + - const: apple,t8140-piodma-diagnostic + deprecated: true + - items: + - const: apple,t8140-pcie-bootstrap + - const: apple,t8140-piodma-diagnostic reg: items: diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index 4e0b4268f3ffa..fcfe475d4dcf2 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -40,6 +40,28 @@ properties: compatible: const: apple,smc-hwmon + apple,soc-die-keys: + $ref: /schemas/types.yaml#/definitions/string-array + minItems: 1 + maxItems: 32 + items: + pattern: "^[A-Za-z0-9]{4}$" + description: + SMC FourCC keys for SoC die temperature sensors. Each key must report a + readable four-byte floating-point temperature in degrees Celsius. The + hottest reading supplies the macsmc_soc_die thermal zone. These channels + follow the temperature-* children in the listed order and are labelled + "SoC die ". Keys absent from the running firmware or with an + unsupported format are skipped. These channels do not change the indices + used by thermal-sensors references to temperature-* children. + + apple,thermal-pressure-key: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: + SMC FourCC key reporting the firmware thermal pressure level as a + readable signed 32-bit integer. This is exposed for monitoring only. + apple,discover-temperatures: type: boolean description: @@ -58,12 +80,10 @@ properties: "#cooling-cells": const: 3 + deprecated: true description: - Makes the fans cooling devices for thermal zones. The first cell is the - zero-based index of a fan, counting the fan-* child nodes in order; the - other two are the lower and upper cooling state. State 0 leaves the fan - to the SMC; states 1 to 8 run it at speeds spread evenly from its - minimum to its maximum. A fan needs all four of its speed and mode keys. + Former fan cooling-device specifier. Fan policy is handled by the SMC + firmware; the driver no longer registers fan cooling devices. patternProperties: "^current-[A-Za-z0-9]{4}$": @@ -95,6 +115,16 @@ patternProperties: pattern: "^[A-Za-z0-9]{4}$" description: Writeable key to enable/disable manual fan control + apple,fan-manual-control: + type: boolean + description: + The SMC firmware of this machine accepts a manual speed for this + fan from the host and returns it to firmware control when asked, + so the fan's target speed can be set from the OS. Requires + apple,fan-target and apple,fan-mode. Without it the fan is only + monitored, and driven by the thermal framework where the SMC + node is a cooling device. + "^power-[A-Za-z0-9]{4}$": $ref: "#/$defs/sensor" unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/iio/imu/apple,aop-motion.yaml b/Documentation/devicetree/bindings/iio/imu/apple,aop-motion.yaml new file mode 100644 index 0000000000000..e358c89ab18a3 --- /dev/null +++ b/Documentation/devicetree/bindings/iio/imu/apple,aop-motion.yaml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iio/imu/apple,aop-motion.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple AOP accelerometer and gyroscope + +maintainers: + - Ryan Murray + +description: | + The Always-On Processor of Apple laptops reads the machine's inertial + sensor and reports acceleration and angular rate as two of its services. + Each node identifies one of those services. It has no application + processor MMIO or interrupt resource of its own. + + A service starts reporting only once it has been sent the machine's three + calibration packets. The bootloader copies them from the Apple device tree + into the calibration properties below at every boot, so they are absent + from the device tree sources. Without them the sensor stays off. + +properties: + compatible: + oneOf: + - items: + - enum: + - apple,t6041-aop-accel + - apple,t8132-aop-accel + - const: apple,aop-accel + - items: + - enum: + - apple,t6041-aop-gyro + - apple,t8132-aop-gyro + - const: apple,aop-gyro + + label: + description: + Where the sensor sits, "accel-base" for an accelerometer in the base + of a laptop. + + apple,range-intrinsic-cal: + $ref: /schemas/types.yaml#/definitions/uint8-array + description: + The sensor's range calibration container, filled in by the bootloader. + + apple,nominal-extrinsic-cal: + $ref: /schemas/types.yaml#/definitions/uint8-array + description: + The calibration container for the sensor's nominal mounting, filled in + by the bootloader. + + apple,range-extrinsic-cal: + $ref: /schemas/types.yaml#/definitions/uint8-array + description: + The sensor's range mounting calibration container, filled in by the + bootloader. + +required: + - compatible + +additionalProperties: false + +examples: + - | + accel { + compatible = "apple,t6041-aop-accel", "apple,aop-accel"; + label = "accel-base"; + }; diff --git a/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml b/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml index e039ebf276123..19e22d47a2db4 100644 --- a/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml +++ b/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml @@ -188,6 +188,23 @@ properties: additionalProperties: false description: The trackpad haptic actuator. + properties: + apple,click-strength: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + Strengths at which the actuator firmware plays its own click + waveforms while it is device-controlled, first on press and then on + release. The coprocessor firmware starts with a weak placeholder + strength, so the host writes these over it. When absent, the + firmware's strength is kept. + items: + - description: Strength of the press click. + minimum: 1 + maximum: 255 + - description: Strength of the release click. + minimum: 1 + maximum: 255 + tp_accel: type: object additionalProperties: false @@ -275,6 +292,7 @@ examples: }; actuator { + apple,click-strength = <0xc0 0x6e>; }; tp_accel { diff --git a/Documentation/devicetree/bindings/media/apple,jpeg.yaml b/Documentation/devicetree/bindings/media/apple,jpeg.yaml new file mode 100644 index 0000000000000..5e31390cf9639 --- /dev/null +++ b/Documentation/devicetree/bindings/media/apple,jpeg.yaml @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/apple,jpeg.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Silicon JPEG codec + +maintainers: + - Ryan Murray + +description: | + Apple Silicon JPEG engines are directly programmed memory-to-memory codecs. + Each engine has a dedicated interrupt and DART stream. Multiple engines can + share a power domain. The power controller manages the gate; no separate + clock or firmware interface is required. + + The DMA window reserves one 4 GiB slice of the DART high alias. The engine + has 32-bit buffer pointer registers; use of this alias for jobs still needs + hardware qualification on each compatible. + +properties: + compatible: + enum: + - apple,t6041-jpeg + - apple,t8140-jpeg + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + apple,dma-range: + $ref: /schemas/types.yaml#/definitions/uint64-array + items: + - const: 0x10000000000 + - const: 0x100000000 + description: Device-visible DMA window, as a 64-bit base and size. + +required: + - compatible + - reg + - interrupts + - iommus + - power-domains + - apple,dma-range + +additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + jpeg@487000000 { + compatible = "apple,t6041-jpeg"; + reg = <0x4 0x87000000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&jpeg0_dart 0>; + power-domains = <&ps_jpg>; + apple,dma-range = /bits/ 64 <0x10000000000 0x100000000>; + status = "disabled"; + }; + }; diff --git a/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml b/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml index 61e10feb5df71..cfc518d4a0237 100644 --- a/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml +++ b/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml @@ -10,14 +10,16 @@ maintainers: - Miguel Cruz <7362641+Acelogic@users.noreply.github.com> description: | - The T6041 and T8140 AVE coprocessors encode video using firmware preloaded by the - bootloader. Its two DART instances translate the same stream. The + The T6000, T6041, T8112 and T8140 AVE coprocessors encode video using firmware + preloaded by the bootloader. Its two DART instances translate the same stream. The bootloader supplies two reserved firmware regions and their IOVA mappings. properties: compatible: enum: + - apple,t6000-ave - apple,t6041-ave + - apple,t8112-ave - apple,t8140-ave reg: diff --git a/Documentation/devicetree/bindings/npu/apple,ane.yaml b/Documentation/devicetree/bindings/npu/apple,ane.yaml index feae5b133ac2f..3167780b88884 100644 --- a/Documentation/devicetree/bindings/npu/apple,ane.yaml +++ b/Documentation/devicetree/bindings/npu/apple,ane.yaml @@ -15,9 +15,11 @@ description: drives the engine through its register window. On T602x and T8112 a coprocessor runs the ANE firmware and the host talks to it through an ASC mailbox. T8140 has two interrupts and one system power domain; its - coprocessor uses a different firmware interface. T6041 likewise needs a - generation-specific firmware backend; it has one interrupt and its CPU - power state supplies the engine through the ANE system domain. + coprocessor is exclave-assigned and cannot be used directly without a + Linux ownership provider. T6041 has one interrupt and preloaded firmware; + its CPU, MPM and task domains depend on the ANE system domain, and its + base domain depends on the task domain. Interrupt roles on T6041 and + T8140 are not yet established. properties: compatible: @@ -34,20 +36,34 @@ properties: reg: minItems: 1 items: - - description: Engine registers. On T602x and T8112 this is the whole - ANE block, including the coprocessor. + - description: Engine registers, including the coprocessor on firmware + managed generations - description: Power manager (PMGR) registers of the ANE - - description: ANE power state registers in the PMGR - - description: eFuse words that give the SoC revision to the ANE - firmware + - description: Power state registers or first auxiliary window + - description: Revision registers or second auxiliary window + - description: Third auxiliary window + - description: Fourth auxiliary window + - description: Fifth auxiliary window reg-names: minItems: 1 - items: - - const: engine - - const: pmgr - - const: set - - const: fuse + maxItems: 7 + + apple,performance-controller: + $ref: /schemas/types.yaml#/definitions/phandle + description: + PMP report provider that owns the ANE clock-level request interface. + Only supported on T6000. The ANE and report SRAM must remain powered + while the driver holds a request. + + interconnects: + maxItems: 1 + + interconnect-names: + const: memory + description: DRAM/fabric path; bandwidth requests come from the level OPP. + + operating-points-v2: true interrupts: minItems: 1 @@ -58,11 +74,16 @@ properties: - const: ane iommus: - maxItems: 3 + description: + Stream 0 of each of the three DARTs, followed on T6041 by stream 11 of + each, which the ANE's MPM uses. + minItems: 3 + maxItems: 6 power-domains: description: - The power states the engine needs, five on T8103 and T6000, eight on + The power states the engine needs, five on T8103, six on T6000 with + ANE level control (including PMP report SRAM), eight on T602x, seven on T8112 and one system domain on T8140. minItems: 1 maxItems: 8 @@ -76,10 +97,47 @@ properties: description: Mailbox of the ANE coprocessor memory-region: - maxItems: 1 + minItems: 1 + maxItems: 2 + description: + On T602x, the IOVA-only reservation of the firmware entry alias. + On T6041 and T8140, bootloader-provided physical TEXT and DATA preload + reservations in that order. Each reservation must have no-map, must + not be reusable, and must remain reserved until reboot. Physical + addresses vary per boot and must not be hard-coded in a board DTS. + + memory-region-names: + items: + - const: firmware-text + - const: firmware-data + + apple,firmware-iovas: + $ref: /schemas/types.yaml#/definitions/uint64-array + items: + - const: 0x10000000000 + - const: 0x100000c0000 + description: + Device addresses of the preloaded TEXT and DATA segments. Their sizes + are 0xc0000 and 0x2ac000 bytes respectively. The bootloader supplies + this property together with the physical reservations. + + apple,ane-type: + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0x110, 0x200] + description: Firmware generation identifier supplied by the bootloader + + apple,firmware-preloaded: + type: boolean + description: + Firmware has been personalized and preloaded by iBoot. The driver + must preserve its runtime patches and must not stage a disk image. + + apple,exclave-assigned: + type: boolean description: - IOVA-only reservation of the firmware entry alias in the ANE DART - address space. + The ANE belongs to an exclave. Direct access requires a separate + Linux ownership provider to acquire a quiesced device and isolate + the exclave DART contexts. Absence does not prove a handoff on T8140. required: - compatible @@ -90,6 +148,24 @@ required: - power-domains allOf: + - if: + required: + - interconnects + then: + required: + - interconnect-names + - operating-points-v2 + - if: + required: + - apple,performance-controller + then: + properties: + compatible: + const: apple,t6000-ane + power-domains: + minItems: 6 + maxItems: 6 + - if: properties: compatible: @@ -98,6 +174,56 @@ allOf: properties: interrupts: minItems: 2 + power-domains: + maxItems: 1 + apple,ane-type: + const: 0x200 + - if: + properties: + compatible: + const: apple,t8140-ane + then: + properties: + reg: + maxItems: 6 + reg-names: + minItems: 1 + items: + - const: engine + - const: pmgr + - const: aux-0 + - const: aux-1 + - const: aux-2 + - const: aux-3 + - if: + required: + - apple,firmware-iovas + then: + properties: + memory-region: + minItems: 2 + required: + - memory-region + - memory-region-names + - apple,firmware-preloaded + - if: + properties: + compatible: + enum: + - apple,t6041-ane + - apple,t8140-ane + required: + - memory-region + then: + required: + - memory-region-names + - apple,firmware-iovas + - if: + required: + - memory-region-names + then: + required: + - apple,firmware-iovas - if: properties: compatible: @@ -106,6 +232,31 @@ allOf: properties: interrupts: maxItems: 1 + reg: + minItems: 7 + reg-names: + items: + - const: engine + - const: pmgr + - const: aux-0 + - const: aux-1 + - const: aux-2 + - const: aux-3 + - const: aux-4 + iommus: + minItems: 6 + power-domains: + minItems: 5 + maxItems: 5 + apple,ane-type: + const: 0x110 + required: + - apple,ane-type + - apple,firmware-preloaded + else: + properties: + iommus: + maxItems: 3 - if: properties: compatible: @@ -115,24 +266,33 @@ allOf: - apple,t6041-ane then: properties: - reg: - maxItems: 1 - reg-names: - maxItems: 1 interrupt-names: false iommus: minItems: 3 - power-domains: - maxItems: 1 resets: false mboxes: false - memory-region: false else: properties: interrupts: maxItems: 1 power-domains: minItems: 5 + reg: + maxItems: 4 + reg-names: + minItems: 1 + items: + - const: engine + - const: pmgr + - const: set + - const: fuse + memory-region: + maxItems: 1 + memory-region-names: false + apple,firmware-iovas: false + apple,ane-type: false + apple,firmware-preloaded: false + apple,exclave-assigned: false required: - interrupt-names @@ -150,7 +310,7 @@ allOf: reg-names: maxItems: 1 power-domains: - maxItems: 5 + maxItems: 6 resets: false mboxes: false memory-region: false diff --git a/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dcs.yaml b/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dcs.yaml new file mode 100644 index 0000000000000..28315f6f799c4 --- /dev/null +++ b/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dcs.yaml @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/opp/operating-points-v2-apple-dcs.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T600x measured DCS bandwidth operating points + +maintainers: + - Ryan Murray + +allOf: + - $ref: opp-v2-base.yaml# + +description: | + Each OPP maps a verified raw PMGR desired performance state to measured + sustainable bandwidth. Values are decimal kB/s, not DRAM clock frequencies, + theoretical peak bandwidth or firmware policy thresholds. State codes and + bandwidths must strictly increase together. The highest OPP must match the + boot firmware's desired state. Unmeasured intermediate states may be omitted. + +properties: + compatible: + const: operating-points-v2-apple-dcs + +patternProperties: + '^opp(-?[0-9]+)*$': + type: object + properties: + opp-supported-hw: false + turbo-mode: false + opp-level: + description: Verified raw PMGR desired state, without a state-index bias + minimum: 7 + maximum: 15 + opp-peak-kBps: + description: Measured sustainable bandwidth at this raw state + items: + - minimum: 1 + required: + - opp-level + - opp-peak-kBps + unevaluatedProperties: false + +required: + - compatible + +unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dram.yaml b/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dram.yaml new file mode 100644 index 0000000000000..76ae3c6f875ae --- /dev/null +++ b/Documentation/devicetree/bindings/opp/operating-points-v2-apple-dram.yaml @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/opp/operating-points-v2-apple-dram.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple firmware-managed DRAM performance states + +maintainers: + - Ryan Murray + +description: | + DCS firmware state IDs are not clock rates. Describe every state with an + explicit opp-level, independent of node order. Rates are optional except + for the highest state. The maximum interface rate and data width are + inferred from the interface topology and published bandwidth class. + + Lower-state rates derived from firmware bandwidth policy thresholds are + inferences, not measured clocks. Those thresholds use Q16.16 decimal GB/s; + they are not a frequency table. Leave lower-state rates absent until the + inference is accepted or independently confirmed by a public decoder. + + opp-hz is CK-equivalent Hz, rounded upwards to the nearest integer, one + half of the interface transfer rate. It does not claim a literal command + clock or LPDDR5 WCK. These OPPs describe firmware states for reporting; + they do not authorize Linux to change clocks, voltages, or firmware votes. + +allOf: + - $ref: opp-v2-base.yaml# + +properties: + compatible: + const: operating-points-v2-apple-dram + + opp-shared: true + +patternProperties: + '^opp(-?[0-9]+)*$': + type: object + additionalProperties: false + + properties: + opp-level: + minimum: 0 + maximum: 31 + + opp-hz: + maxItems: 1 + + apple,data-rate: + $ref: /schemas/types.yaml#/definitions/uint64 + minimum: 1 + description: Interface transfers per second per data pin, rounded to an integer. + + required: + - opp-level + + dependencies: + apple,data-rate: + - opp-hz + opp-hz: + - apple,data-rate + +required: + - compatible + +additionalProperties: false + +examples: + - | + opp-table-dram { + compatible = "operating-points-v2-apple-dram"; + opp-shared; + opp-0 { + opp-level = <0>; + }; + opp-1 { + opp-level = <1>; + opp-hz = /bits/ 64 <3200000000>; + apple,data-rate = /bits/ 64 <6400000000>; + }; + }; diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 02120d1934339..921624dfb9d7a 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -123,7 +123,11 @@ properties: apple,pciec-kernel-init: type: boolean - description: The kernel initializes the tunneled controller from reset. + description: | + The kernel initializes the tunneled controller when firmware has not + already initialized it. The compatible selects the initialization + sequence. T600x and T8112 use the Intr2AXI bridge and require tunables + for their associated DART before the tunnel children can be populated. apple,tunable-rc: $ref: '#/$defs/tunable' diff --git a/Documentation/devicetree/bindings/power/apple,pmp-dram.yaml b/Documentation/devicetree/bindings/power/apple,pmp-dram.yaml new file mode 100644 index 0000000000000..8c0cc99eb2177 --- /dev/null +++ b/Documentation/devicetree/bindings/power/apple,pmp-dram.yaml @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/power/apple,pmp-dram.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple PMP DRAM reporting properties + +maintainers: + - Ryan Murray + +description: | + Static description of the firmware-managed DRAM interface. These properties + reside on the existing PMP report provider, which is available independently + of a Linux PMP firmware start. No extra MMIO resource or ownership change is + implied. The bus width and maximum data rate are inferred physical classes. + + The observed LPDDR5 family string does not distinguish LPDDR5X. Do not + report an inferred X suffix as an observed memory type. Bandwidth is in + decimal bytes per second; usable data width excludes ECC/check bits. + + The OPP table must contain exactly apple,dcs-state-count available entries, + with unique opp-level values covering zero through count minus one. Only + the highest state must have a rate. Peak bandwidth must equal that state's + apple,data-rate multiplied by apple,data-bus-width divided by eight. + The published bandwidth class may be rounded (for example 200 GB/s for + an inferred physical peak of 204.8 GB/s). + + This description does not identify an achieved-state register. PMGR desired + states and PMC floor/ceiling votes are not achieved memory states. The + T6000/T8112 PMP DRAM State/FxS reporter has bucket 3*S+M, where S is DCS + state and M is self-refresh (0), power-down (1), or active (2). That encoding + alone does not specify a transport or a readable register resource. No + current clock may be claimed from a low-power bucket or unknown state. + +select: false + +properties: + operating-points-v2: + description: Reference to an operating-points-v2-apple-dram table. + maxItems: 1 + + apple,dram-type: + $ref: /schemas/types.yaml#/definitions/string + const: lpddr5 + description: Observed DRAM family, without an inferred LPDDR5X suffix. + + apple,data-bus-width: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 8 + multipleOf: 8 + description: Usable data bits, excluding ECC/check bits. + + apple,dcs-state-count: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 32 + description: Number of firmware DCS states. + + apple,peak-bandwidth-bytes-per-second: + $ref: /schemas/types.yaml#/definitions/uint64 + minimum: 1 + description: Physical maximum bandwidth calculated from the highest OPP and data width. + + apple,bandwidth-class-bytes-per-second: + $ref: /schemas/types.yaml#/definitions/uint64 + minimum: 1 + description: Published, potentially rounded, maximum bandwidth class. + +dependencies: + operating-points-v2: + - apple,dram-type + apple,data-bus-width: + - apple,dram-type + apple,dcs-state-count: + - apple,dram-type + apple,peak-bandwidth-bytes-per-second: + - apple,dram-type + apple,bandwidth-class-bytes-per-second: + - apple,dram-type + +allOf: + - if: + required: + - apple,dram-type + then: + required: + - operating-points-v2 + - apple,data-bus-width + - apple,dcs-state-count + - apple,peak-bandwidth-bytes-per-second + - apple,bandwidth-class-bytes-per-second + +additionalProperties: true diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml index e79ea331865b5..c3baafe00064c 100644 --- a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/power/apple,t8140-pmp-report.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple T8140 PMP report aperture +title: Apple PMP report aperture maintainers: - Ryan Murray @@ -18,14 +18,46 @@ description: | documented report entries may be read; the aperture is not a contiguous readable array of registers. +allOf: + - if: + properties: + compatible: + const: apple,t6000-pmp-v2-report + then: + properties: + reg: + maxItems: 2 + else: + properties: + reg: + maxItems: 1 + reg-names: false + - if: + properties: + reg: + minItems: 2 + then: + required: + - reg-names + + - $ref: apple,pmp-dram.yaml# + properties: compatible: enum: + - apple,t6000-pmp-v2-report - apple,t6041-pmp-v2-report + - apple,t8112-pmp-v2-report - apple,t8140-pmp-v2-report reg: - maxItems: 1 + minItems: 1 + maxItems: 2 + + reg-names: + items: + - const: report + - const: ane-dvfs nonposted-mmio: true @@ -38,6 +70,13 @@ properties: '#size-cells': const: 0 + operating-points-v2: true + apple,dram-type: true + apple,data-bus-width: true + apple,dcs-state-count: true + apple,peak-bandwidth-bytes-per-second: true + apple,bandwidth-class-bytes-per-second: true + apple,pmp: $ref: /schemas/types.yaml#/definitions/phandle description: @@ -57,9 +96,12 @@ patternProperties: properties: compatible: oneOf: + - const: apple,t6000-pmp-v2-report-entry - const: apple,t8140-pmp-v2-report-entry - items: - - const: apple,t6041-pmp-v2-report-entry + - enum: + - apple,t6041-pmp-v2-report-entry + - apple,t8112-pmp-v2-report-entry - const: apple,t6000-pmp-v2-report-entry reg: diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml index 588100222f892..e13f8d8394d21 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml @@ -14,6 +14,11 @@ description: | via a different "misc" control block. This includes the fabric and memory controller states. + When the host owns these registers and PMP firmware is disabled, an optional + measured bandwidth OPP table enables one shared interconnect provider on + T600x. Endpoint 0 represents clients and endpoint 1 represents memory. All + clients use the path from 0 to 1, so their bandwidth requests aggregate. + properties: compatible: enum: @@ -28,11 +33,35 @@ properties: - const: fabric-ps - const: dcs-ps + '#interconnect-cells': + const: 1 + + operating-points-v2: + description: | + Measured sustainable bandwidth table following + operating-points-v2-apple-dcs.yaml. Only valid with host ownership of DCS; + do not enable PMP firmware together with this interface. + maxItems: 1 + required: - compatible - reg - reg-names +dependentRequired: + operating-points-v2: ['#interconnect-cells'] + '#interconnect-cells': [operating-points-v2] + +allOf: + - if: + properties: + compatible: + const: apple,t6020-pmgr-misc + then: + properties: + operating-points-v2: false + '#interconnect-cells': false + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t6041-aop.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t6041-aop.yaml index e8a2df805059a..6f31a022c6f35 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,t6041-aop.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,t6041-aop.yaml @@ -86,6 +86,18 @@ properties: required: - compatible + accel: + type: object + $ref: /schemas/iio/imu/apple,aop-motion.yaml# + description: The accelerometer service. + unevaluatedProperties: false + + gyro: + type: object + $ref: /schemas/iio/imu/apple,aop-motion.yaml# + description: The gyroscope service. + unevaluatedProperties: false + required: - compatible - reg @@ -120,5 +132,13 @@ examples: las { compatible = "apple,t6041-aop-las", "apple,aop-las"; }; + + accel { + compatible = "apple,t6041-aop-accel", "apple,aop-accel"; + }; + + gyro { + compatible = "apple,t6041-aop-gyro", "apple,aop-gyro"; + }; }; }; diff --git a/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml b/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml index 9b4e9328d9020..b511202f35c7e 100644 --- a/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,t6030-dpaudio.yaml @@ -4,29 +4,36 @@ $id: http://devicetree.org/schemas/sound/apple,t6030-dpaudio.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple T6030 DisplayPort audio +title: Apple T6030 and T6041 DisplayPort audio maintainers: - Ryan Murray description: | - The external display processor provides the HDMI port's audio service. + The external display processor provides a DisplayPort/HDMI audio service. Its DisplayPort audio interface has no application-processor registers. Samples arrive through SIO, and the graph connects this interface to port 0 of the display processor that owns the audio service. + T6041 interfaces remain disabled until a generation-specific SIO firmware + backend and loader handoff are available and display audio is qualified. + select: properties: compatible: contains: - const: apple,t6030-dpaudio + enum: + - apple,t6030-dpaudio + - apple,t6041-dpaudio required: - compatible properties: compatible: items: - - const: apple,t6030-dpaudio + - enum: + - apple,t6030-dpaudio + - apple,t6041-dpaudio - const: apple,dpaudio dmas: diff --git a/Documentation/driver-api/apple-pmgr-dcs.rst b/Documentation/driver-api/apple-pmgr-dcs.rst new file mode 100644 index 0000000000000..edf6fc4af52f5 --- /dev/null +++ b/Documentation/driver-api/apple-pmgr-dcs.rst @@ -0,0 +1,55 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +Apple T600x DCS bandwidth votes +============================== + +The PMGR miscellaneous driver can provide one shared interconnect path for +host-owned T600x DCS. Every client describes the path ``<&pmgr_misc 0>`` to +``<&pmgr_misc 1>`` and uses the usual interconnect consumer API. Average +requests sum with saturation; peak requests take the maximum. The larger +aggregate selects the lowest measured OPP that can satisfy it. Requests above +the largest OPP fail with ``-ERANGE`` instead of silently under-voting. + +The provider is enabled only by an explicit ``operating-points-v2`` phandle +and ``#interconnect-cells = <1>`` on the existing PMGR miscellaneous node. Its +table uses ``operating-points-v2-apple-dcs``. Each enabled OPP supplies a raw +``opp-level`` and one ``opp-peak-kBps`` value. The driver validates duplicate +states, zero or decreasing capacities, the T600x sleep floor of 7, and the +boot-state ceiling before publishing the provider. It does not derive states +from child order or convert firmware thresholds to bandwidth. Existing DTs +without this table retain their boot and suspend policy. + +The highest table entry must equal the boot desired state. That entry remains +the initial vote until interconnect sync_state releases boot constraints. +Nonzero bandwidth also holds the fabric at its boot state so untracked DMA +clients cannot lose their fabric bandwidth to the idle governor. Requests +change only the desired low nibble and wait for the actual field to +confirm the requested state. A timeout restores the previous confirmed vote. +If restoration also times out, the fabric hold remains asserted and subsequent requests and system sleep fail with +``-EIO`` until reboot. Errors propagate to consumers, which must not start work requiring a +bandwidth request that failed. Firmware ownership and system sleep reject +requests with ``-EBUSY``. Suspend keeps the existing T600x floor; resume +restores the last successful aggregate request. + +Enabling PMP and the host interconnect provider together is rejected. This +interface neither starts PMP nor reclaims a controller from running firmware. +No module parameter supplies bandwidth data or changes ownership. + +Qualification before adding a board table +---------------------------------------- + +No board table is supplied here. A DRAM transfer-rate estimate or a PMP +policy threshold does not establish sustainable client bandwidth. Before +adding one, establish the raw desired/actual state mapping for the exact SoC +and firmware, verify each selected state's fabric dependencies, and measure +sustained read, write and copy bandwidth under controlled CPU and DMA load. +Use a conservative capacity supported by repeated measurements, with the +workload, sample duration, thermal conditions and variance recorded in the +qualification report. Do not publish private target records in source. + +Qualify concurrent consumers, request release, failed raises and rollback, +suspend/resume and idle/load power through the hardware controller. Keep a +bootguard fallback and validate the default policy before enabling any board +DT. The KUnit suite uses synthetic numbers and memory-backed registers; it +validates software behavior only, not state encodings, rates, firmware +ownership or hardware completion. diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst index edb93a23542c1..ee2405d0b8e95 100644 --- a/Documentation/driver-api/index.rst +++ b/Documentation/driver-api/index.rst @@ -104,6 +104,7 @@ Subsystem-specific APIs infiniband input interconnect + apple-pmgr-dcs ipmb ipmi issei/index diff --git a/Documentation/driver-api/media/drivers/apple-isp-j616s.rst b/Documentation/driver-api/media/drivers/apple-isp-j616s.rst index 9a2bf93d7b059..26529b211b79c 100644 --- a/Documentation/driver-api/media/drivers/apple-isp-j616s.rst +++ b/Documentation/driver-api/media/drivers/apple-isp-j616s.rst @@ -41,6 +41,47 @@ firmware-derived output modes still need capture validation on 26.6.2. Only our source and synthetic protocol tests are included; no Apple binary, setfile contents, calibration table or firmware image is added. +Sensor tuning and noise reduction +--------------------------------- + +NV12 setup loads model-wide sensor tuning using ``CH_SET_FILE_LOAD`` +(0x0111). T8140/IMX558 selects ``apple/isp_1925_03XX.dat``; T6040/IMX958 +selects ``apple/isp_2224_01XX.dat``. The T6040 lens-shading fixup is applied +only to the copied firmware buffer. A missing file allows streaming with a +warning. An invalid file, interrupted read or failed firmware upload fails +stream setup instead of silently continuing. A debug message identifies a +successful upload without dumping its contents. +This confirms transport completion only: the NV12 path does not decode the +setfile reply fields, so it does not prove that every coefficient was applied. + +The NV12 command sequence already sends ``CH_CNR_START`` (0x0a2f), +``CH_MBNR_ENABLE`` (0x0a3a, use case 0, mode 1, chroma enabled), +``APPLE_CH_TEMPORAL_FILTER_START`` (0xc100), +``APPLE_CH_MOTION_HISTORY_START`` (0xc102) and +``APPLE_CH_TEMPORAL_FILTER_ENABLE`` (0xc113). The temporal-start record has +one fixed argument of 1 and another supplied by ``apple,temporal-filter`` +(default 0; J700 supplies 1). The latter's meaning is not established here; +it must not be treated as a boolean enable or an output-pool count. +Command submission alone does not establish equivalence to a host-side +motion-compensated filter or measure the firmware's resulting noise reduction. + +The P010 preview still uses its qualified minimal sequence, which skips +these explicit NR commands and setfile upload. An earlier NV12 stream may +have left tuning in the resident firmware. Compare cold-boot P010 with +NV12-to-P010 transitions when assessing image quality. + +Sensor setfiles are distinct from individual camera-module factory +calibration (iCCl). The driver has no iCCl parser, serial binding or upload +command. The installer preserves raw factory records in the local fallback +archive; this does not mean the ISP consumes them. No factory record or +vendor payload belongs in the kernel source or a distributed firmware package. + +The separate ANE video denoiser is not implemented. The board parameter facts +are auto mode 2, total-gain entry/exit thresholds 2304/1792 on J700 and +3584/2816 on J616s, an FPS threshold of 30 and a 30-frame ramp-down. The FPS +comparison direction, gain units and ANE/ISP interchange need a confirmed +interface before implementation. + Public source audit ------------------- @@ -320,3 +361,54 @@ The T6040 device tree's current fixed TEXT/DATA reservations should eventually come from a bootloader handoff describing the loaded image and mapping lifetimes. They are firmware-build-specific. This change does not implement that handoff or replace those reservations with 25G76 addresses. + +Capture size enumeration +------------------------ + +The shared driver preserves the order of its device-tree presets and any +validated firmware output modes. It then adds common video-call sizes that +fit within those presets' processed crop extents. The added preset retains +its source firmware configuration and pixel-format eligibility. Crops are +centred on even coordinates, have the exact output aspect ratio and never +require upscaling. Sensor input dimensions alone are not a processed output +capability: input rows can contain stacked exposures. + +The common sizes include 640x480, 1280x720, 1920x1080, 1080x1920, +1760x1328, 1328x1760, 1552x1552, 1280x960 and 960x1280. A sensor that +cannot provide the larger square also gets the largest even square that +fits its established crops. Existing sizes are not duplicated. This is a +shared output policy, not a list indexed by board or sensor model. + +Neither the crop nor output-configuration command reports scaler limits +or rotation support. The driver does not infer a larger processed image +from raw input dimensions, rotate a portrait image into a landscape one, +or advertise a size that requires enlargement. Consequently, an exact +macOS size may remain unavailable even though a smaller size with the +same aspect ratio can be offered. + +Read-only inspection on the running kernel:: + + v4l2-ctl -d /dev/video0 --list-formats-ext + v4l2-ctl -d /dev/video0 --log-status + dmesg | grep -A20 'sensor config' + +The status ioctl prints already cached firmware configuration replies; +it does not start capture or send additional firmware queries. The cache +is currently populated only on the existing supported P010 discovery path. +A missing cache is not evidence that the sensor has no other modes. + +After booting a test kernel, enumerate again and qualify each added size +that is actually offered. For example, for 640x480 NV12:: + + v4l2-ctl -d /dev/video0 \ + --set-fmt-video=width=640,height=480,pixelformat=NV12 \ + --get-fmt-video --stream-mmap=4 --stream-count=30 --stream-to=/dev/null + gst-launch-1.0 -v v4l2src device=/dev/video0 num-buffers=30 ! \ + video/x-raw,format=NV12,width=640,height=480,framerate=30/1 ! \ + fakesink sync=false + +Repeat with each added width and height and, when offered, P010. Check +that the negotiated dimensions equal the requested dimensions: S_FMT may +otherwise choose a nearby preset. Confirm 30 completed frames, correct +aspect ratio and chroma, and no new ISP or DART errors. Enumeration and unit tests do not establish successful capture at a +new size; capture qualification must be performed separately. diff --git a/Documentation/gpu/apple-dcp-panel-colorimetry.rst b/Documentation/gpu/apple-dcp-panel-colorimetry.rst new file mode 100644 index 0000000000000..ff31c645115c1 --- /dev/null +++ b/Documentation/gpu/apple-dcp-panel-colorimetry.rst @@ -0,0 +1,160 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +Apple internal-panel colour metadata +==================================== + +The panel identity EDID and its colour description are separate inputs. A +checksum-valid EDID with zero RGB primaries does not describe an sRGB panel; +it describes unknown primaries. A Display P3 default profile must not be +inferred merely from the Apple vendor ID or the DCP/SoC generation. + +Sources and publication path +---------------------------- + +The existing paths in appledrm are: + +* External displays: ``epic/dpavservep.c`` obtains an EDID with the existing + DCPAV service command. ``dcp_get_modes()`` in ``iomfb.c`` publishes it. +* Desktop internal panels: the ``DisplayAttributes`` dictionary reaches + ``dcpep_process_chunks()`` in ``iomfb_template.c`` or the existing dictionary + setters in ``iomfb_v26_6_desktop.c``. ``parse_panel_metadata()`` in + ``parser.c`` decodes the description; ``dcp_panel_metadata_replace()`` in + ``dcp.c`` retains it. ``dcp_panel_edid_snapshot()`` prefers a validated raw + panel EDID and otherwise calls ``dcp_panel_edid()`` in + ``dcp-display-metadata.c``. ``dcp_get_modes()`` publishes that snapshot with + ``drm_edid_connector_update()``. The actual mode catalog remains DCP's + ``TimingElements``, rather than the EDID's identity timing summary. +* H17P internal panels currently have no supported internal EDID command + transport and do not enter the desktop identity fallback. An empty EDID on + that path is a distinct missing publication path, not evidence for P3. + +Before the optional-colour fix, the desktop parser retained only +``ProductAttributes`` and the panel builder unconditionally emitted zero RGB +primaries, nominal D65 white and unknown gamma (0xff). The shared colour +encoder now validates and encodes native primaries and white independently +of an optional gamma. Incomplete, +duplicate or malformed optional fields leave panel identity usable. A +standalone ``ProductAttributes`` update for the same panel retains its previous +colour description. A different panel serial cannot inherit that description. +Validated raw EDIDs retain precedence and are not rewritten by this fix. +There is no model/gamut table or new platform switch in this implementation. + +The firmware fields to verify are precisely: + +* ``DisplayAttributes.Chromaticity.Red.X`` and ``.Y``; +* ``DisplayAttributes.Chromaticity.Green.X`` and ``.Y``; +* ``DisplayAttributes.Chromaticity.Blue.X`` and ``.Y``; +* ``DisplayAttributes.DefaultWhitePoint.X``, ``.Y`` and optional ``.Gamma``. + +The existing parser interprets these as unsigned 16.16 values. EDID base-block +chromaticities round to units of 1/1024; gamma rounds to a hundredth. An absent +transfer characteristic remains unknown (0xff) even when valid primaries and +white can be published; it must not be replaced by an invented panel gamma. +The optional values are validated before any colour bytes are changed, +including nonzero/bounded coordinates, nondegenerate primary triangle and +white-point containment. These are metadata validation rules, not panel +calibration or a measurement of its optical output. + +Data still required +------------------ + +The current ``dcp-panel-adt.c`` reader consumes panel identity and dimensions, +not colour calibration. Neither its implementation nor the panel binding +establishes an ADT native-primary property. Do not interpret arbitrary ADT +calibration blobs, display mode colourimetry IDs, or an ICC profile's +D50-adapted matrix as native panel xy coordinates. + +First inspect the cached firmware ``DisplayAttributes`` dictionary for the +fields above. If they are absent, obtain verified native panel xy coordinates, +white and transfer characteristic through a documented ADT/firmware producer. +The fallback then belongs on the existing ``panel`` child of the DCP node, +with a binding in ``Documentation/devicetree/bindings/display/apple,dcp.yaml`` +and a shared reader alongside ``dcp_panel_read_dimensions()`` in ``dcp.c``. +A concrete proposed representation is ``apple,native-chromaticities`` (eight +u32 16.16 cells, Rx/Ry/Gx/Gy/Bx/By/Wx/Wy) plus ``apple,native-gamma`` (one +u32 16.16 cell). These names are a proposal, not supported properties in this +patch. The bootloader can populate them from verified native data; static +board DT values require an authoritative panel specification. Firmware data +should take precedence over a DT fallback. The internal metadata store and +publication path must also be shared with H17P rather than widening the +external-display service/ownership exceptions. + +There is currently no verified runtime colour payload in the investigation +that warrants populating these properties. The optional-colour fix therefore +does not assert that a particular panel now advertises P3, and it does not +manufacture a colour description when firmware omits it. + +Expected gamut and userspace +--------------------------- + +Apple specifies the 2024 16-inch MacBook Pro as wide colour P3 [1]_, but the +MacBook Neo as sRGB [2]_. Neo must not receive a P3 default. Standard Display P3 +uses native xy R (0.680, 0.320), G (0.265, 0.690), B (0.150, 0.060), white +(0.3127, 0.3290) [3]_. These standard values are useful test fixtures, not a +substitute for a particular panel's native calibration. ICC rXYZ/gXYZ/bXYZ +values expressed in a D50 connection space require the profile's chromatic +adaptation to be undone before comparison with a D65 EDID. + +Mutter 51's ``meta-color-device.c`` rejects zero primaries when generating an +EDID ICC profile, and also needs a usable gamma. Its SDR native colour mode +uses EDID primaries and transfer information; its default colour mode still +uses sRGB [4]_. Thus EDID/profile detection alone does not prove wide-gamut +output is selected. The DRM ``Colorspace`` property describes output signal +encoding, not an inventory of ICC profiles; a P3 label need not appear there. + +Fedora, Arch Linux ARM and NixOS need the same usable kernel metadata and +compatible Mutter/colord colour-management stack. With a valid native EDID, +a separate distro-supplied ICC is not inherently required just to identify +the gamut. A measured/complex transfer curve, missing firmware data, or an +older compositor can still require a userspace profile or compositor work. +No three-distro runtime parity or post-patch profile selection is established +by the host selftests. Preserve an existing calibrated profile until native +EDID and the effective compositor colour mode have been checked. + +Read-only verification +---------------------- + +Run these through the hardware controller with explicit device identity and +policy approval. Reading the EDID attribute size with stat is insufficient; +read its contents:: + + for edid in /sys/class/drm/card*-*/edid; do + bytes=$(cat "$edid" | wc -c) + printf '%s: %s bytes\n' "$edid" "$bytes" + if test "$bytes" -gt 0; then + edid-decode "$edid" + fi + done + modetest -M apple -c + colormgr get-devices-by-kind display + colormgr get-profiles + +Sysfs may report zero file size even for a populated EDID. Missing tools +should be reported; do not install packages or change a display mode to obtain +this evidence. Firmware dictionary +inspection reads only the cached ``DisplayAttributes`` debugfs file, after +controller approval. Do not enable new trace streams or copy raw panel data +into public fixtures. + +Check nonzero decoded native primaries/white, gamma, EDID checksum, the internal +connector and its unchanged mode list, then the display device's default +profile in GNOME Colour or colormgr. A successful result records whether the +profile is generated from EDID or explicitly assigned, and the effective +Mutter colour mode. Compare profiles after accounting for ICC chromatic +adaptation. Publish only sanitized colour summaries, not raw EDIDs, serials, +firmware dictionaries or calibration records. + +Host regression command (no hardware access):: + + python3 tools/testing/selftests/drivers/gpu/apple-dcp/run.py \ + --output /tmp/apple-dcp-results + +The ``panel-colorimetry`` test uses synthetic P3 and sRGB primaries with a +synthetic gamma of 2.2 through the production parser and encoder. It checks packed coordinates, gamma, checksums, +missing/malformed/duplicate fields, degenerate geometry and same-panel identity +updates. Existing external EDID and panel identity tests remain relevant. + +.. [1] https://support.apple.com/en-gb/121554 +.. [2] https://www.apple.com/macbook-neo/specs/ +.. [3] https://www.w3.org/TR/css-color-4/#predefined-display-p3 +.. [4] https://gitlab.gnome.org/GNOME/mutter/-/blob/51.0/src/backends/meta-color-device.c diff --git a/Documentation/gpu/apple-dcp-t6041-panel.rst b/Documentation/gpu/apple-dcp-t6041-panel.rst index bdf15e644d68f..b75e6584bc1d2 100644 --- a/Documentation/gpu/apple-dcp-t6041-panel.rst +++ b/Documentation/gpu/apple-dcp-t6041-panel.rst @@ -140,8 +140,11 @@ programmed mode; the original DCP sync positions and mode IDs are unchanged. This also lets a 118-line porch and 82-line notch-expanded pulse publish an identity instead of failing the base EDID size check. The EDID records model year 2024 (not a manufacture date) and a nominal D65 -reference white point required by EDID parsing. No calibrated primaries, gamma, -new mode or machine serial is claimed. The ADT reader, +reference white point required by EDID parsing. When a complete valid optional +firmware colour description is available, the shared encoder supplies its +primaries, white and gamma instead; see :doc:`apple-dcp-panel-colorimetry`. +Missing colour data does not imply calibrated primaries or gamma. No new mode +or machine serial is claimed. The ADT reader, EDID fallback and real EDID setters are restricted to desktop 26.6; T8140 continues to use its existing path. @@ -180,3 +183,32 @@ A failed transaction must publish no partial handoff. Loader host tests should cover supported DT without the chosen property, an unsupported board/DT, and each failed validation, as well as unchanged other-SoC policy. No new policy flag or kernel bypass is needed. This kernel series does not replace Stage 2. + + +Desktop external audio (draft) +------------------------------ + +The desktop audio integration uses the existing DCP PCM and native link +lifecycle, with a separate compact service binding on endpoint 0x2b. Endpoint +0x29 retains AV device control. Only the external firmware role admits the +``dcpav-audio-epic`` unit-zero service; the internal panel does not acquire an +audio client. The existing ``hdmi_audio`` opt-in still controls service startup. + +A PCM opens and pins one service instance. Prepare, Start, Stop and Complete +use that instance's element link data. Sink termination disconnects the card, +drains the DMA and completes its link before closing that instance. Replacement +services cannot inherit an old instance's Close or link completion. Compact +method replies must have the expected body length; serialized property replies +must contain their declared used bytes within the actual received body. + +J616s DPA1 and DPA2 are described with SIO channels 0x66 and 0x68, connected +bidirectionally to DCPEXT0 and DCPEXT1 port zero, respectively. Both clients +remain disabled. This description does not enable SIO, its DART, its mailbox +or any DPA power domain. A generation-specific SIO firmware backend and loader +handoff are still required before a candidate may enable the clients. + +The desktop service name/unit, shared 13.5 command numbers and native 14.7 link +and element layouts are provisional integration choices. Build and synthetic +regression results cannot qualify them against firmware. Qualification must +confirm those contracts and actual sink element catalogs before enabling these +outputs. No sink-specific rates or capabilities are synthesized here. diff --git a/Documentation/gpu/asahi-g17-capacity.rst b/Documentation/gpu/asahi-g17-capacity.rst index 68d261682f842..43c61b6364688 100644 --- a/Documentation/gpu/asahi-g17-capacity.rst +++ b/Documentation/gpu/asahi-g17-capacity.rst @@ -1,7 +1,7 @@ G17 capacity diagnostics ======================= -The read-only ``/sys/kernel/debug/asahi-g17/capacity`` file samples the live +The read-only ``/sys/kernel/debug/aurora-g17/capacity`` file samples the live firmware session. It does not wake firmware or reclaim resources. An absent session reports ``online=0``. The format is diagnostic, not a stable ABI. Counts exclude reserved identities. Sub-pool locks and atomic snapshots mean diff --git a/Documentation/gpu/asahi-usage-stats.rst b/Documentation/gpu/asahi-usage-stats.rst new file mode 100644 index 0000000000000..f5e731e77d0c2 --- /dev/null +++ b/Documentation/gpu/asahi-usage-stats.rst @@ -0,0 +1,80 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +Asahi client engine time +======================== + +Asahi reports the following optional keys in each DRM file's fdinfo, following +:doc:`drm-usage-stats`:: + + drm-engine-vertex: 123456 ns + drm-engine-fragment: 234567 ns + drm-engine-compute: 345678 ns + +``vertex`` is the tiling/vertex stage of render commands, ``fragment`` is their +fragment stage, and ``compute`` is compute dispatch. These are separate timelines: +vertex and fragment can overlap. There is no separately submitted blit engine in +the driver's command ABI. Copies executed by compute or render are included in +the engine that executes them. + +The counters belong to the open DRM file, rather than to a PID or a queue handle. +All queues and VMs of that file contribute to the same ledger. Duplicated or +inherited file descriptors share counters and must be deduplicated using +``drm-client-id``. Closing a queue or VM does not reset its file's counters, and +completions retain the ledger until their firmware work retires. A new DRM file +starts at zero, even when another file in the same process has submitted work. + +Time source and limits +---------------------- + +Only completed stage intervals are reported. A long-running command contributes +after retirement, so polling during execution can temporarily show no change. +Counters accumulate monotonically, saturating at the largest unsigned 64-bit +value. Conversion rounds down by less than one nanosecond per retired stage. + +G13 and G14 use the firmware's internal start/end timestamp cells. The existing +timestamp microsequence operations run even when the application requests no +timestamp output. Internal buffers were already allocated for each command; +fdinfo never reads an application's timestamp buffer. + +The G15 and native G16 backends reuse the internal timestamp cells already read +by their successful retirement statistics path. The shared G17 execution backend, +including T6040/G16S and T8140/G17P, charges the timestamp pairs delivered by its +once-only completion path. These backends use the firmware's 24 MHz command timer; +the classic backend uses its hardware configuration's timer frequency. + +These are firmware stage envelopes, not hardware shader occupancy measurements. +They can include firmware scheduling, preemption and stalls between the stage's +start and end operations. Intervals are summed without attempting to merge +overlapping commands on the same engine. Missing, zero-start or reversed intervals +are ignored. Failed or cancelled commands do not contribute incomplete intervals; +successful stages that retired before a later failure remain counted. An +unexecuted G17 replay contributes only when it eventually retires successfully. +There is no submit-to-completion wall-time fallback in the current backends. + +``drm-cycles-*`` and ``drm-maxfreq-*`` are intentionally absent. The timestamp +clock is independent of the variable GPU execution clock. Treating command timer +ticks as GPU busy cycles, or multiplying time by a sampled/maximum GPU frequency, +would give incorrect frequency-weighted utilization. Per-client execution-cycle +counters would require a separate hardware measurement source. + +Accounting cost +--------------- + +Each file allocates one three-counter ledger. Queues and pending completions +retain references to it. Retirement adds one relaxed, saturating atomic update +per valid stage, with no new locks or allocations. The native backends reuse +existing timestamp reads. Classic completion callbacks read two internal cells +per stage; commands without application timestamps also execute two existing +timestamp microsequence operations per stage. Reading fdinfo loads three counters +and does not traverse queues, wait for fences or access hardware. + +Host arithmetic tests +--------------------- + +The ledger uses only ``core`` and can be tested directly from a source checkout:: + + rustc --edition 2021 --test drivers/gpu/drm/asahi/usage.rs -o /tmp/asahi-usage-test + /tmp/asahi-usage-test + +Tests cover conversion, missing/reset timestamps, saturation, overlapping stages, +client isolation, out-of-order completions and simultaneous queues sharing a file. diff --git a/Documentation/gpu/drivers.rst b/Documentation/gpu/drivers.rst index ced2721b96c75..38f99ddda7c83 100644 --- a/Documentation/gpu/drivers.rst +++ b/Documentation/gpu/drivers.rst @@ -7,8 +7,10 @@ GPU Driver Documentation amdgpu/index apple-dcp-backlight + apple-dcp-panel-colorimetry apple-drm-parameters asahi-reclaim + asahi-usage-stats asahi-m4-bringup apple-dcp-m4 i915 diff --git a/Documentation/hwmon/macsmc-hwmon.rst b/Documentation/hwmon/macsmc-hwmon.rst index 6903f76df62bf..b0c78c3412088 100644 --- a/Documentation/hwmon/macsmc-hwmon.rst +++ b/Documentation/hwmon/macsmc-hwmon.rst @@ -22,10 +22,33 @@ the desktop Macs), sensors present on any given machine are described via Devicetree. The driver picks these up and registers them with hwmon when probed. -Manual fan speed is supported via the fan_control module parameter. This -is disabled by default and marked as unsafe, as it cannot be proven that -the system will fail safe if overheating due to manual fan control being -used. +Manual fan speed is available on the fans whose Devicetree node has +apple,fan-manual-control, which a machine's Devicetree sets once its SMC +firmware is known to accept a manual speed and to take the fan back. On +those fans fanX_target is writable; writing 0 hands the fan back to the +firmware. The fan_control module parameter, disabled by default and marked +as unsafe, allows manual speeds on every fan with a mode key whatever the +Devicetree says. It cannot be proven that the system will fail safe if it +overheats under a manual fan speed. + +The driver leaves fan policy to the SMC until userspace writes a nonzero +``fanX_target``. Thermal zones do not take fans into manual mode. Suspend, +shutdown and removal release only fans taken over by such a request; resume +does not reapply it. A new target write is required after resume. + +SoC die temperatures are described by the ``apple,soc-die-keys`` string list +on the hwmon device node. The driver appends these temperature channels after +the ``temperature-*`` children, in list order, with labels ``SoC die ``. +Only readable four-byte floating-point keys are used. When thermal support is +enabled, their hottest reading supplies the ``macsmc_soc_die`` thermal zone. +A failed or implausible reading fails the zone read rather than reporting a +partial result. Keys absent from the firmware are skipped. + +The optional ``apple,thermal-pressure-key`` on the same node describes a +readable signed 32-bit firmware thermal pressure level. Neither feature needs +a module parameter. Kernels using these properties need an updated device +tree; the former downstream per-board fallback and its ``soc_die_keys`` and +``board_keys`` parameters have been removed. sysfs interface --------------- @@ -69,3 +92,7 @@ tempX_input tempX_label Temperature sensor label +thermal_pressure_level + Firmware thermal pressure level, exposed read-only when the device tree + supplies a valid key. The integer is firmware-defined and is not a + temperature or a percentage. diff --git a/Documentation/iio/aop-sensors.rst b/Documentation/iio/aop-sensors.rst new file mode 100644 index 0000000000000..6c3225024a82a --- /dev/null +++ b/Documentation/iio/aop-sensors.rst @@ -0,0 +1,94 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +================= +Apple AOP sensors +================= + +The AOP firmware announces sensor services over EPIC. The AOP core matches +each service to a child node in the device tree and forwards its fake-HID +reports to the corresponding IIO driver. Ambient light (``als``), lid +angle (``las``), acceleration (``accel``) and angular rate (``gyro``) use +this same transport; no HID bus device is created. + +Lid angle +========= + +Enable ``CONFIG_APPLE_AOP`` and ``CONFIG_IIO_AOP_SENSOR_LAS``. The AOP must +have a ``las`` child with a SoC-specific compatible followed by +``apple,aop-las``. The T6040 device tree already supplies this child for +J616s, and ``asahi.config`` enables the driver. The bootloader must hand +over the AOP firmware memory regions before the AOP can start. + +The device is named ``aop-sensors-las``. Its ``in_angl_raw`` attribute is +the latest valid report-1 angle, in whole degree counts from 0 to 360. +Multiply it by ``in_angl_scale`` (0.017453293) to obtain radians as +specified by the IIO angle ABI. The scale is fixed and can be read before +the first sample; a raw read before the first valid sample returns +``ENODATA``. Other report IDs, truncated reports and out-of-range angles +leave the last valid reading unchanged. + +Report 1 carries a 9-bit little-endian value following the report ID. +The remaining bits in the last byte are padding. Reads return the cached +unsolicited report, rather than issuing a synchronous HID feature request. + +Accelerometer and gyroscope +=========================== + +Enable ``CONFIG_IIO_AOP_SENSOR_MOTION``. The AOP must have ``accel`` and +``gyro`` children with a SoC-specific compatible followed by +``apple,aop-accel`` or ``apple,aop-gyro``. Each service reports only after +it has been sent the machine's three calibration packets, which the +bootloader copies from the Apple device tree into the child's +``apple,range-intrinsic-cal``, ``apple,nominal-extrinsic-cal`` and +``apple,range-extrinsic-cal`` properties. Without them the driver leaves +the sensor off and does not bind. The packets are sent unchanged, then the +50 Hz report interval. + +The devices are named ``aop-sensors-accel`` and ``aop-sensors-gyro``. Each +has ``in_accel_{x,y,z}_raw`` or ``in_anglvel_{x,y,z}_raw``, a shared scale +(acceleration in m/s^2 including gravity, angular rate in rad/s) and +``sampling_frequency``. The axes are the sensor's, in the order it reports +them; the mapping to the machine's axes is not established yet. A raw read +before the first sample returns ``ENODATA``. + +Both devices have a buffer that takes X, Y and Z together, with an optional +timestamp taken when the report arrives. The reports stop when the system +goes to sleep and start again, with the calibration resent, when it wakes. + +If both devices bind but every raw read returns ``ENODATA``, check the AOP +core's ``nCal`` boot argument for the SoC. The T6040 firmware accepts the +calibration and the report interval with ``nCal`` 0, but sends no reports; +its own default, which the core keeps, is 1. + + +Host regression tests require Python 3 and a Rust compiler:: + + tools/testing/selftests/drivers/apple/aop-las.py + tools/testing/selftests/drivers/apple/aop-motion.py + +The tests compile the production decoder and shared IIO read callback +against host mocks. They cover all 16-bit payloads (including padding), +report IDs, short reports, invalid readings, the scale before a sample, +last-valid-sample behaviour and processed channels without a scale. +They do not test EPIC transport or firmware behaviour. + +On J616s, or another supported laptop: + +1. Build with ``asahi.config`` and the board's DTB. Check that the AOP, + LAS and ALS options survived configuration, and that the bootloader + supplies the AOP memory regions. +2. Boot and locate the IIO device whose ``name`` is ``aop-sensors-las``. + Check ``in_angl_scale`` is 0.017453293. Read ``in_angl_raw`` repeatedly + while moving the lid through several safe positions. Expect whole + degree counts that follow the hinge position. Do not force the hinge + beyond its mechanical limit to test values above 255; the host tests + cover that encoding. +3. Confirm the value updates after boot and after resume, and that ALS + still reports illuminance. Check the kernel log for AOP timeouts, + DART faults or warnings. Repeat on an older supported AOP platform + to check the shared receive path. +4. If raw reads keep returning ``ENODATA``, check that firmware announces + ``las`` and sends EPIC report subtype 0xc4 on that service's channel. + A readable feature report alone does not prove unsolicited reports + arrive. Establish that transport behaviour before adding any polling + or firmware setup request. diff --git a/Documentation/iio/apple-aop-als.rst b/Documentation/iio/apple-aop-als.rst new file mode 100644 index 0000000000000..a19625a84be75 --- /dev/null +++ b/Documentation/iio/apple-aop-als.rst @@ -0,0 +1,40 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +Apple AOP ambient light reporting +================================ + +The ALS driver consumes the payload of EPIC fake-HID reports, after the AOP +transport has removed the EPIC headers. These offsets do not describe macOS +host-side event objects. + +Redbird (VD6286) illuminance is the little-endian firmware float at payload +byte 0x28. Linux converts it to microlux using integer arithmetic and exports +it through ``in_illuminance_input``. The driver requests a 200000 microsecond +report interval, or 5 Hz, and loads the device's factory calibration from +``apple/aop-als-cal.bin`` when installed. Calibration is device-specific and +must not be distributed with test fixtures. + +The `public Redbird report description +`_ +identifies a 122-byte report carrying a firmware-computed CCT float at byte +0x45. The driver exports that layout as ``in_cct_input``, in Kelvin, without +recalculating colour temperature from raw channels. Other lengths do not +supply a CCT reading. This layout must be verified on each firmware family +before its CCT readings are used for display policy. The existing CT720 and +CT817 illuminance layouts have no established CCT field in this driver. + +Read both channels by finding the IIO device by name rather than assuming a +device number:: + + for d in /sys/bus/iio/devices/iio:device*; do + [ "$(cat "$d/name" 2>/dev/null)" = aop-sensors-als ] || continue + cat "$d/in_illuminance_input" + [ ! -e "$d/in_cct_input" ] || cat "$d/in_cct_input" + done + +These attributes contain the last valid reading, not a sample counter. Polling +them five times per second cannot prove a 5 Hz arrival cadence. Measure that +at the report callback or transport with tracing, and verify the payload +length and field correspondence there. Keep raw captures private. A CCT value +remaining unchanged while lux changes can also mean an unsupported payload +layout or a report with invalid CCT; it does not prove steady illumination. diff --git a/Documentation/iio/index.rst b/Documentation/iio/index.rst index b02b879b053a1..108146d2b3060 100644 --- a/Documentation/iio/index.rst +++ b/Documentation/iio/index.rst @@ -38,6 +38,8 @@ Industrial I/O Kernel Drivers adxl313 adxl380 adxl345 + aop-sensors + apple-aop-als bno055 ep93xx_adc opt4060 diff --git a/Documentation/networking/device_drivers/wifi/mt7932-j700-antenna.rst b/Documentation/networking/device_drivers/wifi/mt7932-j700-antenna.rst new file mode 100644 index 0000000000000..3884d3fc935e3 --- /dev/null +++ b/Documentation/networking/device_drivers/wifi/mt7932-j700-antenna.rst @@ -0,0 +1,56 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +J700 6 GHz antenna-gain projection +================================= + +The clean t-3573 antenna-gain SPEC (2026-10-31), sections 1, 3 and 6, +establishes these immutable board facts. They apply only to the J700/MT7932 +board admitted by the driver's existing machine, PCI function and supplier +checks. They are not generic PCI 14c3:7932 properties. No external resource, +undocumented device-tree property or module parameter is required. + +Values are signed half-dB gain offsets in SISO, CDD, MIMO order:: + + G1 16 16 10 + G2 14 14 8 + G3 15 16 10 + G4 15 15 9 + G5 16 16 10 + G6 15 16 10 + +The primary-channel/bandwidth lookup is:: + + primary 20 40 80 160 MHz + 2 1 - - - + 1..29 step 4 1 1 1 1 + 33..61 step 4 2 2 2 2 + 65..93 step 4 3 3 3 3 + 97..109 step 4 4 4 4 4 + 113 4 4 5 4 + 117 5 4 5 4 + 121..125 step 4 5 5 5 4 + 129..181 step 4 5 5 5 5 + 185 5 6 5 5 + 189 6 6 5 5 + 193..221 step 4 6 6 6 6 + 225..229 step 4 6 6 - - + 233 6 - - - + +A finite TPE cell subtracts the mapped gain independently for each mode. +The calculation uses a wider signed type and refuses unrepresentable +results. 0x7f is absence of a constraint and survives unchanged; no missing +gain is replaced by zero. Unknown primary/bandwidth mappings refuse finite +cells. The encoder covers four bandwidths; station operation remains at most +80 MHz. Country/class, SAR and hardware ceilings remain independent bounds. + +For example, channel 117 with EIRP limits (40,40,40,40) half-dBm produces +SISO and CDD (24,25,24,25), MIMO (30,31,30,31). The same vectors apply to +LPI and VLP. PSD -1 dBm/MHz converts to EIRP (24,30,36,42) half-dBm; with G1 +this produces SISO/CDD (8,14,20,26), MIMO (14,20,26,32). + +KUnit exercises these vectors, every boundary row, signed arithmetic, +sentinels, unsupported edge bandwidths, and legacy v2 VLP compatibility. +No 2.4/5 GHz mode matrix, OTA resource parser, calibration derivation or +firmware readback is established. Firmware acceptance and conducted/radiated +compliance need separate owner qualification. See :doc:`mt7932-neo` for +class selection, legacy compatibility, lifetime and installation. diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 7925972fe222b..a8fa08c4e3893 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -14,10 +14,11 @@ Co-authored by DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. Platform and configuration -------------------------- -Use the separately reviewed T8140 PCIe bootstrap described in -``Documentation/PCI/controller/apple-t8140.rst``. It runs during boot when -``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``. Public m1n1 must populate the J700 -``wifi0`` endpoint's own ``local-mac-address``; a zero placeholder is rejected. +Use the T8140 PCIe bootstrap described in +``Documentation/PCI/controller/apple-t8140.rst``. Build the provider with +``CONFIG_PCIE_APPLE_T8140_BOOTSTRAP=y`` and select it through reciprocal +host/engine DT links. Public m1n1 must populate the J700 ``wifi0`` endpoint's +own ``local-mac-address``; a zero placeholder is rejected. The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, ``CONFIG_CFG80211=m``, ``CONFIG_BT_MTK7932_PCIE=m``, ``CONFIG_BT_BREDR=y``, @@ -74,12 +75,20 @@ Country packages Every regulatory change selects a country package again, without a reload. For country CC the driver uses the first of these that exists and is valid: -1. ``policy-v2/CC.bin``: J7RP version 2 (version 1 plus 6 GHz VLP pages); -2. ``policy/CC.bin``: J7RP version 1 (2.4 and 5 GHz, at most 16 KiB); -3. ``policy/world-XZ.bin``: the world package. +1. ``policy-v4/CC.bin``: J7RP version 4 (bound LP and optional VLP class data); +2. ``policy-v2/CC.bin``: J7RP version 2 (version 1 plus 6 GHz VLP pages); +3. ``policy/CC.bin``: J7RP version 1 (2.4 and 5 GHz, at most 16 KiB); +4. ``policy/world-XZ.bin``: the world package. + +When valid v4 and v2 packages coexist, the driver keeps v2's original base +and VLP pages and retains v4 as a class sidecar. LP and additional VLP +permissions extend the exposed channel set, but never change the legacy +VLP page bytes or its original numeric clamp. Both files remain held until +publication; the sidecar hash is exposed with the primary package hash. Country 00 uses only the world package. ``policy/`` holds version 1 only and -``policy-v2/`` version 2 only; a package in the wrong directory is refused. +``policy-v2/`` version 2 only and ``policy-v4/`` version 4 only; a package in +the wrong directory is refused. Version 3 archives remain unbound and refused. Kernels without version 2 support never read ``policy-v2/``, so version 2 packages can be installed while those kernels remain bootable. @@ -100,7 +109,8 @@ fallback reason and the result of each lookup. Station security ---------------- -The driver advertises PSK and SAE with CCMP only. Firmware performs SAE +The driver advertises PSK, PSK-SHA256, SAE and OWE with CCMP, plus +BIP-CMAC-128 for management protection. Firmware performs SAE authentication and the four-way handshake and owns PTK, GTK and IGTK; there is no host key-install interface or external-auth exchange. @@ -114,14 +124,20 @@ required or optional management frame protection (MFP). Firmware uses required MFP for the SAE arm. On a PSK+SAE transition network, the first AKM supplied by userspace selects the arm: PSK keeps the existing binary-PMK request and disables MFP, while SAE uses the password and required MFP. PSK with required -MFP and PSK-SHA256 are unsupported. - -BIP-CMAC-128 is not advertised because WPA2 with required MFP has not been -implemented or qualified. Advertising BIP would also make supplicants request -required MFP on WPA2 networks. iwd requires BIP-CMAC-128 before selecting SAE, -so it remains WPA2-only with this driver, including on transition networks; -it cannot join a pure-SAE network. SAE offload support here targets -NetworkManager with wpa_supplicant, pending the hardware qualification below. +MFP is rejected on legacy AKM 2. WPA2 PMF uses the explicitly selected +PSK-SHA256 AKM (6), with a binary 32-byte PMK and optional or required MFP. +The only EA/40 security delta is little-endian 0x0400 at body +0x14. The +host retains the MFP policy; there is no standalone optional/required field. + +Before any association command, the host validates the candidate's RSN IE, +selected AKM, CCMP and MFPC/MFPR. A required request cannot downgrade to +AKM 2 or a non-MFPC AP. Only BIP-CMAC-128 is accepted as an explicit group +management suite; the standard omitted-suite default is also CMAC-128. +Firmware owns IGTK extraction, install/removal and BIP verification. No +add_key or set_default_mgmt_key operation is added. iwd's BIP gate is now +satisfied, but WPA2-PMF and iwd behavior still require hardware qualification. +The admitted PMF state is host policy, not a firmware readback. Selectors +0x40/0x42 retain their normal completion contract; neither reports PMF state. The standard SAE PWE settings (unspecified, hunting-and-pecking, hash-to-element and both) are accepted; firmware chooses the exchange without @@ -182,10 +198,17 @@ Only SAE and OWE with required MFP are admitted on 6 GHz. Hunting-and-pecking only is refused there; firmware owns H2E. These checks extend the SAE and OWE layers without another crypto or authentication implementation. -Wildcard 6 GHz scans listen passively. Hidden SSIDs use directed scanning -when NO_IR permits it. RNR-only requests remain passive until target-list -serialization is implemented. Full-band scans are split into bounded -32-channel commands. Firmware event channel layout is checked before OCA +Legacy VLP channels retain directed hidden-SSID probing and the original +connect discovery sequence. Channels added only by class packages listen +passively until the target supplies its class/TPE. RNR-only requests remain +passive until target-list serialization is implemented. cfg80211 splits the +lower-band and 6 GHz phases, selects PSCs and expands cached RNR entries. +The driver keeps the supplied channel list, including non-PSC RNR targets, +and never treats ``psc_no_listen`` as permission for an unconstrained probe. +Full-band scans are split into bounded 32-channel commands. Internal +discovery allows the entire passive batch's serialized dwell budget plus +the existing five-second completion margin; 32 channels with a 233 ms +budget require a 12456 ms wait. Firmware event channel layout is checked before OCA association calibration, queue binding and the two association power pages. J7RP version 1 remains valid and never authorizes 6 GHz. Version 2 packages @@ -554,11 +577,293 @@ NO_IR, VLP-client restrictions and runtime EIRP/PSD reductions remain authoritative. The driver preserves 0xc4/0x7f sentinels when reducing numeric native half-dBm cells. LPI, SP and AFC are outside this contract. -The active association path submits the two native selector pages after -queue binding and does not build a 60-byte TPE. Generic runtime D7 +The legacy v2 path retains its two native selector pages after queue +binding. Explicit VLP APs on these channels retain their original admission +and command sequence, including ordinary finite or absent TPE and AP-wide +160 MHz announcements (the station still negotiates at most 80 MHz). No +pre-association class/TPE SET is inserted on this compatibility path. Generic runtime D7 requests for band 2 (6 GHz) remain rejected by the raw planner. RF behavior, rate accounting and calibration still require hardware qualification. +6 GHz software proof and qualification boundary +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The association-start OCA group selector and power-on fragments are separate +from runtime D7. The raw planner rejects every runtime band-2 request before +reading a container or producing a partial fragment plan. The runtime worker +latches that error and closes RF admission. Association success does not +prove that firmware will never issue such a request, and the 2.4/5 GHz D7 +layouts must not be reused by changing only the band number. + +The public producer constructs the 32-byte CID5d header pad from +``one_ss_pad()`` and copies it unchanged through J7RP and class publication. +Its bits index channel numbers. It is independent of the primary permission +mask and does not generate the separate unsupported-rate accounting data. +Neither a zero pad nor numeric per-rate cells prove that firmware excludes +unsupported rates. The channel pad must not be relabelled or populated with +guessed rate bits. Synthetic tests check a nonzero pad, class isolation and +the exact first-page header without distributing power-table inputs. + +The strict class path parses default-client EIRP and PSD TPEs, intersects +applicable limits, preserves ``0x7f``, deducts the J700 antenna gains and +submits the separate 60-byte CID5d body before EA/40. A changed or invalid +home beacon envelope disconnects the strict-class link; disconnect and +country/reset handling invalidate its decision. This does not establish +firmware readback or radiated power. The legacy v2 VLP compatibility path +above still bypasses the new class/TPE submission and beacon comparison; +tests preserving that sequence are not proof of AP TPE enforcement there. + +The software regressions can be exercised without a radio:: + + python3 tools/asahi/mt7932-scan-tests.py --out /tmp/mt7932-scan-tests + python3 tools/asahi/run-host-tests.py --only mt7932 --jobs 8 + +The KUnit ``mt7932-six-ghz``, ``mt7932-regulatory`` and +``mt7932-pmf-lpi`` suites also cover runtime D7 rejection, the header pad, +and complete passive-scan/TPE golden bodies. The host scan test executes +the production batcher and internal discovery worker with synthetic +32/27-channel batches, and requires compiling RNR-probe and short-wait +mutants to fail their runtime assertions. + +Before claiming RF qualification, retain failures as well as successful +cells and run this AP matrix with an explicitly approved device identity: + +* PSC and non-PSC targets, with and without lower-band RNR; wildcard and + hidden SSIDs; connected and disconnected scans. Check matching completion, + bounded 32/27-channel batches and no probes from RNR-derived passive scans. +* LPI and VLP at 20/40/80 MHz, both PSD and EIRP TPE, finite and ``0x7f`` + limits, and changed envelopes. Distinguish strict-class operation from the + legacy v2 bypass. Check class pages and TPE precede association, and measure + power against country, AP, SAR and board limits. +* LPI/VLP reconnects, country changes and firmware resets; confirm fresh + policy publication and no stale AP envelope after rollback. +* Calibration events during discovery, association and sustained traffic. + Record only band/channel, procedure outcome and nonsecret body hashes; + band-2 D7 remains unsupported until its own procedure ABI is established. +* Rate/NSS/MCS and bandwidth accounting under each class, including rates + represented by unsupported or unpopulated cells. Static serialization + tests cannot establish the firmware's default rate-selection behavior. + +6 GHz LPI table archive (J7RP v3, unbound) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The class archive producer audits the authenticated text tables and preserves +LPI alongside VLP. Version 3 does **not** enable LPI operation. Its binding +revision zero remains unbound; version 4 below explicitly binds the same data +to the established class/TPE ABI. Do not infer a command selector from +the VLP tag ``0x2a``, use larger LP values with that tag, or transplant a +related softmac driver's CLC command into this fullmac driver. + +Run an audit without producing packages:: + + python3 tools/asahi/mt7932-policy-classes.py --tables /path/to/IZUBA + +Opt in to **host data archives**, separate from operational v1/v2 packages:: + + python3 tools/asahi/mt7932-policy-classes.py --tables /path/to/IZUBA \ + --legacy-dir /path/to/policy --output-dir /path/to/private-archive-root + python3 tools/asahi/mt7932-policy-classes-tests.py + +Archives use ``policy-v3/CC.bin`` exclusively. Never place them in ``policy/`` +or ``policy-v2/``. The driver does not search policy-v3 and rejects version 3 +if it is mistakenly placed in either operational directory. The installer +archive API is explicit and separate from deployment; deployment remains +v1/v2. No Apple input or generated archive may be committed or redistributed. + +Authenticated table coverage +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For the same 25G83 hashes as v2, the class audit distinguishes section labels, +alphabetic countries, empty sections and usable primaries: + +.. list-table:: + :header-rows: 1 + + * - Class + - Sections + - Countries + - Nonempty countries + - Eligible + * - LP (LPI) + - 135 + - 132 + - 129 + - 129 + * - VLP + - 127 + - 124 + - 120 + - 120 + * - SP + - 112 + - 109 + - 8 + - 8 + +A2/A4/A5 are non-country labels in every class. CN/KH/MH have sections but no +6 GHz matrix row; AR has no usable VLP primary. Every eligible LPI country +has a legacy v1 package. LPI adds AE/AR/CO/EG/GT/JO/PE/SV/TN to the VLP set. +SP is audit-only: no SP archive section, command, AFC permission or runtime +support is produced. The report lists countries and primary counts for all +three classes, including the reasons each unusable section was excluded. + +LP uses matrix mode 1; VLP uses mode 2; SP uses mode 3. These are table +operating-class values, **not firmware command selectors**. Require both the +class-specific matrix mode and at least one projected numeric power cell on +a primary; all-sentinel rows never grant permission. Retain the v2 parsing, +row projection, sentinel and one-SS pad rules separately for each class. +Thus the table-derived LP primary counts are AU/GB/DE/JP=24, US/CA=59; +the corresponding VLP counts are 24 and 50. These counts describe table data, +not a live cfg80211 or firmware grant. + +Unbound v3 byte contract +^^^^^^^^^^^^^^^^^^^^^^^ + +All integers are little endian. The outer J7RP header and legacy mode/table +records are identical to v1 except version=3 and the total file length. +After the thirteen legacy tables append this 16-byte class directory:: + + Offset Size Meaning + 0 4 ASCII 6GCL + 4 1 Archive schema revision 1 + 5 1 Section count: 1 or 2 + 6 1 Firmware binding revision: 0 (unbound data) + 7 1 Flags: zero + 8 4 Directory size, including header and every section + 12 4 Reserved: zero + +Each section has this 24-byte header followed by a 32-byte one-SS pad and +110 table records of 122 bytes each (total section size 13,476 bytes):: + + Offset Size Meaning + 0 1 J7RP class ID: 1=VLP, 2=LPI (not a firmware enum) + 1 1 Indoor-only requirement: 0 for VLP, 1 for LPI + 2 1 Cell units contract: 1, native half-dBm and v2 sentinels + 3 1 Table record count: 110 + 4 4 Section size including header: 13476 + 8 8 Nonzero primary bitmap; bit i denotes channel 1+4*i + 16 2 Alpha2 identical to the outer header + 18 6 Reserved: zero + 24 32 One-SS pad, independently derived for this class + 56 13420 110 records: channel byte followed by 121 projected cells + +Records follow the exact v2 110-channel matrix order, including channel 2 +and excluding 231. Sections are ordered by increasing class ID, with no +duplicates. A usable LPI section is mandatory; a usable VLP section is +optional, so countries without VLP data can still be archived. Reject unknown +classes, binding revisions, flags, units, reserved fields, mismatched countries, +wrong channel order/count/size, empty or out-of-range primary masks, a masked +primary with all-sentinel cells, truncation and trailing bytes. The maximum +archive size is 65,536 bytes. SP must never be serialized. These bodies are +**table records, not CID 0x5d or CID 0x95 packets**. + +Build all selected archives before publication. Refuse a nonempty output root. +Use the v2 inventory rules with ``policy-v3/`` replacing ``policy-v2/``; roots +and directories have mode 0700, files 0600. Preserve every legacy byte except +the version and total size. Audit output contains only counts, country names, +skip reasons, hashes and lengths, never source rows or rate values. + +6 GHz runtime binding (J7RP v4) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The clean 2026-10-31 firmware ABI specification establishes CID 0x5d byte ++0x01[6:4] as the power class: 0 SP, 1 LPI, 2 VLP. Normal infrastructure +per-rate pages use submode 0 and byte 1 of 0x1a for LP data or 0x2a for VLP +data. SP stays disabled because AFC is not implemented. No module parameter +or forced power mode is exposed. + +Version 4 is byte-for-byte version 3 table data, except the J7RP version +halfword is 4 and the 6GCL directory binding revision is 1. It uses only +``policy-v4/CC.bin``. Section order, independent class masks, indoor metadata, +units, bitmap and 110 projected records retain the v3 contract. It requires +LP and permits optional VLP; SP, duplicate/reordered classes, reserved fields, +invalid channel order and sentinel-only permitted primaries are rejected. +These files are data, not captured command envelopes. Older kernels never +search this directory, and v1/v2 remain available for rollback. + +Produce bound packages from the same authenticated inputs:: + + python3 tools/asahi/mt7932-policy-classes.py --tables /path/to/IZUBA \ + --legacy-dir /path/to/policy --output-dir /path/to/new-root --runtime + +For new LPI and v4-only VLP associations the driver validates the complete IE +stream, HE Operation size, primary/center/width and default-client TPE. +Regulatory Info 0 selects LPI, 2 selects VLP; SP and other unqualified values +fail closed. Missing/malformed/duplicate HE Operation never defaults to LPI. +The selected class mask and cfg80211 permissions must allow every occupied +20 MHz channel. The exact installed class-package hash is checked again. +Normal class pages precede the separate 60-byte TPE and EA/40. Partial SET +failure closes RF admission and requires checked reset; no rollback ACK is +invented. The single station has no concurrent BSS class arbitration. + +Existing v2 VLP channels instead retain the original startup CID5d/CID95 +pages and the two 1020-byte CID5d pages after queue binding. Their TPE never +selected admission in aurora-wip, so finite, absent or unsupported TPE does +not introduce a new refusal or pre-association SET on this path. HE Regulatory +Info must explicitly select VLP; it can never authorize SP or select LPI by +fallback. Existing SAE/OWE admission is unchanged. + +The new TPE encoder writes the fixed header, enable, band 3, primary and one +to four SISO/CDD/MIMO bandwidth limits. EIRP interpretations retain half-dBm +bytes. PSD uses the minimum of all advertised 20 MHz values, then adds +26/32/38/44 half-dB ticks for 20/40/80/160 MHz; this conservatively bounds +narrower allocations. Multiple applicable envelopes are intersected; PSD +-128 forbids transmission and is rejected. The 0x7f sentinel survives +conversion and gain subtraction. Signed arithmetic underflow or overflow +is rejected. Extended PSD and subordinate-AP envelopes cannot supply a +station limit. Missing applicable TPE is refused on the new class path. + +The t-3573 J700 antenna-gain SPEC supplies immutable board properties. A fixed +six-by-three signed half-dB table is bound to the existing J700-only probe; +no device-tree property or module parameter is added. SISO/CDD/MIMO tuples +are (16,16,10), (14,14,8), (15,16,10), (15,15,9), (16,16,10), (15,16,10). +See :doc:`mt7932-j700-antenna` for the immutable matrix and lookup contract. +The exact primary-channel/bandwidth map includes the boundary exceptions at +113, 117, 121, 125, 185 and 189. Each finite TPE cell subtracts its own mapped +gain. Both LPI and v4-only VLP use this matrix; neither changes the other +country/class, SAR or hardware ceilings. Missing channel/bandwidth mappings +fail closed for finite cells. No 2.4/5 GHz gains are inferred. + +Country changes retire the connection. Each new class connection reparses +AP information. Changed or invalid class/TPE in its connected AP's beacon +closes data admission and schedules disconnect. After peer/DMA retirement, +RF admission stays fenced while the installed VLP policy is restored and a +known all-0x7f TPE body removes the previous AP ceiling. A subsequent legacy +VLP join uses its original association sequence. Firmware acceptance and RF +compliance, including clearing across class transitions, remain unqualified. +There is no authoritative active-class or PMF readback. Debug metadata +excludes BSSIDs and key material. + +Requirements before runtime enablement +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The static ABI now establishes the class selector and 60-byte TPE transport. +The remaining hardware qualification gates are described below, including changes on association, reassociation, AP changes, country +changes, disconnect and reset. Do not send +unbound archive bytes as commands or claim readback of a firmware-active +class from table presence. Public mac80211 handles HE Operation Regulatory +Info separately from TPE; TPE values alone do not identify the AP class. + +Association must use the target BSS's validated HE 6 GHz Operation Information +and applicable client-category TPE, intersect the country/class primary mask +with cfg80211 permissions, channel width, EIRP and PSD, and enforce indoor-only +rules. LPI cannot be selected for a VLP AP. SP requires separate AFC semantics +and stays unsupported. An LPI AP without country LPI permission must never +cause an LPI power increase; VLP fallback must retain all VLP restrictions and +only occur when allowed for that connection. Unknown or malformed HE/TPE +information cannot authorize higher power. Preserve sentinel meanings and +round power caps downward. Revalidate changing BSS/regulatory state before +submission and abort rather than transmit with stale, more permissive limits. + +The Neo test plan requires a 6 GHz AP explicitly advertising LPI in HE 6 GHz +Operation Regulatory Info, with inspectable client TPEs and a legal indoor +configuration. A phone hotspot may advertise only VLP and cannot close this +gate. Capture the advertised class and TPE independently, verify host selection +and firmware acceptance/readback, then test SAE-H2E and OWE, reconnect, VLP +fallback, forbidden-class/country/indoor cases, country changes, reset, power +caps and v1/v2 rollback. Controller approval with explicit Neo identity and +all session/resource/ownership gates is required for every hardware run. + Local firmware inputs --------------------- @@ -623,6 +928,20 @@ over-air MPDU counts, BA-window sizes, TX retries or firmware ACK counters. Duplicate suppression records sequences only after the protection check, so a rejected unprotected frame cannot suppress a later decrypted retry. +Transport counters describe how the host services the device: +``irq_service_passes`` counts interrupt-thread passes and +``rx_ring0_records`` to ``rx_ring8_records`` count descriptors consumed per +receive ring. ``irq_unmasked_bits`` accumulates status bits that were +pending outside the enabled interrupt mask, and ``irq_unmasked_passes`` +counts the passes that saw any. Work announced only by such a bit waits for +an unrelated interrupt, which shows up as latency rather than loss. +``tx_free_events`` counts TX completion reports. ``bss_absent_events`` and +``bss_absent_ms`` count how often and for how long the firmware reported the +BSS absent, during which TX is held. ``home_beacons`` counts beacons from +the associated BSS that reached the host, and ``rx_2ghz_records``, +``rx_5ghz_records`` and ``rx_6ghz_records`` split admitted Ethernet RX by +band. Like the RX counters, they are lifetime values. + Use the current RX bitrate from ``iw dev station dump`` while traffic is running. The driver expires that sample after one second; an idle or unsupported RX vector can leave it absent. TX bitrate is not @@ -658,3 +977,35 @@ Limitations are described under Station security. * SCO/headset microphone, LE Audio/ISO and simultaneous headset audio are unqualified. + +Station power cache and coexistence +---------------------------------- + +Each join submits CID05 awake before EA/40. That pre-join submission is +not the power policy of the subsequent associated firmware session. The +host invalidates its applied-mode cache after pre-join initialization so +that the existing post-association worker submits the desired policy, +including awake when station power save is disabled. Successful repeated +requests within that association still coalesce. Sleep remains gated by +TIM discovery and the normal RF, peer and termination checks. A successful +submission is not firmware readback or a measured coexistence improvement. + +The association builder retains its twenty-beacon listen interval on all +bands. The earlier proposal to reduce it to one on every 2.4 GHz association +is omitted: there is no BT activity notification connecting these drivers, +and a band-only policy would also affect BT-off associations. Twenty to one +permits twenty times as many listen opportunities, or nineteen additional +opportunities per twenty beacons. At a 100 TU beacon interval (102.4 ms), +the advertised span would fall from 2048 ms to 102.4 ms. These values do not +quantify radio wake duty cycle, milliwatts or battery life; those depend on +firmware, TIM/DTIM and actual traffic. No new module parameter, guessed PTA +command or Bluetooth link-policy override is introduced. + +Qualify with paired control/fix boots, BT off/on on 2.4 GHz, local wired +iperf3 in each direction and ten-minute gateway pings. Interleave off/on/off +and on/off/on blocks to expose drift in a busy channel. Compare each +module's BT-on penalty against its adjacent BT-off cells, retaining loss +and tail latency separately. Include a 5 GHz control, functioning BT HID, +and a twenty-second rtc1 suspend/resume followed by a fresh association. +Keep failed cells and independent rollback evidence; host tests alone do +not establish latency, throughput or Bluetooth functionality. diff --git a/Documentation/rust/bindgen-layout-checks.rst b/Documentation/rust/bindgen-layout-checks.rst index 4e0a996bd701d..eace444abbcd8 100644 --- a/Documentation/rust/bindgen-layout-checks.rst +++ b/Documentation/rust/bindgen-layout-checks.rst @@ -58,6 +58,14 @@ builds. Packagers prepare this entry point in their build roots. Make does not fetch tools or install anything system-wide. If the entry point is absent, Kbuild uses ``bindgen`` from PATH and the same mandatory assertions apply. +``make LLVM=1 rustavailable`` probes the selected generator's empty-record +alignment support, including locally patched versions. An affected generator +prints the preparation command above. This is a warning so it cannot silently +disable Rust drivers during configuration; the layout assertions still reject +incorrect bindings when compiling the kernel. Prepare the tool before running +``defconfig`` and merging ``asahi.config``, then keep the resolved configuration +unchanged for the kernel build. + The generated bindings are consumed by the Rust kernel crate and by several drivers. After changing the bindings generator, rebuild the kernel Image and all modules together. diff --git a/MAINTAINERS b/MAINTAINERS index d966b42b5cbe6..2e0aab08c5d6f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2082,6 +2082,7 @@ W: https://aurora-silicon.github.io S: Maintained F: Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml F: Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +F: Documentation/devicetree/bindings/dma/apple,t8140-pcie-bootstrap.yaml F: Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml F: Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml F: drivers/pmdomain/apple/pmp-report.c @@ -2092,6 +2093,7 @@ F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: Documentation/PCI/controller/apple-t8140.rst +F: Documentation/devicetree/bindings/iio/imu/apple,aop-motion.yaml F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml F: Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml @@ -2110,6 +2112,8 @@ F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml F: Documentation/gpu/apple-dcp-backlight.rst F: Documentation/hid/apple-j700-multitouch.rst +F: Documentation/iio/aop-sensors.rst +F: Documentation/networking/device_drivers/wifi/mt7932-j700-antenna.rst F: Documentation/networking/device_drivers/wifi/mt7932-neo.rst F: arch/arm64/boot/dts/apple/t8140* F: arch/arm64/configs/j700*.config @@ -2118,12 +2122,13 @@ F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-parser-test.c F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ +F: drivers/net/wireless/mediatek/mt7932/ +F: drivers/pci/controller/pcie-apple-t8140-bootstrap* F: drivers/phy/apple/atc-j700-dp.h F: drivers/rtc/rtc-apple-pmu.c F: drivers/soc/apple/aop.rs F: drivers/soc/apple/mcc.c F: drivers/tty/serial/apple_dockchannel* -F: drivers/pci/controller/pcie-apple-piodma* F: include/linux/soc/apple/pci-apple-piodma.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs @@ -2131,7 +2136,9 @@ F: sound/soc/apple/aop_t8140.rs F: tools/asahi/mt7932* F: tools/testing/apple/ F: tools/testing/kunit/configs/apple_firmware.config +F: tools/testing/kunit/configs/apple_t8140_bootstrap.config F: tools/testing/kunit/configs/mt7932.config +F: tools/testing/selftests/drivers/apple/aop-las.py F: tools/testing/selftests/drivers/gpu/apple-dcp/ APPLETALK NETWORK LAYER @@ -2705,6 +2712,7 @@ B: https://github.com/AsahiLinux/linux/issues C: irc://irc.oftc.net/asahi-dev T: git https://github.com/AsahiLinux/linux.git F: Documentation/ABI/testing/sysfs-driver-apple-dart +F: Documentation/ABI/testing/sysfs-driver-apple-pmp-dram F: Documentation/devicetree/bindings/arm/apple.yaml F: Documentation/devicetree/bindings/arm/apple/* F: Documentation/devicetree/bindings/clock/apple,nco.yaml @@ -2727,6 +2735,7 @@ F: Documentation/devicetree/bindings/npu/apple,ane.yaml F: Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml F: Documentation/devicetree/bindings/nvmem/apple,efuses.yaml F: Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml +F: Documentation/devicetree/bindings/opp/operating-points-v2-apple-dram.yaml F: Documentation/devicetree/bindings/pci/apple,pcie.yaml F: Documentation/devicetree/bindings/phy/apple,atcphy.yaml F: Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml @@ -2774,6 +2783,7 @@ F: drivers/pmdomain/apple/ F: drivers/power/reset/macsmc-reboot.c F: drivers/power/supply/macsmc-power*.c F: drivers/pwm/pwm-apple.c +F: tools/asahi/apple-pwm-pm-tests.py F: drivers/reset/reset-apple-cio.c F: drivers/rtc/rtc-macsmc.c F: drivers/soc/apple/* @@ -2801,6 +2811,16 @@ F: net/bluetooth/brcm.c F: net/bluetooth/brcm.h F: tools/testing/selftests/drivers/apple/ +ARM/APPLE AVE VIDEO ENCODER DRIVER +M: Ryan Murray +L: linux-media@vger.kernel.org +S: Maintained +W: https://aurora-silicon.github.io +F: Documentation/admin-guide/media/apple-ave.rst +F: Documentation/devicetree/bindings/media/apple,t8140-ave.yaml +F: drivers/media/platform/apple/ave/ +F: tools/asahi/ave*-tests.py + ARM/APPLE QSPIMC DRIVER M: Ryan Murray W: https://aurora-silicon.github.io @@ -8169,9 +8189,13 @@ L: asahi@lists.linux.dev L: dri-devel@lists.freedesktop.org S: Maintained F: Documentation/ABI/testing/sysfs-class-accel-ane +F: Documentation/ABI/testing/sysfs-driver-ane-performance +F: Documentation/accel/apple-ane.rst F: Documentation/devicetree/bindings/npu/apple,ane.yaml F: drivers/accel/ane/ F: include/uapi/drm/ane_accel.h +F: tools/accel/ane-bench.py +F: tools/testing/selftests/drivers/ane/ DRM ACCEL DRIVERS FOR INTEL VPU M: Karol Wachowski diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index f1978b9dff2bb..c0ef0498f1439 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -51,6 +51,14 @@ config ARM64_USER4K_EXPERIMENTAL private prctl numbers are not upstream assignments or a stable ABI promise. Programs that do not use the prctl keep the native granule. +config ARM64_CPUINFO_KUNIT_TEST + bool "KUnit tests for the /proc/cpuinfo model name" if !KUNIT_ALL_TESTS + depends on KUNIT=y && DMI + default KUNIT_ALL_TESTS + help + Test how /proc/cpuinfo reads the processor's name from the SMBIOS + Processor Information. + config ARM64_USER4K_KUNIT_TEST bool "Test mixed userspace granules with larger kernel pages" depends on KUNIT=y && ARM64_USER4K_EXPERIMENTAL diff --git a/arch/arm64/boot/dts/apple/spi1-nvram.dtsi b/arch/arm64/boot/dts/apple/spi1-nvram.dtsi index d2720b3077741..268d9ffb94044 100644 --- a/arch/arm64/boot/dts/apple/spi1-nvram.dtsi +++ b/arch/arm64/boot/dts/apple/spi1-nvram.dtsi @@ -11,6 +11,16 @@ aliases { nvram = &nvram; }; + + /* + * The boot NOR runs from a PMU rail that is on whenever the SoC is + * and that no OS switches. The ADT gives the flash no supply. + */ + nor_flash_vcc: regulator-nor-flash-vcc { + compatible = "regulator-fixed"; + regulator-name = "nor-flash-vcc"; + regulator-always-on; + }; }; &spi1 { @@ -20,6 +30,7 @@ compatible = "jedec,spi-nor"; reg = <0x0>; spi-max-frequency = <25000000>; + vcc-supply = <&nor_flash_vcc>; partitions { compatible = "fixed-partitions"; diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index db7ea6eac5a6f..75b3fcf23b573 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -39,3 +39,54 @@ &gpu { compatible = "apple,agx-t6000", "apple,agx-g13x"; }; + +/* + * DRAM interface class: width and maximum rate are inferred from topology + * and the published bandwidth class. Lower-state rates remain unqualified. + * State IDs below describe firmware DCS states, not PMGR codes. + */ +/ { + dram_opp: opp-table-dram { + compatible = "operating-points-v2-apple-dram"; + opp-shared; + + opp-0 { + opp-level = <0>; + }; + + opp-1 { + opp-level = <1>; + }; + + opp-2 { + opp-level = <2>; + }; + + opp-3 { + opp-level = <3>; + opp-hz = /bits/ 64 <3200000000>; + apple,data-rate = /bits/ 64 <6400000000>; + }; + }; +}; + +&pmp_report { + operating-points-v2 = <&dram_opp>; + apple,dram-type = "lpddr5"; + apple,data-bus-width = <256>; + apple,dcs-state-count = <4>; + apple,peak-bandwidth-bytes-per-second = /bits/ 64 <204800000000>; + apple,bandwidth-class-bytes-per-second = /bits/ 64 <200000000000>; +}; + +&pmp_report { + reg = <0x2 0x8e3c0000 0x0 0x20000>, + <0x2 0x85868000 0x0 0x4000>; + reg-names = "report", "ane-dvfs"; +}; + +&ane { + apple,performance-controller = <&pmp_report>; + power-domains = <&ps_ane_sys_cpu>, <&ps_ane_set1>, + <&ps_ane_set2>, <&ps_ane_set3>, <&ps_ane_set4>, <&ps_pms_sram>; +}; diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index 5da33aa0db9df..e617f7f3f41a9 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -136,3 +136,15 @@ &ps_ane_set4 { status = "okay"; }; + +&pmp_report { + reg = <0x2 0x8e3c0000 0x0 0x20000>, + <0x2 0x85868000 0x0 0x4000>; + reg-names = "report", "ane-dvfs"; +}; + +&ane { + apple,performance-controller = <&pmp_report>; + power-domains = <&ps_ane_sys_cpu>, <&ps_ane_set1>, + <&ps_ane_set2>, <&ps_ane_set3>, <&ps_ane_set4>, <&ps_pms_sram>; +}; diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 2cbe84a3d5625..8357258058339 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -8,6 +8,45 @@ &SOC_NODE { + ave_dart1: iommu@20d030000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x0d030000 0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_venc_sys>; + #iommu-cells = <1>; + status = "disabled"; + }; + + ave_dart0: iommu@20d040000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x0d040000 0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_venc_sys>; + #iommu-cells = <1>; + status = "disabled"; + }; + + ave: video-codec@20d100000 { + compatible = "apple,t6000-ave"; + reg = <0x2 0x0d100000 0 0x45c000>, + <0x2 0x0d050000 0 0x8000>, + <0x2 0x0d800000 0 0x800000>; + reg-names = "codec", "mailbox", "coproc"; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + ; + power-domains = <&ps_venc_dma>, <&ps_venc_pipe4>, + <&ps_venc_pipe5>, <&ps_venc_me0>, <&ps_venc_me1>; + power-domain-names = "dma", "pipe4", "pipe5", "me0", "me1"; + iommus = <&ave_dart0 0>, <&ave_dart1 0>; + status = "disabled"; + }; + ane_dart0: iommu@285800000 { compatible = "apple,t6000-dart"; reg = <0x2 0x85800000 0x0 0x4000>; diff --git a/arch/arm64/boot/dts/apple/t600x-pciec.dtsi b/arch/arm64/boot/dts/apple/t600x-pciec.dtsi index 808048a7fcece..c58c66eda9bc9 100644 --- a/arch/arm64/boot/dts/apple/t600x-pciec.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pciec.dtsi @@ -22,6 +22,7 @@ apciec0: pcie@730000000 { compatible = "apple,t6000-pciec"; + apple,pciec-kernel-init; device_type = "pci"; reg = <0x7 0x30000000 0x0 0x10000000>, @@ -92,6 +93,7 @@ apciec1: pcie@b30000000 { compatible = "apple,t6000-pciec"; + apple,pciec-kernel-init; device_type = "pci"; reg = <0xb 0x30000000 0x0 0x10000000>, @@ -162,6 +164,7 @@ apciec2: pcie@f30000000 { compatible = "apple,t6000-pciec"; + apple,pciec-kernel-init; device_type = "pci"; reg = <0xf 0x30000000 0x0 0x10000000>, @@ -231,6 +234,7 @@ apciec3: pcie@1330000000 { compatible = "apple,t6000-pciec"; + apple,pciec-kernel-init; device_type = "pci"; reg = <0x13 0x30000000 0x0 0x10000000>, diff --git a/arch/arm64/boot/dts/apple/t6021-j414c.dts b/arch/arm64/boot/dts/apple/t6021-j414c.dts index ef90ffe8423f4..2521633009343 100644 --- a/arch/arm64/boot/dts/apple/t6021-j414c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j414c.dts @@ -46,3 +46,21 @@ &mtp_mt { firmware-name = "apple/tpmtfw-j414c.bin"; }; + +/* + * From the J414c platform device tree (/arm-io/spi2/mesa): the power line is + * gpio0 pin 122 (function-mesa_pwr, active high) and the data-ready line is + * pin 121, the same lines as J414s. + */ +&spi2 { + status = "okay"; +}; + +&mesa { + status = "okay"; + enable-gpios = <&pinctrl_ap 122 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <121 IRQ_TYPE_EDGE_RISING>; + spi-cs-setup-delay-ns = <20>; + spi-cs-hold-delay-ns = <20>; +}; diff --git a/arch/arm64/boot/dts/apple/t6030-j516s.dts b/arch/arm64/boot/dts/apple/t6030-j516s.dts index edc00c0c1c4df..39ee54f235c0d 100644 --- a/arch/arm64/boot/dts/apple/t6030-j516s.dts +++ b/arch/arm64/boot/dts/apple/t6030-j516s.dts @@ -97,3 +97,11 @@ &isp { apple,platform-id = <16>; }; + +&smc_hwmon { + /* M3-family die keys, not yet confirmed on J516s; absent keys are skipped. */ + apple,soc-die-keys = "Tf04", "Tf09", "Tf0A", "Tf0B", "Tf0D", + "Tf0E", "Tf14", "Tf18", "Tf19", "Tf1A", + "Tf24", "Tf25", "Tf2A", "Tf2B"; + apple,thermal-pressure-key = "mTPL"; +}; diff --git a/arch/arm64/boot/dts/apple/t6031-j514c.dts b/arch/arm64/boot/dts/apple/t6031-j514c.dts index f4f282e9d3207..42aeffa229c75 100644 --- a/arch/arm64/boot/dts/apple/t6031-j514c.dts +++ b/arch/arm64/boot/dts/apple/t6031-j514c.dts @@ -53,3 +53,12 @@ compatible = "apple,j514-macaudio", "apple,j314-macaudio", "apple,macaudio"; model = "MacBook Pro J514"; }; + +&smc_hwmon { + /* Assumed to match J516c; unavailable firmware keys are skipped. */ + apple,soc-die-keys = "Tf04", "Tf06", "Tf09", "Tf0A", "Tf0B", + "Tf14", "Tf16", "Tf19", "Tf1A", "Tf1B", + "Tf24", "Tf26", "Tf29", "Tf2A", "Tf2B", + "Tf34", "Tf36", "Tf39", "Tf3A", "Tf3B", + "Tf44", "Tf46", "Tf49", "Tf4A", "Tf4B"; +}; diff --git a/arch/arm64/boot/dts/apple/t6031-j516c.dts b/arch/arm64/boot/dts/apple/t6031-j516c.dts index 562491755117f..9b3afec0903e1 100644 --- a/arch/arm64/boot/dts/apple/t6031-j516c.dts +++ b/arch/arm64/boot/dts/apple/t6031-j516c.dts @@ -53,3 +53,12 @@ compatible = "apple,j516-macaudio", "apple,j316-macaudio", "apple,macaudio"; model = "MacBook Pro J516"; }; + +&smc_hwmon { + /* M3 Max die keys; unavailable firmware keys are skipped. */ + apple,soc-die-keys = "Tf04", "Tf06", "Tf09", "Tf0A", "Tf0B", + "Tf14", "Tf16", "Tf19", "Tf1A", "Tf1B", + "Tf24", "Tf26", "Tf29", "Tf2A", "Tf2B", + "Tf34", "Tf36", "Tf39", "Tf3A", "Tf3B", + "Tf44", "Tf46", "Tf49", "Tf4A", "Tf4B"; +}; diff --git a/arch/arm64/boot/dts/apple/t6032-j575d.dts b/arch/arm64/boot/dts/apple/t6032-j575d.dts index 1bf14006807e8..ac36bb2723882 100644 --- a/arch/arm64/boot/dts/apple/t6032-j575d.dts +++ b/arch/arm64/boot/dts/apple/t6032-j575d.dts @@ -14,6 +14,7 @@ / { compatible = "apple,j575d", "apple,t6032", "apple,arm-platform"; model = "Apple Mac Studio (M3 Ultra, 2025)"; + chassis-type = "desktop"; aliases { serial0 = &serial0; diff --git a/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi b/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi new file mode 100644 index 0000000000000..8acd2d04aea8d --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6040-j616s-dpaudio.dtsi @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * DPA1 and DPA2 belong to DCPEXT0 and DCPEXT1, respectively. SIO supplies + * their sample streams; neither interface has AP-side registers. Keep the + * clients disabled until the T6041 SIO firmware backend and loader handoff + * are available and the compact audio protocol has been qualified. + */ + +/ { + dpaudio1: audio-controller-1 { + compatible = "apple,t6041-dpaudio", "apple,dpaudio"; + dmas = <&sio 0x66>; + dma-names = "tx"; + power-domains = <&ps_dpa1>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dpaudio1_dcpext: endpoint { + remote-endpoint = <&dcpext0_audio>; + }; + }; + }; + }; + + dpaudio2: audio-controller-2 { + compatible = "apple,t6041-dpaudio", "apple,dpaudio"; + dmas = <&sio 0x68>; + dma-names = "tx"; + power-domains = <&ps_dpa2>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dpaudio2_dcpext: endpoint { + remote-endpoint = <&dcpext1_audio>; + }; + }; + }; + }; +}; + +&dcpext0 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dcpext0_audio: endpoint { + remote-endpoint = <&dpaudio1_dcpext>; + }; + }; + }; +}; + +&dcpext1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dcpext1_audio: endpoint { + remote-endpoint = <&dpaudio2_dcpext>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-usbc.dtsi b/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-usbc.dtsi index 6fbf8d6a27f89..fb151f79ecd5b 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-usbc.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-j616s-extdisp-usbc.dtsi @@ -152,7 +152,26 @@ }; }; +/* + * As DISPEXT0: dcpext1, its DARTs and its mailbox are in the DISPEXT1 block, + * so they hold DISPEXT1_FE (and with it the PMP request) while they are bound. + * Without it DISPEXT1 is only powered when iBoot left it on for a display + * plugged in at boot, and genpd turns it off as unused under a running dcpext1. + */ +&dcpext1_dart { + power-domains = <&ps_dispext1_fe>; +}; + +&dispext1_dart { + power-domains = <&ps_dispext1_fe>; +}; + +&dcpext1_mbox { + power-domains = <&ps_dispext1_fe>; +}; + &dcpext1 { + power-domains = <&ps_dispext1_fe>; apple,connector-type = "USB-C"; /* as dcpext0: without it apple-dcp's probe fails with -EINVAL (m4.63) */ apple,iomfb-surfaces = <1 1 0 0>; diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 83b29a6c3f866..25cded4aa83df 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -21,6 +21,8 @@ aliases { keyboard = &keyboard; + nvram = &nvram; + spi0 = &qspi; wifi0 = "/pcie@1cb0000000/pci@0,0/wifi@0,0"; bluetooth0 = "/pcie@1cb0000000/pci@0,0/bluetooth@0,1"; /* used by the loader to hand over the display */ @@ -299,16 +301,18 @@ /* * CPU die temperature: SMC key TCMz (CPU die max), sensor 0 of - * smc_hwmon. The PMP's fast die-temperature limit regulates (TCMz - * settles near 111 C); this zone is the backstop above it: passive at - * 113 C, critical (orderly shutdown) at 118 C, polled once a second as - * the SMC refreshes TCMz. The cpufreq cooling device sits on a + * smc_hwmon. The PMP's fast die-temperature limit regulates: its loop + * holds its own hot-spot sensors at 110 C (ADT fast-die-ctrl-loop), + * and under a sustained all-core load TCMz then settles at 113.4 to + * 114.7 C with the fans running. This zone is the backstop above that: + * passive at 116 C, critical (orderly shutdown) at 118 C, polled once a + * second as the SMC refreshes TCMz. The cpufreq cooling device sits on a * cluster's policy CPU, so each map lists every CPU of its cluster. * * Per-cluster and GPU zones follow, one per SMC die-maximum sensor * (smc_hwmon channels 1-23, in the order of the temperature nodes * below): Tex0-3 throttle the E cluster, Tpx0-5 the first P cluster - * and Tpx8-D the second, with the same 113/118 C backstop as TCMz. + * and Tpx8-D the second, with the same 116/118 C backstop as TCMz. * Unlike the fanless J700, this machine has two SMC-driven fans and * the PMP regulates the dies itself, so these zones only catch a * failure of that loop; they never act in normal use. @@ -329,6 +333,17 @@ * highest state any zone asks for and goes back to the SMC once no * zone asks for anything. States 1-8 of a fan run it from its minimum * (1350 rpm) to its maximum (5777 rpm) in even steps. + * + * sustainable-power is what the CPUs dissipate with the die held at + * its operating point: 720 s of all-core load (10 P, 4 E busy loops), + * headless, SMC fans in their own mode. Over the last 600 s TCMz held + * 112.0 to 113.8 C with the PMP limiting the P clusters to a mean of + * about 3.7 GHz and the fans at 3450/3730 rpm (where macOS settles + * too). The SMC core rails then read P0 25.3 W, P1 23.9 W and E 2.1 W + * on average, the units of the energy model. Each zone gets the sum + * over the clusters it throttles. Without it the power allocator + * estimates a budget from the cooling devices' lowest states, which + * throttles far below what this machine sustains. */ #define J616S_FAN_TRIP(label, node, temp, hyst) \ label: node { \ @@ -362,13 +377,14 @@ cpu-die-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <51300>; thermal-sensors = <&smc_hwmon 0>; /* TCMz */ trips { J616S_FAN_TRIPS(cpu_die) cpu_die_passive: cpu-die-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -417,11 +433,12 @@ cpu-e0-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <2100>; thermal-sensors = <&smc_hwmon 1>; /* Tex0 */ trips { cpu_e0_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -448,11 +465,12 @@ cpu-e1-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <2100>; thermal-sensors = <&smc_hwmon 2>; /* Tex1 */ trips { cpu_e1_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -479,11 +497,12 @@ cpu-e2-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <2100>; thermal-sensors = <&smc_hwmon 3>; /* Tex2 */ trips { cpu_e2_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -510,11 +529,12 @@ cpu-e3-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <2100>; thermal-sensors = <&smc_hwmon 4>; /* Tex3 */ trips { cpu_e3_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -541,11 +561,12 @@ cpu-p00-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <25300>; thermal-sensors = <&smc_hwmon 5>; /* Tpx0 */ trips { cpu_p00_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -573,11 +594,12 @@ cpu-p01-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <25300>; thermal-sensors = <&smc_hwmon 6>; /* Tpx1 */ trips { cpu_p01_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -605,11 +627,12 @@ cpu-p02-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <25300>; thermal-sensors = <&smc_hwmon 7>; /* Tpx2 */ trips { cpu_p02_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -637,11 +660,12 @@ cpu-p03-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <25300>; thermal-sensors = <&smc_hwmon 8>; /* Tpx3 */ trips { cpu_p03_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -669,11 +693,12 @@ cpu-p04-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <25300>; thermal-sensors = <&smc_hwmon 9>; /* Tpx4 */ trips { cpu_p04_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -701,11 +726,12 @@ cpu-p05-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <25300>; thermal-sensors = <&smc_hwmon 10>; /* Tpx5 */ trips { cpu_p05_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -733,11 +759,12 @@ cpu-p10-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <23900>; thermal-sensors = <&smc_hwmon 11>; /* Tpx8 */ trips { cpu_p10_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -765,11 +792,12 @@ cpu-p11-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <23900>; thermal-sensors = <&smc_hwmon 12>; /* Tpx9 */ trips { cpu_p11_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -797,11 +825,12 @@ cpu-p12-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <23900>; thermal-sensors = <&smc_hwmon 13>; /* TpxA */ trips { cpu_p12_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -829,11 +858,12 @@ cpu-p13-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <23900>; thermal-sensors = <&smc_hwmon 14>; /* TpxB */ trips { cpu_p13_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -861,11 +891,12 @@ cpu-p14-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <23900>; thermal-sensors = <&smc_hwmon 15>; /* TpxC */ trips { cpu_p14_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -893,11 +924,12 @@ cpu-p15-thermal { polling-delay-passive = <1000>; polling-delay = <1000>; + sustainable-power = <23900>; thermal-sensors = <&smc_hwmon 16>; /* TpxD */ trips { cpu_p15_passive: trip-passive { - temperature = <113000>; + temperature = <116000>; hysteresis = <5000>; type = "passive"; }; @@ -1430,6 +1462,38 @@ status = "okay"; }; +/* + * Boot NOR. The partition is the J616s ADT /arm-io/qspi/spinor/anvram, + * flash-relative; asahi-bless and asahi-nvram find it by its label. + */ +&qspi { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <10000000>; + spi-rx-bus-width = <1>; + spi-tx-bus-width = <1>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + nvram: partition@700000 { + label = "nvram"; + reg = <0x700000 0x100000>; + }; + }; + }; +}; + +/* The QSPI controller cannot probe while its power state is disabled. */ +&ps_qspi { + status = "okay"; +}; + /* * Touch ID sensor (Mesa 0x3356). Its power is NUB GPIO 53 and its * data-ready line is AP GPIO 116, as the J616s ADT describes them. @@ -3851,23 +3915,26 @@ }; }; -/* - * Fan speeds, limits, targets and modes, and the common SMC sensors; every - * key exists on this machine's SMC. Both fans list their target (F0Tg, - * F1Tg) and mode (F0Md, F1Md) keys, so with the driver's fan_control - * parameter fanN_target becomes writable; by default the attributes are - * read-only, and the thermal zones above drive the fans. - */ +/* Fan monitoring and explicit userspace manual control. */ #include "hwmon-common.dtsi" #include "hwmon-fan-dual.dtsi" #include "hwmon-laptop.dtsi" -/* - * Both fans are cooling devices of the CPU die and GPU zones, as - * <&smc_hwmon fan-index lower upper>. - */ &smc_hwmon { - #cooling-cells = <3>; + apple,thermal-pressure-key = "mTPL"; + + /* + * The SMC accepts a manual speed for both fans once its fan test + * key unlocks them, and returns them to system mode when the + * request is dropped. + */ + fan-F0Ac { + apple,fan-manual-control; + }; + + fan-F1Ac { + apple,fan-manual-control; + }; }; &pcie_dart { @@ -4000,6 +4067,11 @@ status = "okay"; }; +/* The sensor is on the logic board, in the base. */ +&aop_accel { + label = "accel-base"; +}; + &admac0 { status = "okay"; }; @@ -4440,6 +4512,43 @@ }; }; +/* The video encoder, its DARTs and its power domains. */ +&ps_venc0_sys { + status = "okay"; +}; + +&ps_venc0_dma { + status = "okay"; +}; + +&ps_venc0_pipe4 { + status = "okay"; +}; + +&ps_venc0_pipe5 { + status = "okay"; +}; + +&ps_venc0_me0 { + status = "okay"; +}; + +&ps_venc0_me1 { + status = "okay"; +}; + +&ave0_dart0 { + status = "okay"; +}; + +&ave0_dart1 { + status = "okay"; +}; + +&ave { + status = "okay"; +}; + #include "t6040-j616s-usb3.dtsi" /* External display chain, all disabled: the gate in drivers/soc/apple/display-gate.c enables it. */ @@ -4447,6 +4556,7 @@ #include "t6040-j616s-extdisp-mem-loader.dtsi" #include "t6040-j616s-extdisp-hdmi.dtsi" #include "t6040-j616s-extdisp-usbc.dtsi" +#include "t6040-j616s-dpaudio.dtsi" /* Additional Abbey PMICs on the two independent NUB SPMI buses. */ &nub_spmi1 { status = "okay"; @@ -4510,3 +4620,113 @@ status = "disabled"; }; }; + +/* + * The SMC owns automatic fan policy. Drop the host fan trip/map instances + * after the board's thermal descriptions, retaining all passive and critical + * protection. Keep this final policy separate from ladder descriptions so + * independent ladder updates cannot restore autonomous SMC writes. + */ + +#define J616S_DELETE_FAN_TRIPS \ + /delete-node/ trip-fan1; \ + /delete-node/ trip-fan2; \ + /delete-node/ trip-fan3; \ + /delete-node/ trip-fan4; \ + /delete-node/ trip-fan5; + +/* Some CPU ladders also map full fan speed to the passive trip. */ +#define J616S_DELETE_FAN_MAPS \ + /delete-node/ map-fan1; \ + /delete-node/ map-fan2; \ + /delete-node/ map-fan3; \ + /delete-node/ map-fan4; \ + /delete-node/ map-fan5; \ + /delete-node/ map-fan6; + +/ { + thermal-zones { + cpu-die-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + + gpu0-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + + gpu1-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + + gpu2-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + + gpu3-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + + gpu4-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + + gpu5-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + + gpu6-thermal { + trips { + J616S_DELETE_FAN_TRIPS + }; + + cooling-maps { + J616S_DELETE_FAN_MAPS + }; + }; + }; +}; + +#undef J616S_DELETE_FAN_TRIPS +#undef J616S_DELETE_FAN_MAPS diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 961af1b7b5bd4..a33ef2e0abf83 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -298,6 +298,14 @@ * in kHz. opp-level is the hardware p-state. Each P cluster has its * own table even though the frequencies are the same: opp-shared * means one clock domain per table. + * + * opp-microwatt is the power of one busy core, measured on a J616s + * from the SMC core-rail meters (P0 PR0b, P1 PP0b, E PP2b) with the + * cluster pinned at each state. The P values follow a k * V^2 * f + c + * fit to the sweep, because the PMP's die-temperature limit clips the + * two top states within seconds. One busy E core does not register on + * its meter below 2352 MHz, so the E values are the idle cluster plus + * a quarter of the four-core increase. */ opp_e: opp-table-e { compatible = "operating-points-v2"; @@ -306,30 +314,37 @@ opp01 { opp-hz = /bits/ 64 <1020000000>; opp-level = <1>; + opp-microwatt = <81900>; }; opp02 { opp-hz = /bits/ 64 <1404000000>; opp-level = <2>; + opp-microwatt = <133100>; }; opp03 { opp-hz = /bits/ 64 <1788000000>; opp-level = <3>; + opp-microwatt = <209400>; }; opp04 { opp-hz = /bits/ 64 <2112000000>; opp-level = <4>; + opp-microwatt = <301400>; }; opp05 { opp-hz = /bits/ 64 <2352000000>; opp-level = <5>; + opp-microwatt = <383900>; }; opp06 { opp-hz = /bits/ 64 <2532000000>; opp-level = <6>; + opp-microwatt = <478500>; }; opp07 { opp-hz = /bits/ 64 <2592000000>; opp-level = <7>; + opp-microwatt = <531000>; }; }; @@ -340,78 +355,97 @@ opp01 { opp-hz = /bits/ 64 <1260000000>; opp-level = <1>; + opp-microwatt = <513000>; }; opp02 { opp-hz = /bits/ 64 <1512000000>; opp-level = <2>; + opp-microwatt = <715000>; }; opp03 { opp-hz = /bits/ 64 <1800000000>; opp-level = <3>; + opp-microwatt = <907500>; }; opp04 { opp-hz = /bits/ 64 <2088000000>; opp-level = <4>; + opp-microwatt = <1158200>; }; opp05 { opp-hz = /bits/ 64 <2352000000>; opp-level = <5>; + opp-microwatt = <1494500>; }; opp06 { opp-hz = /bits/ 64 <2616000000>; opp-level = <6>; + opp-microwatt = <1782600>; }; opp07 { opp-hz = /bits/ 64 <2868000000>; opp-level = <7>; + opp-microwatt = <2209600>; }; opp08 { opp-hz = /bits/ 64 <3096000000>; opp-level = <8>; + opp-microwatt = <2636700>; }; opp09 { opp-hz = /bits/ 64 <3300000000>; opp-level = <9>; + opp-microwatt = <3247200>; }; opp10 { opp-hz = /bits/ 64 <3468000000>; opp-level = <10>; + opp-microwatt = <3668700>; }; opp11 { opp-hz = /bits/ 64 <3624000000>; opp-level = <11>; + opp-microwatt = <4128200>; }; opp12 { opp-hz = /bits/ 64 <3756000000>; opp-level = <12>; + opp-microwatt = <4819300>; }; opp13 { opp-hz = /bits/ 64 <3852000000>; opp-level = <13>; + opp-microwatt = <4977700>; }; opp14 { opp-hz = /bits/ 64 <3924000000>; opp-level = <14>; + opp-microwatt = <5315900>; }; opp15 { opp-hz = /bits/ 64 <3996000000>; opp-level = <15>; + opp-microwatt = <5575900>; }; opp16 { opp-hz = /bits/ 64 <4044000000>; opp-level = <16>; + opp-microwatt = <5773400>; }; opp17 { opp-hz = /bits/ 64 <4104000000>; opp-level = <17>; + opp-microwatt = <5969200>; }; opp18 { opp-hz = /bits/ 64 <4416000000>; opp-level = <18>; + opp-microwatt = <8274900>; }; opp19 { opp-hz = /bits/ 64 <4512000000>; opp-level = <19>; + opp-microwatt = <9256900>; }; }; @@ -422,78 +456,97 @@ opp01 { opp-hz = /bits/ 64 <1260000000>; opp-level = <1>; + opp-microwatt = <561800>; }; opp02 { opp-hz = /bits/ 64 <1512000000>; opp-level = <2>; + opp-microwatt = <772500>; }; opp03 { opp-hz = /bits/ 64 <1800000000>; opp-level = <3>; + opp-microwatt = <988900>; }; opp04 { opp-hz = /bits/ 64 <2088000000>; opp-level = <4>; + opp-microwatt = <1302300>; }; opp05 { opp-hz = /bits/ 64 <2352000000>; opp-level = <5>; + opp-microwatt = <1644100>; }; opp06 { opp-hz = /bits/ 64 <2616000000>; opp-level = <6>; + opp-microwatt = <2003700>; }; opp07 { opp-hz = /bits/ 64 <2868000000>; opp-level = <7>; + opp-microwatt = <2349000>; }; opp08 { opp-hz = /bits/ 64 <3096000000>; opp-level = <8>; + opp-microwatt = <2723000>; }; opp09 { opp-hz = /bits/ 64 <3300000000>; opp-level = <9>; + opp-microwatt = <3333200>; }; opp10 { opp-hz = /bits/ 64 <3468000000>; opp-level = <10>; + opp-microwatt = <3711900>; }; opp11 { opp-hz = /bits/ 64 <3624000000>; opp-level = <11>; + opp-microwatt = <4129900>; }; opp12 { opp-hz = /bits/ 64 <3756000000>; opp-level = <12>; + opp-microwatt = <4666300>; }; opp13 { opp-hz = /bits/ 64 <3852000000>; opp-level = <13>; + opp-microwatt = <4881000>; }; opp14 { opp-hz = /bits/ 64 <3924000000>; opp-level = <14>; + opp-microwatt = <5176700>; }; opp15 { opp-hz = /bits/ 64 <3996000000>; opp-level = <15>; + opp-microwatt = <5306500>; }; opp16 { opp-hz = /bits/ 64 <4044000000>; opp-level = <16>; + opp-microwatt = <5618900>; }; opp17 { opp-hz = /bits/ 64 <4104000000>; opp-level = <17>; + opp-microwatt = <5843100>; }; opp18 { opp-hz = /bits/ 64 <4416000000>; opp-level = <18>; + opp-microwatt = <8075300>; }; opp19 { opp-hz = /bits/ 64 <4512000000>; opp-level = <19>; + opp-microwatt = <8949200>; }; }; @@ -1295,6 +1348,15 @@ aop_las: las { compatible = "apple,t6041-aop-las", "apple,aop-las"; }; + + /* The calibration properties come from the bootloader. */ + aop_accel: accel { + compatible = "apple,t6041-aop-accel", "apple,aop-accel"; + }; + + aop_gyro: gyro { + compatible = "apple,t6041-aop-gyro", "apple,aop-gyro"; + }; }; /* ADT lists the interrupts as 1, 0, 3, 2 of the block: send-empty is the lowest. */ @@ -1377,6 +1439,7 @@ interrupt-parent = <&aic>; interrupts = ; #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; /* ADT "remap": streams 1 2 3 are stream 0 */ apple,remap-streams = <1 0>, <2 0>, <3 0>; status = "disabled"; @@ -1607,6 +1670,7 @@ interrupts = ; power-domains = <&ps_atc0_usb>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; status = "disabled"; }; @@ -1617,6 +1681,7 @@ interrupts = ; power-domains = <&ps_atc0_usb>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; status = "disabled"; }; @@ -1691,6 +1756,7 @@ interrupts = ; power-domains = <&ps_atc1_usb>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; status = "disabled"; }; @@ -1701,6 +1767,7 @@ interrupts = ; power-domains = <&ps_atc1_usb>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; status = "disabled"; }; @@ -1775,6 +1842,7 @@ interrupts = ; power-domains = <&ps_atc2_usb>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; status = "disabled"; }; @@ -1785,6 +1853,7 @@ interrupts = ; power-domains = <&ps_atc2_usb>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0x0 0xffff0000>; status = "disabled"; }; @@ -1816,6 +1885,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x2 0x83080000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x00004000 0x0 0xffffc000>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_venc0_sys>; @@ -1826,6 +1896,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x2 0x830a0000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x00004000 0x0 0xffffc000>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_venc0_sys>; @@ -1836,6 +1907,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x3 0x01020000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_scodec>; @@ -1873,6 +1945,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x3 0xa2300000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; interrupt-parent = <&aic>; interrupts = ; status = "disabled"; @@ -1882,6 +1955,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x3 0xa2340000 0x0 0x20000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; interrupt-parent = <&aic>; interrupts = ; status = "disabled"; @@ -1900,6 +1974,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x4 0x29060000 0x0 0x20000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x1c000 0x2ff 0xfffe4000>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_sio>; @@ -1913,6 +1988,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; + apple,dma-range = <0x100 0x0 0x300 0x0>; status = "disabled"; }; @@ -1923,6 +1999,7 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; + apple,dma-range = <0x100 0x0 0x300 0x0>; status = "disabled"; }; @@ -1933,6 +2010,18 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; + apple,dma-range = <0x100 0x0 0x300 0x0>; + status = "disabled"; + }; + + jpeg0: jpeg@487000000 { + compatible = "apple,t6041-jpeg"; + reg = <0x4 0x87000000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&jpeg0_dart 0>; + power-domains = <&ps_jpg>; + apple,dma-range = /bits/ 64 <0x10000000000 0x100000000>; status = "disabled"; }; @@ -1940,16 +2029,29 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x4 0x87020000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_jpg>; status = "disabled"; }; + jpeg1: jpeg@487060000 { + compatible = "apple,t6041-jpeg"; + reg = <0x4 0x87060000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&jpeg1_dart 0>; + power-domains = <&ps_jpg>; + apple,dma-range = /bits/ 64 <0x10000000000 0x100000000>; + status = "disabled"; + }; + jpeg1_dart: iommu@487080000 { compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x4 0x87080000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_jpg>; @@ -1960,6 +2062,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x5 0x21260000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_msr0>; @@ -1970,6 +2073,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x6 0x02300000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; interrupt-parent = <&aic>; interrupts = ; status = "disabled"; @@ -1979,6 +2083,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x6 0x02340000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; interrupt-parent = <&aic>; interrupts = ; status = "disabled"; @@ -2103,7 +2208,7 @@ power-domains = <&ps_qspi>; #address-cells = <1>; #size-cells = <0>; - /* The firmware NOR needs a qualified partition/write policy. */ + /* Boards enable it with their flash and its nvram partition. */ status = "disabled"; }; @@ -2137,12 +2242,24 @@ /* Firmware interface support is required before enabling the ANE. */ ane: ane@484000000 { compatible = "apple,t6041-ane"; - reg = <0x4 0x84000000 0x0 0x2000000>; - reg-names = "engine"; + reg = <0x4 0x84000000 0x0 0x2000000>, + <0x5 0x02280000 0x0 0x18000>, + <0x5 0x022a0000 0x0 0x4000>, + <0x5 0x003c0000 0x0 0x30000>, + <0x2 0x12000000 0x0 0xff4000>, + <0x5 0x22000000 0x0 0x2000000>, + <0x5 0x08374000 0x0 0x4000>; + reg-names = "engine", "pmgr", "aux-0", "aux-1", "aux-2", + "aux-3", "aux-4"; interrupt-parent = <&aic>; interrupts = ; - iommus = <&ane0_dart0 0>, <&ane0_dart1 0>, <&ane0_dart2 0>; - power-domains = <&ps_ane_cpu>; + iommus = <&ane0_dart0 0>, <&ane0_dart1 0>, <&ane0_dart2 0>, + <&ane0_dart0 11>, <&ane0_dart1 11>, <&ane0_dart2 11>; + power-domains = <&ps_ane_sys>, <&ps_ane_cpu>, <&ps_ane_mpm>, + <&ps_ane_td>, <&ps_ane_base>; + apple,ane-type = <0x110>; + apple,firmware-preloaded; + /* TEXT/DATA reservations and IOVAs must come from iBoot. */ status = "disabled"; }; @@ -2211,6 +2328,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x3 0x81a80000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; status = "disabled"; @@ -2236,6 +2354,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x3 0x89a80000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; status = "disabled"; @@ -2261,6 +2380,7 @@ compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x3 0x91a80000 0x0 0xc000>; #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x300 0x0>; interrupt-parent = <&aic>; interrupts = ; status = "disabled"; @@ -2269,3 +2389,50 @@ }; #include "t6040-pmgr.dtsi" + +/* + * DRAM interface class: width and maximum rate are inferred from topology + * and the published bandwidth class. Lower-state rates remain unqualified. + * State IDs below describe firmware DCS states, not PMGR codes. + */ +/ { + dram_opp: opp-table-dram { + compatible = "operating-points-v2-apple-dram"; + opp-shared; + + opp-0 { + opp-level = <0>; + }; + + opp-1 { + opp-level = <1>; + }; + + opp-2 { + opp-level = <2>; + }; + + opp-3 { + opp-level = <3>; + }; + + opp-4 { + opp-level = <4>; + }; + + opp-5 { + opp-level = <5>; + opp-hz = /bits/ 64 <4266666667>; + apple,data-rate = /bits/ 64 <8533333333>; + }; + }; +}; + +&pmp_report { + operating-points-v2 = <&dram_opp>; + apple,dram-type = "lpddr5"; + apple,data-bus-width = <256>; + apple,dcs-state-count = <6>; + apple,peak-bandwidth-bytes-per-second = /bits/ 64 <273066666656>; + apple,bandwidth-class-bytes-per-second = /bits/ 64 <273000000000>; +}; diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index eb03ebf173d76..8f6f3c6948710 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -251,6 +251,7 @@ }; actuator { + apple,click-strength = <0xc0 0x6e>; }; tp_accel { @@ -261,9 +262,9 @@ /* * Mesa fingerprint sensor on spi2. The J413 platform device tree * (/arm-io/spi2/mesa) describes it as J415's does: power on AP GPIO 178, - * active high, data-ready on GPIO 177. As on J415, the data-ready line is - * left out, so the driver polls until its trigger is confirmed. Native chip - * select only. J413 is untested. + * active high, data-ready on AP GPIO 177. The data-ready edge only wakes the + * capture early; every frame is still gated on the sensor's SPI status. + * Native chip select only. */ &spi2 { status = "okay"; @@ -272,6 +273,8 @@ &mesa { status = "okay"; enable-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <177 IRQ_TYPE_EDGE_RISING>; spi-cs-setup-delay-ns = <20>; spi-cs-hold-delay-ns = <20>; /* Per-device factory data from this Mac's own iBoot system container. */ diff --git a/arch/arm64/boot/dts/apple/t8112-j415.dts b/arch/arm64/boot/dts/apple/t8112-j415.dts index 2e2b982c584e0..cdcb4802396b5 100644 --- a/arch/arm64/boot/dts/apple/t8112-j415.dts +++ b/arch/arm64/boot/dts/apple/t8112-j415.dts @@ -286,8 +286,9 @@ #include "hwmon-laptop.dtsi" /* * Mesa fingerprint sensor on spi2 (J415 ADT /arm-io/spi2/mesa): power on AP - * GPIO 178, active high. The data-ready line (GPIO 177) is left out, so the - * driver polls until its trigger is confirmed. Native chip select only. + * GPIO 178, active high, and data-ready on AP GPIO 177. The data-ready edge + * only wakes the capture early; every frame is still gated on the sensor's + * SPI status. Native chip select only. */ &spi2 { status = "okay"; @@ -296,6 +297,8 @@ &mesa { status = "okay"; enable-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <177 IRQ_TYPE_EDGE_RISING>; spi-cs-setup-delay-ns = <20>; spi-cs-hold-delay-ns = <20>; /* Per-device factory data from this Mac's own iBoot system container. */ diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index 26ab30e219f12..9ff4ed911d482 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -1132,10 +1132,7 @@ power-domains = <&ps_venc_pipe5>, <&ps_venc_pipe4>; }; - /* - * These states serve only the ANE. They stay disabled with the ane - * node in t8112.dtsi; enable them together with it. - */ + /* These states serve only the ANE. */ ps_ane_sys_cpu: power-controller@c008 { compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc008 4>; @@ -1143,7 +1140,6 @@ #reset-cells = <0>; label = "ane_sys_cpu"; power-domains = <&ps_ane_sys>; - status = "disabled"; }; ps_ane_td: power-controller@c010 { @@ -1153,7 +1149,6 @@ #reset-cells = <0>; label = "ane_td"; power-domains = <&ps_ane_sys_cpu>; - status = "disabled"; }; ps_ane_base: power-controller@c018 { @@ -1163,7 +1158,6 @@ #reset-cells = <0>; label = "ane_base"; power-domains = <&ps_ane_td>; - status = "disabled"; }; ps_ane_set1: power-controller@c020 { @@ -1173,7 +1167,6 @@ #reset-cells = <0>; label = "ane_set1"; power-domains = <&ps_ane_base>; - status = "disabled"; }; ps_ane_set2: power-controller@c028 { @@ -1183,7 +1176,6 @@ #reset-cells = <0>; label = "ane_set2"; power-domains = <&ps_ane_set1>; - status = "disabled"; }; ps_ane_set3: power-controller@c030 { @@ -1193,7 +1185,6 @@ #reset-cells = <0>; label = "ane_set3"; power-domains = <&ps_ane_set2>; - status = "disabled"; }; ps_ane_set4: power-controller@c038 { @@ -1203,7 +1194,6 @@ #reset-cells = <0>; label = "ane_set4"; power-domains = <&ps_ane_set3>; - status = "disabled"; }; ps_disp0_cpu0: power-controller@10000 { diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 90f9311d4aede..23369ae0246a2 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -505,6 +505,45 @@ /* Required to get >32-bit DMA via DARTs */ dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; + ave_dart0: iommu@67030000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0 0x67030000 0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_venc_sys>; + #iommu-cells = <1>; + status = "disabled"; + }; + + ave_dart1: iommu@67040000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0 0x67040000 0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_venc_sys>; + #iommu-cells = <1>; + status = "disabled"; + }; + + ave: video-codec@67100000 { + compatible = "apple,t8112-ave"; + reg = <0 0x67100000 0 0x45c000>, + <0 0x67050000 0 0x8000>, + <0 0x67800000 0 0x800000>; + reg-names = "codec", "mailbox", "coproc"; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + ; + power-domains = <&ps_venc_dma>, <&ps_venc_pipe4>, + <&ps_venc_pipe5>, <&ps_venc_me0>, <&ps_venc_me1>; + power-domain-names = "dma", "pipe4", "pipe5", "me0", "me1"; + iommus = <&ave_dart0 0>, <&ave_dart1 0>; + status = "disabled"; + }; + gpu: gpu@206400000 { compatible = "apple,agx-t8112", "apple,agx-g14g"; reg = <0x2 0x6400000 0 0x40000>, @@ -1749,7 +1788,6 @@ interrupts = ; interrupt-names = "recv-not-empty"; #mbox-cells = <0>; - status = "disabled"; }; ane_dart0: iommu@26b800000 { @@ -1759,7 +1797,6 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - status = "disabled"; }; ane_dart1: iommu@26b810000 { @@ -1769,7 +1806,6 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - status = "disabled"; }; ane_dart2: iommu@26b820000 { @@ -1779,7 +1815,6 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - status = "disabled"; }; ane: ane@26a000000 { @@ -1797,7 +1832,6 @@ <&ps_ane_base>, <&ps_ane_set1>, <&ps_ane_set2>, <&ps_ane_set3>, <&ps_ane_set4>; mboxes = <&ane_mbox>; - status = "disabled"; }; dispext0_dart: iommu@271304000 { @@ -2023,6 +2057,7 @@ apciec0: pcie@3b0000000 { compatible = "apple,t8112-pciec", "apple,t8103-pciec"; + apple,pciec-kernel-init; device_type = "pci"; reg = <0x3 0xb0000000 0x0 0x10000000>, @@ -2243,6 +2278,7 @@ apciec1: pcie@530000000 { compatible = "apple,t8112-pciec", "apple,t8103-pciec"; + apple,pciec-kernel-init; device_type = "pci"; reg = <0x5 0x30000000 0x0 0x10000000>, @@ -2522,3 +2558,46 @@ #include "t8112-pmgr.dtsi" #include "hwmon-common.dtsi" + +/* + * DRAM interface class: width and maximum rate are inferred from topology + * and the published bandwidth class. Lower-state rates remain unqualified. + * State IDs below describe firmware DCS states, not PMGR codes. + */ +/ { + dram_opp: opp-table-dram { + compatible = "operating-points-v2-apple-dram"; + opp-shared; + + opp-0 { + opp-level = <0>; + }; + + opp-1 { + opp-level = <1>; + }; + + opp-2 { + opp-level = <2>; + }; + + opp-3 { + opp-level = <3>; + }; + + opp-4 { + opp-level = <4>; + opp-hz = /bits/ 64 <3200000000>; + apple,data-rate = /bits/ 64 <6400000000>; + }; + }; +}; + +&pmp_report { + operating-points-v2 = <&dram_opp>; + apple,dram-type = "lpddr5"; + apple,data-bus-width = <128>; + apple,dcs-state-count = <5>; + apple,peak-bandwidth-bytes-per-second = /bits/ 64 <102400000000>; + apple,bandwidth-class-bytes-per-second = /bits/ 64 <100000000000>; +}; diff --git a/arch/arm64/boot/dts/apple/t8122-j433.dts b/arch/arm64/boot/dts/apple/t8122-j433.dts index 9ceb6a6ebcdf2..d4803de20e04d 100644 --- a/arch/arm64/boot/dts/apple/t8122-j433.dts +++ b/arch/arm64/boot/dts/apple/t8122-j433.dts @@ -16,6 +16,7 @@ / { compatible = "apple,j433", "apple,t8122", "apple,arm-platform"; model = "Apple iMac (24-inch, 2x USB-C, M3, 2023)"; + chassis-type = "all-in-one"; aliases { ethernet0 = ðernet0; diff --git a/arch/arm64/boot/dts/apple/t8122-j434.dts b/arch/arm64/boot/dts/apple/t8122-j434.dts index 1e5d75aa02b41..741833526764e 100644 --- a/arch/arm64/boot/dts/apple/t8122-j434.dts +++ b/arch/arm64/boot/dts/apple/t8122-j434.dts @@ -16,6 +16,7 @@ / { compatible = "apple,j434", "apple,t8122", "apple,arm-platform"; model = "Apple iMac (24-inch, 4x USB-C, M3, 2023)"; + chassis-type = "all-in-one"; }; &wifi0 { diff --git a/arch/arm64/boot/dts/apple/t8122-j504.dts b/arch/arm64/boot/dts/apple/t8122-j504.dts index 4c36e01bae641..58790e56c0cf0 100644 --- a/arch/arm64/boot/dts/apple/t8122-j504.dts +++ b/arch/arm64/boot/dts/apple/t8122-j504.dts @@ -17,6 +17,7 @@ / { compatible = "apple,j504", "apple,t8122", "apple,arm-platform"; model = "Apple MacBook Pro (14-inch, M3, 2023)"; + chassis-type = "laptop"; led-controller { compatible = "pwm-leds"; diff --git a/arch/arm64/boot/dts/apple/t8122-j613-25g83-qspi.dtsi b/arch/arm64/boot/dts/apple/t8122-j613-25g83-qspi.dtsi index 2a46a7da98168..e37fde644ad03 100644 --- a/arch/arm64/boot/dts/apple/t8122-j613-25g83-qspi.dtsi +++ b/arch/arm64/boot/dts/apple/t8122-j613-25g83-qspi.dtsi @@ -31,6 +31,7 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <10000000>; + vcc-supply = <&nor_flash_vcc>; partitions { compatible = "fixed-partitions"; diff --git a/arch/arm64/boot/dts/apple/t8122-j613.dts b/arch/arm64/boot/dts/apple/t8122-j613.dts index 330a02fe9c32a..3768661807e96 100644 --- a/arch/arm64/boot/dts/apple/t8122-j613.dts +++ b/arch/arm64/boot/dts/apple/t8122-j613.dts @@ -17,6 +17,7 @@ / { compatible = "apple,j613", "apple,t8122", "apple,arm-platform"; model = "Apple MacBook Air (13-inch, M3, 2024)"; + chassis-type = "laptop"; aliases { sep = &sep; diff --git a/arch/arm64/boot/dts/apple/t8122-j615.dts b/arch/arm64/boot/dts/apple/t8122-j615.dts index 5e52cd3046e71..34b6db85b4410 100644 --- a/arch/arm64/boot/dts/apple/t8122-j615.dts +++ b/arch/arm64/boot/dts/apple/t8122-j615.dts @@ -17,6 +17,7 @@ / { compatible = "apple,j615", "apple,t8122", "apple,arm-platform"; model = "Apple MacBook Air (15-inch, M3, 2024)"; + chassis-type = "laptop"; aliases { sep = &sep; diff --git a/arch/arm64/boot/dts/apple/t8132-j604.dts b/arch/arm64/boot/dts/apple/t8132-j604.dts index c4be915cb33ab..8a065a6f47a7b 100644 --- a/arch/arm64/boot/dts/apple/t8132-j604.dts +++ b/arch/arm64/boot/dts/apple/t8132-j604.dts @@ -16,6 +16,7 @@ / { compatible = "apple,j604", "apple,t8132", "apple,arm-platform"; model = "Apple MacBook Pro (14-inch, M4, 2024)"; + chassis-type = "laptop"; led-controller { compatible = "pwm-leds"; diff --git a/arch/arm64/boot/dts/apple/t8132-j623.dts b/arch/arm64/boot/dts/apple/t8132-j623.dts index 508e26056022c..9b94a6ae74a20 100644 --- a/arch/arm64/boot/dts/apple/t8132-j623.dts +++ b/arch/arm64/boot/dts/apple/t8132-j623.dts @@ -15,4 +15,5 @@ / { compatible = "apple,j623", "apple,t8132", "apple,arm-platform"; model = "Apple iMac (24-inch, 2x USB-C, M4, 2024)"; + chassis-type = "all-in-one"; }; diff --git a/arch/arm64/boot/dts/apple/t8132-j624.dts b/arch/arm64/boot/dts/apple/t8132-j624.dts index 84bdd8e343eb1..37e1a46f89b84 100644 --- a/arch/arm64/boot/dts/apple/t8132-j624.dts +++ b/arch/arm64/boot/dts/apple/t8132-j624.dts @@ -15,4 +15,5 @@ / { compatible = "apple,j624", "apple,t8132", "apple,arm-platform"; model = "Apple iMac (24-inch, 4x USB-C, M4, 2024)"; + chassis-type = "all-in-one"; }; diff --git a/arch/arm64/boot/dts/apple/t8132-j713.dts b/arch/arm64/boot/dts/apple/t8132-j713.dts index da55888e6d5e4..f62defca0dae5 100644 --- a/arch/arm64/boot/dts/apple/t8132-j713.dts +++ b/arch/arm64/boot/dts/apple/t8132-j713.dts @@ -20,6 +20,7 @@ / { compatible = "apple,j713", "apple,t8132", "apple,arm-platform"; model = "Apple MacBook Air (13-inch, M4, 2025)"; + chassis-type = "laptop"; aliases { ans = &nvme; @@ -54,6 +55,16 @@ default-state = "keep"; }; }; + + /* + * The boot NOR runs from a PMU rail that is on whenever the SoC is + * and that no OS switches. The ADT gives the flash no supply. + */ + nor_flash_vcc: regulator-nor-flash-vcc { + compatible = "regulator-fixed"; + regulator-name = "nor-flash-vcc"; + regulator-always-on; + }; }; /* Enable DVFS only with the J713 board thermal owner. */ @@ -72,6 +83,7 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <10000000>; + vcc-supply = <&nor_flash_vcc>; spi-rx-bus-width = <1>; spi-tx-bus-width = <1>; diff --git a/arch/arm64/boot/dts/apple/t8132-j715.dts b/arch/arm64/boot/dts/apple/t8132-j715.dts index 2672914ea76c9..1a387d273e8ff 100644 --- a/arch/arm64/boot/dts/apple/t8132-j715.dts +++ b/arch/arm64/boot/dts/apple/t8132-j715.dts @@ -16,6 +16,7 @@ / { compatible = "apple,j715", "apple,t8132", "apple,arm-platform"; model = "Apple MacBook Air (15-inch, M4, 2025)"; + chassis-type = "laptop"; led-controller { compatible = "pwm-leds"; diff --git a/arch/arm64/boot/dts/apple/t8132-j773g.dts b/arch/arm64/boot/dts/apple/t8132-j773g.dts index 0583cb9d19f62..2f16f8e62c06c 100644 --- a/arch/arm64/boot/dts/apple/t8132-j773g.dts +++ b/arch/arm64/boot/dts/apple/t8132-j773g.dts @@ -15,6 +15,7 @@ / { compatible = "apple,j773g", "apple,t8132", "apple,arm-platform"; model = "Apple Mac mini (M4, 2024)"; + chassis-type = "desktop"; }; /* diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index ece0d9c6bfefb..4034b774388c2 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -15,6 +15,7 @@ / { compatible = "apple,j700", "apple,t8140", "apple,arm-platform"; model = "Apple MacBook Neo"; + chassis-type = "laptop"; aliases { nvram = &nvram; @@ -69,6 +70,16 @@ reg = <0x100 0 0x2 0>; /* To be filled by loader */ }; + /* + * The boot NOR runs from a PMU rail that is on whenever the SoC is + * and that no OS switches. The ADT gives the flash no supply. + */ + nor_flash_vcc: regulator-nor-flash-vcc { + compatible = "regulator-fixed"; + regulator-name = "nor-flash-vcc"; + regulator-always-on; + }; + /* * The J700 is fanless: the CPUs are the only cooling devices. Every * die sensor gets a zone that throttles both clusters, since they share @@ -384,10 +395,33 @@ }; }; }; + + /* + * The headphone codec runs from PMU rails that are on whenever the + * SoC is and that no OS switches. The ADT gives the codec a reset + * line and no supply or power function. + */ + jack_codec_supply: regulator-jack-codec { + compatible = "regulator-fixed"; + regulator-name = "jack-codec"; + regulator-always-on; + }; }; #include "isp-imx558.dtsi" +&ave_dart0 { + status = "okay"; +}; + +&ave_dart1 { + status = "okay"; +}; + +&ave { + status = "okay"; +}; + &serial0 { status = "okay"; }; @@ -448,6 +482,7 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <25000000>; + vcc-supply = <&nor_flash_vcc>; partitions { compatible = "fixed-partitions"; @@ -490,6 +525,11 @@ jack_codec: codec@48 { compatible = "cirrus,cs42l83"; reg = <0x48>; + VA-supply = <&jack_codec_supply>; + VCP-supply = <&jack_codec_supply>; + VD_FILT-supply = <&jack_codec_supply>; + VL-supply = <&jack_codec_supply>; + VP-supply = <&jack_codec_supply>; reset-gpios = <&pinctrl_ap 103 GPIO_ACTIVE_HIGH>; interrupt-parent = <&pinctrl_ap>; interrupts = <104 IRQ_TYPE_LEVEL_LOW>; @@ -613,6 +653,8 @@ }; &smc_hwmon { + apple,thermal-pressure-key = "mTPL"; + /* * Further channels, reported to userspace. They follow the six CPU * die temperatures in t8140.dtsi, which the thermal zones above @@ -1812,9 +1854,23 @@ #include "hwmon-common.dtsi" #include "hwmon-laptop.dtsi" +/* + * The J700 SMC has no keys for these shared laptop sensors. On T8140 the + * temperature channels are indexed for the thermal zones, so a missing + * temperature key keeps an empty, hidden channel and is reported at every + * boot. The NAND temperature is described above as Storage (NAND). + */ +&smc_hwmon { + /delete-node/ power-PMVR; + /delete-node/ power-PHPC; + /delete-node/ temperature-TH0x; + /delete-node/ temperature-TCHP; + /delete-node/ temperature-TW0P; +}; + &pcie0 { /* The PIODMA guard stays closed until the retained bootstrap succeeds. */ - apple,piodma = <&pcie0_piodma_diag>; + apple,piodma = <&pcie0_bootstrap>; iommu-map = <0x100 &pcie0_dart 1 2>; iommu-map-mask = <0xffff>; pinctrl-0 = <&pcie0_clkreq>; @@ -1842,8 +1898,9 @@ }; / { - pcie0_piodma_diag: piodma@390030000 { - compatible = "apple,t8140-piodma-diagnostic"; + pcie0_bootstrap: piodma@390030000 { + compatible = "apple,t8140-pcie-bootstrap", + "apple,t8140-piodma-diagnostic"; reg = <0x3 0x90030000 0x0 0x4000>, <0x3 0x90034000 0x0 0x4000>, <0x3 0x90038000 0x0 0x4000>; diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 8a1864e0fabbd..0a479f7e8e8e5 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -69,7 +69,6 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "venc0_sys"; - status = "disabled"; }; ps_apcie_gp: power-controller@258 { @@ -408,7 +407,6 @@ #reset-cells = <0>; label = "venc0_dma"; power-domains = <&ps_venc0_sys>; - status = "disabled"; }; ps_venc0_pipe4: power-controller@8008 { @@ -418,7 +416,6 @@ #reset-cells = <0>; label = "venc0_pipe4"; power-domains = <&ps_venc0_dma>; - status = "disabled"; }; ps_venc0_pipe5: power-controller@8010 { @@ -428,7 +425,6 @@ #reset-cells = <0>; label = "venc0_pipe5"; power-domains = <&ps_venc0_dma>; - status = "disabled"; }; ps_venc0_me0: power-controller@8018 { @@ -438,7 +434,6 @@ #reset-cells = <0>; label = "venc0_me0"; power-domains = <&ps_venc0_dma>; - status = "disabled"; }; ps_venc0_me1: power-controller@8020 { @@ -448,7 +443,6 @@ #reset-cells = <0>; label = "venc0_me1"; power-domains = <&ps_venc0_me0>; - status = "disabled"; }; }; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 9fdd91a302412..cbdbf4ab8a36b 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -448,6 +448,17 @@ nonposted-mmio; ranges; + jpeg0: jpeg@28d000000 { + compatible = "apple,t8140-jpeg"; + reg = <0x2 0x8d000000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&jpeg_dart 0>; + power-domains = <&ps_jpg>; + apple,dma-range = /bits/ 64 <0x10000000000 0x100000000>; + status = "disabled"; + }; + jpeg_dart: iommu@28d020000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x2 0x8d020000 0x0 0xc000>; @@ -459,6 +470,28 @@ status = "disabled"; }; + jpeg1: jpeg@28d060000 { + compatible = "apple,t8140-jpeg"; + reg = <0x2 0x8d060000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&jpeg1_dart 0>; + power-domains = <&ps_jpg>; + apple,dma-range = /bits/ 64 <0x10000000000 0x100000000>; + status = "disabled"; + }; + + jpeg1_dart: iommu@28d080000 { + compatible = "apple,t8140-dart", "apple,t8110-dart"; + reg = <0x2 0x8d080000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = /bits/ 64 <0x10000000000 0x30000000000>; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + apr_dart: iommu@295020000 { compatible = "apple,t8140-dart", "apple,t8110-dart"; reg = <0x2 0x95020000 0x0 0x20000>; @@ -982,7 +1015,7 @@ interrupts = ; power-domains = <&ps_venc0_sys>; #iommu-cells = <1>; - apple,dma-range = <0x100 0x00004000 0 0x0fffc000>; + apple,dma-range = <0x100 0x00004000 0 0xffffc000>; status = "disabled"; }; @@ -993,7 +1026,7 @@ interrupts = ; power-domains = <&ps_venc0_sys>; #iommu-cells = <1>; - apple,dma-range = <0x100 0x00004000 0 0x0fffc000>; + apple,dma-range = <0x100 0x00004000 0 0xffffc000>; status = "disabled"; }; @@ -1778,3 +1811,46 @@ }; #include "t8140-pmgr.dtsi" + +/* + * DRAM interface class: width and maximum rate are inferred from topology + * and the published bandwidth class. Lower-state rates remain unqualified. + * State IDs below describe firmware DCS states, not PMGR codes. + */ +/ { + dram_opp: opp-table-dram { + compatible = "operating-points-v2-apple-dram"; + opp-shared; + + opp-0 { + opp-level = <0>; + }; + + opp-1 { + opp-level = <1>; + }; + + opp-2 { + opp-level = <2>; + }; + + opp-3 { + opp-level = <3>; + }; + + opp-4 { + opp-level = <4>; + opp-hz = /bits/ 64 <3750000000>; + apple,data-rate = /bits/ 64 <7500000000>; + }; + }; +}; + +&pmp_report { + operating-points-v2 = <&dram_opp>; + apple,dram-type = "lpddr5"; + apple,data-bus-width = <64>; + apple,dcs-state-count = <5>; + apple,peak-bandwidth-bytes-per-second = /bits/ 64 <60000000000>; + apple,bandwidth-class-bytes-per-second = /bits/ 64 <60000000000>; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 20df27fa1bfe3..576c4637f37df 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -14,6 +14,7 @@ / { compatible = "apple,j813", "apple,t8142", "apple,arm-platform"; model = "Apple MacBook Air (13-inch, M5, 2026)"; + chassis-type = "laptop"; #address-cells = <2>; #size-cells = <2>; diff --git a/arch/arm64/boot/dts/apple/t8152-j873g.dts b/arch/arm64/boot/dts/apple/t8152-j873g.dts index 6b5e14036c98a..0a0818d85d085 100644 --- a/arch/arm64/boot/dts/apple/t8152-j873g.dts +++ b/arch/arm64/boot/dts/apple/t8152-j873g.dts @@ -14,6 +14,7 @@ / { compatible = "apple,j873g", "apple,t8152", "apple,arm-platform"; model = "Apple Mac mini (M6)"; + chassis-type = "desktop"; #address-cells = <2>; #size-cells = <2>; diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 25c4ee4183cc6..49f8a5d2b4b56 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -27,7 +27,7 @@ CONFIG_NVME_APPLE=m CONFIG_MMC_SDHCI_PCI=m CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y -CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_PCIE_APPLE_T8140_BOOTSTRAP=y CONFIG_PCIEASPM=y CONFIG_NET_VENDOR_INTEL=y CONFIG_IGB=m @@ -51,7 +51,10 @@ CONFIG_APPLE_WATCHDOG=y # m1n1 hands the ADT and its own log over as phram regions (/dev/mtd*). CONFIG_MTD=y CONFIG_MTD_PHRAM=m +# Apple NVRAM partitions are 4 KiB aligned (J700: 0x24000). +CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y CONFIG_VIDEO_APPLE_AVD=m +CONFIG_VIDEO_APPLE_AVE=m CONFIG_VIDEO_APPLE_ISP=m CONFIG_DRM=y CONFIG_DRM_ASAHI=m @@ -117,6 +120,7 @@ CONFIG_APPLE_PMP_THERMAL=m CONFIG_APPLE_T8132_PLATFORM_HANDOFF=y CONFIG_IIO_AOP_SENSOR_LAS=m CONFIG_IIO_AOP_SENSOR_ALS=m +CONFIG_IIO_AOP_SENSOR_MOTION=m CONFIG_RUST_FW_LOADER_ABSTRACTIONS=y CONFIG_PWM_APPLE=m CONFIG_APPLE_AIC=y diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 45c63f3d75c53..ba7efe1b93258 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -24,6 +24,8 @@ #include #include #include +#include +#include /* * In case the boot CPU is hotpluggable, we record its initial state and @@ -219,6 +221,58 @@ static const char *const compat_hwcap2_str[] = { }; #endif /* CONFIG_COMPAT */ +/* + * The processor's name as the firmware gives it: the Version string of the + * SMBIOS Processor Information (type 4), e.g. "Apple M4 Pro". x86 prints the + * CPU's brand string as "model name"; arm64 CPUs have no brand string, so the + * firmware's name is the only one there is, and desktop tools (GNOME's About + * page, KDE Info Center, inxi, lscpu for cores it has no name for) read it + * from here. Empty, and the line left out, when the firmware gives none. + */ +static char cpuinfo_model_name[64]; + +/* + * Copy the Version string of an SMBIOS type 4 structure into @buf, without + * leading or trailing white space. Leaves @buf empty if the structure is too + * short to have the field, the field names no string or the string is blank. + */ +static void dmi_processor_version(const struct dmi_header *dm, + char *buf, size_t size) +{ + const char *s = (const char *)dm + dm->length; + u8 index; + + buf[0] = '\0'; + if (dm->type != DMI_ENTRY_PROCESSOR || dm->length <= 0x10) + return; + + /* String numbers count from 1 in the unformatted area after the structure */ + index = ((const u8 *)dm)[0x10]; + if (!index) + return; + while (--index && *s) + s += strlen(s) + 1; + if (!*s) + return; + + strscpy(buf, skip_spaces(s), size); + strim(buf); +} + +static void __init cpuinfo_find_model_name(const struct dmi_header *dm, void *data) +{ + /* The first processor names the package; later ones repeat it */ + if (dm->type == DMI_ENTRY_PROCESSOR && !cpuinfo_model_name[0]) + dmi_processor_version(dm, cpuinfo_model_name, sizeof(cpuinfo_model_name)); +} + +static int __init cpuinfo_model_name_init(void) +{ + dmi_walk(cpuinfo_find_model_name, NULL); + return 0; +} +late_initcall(cpuinfo_model_name_init); + static int c_show(struct seq_file *m, void *v) { int j; @@ -236,6 +290,8 @@ static int c_show(struct seq_file *m, void *v) if (compat) seq_printf(m, "model name\t: ARMv8 Processor rev %d (%s)\n", MIDR_REVISION(midr), COMPAT_ELF_PLATFORM); + else if (cpuinfo_model_name[0]) + seq_printf(m, "model name\t: %s\n", cpuinfo_model_name); seq_printf(m, "BogoMIPS\t: %lu.%02lu\n", loops_per_jiffy / (500000UL/HZ), @@ -542,3 +598,7 @@ void __init cpuinfo_store_boot_cpu(void) boot_cpu_data = *info; init_cpu_features(&boot_cpu_data); } + +#if IS_ENABLED(CONFIG_ARM64_CPUINFO_KUNIT_TEST) +#include "cpuinfo_kunit.c" +#endif diff --git a/arch/arm64/kernel/cpuinfo_kunit.c b/arch/arm64/kernel/cpuinfo_kunit.c new file mode 100644 index 0000000000000..0c4731fc05060 --- /dev/null +++ b/arch/arm64/kernel/cpuinfo_kunit.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the "model name" that /proc/cpuinfo takes from the SMBIOS + * Processor Information. Included from cpuinfo.c, so it can test its static + * parser. + */ +#include + +/* A type 4 structure of @len formatted bytes, Version string number @version */ +static size_t make_type4(u8 *buf, u8 len, u8 version, const char *strings, size_t slen) +{ + memset(buf, 0, len); + buf[0] = DMI_ENTRY_PROCESSOR; + buf[1] = len; + if (len > 0x10) + buf[0x10] = version; + memcpy(buf + len, strings, slen); + return len + slen; +} + +static void parse(u8 len, u8 version, const char *strings, size_t slen, char *out, size_t size) +{ + u8 buf[128]; + + make_type4(buf, len, version, strings, slen); + dmi_processor_version((const struct dmi_header *)buf, out, size); +} + +#define STRINGS(s) s, sizeof(s) + +static void cpuinfo_model_name_first_string(struct kunit *test) +{ + char out[64]; + + parse(0x30, 1, STRINGS("Apple M4 Pro\0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, "Apple M4 Pro"); +} + +static void cpuinfo_model_name_later_string(struct kunit *test) +{ + char out[64]; + + /* socket, manufacturer, version: the version is string 3 */ + parse(0x30, 3, STRINGS("SoC\0Apple Inc.\0Apple A18 Pro\0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, "Apple A18 Pro"); +} + +static void cpuinfo_model_name_trimmed(struct kunit *test) +{ + char out[64]; + + parse(0x30, 1, STRINGS(" Apple M2 \t\0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, "Apple M2"); +} + +static void cpuinfo_model_name_absent(struct kunit *test) +{ + char out[64] = "stale"; + + /* no string number */ + parse(0x30, 0, STRINGS("Apple M1\0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, ""); + + /* a string number past the strings there are */ + strscpy(out, "stale", sizeof(out)); + parse(0x30, 4, STRINGS("SoC\0Apple Inc.\0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, ""); + + /* only blanks */ + strscpy(out, "stale", sizeof(out)); + parse(0x30, 1, STRINGS(" \0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, ""); + + /* a structure too short to have the Version field */ + strscpy(out, "stale", sizeof(out)); + parse(0x10, 1, STRINGS("Apple M1\0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, ""); +} + +static void cpuinfo_model_name_other_type(struct kunit *test) +{ + char out[64] = "stale"; + u8 buf[128]; + + make_type4(buf, 0x30, 1, STRINGS("Apple M4 Pro\0\0")); + buf[0] = 1; /* System Information */ + dmi_processor_version((const struct dmi_header *)buf, out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, ""); +} + +static void cpuinfo_model_name_truncated(struct kunit *test) +{ + char out[8]; + + parse(0x30, 1, STRINGS("Apple M4 Max\0\0"), out, sizeof(out)); + KUNIT_EXPECT_STREQ(test, out, "Apple M"); +} + +static struct kunit_case cpuinfo_model_name_cases[] = { + KUNIT_CASE(cpuinfo_model_name_first_string), + KUNIT_CASE(cpuinfo_model_name_later_string), + KUNIT_CASE(cpuinfo_model_name_trimmed), + KUNIT_CASE(cpuinfo_model_name_absent), + KUNIT_CASE(cpuinfo_model_name_other_type), + KUNIT_CASE(cpuinfo_model_name_truncated), + {} +}; + +static struct kunit_suite cpuinfo_model_name_suite = { + .name = "arm64-cpuinfo-model-name", + .test_cases = cpuinfo_model_name_cases, +}; +kunit_test_suite(cpuinfo_model_name_suite); diff --git a/drivers/accel/Makefile b/drivers/accel/Makefile index 235ebf723025c..2aa8f41d91dfc 100644 --- a/drivers/accel/Makefile +++ b/drivers/accel/Makefile @@ -2,6 +2,8 @@ obj-$(CONFIG_DRM_ACCEL_AMDXDNA) += amdxdna/ obj-$(CONFIG_DRM_ACCEL_ANE) += ane/ +obj-$(CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST) += ane/ +obj-$(CONFIG_DRM_ACCEL_ANE_PM_KUNIT_TEST) += ane/ obj-$(CONFIG_DRM_ACCEL_ARM_ETHOSU) += ethosu/ obj-$(CONFIG_DRM_ACCEL_HABANALABS) += habanalabs/ obj-$(CONFIG_DRM_ACCEL_IVPU) += ivpu/ diff --git a/drivers/accel/ane/Kconfig b/drivers/accel/ane/Kconfig index a14e621707a54..2e804e63199ce 100644 --- a/drivers/accel/ane/Kconfig +++ b/drivers/accel/ane/Kconfig @@ -7,11 +7,16 @@ config DRM_ACCEL_ANE depends on APPLE_RTKIT depends on APPLE_DART || COMPILE_TEST depends on MMU + select INTERCONNECT + select PM_OPP select CRYPTO_LIB_SHA256 + select DRM_ACCEL_ANE_H16_VALIDATION help Choose this option if you have an Apple SoC with a Neural Engine: M1 (T8103), M1 Pro, Max and Ultra (T6000, T6001, T6002), M2 (T8112), or M2 Pro, Max and Ultra (T6020, T6021, T6022). + H16 (M4 Pro) and H17 are recognized for firmware preflight and + ownership refusal only; execution support is not available. The interface exposed to userspace is described in include/uapi/drm/ane_accel.h. @@ -33,3 +38,32 @@ config DRM_ACCEL_ANE_KUNIT_TEST past them and with a size product that overflows. If unsure, say N. + +config DRM_ACCEL_ANE_H16_VALIDATION + tristate + select CRYPTO_LIB_SHA256 + +config DRM_ACCEL_ANE_FW_KUNIT_TEST + tristate "KUnit tests for ANE firmware validation" if !KUNIT_ALL_TESTS + depends on DRM_ACCEL && KUNIT + select DRM_ACCEL_ANE_H16_VALIDATION + default KUNIT_ALL_TESTS + help + Check preload ranges, 27.x channel and reply validation, RTKit + management ordering, H16 HWX/resource bounds and UUID retirement + without hardware. + These tests can run on architectures other than Apple arm64. + + If unsure, say N. + +config DRM_ACCEL_ANE_PM_KUNIT_TEST + tristate "KUnit tests for ANE runtime-PM recovery" if !KUNIT_ALL_TESTS + depends on DRM_ACCEL && KUNIT && PM + default KUNIT_ALL_TESTS + help + Exercise supplier suspension, power-off verification failure and + runtime resume errors against the runtime-PM and device-link core. + No ANE hardware is required. These tests can run independently of + the ANE driver on non-Apple platforms. + + If unsure, say N. diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile index 8103e33a112a4..1d830b9af9235 100644 --- a/drivers/accel/ane/Makefile +++ b/drivers/accel/ane/Makefile @@ -1,15 +1,21 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_DRM_ACCEL_ANE) += ane.o ane_t6021.o +obj-$(CONFIG_DRM_ACCEL_ANE_PM_KUNIT_TEST) += ane_pm_test.o +obj-$(CONFIG_DRM_ACCEL_ANE_H16_VALIDATION) += ane_h16_hwx.o obj-$(CONFIG_DRM_ACCEL_ANE_KUNIT_TEST) += ane_drv_test.o +obj-$(CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST) += ane_fw_platform_test.o \ + ane_fw_protocol_test.o ane_h16_tasks_test.o ane_h16_execute_test.o ane_h16_pending_test.o ane_h16_hwx_test.o ane-y := \ ane_drv.o \ ane_tm.o \ ane_dart.o \ + ane_perf.o \ ane_boost.o ane_t6021-y := \ + ane_fw_platform.o \ ane_t6021_rtclient_main.o \ ane_t6021_fwload.o \ ane_t6021_boot.o diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index f974f8a31194e..913fa0a7a8d94 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -64,6 +64,8 @@ struct ane_device { * and protects bo_list, preserved_list and removed. */ struct mutex engine_lock; + /* Serializes runtime-PM references across a recovery cycle. */ + struct mutex pm_lock; struct list_head bo_list; /* @@ -110,11 +112,32 @@ struct ane_device { */ bool tm_retention; + /* Protected independently of engine_lock so idle work can release votes. */ + struct ane_perf { + struct mutex lock; /* protects votes, idle work and status */ + struct delayed_work idle; + struct device_node *report; + struct icc_path *memory; + unsigned int high_level; + unsigned int level; + u32 avg_bw; + u32 peak_bw; + int error; + unsigned int release_retries; + bool initialized; + bool busy; + bool voted; + bool clock_held; + bool pinned; + bool cpu_boost; + } perf; + /* * Engine-busy CPU cluster boost (ane_boost.c): min-frequency QoS on * every cpufreq policy while a submit runs and until boost_idle_ms - * after the last one completes: on T8103 bandwidth-bound programs - * run ~2x slower while schedutil parks the idle clusters low. + * after the last one completes. Used only when the DT binds no + * performance controller, so the engine keeps the boost it had + * until it gets a clock level instead. */ struct ane_boost { struct mutex lock; /* protects the fields below */ @@ -159,6 +182,12 @@ int ane_submit_check_sizes(const struct drm_ane_submit *args, size_t cmd_size, */ void ane_reclaim_preserved(struct ane_device *ane); +int ane_perf_init(struct ane_device *ane, unsigned int high_level); +int ane_perf_exit(struct ane_device *ane); +int ane_perf_begin(struct ane_device *ane); +void ane_perf_end(struct ane_device *ane); +int ane_perf_suspend(struct ane_device *ane); + int ane_boost_init(struct ane_device *ane); void ane_boost_exit(struct ane_device *ane); void ane_boost_begin(struct ane_device *ane); diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 1b61d3f981bd9..120d900973cd2 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -690,17 +690,24 @@ static int ane_submit(struct drm_device *drm, void *data, struct drm_file *file) goto unlock; } + err = ane_perf_begin(ane); + if (err) { + dev_err_ratelimited(ane->dev, "performance request failed: %d\n", err); + ane_perf_end(ane); + goto unlock; + } err = ane_tm_enqueue(ane, &req); - if (err < 0) + if (err < 0) { + ane_perf_end(ane); goto unlock; + } if (ane->stats_slots) stats_idx = ane_stats_begin(&ane->stats_ctrs, &ane->stats_ring, ktime_get_ns(), req.td_count); - ane_boost_begin(ane); err = ane_tm_execute(ane, &req); - ane_boost_end(ane); + ane_perf_end(ane); if (ane->stats_slots) ane_stats_complete(&ane->stats_ctrs, &ane->stats_ring, @@ -746,18 +753,19 @@ static ssize_t reset_store(struct device *dev, struct device_attribute *attr, /* Recovery power-cycles the partitions and then writes the engine: * hold a reference so that autosuspend cannot gate it in between. */ + mutex_lock(&ane->pm_lock); err = pm_runtime_resume_and_get(dev); if (err < 0) - return err; + goto out_pm; mutex_lock(&ane->engine_lock); if (!ane->removed) err = ane_tm_recover(ane); mutex_unlock(&ane->engine_lock); pm_runtime_put_autosuspend(dev); - if (err) - return err; - return count; +out_pm: + mutex_unlock(&ane->pm_lock); + return err < 0 ? err : count; } static DEVICE_ATTR_WO(reset); @@ -811,12 +819,13 @@ static int ane_drm_open(struct drm_device *drm, struct drm_file *file) /* Bring up power while the file context is created. A failed * resume propagates; nothing is released that was not acquired. */ + mutex_lock(&ane->pm_lock); err = pm_runtime_resume_and_get(ane->dev); - if (err < 0) - return err; + if (err >= 0) + pm_runtime_put_autosuspend(ane->dev); - pm_runtime_put_autosuspend(ane->dev); - return 0; + mutex_unlock(&ane->pm_lock); + return err < 0 ? err : 0; } /* Retry recovery on close, retaining quarantine if DMA cannot be quiesced. */ @@ -872,14 +881,17 @@ static long ane_drm_ioctl(struct file *file, unsigned int cmd, _IOC_NR(cmd) >= DRM_COMMAND_END) return -ENOTTY; + mutex_lock(&ane->pm_lock); err = pm_runtime_resume_and_get(ane->dev); if (err < 0) - return err; + goto unlock_pm; err = dispatch(file, cmd, arg); pm_runtime_put_autosuspend(ane->dev); +unlock_pm: + mutex_unlock(&ane->pm_lock); return err; } @@ -911,11 +923,19 @@ static int ane_drm_release(struct inode *inode, struct file *file) { struct drm_file *priv = file->private_data; struct device *dev = get_device(priv->minor->dev->dev); + struct ane_device *ane = priv->minor->dev->dev_private; int err; + /* drm_release may drop the last drm reference; keep ane alive until + * pm_lock is unlocked. + */ + drm_dev_get(&ane->drm); + mutex_lock(&ane->pm_lock); pm_runtime_get_sync(dev); err = drm_release(inode, file); pm_runtime_put_autosuspend(dev); + mutex_unlock(&ane->pm_lock); + drm_dev_put(&ane->drm); put_device(dev); return err; } @@ -1113,6 +1133,7 @@ struct ane_soc { * retention and recovery must drain it (see ane_tm_drain_retained). */ bool tm_retention; + unsigned int high_level; }; static const struct ane_soc ane_soc_t8103 = { @@ -1128,6 +1149,7 @@ static const struct ane_soc ane_soc_t6000 = { */ .ps_base = 0x28e08c000ULL, .tm_retention = true, + .high_level = 5, }; static const struct of_device_id ane_of_match[] = { @@ -1217,9 +1239,10 @@ static int ane_platform_probe(struct platform_device *pdev) mutex_init(&ane->iommu_lock); mutex_init(&ane->engine_lock); + mutex_init(&ane->pm_lock); INIT_LIST_HEAD(&ane->bo_list); INIT_LIST_HEAD(&ane->preserved_list); - err = ane_boost_init(ane); + err = ane_perf_init(ane, soc->high_level); if (err < 0) goto detach_genpd; @@ -1279,7 +1302,7 @@ static int ane_platform_probe(struct platform_device *pdev) pm_runtime_dont_use_autosuspend(dev); drm_mm_takedown(&ane->mm); detach_genpd: - ane_boost_exit(ane); + ane_perf_exit(ane); ane_detach_genpd(ane); return err; } @@ -1288,10 +1311,12 @@ static void ane_platform_remove(struct platform_device *pdev) { struct ane_device *ane = platform_get_drvdata(pdev); struct ane_bo *bo, *tmp; + int err; /* Powered and pinned for the teardown: the unmaps below flush the * DART TLBs, and no autosuspend may race the genpd detach. */ + mutex_lock(&ane->pm_lock); pm_runtime_get_sync(ane->dev); mutex_lock(&ane->engine_lock); @@ -1302,6 +1327,15 @@ static void ane_platform_remove(struct platform_device *pdev) dev_err(ane->dev, "forced removal of a wedged device is unsafe; reboot required\n"); mutex_unlock(&ane->engine_lock); + mutex_unlock(&ane->pm_lock); + return; + } + + err = ane_perf_exit(ane); + if (err) { + dev_err(ane->dev, + "retaining power-domain attachments after failed performance release; reboot required\n"); + mutex_unlock(&ane->engine_lock); return; } @@ -1310,13 +1344,13 @@ static void ane_platform_remove(struct platform_device *pdev) ane_reclaim_preserved(ane); drm_mm_takedown(&ane->mm); - ane_boost_exit(ane); ane_detach_genpd(ane); pm_runtime_disable(ane->dev); pm_runtime_dont_use_autosuspend(ane->dev); pm_runtime_put_noidle(ane->dev); mutex_unlock(&ane->engine_lock); + mutex_unlock(&ane->pm_lock); } static int __maybe_unused ane_runtime_suspend(struct device *dev) @@ -1331,7 +1365,7 @@ static int __maybe_unused ane_runtime_suspend(struct device *dev) if (atomic_read(&ane->wedged) && !ane->recovering) return -EBUSY; - return 0; + return ane_perf_suspend(ane); } static int __maybe_unused ane_runtime_resume(struct device *dev) @@ -1365,6 +1399,10 @@ static int __maybe_unused ane_runtime_resume(struct device *dev) eng); } + /* Recovery verifies the power-on ACK before its first engine access. */ + if (ane->recovering) + return 0; + ane_tm_enable(ane, first); /* First enable is the engine's fresh signature; recovery compares diff --git a/drivers/accel/ane/ane_fw_platform.c b/drivers/accel/ane/ane_fw_platform.c new file mode 100644 index 0000000000000..1a8f69a329abe --- /dev/null +++ b/drivers/accel/ane/ane_fw_platform.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * H16/H17 facts from the 2026-10-31 H16/H17 specification, sections 3-7, + * 12 and 15. This preflight performs no MMIO, DMA mapping or power access. + */ +#include +#include +#include +#include +#include + +#include "ane_fw_platform.h" + +static const char * const ane_h16_reg_names[] = { + "engine", "pmgr", "aux-0", "aux-1", "aux-2", "aux-3", "aux-4", +}; + +static const u32 ane_h16_reg_sizes[] = { + 0x2000000, 0x18000, 0x4000, 0x30000, 0xff4000, 0x2000000, 0x4000, +}; + +/* Offsets in reg[1]; genpd, never the ANE driver, owns transitions. */ +static const u32 ane_h16_power_words[] = { + 0x3c0, 0xc008, 0xc000, 0xc010, 0xc018, +}; + +const struct ane_fw_platform ane_h16_platform = { + /* The DT compatible is t6041; the physical M4 Pro is T6040. */ + .soc = 0x6040, + .chip_family = 16, + .ane_type = 0x110, + .hwx_subtype = 7, + .task_generation = 17, + .interrupt_count = 1, + /* J616s IRQ 1054 drains replies and events from all F2H channels. */ + .irq_role = ANE_FW_IRQ_CHANNELS, + .owner = ANE_FW_OWNER_AP, + .reg_names = ane_h16_reg_names, + .reg_sizes = ane_h16_reg_sizes, + .reg_count = ARRAY_SIZE(ane_h16_reg_names), + .wrapper = 0x1600000, + .mailbox = 0x1608000, + .scratch = 0x1880020, + .doorbell = 0x1884000, + .irq_status = 0x188c000, + .irq_ack = 0x1890000, + .power_words = ane_h16_power_words, + .power_count = ARRAY_SIZE(ane_h16_power_words), + .preload_dva = { 0x10000000000, 0x100000c0000 }, + .preload_size = { 0xc0000, 0x2ac000 }, + .stream_mask = BIT(0) | BIT(11), + .firmware_preloaded = true, + .safe_reset = false, +}; + +const struct ane_fw_platform ane_h17_platform = { + .soc = 0x8140, + .chip_family = 17, + .ane_type = 0x200, + .hwx_subtype = 9, + .task_generation = 19, + .interrupt_count = 2, + .owner = ANE_FW_OWNER_EXCLAVE, + .preload_dva = { 0x10000000000, 0x100000c0000 }, + .preload_size = { 0xc0000, 0x2ac000 }, + .stream_mask = BIT(0) | BIT(11), + .firmware_preloaded = true, + .safe_reset = false, + /* No H16 register-map inference is executable on H17. */ +}; + +static int ane_fw_preload_check(struct device *dev, + const struct ane_fw_platform *soc) +{ + struct device_node *np = dev->of_node; + struct device_node *reserved __free(device_node) = + of_find_node_by_path("/reserved-memory"); + u64 phys[2], size[2], dva[2]; + unsigned int i; + + if (!reserved || + of_property_count_u32_elems(np, "memory-region") != 2 || + of_property_count_strings(np, "memory-region-names") != 2 || + of_property_count_u64_elems(np, "apple,firmware-iovas") != 2 || + of_property_read_u64_array(np, "apple,firmware-iovas", dva, 2)) + return -ENODATA; + + for (i = 0; i < 2; i++) { + struct device_node *mem; + struct reserved_mem *rmem; + const char *name; + bool valid; + + if (of_property_read_string_index(np, "memory-region-names", i, + &name) || + strcmp(name, i ? "firmware-data" : "firmware-text")) + return -EINVAL; + mem = of_parse_phandle(np, "memory-region", i); + if (!mem) + return -EINVAL; + rmem = of_reserved_mem_lookup(mem); + valid = rmem && mem->parent == reserved && + of_device_is_available(mem) && + of_property_read_bool(mem, "no-map") && + !of_property_read_bool(mem, "reusable"); + if (valid) { + phys[i] = rmem->base; + size[i] = rmem->size; + } + of_node_put(mem); + if (!valid) + return -EINVAL; + } + + return ane_fw_preload_ranges_valid(soc, phys, size, dva) ? 0 : -EINVAL; +} + +int ane_fw_platform_preflight(struct platform_device *pdev, + const struct ane_fw_platform *soc) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct resource *res; + u32 ane_type; + unsigned int i; + int ret; + + /* No backend has acquired ownership, even if DT omits the marker. + * Check before resources, firmware, power, IOMMU or engine access. + */ + if (soc->owner == ANE_FW_OWNER_EXCLAVE) { + dev_info(dev, "H17 ANE: Linux exclave ownership handoff unavailable; refusing probe\n"); + return -EOPNOTSUPP; + } + if (of_property_read_bool(np, "apple,exclave-assigned")) { + dev_info(dev, "ANE: exclave-assigned device has no Linux ownership provider\n"); + return -EOPNOTSUPP; + } + + if (!of_property_read_bool(np, "apple,firmware-preloaded") || + of_property_read_u32(np, "apple,ane-type", &ane_type) || + ane_type != soc->ane_type) + return dev_err_probe(dev, -EINVAL, "H16 firmware identity missing or invalid\n"); + + if (of_property_count_strings(np, "reg-names") != soc->reg_count || + platform_get_resource(pdev, IORESOURCE_MEM, soc->reg_count) || + of_irq_count(np) != soc->interrupt_count) + return dev_err_probe(dev, -EINVAL, "H16 resource count mismatch\n"); + + for (i = 0; i < soc->reg_count; i++) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + soc->reg_names[i]); + if (!res || resource_size(res) != soc->reg_sizes[i]) + return dev_err_probe(dev, -EINVAL, "H16 resource geometry mismatch\n"); + } + + ret = ane_fw_preload_check(dev, soc); + if (ret) + return dev_err_probe(dev, ret, + "H16 requires bootloader-reserved no-map TEXT/DATA preload ranges\n"); + + /* Original HWX validation does not establish firmware ownership, + * preload integrity, startup state or a decoded process resource map. + * Keep the section-only ABI and incomplete boot backend refused. + */ + dev_info(dev, "H16 ANE: preload validation, HWX ABI and boot backend incomplete; refusing before power access\n"); + return -EOPNOTSUPP; +} diff --git a/drivers/accel/ane/ane_fw_platform.h b/drivers/accel/ane/ane_fw_platform.h new file mode 100644 index 0000000000000..a7d4ed4f344f0 --- /dev/null +++ b/drivers/accel/ane/ane_fw_platform.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_FW_PLATFORM_H__ +#define __ANE_FW_PLATFORM_H__ + +#include +#include +#include +#include +#include +#include + +struct platform_device; + +enum ane_fw_owner { + ANE_FW_OWNER_AP, + ANE_FW_OWNER_EXCLAVE, +}; + +enum ane_fw_irq_role { + ANE_FW_IRQ_UNKNOWN, + ANE_FW_IRQ_CHANNELS, +}; + +/* Later firmware generations must never enter the H14 staging path. */ +struct ane_fw_platform { + u32 soc; + u16 chip_family; + u16 ane_type; + u8 hwx_subtype; + u8 task_generation; + u8 interrupt_count; + enum ane_fw_irq_role irq_role; + enum ane_fw_owner owner; + const char * const *reg_names; + const u32 *reg_sizes; + u32 reg_count; + u32 wrapper; + u32 mailbox; + u32 scratch; + u32 doorbell; + u32 irq_status; + u32 irq_ack; + const u32 *power_words; + u32 power_count; + u64 preload_dva[2]; + u64 preload_size[2]; + u16 stream_mask; + bool firmware_preloaded; + bool safe_reset; +}; + +/* Pure range check: the boot loader supplies the physical addresses. */ +static inline bool +ane_fw_preload_ranges_valid(const struct ane_fw_platform *soc, const u64 phys[2], + const u64 size[2], const u64 dva[2]) +{ + u64 end[2]; + unsigned int i; + + for (i = 0; i < 2; i++) { + if (!phys[i] || !IS_ALIGNED(phys[i], SZ_16K) || + size[i] != soc->preload_size[i] || + dva[i] != soc->preload_dva[i] || + check_add_overflow(phys[i], size[i], &end[i])) + return false; + } + return end[0] <= phys[1] || end[1] <= phys[0]; +} + +extern const struct ane_fw_platform ane_h16_platform, ane_h17_platform; +int ane_fw_platform_preflight(struct platform_device *pdev, + const struct ane_fw_platform *soc); + +#endif /* __ANE_FW_PLATFORM_H__ */ diff --git a/drivers/accel/ane/ane_fw_platform_test.c b/drivers/accel/ane/ane_fw_platform_test.c new file mode 100644 index 0000000000000..0f7171ec95a7e --- /dev/null +++ b/drivers/accel/ane/ane_fw_platform_test.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "ane_fw_platform.h" + +static void ane_preload_ranges_test(struct kunit *test) +{ + const struct ane_fw_platform soc = { + .preload_dva = { 0x10000000000, 0x100000c0000 }, + .preload_size = { 0xc0000, 0x2ac000 }, + }; + u64 phys[] = { 0x100000000, 0x200000000 }; + u64 size[] = { 0xc0000, 0x2ac000 }; + u64 dva[] = { 0x10000000000, 0x100000c0000 }; + + KUNIT_EXPECT_TRUE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + /* Physical placement may change independently on every boot. */ + phys[0] = 0x300000000; + KUNIT_EXPECT_TRUE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + phys[0] = phys[1] + SZ_16K; + KUNIT_EXPECT_FALSE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + phys[0] = phys[1] - size[0]; + KUNIT_EXPECT_TRUE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + phys[0]++; + KUNIT_EXPECT_FALSE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + phys[0] = 0; + KUNIT_EXPECT_FALSE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + phys[0] = U64_MAX - SZ_16K + 1; + KUNIT_EXPECT_FALSE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + phys[0] = 0x100000000; + size[1] -= SZ_16K; + KUNIT_EXPECT_FALSE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); + size[1] = 0x2ac000; + dva[0] = 0x100000c0000; + dva[1] = 0x10000000000; + KUNIT_EXPECT_FALSE(test, ane_fw_preload_ranges_valid(&soc, phys, size, dva)); +} + +static struct kunit_case ane_fw_platform_cases[] = { + KUNIT_CASE(ane_preload_ranges_test), + {} +}; + +static struct kunit_suite ane_fw_platform_suite = { + .name = "ane-fw-platform", + .test_cases = ane_fw_platform_cases, +}; + +kunit_test_suite(ane_fw_platform_suite); + +MODULE_DESCRIPTION("KUnit tests for ANE firmware preload range validation"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_fw_protocol.h b/drivers/accel/ane/ane_fw_protocol.h new file mode 100644 index 0000000000000..ec2aca2b35845 --- /dev/null +++ b/drivers/accel/ane/ane_fw_protocol.h @@ -0,0 +1,257 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_FW_PROTOCOL_H__ +#define __ANE_FW_PROTOCOL_H__ + +#include +#include +#include +#include +#include +#include + +#define ANE_FW_CHANNELS 8 +#define ANE_FW_DESCRIPTOR_SIZE 0x100 +#define ANE_FW_TABLE_SIZE (ANE_FW_CHANNELS * ANE_FW_DESCRIPTOR_SIZE) +#define ANE_FW_SLOT_SIZE 64 +#define ANE_FW_TICK_RATE 24000000U + +struct ane_fw_channel { + u64 offset; + u32 slots; + u32 type; + u32 bit; + u32 index; +}; + +static inline bool ane_fw_contains(u64 base, u64 bytes, u64 addr, u64 size) +{ + return size <= bytes && addr >= base && addr - base <= bytes - size; +} + +/* Parse a host-owned descriptor snapshot; output indices are logical names. */ +static inline int +ane_fw_channels_parse(const void *table, u64 ipc_dva, size_t ipc_size, + u64 table_dva, struct ane_fw_channel channels[ANE_FW_CHANNELS]) +{ + static const char * const names[ANE_FW_CHANNELS] = { + "TERMINAL", "IO", "DEBUG", "BUF_H2T", "BUF_T2H", + "SHAREDMALLOC", "IO_T2H", "DATA_CHAIN_H2T", + }; + struct ane_fw_channel parsed[ANE_FW_CHANNELS] = {}; + u64 table_off; + u32 seen = 0; + unsigned int i, j; + + if (!table || !ane_fw_contains(ipc_dva, ipc_size, table_dva, ANE_FW_TABLE_SIZE)) + return -EPROTO; + table_off = table_dva - ipc_dva; + for (i = 0; i < ANE_FW_CHANNELS; i++) { + const u8 *desc = (const u8 *)table + i * ANE_FW_DESCRIPTOR_SIZE; + struct ane_fw_channel *c; + u64 slots, ring, end; + + if (!memchr(desc, 0, 64)) + return -EPROTO; + for (j = 0; j < ANE_FW_CHANNELS; j++) + if (!strcmp((const char *)desc, names[j])) + break; + if (j == ANE_FW_CHANNELS || (seen & BIT(j))) + return -EPROTO; + seen |= BIT(j); + c = &parsed[j]; + c->index = i; + c->type = get_unaligned_le32(desc + 0x40); + c->bit = get_unaligned_le32(desc + 0x44); + slots = get_unaligned_le64(desc + 0x48); + ring = get_unaligned_le64(desc + 0x50); + if (c->type > 2 || c->bit >= 32 || !slots || slots > 0x400 || + (ring & (ANE_FW_SLOT_SIZE - 1)) || + !ane_fw_contains(ipc_dva, ipc_size, ring, + slots * ANE_FW_SLOT_SIZE)) + return -EPROTO; + c->offset = ring - ipc_dva; + c->slots = slots; + end = c->offset + slots * ANE_FW_SLOT_SIZE; + if (c->offset < table_off + ANE_FW_TABLE_SIZE && end > table_off) + return -EPROTO; + } + if (seen != GENMASK(ANE_FW_CHANNELS - 1, 0) || + parsed[1].type != 0 || parsed[5].type != 1) + return -EPROTO; + for (i = 0; i < ANE_FW_CHANNELS; i++) { + u64 end = parsed[i].offset + parsed[i].slots * ANE_FW_SLOT_SIZE; + + for (j = i + 1; j < ANE_FW_CHANNELS; j++) + if (parsed[i].offset < parsed[j].offset + + parsed[j].slots * ANE_FW_SLOT_SIZE && + parsed[j].offset < end) + return -EPROTO; + } + memcpy(channels, parsed, sizeof(parsed)); + return 0; +} + +/* Call only after parsing the table and checking its immutable snapshot. */ +static inline void +ane_fw_channels_host_init(void *ipc, const struct ane_fw_channel *channels) +{ + unsigned int i, slot; + + for (i = 0; i < ANE_FW_CHANNELS; i++) { + const struct ane_fw_channel *c = &channels[i]; + + if (c->type) + continue; + for (slot = 0; slot < c->slots; slot++) { + u8 *p = (u8 *)ipc + c->offset + slot * ANE_FW_SLOT_SIZE; + + memset(p, 0, 24); + put_unaligned_le64(1, p); + } + } +} + +/* H14 preserves its reserved word; H16 requires bytes 0..3 to be zero. */ +static inline void ane_fw_command_header(void *command, u16 opcode, bool zero_reserved) +{ + u8 *p = command; + + if (zero_reserved) + put_unaligned_le32(0, p); + put_unaligned_le16(opcode, p + 4); + put_unaligned_le16(0, p + 6); +} + +/* A firmware refusal is distinct from corruption and never means success. */ +static inline int +ane_fw_reply_status(u64 header, u64 slot_length, u64 slot_result, + u64 command_dva, size_t length, u16 opcode, const void *command, + bool zero_reserved) +{ + const u8 *p = command; + + if (!(header & 1)) + return -EAGAIN; + if (!command || length < 8 || (command_dva & 1) || header != (command_dva | 1) || + slot_length != length || slot_result || + (zero_reserved && get_unaligned_le32(p)) || + get_unaligned_le16(p + 4) != opcode) + return -EPROTO; + return get_unaligned_le16(p + 6) ? -EREMOTEIO : 0; +} + +static inline int ane_fw_config_status(const void *command, size_t length) +{ + if (!command || length != 16 || get_unaligned_le32((const u8 *)command + 8) != + ANE_FW_TICK_RATE) + return -EPROTO; + return 0; +} + +/* Legacy H14/comparator layout only; H16 27.0 reports are 0x60 bytes. */ +static inline int +ane_fw_finished_event(const void *event, size_t length, u64 cookie, + u32 program, u32 process) +{ + const u8 *p = event; + u32 state; + + if (!p || length != 0x28 || get_unaligned_le32(p + 4) != 0x300 || + get_unaligned_le64(p + 8) != cookie || + get_unaligned_le32(p + 0x10) != program || + get_unaligned_le32(p + 0x14) != process || + get_unaligned_le32(p + 0x18)) + return -EPROTO; + state = get_unaligned_le32(p + 0x1c); + return state > 1 ? -EPROTO : state; +} + +enum ane_fw_rtk_phase { + ANE_FW_RTK_HELLO, + ANE_FW_RTK_EPMAP, + ANE_FW_RTK_IOP_ACK, + ANE_FW_RTK_AP_ACK, + ANE_FW_RTK_READY, +}; + +struct ane_fw_rtk { + u32 endpoints[8]; + u8 fragments; + enum ane_fw_rtk_phase phase; +}; + +#define ANE_FW_RTK_TYPE(t) ((u64)(t) << 52) +#define ANE_FW_RTK_LAST BIT_ULL(51) + +/* The caller owns one absolute deadline for the complete management session. */ +static inline int +ane_fw_rtk_receive(struct ane_fw_rtk *rtk, u64 word0, u64 word1, + void *ctx, int (*send)(void *ctx, u64 word0, u64 word1)) +{ + static const u8 system_eps[] = { 1, 2, 3, 4, 8, 10 }; + u32 type = (word0 >> 52) & 0xff; + unsigned int base, ep, i; + int ret; + + if (word1 & 0xff) + return -EOPNOTSUPP; + switch (type) { + case 1: + if (rtk->phase != ANE_FW_RTK_HELLO || (word0 & 0xffff) > 12 || + ((word0 >> 16) & 0xffff) < 12) + return -EPROTO; + ret = send(ctx, ANE_FW_RTK_TYPE(2) | (12 << 16) | 12, 0); + if (!ret) + rtk->phase = ANE_FW_RTK_EPMAP; + return ret; + case 8: + if (rtk->phase != ANE_FW_RTK_EPMAP) + return -EPROTO; + base = (word0 >> 32) & 7; + if (rtk->fragments & BIT(base)) + return -EPROTO; + rtk->fragments |= BIT(base); + rtk->endpoints[base] = word0; + ret = send(ctx, ANE_FW_RTK_TYPE(8) | ((u64)base << 32) | + (word0 & ANE_FW_RTK_LAST ? ANE_FW_RTK_LAST : 1), 0); + if (ret || !(word0 & ANE_FW_RTK_LAST)) + return ret; + for (i = 0; i < ARRAY_SIZE(system_eps); i++) { + ep = system_eps[i]; + if (!(rtk->endpoints[0] & BIT(ep))) + continue; + ret = send(ctx, ANE_FW_RTK_TYPE(5) | ((u64)ep << 32) | 2, 0); + if (ret) + return ret; + } + ret = send(ctx, ANE_FW_RTK_TYPE(6) | 0x20, 0); + if (!ret) + rtk->phase = ANE_FW_RTK_IOP_ACK; + return ret; + case 7: + if (rtk->phase != ANE_FW_RTK_IOP_ACK || (word0 & 0xffff) != 0x20) + return -EPROTO; + ret = send(ctx, ANE_FW_RTK_TYPE(0xb) | 0x20, 0); + if (!ret) + rtk->phase = ANE_FW_RTK_AP_ACK; + return ret; + case 0xb: + if (rtk->phase != ANE_FW_RTK_AP_ACK || (word0 & 0xffff) != 0x20) + return -EPROTO; + for (ep = 0x20; ep < 0x100; ep++) { + if (!(rtk->endpoints[ep / 32] & BIT(ep % 32))) + continue; + ret = send(ctx, ANE_FW_RTK_TYPE(5) | ((u64)ep << 32) | 2, 0); + if (ret) + return ret; + } + rtk->phase = ANE_FW_RTK_READY; + return 0; + case 0xc: + return 0; + default: + return -EOPNOTSUPP; + } +} + +#endif /* __ANE_FW_PROTOCOL_H__ */ diff --git a/drivers/accel/ane/ane_fw_protocol_test.c b/drivers/accel/ane/ane_fw_protocol_test.c new file mode 100644 index 0000000000000..fb98428f3f238 --- /dev/null +++ b/drivers/accel/ane/ane_fw_protocol_test.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include + +#include "ane_fw_protocol.h" +#include "ane_h16_protocol.h" + +static void ane_h16_id_table_test(struct kunit *test) +{ + static const struct ane_h16_command_info expected[] = { + { 0x200, 0x1c0, 0x1c0, 1, 1, false }, + { 0x201, 0xc, 0xc, 1, 1, false }, + { 0x202, 0x258, 0x258, 1, 1, false }, + { 0x203, 0x10, 0x10, 1, 1, false }, + { 0x20c, 0xca8, 0xcd8, 0x18, 7, false }, + { 0x2d, 0xc, 0xc, 1, 7, false }, + { 0x300, 0x60, 0x60, 1, 6, true }, + }; + u8 *command = kunit_kzalloc(test, 0xcd8, GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, command); + for (i = 0; i < ARRAY_SIZE(expected); i++) { + const struct ane_h16_command_info *e = &expected[i]; + const struct ane_h16_command_info *info = ane_h16_command_info(e->id); + size_t length; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, info); + KUNIT_EXPECT_EQ(test, info->min_size, e->min_size); + KUNIT_EXPECT_EQ(test, info->max_size, e->max_size); + KUNIT_EXPECT_EQ(test, info->size_step, e->size_step); + KUNIT_EXPECT_EQ(test, info->channel, e->channel); + KUNIT_EXPECT_EQ(test, info->from_firmware, e->from_firmware); + if (e->from_firmware) { + ret = ane_h16_command_prepare(command, e->min_size, e->id, e->channel); + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + continue; + } + for (length = e->min_size; length <= e->max_size; length += e->size_step) { + ret = ane_h16_command_prepare(command, length, e->id, e->channel); + KUNIT_EXPECT_EQ(test, ret, 0); + } + ret = ane_h16_command_prepare(command, e->min_size - 1, e->id, e->channel); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = ane_h16_command_prepare(command, e->max_size + 1, e->id, e->channel); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = ane_h16_command_prepare(command, e->min_size, e->id, e->channel ^ 1); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + } + KUNIT_EXPECT_PTR_EQ(test, ane_h16_command_info(0x204), NULL); + KUNIT_EXPECT_PTR_EQ(test, ane_h16_command_info(0x301), NULL); + KUNIT_EXPECT_EQ(test, ane_h16_command_prepare(command, 0xca8, 0x204, 7), + -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, ane_h16_command_prepare(command, 0xca9, 0x20c, 7), -EINVAL); + /* The H14 CREATE_PROCESS size must never be sent to H16. */ + KUNIT_EXPECT_EQ(test, ane_h16_command_prepare(command, 0x10, 0x202, 1), -EINVAL); +} + +static void ane_h16_envelope_test(struct kunit *test) +{ + u8 command[0xc]; + const u64 dva = 0x30000000000; + + memset(command, 0xa5, sizeof(command)); + put_unaligned_le32(1, command + 8); + KUNIT_ASSERT_EQ(test, ane_h16_command_prepare(command, sizeof(command), 0x2d, 7), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(command), 0U); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(command + 4), (u16)0x2d); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(command + 6), (u16)0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(command + 8), 1U); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 0xc, 0, dva, 0xc, + 0x2d, command, true), 0); + /* Test the upper byte too: acknowledgment is u16, not u8. */ + put_unaligned_le16(0x100, command + 6); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 0xc, 0, dva, 0xc, + 0x2d, command, true), -EREMOTEIO); + put_unaligned_le32(0, command + 8); + KUNIT_ASSERT_EQ(test, ane_h16_command_prepare(command, sizeof(command), 0x2d, 7), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(command + 6), (u16)0); + put_unaligned_le32(2, command + 8); + KUNIT_EXPECT_EQ(test, ane_h16_command_prepare(command, sizeof(command), 0x2d, 7), + -EINVAL); + KUNIT_EXPECT_EQ(test, ane_h16_command_prepare(NULL, sizeof(command), 0x2d, 7), + -EINVAL); +} + +static void ane_h16_report_test(struct kunit *test) +{ + u8 event[0x60] = {}; + unsigned int n; + + put_unaligned_le16(0x300, event + 4); + for (n = 0; n < 4; n++) { + put_unaligned_le32(0xfe00 | n, event); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, sizeof(event), 6), 0); + /* A report can never satisfy the legacy completion predicate. */ + KUNIT_EXPECT_EQ(test, ane_fw_finished_event(event, sizeof(event), 0, 0, 0), + -EPROTO); + } + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, sizeof(event), 7), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, 0x28, 6), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, 0xc, 6), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(NULL, 0x60, 6), -EPROTO); + put_unaligned_le32(0xfe04, event); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, sizeof(event), 6), 0); + put_unaligned_le32(0xfe00, event); + put_unaligned_le16(0x301, event + 4); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, sizeof(event), 6), -EPROTO); + put_unaligned_le16(0x300, event + 4); + put_unaligned_le16(0x100, event + 6); + KUNIT_EXPECT_EQ(test, ane_h16_event_report_status(event, sizeof(event), 6), -EPROTO); +} + +#define TEST_IPC_SIZE 0x10000 +#define TEST_IPC_DVA 0x20000000000ULL +#define TEST_TABLE_OFF 0x1000 + +static void *ane_test_table(struct kunit *test) +{ + static const char * const names[] = { + "TERMINAL", "IO", "DEBUG", "BUF_H2T", "BUF_T2H", + "SHAREDMALLOC", "IO_T2H", "DATA_CHAIN_H2T", + }; + void *ipc = kunit_kzalloc(test, TEST_IPC_SIZE, GFP_KERNEL); + unsigned int i; + + if (!ipc) + return NULL; + for (i = 0; i < ANE_FW_CHANNELS; i++) { + u8 *d = ipc + TEST_TABLE_OFF + i * ANE_FW_DESCRIPTOR_SIZE; + + strscpy((char *)d, names[i], 64); + put_unaligned_le32(i == 0 ? 2 : (i == 4 || i == 5 || i == 6), d + 0x40); + put_unaligned_le32(31 - i, d + 0x44); + put_unaligned_le64(i + 1, d + 0x48); + put_unaligned_le64(TEST_IPC_DVA + 0x2000 + i * 0x400, d + 0x50); + } + return ipc; +} + +static int ane_test_channels_parse(void *ipc, struct ane_fw_channel *channels) +{ + return ane_fw_channels_parse(ipc + TEST_TABLE_OFF, TEST_IPC_DVA, TEST_IPC_SIZE, + TEST_IPC_DVA + TEST_TABLE_OFF, channels); +} + +static void ane_dynamic_channels_test(struct kunit *test) +{ + struct ane_fw_channel channels[ANE_FW_CHANNELS]; + u8 *ipc = ane_test_table(test); + u8 *io; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ipc); + KUNIT_ASSERT_EQ(test, ane_test_channels_parse(ipc, channels), 0); + KUNIT_EXPECT_EQ(test, channels[1].bit, 30U); + KUNIT_EXPECT_EQ(test, channels[1].slots, 2U); + KUNIT_EXPECT_EQ(test, channels[1].offset, 0x2400ULL); + KUNIT_EXPECT_EQ(test, channels[1].index, 1U); + { + u8 saved[ANE_FW_DESCRIPTOR_SIZE]; + u8 *first = ipc + TEST_TABLE_OFF + ANE_FW_DESCRIPTOR_SIZE; + u8 *last = ipc + TEST_TABLE_OFF + 7 * ANE_FW_DESCRIPTOR_SIZE; + + memcpy(saved, first, sizeof(saved)); + memcpy(first, last, sizeof(saved)); + memcpy(last, saved, sizeof(saved)); + KUNIT_ASSERT_EQ(test, ane_test_channels_parse(ipc, channels), 0); + KUNIT_EXPECT_EQ(test, channels[1].offset, 0x2400ULL); + KUNIT_EXPECT_EQ(test, channels[1].index, 7U); + } + memset(ipc + 0x2000, 0xa5, 0x2000); + ane_fw_channels_host_init(ipc, channels); + io = ipc + channels[1].offset; + KUNIT_EXPECT_EQ(test, get_unaligned_le64(io), 1ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(io + 8), 0ULL); + KUNIT_EXPECT_EQ(test, io[24], (u8)0xa5); + for (i = 0; i < 24; i++) + KUNIT_EXPECT_EQ(test, ipc[0x2000 + i], (u8)0xa5); +} + +static void ane_channel_snapshot_test(struct kunit *test) +{ + struct ane_fw_channel channels[ANE_FW_CHANNELS]; + u8 *ipc = ane_test_table(test); + u8 *snapshot = kunit_kmalloc(test, ANE_FW_TABLE_SIZE, GFP_KERNEL); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, ipc); + KUNIT_ASSERT_NOT_NULL(test, snapshot); + memcpy(snapshot, ipc + TEST_TABLE_OFF, ANE_FW_TABLE_SIZE); + put_unaligned_le64(0, ipc + TEST_TABLE_OFF + ANE_FW_DESCRIPTOR_SIZE + 0x48); + ret = ane_fw_channels_parse(snapshot, TEST_IPC_DVA, TEST_IPC_SIZE, + TEST_IPC_DVA + TEST_TABLE_OFF, channels); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, channels[1].slots, 2U); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); +} + +static void ane_channel_rejections_test(struct kunit *test) +{ + struct ane_fw_channel channels[ANE_FW_CHANNELS]; + u8 *ipc = ane_test_table(test); + u8 *d; + + KUNIT_ASSERT_NOT_NULL(test, ipc); + d = ipc + TEST_TABLE_OFF + ANE_FW_DESCRIPTOR_SIZE; + put_unaligned_le64(0, d + 0x48); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + put_unaligned_le64(0x401, d + 0x48); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + put_unaligned_le64(2, d + 0x48); + put_unaligned_le64(TEST_IPC_DVA + TEST_TABLE_OFF, d + 0x50); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + put_unaligned_le64(TEST_IPC_DVA + 0x2000, d + 0x50); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + put_unaligned_le64(TEST_IPC_DVA + TEST_IPC_SIZE - 64, d + 0x50); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + put_unaligned_le64(TEST_IPC_DVA + 0x2400, d + 0x50); + put_unaligned_le32(32, d + 0x44); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + put_unaligned_le32(30, d + 0x44); + put_unaligned_le32(1, d + 0x40); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + put_unaligned_le32(0, d + 0x40); + strscpy((char *)d, "TERMINAL", 64); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + memset(d, 0x41, 64); + KUNIT_EXPECT_EQ(test, ane_test_channels_parse(ipc, channels), -EPROTO); + KUNIT_EXPECT_FALSE(test, ane_fw_contains(16, 64, U64_MAX, 32)); + KUNIT_EXPECT_FALSE(test, ane_fw_contains(16, 64, 16, 65)); +} + +static void ane_reply_status_test(struct kunit *test) +{ + u8 command[16] = {}; + const u64 dva = 0x30000000000; + + put_unaligned_le16(3, command + 4); + put_unaligned_le32(ANE_FW_TICK_RATE, command + 8); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva, 16, 16, dva, 16, 3, + command, true), -EAGAIN); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 16, 0, dva, 16, 3, + command, true), 0); + KUNIT_EXPECT_EQ(test, ane_fw_config_status(command, sizeof(command)), 0); + put_unaligned_le16(1, command + 6); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 16, 0, dva, 16, 3, + command, true), -EREMOTEIO); + put_unaligned_le16(0xffff, command + 6); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 16, 0, dva, 16, 3, + command, true), -EREMOTEIO); + put_unaligned_le16(0, command + 6); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 15, 0, dva, 16, 3, + command, true), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 16, 1, dva, 16, 3, + command, true), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 16, 0, dva, 16, 4, + command, true), -EPROTO); + put_unaligned_le32(1, command); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 16, 0, dva, 16, 3, + command, true), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(dva | 1, 16, 0, dva, 16, 3, + command, false), 0); + put_unaligned_le32(1, command + 8); + KUNIT_EXPECT_EQ(test, ane_fw_config_status(command, sizeof(command)), -EPROTO); +} + +static void ane_finished_event_test(struct kunit *test) +{ + u8 event[0x28] = {}; + + put_unaligned_le32(0x300, event + 4); + put_unaligned_le64(42, event + 8); + put_unaligned_le32(3, event + 0x10); + put_unaligned_le32(4, event + 0x14); + KUNIT_EXPECT_EQ(test, ane_fw_finished_event(event, sizeof(event), 42, 3, 4), 0); + put_unaligned_le32(1, event + 0x1c); + KUNIT_EXPECT_EQ(test, ane_fw_finished_event(event, sizeof(event), 42, 3, 4), 1); + KUNIT_EXPECT_EQ(test, ane_fw_finished_event(event, sizeof(event), 41, 3, 4), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_fw_finished_event(event, sizeof(event), 42, 3, 5), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_fw_finished_event(event, 0x20, 42, 3, 4), -EPROTO); + put_unaligned_le32(2, event + 0x1c); + KUNIT_EXPECT_EQ(test, ane_fw_finished_event(event, sizeof(event), 42, 3, 4), -EPROTO); +} + +struct ane_test_tx { + u64 words[16]; + unsigned int count; +}; + +static int ane_test_send(void *ctx, u64 word0, u64 word1) +{ + struct ane_test_tx *tx = ctx; + + if (word1 || tx->count == ARRAY_SIZE(tx->words)) + return -EOVERFLOW; + tx->words[tx->count++] = word0; + return 0; +} + +static void ane_rtk_order_test(struct kunit *test) +{ + struct ane_fw_rtk rtk = {}; + struct ane_test_tx tx = {}; + + KUNIT_ASSERT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(1) | + (13 << 16) | 11, 0, &tx, ane_test_send), 0); + KUNIT_EXPECT_EQ(test, tx.words[0], ANE_FW_RTK_TYPE(2) | (12 << 16) | 12); + KUNIT_ASSERT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(8) | + BIT(2) | BIT(8), 0, &tx, ane_test_send), 0); + KUNIT_EXPECT_EQ(test, tx.words[1], ANE_FW_RTK_TYPE(8) | 1); + KUNIT_ASSERT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(8) | + BIT_ULL(32) | ANE_FW_RTK_LAST | BIT(0) | BIT(5), 0, + &tx, ane_test_send), 0); + KUNIT_EXPECT_EQ(test, rtk.phase, ANE_FW_RTK_IOP_ACK); + KUNIT_EXPECT_EQ(test, tx.count, 6U); + KUNIT_EXPECT_EQ(test, tx.words[2], ANE_FW_RTK_TYPE(8) | BIT_ULL(32) | + ANE_FW_RTK_LAST); + KUNIT_EXPECT_EQ(test, tx.words[3], ANE_FW_RTK_TYPE(5) | (2ULL << 32) | 2); + KUNIT_EXPECT_EQ(test, tx.words[4], ANE_FW_RTK_TYPE(5) | (8ULL << 32) | 2); + KUNIT_ASSERT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(7) | 0x20, + 0, &tx, ane_test_send), 0); + KUNIT_EXPECT_EQ(test, tx.words[6], ANE_FW_RTK_TYPE(0xb) | 0x20); + KUNIT_ASSERT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(0xb) | 0x20, + 0, &tx, ane_test_send), 0); + KUNIT_EXPECT_EQ(test, rtk.phase, ANE_FW_RTK_READY); + KUNIT_EXPECT_EQ(test, tx.words[7], ANE_FW_RTK_TYPE(5) | (32ULL << 32) | 2); + KUNIT_EXPECT_EQ(test, tx.words[8], ANE_FW_RTK_TYPE(5) | (37ULL << 32) | 2); +} + +static void ane_rtk_rejections_test(struct kunit *test) +{ + struct ane_fw_rtk rtk = {}; + struct ane_test_tx tx = {}; + u64 hello = ANE_FW_RTK_TYPE(1) | (12 << 16) | 12; + int ret; + + KUNIT_EXPECT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(1) | + (2 << 16) | 1, 0, &tx, ane_test_send), -EPROTO); + KUNIT_EXPECT_EQ(test, tx.count, 0U); + ret = ane_fw_rtk_receive(&rtk, hello, 2, &tx, ane_test_send); + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, tx.count, 0U); + KUNIT_EXPECT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(7) | 0x20, + 0, &tx, ane_test_send), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_fw_rtk_receive(&rtk, ANE_FW_RTK_TYPE(0xff), + 0, &tx, ane_test_send), -EOPNOTSUPP); +} + +static struct kunit_case ane_protocol_cases[] = { + KUNIT_CASE(ane_h16_id_table_test), + KUNIT_CASE(ane_h16_envelope_test), + KUNIT_CASE(ane_h16_report_test), + KUNIT_CASE(ane_dynamic_channels_test), + KUNIT_CASE(ane_channel_snapshot_test), + KUNIT_CASE(ane_channel_rejections_test), + KUNIT_CASE(ane_reply_status_test), + KUNIT_CASE(ane_finished_event_test), + KUNIT_CASE(ane_rtk_order_test), + KUNIT_CASE(ane_rtk_rejections_test), + {} +}; + +static struct kunit_suite ane_protocol_suite = { + .name = "ane-fw-protocol", + .test_cases = ane_protocol_cases, +}; + +kunit_test_suite(ane_protocol_suite); +MODULE_DESCRIPTION("KUnit tests for ANE 27.x channel and RTKit contracts"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_execute.h b/drivers/accel/ane/ane_h16_execute.h new file mode 100644 index 0000000000000..df3cb515e9e2c --- /dev/null +++ b/drivers/accel/ane/ane_h16_execute.h @@ -0,0 +1,208 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_EXECUTE_H__ +#define __ANE_H16_EXECUTE_H__ + +#include + +#include "ane_h16_protocol.h" + +#define ANE_H16_EXECUTE_BASE_SIZE 0xca8 +#define ANE_H16_BUFFER_OFFSET 0x60 +#define ANE_H16_BUFFER_STRIDE 0x30 +#define ANE_H16_MAX_BUFFERS 64 +#define ANE_H16_CUSTOM_BAR_STRIDE 0x18 +/* Conservative observed-version ceiling, not a firmware maximum. */ +#define ANE_H16_MAX_CUSTOM_BARS 2 + +struct ane_h16_request_key { + u64 uuid; + u32 program; + u32 process; + u32 procedure; +}; + +/* Kernel-owned, validated process resources and pinned mapping geometry. */ +struct ane_h16_execute_buffer { + u32 index; + u32 class; + u64 mapping_dva; + u64 mapping_size; + u64 offset; + u64 size; +}; + +/* Selectors and value interpretation must come from validated program metadata. */ +struct ane_h16_custom_bar { + u32 configuration; + u32 value_type; + u32 bar; + u8 bit_offset; + u8 bit_width; + u16 flags; + u64 value; +}; + +struct ane_h16_execute { + struct ane_h16_request_key key; + const struct ane_h16_execute_buffer *buffers; + unsigned int buffer_count; + const struct ane_h16_custom_bar *bars; + unsigned int bar_count; +}; + +/* + * Encode only after matching resources, permissions and BAR selectors to the + * immutable loaded program. This helper checks geometry, not mapping ownership. + * Inputs must remain immutable and must not alias the destination. On error the + * destination is untouched. No mapping is published or released by this helper. + */ +static inline int +ane_h16_execute_encode(void *command, size_t length, const struct ane_h16_execute *call) +{ + u8 *p = command; + unsigned int i, j; + size_t expected; + + if (!p || !call || !call->buffers || !call->buffer_count || + call->buffer_count > ANE_H16_MAX_BUFFERS || + call->bar_count > ANE_H16_MAX_CUSTOM_BARS || + (call->bar_count && !call->bars)) + return -EINVAL; + expected = ANE_H16_EXECUTE_BASE_SIZE + call->bar_count * ANE_H16_CUSTOM_BAR_STRIDE; + if (length != expected) + return -EINVAL; + for (i = 0; i < call->buffer_count; i++) { + const struct ane_h16_execute_buffer *b = &call->buffers[i]; + + if (!b->size || b->mapping_size > U64_MAX - b->mapping_dva || + b->offset > b->mapping_size || b->size > b->mapping_size - b->offset) + return -EINVAL; + } + for (i = 0; i < call->bar_count; i++) { + const struct ane_h16_custom_bar *b = &call->bars[i]; + unsigned int end = b->bit_offset + b->bit_width; + + /* Conservative 64-bit field bound until wider fields are established. */ + if (!b->bit_width || end > 64) + return -EINVAL; + for (j = 0; j < i; j++) { + const struct ane_h16_custom_bar *a = &call->bars[j]; + + if (a->configuration == b->configuration && a->bar == b->bar && + a->bit_offset < end && + b->bit_offset < a->bit_offset + a->bit_width) + return -EINVAL; + } + } + + memset(p, 0, length); + put_unaligned_le16(ANE_H16_EXECUTE, p + 4); + put_unaligned_le32(call->key.program, p + 8); + put_unaligned_le32(call->key.process, p + 0xc); + put_unaligned_le32(call->key.procedure, p + 0x10); + put_unaligned_le32(5, p + 0x18); + put_unaligned_le64(call->key.uuid, p + 0x20); + put_unaligned_le32(call->buffer_count, p + 0x28); + for (i = 0; i < call->buffer_count; i++) { + const struct ane_h16_execute_buffer *b = &call->buffers[i]; + u8 *d = p + ANE_H16_BUFFER_OFFSET + i * ANE_H16_BUFFER_STRIDE; + + put_unaligned_le32(1, d); + put_unaligned_le32(b->index, d + 4); + put_unaligned_le32(b->class, d + 8); + put_unaligned_le32(3, d + 0x10); + put_unaligned_le64(b->mapping_dva + b->offset, d + 0x18); + put_unaligned_le64(b->size, d + 0x20); + } + put_unaligned_le32(call->bar_count, p + 0xca0); + for (i = 0; i < call->bar_count; i++) { + const struct ane_h16_custom_bar *b = &call->bars[i]; + u8 *d = p + ANE_H16_EXECUTE_BASE_SIZE + i * ANE_H16_CUSTOM_BAR_STRIDE; + + put_unaligned_le32(b->configuration, d); + put_unaligned_le32(b->value_type, d + 4); + put_unaligned_le32(b->bar, d + 8); + d[0xc] = b->bit_offset; + d[0xd] = b->bit_width; + put_unaligned_le16(b->flags, d + 0xe); + put_unaligned_le64(b->value, d + 0x10); + } + return 0; +} + +enum ane_h16_event_state { + ANE_H16_EVENT_START, + ANE_H16_EVENT_SUCCESS, + ANE_H16_EVENT_FAILURE, + ANE_H16_EVENT_HW_START, +}; + +struct ane_h16_event { + struct ane_h16_request_key key; + u32 state; + u32 cpu_load; + u64 timestamp; + u8 destination; +}; + +/* Parse a bounded host-owned snapshot. Diagnostic words are not predicates. */ +static inline int +ane_h16_event_decode(const void *event, size_t length, unsigned int channel, + struct ane_h16_event *decoded) +{ + const u8 *p = event; + struct ane_h16_event e = {}; + + if (!decoded || ane_h16_event_report_status(event, length, channel)) + return -EPROTO; + e.key.uuid = get_unaligned_le64(p + 8); + e.key.program = get_unaligned_le32(p + 0x10); + e.key.process = get_unaligned_le32(p + 0x14); + e.key.procedure = get_unaligned_le32(p + 0x18); + e.state = get_unaligned_le32(p + 0x1c); + e.cpu_load = get_unaligned_le32(p + 0x20); + e.timestamp = get_unaligned_le64(p + 0x28); + e.destination = p[0x30]; + if (e.state > ANE_H16_EVENT_HW_START || e.destination > 1) + return -EPROTO; + *decoded = e; + return 0; +} + +enum ane_h16_event_result { + ANE_H16_EVENT_PROGRESS, + ANE_H16_EVENT_COMPLETE, + ANE_H16_EVENT_FAILED, +}; + +/* + * For the general scheduler only. Look up an outstanding UUID before calling; + * remove it atomically on terminal state to reject duplicates and late reports. + * COMPLETE/FAILED classify an event, not permission to unmap: DMA sync and + * failure quarantine remain the caller's responsibility. Never call on ch7 ACK. + */ +static inline int +ane_h16_event_result(const void *event, size_t length, unsigned int channel, + const struct ane_h16_request_key *pending) +{ + struct ane_h16_event e; + int ret; + + if (!pending) + return -EINVAL; + ret = ane_h16_event_decode(event, length, channel, &e); + if (ret) + return ret; + if (e.destination) + return -ENOMSG; + if (e.key.uuid != pending->uuid || e.key.program != pending->program || + e.key.process != pending->process || e.key.procedure != pending->procedure) + return -EPROTO; + if (e.state == ANE_H16_EVENT_SUCCESS) + return ANE_H16_EVENT_COMPLETE; + if (e.state == ANE_H16_EVENT_FAILURE) + return ANE_H16_EVENT_FAILED; + return ANE_H16_EVENT_PROGRESS; +} + +#endif /* __ANE_H16_EXECUTE_H__ */ diff --git a/drivers/accel/ane/ane_h16_execute_test.c b/drivers/accel/ane/ane_h16_execute_test.c new file mode 100644 index 0000000000000..933011c18a77a --- /dev/null +++ b/drivers/accel/ane/ane_h16_execute_test.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include + +#include "ane_h16_execute.h" + +#define TEST_COMMAND_SIZE 0xcd8 + +static const struct ane_h16_request_key test_key = { + .uuid = 0x8877665544332211ULL, + .program = 0x10203040, + .process = 0x50607080, + .procedure = 0x90a0b0c0, +}; + +static struct ane_h16_execute_buffer test_buffer(void) +{ + return (struct ane_h16_execute_buffer) { + .index = 7, + .class = 1, + .mapping_dva = 0x12300000000ULL, + .mapping_size = 0x300000000ULL, + .offset = 0x10000, + .size = 0x200000000ULL, + }; +} + +static void ane_h16_execute_layout_test(struct kunit *test) +{ + static const u8 header[] = { + 0, 0, 0, 0, 0x0c, 2, 0, 0, + 0x40, 0x30, 0x20, 0x10, 0x80, 0x70, 0x60, 0x50, + 0xc0, 0xb0, 0xa0, 0x90, 0, 0, 0, 0, + 5, 0, 0, 0, 0, 0, 0, 0, + 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, + 1, 0, 0, 0, + }; + static const u8 descriptor[] = { + 1, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0x23, 1, 0, 0, + 0, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + }; + struct ane_h16_execute_buffer buffer = test_buffer(); + struct ane_h16_execute call = { test_key, &buffer, 1, NULL, 0 }; + u8 *storage = kunit_kmalloc(test, TEST_COMMAND_SIZE + 2, GFP_KERNEL); + u8 *p; + + KUNIT_ASSERT_NOT_NULL(test, storage); + memset(storage, 0xa5, TEST_COMMAND_SIZE + 2); + p = storage + 1; /* Exercise unaligned output and both destination guards. */ + KUNIT_ASSERT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), 0); + KUNIT_EXPECT_MEMEQ(test, p, header, sizeof(header)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(p + sizeof(header), 0, 0x60 - sizeof(header)), NULL); + KUNIT_EXPECT_MEMEQ(test, p + 0x60, descriptor, sizeof(descriptor)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(p + 0x90, 0, 0xca8 - 0x90), NULL); + KUNIT_EXPECT_EQ(test, storage[0], (u8)0xa5); + KUNIT_EXPECT_EQ(test, p[0xca8], (u8)0xa5); +} + +static void ane_h16_execute_counts_test(struct kunit *test) +{ + struct ane_h16_execute_buffer *buffers; + struct ane_h16_execute call = { .key = test_key }; + u8 *p = kunit_kmalloc(test, TEST_COMMAND_SIZE, GFP_KERNEL); + unsigned int n, i; + + buffers = kunit_kcalloc(test, 64, sizeof(*buffers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, p); + KUNIT_ASSERT_NOT_NULL(test, buffers); + for (i = 0; i < 64; i++) { + buffers[i] = test_buffer(); + buffers[i].index = i; + buffers[i].class = 8; /* Numeric classes have no guessed semantic name. */ + } + call.buffers = buffers; + for (n = 1; n <= 64; n++) { + size_t end = 0x60 + n * 0x30; + + call.buffer_count = n; + memset(p, 0xa5, TEST_COMMAND_SIZE); + KUNIT_ASSERT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(p + 0x28), n); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(p + end - 0x30 + 4), n - 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(p + end - 0x30 + 8), 8U); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(p + end, 0, 0xca8 - end), NULL); + } +} + +static void ane_h16_custom_bar_test(struct kunit *test) +{ + static const u8 first_bar[] = { + 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, + 0, 32, 0x34, 0x12, 0x18, 0x17, 0x16, 0x15, 0x14, 0x13, 0x12, 0x11, + }; + struct ane_h16_execute_buffer buffer = test_buffer(); + struct ane_h16_custom_bar bars[2] = { + { 0x01020304, 0x05060708, 0x090a0b0c, 0, 32, 0x1234, + 0x1112131415161718ULL }, + { 0x01020304, 0x05060708, 0x090a0b0c, 32, 32, 0, 0x2122232425262728ULL }, + }; + struct ane_h16_execute call = { test_key, &buffer, 1, bars, 0 }; + u8 *p = kunit_kmalloc(test, TEST_COMMAND_SIZE + 1, GFP_KERNEL); + unsigned int n; + + KUNIT_ASSERT_NOT_NULL(test, p); + for (n = 0; n <= 2; n++) { + size_t length = 0xca8 + n * 0x18; + + call.bar_count = n; + memset(p, 0xa5, TEST_COMMAND_SIZE + 1); + KUNIT_ASSERT_EQ(test, ane_h16_execute_encode(p, length, &call), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(p + 0xca0), n); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(p + 0xca4), 0U); + KUNIT_EXPECT_EQ(test, p[length], (u8)0xa5); + if (n) + KUNIT_EXPECT_MEMEQ(test, p + 0xca8, first_bar, sizeof(first_bar)); + if (n == 2) { + KUNIT_EXPECT_EQ(test, p[0xccc], (u8)32); + KUNIT_EXPECT_EQ(test, p[0xccd], (u8)32); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p + 0xcd0), + 0x2122232425262728ULL); + } + } +} + +static void ane_h16_execute_rejections_test(struct kunit *test) +{ + struct ane_h16_execute_buffer buffer = test_buffer(); + struct ane_h16_execute call = { test_key, &buffer, 1, NULL, 0 }; + u8 *p = kunit_kmalloc(test, TEST_COMMAND_SIZE, GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, p); + memset(p, 0xa5, TEST_COMMAND_SIZE); + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(NULL, 0xca8, &call), -EINVAL); + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, NULL), -EINVAL); + for (i = 0; i < 4; i++) { + static const size_t lengths[] = { 0, 0xca7, 0xca9, 0xcc0 }; + + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, lengths[i], &call), -EINVAL); + } + call.buffer_count = 0; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + call.buffer_count = 65; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + call.buffer_count = 1; + call.buffers = NULL; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + call.buffers = &buffer; + call.bar_count = 3; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcf0, &call), -EINVAL); + call.bar_count = 1; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcc0, &call), -EINVAL); + call.bar_count = 0; + buffer.size = 0; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + buffer = test_buffer(); + buffer.offset = buffer.mapping_size + 1; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + buffer.offset = buffer.mapping_size; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + buffer = test_buffer(); + buffer.size = buffer.mapping_size - buffer.offset + 1; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + buffer = test_buffer(); + buffer.mapping_dva = U64_MAX - buffer.mapping_size + 1; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(p, 0xa5, TEST_COMMAND_SIZE), NULL); + /* The exact last byte of a mapping is valid. */ + buffer = test_buffer(); + buffer.offset = buffer.mapping_size - 1; + buffer.size = 1; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), 0); +} + +static void ane_h16_bar_rejections_test(struct kunit *test) +{ + struct ane_h16_execute_buffer buffer = test_buffer(); + struct ane_h16_custom_bar bars[2] = { { .bit_width = 32 }, { .bit_width = 32 } }; + struct ane_h16_execute call = { test_key, &buffer, 1, bars, 2 }; + u8 *p = kunit_kmalloc(test, TEST_COMMAND_SIZE, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, p); + memset(p, 0xa5, TEST_COMMAND_SIZE); + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), -EINVAL); + bars[1].bit_offset = 31; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), -EINVAL); + bars[1].bit_offset = 33; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), -EINVAL); + bars[1].bit_offset = 255; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), -EINVAL); + bars[1].bit_offset = 32; + bars[1].bit_width = 0; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(p, 0xa5, TEST_COMMAND_SIZE), NULL); + bars[1].bit_width = 32; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), 0); + bars[1].bit_offset = 0; + bars[1].bar = 1; /* Distinct BARs do not overlap. */ + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), 0); + bars[1].bar = 0; + bars[1].configuration = 1; + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xcd8, &call), 0); +} + +static void ane_h16_test_event(u8 *p, u32 state) +{ + memset(p, 0, 0x60); + put_unaligned_le32(0x12345678, p); /* A rolling word, not fe0N. */ + put_unaligned_le16(0x300, p + 4); + put_unaligned_le64(test_key.uuid, p + 8); + put_unaligned_le32(test_key.program, p + 0x10); + put_unaligned_le32(test_key.process, p + 0x14); + put_unaligned_le32(test_key.procedure, p + 0x18); + put_unaligned_le32(state, p + 0x1c); + put_unaligned_le32(10, p + 0x20); + put_unaligned_le64(0x123456789abcdef0ULL, p + 0x28); + memset(p + 0x31, 0xff, 0x2f); /* Diagnostics do not decide completion. */ +} + +static void ane_h16_event_correlation_test(struct kunit *test) +{ + static const int results[] = { 0, 1, 2, 0 }; + struct ane_h16_event e; + u8 storage[0x61], *p = storage + 1; + unsigned int state, n; + + for (state = 0; state < 4; state++) { + ane_h16_test_event(p, state); + KUNIT_ASSERT_EQ(test, ane_h16_event_decode(p, 0x60, 6, &e), 0); + KUNIT_EXPECT_EQ(test, e.key.uuid, test_key.uuid); + KUNIT_EXPECT_EQ(test, e.state, state); + KUNIT_EXPECT_EQ(test, e.cpu_load, 10U); + KUNIT_EXPECT_EQ(test, e.timestamp, 0x123456789abcdef0ULL); + KUNIT_EXPECT_EQ(test, ane_h16_event_result(p, 0x60, 6, &test_key), results[state]); + for (n = 0; n < 4; n++) { + put_unaligned_le32(0xfe00 + n, p); + KUNIT_EXPECT_EQ(test, ane_h16_event_result(p, 0x60, 6, &test_key), + results[state]); + } + } + ane_h16_test_event(p, 1); + for (n = 0; n < 4; n++) { + struct ane_h16_request_key other = test_key; + + switch (n) { + case 0: + other.uuid ^= BIT_ULL(63); + break; + case 1: + other.program++; + break; + case 2: + other.process++; + break; + case 3: + other.procedure++; + break; + } + KUNIT_EXPECT_EQ(test, ane_h16_event_result(p, 0x60, 6, &other), -EPROTO); + } + p[0x30] = 1; /* ISP traffic cannot retire a general scheduler request. */ + KUNIT_EXPECT_EQ(test, ane_h16_event_result(p, 0x60, 6, &test_key), -ENOMSG); +} + +static void ane_h16_event_rejections_test(struct kunit *test) +{ + struct ane_h16_event e; + u8 p[0x61]; + size_t n; + + ane_h16_test_event(p, 1); + memset(&e, 0xa5, sizeof(e)); + for (n = 0; n < 0x60; n++) + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(p, n, 6, &e), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(p, sizeof(p), 6, &e), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(p, 0x60, 7, &e), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(NULL, 0x60, 6, &e), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(p, 0x60, 6, NULL), -EPROTO); + put_unaligned_le16(0x20c, p + 4); /* An ACK is never a terminal event. */ + KUNIT_EXPECT_EQ(test, ane_h16_event_result(p, 0x60, 7, &test_key), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_result(p, 0x60, 6, &test_key), -EPROTO); + put_unaligned_le16(0x300, p + 4); + put_unaligned_le16(0x100, p + 6); + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(p, 0x60, 6, &e), -EPROTO); + put_unaligned_le16(0, p + 6); + put_unaligned_le32(4, p + 0x1c); + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(p, 0x60, 6, &e), -EPROTO); + put_unaligned_le32(1, p + 0x1c); + p[0x30] = 2; + KUNIT_EXPECT_EQ(test, ane_h16_event_decode(p, 0x60, 6, &e), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_event_result(p, 0x60, 6, NULL), -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(&e, 0xa5, sizeof(e)), NULL); +} + +static struct kunit_case ane_h16_execute_cases[] = { + KUNIT_CASE(ane_h16_execute_layout_test), + KUNIT_CASE(ane_h16_execute_counts_test), + KUNIT_CASE(ane_h16_custom_bar_test), + KUNIT_CASE(ane_h16_execute_rejections_test), + KUNIT_CASE(ane_h16_bar_rejections_test), + KUNIT_CASE(ane_h16_event_correlation_test), + KUNIT_CASE(ane_h16_event_rejections_test), + {} +}; + +static struct kunit_suite ane_h16_execute_suite = { + .name = "ane-h16-execute", + .test_cases = ane_h16_execute_cases, +}; + +kunit_test_suite(ane_h16_execute_suite); +MODULE_DESCRIPTION("KUnit tests for H16 execute serialization and event correlation"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_hwx.c b/drivers/accel/ane/ane_h16_hwx.c new file mode 100644 index 0000000000000..a29e5f81f0aa2 --- /dev/null +++ b/drivers/accel/ane/ane_h16_hwx.c @@ -0,0 +1,619 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include + +#include "ane_h16_hwx.h" +#include "ane_h16_tasks.h" + +#define HWX_COMMANDS_MAX 64 +#define HWX_SYMBOLS_MAX 4096 +#define HWX_SEGMENTS_MAX 8 + +struct hwx_range { + u64 start, size; +}; + +struct hwx_parser { + const u8 *bytes; + size_t size, command_end; + struct ane_h16_hwx info; + struct hwx_range vm[HWX_SEGMENTS_MAX], file[HWX_SEGMENTS_MAX + 4]; + struct ane_h16_hwx_section sections[ANE_H16_SECTIONS]; + u32 tensors[ANE_H16_SURFACES], refs[ANE_H16_SURFACES]; + u32 tensor_count, ref_count, segment_count, file_count, section_count; + u32 symoff, nsyms, stroff, strsize; + bool guard, symtab, metadata; +}; + +static bool hwx_fits(u64 start, u64 length, u64 size) +{ + return start <= size && length <= size - start; +} + +static bool hwx_name(const u8 *p, const char *name) +{ + size_t length = strlen(name); + + return length < 16 && !memcmp(p, name, length) && + !memchr_inv(p + length, 0, 16 - length); +} + +static bool hwx_zero(const u8 *p, size_t start, size_t end) +{ + return !memchr_inv(p + start, 0, end - start); +} + +static int hwx_range_add(struct hwx_range *ranges, u32 *count, u32 max, + u64 start, u64 size) +{ + u64 end; + unsigned int i; + + if (!size || check_add_overflow(start, size, &end) || *count == max) + return -ENOEXEC; + for (i = 0; i < *count; i++) + if (start < ranges[i].start + ranges[i].size && + ranges[i].start < end) + return -ENOEXEC; + ranges[(*count)++] = (struct hwx_range) { start, size }; + return 0; +} + +static int hwx_file_add(struct hwx_parser *p, u64 start, u64 size) +{ + if (start < p->command_end || !hwx_fits(start, size, p->size)) + return -ENOEXEC; + return hwx_range_add(p->file, &p->file_count, ARRAY_SIZE(p->file), start, size); +} + +static int hwx_segment(struct hwx_parser *p, const u8 *c, u32 length) +{ + struct ane_h16_hwx_section *dest; + u64 vm, size, file, filesz; + u32 nsects, prot, i; + bool text, kernel, scratch, surface, guard; + + if (length < 72) + return -ENOEXEC; + vm = get_unaligned_le64(c + 24); + size = get_unaligned_le64(c + 32); + file = get_unaligned_le64(c + 40); + filesz = get_unaligned_le64(c + 48); + prot = get_unaligned_le32(c + 60); + nsects = get_unaligned_le32(c + 64); + if (nsects > ANE_H16_SECTIONS || length != 72 + nsects * 80 || + prot != get_unaligned_le32(c + 56) || + hwx_range_add(p->vm, &p->segment_count, ARRAY_SIZE(p->vm), vm, size)) + return -ENOEXEC; + text = hwx_name(c + 8, "__TEXT"); + kernel = hwx_name(c + 8, "__KERN_0"); + scratch = hwx_name(c + 8, "__DATA"); + surface = hwx_name(c + 8, "__FVMLIB"); + guard = hwx_name(c + 8, "__PAGEZERO"); + if (!text && !kernel && !scratch && !surface && !guard) + return -ENOEXEC; + if (guard) { + if (p->guard || vm || size != SZ_16K || file || filesz || prot || nsects) + return -ENOEXEC; + if (get_unaligned_le32(c + 68) != 4) + return -ENOEXEC; + p->guard = true; + return 0; + } + if (surface || scratch) { + if (file || filesz || !IS_ALIGNED(vm, SZ_16K) || + !IS_ALIGNED(size, SZ_16K) || nsects != 1 || size > SZ_256M) + return -ENOEXEC; + } else if (!filesz || filesz > size || size > SZ_256M || + !IS_ALIGNED(vm, SZ_16K) || !IS_ALIGNED(file, SZ_16K) || + hwx_file_add(p, file, filesz)) { + return -ENOEXEC; + } + if (get_unaligned_le32(c + 68) != (surface ? 6 : text ? + (p->info.scratch.size ? 4 : 0) : 4)) + return -ENOEXEC; + if ((text && (prot != 5 || nsects != 2 || p->info.text.size)) || + (kernel && (prot != 1 || nsects != 1 || p->info.kernel.size)) || + (scratch && (prot != 3 || p->info.scratch.size)) || + (surface && (prot < 1 || prot > 2 || + p->info.surface_count == ANE_H16_SURFACES))) + return -ENOEXEC; + + for (i = 0; i < nsects; i++) { + const u8 *s = c + 72 + i * 80; + struct ane_h16_hwx_section section = { + .vm = get_unaligned_le64(s + 32), + .size = get_unaligned_le64(s + 40), + .offset = get_unaligned_le32(s + 48), + .reloff = get_unaligned_le32(s + 56), + .nreloc = get_unaligned_le32(s + 60), + .ordinal = p->section_count + 1, + }; + u32 alignment = get_unaligned_le32(s + 52), j; + bool task = text && hwx_name(s, "__text"); + + if (p->section_count == ANE_H16_SECTIONS || !section.size || + memcmp(s + 16, c + 8, 16) || alignment > 31 || + !IS_ALIGNED(section.vm, 1ULL << alignment) || + section.vm < vm || !hwx_fits(section.vm - vm, section.size, size) || + !hwx_zero(s, 68, 80) || section.reloff > p->size) + return -ENOEXEC; + for (j = 0; j < p->section_count; j++) { + const struct ane_h16_hwx_section *a = &p->sections[j]; + + if (section.vm < a->vm + a->size && a->vm < section.vm + section.size) + return -ENOEXEC; + } + if (surface || scratch) { + if (section.offset || section.reloff || section.nreloc || section.vm != vm) + return -ENOEXEC; + } else if (section.offset < file || + !IS_ALIGNED(section.offset, 1ULL << alignment) || + !hwx_fits(section.offset - file, section.size, filesz) || + section.offset - file != section.vm - vm) { + return -ENOEXEC; + } + if (section.nreloc) { + if (!task || section.nreloc > ANE_H16_TASK_MAX / 4 || + hwx_file_add(p, section.reloff, (u64)section.nreloc * 8)) + return -ENOEXEC; + } + if (text) { + if (task) + dest = &p->info.text; + else if (hwx_name(s, "__const")) + dest = &p->info.constants; + else + return -ENOEXEC; + } else if (kernel && hwx_name(s, "__kern_0")) { + dest = &p->info.kernel; + if (!IS_ALIGNED(section.vm, 64) || section.size > SZ_256M) + return -ENOEXEC; + } else if (scratch && hwx_name(s, "__bss")) { + dest = &p->info.scratch; + } else if (surface && hwx_name(s, prot == 1 ? "__const" : "__data")) { + struct ane_h16_hwx_surface *b = &p->info.surfaces[p->info.surface_count++]; + + b->allocation = size; + b->direction = prot; + dest = &b->section; + } else { + return -ENOEXEC; + } + if (get_unaligned_le32(s + 64) != (task ? + (section.nreloc ? 0x128 : 0x28) : surface ? + (prot == 1 ? 0x21 : 0x23) : scratch ? 0x25 : 0x26)) + return -ENOEXEC; + if (dest->size) + return -ENOEXEC; + *dest = section; + p->sections[p->section_count++] = section; + } + return 0; +} + +static bool hwx_format(u32 code) +{ + static const u8 known[] = { + 0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 15, 16, 17, + 18, 20, 21, 23, 27, 30, 31, 36, 41, 43, 72, 145, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(known); i++) + if (known[i] == code) + return true; + return false; +} + +static int hwx_program(struct hwx_parser *p) +{ + const u8 *c = p->bytes + p->info.program_offset; + struct ane_h16_hwx *info = &p->info; + u32 length = get_unaligned_le32(c + 4), records, i; + bool mixed = length == 0x8c0; + /* Nonzero cells only; every other byte is checked below. */ + static const u16 common[] = { + 0, 4, 8, 0xc, 0x10, 0x14, 0x20, 0x24, 0x40, 0x44, + 0x810, 0x814, 0x820, 0x824, 0x830, 0x838, 0x83c, + 0x844, 0x848, 0x858, 0x860, + }; + u8 *expected; + int ret = -ENOEXEC; + + if (length != 0x8a0 && !mixed) + return -ENOEXEC; + if (get_unaligned_le32(c + 0xc) != (mixed ? 0x22a : 0x222) || + get_unaligned_le64(c + 0x10) != info->text.vm || + get_unaligned_le64(c + 0x810) != info->text.vm || + get_unaligned_le64(c + 0x20) != info->constants.vm || + get_unaligned_le32(c + 0x820) != 4 || + (u64)get_unaligned_le32(c + 0x824) * 4 != info->text.size || + get_unaligned_le32(c + 0x838) != U32_MAX || + get_unaligned_le32(c + 0x83c) != U32_MAX || + get_unaligned_le32(c + 0x844) != 0xffff || + get_unaligned_le32(c + 0x848) != (mixed ? 0x8b8 : 0x898)) + return -ENOEXEC; + records = get_unaligned_le32(c + 0x860); + if (records != 0x10 && records != 0x12 && records != 0x1f && records != 0x20) + return -ENOEXEC; + info->tasks = get_unaligned_le32(c + 0x830); + if (mixed) { + u32 usable = get_unaligned_le32(c + 0x858); + + if (!info->scratch.size || !usable || usable > info->scratch.size || + get_unaligned_le64(c + 0x40) != info->scratch.vm || + get_unaligned_le32(c + 0x880) != usable) + return -ENOEXEC; + } else if (info->scratch.size || get_unaligned_le64(c + 0x40) || + get_unaligned_le32(c + 0x858)) { + return -ENOEXEC; + } + for (i = 0; i < 5; i++) { + u64 base = i < info->surface_count ? info->surfaces[i].section.vm : + i == info->surface_count ? info->kernel.vm : 0; + + if (get_unaligned_le64(c + 0x70 + i * 16) != base || + get_unaligned_le64(c + 0x78 + i * 16)) + return -ENOEXEC; + } + if (!hwx_format(get_unaligned_le32(c + (mixed ? 0x8b0 : 0x890)))) + return -ENOEXEC; + + /* Construct the exact fixed trailer and reserved-space expectation. */ + expected = kzalloc(length, GFP_KERNEL); + if (!expected) + return -ENOMEM; + for (i = 0; i < ARRAY_SIZE(common); i++) + memcpy(expected + common[i], c + common[i], 4); + for (i = 0; i < 5; i++) + memcpy(expected + 0x70 + i * 16, c + 0x70 + i * 16, 8); + put_unaligned_le32(4, expected + 0x868); + put_unaligned_le32(mixed ? 4 : 2, expected + 0x870); + put_unaligned_le32(mixed ? 2 : 7, expected + 0x878); + put_unaligned_le32(8, expected + 0x87c); + put_unaligned_le32(mixed ? get_unaligned_le32(c + 0x858) : 0x461c4000, + expected + 0x880); + put_unaligned_le32(mixed ? 3 : 9, expected + 0x888); + put_unaligned_le32(8, expected + 0x88c); + if (mixed) { + put_unaligned_le32(0x08020004, expected + 0x890); + put_unaligned_le32(8, expected + 0x894); + put_unaligned_le32(7, expected + 0x898); + put_unaligned_le32(8, expected + 0x89c); + put_unaligned_le32(0x461c4000, expected + 0x8a0); + put_unaligned_le32(9, expected + 0x8a8); + put_unaligned_le32(8, expected + 0x8ac); + } + memcpy(expected + (mixed ? 0x8b0 : 0x890), c + (mixed ? 0x8b0 : 0x890), 4); + memcpy(expected + (mixed ? 0x8b8 : 0x898), "main", 5); + if (!memcmp(c, expected, length)) + ret = ane_h16_task_stream_validate(p->bytes + info->text.offset, + info->text.size, info->tasks); + kfree(expected); + return ret; +} + +static int hwx_tensor(struct hwx_parser *p, u32 index) +{ + struct ane_h16_hwx_surface *b = &p->info.surfaces[index]; + const u8 *c = p->bytes + p->tensors[index]; + const u8 *ref = p->bytes + p->refs[index]; + u32 length = get_unaligned_le32(c + 4), shortoff; + u64 n, channels, height, width, row, plane, batch, minimum; + size_t symbol_length, short_length; + unsigned int i; + + if (length < 0xd48 || length > SZ_16K || get_unaligned_le32(c + 0xc) != 0x34a || + get_unaligned_le32(c + 0x10) != 3 || + get_unaligned_le32(c + 0x14) != b->direction || + get_unaligned_le32(c + 0x18) != 0xd38 || !hwx_zero(c, 0x1c, 0x20) || + get_unaligned_le32(c + 0x20) != 0xd3d || + get_unaligned_le32(c + 0x24) != 5 || + get_unaligned_le32(c + 0x38) != 1 || !hwx_zero(c, 0x3c, 0x50) || + get_unaligned_le64(c + 0x68) != 2 || + get_unaligned_le32(c + 0x78) != 1 || !hwx_zero(c, 0x88, 0xd38) || + memcmp(c + 0xd38, "main", 5)) + return -ENOEXEC; + n = get_unaligned_le32(c + 0x28); + channels = get_unaligned_le32(c + 0x2c); + height = get_unaligned_le32(c + 0x30); + width = get_unaligned_le32(c + 0x34); + batch = get_unaligned_le64(c + 0x50); + plane = get_unaligned_le64(c + 0x58); + row = get_unaligned_le64(c + 0x60); + if (!n || !channels || !height || !width || + check_mul_overflow(width, 2ULL, &minimum) || row < minimum || + check_mul_overflow(height, row, &minimum) || plane < minimum || + check_mul_overflow(channels, plane, &minimum) || batch < minimum || + get_unaligned_le64(c + 0x70) != batch || + get_unaligned_le64(c + 0x80) != batch || + check_mul_overflow(n, batch, &b->span) || b->span > b->section.size) + return -ENOEXEC; + symbol_length = strnlen(c + 0xd3d, min_t(u32, 256, length - 0xd3d)); + if (!symbol_length || symbol_length == min_t(u32, 256, length - 0xd3d)) + return -ENOEXEC; + shortoff = get_unaligned_le32(c + 0x7c); + if (shortoff != 0xd3d + symbol_length + 1 || shortoff >= length) + return -ENOEXEC; + short_length = strnlen(c + shortoff, length - shortoff); + if (!short_length || short_length == length - shortoff || + short_length >= sizeof(b->name) || + !hwx_zero(c, shortoff + short_length + 1, length)) + return -ENOEXEC; + memcpy(b->name, c + shortoff, short_length); + b->symbol_offset = p->tensors[index] + 0xd3d; + if (get_unaligned_le64(ref + 8) != 0x18 || + get_unaligned_le64(ref + 0x10) != b->section.vm || + memcmp(ref + 0x18, b->name, sizeof(b->name))) + return -ENOEXEC; + for (i = 0; i < index; i++) + if (!strcmp(b->name, p->info.surfaces[i].name) || + !strcmp(p->bytes + b->symbol_offset, + p->bytes + p->info.surfaces[i].symbol_offset)) + return -ENOEXEC; + return 0; +} + +static int hwx_symbols(struct hwx_parser *p) +{ + u32 found = 0, i, j; + + if (!p->symtab || !p->nsyms || p->nsyms > HWX_SYMBOLS_MAX || + !p->strsize || p->strsize > SZ_256K || + hwx_file_add(p, p->symoff, (u64)p->nsyms * 16) || + hwx_file_add(p, p->stroff, p->strsize) || p->bytes[p->stroff]) + return -ENOEXEC; + for (i = 0; i < p->nsyms; i++) { + const u8 *s = p->bytes + p->symoff + i * 16; + u32 strx = get_unaligned_le32(s); + u64 value = get_unaligned_le64(s + 8); + const struct ane_h16_hwx_section *section; + const char *name; + + if (strx >= p->strsize || !s[5] || s[5] > p->section_count || + (s[4] & ~1) != 0xe) + return -ENOEXEC; + name = p->bytes + p->stroff + strx; + if (!*name || strnlen(name, min_t(u32, 256, p->strsize - strx)) == + min_t(u32, 256, p->strsize - strx)) + return -ENOEXEC; + for (j = 0; j < i; j++) { + u32 previous = get_unaligned_le32(p->bytes + p->symoff + j * 16); + + if (!strcmp(name, p->bytes + p->stroff + previous)) + return -ENOEXEC; + } + section = &p->sections[s[5] - 1]; + if (value < section->vm || value - section->vm >= section->size) + return -ENOEXEC; + for (j = 0; j < p->info.surface_count; j++) { + const struct ane_h16_hwx_surface *b = &p->info.surfaces[j]; + + if (strcmp(name, b->name)) + continue; + if ((found & BIT(j)) || s[5] != b->section.ordinal || + value != b->section.vm) + return -ENOEXEC; + found |= BIT(j); + } + } + return found == BIT(p->info.surface_count) - 1 ? 0 : -ENOEXEC; +} + +/* The stream has already passed the complete structural walk. Clear every + * payload site from the relocation bitmap: any remaining bit names a prefix, + * header, register selector or padding byte, which must never be relocated. + */ +static bool hwx_relocations_in_payload(const struct hwx_parser *p, unsigned long *sites) +{ + const u8 *text = p->bytes + p->info.text.offset; + size_t pos = 16; + + while (pos < p->info.text.size) { + u32 words = (get_unaligned_le32(text + pos) >> 16) & 0x7ff; + size_t end, record; + + if (!words) { + pos += 16; + continue; + } + end = pos + words * 4; + record = pos + 9 * 4; + if ((get_unaligned_le32(text + pos + 8 * 4) & 3) == 3) + record += 4; + while (record < end) { + u32 header = get_unaligned_le32(text + record); + u32 count = header & BIT(31) ? + 1 + hweight32((header >> 15) & 0xffff) : + 1 + ((header >> 15) & 0x3f); + + record += 4; + bitmap_clear(sites, record / 4, count); + record += count * 4; + } + pos = ALIGN(end, 16); + } + return bitmap_empty(sites, p->info.text.size / 4); +} + +int ane_h16_hwx_validate(const void *bytes, size_t size, struct ane_h16_hwx *info) +{ + struct hwx_parser *p; + const void *relocations; + const u8 *c = bytes; + size_t pos = 32; + u32 count, length, i; + unsigned long *seen = NULL; + int ret = -ENOEXEC; + + if (!c || !info || size < 32 || size > ANE_H16_HWX_MAX || + get_unaligned_le32(c) != 0xbeefface || get_unaligned_le32(c + 4) != 0x80 || + get_unaligned_le32(c + 8) != 7 || get_unaligned_le32(c + 12) != 2 || + get_unaligned_le32(c + 24) != 0x200000 || get_unaligned_le32(c + 28)) + return -ENOEXEC; + count = get_unaligned_le32(c + 16); + length = get_unaligned_le32(c + 20); + if (!count || count > HWX_COMMANDS_MAX || !hwx_fits(32, length, size)) + return -ENOEXEC; + p = kzalloc_obj(*p); + if (!p) + return -ENOMEM; + p->bytes = bytes; + p->size = size; + p->command_end = 32 + (size_t)length; + for (i = 0; i < count; i++) { + u32 cmd, len; + + if (!hwx_fits(pos, 8, p->command_end)) + goto invalid; + c = p->bytes + pos; + cmd = get_unaligned_le32(c); + len = get_unaligned_le32(c + 4); + if (len < 8 || (len & 7) || !hwx_fits(pos, len, p->command_end)) + goto invalid; + switch (cmd) { + case 0x19: + ret = hwx_segment(p, c, len); + if (ret) + goto out; + break; + case 2: + if (p->symtab || len != 24) + goto invalid; + p->symtab = true; + p->symoff = get_unaligned_le32(c + 8); + p->nsyms = get_unaligned_le32(c + 12); + p->stroff = get_unaligned_le32(c + 16); + p->strsize = get_unaligned_le32(c + 20); + break; + case 4: + if (len < 16) + goto invalid; + if (get_unaligned_le32(c + 8) == 3) { + if (p->tensor_count == ANE_H16_SURFACES || len < 0xd48) + goto invalid; + p->tensors[p->tensor_count++] = pos; + } else if (get_unaligned_le32(c + 8) == 4) { + if (p->info.program_offset || (len != 0x8a0 && len != 0x8c0)) + goto invalid; + p->info.program_offset = pos; + } else { + goto invalid; + } + break; + case 0x40: + if (len != 32 || p->ref_count == ANE_H16_SURFACES) + goto invalid; + p->refs[p->ref_count++] = pos; + break; + case 8: { + size_t annotation; + + /* Compiler annotation has no authority over mappings. */ + if (p->metadata || len > SZ_4K || len == 8) + goto invalid; + annotation = strnlen(c + 8, len - 8); + if (annotation == len - 8 || !hwx_zero(c, 8 + annotation, len)) + goto invalid; + p->metadata = true; + break; + } + default: + goto invalid; + } + pos += len; + } + if (pos != p->command_end || !p->guard || !p->info.program_offset || + !p->info.text.size || !p->info.constants.size || + p->info.surface_count < 2 || p->ref_count != p->info.surface_count || + p->tensor_count != p->info.surface_count) + goto invalid; + for (i = 0; i < p->info.surface_count; i++) { + /* Inputs in segment order, exactly one output last. */ + if (p->info.surfaces[i].direction != (i == p->info.surface_count - 1 ? 2 : 1)) + goto invalid; + ret = hwx_tensor(p, i); + if (ret) + goto out; + } + ret = hwx_program(p); + if (ret) + goto out; + ret = hwx_symbols(p); + if (ret) + goto out; + if (p->info.text.nreloc) { + seen = bitmap_zalloc(p->info.text.size / 4, GFP_KERNEL); + if (!seen) { + ret = -ENOMEM; + goto out; + } + } + relocations = p->info.text.nreloc ? p->bytes + p->info.text.reloff : NULL; + ret = ane_h16_relocations_validate(p->bytes + p->info.text.offset, + p->info.text.size, relocations, + p->info.text.nreloc, p->info.kernel.ordinal, + p->info.kernel.vm, p->info.kernel.size, + seen, p->info.text.size / 4); + if (!ret && p->info.text.nreloc && !hwx_relocations_in_payload(p, seen)) + ret = -ENOEXEC; + if (!ret) + *info = p->info; + goto out; +invalid: + ret = -ENOEXEC; +out: + bitmap_free(seen); + kfree(p); + return ret; +} +EXPORT_SYMBOL_GPL(ane_h16_hwx_validate); + +int ane_h16_hwx_snapshot_create(const void *bytes, size_t size, + struct ane_h16_hwx_snapshot *snapshot) +{ + struct ane_h16_hwx_snapshot copy = {}; + int ret; + + if (!bytes || !snapshot || size < 32 || size > ANE_H16_HWX_MAX) + return -EINVAL; + copy.bytes = kvmalloc(size, GFP_KERNEL); + if (!copy.bytes) + return -ENOMEM; + memcpy(copy.bytes, bytes, size); + copy.size = size; + ret = ane_h16_hwx_validate(copy.bytes, size, ©.info); + if (ret) { + kvfree(copy.bytes); + return ret; + } + sha256(copy.bytes, size, copy.digest); + *snapshot = copy; + return 0; +} +EXPORT_SYMBOL_GPL(ane_h16_hwx_snapshot_create); + +int ane_h16_hwx_snapshot_check(const struct ane_h16_hwx_snapshot *snapshot) +{ + u8 digest[SHA256_DIGEST_SIZE]; + + if (!snapshot || !snapshot->bytes || snapshot->size < 32 || + snapshot->size > ANE_H16_HWX_MAX) + return -ENOEXEC; + sha256(snapshot->bytes, snapshot->size, digest); + return memcmp(digest, snapshot->digest, sizeof(digest)) ? -ENOEXEC : 0; +} +EXPORT_SYMBOL_GPL(ane_h16_hwx_snapshot_check); + +void ane_h16_hwx_snapshot_destroy(struct ane_h16_hwx_snapshot *snapshot) +{ + kvfree(snapshot->bytes); + memset(snapshot, 0, sizeof(*snapshot)); +} +EXPORT_SYMBOL_GPL(ane_h16_hwx_snapshot_destroy); + +MODULE_DESCRIPTION("H16 original HWX validation"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_hwx.h b/drivers/accel/ane/ane_h16_hwx.h new file mode 100644 index 0000000000000..ba913c0f25d12 --- /dev/null +++ b/drivers/accel/ane/ane_h16_hwx.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_HWX_H__ +#define __ANE_H16_HWX_H__ + +#include +#include +#include + +#define ANE_H16_SURFACES 4 +#define ANE_H16_SECTIONS 8 +#define ANE_H16_HWX_MAX (SZ_256M + SZ_1M) + +struct ane_h16_hwx_section { + u64 vm; + u64 size; + u32 offset; + u32 reloff; + u32 nreloc; + u32 ordinal; +}; + +struct ane_h16_hwx_surface { + struct ane_h16_hwx_section section; + u64 allocation; + u64 span; + u32 direction; + u32 symbol_offset; + char name[8]; +}; + +struct ane_h16_hwx { + struct ane_h16_hwx_section text, constants, kernel, scratch; + struct ane_h16_hwx_surface surfaces[ANE_H16_SURFACES]; + u32 surface_count; + u32 tasks; + u32 program_offset; +}; + +/* Input must be one immutable kernel-owned snapshot of the original file. */ +int ane_h16_hwx_validate(const void *bytes, size_t size, struct ane_h16_hwx *info); + +struct ane_h16_hwx_snapshot { + void *bytes; + size_t size; + u8 digest[SHA256_DIGEST_SIZE]; + struct ane_h16_hwx info; +}; + +/* Snapshot before validation; callers never publish or mutate snapshot bytes. */ +int ane_h16_hwx_snapshot_create(const void *bytes, size_t size, + struct ane_h16_hwx_snapshot *snapshot); +int ane_h16_hwx_snapshot_check(const struct ane_h16_hwx_snapshot *snapshot); +void ane_h16_hwx_snapshot_destroy(struct ane_h16_hwx_snapshot *snapshot); + +#endif /* __ANE_H16_HWX_H__ */ diff --git a/drivers/accel/ane/ane_h16_hwx_test.c b/drivers/accel/ane/ane_h16_hwx_test.c new file mode 100644 index 0000000000000..b7f434748b829 --- /dev/null +++ b/drivers/accel/ane/ane_h16_hwx_test.c @@ -0,0 +1,427 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "ane_h16_hwx.h" +#include "ane_h16_tasks.h" + +struct hwx_fixture { + u8 *bytes; + size_t size, cursor; + u32 commands, program, tensor[2], segment[2], text_command, symtab, reloc; + u32 symbols, strings; +}; + +static u8 *fixture_command(struct hwx_fixture *f, u32 command, u32 length) +{ + u8 *c = f->bytes + f->cursor; + + put_unaligned_le32(command, c); + put_unaligned_le32(length, c + 4); + f->cursor += length; + f->commands++; + return c; +} + +static u8 *fixture_segment(struct hwx_fixture *f, const char *name, u64 vm, + u64 size, u32 offset, u32 prot, u32 flags, u32 sections) +{ + u8 *c = fixture_command(f, 0x19, 72 + sections * 80); + + strscpy(c + 8, name, 16); + put_unaligned_le64(vm, c + 24); + put_unaligned_le64(size, c + 32); + put_unaligned_le64(offset, c + 40); + put_unaligned_le64(offset ? size : 0, c + 48); + put_unaligned_le32(prot, c + 56); + put_unaligned_le32(prot, c + 60); + put_unaligned_le32(sections, c + 64); + put_unaligned_le32(flags, c + 68); + return c + 72; +} + +static void fixture_section(u8 *s, const char *segment, const char *name, + u64 vm, u64 size, u32 offset, u32 alignment, u32 flags) +{ + strscpy(s, name, 16); + strscpy(s + 16, segment, 16); + put_unaligned_le64(vm, s + 32); + put_unaligned_le64(size, s + 40); + put_unaligned_le32(offset, s + 48); + put_unaligned_le32(alignment, s + 52); + put_unaligned_le32(flags, s + 64); +} + +static struct hwx_fixture *fixture_create(struct kunit *test, bool mixed, bool kernel) +{ + struct hwx_fixture *f; + u64 surface = 0x30000000 + (mixed ? SZ_16K : 0); + u64 text = surface + 2 * SZ_16K, kern = text + SZ_32K; + u32 i, strings_length = 1; + u8 *c, *s, *text_section; + static const char * const names[] = { "in", "out" }; + static const char * const symbols[] = { "input", "output" }; + + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + if (!f) + return NULL; + f->size = kernel ? SZ_64K : SZ_32K + SZ_16K; + f->bytes = kunit_kzalloc(test, f->size, GFP_KERNEL); + if (!f->bytes) + return NULL; + f->cursor = 32; + fixture_segment(f, "__PAGEZERO", 0, SZ_16K, 0, 0, 4, 0); + if (mixed) { + s = fixture_segment(f, "__DATA", 0x30000000, SZ_16K, 0, 3, 4, 1); + fixture_section(s, "__DATA", "__bss", 0x30000000, SZ_16K, 0, 6, 0x25); + } + for (i = 0; i < 2; i++) { + f->segment[i] = f->cursor; + s = fixture_segment(f, "__FVMLIB", surface + i * SZ_16K, SZ_16K, + 0, i + 1, 6, 1); + fixture_section(s, "__FVMLIB", i ? "__data" : "__const", + surface + i * SZ_16K, SZ_16K, 0, 14, i ? 0x23 : 0x21); + } + f->text_command = f->cursor; + text_section = fixture_segment(f, "__TEXT", text, SZ_32K, SZ_16K, 5, + mixed ? 4 : 0, 2); + fixture_section(text_section, "__TEXT", "__text", text, 64, SZ_16K, + 14, kernel ? 0x128 : 0x28); + fixture_section(text_section + 80, "__TEXT", "__const", text + 64, + SZ_16K, SZ_16K + 64, 6, 0x26); + if (kernel) { + s = fixture_segment(f, "__KERN_0", kern, SZ_16K, SZ_32K + SZ_16K, 1, 4, 1); + fixture_section(s, "__KERN_0", "__kern_0", kern, 64, + SZ_32K + SZ_16K, 6, 0x26); + } + for (i = 0; i < 2; i++) { + c = fixture_command(f, 0x40, 32); + put_unaligned_le64(0x18, c + 8); + put_unaligned_le64(surface + i * SZ_16K, c + 0x10); + strscpy(c + 0x18, names[i], 8); + } + f->program = f->cursor; + c = fixture_command(f, 4, mixed ? 0x8c0 : 0x8a0); + put_unaligned_le32(4, c + 8); + put_unaligned_le32(mixed ? 0x22a : 0x222, c + 0xc); + put_unaligned_le64(text, c + 0x10); + put_unaligned_le64(text + 64, c + 0x20); + if (mixed) + put_unaligned_le64(0x30000000, c + 0x40); + for (i = 0; i < 2; i++) + put_unaligned_le64(surface + i * SZ_16K, c + 0x70 + i * 16); + if (kernel) + put_unaligned_le64(kern, c + 0x90); + put_unaligned_le64(text, c + 0x810); + put_unaligned_le32(4, c + 0x820); + put_unaligned_le32(16, c + 0x824); + put_unaligned_le32(1, c + 0x830); + put_unaligned_le32(U32_MAX, c + 0x838); + put_unaligned_le32(U32_MAX, c + 0x83c); + put_unaligned_le32(0xffff, c + 0x844); + put_unaligned_le32(mixed ? 0x8b8 : 0x898, c + 0x848); + put_unaligned_le32(0x10, c + 0x860); + put_unaligned_le32(4, c + 0x868); + put_unaligned_le32(mixed ? 4 : 2, c + 0x870); + put_unaligned_le32(mixed ? 2 : 7, c + 0x878); + put_unaligned_le32(8, c + 0x87c); + put_unaligned_le32(mixed ? 256 : 0x461c4000, c + 0x880); + put_unaligned_le32(mixed ? 3 : 9, c + 0x888); + put_unaligned_le32(8, c + 0x88c); + if (mixed) { + put_unaligned_le32(256, c + 0x858); + put_unaligned_le32(0x08020004, c + 0x890); + put_unaligned_le32(8, c + 0x894); + put_unaligned_le32(7, c + 0x898); + put_unaligned_le32(8, c + 0x89c); + put_unaligned_le32(0x461c4000, c + 0x8a0); + put_unaligned_le32(9, c + 0x8a8); + put_unaligned_le32(8, c + 0x8ac); + } + memcpy(c + (mixed ? 0x8b8 : 0x898), "main", 5); + for (i = 0; i < 2; i++) { + u32 shortoff = 0xd3d + strlen(symbols[i]) + 1; + + f->tensor[i] = f->cursor; + c = fixture_command(f, 4, 0xd48); + put_unaligned_le32(3, c + 8); + put_unaligned_le32(0x34a, c + 0xc); + put_unaligned_le32(3, c + 0x10); + put_unaligned_le32(i + 1, c + 0x14); + put_unaligned_le32(0xd38, c + 0x18); + put_unaligned_le32(0xd3d, c + 0x20); + put_unaligned_le32(5, c + 0x24); + put_unaligned_le32(1, c + 0x28); + put_unaligned_le32(1, c + 0x2c); + put_unaligned_le32(1, c + 0x30); + put_unaligned_le32(128, c + 0x34); + put_unaligned_le32(1, c + 0x38); + put_unaligned_le64(256, c + 0x50); + put_unaligned_le64(256, c + 0x58); + put_unaligned_le64(256, c + 0x60); + put_unaligned_le64(2, c + 0x68); + put_unaligned_le64(256, c + 0x70); + put_unaligned_le32(1, c + 0x78); + put_unaligned_le32(shortoff, c + 0x7c); + put_unaligned_le64(256, c + 0x80); + memcpy(c + 0xd38, "main", 5); + memcpy(c + 0xd3d, symbols[i], strlen(symbols[i]) + 1); + memcpy(c + shortoff, names[i], strlen(names[i]) + 1); + } + f->symtab = f->cursor; + c = fixture_command(f, 2, 24); + f->symbols = f->cursor; + f->strings = f->symbols + 2 * 16; + for (i = 0; i < 2; i++) { + u8 *symbol = f->bytes + f->symbols + i * 16; + + put_unaligned_le32(strings_length, symbol); + symbol[4] = 0xf; + symbol[5] = (mixed ? 2 : 1) + i; + put_unaligned_le64(surface + i * SZ_16K, symbol + 8); + memcpy(f->bytes + f->strings + strings_length, names[i], strlen(names[i]) + 1); + strings_length += strlen(names[i]) + 1; + } + put_unaligned_le32(f->symbols, c + 8); + put_unaligned_le32(2, c + 12); + put_unaligned_le32(f->strings, c + 16); + put_unaligned_le32(strings_length, c + 20); + f->reloc = ALIGN(f->strings + strings_length, 8); + if (kernel) { + put_unaligned_le32(f->reloc, text_section + 56); + put_unaligned_le32(1, text_section + 60); + put_unaligned_le32(56, f->bytes + f->reloc); + put_unaligned_le32(0x07000000 | (mixed ? 6 : 5), f->bytes + f->reloc + 4); + put_unaligned_le32(32, f->bytes + SZ_16K + 56); + } + put_unaligned_le32(11 << 16, f->bytes + SZ_16K + 16); + put_unaligned_le32(0x4100 / 4, f->bytes + SZ_16K + 52); + put_unaligned_le32(0xbeefface, f->bytes); + put_unaligned_le32(0x80, f->bytes + 4); + put_unaligned_le32(7, f->bytes + 8); + put_unaligned_le32(2, f->bytes + 12); + put_unaligned_le32(f->commands, f->bytes + 16); + put_unaligned_le32(f->cursor - 32, f->bytes + 20); + put_unaligned_le32(0x200000, f->bytes + 24); + return f; +} + +static void hwx_forms_test(struct kunit *test) +{ + unsigned int variant; + + for (variant = 0; variant < 4; variant++) { + struct hwx_fixture *f = fixture_create(test, variant & 1, variant & 2); + struct ane_h16_hwx info = {}; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), 0); + KUNIT_EXPECT_EQ(test, info.surface_count, 2U); + KUNIT_EXPECT_EQ(test, info.surfaces[0].span, 256ULL); + KUNIT_EXPECT_EQ(test, info.tasks, 1U); + KUNIT_EXPECT_EQ(test, info.text.ordinal, (variant & 1) ? 4U : 3U); + KUNIT_EXPECT_EQ(test, info.text.nreloc, (variant & 2) ? 1U : 0U); + } +} + +static void hwx_header_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx info; + u32 offset; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (offset = 0; offset < 32; offset += 4) { + u32 saved = get_unaligned_le32(f->bytes + offset); + + put_unaligned_le32(U32_MAX, f->bytes + offset); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(saved, f->bytes + offset); + } + for (offset = 0; offset < 32; offset++) + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, offset, &info), -ENOEXEC); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size - 1, &info), -ENOEXEC); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, ANE_H16_HWX_MAX + 1, &info), -ENOEXEC); + /* A truncated command must not inherit a previous command's success. */ + put_unaligned_le32(f->cursor - 33, f->bytes + 20); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); +} + +static void hwx_command_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx info; + u32 pos; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (pos = 32; pos < f->cursor;) { + u32 command = get_unaligned_le32(f->bytes + pos); + u32 length = get_unaligned_le32(f->bytes + pos + 4); + + put_unaligned_le32(0xffffffff, f->bytes + pos); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(command, f->bytes + pos); + put_unaligned_le32(0, f->bytes + pos + 4); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(length, f->bytes + pos + 4); + pos += length; + } +} + +static void hwx_tensor_test(struct kunit *test) +{ + static const u32 fields[] = { + 0xc, 0x10, 0x14, 0x18, 0x1c, 0x20, 0x24, 0x28, 0x2c, + 0x30, 0x34, 0x38, 0x3c, 0x50, 0x58, 0x60, 0x68, 0x70, + 0x78, 0x7c, 0x80, 0x88, 0xd38, 0xd44, + }; + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx info; + u8 *tensor; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + tensor = f->bytes + f->tensor[0]; + for (i = 0; i < ARRAY_SIZE(fields); i++) { + u32 saved = get_unaligned_le32(tensor + fields[i]); + + put_unaligned_le32(U32_MAX, tensor + fields[i]); + KUNIT_EXPECT_EQ_MSG(test, ane_h16_hwx_validate(f->bytes, f->size, &info), + -ENOEXEC, "tensor field %#x", fields[i]); + put_unaligned_le32(saved, tensor + fields[i]); + } + /* Four batches fit in the allocation; span is not just one batch. */ + put_unaligned_le32(4, tensor + 0x28); + KUNIT_ASSERT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), 0); + KUNIT_EXPECT_EQ(test, info.surfaces[0].span, 1024ULL); + put_unaligned_le64(U64_MAX, tensor + 0x50); + put_unaligned_le64(U64_MAX, tensor + 0x70); + put_unaligned_le64(U64_MAX, tensor + 0x80); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); +} + +static void hwx_program_test(struct kunit *test) +{ + static const u32 fields[] = { + 0xc, 0x10, 0x18, 0x20, 0x28, 0x40, 0x48, 0x70, 0x78, 0x80, 0x88, + 0x90, 0xa0, 0xb0, 0xb8, 0xc0, 0x80c, 0x810, 0x818, 0x820, 0x824, + 0x828, 0x830, 0x834, 0x838, 0x83c, 0x840, 0x844, 0x848, 0x84c, + 0x858, 0x85c, 0x860, 0x864, 0x868, 0x870, 0x878, 0x880, 0x890, 0x89c, + }; + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx info; + u8 *program; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + program = f->bytes + f->program; + for (i = 0; i < ARRAY_SIZE(fields); i++) { + u32 saved = get_unaligned_le32(program + fields[i]); + + put_unaligned_le32(saved ^ U32_MAX, program + fields[i]); + KUNIT_EXPECT_EQ_MSG(test, ane_h16_hwx_validate(f->bytes, f->size, &info), + -ENOEXEC, "program field %#x", fields[i]); + put_unaligned_le32(saved, program + fields[i]); + } +} + +static void hwx_geometry_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx info; + u8 *segment; + unsigned int variant; + + KUNIT_ASSERT_NOT_NULL(test, f); + segment = f->bytes + f->segment[0]; + for (variant = 0; variant < 8; variant++) { + u32 fields[] = { 24, 32, 40, 48, 56, 60, 64, 72 + 52 }; + u32 saved = get_unaligned_le32(segment + fields[variant]); + + put_unaligned_le32(U32_MAX, segment + fields[variant]); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(saved, segment + fields[variant]); + } + put_unaligned_le64(U64_MAX - SZ_16K + 1, segment + 24); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); +} + +static void hwx_symbols_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx info; + u32 field; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (field = 8; field < 24; field += 4) { + u32 saved = get_unaligned_le32(f->bytes + f->symtab + field); + + put_unaligned_le32(U32_MAX, f->bytes + f->symtab + field); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(saved, f->bytes + f->symtab + field); + } + f->bytes[f->symbols + 5] = 0; + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + f->bytes[f->symbols + 5] = 1; + f->bytes[f->strings + 1] = 'X'; + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); +} + +static void hwx_relocations_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, true, true); + struct ane_h16_hwx info; + + KUNIT_ASSERT_NOT_NULL(test, f); + /* Scratch changes the section ordinal; a fixed ordinal is invalid. */ + put_unaligned_le32(0x07000005, f->bytes + f->reloc + 4); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(0x07000006, f->bytes + f->reloc + 4); + put_unaligned_le32(64, f->bytes + SZ_16K + 56); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(32, f->bytes + SZ_16K + 56); + put_unaligned_le32(57, f->bytes + f->reloc); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + /* A small addend in a header or the prefix must not be patched. */ + put_unaligned_le32(20, f->bytes + f->reloc); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + put_unaligned_le32(4, f->bytes + f->reloc); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); +} + +static void hwx_snapshot_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx_snapshot snapshot = {}; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, ane_h16_hwx_snapshot_create(f->bytes, f->size, &snapshot), 0); + f->bytes[f->tensor[0] + 0x24] = 0; + KUNIT_EXPECT_EQ(test, ane_h16_hwx_snapshot_check(&snapshot), 0); + /* Even padding and original command bytes are covered by the digest. */ + ((u8 *)snapshot.bytes)[f->program + 0x100] = 1; + KUNIT_EXPECT_EQ(test, ane_h16_hwx_snapshot_check(&snapshot), -ENOEXEC); + ane_h16_hwx_snapshot_destroy(&snapshot); + KUNIT_EXPECT_PTR_EQ(test, snapshot.bytes, NULL); +} + +static struct kunit_case hwx_cases[] = { + KUNIT_CASE(hwx_forms_test), + KUNIT_CASE(hwx_header_test), + KUNIT_CASE(hwx_command_test), + KUNIT_CASE(hwx_tensor_test), + KUNIT_CASE(hwx_program_test), + KUNIT_CASE(hwx_geometry_test), + KUNIT_CASE(hwx_symbols_test), + KUNIT_CASE(hwx_relocations_test), + KUNIT_CASE(hwx_snapshot_test), + {} +}; + +static struct kunit_suite hwx_suite = { + .name = "ane-h16-hwx", + .test_cases = hwx_cases, +}; + +kunit_test_suite(hwx_suite); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_pending.h b/drivers/accel/ane/ane_h16_pending.h new file mode 100644 index 0000000000000..b74b20a0b0ea2 --- /dev/null +++ b/drivers/accel/ane/ane_h16_pending.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_PENDING_H__ +#define __ANE_H16_PENDING_H__ + +#include + +#include "ane_h16_execute.h" + +#define ANE_H16_PENDING_MAX 64 + +struct ane_h16_pending { + struct ane_h16_request_key key; + unsigned long deadline; + int status; + bool active; + bool terminal; +}; + +/* All operations, including event dispatch, require the firmware mutex. */ +struct ane_h16_pending_table { + struct ane_h16_pending *slots[ANE_H16_PENDING_MAX]; + u64 next_uuid; + int error; + bool quarantined; +}; + +static inline int +ane_h16_pending_insert(struct ane_h16_pending_table *table, + struct ane_h16_pending *request, unsigned long now) +{ + unsigned int i; + + if (table->quarantined) + return -EIO; + if (!request || request->active || time_after_eq(now, request->deadline)) + return -EINVAL; + for (i = 0; i < ANE_H16_PENDING_MAX; i++) + if (table->slots[i] == request) + return -EEXIST; + if (table->next_uuid == U64_MAX) + return -EOVERFLOW; + for (i = 0; i < ANE_H16_PENDING_MAX; i++) { + if (table->slots[i]) + continue; + /* Never reuse a UUID, including after pre-publication failure. */ + request->key.uuid = ++table->next_uuid; + request->status = -EINPROGRESS; + request->terminal = false; + request->active = true; + table->slots[i] = request; + return 0; + } + return -ENOSPC; +} + +/* Removal is not permission to release DMA backing. */ +static inline void +ane_h16_pending_remove(struct ane_h16_pending_table *table, + struct ane_h16_pending *request) +{ + unsigned int i; + + for (i = 0; i < ANE_H16_PENDING_MAX; i++) + if (table->slots[i] == request) + table->slots[i] = NULL; + request->active = false; +} + +/* A published command with an uncertain outcome pins all its mappings. */ +static inline void +ane_h16_pending_abort(struct ane_h16_pending_table *table, + struct ane_h16_pending *request, int status) +{ + table->quarantined = true; + table->error = status; + ane_h16_pending_remove(table, request); + request->status = status; +} + +static inline int +ane_h16_pending_event(struct ane_h16_pending_table *table, const void *bytes, + size_t length, unsigned int channel, unsigned long now) +{ + struct ane_h16_pending *request = NULL; + struct ane_h16_event event; + unsigned int i; + int ret; + + if (table->quarantined) + return -EIO; + ret = ane_h16_event_decode(bytes, length, channel, &event); + if (ret) + goto corrupt; + if (event.destination) + return -ENOMSG; + for (i = 0; i < ANE_H16_PENDING_MAX; i++) + if (table->slots[i] && table->slots[i]->key.uuid == event.key.uuid) { + request = table->slots[i]; + break; + } + /* Duplicates and late/foreign UUIDs can never retire another call. */ + if (!request) + return -ENOENT; + ret = ane_h16_event_result(bytes, length, channel, &request->key); + if (ret < 0) + goto corrupt; + if (time_after_eq(now, request->deadline)) { + ane_h16_pending_abort(table, request, -ETIMEDOUT); + return -ETIMEDOUT; + } + if (ret == ANE_H16_EVENT_PROGRESS) + return ret; + ane_h16_pending_remove(table, request); + request->terminal = true; + request->status = ret == ANE_H16_EVENT_COMPLETE ? 0 : -EIO; + if (request->status) { + table->error = request->status; + table->quarantined = true; + } + return ret; + +corrupt: + table->error = ret; + table->quarantined = true; + return ret; +} + +/* Channel-6 CPU-load notifications are diagnostics, never completions. */ +static inline int +ane_h16_pending_dispatch(struct ane_h16_pending_table *table, const void *bytes, + size_t length, unsigned int channel, unsigned long now) +{ + const u8 *p = bytes; + + if (table->quarantined) + return -EIO; + if (p && channel == 6 && length == 0xc && + get_unaligned_le16(p + 4) == 0x0107 && !get_unaligned_le16(p + 6)) + return ANE_H16_EVENT_PROGRESS; + return ane_h16_pending_event(table, bytes, length, channel, now); +} + +#endif /* __ANE_H16_PENDING_H__ */ diff --git a/drivers/accel/ane/ane_h16_pending_test.c b/drivers/accel/ane/ane_h16_pending_test.c new file mode 100644 index 0000000000000..891342b9a7a37 --- /dev/null +++ b/drivers/accel/ane/ane_h16_pending_test.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "ane_h16_pending.h" +#include "ane_h16_resources.h" + +static void make_event(u8 *bytes, const struct ane_h16_pending *request, u32 state) +{ + memset(bytes, 0, 0x60); + put_unaligned_le32(0x12345678, bytes); + put_unaligned_le16(ANE_H16_EVENT_REPORT, bytes + 4); + put_unaligned_le64(request->key.uuid, bytes + 8); + put_unaligned_le32(request->key.program, bytes + 0x10); + put_unaligned_le32(request->key.process, bytes + 0x14); + put_unaligned_le32(request->key.procedure, bytes + 0x18); + put_unaligned_le32(state, bytes + 0x1c); +} + +static void pending_sequence_test(struct kunit *test) +{ + struct ane_h16_pending_table table = {}; + struct ane_h16_pending a = { .key = { .program = 7, .process = 8, .procedure = 9 }, + .deadline = 100 }; + struct ane_h16_pending b = { .key = a.key, .deadline = 100 }; + u8 event[0x60], ack[8] = {}; + u32 state; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &a, 0), 0); + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &b, 0), 0); + KUNIT_EXPECT_NE(test, a.key.uuid, b.key.uuid); + /* A valid transport acknowledgment leaves both UUIDs pending. */ + put_unaligned_le16(ANE_H16_EXECUTE, ack + 4); + KUNIT_EXPECT_EQ(test, ane_fw_reply_status(0x10001, 0xca8, 0, + 0x10000, 0xca8, ANE_H16_EXECUTE, ack, true), 0); + KUNIT_EXPECT_TRUE(test, a.active); + KUNIT_EXPECT_TRUE(test, b.active); + for (state = 0; state <= 3; state += 3) { + make_event(event, &a, state); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 1), + ANE_H16_EVENT_PROGRESS); + KUNIT_EXPECT_TRUE(test, a.active); + KUNIT_EXPECT_FALSE(test, a.terminal); + } + make_event(event, &b, 1); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 2), + ANE_H16_EVENT_COMPLETE); + KUNIT_EXPECT_TRUE(test, b.terminal); + KUNIT_EXPECT_EQ(test, b.status, 0); + KUNIT_EXPECT_FALSE(test, b.active); + KUNIT_EXPECT_TRUE(test, a.active); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 3), -ENOENT); + make_event(event, &a, 1); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 4), + ANE_H16_EVENT_COMPLETE); + KUNIT_EXPECT_FALSE(test, table.quarantined); +} + +static void pending_mismatched_ids_test(struct kunit *test) +{ + unsigned int offset; + + for (offset = 0x10; offset <= 0x18; offset += 4) { + struct ane_h16_pending_table table = {}; + struct ane_h16_pending request = { .deadline = 100 }; + u8 event[0x60]; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &request, 0), 0); + make_event(event, &request, 1); + put_unaligned_le32(1, event + offset); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 1), -EPROTO); + KUNIT_EXPECT_TRUE(test, request.active); + KUNIT_EXPECT_FALSE(test, request.terminal); + KUNIT_EXPECT_TRUE(test, table.quarantined); + KUNIT_EXPECT_EQ(test, table.error, -EPROTO); + ane_h16_pending_abort(&table, &request, -EIO); + KUNIT_EXPECT_FALSE(test, request.active); + } +} + +static void pending_malformed_test(struct kunit *test) +{ + unsigned int variant; + + for (variant = 0; variant < 9; variant++) { + struct ane_h16_pending_table table = {}; + struct ane_h16_pending request = { .deadline = 100 }; + u8 event[0x60]; + unsigned int channel = 6; + size_t length = sizeof(event); + const void *bytes = event; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &request, 0), 0); + make_event(event, &request, 1); + switch (variant) { + case 0: + bytes = NULL; + break; + case 1: + length--; + break; + case 2: + length++; + break; + case 3: + channel = 7; + break; + case 4: + put_unaligned_le16(ANE_H16_EXECUTE, event + 4); + break; + case 5: + event[6] = 1; + break; + case 6: + event[7] = 1; + break; + case 7: + put_unaligned_le32(4, event + 0x1c); + break; + case 8: + event[0x30] = 2; + break; + } + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, bytes, length, channel, 1), + -EPROTO); + KUNIT_EXPECT_TRUE(test, request.active); + KUNIT_EXPECT_TRUE(test, table.quarantined); + } +} + +static void pending_routing_test(struct kunit *test) +{ + struct ane_h16_pending_table table = {}; + struct ane_h16_pending request = { .deadline = 100 }; + u8 event[0x60]; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &request, 0), 0); + make_event(event, &request, 1); + event[0x30] = 1; + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 1), -ENOMSG); + event[0x30] = 0; + put_unaligned_le64(request.key.uuid + 1, event + 8); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 1), -ENOENT); + memset(event, 0, sizeof(event)); + put_unaligned_le16(0x0107, event + 4); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0xc, 6, 1), + ANE_H16_EVENT_PROGRESS); + KUNIT_EXPECT_TRUE(test, request.active); + KUNIT_EXPECT_FALSE(test, table.quarantined); + event[7] = 1; + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0xc, 6, 1), -EPROTO); + KUNIT_EXPECT_TRUE(test, table.quarantined); +} + +static void pending_failure_test(struct kunit *test) +{ + struct ane_h16_pending_table table = {}; + struct ane_h16_pending request = { .deadline = 100 }; + struct ane_h16_pending next = { .deadline = 100 }; + u8 event[0x60]; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &request, 0), 0); + make_event(event, &request, 2); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 1), + ANE_H16_EVENT_FAILED); + KUNIT_EXPECT_FALSE(test, request.active); + KUNIT_EXPECT_TRUE(test, request.terminal); + KUNIT_EXPECT_EQ(test, request.status, -EIO); + KUNIT_EXPECT_TRUE(test, table.quarantined); + KUNIT_EXPECT_EQ(test, ane_h16_pending_insert(&table, &next, 2), -EIO); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 2), -EIO); +} + +static void pending_timeout_test(struct kunit *test) +{ + struct ane_h16_pending_table table = {}; + struct ane_h16_pending request = { .deadline = 100 }; + u8 event[0x60]; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &request, 0), 0); + make_event(event, &request, 1); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 100), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, request.status, -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, table.error, -ETIMEDOUT); + KUNIT_EXPECT_FALSE(test, request.terminal); + KUNIT_EXPECT_FALSE(test, request.active); + KUNIT_EXPECT_TRUE(test, table.quarantined); + /* Late success never makes output readable after timeout. */ + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 101), -EIO); +} + +static void pending_transport_failure_test(struct kunit *test) +{ + static const int errors[] = { -ETIMEDOUT, -EPROTO, -EREMOTEIO, -EIO }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(errors); i++) { + struct ane_h16_pending_table table = {}; + struct ane_h16_pending request = { .deadline = 100 }; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &request, 0), 0); + ane_h16_pending_abort(&table, &request, errors[i]); + KUNIT_EXPECT_FALSE(test, request.active); + KUNIT_EXPECT_FALSE(test, request.terminal); + KUNIT_EXPECT_EQ(test, request.status, errors[i]); + KUNIT_EXPECT_TRUE(test, table.quarantined); + } +} + +static void pending_capacity_test(struct kunit *test) +{ + struct ane_h16_pending_table table = {}; + struct ane_h16_pending *requests; + unsigned int i; + + requests = kunit_kcalloc(test, ANE_H16_PENDING_MAX + 1, sizeof(*requests), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, requests); + for (i = 0; i <= ANE_H16_PENDING_MAX; i++) + requests[i].deadline = 100; + for (i = 0; i < ANE_H16_PENDING_MAX; i++) { + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &requests[i], 0), 0); + KUNIT_EXPECT_EQ(test, requests[i].key.uuid, (u64)i + 1); + } + KUNIT_EXPECT_EQ(test, ane_h16_pending_insert(&table, &requests[i], 0), -ENOSPC); + ane_h16_pending_remove(&table, &requests[0]); + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &requests[i], 0), 0); + KUNIT_EXPECT_EQ(test, requests[i].key.uuid, (u64)ANE_H16_PENDING_MAX + 1); + ane_h16_pending_remove(&table, &requests[i]); + table.next_uuid = U64_MAX; + KUNIT_EXPECT_EQ(test, ane_h16_pending_insert(&table, &requests[i], 0), -EOVERFLOW); +} + +static void pending_deadline_wrap_test(struct kunit *test) +{ + struct ane_h16_pending_table table = {}; + struct ane_h16_pending request = { .deadline = 3 }; + u8 event[0x60]; + + KUNIT_ASSERT_EQ(test, ane_h16_pending_insert(&table, &request, ULONG_MAX - 4), 0); + make_event(event, &request, 1); + KUNIT_EXPECT_EQ(test, ane_h16_pending_dispatch(&table, event, 0x60, 6, 1), + ANE_H16_EVENT_COMPLETE); + request.deadline = 3; + KUNIT_EXPECT_EQ(test, ane_h16_pending_insert(&table, &request, 3), -EINVAL); +} + +static void resources_test(struct kunit *test) +{ + struct ane_h16_hwx hwx = { .surface_count = 2 }; + struct ane_h16_resource resources[2] = {}; + unsigned int i, variant; + + for (i = 0; i < 2; i++) { + hwx.surfaces[i].section.vm = 0x30000000 + i * SZ_16K; + hwx.surfaces[i].section.size = SZ_16K; + hwx.surfaces[i].allocation = SZ_16K; + hwx.surfaces[i].span = 256; + hwx.surfaces[i].direction = i + 1; + resources[i].hwx_vm = hwx.surfaces[i].section.vm; + resources[i].permissions = i + 1; + resources[i].buffer = (struct ane_h16_execute_buffer) { + .index = i, .class = i, .mapping_dva = 0x100000000ULL + i * SZ_16K, + .mapping_size = SZ_16K, .size = 256, + }; + } + KUNIT_EXPECT_EQ(test, ane_h16_resources_validate(&hwx, resources, 2), 0); + KUNIT_EXPECT_EQ(test, ane_h16_resources_validate(&hwx, resources, 1), -EINVAL); + for (variant = 0; variant < 9; variant++) { + struct ane_h16_resource bad[2]; + + memcpy(bad, resources, sizeof(bad)); + switch (variant) { + case 0: + bad[0].permissions = 2; + break; + case 1: + bad[0].hwx_vm++; + break; + case 2: + bad[0].buffer.class = 8; + break; + case 3: + bad[0].buffer.size++; + break; + case 4: + bad[0].buffer.offset = 1; + break; + case 5: + bad[0].buffer.mapping_dva = U64_MAX - SZ_16K + 1; + break; + case 6: + bad[1].buffer.index = bad[0].buffer.index; + break; + case 7: + bad[1].buffer.mapping_dva = bad[0].buffer.mapping_dva; + break; + case 8: + bad[0].buffer.mapping_size--; + break; + } + KUNIT_EXPECT_EQ(test, ane_h16_resources_validate(&hwx, bad, 2), -EINVAL); + } +} + +static struct kunit_case pending_cases[] = { + KUNIT_CASE(pending_sequence_test), + KUNIT_CASE(pending_mismatched_ids_test), + KUNIT_CASE(pending_malformed_test), + KUNIT_CASE(pending_routing_test), + KUNIT_CASE(pending_failure_test), + KUNIT_CASE(pending_timeout_test), + KUNIT_CASE(pending_transport_failure_test), + KUNIT_CASE(pending_capacity_test), + KUNIT_CASE(pending_deadline_wrap_test), + KUNIT_CASE(resources_test), + {} +}; + +static struct kunit_suite pending_suite = { + .name = "ane-h16-pending", + .test_cases = pending_cases, +}; + +kunit_test_suite(pending_suite); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_protocol.h b/drivers/accel/ane/ane_h16_protocol.h new file mode 100644 index 0000000000000..a3ef6a9c25ae8 --- /dev/null +++ b/drivers/accel/ane/ane_h16_protocol.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_PROTOCOL_H__ +#define __ANE_H16_PROTOCOL_H__ + +#include "ane_fw_protocol.h" + +/* Derived field facts from H16/macOS 27.0, 2026-10-31. No payload bytes. */ +enum ane_h16_command { + ANE_H16_LOAD_PROGRAM = 0x0200, + ANE_H16_UNLOAD_PROGRAM = 0x0201, + ANE_H16_CREATE_PROCESS = 0x0202, + ANE_H16_TERMINATE_PROCESS = 0x0203, + ANE_H16_EXECUTE = 0x020c, + ANE_H16_POWER_GATE = 0x002d, + ANE_H16_EVENT_REPORT = 0x0300, +}; + +struct ane_h16_command_info { + u16 id; + u16 min_size; + u16 max_size; + u8 size_step; + u8 channel; + bool from_firmware; +}; + +static inline const struct ane_h16_command_info *ane_h16_command_info(u16 id) +{ + static const struct ane_h16_command_info commands[] = { + { ANE_H16_LOAD_PROGRAM, 0x1c0, 0x1c0, 1, 1, false }, + { ANE_H16_UNLOAD_PROGRAM, 0xc, 0xc, 1, 1, false }, + { ANE_H16_CREATE_PROCESS, 0x258, 0x258, 1, 1, false }, + { ANE_H16_TERMINATE_PROCESS, 0x10, 0x10, 1, 1, false }, + { ANE_H16_EXECUTE, 0xca8, 0xcd8, 0x18, 7, false }, + { ANE_H16_POWER_GATE, 0xc, 0xc, 1, 7, false }, + { ANE_H16_EVENT_REPORT, 0x60, 0x60, 1, 6, true }, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(commands); i++) + if (commands[i].id == id) + return &commands[i]; + return NULL; +} + +/* Validate the runtime wire shape, never infer a body from the H14 ABI. */ +static inline int +ane_h16_command_prepare(void *command, size_t length, u16 id, unsigned int channel) +{ + const struct ane_h16_command_info *info = ane_h16_command_info(id); + + if (!info || info->from_firmware) + return -EOPNOTSUPP; + if (!command || channel != info->channel || length < info->min_size || + length > info->max_size || (length - info->min_size) % info->size_step) + return -EINVAL; + if (id == ANE_H16_POWER_GATE && + get_unaligned_le32((u8 *)command + 8) > 1) + return -EINVAL; + ane_fw_command_header(command, id, true); + return 0; +} + +/* The first word selects report slots; rolling words are also valid. */ +static inline int +ane_h16_event_report_status(const void *event, size_t length, unsigned int channel) +{ + const u8 *p = event; + const struct ane_h16_command_info *info = + ane_h16_command_info(ANE_H16_EVENT_REPORT); + + if (!p || channel != info->channel || length != info->min_size || + get_unaligned_le16(p + 4) != info->id || get_unaligned_le16(p + 6)) + return -EPROTO; + return 0; +} + +#endif /* __ANE_H16_PROTOCOL_H__ */ diff --git a/drivers/accel/ane/ane_h16_resources.h b/drivers/accel/ane/ane_h16_resources.h new file mode 100644 index 0000000000000..46fa17d506e2c --- /dev/null +++ b/drivers/accel/ane/ane_h16_resources.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_RESOURCES_H__ +#define __ANE_H16_RESOURCES_H__ + +#include "ane_h16_execute.h" +#include "ane_h16_hwx.h" + +/* Created only from a validated HWX and a decoded firmware process resource + * map. No userspace field may supply this authority. Each mapping is private, + * coherent DMA memory with no userspace alias, retained for process lifetime. + */ +struct ane_h16_resource { + u64 hwx_vm; + u32 permissions; + struct ane_h16_execute_buffer buffer; +}; + +static inline int +ane_h16_resources_validate(const struct ane_h16_hwx *hwx, + const struct ane_h16_resource *resources, unsigned int count) +{ + unsigned int i, j; + + if (!hwx || !resources || count < 2 || count > ANE_H16_SURFACES || + count != hwx->surface_count) + return -EINVAL; + for (i = 0; i < count; i++) { + const struct ane_h16_hwx_surface *s = &hwx->surfaces[i]; + const struct ane_h16_resource *r = &resources[i]; + const struct ane_h16_execute_buffer *b = &r->buffer; + u64 end; + + /* Only the established input/output classes are executable. */ + if (s->direction < 1 || s->direction > 2 || + r->hwx_vm != s->section.vm || r->permissions != s->direction || + b->class != s->direction - 1 || !s->span || + s->span > s->section.size || s->section.size > s->allocation || + !IS_ALIGNED(s->allocation, SZ_16K) || + b->size != s->span || b->offset || !b->mapping_dva || + !IS_ALIGNED(b->mapping_dva, SZ_16K) || + b->mapping_size != s->allocation || b->size > b->mapping_size || + check_add_overflow(b->mapping_dva, b->mapping_size, &end)) + return -EINVAL; + for (j = 0; j < i; j++) { + const struct ane_h16_execute_buffer *a = &resources[j].buffer; + + if (a->index == b->index || + (b->mapping_dva < a->mapping_dva + a->mapping_size && + a->mapping_dva < end)) + return -EINVAL; + } + } + return 0; +} + +#endif /* __ANE_H16_RESOURCES_H__ */ diff --git a/drivers/accel/ane/ane_h16_tasks.h b/drivers/accel/ane/ane_h16_tasks.h new file mode 100644 index 0000000000000..c270d748fbf0d --- /dev/null +++ b/drivers/accel/ane/ane_h16_tasks.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_TASKS_H__ +#define __ANE_H16_TASKS_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define ANE_H16_TASK_MAX 0x60000 + +/* Byte addresses in the source-backed H16 register image, exclusive ends. */ +static inline bool ane_h16_register_valid(u32 word) +{ + static const struct { + u32 start; + u32 end; + } blocks[] = { + { 0x0000, 0x005c }, { 0x4100, 0x41a4 }, { 0x44d0, 0x44d4 }, + { 0x4500, 0x453c }, { 0x4900, 0x4930 }, { 0x4d00, 0x4e44 }, + { 0x5100, 0x5154 }, { 0x5500, 0x5620 }, { 0x5900, 0x5930 }, + }; + u32 addr = word * 4; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(blocks); i++) + if (addr >= blocks[i].start && addr < blocks[i].end) + return true; + return false; +} + +/* + * H16 streams begin with little-endian u32 1 and twelve reserved zero bytes. + * This structural gate does not replace container/tensor/aperture validation. + */ +static inline int +ane_h16_task_stream_validate(const void *text, size_t size, u32 expected_tasks) +{ + const u8 *p = text; + size_t pos = 16; + u32 tasks = 0; + + if (!p || !expected_tasks || size < 16 || size > ANE_H16_TASK_MAX || + (size & 3) || get_unaligned_le32(p) != 1 || + memchr_inv(p + 4, 0, 12)) + return -ENOEXEC; + while (pos < size) { + u32 words, header_words = 9; + size_t end, padded, rec; + + if (size - pos < 16) + return -ENOEXEC; + words = (get_unaligned_le32(p + pos) >> 16) & 0x7ff; + if (!words) { + if (memchr_inv(p + pos, 0, 16)) + return -ENOEXEC; + pos += 16; + continue; + } + if (words < 9 || words > (size - pos) / 4) + return -ENOEXEC; + end = pos + words * 4; + if ((get_unaligned_le32(p + pos + 8 * 4) & 3) == 3) + header_words++; + if (words < header_words) + return -ENOEXEC; + rec = pos + header_words * 4; + while (rec < end) { + u32 h = get_unaligned_le32(p + rec); + u32 base = h & 0x7fff; + u32 mask = (h >> 15) & 0xffff; + u32 count, i; + + rec += 4; + if (h & BIT(31)) + count = 1 + hweight32(mask); + else + count = ((h >> 15) & 0x3f) + 1; + if (count > (end - rec) / 4 || !ane_h16_register_valid(base)) + return -ENOEXEC; + if (h & BIT(31)) { + for (i = 0; i < 16; i++) + if ((mask & BIT(i)) && + !ane_h16_register_valid(base + i + 1)) + return -ENOEXEC; + } else { + for (i = 1; i < count; i++) + if (!ane_h16_register_valid(base + i)) + return -ENOEXEC; + } + rec += count * 4; + } + padded = ALIGN(end, 16); + if (padded > size || memchr_inv(p + end, 0, padded - end)) + return -ENOEXEC; + pos = padded; + if (++tasks > expected_tasks) + return -ENOEXEC; + } + return tasks == expected_tasks ? 0 : -ENOEXEC; +} + +/* Section ordinal is supplied only by the validated complete section table. + * This checks Mach-O relocation entries, not the distinct aperture records. + */ +static inline int +ane_h16_relocations_validate(const void *text, size_t text_size, + const void *relocs, size_t count, u32 ordinal, + u64 kernel_vm, u64 kernel_size, + unsigned long *seen, size_t seen_bits) +{ + const u8 *r = relocs; + u64 kernel_end; + size_t i; + + if (!count) + return 0; + if (!text || !relocs || !seen || !ordinal || ordinal > 0xffffff || + !kernel_size || text_size > ANE_H16_TASK_MAX || (text_size & 3) || + count > text_size / 4 || seen_bits < text_size / 4 || + check_add_overflow(kernel_vm, kernel_size, &kernel_end)) + return -ENOEXEC; + bitmap_zero(seen, text_size / 4); + for (i = 0; i < count; i++) { + u32 patch = get_unaligned_le32(r + i * 8); + u32 info = get_unaligned_le32(r + i * 8 + 4); + u32 addend; + u64 target; + + if ((patch & 3) || text_size < 4 || patch > text_size - 4 || + (info >> 24) != 7 || (info & 0xffffff) != ordinal || + test_and_set_bit(patch / 4, seen)) + return -ENOEXEC; + addend = get_unaligned_le32((const u8 *)text + patch); + if (addend >= kernel_size || + check_add_overflow(kernel_vm, (u64)addend, &target) || + target >= kernel_end) + return -ENOEXEC; + } + return 0; +} + +#endif /* __ANE_H16_TASKS_H__ */ diff --git a/drivers/accel/ane/ane_h16_tasks_test.c b/drivers/accel/ane/ane_h16_tasks_test.c new file mode 100644 index 0000000000000..f11aab0119c10 --- /dev/null +++ b/drivers/accel/ane/ane_h16_tasks_test.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "ane_h16_tasks.h" + +static void ane_h16_dense_task_test(struct kunit *test) +{ + u8 text[64] = {}; + + put_unaligned_le32(1, text); + /* Nine header words, one dense record and its one-word payload. */ + put_unaligned_le32(11 << 16, text + 16); + put_unaligned_le32(0x4100 / 4, text + 16 + 9 * 4); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), 0); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 2), -ENOEXEC); + put_unaligned_le32(0x41a4 / 4, text + 16 + 9 * 4); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + put_unaligned_le32(0x419c / 4 | (1 << 15), text + 16 + 9 * 4); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + put_unaligned_le32(0x4100 / 4, text + 16 + 9 * 4); + text[63] = 1; + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); +} + +static void ane_h16_scatter_task_test(struct kunit *test) +{ + u8 text[64] = {}; + + put_unaligned_le32(1, text); + put_unaligned_le32(12 << 16, text + 16); + put_unaligned_le32(BIT(31) | BIT(15) | 21, text + 16 + 9 * 4); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), 0); + /* The second scatter write would leave the common register block. */ + put_unaligned_le32(BIT(31) | BIT(15) | 22, text + 16 + 9 * 4); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); +} + +static void ane_h16_extra_header_test(struct kunit *test) +{ + u8 text[64] = {}; + + put_unaligned_le32(1, text); + put_unaligned_le32(10 << 16, text + 16); + put_unaligned_le32(3, text + 16 + 8 * 4); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), 0); + put_unaligned_le32(9 << 16, text + 16); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + KUNIT_EXPECT_FALSE(test, ane_h16_register_valid(0x44d4 / 4)); + KUNIT_EXPECT_TRUE(test, ane_h16_register_valid(0x44d0 / 4)); + KUNIT_EXPECT_FALSE(test, ane_h16_register_valid(0x5930 / 4)); +} + +static void ane_h16_prologue_test(struct kunit *test) +{ + u8 text[64] = {}; + unsigned int i; + + put_unaligned_le32(11 << 16, text + 16); + put_unaligned_le32(0x4100 / 4, text + 16 + 9 * 4); + /* A zero prologue is not the H16 stream marker. */ + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + put_unaligned_le32(1, text); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), 0); + put_unaligned_le32(2, text); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + put_unaligned_le32(0x01000000, text); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + put_unaligned_le32(1, text); + for (i = 4; i < 16; i++) { + text[i] = 1; + KUNIT_EXPECT_EQ(test, + ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + text[i] = 0; + } + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, 15, 1), -ENOEXEC); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, 16, 1), -ENOEXEC); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(NULL, sizeof(text), 1), -ENOEXEC); +} + +static void ane_h16_relocation_test(struct kunit *test) +{ + u8 text[64] = {}; + u8 relocs[16] = {}; + DECLARE_BITMAP(seen, 16); + int ret; + + put_unaligned_le32(4, relocs); + put_unaligned_le32(0x07000003, relocs + 4); + put_unaligned_le32(0x20, text + 4); + ret = ane_h16_relocations_validate(text, sizeof(text), + relocs, 1, 3, 0x30020000, 0x100, seen, 16); + KUNIT_EXPECT_EQ(test, ret, 0); + memcpy(relocs + 8, relocs, 8); + ret = ane_h16_relocations_validate(text, sizeof(text), + relocs, 2, 3, 0x30020000, 0x100, seen, 16); + KUNIT_EXPECT_EQ(test, ret, -ENOEXEC); + ret = ane_h16_relocations_validate(text, sizeof(text), + relocs, 1, 4, 0x30020000, 0x100, seen, 16); + KUNIT_EXPECT_EQ(test, ret, -ENOEXEC); + put_unaligned_le32(0x0f000003, relocs + 4); + ret = ane_h16_relocations_validate(text, sizeof(text), + relocs, 1, 3, 0x30020000, 0x100, seen, 16); + KUNIT_EXPECT_EQ(test, ret, -ENOEXEC); + put_unaligned_le32(0x07000003, relocs + 4); + put_unaligned_le32(0x100, text + 4); + ret = ane_h16_relocations_validate(text, sizeof(text), + relocs, 1, 3, 0x30020000, 0x100, seen, 16); + KUNIT_EXPECT_EQ(test, ret, -ENOEXEC); + ret = ane_h16_relocations_validate(text, sizeof(text), + relocs, 1, 3, U64_MAX - 8, 0x100, seen, 16); + KUNIT_EXPECT_EQ(test, ret, -ENOEXEC); + ret = ane_h16_relocations_validate(text, sizeof(text), + relocs, 1, 3, 0x30020000, 0, seen, 16); + KUNIT_EXPECT_EQ(test, ret, -ENOEXEC); +} + +static struct kunit_case ane_h16_task_cases[] = { + KUNIT_CASE(ane_h16_prologue_test), + KUNIT_CASE(ane_h16_dense_task_test), + KUNIT_CASE(ane_h16_scatter_task_test), + KUNIT_CASE(ane_h16_extra_header_test), + KUNIT_CASE(ane_h16_relocation_test), + {} +}; + +static struct kunit_suite ane_h16_task_suite = { + .name = "ane-h16-tasks", + .test_cases = ane_h16_task_cases, +}; + +kunit_test_suite(ane_h16_task_suite); +MODULE_DESCRIPTION("KUnit tests for H16 task and relocation validation"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_perf.c b/drivers/accel/ane/ane_perf.c new file mode 100644 index 0000000000000..468fd151c4886 --- /dev/null +++ b/drivers/accel/ane/ane_perf.c @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Workload-scoped ANE clock and memory bandwidth requests. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane.h" + +/* Amortize clock transitions across a stream of short submissions. */ +#define ANE_PERF_IDLE_MS 32 + +static int ane_perf_drop(struct ane_device *ane) +{ + struct ane_perf *p = &ane->perf; + int ret; + + if (p->clock_held) { + ret = apple_pmp_report_ane_perf(p->report, ane->dev, 0, &p->clock_held); + p->level = 0; + if (ret) + return ret; + } + if (p->voted) { + ret = icc_set_bw(p->memory, 0, 0); + if (ret) + return ret; + p->voted = false; + } + return 0; +} + +static void ane_perf_idle(struct work_struct *work) +{ + struct ane_device *ane = container_of(to_delayed_work(work), + struct ane_device, perf.idle); + struct ane_perf *p = &ane->perf; + + mutex_lock(&p->lock); + if (!p->busy && !atomic_read(&ane->wedged)) { + p->error = ane_perf_drop(ane); + if (!p->error && p->pinned) { + p->pinned = false; + pm_runtime_put_autosuspend(ane->dev); + } + if (p->error && ++p->release_retries < 3) + mod_delayed_work(system_dfl_wq, &p->idle, + msecs_to_jiffies(ANE_PERF_IDLE_MS)); + } + mutex_unlock(&p->lock); +} + +int ane_perf_begin(struct ane_device *ane) +{ + struct ane_perf *p = &ane->perf; + int ret = 0; + + if (p->cpu_boost) + ane_boost_begin(ane); + + /* The submit ioctl already holds a runtime PM reference. */ + mutex_lock(&p->lock); + p->busy = true; + if (p->error) { + ret = ane_perf_drop(ane); + if (ret) + goto out; + } + if (p->memory && !p->voted) { + ret = icc_set_bw(p->memory, p->avg_bw, p->peak_bw); + if (ret) + goto out; + p->voted = true; + } + if (p->report && !p->clock_held) { + ret = apple_pmp_report_ane_perf(p->report, ane->dev, p->high_level, + &p->clock_held); + if (ret) + goto out; + p->level = p->high_level; + } +out: + p->error = ret; + /* Keep the windows powered until delayed release, including errors. */ + if (!p->pinned && (p->clock_held || p->voted)) { + pm_runtime_get_noresume(ane->dev); + p->pinned = true; + } + mutex_unlock(&p->lock); + return ret; +} + +void ane_perf_end(struct ane_device *ane) +{ + struct ane_perf *p = &ane->perf; + + if (p->cpu_boost) + ane_boost_end(ane); + + mutex_lock(&p->lock); + p->busy = false; + p->release_retries = 0; + mod_delayed_work(system_dfl_wq, &p->idle, msecs_to_jiffies(ANE_PERF_IDLE_MS)); + mutex_unlock(&p->lock); +} + +static ssize_t performance_level_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ane_device *ane = dev_get_drvdata(dev); + struct ane_perf *p = &ane->perf; + ssize_t ret; + + mutex_lock(&p->lock); + ret = p->report ? sysfs_emit(buf, "%u\n", p->level) : + sysfs_emit(buf, "unsupported\n"); + mutex_unlock(&p->lock); + return ret; +} +static DEVICE_ATTR_RO(performance_level); + +static ssize_t memory_vote_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ane_device *ane = dev_get_drvdata(dev); + struct ane_perf *p = &ane->perf; + ssize_t ret; + + mutex_lock(&p->lock); + ret = p->memory ? sysfs_emit(buf, "%u %u\n", + p->voted ? p->avg_bw : 0, + p->voted ? p->peak_bw : 0) : + sysfs_emit(buf, "unsupported\n"); + mutex_unlock(&p->lock); + return ret; +} +static DEVICE_ATTR_RO(memory_vote); + +static ssize_t performance_error_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ane_device *ane = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", READ_ONCE(ane->perf.error)); +} +static DEVICE_ATTR_RO(performance_error); + +static struct attribute *ane_perf_attrs[] = { + &dev_attr_performance_level.attr, + &dev_attr_memory_vote.attr, + &dev_attr_performance_error.attr, + NULL, +}; + +static const struct attribute_group ane_perf_group = { .attrs = ane_perf_attrs }; + +int ane_perf_init(struct ane_device *ane, unsigned int high_level) +{ + struct ane_perf *p = &ane->perf; + struct platform_device *supplier; + struct device_link *link; + struct dev_pm_opp *opp; + int ret; + + mutex_init(&p->lock); + p->initialized = true; + INIT_DELAYED_WORK(&p->idle, ane_perf_idle); + p->high_level = high_level; + p->report = of_parse_phandle(ane->dev->of_node, "apple,performance-controller", 0); + if (p->report) { + if (!high_level) + return -EINVAL; + supplier = of_find_device_by_node(p->report); + if (!supplier) + return -EPROBE_DEFER; + link = device_link_add(ane->dev, &supplier->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + put_device(&supplier->dev); + if (!link) + return -ENOMEM; + ret = apple_pmp_report_ane_available(p->report); + if (ret) + return ret; + } + + if (of_property_present(ane->dev->of_node, "interconnects")) { + p->memory = devm_of_icc_get(ane->dev, "memory"); + if (IS_ERR(p->memory)) + return PTR_ERR(p->memory); + if (!p->memory) + return -ENODEV; + ret = devm_pm_opp_of_add_table(ane->dev); + if (ret) + return ret; + opp = dev_pm_opp_find_level_exact(ane->dev, high_level); + if (IS_ERR(opp)) + return PTR_ERR(opp); + p->avg_bw = dev_pm_opp_get_bw(opp, false, 0); + p->peak_bw = dev_pm_opp_get_bw(opp, true, 0); + dev_pm_opp_put(opp); + if (!p->peak_bw || p->avg_bw > p->peak_bw) + return -EINVAL; + } + + ret = devm_device_add_group(ane->dev, &ane_perf_group); + if (ret) + return ret; + + /* + * Without a performance controller the engine runs at its boot + * clock, and the CPU cluster boost is what keeps bandwidth-bound + * programs and the submit tail fast. Keep it until the DT binds a + * controller. + */ + if (!p->report) { + ret = ane_boost_init(ane); + if (ret) + return ret; + p->cpu_boost = true; + dev_info(ane->dev, "no performance controller; keeping the CPU frequency boost\n"); + } + return 0; +} + +int ane_perf_exit(struct ane_device *ane) +{ + struct ane_perf *p = &ane->perf; + int ret; + + if (!p->initialized) + return 0; + if (p->cpu_boost) { + ane_boost_exit(ane); + p->cpu_boost = false; + } + cancel_delayed_work_sync(&p->idle); + mutex_lock(&p->lock); + p->error = ane_perf_drop(ane); + if (p->error) + dev_err(ane->dev, "performance release failed during removal: %d\n", + p->error); + if (p->pinned && !p->error) { + pm_runtime_put_noidle(ane->dev); + p->pinned = false; + } + ret = p->error; + mutex_unlock(&p->lock); + if (ret) + return ret; + of_node_put(p->report); + p->report = NULL; + return 0; +} + +int ane_perf_suspend(struct ane_device *ane) +{ + struct ane_perf *p = &ane->perf; + int ret; + + cancel_delayed_work_sync(&p->idle); + mutex_lock(&p->lock); + if (p->busy && !ane->recovering) { + ret = -EBUSY; + goto out; + } + ret = ane_perf_drop(ane); + p->error = ret; + if (!ret && p->pinned) { + pm_runtime_put_noidle(ane->dev); + p->pinned = false; + } +out: + mutex_unlock(&p->lock); + return ret; +} diff --git a/drivers/accel/ane/ane_pm.h b/drivers/accel/ane/ane_pm.h new file mode 100644 index 0000000000000..ccc08c88057a4 --- /dev/null +++ b/drivers/accel/ane/ane_pm.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_PM_H__ +#define __ANE_PM_H__ + +#include +#include + +/* + * The caller serializes PM entries and holds exactly one consumer reference. + * Its suspend callback must permit recovery and its resume callback must defer + * engine access until the caller verifies power-on. verify_off may only read + * always-accessible provider registers, never the powered-down engine. + * + * Kept independent of the H13 task manager: a firmware client can use the same + * PM transaction after stopping its RX worker and fencing submissions. + */ +static inline int ane_pm_cycle(struct device *dev, struct device **domains, + int count, int (*verify_off)(void *), void *ctx) +{ + int err, resume_err; + + if (count > 1) { + for (int i = 0; i < count; i++) + dev_pm_genpd_synced_poweroff(domains[i]); + } else { + dev_pm_genpd_synced_poweroff(dev); + } + + /* Normal runtime PM releases the consumer's supplier references. */ + err = pm_runtime_put_sync_suspend(dev); + if (err < 0) + goto resume; + + /* Finish asynchronous supplier idle work, then suspend leaves first. + * Runtime PM stays enabled so genpd can also suspend their parents. + */ + for (int i = count - 1; i >= 0 && count > 1; i--) { + pm_runtime_barrier(domains[i]); + err = pm_runtime_suspend(domains[i]); + if (err < 0) + goto resume; + } + err = verify_off(ctx); + +resume: + /* Restore the caller's reference even if resume itself fails. */ + resume_err = pm_runtime_get_sync(dev); + return err < 0 ? err : (resume_err < 0 ? resume_err : 0); +} + +#endif /* __ANE_PM_H__ */ diff --git a/drivers/accel/ane/ane_pm_test.c b/drivers/accel/ane/ane_pm_test.c new file mode 100644 index 0000000000000..5c372a2a73011 --- /dev/null +++ b/drivers/accel/ane/ane_pm_test.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Exercise recovery against the real runtime-PM and device-link core. */ +#include +#include +#include + +#include "ane_pm.h" + +struct ane_pm_test_dev { + struct device dev; + int suspend_error; + int resume_error; + unsigned int suspends; + unsigned int resumes; +}; + +struct ane_pm_test_ctx { + struct kunit *test; + struct device *consumer; + struct device *domains[2]; + struct device_link *links[2]; + unsigned int verified; + int verify_error; +}; + +static int ane_pm_test_suspend(struct device *dev) +{ + struct ane_pm_test_dev *d = container_of(dev, struct ane_pm_test_dev, dev); + + d->suspends++; + return d->suspend_error; +} + +static int ane_pm_test_resume(struct device *dev) +{ + struct ane_pm_test_dev *d = container_of(dev, struct ane_pm_test_dev, dev); + + d->resumes++; + return d->resume_error; +} + +static const struct dev_pm_ops ane_pm_test_ops = { + .runtime_suspend = ane_pm_test_suspend, + .runtime_resume = ane_pm_test_resume, +}; + +static const struct device_type ane_pm_test_type = { + .pm = &ane_pm_test_ops, +}; + +static void ane_pm_test_release(struct device *dev) +{ + kfree(container_of(dev, struct ane_pm_test_dev, dev)); +} + +static void ane_pm_test_unregister(void *data) +{ + struct device *dev = data; + + pm_runtime_barrier(dev); + pm_runtime_disable(dev); + /* Also releases supplier votes after an injected resume failure. */ + pm_runtime_set_suspended(dev); + device_unregister(dev); +} + +static struct device *ane_pm_test_register(struct kunit *test, const char *name) +{ + struct ane_pm_test_dev *d; + int ret; + + d = kzalloc_obj(*d); + if (!d) + return NULL; + device_initialize(&d->dev); + d->dev.release = ane_pm_test_release; + d->dev.type = &ane_pm_test_type; + ret = dev_set_name(&d->dev, "%s", name); + if (!ret) + ret = device_add(&d->dev); + if (ret) { + put_device(&d->dev); + return NULL; + } + pm_runtime_enable(&d->dev); + ret = kunit_add_action_or_reset(test, ane_pm_test_unregister, &d->dev); + return ret ? NULL : &d->dev; +} + +static void ane_pm_test_unlink(void *data) +{ + device_link_del(data); +} + +static int ane_pm_test_verify(void *data) +{ + struct ane_pm_test_ctx *ctx = data; + + ctx->verified++; + KUNIT_EXPECT_TRUE(ctx->test, pm_runtime_suspended(ctx->consumer)); + KUNIT_EXPECT_EQ(ctx->test, atomic_read(&ctx->consumer->power.usage_count), 0); + for (int i = 0; i < ARRAY_SIZE(ctx->domains); i++) { + if (!ctx->domains[i]) + continue; + KUNIT_EXPECT_TRUE(ctx->test, pm_runtime_suspended(ctx->domains[i])); + KUNIT_EXPECT_EQ(ctx->test, atomic_read(&ctx->domains[i]->power.usage_count), 0); + } + return ctx->verify_error; +} + +static int ane_pm_test_setup(struct kunit *test, struct ane_pm_test_ctx *ctx, + bool multiple) +{ + static const char * const names[] = { "ane-pm-domain0", "ane-pm-domain1" }; + + ctx->test = test; + ctx->consumer = ane_pm_test_register(test, "ane-pm-consumer"); + if (!ctx->consumer) + return -ENOMEM; + if (multiple) { + for (int i = 0; i < ARRAY_SIZE(ctx->domains); i++) { + ctx->domains[i] = ane_pm_test_register(test, names[i]); + if (!ctx->domains[i]) + return -ENOMEM; + ctx->links[i] = device_link_add(ctx->consumer, ctx->domains[i], + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!ctx->links[i]) + return -ENOMEM; + if (kunit_add_action_or_reset(test, ane_pm_test_unlink, + ctx->links[i])) + return -ENOMEM; + } + } + return pm_runtime_resume_and_get(ctx->consumer); +} + +static void ane_pm_test_finish(struct kunit *test, struct ane_pm_test_ctx *ctx) +{ + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->consumer->power.usage_count), 1); + KUNIT_EXPECT_GE(test, pm_runtime_put_sync_suspend(ctx->consumer), 0); +} + +static void ane_pm_cycle_domains_test(struct kunit *test) +{ + struct ane_pm_test_ctx ctx = {}; + + KUNIT_ASSERT_GE(test, ane_pm_test_setup(test, &ctx, true), 0); + for (int repeat = 0; repeat < 3; repeat++) { + KUNIT_EXPECT_EQ(test, ane_pm_cycle(ctx.consumer, ctx.domains, 2, + ane_pm_test_verify, &ctx), 0); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(ctx.consumer)); + for (int i = 0; i < ARRAY_SIZE(ctx.domains); i++) { + KUNIT_EXPECT_TRUE(test, pm_runtime_active(ctx.domains[i])); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx.domains[i]->power.usage_count), 1); + } + } + KUNIT_EXPECT_EQ(test, ctx.verified, 3U); + ane_pm_test_finish(test, &ctx); +} + +static void ane_pm_cycle_single_test(struct kunit *test) +{ + struct ane_pm_test_ctx ctx = {}; + + KUNIT_ASSERT_GE(test, ane_pm_test_setup(test, &ctx, false), 0); + KUNIT_EXPECT_EQ(test, ane_pm_cycle(ctx.consumer, NULL, 1, + ane_pm_test_verify, &ctx), 0); + KUNIT_EXPECT_EQ(test, ctx.verified, 1U); + ane_pm_test_finish(test, &ctx); +} + +static void ane_pm_cycle_suspend_error_test(struct kunit *test) +{ + struct ane_pm_test_ctx ctx = {}; + struct ane_pm_test_dev *d; + + KUNIT_ASSERT_GE(test, ane_pm_test_setup(test, &ctx, true), 0); + d = container_of(ctx.consumer, struct ane_pm_test_dev, dev); + d->suspend_error = -EBUSY; + KUNIT_EXPECT_EQ(test, ane_pm_cycle(ctx.consumer, ctx.domains, 2, + ane_pm_test_verify, &ctx), -EBUSY); + KUNIT_EXPECT_EQ(test, ctx.verified, 0U); + d->suspend_error = 0; + ane_pm_test_finish(test, &ctx); +} + +static void ane_pm_cycle_supplier_error_test(struct kunit *test) +{ + struct ane_pm_test_ctx ctx = {}; + struct ane_pm_test_dev *d; + + KUNIT_ASSERT_GE(test, ane_pm_test_setup(test, &ctx, true), 0); + d = container_of(ctx.domains[1], struct ane_pm_test_dev, dev); + d->suspend_error = -EBUSY; + KUNIT_EXPECT_EQ(test, ane_pm_cycle(ctx.consumer, ctx.domains, 2, + ane_pm_test_verify, &ctx), -EBUSY); + KUNIT_EXPECT_EQ(test, ctx.verified, 0U); + d->suspend_error = 0; + ane_pm_test_finish(test, &ctx); +} + +static void ane_pm_cycle_verify_error_test(struct kunit *test) +{ + struct ane_pm_test_ctx ctx = { .verify_error = -ETIMEDOUT }; + + KUNIT_ASSERT_GE(test, ane_pm_test_setup(test, &ctx, true), 0); + KUNIT_EXPECT_EQ(test, ane_pm_cycle(ctx.consumer, ctx.domains, 2, + ane_pm_test_verify, &ctx), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, ctx.verified, 1U); + KUNIT_EXPECT_TRUE(test, pm_runtime_active(ctx.consumer)); + ane_pm_test_finish(test, &ctx); +} + +static void ane_pm_cycle_resume_error_test(struct kunit *test) +{ + struct ane_pm_test_ctx ctx = {}; + struct ane_pm_test_dev *d; + + KUNIT_ASSERT_GE(test, ane_pm_test_setup(test, &ctx, false), 0); + d = container_of(ctx.consumer, struct ane_pm_test_dev, dev); + d->resume_error = -EIO; + KUNIT_EXPECT_EQ(test, ane_pm_cycle(ctx.consumer, NULL, 1, + ane_pm_test_verify, &ctx), -EIO); + KUNIT_EXPECT_EQ(test, ctx.verified, 1U); + /* get_sync must leave exactly the reference the caller will put. */ + KUNIT_EXPECT_EQ(test, atomic_read(&ctx.consumer->power.usage_count), 1); + pm_runtime_put_noidle(ctx.consumer); +} + +static struct kunit_case ane_pm_test_cases[] = { + KUNIT_CASE(ane_pm_cycle_domains_test), + KUNIT_CASE(ane_pm_cycle_single_test), + KUNIT_CASE(ane_pm_cycle_suspend_error_test), + KUNIT_CASE(ane_pm_cycle_supplier_error_test), + KUNIT_CASE(ane_pm_cycle_verify_error_test), + KUNIT_CASE(ane_pm_cycle_resume_error_test), + {} +}; + +static struct kunit_suite ane_pm_test_suite = { + .name = "ane-pm-recovery", + .test_cases = ane_pm_test_cases, +}; + +kunit_test_suite(ane_pm_test_suite); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("ANE recovery runtime-PM transaction tests"); diff --git a/drivers/accel/ane/ane_t6021.h b/drivers/accel/ane/ane_t6021.h index cf04c6699710e..bf6d0f3e44202 100644 --- a/drivers/accel/ane/ane_t6021.h +++ b/drivers/accel/ane/ane_t6021.h @@ -742,6 +742,7 @@ bool ane_t6021_fwload_options_ok(void); bool ane_t6021_fwload_placement_ok(struct device *dev); bool ane_t6021_fwload_requested(void); +struct ane_fw_platform; struct ane_fw_image; struct ane_asc_tunables; @@ -753,8 +754,9 @@ struct ane_asc_tunables; * for this SoC (T6021 only), so fw_alias_reserved=1 may map it; the * other SoCs always run the firmware from driver-owned memory. * ps_cpu_off: the ANE CPU ps word in the DT "pmgr" window (probe guard). - * pwgate_off: the PWGATE word in the DT "set" window, checked - * open before any engine read; 0 = not checked. + * pwgate_off: the power gate word in the DT "set" window. The driver + * opens it before ane_sys_cpu powers on, closes it after that state + * powers off, and checks it open before any engine read; 0 = no gate. * pmu_pa: the page of the seven ANE ps words; the firmware's power * service writes them through its DART at IOVA == PA. ps_off: the * first of the seven in that page. @@ -762,6 +764,7 @@ struct ane_asc_tunables; * trace_td; 0 = trace_td unsupported. */ struct ane_t602x_soc { + const struct ane_fw_platform *platform; u32 soc; u32 soc_revision; bool revision_fuse; diff --git a/drivers/accel/ane/ane_t6021_fwload.c b/drivers/accel/ane/ane_t6021_fwload.c index 4dc4ce27b1674..f39e4b7e8b05c 100644 --- a/drivers/accel/ane/ane_t6021_fwload.c +++ b/drivers/accel/ane/ane_t6021_fwload.c @@ -533,11 +533,20 @@ int ane_t6021_fwload_check(struct device *dev) if (!fw_load) return 0; - ret = request_firmware(&fw, img->name, dev); + /* + * The image is installed on the machine itself from Apple's files, so + * a missing file is the normal state of a system that has not fetched + * it yet: say so once and leave the ANE off. Other errors are errors. + */ + ret = firmware_request_nowarn(&fw, img->name, dev); + if (ret == -ENOENT) { + dev_info(dev, + "firmware %s is not installed, so the ANE stays off; install it with omarchy-ane-firmware-fetch\n", + img->name); + return -ENODEV; + } if (ret) { - dev_err(dev, - "cannot load firmware %s: %d; install it with omarchy-ane-firmware-fetch (or equivalent for your distribution)\n", - img->name, ret); + dev_err(dev, "cannot load firmware %s: %d\n", img->name, ret); return ret; } diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index 40e643f58a8ca..ed08c6506b324 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -65,6 +65,8 @@ #include #include #include +#include +#include #include #include #include @@ -90,6 +92,11 @@ #include #include +#include "ane_fw_platform.h" +#include "ane_h16_protocol.h" +#include "ane_h16_pending.h" +#include "ane_h16_resources.h" +#include "ane_fw_protocol.h" #include "ane_stats.h" #include "ane_t6021.h" #include "ane_t6021_boot.h" @@ -191,6 +198,22 @@ struct ane_legacy_buffer { size_t size; }; +struct ane_t6021_fd; + +/* Populated only by an original-container loader and decoded process reply. + * Those interfaces remain refused until their complete wire contract exists. + * The firmware mutex protects this list and every pending UUID operation. + */ +struct ane_h16_process { + struct list_head node; + struct list_head owner_node; + struct ane_h16_hwx_snapshot original; + struct ane_h16_request_key key; + struct ane_h16_resource resources[ANE_H16_SURFACES]; + struct ane_legacy_buffer mappings[ANE_H16_SURFACES]; + u32 resource_count; +}; + struct ane_rtclient { struct device *dev; void __iomem *engine; @@ -213,6 +236,12 @@ struct ane_rtclient { /* ChMan descriptor table validated against the 'IPC ' surface. */ bool chman_ok; + struct ane_fw_channel channels[ANE_FW_CHANNELS]; + u8 channel_snapshot[ANE_FW_TABLE_SIZE]; + u64 channel_table_offset; + struct ane_h16_pending_table pending; + struct ane_h16_pending *waiting; + struct list_head h16_processes; struct ane_legacy_buffer *legacy_buffers; u32 legacy_allocated; size_t legacy_bytes; @@ -245,6 +274,7 @@ struct ane_rtclient { */ struct ane_t6021_fd { struct list_head bos; + struct list_head h16_processes; }; /* Per-file handle counter and every per-fd list against same-fd concurrent @@ -400,7 +430,10 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, struct ane_legacy_buffer *command, size_t length, u16 opcode, unsigned int channel, - unsigned int timeout_ms); + unsigned long deadline); + +static bool ane_rtclient_drain_t2h(struct ane_rtclient *ane, + unsigned int channel); /* ---- boot helpers (shared with boot.c / fwload.c) ---- */ @@ -442,6 +475,31 @@ static void ane_rtclient_validate_chman(struct ane_rtclient *ane) "chman: no host IPC surface — table not validated\n"); return; } + if (ane->soc->platform) { + int ret; + + if (!ane_fw_contains(a->boot_ipc_iova, a->boot_ipc_size, + a->boot_scratch_result, ANE_FW_TABLE_SIZE)) { + dev_err(ane->dev, "27.x channel table outside IPC\n"); + return; + } + ane->channel_table_offset = a->boot_scratch_result - a->boot_ipc_iova; + dma_rmb(); + memcpy(ane->channel_snapshot, + a->boot_ipc + ane->channel_table_offset, ANE_FW_TABLE_SIZE); + ret = ane_fw_channels_parse(ane->channel_snapshot, a->boot_ipc_iova, + a->boot_ipc_size, a->boot_scratch_result, + ane->channels); + dma_rmb(); + if (ret || memcmp(ane->channel_snapshot, + a->boot_ipc + ane->channel_table_offset, + ANE_FW_TABLE_SIZE)) { + dev_err(ane->dev, "27.x channel table rejected or changed\n"); + return; + } + ane->chman_ok = true; + return; + } if (a->boot_ipc_size < ANE_T6021_CHMAN_TOTAL) { dev_warn(ane->dev, "chman: IPC surface %#llx bytes < fw layout %#x — table not validated\n", @@ -465,10 +523,39 @@ static void ane_rtclient_validate_chman(struct ane_rtclient *ane) } ane->chman_ok = !bad; + if (!bad) + for (i = 0; i < ANE_FW_CHANNELS; i++) { + const struct ane_t6021_chman_static *c = &ane_t6021_chman_layout[i]; + + ane->channels[i] = (struct ane_fw_channel) { + .offset = c->off, .slots = c->size, + .type = c->type, .bit = c->bit, + }; + } dev_info(ane->dev, "chman: table %s (mismatch mask %#x)\n", bad ? "NOT VALIDATED" : "VALIDATED", bad); } +static bool ane_rtclient_channels_unchanged(struct ane_rtclient *ane) +{ + struct ane_t6021 *a = ane->fw; + + if (!a || !ane->chman_ok) + return false; + dma_rmb(); + if (ane->soc->platform) + return !memcmp(ane->channel_snapshot, + a->boot_ipc + ane->channel_table_offset, + ANE_FW_TABLE_SIZE); + return !ane_t6021_chman_check(a->boot_ipc, a->boot_ipc_iova); +} + +static void __iomem *ane_rtclient_doorbell(struct ane_rtclient *ane) +{ + return ane->engine + (ane->soc->platform ? + ane->soc->platform->doorbell : ANE_IPI_OFF); +} + /* Legacy ChMan exchange (post-DONE 13.5 transport). Resends to the same * 64-byte IO slot would wedge the ring; cursor advances after each * completed command (the fw treats bit0 as host-owned). @@ -477,46 +564,66 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, struct ane_legacy_buffer *command, size_t length, u16 opcode, unsigned int channel, - unsigned int timeout_ms) + unsigned long deadline) { struct ane_t6021 *a = ane->fw; u64 *io, *malloc_ring, header; - u64 *t2h_buf, *t2h_ioq; - void __iomem *ipi = ane->engine + ANE_IPI_OFF; + void __iomem *ipi = ane_rtclient_doorbell(ane); unsigned int cursor = ane->legacy_malloc_cursor; - unsigned long deadline; - u32 *reply; int result = -ETIMEDOUT; - if (!ane->held || !ane->chman_ok || - ane_t6021_chman_check(a->boot_ipc, a->boot_ipc_iova)) + if (!ane->held || !ane_rtclient_channels_unchanged(ane)) return -EPROTO; if (length < 8 || length > command->size || channel >= ANE_T6021_CHMAN_COUNT) { result = -EINVAL; goto out; } - io = a->boot_ipc + ane_t6021_chman_layout[channel].off + + io = a->boot_ipc + ane->channels[channel].offset + (size_t)ane->legacy_cmd_cursor[channel] * 64; - malloc_ring = a->boot_ipc + ane_t6021_chman_layout[5].off; - t2h_buf = a->boot_ipc + ane_t6021_chman_layout[4].off; - t2h_ioq = a->boot_ipc + ane_t6021_chman_layout[6].off; + malloc_ring = a->boot_ipc + ane->channels[5].offset; if (!(READ_ONCE(io[0]) & 1)) { result = -EBUSY; goto out; } - reply = command->cpu; - ((u16 *)reply)[2] = opcode; + if (time_after_eq(jiffies, deadline)) + goto out; + if (ane->soc->platform) { + result = ane_h16_command_prepare(command->cpu, length, opcode, + ane->channels[channel].index); + if (result) + goto out; + } else { + ane_fw_command_header(command->cpu, opcode, false); + } + if (ane->soc->platform) + memset((u8 *)command->cpu + length, 0xa5, command->size - length); WRITE_ONCE(io[1], length); WRITE_ONCE(io[2], length); dma_wmb(); WRITE_ONCE(io[0], command->dma); dma_wmb(); - writel(BIT(ane_t6021_chman_layout[channel].bit), ipi); - deadline = jiffies + msecs_to_jiffies(timeout_ms); + writel(BIT(ane->channels[channel].bit), ipi); while (time_before(jiffies, deadline)) { - u32 pending = readl(ipi + 0x8000); + u32 pending; + + if (atomic_read(&ane_t6021_quarantined)) { + result = -EIO; + goto out; + } + if (!ane_rtclient_channels_unchanged(ane)) { + result = -EPROTO; + goto out; + } + if (ane->soc->platform && opcode == ANE_H16_EXECUTE) { + ane_rtclient_drain_t2h(ane, 6); + if (ane->pending.quarantined) { + result = ane->pending.error ?: -EIO; + goto out; + } + } + pending = readl(ipi + 0x8000); if (pending) { writel(pending, ipi + 0xc000); @@ -538,6 +645,13 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, u64 size, tag; struct ane_legacy_buffer *buf; + /* H16 needs its own validated allocation/reply provider. + * Never answer it with the legacy H14 allocation index. + */ + if (ane->soc->platform) { + result = -EOPNOTSUPP; + goto out; + } dma_rmb(); size = READ_ONCE(slot[1]); tag = READ_ONCE(slot[2]); @@ -546,7 +660,7 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, * is a malformed ring and we stop. */ if (alloc_hdr || tag > U32_MAX) { - result = -EOPNOTSUPP; + result = -EPROTO; goto out; } if (ane_rtclient_legacy_alloc(ane, size)) { @@ -559,22 +673,36 @@ static int ane_rtclient_legacy_exchange(struct ane_rtclient *ane, dma_wmb(); WRITE_ONCE(slot[0], buf->dma | 1); dma_wmb(); - writel(BIT(5), ipi); + writel(BIT(ane->channels[5].bit), ipi); cursor = (cursor + 1) % - ane_t6021_chman_layout[5].size; + ane->channels[5].slots; } } + /* Allocation service cannot extend the command deadline. */ + if (time_after_eq(jiffies, deadline)) + goto out; header = READ_ONCE(io[0]); if (header & 1) { dma_rmb(); - result = (header == (command->dma | 1) && - READ_ONCE(io[1]) == length && - READ_ONCE(io[2]) == 0 && - ((u16 *)reply)[2] == opcode && - ((u16 *)reply)[3] == 0) ? 0 : -EPROTO; - ane->legacy_cmd_cursor[channel] = - (ane->legacy_cmd_cursor[channel] + 1) % - ane_t6021_chman_layout[channel].size; + result = ane_fw_reply_status(header, READ_ONCE(io[1]), + READ_ONCE(io[2]), command->dma, + length, opcode, command->cpu, + !!ane->soc->platform); + /* Corruption does not retire the unknown slot. A refusal + * retires ownership but remains a nonzero result. + */ + if (!ane_rtclient_channels_unchanged(ane) || + (ane->soc->platform && + memchr_inv((u8 *)command->cpu + length, 0xa5, + command->size - length))) + result = -EPROTO; + if (result == -EREMOTEIO) + dev_info(ane->dev, "firmware refused op=%#x ack=%#x; no retry\n", + opcode, get_unaligned_le16((u8 *)command->cpu + 6)); + if (!result || result == -EREMOTEIO) + ane->legacy_cmd_cursor[channel] = + (ane->legacy_cmd_cursor[channel] + 1) % + ane->channels[channel].slots; goto out; } /* Fast poll: check the rings every 50 us instead of @@ -766,18 +894,24 @@ static void ane_t6021_trace_free(void) static const void *ane_rtclient_fw_cpu(struct ane_rtclient *ane, u64 iova, size_t len) { + const void *found = NULL; u32 i; for (i = 0; i < ane->legacy_allocated; i++) { const struct ane_legacy_buffer *b = &ane->legacy_buffers[i]; - if (iova >= b->dma && iova - b->dma + len <= b->size) - return b->cpu + (iova - b->dma); + if (ane_fw_contains(b->dma, b->size, iova, len)) { + if (!ane->soc->platform) + return b->cpu + (iova - b->dma); + if (found) + return NULL; + found = b->cpu + (iova - b->dma); + } } - if (iova >= ane->fw->boot_ipc_iova && - iova - ane->fw->boot_ipc_iova + len <= ane->fw->boot_ipc_size) - return ane->fw->boot_ipc + (iova - ane->fw->boot_ipc_iova); - return NULL; + if (ane_fw_contains(ane->fw->boot_ipc_iova, ane->fw->boot_ipc_size, + iova, len)) + return found ? NULL : ane->fw->boot_ipc + (iova - ane->fw->boot_ipc_iova); + return found; } /* Return every firmware-owned slot on a target-to-host ring to the fw: @@ -790,23 +924,62 @@ static const void *ane_rtclient_fw_cpu(struct ane_rtclient *ane, u64 iova, static bool ane_rtclient_drain_t2h(struct ane_rtclient *ane, unsigned int channel) { - const struct ane_t6021_chman_static *c = &ane_t6021_chman_layout[channel]; + const struct ane_fw_channel *c = &ane->channels[channel]; unsigned int n = 0, slot_i = ane->legacy_cmd_cursor[channel]; - void __iomem *ipi = ane->engine + ANE_IPI_OFF; + void __iomem *ipi = ane_rtclient_doorbell(ane); bool finished = false; - if (!ane->fw || !ane->fw->boot_ipc) + if (!ane_rtclient_channels_unchanged(ane)) { + atomic_set(&ane_t6021_quarantined, 1); + if (ane->soc->platform) { + ane->pending.error = -EPROTO; + ane->pending.quarantined = true; + } + return false; + } + if (c->type != 1) return false; - while (n < c->size) { - u64 *slot = ane->fw->boot_ipc + c->off + (size_t)slot_i * 64; - u64 hdr = READ_ONCE(slot[0]), len = READ_ONCE(slot[1]); + while (n < c->slots) { + u64 *slot = ane->fw->boot_ipc + c->offset + (size_t)slot_i * 64; + u64 hdr = READ_ONCE(slot[0]), len; if (hdr & 1) break; dma_rmb(); - dev_dbg(ane->dev, "T2H ch=%s slot=%u hdr=%016llx len=%#llx\n", - c->name, slot_i, hdr, len); - if (channel == 6 && len >= ANE_T2H_CALL_STATE_OFF + 4) { + len = READ_ONCE(slot[1]); + dev_dbg(ane->dev, "T2H ch=%u slot=%u len=%#llx\n", + channel, slot_i, len); + if (ane->soc->platform) { + u8 snapshot[0x60]; + const void *ev; + int status; + + /* Other channels have different message types; never apply + * the event predicate to logs or allocation traffic. + */ + if (c->index != 6 || (len != sizeof(snapshot) && len != 0xc)) { + atomic_set(&ane_t6021_quarantined, 1); + ane->pending.error = -EPROTO; + ane->pending.quarantined = true; + return false; + } + ev = ane_rtclient_fw_cpu(ane, hdr, len); + if (!ev) { + atomic_set(&ane_t6021_quarantined, 1); + ane->pending.error = -EPROTO; + ane->pending.quarantined = true; + return false; + } + memcpy(snapshot, ev, len); + status = ane_h16_pending_dispatch(&ane->pending, snapshot, + len, c->index, jiffies); + if (ane->pending.quarantined) { + atomic_set(&ane_t6021_quarantined, 1); + return false; + } + if (status == ANE_H16_EVENT_COMPLETE) + finished = true; + } else if (channel == 6 && len >= ANE_T2H_CALL_STATE_OFF + 4) { const u8 *ev = ane_rtclient_fw_cpu(ane, hdr, ANE_T2H_CALL_STATE_OFF + 4); @@ -826,7 +999,7 @@ static bool ane_rtclient_drain_t2h(struct ane_rtclient *ane, WRITE_ONCE(slot[0], hdr | 1); dma_wmb(); writel(BIT(c->bit), ipi); - slot_i = (slot_i + 1) % c->size; + slot_i = (slot_i + 1) % c->slots; ane->legacy_cmd_cursor[channel] = slot_i; } return finished; @@ -885,24 +1058,38 @@ static int ane_rtclient_call_wait_traced(struct ane_rtclient *ane, * Return zero on the event, or -ETIMEDOUT. */ static int ane_rtclient_call_wait(struct ane_rtclient *ane, - unsigned int timeout_ms) + unsigned long deadline) { - unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); - unsigned int poll_us = READ_ONCE(call_poll_us); + unsigned int poll_us = ane->soc->platform ? 0 : READ_ONCE(call_poll_us); ktime_t tight_end = ktime_add_ns(ktime_get(), (u64)poll_us * NSEC_PER_USEC); if (ane_t6021_tracing) return ane_rtclient_call_wait_traced(ane, deadline); + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; do { - if (ane_rtclient_drain_t2h(ane, 6)) + if (atomic_read(&ane_t6021_quarantined)) + return -EIO; + if (ane->soc->platform) { + ane_rtclient_drain_t2h(ane, 6); + if (atomic_read(&ane_t6021_quarantined)) + return ane->pending.error ?: -EIO; + if (ane->waiting && ane->waiting->terminal) + return ane->waiting->status; + } else if (ane_rtclient_drain_t2h(ane, 6)) { return 0; + } if (poll_us && ktime_before(ktime_get(), tight_end)) usleep_range(1, 2); else usleep_range(50, 100); } while (time_before(jiffies, deadline)); + if (atomic_read(&ane_t6021_quarantined)) + return -EIO; + if (ane->soc->platform) + return -ETIMEDOUT; return ane_rtclient_drain_t2h(ane, 6) ? 0 : -ETIMEDOUT; } @@ -912,9 +1099,12 @@ static int ane_rtclient_command(struct ane_rtclient *ane, unsigned int channel, unsigned int timeout_ms) { + unsigned long deadline; int ret; u64 stats_ticket = 0; u64 stats_submit_ns = 0; + bool execute = ane->soc->platform ? opcode == ANE_H16_EXECUTE : + opcode == CSNE_CMD_PROCEDURE_CALL; ane_t6021_tracing = opcode == CSNE_CMD_PROCEDURE_CALL && trace_td && ane->soc->trace_td_off; @@ -937,8 +1127,16 @@ static int ane_rtclient_command(struct ane_rtclient *ane, * ring, as on ane.ko: stats=0 or a failed probe allocation leaves * stats_slots NULL, and the hooks must not run on a NULL ring. */ - bool stats_call = ane->stats_slots && - opcode == CSNE_CMD_PROCEDURE_CALL; + bool stats_call = ane->stats_slots && execute; + + if (ane->soc->platform && execute && + (!ane->waiting || !ane->waiting->active || ane->pending.quarantined)) + return -EIO; + + /* One bounded budget covers both ACK and finish, including H14. */ + timeout_ms = min(timeout_ms, 5000U); + deadline = ane->soc->platform && ane->waiting ? + ane->waiting->deadline : jiffies + msecs_to_jiffies(timeout_ms); if (stats_call) { stats_submit_ns = ktime_get_ns(); @@ -947,7 +1145,7 @@ static int ane_rtclient_command(struct ane_rtclient *ane, stats_submit_ns, 1); } ret = ane_rtclient_legacy_exchange(ane, command, length, opcode, - channel, timeout_ms); + channel, deadline); if (ret) { ane_t6021_tracing = false; if (stats_call) @@ -959,12 +1157,14 @@ static int ane_rtclient_command(struct ane_rtclient *ane, atomic_set(&ane_t6021_quarantined, 1); return ret; } - if (opcode == CSNE_CMD_PROCEDURE_CALL) { + if (execute) { if (ane_t6021_tracing) ane_t6021_trace_add(ANE_TR_ACK, 0); - ret = ane_rtclient_call_wait(ane, timeout_ms); + ret = ane_rtclient_call_wait(ane, deadline); ane_t6021_tracing = false; - if (!ret && call_settle_us) + if (!ret) + dma_rmb(); + if (!ret && !ane->soc->platform && call_settle_us) usleep_range(call_settle_us, call_settle_us + 100); if (ret) { if (stats_call) @@ -974,7 +1174,7 @@ static int ane_rtclient_command(struct ane_rtclient *ane, ktime_get_ns(), (u32)ret, 0); dev_err(ane->dev, - "call completion wait failed %d: no finish event in %u ms\n", + "call completion failed %d within %u ms\n", ret, timeout_ms); atomic_set(&ane_t6021_quarantined, 1); return ret; @@ -987,8 +1187,11 @@ static int ane_rtclient_command(struct ane_rtclient *ane, * hand those slots back so the rings never fill (channels 4 * and 6). */ - ane_rtclient_drain_t2h(ane, 4); + if (!ane->soc->platform) + ane_rtclient_drain_t2h(ane, 4); ane_rtclient_drain_t2h(ane, 6); + if (atomic_read(&ane_t6021_quarantined)) + return -EIO; /* BOs are dma_alloc_coherent memory mapped write-combined for the * CPU, so no cache maintenance is needed on either side. */ @@ -1058,6 +1261,9 @@ static int ane_rtclient_load_program(struct ane_rtclient *ane, int ret, i, j; BUILD_BUG_ON(DRM_ANE_MAX_SECTIONS != ANE_T6021_LOAD_SEC_COUNT); + /* H16 requires the original HWX, not caller-asserted section identity. */ + if (ane->soc->platform) + return -EOPNOTSUPP; if (user->pad || user->section_count < 1 || user->section_count > ANE_T6021_LOAD_SEC_COUNT || user->generic_count > DRM_ANE_MAX_BINDS || !fd) @@ -1280,6 +1486,9 @@ static int ane_rtclient_create_process(struct ane_rtclient *ane, struct ane_t6021_prog *prog = ane_t6021_prog_by_id(prog_id); int ret; + /* H16 uses 0x258 bytes; its body is not the H14 0x10-byte body. */ + if (ane->soc->platform) + return -EOPNOTSUPP; if (!prog) return -ENOENT; if (prog->proc_id != U32_MAX) { @@ -1311,6 +1520,140 @@ static int ane_rtclient_create_process(struct ane_rtclient *ane, return ret; } +/* H16 consumes only backend-owned process resources. The ABI-2 section + * loader and short process packet cannot create an entry in h16_processes. + */ +static int ane_rtclient_h16_call(struct ane_rtclient *ane, struct drm_file *file, + const struct drm_ane_exec *user) +{ + struct ane_t6021_fd *fd = file->driver_priv; + struct ane_h16_process *process = NULL, *p; + struct ane_h16_execute_buffer buffers[ANE_H16_SURFACES]; + struct ane_t6021_bo *bos[ANE_H16_SURFACES] = {}; + struct drm_ane_exec_io *ios; + struct ane_h16_pending request = {}; + struct ane_h16_execute call = {}; + unsigned int i, j, pinned = 0; + int ret = -EOPNOTSUPP; + + if (!fd || user->pad || user->priority != 5 || user->count < 2 || + user->count > ANE_H16_SURFACES || user->timeout_ms > 30000) + return -EINVAL; + ios = memdup_user(u64_to_user_ptr(user->io_ptr), + array_size(user->count, sizeof(*ios))); + if (IS_ERR(ios)) + return PTR_ERR(ios); + mutex_lock(&ane_t6021_fw_lock); + if (atomic_read(&ane_t6021_quarantined) || ane->pending.quarantined) { + ret = -EIO; + goto out; + } + list_for_each_entry(p, &fd->h16_processes, owner_node) { + if (p->key.program == user->prog_id && + p->key.process == user->proc_id) { + process = p; + break; + } + } + if (!process) + goto out; + ret = ane_h16_hwx_snapshot_check(&process->original); + if (!ret) + ret = ane_h16_resources_validate(&process->original.info, + process->resources, process->resource_count); + if (ret) + goto out; + if (user->count != process->resource_count || !ane->cmd_buf || + process->key.program == U32_MAX || process->key.process == U32_MAX || + process->key.procedure == U32_MAX) { + ret = -EINVAL; + goto out; + } + for (i = 0; i < user->count; i++) { + struct ane_t6021_bo *b; + const struct ane_h16_resource *r = &process->resources[i]; + const struct ane_legacy_buffer *m = &process->mappings[i]; + + if (ios[i].flags || ios[i].reserved || + ios[i].buffer_id != r->buffer.index || ios[i].type != r->buffer.class || + ios[i].size != r->buffer.size || !m->cpu || + m->dma != r->buffer.mapping_dva || m->size != r->buffer.mapping_size) { + ret = -EINVAL; + goto out; + } + for (j = 0; j < i; j++) { + if (ios[j].bo_handle == ios[i].bo_handle) { + ret = -EINVAL; + goto out; + } + } + mutex_lock(&ane_t6021_bo_lock); + list_for_each_entry(b, &fd->bos, node) { + if (b->handle == ios[i].bo_handle && b->owner == fd && + b->size >= ios[i].size) { + kref_get(&b->refcount); + bos[pinned++] = b; + break; + } + } + mutex_unlock(&ane_t6021_bo_lock); + if (pinned != i + 1) { + ret = -ENOENT; + goto out; + } + buffers[i] = r->buffer; + } + /* All user-facing BOs are staging only. Concurrent mmap writes cannot + * change code, input snapshots or firmware-visible output allocations. + */ + for (i = 0; i < user->count; i++) { + if (process->resources[i].permissions == 1) + memcpy(process->mappings[i].cpu, bos[i]->cpu, buffers[i].size); + else + memset(process->mappings[i].cpu, 0xff, buffers[i].size); + } + request.key = process->key; + request.deadline = jiffies + msecs_to_jiffies(user->timeout_ms ?: 5000); + ret = ane_h16_pending_insert(&ane->pending, &request, jiffies); + if (ret) + goto out; + call.key = request.key; + call.buffers = buffers; + call.buffer_count = user->count; + /* Custom BAR selectors/values have no validated process map yet. */ + ret = ane_h16_execute_encode(ane->cmd_buf->cpu, ANE_H16_EXECUTE_BASE_SIZE, &call); + if (ret) { + ane_h16_pending_remove(&ane->pending, &request); + goto out; + } + ane->waiting = &request; + /* Coherent allocations require ordering, not streaming-map sync calls. */ + wmb(); + dma_wmb(); + ret = ane_rtclient_command(ane, ane->cmd_buf, ANE_H16_EXECUTE_BASE_SIZE, + ANE_H16_EXECUTE, 7, user->timeout_ms ?: 5000); + ane->waiting = NULL; + if (ret || !request.terminal || request.status) { + ret = ret ?: -EIO; + ane_h16_pending_abort(&ane->pending, &request, ret); + atomic_set(&ane_t6021_quarantined, 1); + goto out; + } + dma_rmb(); + for (i = 0; i < user->count; i++) + if (process->resources[i].permissions == 2) + memcpy(bos[i]->cpu, process->mappings[i].cpu, buffers[i].size); + /* Private DMA mappings remain resident for the entire process lifetime, + * including after terminal failure or an uncertain transport outcome. + */ +out: + while (pinned) + kref_put(&bos[--pinned]->refcount, ane_t6021_bo_release); + mutex_unlock(&ane_t6021_fw_lock); + kfree(ios); + return ret; +} + static int ane_rtclient_procedure_call(struct ane_rtclient *ane, struct drm_file *file, const struct drm_ane_exec *user) @@ -1323,6 +1666,8 @@ static int ane_rtclient_procedure_call(struct ane_rtclient *ane, size_t cmd_size; int ret, i, pinned = 0; + if (ane->soc->platform) + return ane_rtclient_h16_call(ane, file, user); if (user->pad || user->count < 1 || user->count > DRM_ANE_MAX_BINDS || user->priority < 2 || user->priority > 7 || !fd) return -EINVAL; @@ -1451,6 +1796,8 @@ static int ane_t6021_bo_init_ioctl(struct drm_device *drm, void *data, struct ane_t6021_bo *bo; struct ane_rtclient *ane; + if (atomic_read(&ane_t6021_quarantined)) + return -EIO; if (args->pad || args->size == 0 || args->size > ANE_T6021_BO_MAX || !fd) return -EINVAL; @@ -1611,6 +1958,7 @@ static int ane_t6021_open(struct drm_device *drm, struct drm_file *file) if (!fd) return -ENOMEM; INIT_LIST_HEAD(&fd->bos); + INIT_LIST_HEAD(&fd->h16_processes); file->driver_priv = fd; return 0; } @@ -1619,13 +1967,19 @@ static void ane_t6021_postclose(struct drm_device *drm, struct drm_file *file) { struct ane_t6021_fd *fd = file->driver_priv; struct ane_t6021_bo *bo, *tmp; + struct ane_h16_process *process, *next; if (!fd) return; + mutex_lock(&ane_t6021_fw_lock); + /* Revoke per-file authority, retaining device-owned DMA allocations. */ + list_for_each_entry_safe(process, next, &fd->h16_processes, owner_node) + list_del_init(&process->owner_node); mutex_lock(&ane_t6021_bo_lock); list_for_each_entry_safe(bo, tmp, &fd->bos, node) ane_t6021_bo_drop(bo); mutex_unlock(&ane_t6021_bo_lock); + mutex_unlock(&ane_t6021_fw_lock); kfree(fd); file->driver_priv = NULL; } @@ -1921,6 +2275,13 @@ struct ane_rtclient_pd { struct device **pd_dev; struct device_link **pd_link; int count; + + /* T8112 power gate, kept in step with power-domains[0]. */ + struct notifier_block pwgate_nb; + void __iomem *pwgate_base; + void __iomem *pwgate; + bool pwgate_nb_added; + bool pwgate_opened; }; static LIST_HEAD(ane_rtclient_pd_list); @@ -1932,11 +2293,103 @@ static DEFINE_MUTEX(ane_rtclient_probe_guard_lock); static void ane_rtclient_pd_free(struct ane_rtclient_pd *pd) { list_del(&pd->list); + if (pd->pwgate_base) + iounmap(pd->pwgate_base); kfree(pd->pd_dev); kfree(pd->pd_link); kfree(pd); } +/* + * T8112 power gate. A word in the "set" window gates the ANE: bits 29:28 + * set (the word reads 0x30000000) is closed, both clear is open. It must be + * open before the ANE CPU power state (ane_sys_cpu) turns on and is closed + * again after that state turns off, with ane_sys, the parent, on both + * times. power-domains[0] is ane_sys_cpu, and genpd calls this notifier + * after ane_sys is on and before ane_sys_cpu is written (PRE_ON), and + * after ane_sys_cpu is off and before ane_sys goes off (OFF), which is that + * order. Only the two known values are written. Any other word fails the + * power-up without a write. + */ +#define ANE_PWGATE_CLOSED GENMASK(29, 28) +#define ANE_PWGATE_POLL_US 10 +#define ANE_PWGATE_TIMEOUT_US 100000 + +static int ane_rtclient_pwgate_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct ane_rtclient_pd *pd = + container_of(nb, struct ane_rtclient_pd, pwgate_nb); + u32 gate; + int ret; + + switch (action) { + case GENPD_NOTIFY_PRE_ON: + gate = readl(pd->pwgate); + if (!(gate & ANE_PWGATE_CLOSED)) + return NOTIFY_OK; + if (gate != ANE_PWGATE_CLOSED) { + dev_err(pd->dev, "power gate word %#x is not a known state\n", + gate); + return notifier_from_errno(-EIO); + } + writel(0, pd->pwgate); + ret = readl_poll_timeout_atomic(pd->pwgate, gate, + !(gate & ANE_PWGATE_CLOSED), + ANE_PWGATE_POLL_US, + ANE_PWGATE_TIMEOUT_US); + if (ret) { + writel(ANE_PWGATE_CLOSED, pd->pwgate); + dev_err(pd->dev, "power gate did not open (word %#x)\n", + gate); + return notifier_from_errno(ret); + } + pd->pwgate_opened = true; + return NOTIFY_OK; + case GENPD_NOTIFY_OFF: + if (!pd->pwgate_opened) + return NOTIFY_OK; + pd->pwgate_opened = false; + writel(ANE_PWGATE_CLOSED, pd->pwgate); + ret = readl_poll_timeout_atomic(pd->pwgate, gate, + (gate & ANE_PWGATE_CLOSED) == + ANE_PWGATE_CLOSED, + ANE_PWGATE_POLL_US, + ANE_PWGATE_TIMEOUT_US); + if (ret) + dev_warn(pd->dev, "power gate did not close (word %#x)\n", + gate); + return NOTIFY_OK; + default: + return NOTIFY_DONE; + } +} + +/* + * Register the power gate notifier on power-domains[0] before its device + * link is created: the link resumes the domain at once. + */ +static int ane_rtclient_pwgate_add(struct ane_rtclient *ane, + struct ane_rtclient_pd *pd) +{ + int ret; + + if (!ane->soc->pwgate_off || pd->pwgate_nb_added) + return 0; + if (!pd->pwgate_base) { + pd->pwgate_base = of_iomap(ane->dev->of_node, 2); + if (!pd->pwgate_base) + return -ENOMEM; + pd->pwgate = pd->pwgate_base + ane->soc->pwgate_off; + } + pd->pwgate_nb.notifier_call = ane_rtclient_pwgate_notify; + ret = dev_pm_genpd_add_notifier(pd->pd_dev[0], &pd->pwgate_nb); + if (ret) + return ret; + pd->pwgate_nb_added = true; + return 0; +} + static int ane_rtclient_attach_genpd(struct ane_rtclient *ane) { struct device *dev = ane->dev; @@ -1985,6 +2438,11 @@ static int ane_rtclient_attach_genpd(struct ane_rtclient *ane) goto out; } } + if (i == 0) { + err = ane_rtclient_pwgate_add(ane, pd); + if (err) + goto out; + } if (!pd->pd_link[i]) { if (!ane_rtclient_pinned) { if (!try_module_get(THIS_MODULE)) { @@ -2029,6 +2487,15 @@ static void ane_rtclient_detach_genpd(struct device *dev) for (i = count - 1; i >= 0; i--) { if (pd->pd_link[i]) device_link_del(pd->pd_link[i]); + /* + * genpd keeps a notifier past detach, so drop it first. + * The domain may then go off with the gate open, the + * state between the two power-up steps. + */ + if (i == 0 && pd->pwgate_nb_added) { + dev_pm_genpd_remove_notifier(pd->pd_dev[0]); + pd->pwgate_nb_added = false; + } if (pd->pd_dev[i]) dev_pm_domain_detach(pd->pd_dev[i], true); } @@ -2093,6 +2560,7 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, if (!ane->legacy_buffers) return -ENOMEM; ane->dev = dev; + INIT_LIST_HEAD(&ane->h16_processes); ane->soc = of_device_get_match_data(dev); platform_set_drvdata(pdev, ane); INIT_DELAYED_WORK(&ane->poll_work, ane_rtclient_post_boot); @@ -2140,9 +2608,10 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, pm_runtime_disable(dev); return -EPROBE_DEFER; } - /* T8112: macOS (type 0x70) opens PWGATE (bits 29:28 = 0) before - * the ps words. This driver only reads it: an engine read behind a - * closed gate is untested, so it refuses first. + /* T8112: the power gate notifier opened the gate before ane_sys_cpu + * turned on. A gate that still reads closed means the state was + * already on without it (no power-up ran), so refuse before any + * engine read. */ if (ane->soc->pwgate_off) { void __iomem *set = devm_of_iomap(dev, dev->of_node, 2, NULL); @@ -2335,14 +2804,13 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, * reboot. */ struct ane_legacy_buffer *command = ane->cmd_buf; + unsigned long query_deadline = jiffies + msecs_to_jiffies(3000); int qret; if (command) { memset(command->cpu, 0, SZ_16K); - qret = ane_rtclient_legacy_exchange(ane, - command, - 16, 0x03, - 1, 3000); + qret = ane_rtclient_legacy_exchange(ane, command, 16, 0x03, 1, + query_deadline); dev_dbg(dev, "LEGACY CONFIG_GET words %08x %08x result=%d (DMA remains held)\n", READ_ONCE(((u32 *)command->cpu)[1]), @@ -2483,9 +2951,16 @@ static int ane_rtclient_probe_inner(struct platform_device *pdev, static int ane_rtclient_probe(struct platform_device *pdev) { + const struct ane_t602x_soc *soc = of_device_get_match_data(&pdev->dev); bool hardware_touched = false; int ret; + if (!soc) + return -ENODEV; + /* Later generations cannot enter H14 boot or parameter paths. */ + if (soc->platform) + return ane_fw_platform_preflight(pdev, soc->platform); + mutex_lock(&ane_rtclient_probe_guard_lock); if (ane_rtclient_probe_failed_dirty) { mutex_unlock(&ane_rtclient_probe_guard_lock); @@ -2526,11 +3001,21 @@ static void ane_rtclient_remove(struct platform_device *pdev) "remove HELD: no teardown — reboot reclaims\n"); } +static const struct ane_t602x_soc ane_h16_soc = { + .platform = &ane_h16_platform, +}; + +static const struct ane_t602x_soc ane_h17_soc = { + .platform = &ane_h17_platform, +}; + static const struct of_device_id ane_rtclient_of_match[] = { { .compatible = "apple,t6020-ane", .data = &ane_t6020_soc }, { .compatible = "apple,t6021-ane", .data = &ane_t6021_soc }, { .compatible = "apple,t6022-ane", .data = &ane_t6022_soc }, { .compatible = "apple,t8112-ane", .data = &ane_t8112_soc }, + { .compatible = "apple,t6041-ane", .data = &ane_h16_soc }, + { .compatible = "apple,t8140-ane", .data = &ane_h17_soc }, { } }; MODULE_DEVICE_TABLE(of, ane_rtclient_of_match); @@ -2588,8 +3073,38 @@ static struct platform_driver ane_rtclient_driver = { static int __init ane_rtclient_init(void) { + const struct of_device_id *match; + struct device_node *np; int ret; + /* platform_probe() powers a single genpd, and DMA configuration + * happens even earlier. Refuse later generations before registering + * the driver, including disabled nodes, so these core paths cannot + * precede the ownership and firmware preflight. These SoCs have one + * ANE generation; refusing registration cannot hide an H14 engine. + */ + for_each_matching_node_and_match(np, ane_rtclient_of_match, &match) { + const struct ane_t602x_soc *soc = match->data; + struct platform_device *pdev; + + if (!soc->platform) + continue; + if (soc->platform->owner == ANE_FW_OWNER_EXCLAVE) { + of_node_put(np); + pr_info("ane: H17 ANE: Linux exclave ownership handoff unavailable; refusing registration\n"); + return -EOPNOTSUPP; + } + pdev = of_find_device_by_node(np); + of_node_put(np); + if (!pdev) { + pr_info("ane: H16 ANE: firmware transport unvalidated; keeping disabled\n"); + return -EOPNOTSUPP; + } + ret = ane_fw_platform_preflight(pdev, soc->platform); + put_device(&pdev->dev); + return ret; + } + WRITE_ONCE(ane_rtclient_probe_failed_dirty, false); ret = platform_driver_register(&ane_rtclient_driver); @@ -2607,4 +3122,4 @@ static void __exit ane_rtclient_exit(void) module_exit(ane_rtclient_exit); MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("Apple Neural Engine (T6021/M2) installed module"); +MODULE_DESCRIPTION("Apple Neural Engine firmware driver with H16/H17 preflight"); diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index e4fab5e3225ff..63ce4037d0ca7 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -6,6 +6,7 @@ #include #include #include "ane_tm.h" +#include "ane_pm.h" #define ANE_TQ_COUNT 8 static const int TQ_PRTY_TABLE[ANE_TQ_COUNT] = { 0x1, 0x2, 0x3, 0x4, @@ -266,6 +267,7 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) struct ane_dart_scratch *scratch = ane->scratch; bool faulted = false; int err, status; + ktime_t deadline = ktime_add_ms(ktime_get(), 1000); lockdep_assert_held(&ane->engine_lock); ane_dart_mask(ane); @@ -280,6 +282,10 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) ane_tm_push_tq(ane, req); for (;;) { + if (ktime_compare(ktime_get(), deadline) >= 0) { + err = -ETIMEDOUT; + break; + } /* Latch-first completion poll. Engine register reads hang * the SoC (T8103) once a DART fault has landed, inside the * 1-microsecond completion poll and before any fault @@ -289,10 +295,13 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) * iteration's latch read and engine read. */ err = read_poll_timeout(ane_tm_poll_step, status, - status != 0, 1, 1000000, false, + status != 0 || ktime_compare(ktime_get(), deadline) >= 0, + 1, 1000000, false, ane, req, &finished); if (!err && status < 0) err = status; + if (!err && !status) + err = -ETIMEDOUT; if (ane_dart_faulted(ane, &fault_iova, &fault_status)) { faulted = true; if (ane->scratch_pages >= ANE_DART_SCRATCH_MAX) @@ -366,38 +375,18 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) * and return to accepting work. Only a failed reset preserves * resources until reboot. */ - if (ane_tm_recover(ane) < 0) { - if (ane->tm_retention) - dev_err(ane->dev, - "recovery failed; preserving resources until module reload or reboot (tm/tq file retained through the cycle: the latched task error parks the TM in place; rmmod+insmod restores service)\n"); - else - dev_err(ane->dev, - "recovery failed; preserving resources until reboot\n"); - } + if (ane_tm_recover(ane) < 0) + dev_err(ane->dev, + "recovery failed; submissions disabled, resources retained while DMA may be active; retry reset or reboot\n"); return err; } /* - * Bounded recovery after a failed task. The only reset this driver knows - * is the partition power cycle system sleep already performs on this - * hardware. With the several ANE power partitions attached as genpd - * devices (T8103 five, T6001 similar), gating happens on those devices; - * with a single domain attached directly to the device, on the device - * itself. Gating the partitions stops any DMA still fetching the dead - * task and clears the tm register file; ungating brings the engine back - * through the probe resume path (ane_tm_enable), then the task manager - * must report idle. BO mappings deliberately stay in place: they remain - * coherent through the cycle and valid for the next submit, and once the - * wedge clears, BO_FREE unmaps them through the normal path again. - * Callers hold engine_lock. - * - * T6001 note: its set0/base islands are - * firmware-locked (direct pmgr writes external-abort; see the SET block - * comment above) and hold the tm/tq file in retention through any - * genpd cycle - TQ_EN still reads 0x3000 after a 20 ms and a 2 s gate, - * and tasks dispatched afterwards complete but compute nondeterministic - * garbage. The idle-or-fresh poll below therefore fails on T6001 and - * recovery preserves until reboot instead of serving wrong data. + * Recovery runs with engine_lock and pm_lock held and one consumer PM + * reference. Suspend through runtime PM, just as autosuspend does, so the + * supplier references and the domain hierarchy are released together. + * Never use the system-sleep force helpers here: an in-use forced supplier + * retains RPM_ACTIVE, which prevents its parent domain from powering off. */ /* No engine MMIO unless the owning partitions read powered on (ACTUAL @@ -426,67 +415,60 @@ static int ane_ps_verify_on(struct ane_device *ane) return err; } +static bool ane_partitions_off(struct ane_device *ane) +{ + if (ane_ps_act(ane)) + return false; + + /* The SET window does not include the shared SYS_CPU parent. Its + * DART consumers suspend asynchronously after our references drop. + * Wait for every attached domain as well, so resuming the consumer + * cannot race that work and leave the TM register file powered. + */ + if (ane->pd_count == 1) + return !dev_pm_genpd_is_on(ane->dev); + for (int i = 0; i < ane->pd_count; i++) + if (dev_pm_genpd_is_on(ane->pd_dev[i])) + return false; + return true; +} + +static int ane_verify_off(void *ctx) +{ + struct ane_device *ane = ctx; + bool off; + int err; + + /* genpd may retain a shared or busy parent even after suspend succeeds. + * Only PMGR and provider state are safe to read while the engine is off. + */ + if (!ane->ps) + return -EOPNOTSUPP; + err = read_poll_timeout(ane_partitions_off, off, off, + 100, 100000, false, ane); + dev_info(ane->dev, "recovery: engine power-off actual=%#x rc=%d\n", + ane_ps_act(ane), err); + return err; +} + static int ane_pd_cycle(struct ane_device *ane) { - int err = 0; + int err; + lockdep_assert_held(&ane->pm_lock); ane->recovering = true; - if (ane->pd_count > 1) { - int gated = 0; - - for (int i = 0; i < ane->pd_count; i++) { - pm_runtime_get_noresume(ane->pd_dev[i]); - gated = i + 1; - dev_dbg(ane->dev, - "ANERD pd[%d] %s force_suspend begin (ps act %#x)\n", - i, dev_name(ane->pd_dev[i]), ane_ps_act(ane)); - err = pm_runtime_force_suspend(ane->pd_dev[i]); - dev_dbg(ane->dev, - "ANERD pd[%d] force_suspend -> %d (ps act %#x)\n", - i, err, ane_ps_act(ane)); - if (err) - break; - } - if (!err) - for (int i = 0; i < ane->pd_count; i++) { - dev_dbg(ane->dev, - "ANERD pd[%d] %s force_resume begin (ps act %#x)\n", - i, dev_name(ane->pd_dev[i]), - ane_ps_act(ane)); - err = pm_runtime_force_resume(ane->pd_dev[i]); - dev_dbg(ane->dev, - "ANERD pd[%d] force_resume -> %d (ps act %#x)\n", - i, err, ane_ps_act(ane)); - if (err) - break; - } - while (gated--) - pm_runtime_put_noidle(ane->pd_dev[gated]); - } else { - /* Pin a second usage ref for the cycle: force_suspend only - * marks needs_force_resume when a ref beyond the caller's - * one (submit ioctl or reset write) is held, and without - * that mark force_resume would leave the partition gated. - */ - pm_runtime_get_noresume(ane->dev); - dev_dbg(ane->dev, "ANERD dev force_suspend begin\n"); - err = pm_runtime_force_suspend(ane->dev); - dev_dbg(ane->dev, "ANERD dev force_suspend -> %d\n", err); - if (!err) { - dev_dbg(ane->dev, "ANERD dev force_resume begin\n"); - err = pm_runtime_force_resume(ane->dev); - dev_dbg(ane->dev, "ANERD dev force_resume -> %d\n", - err); - } - pm_runtime_put_noidle(ane->dev); - } + /* Release the performance request before the domains go down. */ + err = ane_perf_suspend(ane); + if (!err) + err = ane_pm_cycle(ane->dev, ane->pd_dev, ane->pd_count, + ane_verify_off, ane); ane->recovering = false; - return err; } /* - * Drain the task-manager state a timed-out task leaves behind when the + * Drain the task-manager state a timed-out task leaves behind (from + * omarchy-ane ane_tm_drain_retained by Joshua Warren, MIT) when the * tm/tq file survives the power cycle in retention (T6001; descriptor * tm_retention). T8103 never runs this: its cycle is a full POR of the * file, and the re-arm below starts it clean. @@ -601,23 +583,23 @@ int ane_tm_recover(struct ane_device *ane) return err; } - /* T6001 (tm_retention): the cycle did NOT clear the tm/tq file — - * the timed-out task's events and queue entry are still latched. - * Drain them, or the TM below never reads idle and recovery - * degrades to preserve-until-reboot. + /* On T600x the tm/tq file survives the acknowledged power-off in + * retention (TQ_EN reads 0x3000 after every resume, healthy or not, + * and PMGR reset pulses on the ANE domains do not clear it). Drain + * the retained state there; this is what recovers a failed task + * ("-5, finish lines=1"). Anything the drain cannot clear fails the + * idle check below and stays wedged. */ if (ane->tm_retention) ane_tm_drain_retained(ane); - /* Power-on reset cleared the tm register file; re-arm it exactly - * like the probe resume path does. - */ ane_tm_enable(ane, true); err = readl_poll_timeout(ane->engine + ANE_TM_BASE + TM_STATUS, status, (status & TM_IS_IDLE) || - status == ane->tm_status_fresh, + (ane->tm_status_known && + status == ane->tm_status_fresh), 100, 1000000); status = ane_rec_read32(ane, "TM_STATUS tm+0x54", ane->engine + ANE_TM_BASE + TM_STATUS); @@ -628,7 +610,7 @@ int ane_tm_recover(struct ane_device *ane) } if (atomic_xchg(&ane->wedged, 0)) { - module_put(THIS_MODULE); /* drop the wedge pin */ + module_put(THIS_MODULE); /* verified reset releases the wedge pin */ /* The power cycle above quiesced every transaction, so the * wedge-preserved mappings (dead BOs kept alive only to * stop IOVA teardown under active DMA) can now be unmapped diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index a24bd66c86d81..90d51641a6a35 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -680,6 +681,11 @@ static int admac_alloc_chan_resources(struct dma_chan *chan) struct admac_data *ad = adchan->host; int ret; + /* Keep MMIO accessible for config, status and the atomic DMA callbacks. */ + ret = pm_runtime_resume_and_get(ad->dev); + if (ret < 0) + return ret; + dma_cookie_init(&adchan->chan); /* @@ -699,8 +705,10 @@ static int admac_alloc_chan_resources(struct dma_chan *chan) ret = admac_alloc_sram_carveout(ad, admac_chan_direction(adchan->no), &adchan->carveout); - if (ret < 0) + if (ret < 0) { + pm_runtime_put_autosuspend(ad->dev); return ret; + } writel_relaxed(adchan->carveout, ad->base + REG_CHAN_SRAM_CARVEOUT(ad, adchan->no)); @@ -715,6 +723,7 @@ static void admac_free_chan_resources(struct dma_chan *chan) admac_synchronize(chan); admac_free_sram_carveout(adchan->host, admac_chan_direction(adchan->no), adchan->carveout); + pm_runtime_put_autosuspend(adchan->host->dev); } static struct dma_chan *admac_dma_of_xlate(struct of_phandle_args *dma_spec, @@ -976,6 +985,37 @@ static int admac_device_config(struct dma_chan *chan, return 0; } +static int admac_runtime_suspend(struct device *dev) +{ + struct admac_data *ad = dev_get_drvdata(dev); + + /* No channel resources remain; drain handlers before genpd gates MMIO. */ + disable_irq(ad->irq); + if (dev->pm_domain) + ad->ctl_programmed = false; + + return 0; +} + +static int admac_runtime_resume(struct device *dev) +{ + struct admac_data *ad = dev_get_drvdata(dev); + + /* + * Defer register restoration to alloc/config, before any transfer. Some + * controllers are powered by firmware only after the audio handshake. + * Restore only the requested channel, never reset the shared fabric. + */ + enable_irq(ad->irq); + + return 0; +} + +/* Allocated channels must stay powered, including paused cyclic transfers. */ +static const struct dev_pm_ops admac_pm_ops = { + RUNTIME_PM_OPS(admac_runtime_suspend, admac_runtime_resume, NULL) +}; + static int admac_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; @@ -1093,24 +1133,37 @@ static int admac_probe(struct platform_device *pdev) goto free_reset; } + pm_runtime_get_noresume(ad->dev); + err = pm_runtime_set_active(ad->dev); + if (err) + goto put_pm; + pm_runtime_set_autosuspend_delay(ad->dev, 100); + pm_runtime_use_autosuspend(ad->dev); + pm_runtime_enable(ad->dev); + err = dma_async_device_register(&ad->dma); if (err) { dev_err_probe(&pdev->dev, err, "failed to register DMA device\n"); - goto free_irq; + goto disable_pm; } err = of_dma_controller_register(pdev->dev.of_node, admac_dma_of_xlate, ad); if (err) { dma_async_device_unregister(&ad->dma); dev_err_probe(&pdev->dev, err, "failed to register with OF\n"); - goto free_irq; + goto disable_pm; } dev_info(&pdev->dev, "Audio DMA Controller\n"); + pm_runtime_put_autosuspend(ad->dev); return 0; -free_irq: +disable_pm: + pm_runtime_disable(ad->dev); + pm_runtime_dont_use_autosuspend(ad->dev); +put_pm: + pm_runtime_put_noidle(ad->dev); free_irq(ad->irq, ad); free_reset: reset_control_rearm(ad->rstc); @@ -1123,6 +1176,8 @@ static void admac_remove(struct platform_device *pdev) of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&ad->dma); + pm_runtime_disable(ad->dev); + pm_runtime_dont_use_autosuspend(ad->dev); free_irq(ad->irq, ad); reset_control_rearm(ad->rstc); } @@ -1163,6 +1218,7 @@ static struct platform_driver apple_admac_driver = { .driver = { .name = "apple-admac", .of_match_table = admac_of_match, + .pm = pm_ptr(&admac_pm_ops), }, .probe = admac_probe, .remove = admac_remove, diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 06415026853f9..7575fb03695ec 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -32,16 +32,17 @@ config DRM_APPLE_AUDIO select SND_DMAENGINE_PCM config DRM_APPLE_DCP_T6030_AUDIO - bool "HDMI audio through the T6030 external display processor" + bool "Native audio links through external display processors" default y depends on DRM_APPLE_AUDIO help DP/HDMI audio of the external display processors of the M3 Pro (T6030, MacBook Pro J514S/J516S built-in HDMI port), whose DCP firmware 14.7 external role takes 32-byte audio link data and the - native driver's link lifecycle. The audio service of every other - DCP is driven unchanged. Say N to build the audio and element - parser code exactly as before. + native driver's link lifecycle. Also build the compact audio service + binding for desktop 26.x external processors (T6041). Those audio + interfaces remain disabled until SIO firmware support is available. + Say N to omit native links and the desktop compact audio binding. config DRM_APPLE_DCP_T8132 bool "MacBook Air M4 (J713) internal panel" diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index d71c5c0370f2e..75e52d2454385 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -553,7 +553,18 @@ apple_connector_atomic_best_encoder(struct drm_connector *conn, return NULL; } -/* The output color element follows the HDR metadata: a change is a modeset. */ +/* The EOTF a connector state asks for, an HDMI_EOTF_* value. */ +static u8 apple_connector_eotf(const struct drm_connector_state *state) +{ + const struct hdr_output_metadata *hdr; + + if (!state->hdr_output_metadata || + state->hdr_output_metadata->length != sizeof(*hdr)) + return HDMI_EOTF_TRADITIONAL_GAMMA_SDR; + hdr = state->hdr_output_metadata->data; + return hdr->hdmi_metadata_type1.eotf; +} + static int apple_connector_atomic_check(struct drm_connector *conn, struct drm_atomic_commit *state) { @@ -567,6 +578,17 @@ static int apple_connector_atomic_check(struct drm_connector *conn, crtc_state = drm_atomic_get_crtc_state(state, new->crtc); if (IS_ERR(crtc_state)) return PTR_ERR(crtc_state); + /* + * The output colour element follows the EOTF and colorspace, so a + * change of either is a mode set. Where the firmware takes HDR10 + * static metadata on a live link, a new block alone is sent with this + * commit's flush (dcp_hdr_metadata_sync()) and the colour mode stays. + */ + if (old->colorspace == new->colorspace && old->crtc == new->crtc && + apple_connector_eotf(old) == apple_connector_eotf(new) && + to_apple_connector(conn)->dcp && + dcp_hdr_metadata_live(to_apple_connector(conn)->dcp)) + return 0; crtc_state->mode_changed = true; return 0; } diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 2b507dc1fa1b0..ccb10dcf3768c 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -84,7 +84,7 @@ struct dcp_audio { #endif }; -/* The 14.7 external role's audio service (DRM_APPLE_DCP_T6030_AUDIO only). */ +/* Native external audio service (DRM_APPLE_DCP_T6030_AUDIO only). */ static inline bool dcpaud_native_link(struct dcp_audio *dcpaud) { #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) @@ -1057,7 +1057,7 @@ static int dcpaud_comp_bind(struct device *dev, struct device *main, void *data) dcpaud->dcp_dev = &dcp_pdev->dev; #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) - /* The 14.7 external role: its native links and element catalog. */ + /* Native external roles share the link lifecycle and element catalog. */ if (dcp_audiosrv_native_link(dcpaud->dcp_dev)) { void *elements = devm_kzalloc(dev, DCPAUD_ELEMENTS_V14_7, GFP_KERNEL); diff --git a/drivers/gpu/drm/apple/audio.h b/drivers/gpu/drm/apple/audio.h index 58633598253ea..88e9610363234 100644 --- a/drivers/gpu/drm/apple/audio.h +++ b/drivers/gpu/drm/apple/audio.h @@ -18,7 +18,7 @@ void dcpaud_connect(struct platform_device *pdev, bool connected); void dcpaud_disconnect(struct platform_device *pdev); #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) -/* The native link methods of the DCP firmware 14.7 external role. */ +/* Native link methods for 14.7 and desktop 26.x external roles. */ enum dcp_audio_link { DCP_AUDIO_LINK_PREPARE, DCP_AUDIO_LINK_START, diff --git a/drivers/gpu/drm/apple/av-v14-test.c b/drivers/gpu/drm/apple/av-v14-test.c index 8d78113fb8f17..362bb77db6465 100644 --- a/drivers/gpu/drm/apple/av-v14-test.c +++ b/drivers/gpu/drm/apple/av-v14-test.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* - * Included by av.c: which instance the 14.7 external role's audio client + * Included by av.c: which instance a native external role's audio client * opens and closes. One reusable service slot is published, torn down and * published again by the production init and teardown; the production open, * close, Terminate report and Terminate work run on it, with only the @@ -27,6 +27,8 @@ struct av_v14_test { u64 release_gen; /* Within a link release: the old PCM closes, a successor is published and opened */ bool successor_during_release; + u32 link_status; + u8 link_body[64]; }; static struct av_v14_test *av_v14_current; @@ -45,6 +47,10 @@ static int av_v14_call(struct apple_epic_service *srv, u32 command, const void * /* Open: status 0 at +0, opened at +16 */ if (command == t->asrv->cmds.open && reply_len >= 24) put_unaligned_le64(1, reply + 16); + if (body_len == sizeof(t->link_body)) { + memcpy(t->link_body, body, body_len); + put_unaligned_le32(t->link_status, reply + 48); + } return 0; } @@ -114,7 +120,8 @@ static struct av_v14_test *av_v14_setup(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); dev_set_drvdata(dev, t->dcp); t->dcp->dev = dev; - t->dcp->fw_compat = DCP_FIRMWARE_V_14_7; + t->dcp->fw_compat = test->param_value && *(const bool *)test->param_value ? + DCP_FIRMWARE_V_26_6_DESKTOP : DCP_FIRMWARE_V_14_7; t->dcp->external = true; t->dcp->external_native = true; @@ -130,6 +137,8 @@ static struct av_v14_test *av_v14_setup(struct kunit *test) t->dcp->audiosrv = asrv; t->ep.dcp = t->dcp; + t->ep.endpoint = AV_AUDIO_ENDPOINT_DESKTOP; + t->ep.compact = t->dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP; spin_lock_init(&t->svc.lock); t->svc.ep = &t->ep; spin_lock_init(&t->other.lock); @@ -324,21 +333,84 @@ static void av_v14_terminate_unpublished(struct kunit *test) KUNIT_EXPECT_EQ(test, t->ncalls, 2U); } +/* Rejected prepares must not create a link which Stop/Complete can act on. */ +static void av_native_link_payload_status(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + struct dcp_sound_cookie cookie; + + memset(&cookie, 0xa5, sizeof(cookie)); + KUNIT_ASSERT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + t->link_status = 1; + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_PREPARE, + &cookie), -EIO); + KUNIT_EXPECT_FALSE(test, dcp_audiosrv_link_on(t->dcp->dev, t->asrv->gen)); + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_START, + &cookie), -ENODEV); + t->link_status = 0; + for (enum dcp_audio_link op = DCP_AUDIO_LINK_PREPARE; + op <= DCP_AUDIO_LINK_COMPLETE; op++) { + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, op, &cookie), 0); + KUNIT_EXPECT_MEMEQ(test, t->link_body, &cookie, sizeof(cookie)); + for (size_t i = sizeof(cookie); i < sizeof(t->link_body); i++) + KUNIT_EXPECT_EQ(test, t->link_body[i], 0); + } + KUNIT_EXPECT_EQ(test, t->calls[2], t->asrv->cmds.prepare); + KUNIT_EXPECT_EQ(test, t->calls[3], t->asrv->cmds.start_link); + KUNIT_EXPECT_EQ(test, t->calls[4], t->asrv->cmds.stop_link); + KUNIT_EXPECT_EQ(test, t->calls[5], t->asrv->cmds.unprepare); + dcp_audiosrv_close(t->dcp->dev); +} + +static void av_desktop_service_admission(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + const char *name = "dcpav-audio-epic"; + bool desktop = t->dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP; + + KUNIT_EXPECT_TRUE(test, dcp_audiosrv_native_link(t->dcp->dev)); + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, name, name, 0), + desktop ? 0 : -EINVAL); + t->dcp->fw_compat = DCP_FIRMWARE_V_26_6_DESKTOP; + t->ep.endpoint = AV_ENDPOINT; + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, name, name, 0), -EINVAL); + t->ep.endpoint = AV_AUDIO_ENDPOINT_DESKTOP; + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, "dcpav-device-epic", name, 0), + -EINVAL); + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, NULL, name, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, name, name, 1), -EINVAL); + t->dcp->external = false; + KUNIT_EXPECT_FALSE(test, dcp_audiosrv_native_link(t->dcp->dev)); + KUNIT_EXPECT_EQ(test, av_desktop_audio_prepare(&t->svc, name, name, 0), -EINVAL); + dcp_audiosrv_close(t->dcp->dev); +} + +static const bool av_native_desktop[] = { false, true }; + +static void av_native_desc(const bool *desktop, char *desc) +{ + scnprintf(desc, KUNIT_PARAM_DESC_SIZE, "%s", *desktop ? "desktop-26" : "external-14.7"); +} + +KUNIT_ARRAY_PARAM(av_native, av_native_desktop, av_native_desc); + static struct kunit_case av_v14_cases[] = { - KUNIT_CASE(av_v14_open_close), - KUNIT_CASE(av_v14_successor_opened), - KUNIT_CASE(av_v14_old_close_not_to_successor), - KUNIT_CASE(av_v14_old_terminate_vs_successor), - KUNIT_CASE(av_v14_republished), - KUNIT_CASE(av_v14_successor_during_release), - KUNIT_CASE(av_v14_terminate_after_successor), - KUNIT_CASE(av_v14_link_instance), - KUNIT_CASE(av_v14_terminate_unpublished), + KUNIT_CASE_PARAM(av_v14_open_close, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_successor_opened, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_old_close_not_to_successor, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_old_terminate_vs_successor, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_republished, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_successor_during_release, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_terminate_after_successor, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_link_instance, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_terminate_unpublished, av_native_gen_params), + KUNIT_CASE_PARAM(av_native_link_payload_status, av_native_gen_params), + KUNIT_CASE_PARAM(av_desktop_service_admission, av_native_gen_params), {} }; static struct kunit_suite av_v14_suite = { - .name = "apple-av-v14-audio", + .name = "apple-av-native-audio", .test_cases = av_v14_cases, }; diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index d439ce69368b0..afb918fb58f89 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -254,10 +254,13 @@ int dcp_audiosrv_unprepare(struct device *dev) } #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) -/* The audio service of the 14.7 external role, known before it starts. */ +/* Native audio links are selected from the firmware role, before service start. */ bool dcp_audiosrv_native_link(struct device *dev) { - return dcp_is_v14_7_external(dev_get_drvdata(dev)); + struct apple_dcp *dcp = dev_get_drvdata(dev); + + return dcp_is_v14_7_external(dcp) || + (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && dcp->external); } /* The 14.7 external role's audio service calls; KUnit stubs them. */ @@ -266,6 +269,14 @@ static int av_audiosrv_call(struct apple_epic_service *srv, u32 command, const v { KUNIT_STATIC_STUB_REDIRECT(av_audiosrv_call, srv, command, body, body_len, body_pad, reply, reply_len); + if (READ_ONCE(srv->ep->compact)) { + size_t received; + int ret; + + ret = afk_service_call_with_reply_len(srv, 0, command, body, body_len, + body_pad, reply, reply_len, 0, &received); + return ret ?: (received == reply_len ? 0 : -EMSGSIZE); + } return afk_service_call(srv, 0, command, body, body_len, body_pad, reply, reply_len, 0); } @@ -395,6 +406,10 @@ int dcp_audiosrv_link(struct device *dev, enum dcp_audio_link op, KUNIT_STATIC_STUB_REDIRECT(dcp_audiosrv_link, dev, op, cookie); dcp = dev_get_drvdata(dev); asrv = dcp->audiosrv; + if (!asrv) + return -ENODEV; + if (!cookie) + return -EINVAL; switch (op) { case DCP_AUDIO_LINK_PREPARE: @@ -423,16 +438,16 @@ int dcp_audiosrv_link(struct device *dev, enum dcp_audio_link op, return -ENODEV; ret = av_audiosrv_call(asrv->open_srv, command, body, sizeof(body), 0, reply, sizeof(reply)); - if (!ret && op == DCP_AUDIO_LINK_PREPARE) + if (ret) + return ret; + status = get_unaligned_le32(reply + 48); + if (status) { + dev_err(dev, "audiosrv: link method %u answered %#x\n", command, status); + return -EIO; + } + if (op == DCP_AUDIO_LINK_PREPARE) asrv->link_gen = asrv->open_gen; } - if (ret) - return ret; - status = get_unaligned_le32(reply + 48); - if (status) { - dev_err(dev, "audiosrv: link method %u answered %#x\n", command, status); - return -EIO; - } return 0; } #endif @@ -449,6 +464,7 @@ dcp_audiosrv_osobject_call(struct apple_epic_service *service, u16 group, u8 _pad2[8]; } __attribute__((packed)) *hdr; static_assert(sizeof(*hdr) == 48); + size_t received = 0, used; size_t bfr_len = output_maxsize + sizeof(*hdr); void *bfr __free(kfree) = NULL; int ret; @@ -459,16 +475,30 @@ dcp_audiosrv_osobject_call(struct apple_epic_service *service, u16 group, hdr = bfr; hdr->max_size = cpu_to_le64(output_maxsize); - ret = afk_service_call(service, group, command, hdr, sizeof(*hdr), output_maxsize, - bfr, sizeof(*hdr) + output_maxsize, 0); + if (READ_ONCE(service->ep->compact)) + ret = afk_service_call_with_reply_len(service, group, command, hdr, + sizeof(*hdr), output_maxsize, bfr, + bfr_len, 0, &received); + else + ret = afk_service_call(service, group, command, hdr, sizeof(*hdr), + output_maxsize, bfr, bfr_len, 0); if (ret) return ret; - if (output) + used = le64_to_cpu(hdr->used_size); + if (READ_ONCE(service->ep->compact)) { + if (used > output_maxsize || received < sizeof(*hdr) || + used > received - sizeof(*hdr)) + return -EMSGSIZE; + if (output) { + memset(output, 0, output_maxsize); + memcpy(output, bfr + sizeof(*hdr), used); + } + } else if (output) { memcpy(output, bfr + sizeof(*hdr), output_maxsize); - + } if (output_size) - *output_size = le64_to_cpu(hdr->used_size); + *output_size = used; return 0; } @@ -477,13 +507,19 @@ int dcp_audiosrv_get_elements(struct device *dev, void *elements, size_t maxsize { struct apple_dcp *dcp = dev_get_drvdata(dev); struct audiosrv_data *asrv = dcp->audiosrv; + struct apple_epic_service *srv; size_t size; int ret; - down_write(&asrv->srv_rwsem); - ret = dcp_audiosrv_osobject_call(asrv->srv, 1, asrv->cmds.get_elements, + if (!asrv) + return -ENODEV; + scoped_guard(rwsem_read, &asrv->srv_rwsem) + srv = afk_service_get(asrv->srv); + if (!srv) + return -ENODEV; + ret = dcp_audiosrv_osobject_call(srv, 1, asrv->cmds.get_elements, elements, maxsize, &size); - up_write(&asrv->srv_rwsem); + afk_service_put(srv); if (ret && !asrv->warned_get_elements) { dev_err(dev, "audiosrv: error getting elements: %d\n", ret); @@ -499,14 +535,20 @@ int dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsi { struct apple_dcp *dcp = dev_get_drvdata(dev); struct audiosrv_data *asrv = dcp->audiosrv; + struct apple_epic_service *srv; size_t size; int ret; - down_write(&asrv->srv_rwsem); - ret = dcp_audiosrv_osobject_call(asrv->srv, 1, + if (!asrv) + return -ENODEV; + scoped_guard(rwsem_read, &asrv->srv_rwsem) + srv = afk_service_get(asrv->srv); + if (!srv) + return -ENODEV; + ret = dcp_audiosrv_osobject_call(srv, 1, asrv->cmds.get_product_attrs, attrs, maxsize, &size); - up_write(&asrv->srv_rwsem); + afk_service_put(srv); if (ret && !asrv->warned_get_product_attrs) { dev_err(dev, "audiosrv: error getting product attributes: %d\n", ret); @@ -542,7 +584,7 @@ static int av_audiosrv_report(struct apple_epic_service *service, u32 idx, * 10 s for that before it removes the interface, and the sink's link * with it. A later publication brings the new instance. */ - if (dcp_is_v14_7_external(dcp) && av_report_terminate(data, data_size)) { + if (dcp_audiosrv_native_link(dcp->dev) && av_report_terminate(data, data_size)) { struct audiosrv_data *asrv = dcp->audiosrv; bool published; @@ -590,6 +632,32 @@ static void av_cec_teardown(struct apple_epic_service *service) } #endif +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) +static int av_desktop_audio_prepare(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + + if (dcp->fw_compat != DCP_FIRMWARE_V_26_6_DESKTOP || !dcp->external || + service->ep->endpoint != AV_AUDIO_ENDPOINT_DESKTOP || + !name || strcmp(name, "dcpav-audio-epic") || unit != 0) + return -EINVAL; + return 0; +} + +static const struct apple_epic_service_ops av_desktop_ops[] = { + { + .name = "dcpav-audio-epic", + .reusable = true, + .prepare = av_desktop_audio_prepare, + .init = av_audiosrv_init, + .report = av_audiosrv_report, + .teardown = av_audiosrv_teardown, + }, + {} +}; +#endif + static const struct apple_epic_service_ops avep_ops[] = { { .name = "DCPAVSimpleVideoInterface", @@ -641,8 +709,8 @@ void av_service_connect(struct apple_dcp *dcp) * the display's power (A472 on): the CRTC enable connects before the * mode is set, and the modeset connects again once the display is on. */ - if (dcp_is_v14_7_external(dcp)) { - if (!iomfb_v14_7_external_powered(dcp)) + if (dcp_audiosrv_native_link(dcp->dev)) { + if (dcp_is_v14_7_external(dcp) && !iomfb_v14_7_external_powered(dcp)) goto out; scoped_guard(mutex, &asrv->plug_lock) { if (asrv->audio_dev) @@ -689,7 +757,7 @@ static void __av_service_disconnect(struct apple_dcp *dcp, const u64 *gen) dev_info(dcp->dev, "%s: stopping audio service\n", __func__); #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) /* The 14.7 external role's card goes with the display, and an open PCM's service. */ - if (dcp_is_v14_7_external(dcp)) { + if (dcp_audiosrv_native_link(dcp->dev)) { struct platform_device *audio_dev = NULL; bool published = true; @@ -866,6 +934,10 @@ static void av_audiosrv_cleanup(void *data) av_audiosrv_drain_owner(asrv); } afk_quiesce(cleanup->ep); + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && + !IS_ERR_OR_NULL(cleanup->ep) && + READ_ONCE(dcp->desktop->graph[cleanup->ep->endpoint - 0x20]) == cleanup->ep) + WRITE_ONCE(dcp->desktop->graph[cleanup->ep->endpoint - 0x20], NULL); /* An old devres group must not unpublish a later session. */ if (dcp->audiosrv == asrv) { if (dcp->avep == cleanup->ep) @@ -923,8 +995,21 @@ int avep_init(struct apple_dcp *dcp) struct platform_device *audio_pdev; struct device *dev = dcp->dev; struct device_node *endpoint, *audio_node = NULL; + const struct apple_epic_service_ops *ops = avep_ops; + u32 av_endpoint = AV_ENDPOINT; int ret; + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) { +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + if (!dcp->external) + return -EOPNOTSUPP; + ops = av_desktop_ops; + av_endpoint = AV_AUDIO_ENDPOINT_DESKTOP; +#else + return -EOPNOTSUPP; +#endif + } + audiosrv_data = devm_kzalloc(dcp->dev, sizeof(*audiosrv_data), GFP_KERNEL); if (!audiosrv_data) return -ENOMEM; @@ -986,7 +1071,7 @@ int avep_init(struct apple_dcp *dcp) device_link_add(&audio_pdev->dev, dev, DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); - dcp->avep = afk_init(dcp, AV_ENDPOINT, avep_ops); + dcp->avep = afk_init(dcp, av_endpoint, ops); if (IS_ERR(dcp->avep)) { ret = PTR_ERR(dcp->avep); if (!av_audiosrv_register_cleanup(dcp, audiosrv_data, dcp->avep)) @@ -997,7 +1082,10 @@ int avep_init(struct apple_dcp *dcp) ret = av_audiosrv_register_cleanup(dcp, audiosrv_data, dcp->avep); if (ret) return ret; - dcp->avep->debugfs_entry = dcp->ep_debugfs[AV_ENDPOINT - 0x20]; + dcp->avep->debugfs_entry = dcp->ep_debugfs[av_endpoint - 0x20]; + /* afk_start waits for RX: publish the desktop route before starting. */ + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) + WRITE_ONCE(dcp->desktop->graph[av_endpoint - 0x20], dcp->avep); ret = afk_start(dcp->avep); if (ret) av_audiosrv_cleanup(&audiosrv_data->cleanup); diff --git a/drivers/gpu/drm/apple/dcp-display-metadata.c b/drivers/gpu/drm/apple/dcp-display-metadata.c index 3a40d288b024c..3be02ae5df3fa 100644 --- a/drivers/gpu/drm/apple/dcp-display-metadata.c +++ b/drivers/gpu/drm/apple/dcp-display-metadata.c @@ -89,26 +89,14 @@ static s64 dcp_metadata_side(const u32 *a, const u32 *b, const u32 *point) ((s64)b[1] - a[1]) * ((s64)point[0] - a[0]); } -/* Build only a base block: the authoritative firmware mode list stays separate. */ -static int dcp_metadata_block(struct edid *edid, const struct dcp_display_metadata *metadata, - const struct drm_display_mode *mode) +/* Validate the complete colour description before changing any EDID bytes. */ +static int dcp_metadata_color(struct edid *edid, const struct dcp_display_metadata *metadata) { - u32 coordinate[8], manufacturer = 0, gamma; - u8 checksum = 0; - int ret; + u32 coordinate[8], gamma; - if (!metadata->width_cm || metadata->width_cm > 255 || - !metadata->height_cm || metadata->height_cm > 255 || - metadata->product > U16_MAX || metadata->week > 53 || - metadata->year < 1990 || metadata->year > 2245 || - metadata->gamma < 65536 || metadata->gamma > (354U << 16) / 100 || - strnlen(metadata->manufacturer, 4) != 3) + if (metadata->gamma && + (metadata->gamma < 65536 || metadata->gamma > (354U << 16) / 100)) return -EINVAL; - for (unsigned int i = 0; i < 3; i++) { - if (metadata->manufacturer[i] < 'A' || metadata->manufacturer[i] > 'Z') - return -EINVAL; - manufacturer = manufacturer << 5 | (metadata->manufacturer[i] - '@'); - } for (unsigned int i = 0; i < 8; i++) { u32 value = i < 6 ? metadata->chromaticity[i / 2][i % 2] : metadata->white[i - 6]; @@ -134,10 +122,48 @@ static int dcp_metadata_block(struct edid *edid, const struct dcp_display_metada return -EINVAL; } } - gamma = DIV_ROUND_CLOSEST(metadata->gamma * 100, 65536) - 100; - if (gamma > 254) + gamma = metadata->gamma ? + DIV_ROUND_CLOSEST(metadata->gamma * 100, 65536) - 100 : 255; + if (metadata->gamma && gamma > 254) return -EINVAL; + edid->gamma = gamma; + edid->red_green_lo = (coordinate[0] & 3) << 6 | (coordinate[1] & 3) << 4 | + (coordinate[2] & 3) << 2 | (coordinate[3] & 3); + edid->blue_white_lo = (coordinate[4] & 3) << 6 | (coordinate[5] & 3) << 4 | + (coordinate[6] & 3) << 2 | (coordinate[7] & 3); + edid->red_x = coordinate[0] >> 2; + edid->red_y = coordinate[1] >> 2; + edid->green_x = coordinate[2] >> 2; + edid->green_y = coordinate[3] >> 2; + edid->blue_x = coordinate[4] >> 2; + edid->blue_y = coordinate[5] >> 2; + edid->white_x = coordinate[6] >> 2; + edid->white_y = coordinate[7] >> 2; + return 0; +} + +/* Build only a base block: the authoritative firmware mode list stays separate. */ +static int dcp_metadata_block(struct edid *edid, const struct dcp_display_metadata *metadata, + const struct drm_display_mode *mode) +{ + u32 manufacturer = 0; + u8 checksum = 0; + int ret; + + if (!metadata->width_cm || metadata->width_cm > 255 || + !metadata->height_cm || metadata->height_cm > 255 || + metadata->product > U16_MAX || metadata->week > 53 || + metadata->year < 1990 || metadata->year > 2245 || + metadata->gamma < 65536 || metadata->gamma > (354U << 16) / 100 || + strnlen(metadata->manufacturer, 4) != 3) + return -EINVAL; + for (unsigned int i = 0; i < 3; i++) { + if (metadata->manufacturer[i] < 'A' || metadata->manufacturer[i] > 'Z') + return -EINVAL; + manufacturer = manufacturer << 5 | (metadata->manufacturer[i] - '@'); + } + memset(edid, 0, sizeof(*edid)); memcpy(edid->header, "\x00\xff\xff\xff\xff\xff\xff\x00", 8); edid->product_id.manufacturer_name = cpu_to_be16(manufacturer); @@ -151,20 +177,10 @@ static int dcp_metadata_block(struct edid *edid, const struct dcp_display_metada edid->input = DRM_EDID_INPUT_DIGITAL; edid->width_cm = metadata->width_cm; edid->height_cm = metadata->height_cm; - edid->gamma = gamma; + ret = dcp_metadata_color(edid, metadata); + if (ret) + return ret; edid->features = DRM_EDID_FEATURE_PREFERRED_TIMING | DRM_EDID_FEATURE_RGB; - edid->red_green_lo = (coordinate[0] & 3) << 6 | (coordinate[1] & 3) << 4 | - (coordinate[2] & 3) << 2 | (coordinate[3] & 3); - edid->blue_white_lo = (coordinate[4] & 3) << 6 | (coordinate[5] & 3) << 4 | - (coordinate[6] & 3) << 2 | (coordinate[7] & 3); - edid->red_x = coordinate[0] >> 2; - edid->red_y = coordinate[1] >> 2; - edid->green_x = coordinate[2] >> 2; - edid->green_y = coordinate[3] >> 2; - edid->blue_x = coordinate[4] >> 2; - edid->blue_y = coordinate[5] >> 2; - edid->white_x = coordinate[6] >> 2; - edid->white_y = coordinate[7] >> 2; memset(edid->standard_timings, 1, sizeof(edid->standard_timings)); ret = dcp_metadata_timing(&edid->detailed_timings[0], mode, metadata); if (!ret) @@ -400,6 +416,8 @@ const struct drm_edid *dcp_panel_edid(const struct dcp_display_metadata *metadat block.white_x = 320 >> 2; block.white_y = 337 >> 2; block.blue_white_lo = (320 & 3) << 2 | (337 & 3); + /* Missing or malformed optional colour data leaves the identity usable. */ + dcp_metadata_color(&block, metadata); block.features = DRM_EDID_FEATURE_PREFERRED_TIMING; memset(block.standard_timings, 1, sizeof(block.standard_timings)); snprintf(name, sizeof(name), "%.3s %04x", metadata->manufacturer, metadata->product); diff --git a/drivers/gpu/drm/apple/dcp-hdr-metadata.h b/drivers/gpu/drm/apple/dcp-hdr-metadata.h new file mode 100644 index 0000000000000..f7590c32b13c8 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-hdr-metadata.h @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * HDR10 static metadata for the DCPAV service of desktop 26.x DCPs. + * + * The display's Dynamic Range and Mastering InfoFrame is link state the + * firmware builds itself: the host sends DCPAV selector 0x15 + * (setHDRStaticMetadata) to the dcpav-service-epic service, an AFK standard + * service command carrying a serialized two-key dictionary, + * { "Type": "CEA-861:Type1", "Data": <24 bytes> }. The 24 bytes are the + * CTA-861.3 Type 1 body in the order of struct hdr_metadata_infoframe from + * display_primaries to max_fall, little-endian. The EOTF is not in it: the + * firmware takes it from the colour mode the link runs. + */ +#ifndef __APPLE_DCP_HDR_METADATA_H__ +#define __APPLE_DCP_HDR_METADATA_H__ + +#include +#include +#include +#include +#include + +#define DCP_HDR_SELECTOR_SET_STATIC 0x15 +#define DCP_HDR_IPC_MAGIC 0x69706378 /* "xcpi" */ +#define DCP_HDR_DATA_SIZE 24 +#define DCP_HDR_TYPE "CEA-861:Type1" + +/* + * The dictionary in the compact object format (header 0xd3, then 32-bit + * tags: bit 31 marks the last object of a collection, bits 24-30 the type, + * the low 24 bits the size; data is padded to 32 bits). The keys are + * symbols (type 8) whose size counts the terminating NUL, the type a string + * (type 9) without one, the data a blob (type 10). + */ +#define DCP_HDR_OBJ_DICT 0x81000002 /* last, dictionary, 2 pairs */ +#define DCP_HDR_OBJ_KEY 0x08000005 /* symbol, "Type\0" / "Data\0" */ +#define DCP_HDR_OBJ_TYPE (0x09000000 | (sizeof(DCP_HDR_TYPE) - 1)) +#define DCP_HDR_OBJ_DATA (0x8a000000 | DCP_HDR_DATA_SIZE) + +#define DCP_HDR_OFF_TYPE_KEY 8 +#define DCP_HDR_OFF_TYPE_VALUE 20 +#define DCP_HDR_OFF_DATA_KEY 40 +#define DCP_HDR_OFF_DATA_VALUE 52 +#define DCP_HDR_OFF_DATA (DCP_HDR_OFF_DATA_VALUE + 4) +#define DCP_HDR_OBJECT_SIZE (DCP_HDR_OFF_DATA + DCP_HDR_DATA_SIZE) + +/* The DCPAV IPC message: a 0x60-byte header, then the serialized object. */ +#define DCP_HDR_MSG_HEADER 0x60 +#define DCP_HDR_MSG_STATUS 0x40 +#define DCP_HDR_MSG_SIZE (DCP_HDR_MSG_HEADER + DCP_HDR_OBJECT_SIZE) + +/* The 24-byte Type 1 body of @hdr; -EINVAL unless it is a Type 1 block. */ +static inline int dcp_hdr_data(u8 data[DCP_HDR_DATA_SIZE], + const struct hdr_output_metadata *hdr) +{ + const struct hdr_metadata_infoframe *t1 = &hdr->hdmi_metadata_type1; + unsigned int i; + + if (hdr->metadata_type || t1->metadata_type) + return -EINVAL; + for (i = 0; i < 3; i++) { + put_unaligned_le16(t1->display_primaries[i].x, data + i * 4); + put_unaligned_le16(t1->display_primaries[i].y, data + i * 4 + 2); + } + put_unaligned_le16(t1->white_point.x, data + 12); + put_unaligned_le16(t1->white_point.y, data + 14); + put_unaligned_le16(t1->max_display_mastering_luminance, data + 16); + put_unaligned_le16(t1->min_display_mastering_luminance, data + 18); + put_unaligned_le16(t1->max_cll, data + 20); + put_unaligned_le16(t1->max_fall, data + 22); + return 0; +} + +/* The selector 0x15 message carrying @data. */ +static inline void dcp_hdr_message(u8 msg[DCP_HDR_MSG_SIZE], + const u8 data[DCP_HDR_DATA_SIZE]) +{ + u8 *obj = msg + DCP_HDR_MSG_HEADER; + + memset(msg, 0, DCP_HDR_MSG_SIZE); + put_unaligned_le16(1, msg + 0x02); + put_unaligned_le32(DCP_HDR_SELECTOR_SET_STATIC, msg + 0x04); + put_unaligned_le32(DCP_HDR_OBJECT_SIZE + 0x20, msg + 0x08); + put_unaligned_le32(DCP_HDR_IPC_MAGIC, msg + 0x0c); + put_unaligned_le32(DCP_HDR_OBJECT_SIZE, msg + 0x50); + + put_unaligned_le32(0xd3, obj); + put_unaligned_le32(DCP_HDR_OBJ_DICT, obj + 4); + put_unaligned_le32(DCP_HDR_OBJ_KEY, obj + DCP_HDR_OFF_TYPE_KEY); + memcpy(obj + DCP_HDR_OFF_TYPE_KEY + 4, "Type", 5); + put_unaligned_le32(DCP_HDR_OBJ_TYPE, obj + DCP_HDR_OFF_TYPE_VALUE); + memcpy(obj + DCP_HDR_OFF_TYPE_VALUE + 4, DCP_HDR_TYPE, + sizeof(DCP_HDR_TYPE) - 1); + put_unaligned_le32(DCP_HDR_OBJ_KEY, obj + DCP_HDR_OFF_DATA_KEY); + memcpy(obj + DCP_HDR_OFF_DATA_KEY + 4, "Data", 5); + put_unaligned_le32(DCP_HDR_OBJ_DATA, obj + DCP_HDR_OFF_DATA_VALUE); + memcpy(obj + DCP_HDR_OFF_DATA, data, DCP_HDR_DATA_SIZE); +} + +/* The result of a selector 0x15 command: transport, AFK, then DCP status. */ +static inline int dcp_hdr_reply(int transport, u32 retcode, const u8 *reply, + size_t length) +{ + if (transport) + return transport; + if (retcode) + return -EIO; + if (length < DCP_HDR_MSG_STATUS + 4) + return -EPROTO; + return get_unaligned_le32(reply + DCP_HDR_MSG_STATUS) ? -EIO : 0; +} + +#endif /* __APPLE_DCP_HDR_METADATA_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 1c08764d767e7..7e234e2bfac43 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -97,6 +97,8 @@ enum { DPAVSERV_ENDPOINT = 0x28, DPAVSERV_ENDPOINT_V2 = 0x2a, AV_ENDPOINT = 0x29, + /* Desktop 26.x separates the audio service from AV device control. */ + AV_AUDIO_ENDPOINT_DESKTOP = 0x2b, DPTX_ENDPOINT = 0x2a, DPTX_ENDPOINT_V2 = 0x2c, HDCP_ENDPOINT = 0x2b, @@ -266,6 +268,23 @@ struct dcp_desktop_state { bool modeset_callback_error; /* An adaptive swap that timed out keeps fixed refresh until the next boot. */ bool vrr_failed; + /* + * Adaptive sync of a VRR timing installed by this pipe alone: the + * minimum rates for VRR_ENABLED on and off (16.16 Hz), and the one + * the firmware was last given. Zero when not installed that way. + */ + u32 vrr_min_on, vrr_min_off, vrr_min_sent; + /* The frame's colour matrix, sent after an adaptive sync update. */ + struct iomfb_set_matrix_req matrix; + /* + * HDR10 static metadata (external processors): the pinned + * dcpav-service-epic service, and the Type 1 body last sent on this + * link (taken or refused), if any. hdr_lock protects both. + */ + spinlock_t hdr_lock; + struct apple_epic_service *av_service; + u8 hdr_sent[24]; + bool hdr_sent_valid; u32 present_id; bool present_pending; /* @@ -898,6 +917,17 @@ int dcp_backlight_update(struct apple_dcp *dcp); s32 dcp_ext_backlight_value(struct apple_dcp *dcp); bool dcp_has_panel(struct apple_dcp *dcp); +/* + * The desktop 26.6 panel takes VRR_ENABLED on its live mode, through the + * adaptive sync minimum rate (see dcp_vrr_update()). The external + * processors ignore a new minimum on a live mode. + */ +static inline bool dcp_vrr_live(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && dcp->main_display && + dcp_has_panel(dcp); +} + /* * Does @dcp's firmware speak the 14.7 AFK, DPTX and DPAV endpoint protocol, * whichever IOMFB path drives it? DCP_FIRMWARE_V_14_7 is the native DCPLink diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 2c3404593e69a..d575a503c6d8b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -548,8 +548,8 @@ void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) } /* - * A desktop 26.x external processor has no dcpav-service-epic to read the - * sink's EDID through, but exports the TimingElements and DisplayAttributes + * A desktop 26.x external processor does not hand out the sink's EDID through + * its DCPAV service, but exports the TimingElements and DisplayAttributes * of every sink it drives, on HDMI and Type-C alike. */ static bool dcp_sink_metadata(struct apple_dcp *dcp) @@ -909,6 +909,12 @@ void dcp_panel_metadata_replace(struct apple_dcp *dcp, merged.serial = old->serial; if (same_serial && !merged.serial_text[0]) memcpy(merged.serial_text, old->serial_text, sizeof(merged.serial_text)); + /* Standalone ProductAttributes setters omit the colour description. */ + if (same_serial && !merged.chromaticity[0][0] && !merged.white[0]) { + memcpy(merged.chromaticity, old->chromaticity, sizeof(merged.chromaticity)); + memcpy(merged.white, old->white, sizeof(merged.white)); + merged.gamma = old->gamma; + } if (!merged.year) merged.year = old->year; if (!merged.week) @@ -1503,8 +1509,14 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commi crtc_state->vrr_enabled) return -EINVAL; - /* The desktop 26.6 firmware applies adaptive sync with a mode set. */ - if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && crtc_state->enable) { + /* + * On desktop 26.6 firmware the panel takes VRR_ENABLED with the next + * frame, without a mode set (see dcp_vrr_update()). The external + * processors ignore a new minimum rate on a live mode, so they still + * change adaptive sync with a mode set. + */ + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && crtc_state->enable && + !dcp_vrr_live(dcp)) { struct drm_crtc_state *old = drm_atomic_get_old_crtc_state(commit, crtc); @@ -3236,13 +3248,21 @@ bool dcp_fw_compat_is_14_7(struct platform_device *pdev) dcp->fw_compat == DCP_FIRMWARE_V_26_6; } -/* GAMMA_LUT entries, 0 where the driver cannot program the gamma table */ +/* + * GAMMA_LUT entries, 0 where the driver cannot program the gamma table. + * + * Desktop 26.x composes the LUT through the firmware's own table (see + * dcp_gamma_set()), but no desktop 26.x DCP hands out a usable one: the + * J616s panel's A420 table does not rise monotonically and the external + * processors return none (status 2). Offering GAMMA_LUT there meant + * accepting a LUT and dropping it, so compositors that move work into it + * got wrong output: KWin's HDR encoding curve went missing (PQ content + * washed out) and night light did nothing. Without the property KWin + * applies its curves while rendering and mutter uses CTM for night light. + */ unsigned int dcp_gamma_lut_size(struct platform_device *pdev) { - struct apple_dcp *dcp = platform_get_drvdata(pdev); - - return dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP ? - IOMFB_GAMMA_ENTRIES : 0; + return 0; } unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev) @@ -3289,6 +3309,55 @@ static const struct apple_epic_service_ops dcp_compact_aux_ops[] = { {}, }; +/* The DCPAV service of an external processor: HDR10 static metadata. */ +static void dcp_av_service_init(struct apple_epic_service *service, + const char *name, const char *class, s64 unit) +{ + struct dcp_desktop_state *d = service->ep->dcp->desktop; + struct apple_epic_service *old; + + service = afk_service_get(service); + if (!service) + return; + scoped_guard(spinlock_irqsave, &d->hdr_lock) { + old = d->av_service; + d->av_service = service; + d->hdr_sent_valid = false; + } + if (old) + afk_service_put(old); +} + +static void dcp_av_service_teardown(struct apple_epic_service *service) +{ + struct dcp_desktop_state *d = service->ep->dcp->desktop; + bool owned; + + scoped_guard(spinlock_irqsave, &d->hdr_lock) { + owned = d->av_service == service; + if (owned) { + d->av_service = NULL; + d->hdr_sent_valid = false; + } + } + if (owned) + afk_service_put(service); +} + +static const struct apple_epic_service_ops dcp_ext_av_service_ops[] = { + { + .name = "dcpav-service-epic", + .reusable = true, + .init = dcp_av_service_init, + .teardown = dcp_av_service_teardown, + }, + { + .name = "*", + .init = dcp_compact_aux_init, + }, + {}, +}; + static void dcp_dpavctrl_service_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) @@ -3438,19 +3507,38 @@ static int dcp_start_desktop_external(struct apple_dcp *dcp, u32 map) ret = dcp_desktop_start_ext_ep(dcp, map, DPSAC_ENDPOINT, "DPSAC"); if (ret) return ret; + ret = dcp_desktop_start_ext_ep(dcp, map, AV_ENDPOINT, "AV"); + if (ret) + return ret; #if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) - if (hdmi_audio) + if (hdmi_audio && (map & BIT(AV_AUDIO_ENDPOINT_DESKTOP - 0x20))) { ret = avep_init(dcp); - else + if (ret) + dcp_desktop_graph_skip(dcp, AV_AUDIO_ENDPOINT_DESKTOP, + "AUDIO", "audio init failed", ret); + else if (dcp->avep) + d->graph_started |= BIT(AV_AUDIO_ENDPOINT_DESKTOP - 0x20); + } #endif - ret = dcp_desktop_start_ext_ep(dcp, map, AV_ENDPOINT, "AV"); - if (ret) - return ret; ret = dcp_desktop_start_ext_ep(dcp, map, REMOTE_ALLOC_ENDPOINT, "REMOTEALLOC"); if (ret) return ret; + /* + * DCPAV service (dcpav-service-epic, HDR10 static metadata). Without + * it a PQ link still works, only without mastering data. + */ + if (map & BIT(DPAVSERV_ENDPOINT_V2 - 0x20)) { + ret = dcp_desktop_start_ep(dcp, DPAVSERV_ENDPOINT_V2, + dcp_ext_av_service_ops); + if (ret) + dcp_desktop_graph_skip(dcp, DPAVSERV_ENDPOINT_V2, + "DPAVSERV", "start failed", ret); + else + d->graph_started |= BIT(DPAVSERV_ENDPOINT_V2 - 0x20); + } + /* An idle Type-C pipeline has no PHY until a route is claimed. */ if (!dcp->phy && !dcp->nr_typec_routes) { dev_warn(dcp->dev, "no DP PHY and no Type-C route: no display link\n"); @@ -3543,10 +3631,6 @@ static int dcp_start_desktop(struct apple_dcp *dcp) if (endpoint == DPAV_CTRL_ENDPOINT) ret = dcp_desktop_start_dpavctrl(dcp); -#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) - else if (endpoint == AV_ENDPOINT && hdmi_audio) - ret = avep_init(dcp); -#endif else ret = dcp_desktop_start_ep(dcp, endpoint, dcp_compact_aux_ops); @@ -3751,6 +3835,8 @@ static void _dcp_poweroff(struct apple_dcp *dcp) break; case DCP_FIRMWARE_V_26_6_DESKTOP: iomfb_poweroff_v26_6_desktop(dcp); + /* The link set up again starts without HDR10 metadata. */ + dcp_hdr_metadata_forget(dcp); break; case DCP_FIRMWARE_V_26_6: iomfb_v26_6_poweroff(dcp); @@ -5550,6 +5636,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) dcp->desktop = devm_kzalloc(dev, sizeof(*dcp->desktop), GFP_KERNEL); if (!dcp->desktop) return -ENOMEM; + spin_lock_init(&dcp->desktop->hdr_lock); /* * The opaque client identity of the IOMFB calls. The firmware's * own client passes a kernel pointer there; pass a random value diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index bcdbf49318a6a..d65367c8af13f 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -76,6 +76,7 @@ int dcp_start(struct platform_device *pdev); int dcp_activate(struct platform_device *pdev); int dcp_wait_ready(struct platform_device *pdev, u64 timeout); void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit); +bool dcp_hdr_metadata_live(struct platform_device *pdev); void dcp_retain_framebuffer(struct platform_device *pdev, struct dcp_fb_reference *entry); void dcp_arm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 76a53d1f91556..8c464d03a0508 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -34,6 +34,7 @@ #include #include "dcp.h" +#include "dcp-hdr-metadata.h" #include "dcp-internal.h" #include "iomfb.h" #include "iomfb_internal.h" @@ -2171,7 +2172,8 @@ static int dcp_desktop_modeset(struct apple_dcp *dcp, if (d->trial_modeset_ok) { d->trial_modeset_ok = false; - if (!crtc_state->vrr_enabled && d->mode_live && + /* The trial installed a VRR timing variable too (see the mode set). */ + if (d->mode_live && lookup_mode(dcp, &crtc_state->mode, &matched) && matched.timing_mode_id == d->trial_timing_mode_id && matched.color_mode_id == d->trial_color_mode_id) { @@ -2354,6 +2356,102 @@ u8 dcp_crtc_eotf(struct apple_dcp *dcp, struct drm_crtc *crtc, return hdr->hdmi_metadata_type1.eotf; } +/* + * HDR10 static metadata is link state on the external processors: a link + * set up again (mode set, power off) starts without it. + */ +void dcp_hdr_metadata_forget(struct apple_dcp *dcp) +{ + struct dcp_desktop_state *d = dcp->desktop; + + if (!d) + return; + scoped_guard(spinlock_irqsave, &d->hdr_lock) + d->hdr_sent_valid = false; +} + +/* Whether @pdev takes a change of HDR_OUTPUT_METADATA without a mode set. */ +bool dcp_hdr_metadata_live(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && + !dcp_has_panel(dcp); +} + +/* + * Send the connector's HDR10 static metadata to the DCPAV service once the + * link carries PQ: after the mode set that selected a PQ colour mode, and + * whenever the metadata changes on it. Never per frame, and never on an + * SDR link (the firmware would build a DRM InfoFrame with the SDR EOTF). + * Going back to SDR is a mode set, and the link set up again carries no + * InfoFrame. + */ +static void dcp_hdr_metadata_sync(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *commit) +{ + struct dcp_desktop_state *d = dcp->desktop; + const struct drm_connector_state *conn_state = NULL; + struct drm_connector_state *state; + struct drm_connector *connector; + struct apple_epic_service *service; + u8 data[DCP_HDR_DATA_SIZE], msg[DCP_HDR_MSG_SIZE]; + u8 reply[DCP_HDR_MSG_SIZE] = {}; + size_t received = 0; + u32 retcode = 0; + bool same; + int i, ret; + + if (!d || !dcp->output_pq || READ_ONCE(dcp->crashed) || + !dcp_hdr_metadata_live(to_apple_crtc(crtc)->dcp)) + return; + for_each_new_connector_in_state(commit, connector, state, i) + if (state->crtc == crtc) + conn_state = state; + if (!conn_state && dcp->connector && dcp->connector->base.state && + dcp->connector->base.state->crtc == crtc) + conn_state = dcp->connector->base.state; + if (!conn_state || !conn_state->hdr_output_metadata || + conn_state->hdr_output_metadata->length != sizeof(struct hdr_output_metadata)) + return; + if (((const struct hdr_output_metadata *)conn_state->hdr_output_metadata->data) + ->hdmi_metadata_type1.eotf != HDMI_EOTF_SMPTE_ST2084 || + dcp_hdr_data(data, conn_state->hdr_output_metadata->data)) + return; + + /* Most frames change nothing: compare before pinning the service. */ + scoped_guard(spinlock_irqsave, &d->hdr_lock) { + same = d->hdr_sent_valid && !memcmp(d->hdr_sent, data, sizeof(data)); + service = same ? NULL : afk_service_get(d->av_service); + } + if (!service) + return; + + dcp_hdr_message(msg, data); + ret = afk_send_command_with_reply_len(service, EPIC_SUBTYPE_STD_SERVICE, + msg, sizeof(msg), reply, sizeof(reply), + &retcode, &received); + ret = dcp_hdr_reply(ret, retcode, reply, received); + /* + * A block the firmware refused is not sent again on every frame (the + * command is synchronous): only a new block, a new mode set or a new + * service instance retries. + */ + if (ret) + dev_warn_ratelimited(dcp->dev, "HDR static metadata not taken: %d\n", ret); + else + dev_dbg(dcp->dev, "HDR static metadata: MaxCLL %u MaxFALL %u mastering %u/%u\n", + get_unaligned_le16(data + 20), get_unaligned_le16(data + 22), + get_unaligned_le16(data + 16), get_unaligned_le16(data + 18)); + scoped_guard(spinlock_irqsave, &d->hdr_lock) { + if (d->av_service == service) { + memcpy(d->hdr_sent, data, sizeof(data)); + d->hdr_sent_valid = true; + } + } + afk_service_put(service); +} + /* Only an SDR or a PQ signal, and PQ only where the timing has an element. */ int dcp_crtc_check_hdr(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_commit *commit) @@ -2517,6 +2615,7 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, dcp->pq_requested = dcp_crtc_eotf(dcp, crtc, commit) == HDMI_EOTF_SMPTE_ST2084; dcp->output_edr = dcp->pq_requested && dcp_panel_edr_supported(dcp); dcp->pq_requested &= dcp_hdr_output_supported(dcp); + dcp_hdr_metadata_forget(dcp); dcp_mode_begin(&dcp->mode_state); switch (dcp->fw_compat) { case DCP_FIRMWARE_V_12_3: @@ -3064,6 +3163,11 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) return; } + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && + READ_ONCE(dcp->mode_state.valid) && dcp->connector && + READ_ONCE(dcp->connector->connected)) + dcp_hdr_metadata_sync(dcp, crtc, commit); + if (iomfb_uses_queue(dcp)) { iomfb_queue_atomic(dcp, crtc, commit); return; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 7c60a81207ad7..7fb06f8d5eea2 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -476,7 +476,8 @@ struct dcp_bool_arg { /* * A420 get_gamma_table / A421 set_gamma_table: three channels (red, green, - * blue) of 257 u32 entries followed by a null flag, 3088 bytes in and out. + * blue) of 257 u32 entries. The request appends an input pointer null flag; + * the reply appends a return status instead. Both are 3088 bytes. */ #define IOMFB_GAMMA_ENTRIES 257 @@ -488,8 +489,7 @@ struct iomfb_gamma_table_req { struct iomfb_gamma_table_resp { __le32 table[3][IOMFB_GAMMA_ENTRIES]; - u8 table_null; - u8 padding[3]; + __le32 ret; } __packed; #endif diff --git a/drivers/gpu/drm/apple/iomfb_internal.h b/drivers/gpu/drm/apple/iomfb_internal.h index feb4562f27a35..b34a1171123e3 100644 --- a/drivers/gpu/drm/apple/iomfb_internal.h +++ b/drivers/gpu/drm/apple/iomfb_internal.h @@ -117,3 +117,4 @@ u8 dcp_crtc_eotf(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_commit *commit); int dcp_crtc_check_hdr(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_commit *commit); +void dcp_hdr_metadata_forget(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index b649590127626..e1cb13e7f8517 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2875,6 +2875,11 @@ static u32 dcp_gamma_lookup(const __le32 *fw, u16 v) static void dcp_gamma_set_done(struct apple_dcp *dcp, void *out, void *cookie) { + struct iomfb_gamma_table_resp *resp = out; + + if (resp) + dev_dbg(dcp->dev, "A421 gamma table status %u\n", + le32_to_cpu(resp->ret)); do_swap(dcp, NULL, NULL); } @@ -2905,23 +2910,31 @@ static void dcp_gamma_read(struct apple_dcp *dcp, void *out, void *cookie) u32 ch, i; /* A table to compose with rises from its first to its last entry. */ - for (ch = 0; resp && ch < 3; ch++) { + for (ch = 0; resp && !le32_to_cpu(resp->ret) && ch < 3; ch++) { for (i = 1; i < IOMFB_GAMMA_ENTRIES; i++) if (le32_to_cpu(resp->table[ch][i]) < le32_to_cpu(resp->table[ch][i - 1])) break; - if (resp->table_null || i < IOMFB_GAMMA_ENTRIES || + if (i < IOMFB_GAMMA_ENTRIES || !le32_to_cpu(resp->table[ch][IOMFB_GAMMA_ENTRIES - 1])) break; } if (!resp || ch < 3) { - dev_warn(dcp->dev, "unusable firmware gamma table, GAMMA_LUT ignored\n"); + /* External firmware can return no table with status 2. */ + if (resp && !dcp_has_panel(dcp) && le32_to_cpu(resp->ret) == 2 && + !memchr_inv(resp->table, 0, sizeof(resp->table))) + dev_dbg(dcp->dev, + "A420 returned no gamma table (status 2), GAMMA_LUT ignored\n"); + else + dev_warn(dcp->dev, "unusable firmware gamma table, GAMMA_LUT ignored\n"); d->gamma_unusable = true; do_swap(dcp, NULL, NULL); return; } - memcpy(d->gamma_fw, resp, sizeof(*d->gamma_fw)); + memset(d->gamma_fw, 0, sizeof(*d->gamma_fw)); + memcpy(d->gamma_fw->table, resp->table, sizeof(resp->table)); d->gamma_fw_read = true; + dev_dbg(dcp->dev, "A420 firmware gamma table read\n"); dcp_gamma_set(dcp); } @@ -3224,9 +3237,19 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, return -ENOMEM; cookie = &modeset->wait; modeset->actual = dcp->mode; - /* After an adaptive swap timed out (dcp_swap_watchdog), stay fixed. */ - modeset->vrr = mode->vrr && crtc_state->vrr_enabled && + /* + * A VRR timing goes in with adaptive sync on whatever VRR_ENABLED + * says, at the mode's minimum rate. Changing adaptive sync through a + * mode set tears the link down (the panel goes dark for about 200 ms), + * so VRR_ENABLED only moves the minimum rate on the live mode, see + * dcp_vrr_update(). Only the panel takes a new minimum on a live mode; + * an external processor keeps installing what VRR_ENABLED asks. After + * an adaptive swap timed out (dcp_swap_watchdog), stay fixed. + */ + modeset->vrr = mode->vrr && + (dcp_vrr_live(dcp) || crtc_state->vrr_enabled) && !READ_ONCE(dcp->desktop->vrr_failed); + dcp->desktop->vrr_min_on = 0; cookie->status = ~0U; init_completion(&cookie->done); @@ -3303,6 +3326,18 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, } WRITE_ONCE(dcp->desktop->modeset_active, false); dcp->vrr_enabled = !ret && modeset->vrr; + if (dcp->vrr_enabled && dcp_vrr_live(dcp)) { + /* + * Fixed refresh holds the minimum at the timing's own rate, + * never above the sink's maximum (a 280.001 Hz timing reports + * a maximum just over 280 Hz). + */ + dcp->desktop->vrr_min_on = mode->min_vrr; + dcp->desktop->vrr_min_off = drm_mode_vrefresh(&crtc_state->mode) << 16; + if (mode->max_vrr && mode->max_vrr < dcp->desktop->vrr_min_off) + dcp->desktop->vrr_min_off = mode->max_vrr; + dcp->desktop->vrr_min_sent = mode->min_vrr; + } kref_put(&cookie->refcount, release_wait_cookie); dcp->desktop->mode_live = !ret; @@ -3464,6 +3499,45 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, } #endif +#if DCP_FW_DESKTOP_26(DCP_FW_VER) +/* + * VRR_ENABLED on a live VRR timing. The firmware takes a new minimum rate + * (IOMFBParameter 14) without a mode set: the mode's own minimum lets the + * display follow the frames, the nominal rate holds it fixed. Sent ahead + * of the frame's colour updates and swap, which @next continues with. + * Returns true when it took over the frame. + */ +static bool dcp_vrr_update(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state, + dcp_callback_t next) +{ + struct dcp_desktop_state *d = dcp->desktop; + struct dcp_set_parameter_dcp param = { + .param = IOMFBPARAM_ADAPTIVE_SYNC, + .count = 3, + }; + u32 min; + + if (!dcp->vrr_enabled || !d->vrr_min_on) + return false; + min = crtc_state->vrr_enabled ? d->vrr_min_on : d->vrr_min_off; + if (min == d->vrr_min_sent) + return false; + d->vrr_min_sent = min; + /* minRR (16.16 Hz), then mediaTargetRate and the fractional rate */ + param.value[0] = min; + dev_dbg(dcp->dev, "adaptive sync minimum %#x (VRR_ENABLED %u)\n", min, + crtc_state->vrr_enabled); + dcp_set_parameter_dcp(dcp, false, ¶m, next, NULL); + return true; +} + +/* After an adaptive sync update: the frame's colour matrix, gamma and swap. */ +static void dcp_matrix_then_swap(struct apple_dcp *dcp, void *out, void *cookie) +{ + iomfb_set_matrix(dcp, false, &dcp->desktop->matrix, dcp_gamma_then_swap, NULL); +} +#endif + void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_commit *commit) { struct drm_plane *plane; @@ -3660,12 +3734,20 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru iomfb_set_matrix(dcp, false, &mat, dcp_pre_swap_a428, NULL); #elif DCP_FW_DESKTOP_26(DCP_FW_VER) dcp->desktop->gamma_pending = dcp_gamma_update(dcp, crtc_state); + dcp->desktop->matrix = mat; + if (dcp_vrr_update(dcp, crtc_state, dcp_matrix_then_swap)) + return; iomfb_set_matrix(dcp, false, &mat, dcp_gamma_then_swap, NULL); #else iomfb_set_matrix(dcp, false, &mat, do_swap, NULL); #endif - } else + } else { +#if DCP_FW_DESKTOP_26(DCP_FW_VER) + if (dcp_vrr_update(dcp, crtc_state, do_swap)) + return; +#endif do_swap(dcp, NULL, NULL); + } } static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index ca0883907a7d6..fff8b3465c60c 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -1380,7 +1380,8 @@ static int parse_metadata_number(struct dcp_parse_ctx *handle, u32 *out, u32 min return ret; } -static int parse_metadata_point(struct dcp_parse_ctx *handle, u32 point[2], u32 *gamma) +static int parse_metadata_point(struct dcp_parse_ctx *handle, u32 point[2], u32 *gamma, + bool optional_gamma) { struct iterator it; unsigned int seen = 0; @@ -1419,7 +1420,8 @@ static int parse_metadata_point(struct dcp_parse_ctx *handle, u32 point[2], u32 * A white point is a chromaticity. A primary is passed on as the * sink reports it: real EDIDs carry primaries with x + y above 1. */ - if (seen != (gamma ? 7 : 3) || (gamma && point[0] + point[1] > 65536)) + if ((seen & 3) != 3 || (gamma && !optional_gamma && !(seen & 4)) || + (gamma && point[0] + point[1] > 65536)) return -EINVAL; return 0; } @@ -1448,7 +1450,7 @@ static int parse_metadata_chromaticity(struct dcp_parse_ctx *handle, if (seen & 1U << color) return -EINVAL; seen |= 1U << color; - ret = parse_metadata_point(handle, out->chromaticity[color], NULL); + ret = parse_metadata_point(handle, out->chromaticity[color], NULL, false); } else { ret = skip(handle); } @@ -1667,7 +1669,7 @@ int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_meta if (seen & 8) return -EINVAL; seen |= 8; - ret = parse_metadata_point(handle, parsed.white, &parsed.gamma); + ret = parse_metadata_point(handle, parsed.white, &parsed.gamma, false); } else if (!strcmp(key, "ProductAttributes")) { if (seen & 16) return -EINVAL; @@ -1709,7 +1711,8 @@ int parse_panel_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metada { struct dcp_display_metadata parsed = {}; struct iterator it; - bool product = false; + bool product = false, chromaticity = false, white = false; + unsigned int color_seen = 0; int ret = iterator_begin(handle, &it, true); if (ret) @@ -1725,6 +1728,21 @@ int parse_panel_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metada return -EINVAL; product = true; ret = parse_metadata_product(handle, &parsed, true); + } else if (!strcmp(key, "Chromaticity")) { + struct dcp_parse_ctx color = *handle; + + /* Optional colour data must not invalidate panel identity. */ + chromaticity = !parse_metadata_chromaticity(&color, &parsed) && + !(color_seen & 1); + color_seen |= 1; + ret = skip(handle); + } else if (!strcmp(key, "DefaultWhitePoint")) { + struct dcp_parse_ctx color = *handle; + + white = !parse_metadata_point(&color, parsed.white, &parsed.gamma, true) && + !(color_seen & 2); + color_seen |= 2; + ret = skip(handle); } else { ret = skip(handle); } @@ -1733,6 +1751,12 @@ int parse_panel_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metada } if (!product) return -ENODATA; + /* Never combine incomplete or duplicate descriptions with a nominal white. */ + if (!chromaticity || !white) { + memset(parsed.chromaticity, 0, sizeof(parsed.chromaticity)); + memset(parsed.white, 0, sizeof(parsed.white)); + parsed.gamma = 0; + } *out = parsed; return 0; } diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 3903ee68b162b..647f8f37507d4 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -9,7 +9,7 @@ config RUST_DRM_GPUVM select DRM_GPUVM config DRM_ASAHI - tristate "Asahi (DRM support for Apple AGX GPUs)" + tristate "Aurora (DRM support for Apple AGX GPUs)" depends on RUST depends on DRM=y depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) @@ -23,7 +23,10 @@ config DRM_ASAHI select RUST_APPLE_RTKIT select CRYPTO_LIB_SHA256 help - DRM driver for Apple AGX GPUs (G13x, found in the M1 SoC family) + DRM driver for Apple AGX GPUs, from the M1 family onwards. + + To compile this driver as a module, choose M here: the module will + be called aurora. It keeps "asahi" as a module alias. config DRM_ASAHI_G16 bool "Apple M4 (G16G) support (experimental)" @@ -31,8 +34,8 @@ config DRM_ASAHI_G16 depends on APPLE_PMP=y || APPLE_PMP=DRM_ASAHI depends on THERMAL_OF help - Rust Asahi backend for the qualified J713/T8132 GPU firmware. - Load asahi explicitly after the native display and GFX TVM supplier + Rust backend for the qualified J713/T8132 GPU firmware. + Load aurora explicitly after the native display and GFX TVM supplier are ready. Firmware-visible storage remains retained until a verified retirement or reboot. Keep the working kernel available during bring-up. A rendering VM pins about 0.7 GiB; memory-cgroup limits are advisable. diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index fd8421f27590d..db13d06336b3c 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,9 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o +obj-$(CONFIG_DRM_ASAHI) += aurora.o +aurora-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o g17/clpc.o CFLAGS_g17/trace.o := -I$(src)/g17 -asahi-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o +aurora-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o -asahi-$(CONFIG_DRM_ASAHI_G16) += g16/glue.o +aurora-$(CONFIG_DRM_ASAHI_G16) += g16/glue.o diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs index d815fdf1040ac..10e367b1a15fa 100644 --- a/drivers/gpu/drm/asahi/alloc.rs +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -162,14 +162,14 @@ impl Drop for GenericAlloc { // the invariant, and greater than debug_len as checked above. unsafe { let p = p.as_ptr().add(self.debug_offset - debug_len); - (p as *mut u32).write(STATE_DEAD); + p.cast::().write(STATE_DEAD); } } } if debug_enabled(DebugFlags::FillAllocations) { if let Some(p) = self.ptr() { // SAFETY: Writing to our inner base pointer with our known inner size is safe. - unsafe { (p.as_ptr() as *mut u8).write_bytes(0xde, self.size()) }; + unsafe { p.as_ptr().cast::().write_bytes(0xde, self.size()) }; } } if self.padding != 0 { @@ -178,7 +178,7 @@ impl Drop for GenericAlloc { // the inner base pointer, after `size()` bytes. let guard = unsafe { core::slice::from_raw_parts( - (p.as_ptr() as *mut u8 as *const u8).add(self.size()), + p.as_ptr().cast::().cast_const().add(self.size()), self.padding, ) }; @@ -313,8 +313,8 @@ pub(crate) trait Allocator { unsafe { let p = p.as_ptr(); p.write_bytes(0x42, debug_offset - 2 * debug_len); - let cur = p.add(debug_offset - debug_len) as *mut AllocDebugData; - let prev = p.add(debug_offset - 2 * debug_len) as *mut AllocDebugData; + let cur = p.add(debug_offset - debug_len).cast::(); + let prev = p.add(debug_offset - 2 * debug_len).cast::(); prev.copy_from(cur, 1); cur.copy_from(&debug, 1); }; @@ -344,7 +344,7 @@ pub(crate) trait Allocator { if debug_enabled(DebugFlags::FillAllocations) { if let Some(p) = ret.ptr() { // SAFETY: Writing to our inner base pointer with our known inner size is safe. - unsafe { (p.as_ptr() as *mut u8).write_bytes(0xaa, ret.size()) }; + unsafe { p.as_ptr().cast::().write_bytes(0xaa, ret.size()) }; } } @@ -354,7 +354,7 @@ pub(crate) trait Allocator { // the inner base pointer, after `size()` bytes. let guard = unsafe { core::slice::from_raw_parts_mut( - (p.as_ptr() as *mut u8).add(ret.size()), + p.as_ptr().cast::().add(ret.size()), padding, ) }; @@ -536,7 +536,7 @@ impl Allocator for SimpleAllocator { ); let mut obj = crate::gem::new_kernel_object(&self.dev, size_aligned)?; - let p = obj.vmap()?.addr as *mut u8; + let p = obj.vmap()?.addr.cast::(); if debug_enabled(DebugFlags::FillAllocations) { let obj_size = obj.size(); if let Ok(vmap) = obj.vmap() { @@ -990,7 +990,7 @@ impl HeapAllocator { let (obj_start, obj_size, p) = self.mm.with_inner(|inner| -> Result<_> { let idx = idx.unwrap_or(inner.backing_objects.len() - 1); let obj = &mut inner.backing_objects[idx]; - let p = obj.0.vmap()?.addr as *mut u8; + let p = obj.0.vmap()?.addr.cast::(); Ok((obj.2, obj.0.size(), p)) })?; assert!(obj_start <= start); diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index e6b4d0c9a088f..42843e3459217 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -4,6 +4,7 @@ //! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. mod alloc; +mod binding; mod buffer; mod channel; mod cleanup; @@ -12,21 +13,21 @@ mod driver; mod event; mod file; mod float; -mod fwid; mod fw; -mod gem; -mod g17; +mod fwid; #[cfg(CONFIG_DRM_ASAHI_G15)] mod g15; +mod g17; +mod gem; #[cfg(CONFIG_DRM_ASAHI_G16)] mod g16; mod gpu; mod hw; mod initdata; +mod m3; mod mem; mod memacct; -mod m3; mod microseq; mod mmu; mod object; @@ -37,6 +38,7 @@ mod slotalloc; mod stats; mod sysfs_exports; mod timestamp; +mod usage; mod util; #[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] mod vm; @@ -84,11 +86,13 @@ struct DriverModule { impl kernel::InPlaceModule for DriverModule { fn init(module: &'static kernel::ThisModule) -> impl PinInit { + // SAFETY: Module init runs once, before either adapter is registered below. + unsafe { binding::init() }; try_pin_init!(Self { // Initialize the queue first, before registration can invoke probe. _cleanup: cleanup::QueueOwner::new()?, #[cfg(CONFIG_DRM_ASAHI_G15)] - _g15 <- kernel::driver::Registration::new(c"asahi-g15", module), + _g15 <- kernel::driver::Registration::new(c"aurora-g15", module), _driver <- kernel::driver::Registration::new( ::NAME, module, @@ -103,9 +107,10 @@ impl kernel::InPlaceModule for DriverModule { module! { type: DriverModule, - name: "asahi", + name: "aurora", description: "AGX GPU driver for Apple silicon SoCs", license: "Dual MIT/GPL", + alias: ["asahi"], params: { debug_flags: u64 { default: 0, diff --git a/drivers/gpu/drm/asahi/binding.rs b/drivers/gpu/drm/asahi/binding.rs new file mode 100644 index 0000000000000..5e0fac1f29ab2 --- /dev/null +++ b/drivers/gpu/drm/asahi/binding.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scoped DMA-device access during probe and runtime mapping preparation. +//! +//! The platform owner closes admission and joins leases before unbind returns. +//! VMs retain the gate, so an old VM cannot acquire access after removal/rebind. + +use crate::driver::AsahiDrmDevice; +use kernel::{ + device::{Bound, Device}, + new_condvar, new_mutex, + prelude::*, + sync::{aref::ARef, Arc, CondVar, Mutex}, +}; + +kernel::sync::global_lock! { + // SAFETY: Module initialization initializes this lock before registering adapters. + unsafe(uninit) static DEVICES: Mutex>> = KVec::new(); +} + +/// Initialize the registry before a platform adapter can probe. +/// +/// # Safety +/// +/// Called exactly once per module instance, before any other binding operation. +pub(crate) unsafe fn init() { + // SAFETY: The module initializer establishes the single-call ordering above. + unsafe { DEVICES.init() }; +} + +struct State { + live: bool, + users: usize, +} + +#[pin_data] +pub(crate) struct Gate { + key: usize, + dev: ARef, + #[pin] + state: Mutex, + #[pin] + changed: CondVar, +} + +/// The platform binding's owner; Drop also covers errors during probe. +pub(crate) struct Owner(Arc); + +impl Owner { + /// Register while probe supplies the parent bound-device proof. + /// + /// # Safety + /// + /// The platform adapter must call close before unbind returns, independently + /// of this owner's Drop. On probe error the owner must be closed/dropped + /// before the bound probe callback returns. + pub(crate) unsafe fn new(dev: &Device, drm: &AsahiDrmDevice) -> Result { + let gate = Arc::pin_init( + pin_init!(Gate { + key: core::ptr::from_ref(drm) as usize, + dev: dev.into(), + state <- new_mutex!(State { live: true, users: 0 }, "asahi::binding::state"), + changed <- new_condvar!("asahi::binding::changed"), + }), + GFP_KERNEL, + )?; + DEVICES.lock().push(gate.clone(), GFP_KERNEL)?; + Ok(Self(gate)) + } + + /// Called from platform unbind before devres releases the parent's state. + pub(crate) fn close(&self) { + DEVICES.lock().retain(|gate| !Arc::ptr_eq(gate, &self.0)); + let mut state = self.0.state.lock(); + state.live = false; + while state.users != 0 { + self.0.changed.wait(&mut state); + } + } +} + +impl Drop for Owner { + fn drop(&mut self) { + self.close(); + } +} + +/// Look up the binding for this DRM instance, not just its reusable parent address. +pub(crate) fn lookup(drm: &AsahiDrmDevice) -> Result> { + let key = core::ptr::from_ref(drm) as usize; + DEVICES + .lock() + .iter() + .find(|gate| gate.key == key) + .cloned() + .ok_or(ENODEV) +} + +impl Gate { + /// Covers only SG-table acquisition/segment preparation, not firmware waits. + pub(crate) fn enter(&self) -> Result> { + let mut state = self.state.lock(); + if !state.live { + return Err(ENODEV); + } + state.users = state.users.checked_add(1).ok_or(EOVERFLOW)?; + Ok(Lease(self)) + } +} + +pub(crate) struct Lease<'a>(&'a Gate); + +impl Lease<'_> { + pub(crate) fn device(&self) -> &Device { + // SAFETY: Owner starts during bound probe, rejects new leases on close, + // and joins existing leases before unbind returns. This borrow cannot + // outlive its lease; the ARef also retains the device allocation. + unsafe { self.0.dev.as_bound() } + } +} + +impl Drop for Lease<'_> { + fn drop(&mut self) { + let mut state = self.0.state.lock(); + state.users -= 1; + if state.users == 0 { + self.0.changed.notify_all(); + } + } +} diff --git a/drivers/gpu/drm/asahi/cleanup.rs b/drivers/gpu/drm/asahi/cleanup.rs index e3dd5ad95dc0c..52436b40304c0 100644 --- a/drivers/gpu/drm/asahi/cleanup.rs +++ b/drivers/gpu/drm/asahi/cleanup.rs @@ -33,7 +33,7 @@ impl QueueOwner { // SAFETY: The name is NUL terminated and contains no format specifiers. let raw = unsafe { bindings::alloc_workqueue_noprof( - c_str!("asahi-ts-cleanup").as_char_ptr(), + c_str!("aurora-ts-cleanup").as_char_ptr(), bindings::wq_flags_WQ_UNBOUND | bindings::wq_flags_WQ_MEM_RECLAIM, 0, ) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index ab68573881659..a89728cc2f2b3 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -45,6 +45,7 @@ pub(crate) struct AsahiPlatformDriver; // #[pin_data(PinnedDrop)] pub(crate) struct AsahiPlatformDriverData<'bound> { + binding: crate::binding::Owner, _device: ARef, _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. @@ -78,12 +79,18 @@ pub(crate) type AsahiDevRef = ARef; pub(crate) type AsahiRegData<'drm> = AsahiDrmRegistrationData<'drm>; /// DRM Driver metadata +/// +/// The major version names the UAPI generation in include/uapi/drm/asahi_drm.h +/// and only changes if that ABI ever breaks. The minor version goes up with +/// each new ioctl, flag or feature bit, so userspace can tell what a kernel +/// accepts without probing. The patch level marks fixes userspace may need to +/// detect. const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { - major: 0, + major: 1, minor: 0, patchlevel: 0, - name: c"asahi", - desc: c"Apple AGX Graphics", + name: c"aurora", + desc: c"Aurora GPU for Apple AGX", }; /// DRM Driver implementation for `AsahiDriver`. @@ -220,14 +227,12 @@ impl platform::Driver for AsahiPlatformDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; - fn unbind<'bound>( - _pdev: &'bound platform::Device>, - this: Pin<&Self::Data<'bound>>, - ) { + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { if let Some(gpu) = this.g17.as_ref() { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); } + this.close_binding(); } /// Device probe function. @@ -246,6 +251,7 @@ impl platform::Driver for AsahiPlatformDriver { ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), }; + // SAFETY: The selected SoC's UAT output width bounds all device DMA. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; @@ -262,6 +268,9 @@ impl platform::Driver for AsahiPlatformDriver { .required_by(pdev.as_ref())?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { @@ -310,6 +319,7 @@ impl platform::Driver for AsahiPlatformDriver { )?; let driver = AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, @@ -334,6 +344,9 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::g15::Gpu::new(pdev, &drm, cfg)?; let data = try_pin_init!(AsahiDrmRegistrationData { gpu: gpu as Arc, @@ -343,6 +356,7 @@ impl AsahiPlatformDriver { // G15 retains its firmware-visible state independently until reboot, even on error. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, @@ -363,6 +377,9 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res)?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { @@ -378,9 +395,14 @@ impl AsahiPlatformDriver { return Err(error); } }; - // G17's firmware channel format has no supported stats decoder yet. + let stats = gpu::Gpu::stats_snapshot(&*gpu); let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { + let snapshot = stats.as_ref().map_or(core::ptr::null(), Arc::as_ptr); + let raw_stats = match crate::sysfs_exports::register_g17( + raw_dev, + snapshot, + module_parameters::stats_export.value() != 0, + ) { Ok(handle) => handle, Err(error) => { gpu.shutdown(); @@ -390,9 +412,14 @@ impl AsahiPlatformDriver { if let Err(error) = crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) { - dev_warn!(pdev.as_ref(), "idle_off_delay_ms not registered: {:?}\n", error); + dev_warn!( + pdev.as_ref(), + "idle_off_delay_ms not registered: {:?}\n", + error + ); } Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: Some(gpu), @@ -400,7 +427,7 @@ impl AsahiPlatformDriver { g16: None, raw_dev, raw_stats, - stats: None, + stats, }) } } @@ -416,6 +443,9 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G16Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::g16::Gpu::new(pdev, &drm, cfg, res)?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { @@ -427,6 +457,7 @@ impl AsahiPlatformDriver { let raw_dev = pdev.as_ref().as_raw(); let raw_stats = crate::sysfs_exports::register(raw_dev, core::ptr::null(), false)?; Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, @@ -438,8 +469,15 @@ impl AsahiPlatformDriver { } } +impl AsahiPlatformDriverData<'_> { + pub(crate) fn close_binding(&self) { + self.binding.close(); + } +} + impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { + self.binding.close(); // `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 01598b691ded3..244dca39e41ac 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,8 +16,8 @@ use crate::{ AsahiDrmDevice, // }, gem, - memacct, gpu, + memacct, mmu, queue, util::{ @@ -132,11 +132,16 @@ pub(crate) struct SyncItem { } impl SyncItem { - fn parse_one(file: &DrmFile, mut data: uapi::drm_asahi_sync, out: bool, - bundled: bool) -> Result - { + fn parse_one( + file: &DrmFile, + mut data: uapi::drm_asahi_sync, + out: bool, + bundled: bool, + ) -> Result { if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { - if !out || !bundled || data.timeline_value == 0 { return Err(EINVAL); } + if !out || !bundled || data.timeline_value == 0 { + return Err(EINVAL); + } data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; } match data.sync_type { @@ -210,15 +215,15 @@ impl SyncItem { let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; let mut anchor = None; - // SAFETY: We only read this once, so there are no TOCTOU issues. - let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); + // Read each record once so validation and use see the same retained bytes. + let mut reader = UserSlice::new(UserPtr::from_addr(ptr as usize), size).reader(); for i in 0..count { let mut sync: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE reader.read_raw(unsafe { - core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) + core::slice::from_raw_parts_mut(sync.as_mut_ptr().cast::>(), STRIDE) })?; // SAFETY: All bit patterns in the struct are valid @@ -230,12 +235,16 @@ impl SyncItem { } anchor = Some(i as usize); } - vec.push(Self::parse_one(file, sync, i >= in_count, - allow_cumulative)?, GFP_KERNEL)?; + vec.push( + Self::parse_one(file, sync, i >= in_count, allow_cumulative)?, + GFP_KERNEL, + )?; } if let Some(anchor) = anchor { let bundle = crate::g17::sync_output::SignalBundle::new( - &vec[in_count as usize..], anchor - in_count as usize)?; + &vec[in_count as usize..], + anchor - in_count as usize, + )?; vec[anchor].cumulative = Some(bundle); } else { for sync in &mut vec[in_count as usize..] { @@ -268,6 +277,8 @@ pub(crate) struct File { objects: xarray::XArray>, /// GPU memory this client created and holds. mem: Arc, + /// Completed engine time, retained by queues after handle destruction. + usage: Arc, } /// Convenience type alias for our DRM `File` type. @@ -288,9 +299,10 @@ impl drm::file::DriverFile for File { let id = gpu.ids().file.next(); let user_range = gpu.user_range(); let mem = memacct::Client::new()?; + let usage = Arc::new(crate::usage::Client::new(), GFP_KERNEL)?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - KBox::pin_init(File::new(id, user_range, mem), GFP_KERNEL) + KBox::pin_init(File::new(id, user_range, mem, usage), GFP_KERNEL) }) } @@ -304,6 +316,17 @@ impl drm::file::DriverFile for File { }) } + fn engine_stats(&self, mut emit: impl FnMut(&CStr, u64)) { + let names = [ + kernel::c_str!("vertex"), + kernel::c_str!("fragment"), + kernel::c_str!("compute"), + ]; + for (name, ns) in names.into_iter().zip(self.usage.snapshot()) { + emit(name, ns); + } + } + fn as_raw(&self) -> *mut bindings::drm_file { todo!() } @@ -317,7 +340,11 @@ impl File { id: u64, user_range: Range, mem: Arc, + usage: Arc, ) -> impl PinInit { + // SAFETY: The initializer supplies aligned, pinned storage for Self. + // XArray::new is infallible; each pinned array is initialized in place, + // then every remaining field is written before returning success. unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); @@ -343,6 +370,7 @@ impl File { (*slot).id = id; (*slot).user_range = user_range; addr_of_mut!((*slot).mem).write(mem); + addr_of_mut!((*slot).usage).write(usage); Ok(()) }) } @@ -437,13 +465,13 @@ impl File { let size = core::mem::size_of::().min(data.size.try_into()?); - // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. + // Write the retained result to this user slice once. let mut params_writer = UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { - core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) + core::slice::from_raw_parts(core::ptr::from_ref(¶ms).cast::(), size) })?; Ok(0) @@ -698,22 +726,28 @@ impl File { }; let size = gem::checked_object_size(data.size.try_into()?)?; - let commit = memacct::LEDGER.admit(&file.inner().mem, size as u64).inspect_err(|_| { - cls_pr_debug!( - Errors, - "gem_create: {:#x} bytes refused by the GPU memory limits\n", - size - ); - })?; + let commit = memacct::LEDGER + .admit(&file.inner().mem, size as u64) + .inspect_err(|_| { + cls_pr_debug!( + Errors, + "gem_create: {:#x} bytes refused by the GPU memory limits\n", + size + ); + })?; // Large G17 user objects are populated in full by the first VM_BIND. // Do not multiply direct reclaim/compaction latency by their folio // count. Userspace can evict its BO cache and retry after ENOMEM. - let pages_no_reclaim = gem::user_pages_no_reclaim( - size, - reg_data.gpu.as_any().is::(), - ); + let pages_no_reclaim = + gem::user_pages_no_reclaim(size, reg_data.gpu.as_any().is::()); let gem = gem::new_object( - device, size, data.flags, resv_obj, reclaim_cpu, pages_no_reclaim, commit, + device, + size, + data.flags, + resv_obj, + reclaim_cpu, + pages_no_reclaim, + commit, )?; if reg_data.gpu.prepare_new_objects() { #[cfg(CONFIG_DRM_ASAHI_G15)] @@ -1306,15 +1340,21 @@ impl File { if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH && !reg_data.gpu.as_any().is::() { - cls_pr_debug!(Errors, "queue_create: HIGH priority is not supported on this GPU\n"); + cls_pr_debug!( + Errors, + "queue_create: HIGH priority is not supported on this GPU\n" + ); return Err(EINVAL); } - // SAFETY: capable() only reads the calling task's credentials (ioctl process context). if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as _) } + // SAFETY: capable() reads the current ioctl task's credentials. + && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as i32) } && !file.is_current_master() { - cls_pr_debug!(Errors, "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n"); + cls_pr_debug!( + Errors, + "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n" + ); return Err(EACCES); } @@ -1337,6 +1377,7 @@ impl File { // TODO: Plumb deeper the enum uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority, data.usc_exec_base, + file.inner().usage.clone(), )?; queue.reserve_render_scs(data.render_scs_layout)?; @@ -1425,9 +1466,13 @@ impl File { if guarded { copy_commands(&mut vec)?; } - let syncs = - SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count, - gpu.as_any().is::())?; + let syncs = SyncItem::parse_array( + file, + data.syncs, + data.in_sync_count, + data.out_sync_count, + gpu.as_any().is::(), + )?; mod_dev_dbg!( device, diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index 224e92c6d97e2..52261d10757d1 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -140,12 +140,12 @@ macro_rules! from_ints { ($u:ty, $i:ty) => { impl From<$i> for F32 { fn from(v: $i) -> F32 { - F32U::from_i64(v as i64).pack() + F32U::from_i64(i64::from(v)).pack() } } impl From<$u> for F32 { fn from(v: $u) -> F32 { - F32U::from_u64(v as u64).pack() + F32U::from_u64(u64::from(v)).pack() } } }; diff --git a/drivers/gpu/drm/asahi/fw/event.rs b/drivers/gpu/drm/asahi/fw/event.rs index 52bc456f58707..e26bc6ceb3448 100644 --- a/drivers/gpu/drm/asahi/fw/event.rs +++ b/drivers/gpu/drm/asahi/fw/event.rs @@ -73,7 +73,7 @@ pub(crate) mod raw { // We could use fetch_add, but the non-LSE atomic // sequence Rust produces confuses the hypervisor. let v = self.0.load(Ordering::Relaxed); - self.0.store(v + (amount as u64), Ordering::Relaxed); + self.0.store(v + u64::from(amount), Ordering::Relaxed); } } diff --git a/drivers/gpu/drm/asahi/fw/job.rs b/drivers/gpu/drm/asahi/fw/job.rs index e4f2f9225ea05..aaa83355b437d 100644 --- a/drivers/gpu/drm/asahi/fw/job.rs +++ b/drivers/gpu/drm/asahi/fw/job.rs @@ -139,10 +139,6 @@ pub(crate) struct UserTimestamps { } impl UserTimestamps { - pub(crate) fn any(&self) -> bool { - self.start.is_some() || self.end.is_some() - } - pub(crate) fn pointers(&self) -> Result> { Ok(raw::TimestampPointers { start_addr: self diff --git a/drivers/gpu/drm/asahi/fw/types.rs b/drivers/gpu/drm/asahi/fw/types.rs index f55d6cec6b8ca..11def551029f5 100644 --- a/drivers/gpu/drm/asahi/fw/types.rs +++ b/drivers/gpu/drm/asahi/fw/types.rs @@ -117,6 +117,8 @@ macro_rules! default_zeroed { (<$($lt:lifetime),*>, $type:ty) => { impl<$($lt),*> Default for $type { fn default() -> $type { + // SAFETY: The macro's contract requires every field of this + // type to admit the all-zero bit pattern. unsafe { core::mem::zeroed() } } } @@ -126,6 +128,8 @@ macro_rules! default_zeroed { ($type:ty) => { impl Default for $type { fn default() -> $type { + // SAFETY: The macro's contract requires every field of this + // type to admit the all-zero bit pattern. unsafe { core::mem::zeroed() } } } @@ -144,6 +148,7 @@ unsafe impl Zeroable for Pad {} impl Default for Pad { fn default() -> Self { + // SAFETY: The wrapper contains only an array of bytes. unsafe { core::mem::zeroed() } } } @@ -170,6 +175,7 @@ unsafe impl Zeroable for Array {} impl Default for Array { fn default() -> Self { + // SAFETY: Every element admits the all-zero bit pattern through T: Zeroable. unsafe { core::mem::zeroed() } } } diff --git a/drivers/gpu/drm/asahi/fwid.rs b/drivers/gpu/drm/asahi/fwid.rs index be81eac5dfb74..34c65a0d8a097 100644 --- a/drivers/gpu/drm/asahi/fwid.rs +++ b/drivers/gpu/drm/asahi/fwid.rs @@ -50,7 +50,7 @@ impl fmt::Display for Hex<'_> { pub(crate) fn copy_physical(phys: u64, size: usize) -> Result> { // SAFETY: memremap() of a boot-loader range creates a temporary mapping; it has no other // side effects. - let map = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WC as _) }; + let map = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WC as usize) }; if map.is_null() { return Err(ENOMEM); } diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index e5c18a15e0c3b..51b3cfd10d594 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -221,7 +221,7 @@ struct Prefix { fs_ts: bool, } -/// Engine statistics, printed by `asahi-g15/stats`. +/// Engine statistics, printed by `aurora-g15/stats`. #[derive(Default)] pub(crate) struct Stats { pub(crate) pub_ns: u64, @@ -271,7 +271,7 @@ pub(crate) struct Stats { } /// Host-side phase timers of publication and polling, on with `debug_flags` bit 49 -/// (`Debug1`); printed as the `prof:` line of `asahi-g15/stats`. A measurement aid: nothing is +/// (`Debug1`); printed as the `prof:` line of `aurora-g15/stats`. A measurement aid: nothing is /// timed while the flag is off. #[derive(Default)] pub(crate) struct Prof { @@ -391,9 +391,9 @@ pub(crate) struct Engine { pub(crate) stats: Stats, /// Phase timers (debug_flags bit 49). pub(crate) prof: Prof, - /// The `asahi-g15/stats` file. + /// The `aurora-g15/stats` file. pub(crate) debugfs: Option>>>>, - /// The `asahi-g15/pstate` file. + /// The `aurora-g15/pstate` file. pub(crate) debugfs_pstate: Option>>>>, scratch: Option>, } diff --git a/drivers/gpu/drm/asahi/g15/fence.rs b/drivers/gpu/drm/asahi/g15/fence.rs index a959ff00d4bb1..bb85764228070 100644 --- a/drivers/gpu/drm/asahi/g15/fence.rs +++ b/drivers/gpu/drm/asahi/g15/fence.rs @@ -32,7 +32,7 @@ pub(crate) struct JobFenceOps; #[vtable] impl FenceOps for JobFenceOps { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("asahi") + c_str!("aurora") } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { @@ -63,11 +63,11 @@ pub(crate) struct Timeline { impl Timeline { pub(crate) fn new() -> Result> { - let errors = FenceContexts::new(1, c_str!("asahi_g15_refused"), ERROR_KEY)?; + let errors = FenceContexts::new(1, c_str!("aurora_g15_refused"), ERROR_KEY)?; let allocation_failure = error_fence(&errors, ENOMEM)?; Arc::pin_init( try_pin_init!(Self { - jobs: FenceContexts::new(1, c_str!("asahi_g15_queue"), TIMELINE_KEY)?, + jobs: FenceContexts::new(1, c_str!("aurora_g15_queue"), TIMELINE_KEY)?, errors, pending <- new_mutex!(KVec::new(), "g15::Timeline::pending"), allocation_failure, diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index 662c9c8258e21..b949a4b9b61ed 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -267,12 +267,12 @@ void asahi_g15_reprobe(void) const struct device_driver *drv; int ret; - drv = driver_find("asahi-g15", &platform_bus_type); + drv = driver_find("aurora-g15", &platform_bus_type); if (!drv) { - pr_err("asahi-g15: registered adapter not found for activation\n"); + pr_err("aurora-g15: registered adapter not found for activation\n"); return; } ret = driver_attach(drv); if (ret) - pr_err("asahi-g15: activation attach failed: %d\n", ret); + pr_err("aurora-g15: activation attach failed: %d\n", ret); } diff --git a/drivers/gpu/drm/asahi/g15/job.rs b/drivers/gpu/drm/asahi/g15/job.rs index 159eea1dec3bc..5f6669f06d1b0 100644 --- a/drivers/gpu/drm/asahi/g15/job.rs +++ b/drivers/gpu/drm/asahi/g15/job.rs @@ -243,7 +243,7 @@ impl Scheduler { hw::sizes::HORIZON_JOBS, 0, hw::tuning::SCHED_TIMEOUT_MS, - c_str!("asahi_g15"), + c_str!("aurora_g15"), )?; // With one run queue, only the kernel priority is valid. let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 371bc41b147ce..713a1360b5687 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -176,6 +176,12 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { + if let Some(driver) = &this._driver { + driver.close_binding(); + } + } + /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true @@ -210,7 +216,6 @@ impl platform::Driver for Driver<'_> { }, }) } - } /// Firmware starts only after every fallible module/driver registration has succeeded. @@ -436,7 +441,7 @@ impl Gpu { *shared.state.lock() = Some(firmware); // The engine's statistics file (a debugfs failure does not fail the probe). if let Err(error) = shared.start_engine() { - dev_warn!(dev.as_ref(), "G15: no asahi-g15 debugfs ({:?})\n", error); + dev_warn!(dev.as_ref(), "G15: no aurora-g15 debugfs ({:?})\n", error); } let gpu = Arc::new( Gpu { @@ -644,9 +649,10 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { Ok(KBox::new( - queue::Queue::new(self, vm, priority, usc_exec_base)?, + queue::Queue::new(self, vm, priority, usc_exec_base, usage)?, GFP_KERNEL, )?) } diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index cff5d87b2c84e..566995eb342c9 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -54,7 +54,7 @@ impl HrTimerCallback for Shared { } } -/// The `asahi-g15/stats` debugfs file contents. +/// The `aurora-g15/stats` debugfs file contents. fn stats_show(shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { let state = shared.state.lock(); match state.as_deref() { @@ -63,7 +63,7 @@ fn stats_show(shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { } } -/// The `asahi-g15/pstate` file contents. +/// The `aurora-g15/pstate` file contents. fn pstate_show(shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { let state = shared.state.lock(); match state.as_deref() { @@ -160,10 +160,10 @@ impl Shared { self.mod_poll(1); } - /// Starts what the engine needs besides the firmware state: the `asahi-g15/stats` and - /// `asahi-g15/pstate` files. + /// Starts what the engine needs besides the firmware state: the `aurora-g15/stats` and + /// `aurora-g15/pstate` files. pub(crate) fn start_engine(self: &Arc) -> Result { - let dir = debugfs::Dir::new(c_str!("asahi-g15")); + let dir = debugfs::Dir::new(c_str!("aurora-g15")); let file = KBox::pin_init( dir.read_callback_file(c_str!("stats"), self.clone(), &stats_show), GFP_KERNEL, @@ -529,25 +529,38 @@ impl Firmware { let s = &mut self.engine.stats; // Add one stage interval read from internal timestamp cells (start, end). - let mut stage = |o: &super::object::FwObject, off: usize, sum: &mut u64| { - let rd = |x: usize| o.r32(x).unwrap_or(0) as u64; - let start = rd(off + 4) << 32 | rd(off); - let end = rd(off + 12) << 32 | rd(off + 8); + let mut stage = |o: &super::object::FwObject, + off: usize, + sum: &mut u64, + engine: crate::usage::Engine| { + let (Ok(start_lo), Ok(start_hi), Ok(end_lo), Ok(end_hi)) = + (o.r32(off), o.r32(off + 4), o.r32(off + 8), o.r32(off + 12)) + else { + return; + }; + let start = u64::from(start_hi) << 32 | u64::from(start_lo); + let end = u64::from(end_hi) << 32 | u64::from(end_lo); if start == 0 || end < start { return; } + f.job.queue.usage.add(engine, start, end, 24_000_000); *sum += end - start; lo = lo.min(start); hi = hi.max(end); }; for k in 0..f.renders as usize { let p = &scrub.passes[f.claims.pass[k] as usize]; - stage(&p.its, 0, &mut s.ta_ticks); - stage(&p.its, 0x10, &mut s.fs_ticks); + stage(&p.its, 0, &mut s.ta_ticks, crate::usage::Engine::Vertex); + stage( + &p.its, + 0x10, + &mut s.fs_ticks, + crate::usage::Engine::Fragment, + ); } for k in 0..(f.computes as usize).min(f.claims.slot_count as usize) { let c = &scrub.compute[f.claims.slot[k] as usize]; - stage(&c.its, 0, &mut s.cdm_ticks); + stage(&c.its, 0, &mut s.cdm_ticks, crate::usage::Engine::Compute); } if hi > lo { s.span_ticks += hi - lo; @@ -641,7 +654,7 @@ impl Firmware { } } - /// The `asahi-g15/stats` contents. Scripts parse the keys: keep them stable. + /// The `aurora-g15/stats` contents. Scripts parse the keys: keep them stable. pub(crate) fn write_stats(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let s = &self.engine.stats; let slots = &self.slots.stats; diff --git a/drivers/gpu/drm/asahi/g15/power.rs b/drivers/gpu/drm/asahi/g15/power.rs index 58b6636fde383..69680f4636499 100644 --- a/drivers/gpu/drm/asahi/g15/power.rs +++ b/drivers/gpu/drm/asahi/g15/power.rs @@ -13,7 +13,7 @@ //! - +0x2c idle decision enabled (template 1, unchanged). //! //! DVFS is firmware-run on the ADT tables. The state it runs at is read back by -//! `asahi-g15/pstate` ([`pstate`], [`super::Firmware::write_pstate`]): the newest power-state +//! `aurora-g15/pstate` ([`pstate`], [`super::Firmware::write_pstate`]): the newest power-state //! record of the stats ring, and HardwareDataA's state words. mod idle; @@ -50,7 +50,7 @@ impl Firmware { }) } - /// The `asahi-g15/pstate` file ([`pstate::write`]): the stats ring's records so far, those + /// The `aurora-g15/pstate` file ([`pstate::write`]): the stats ring's records so far, those /// it holds unconsumed (read, not consumed), HardwareDataA's words and table A of the /// device tree's operating points. Read-only: nothing firmware-visible is written. pub(crate) fn write_pstate(&self, f: &mut impl fmt::Write) -> fmt::Result { diff --git a/drivers/gpu/drm/asahi/g15/power/pstate.rs b/drivers/gpu/drm/asahi/g15/power/pstate.rs index 411fe22413893..0ea68f02ac180 100644 --- a/drivers/gpu/drm/asahi/g15/power/pstate.rs +++ b/drivers/gpu/drm/asahi/g15/power/pstate.rs @@ -12,7 +12,7 @@ //! An index counts the firmware's DVFS states, 0 (off) to 8; its frequency is the state's //! table-A operating point ([`super::super::board`]), taken from the device tree. //! -//! [`write`] prints the `asahi-g15/pstate` file. Its first line is what a sampler reads: +//! [`write`] prints the `aurora-g15/pstate` file. Its first line is what a sampler reads: //! `pstate freq_khz source `, from the newest power-state record //! seen, or HardwareDataA's current state when the stats ring has carried none. //! @@ -181,7 +181,7 @@ impl fmt::Display for Khz { } } -/// The `asahi-g15/pstate` file: +/// The `aurora-g15/pstate` file: /// /// ```text /// pstate freq_khz source diff --git a/drivers/gpu/drm/asahi/g15/queue.rs b/drivers/gpu/drm/asahi/g15/queue.rs index cc87f1ccbe084..03a8f46c8a0fc 100644 --- a/drivers/gpu/drm/asahi/g15/queue.rs +++ b/drivers/gpu/drm/asahi/g15/queue.rs @@ -42,6 +42,8 @@ pub(crate) struct QueueState { /// Host priority: `DRM_ASAHI_PRIORITY_REALTIME - priority` as `file.rs` passes it (MEDIUM 2, /// LOW 3; `file.rs` refuses HIGH and REALTIME). Smaller is served first. pub(crate) priority: u32, + /// Completed stage time of the DRM file that created this queue. + pub(crate) usage: Arc, /// USC executable window base. pub(crate) usc_base: u64, /// Engine service turn of this queue (least recently served first). @@ -103,7 +105,13 @@ impl Drop for Queue { impl Queue { /// Creates a logical queue. `priority` is `DRM_ASAHI_PRIORITY_REALTIME - priority`, as /// `file.rs` passes it. - pub(crate) fn new(gpu: &Gpu, vm: mmu::Vm, priority: u32, usc_exec_base: u64) -> Result { + pub(crate) fn new( + gpu: &Gpu, + vm: mmu::Vm, + priority: u32, + usc_exec_base: u64, + usage: Arc, + ) -> Result { if priority > uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { return Err(EINVAL); } @@ -129,6 +137,7 @@ impl Queue { let state = Arc::new( QueueState { priority, + usage, usc_base: usc_exec_base, turn: AtomicU64::new(0), owned: [const { AtomicU32::new(0) }; 32], diff --git a/drivers/gpu/drm/asahi/g15/recovery.rs b/drivers/gpu/drm/asahi/g15/recovery.rs index 0827385e92c5f..b81734b1d5f2d 100644 --- a/drivers/gpu/drm/asahi/g15/recovery.rs +++ b/drivers/gpu/drm/asahi/g15/recovery.rs @@ -197,7 +197,7 @@ impl Firmware { self.recovery.armed.is_none() && !self.new_errors() } - /// `(recoveries, hang_kills)` for `asahi-g15/stats`. + /// `(recoveries, hang_kills)` for `aurora-g15/stats`. pub(crate) fn recovery_counts(&self) -> (u64, u64) { (self.recovery.recoveries, self.recovery.hang_kills) } diff --git a/drivers/gpu/drm/asahi/g15/slots.rs b/drivers/gpu/drm/asahi/g15/slots.rs index aae6d8804ee7f..8767eea321a0d 100644 --- a/drivers/gpu/drm/asahi/g15/slots.rs +++ b/drivers/gpu/drm/asahi/g15/slots.rs @@ -376,7 +376,7 @@ impl ClearQueue { // SAFETY: The name is a valid NUL-terminated string the C side copies. let ptr = unsafe { bindings::alloc_ordered_workqueue( - c_str!("asahi-g15-clear").as_char_ptr(), + c_str!("aurora-g15-clear").as_char_ptr(), bindings::wq_flags_WQ_MEM_RECLAIM, ) }; @@ -535,7 +535,7 @@ impl Claims { } } -/// Slot accounting statistics (`asahi-g15/stats`). +/// Slot accounting statistics (`aurora-g15/stats`). #[derive(Copy, Clone, Default)] pub(crate) struct SlotStats { /// Clustering buffers zeroed at claim, pass slots reset at claim, compute slots scrubbed diff --git a/drivers/gpu/drm/asahi/g16/adt.rs b/drivers/gpu/drm/asahi/g16/adt.rs index a34ecc901f615..804703b19cecc 100644 --- a/drivers/gpu/drm/asahi/g16/adt.rs +++ b/drivers/gpu/drm/asahi/g16/adt.rs @@ -94,7 +94,7 @@ fn performance_source(regions: &[Range]) -> Result<(KVVec, u64)> { let (start, size) = source.ok_or(ENODEV)?; // SAFETY: The sole accepted DT source above is the bounded reserved original ADT export. // WB matches its System RAM mapping; no writes are made through this temporary mapping. - let mapping = unsafe { bindings::memremap(start, size, bindings::MEMREMAP_WB as _) }; + let mapping = unsafe { bindings::memremap(start, size, bindings::MEMREMAP_WB as usize) }; if mapping.is_null() { return Err(ENOMEM); } diff --git a/drivers/gpu/drm/asahi/g16/boot.rs b/drivers/gpu/drm/asahi/g16/boot.rs index 450d1c25268ab..e43778bb02a1f 100644 --- a/drivers/gpu/drm/asahi/g16/boot.rs +++ b/drivers/gpu/drm/asahi/g16/boot.rs @@ -147,6 +147,10 @@ unsafe fn shared<'a>(cookie: *mut c_void) -> ArcBorrow<'a, Shared> { } /// RTKit crash callback: latches fail-stop. +/// +/// # Safety +/// +/// `cookie` must be the live Shared cookie registered with these RTKit operations. unsafe extern "C" fn rtkit_crashed(cookie: *mut c_void, _log: *const c_void, _size: usize) { // SAFETY: RTKit passes back the registered cookie. let shared = unsafe { shared(cookie) }; @@ -158,6 +162,10 @@ unsafe extern "C" fn rtkit_crashed(cookie: *mut c_void, _log: *const c_void, _si } /// RTKit message callback. +/// +/// # Safety +/// +/// `cookie` must be the live Shared cookie registered with these RTKit operations. unsafe extern "C" fn rtkit_recv(cookie: *mut c_void, endpoint: u8, message: u64) { // SAFETY: RTKit passes back the registered cookie. let shared = unsafe { shared(cookie) }; @@ -179,6 +187,10 @@ unsafe extern "C" fn rtkit_recv(cookie: *mut c_void, endpoint: u8, message: u64) /// EP 0x20 ring-service request, in the mailbox interrupt: wake only. The drain and the /// backend's completion check run in the poll work; the shared state outlives the mailbox. +/// +/// # Safety +/// +/// `cookie` must be the live Shared cookie registered with these RTKit operations. unsafe extern "C" fn rtkit_recv_early(cookie: *mut c_void, endpoint: u8, message: u64) -> bool { // SAFETY: RTKit passes back the registered cookie. let shared = unsafe { shared(cookie) }; @@ -194,6 +206,11 @@ unsafe extern "C" fn rtkit_recv_early(cookie: *mut c_void, endpoint: u8, message } /// Only the inherited crash buffer inside this session's firmware DATA, once. +/// +/// # Safety +/// +/// `cookie` must be the registered live Shared cookie. `bfr` must point to the +/// writable RTKit buffer descriptor, exclusively borrowed for this callback. unsafe extern "C" fn rtkit_shmem_setup( cookie: *mut c_void, endpoint: u8, @@ -203,7 +220,7 @@ unsafe extern "C" fn rtkit_shmem_setup( let shared = unsafe { shared(cookie) }; // SAFETY: RTKit passes a valid buffer descriptor it owns for the duration of the call. let bfr = unsafe { &mut *bfr }; - let (iova, size) = (bfr.iova as u64, bfr.size as u64); + let (iova, size) = (bfr.iova, bfr.size as u64); let data = &shared.data; let data_size = data.end - data.start; if endpoint != 1 @@ -245,6 +262,10 @@ unsafe extern "C" fn rtkit_shmem_setup( } /// Retained until reboot: no qualified stop path. +/// +/// # Safety +/// +/// This no-op callback accesses neither pointer and releases no storage. unsafe extern "C" fn rtkit_shmem_destroy( _cookie: *mut c_void, _bfr: *mut bindings::apple_rtkit_shmem, @@ -436,8 +457,13 @@ pub(crate) fn admit( } let adt = adt::read(dev).inspect_err(|e| dev_err!(dev, "ADT GPU regions not admitted: {:?}\n", e))?; - crate::fwid::identify(dev, adt.text.clone(), adt.data.end - adt.data.start, cfg.firmware) - .inspect_err(|e| dev_err!(dev, "loaded GFX TEXT identity mismatch: {:?}\n", e))?; + crate::fwid::identify( + dev, + adt.text.clone(), + adt.data.end - adt.data.start, + cfg.firmware, + ) + .inspect_err(|e| dev_err!(dev, "loaded GFX TEXT identity mismatch: {:?}\n", e))?; dev_info!(dev, "M4: firmware identity accepted\n"); // Firmware objects are cleaned to the point of coherency by their DMA mapping: that needs // a device that is not DMA-coherent, as the synthesized node is. diff --git a/drivers/gpu/drm/asahi/g16/encode.rs b/drivers/gpu/drm/asahi/g16/encode.rs index 370f1848e1cb7..a9bf84943b805 100644 --- a/drivers/gpu/drm/asahi/g16/encode.rs +++ b/drivers/gpu/drm/asahi/g16/encode.rs @@ -118,23 +118,27 @@ impl Geometry { /// The geometry of render `u` with layer metadata at `heap`. The render was validated /// (width, height, layers >= 1, samples 1, 2 or 4, utiles 16 or 32). pub(crate) fn new(u: &uapi::drm_asahi_cmd_render, heap: u64) -> Self { - let (w, h, l) = (u.width_px as u32, u.height_px as u32, u.layers as u32); + let (w, h, l) = ( + u32::from(u.width_px), + u32::from(u.height_px), + u32::from(u.layers), + ); let empty = u.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES != 0; - let log2s = (u.samples as u32).max(1).ilog2(); + let log2s = u32::from(u.samples).max(1).ilog2(); let mut g = Geometry::default(); - g.tx = div_round_up(w as u64, 32) as u32; - g.ty = div_round_up(h as u64, 32) as u32; - g.bx = align(div_round_up(g.tx as u64, 4), 4) as u32; - g.by = align(div_round_up(g.ty as u64, 4), 4) as u32; + g.tx = div_round_up(u64::from(w), 32) as u32; + g.ty = div_round_up(u64::from(h), 32) as u32; + g.bx = align(div_round_up(u64::from(g.tx), 4), 4) as u32; + g.by = align(div_round_up(u64::from(g.ty), 4), 4) as u32; g.sx = g.bx * if u.utile_width_px == 16 { 2 } else { 1 }; g.sy = g.by * if u.utile_height_px == 16 { 2 } else { 1 }; - g.e = (4 * g.sx as u64) * (4 * g.sy as u64); + g.e = (4 * u64::from(g.sx)) * (4 * u64::from(g.sy)); g.pixels = w.wrapping_sub(1) | h.wrapping_sub(1) << 16; g.screen = g.tx.wrapping_sub(1) | g.ty.wrapping_sub(1) << 12; - g.x_blocks = (g.bx as u64) << 10 | (g.bx as u64) << 18 | 3 * g.bx as u64; - g.y_blocks = (g.by as u64) << 18 | (g.by as u64) << 10 | 3 * g.by as u64; + g.x_blocks = u64::from(g.bx) << 10 | u64::from(g.bx) << 18 | (3 * u64::from(g.bx)); + g.y_blocks = u64::from(g.by) << 18 | u64::from(g.by) << 10 | (3 * u64::from(g.by)); g.blocks = g.bx * g.by; g.tiles = g.tx * g.ty; g.region_stride = (5 * g.e / 64) as u32; @@ -151,8 +155,8 @@ impl Geometry { 0x4000 }; g.utile_config = utile_w | utile_h | log2s; - g.tilemap_bytes = align(5 * g.e * l as u64, 0x1000); - g.tpc_bytes = g.e * 8 * l as u64; + g.tilemap_bytes = align(5 * g.e * u64::from(l), 0x1000); + g.tpc_bytes = g.e * 8 * u64::from(l); g.layer_metadata = heap + if l > 1 { 0x200 } else { 0 }; g.layer_mode = if l == 1 { 0x8000 @@ -160,21 +164,21 @@ impl Geometry { 0xc000 | (if empty { 0x2000 } else { 0x1000 }) | l.wrapping_sub(1) }; g.tilebuffer_control = div_round_up( - u.sample_size_B as u64 - * u.utile_width_px as u64 - * u.utile_height_px as u64 - * u.samples as u64, + u64::from(u.sample_size_B) + * u64::from(u.utile_width_px) + * u64::from(u.utile_height_px) + * u64::from(u.samples), 2048, ) as u32 | log2s << 17; - g.tile_mode = 0x280 | (l > 1) as u32 | if empty { 1 << 16 } else { 0 }; + g.tile_mode = 0x280 | u32::from(l > 1) | if empty { 1 << 16 } else { 0 }; g.isp_control = 0xc000 | (u.flags & (1 << 18)); g.native_tiles = 0x3717fu64 - | ((l > 1) as u64) << 32 - | (g.tx.wrapping_sub(1) as u64) << 44 - | (g.ty.wrapping_sub(1) as u64) << 53 + | u64::from(l > 1) << 32 + | u64::from(g.tx.wrapping_sub(1)) << 44 + | u64::from(g.ty.wrapping_sub(1)) << 53 | 137438953472u64 - | ((g.utile_config & 0xf000) as u64) << 28; + | u64::from(g.utile_config & 0xf000) << 28; g } } @@ -223,7 +227,7 @@ pub(crate) fn cluster_eligible(_u: &uapi::drm_asahi_cmd_render, _geo: &Geometry) /// Program address. pub(crate) fn prog_addr(usc_base: u64, p: &uapi::drm_asahi_bg_eot) -> u64 { - usc_base.wrapping_add((p.usc & !63) as u64) + usc_base.wrapping_add(u64::from(p.usc & !63)) } /// Program resource word, 64-bit with RSRC_SPEC_HI. @@ -233,9 +237,9 @@ pub(crate) fn prog_rsrc( hi: u32, ) -> u64 { if u.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI != 0 { - (hi as u64) << 32 | p.rsrc_spec as u64 + u64::from(hi) << 32 | u64::from(p.rsrc_spec) } else { - p.rsrc_spec as u64 + u64::from(p.rsrc_spec) } } diff --git a/drivers/gpu/drm/asahi/g16/encode/compute.rs b/drivers/gpu/drm/asahi/g16/encode/compute.rs index 4089533f67ef7..08a4f9ee9cefd 100644 --- a/drivers/gpu/drm/asahi/g16/encode/compute.rs +++ b/drivers/gpu/drm/asahi/g16/encode/compute.rs @@ -221,7 +221,7 @@ pub(crate) fn encode_compute( let start = a.pointer & !127; if a.size == 0 || end <= start - || (end - start) / 128 > u32::MAX as u64 + || (end - start) / 128 > u64::from(u32::MAX) || !gpu(start, end - start, 128) { return Err(ComputeEncodeError::Attachment); diff --git a/drivers/gpu/drm/asahi/g16/encode/render.rs b/drivers/gpu/drm/asahi/g16/encode/render.rs index ec143a818c46e..b054ec08198b7 100644 --- a/drivers/gpu/drm/asahi/g16/encode/render.rs +++ b/drivers/gpu/drm/asahi/g16/encode/render.rs @@ -112,7 +112,7 @@ fn timestamps( put64(q, base + 0x24, if user { work + user_pair } else { 0 }); put64(q, base + 0x2c, work + auxiliary); put64(q, base + 0x34, work + context); - put64(q, base + 0x44, uuid as u64); + put64(q, base + 0x44, u64::from(uuid)); } put32(q, off + 0x4c, 1); } @@ -160,7 +160,7 @@ pub(crate) fn encode_ta( compact(p.heap_gva) | 1 << 63, 0x003a0012006b0003, 1, - geo.utile_config as u64, + u64::from(geo.utile_config), u.ppp_multisamplectl, compact(p.pre_gva[0]), compact(p.pre_gva[0]), @@ -179,18 +179,18 @@ pub(crate) fn encode_ta( 0, 0, 0x8860, - geo.region_stride as u64, - geo.region_stride as u64, + u64::from(geo.region_stride), + u64::from(geo.region_stride), 0x88, - (2 | (u.ppp_ctrl & 0x201)) as u64, - geo.pixels as u64, - geo.screen as u64, + u64::from(2 | (u.ppp_ctrl & 0x201)), + u64::from(geo.pixels), + u64::from(geo.screen), geo.x_blocks, geo.y_blocks, - geo.blocks as u64, - geo.tpc_stride as u64, + u64::from(geo.blocks), + u64::from(geo.tpc_stride), 0x100, - geo.layer_mode as u64, + u64::from(geo.layer_mode), 0, u64::MAX, u64::MAX, @@ -207,15 +207,15 @@ pub(crate) fn encode_ta( 0, 8, 0xff0000, - r.parameter_id as u64, + u64::from(r.parameter_id), compact(r.scene_list), compact(r.scene_list), scratch_word, 0x6000400020, 0x200000001, - r.uuid as u64, - r.uuid as u64, - r.uuid as u64, + u64::from(r.uuid), + u64::from(r.uuid), + u64::from(r.uuid), // USC base is a shared queue property carried by the render refs. r.usc_base, ]; @@ -307,33 +307,33 @@ pub(crate) fn encode_fs( q.fill(0); let val: [u64; 82] = [ 1, - geo.utile_config as u64, + u64::from(geo.utile_config), prog_rsrc(u, &u.eot, u.bg_eot_rsrc_spec_hi), prog_addr(usc, &u.eot), prog_rsrc(u, &u.bg, u.bg_rsrc_spec_hi), prog_addr(usc, &u.bg), - u.isp_merge_upper_x as u64, - u.isp_merge_upper_y as u64, + u64::from(u.isp_merge_upper_x), + u64::from(u.isp_merge_upper_y), 0, 0, 0, 0, u.ppp_multisamplectl, - geo.isp_blocks as u64, - geo.isp_blocks as u64, - geo.screen as u64, + u64::from(geo.isp_blocks), + u64::from(geo.isp_blocks), + u64::from(geo.screen), 0, 0xff0200, p.heap_gva & !63, u.isp_scissor_base & !3, u.isp_dbias_base & !3, - geo.isp_control as u64, - (u.height_px as u64) << 32 | u.width_px as u64, + u64::from(geo.isp_control), + u64::from(u.height_px) << 32 | u64::from(u.width_px), 0x100000, - geo.tilebuffer_control as u64, - u.isp_zls_pixels as u64, - u.isp_bgobjdepth as u64, - u.isp_bgobjvals as u64, + u64::from(geo.tilebuffer_control), + u64::from(u.isp_zls_pixels), + u64::from(u.isp_bgobjdepth), + u64::from(u.isp_bgobjvals), u.isp_oclqry_base & !15, u.zls_ctrl, 0x04040404, @@ -346,10 +346,10 @@ pub(crate) fn encode_fs( 0, 0, 0, - u.depth.stride as u64, - u.depth.stride as u64, - u.stencil.stride as u64, - u.stencil.stride as u64, + u64::from(u.depth.stride), + u64::from(u.depth.stride), + u64::from(u.stencil.stride), + u64::from(u.stencil.stride), 0, 0, 0, @@ -360,8 +360,8 @@ pub(crate) fn encode_fs( 0, p.tilemap_gva, geo.layer_metadata, - (geo.region_stride as u64) << 26, - geo.tile_mode as u64, + u64::from(geo.region_stride) << 26, + u64::from(geo.tile_mode), 0, 0, 0, @@ -382,10 +382,10 @@ pub(crate) fn encode_fs( 8, 0x6000400020, 0x200000001, - r.uuid as u64, - r.uuid as u64, - r.uuid as u64, - r.parameter_id as u64, + u64::from(r.uuid), + u64::from(r.uuid), + u64::from(r.uuid), + u64::from(r.parameter_id), compact(r.scene_list), usc, ]; @@ -423,18 +423,22 @@ pub(crate) fn encode_fs( put64(w, 0x938, u.zls_ctrl); put64(w, 0x940, 0x04040404); put64(w, 0x948, u.depth.base); - put64(w, 0x950, u.depth.stride as u64); + put64(w, 0x950, u64::from(u.depth.stride)); put64(w, 0x960, u.depth.base); put64(w, 0x968, u.depth.base); put64(w, 0x978, u.stencil.base); - put64(w, 0x980, u.stencil.stride as u64); + put64(w, 0x980, u64::from(u.stencil.stride)); put64(w, 0x990, u.stencil.base); put64(w, 0x998, u.stencil.base); - put64(w, 0x9b8, geo.tilebuffer_control as u64); - put64(w, 0x9c0, geo.isp_control as u64); - put64(w, 0x9c8, (u.height_px as u64) << 32 | u.width_px as u64); + put64(w, 0x9b8, u64::from(geo.tilebuffer_control)); + put64(w, 0x9c0, u64::from(geo.isp_control)); + put64( + w, + 0x9c8, + u64::from(u.height_px) << 32 | u64::from(u.width_px), + ); put64(w, 0x9d0, 0x100000); - put64(w, 0x9d8, geo.tile_mode as u64); + put64(w, 0x9d8, u64::from(geo.tile_mode)); put64(w, 0x9f4, prog_rsrc(u, &u.eot, u.bg_eot_rsrc_spec_hi)); put64(w, 0x9fc, prog_addr(usc, &u.eot)); put64( @@ -446,9 +450,9 @@ pub(crate) fn encode_fs( put64( w, 0xa28, - (u.isp_bgobjvals as u64) << 32 | u.isp_bgobjdepth as u64, + u64::from(u.isp_bgobjvals) << 32 | u64::from(u.isp_bgobjdepth), ); - put64(w, 0xa50, u.isp_zls_pixels as u64); + put64(w, 0xa50, u64::from(u.isp_zls_pixels)); metadata(w, 0xbc0, r, 1, target); internal_pair(w, p, 1, ts_va); put64(w, 0xc46, r.uma_mgr[0]); diff --git a/drivers/gpu/drm/asahi/g16/engine.rs b/drivers/gpu/drm/asahi/g16/engine.rs index 6e5700b78088e..f53293578316f 100644 --- a/drivers/gpu/drm/asahi/g16/engine.rs +++ b/drivers/gpu/drm/asahi/g16/engine.rs @@ -241,7 +241,7 @@ enum PublicationError { Uncertain(Error), } -/// Engine statistics, printed by `asahi-g16/stats`. +/// Engine statistics, printed by `aurora-g16/stats`. #[derive(Default)] pub(crate) struct Stats { pub(crate) pub_ns: u64, @@ -294,7 +294,7 @@ pub(crate) struct Stats { } /// Host-side phase timers of publication and polling, on with `debug_flags` bit 49 -/// (`Debug1`); printed as the `prof:` line of `asahi-g16/stats`. A measurement aid: nothing is +/// (`Debug1`); printed as the `prof:` line of `aurora-g16/stats`. A measurement aid: nothing is /// timed while the flag is off. #[derive(Default)] pub(crate) struct Prof { @@ -411,7 +411,7 @@ pub(crate) struct Engine { pub(crate) stats: Stats, /// Phase timers (debug_flags bit 49). pub(crate) prof: Prof, - /// The `asahi-g16/stats` file. + /// The `aurora-g16/stats` file. pub(crate) debugfs: Option>>>>, scratch: Option>, } @@ -1096,7 +1096,7 @@ impl Firmware { kind: channel as u32, head: head as u16, event, - new_queue: fresh as u8, + new_queue: u8::from(fresh), }; self.channels.post(channel, &record)?; self.session.ring(channel)?; diff --git a/drivers/gpu/drm/asahi/g16/event.rs b/drivers/gpu/drm/asahi/g16/event.rs index 4fabea819b854..26ef79a2c3a9a 100644 --- a/drivers/gpu/drm/asahi/g16/event.rs +++ b/drivers/gpu/drm/asahi/g16/event.rs @@ -30,7 +30,7 @@ pub(crate) struct ErrorEvent { impl ErrorEvent { pub(crate) fn generation(&self) -> u64 { - self.raw[1] as u64 | ((self.raw[2] as u64) << 32) + u64::from(self.raw[1]) | (u64::from(self.raw[2]) << 32) } } diff --git a/drivers/gpu/drm/asahi/g16/fence.rs b/drivers/gpu/drm/asahi/g16/fence.rs index afd89cc588a6a..01ba768b313b7 100644 --- a/drivers/gpu/drm/asahi/g16/fence.rs +++ b/drivers/gpu/drm/asahi/g16/fence.rs @@ -76,7 +76,7 @@ pub(crate) struct JobFenceOps; #[vtable] impl FenceOps for JobFenceOps { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("asahi") + c_str!("aurora") } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { @@ -107,11 +107,11 @@ pub(crate) struct Timeline { impl Timeline { pub(crate) fn new() -> Result> { - let errors = FenceContexts::new(1, c_str!("asahi_g16_refused"), ERROR_KEY)?; + let errors = FenceContexts::new(1, c_str!("aurora_g16_refused"), ERROR_KEY)?; let allocation_failure = error_fence(&errors, ENOMEM)?; Arc::pin_init( try_pin_init!(Self { - jobs: FenceContexts::new(1, c_str!("asahi_g16_queue"), TIMELINE_KEY)?, + jobs: FenceContexts::new(1, c_str!("aurora_g16_queue"), TIMELINE_KEY)?, errors, pending <- new_mutex!(KVec::new(), "g16::Timeline::pending"), allocation_failure, diff --git a/drivers/gpu/drm/asahi/g16/fw/channels.rs b/drivers/gpu/drm/asahi/g16/fw/channels.rs index 87b207f5b5ea7..9b5ae1e7c4b08 100644 --- a/drivers/gpu/drm/asahi/g16/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g16/fw/channels.rs @@ -34,7 +34,7 @@ const _: () = assert!(core::mem::offset_of!(SubmitRecord, new_queue) == 0x17); impl SubmitRecord { /// The little-endian record bytes. - pub(crate) fn to_bytes(&self) -> [u8; 0x18] { + pub(crate) fn to_bytes(self) -> [u8; 0x18] { let mut b = [0u8; 0x18]; b[0..8].copy_from_slice(&self.unk_0.to_le_bytes()); b[8..16].copy_from_slice(&self.qinfo.to_le_bytes()); @@ -80,7 +80,7 @@ impl DevCtl { } /// The little-endian record bytes. - pub(crate) fn to_bytes(&self) -> [u8; 0x40] { + pub(crate) fn to_bytes(self) -> [u8; 0x40] { let mut b = [0u8; 0x40]; for (i, w) in self.words.iter().enumerate() { b[4 * i..4 * i + 4].copy_from_slice(&w.to_le_bytes()); diff --git a/drivers/gpu/drm/asahi/g16/fw/initdata.rs b/drivers/gpu/drm/asahi/g16/fw/initdata.rs index 40fb02cd3bda5..9acbb4ac763a5 100644 --- a/drivers/gpu/drm/asahi/g16/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g16/fw/initdata.rs @@ -272,7 +272,11 @@ pub(crate) fn loader_span(slot: usize, bank: usize) -> Result<(u64, u32, u32), ( return Err(()); } let offset = e.pa & (PAGE - 1); - let bytes = if slot == 3 { 0x18000 } else { e.length as u64 }; + let bytes = if slot == 3 { + 0x18000 + } else { + u64::from(e.length) + }; let size = (offset + bytes + PAGE - 1) & !(PAGE - 1); Ok(( (e.pa & PA_MASK) + if bank != 0 { 0x2000000 } else { 0 }, @@ -305,9 +309,8 @@ pub(crate) fn loader_encode(slot: usize, va: u64, record: &mut [u8]) -> Result<( return Ok(()); } let offset = e.pa & (PAGE - 1); - let span = (offset + e.length as u64 + PAGE - 1) & !(PAGE - 1); - if va < HOST_START - || va >= HOST_END + let span = (offset + u64::from(e.length) + PAGE - 1) & !(PAGE - 1); + if !(HOST_START..HOST_END).contains(&va) || va & (PAGE - 1) != offset || span > HOST_END - (va - offset) { @@ -318,7 +321,7 @@ pub(crate) fn loader_encode(slot: usize, va: u64, record: &mut [u8]) -> Result<( put32(out, 0x10, e.length)?; put32(out, 0x14, e.requested)?; put64(out, 0x18, e.tag)?; - put64(out, 0x20, e.mode as u64) + put64(out, 0x20, u64::from(e.mode)) } pub(crate) const PERF_STATES: usize = 10; @@ -371,10 +374,11 @@ pub(crate) struct Input { } pub(crate) fn object_va(va: u64, size: usize) -> bool { - va >= HOST_START && va < HOST_END && va & 7 == 0 && size != 0 && size as u64 <= HOST_END - va + (HOST_START..HOST_END).contains(&va) && va & 7 == 0 && size != 0 && size as u64 <= HOST_END - va } fn bootstrap_alias(va: u64) -> bool { - va >= BOOTSTRAP_START && va <= BOOTSTRAP_START + BOOTSTRAP_SIZE - 0x8000 && va & (PAGE - 1) == 0 + (BOOTSTRAP_START..=BOOTSTRAP_START + BOOTSTRAP_SIZE - 0x8000).contains(&va) + && va & (PAGE - 1) == 0 } pub(crate) fn encode_root(out: &mut [u8], va: &[u64; OBJECT_COUNT]) -> Result<(), ()> { @@ -410,7 +414,7 @@ pub(crate) fn encode_root(out: &mut [u8], va: &[u64; OBJECT_COUNT]) -> Result<() out[at..at + 8].copy_from_slice(&[8, 14, 14, *shift, c[0], c[1], 0, 0x40]); put64(out, at + 8, 1)?; put64(out, at + 0x10, PA_MASK)?; - put64(out, at + 0x18, (*count as u64 - 1) << shift)?; + put64(out, at + 0x18, (u64::from(*count) - 1) << shift)?; } Ok(()) } diff --git a/drivers/gpu/drm/asahi/g16/glue.c b/drivers/gpu/drm/asahi/g16/glue.c index 0160187a0dd2c..db0576a1e0fe0 100644 --- a/drivers/gpu/drm/asahi/g16/glue.c +++ b/drivers/gpu/drm/asahi/g16/glue.c @@ -101,7 +101,7 @@ struct asahi_g16_cooling *asahi_g16_cooling_register(struct device *dev) goto put_node; } cooling->cdev = thermal_of_cooling_device_register(dev->of_node, 0, - "asahi-g16g", cooling, + "aurora-g16g", cooling, &asahi_g16_cooling_ops); if (IS_ERR(cooling->cdev)) { ret = PTR_ERR(cooling->cdev); diff --git a/drivers/gpu/drm/asahi/g16/initdata.rs b/drivers/gpu/drm/asahi/g16/initdata.rs index 24e7f1f83b78c..ecdde0579133a 100644 --- a/drivers/gpu/drm/asahi/g16/initdata.rs +++ b/drivers/gpu/drm/asahi/g16/initdata.rs @@ -158,11 +158,12 @@ pub(crate) fn build( continue; } let offset = entry.pa & (wire::PAGE - 1); - let at = objects.reserve_io((offset + entry.length as u64).next_multiple_of(wire::PAGE))?; + let at = + objects.reserve_io((offset + u64::from(entry.length)).next_multiple_of(wire::PAGE))?; input.loader_va[slot] = at + offset; for bank in 0..if slot == 3 { 2 } else { 1 } { - let (pa, size, delta) = wire::loader_span(slot, bank).map_err(|_| EINVAL)?; - objects.map_mmio_class(at + delta as u64, pa, size as u64, entry.attrs)?; + let (pa, size, delta) = wire::loader_span(slot, bank).map_err(|()| EINVAL)?; + objects.map_mmio_class(at + u64::from(delta), pa, u64::from(size), entry.attrs)?; } } input.scratch = objects.reserve_io(0x8000)?; @@ -183,7 +184,7 @@ pub(crate) fn build( } let array: &mut [&mut [u8]; wire::OBJECT_COUNT] = (&mut slices[..]).try_into().map_err(|_| EINVAL)?; - wire::encode(array, &input, 9).map_err(|_| EINVAL)?; + wire::encode(array, &input, 9).map_err(|()| EINVAL)?; } let mut loader = [0u8; wire::LOADER_BYTES]; loader.copy_from_slice(&images[id::MAIN][0x640..0x640 + wire::LOADER_BYTES]); diff --git a/drivers/gpu/drm/asahi/g16/job.rs b/drivers/gpu/drm/asahi/g16/job.rs index fb22514e1c61b..31f06c80ab8bf 100644 --- a/drivers/gpu/drm/asahi/g16/job.rs +++ b/drivers/gpu/drm/asahi/g16/job.rs @@ -121,6 +121,10 @@ pub(crate) struct Job { } impl Job { + #[expect( + clippy::too_many_arguments, + reason = "The job owns independently prepared command, mapping and fence resources." + )] pub(crate) fn new( id: u64, queue: Arc, @@ -467,7 +471,7 @@ impl Scheduler { hw::sizes::SCHEDULER_CREDITS, 0, hw::tuning::SCHED_TIMEOUT_MS, - c_str!("asahi_g16"), + c_str!("aurora_g16"), )?; // With one run queue, only the kernel priority is valid. let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; diff --git a/drivers/gpu/drm/asahi/g16/mod.rs b/drivers/gpu/drm/asahi/g16/mod.rs index c5c9829880e98..2c10fedd1e0be 100644 --- a/drivers/gpu/drm/asahi/g16/mod.rs +++ b/drivers/gpu/drm/asahi/g16/mod.rs @@ -91,7 +91,7 @@ use crate::driver::{AsahiDrmDevice, AsahiPlatformDriver}; use crate::{alloc, gem, gpu, mmu, regs as shared_regs}; /// Name of the G16 platform driver. -pub(crate) const DRIVER_NAME: &CStr = c_str!("asahi-g16"); +pub(crate) const DRIVER_NAME: &CStr = c_str!("aurora-g16"); /// GPU generation reported to userspace. pub(crate) const GPU_GENERATION: u32 = 16; @@ -171,6 +171,7 @@ impl platform::Driver for Driver { if let Some(gpu) = &driver.g16 { gpu.retain(); } + driver.close_binding(); } } } @@ -397,7 +398,7 @@ impl Gpu { *shared.state.lock() = Some(firmware); // The engine's statistics file (a debugfs failure does not fail the probe). if let Err(error) = shared.start_engine() { - dev_warn!(dev.as_ref(), "G16: no asahi-g16 debugfs ({:?})\n", error); + dev_warn!(dev.as_ref(), "G16: no aurora-g16 debugfs ({:?})\n", error); } let gpu = Arc::new( Gpu { @@ -616,9 +617,10 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { Ok(KBox::new( - queue::Queue::new(self, vm, priority, usc_exec_base)?, + queue::Queue::new(self, vm, priority, usc_exec_base, usage)?, GFP_KERNEL, )?) } diff --git a/drivers/gpu/drm/asahi/g16/object.rs b/drivers/gpu/drm/asahi/g16/object.rs index 11072b883e288..0f26cb3276738 100644 --- a/drivers/gpu/drm/asahi/g16/object.rs +++ b/drivers/gpu/drm/asahi/g16/object.rs @@ -175,7 +175,7 @@ impl FwObject { let p = self.ptr(offset, 4)?; if p as usize % 4 == 0 { // SAFETY: `p` is valid and aligned for a u32. - unsafe { (p as *mut u32).write_volatile(value.to_le()) }; + unsafe { p.cast::().write_volatile(value.to_le()) }; Ok(()) } else { self.write(offset, &value.to_le_bytes()) @@ -187,7 +187,7 @@ impl FwObject { let p = self.ptr(offset, 8)?; if p as usize % 8 == 0 { // SAFETY: `p` is valid and aligned for a u64. - unsafe { (p as *mut u64).write_volatile(value.to_le()) }; + unsafe { p.cast::().write_volatile(value.to_le()) }; Ok(()) } else { self.write(offset, &value.to_le_bytes()) @@ -624,8 +624,8 @@ impl Objects { let bound = unsafe { self.dev.as_ref().as_ref().as_bound() }; let sgt = gem::held_sg_table(&backing.gem, bound)?; for range in sgt.iter() { - let pa = range.dma_address() as u64; - let len = range.dma_len() as u64; + let pa = range.dma_address(); + let len = range.dma_len(); // EPERM for a page the UAT cannot name. if pa == 0 || (pa | len) & (hw::PAGE - 1) != 0 diff --git a/drivers/gpu/drm/asahi/g16/params.rs b/drivers/gpu/drm/asahi/g16/params.rs index 90540338f39a2..d2ef01e26b3a3 100644 --- a/drivers/gpu/drm/asahi/g16/params.rs +++ b/drivers/gpu/drm/asahi/g16/params.rs @@ -60,8 +60,8 @@ pub(crate) fn fill(gpu: &Gpu, params: &mut uapi::drm_asahi_params_global) -> Res if gpu.cfg.chip_id != 0x8132 || gpu.cfg.gpu_revision != 0x10 { return Err(ENODEV); } - params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64; + params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS); params.gpu_generation = GPU_GENERATION; params.gpu_variant = gpu.cfg.gpu_variant as u32; params.gpu_revision = gpu.cfg.gpu_revision; diff --git a/drivers/gpu/drm/asahi/g16/poll.rs b/drivers/gpu/drm/asahi/g16/poll.rs index 172296263398f..b94fb3a4cd36f 100644 --- a/drivers/gpu/drm/asahi/g16/poll.rs +++ b/drivers/gpu/drm/asahi/g16/poll.rs @@ -56,7 +56,7 @@ impl HrTimerCallback for Shared { } } -/// The `asahi-g16/stats` debugfs file contents. +/// The `aurora-g16/stats` debugfs file contents. fn stats_show(shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { let state = shared.state.lock(); match state.as_deref() { @@ -172,14 +172,14 @@ impl Shared { // Dropping the previous handle cancels a still pending expiry before the timer // is started again. drop(handle.take()); - *handle = Some(self.clone().start(Delta::from_micros(us as i64))); + *handle = Some(self.clone().start(Delta::from_micros(i64::from(us)))); } self.mod_poll(1); } - /// Starts what the engine needs besides the firmware state: the `asahi-g16/stats` file. + /// Starts what the engine needs besides the firmware state: the `aurora-g16/stats` file. pub(crate) fn start_engine(self: &Arc) -> Result { - let dir = debugfs::Dir::new(c_str!("asahi-g16")); + let dir = debugfs::Dir::new(c_str!("aurora-g16")); let file = KBox::pin_init( dir.read_callback_file(c_str!("stats"), self.clone(), &stats_show), GFP_KERNEL, @@ -572,8 +572,8 @@ impl Firmware { let s = &mut self.engine.stats; if result.is_ok() { s.jobs += 1; - s.job_cmds += p.count() as u64; - s.job_max = s.job_max.max(p.count() as u64); + s.job_cmds += u64::from(p.count()); + s.job_max = s.job_max.max(u64::from(p.count())); } complete(done, &job, Ok(())); // The rest of a compute horizon, after the first job (fence order). @@ -584,8 +584,8 @@ impl Firmware { Err(error) => complete(done, j2, Err(error)), Ok(()) => { s.jobs += 1; - s.job_cmds += p2.count() as u64; - s.job_max = s.job_max.max(p2.count() as u64); + s.job_cmds += u64::from(p2.count()); + s.job_max = s.job_max.max(u64::from(p2.count())); } } complete(done, j2, Ok(())); @@ -694,13 +694,24 @@ impl Firmware { let s = &mut self.engine.stats; // Add one stage interval read from internal timestamp cells (start, end). - let mut stage = |o: &super::object::FwObject, off: usize, sum: &mut u64| { - let rd = |x: usize| o.r32(x).unwrap_or(0) as u64; - let start = rd(off + 4) << 32 | rd(off); - let end = rd(off + 12) << 32 | rd(off + 8); + let mut stage = |o: &super::object::FwObject, + off: usize, + sum: &mut u64, + engine: crate::usage::Engine| { + let (Ok(start_lo), Ok(start_hi), Ok(end_lo), Ok(end_hi)) = + (o.r32(off), o.r32(off + 4), o.r32(off + 8), o.r32(off + 12)) + else { + return; + }; + let start = u64::from(start_hi) << 32 | u64::from(start_lo); + let end = u64::from(end_hi) << 32 | u64::from(end_lo); if start == 0 || end < start { return; } + f.job + .queue + .usage + .add(engine, start, end, super::hw::COMMAND_TIMESTAMP_HZ as u32); *sum = sum.saturating_add(end - start); lo = lo.min(start); hi = hi.max(end); @@ -710,15 +721,25 @@ impl Firmware { return; }; let p = &arena.passes[f.claims.pass[k] as usize]; - stage(&p.scene, 0x600, &mut s.ta_ticks); - stage(&p.scene, 0x620, &mut s.fs_ticks); + stage( + &p.scene, + 0x600, + &mut s.ta_ticks, + crate::usage::Engine::Vertex, + ); + stage( + &p.scene, + 0x620, + &mut s.fs_ticks, + crate::usage::Engine::Fragment, + ); } for k in 0..(f.computes as usize).min(f.claims.slot_count as usize) { let Some(arena) = f.job.resources.compute.as_ref() else { return; }; let c = &arena.compute[f.claims.slot[k] as usize]; - stage(&c.its, 0, &mut s.cdm_ticks); + stage(&c.its, 0, &mut s.cdm_ticks, crate::usage::Engine::Compute); } if hi > lo { s.span_ticks = s.span_ticks.saturating_add(hi - lo); @@ -740,8 +761,8 @@ impl Firmware { s.first_ns[kind] += first - f.published_ns; s.last_ns[kind] += last - f.published_ns; } - s.count[0] += f.computes as u64; - s.count[1] += f.renders as u64; + s.count[0] += u64::from(f.computes); + s.count[1] += u64::from(f.renders); if kind == 2 { s.count[2] += f.count as u64; } @@ -813,7 +834,7 @@ impl Firmware { } } - /// The `asahi-g16/stats` contents. Scripts parse the keys: keep them stable. + /// The `aurora-g16/stats` contents. Scripts parse the keys: keep them stable. pub(crate) fn write_stats(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let s = &self.engine.stats; let slots = &self.slots.stats; diff --git a/drivers/gpu/drm/asahi/g16/pools.rs b/drivers/gpu/drm/asahi/g16/pools.rs index 1b0406aae6d51..d4a85cc6f3d36 100644 --- a/drivers/gpu/drm/asahi/g16/pools.rs +++ b/drivers/gpu/drm/asahi/g16/pools.rs @@ -99,7 +99,7 @@ fn read64(o: &FwObject, at: usize) -> Result { let high = o.r32(at + 4)?; let low = o.r32(at)?; if high == o.r32(at + 4)? { - return Ok((high as u64) << 32 | low as u64); + return Ok(u64::from(high) << 32 | u64::from(low)); } } Err(EAGAIN) @@ -347,7 +347,7 @@ impl PbEpoch { buffer::page_record(heap, i as u32).to_le_bytes() })?; write_records(&self.blocks, buffer::MAX_BLOCKS as usize, |i| { - (buffer::block_record(heap, i as u32) as u64).to_le_bytes() + u64::from(buffer::block_record(heap, i as u32)).to_le_bytes() })?; self.ring.w32(0, buffer::MAX_BLOCKS)?; self.ring.w32(4, buffer::MAX_BLOCKS)?; diff --git a/drivers/gpu/drm/asahi/g16/pools/lifetime.rs b/drivers/gpu/drm/asahi/g16/pools/lifetime.rs index 58d482f1ea94b..baf623b2722d7 100644 --- a/drivers/gpu/drm/asahi/g16/pools/lifetime.rs +++ b/drivers/gpu/drm/asahi/g16/pools/lifetime.rs @@ -66,7 +66,7 @@ impl Lifetime { State::Retired => counts.retired += 1, State::Quarantined => counts.quarantined += 1, } - counts.reusable += self.available(k) as u32; + counts.reusable += u32::from(self.available(k)); } counts } diff --git a/drivers/gpu/drm/asahi/g16/queue.rs b/drivers/gpu/drm/asahi/g16/queue.rs index 08a2093ef527b..41da5b94c7c50 100644 --- a/drivers/gpu/drm/asahi/g16/queue.rs +++ b/drivers/gpu/drm/asahi/g16/queue.rs @@ -43,6 +43,8 @@ pub(crate) struct QueueState { /// Host priority: `DRM_ASAHI_PRIORITY_REALTIME - priority` as `file.rs` passes it (MEDIUM 2, /// LOW 3; `file.rs` refuses HIGH and REALTIME). Smaller is served first. pub(crate) priority: u32, + /// Completed stage time of the DRM file that created this queue. + pub(crate) usage: Arc, /// USC executable window base. pub(crate) usc_base: u64, /// First cancelled accepted ordinal per engine; zero means none. Later scheduler @@ -102,7 +104,13 @@ pub(crate) struct Queue { impl Queue { /// Creates a logical queue. `priority` is `DRM_ASAHI_PRIORITY_REALTIME - priority`, as /// `file.rs` passes it. - pub(crate) fn new(gpu: &Gpu, vm: mmu::Vm, priority: u32, usc_exec_base: u64) -> Result { + pub(crate) fn new( + gpu: &Gpu, + vm: mmu::Vm, + priority: u32, + usc_exec_base: u64, + usage: Arc, + ) -> Result { if priority > uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { return Err(EINVAL); } @@ -127,6 +135,7 @@ impl Queue { let state = Arc::new( QueueState { priority, + usage, usc_base: usc_exec_base, cancelled_at: [const { AtomicU64::new(0) }; 2], turn: AtomicU64::new(0), diff --git a/drivers/gpu/drm/asahi/g16/recovery.rs b/drivers/gpu/drm/asahi/g16/recovery.rs index cc3f3525c101d..e15590048dacb 100644 --- a/drivers/gpu/drm/asahi/g16/recovery.rs +++ b/drivers/gpu/drm/asahi/g16/recovery.rs @@ -126,8 +126,7 @@ impl Firmware { } pub(crate) fn recovery_pending(&self) -> Result { - Ok(self.recovery.running || self.new_recovery() - || self.firmware_r32(logic::PHASE)? != 0) + Ok(self.recovery.running || self.new_recovery() || self.firmware_r32(logic::PHASE)? != 0) } pub(crate) fn recovery_idle(&self) -> Result { @@ -289,7 +288,7 @@ impl Firmware { // The firmware is stopped while its published generation is consumed. let lo = self.firmware_r32(logic::GENERATION)?; let hi = self.firmware_r32(logic::GENERATION + 4)?; - Ok(lo as u64 | ((hi as u64) << 32)) + Ok(u64::from(lo) | (u64::from(hi) << 32)) } /// Resolve the snapshot while the firmware is stopped. A missing retained diff --git a/drivers/gpu/drm/asahi/g16/recovery/logic.rs b/drivers/gpu/drm/asahi/g16/recovery/logic.rs index 2d29a39337df8..4d8bcb716eda5 100644 --- a/drivers/gpu/drm/asahi/g16/recovery/logic.rs +++ b/drivers/gpu/drm/asahi/g16/recovery/logic.rs @@ -142,7 +142,7 @@ fn host_deadline(now: u64, milliseconds: u32) -> u64 { if milliseconds == 0 { u64::MAX } else { - now.saturating_add(milliseconds.max(100) as u64 * 1_000_000) + now.saturating_add(u64::from(milliseconds.max(100)) * 1_000_000) } } diff --git a/drivers/gpu/drm/asahi/g16/regs.rs b/drivers/gpu/drm/asahi/g16/regs.rs index 86c98be692c73..e264e319c2a64 100644 --- a/drivers/gpu/drm/asahi/g16/regs.rs +++ b/drivers/gpu/drm/asahi/g16/regs.rs @@ -104,7 +104,7 @@ impl Mapping { } // SAFETY: memremap() of a range outside System RAM creates a new mapping; the range is // an ADT-described GPU region that the caller admitted. - let addr = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WC as _) }; + let addr = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WC as usize) }; if addr.is_null() { return Err(ENOMEM); } diff --git a/drivers/gpu/drm/asahi/g16/uat.rs b/drivers/gpu/drm/asahi/g16/uat.rs index 54104712c294c..72ca9cb94b846 100644 --- a/drivers/gpu/drm/asahi/g16/uat.rs +++ b/drivers/gpu/drm/asahi/g16/uat.rs @@ -294,7 +294,7 @@ impl Window { let address = va | ((index as u64) << shift); let off = address.checked_sub(hw::UPPER).ok_or(EINVAL)?; let expected = if off < 0x60000 { - self.text_pa + off + (self.text_pa + off) | if (0x50000..0x58000).contains(&off) { 0x483 } else { @@ -302,7 +302,7 @@ impl Window { } } else if off < 0x19c000 { let data_offset = off - 0x60000; - self.data_pa + data_offset + (self.data_pa + data_offset) | if data_offset < 0xc000 { 0x00200000000004c3 } else { @@ -407,9 +407,9 @@ impl Window { continue; } let expected = if index == 0 { - self.root.phys() + hw::PAGE | 0x403 + (self.root.phys() + hw::PAGE) | 0x403 } else { - self.root.phys() + 0x10000 | 0x00200000000004cb + (self.root.phys() + 0x10000) | 0x00200000000004cb }; if self.firmware_root[index] != 0 || observed != expected { return Err(EIO); @@ -460,7 +460,7 @@ impl Window { let handoff = uat.lock_m4_handoff()?; let result = self .verify_root() - .and_then(|_| self.sync_inner(shadow_l2, Some(range.clone()))); + .and_then(|()| self.sync_inner(shadow_l2, Some(range.clone()))); drop(handoff); result?; // This includes global final-level entries. Never replace this with ASID-only TLBI: diff --git a/drivers/gpu/drm/asahi/g17/availability.rs b/drivers/gpu/drm/asahi/g17/availability.rs index 8014266f0b0ab..c76a20db7007a 100644 --- a/drivers/gpu/drm/asahi/g17/availability.rs +++ b/drivers/gpu/drm/asahi/g17/availability.rs @@ -40,7 +40,7 @@ impl Availability { State { generation: 0, waiters: None }, "g17::Availability" ), - fences: FenceContexts::new(1, c_str!("asahi_resource"), kernel::static_lock_class!())?, + fences: FenceContexts::new(1, c_str!("aurora_resource"), kernel::static_lock_class!())?, }), GFP_KERNEL, ) diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 568647ebb444a..7b4c9de0e0fd9 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -46,18 +46,59 @@ impl ReportRing { pub(crate) struct Rings { quarantined: [[bool; 2]; 2], reports: super::reports::Reports, + telemetry: super::telemetry::Telemetry, + max_age_ns: u64, + suspended: bool, + discard_statistics: [bool; 2], + statistics_epoch: u64, } impl Rings { - pub(crate) fn new() -> Result { + pub(crate) fn new(graph: super::fw::variant::InitGraph) -> Result { Ok(Self { quarantined: [[false; 2]; 2], reports: super::reports::Reports::new()?, + telemetry: super::telemetry::Telemetry::default(), + max_age_ns: super::stats::max_age_ns(graph), + suspended: false, + discard_statistics: [false; 2], + statistics_epoch: 0, }) } + pub(super) fn invalidate_statistics(&mut self) { + self.telemetry.invalidate(); + // The packets have no epoch marker. Discard each role's already + // queued records before accepting a new baseline after the boundary. + self.discard_statistics = [true; 2]; + } + + pub(super) fn suspend_statistics(&mut self, suspended: bool) { + self.suspended = suspended; + self.invalidate_statistics(); + } + + pub(super) fn statistics_suspended(&self) -> bool { + self.suspended + } + + pub(super) fn observe_statistics_power(&mut self, powered: bool, epoch: u64) { + if !powered || epoch != self.statistics_epoch { + self.invalidate_statistics(); + } + self.statistics_epoch = epoch; + } + + pub(super) fn read_statistics(&self, now_ns: u64) -> super::telemetry::Readout { + self.telemetry.read(now_ns, self.max_age_ns) + } + pub(super) fn write_reports(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - self.reports.write(f) + self.reports.write(f)?; + let view = self.read_statistics(super::now_ns()); + writeln!(f, "qualified max_age_ns={} util_status={} util_host_ns={} util_samples={} pstate_status={} pstate_host_ns={} pstate_timestamp={} delta_ticks={:?}", + self.max_age_ns, view.util_status, view.util_host_ns, view.util_samples, + view.pstate_status, view.pstate_host_ns, view.pstate_timestamp, self.telemetry.delta_ticks) } pub(crate) fn drain_trace(&mut self, dev: &Device, init: &InitData, role: Role) { @@ -69,12 +110,29 @@ impl Rings { } fn drain_report(&mut self, dev: &Device, init: &InitData, role: Role, ring: ReportRing) { - let quarantined = &mut self.quarantined[role as usize][ring as usize]; - if *quarantined { + if self.quarantined[role as usize][ring as usize] { return; } - if let Err(error) = Self::consume_report(init, role, ring, &mut self.reports) { - *quarantined = true; + let statistics = matches!(ring, ReportRing::Statistics); + let accept = !self.suspended + && !self.discard_statistics[role as usize] + && !self + .quarantined + .iter() + .any(|rings| rings[ReportRing::Statistics as usize]); + if let Err(error) = Self::consume_report( + init, + role, + ring, + &mut self.reports, + &mut self.telemetry, + self.max_age_ns, + accept, + ) { + self.quarantined[role as usize][ring as usize] = true; + if statistics { + self.invalidate_statistics(); + } dev_err!( dev.as_ref(), "G17 {:?} {:?} ring stopped: {:?}\n", @@ -82,10 +140,20 @@ impl Rings { ring, error ); + } else if statistics && !self.suspended { + self.discard_statistics[role as usize] = false; } } - fn consume_report(init: &InitData, role: Role, ring: ReportRing, reports: &mut super::reports::Reports) -> Result { + fn consume_report( + init: &InitData, + role: Role, + ring: ReportRing, + reports: &mut super::reports::Reports, + telemetry: &mut super::telemetry::Telemetry, + max_age_ns: u64, + accept_statistics: bool, + ) -> Result { let (state, entries, capacity, allowed_types) = ring.geometry(); let consumer_word = init.status_word(role, state)?; let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; @@ -99,8 +167,8 @@ impl Rings { } while consumer != producer { let offset = entries + consumer as usize * EVENT_RECORD_SIZE; - // Copy all words before returning the slot. Interpretation is a - // separate firmware qualification step, never a scheduling input. + // Decode only the complete copy, before returning the slot. + // Statistics never affect scheduling or power control. let words = init.status_read_words::(role, offset)?; super::trace::report(role as u8, ring as u8, &words); let tag = words[0]; @@ -108,7 +176,11 @@ impl Rings { return Err(EIO); } if matches!(ring, ReportRing::Statistics) { - reports.record(role, words, super::now_ns()); + let host_ns = super::now_ns(); + reports.record(role, words, host_ns); + if accept_statistics { + telemetry.record(&words, host_ns, max_age_ns); + } } consumer = (consumer + 1) & (capacity - 1); fence(Ordering::SeqCst); diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 362cf507e1575..7d887f17d22c6 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -13,7 +13,12 @@ use super::{ recovery::Class, status::VmStatus, }; -use crate::{driver::AsahiDrmDevice, file, hw::t8140::{dynamic, resources as cfg}, mmu}; +use crate::{ + driver::AsahiDrmDevice, + file, + hw::t8140::{dynamic, resources as cfg}, + mmu, +}; use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; use kernel::{ dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, @@ -86,7 +91,9 @@ impl Destinations { Ok(Self(destinations)) } - pub(crate) fn has_outputs(&self) -> bool { self.0.iter().any(Option::is_some) } + pub(crate) fn has_outputs(&self) -> bool { + self.0.iter().any(Option::is_some) + } fn iter(&self) -> core::slice::Iter<'_, Option> { self.0.iter() @@ -248,6 +255,7 @@ pub(crate) struct Completion { terminal: Option>, compute: Option, feed: Option>, + usage: Arc, sources: Option>, #[pin] vm_job: Mutex>, @@ -285,16 +293,20 @@ impl Completion { host_predecessors: bool, feed: Option>, sources: Option>, + usage: Arc, ) -> Result> { let fence = fence::independent(contexts)?; - let ordered = !compute - && (host_predecessors || destinations.iter().any(Option::is_some)); + let ordered = !compute && (host_predecessors || destinations.iter().any(Option::is_some)); let terminal = if host_predecessors { Some(fence::independent(contexts)?) } else { None }; - let output = Output { fence, member, destinations }; + let output = Output { + fence, + member, + destinations, + }; let output = if ordered { HostOutput::Ordered(Publication::new(output, context.status().clone())?) } else { @@ -312,6 +324,7 @@ impl Completion { terminal, compute, feed, + usage, sources, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), @@ -334,7 +347,9 @@ impl Completion { Fence::from_fence(&self.output.output().fence) } pub(crate) fn scheduler_fence(&self) -> Fence { - self.terminal.as_ref().map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) + self.terminal + .as_ref() + .map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) } pub(crate) fn publication(&self) -> Option> { match &self.output { @@ -382,11 +397,15 @@ impl Completion { } match self.check_source_inputs() { Ok(()) => self.output.output().member.note_retired(), - Err(error) => { self.note_input_error(error); } + Err(error) => { + self.note_input_error(error); + } } } pub(super) fn check_source_inputs(&self) -> Result { - if let Some(sources) = &self.sources { sources.check()?; } + if let Some(sources) = &self.sources { + sources.check()?; + } Ok(()) } @@ -394,7 +413,11 @@ impl Completion { self.output.output().member.work_state() } pub(crate) fn has_timestamps(&self) -> bool { - self.output.output().destinations.iter().any(Option::is_some) + self.output + .output() + .destinations + .iter() + .any(Option::is_some) } pub(crate) fn compute_timestamp_va(&self) -> Result { @@ -597,7 +620,9 @@ impl Completion { /// An exact whole-pair drain or processor stop allows rehabilitation. The /// VM pin goes first; the still-held lease excludes reuse until both settle. pub(crate) fn release_after_retirement(&self) { - if let Some(sources) = &self.sources { sources.release(); } + if let Some(sources) = &self.sources { + sources.release(); + } drop(self.vm_job.lock().take()); let lease = self.render.lock().take(); if let Some(lease) = lease.as_ref() { @@ -612,7 +637,9 @@ impl Completion { } fn release_healthy(&self) { - if let Some(sources) = &self.sources { sources.release(); } + if let Some(sources) = &self.sources { + sources.release(); + } drop(self.render.lock().take()); drop(self.vm_job.lock().take()); } @@ -622,7 +649,9 @@ impl Completion { #[track_caller] pub(crate) fn complete(&self, mut result: Result<[u64; 4]>) { if result.is_ok() { - if let Err(error) = self.check_source_inputs() { result = Err(error); } + if let Err(error) = self.check_source_inputs() { + result = Err(error); + } } if self.signalled.swap(true, Ordering::AcqRel) { return; @@ -630,6 +659,16 @@ impl Completion { if result.is_ok() && self.interrupted() { result = Err(ECANCELED); } + if let Ok(ticks) = &result { + use crate::usage::Engine; + let hz = cfg::COMMAND_TIMESTAMP_HZ as u32; + if self.compute.is_some() { + self.usage.add(Engine::Compute, ticks[0], ticks[1], hz); + } else { + self.usage.add(Engine::Vertex, ticks[0], ticks[1], hz); + self.usage.add(Engine::Fragment, ticks[2], ticks[3], hz); + } + } if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { // Compute reports its dispatch envelope in the first pair, a // render its fragment pass in the second. diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 8245e8bca71e6..71107371640bd 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -32,7 +32,7 @@ fn signal_exported(fence: &dyn RawDmaFence) { #[vtable] impl FenceOps for CompletionFence { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("asahi") + c_str!("aurora") } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { @@ -68,7 +68,9 @@ static EXPORT_OPS: AtomicUsize = AtomicUsize::new(0); #[vtable] impl FenceOps for ExportFence { - fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("asahi") } + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("aurora") + } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("g17-aggregate") } @@ -76,7 +78,13 @@ impl FenceOps for ExportFence { #[inline(never)] fn export_fallback(contexts: &FenceContexts) -> Result> { - let fence = contexts.new_fence(0, ExportFence { source: None, scoped: false })?; + let fence = contexts.new_fence( + 0, + ExportFence { + source: None, + scoped: false, + }, + )?; // SAFETY: Unique, initialized and still private. Only this constructor // publishes ExportFence objects; its payload is immutable after sharing. let ops = unsafe { @@ -86,8 +94,8 @@ fn export_fallback(contexts: &FenceContexts) -> Result> let previous = EXPORT_OPS.load(Ordering::Acquire); if previous == 0 { match EXPORT_OPS.compare_exchange(0, ops, Ordering::AcqRel, Ordering::Acquire) { - Ok(_) => {}, - Err(previous) if previous == ops => {}, + Ok(_) => {} + Err(previous) if previous == ops => {} Err(_) => return Err(EIO), } } else if previous != ops { @@ -96,21 +104,32 @@ fn export_fallback(contexts: &FenceContexts) -> Result> Ok(fence) } -/// Caller retains the leaf fence throughout lookup. Exact registered operations -/// identify the repr(C) FenceObject payload, not a driver or timeline name. -pub(super) unsafe fn exported_source_raw(raw: *mut bindings::dma_fence) - -> Option> -{ +/// Find the submission behind a leaf fence with the exact registered operations. +/// Those operations identify the repr(C) FenceObject payload, not its name. +/// +/// # Safety +/// +/// `raw` must be non-null and point to an initialized dma_fence. The caller +/// must retain a fence reference throughout this lookup. +pub(super) unsafe fn exported_source_raw(raw: *mut bindings::dma_fence) -> Option> { let expected = EXPORT_OPS.load(Ordering::Acquire); - if expected == 0 { return None; } + if expected == 0 { + return None; + } // SAFETY: The caller owns a live fence reference. ExportFence has a unique // timeline callback and nontrivial payload drop. No cast precedes the exact // registered vtable check. FenceObject begins with its dma_fence. let object = unsafe { - if (*raw).ops as usize != expected { return None; } + if (*raw).ops as usize != expected { + return None; + } &*(raw as *const FenceObject) }; - object.source.as_ref().filter(|source| source.firmware_source).cloned() + object + .source + .as_ref() + .filter(|source| source.firmware_source) + .cloned() } struct State { @@ -144,33 +163,50 @@ impl Submission { Self::with_work(Some(work), firmware_source) } fn with_work(work: Option>, firmware_source: bool) -> Result> { - Arc::pin_init(pin_init!(Self { - state <- new_mutex!(State { work, ready: None }, "G17 submission work state"), - remaining: AtomicU8::new(1), - error: AtomicI32::new(0), - owners: AtomicU8::new(1), - firmware_source, - retired: AtomicBool::new(false), - }), GFP_KERNEL) + Arc::pin_init( + pin_init!(Self { + state <- new_mutex!(State { work, ready: None }, "G17 submission work state"), + remaining: AtomicU8::new(1), + error: AtomicI32::new(0), + owners: AtomicU8::new(1), + firmware_source, + retired: AtomicBool::new(false), + }), + GFP_KERNEL, + ) } pub(super) fn source_status(&self) -> Result { - let complete = self.remaining.load(Ordering::Acquire) == 0 - || self.retired.load(Ordering::Acquire); + let complete = + self.remaining.load(Ordering::Acquire) == 0 || self.retired.load(Ordering::Acquire); let error = self.error.load(Ordering::Acquire); - if error != 0 { return Err(super::dependency::inherited_error(Error::from_errno(error))); } + if error != 0 { + return Err(super::dependency::inherited_error(Error::from_errno(error))); + } Ok(complete) } pub(super) fn same_vm(&self, status: &Arc) -> bool { - self.state.lock().work.as_ref().is_some_and(|work| Arc::ptr_eq(work.status(), status)) + self.state + .lock() + .work + .as_ref() + .is_some_and(|work| Arc::ptr_eq(work.status(), status)) + } + pub(super) fn ready(&self) -> Option { + self.state.lock().ready.clone() } - pub(super) fn ready(&self) -> Option { self.state.lock().ready.clone() } fn set_ready(&self, ready: Fence) { - if !self.firmware_source { return; } + if !self.firmware_source { + return; + } let mut state = self.state.lock(); - if state.work.is_some() && state.ready.is_none() { state.ready = Some(ready); } + if state.work.is_some() && state.ready.is_none() { + state.ready = Some(ready); + } } fn note_error(&self, error: Error) -> bool { - self.error.compare_exchange(0, error.to_errno(), Ordering::AcqRel, Ordering::Acquire).is_ok() + self.error + .compare_exchange(0, error.to_errno(), Ordering::AcqRel, Ordering::Acquire) + .is_ok() } fn release_state(&self) { let old = { @@ -182,15 +218,26 @@ impl Submission { fn owner_done(&self) { // Every hardware owner retains its Completion/Member through exact // retirement or stop. Exported fence FDs must not retain failed pins. - if self.owners.fetch_sub(1, Ordering::AcqRel) == 1 { self.release_state(); } + if self.owners.fetch_sub(1, Ordering::AcqRel) == 1 { + self.release_state(); + } } fn member_done(&self, fence: &UserFence, result: Result) { - if let Err(error) = result { self.note_error(error); } + if let Err(error) = result { + self.note_error(error); + } if self.remaining.fetch_sub(1, Ordering::AcqRel) == 1 { let error = self.error.load(Ordering::Acquire); - if error != 0 { fence.set_error(Error::from_errno(error)); } - else { self.release_state(); } - if fence.scoped { signal_exported(fence); } else { fence.signal(); } + if error != 0 { + fence.set_error(Error::from_errno(error)); + } else { + self.release_state(); + } + if fence.scoped { + signal_exported(fence); + } else { + fence.signal(); + } } } } @@ -200,16 +247,28 @@ pub(crate) struct Aggregate(UserFence); impl Aggregate { fn source(&self) -> &Arc { // Only Outputs::publish constructs Aggregate, after setting the payload. - self.0.source.as_ref().expect("published aggregate has submission state") + self.0 + .source + .as_ref() + .expect("published aggregate has submission state") + } + fn fence(&self) -> Fence { + Fence::from_fence(&self.0) } - fn fence(&self) -> Fence { Fence::from_fence(&self.0) } pub(crate) fn member(&self) -> Member { self.source().remaining.fetch_add(1, Ordering::Relaxed); self.source().owners.fetch_add(1, Ordering::Relaxed); - Member { aggregate: self.clone(), complete: AtomicBool::new(false) } + Member { + aggregate: self.clone(), + complete: AtomicBool::new(false), + } } fn finish_enqueue(&self, result: Result) { - if let Err(error) = result { self.source().error.store(error.to_errno(), Ordering::Release); } + if let Err(error) = result { + self.source() + .error + .store(error.to_errno(), Ordering::Release); + } self.source().member_done(&self.0, result); self.source().owner_done(); } @@ -220,22 +279,38 @@ pub(crate) struct Member { complete: AtomicBool, } impl Member { - pub(super) fn needs_source_ready(&self) -> bool { self.aggregate.source().firmware_source } + pub(super) fn needs_source_ready(&self) -> bool { + self.aggregate.source().firmware_source + } pub(crate) fn work_state(&self) -> Result> { - self.aggregate.source().state.lock().work.clone().ok_or(ECANCELED) + self.aggregate + .source() + .state + .lock() + .work + .clone() + .ok_or(ECANCELED) } pub(super) fn is_source(&self, source: &Arc) -> bool { self.aggregate.source().firmware_source && Arc::ptr_eq(self.aggregate.source(), source) } - pub(super) fn set_source_ready(&self, ready: Fence) { self.aggregate.source().set_ready(ready); } - pub(super) fn note_error(&self, error: Error) -> bool { self.aggregate.source().note_error(error) } + pub(super) fn set_source_ready(&self, ready: Fence) { + self.aggregate.source().set_ready(ready); + } + pub(super) fn note_error(&self, error: Error) -> bool { + self.aggregate.source().note_error(error) + } pub(super) fn note_retired(&self) { let source = self.aggregate.source(); - if source.firmware_source { source.retired.store(true, Ordering::Release); } + if source.firmware_source { + source.retired.store(true, Ordering::Release); + } } pub(crate) fn complete(&self, result: Result) { if !self.complete.swap(true, Ordering::AcqRel) { - self.aggregate.source().member_done(&self.aggregate.0, result); + self.aggregate + .source() + .member_done(&self.aggregate.0, result); } } } @@ -278,7 +353,10 @@ mod tests { let member = aggregate.member(); aggregate.finish_enqueue(Ok(())); member.complete(Err(EIO)); - assert_eq!(super::super::job::fence_status(&aggregate.fence()), EIO.to_errno()); + assert_eq!( + super::super::job::fence_status(&aggregate.fence()), + EIO.to_errno() + ); assert!(source.state.lock().ready.is_some()); drop(member); // Keep both the exported fence and source state alive, as userspace can. @@ -291,7 +369,10 @@ mod tests { let aggregate = aggregate(&contexts()); drop(aggregate.member()); aggregate.finish_enqueue(Err(ENOMEM)); - assert_eq!(super::super::job::fence_status(&aggregate.fence()), ENOMEM.to_errno()); + assert_eq!( + super::super::job::fence_status(&aggregate.fence()), + ENOMEM.to_errno() + ); } #[test] @@ -318,7 +399,9 @@ mod tests { } assert!(super::super::job::fence_status(&aggregate.fence()) > 0); assert_eq!(source.remaining.load(Ordering::Acquire), 0); - for member in &mut members { drop(member.take()); } + for member in &mut members { + drop(member.take()); + } assert_eq!(source.owners.load(Ordering::Acquire), 0); } } @@ -333,7 +416,9 @@ fn install(outputs: impl Iterator, fence: &Fence) { for mut sync in outputs { if let Some(chain) = sync.chain_fence.take() { sync.syncobj.add_point(chain, fence, sync.timeline_value); - } else { sync.syncobj.replace_fence(Some(fence)); } + } else { + sync.syncobj.replace_fence(Some(fence)); + } } } @@ -346,10 +431,15 @@ pub(crate) struct Outputs { cumulative: Option, } impl Outputs { - pub(crate) fn new(contexts: &FenceContexts, mut syncs: KVec, - input_count: usize, allocation_failure: &Fence) -> Result - { - if input_count > syncs.len() { return Err(EINVAL); } + pub(crate) fn new( + contexts: &FenceContexts, + mut syncs: KVec, + input_count: usize, + allocation_failure: &Fence, + ) -> Result { + if input_count > syncs.len() { + return Err(EINVAL); + } let mut cumulative = syncs.iter_mut().find_map(|sync| sync.cumulative.take()); let fallback = match export_fallback(contexts) { Ok(fence) => fence, @@ -362,11 +452,23 @@ impl Outputs { return Err(error); } }; - Ok(Self { fallback: Some(fallback), syncs, input_count, - submission: None, result: Err(ECANCELED), cumulative }) - } - pub(crate) fn inputs(&self) -> &[SyncItem] { &self.syncs[..self.input_count] } - pub(crate) fn publish(&mut self, context: Arc, firmware_source: bool) -> Result { + Ok(Self { + fallback: Some(fallback), + syncs, + input_count, + submission: None, + result: Err(ECANCELED), + cumulative, + }) + } + pub(crate) fn inputs(&self) -> &[SyncItem] { + &self.syncs[..self.input_count] + } + pub(crate) fn publish( + &mut self, + context: Arc, + firmware_source: bool, + ) -> Result { let source = Submission::new(context, firmware_source)?; let mut fence = self.fallback.take().ok_or(EINVAL)?; fence.source = Some(source); @@ -381,7 +483,10 @@ impl Outputs { } Ok(submission) } - pub(crate) fn finish(mut self, result: Result) -> Result { self.result = result; result } + pub(crate) fn finish(mut self, result: Result) -> Result { + self.result = result; + result + } } impl Drop for Outputs { fn drop(&mut self) { @@ -393,11 +498,16 @@ impl Drop for Outputs { return; } if let Some(fallback) = self.fallback.take() { - if let Err(error) = self.result { fallback.set_error(error); } + if let Err(error) = self.result { + fallback.set_error(error); + } if let Some(bundle) = &mut self.cumulative { bundle.fail(&Fence::from_fence(&fallback)); } else { - install(self.syncs.drain(self.input_count..), &Fence::from_fence(&fallback)); + install( + self.syncs.drain(self.input_count..), + &Fence::from_fence(&fallback), + ); } fallback.signal(); } diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index c8763f40871d1..578bb7533f3c9 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -41,7 +41,11 @@ pub(crate) struct ComputeCompletion<'a> { /// Records in the compute completion ring that the host scans. pub(super) const COMPUTE_COMPLETION_SLOTS: usize = { let slots = abi::COMPLETION_RING_SIZE / 0x40; - if slots < 0x800 { slots } else { 0x800 } + if slots < 0x800 { + slots + } else { + 0x800 + } }; /// Timestamp bits of a completion record's start and end words. @@ -203,7 +207,13 @@ struct BuildEnv<'a> { impl BuildEnv<'_> { /// An object at a fixed offset of the kernel window. - fn fixed(&self, offset: u64, size: usize, prot: mmu::Prot, cpu: CpuMap) -> Result { + fn fixed( + &self, + offset: u64, + size: usize, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { KernelObject::new( self.dev, self.vm, @@ -215,7 +225,13 @@ impl BuildEnv<'_> { } /// An object placed in the dynamic part of the kernel window. - fn dynamic(&self, size: usize, align: u64, prot: mmu::Prot, cpu: CpuMap) -> Result { + fn dynamic( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { KernelObject::new( self.dev, self.vm, @@ -320,7 +336,12 @@ impl InitData { /// Initialize a fresh physical compute QID's row before exposing its queue. An installed /// or rebound queue keeps the row's firmware-owned counters and must never call this again. /// Called for a fresh row or after its free-list release was consumed. - pub(super) fn initialize_compute_pool_blocks(&self, slot: u16, page_list_va: u64, blocks: usize) -> Result { + pub(super) fn initialize_compute_pool_blocks( + &self, + slot: u16, + page_list_va: u64, + blocks: usize, + ) -> Result { if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { return Err(EINVAL); } @@ -682,23 +703,21 @@ impl InitData { /// Raw views of the memory publishers share without the device mutex: every work ring, /// the completion stamps, the QoS table and the recovery state word. They stay valid for /// this session. - pub(super) fn port_views( - &self, - mailbox: *const super::Mailbox, - ) -> Result { - use super::fw::{channels, queue::{DataMaster, Policy}}; + pub(super) fn port_views(&self, mailbox: *const super::Mailbox) -> Result { + use super::fw::{ + channels, + queue::{DataMaster, Policy}, + }; let ring_of = |priority: u8, engine: DataMaster| -> Result { let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; - let word = |offset: usize| -> Result<*const AtomicU32> { - Ok(state.word(base + offset)?) - }; + let word = + |offset: usize| -> Result<*const AtomicU32> { Ok(state.word(base + offset)?) }; Ok(super::port::RingView { slots: self.object(self.bundle)?.pointer( abi::work_ring_offset(channel), - channels::WORK_RING_SLOTS as usize - * core::mem::size_of::(), + channels::WORK_RING_SLOTS as usize * core::mem::size_of::(), )?, consumers: [ word(abi::ChannelState::CONSUMER)?, @@ -707,13 +726,14 @@ impl InitData { producer: word(abi::ChannelState::PRODUCER)?, }) }; - let rings = |engine: DataMaster| -> Result<[super::port::RingView; Policy::PRIORITIES as usize]> { - let mut views = [ring_of(0, engine)?; Policy::PRIORITIES as usize]; - for (priority, view) in views.iter_mut().enumerate().skip(1) { - *view = ring_of(priority as u8, engine)?; - } - Ok(views) - }; + let rings = + |engine: DataMaster| -> Result<[super::port::RingView; Policy::PRIORITIES as usize]> { + let mut views = [ring_of(0, engine)?; Policy::PRIORITIES as usize]; + for (priority, view) in views.iter_mut().enumerate().skip(1) { + *view = ring_of(priority as u8, engine)?; + } + Ok(views) + }; Ok(super::port::Views { compute: rings(DataMaster::Compute)?, tiling: rings(DataMaster::Tiling)?, @@ -775,13 +795,13 @@ impl InitData { } fence(Ordering::Acquire); let mut newest = None; - for slot in 0..COMPUTE_COMPLETION_SLOTS { + for (slot, seen_control) in seen.iter_mut().enumerate() { let completion = ComputeCompletion { object, offset: slot * 0x40, }; let control = completion.read(0)?; - seen[slot] = control; + *seen_control = control; if control != 0 { let end = completion.read(5)? & COMPLETION_TIME_MASK; if newest.is_none_or(|(_, latest)| end > latest) { @@ -814,6 +834,10 @@ impl InitData { } /// Constructs the graph after the firmware has published the high-root entries. + #[expect( + clippy::too_many_arguments, + reason = "The firmware graph builder keeps its independent input objects explicit." + )] pub(super) fn build( &mut self, dev: &AsahiDrmDevice, @@ -891,7 +915,8 @@ impl InitData { table.init(); Ok(()) })?; - let page_pool = self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + let page_pool = + self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(page_pool)? .initialize::(0, |table| { table.init(low::RENDER_PAGE_LIST, low::COMPUTE_POOL_PAGE_LIST); @@ -975,7 +1000,11 @@ impl InitData { fn build_render_pool(&mut self, env: &BuildEnv<'_>) -> Result { use mmu::PROT_GPU_SHARED_RW as GS; use CpuMap::WriteBack as WB; - let pages = self.add(KernelObject::backing(env.dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; + let pages = self.add(KernelObject::backing( + env.dev, + low::RENDER_PAGE_LIST_SIZE, + WB, + )?)?; self.object_mut(pages)? .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |list| { abi::fill_page_list( @@ -983,10 +1012,18 @@ impl InitData { (0..low::RENDER_BLOCKS_INITIAL as usize).map(low::render_block), ) })?; - let runs = self.add(KernelObject::backing(env.dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; + let runs = self.add(KernelObject::backing( + env.dev, + low::RENDER_RUN_LIST_SIZE, + WB, + )?)?; let first_block = self.objects.len(); for _ in 0..low::RENDER_BLOCKS_ALLOCATED { - self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + self.add(KernelObject::backing( + env.dev, + abi::FREE_LIST_BLOCK_SIZE, + WB, + )?)?; } self.render_pages = pages; self.render_runs = runs; @@ -994,7 +1031,12 @@ impl InitData { self.alias(pages, env.render, low::RENDER_PAGE_LIST, GS)?; self.alias(runs, env.render, low::RENDER_RUN_LIST, GS)?; for block in 0..low::RENDER_BLOCKS_MAPPED { - self.alias(first_block + block, env.render, low::render_block(block), GS)?; + self.alias( + first_block + block, + env.render, + low::render_block(block), + GS, + )?; } Ok(first_block) } @@ -1031,12 +1073,8 @@ impl InitData { .initialize::<[abi::FreeListRunSlot; mmu::UAT_PGSZ / 0x40]>(0, |list| { abi::fill_run_list(list, low::COMPUTE_FREE_LIST_BLOCKS) })?; - let ready = self.add(env.fixed( - high::COMPUTE_READY_FREE_LIST, - mmu::UAT_PGSZ, - GFS, - WC, - )?)?; + let ready = + self.add(env.fixed(high::COMPUTE_READY_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; self.object_mut(ready)?.write( 0, abi::FreeListControl::new(&abi::FreeListArgs { @@ -1427,15 +1465,15 @@ impl InitData { /// Whether a firmware that acknowledges in status A accepted the graph (`+0x14` = 1). pub(super) fn init_accepted(&self) -> Result { - Ok(self.status_word(Role::Primary, 0x14)?.load(Ordering::Acquire) == 1) + Ok(self + .status_word(Role::Primary, 0x14)? + .load(Ordering::Acquire) + == 1) } pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { - self.object(self.cluster)?.word( - self.status_offset(role) - .checked_add(offset) - .ok_or(EINVAL)?, - ) + self.object(self.cluster)? + .word(self.status_offset(role).checked_add(offset).ok_or(EINVAL)?) } pub(super) fn status_read_words( @@ -1443,11 +1481,8 @@ impl InitData { role: Role, offset: usize, ) -> Result<[u32; N]> { - self.object(self.cluster)?.read_words( - self.status_offset(role) - .checked_add(offset) - .ok_or(EINVAL)?, - ) + self.object(self.cluster)? + .read_words(self.status_offset(role).checked_add(offset).ok_or(EINVAL)?) } pub(super) fn control_counters(&self) -> Result<[u32; 3]> { @@ -1540,13 +1575,15 @@ impl InitData { Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), }; - let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) + let end = offset + .checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) .ok_or(EOVERFLOW)?; if end > extent { return Err(EINVAL); } let pointer_at = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + core::mem::offset_of!(abi::MainConfig, view_va) + view * size_of::(); + + core::mem::offset_of!(abi::MainConfig, view_va) + + view * size_of::(); fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; @@ -1559,7 +1596,8 @@ impl InitData { impl super::recovery::Memory for InitData { fn fault_sources(&self) -> Result { - let mask = self.object(self.cluster)? + let mask = self + .object(self.cluster)? .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { @@ -1567,13 +1605,19 @@ impl super::recovery::Memory for InitData { // address is unknown. Zero keeps ownership unknown and cannot override // firmware blame, but preserves the progress evidence for diagnostics. let key = super::recovery::slot_key(slot, self.recovery_slot_keys).unwrap_or(0); - let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, - abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) - .ok().map(|words| words[0])?; + let monitor = self + .recovery_view_words::<6>( + abi::recovery::PROGRESS_VIEW, + abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE, + ) + .ok() + .map(|words| words[0])?; Some((key, monitor)) }); - sources.reason = self.recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) - .ok().map(|words| words[0]); + sources.reason = self + .recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) + .ok() + .map(|words| words[0]); Ok(sources) } diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 0c940f74213b0..711dd59b0ecc2 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -425,8 +425,8 @@ impl Fences { impl Scheduler { pub(crate) fn new(dev: &AsahiDrmDevice, backend: Arc, engine: Engine) -> Result { let (credits, name) = match engine { - Engine::Render => (RENDER_CREDITS, c_str!("asahi_g17_render")), - Engine::Compute => (COMPUTE_CREDITS, c_str!("asahi_g17_compute")), + Engine::Render => (RENDER_CREDITS, c_str!("aurora_g17_render")), + Engine::Compute => (COMPUTE_CREDITS, c_str!("aurora_g17_compute")), }; let scheduler = sched::Scheduler::new(dev.as_ref().as_ref(), 1, credits, 0, TIMEOUT_MS, name)?; let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 6495ad2d98c13..cc3b1297f0467 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -36,29 +36,31 @@ pub(crate) mod sync_output; mod trace; #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) use dependency::inherited_error; +mod capacity; mod event; mod feed; mod fence; mod freelist; pub(crate) mod fw; -mod ids; pub(crate) mod hazard; +mod ids; mod initdata; mod job; -mod capacity; mod kick; mod object; mod port; pub(crate) mod power; -mod reports; mod preparation; mod qos; mod queue; mod recovery; mod replay; +mod reports; mod runtime; +pub(crate) mod stats; pub(crate) mod status; mod teardown; +pub(crate) mod telemetry; mod timeout; mod validation; @@ -75,12 +77,12 @@ use kernel::{ c_str, device::{self, Core}, drm::gem::shmem, + impl_has_delayed_work, impl_has_work, io::{ IoBase, IoSysMap, Region, // }, - impl_has_delayed_work, impl_has_work, new_condvar, new_delayed_work, new_mutex, new_work, platform, prelude::*, soc::apple::rtkit, @@ -734,6 +736,16 @@ impl Firmware { /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, /// unretired work; quarantined work retaining mappings is excluded. Submissions published /// without the device mutex count as activity, and a clear that races one is abandoned. + fn observe_statistics_power(&mut self) { + if self.rings.statistics_suspended() { + return; + } + self.rings.observe_statistics_power( + self.bridge.regs.g17_core_power() & 0xf != 0, + self.port.statistics_epoch(), + ); + } + fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { let submissions = self.port.submissions(); let submitted = submissions != self.seen_submissions; @@ -755,10 +767,12 @@ impl Firmware { } power::IdleAction::Recheck(delay) => Some(delay), }; - Ok(match (effort, self.idle_hold(now_ns, in_flight || submitted)) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - }) + Ok( + match (effort, self.idle_hold(now_ns, in_flight || submitted)) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }, + ) } /// Drives the idle hold (see power::Hold); returns the delay in ns for its next check. A @@ -767,7 +781,10 @@ impl Firmware { const RETRY_NS: u64 = 1_000_000; let hold_ms = power::IDLE_HOLD_MS.load(Ordering::Relaxed); for _ in 0..2 { - match self.hold.step(now_ns, in_flight, self.effort.submissions(), hold_ms) { + match self + .hold + .step(now_ns, in_flight, self.effort.submissions(), hold_ms) + { power::HoldAction::None => return None, power::HoldAction::Recheck(delay) => return Some(delay), power::HoldAction::Assert => { @@ -994,7 +1011,6 @@ impl Shared { self.renders.notify(); } - /// The session's port, valid while `lease` is held: shutdown and device loss join every /// lease before they detach the session. fn port<'a>(&self, lease: &'a preparation::Lease) -> Option<&'a port::Port> { @@ -1126,6 +1142,7 @@ pub(crate) struct Gpu { geometry: mmu::UatGeometry, ids: gpu::SequenceIDs, shared: Arc, + stats: Arc, _capacity: Pin>>, _reports: Pin>>, _power: Pin>>, @@ -1141,7 +1158,10 @@ impl Gpu { ) -> Result> { let calibration = if cfg.chip_id == crate::hw::t6040::CHIP_ID { Some(calibration::Calibration::read(dev).inspect_err(|_| { - dev_err!(dev.as_ref(), "Missing or invalid T6040 GPU calibration handoff\n"); + dev_err!( + dev.as_ref(), + "Missing or invalid T6040 GPU calibration handoff\n" + ); })?) } else { None @@ -1172,11 +1192,14 @@ impl Gpu { }; let max_frequency_khz = Self::max_frequency_khz(dev)?; let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; - let idle_delay = power::idle_delay( - node.get_opt_property(c_str!("apple,idle-off-delay-ms"))?, - )?; + let idle_delay = + power::idle_delay(node.get_opt_property(c_str!("apple,idle-off-delay-ms"))?)?; power::IDLE_HOLD_MS.store(idle_delay, Ordering::Relaxed); - dev_info!(dev.as_ref(), "GPU idle power-off delay: {} ms\n", idle_delay); + dev_info!( + dev.as_ref(), + "GPU idle power-off delay: {} ms\n", + idle_delay + ); let channels = availability::Availability::new()?; let progress = availability::Availability::new()?; let queues = availability::Availability::new()?; @@ -1233,7 +1256,7 @@ impl Gpu { GFP_KERNEL, )?; - let directory = kernel::debugfs::Dir::new(c"asahi-g17"); + let directory = kernel::debugfs::Dir::new(c"aurora-g17"); let capacity = KBox::pin_init( directory.read_only_file(c"capacity", capacity::View(shared.clone())), GFP_KERNEL, @@ -1246,6 +1269,13 @@ impl Gpu { directory.read_only_file(c"power", reports::PowerView(shared.clone())), GFP_KERNEL, )?; + let stats = Arc::new( + crate::stats::StatsSnapshot { + g17_reader: Some(stats::Reader(shared.clone())), + ..Default::default() + }, + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -1254,6 +1284,7 @@ impl Gpu { geometry, ids: Default::default(), shared, + stats, _capacity: capacity, _reports: reports, _power: power, @@ -1282,8 +1313,12 @@ impl Gpu { let version = Self::os_firmware_version(dev); // iBoot strings can be shared by OS releases. Prefer the reported release's // candidate, but admit only an exact loaded image, regardless of that hint. - let preferred = builds.iter().filter(|build| Some(build.os_version) == version); - let others = builds.iter().filter(|build| Some(build.os_version) != version); + let preferred = builds + .iter() + .filter(|build| Some(build.os_version) == version); + let others = builds + .iter() + .filter(|build| Some(build.os_version) != version); for build in Iterator::chain(preferred, others) { if Self::identify_image(dev, build).is_ok() { dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); @@ -1306,13 +1341,10 @@ impl Gpu { let data = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; let start = text.start(); let end = start.checked_add(text.size()).ok_or(EINVAL)?; - crate::fwid::identify( - dev.as_ref().as_ref(), - start..end, - data.size(), - build.images, - ) - .inspect_err(|e| dev_err!(dev.as_ref(), "GPU firmware image not admitted: {:?}\n", e))?; + crate::fwid::identify(dev.as_ref().as_ref(), start..end, data.size(), build.images) + .inspect_err(|e| { + dev_err!(dev.as_ref(), "GPU firmware image not admitted: {:?}\n", e) + })?; Ok(()) } @@ -1372,13 +1404,21 @@ impl Gpu { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) - .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; + let primary = + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary).inspect_err(|e| { + dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e) + })?; let secondary = match cfg.secondary_root_offset { Some(_) => Some( - Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err(|e| { - dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e) - })?, + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err( + |e| { + dev_err!( + dev.as_ref(), + "Secondary coprocessor setup failed: {:?}\n", + e + ) + }, + )?, ), None => None, }; @@ -1401,7 +1441,7 @@ impl Gpu { shared.feed.clone(), )?, seen_submissions: 0, - rings: channel::Rings::new()?, + rings: channel::Rings::new(shared.variant.graph)?, recovery: recovery::State::new(), effort: power::Effort::new(), hold: power::Hold::new(), @@ -1450,13 +1490,18 @@ impl Gpu { session.uat.confirm_firmware_root_entries(root_entries)?; } let mailbox: *const Mailbox = &*session.primary.rtkit; - session.port.attach(session.init.port_views(mailbox)?, session.bridge.regs.clone())?; + session.port.attach( + session.init.port_views(mailbox)?, + session.bridge.regs.clone(), + )?; session.uat.mirror_secondary_root()?; if cfg.fabric_bridge { session.bridge.regs.enable_g17_gating()?; } let ack = shared.variant.init_ack; - session.primary.init(session.init.root_va(Role::Primary)?, ack)?; + session + .primary + .init(session.init.root_va(Role::Primary)?, ack)?; if ack == fw::variant::InitAck::Status { Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; } @@ -1469,7 +1514,12 @@ impl Gpu { secondary.notify(MSG_POWER_ON_DONE)?; } } - Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref())?; + Self::wait_for_opening( + dev, + &session.init, + &session.primary, + session.secondary.as_ref(), + )?; session .uat .activate_render_context(&session._render_global)?; @@ -1562,7 +1612,7 @@ impl gpu::Gpu for Gpu { } fn stats_snapshot(&self) -> Option> { - None + Some(self.stats.clone()) } fn manager(&self) -> Option<&dyn gpu::GpuManager> { @@ -1590,8 +1640,7 @@ impl gpu::Gpu for Gpu { | sync_output::FEATURE_CUMULATIVE_SIGNALS; let variant = self.shared.variant; if variant.few_primitives { - params.features |= - u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); + params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); } if variant.ray_traversal { params.features |= @@ -1673,6 +1722,7 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { let dev = self.shared.drm()?; let owner = self.ids.queue.next(); @@ -1693,13 +1743,17 @@ impl gpu::Gpu for Gpu { // until the scheduler release is consumed and reclaimed. // Backend construction has rolled back all temporary leases. let waited = now_ns().saturating_sub(admission_started); - let limit = job::TIMEOUT_MS as u64 * 1_000_000; - if waited >= limit { return Err(EBUSY); } + let limit = u64::from(job::TIMEOUT_MS) * 1_000_000; + if waited >= limit { + return Err(EBUSY); + } self.shared.queue_events(); // A returned queue slot or reclaimed context wakes this. - self.shared - .queues - .block(observed, (limit - waited).div_ceil(1_000_000) as u32, true)?; + self.shared.queues.block( + observed, + (limit - waited).div_ceil(1_000_000) as u32, + true, + )?; } result => break result?, } @@ -1710,6 +1764,7 @@ impl gpu::Gpu for Gpu { backend.clone(), frontend, slot, + usage, ) { Ok(frontend) => frontend, Err(error) => { diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 6d0e87c14b947..6f9d1c7d956c0 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -116,6 +116,7 @@ pub(super) struct Port { /// Full firmware effort is applied; cleared before an idle clear rechecks `submissions`. effort_full: AtomicBool, demand: super::power::Demand, + statistics_epoch: AtomicU64, last_submission_ns: AtomicU64, submissions: AtomicU64, } @@ -172,6 +173,7 @@ impl Port { publications: AtomicU64::new(0), effort_full: AtomicBool::new(false), demand, + statistics_epoch: AtomicU64::new(0), last_submission_ns: AtomicU64::new(0), submissions: AtomicU64::new(0), }), @@ -225,7 +227,11 @@ impl Port { /// Exposes `record` in its reserved slot. Inner queue records must be visible first. Returns /// the new producer, the retained consumer witness for this slot. - pub(super) fn publish_compute(&self, mut claim: OuterClaim<'_>, record: &WorkSlot) -> Result { + pub(super) fn publish_compute( + &self, + mut claim: OuterClaim<'_>, + record: &WorkSlot, + ) -> Result { if record.data_master != DataMaster::Compute as u32 || !core::ptr::eq(claim.port, self) { return Err(EINVAL); } @@ -242,7 +248,8 @@ impl Port { unsafe { core::ptr::copy_nonoverlapping( core::ptr::from_ref(record).cast::(), - ring.slots.add(producer as usize * core::mem::size_of::()), + ring.slots + .add(producer as usize * core::mem::size_of::()), core::mem::size_of::(), ); } @@ -280,13 +287,16 @@ impl Port { let (fragment, tiling) = self.render_rings(priority)?; let cursors = [fragment.cursors()?, tiling.cursors()?]; let room = |[first, second, producer]: [u32; 3]| { - let room = |consumer: u32| (consumer + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS; + let room = + |consumer: u32| (consumer + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS; room(first).min(room(second)) }; - Ok((room(cursors[0]) > 0 && room(cursors[1]) > 0).then_some(RenderRoom { - priority, - producers: [cursors[0][2], cursors[1][2]], - })) + Ok( + (room(cursors[0]) > 0 && room(cursors[1]) > 0).then_some(RenderRoom { + priority, + producers: [cursors[0][2], cursors[1][2]], + }), + ) } /// Copies a render pair's fragment, then tiling record into the slots after the producers @@ -312,7 +322,10 @@ impl Port { return Err(EINVAL); } // SAFETY: The ring view spans WORK_RING_SLOTS records; `producer` is in range. - let slot = unsafe { ring.slots.add(producer as usize * core::mem::size_of::()) }; + let slot = unsafe { + ring.slots + .add(producer as usize * core::mem::size_of::()) + }; Ok((slot, (producer + 1) % WORK_RING_SLOTS)) }; let (fragment_slot, fragment_next) = slot(&fragment_ring, room.producers[0])?; @@ -385,7 +398,10 @@ impl Port { policy: Policy, ) -> Result { let view = self.qos_view()?; - let publication = self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock)?; + let publication = self + .qos + .lock() + .publish(&view, owner, scheduler, policy, qos::clock)?; self.rx_update(); Ok(publication) } @@ -399,7 +415,9 @@ impl Port { ) -> Result<[qos::Publication; 2]> { let view = self.qos_view()?; let publications = - self.qos.lock().publish_owners(&view, owners, scheduler, policy, qos::clock)?; + self.qos + .lock() + .publish_owners(&view, owners, scheduler, policy, qos::clock)?; self.rx_update(); Ok(publications) } @@ -484,14 +502,33 @@ impl Port { self.demand.set_active(full); } + pub(super) fn statistics_epoch(&self) -> u64 { + self.statistics_epoch.load(Ordering::Acquire) + } + pub(super) fn note_gpu_activity(&self) { - self.last_submission_ns.store(super::now_ns(), Ordering::Relaxed); + // Observe gating before activity can wake the cores. A lockless + // publisher cannot clear the decoder under the device mutex; this + // epoch makes both the next drain and the next reader discard it. + if self + .regs + .as_ref() + .is_some_and(|regs| regs.g17_core_power() & 0xf == 0) + { + self.statistics_epoch.fetch_add(1, Ordering::Release); + } + self.last_submission_ns + .store(super::now_ns(), Ordering::Relaxed); self.demand.set_active(true); } pub(super) fn write_power(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "effort_full={} direct_submissions={} last_submission_ns={}", - self.effort_full.load(Ordering::SeqCst), self.submissions(), - self.last_submission_ns.load(Ordering::Relaxed)) + writeln!( + f, + "effort_full={} direct_submissions={} last_submission_ns={}", + self.effort_full.load(Ordering::SeqCst), + self.submissions(), + self.last_submission_ns.load(Ordering::Relaxed) + ) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 7ff828b1e3602..368e95f144c24 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -35,7 +35,8 @@ const NATIVE_WAIT_STRIDE: usize = 0x60; // A submission can name every physical compute channel. Keep the complete // wait list inside its own cached slot, including the last retained slot. const CACHED_RECORD_STRIDE: usize = (NATIVE_WAIT_OFFSET - + crate::g17::kick::COMPUTE_CHANNELS * NATIVE_WAIT_STRIDE).next_power_of_two(); + + crate::g17::kick::COMPUTE_CHANNELS * NATIVE_WAIT_STRIDE) + .next_power_of_two(); const CONFIG: usize = 0; const ANNOUNCE: usize = 0x800; const PREDECESSOR: usize = 0x400; @@ -280,7 +281,10 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self.graph.word(CURSORS + self.pointers.producer)?.load(Ordering::Relaxed), + producer: self + .graph + .word(CURSORS + self.pointers.producer)? + .load(Ordering::Relaxed), consumer: self.graph.word(CURSORS)?.load(Ordering::Relaxed), count: self.graph.word(CURSORS + 0x60)?.load(Ordering::Relaxed), }) @@ -332,7 +336,9 @@ impl Graph { return Err(EIO); } for index in 0..ITEM_RING_ENTRIES as usize { - self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); + self.graph + .dword(ITEMS + index * 8)? + .store(0, Ordering::Relaxed); } fence(Ordering::SeqCst); self.graph @@ -685,7 +691,10 @@ impl Queue { /// never runs, or the failed command itself, whose ring entry it does not consume (a ray /// traversal's, on Neo). Only a kill of the queue's context gives them back. pub(crate) fn fault_pending(&self) -> bool { - self.fault_stamp.is_some() && self.quarantined && self.awaiting_witness && self.exit.is_none() + self.fault_stamp.is_some() + && self.quarantined + && self.awaiting_witness + && self.exit.is_none() } /// The stamp that proves retirement: the stored one, or a newer reported fault's. @@ -724,7 +733,9 @@ impl Queue { } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { - !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + !self + .context() + .is_some_and(|bound| Arc::ptr_eq(bound, context)) || (self.active.len == 0 && self.replays.len == 0) } pub(crate) fn in_flight(&self) -> bool { @@ -801,7 +812,10 @@ impl Queue { qid: self.qid(), // Accounting completes only for bound owners; a detached queue has no // pending work and never reaches this. - qos: self.binding.as_ref().map_or(0, |binding| binding.context.qos_id()), + qos: self + .binding + .as_ref() + .map_or(0, |binding| binding.context.qos_id()), data_master: DataMaster::Compute as u8, } } @@ -818,7 +832,8 @@ impl Queue { } pub(crate) fn pool_release_stalled(&self, now: u64) -> bool { - self.pool_release_pending() && crate::g17::timeout::background_stalled(self.release_sent_ns, now) + self.pool_release_pending() + && crate::g17::timeout::background_stalled(self.release_sent_ns, now) } /// A closed owner can still retain its last commands or their completion @@ -827,7 +842,12 @@ impl Queue { pub(crate) fn release_settling(&self) -> bool { // release_owner keeps the old owner while its last commands retire. // Waiting only once owner is None misses precisely that interval. - release_pending(self.released, self.quarantined, self.active.len, self.retire_pending) + release_pending( + self.released, + self.quarantined, + self.active.len, + self.retire_pending, + ) } fn selected(&self, masks: Option<[u64; 2]>) -> bool { @@ -1029,8 +1049,13 @@ impl Queue { let front = self.active.front().ok_or(EIO)?; if let Some(ticket) = front.ticket { crate::g17::recovery::note_retired(); - crate::g17::trace::complete(crate::g17::trace::COMPUTE, ticket.qid, - ticket.ordinal, &front.packet, [timestamps[0], timestamps[1]]); + crate::g17::trace::complete( + crate::g17::trace::COMPUTE, + ticket.qid, + ticket.ordinal, + &front.packet, + [timestamps[0], timestamps[1]], + ); } self.retire_pending = true; defer(Deferred::Retired( @@ -1251,7 +1276,9 @@ impl Queue { /// Validate the complete unpublished item budget before changing descriptors, /// work counts or queue producers. Leave one item empty across ring wrap. fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { - if cursors.count != ITEM_RING_ENTRIES { return Err(EIO); } + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { return Err(ERANGE); } @@ -1260,14 +1287,18 @@ fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { let _ = crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeItems); return Err(EAGAIN); } - Ok(()) + Ok(()) } /// Retained commands are capacity, not a failed VM. Nothing has been written /// when this check runs, so the outer publisher can wait for retirement under /// its original admission deadline, just as it does for a full firmware ring. fn retained_capacity( - quarantined: bool, released: bool, active: usize, replays: usize, replay_front: bool, + quarantined: bool, + released: bool, + active: usize, + replays: usize, + replay_front: bool, ) -> Result { if quarantined || released { return Err(EBUSY); @@ -1293,8 +1324,13 @@ mod admission_tests { let waits = crate::g17::kick::COMPUTE_CHANNELS; for slot in 0..RECORD_SLOTS { for index in 0..waits { - let start = record_start(slot, NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, - NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).unwrap(); + let start = record_start( + slot, + NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, + CACHED_RECORD_STRIDE, + ) + .unwrap(); assert!(start >= slot * CACHED_RECORD_STRIDE); assert!(start + NATIVE_WAIT_STRIDE <= (slot + 1) * CACHED_RECORD_STRIDE); } @@ -1305,13 +1341,27 @@ mod admission_tests { fn record_window_cannot_overwrite_the_next_slot() { // The nineteenth wait used to cross the old slot boundary silently. let offset = NATIVE_WAIT_OFFSET + 18 * NATIVE_WAIT_STRIDE; - assert_eq!(record_start(0, offset, NATIVE_WAIT_STRIDE, RECORD_STRIDE), Err(EINVAL)); + assert_eq!( + record_start(0, offset, NATIVE_WAIT_STRIDE, RECORD_STRIDE), + Err(EINVAL) + ); assert!(record_start(0, offset, NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).is_ok()); - assert_eq!(record_start(RECORD_SLOTS, 0, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); - assert_eq!(record_start(0, usize::MAX, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); - assert_eq!(record_start(0, CACHED_RECORD_STRIDE, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); - assert_eq!(record_start(0, CACHED_RECORD_STRIDE - 1, 1, CACHED_RECORD_STRIDE), - Ok(CACHED_RECORD_STRIDE - 1)); + assert_eq!( + record_start(RECORD_SLOTS, 0, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, usize::MAX, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, CACHED_RECORD_STRIDE, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, CACHED_RECORD_STRIDE - 1, 1, CACHED_RECORD_STRIDE), + Ok(CACHED_RECORD_STRIDE - 1) + ); } #[test] @@ -1320,11 +1370,17 @@ mod admission_tests { for sequence in 0..IN_FLIGHT { active.push(sequence).unwrap(); } - assert_eq!(retained_capacity(false, false, active.len, 0, false), Err(EAGAIN)); + assert_eq!( + retained_capacity(false, false, active.len, 0, false), + Err(EAGAIN) + ); assert_eq!(active.front(), Some(&0)); assert_eq!(active.len, IN_FLIGHT); assert_eq!(active.pop(), Some(0)); - assert_eq!(retained_capacity(false, false, active.len, 0, false), Ok(())); + assert_eq!( + retained_capacity(false, false, active.len, 0, false), + Ok(()) + ); assert_eq!(active.front(), Some(&1)); } @@ -1332,28 +1388,114 @@ mod admission_tests { fn replay_front_and_terminal_ownership_remain_distinct() { assert_eq!(retained_capacity(false, false, 0, 1, false), Err(EAGAIN)); assert_eq!(retained_capacity(false, false, 0, 1, true), Ok(())); - assert_eq!(retained_capacity(false, false, IN_FLIGHT, 1, true), Err(EAGAIN)); + assert_eq!( + retained_capacity(false, false, IN_FLIGHT, 1, true), + Err(EAGAIN) + ); for (quarantined, released) in [(true, false), (false, true), (true, true)] { - assert_eq!(retained_capacity(quarantined, released, 0, 1, true), Err(EBUSY)); + assert_eq!( + retained_capacity(quarantined, released, 0, 1, true), + Err(EBUSY) + ); } } #[test] fn item_refusal_is_retryable_and_preserves_the_empty_slot() { let n = ITEM_RING_ENTRIES; - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, false), Ok(())); - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, true), Err(EAGAIN)); - assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, false), Ok(())); - assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, true), Err(EAGAIN)); - assert_eq!(item_capacity(Cursors { producer: 10, consumer: 10, count: n }, true), Ok(())); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 4, + count: n + }, + false + ), + Ok(()) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 4, + count: n + }, + true + ), + Err(EAGAIN) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n - 2, + consumer: 2, + count: n + }, + false + ), + Ok(()) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n - 2, + consumer: 2, + count: n + }, + true + ), + Err(EAGAIN) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 10, + consumer: 10, + count: n + }, + true + ), + Ok(()) + ); } #[test] fn bad_item_cursors_are_not_backpressure() { let n = ITEM_RING_ENTRIES; - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 0, count: 0 }, false), Err(EIO)); - assert_eq!(item_capacity(Cursors { producer: n, consumer: 0, count: n }, false), Err(ERANGE)); - assert_eq!(item_capacity(Cursors { producer: 0, consumer: n, count: n }, true), Err(ERANGE)); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 0, + count: 0 + }, + false + ), + Err(EIO) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n, + consumer: 0, + count: n + }, + false + ), + Err(ERANGE) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: n, + count: n + }, + true + ), + Err(ERANGE) + ); } #[test] @@ -1389,7 +1531,9 @@ impl Queue { Ok(true) } pub(crate) fn reusable(&self) -> bool { - if self.growing { return false; } + if self.growing { + return false; + } self.installed // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) @@ -1454,8 +1598,12 @@ impl Queue { } context.mark_published(); context.note_compute_binding(true); - crate::g17::trace::bind(crate::g17::trace::COMPUTE, self.qid(), context.id(), - if same_vm { 2 } else { 1 }); + crate::g17::trace::bind( + crate::g17::trace::COMPUTE, + self.qid(), + context.id(), + if same_vm { 2 } else { 1 }, + ); self.owner = Some(owner); self.unpublished_binding = true; self.released = false; @@ -1523,7 +1671,8 @@ impl Queue { } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { - self.pool_phase == PoolPhase::Populated && self.pool.populated() + self.pool_phase == PoolPhase::Populated + && self.pool.populated() && (self.reusable() || self.recycled_pending()) && !self.source_hint.has_readers(&self.source_readers) } @@ -1619,9 +1768,8 @@ impl Queue { if !self.vacate_candidate() { return Err(EBUSY); } - let record = ControlRecord::FreeListRelease( - self.pool.release_record(self.free_list_generation), - ); + let record = + ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); self.release_sent_ns = Some(crate::g17::now_ns()); @@ -1636,7 +1784,9 @@ impl Queue { consumer: u32, producer: u32, ) -> Result> { - if self.growing { return Ok(None); } + if self.growing { + return Ok(None); + } let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { return Ok(None); }; @@ -1645,11 +1795,18 @@ impl Queue { } // The retained owner binding's client alias pins the backing: detach it // first (the owner has exited; `reusable()` proved no previous binding). - let alias = self.binding.as_mut().and_then(|binding| binding.pool_alias.take()); + let alias = self + .binding + .as_mut() + .and_then(|binding| binding.pool_alias.take()); let pages = self.pool.take_pages(); - let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); + let descriptors = self + .binding + .as_mut() + .and_then(|binding| binding.descriptors.take()); let ring = self.graph.vacate_ring(); - self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; + self.pool_phase = PoolPhase::Vacant; + self.release_sent_ns = None; Ok(Some(Vacated { _alias: alias, _pages: pages, @@ -1664,7 +1821,9 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if !self.can_rearm() { return Err(EAGAIN); } + if !self.can_rearm() { + return Err(EAGAIN); + } self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; @@ -1678,27 +1837,50 @@ impl Queue { /// Only a capacity increase waits for this queue's previously published work. /// Other queues and ordinary submissions keep their existing concurrency. - pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { - if self.growing { return Err(EAGAIN); } - if self.quarantined || self.released || self.exit.is_some() - || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() + pub(crate) fn begin_growth( + &mut self, + init: &crate::g17::initdata::InitData, + blocks: usize, + ) -> Result { + if self.growing { + return Err(EAGAIN); + } + if self.quarantined + || self.released + || self.exit.is_some() + || self.active.len != 0 + || self.replays.len != 0 + || self.previous.is_some() || (self.installed && !self.retirement_ready) - || self.pool_phase != PoolPhase::Populated { return Err(EAGAIN); } + || self.pool_phase != PoolPhase::Populated + { + return Err(EAGAIN); + } if self.installed { - let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); + let record = + ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); - self.release_sent_ns = Some(crate::g17::now_ns()); - } else { self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; } + self.release_sent_ns = Some(crate::g17::now_ns()); + } else { + self.pool_phase = PoolPhase::Vacant; + self.release_sent_ns = None; + } self.growing = true; self.growth_blocks = blocks; Ok(self.installed) } - pub(crate) fn finish_growth(&mut self, init: &crate::g17::initdata::InitData, + pub(crate) fn finish_growth( + &mut self, + init: &crate::g17::initdata::InitData, pages: &mut Option, - alias: &mut Option, blocks: usize) -> Result { - if !self.growing || blocks < self.growth_blocks { return Err(EAGAIN); } + alias: &mut Option, + blocks: usize, + ) -> Result { + if !self.growing || blocks < self.growth_blocks { + return Err(EAGAIN); + } if let PoolPhase::ReleaseSent(cursor) = self.pool_phase { let consumer = init.control_consumer()?.load(Ordering::Acquire); let producer = init.control_producer()?.load(Ordering::Acquire); @@ -1706,7 +1888,9 @@ impl Queue { return Err(EAGAIN); } } - if alias.is_none() { return Err(EINVAL); } + if alias.is_none() { + return Err(EINVAL); + } let fresh = pages.as_ref().ok_or(EINVAL)?; let binding = self.binding.as_mut().ok_or(EFAULT)?; // No ownership moves before every fallible descriptor/header operation. @@ -1718,11 +1902,19 @@ impl Queue { self.release_sent_ns = None; self.growing = false; self.growth_blocks = 0; - Ok(Vacated { _alias: old_alias, _pages: pages.take(), _descriptors: None, _ring: None }) + Ok(Vacated { + _alias: old_alias, + _pages: pages.take(), + _descriptors: None, + _ring: None, + }) } pub(crate) fn growth_token(&self) -> Option { - self.growing.then_some(Growth { qid: self.qid(), phase: self.pool_phase }) + self.growing.then_some(Growth { + qid: self.qid(), + phase: self.pool_phase, + }) } /// Transfer an abandoned builder's release back to the event worker. Do not @@ -1885,7 +2077,9 @@ fn record_start(slot: usize, offset: usize, window: usize, stride: usize) -> Res if slot >= RECORD_SLOTS || offset > stride || window > stride - offset { return Err(EINVAL); } - slot.checked_mul(stride).and_then(|start| start.checked_add(offset)).ok_or(EINVAL) + slot.checked_mul(stride) + .and_then(|start| start.checked_add(offset)) + .ok_or(EINVAL) } impl Graph { @@ -1957,8 +2151,7 @@ pub(crate) trait Host { ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; - fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) - -> Result; + fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; fn notify(&mut self, message: u64) -> Result; fn note_submission(&mut self) -> Result; } @@ -2023,8 +2216,12 @@ impl Queue { let first = !self.installed && self.ordinal == 0; // The first prototype only waits on an installed consumer channel. // A retry without a published source falls back to completion on CPU. - if first && !native.is_empty() { return Err(EAGAIN); } - if native.len() > crate::g17::kick::COMPUTE_CHANNELS { return Err(EINVAL); } + if first && !native.is_empty() { + return Err(EAGAIN); + } + if native.len() > crate::g17::kick::COMPUTE_CHANNELS { + return Err(EINVAL); + } let entry_signal = first || dependencies .iter() @@ -2081,7 +2278,9 @@ impl Queue { let policy = packet.context.policy(); let ring = 1u8 << policy.priority(); let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; - let cached = self.config_cache.and_then(|(config, at)| (at == epoch).then_some(config)); + let cached = self + .config_cache + .and_then(|(config, at)| (at == epoch).then_some(config)); let send_config = first || super::render::publication::required(cached, config); let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; @@ -2155,10 +2354,16 @@ impl Queue { let cursors = self.graph.cursors()?; let mut wait_vas = KVec::with_capacity(native.len(), GFP_KERNEL)?; for (index, wait) in native.iter().enumerate() { - let record = wait.bytes(crate::g17::fw::stamp_value(kick_timestamp.get() as u32), - packet.order.sequence); - let address = self.graph.record(record_slot, NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, - NATIVE_WAIT_STRIDE, record)?; + let record = wait.bytes( + crate::g17::fw::stamp_value(kick_timestamp.get() as u32), + packet.order.sequence, + ); + let address = self.graph.record( + record_slot, + NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, + record, + )?; wait_vas.push(address, GFP_KERNEL)?; } let prepared_kick = self.kick.prepare_entry(&kick_args)?; @@ -2207,8 +2412,11 @@ impl Queue { packet.mark_published(); visible = true; }) - })().or_else(|error| { - if !visible { self.pool.cancel_unpublished(target)?; } + })() + .or_else(|error| { + if !visible { + self.pool.cancel_unpublished(target)?; + } Err(error) })?; visible = true; @@ -2336,8 +2544,13 @@ impl Queue { .replays .front() .is_some_and(|front| Arc::ptr_eq(front, packet)); - retained_capacity(self.quarantined, self.released, self.active.len, - self.replays.len, replay_front)?; + retained_capacity( + self.quarantined, + self.released, + self.active.len, + self.replays.len, + replay_front, + )?; if !packet.context.is_current() || !self .binding @@ -2425,17 +2638,30 @@ impl Queue { u8::from(self.quarantined), u8::from(self.awaiting_witness), u8::from(self.previous.is_some()), u8::from(self.exit.is_some()), u8::from(self.growing), self.pool_phase, self.active.len, IN_FLIGHT)?; match self.graph.cursors() { - Ok(cursors) if cursors.count != 0 => writeln!(f, "items_used={} items_total={}", - cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, cursors.count - 1), + Ok(cursors) if cursors.count != 0 => writeln!( + f, + "items_used={} items_total={}", + cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, + cursors.count - 1 + ), _ => writeln!(f, "items_unavailable=1"), }?; - writeln!(f, "compute_settling qid={} release={} pool_release={}", self.qid(), - u8::from(self.release_settling()), u8::from(self.pool_release_pending()))?; + writeln!( + f, + "compute_settling qid={} release={} pool_release={}", + self.qid(), + u8::from(self.release_settling()), + u8::from(self.pool_release_pending()) + )?; if let Some(binding) = self.binding.as_ref() { - binding.context.write_capacity(f, "compute", usize::from(self.qid()))?; + binding + .context + .write_capacity(f, "compute", usize::from(self.qid()))?; } if let Some(previous) = self.previous.as_ref() { - previous.context.write_capacity(f, "compute_previous", usize::from(self.qid()))?; + previous + .context + .write_capacity(f, "compute_previous", usize::from(self.qid()))?; } Ok(()) } @@ -2544,10 +2770,18 @@ impl Queue { /// An open client's idle channel that [`Self::share_idle`] can hand over. pub(crate) fn shareable(&self) -> bool { - self.claims == 0 && self.installed && self.owner.is_some() && self.retirement_ready - && !self.growing && !self.quarantined && !self.awaiting_witness - && self.exit.is_none() && self.active.len == 0 && self.replays.len == 0 - && self.previous.is_none() && self.pool_phase == PoolPhase::Populated + self.claims == 0 + && self.installed + && self.owner.is_some() + && self.retirement_ready + && !self.growing + && !self.quarantined + && !self.awaiting_witness + && self.exit.is_none() + && self.active.len == 0 + && self.replays.len == 0 + && self.previous.is_none() + && self.pool_phase == PoolPhase::Populated && !self.source_hint.has_readers(&self.source_readers) } @@ -2560,7 +2794,8 @@ impl Queue { impl Queue { pub(crate) fn can_rearm(&self) -> bool { - self.pool_phase == PoolPhase::Vacant && (self.reusable() || self.recycled_pending()) + self.pool_phase == PoolPhase::Vacant + && (self.reusable() || self.recycled_pending()) && !self.source_hint.has_readers(&self.source_readers) } } @@ -2571,19 +2806,35 @@ impl Queue { /// caller validates the current preparation epoch and cancels only a true /// result's never-published QID; installed rings and identities stay intact. pub(crate) fn rollback_unpublished( - &mut self, owner: u64, context: &Arc, + &mut self, + owner: u64, + context: &Arc, ) -> Result<(bool, Option)> { - if self.owner != Some(owner) || !self.unpublished_binding - || self.binding.as_ref().is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) - || self.growing || self.quarantined || self.awaiting_witness - || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 - || self.retire_pending || self.failure_status_pending + if self.owner != Some(owner) + || !self.unpublished_binding + || self + .binding + .as_ref() + .is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) + || self.growing + || self.quarantined + || self.awaiting_witness + || self.exit.is_some() + || self.active.len != 0 + || self.replays.len != 0 + || self.retire_pending + || self.failure_status_pending || self.pool_phase != PoolPhase::Populated - || self.previous.as_ref().is_some_and(|previous| previous.publication.is_some()) + || self + .previous + .as_ref() + .is_some_and(|previous| previous.publication.is_some()) { return Ok((false, None)); } - if !self.installed { return Ok((true, None)); } + if !self.installed { + return Ok((true, None)); + } let mut retired = None; if let Some(previous) = self.previous.as_ref() { self.graph.set_owner(&previous.context)?; @@ -2604,37 +2855,61 @@ impl Queue { } } - impl Queue { /// Same immutable singleton identity and current physical epoch. - pub(crate) fn has_input_source(&self, source: &Arc, - status: &Arc, epoch: u64) -> bool - { - !self.quarantined && self.replays.len == 0 && self.active.iter().any(|active| - active.ticket.is_some_and(|ticket| ticket.epoch == epoch) - && Arc::ptr_eq(active.packet.completion.status(), status) - && active.packet.completion.is_input_source(source)) - } - - pub(crate) fn source_dependency(&mut self, source: &crate::g17::sync_input::Source, - status: &Arc, epoch: u64) -> Result> - { + pub(crate) fn has_input_source( + &self, + source: &Arc, + status: &Arc, + epoch: u64, + ) -> bool { + !self.quarantined + && self.replays.len == 0 + && self.active.iter().any(|active| { + active.ticket.is_some_and(|ticket| ticket.epoch == epoch) + && Arc::ptr_eq(active.packet.completion.status(), status) + && active.packet.completion.is_input_source(source) + }) + } + + pub(crate) fn source_dependency( + &mut self, + source: &crate::g17::sync_input::Source, + status: &Arc, + epoch: u64, + ) -> Result> { for active in self.active.iter() { - if !active.packet.completion.is_input_source(&source.submission) { continue; } - if !Arc::ptr_eq(active.packet.completion.status(), status) { return Err(EINVAL); } - if self.quarantined || self.replays.len != 0 { return Err(EAGAIN); } + if !active.packet.completion.is_input_source(&source.submission) { + continue; + } + if !Arc::ptr_eq(active.packet.completion.status(), status) { + return Err(EINVAL); + } + if self.quarantined || self.replays.len != 0 { + return Err(EAGAIN); + } let ticket = active.ticket.ok_or(EAGAIN)?; - if ticket.epoch != epoch { return Err(EAGAIN); } + if ticket.epoch != epoch { + return Err(EAGAIN); + } self.valid_ticket(active, &ticket)?; - if active.packet.cancel_requested() { return Err(ECANCELED); } + if active.packet.cancel_requested() { + return Err(ECANCELED); + } source.submission.source_status()?; - let dependency = crate::g17::fw::kick::KickDependency::new(self.qid(), - crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?).ok_or(EIO)?; + let dependency = crate::g17::fw::kick::KickDependency::new( + self.qid(), + crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?, + ) + .ok_or(EIO)?; let hold = self.source_hint.hold(&self.source_readers)?; - return Ok(Some((active.published, dependency, + return Ok(Some(( + active.published, + dependency, self.graph.support.gpu_va() + STAMP as u64, - u32::from(self.qid()), hold))); + u32::from(self.qid()), + hold, + ))); } Ok(None) } @@ -2650,3 +2925,12 @@ impl Queue { } } } + +/// Publication, dependency, stamp address, queue and the retained source hold. +pub(crate) type SourceDependency = ( + u64, + crate::g17::fw::kick::KickDependency, + u64, + u32, + crate::g17::sync_input::Hold, +); diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index f6637f2858c26..b7037a1c35f75 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -70,7 +70,7 @@ impl Frontend { return Err(EINVAL); } let window = UscWindow::new(usc_base, range.clone())?; - let fences = FenceContexts::new(1, c_str!("asahi_g17_queue"), FENCE_KEY)?; + let fences = FenceContexts::new(1, c_str!("aurora_g17_queue"), FENCE_KEY)?; let allocation_failure = super::fence::allocation_failure(&fences)?; Ok(Self { fences, @@ -85,6 +85,7 @@ impl Frontend { pub(crate) struct Queue { dev: AsahiDevRef, context: Arc, + usage: Arc, backend: Arc, render: Option>, compute: Option>, @@ -105,7 +106,10 @@ pub(crate) struct Queue { _slot: admission::QueueSlot, } -type ValidatedBatch = (KVec<(Validated, Order, Destinations)>, KVec); +type ValidatedBatch = ( + KVec<(Validated, Order, Destinations)>, + KVec, +); impl Queue { pub(crate) fn new( @@ -114,6 +118,7 @@ impl Queue { backend: Arc, frontend: Frontend, slot: admission::QueueSlot, + usage: Arc, ) -> Result { let Frontend { fences, @@ -126,6 +131,7 @@ impl Queue { Ok(Self { dev: dev.into(), context, + usage, backend, render, compute: None, @@ -160,7 +166,11 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; // Without room the entries are released here, as they always could be. - let mut finished = if self.finished.reserve(self.frontiers[index].len(), GFP_KERNEL).is_ok() { + let mut finished = if self + .finished + .reserve(self.frontiers[index].len(), GFP_KERNEL) + .is_ok() + { Some(&mut self.finished) } else { None @@ -198,13 +208,19 @@ impl Queue { }; let index = engine.index(); if let Validated::Render { ray_query, .. } = &command { - let blocks = self.backend.render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; - if blocks > self.context.render_blocks() { return Err(EINVAL); } + let blocks = self + .backend + .render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; + if blocks > self.context.render_blocks() { + return Err(EINVAL); + } } self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { - let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; + let Validated::Compute { scratch, .. } = &command else { + return Err(EINVAL); + }; self.ensure_compute(scratch.pages())?; } // Only convert inputs when no independent host-prefix wait could keep @@ -212,21 +228,34 @@ impl Queue { if engine == Engine::Compute { super::trace::inputs(self.context.id(), order.sequence, 0, inputs.len(), None); } - let sources = if engine == Engine::Compute && !inputs.is_empty() + let sources = if engine == Engine::Compute + && !inputs.is_empty() && !destinations.has_outputs() - && self.frontiers[0].iter().all(|(sequence, fences)| - !Order::contains(order.wait_through[0], *sequence) - || fences.status() > 0) - && self.frontiers[1].iter().all(|(sequence, fences)| + && self.frontiers[0].iter().all(|(sequence, fences)| { + !Order::contains(order.wait_through[0], *sequence) || fences.status() > 0 + }) + && self.frontiers[1].iter().all(|(sequence, fences)| { !Order::contains(order.wait_through[1], *sequence) || fences.ready.raw() != fences.completed.raw() - || fences.status() > 0) - && self.failures.iter_mut().enumerate().all(|(index, failure)| - failure.earliest().is_none_or(|(sequence, _)| - !Order::contains(order.wait_through[index], sequence))) - { self.backend.input_sources(inputs) } else { None }; + || fences.status() > 0 + }) + && self.failures.iter_mut().enumerate().all(|(index, failure)| { + failure.earliest().is_none_or(|(sequence, _)| { + !Order::contains(order.wait_through[index], sequence) + }) + }) { + self.backend.input_sources(inputs) + } else { + None + }; if engine == Engine::Compute { - super::trace::inputs(self.context.id(), order.sequence, 1, inputs.len(), sources.as_deref()); + super::trace::inputs( + self.context.id(), + order.sequence, + 1, + inputs.len(), + sources.as_deref(), + ); } // Only earlier render ordinals covered by the explicit logical prefix @@ -234,8 +263,7 @@ impl Queue { let host_predecessors = self.frontiers[index] .iter() .take_while(|(sequence, _)| { - engine == Engine::Render - && Order::contains(order.wait_through[index], *sequence) + engine == Engine::Render && Order::contains(order.wait_through[index], *sequence) }) .filter_map(|(_, fences)| fences.publication.as_ref()); let completion = Completion::new( @@ -249,8 +277,10 @@ impl Queue { host_predecessors.clone().next().is_some(), self.backend.feed(), sources.clone(), + self.usage.clone(), )?; - let publication = completion.publication() + let publication = completion + .publication() .map(|target| PreparedPublication::new(target, host_predecessors)) .transpose()?; if engine == Engine::Compute { @@ -288,8 +318,10 @@ impl Queue { }); } let failure_count = failures.iter().flatten().count(); - let mut dependencies = - KVec::with_capacity(inputs.len() + count + timestamp_count + failure_count, GFP_KERNEL)?; + let mut dependencies = KVec::with_capacity( + inputs.len() + count + timestamp_count + failure_count, + GFP_KERNEL, + )?; let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count + failure_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; @@ -327,7 +359,10 @@ impl Queue { if job::fence_status(fence) > 0 { continue; } - if sources.as_ref().is_some_and(|sources| sources.contains_input(fence)) { + if sources + .as_ref() + .is_some_and(|sources| sources.contains_input(fence)) + { continue; } dependencies.push(fence.clone(), GFP_KERNEL)?; @@ -357,10 +392,11 @@ impl Queue { Engine::Render => &mut self.render, Engine::Compute => &mut self.compute, }; - let fences = scheduler - .as_mut() - .ok_or(ENODEV)? - .enqueue(packet, dependencies, early, publication)?; + let fences = + scheduler + .as_mut() + .ok_or(ENODEV)? + .enqueue(packet, dependencies, early, publication)?; self.sequence[index] = order.sequence; self.frontiers[index] .push_within_capacity((order.sequence, fences)) @@ -427,8 +463,10 @@ impl Queue { impl crate::queue::Queue for Queue { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - let pages = [super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), - super::fw::compute::ScratchRequest::new(layouts[1])?.pages()]; + let pages = [ + super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), + super::fw::compute::ScratchRequest::new(layouts[1])?.pages(), + ]; let blocks = self.backend.render_scs_blocks(pages)?; self.context.reserve_render_blocks(blocks) } @@ -442,22 +480,22 @@ impl crate::queue::Queue for Queue { bytes: &[u8], objects: Pin<&xarray::XArray>>, ) -> Result { - let mut outputs = Outputs::new( - &self.fences, - syncs, - input_count, - &self.allocation_failure, - )?; + let mut outputs = Outputs::new(&self.fences, syncs, input_count, &self.allocation_failure)?; let result = (|| { if self.context.status().get() != 0 { return Err(EIO); } let (mut pending, guards) = self.prepare_submission(bytes, objects)?; - if pending.iter().any(|(command, _, _)| matches!(command, Validated::Render { .. })) { + if pending + .iter() + .any(|(command, _, _)| matches!(command, Validated::Render { .. })) + { self.context.note_render(); } if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { - for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } + for (command, _, _) in pending.iter_mut() { + command.allow_long_job(); + } } let mut guards = guards.into_iter(); let firmware_source = pending.len() == 1 diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 540f7cc76e8e7..2436cfd33ed35 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -248,8 +248,7 @@ impl Pair { let layout = target::Layout::new(pass, clusters, variant.programs)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; - let (fragment, fragment_alias) = - kick::Queue::render(alloc, ids[1], context.vm(), rings)?; + let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm(), rings)?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -336,7 +335,11 @@ impl Pair { self.replay.front().cloned() } pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { - if !self.replay.front().is_some_and(|front| Arc::ptr_eq(front, packet)) { + if !self + .replay + .front() + .is_some_and(|front| Arc::ptr_eq(front, packet)) + { return Err(EIO); } self.replay.pop(); @@ -363,14 +366,20 @@ impl Pair { addresses: RenderAddresses, ) -> Result { if !self.room() - || self.replay.front().is_some_and(|front| !Arc::ptr_eq(front, packet)) + || self + .replay + .front() + .is_some_and(|front| !Arc::ptr_eq(front, packet)) || !Arc::ptr_eq(&self.memory.context, &packet.context) || !packet.context.is_current() { return Err(EBUSY); } let Validated::Render { - pass, attachments, ray_query, .. + pass, + attachments, + ray_query, + .. } = &packet.command else { return Err(EINVAL); @@ -465,11 +474,9 @@ impl Pair { true => attachments[1].as_slice(), false => &[][..], }; - let result = self.memory.write_descriptors( - &args, - attachments, - packet.context.id().try_into()?, - ); + let result = + self.memory + .write_descriptors(&args, attachments, packet.context.id().try_into()?); let mcache = match result { Ok(mcache) => mcache, Err(error) => { @@ -669,7 +676,9 @@ impl Pair { .filter(|_| self.recovery_generation == epoch); let present = [0, 1].map(|stage| { let mut config = configs[stage]; - let reload = kicks[stage].prev().completion_value(payload.wrapping_sub(1)); + let reload = kicks[stage] + .prev() + .completion_value(payload.wrapping_sub(1)); if config.completion_seed_valid != 0 && self.installed[stage] && held.is_some_and(|held| held[stage] == reload) @@ -687,7 +696,10 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - let Validated::Render { pass, ray_query, .. } = &packet.command else { + let Validated::Render { + pass, ray_query, .. + } = &packet.command + else { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; @@ -790,10 +802,7 @@ impl Pair { windows[0].target().try_into()?, self.outer_started & ring == 0, )?; - let event_slot = memory - .graph - .support - .word(abi::storage::EVENT_SLOTS + 4)?; + let event_slot = memory.graph.support.word(abi::storage::EVENT_SLOTS + 4)?; let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); let inner = memory.graph.support.word(0x28000)?; let next_inner = (next_ordinal as u32).wrapping_mul(2); @@ -948,7 +957,11 @@ impl Pair { let policy = packet.context.policy(); // A registration of another class can only follow a rebind, which // requires the pair to have drained. - if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { + if self + .registered + .is_some_and(|class| class != policy.priority()) + && self.in_flight() + { return Err(EIO); } Ok((epoch, prepared.mcache, policy)) @@ -986,8 +999,12 @@ impl Pair { host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; host.notify_render(priority)?; - crate::g17::trace::publish(crate::g17::trace::RENDER, self.qids()[1], self.ordinal as u32, - packet); + crate::g17::trace::publish( + crate::g17::trace::RENDER, + self.qids()[1], + self.ordinal as u32, + packet, + ); host.note_submission() })(); if let Err(error) = publish { @@ -1053,9 +1070,14 @@ impl Pair { active.ticket.ordinal, pass.width, pass.height, pass.layers, target::Layout::new(pass, self.clusters, self.variant.programs)); } - pr_warn!("G17: recovery render ordinal={} published={} ctx={} seq={} snapshot={:?}\n", - active.ticket.ordinal, active.outer_published, active.packet.context.id(), - active.packet.order.sequence, observation::snapshot(&self.memory, active.ticket.ordinal)); + pr_warn!( + "G17: recovery render ordinal={} published={} ctx={} seq={} snapshot={:?}\n", + active.ticket.ordinal, + active.outer_published, + active.packet.context.id(), + active.packet.order.sequence, + observation::snapshot(&self.memory, active.ticket.ordinal) + ); } } @@ -1063,7 +1085,12 @@ impl Pair { /// `stamps` still lag it, and reports whether that has held for /// [`retirement::STAMP_FALLBACK_NS`] (`now`, host ns). The observation scan then retires it /// on its pass record instead of waiting for stamps no later event will move. - pub(crate) fn note_stamp_wait(&mut self, stamps: [u32; 2], state: u32, now: u64) -> Result { + pub(crate) fn note_stamp_wait( + &mut self, + stamps: [u32; 2], + state: u32, + now: u64, + ) -> Result { if self.quarantined || self.prepared.is_some() || state != 0 { return Ok(false); } @@ -1074,7 +1101,10 @@ impl Pair { return Ok(false); } let published = front.ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; - if stamps.into_iter().all(|stamp| retirement::stamp_covers(stamp, published)) { + if stamps + .into_iter() + .all(|stamp| retirement::stamp_covers(stamp, published)) + { front.stamp_wait_since = None; return Ok(false); } @@ -1113,14 +1143,25 @@ impl Pair { /// Whether the front group's stamp wait is latched: the observation scan rechecks the pair /// without a completion notification, which may never come for such a group. pub(crate) fn stamp_waiting(&self) -> bool { - self.active.front().is_some_and(|front| front.stamp_wait_since.is_some()) + self.active + .front() + .is_some_and(|front| front.stamp_wait_since.is_some()) } /// The watchdog of `packet` expired while it is this pair's front: if the firmware executed /// it and consumed its ring entries while the stamps lag, it retires on its pass record now /// instead of failing. Failing would give up on the stamps as well. - pub(crate) fn expire_stamp_wait(&mut self, packet: &Arc, stamps: [u32; 2], state: u32) -> Result { - if !self.active.front().is_some_and(|front| Arc::ptr_eq(&front.packet, packet)) { + pub(crate) fn expire_stamp_wait( + &mut self, + packet: &Arc, + stamps: [u32; 2], + state: u32, + ) -> Result { + if !self + .active + .front() + .is_some_and(|front| Arc::ptr_eq(&front.packet, packet)) + { return Ok(false); } if !self.note_stamp_wait(stamps, state, crate::g17::now_ns())? && !self.stamp_waiting() { @@ -1183,7 +1224,9 @@ impl Pair { return Err(EINVAL); }; let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; - self.memory.manager.note_required(required, crate::g17::now_ns()); + self.memory + .manager + .note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); } @@ -1246,7 +1289,10 @@ impl Pair { /// A slot stamp must name retained work before its execution markers are meaningful. pub(crate) fn pass_started(&self, stamp: u64) -> Option { const MASK: u64 = (1 << 40) - 1; - let ordinal = self.active.iter().map(|active| active.ticket.ordinal) + let ordinal = self + .active + .iter() + .map(|active| active.ticket.ordinal) .find(|ordinal| ordinal.wrapping_add(1) & MASK == stamp & MASK)?; Some(observation::snapshot(&self.memory, ordinal).ok()?.pass[0] != 0) } @@ -1278,7 +1324,11 @@ impl Pair { self.quarantined = true; // Successors published after recovery can still fail behind its spared // head. Classify the whole failing group before any fence can settle. - if self.active.iter().any(|active| active.packet.completion.spared()) { + if self + .active + .iter() + .any(|active| active.packet.completion.spared()) + { for active in self.active.iter() { active.packet.completion.classify(recovery::Class::Spared); } @@ -1394,8 +1444,13 @@ impl Pair { front.flist_completed = true; } crate::g17::recovery::note_retired(); - crate::g17::trace::complete(crate::g17::trace::RENDER, qids[1], ordinal as u32, - &front.packet, [timestamps[0], timestamps[3]]); + crate::g17::trace::complete( + crate::g17::trace::RENDER, + qids[1], + ordinal as u32, + &front.packet, + [timestamps[0], timestamps[3]], + ); defer(Deferred::Retired( front.packet.completion.clone(), Ok(timestamps), @@ -1455,7 +1510,13 @@ impl Pair { // A torn sample is no proof; a later poll or the timeout looks again. let witness = |executed| { matches!( - retirement::abandoned(&active.ticket, snapshot, stamps, state, executed), + retirement::abandoned( + &active.ticket, + snapshot, + stamps, + state, + executed + ), Ok(true) ) }; @@ -1464,7 +1525,11 @@ impl Pair { // another queue waited on never: the wait ended when firmware retired it. let lost = (active.packet.completion.interrupted() && !snapshot.untouched()) || active.packet.completion.has_firmware_consumer(); - Ok((unexecuted && !lost, !unexecuted && witness(true), unexecuted && lost)) + Ok(( + unexecuted && !lost, + !unexecuted && witness(true), + unexecuted && lost, + )) })() .unwrap_or_else(|error: Error| { failure = Some(error); @@ -1728,7 +1793,8 @@ impl Pair { /// The firmware acknowledged a kill of this pair's context (scheduler page): it no longer /// walks that context's work list, so the commands it never unlinked can be released. fn context_killed(&self) -> bool { - crate::g17::queue::compute::exit::State::read(&self.memory.context).is_ok_and(|state| state.killed != 0) + crate::g17::queue::compute::exit::State::read(&self.memory.context) + .is_ok_and(|state| state.killed != 0) } /// The pair's context may be killed although its commands were not retired: they all ran @@ -1867,15 +1933,13 @@ impl Pair { // own even with successors queued, so the pair is not quarantined into a sticky EIO. It // fails with ECANCELED, which spared work never reports as device loss; consecutive // abandoned spared successors are retired the same way. - let front_spared = self.active.front().is_some_and(|a| { - Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared() - }); + let front_spared = self + .active + .front() + .is_some_and(|a| Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared()); if !self.quarantined && (self.active.len() == 1 || front_spared) { let mut retired = false; - loop { - let Some(active) = self.active.front() else { - break; - }; + while let Some(active) = self.active.front() { let spared = active.packet.completion.spared(); if retired && !spared { break; @@ -2146,12 +2210,25 @@ impl Pair { } /// The render prefix of `packet` on this pair, as the registry collects it /// for an owner whose only pair this is. - pub(crate) fn own_dependencies(&self, owner: u64, packet: &Arc) -> Result { + pub(crate) fn own_dependencies( + &self, + owner: u64, + packet: &Arc, + ) -> Result { let mut dependencies = RenderDependencies::EMPTY; let mut frontier = crate::g17::dependency::Frontier::new(); - self.collect_dependencies(owner, packet.completion.status(), packet.order.wait_through[0], &mut frontier, false)?; + self.collect_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + false, + )?; for dependency in frontier.as_slice() { - *dependencies.render.get_mut(dependencies.render_count).ok_or(EOVERFLOW)? = *dependency; + *dependencies + .render + .get_mut(dependencies.render_count) + .ok_or(EOVERFLOW)? = *dependency; dependencies.render_count += 1; } if dependencies.render_count != 0 @@ -2165,7 +2242,10 @@ impl Pair { &mut tiling, )?; for dependency in tiling.as_slice() { - *dependencies.tiling.get_mut(dependencies.tiling_count).ok_or(EOVERFLOW)? = *dependency; + *dependencies + .tiling + .get_mut(dependencies.tiling_count) + .ok_or(EOVERFLOW)? = *dependency; dependencies.tiling_count += 1; } } @@ -2178,8 +2258,12 @@ impl Pair { /// not left the pair yet. pub(crate) fn switch_in_flight(&self) -> Option { let (from, first) = self.switch?; - (self.replay.len() != 0 || self.active.iter().any(|active| active.ticket.ordinal == first)) - .then_some(from) + (self.replay.len() != 0 + || self + .active + .iter() + .any(|active| active.ticket.ordinal == first)) + .then_some(from) } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { @@ -2344,7 +2428,9 @@ impl Pair { self.memory.context = binding.context.clone(); self.owner = Some(binding.owner); self.initial_packet = binding.initial_packet; - if foreign { self.graph_started = false; } + if foreign { + self.graph_started = false; + } self.context_update = true; if foreign { self.cursors = [0; 2]; @@ -2395,4 +2481,3 @@ mod completion_noted_tests { assert!(completion_noted([3, 68], both)); } } - diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index c1985c72f6ec1..d65c40c7cfade 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -41,6 +41,10 @@ pub(super) struct Lane<'a> { pub(super) window: Window, } impl<'a> Lane<'a> { + #[expect( + clippy::too_many_arguments, + reason = "The publication lane describes independent fields of the firmware wire record." + )] pub(super) fn new( graph: &'a memory::Graph, stage: usize, @@ -94,7 +98,9 @@ impl<'a> Lane<'a> { push(announce_va); Ok(Self { items, - producer: graph.queues.word(memory::POINTERS[stage] + graph.pointers.producer)?, + producer: graph + .queues + .word(memory::POINTERS[stage] + graph.pointers.producer)?, values, config, config_destination, diff --git a/drivers/gpu/drm/asahi/g17/reports.rs b/drivers/gpu/drm/asahi/g17/reports.rs index b67512f7eec6b..246aa5f02edbf 100644 --- a/drivers/gpu/drm/asahi/g17/reports.rs +++ b/drivers/gpu/drm/asahi/g17/reports.rs @@ -4,7 +4,10 @@ //! treating another firmware's pstate report as an achieved GPU frequency. //! Writers and readers hold the device mutex; reads never power the GPU up. -use super::{fw::channels::{EVENT_WORDS, STATISTICS_TYPES}, Role}; +use super::{ + fw::channels::{EVENT_WORDS, STATISTICS_TYPES}, + Role, +}; use kernel::prelude::*; #[derive(Clone, Copy, Default)] @@ -40,10 +43,25 @@ impl Reports { } pub(super) fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "format=1 units=raw host_time=monotonic_ns achieved_frequency=unqualified")?; - for (index, sample) in self.samples.iter().enumerate().filter(|(_, s)| s.count != 0) { - writeln!(f, "role={} tag={:#x} count={} host_ns={} words={:08x?}", - index / 32, index % 32, sample.count, sample.host_ns, sample.words)?; + writeln!( + f, + "format=1 units=raw host_time=monotonic_ns achieved_frequency=unqualified" + )?; + for (index, sample) in self + .samples + .iter() + .enumerate() + .filter(|(_, s)| s.count != 0) + { + writeln!( + f, + "role={} tag={:#x} count={} host_ns={} words={:08x?}", + index / 32, + index % 32, + sample.count, + sample.host_ns, + sample.words + )?; } Ok(()) } @@ -54,15 +72,29 @@ pub(super) struct View(pub(super) kernel::sync::Arc); impl kernel::debugfs::Writer for View { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - let state = self.0.state.lock(); - let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; - writeln!(f, "online=1 graph={:?} sampled_ns={}", self.0.variant.graph, super::now_ns())?; + let mut state = self.0.state.lock(); + let Some(firmware) = state.as_deref_mut() else { + return writeln!(f, "online=0"); + }; + firmware.observe_statistics_power(); + writeln!( + f, + "online=1 graph={:?} sampled_ns={}", + self.0.variant.graph, + super::now_ns() + )?; write_power(firmware, f)?; firmware.rings.write_reports(f) } } -fn write_power(firmware: &super::Firmware, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { +fn write_power( + firmware: &super::Firmware, + f: &mut kernel::fmt::Formatter<'_>, +) -> kernel::fmt::Result { + if firmware.rings.statistics_suspended() { + return writeln!(f, "power_management=firmware suspended=1"); + } let core_power = firmware.bridge.regs.g17_core_power(); writeln!(f, "power_management=firmware sampled_ns={} core_power={:#x} cores_powered={} host_hold={} idle_hold_ms={}", super::now_ns(), core_power, core_power & 0xf != 0, firmware.hold.active(), @@ -76,7 +108,9 @@ pub(super) struct PowerView(pub(super) kernel::sync::Arc); impl kernel::debugfs::Writer for PowerView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let state = self.0.state.lock(); - let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; + let Some(firmware) = state.as_deref() else { + return writeln!(f, "online=0"); + }; write_power(firmware, f) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index a59f496ab4f3b..3e779bba4fbbf 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -38,16 +38,19 @@ impl event::Transport for Service<'_> { } fn statistics(&mut self, role: Role) { let firmware = &mut *self.firmware; + firmware.observe_statistics_power(); firmware .rings .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); + firmware.observe_statistics_power(); } fn drain( &mut self, role: Role, mut consume: impl FnMut(&mut Self, Event) -> Result, ) -> Result { - let mut events = channel::Events::new(role, self.firmware.primary.state.shared.variant.graph); + let mut events = + channel::Events::new(role, self.firmware.primary.state.shared.variant.graph); while let Some(event) = events.next(&self.firmware.primary.state.shared.dev, &self.firmware.init)? { @@ -232,7 +235,9 @@ impl recovery::Queues for Service<'_> { }) } fn report_blamed_render(&self, qid: u8) { - self.firmware.queues.report_render_recovery(qid, &self.firmware.port); + self.firmware + .queues + .report_render_recovery(qid, &self.firmware.port); } fn settle_blamed_render(&mut self, qid: u8, error: Error) { let result = self @@ -264,11 +269,9 @@ impl Firmware { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - queue.release_witnessed( - state, - |owner| port.qos_complete(&[owner]), - &mut |event| deferred.push(event), - )?; + queue.release_witnessed(state, |owner| port.qos_complete(&[owner]), &mut |event| { + deferred.push(event) + })?; } Ok(()) } @@ -297,6 +300,7 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { + self.rings.invalidate_statistics(); if self.recovery.restart_limit(super::now_ns()) { pr_warn!("G17: restart storm limit reached\n"); return Err(ETIMEDOUT); @@ -313,7 +317,11 @@ impl Firmware { }); // Publishers without the device mutex stamp their tickets with this // mirror; they rejoin only after preparation admission reopens. - service.firmware.port.set_generation(service.firmware.recovery.generation()); + service + .firmware + .port + .set_generation(service.firmware.recovery.generation()); + service.firmware.rings.invalidate_statistics(); let result = result?; if let Some(error) = result.target_error { dev_warn!( @@ -394,7 +402,8 @@ impl crate::g17::Shared { deferred.finish(); crate::g17::trace::event(1); if recover { - self.preparations.wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; + self.preparations + .wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; } let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -451,8 +460,10 @@ impl crate::g17::Shared { || self.progress.has_waiters() || self.queues.has_waiters() || self.renders.has_waiters() - || now.saturating_sub(self.housekeeping_ns.load(core::sync::atomic::Ordering::Relaxed)) - >= HOUSEKEEPING_NS; + || now.saturating_sub( + self.housekeeping_ns + .load(core::sync::atomic::Ordering::Relaxed), + ) >= HOUSEKEEPING_NS; let mut lingers = None; let mut pools = None; let mut shrinking = false; diff --git a/drivers/gpu/drm/asahi/g17/stats.rs b/drivers/gpu/drm/asahi/g17/stats.rs new file mode 100644 index 0000000000000..4541ab8d40220 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/stats.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! A sysfs reader owns the session, never a pointer into detachable Firmware. + +use super::{fw::variant::InitGraph, telemetry::Readout, Shared}; +use core::sync::atomic::Ordering; +use kernel::sync::Arc; + +pub(crate) struct Reader(pub(super) Arc); + +impl Reader { + pub(crate) fn read(&self) -> Readout { + let mut state = self.0.state.lock(); + let Some(firmware) = state.as_deref_mut() else { + return Readout::default(); + }; + if self.0.crashed.load(Ordering::Acquire) || firmware.recovery.pending() { + firmware.rings.invalidate_statistics(); + return firmware.rings.read_statistics(super::now_ns()); + } + firmware.observe_statistics_power(); + firmware.rings.read_statistics(super::now_ns()) + } + + pub(crate) fn suspend(&self, suspended: bool) { + if let Some(firmware) = self.0.state.lock().as_deref_mut() { + firmware.rings.suspend_statistics(suspended); + } + } +} + +/// Two nominal periods is a conservative host age/window policy. The firmware +/// does not guarantee a packet duration or a pstate report cadence. Logical +/// pstate validity is controlled by records and epoch boundaries, not this age. +pub(super) const fn max_age_ns(graph: InitGraph) -> u64 { + match graph { + InitGraph::V25G83 => 16_000_000, + InitGraph::G17P => 32_000_000, + } +} diff --git a/drivers/gpu/drm/asahi/g17/sync_scope.c b/drivers/gpu/drm/asahi/g17/sync_scope.c index a3153f3122b94..73dc096d119c9 100644 --- a/drivers/gpu/drm/asahi/g17/sync_scope.c +++ b/drivers/gpu/drm/asahi/g17/sync_scope.c @@ -57,7 +57,7 @@ static struct asahi_g17_scope *to_scope(struct dma_fence *fence) static const char *scope_driver_name(struct dma_fence *fence) { - return "asahi"; + return "aurora"; } static const char *scope_timeline_name(struct dma_fence *fence) diff --git a/drivers/gpu/drm/asahi/g17/telemetry.rs b/drivers/gpu/drm/asahi/g17/telemetry.rs new file mode 100644 index 0000000000000..a048665dcab5c --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/telemetry.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Qualified V25G83/V26A428 statistics, decoded from complete copied records. +//! No record measures busy time, power, temperature or achieved frequency. +//! A/B table selection is absent, so neither nominal candidate is exported as +//! a requested or achieved frequency. All state is protected by the device mutex. + +pub(super) const UNAVAILABLE: u32 = 0; +pub(super) const VALID: u32 = 1; +pub(super) const STALE: u32 = 2; + +/// C read callback result; independent capabilities never imply one another. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(C)] +pub(crate) struct Readout { + pub(crate) util_status: u32, + pub(crate) pstate_status: u32, + pub(crate) tiler: u32, + pub(crate) renderer: u32, + pub(crate) device: u32, + pub(crate) pstate: u32, + pub(crate) util_host_ns: u64, + pub(crate) pstate_host_ns: u64, + pub(crate) util_samples: u64, + pub(crate) pstate_timestamp: u64, +} + +const _: () = { + assert!(core::mem::offset_of!(Readout, util_host_ns) == 24); + assert!(core::mem::size_of::() == 56); +}; + +#[derive(Default)] +pub(super) struct Telemetry { + view: Readout, + sums: [u64; 3], + window_ns: u64, + timestamp: Option, + /// Diagnostic interval only, never used as busy time. + pub(super) delta_ticks: Option, +} + +impl Telemetry { + /// End an epoch. Preserve the reason that formerly valid data is stale, + /// but never carry an average or a timestamp baseline into a new epoch. + pub(super) fn invalidate(&mut self) { + if self.view.util_status != UNAVAILABLE { + self.view.util_status = STALE; + } + if self.view.pstate_status != UNAVAILABLE { + self.view.pstate_status = STALE; + } + self.view.util_samples = 0; + self.timestamp = None; + self.delta_ticks = None; + } + + /// `max_age_ns` is a host policy (two nominal periods), not a packet duration. + /// Every role contributes one equally weighted sample to the same bounded + /// collection window. Reads do not consume samples or change the mean. + pub(super) fn record(&mut self, words: &[u32; 18], host_ns: u64, max_age_ns: u64) { + match words[0] { + 0x13 => { + let values = [words[3], words[4], words[6]]; + if values.iter().any(|&value| value > 100) { + self.view.util_status = UNAVAILABLE; + self.view.util_samples = 0; + return; + } + if self.view.util_samples == 0 + || host_ns.saturating_sub(self.window_ns) >= max_age_ns + || host_ns < self.window_ns + || self.view.util_samples >= u64::MAX / 100 + { + self.sums = [0; 3]; + self.view.util_samples = 0; + self.window_ns = host_ns; + } + for (sum, value) in self.sums.iter_mut().zip(values) { + *sum += u64::from(value); + } + self.view.util_samples += 1; + self.view.util_host_ns = host_ns; + self.view.util_status = VALID; + self.view.tiler = (self.sums[0] / self.view.util_samples) as u32; + self.view.renderer = (self.sums[1] / self.view.util_samples) as u32; + self.view.device = (self.sums[2] / self.view.util_samples) as u32; + } + 0x10 => { + let timestamp = u64::from(words[1]) | (u64::from(words[2]) << 32); + if self.timestamp.is_some_and(|last| timestamp < last) { + self.invalidate(); + self.view.pstate_status = UNAVAILABLE; + return; + } + self.delta_ticks = self.timestamp.map(|last| timestamp - last); + self.timestamp = Some(timestamp); + self.view.pstate_host_ns = host_ns; + self.view.pstate_timestamp = timestamp; + self.view.pstate_status = if words[6] <= 10 { VALID } else { UNAVAILABLE }; + self.view.pstate = words[6]; + } + _ => {} + } + } + + pub(super) fn read(&self, now_ns: u64, max_age_ns: u64) -> Readout { + let mut view = self.view; + let expired = |received| now_ns < received || now_ns - received >= max_age_ns; + // The oldest contributing sample bounds the whole averaging window. + if view.util_status == VALID && expired(self.window_ns) { + view.util_status = STALE; + } + // A pstate report describes a state transition, with no qualified + // cadence. Its validity ends on replacement or epoch invalidation, + // rather than an invented periodic-report deadline. + view + } +} + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_telemetry))] +#[cfg(any(test, CONFIG_KUNIT))] +mod tests { + use super::*; + + const AGE: u64 = 16_000_000; + const U1: [u32; 18] = [ + 0x13, 0, 0, 25, 75, 0xdeadbeef, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + const U2: [u32; 18] = [ + 0x13, 0, 0, 35, 85, 0x12345678, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + const U3: [u32; 18] = [0x13, 0, 0, 101, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + const P1: [u32; 18] = [0x10, 0x10, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + const P2A: [u32; 18] = [0x10, 0x1000, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + const P2B: [u32; 18] = [0x10, 0x2500, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + const P3: [u32; 18] = [ + 0x10, 0x3000, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + + #[test] + fn spec_u1_u2_u3_both_firmware_policies() { + for age in [AGE, 2 * AGE] { + let mut stats = Telemetry::default(); + assert_eq!(stats.read(0, age), Readout::default()); + stats.record(&U1, 100, age); + let view = stats.read(100, age); + assert_eq!((view.tiler, view.renderer, view.device), (25, 75, 50)); + assert_eq!(view.util_status, VALID); + stats.record(&U2, 101, age); // May be the other role; no deduplication. + let view = stats.read(101, age); + assert_eq!((view.tiler, view.renderer, view.device), (30, 80, 57)); + assert_eq!(view.util_samples, 2); + stats.record(&U3, 102, age); + assert_eq!(stats.read(102, age).util_status, UNAVAILABLE); + stats.record(&U2, 103, age); + assert_eq!(stats.read(103, age).device, 65); + } + } + + #[test] + fn spec_p1_p2_p3_and_regression() { + let mut stats = Telemetry::default(); + stats.record(&P1, 100, AGE); + let view = stats.read(100, AGE); + assert_eq!(view.pstate, 6); + assert_eq!(view.pstate_status, VALID); + assert_eq!(view.pstate_timestamp, 0x1_0000_0010); + assert_eq!(stats.delta_ticks, None); // First sample is only a baseline. + stats.invalidate(); + stats.record(&P2A, 101, AGE); + stats.record(&P2B, 102, AGE); + assert_eq!(stats.delta_ticks, Some(5376)); + assert_eq!( + stats.delta_ticks.unwrap() * 1_000_000_000 / 24_000_000, + 224_000 + ); + stats.record(&P3, 103, AGE); + assert_eq!(stats.read(103, AGE).pstate_status, UNAVAILABLE); + stats.record(&U1, 104, AGE); + stats.record(&P2A, 105, AGE); // Backwards, including across role drain order. + assert_eq!(stats.read(105, AGE).pstate_status, UNAVAILABLE); + assert_eq!(stats.read(105, AGE).util_status, STALE); + assert_eq!(stats.delta_ticks, None); + stats.record(&P2A, 106, AGE); // New epoch establishes a new baseline. + assert_eq!(stats.read(106, AGE).pstate_status, VALID); + assert_eq!(stats.delta_ticks, None); + stats.record(&P2A, 107, AGE); // Equal timestamps are non-regressing. + assert_eq!(stats.delta_ticks, Some(0)); + } + + #[test] + fn age_epoch_and_zero_are_independent() { + let mut stats = Telemetry::default(); + stats.record(&U1, 100, AGE); + stats.record(&P1, 200, AGE); + assert_eq!(stats.read(99, AGE).util_status, STALE); + assert_eq!(stats.read(100 + AGE - 1, AGE).util_status, VALID); + let view = stats.read(100 + AGE, AGE); + assert_eq!(view.util_status, STALE); + assert_eq!(view.pstate_status, VALID); + assert_eq!(stats.read(200 + AGE, AGE).pstate_status, VALID); + assert_eq!(stats.read(u64::MAX, AGE).pstate_status, VALID); + stats.record(&U2, 100 + AGE, AGE); + assert_eq!(stats.read(100 + AGE, AGE).device, 65); // Old window was discarded. + stats.invalidate(); // Suspend, restart, quarantine, gate or teardown. + let view = stats.read(100 + AGE, AGE); + assert_eq!(view.util_status, STALE); + assert_eq!(view.pstate_status, STALE); + let mut zero = [0; 18]; + zero[0] = 0x13; + stats.record(&zero, 101 + AGE, AGE); + let view = stats.read(101 + AGE, AGE); + assert_eq!(view.util_status, VALID); + assert_eq!((view.tiler, view.renderer, view.device), (0, 0, 0)); + zero[0] = 0x10; + stats.record(&zero, 102 + AGE, AGE); + assert_eq!(stats.read(102 + AGE, AGE).pstate, 0); + assert_eq!(stats.read(102 + AGE, AGE).pstate_status, VALID); + } + + #[test] + fn invalid_components_and_selectors_never_clamp() { + for word in [3, 4, 6] { + let mut stats = Telemetry::default(); + let mut words = U1; + words[word] = u32::MAX; + stats.record(&words, 1, AGE); + assert_eq!(stats.read(1, AGE).util_status, UNAVAILABLE); + } + for selector in [11, 15, 16, u32::MAX] { + let mut stats = Telemetry::default(); + let mut words = P1; + words[6] = selector; + stats.record(&words, 1, AGE); + assert_eq!(stats.read(1, AGE).pstate_status, UNAVAILABLE); + } + let mut stats = Telemetry::default(); + for tag in [0x11, 0x12, 0x18, 0x1d, u32::MAX] { + let mut words = U1; + words[0] = tag; + stats.record(&words, 1, AGE); + } + assert_eq!(stats.read(1, AGE), Readout::default()); + } +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 00b44cbe62887..f100820a0c75e 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -41,9 +41,7 @@ use crate::{ AsahiDrmDevice, AsahiDrmDriver, // }, - file, - memacct, - mmu, + file, memacct, mmu, }; const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; @@ -133,9 +131,6 @@ pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { Ok(()) } -unsafe impl Send for AsahiObject {} -unsafe impl Sync for AsahiObject {} - /// A shared reference to a GEM object for this driver. pub(crate) struct ObjectRef { /// The underlying GEM object reference @@ -503,6 +498,7 @@ pub(crate) fn prepare_new_object_g15(gem: &Object) -> Result { Ok(()) } - #[cfg(CONFIG_DRM_ASAHI_G15)] -extern "C" { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } +extern "C" { + fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; +} diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 3f2672f803ce9..e9f1008fc5a72 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -311,7 +311,7 @@ pub(crate) trait Gpu: Send + Sync { flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES); } if self.soft_faults() && self.supports_soft_faults() { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS); } flags } @@ -367,6 +367,7 @@ pub(crate) trait Gpu: Send + Sync { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result>; /// Bind a BO range as a firmware- or CPU-written timestamp buffer. fn map_timestamp_buffer( @@ -1099,7 +1100,7 @@ impl GpuManager::ver { virt_addr: U64(iova + off as u64), total_size: (map.size * map.count * dies) as u32, element_size: map.size as u32, - readwrite: U64(map.writable as u64), + readwrite: U64(u64::from(map.writable)), }; }); @@ -1149,20 +1150,14 @@ impl GpuManager::ver { /// Fetch the GPU MMU fault information from the hardware registers. fn get_fault_info(&self) -> Option { // TODO: handle differently - let Some(guard) = self.dev.registration_guard() else { - return None; - }; + let guard = self.dev.registration_guard()?; guard.registration_data_with(|reg_data| { let res = ®_data.resources; let info = res.get_fault_info(self.cfg); - if info.is_some() { - dev_err!( - self.dev.as_ref(), - " Fault info: {:#x?}\n", - info.as_ref().unwrap() - ); + if let Some(fault) = &info { + dev_err!(self.dev.as_ref(), " Fault info: {:#x?}\n", fault); } info }) @@ -1411,6 +1406,7 @@ impl Gpu for GpuManager::ver { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { let mut kalloc = self.alloc(); let id = self.ids.queue.next(); @@ -1428,6 +1424,7 @@ impl Gpu for GpuManager::ver { id, priority, usc_exec_base, + usage, )?, GFP_KERNEL, )?) diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index bc79ac73739a5..75cbb1c03c6b1 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -137,11 +137,11 @@ impl<'a> InitDataBuilder::ver<'a> { continue; } - let f_khz = (ps.freq_hz / 1000) as u64; - let v_max = ps.max_volt_mv() as u64; + let f_khz = u64::from(ps.freq_hz / 1000); + let v_max = u64::from(ps.max_volt_mv()); t1.push( - (1000000000 * (curve_cfg.t1_coef as u64) / (f_khz * v_max)) + (1000000000 * u64::from(curve_cfg.t1_coef) / (f_khz * v_max)) .try_into() .unwrap(), GFP_KERNEL, @@ -149,7 +149,7 @@ impl<'a> InitDataBuilder::ver<'a> { for (j, scale) in curve_cfg.t3_scales.iter().enumerate() { t3[j].push( - (t3_coef as u64 * 1000000100 * *scale as u64 / (f_khz * v_max * 6)) + (u64::from(t3_coef) * 1000000100 * u64::from(*scale) / (f_khz * v_max * 6)) .try_into() .unwrap(), GFP_KERNEL, @@ -423,7 +423,7 @@ impl<'a> InitDataBuilder::ver<'a> { #[ver(G >= G14X)] fast_die1_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[1]), #[ver(V < V13_0B4)] - fast_die0_sensor_present: U64(cfg.fast_die0_sensor_present as u64), + fast_die0_sensor_present: U64(u64::from(cfg.fast_die0_sensor_present)), unk_163c: 1, unk_3644: 0, hws1 <- Self::hw_shared1(cfg), @@ -487,10 +487,12 @@ impl<'a> InitDataBuilder::ver<'a> { } if !dyncfg.hw_data_b.is_empty() { + // SAFETY: This closure owns the fully sized, initially zeroed + // wire allocation; the byte view remains within that allocation. unsafe { let mut matches: bool = true; let sla = core::slice::from_raw_parts( - raw as *const raw::HwDataA::ver as *const u8, + core::ptr::from_ref(raw).cast::(), core::mem::size_of::(), ); if sla.len() != dyncfg.hw_data_a.len() { @@ -502,8 +504,10 @@ impl<'a> InitDataBuilder::ver<'a> { dyncfg.hw_data_a.len(), ); } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_a.len()) { - if sla[i] != dyncfg.hw_data_a[i] { + for (i, (actual, expected)) in + sla.iter().zip(&dyncfg.hw_data_a).enumerate() + { + if actual != expected { matches = false; dev_err!(self.dev.as_ref(), "!!! Hwdata A first mismatch: {i}"); break; @@ -670,10 +674,12 @@ impl<'a> InitDataBuilder::ver<'a> { } if !dyncfg.hw_data_b.is_empty() { + // SAFETY: This closure owns the fully sized, initially zeroed + // wire allocation; the byte view remains within that allocation. unsafe { let mut matches: bool = true; let sla = core::slice::from_raw_parts( - raw as *const raw::HwDataB::ver as *const u8, + core::ptr::from_ref(raw).cast::(), core::mem::size_of::(), ); if sla.len() != dyncfg.hw_data_b.len() { @@ -685,8 +691,10 @@ impl<'a> InitDataBuilder::ver<'a> { dyncfg.hw_data_b.len(), ); } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_b.len()) { - if sla[i] != dyncfg.hw_data_b[i] { + for (i, (actual, expected)) in + sla.iter().zip(&dyncfg.hw_data_b).enumerate() + { + if actual != expected { matches = false; dev_err!(self.dev.as_ref(), "!!! Hwdata B first mismatch: {i}"); break; @@ -824,10 +832,12 @@ impl<'a> InitDataBuilder::ver<'a> { } if !dyncfg.hw_globals.is_empty() { + // SAFETY: This closure owns the fully sized, initially zeroed + // wire allocation; the byte view remains within that allocation. unsafe { let mut matches: bool = true; let sla = core::slice::from_raw_parts( - raw as *const raw::Globals::ver as *const u8, + core::ptr::from_ref(raw).cast::(), core::mem::size_of::(), ); if sla.len() != dyncfg.hw_globals.len() { @@ -839,8 +849,10 @@ impl<'a> InitDataBuilder::ver<'a> { dyncfg.hw_globals.len(), ); } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_globals.len()) { - if sla[i] != dyncfg.hw_globals[i] { + for (i, (actual, expected)) in + sla.iter().zip(&dyncfg.hw_globals).enumerate() + { + if actual != expected { matches = false; dev_err!(self.dev.as_ref(), "!!! Globals first mismatch: {i}"); break; @@ -971,12 +983,12 @@ impl<'a> InitDataBuilder::ver<'a> { num_entries: usize, ) -> raw::UatLevelInfo { raw::UatLevelInfo { - index_shift: index_shift as _, + index_shift: index_shift as u8, unk_1: 14, unk_2: 14, unk_3: 8, unk_4: 0x4000, - num_entries: num_entries as _, + num_entries: num_entries as u16, unk_8: U64(1), unk_10: U64(((1u64 << cfg.uat_oas) - 1) & !(mmu::UAT_PGMSK as u64)), index_mask: U64(((num_entries - 1) << index_shift) as u64), diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 5f0ab2b4aea4f..79071cbc434a0 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -71,7 +71,7 @@ pub(crate) fn tlbi_asid(asid: Asid) { asm!( ".arch armv8.4-a", "tlbi aside1os, {x}", - x = in(reg) ((asid as u64) << 48) + x = in(reg) (u64::from(asid) << 48) ); } } @@ -85,7 +85,7 @@ pub(crate) fn tlbi_page(asid: Asid, va: usize) { return; } - let val: u64 = ((asid as u64) << 48) | ((va as u64 >> 12) & 0xffffffffffc); + let val: u64 = (u64::from(asid) << 48) | ((va as u64 >> 12) & 0xffffffffffc); // SAFETY: tlbi is always safe by definition unsafe { asm!( @@ -147,7 +147,7 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { let bits = 5 * exp; let mut base = (num + (1 << bits) - 1) >> bits; - val |= (exp as u64) << 44; + val |= u64::from(exp) << 44; while base > 32 { // SAFETY: tlbi is always safe by definition diff --git a/drivers/gpu/drm/asahi/memacct.rs b/drivers/gpu/drm/asahi/memacct.rs index b7c996d0c6f6e..12fc90b23d8eb 100644 --- a/drivers/gpu/drm/asahi/memacct.rs +++ b/drivers/gpu/drm/asahi/memacct.rs @@ -131,7 +131,7 @@ fn try_add_within(counter: &AtomicU64, bytes: u64, limit: u64, extra: u64) -> bo fn sub(counter: &AtomicU64, bytes: u64) { let prev = counter.fetch_sub(bytes, Ordering::AcqRel); if prev < bytes { - pr_warn!("asahi: GPU memory accounting underflow ({} < {})\n", prev, bytes); + pr_warn!("GPU memory accounting underflow ({} < {})\n", prev, bytes); counter.store(0, Ordering::Relaxed); } } @@ -162,7 +162,7 @@ impl Ledger { ); self.set_limits(limits); pr_info!( - "asahi: GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", + "GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", limits.device >> 20, limits.client >> 20 ); diff --git a/drivers/gpu/drm/asahi/microseq.rs b/drivers/gpu/drm/asahi/microseq.rs index cbdb5de62e921..42cd40b8220f6 100644 --- a/drivers/gpu/drm/asahi/microseq.rs +++ b/drivers/gpu/drm/asahi/microseq.rs @@ -45,7 +45,7 @@ impl Builder { pub(crate) fn add(&mut self, op: T) -> Result { let off = self.ops.len(); let p: *const T = &op; - let p: *const u8 = p as *const u8; + let p: *const u8 = p.cast::(); // SAFETY: Microseq operations always have no padding bytes, so it is safe to // access them as a byte slice. let s: &[u8] = unsafe { core::slice::from_raw_parts(p, core::mem::size_of::()) }; diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index c4afbc0127d70..6baaa97977f5a 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -108,8 +108,6 @@ pub(crate) use pgtable::{ UAT_TOP_LEVEL_SHIFT, // }; -use pin_init; - const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; @@ -992,6 +990,7 @@ impl VmInner { /// Shared reference to a virtual memory address space ([`Vm`]). #[derive(Clone)] pub(crate) struct Vm { + device_binding: Arc, dev: ARef, id: u64, mapping_epoch: Option>, @@ -1220,14 +1219,17 @@ impl TimestampVmap { /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused /// when the destination is prepared, and ignored here. pub(crate) fn write(&self, offset: usize, value: u64) { - if offset.checked_add(size_of::()).is_none_or(|end| end > self.size()) { + if offset + .checked_add(size_of::()) + .is_none_or(|end| end > self.size()) + { return; } // Completion runs in task context. Bindings of the same BO share this lock, // so aligned u64 and overlapping unaligned byte stores never race in Rust. let _guard = self.0.owner().timestamp_lock.lock(); - // SAFETY: The complete extent was checked above, inside this pinned vmap. let base = self.0.addr.cast::(); + // SAFETY: The complete extent was checked above, inside this pinned vmap. let pointer = unsafe { base.add(offset) }; if pointer as usize % core::mem::align_of::() == 0 { // SAFETY: All eight bytes were checked above; the vmap pins the backing. @@ -1448,7 +1450,7 @@ impl KernelMapping { } else { // Otherwise, check if this slot is the active one, otherwise return // Also check that we actually own this slot - let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; + let ttb = owner.ttb() | TTBR_VALID | u64::from(slot) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); uat_inner.handoff_lock(); @@ -1691,7 +1693,7 @@ impl UatShared { /// Writes back context-table slot `idx` after it changed, where this SoC needs it. fn publish_slot(&self, idx: usize) { if self.quirks.clean_tables { - let slot = &self.ttbs()[idx] as *const SlotTTBS as *const u8; + let slot = core::ptr::from_ref(&self.ttbs()[idx]).cast::(); pgtable::clean_poc(slot, size_of::()); pgtable::poc_barrier(); } @@ -1700,13 +1702,20 @@ impl UatShared { /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant - unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() + unsafe { self.handoff_rgn.map.ptr().cast::().as_ref() }.unwrap() } /// Returns the TTBAT area fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { // SAFETY: pointer is non-null per the type invariant - unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() + unsafe { + self.ttbs_rgn + .map + .ptr() + .cast::<[SlotTTBS; UAT_NUM_CTX]>() + .as_ref() + } + .unwrap() } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. @@ -1894,7 +1903,9 @@ impl Handoff { while self.lock_fw.load(Ordering::Relaxed) != 0 { if self.turn.load(Ordering::Relaxed) != 0 { self.lock_ap.store(0, Ordering::Relaxed); - while self.turn.load(Ordering::Relaxed) != 0 {} + while self.turn.load(Ordering::Relaxed) != 0 { + core::hint::spin_loop() + } self.lock_ap.store(1, Ordering::Relaxed); fence(Ordering::SeqCst); } @@ -2001,10 +2012,7 @@ impl HandoffFlush { } } -fn prefetch_sgt_segments( - dev: &device::Device, - gem: &gem::Object, -) -> Result { +fn prefetch_sgt_segments(dev: &device::Device, gem: &gem::Object) -> Result { let sgt = crate::gem::held_sg_table(gem, dev)?; let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; @@ -2012,7 +2020,10 @@ fn prefetch_sgt_segments( segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(SgSegments { segments, _lease: sgt.into_lease() }) + Ok(SgSegments { + segments, + _lease: sgt.into_lease(), + }) } /// Reuse cached segments without allocating another copy for each bind. The bind lease was @@ -2122,7 +2133,9 @@ impl Vm { )?; let gpuvm = ARef::from(&*unique_gpuvm); + let device_binding = crate::binding::lookup(&dev)?; Ok(Vm { + device_binding, dev: dev.as_ref().as_ref().into(), id, gpuvm, @@ -2328,7 +2341,13 @@ impl Vm { guard_size: usize, ) -> Result { self.map_in_range_with_mode( - gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Best, + gem, + object_range, + alignment, + range, + prot, + guard_size, + mm::InsertMode::Best, ) } @@ -2343,7 +2362,13 @@ impl Vm { guard_size: usize, ) -> Result { self.map_in_range_with_mode( - gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Low, + gem, + object_range, + alignment, + range, + prot, + guard_size, + mm::InsertMode::Low, ) } @@ -2371,15 +2396,24 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let vm_bo = self - .gpuvm - .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; + // End the bound-device lease before entering firmware/GPUVM waits. + let (_bind_lease, vm_bo) = { + let binding = self.device_binding.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = self.gpuvm.obtain( + gem, + pin_init!(VmBoData { + segments <- new_mutex!(None) + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; let size = object_range.range(); + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(/*Some(gem)*/ None, false)? }; @@ -2450,13 +2484,20 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let vm_bo = self - .gpuvm - .obtain(&gem, pin_init!(VmBoData { segments <-new_mutex!(None) }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; + // End the bound-device lease before entering firmware/GPUVM waits. + let (_bind_lease, vm_bo) = { + let binding = self.device_binding.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = self.gpuvm.obtain( + gem, + pin_init!(VmBoData { + segments <- new_mutex!(None) + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { @@ -2530,7 +2571,7 @@ impl Vm { gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; + let dev: &device::Device = device.as_ref().as_ref(); let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm @@ -2548,6 +2589,8 @@ impl Vm { // Mapping needs a complete context let mut ctx = StepContext::new(Some(device.as_ref().as_ref()), prot, &mut resources)?; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(/*Some(&gem)*/ None, false)? }; @@ -2620,6 +2663,8 @@ impl Vm { size: usize, prot: Prot, ) -> Result { + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; if (iova as usize | phys | size) & UAT_PGMSK != 0 { @@ -2724,6 +2769,8 @@ impl Vm { let mut resources = VmOpResources::for_unmap()?; let mut ctx = StepContext::new(dev, Prot::default(), &mut resources)?; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; mod_dev_dbg!(self.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); @@ -2900,7 +2947,11 @@ impl Uat { #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Map a region given by its physical range, write-combined. - fn map_physical(range: &Range, min_size: usize, reserved_ram: bool) -> Result { + fn map_physical( + range: &Range, + min_size: usize, + reserved_ram: bool, + ) -> Result { let size: usize = (range.end.checked_sub(range.start).ok_or(EINVAL)?).try_into()?; if size < min_size || range.start & (UAT_PGMSK as PhysicalAddr) != 0 { return Err(EINVAL); @@ -2909,13 +2960,11 @@ impl Uat { let mut raw: kernel::bindings::resource = unsafe { core::mem::zeroed() }; raw.start = range.start; raw.end = range.end - 1; - raw.flags = kernel::bindings::IORESOURCE_MEM as _; + raw.flags = kernel::bindings::IORESOURCE_MEM as usize; // SAFETY: `Resource` is a transparent wrapper around `Opaque`, and // `raw` describes a valid memory range. let res: io::resource::Resource = unsafe { core::mem::transmute(kernel::types::Opaque::new(raw)) }; - // SAFETY: The UAT regions are only accessed through this mapping by the CPU; the GPU - // reaches them through its own translation. let flags = io::mem::MemFlag::WC; #[cfg(CONFIG_DRM_ASAHI_G15)] let flags = if reserved_ram { @@ -2932,6 +2981,8 @@ impl Uat { }; #[cfg(not(CONFIG_DRM_ASAHI_G15))] let _ = reserved_ram; + // SAFETY: The UAT regions are only accessed through this mapping by the CPU; + // the GPU reaches them through its own translation. let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }?; Ok(UatRegion { base: range.start, @@ -2988,7 +3039,7 @@ impl Uat { // SAFETY: The Handoff struct layout matches the firmware's view of memory at this // address, and the region is at least large enough per the check above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; let inner = Arc::pin_init( try_pin_init!(UatInner { @@ -3258,9 +3309,9 @@ impl Uat { { return Err(EIO); } - for index in UAT_USER_CTX_START..UAT_NUM_CTX { - if ttbs[index].ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] - || ttbs[index].ttb1.load(Ordering::Acquire) != 0 + for (index, table) in ttbs.iter().enumerate().skip(UAT_USER_CTX_START) { + if table.ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] + || table.ttb1.load(Ordering::Acquire) != 0 { return Err(EIO); } @@ -3361,14 +3412,19 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { - for (value, pte) in snapshot.iter_mut().zip(ptes) { - *value = pte.load(Ordering::Relaxed); - } - for pte in &ptes[owned..] { - pte.store(0, Ordering::Relaxed); - } - }) + pgtable::with_root_entries( + self.inner.reserved.as_deref(), + self.kernel_vm.ttb(), + UAT_PGSZ / size_of::(), + |ptes| { + for (value, pte) in snapshot.iter_mut().zip(ptes) { + *value = pte.load(Ordering::Relaxed); + } + for pte in &ptes[owned..] { + pte.store(0, Ordering::Relaxed); + } + }, + ) }?; mem::sync(); @@ -3381,11 +3437,16 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), snapshot.len(), |ptes| { - ptes.iter() - .zip(snapshot) - .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) - }) + pgtable::with_root_entries( + self.inner.reserved.as_deref(), + self.kernel_vm.ttb(), + snapshot.len(), + |ptes| { + ptes.iter() + .zip(snapshot) + .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) + }, + ) }?; if !matches { return Err(EIO); @@ -3473,7 +3534,7 @@ impl Uat { // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, // and the region is at least large enough per the size specified above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); let shared_quirks = quirks; @@ -3602,7 +3663,10 @@ impl Uat { inner.handoff().init(firmware, quirks.handoff_slot)?; if quirks.clean_tables { - pgtable::clean_poc(inner.handoff() as *const Handoff as *const u8, HANDOFF_SIZE); + pgtable::clean_poc( + core::ptr::from_ref(inner.handoff()).cast::(), + HANDOFF_SIZE, + ); pgtable::poc_barrier(); } @@ -3630,7 +3694,7 @@ impl Uat { .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } if quirks.clean_tables { - pgtable::clean_poc(ttbs.as_ptr() as *const u8, SLOTS_SIZE); + pgtable::clean_poc(ttbs.as_ptr().cast::(), SLOTS_SIZE); pgtable::poc_barrier(); } diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 49c2241bc399f..1df852184b31d 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -266,7 +266,7 @@ impl VmShrinker { )?; // SAFETY: The format is static and has no format arguments. let raw = - unsafe { kernel::bindings::shrinker_alloc(0, c_str!("asahi-idle-vm").as_char_ptr()) }; + unsafe { kernel::bindings::shrinker_alloc(0, c_str!("aurora-idle-vm").as_char_ptr()) }; if raw.is_null() { return Err(ENOMEM); } diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index ecf040670cfaa..0c9e6f62b5bbe 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -152,9 +152,9 @@ macro_rules! inner_ptr { // we don't actually read from `outer` (which would be UB) nor create an intermediate // reference. let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p as *const u8; + let inner = p.cast::(); // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer as *const u8) }; + let off = unsafe { inner.offset_from(outer.cast::()) }; // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer // object. unsafe { $gpuva.offset(off.try_into().unwrap(), p) } @@ -234,9 +234,9 @@ macro_rules! inner_weak_ptr { // we don't actually read from `outer` (which would be UB) nor create an intermediate // reference. let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p as *const u8; + let inner = p.cast::(); // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer as *const u8) }; + let off = unsafe { inner.offset_from(outer.cast::()) }; // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer // object. unsafe { $gpuva.offset(off.try_into().unwrap(), p) } @@ -298,11 +298,15 @@ impl> GpuObject { core::any::type_name::(), alloc.gpu_ptr() ); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'static>; + let p = alloc + .ptr() + .ok_or(EINVAL)? + .as_ptr() + .cast::>(); let mut raw = callback(&inner); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant, and the type is // identical to the type of `raw` other than the lifetime. - unsafe { p.copy_from(&mut raw as *mut _ as *mut u8 as *mut _, 1) }; + unsafe { p.copy_from(core::ptr::from_mut(&mut raw).cast::().cast(), 1) }; mem::forget(raw); Ok(Self { raw: p, @@ -337,10 +341,14 @@ impl> GpuObject { core::any::type_name::(), alloc.gpu_ptr() ); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; + let p = alloc + .ptr() + .ok_or(EINVAL)? + .as_ptr() + .cast::>>(); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. let raw = callback(&inner, unsafe { &mut *p })?; - if p as *mut T::Raw<'_> != raw as *mut _ { + if !core::ptr::eq(p.cast::>(), raw) { dev_err!( alloc.device().as_ref(), "Allocation callback returned a mismatched reference ({})\n", @@ -349,7 +357,7 @@ impl> GpuObject { return Err(EINVAL); } Ok(Self { - raw: p as *mut u8 as *mut T::Raw<'static>, + raw: p.cast::().cast::>(), gpu_ptr, alloc, inner, @@ -400,14 +408,14 @@ impl> GpuObject { alloc.gpu_ptr() ); let inner = inner_init(gpu_ptr); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'_>; + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr().cast::>(); let ret = Self { - raw: p as *mut u8 as *mut T::Raw<'static>, + raw: p.cast::().cast::>(), gpu_ptr, alloc, inner: KBox::init(inner, GFP_KERNEL)?, }; - let q = &*ret.inner as *const T; + let q = core::ptr::from_ref::(&*ret.inner); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. unsafe { pin_init::raw_try_init(p, raw_init(&*q, gpu_ptr)) }?; Ok(ret) @@ -437,7 +445,7 @@ impl> GpuObject { ) -> RetVal { // SAFETY: `self.raw` is valid per the type invariant, and the second half is just // converting lifetimes. - unsafe { callback(&mut *self.raw, &mut *(&mut *self.inner as *mut _)) } + unsafe { callback(&mut *self.raw, &mut *core::ptr::from_mut(&mut *self.inner)) } } /// Access the inner `Raw` data given a user-supplied callback. @@ -449,7 +457,7 @@ impl> GpuObject { ) -> RetVal { // SAFETY: `self.raw` is valid per the type invariant, and the second half is just // converting lifetimes. - unsafe { callback(&*self.raw, &*(&*self.inner as *const _)) } + unsafe { callback(&*self.raw, &*core::ptr::from_ref(&*self.inner)) } } } @@ -475,8 +483,8 @@ where f.debug_struct(core::any::type_name::()) // SAFETY: `self.raw` is valid per the type invariant. .field("raw", &format_args!("{:#X?}", unsafe { &*self.raw })) - .field("inner", &format_args!("{:#X?}", &self.inner)) - .field("alloc", &format_args!("{:?}", &self.alloc)) + .field("inner", &format_args!("{:#X?}", self.inner)) + .field("alloc", &format_args!("{:?}", self.alloc)) .finish() } } diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 2fabe6bee88d6..0f915607cb1f4 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -153,9 +153,8 @@ impl ReservedTables { pub(crate) unsafe fn add(&mut self, res: kernel::io::resource::Resource) -> Result { let start = res.start(); // SAFETY: The caller guarantees the region; write-back matches how the firmware maps it. - let map = unsafe { - kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? - }; + let map = + unsafe { kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? }; self.starts.push(start, GFP_KERNEL)?; self.regions.push(map, GFP_KERNEL)?; Ok(()) @@ -671,7 +670,7 @@ impl UatPageTable { let driver_tables = &mut self.driver_tables; let oas_mask = self.oas_mask; let entry = |p: *mut u8| -> Result> { - let uptep = p as *const _ as *const Pte; + let uptep = p.cast_const().cast::(); // SAFETY: with_table() ensures the pointer is valid, and our index is // aligned so it is safe to deref as an AtomicU64. let upte = unsafe { &*uptep }; @@ -679,7 +678,10 @@ impl UatPageTable { // Allocate if requested if upte_val == 0 && alloc { let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + mod_pr_debug!( + "UATPageTable::with_pages: alloc PT at {:#x}\n", + pt_page.phys() + ); if clean { // A non-snooping walker must never follow a parent to a // table whose zeroes are still in the CPU caches. @@ -755,7 +757,7 @@ impl UatPageTable { iova ); let leaves = |p: *mut u8| -> Result { - let ptep = p as *const _ as *const Pte; + let ptep = p.cast_const().cast::(); // SAFETY: We know this is a valid pointer to PTEs and the range is valid and // checked by with_table(). let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; @@ -769,7 +771,13 @@ impl UatPageTable { // SAFETY: Page table addresses are either allocated by us, or // firmware-managed tables in RAM or in a reserved region. unsafe { - with_table(reserved.as_deref(), phys, idx * PTE_SIZE, count * PTE_SIZE, leaves) + with_table( + reserved.as_deref(), + phys, + idx * PTE_SIZE, + count * PTE_SIZE, + leaves, + ) }?; let block = 1 << (UAT_PGBIT + UAT_LVBIT); @@ -806,7 +814,10 @@ impl UatPageTable { } if let Some((parent_phys, parent_index)) = parent { let oas_mask = self.oas_mask; - let in_reserved = self.reserved.as_ref().is_some_and(|r| r.contains(parent_phys)); + let in_reserved = self + .reserved + .as_ref() + .is_some_and(|r| r.contains(parent_phys)); // SAFETY: Only looks the page up; nothing is accessed through it. if !in_reserved && unsafe { Page::borrow_phys(&parent_phys) }.is_none() { return Err(EIO); @@ -832,7 +843,15 @@ impl UatPageTable { // its child descriptor is detached. A changed descriptor indicates // an ownership violation; fail closed instead of freeing a table // that may still be reachable. - unsafe { with_table(reserved, parent_phys, parent_index * PTE_SIZE, PTE_SIZE, detach) }?; + unsafe { + with_table( + reserved, + parent_phys, + parent_index * PTE_SIZE, + PTE_SIZE, + detach, + ) + }?; } if let Some(tables) = self.driver_tables.as_mut() { let index = tables.iter().position(|&table| table == phys).ok_or(EIO)?; diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index c9f205708e313..a89fcaf0b8dcb 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -35,6 +35,10 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::Compute; #[versions(AGX)] impl super::QueueInner::ver { /// Submit work to a compute queue. + #[expect( + clippy::too_many_arguments, + reason = "Submission inputs are retained separately by the existing queue interface." + )] pub(super) fn submit_compute( &self, gpu: &gpu::GpuManager::ver, @@ -54,9 +58,10 @@ impl super::QueueInner::ver { return Err(EINVAL); } - let mut user_timestamps: fw::job::UserTimestamps = Default::default(); - user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts.start)?; - user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts.end)?; + let user_timestamps = fw::job::UserTimestamps { + start: common::get_timestamp_object(objects, cmdbuf.ts.start)?, + end: common::get_timestamp_object(objects, cmdbuf.ts.end)?, + }; // This sequence number increases per new client/VM? assigned to some slot, // but it's unclear *which* slot... @@ -102,6 +107,10 @@ impl super::QueueInner::ver { GFP_KERNEL, )?; + let usage_timestamps = timestamps.clone(); + let usage = self.usage.clone(); + let timestamp_hz = gpu.get_cfg().base_clock_hz; + let uuid = 0; mod_dev_dbg!(self.dev, "[Submission {}] UUID = {:#x?}\n", id, uuid); @@ -150,20 +159,18 @@ impl super::QueueInner::ver { notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), })?; - if user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), - work_queue: ev_comp.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, context_store_req), - uuid, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_comp.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, context_store_req), + uuid, + unk_30_padding: 0, + })?; #[ver(G < G14X)] builder.add(microseq::WaitForIdle { @@ -174,20 +181,18 @@ impl super::QueueInner::ver { header: microseq::op::WaitForIdle2::HEADER, })?; - if user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), - work_queue: ev_comp.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, context_store_req), - uuid, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_comp.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, context_store_req), + uuid, + unk_30_padding: 0, + })?; let off = builder.offset_to(start_comp); builder.add(microseq::FinalizeCompute::ver { @@ -214,7 +219,7 @@ impl super::QueueInner::ver { #[ver(G == G14 && V < V13_0B4)] unk_5c_g14: U64(0), restart_branch_offset: off, - has_attachments: (attachments.count > 0) as u32, + has_attachments: u32::from(attachments.count > 0), #[ver(V >= V13_0B4)] unk_64: Default::default(), #[ver(V >= V13_0B4)] @@ -316,8 +321,8 @@ impl super::QueueInner::ver { unk_18: 0x0, // fixed unk_mask: 0xffffffff, sampler_array: U64(cmdbuf.sampler_heap), - sampler_count: cmdbuf.sampler_count as u32, - sampler_max: (cmdbuf.sampler_count as u32) + 1, + sampler_count: cmdbuf.sampler_count, + sampler_max: cmdbuf.sampler_count + 1, }), meta <- try_init!(fw::job::raw::JobMeta { unk_0: 0, @@ -328,7 +333,7 @@ impl super::QueueInner::ver { stamp_value: ev_comp.value.next(), stamp_slot: ev_comp.slot, evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, + flush_stamps: u32::from(flush_stamps), uuid, event_seq: ev_comp.event_seq as u32, }), @@ -362,6 +367,15 @@ impl super::QueueInner::ver { comp_job.add_cb(comp, vm_bind.slot(), move |error| { if let Some(err) = error { fence.set_error(err.into()) + } else { + usage_timestamps.with(|raw, _| { + usage.add( + crate::usage::Engine::Compute, + raw.start.load(Ordering::Relaxed), + raw.end.load(Ordering::Relaxed), + timestamp_hz, + ); + }); } fence.command_complete(); diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 2d9052c2edb51..aa5dc1fba4723 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -67,7 +67,9 @@ mod render; /// Trait implemented by all versioned queues. pub(crate) trait Queue: Send + Sync { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - if layouts != [0; 2] { return Err(EINVAL); } + if layouts != [0; 2] { + return Err(EINVAL); + } Ok(()) } @@ -151,6 +153,7 @@ pub(crate) struct QueueInner { notifier_list: Arc>, notifier: Arc>, usc_exec_base: u64, + usage: Arc, id: u64, #[ver(V >= V13_0B4)] counter: AtomicU64, @@ -191,7 +194,7 @@ impl JobFence::ver { #[vtable] impl dma_fence::FenceOps for JobFence::ver { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("asahi") + c_str!("aurora") } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("queue") @@ -266,7 +269,10 @@ impl sched::JobImpl for QueueJob::ver { Some(kernel::module::this_module::()); fn timeout_data(&self) -> QueueTimeout { - QueueTimeout { dev: self.dev.clone(), id: self.id } + QueueTimeout { + dev: self.dev.clone(), + id: self.id, + } } fn prepare(job: &mut Self) -> Option { @@ -358,7 +364,7 @@ impl sched::JobImpl for QueueJob::ver { return Err(ENODEV); }; - return guard.registration_data_with(|reg_data| { + guard.registration_data_with(|reg_data| { let gpu = match reg_data .gpu .clone() @@ -433,7 +439,7 @@ impl sched::JobImpl for QueueJob::ver { core::mem::drop(frag_job); Ok(Some(Fence::from_fence(&job.fence))) - }); + }) } fn timed_out(job: &mut QueueTimeout, _finished: bool) -> sched::Status { @@ -462,7 +468,7 @@ impl Drop for QueueJob::ver { } } -static QUEUE_NAME: &CStr = c_str!("asahi_fence"); +static QUEUE_NAME: &CStr = c_str!("aurora_fence"); static QUEUE_CLASS_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); #[versions(AGX)] @@ -482,6 +488,7 @@ impl Queue::ver { id: u64, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result { mod_dev_dbg!(dev, "[Queue {}] Creating queue\n", id); @@ -519,7 +526,7 @@ impl Queue::ver { WQ_SIZE, 0, 100000, - c_str!("asahi_sched"), + c_str!("aurora_sched"), )?; let entity = sched::Entity::new(&sched, sched::Priority::Kernel)?; @@ -552,6 +559,7 @@ impl Queue::ver { notifier_list: Arc::new(notifier_list, GFP_KERNEL)?, notifier, usc_exec_base, + usage, id, #[ver(V >= V13_0B4)] counter: AtomicU64::new(0), @@ -629,8 +637,11 @@ const COMPUTE_SUPPORTED_SIZE: usize = // SAFETY: All bit patterns are valid by construction. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_header {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_render {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_compute {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_attachment {} fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result { @@ -641,8 +652,10 @@ fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result, size: usize) -> Result> 7; - attachments.list[i as usize] = microseq::Attachment { + attachments.list[i] = microseq::Attachment { address: U64(att.pointer), size: cache_lines.try_into()?, unk_c: 0x17, @@ -761,16 +774,24 @@ impl Queue for Queue::ver { while !cmdbuf.is_empty() { let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; let payload_size = header.size as usize; - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(payload_size)?; - if render.ts_frag_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + if render + .ts_frag_end_copies + .iter() + .any(|ts| ts.handle != 0 || ts.offset != 0) + { return Err(EINVAL); } } uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(payload_size)?; - if compute.ts_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + if compute + .ts_end_copies + .iter() + .any(|ts| ts.handle != 0 || ts.offset != 0) + { return Err(EINVAL); } } @@ -778,7 +799,7 @@ impl Queue for Queue::ver { } nr_commands += 1; - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { last_render = nr_commands; nr_render += 1; @@ -870,7 +891,7 @@ impl Queue for Queue::ver { EINVAL })? { let mut alloc = gpu.alloc(); - let queue_job = match header.cmd_type as u32 { + let queue_job = match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => job.get_vtx()?, uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => job.get_comp()?, _ => return Err(EINVAL), @@ -900,7 +921,7 @@ impl Queue for Queue::ver { } } - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { let render: uapi::drm_asahi_cmd_render = cmdbuf .read_supported(header_size, RENDER_SUPPORTED_SIZE) diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 68c4856ca4200..e02411e2417c0 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -47,9 +47,9 @@ impl super::QueueInner::ver { cmdbuf: &uapi::drm_asahi_cmd_render, num_clusters: u32, ) -> Result { - let width: u32 = cmdbuf.width_px as u32; - let height: u32 = cmdbuf.height_px as u32; - let layers: u32 = cmdbuf.layers as u32; + let width: u32 = u32::from(cmdbuf.width_px); + let height: u32 = u32::from(cmdbuf.height_px); + let layers: u32 = u32::from(cmdbuf.layers); if layers == 0 || layers > 2048 { cls_pr_debug!(Errors, "Layer count invalid ({})\n", layers); @@ -68,8 +68,8 @@ impl super::QueueInner::ver { let tile_width = 32u32; let tile_height = 32u32; - let utile_width = cmdbuf.utile_width_px as u32; - let utile_height = cmdbuf.utile_height_px as u32; + let utile_width = u32::from(cmdbuf.utile_width_px); + let utile_height = u32::from(cmdbuf.utile_height_px); match (utile_width, utile_height) { (32, 32) | (32, 16) | (16, 16) => (), @@ -153,7 +153,7 @@ impl super::QueueInner::ver { tpc_size, meta1_layer_stride, #[ver(G < G14X)] - meta1_blocks: meta1_layer_stride * (cmdbuf.layers as u32), + meta1_blocks: meta1_layer_stride * u32::from(cmdbuf.layers), #[ver(G >= G14X)] meta1_blocks: meta1_layer_stride, layermeta_size: if layers > 1 { 0x100 } else { 0 }, @@ -182,6 +182,10 @@ impl super::QueueInner::ver { } /// Submit work to a render queue. + #[expect( + clippy::too_many_arguments, + reason = "Submission inputs are retained separately by the existing queue interface." + )] pub(super) fn submit_render( &self, gpu: &gpu::GpuManager::ver, @@ -199,7 +203,7 @@ impl super::QueueInner::ver { & !(uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING - | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT) as u32 + | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT) != 0 { cls_pr_debug!(Errors, "Invalid flags ({:#x})\n", cmdbuf.flags); @@ -235,8 +239,7 @@ impl super::QueueInner::ver { let mut clustering = nclusters > 1; if debug_enabled(debug::DebugFlags::DisableClustering) - || cmdbuf.flags - & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING as u32 + || cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING != 0 { clustering = false; @@ -260,7 +263,7 @@ impl super::QueueInner::ver { // but it's unclear *which* slot... let slot_client_seq: u8 = (self.id & 0xff) as u8; - let tile_info = Self::get_tiling_params(&cmdbuf, if clustering { nclusters } else { 1 })?; + let tile_info = Self::get_tiling_params(cmdbuf, if clustering { nclusters } else { 1 })?; let buffer = &self.buffer; let notifier = self.notifier.clone(); @@ -375,9 +378,7 @@ impl super::QueueInner::ver { if cmdbuf.layers > 1 { tile_config |= 1; } - if cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES as u32 - != 0 - { + if cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES != 0 { tile_config |= 0x10000; } @@ -398,19 +399,19 @@ impl super::QueueInner::ver { // Calculate the number of 2KiB blocks to allocate per utile. This is // just a bit of dimensional analysis. let pixels_per_utile: u32 = - (cmdbuf.utile_width_px as u32) * (cmdbuf.utile_height_px as u32); + u32::from(cmdbuf.utile_width_px) * u32::from(cmdbuf.utile_height_px); let samples_per_utile: u32 = pixels_per_utile << samples_log2; - let utile_size_bytes: u32 = (cmdbuf.sample_size_B as u32) * samples_per_utile; + let utile_size_bytes: u32 = u32::from(cmdbuf.sample_size_B) * samples_per_utile; let block_size_bytes: u32 = 2048; let blocks_per_utile: u32 = utile_size_bytes.div_ceil(block_size_bytes); #[ver(G >= G14X)] let frg_tilecfg = 0x0000000_00036011 - | (((tile_info.tiles_x - 1) as u64) << 44) - | (((tile_info.tiles_y - 1) as u64) << 53) + | (u64::from(tile_info.tiles_x - 1) << 44) + | (u64::from(tile_info.tiles_y - 1) << 53) | (if unk1 { 0 } else { 0x20_00000000 }) | (if cmdbuf.layers > 1 { 0x1_00000000 } else { 0 }) - | ((utile_config as u64 & 0xf000) << 28); + | ((u64::from(utile_config) & 0xf000) << 28); // TODO: check #[ver(V >= V13_0B4)] @@ -428,9 +429,9 @@ impl super::QueueInner::ver { let frg_unk_140 = 0x8c60; let frg_unk_158 = 0x1c; #[ver(G >= G14)] - let load_bgobjvals = cmdbuf.isp_bgobjvals as u64; + let load_bgobjvals = u64::from(cmdbuf.isp_bgobjvals); #[ver(G < G14)] - let load_bgobjvals = cmdbuf.isp_bgobjvals as u64 | 0x400; + let load_bgobjvals = u64::from(cmdbuf.isp_bgobjvals) | 0x400; let reload_zlsctrl = cmdbuf.zls_ctrl; let iogpu_unk54 = 0x3a0012006b0003; let iogpu_unk56 = 1; @@ -441,12 +442,12 @@ impl super::QueueInner::ver { #[ver(G >= G14X)] let vtx_unk_f0 = 0x1c; #[ver(G < G14)] - let vtx_unk_f0 = 0x1c + (align(tile_info.meta1_blocks, 4) as u64); + let vtx_unk_f0 = 0x1c + u64::from(align(tile_info.meta1_blocks, 4)); let vtx_unk_118 = 0x1c; // DRM_ASAHI_RENDER_DBIAS_IS_INT chosen to match hardware bit. - let isp_ctl = 0xc000u32 - | (cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT as u32); + let isp_ctl = + 0xc000u32 | (cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT); // Always allow preemption at the UAPI level let no_preemption = false; @@ -511,20 +512,18 @@ impl super::QueueInner::ver { notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), })?; - if frg_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), - work_queue: ev_frag.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_3d, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_frag.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; #[ver(G < G14X)] builder.add(microseq::WaitForIdle { @@ -535,20 +534,18 @@ impl super::QueueInner::ver { header: microseq::op::WaitForIdle2::HEADER, })?; - if frg_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), - work_queue: ev_frag.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_3d, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_frag.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; let off = builder.offset_to(start_frag); builder.add(microseq::FinalizeFragment::ver { @@ -579,7 +576,7 @@ impl super::QueueInner::ver { #[ver(G == G14 && V < V13_0B4)] unk_8c_g14: U64(0), restart_branch_offset: off, - has_attachments: (fragment_attachments.count > 0) as u32, + has_attachments: u32::from(fragment_attachments.count > 0), #[ver(V >= V13_0B4)] unk_9c: Default::default(), })?; @@ -601,10 +598,10 @@ impl super::QueueInner::ver { |inner, _ptr| { let vm_slot = vm_bind.slot(); let aux_fb_info = fw::fragment::raw::AuxFBInfo::ver { - isp_ctl: isp_ctl, + isp_ctl, unk2: 0, - width: cmdbuf.width_px as u32, - height: cmdbuf.height_px as u32, + width: u32::from(cmdbuf.width_px), + height: u32::from(cmdbuf.height_px), #[ver(V >= V13_0B4)] unk3: U64(0x100000), }; @@ -623,7 +620,7 @@ impl super::QueueInner::ver { unk_buffer_buf: inner.scene.kernel_buffer_pointer(), tvb_tilemap: inner.scene.tvb_tilemap_pointer(), ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), - samples: cmdbuf.samples as u32, + samples: u32::from(cmdbuf.samples), tiles_per_mtile_y: tile_info.tiles_per_mtile_y as u16, tiles_per_mtile_x: tile_info.tiles_per_mtile_x as u16, unk_50: U64(0), @@ -631,21 +628,21 @@ impl super::QueueInner::ver { isp_merge_upper_x: F32::from_bits(cmdbuf.isp_merge_upper_x), isp_merge_upper_y: F32::from_bits(cmdbuf.isp_merge_upper_y), unk_68: U64(0), - tile_count: U64(tile_info.tiles as u64), + tile_count: U64(u64::from(tile_info.tiles)), #[ver(G < G14X)] job_params1 <- try_init!(fw::fragment::raw::JobParameters1::ver { utile_config, unk_4: 0, bg: fw::fragment::raw::BackgroundProgram { - rsrc_spec: U64(cmdbuf.bg.rsrc_spec as u64), - address: U64(cmdbuf.bg.usc as u64), + rsrc_spec: U64(u64::from(cmdbuf.bg.rsrc_spec)), + address: U64(u64::from(cmdbuf.bg.usc)), }, ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), isp_scissor_base: U64(cmdbuf.isp_scissor_base), isp_dbias_base: U64(cmdbuf.isp_dbias_base), isp_oclqry_base: U64(cmdbuf.isp_oclqry_base), aux_fb_info, - isp_zls_pixels: U64(cmdbuf.isp_zls_pixels as u64), + isp_zls_pixels: U64(u64::from(cmdbuf.isp_zls_pixels)), zls_ctrl: U64(cmdbuf.zls_ctrl), #[ver(G >= G14)] unk_58_g14_0: U64(g14_unk), @@ -657,21 +654,21 @@ impl super::QueueInner::ver { s_store: U64(cmdbuf.stencil.base), #[ver(G >= G14)] unk_68_g14_0: Default::default(), - z_load_stride: U64(cmdbuf.depth.stride as u64), - z_store_stride: U64(cmdbuf.depth.stride as u64), - s_load_stride: U64(cmdbuf.stencil.stride as u64), - s_store_stride: U64(cmdbuf.stencil.stride as u64), + z_load_stride: U64(u64::from(cmdbuf.depth.stride)), + z_store_stride: U64(u64::from(cmdbuf.depth.stride)), + s_load_stride: U64(u64::from(cmdbuf.stencil.stride)), + s_store_stride: U64(u64::from(cmdbuf.stencil.stride)), z_load_comp: U64(cmdbuf.depth.comp_base), - z_load_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_load_comp_stride: U64(u64::from(cmdbuf.depth.comp_stride)), z_store_comp: U64(cmdbuf.depth.comp_base), - z_store_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_store_comp_stride: U64(u64::from(cmdbuf.depth.comp_stride)), s_load_comp: U64(cmdbuf.stencil.comp_base), - s_load_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_load_comp_stride: U64(u64::from(cmdbuf.stencil.comp_stride)), s_store_comp: U64(cmdbuf.stencil.comp_base), - s_store_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_store_comp_stride: U64(u64::from(cmdbuf.stencil.comp_stride)), tvb_tilemap: inner.scene.tvb_tilemap_pointer(), tvb_layermeta: inner.scene.tvb_layermeta_pointer(), - mtile_stride_dwords: U64((4 * tile_info.params.rgn_size as u64) << 24), + mtile_stride_dwords: U64((4 * u64::from(tile_info.params.rgn_size)) << 24), tvb_heapmeta: inner.scene.tvb_heapmeta_pointer(), tile_config: U64(tile_config), aux_fb: inner.aux_fb.gpu_pointer(), @@ -741,7 +738,7 @@ impl super::QueueInner::ver { r.add(0x15021, isp_ctl.into()); // aux_fb_info.unk_1 r.add( 0x15211, - ((cmdbuf.height_px as u64) << 32) | cmdbuf.width_px as u64, + (u64::from(cmdbuf.height_px) << 32) | u64::from(cmdbuf.width_px), ); // aux_fb_info.{width, heigh r.add(0x15049, 0x100000); // s2.aux_fb_info.unk3 r.add(0x10051, blocks_per_utile.into()); // s1.unk_2c @@ -760,21 +757,21 @@ impl super::QueueInner::ver { r.add(0x15221, 0); r.add(0x15239, 0); r.add(0x15229, 0); - r.add(0x15401, cmdbuf.depth.stride as u64); // load - r.add(0x15421, cmdbuf.depth.stride as u64); // store - r.add(0x15409, cmdbuf.stencil.stride as u64); // load - r.add(0x15429, cmdbuf.stencil.stride as u64); + r.add(0x15401, u64::from(cmdbuf.depth.stride)); // load + r.add(0x15421, u64::from(cmdbuf.depth.stride)); // store + r.add(0x15409, u64::from(cmdbuf.stencil.stride)); // load + r.add(0x15429, u64::from(cmdbuf.stencil.stride)); r.add(0x153c1, cmdbuf.depth.comp_base); // load - r.add(0x15411, cmdbuf.depth.comp_stride as u64); // load + r.add(0x15411, u64::from(cmdbuf.depth.comp_stride)); // load r.add(0x153c9, cmdbuf.depth.comp_base); // store - r.add(0x15431, cmdbuf.depth.comp_stride as u64); // store + r.add(0x15431, u64::from(cmdbuf.depth.comp_stride)); // store r.add(0x153d1, cmdbuf.stencil.comp_base); // load - r.add(0x15419, cmdbuf.stencil.comp_stride as u64); // load + r.add(0x15419, u64::from(cmdbuf.stencil.comp_stride)); // load r.add(0x153d9, cmdbuf.stencil.comp_base); // store - r.add(0x15439, cmdbuf.stencil.comp_stride as u64); // store + r.add(0x15439, u64::from(cmdbuf.stencil.comp_stride)); // store r.add(0x16429, inner.scene.tvb_tilemap_pointer().into()); r.add(0x16060, inner.scene.tvb_layermeta_pointer().into()); - r.add(0x16431, (4 * tile_info.params.rgn_size as u64) << 24); // ISP_RGN? + r.add(0x16431, (4 * u64::from(tile_info.params.rgn_size)) << 24); // ISP_RGN? r.add(0x10039, tile_config); // tile_config ISP_CTL? r.add(0x16451, 0x0); // ISP_RENDER_ORIGIN r.add(0x11821, cmdbuf.fragment_helper.binary.into()); @@ -811,28 +808,28 @@ impl super::QueueInner::ver { unk_118: U64(0x0), unk_120: Default::default(), unk_partial_bg: fw::fragment::raw::BackgroundProgram { - rsrc_spec: U64(cmdbuf.partial_bg.rsrc_spec as u64), - address: U64(cmdbuf.partial_bg.usc as u64), + rsrc_spec: U64(u64::from(cmdbuf.partial_bg.rsrc_spec)), + address: U64(u64::from(cmdbuf.partial_bg.usc)), }, unk_258: U64(0), unk_260: U64(0), unk_268: U64(0), unk_270: U64(0), partial_bg: fw::fragment::raw::BackgroundProgram { - rsrc_spec: U64(cmdbuf.partial_bg.rsrc_spec as u64), - address: U64(cmdbuf.partial_bg.usc as u64), + rsrc_spec: U64(u64::from(cmdbuf.partial_bg.rsrc_spec)), + address: U64(u64::from(cmdbuf.partial_bg.usc)), }, zls_ctrl: U64(reload_zlsctrl), unk_290: U64(g14_unk), z_load: U64(cmdbuf.depth.base), - z_partial_stride: U64(cmdbuf.depth.stride as u64), - z_partial_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_partial_stride: U64(u64::from(cmdbuf.depth.stride)), + z_partial_comp_stride: U64(u64::from(cmdbuf.depth.comp_stride)), z_store: U64(cmdbuf.depth.base), z_partial: U64(cmdbuf.depth.base), z_partial_comp: U64(cmdbuf.depth.comp_base), s_load: U64(cmdbuf.stencil.base), - s_partial_stride: U64(cmdbuf.stencil.stride as u64), - s_partial_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_partial_stride: U64(u64::from(cmdbuf.stencil.stride)), + s_partial_comp_stride: U64(u64::from(cmdbuf.stencil.comp_stride)), s_store: U64(cmdbuf.stencil.base), s_partial: U64(cmdbuf.stencil.base), s_partial_comp: U64(cmdbuf.stencil.comp_base), @@ -852,13 +849,13 @@ impl super::QueueInner::ver { ), isp_bgobjdepth: cmdbuf.isp_bgobjdepth, isp_bgobjvals: cmdbuf.isp_bgobjvals, - sample_size: cmdbuf.sample_size_B as u32, + sample_size: u32::from(cmdbuf.sample_size_B), unk_37c: 0x0, unk_380: U64(0x0), unk_388: U64(0x0), #[ver(V >= V13_0B4)] unk_390_0: U64(0x0), - isp_zls_pixels: U64(cmdbuf.isp_zls_pixels as u64), + isp_zls_pixels: U64(u64::from(cmdbuf.isp_zls_pixels)), }), unk_758_flag: 0, unk_75c_flag: 0, @@ -875,12 +872,11 @@ impl super::QueueInner::ver { unk_18: 0x0, // fixed unk_mask: 0xffffffffu32, sampler_array: U64(cmdbuf.sampler_heap), - sampler_count: cmdbuf.sampler_count as u32, - sampler_max: (cmdbuf.sampler_count as u32) + 1, + sampler_count: u32::from(cmdbuf.sampler_count), + sampler_max: u32::from(cmdbuf.sampler_count) + 1, }), - process_empty_tiles: (cmdbuf.flags - & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES as u32 - != 0) as u32, + process_empty_tiles: u32::from(cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES != 0), // TODO: needs to be investigated no_clear_pipeline_textures: 1, // TODO: needs to be investigated @@ -891,13 +887,13 @@ impl super::QueueInner::ver { meta <- try_init!(fw::job::raw::JobMeta { unk_0: 0, unk_2: 0, - no_preemption: no_preemption as u8, + no_preemption: u8::from(no_preemption), stamp: ev_frag.stamp_pointer, fw_stamp: ev_frag.fw_stamp_pointer, stamp_value: ev_frag.value.next(), stamp_slot: ev_frag.slot, evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, + flush_stamps: u32::from(flush_stamps), uuid: uuid_3d, event_seq: ev_frag.event_seq as u32, }), @@ -929,9 +925,21 @@ impl super::QueueInner::ver { mod_dev_dbg!(self.dev, "[Submission {}] Add Frag\n", id); fence.add_command(); + let usage = self.usage.clone(); + let usage_timestamps = timestamps.clone(); + let timestamp_hz = gpu.get_cfg().base_clock_hz; frag_job.add_cb(frag, vm_bind.slot(), move |error| { if let Some(err) = error { fence.set_error(err.into()); + } else { + usage_timestamps.with(|raw, _| { + usage.add( + crate::usage::Engine::Fragment, + raw.frag.start.load(Ordering::Relaxed), + raw.frag.end.load(Ordering::Relaxed), + timestamp_hz, + ); + }); } fence.command_complete(); @@ -1019,23 +1027,21 @@ impl super::QueueInner::ver { #[ver(V < V13_0B4)] unk_178: 0x0, // padding? #[ver(V >= V13_0B4)] - unk_178: (!clustering) as u32, + unk_178: u32::from(!clustering), })?; - if vtx_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), - work_queue: ev_vtx.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_ta, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_vtx.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; #[ver(G < G14X)] builder.add(microseq::WaitForIdle { @@ -1046,20 +1052,18 @@ impl super::QueueInner::ver { header: microseq::op::WaitForIdle2::HEADER, })?; - if vtx_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), - work_queue: ev_vtx.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_ta, - unk_30_padding: 0, - })?; - } + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_vtx.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; let off = builder.offset_to(start_vtx); builder.add(microseq::FinalizeVertex::ver { @@ -1085,7 +1089,7 @@ impl super::QueueInner::ver { #[ver(G >= G14 && V < V13_0B4)] unk_68_g14: U64(0), restart_branch_offset: off, - has_attachments: (vertex_attachments.count > 0) as u32, + has_attachments: u32::from(vertex_attachments.count > 0), #[ver(V >= V13_0B4)] unk_74: Default::default(), // Ventura })?; @@ -1136,7 +1140,7 @@ impl super::QueueInner::ver { tvb_cluster_meta1: inner .scene .meta_1_pointer() - .map(|x| x.or((tile_info.meta1_layer_stride as u64) << 50)), + .map(|x| x.or(u64::from(tile_info.meta1_layer_stride) << 50)), utile_config, unk_4c: 0, ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), // fixed @@ -1264,8 +1268,8 @@ impl super::QueueInner::ver { r.add(0x10121, tile_info.params.ppp_ctrl.into()); // PPP_CTRL r.add( 0x10129, - tile_info.params.x_max as u64 - | ((tile_info.params.y_max as u64) << 16), + u64::from(tile_info.params.x_max) + | (u64::from(tile_info.params.y_max) << 16), ); // PPP_SCREEN r.add(0x101b9, tile_info.params.te_screen.into()); // TE_SCREEN r.add(0x1c069, tile_info.params.te_mtile1.into()); // TE_MTILE1 @@ -1319,26 +1323,25 @@ impl super::QueueInner::ver { unk_18: 0x0, // fixed unk_mask: 0xffffffffu32, sampler_array: U64(cmdbuf.sampler_heap), - sampler_count: cmdbuf.sampler_count as u32, - sampler_max: (cmdbuf.sampler_count as u32) + 1, + sampler_count: u32::from(cmdbuf.sampler_count), + sampler_max: u32::from(cmdbuf.sampler_count) + 1, }), unk_55c: 0, unk_560: 0, sync_grow: 0, unk_568: 0, - uses_scratch: (cmdbuf.flags - & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH as u32 - != 0) as u32, + uses_scratch: u32::from(cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH != 0), meta <- try_init!(fw::job::raw::JobMeta { unk_0: 0, unk_2: 0, - no_preemption: no_preemption as u8, + no_preemption: u8::from(no_preemption), stamp: ev_vtx.stamp_pointer, fw_stamp: ev_vtx.fw_stamp_pointer, stamp_value: ev_vtx.value.next(), stamp_slot: ev_vtx.slot, evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, + flush_stamps: u32::from(flush_stamps), uuid: uuid_ta, event_seq: ev_vtx.event_seq as u32, }), @@ -1368,9 +1371,20 @@ impl super::QueueInner::ver { mod_dev_dbg!(self.dev, "[Submission {}] Add Vertex\n", id); fence.add_command(); + let usage = self.usage.clone(); + let usage_timestamps = timestamps.clone(); vtx_job.add_cb(vtx, vm_bind.slot(), move |error| { if let Some(err) = error { fence.set_error(err.into()) + } else { + usage_timestamps.with(|raw, _| { + usage.add( + crate::usage::Engine::Vertex, + raw.vtx.start.load(Ordering::Relaxed), + raw.vtx.end.load(Ordering::Relaxed), + timestamp_hz, + ); + }); } fence.command_complete(); diff --git a/drivers/gpu/drm/asahi/stats.rs b/drivers/gpu/drm/asahi/stats.rs index 772d969d9baff..d07de2c6e7384 100644 --- a/drivers/gpu/drm/asahi/stats.rs +++ b/drivers/gpu/drm/asahi/stats.rs @@ -11,7 +11,8 @@ //! the versioned context where `StatsMsg` is nameable; `note_job()` is called //! from the `recv_message` rtkit callback (so the firmware mailbox IRQ //! context); the sysfs `show` callback runs in arbitrary process context. All -//! fields use `AtomicU*` so neither side takes a lock. +//! legacy fields use `AtomicU*`. G17 instead uses the optional reader under +//! its device mutex and does not populate these legacy atomic fields. //! //! Field semantics (firmware units, late-validated): //! - `util1..util4`: raw `Utilization` u32s from the firmware. Meaning @@ -58,6 +59,9 @@ pub(crate) struct StatsSnapshot { /// `JobFence::command_complete` when the last command of a submission /// completes. pub(crate) jobs: AtomicU64, + /// Optional qualified reader. The legacy C layout above remains a prefix; + /// C accesses this extension only through the registered Rust callbacks. + pub(crate) g17_reader: Option, } // Keep the Rust half of the C sysfs layout contract checked as well. @@ -65,7 +69,7 @@ const _: () = { assert!(core::mem::offset_of!(StatsSnapshot, pstate) == 16); assert!(core::mem::offset_of!(StatsSnapshot, busy_ns) == 40); assert!(core::mem::offset_of!(StatsSnapshot, jobs) == 48); - assert!(core::mem::size_of::() == 56); + assert!(core::mem::offset_of!(StatsSnapshot, g17_reader) == 56); }; impl StatsSnapshot { diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 7d7d785a83133..7132e6de41c09 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -16,7 +18,7 @@ #include "sysfs.h" -/* Mirror of repr(C) Rust atomics; READ_ONCE accesses the current values. */ +/* Legacy prefix of repr(C) Rust atomics; READ_ONCE accesses current values. */ struct asahi_stats_snapshot { u32 util1; u32 util2; @@ -48,8 +50,69 @@ struct asahi_stats_attribute { const struct asahi_stats_snapshot *snapshot; const struct asahi_gpu_memory *memory; bool export_enabled; + asahi_stats_read_fn read; + asahi_stats_suspend_fn suspend; + struct notifier_block pm_notifier; }; +/* 0: unavailable, 1: valid, 2: stale; matches g17::telemetry. */ +static const char *asahi_stats_quality(u32 status) +{ + return status == 1 ? "valid" : status == 2 ? "stale" : "unavailable"; +} + +static ssize_t agx_qualified_stats_show(struct asahi_stats_attribute *stats, char *buf) +{ + struct asahi_qualified_stats snap; + ssize_t n = 0; + + stats->read(stats->snapshot, &snap); + n += sysfs_emit_at(buf, n, "utilisation_status %s\n", + asahi_stats_quality(snap.util_status)); + if (snap.util_status == 1) { + n += sysfs_emit_at(buf, n, "util_tiler_percent %u\n", snap.tiler); + n += sysfs_emit_at(buf, n, "util_renderer_percent %u\n", snap.renderer); + n += sysfs_emit_at(buf, n, "util_device_percent %u\n", snap.device); + } else { + n += sysfs_emit_at(buf, n, "util_tiler_percent unsupported\n" + "util_renderer_percent unsupported\n" + "util_device_percent unsupported\n"); + } + n += sysfs_emit_at(buf, n, "pstate_status %s\n", + asahi_stats_quality(snap.pstate_status)); + if (snap.pstate_status == 1) + n += sysfs_emit_at(buf, n, "pstate %u\n", snap.pstate); + else + n += sysfs_emit_at(buf, n, "pstate unsupported\n"); + /* No A/B selector, packet duration, measured power or temperature. */ + n += sysfs_emit_at(buf, n, "requested_frequency_hz unsupported\n" + "achieved_frequency_hz unsupported\n" + "busy_ns unsupported\n" + "power_mw unsupported\n" + "temperature_raw unsupported\n"); + return n; +} + +static int asahi_stats_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +{ + struct asahi_stats_attribute *stats = + container_of(nb, struct asahi_stats_attribute, pm_notifier); + + switch (action) { + case PM_SUSPEND_PREPARE: + case PM_HIBERNATION_PREPARE: + case PM_RESTORE_PREPARE: + stats->suspend(stats->snapshot, true); + break; + case PM_POST_SUSPEND: + case PM_POST_HIBERNATION: + case PM_POST_RESTORE: + stats->suspend(stats->snapshot, false); + break; + } + return NOTIFY_OK; +} + static ssize_t agx_stats_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -60,6 +123,8 @@ static ssize_t agx_stats_show(struct device *dev, if (!stats->export_enabled) return sysfs_emit(buf, "unsupported\n"); + if (stats->read) + return agx_qualified_stats_show(stats, buf); n += scnprintf(buf + n, PAGE_SIZE - n, "busy_ns %llu\n", (unsigned long long)READ_ONCE(snap->busy_ns)); n += scnprintf(buf + n, PAGE_SIZE - n, "jobs %llu\n", @@ -118,14 +183,18 @@ static ssize_t gpu_memory_show(struct device *dev, * registered. Returns 0 on success, or a negative errno. */ int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, const void *memory, void **handle) + int export_enabled, const void *memory, void **handle, + asahi_stats_read_fn read, asahi_stats_suspend_fn suspend) { struct asahi_stats_attribute *stats; int ret; - if (!dev || !handle || !memory || (export_enabled && !snapshot)) + if (!dev || !handle || !memory || ((export_enabled || read) && !snapshot) || + (!!read != !!suspend)) return -ENODEV; + BUILD_BUG_ON(offsetof(struct asahi_qualified_stats, util_host_ns) != 24); + BUILD_BUG_ON(sizeof(struct asahi_qualified_stats) != 56); BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, refused) != 24); BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, client_limit) != 40); BUILD_BUG_ON(sizeof(struct asahi_gpu_memory) != 48); @@ -145,6 +214,16 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, stats->attr.show = agx_stats_show; stats->snapshot = snapshot; stats->export_enabled = export_enabled; + stats->read = read; + stats->suspend = suspend; + if (suspend) { + stats->pm_notifier.notifier_call = asahi_stats_pm_notify; + ret = register_pm_notifier(&stats->pm_notifier); + if (ret) { + kfree(stats); + return ret; + } + } sysfs_attr_init(&stats->memory_attr.attr); stats->memory_attr.attr.name = "gpu_memory"; @@ -154,6 +233,8 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, ret = device_create_file(dev, &stats->attr); if (ret) { + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); kfree(stats); return ret; } @@ -161,6 +242,8 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, ret = device_create_file(dev, &stats->memory_attr); if (ret) { device_remove_file(dev, &stats->attr); + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); kfree(stats); return ret; } @@ -217,6 +300,9 @@ void asahi_sysfs_unregister(struct device *dev, void *handle) if (!dev || !stats) return; + /* Both notifier removal and file removal join readers of the Rust Arc. */ + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); /* Removal drains active show callbacks before either allocation dies. */ device_remove_file(dev, &stats->memory_attr); device_remove_file(dev, &stats->attr); @@ -230,4 +316,4 @@ EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); MODULE_AUTHOR("AGX driver maintainer"); MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("Asahi AGX firmware stats sysfs shim"); +MODULE_DESCRIPTION("AGX firmware stats sysfs shim"); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index f1873f9be6caa..d43f482903e55 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -3,10 +3,30 @@ #ifndef _ASAHI_SYSFS_H #define _ASAHI_SYSFS_H +#include + struct device; +/* Mirrors g17::telemetry::Readout, filled under the Rust device mutex. */ +struct asahi_qualified_stats { + u32 util_status; + u32 pstate_status; + u32 tiler; + u32 renderer; + u32 device; + u32 pstate; + u64 util_host_ns; + u64 pstate_host_ns; + u64 util_samples; + u64 pstate_timestamp; +}; + +typedef void (*asahi_stats_read_fn)(const void *snapshot, struct asahi_qualified_stats *out); +typedef void (*asahi_stats_suspend_fn)(const void *snapshot, bool suspended); + int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, const void *memory, void **handle); + int export_enabled, const void *memory, void **handle, + asahi_stats_read_fn read, asahi_stats_suspend_fn suspend); int asahi_sysfs_register_idle(struct device *dev, u32 *word); void asahi_sysfs_unregister(struct device *dev, void *handle); diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index cee5d52a4e261..7c34f0627b020 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -3,16 +3,21 @@ //! Per-device sysfs registration. The caller retains the snapshot Arc until //! unregister has drained active readers and freed the C attribute wrapper. -use crate::{memacct::Ledger, stats::StatsSnapshot}; +use crate::{g17::telemetry::Readout, memacct::Ledger, stats::StatsSnapshot}; use core::ffi::{c_int, c_void}; +type ReadStats = unsafe extern "C" fn(*const c_void, *mut Readout); +type SuspendStats = unsafe extern "C" fn(*const c_void, bool); + extern "C" { fn asahi_sysfs_register( dev: *mut c_void, - snapshot: *const StatsSnapshot, + snapshot: *const c_void, export_enabled: c_int, memory: *const Ledger, handle: *mut *mut c_void, + read: Option, + suspend: Option, ) -> c_int; fn asahi_sysfs_unregister(dev: *mut c_void, handle: *mut c_void); } @@ -29,7 +34,11 @@ pub(crate) fn register_idle( // SAFETY: `word` is a static atomic that outlives the attribute; the C side reads and writes // it with READ_ONCE/WRITE_ONCE. let ret = unsafe { asahi_sysfs_register_idle(dev.cast(), word) }; - if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { Ok(()) } + if ret < 0 { + Err(kernel::error::Error::from_errno(ret)) + } else { + Ok(()) + } } /// Register only after initialization succeeds, keeping the snapshot Arc alive. @@ -39,6 +48,66 @@ pub(crate) fn register( dev: *mut kernel::bindings::device, snapshot: *const StatsSnapshot, export_enabled: bool, +) -> kernel::error::Result<*mut c_void> { + register_with_callbacks(dev, snapshot, export_enabled, None, None) +} + +/// The caller retains the G17 snapshot Arc until unregister joins both sysfs +/// readers and the PM notifier. The callbacks never power the GPU up. +pub(crate) fn register_g17( + dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, + export_enabled: bool, +) -> kernel::error::Result<*mut c_void> { + register_with_callbacks( + dev, + snapshot, + export_enabled, + Some(read_g17), + Some(suspend_g17), + ) +} + +/// Copy the locked qualified snapshot to the C formatter's local output. +/// +/// # Safety +/// +/// `snapshot` must name a live StatsSnapshot retained by the registration. +/// `out` must be aligned and writable for one Readout for this call. +unsafe extern "C" fn read_g17(snapshot: *const c_void, out: *mut Readout) { + // SAFETY: C calls only while registration retains the snapshot Arc, and + // supplies a writable, aligned output for this call. + let snapshot = unsafe { &*snapshot.cast::() }; + let view = snapshot + .g17_reader + .as_ref() + .map_or_else(Readout::default, |reader| reader.read()); + // SAFETY: As above; the C output is initialized in one locked read. + unsafe { out.write(view) }; +} + +/// End the telemetry epoch at either side of system sleep. +/// +/// # Safety +/// +/// `snapshot` must name a live StatsSnapshot. The caller must join this +/// callback before releasing the registration's owning Arc. +unsafe extern "C" fn suspend_g17(snapshot: *const c_void, suspended: bool) { + // SAFETY: The notifier is unregistered and joined before the Arc is dropped. + if let Some(reader) = unsafe { &*snapshot.cast::() } + .g17_reader + .as_ref() + { + reader.suspend(suspended); + } +} + +fn register_with_callbacks( + dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, + export_enabled: bool, + read: Option, + suspend: Option, ) -> kernel::error::Result<*mut c_void> { let mut handle = core::ptr::null_mut(); // SAFETY: The bound driver keeps the device and optional snapshot alive until @@ -46,10 +115,12 @@ pub(crate) fn register( let ret = unsafe { asahi_sysfs_register( dev.cast(), - snapshot, + snapshot.cast(), c_int::from(export_enabled), &crate::memacct::LEDGER, &mut handle, + read, + suspend, ) }; if ret < 0 { diff --git a/drivers/gpu/drm/asahi/usage.rs b/drivers/gpu/drm/asahi/usage.rs new file mode 100644 index 0000000000000..496c722dba651 --- /dev/null +++ b/drivers/gpu/drm/asahi/usage.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-file engine time, shared by all firmware backends. +//! +//! Queues retain their file's ledger until their last completion. Only authenticated, +//! successful completion timestamps are charged, before signalling the completion fence. +//! These are command timer ticks, not shader clock cycles. Never export them as drm-cycles. + +use core::sync::atomic::{AtomicU64, Ordering}; + +#[derive(Clone, Copy)] +pub(crate) enum Engine { + Vertex, + Fragment, + Compute, +} + +/// One ledger per DRM file, shared by all of its queues and VMs. +pub(crate) struct Client { + ns: [AtomicU64; 3], +} + +impl Client { + pub(crate) const fn new() -> Self { + Self { + ns: [const { AtomicU64::new(0) }; 3], + } + } + + /// Called exactly once per retired stage, while its internal timestamps are retained. + /// Missing, reversed or reset timestamps contribute nothing. Sub-nanosecond fractions + /// are rounded down per stage. Saturation keeps fdinfo monotonic even on overflow. + pub(crate) fn add(&self, engine: Engine, start: u64, end: u64, frequency_hz: u32) { + if start == 0 || end <= start || frequency_hz == 0 { + return; + } + let ns = ticks_to_ns(end - start, frequency_hz); + let counter = &self.ns[engine as usize]; + let mut old = counter.load(Ordering::Relaxed); + loop { + match counter.compare_exchange_weak( + old, + old.saturating_add(ns), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(value) => old = value, + } + } + } + + /// Vertex, fragment and compute, in ns. No cross-engine atomic snapshot is promised. + pub(crate) fn snapshot(&self) -> [u64; 3] { + self.ns.each_ref().map(|ns| ns.load(Ordering::Relaxed)) + } +} + +fn ticks_to_ns(ticks: u64, frequency_hz: u32) -> u64 { + const NSEC_PER_SEC: u64 = 1_000_000_000; + let hz = u64::from(frequency_hz); + // The remainder is smaller than a u32 frequency, so its product fits in u64. + (ticks / hz) + .saturating_mul(NSEC_PER_SEC) + .saturating_add(ticks % hz * NSEC_PER_SEC / hz) +} + +#[cfg(test)] +mod tests { + use super::*; + + const HZ: u32 = 24_000_000; + + #[test] + fn idle_and_invalid_intervals() { + let client = Client::new(); + assert_eq!(client.snapshot(), [0; 3]); + for (start, end, hz) in [ + (0, 100, HZ), + (100, 0, HZ), + (20, 10, HZ), + (1, 1, HZ), + (1, 2, 0), + ] { + client.add(Engine::Compute, start, end, hz); + } + assert_eq!(client.snapshot(), [0; 3]); + } + + #[test] + fn engine_and_client_isolation() { + let a = Client::new(); + let b = Client::new(); + a.add(Engine::Vertex, 10, 24_010, HZ); + a.add(Engine::Fragment, 20, 48_020, HZ); + b.add(Engine::Compute, 30, 72_030, HZ); + assert_eq!(a.snapshot(), [1_000_000, 2_000_000, 0]); + assert_eq!(b.snapshot(), [0, 0, 3_000_000]); + } + + #[test] + fn overlapping_stages_are_independent() { + let client = Client::new(); + client.add(Engine::Vertex, 1, 25, HZ); + client.add(Engine::Fragment, 13, 37, HZ); + assert_eq!(client.snapshot(), [1_000, 1_000, 0]); + } + + #[test] + fn conversion_matches_wide_reference() { + for hz in [1, HZ, 1_000_000_000, u32::MAX] { + for ticks in [0, 1, 23, 24, 25, u32::MAX as u64, u64::MAX / 2, u64::MAX] { + let expected = (u128::from(ticks) * 1_000_000_000 / u128::from(hz)) + .min(u128::from(u64::MAX)) as u64; + assert_eq!(ticks_to_ns(ticks, hz), expected); + } + } + } + + #[test] + fn fractional_ticks_round_down() { + assert_eq!(ticks_to_ns(1, HZ), 41); + assert_eq!(ticks_to_ns(3, HZ), 125); + } + + #[test] + fn out_of_order_completions_accumulate() { + let client = Client::new(); + client.add(Engine::Compute, 100, 124, HZ); + client.add(Engine::Compute, 1, 25, HZ); + assert_eq!(client.snapshot(), [0, 0, 2_000]); + } + + #[test] + fn saturation_is_monotonic() { + let client = Client::new(); + client.add(Engine::Compute, 1, u64::MAX, 1); + assert_eq!(client.snapshot()[2], u64::MAX); + client.add(Engine::Compute, 1, 25, HZ); + assert_eq!(client.snapshot()[2], u64::MAX); + } + + #[test] + fn concurrent_queues_share_one_file() { + let client = std::sync::Arc::new(Client::new()); + let threads: std::vec::Vec<_> = (0..8) + .map(|_| { + let client = client.clone(); + std::thread::spawn(move || { + for _ in 0..10_000 { + client.add(Engine::Compute, 1, 25, HZ); + } + }) + }) + .collect(); + for thread in threads { + thread.join().unwrap(); + } + assert_eq!(client.snapshot(), [0, 0, 80_000_000]); + } +} diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs index b51b027eb573c..6e5540caeac3f 100644 --- a/drivers/gpu/drm/asahi/util.rs +++ b/drivers/gpu/drm/asahi/util.rs @@ -96,6 +96,13 @@ pub(crate) fn gcd(in_n: u64, in_m: u64) -> u64 { m } +/// A value that can be populated from an untrusted byte record. +/// +/// # Safety +/// +/// Every bit pattern must be valid for Self. Its byte representation must +/// contain no uninitialized bytes (including padding) when produced by Default, +/// since Reader temporarily views that storage as a byte slice. pub(crate) unsafe trait AnyBitPattern: Default + Sized + Copy + 'static {} pub(crate) struct Reader<'a> { @@ -117,7 +124,9 @@ impl<'a> Reader<'a> { // SAFETY: The output pointer is valid, and the size does not exceed // the type size, and all bit patterns are valid. - let dst = unsafe { core::slice::from_raw_parts_mut(&mut obj as *mut _ as *mut u8, size) }; + let dst = unsafe { + core::slice::from_raw_parts_mut(core::ptr::from_mut(&mut obj).cast::(), size) + }; dst.copy_from_slice(&src[..size]); self.offset = end; diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index d6695a1b3ee96..85281346296d5 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -803,7 +803,7 @@ impl WorkQueue::ver { )?; let inner = WorkQueueInner::ver { - dev: dev.into(), + dev, event_manager, info: Some(KBox::new(info, GFP_KERNEL)?), gpu_context, diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 11450d904fc8a..da0f2ab1403d9 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -1698,7 +1698,11 @@ static int dockchannel_hid_probe(struct platform_device *pdev) if (!dchid->ring) return -ENOMEM; } - dchid->new_iface_wq = alloc_ordered_workqueue("dchid-new", 0); + /* + * Freeze interface creation across system sleep so the HID devices + * stay stable while the suspend and resume callbacks walk them. + */ + dchid->new_iface_wq = alloc_ordered_workqueue("dchid-new", WQ_FREEZABLE); if (!dchid->new_iface_wq) return -ENOMEM; @@ -1755,17 +1759,50 @@ static void dockchannel_hid_remove(struct platform_device *pdev) static int dockchannel_hid_suspend(struct device *dev) { struct dockchannel_hid *dchid = dev_get_drvdata(dev); + int i, ret; + + /* HID drivers may still need the transport while preparing for sleep. */ + for (i = 0; i < MAX_INTERFACES; i++) { + struct dchid_iface *iface = dchid->ifaces[i]; + if (!iface || !iface->hid) + continue; + ret = hid_driver_suspend(iface->hid, PMSG_SUSPEND); + if (ret) + goto unwind; + } WRITE_ONCE(dchid->suspended, true); return 0; + +unwind: + /* PM does not call our resume when this callback fails. */ + while (--i >= 0) { + struct dchid_iface *iface = dchid->ifaces[i]; + + if (iface && iface->hid) + hid_driver_resume(iface->hid); + } + return ret; } static int dockchannel_hid_resume(struct device *dev) { struct dockchannel_hid *dchid = dev_get_drvdata(dev); + int i, ret = 0; + /* Allow feature requests again before notifying any HID driver. */ WRITE_ONCE(dchid->suspended, false); - return 0; + for (i = 0; i < MAX_INTERFACES; i++) { + struct dchid_iface *iface = dchid->ifaces[i]; + int err; + + if (!iface || !iface->hid) + continue; + err = hid_driver_resume(iface->hid); + if (!ret) + ret = err; + } + return ret; } static DEFINE_SIMPLE_DEV_PM_OPS(dockchannel_hid_pm_ops, dockchannel_hid_suspend, diff --git a/drivers/hid/hid-apple-mtp-haptic.c b/drivers/hid/hid-apple-mtp-haptic.c index 42ab019f7a27a..b8e3ee2f605f2 100644 --- a/drivers/hid/hid-apple-mtp-haptic.c +++ b/drivers/hid/hid-apple-mtp-haptic.c @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#include #include +#include +#include #include #include "hid-ids.h" @@ -56,6 +59,92 @@ int apple_taptic_switch_modes(struct hid_device *haptic_hdev, bool enable_host_c } EXPORT_SYMBOL_GPL(apple_taptic_switch_modes); +/* + * While the actuator is device-controlled, its firmware plays a click + * waveform of its own on every press and release: feature report 0x22 on + * press and 0x23 on release. Both begin with the report ID, a version byte + * and the strength the waveform is played at. The MTP firmware boots with a + * weak strength there and nothing else sets one, so the click is much + * fainter than the same pad gives under macOS. A machine whose device tree + * gives the actuator a click strength has it written over the firmware's, + * leaving the shape of each waveform as the firmware has it. + * + * The strength is firmware state, so it is checked again on every resume + * the transport passes on, and written if the firmware no longer has it. + */ +#define APPLE_TAPTIC_PRESS_WAVEFORM 0x22 +#define APPLE_TAPTIC_RELEASE_WAVEFORM 0x23 +#define APPLE_TAPTIC_WAVEFORM_STRENGTH 2 +#define APPLE_TAPTIC_WAVEFORM_MAX 64 + +static int apple_taptic_set_strength(struct hid_device *haptic_hdev, u8 report, + u8 strength) +{ + u8 *buf __free(kfree) = kzalloc(APPLE_TAPTIC_WAVEFORM_MAX, GFP_KERNEL); + int len; + + if (!buf) + return -ENOMEM; + + len = hid_hw_raw_request(haptic_hdev, report, buf, + APPLE_TAPTIC_WAVEFORM_MAX, HID_FEATURE_REPORT, + HID_REQ_GET_REPORT); + if (len < 0) + return len; + if (len <= APPLE_TAPTIC_WAVEFORM_STRENGTH || buf[0] != report) + return -EPROTO; + if (buf[APPLE_TAPTIC_WAVEFORM_STRENGTH] == strength) + return 0; + + hid_dbg(haptic_hdev, "click waveform 0x%02x strength 0x%02x -> 0x%02x\n", + report, buf[APPLE_TAPTIC_WAVEFORM_STRENGTH], strength); + buf[APPLE_TAPTIC_WAVEFORM_STRENGTH] = strength; + len = hid_hw_raw_request(haptic_hdev, report, buf, len, + HID_FEATURE_REPORT, HID_REQ_SET_REPORT); + + return len < 0 ? len : 0; +} + +static void apple_taptic_set_click_strength(struct hid_device *haptic_hdev) +{ + static const u8 waveforms[] = { + APPLE_TAPTIC_PRESS_WAVEFORM, + APPLE_TAPTIC_RELEASE_WAVEFORM, + }; + struct device_node *np __free(device_node) = NULL; + u32 strength[ARRAY_SIZE(waveforms)]; + unsigned int i; + int ret; + + if (haptic_hdev->dev.parent) + np = of_get_child_by_name(haptic_hdev->dev.parent->of_node, + "actuator"); + if (!np) + return; + + ret = of_property_read_u32_array(np, "apple,click-strength", strength, + ARRAY_SIZE(strength)); + if (ret == -EINVAL) + return; + + for (i = 0; !ret && i < ARRAY_SIZE(strength); i++) + if (!strength[i] || strength[i] > U8_MAX) + ret = -ERANGE; + if (ret) { + hid_warn(haptic_hdev, "invalid apple,click-strength (%d)\n", ret); + return; + } + + for (i = 0; i < ARRAY_SIZE(waveforms); i++) { + ret = apple_taptic_set_strength(haptic_hdev, waveforms[i], + strength[i]); + if (ret) + hid_warn(haptic_hdev, + "cannot set the strength of click waveform 0x%02x (%d)\n", + waveforms[i], ret); + } +} + static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid_device_id *id) { int ret; @@ -87,6 +176,14 @@ static int apple_actuator_probe(struct hid_device *haptic_hdev, const struct hid "cannot return the actuator to device-controlled mode (%d)\n", ret); + apple_taptic_set_click_strength(haptic_hdev); + + return 0; +} + +static int apple_actuator_resume(struct hid_device *haptic_hdev) +{ + apple_taptic_set_click_strength(haptic_hdev); return 0; } @@ -115,7 +212,9 @@ static struct hid_driver apple_actuator_driver = { .id_table = apple_haptic_devices, .probe = apple_actuator_probe, .match = actuator_match, - .remove = actuator_remove + .remove = actuator_remove, + .resume = pm_ptr(apple_actuator_resume), + .reset_resume = pm_ptr(apple_actuator_resume), }; module_hid_driver(apple_actuator_driver); diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 2d23290497b51..c2d96ac9dd4bd 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -2481,6 +2481,15 @@ static void magicmouse_remove(struct hid_device *hdev) } #ifdef CONFIG_PM +static int magicmouse_resume(struct hid_device *hdev) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + if (msc) + magicmouse_haptic_forget_mode(msc); + return 0; +} + static int magicmouse_reset_resume(struct hid_device *hdev) { struct magicmouse_sc *msc = hid_get_drvdata(hdev); @@ -2567,6 +2576,7 @@ static struct hid_driver magicmouse_driver = { .input_mapping = magicmouse_input_mapping, .input_configured = magicmouse_input_configured, #ifdef CONFIG_PM + .resume = magicmouse_resume, .reset_resume = magicmouse_reset_resume, #endif }; diff --git a/drivers/hwmon/macsmc-hwmon-test.c b/drivers/hwmon/macsmc-hwmon-test.c index 312e752565e04..e549c6d61441d 100644 --- a/drivers/hwmon/macsmc-hwmon-test.c +++ b/drivers/hwmon/macsmc-hwmon-test.c @@ -2,7 +2,7 @@ /* * KUnit tests for the Apple SMC hwmon driver: channel numbering of the * temperature sensors, which thermal zones depend on, and manual fan control - * by userspace and by the thermal framework. + * by userspace. * * Included by macsmc-hwmon.c. */ @@ -106,8 +106,12 @@ struct macsmc_fan_test { u8 mode[2]; u32 target[2]; u8 ftst; + int writes; int ftst_writes; int refused; + int mode_reads; + int manual_busy[2]; + int target_writes; smc_key fail_key; }; @@ -124,6 +128,7 @@ static int fan_test_read(struct apple_smc *smc, smc_key key, void *buf, size_t s switch (key & 0xff00ffff) { case __SMC_KEY('F', 0, 'M', 'd'): + t->mode_reads++; value = t->mode[i]; break; case __SMC_KEY('F', 0, 'M', 'n'): @@ -149,6 +154,7 @@ static int fan_test_write(struct apple_smc *smc, smc_key key, const void *buf, s u32 value = 0; int i = fan_test_index(key); + t->writes++; memcpy(&value, buf, min(size, sizeof(value))); if (key == t->fail_key) return -EIO; @@ -171,9 +177,15 @@ static int fan_test_write(struct apple_smc *smc, smc_key key, const void *buf, s t->refused++; return -EIO; } + if (value == MACSMC_FAN_MODE_MANUAL && t->manual_busy[i]) { + t->manual_busy[i]--; + return -EIO; + } t->mode[i] = value; return size; case __SMC_KEY('F', 0, 'T', 'g'): + KUNIT_EXPECT_EQ(t->test, t->mode[i], MACSMC_FAN_MODE_MANUAL); + t->target_writes++; t->target[i] = value; return size; } @@ -218,9 +230,6 @@ static int macsmc_fan_test_init(struct kunit *test) t->hwmon.fan.count = 2; t->hwmon.fan.unlock_key = SMC_KEY(Ftst); mutex_init(&t->hwmon.fan.lock); - INIT_WORK(&t->hwmon.fan.cool_work, macsmc_hwmon_fans_cool_work); - for (i = 0; i < 2; i++) - t->fans[i].hwmon = &t->hwmon; test->priv = t; fan_control = true; kunit_activate_static_stub(test, apple_smc_read, fan_test_read); @@ -261,6 +270,7 @@ static void macsmc_fan_test_unlock_and_release(struct kunit *test) KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 0), 0); fan_test_released(test, t); KUNIT_EXPECT_EQ(test, t->ftst_writes, 2); + KUNIT_EXPECT_EQ(test, t->refused, 0); } /* Targets are clamped to [min, max]; negative ones are refused. */ @@ -303,14 +313,14 @@ static void macsmc_fan_test_two_fans(struct kunit *test) fan_test_released(test, t); } -/* Locked firmware without the unlock key: refused, nothing else written. */ +/* System mode without an unlock key: fail without attempting a mode write. */ static void macsmc_fan_test_no_unlock_key(struct kunit *test) { struct macsmc_fan_test *t = test->priv; t->hwmon.fan.unlock_key = 0; - KUNIT_EXPECT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), -EIO); - KUNIT_EXPECT_EQ(test, t->ftst_writes, 0); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, t->writes, 0); KUNIT_EXPECT_EQ(test, t->mode[0], MACSMC_FAN_MODE_SYSTEM); KUNIT_EXPECT_EQ(test, t->hwmon.fan.manual, 0); } @@ -330,111 +340,133 @@ static void macsmc_fan_test_legacy(struct kunit *test) KUNIT_EXPECT_EQ(test, t->refused, 0); } -/* Cooling states spread over [min, max]; 0 asks for nothing. */ -static void macsmc_fan_test_state_rpm(struct kunit *test) -{ - KUNIT_EXPECT_EQ(test, macsmc_hwmon_fan_state_rpm(1350, 5777, 0), 0); - KUNIT_EXPECT_EQ(test, macsmc_hwmon_fan_state_rpm(1350, 5777, 1), 1350); - KUNIT_EXPECT_EQ(test, macsmc_hwmon_fan_state_rpm(1350, 5777, 4), 3247); - KUNIT_EXPECT_EQ(test, macsmc_hwmon_fan_state_rpm(1350, 5777, - MACSMC_FAN_COOLING_STATES), 5777); - /* Never past the maximum, whatever the state. */ - KUNIT_EXPECT_EQ(test, macsmc_hwmon_fan_state_rpm(1350, 5777, - MACSMC_FAN_COOLING_STATES + 5), 5777); -} - -/* Cooling needs no fan_control: unlock for a state, release at state 0. */ -static void macsmc_fan_test_cooling(struct kunit *test) +/* A legacy write-only mode key keeps working without a new read requirement. */ +static void macsmc_fan_test_legacy_write_only(struct kunit *test) { struct macsmc_fan_test *t = test->priv; - fan_control = false; - t->fans[0].cool_state = 1; - macsmc_hwmon_fans_apply(&t->hwmon); - KUNIT_EXPECT_EQ(test, t->ftst, 1); + t->legacy = true; + t->mode[0] = MACSMC_FAN_MODE_AUTO; + t->fans[0].mode.info.flags &= ~APPLE_SMC_READABLE; + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), 0); KUNIT_EXPECT_EQ(test, t->mode[0], MACSMC_FAN_MODE_MANUAL); - KUNIT_EXPECT_EQ(test, t->target[0], FAN_TEST_F1350); - /* The other fan asks for nothing and stays with the firmware. */ - KUNIT_EXPECT_EQ(test, t->mode[1], MACSMC_FAN_MODE_AUTO); - KUNIT_EXPECT_FALSE(test, t->fans[1].manual); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 0), 0); + KUNIT_EXPECT_EQ(test, t->mode[0], MACSMC_FAN_MODE_AUTO); + KUNIT_EXPECT_EQ(test, t->mode_reads, 0); + KUNIT_EXPECT_EQ(test, t->ftst_writes, 0); +} - t->fans[0].cool_state = MACSMC_FAN_COOLING_STATES; - macsmc_hwmon_fans_apply(&t->hwmon); - KUNIT_EXPECT_EQ(test, t->target[0], FAN_TEST_F5777); - KUNIT_EXPECT_EQ(test, t->ftst_writes, 1); +/* Suspend, resume and shutdown do not write untouched fan keys. */ +static void macsmc_fan_test_untouched(struct kunit *test) +{ + struct macsmc_fan_test *t = test->priv; + struct platform_device pdev = {}; - t->fans[0].cool_state = 0; - macsmc_hwmon_fans_apply(&t->hwmon); + platform_set_drvdata(&pdev, &t->hwmon); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_suspend(&pdev.dev), 0); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_resume(&pdev.dev), 0); + macsmc_hwmon_shutdown(&pdev); + KUNIT_EXPECT_EQ(test, t->writes, 0); fan_test_released(test, t); } -/* A fan runs at the higher of the userspace target and the cooling state. */ -static void macsmc_fan_test_cooling_and_user(struct kunit *test) +/* Suspend releases a manual target; resume never reapplies it. */ +static void macsmc_fan_test_suspend_manual(struct kunit *test) { struct macsmc_fan_test *t = test->priv; + struct device dev = {}; + int writes; + dev_set_drvdata(&dev, &t->hwmon); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), 0); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_suspend(&dev), 0); + fan_test_released(test, t); + writes = t->writes; + KUNIT_EXPECT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), -EBUSY); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_resume(&dev), 0); + KUNIT_EXPECT_EQ(test, t->writes, writes); + fan_test_released(test, t); + /* A fresh userspace request is still accepted after resume. */ KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), 0); - t->fans[0].cool_state = 1; - macsmc_hwmon_fans_apply(&t->hwmon); - KUNIT_EXPECT_EQ(test, t->target[0], FAN_TEST_F2500); - - t->fans[0].cool_state = MACSMC_FAN_COOLING_STATES; - macsmc_hwmon_fans_apply(&t->hwmon); - KUNIT_EXPECT_EQ(test, t->target[0], FAN_TEST_F5777); - /* Userspace cannot take the fan below what cooling asks for. */ - KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 1350), 0); - KUNIT_EXPECT_EQ(test, t->target[0], FAN_TEST_F5777); KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 0), 0); - KUNIT_EXPECT_EQ(test, t->mode[0], MACSMC_FAN_MODE_MANUAL); + KUNIT_EXPECT_EQ(test, t->refused, 0); +} - t->fans[0].cool_state = 0; - macsmc_hwmon_fans_apply(&t->hwmon); +/* A failed unlock leaves the fan with the firmware. */ +static void macsmc_fan_test_unlock_fails(struct kunit *test) +{ + struct macsmc_fan_test *t = test->priv; + + t->fail_key = SMC_KEY(Ftst); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), -EIO); fan_test_released(test, t); + KUNIT_EXPECT_EQ(test, t->refused, 0); } -/* Once stopped (suspend, shutdown, removal), no state takes a fan back. */ -static void macsmc_fan_test_cooling_stopped(struct kunit *test) +/* A stopped fan may leave mode 3 before the manual-mode write is ready. */ +static void macsmc_fan_test_stopped_manual_busy(struct kunit *test) { struct macsmc_fan_test *t = test->priv; - t->fans[0].cool_state = 4; - t->fans[1].cool_state = 4; - macsmc_hwmon_fans_apply(&t->hwmon); - KUNIT_EXPECT_EQ(test, t->hwmon.fan.manual, 2); - - macsmc_hwmon_fans_stop(&t->hwmon); + KUNIT_ASSERT_EQ(test, t->target[0], 0); + t->manual_busy[0] = 2; + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), 0); + KUNIT_EXPECT_EQ(test, t->manual_busy[0], 0); + KUNIT_EXPECT_EQ(test, t->target[0], FAN_TEST_F2500); + KUNIT_EXPECT_EQ(test, t->target_writes, 1); + KUNIT_EXPECT_EQ(test, t->hwmon.fan.manual, 1); + /* Unlock, three mode attempts, then the target. No locked-mode write. */ + KUNIT_EXPECT_EQ(test, t->writes, 5); + KUNIT_EXPECT_EQ(test, t->refused, 0); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 0), 0); fan_test_released(test, t); + KUNIT_EXPECT_EQ(test, t->ftst_writes, 2); +} - macsmc_hwmon_fans_apply(&t->hwmon); +/* Exhausting the post-unlock retries must release Ftst, without a target. */ +static void macsmc_fan_test_manual_busy_timeout(struct kunit *test) +{ + struct macsmc_fan_test *t = test->priv; + + t->fail_key = SMC_KEY(F0Md); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), -EIO); fan_test_released(test, t); + KUNIT_EXPECT_GT(test, t->writes, 3); + KUNIT_EXPECT_EQ(test, t->target_writes, 0); KUNIT_EXPECT_EQ(test, t->ftst_writes, 2); + KUNIT_EXPECT_EQ(test, t->refused, 0); } -/* A failed unlock leaves the fan with the firmware. */ -static void macsmc_fan_test_cooling_unlock_fails(struct kunit *test) +/* Taking over an already spinning firmware fan needs no unlock or retry. */ +static void macsmc_fan_test_spinning(struct kunit *test) { struct macsmc_fan_test *t = test->priv; - t->fail_key = SMC_KEY(Ftst); - t->fans[0].cool_state = MACSMC_FAN_COOLING_STATES; - macsmc_hwmon_fans_apply(&t->hwmon); - KUNIT_EXPECT_EQ(test, t->mode[0], MACSMC_FAN_MODE_SYSTEM); + t->mode[0] = MACSMC_FAN_MODE_AUTO; + t->target[0] = FAN_TEST_F1350; + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 2500), 0); + KUNIT_EXPECT_EQ(test, t->target[0], FAN_TEST_F2500); + KUNIT_EXPECT_EQ(test, t->writes, 2); + KUNIT_EXPECT_EQ(test, t->ftst_writes, 0); + KUNIT_ASSERT_EQ(test, macsmc_hwmon_set_fan(&t->hwmon, 0, 0), 0); + KUNIT_EXPECT_EQ(test, t->mode[0], MACSMC_FAN_MODE_AUTO); KUNIT_EXPECT_EQ(test, t->hwmon.fan.manual, 0); - KUNIT_EXPECT_FALSE(test, t->hwmon.fan.unlocked); } static struct kunit_case macsmc_fan_test_cases[] = { + KUNIT_CASE(macsmc_fan_test_stopped_manual_busy), + KUNIT_CASE(macsmc_fan_test_manual_busy_timeout), + KUNIT_CASE(macsmc_fan_test_spinning), KUNIT_CASE(macsmc_fan_test_unlock_and_release), KUNIT_CASE(macsmc_fan_test_clamp), KUNIT_CASE(macsmc_fan_test_target_write_fails), KUNIT_CASE(macsmc_fan_test_two_fans), KUNIT_CASE(macsmc_fan_test_no_unlock_key), KUNIT_CASE(macsmc_fan_test_legacy), - KUNIT_CASE(macsmc_fan_test_state_rpm), - KUNIT_CASE(macsmc_fan_test_cooling), - KUNIT_CASE(macsmc_fan_test_cooling_and_user), - KUNIT_CASE(macsmc_fan_test_cooling_stopped), - KUNIT_CASE(macsmc_fan_test_cooling_unlock_fails), + KUNIT_CASE(macsmc_fan_test_legacy_write_only), + KUNIT_CASE(macsmc_fan_test_untouched), + KUNIT_CASE(macsmc_fan_test_suspend_manual), + KUNIT_CASE(macsmc_fan_test_unlock_fails), {} }; diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index c0ca04e88d3d2..cb824b252ddcd 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -36,7 +36,6 @@ #include #include #include -#include #define MAX_LABEL_LENGTH 32 @@ -49,68 +48,15 @@ #define FLT_MANT_MASK GENMASK(22, 0) #define FLT_SIGN_MASK BIT(31) +/* + * Fans whose device tree node has apple,fan-manual-control take a target speed + * from userspace. This parameter, kept for compatibility, allows it for every + * fan with a mode key, whatever the device tree says. + */ static bool fan_control; module_param_unsafe(fan_control, bool, 0644); MODULE_PARM_DESC(fan_control, - "Override the SMC to set your own fan speeds on supported machines"); - -/* - * SoC die temperature keys: temperature channels labelled "SoC die " - * whose hottest reading is the SoC die thermal zone (see - * macsmc_hwmon_die_get_temp()). They come from this parameter or, without it, - * from the machine's entry in macsmc_hwmon_boards[]. Keys the SMC does not - * have are skipped. - */ -static char *soc_die_keys; -module_param(soc_die_keys, charp, 0444); -MODULE_PARM_DESC(soc_die_keys, - "Comma-separated SMC temperature keys whose hottest reading is the SoC die thermal zone, \"none\" for no zone (default: the machine's list)"); - -/* - * Keys a machine's device tree does not list yet. The device tree comes from - * the boot loader, so these let a kernel use keys before its device tree - * catches up. board_keys=0 turns them off: the device tree alone then decides - * (soc_die_keys still applies). - */ -static bool board_keys = true; -module_param(board_keys, bool, 0444); -MODULE_PARM_DESC(board_keys, - "Use the built-in per-machine SoC die and thermal pressure keys the device tree does not list (default: on)"); - -static const struct macsmc_hwmon_board { - const char *compatible; - const char *die_keys; - const char *pressure_key; -} macsmc_hwmon_boards[] = { - /* - * MacBook Pro (16-inch, M3 Pro): key names used for the CPU - * performance-core and GPU die sensors of M3-family machines. They - * are not confirmed on this machine: the debugfs key list shows which - * temperature keys it has. - */ - { - .compatible = "apple,j516s", - .die_keys = "Tf04,Tf09,Tf0A,Tf0B,Tf0D,Tf0E,Tf14,Tf18,Tf19,Tf1A,Tf24,Tf25,Tf2A,Tf2B", - .pressure_key = "mTPL", - }, - /* - * MacBook Pro (14-inch and 16-inch, M3 Max): the same key family in - * five groups. On a J516C every one of these keys read a die - * temperature (47-62 C at idle); the keys ending in D and E repeated - * the one ending in 4 and are left out. The J514C list is assumed to - * be the same; keys the SMC does not have are skipped. - */ - { - .compatible = "apple,j516c", - .die_keys = "Tf04,Tf06,Tf09,Tf0A,Tf0B,Tf14,Tf16,Tf19,Tf1A,Tf1B,Tf24,Tf26,Tf29,Tf2A,Tf2B," - "Tf34,Tf36,Tf39,Tf3A,Tf3B,Tf44,Tf46,Tf49,Tf4A,Tf4B", - }, - { - .compatible = "apple,j514c", - .die_keys = "Tf04,Tf06,Tf09,Tf0A,Tf0B,Tf14,Tf16,Tf19,Tf1A,Tf1B,Tf24,Tf26,Tf29,Tf2A,Tf2B," - "Tf34,Tf36,Tf39,Tf3A,Tf3B,Tf44,Tf46,Tf49,Tf4A,Tf4B", - }, -}; + "Allow manual fan speeds on every fan with a mode key, also where the device tree does not (default: the device tree decides)"); static bool macsmc_hwmon_is_m3(void) { @@ -121,52 +67,6 @@ static bool macsmc_hwmon_is_m3(void) of_machine_is_compatible("apple,t8122"); } -static const struct macsmc_hwmon_board *macsmc_hwmon_board(void) -{ - unsigned int i; - - if (!board_keys) - return NULL; - for (i = 0; i < ARRAY_SIZE(macsmc_hwmon_boards); i++) - if (of_machine_is_compatible(macsmc_hwmon_boards[i].compatible)) - return &macsmc_hwmon_boards[i]; - return NULL; -} - -/* The list of SoC die temperature keys, or NULL for none. */ -static const char *macsmc_hwmon_die_key_list(void) -{ - const struct macsmc_hwmon_board *board; - - if (!macsmc_hwmon_is_m3()) - return NULL; - if (soc_die_keys) - return strcmp(soc_die_keys, "none") ? soc_die_keys : NULL; - board = macsmc_hwmon_board(); - return board ? board->die_keys : NULL; -} - -/* - * The next key of a comma-separated list: returns 1 and the key, 0 at the end - * of the list, or -EINVAL for an entry that is not four characters long. - */ -static int macsmc_hwmon_next_key(const char **list, char key[5]) -{ - const char *p = *list, *end; - - while (*p == ',' || *p == ' ') - p++; - if (!*p) - return 0; - end = strchrnul(p, ','); - *list = end; - if (end - p != 4) - return -EINVAL; - memcpy(key, p, 4); - key[4] = '\0'; - return 1; -} - struct macsmc_hwmon_sensor { struct apple_smc_key_info info; smc_key macsmc_key; @@ -185,13 +85,8 @@ struct macsmc_hwmon_fan { char label[MAX_LABEL_LENGTH]; u32 attrs; bool manual; - /* Index of the fan node among the fan-* nodes: its cooling device id. */ - u32 index; - struct macsmc_hwmon *hwmon; - /* Speed asked for through the fan_control target, 0 for none. */ - long user_rpm; - /* Cooling state asked for by the thermal framework, 0 for none. */ - unsigned long cool_state; + /* The device tree allows a manual speed from userspace. */ + bool manual_control; }; struct macsmc_hwmon_sensors { @@ -206,13 +101,6 @@ struct macsmc_hwmon_sensors { /* The firmware's own fan policy; it refuses a manual mode request. */ #define MACSMC_FAN_MODE_SYSTEM 3 -/* - * Cooling states of a fan with #cooling-cells: 0 leaves the fan to the - * firmware, 1 to MACSMC_FAN_COOLING_STATES run it at speeds spread evenly - * from its minimum to its maximum. - */ -#define MACSMC_FAN_COOLING_STATES 8 - /* How long the firmware gets to leave or re-enter its system mode */ #define MACSMC_FAN_MODE_TIMEOUT_MS 10000 #define MACSMC_FAN_MODE_POLL_MS 100 @@ -231,8 +119,6 @@ struct macsmc_hwmon_fans { bool unlocked; /* Fans in manual mode */ u32 manual; - /* Applies the cooling states, which can take seconds to unlock. */ - struct work_struct cool_work; /* Suspended, shut down or going away: every fan stays with the SMC. */ bool stopped; }; @@ -594,118 +480,106 @@ static int macsmc_hwmon_fan_manual(struct macsmc_hwmon *hwmon, if (fan->manual) return 0; - ret = macsmc_hwmon_write_key(hwmon->smc, &fan->mode, MACSMC_FAN_MODE_MANUAL); - if (ret < 0 && hwmon->fan.unlock_key && !hwmon->fan.unlocked && - !macsmc_hwmon_fan_read_mode(hwmon, fan, &mode) && - mode == MACSMC_FAN_MODE_SYSTEM) { - ret = apple_smc_write_u8(hwmon->smc, hwmon->fan.unlock_key, 1); - if (ret < 0) { - /* The key may have been set anyway; clear it. */ - apple_smc_write_u8(hwmon->smc, hwmon->fan.unlock_key, 0); - return ret; + /* Older firmware may expose a write-only mode key. */ + if (!(fan->mode.info.flags & APPLE_SMC_READABLE)) + goto set_manual; + + /* Do not send a mode write the firmware is known to refuse. */ + ret = macsmc_hwmon_fan_read_mode(hwmon, fan, &mode); + if (ret) + return ret; + + if (mode == MACSMC_FAN_MODE_SYSTEM) { + if (!hwmon->fan.unlock_key) + return -EOPNOTSUPP; + + if (!hwmon->fan.unlocked) { + ret = apple_smc_write_u8(hwmon->smc, hwmon->fan.unlock_key, 1); + if (ret < 0) { + /* The key may have been set anyway; clear it. */ + apple_smc_write_u8(hwmon->smc, hwmon->fan.unlock_key, 0); + return ret; + } + hwmon->fan.unlocked = true; } - hwmon->fan.unlocked = true; - /* Ask again once the fan has left system mode. */ timeout = jiffies + msecs_to_jiffies(MACSMC_FAN_MODE_TIMEOUT_MS); for (;;) { ret = macsmc_hwmon_fan_read_mode(hwmon, fan, &mode); - if (!ret && mode == MACSMC_FAN_MODE_SYSTEM) - ret = -EBUSY; - else if (!ret) + if (ret) + goto relock; + if (mode != MACSMC_FAN_MODE_SYSTEM) { + /* Leaving system mode need not mean writes are ready. */ ret = macsmc_hwmon_write_key(hwmon->smc, &fan->mode, MACSMC_FAN_MODE_MANUAL); - if (ret >= 0 || time_after(jiffies, timeout)) - break; + if (ret >= 0) + goto manual; + if (ret != -EIO && ret != -EBUSY) + goto relock; + } + if (time_after(jiffies, timeout)) { + if (mode == MACSMC_FAN_MODE_SYSTEM) + ret = -ETIMEDOUT; + goto relock; + } msleep(MACSMC_FAN_MODE_POLL_MS); } - if (ret < 0) { - dev_err(hwmon->dev, "%s refused manual mode after unlock (%d)\n", - fan->label, ret); - if (!hwmon->fan.manual) - macsmc_hwmon_fans_relock(hwmon); - return ret; - } } + +set_manual: + ret = macsmc_hwmon_write_key(hwmon->smc, &fan->mode, MACSMC_FAN_MODE_MANUAL); if (ret < 0) - return ret; + goto relock; +manual: fan->manual = true; hwmon->fan.manual++; return 0; -} -/* The speed of a cooling state of a fan running from @min to @max. */ -static long macsmc_hwmon_fan_state_rpm(long min, long max, unsigned long state) -{ - if (!state) - return 0; - state = min(state, MACSMC_FAN_COOLING_STATES); - return min + DIV_ROUND_CLOSEST((max - min) * (long)(state - 1), - MACSMC_FAN_COOLING_STATES - 1); +relock: + if (hwmon->fan.unlocked && !hwmon->fan.manual) + macsmc_hwmon_fans_relock(hwmon); + return ret; } -/* - * Run a fan at the higher of the userspace target and the speed of its - * cooling state, or hand it to the firmware when neither asks for anything. - * Any failure leaves the fan with the firmware. Called with the fan lock held. - */ -static int macsmc_hwmon_fan_apply(struct macsmc_hwmon *hwmon, - struct macsmc_hwmon_fan *fan) +/* Whether userspace may set this fan's speed. */ +static bool macsmc_hwmon_fan_user_control(const struct macsmc_hwmon_fan *fan) { - unsigned long state = READ_ONCE(fan->cool_state); - long min, max, rpm; - int ret; - - if (hwmon->fan.stopped || (!fan->user_rpm && !state)) - return macsmc_hwmon_fan_auto(hwmon, fan); - - /* - * The SMC does no sanity checks on requested fan speeds, so we need to. - */ - ret = macsmc_hwmon_read_key(hwmon->smc, &fan->min, 1, &min); - if (!ret) - ret = macsmc_hwmon_read_key(hwmon->smc, &fan->max, 1, &max); - if (!ret && (min <= 0 || max < min)) - ret = -EIO; - if (ret) { - macsmc_hwmon_fan_auto(hwmon, fan); - return ret; - } - - rpm = max(fan->user_rpm, macsmc_hwmon_fan_state_rpm(min, max, state)); - rpm = clamp(rpm, min, max); - - ret = macsmc_hwmon_fan_manual(hwmon, fan); - if (ret) - return ret; - - ret = macsmc_hwmon_write_key(hwmon->smc, &fan->set, rpm); - if (ret < 0) { - macsmc_hwmon_fan_auto(hwmon, fan); - return ret; - } - - return 0; + return fan->mode.macsmc_key && (fan->manual_control || fan_control); } static int macsmc_hwmon_set_fan(struct macsmc_hwmon *hwmon, int channel, long val) { struct macsmc_hwmon_fan *fan = &hwmon->fan.fans[channel]; + long min, max; int ret; - if (!fan_control || fan->mode.macsmc_key == 0) + if (!macsmc_hwmon_fan_user_control(fan)) return -EOPNOTSUPP; if (val < 0) return -EINVAL; guard(mutex)(&hwmon->fan.lock); - /* 0 drops the request; the fan goes back to the firmware unless cooling. */ - fan->user_rpm = val; - ret = macsmc_hwmon_fan_apply(hwmon, fan); - if (ret) - fan->user_rpm = 0; + if (!val) + return macsmc_hwmon_fan_auto(hwmon, fan); + if (hwmon->fan.stopped) + return -EBUSY; + + /* The SMC does no sanity checks on requested fan speeds. */ + ret = macsmc_hwmon_read_key(hwmon->smc, &fan->min, 1, &min); + if (!ret) + ret = macsmc_hwmon_read_key(hwmon->smc, &fan->max, 1, &max); + if (!ret && (min <= 0 || max < min)) + ret = -EIO; + if (!ret) + ret = macsmc_hwmon_fan_manual(hwmon, fan); + if (!ret) { + ret = macsmc_hwmon_write_key(hwmon->smc, &fan->set, clamp(val, min, max)); + if (ret >= 0) + return 0; + } + macsmc_hwmon_fan_auto(hwmon, fan); return ret; } @@ -726,129 +600,15 @@ static void macsmc_hwmon_fans_restore(struct macsmc_hwmon *hwmon) macsmc_hwmon_fan_auto(hwmon, &hwmon->fan.fans[i]); } -/* Bring every fan in line with its requests, unless stopped. */ -static void macsmc_hwmon_fans_apply(struct macsmc_hwmon *hwmon) -{ - u32 i; - int ret; - - guard(mutex)(&hwmon->fan.lock); - for (i = 0; i < hwmon->fan.count; i++) { - struct macsmc_hwmon_fan *fan = &hwmon->fan.fans[i]; - - ret = macsmc_hwmon_fan_apply(hwmon, fan); - if (ret) - dev_warn_ratelimited(hwmon->dev, - "%s left to the SMC: cannot run cooling state %lu (%d)\n", - fan->label, READ_ONCE(fan->cool_state), ret); - } -} - -static void macsmc_hwmon_fans_cool_work(struct work_struct *work) +static void macsmc_hwmon_fans_restore_action(void *data) { - macsmc_hwmon_fans_apply(container_of(work, struct macsmc_hwmon, fan.cool_work)); -} + struct macsmc_hwmon *hwmon = data; -/* - * Stop applying cooling states and hand every fan back to the firmware, for - * suspend, shutdown and removal. - */ -static void macsmc_hwmon_fans_stop(struct macsmc_hwmon *hwmon) -{ scoped_guard(mutex, &hwmon->fan.lock) hwmon->fan.stopped = true; - cancel_work_sync(&hwmon->fan.cool_work); macsmc_hwmon_fans_restore(hwmon); } -static void macsmc_hwmon_fans_restore_action(void *data) -{ - macsmc_hwmon_fans_stop(data); -} - -#if IS_ENABLED(CONFIG_THERMAL_OF) -static int macsmc_hwmon_fan_get_max_state(struct thermal_cooling_device *cdev, - unsigned long *state) -{ - *state = MACSMC_FAN_COOLING_STATES; - return 0; -} - -static int macsmc_hwmon_fan_get_cur_state(struct thermal_cooling_device *cdev, - unsigned long *state) -{ - struct macsmc_hwmon_fan *fan = cdev->devdata; - - *state = READ_ONCE(fan->cool_state); - return 0; -} - -/* - * Unlocking the fans takes the firmware seconds, so the state is applied - * from a work item rather than from the thermal zone update. - */ -static int macsmc_hwmon_fan_set_cur_state(struct thermal_cooling_device *cdev, - unsigned long state) -{ - struct macsmc_hwmon_fan *fan = cdev->devdata; - - if (state > MACSMC_FAN_COOLING_STATES) - return -EINVAL; - - WRITE_ONCE(fan->cool_state, state); - queue_work(system_freezable_wq, &fan->hwmon->fan.cool_work); - return 0; -} - -static const struct thermal_cooling_device_ops macsmc_hwmon_fan_cooling_ops = { - .get_max_state = macsmc_hwmon_fan_get_max_state, - .get_cur_state = macsmc_hwmon_fan_get_cur_state, - .set_cur_state = macsmc_hwmon_fan_set_cur_state, -}; - -/* - * With #cooling-cells = <3> on the hwmon node, every fan is a cooling device - * for the thermal zones that map it as <&hwmon fan-index lower upper>, the - * index counting the fan-* nodes in order. Without it the fans stay with the - * firmware as before. - */ -static void macsmc_hwmon_fans_register_cooling(struct macsmc_hwmon *hwmon) -{ - const u32 needed = HWMON_F_MIN | HWMON_F_MAX | HWMON_F_TARGET; - u32 cells, i; - - if (!hwmon->fan.count || - of_property_read_u32(hwmon->dev->of_node, "#cooling-cells", &cells)) - return; - if (cells != 3) { - dev_warn(hwmon->dev, "#cooling-cells must be 3; fans left to the SMC\n"); - return; - } - - for (i = 0; i < hwmon->fan.count; i++) { - struct macsmc_hwmon_fan *fan = &hwmon->fan.fans[i]; - struct thermal_cooling_device *cdev; - - if ((fan->attrs & needed) != needed || !fan->mode.macsmc_key) { - dev_warn(hwmon->dev, "%s has no speed or mode keys; not a cooling device\n", - fan->label); - continue; - } - - cdev = devm_thermal_of_cooling_device_register(hwmon->dev, fan->index, - "macsmc-fan", fan, - &macsmc_hwmon_fan_cooling_ops); - if (IS_ERR(cdev)) - dev_warn(hwmon->dev, "%s: cannot register a cooling device (%pe)\n", - fan->label, cdev); - } -} -#else -static void macsmc_hwmon_fans_register_cooling(struct macsmc_hwmon *hwmon) -{ -} -#endif - static int macsmc_hwmon_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long *val) { @@ -899,7 +659,7 @@ static umode_t macsmc_hwmon_fan_is_visible(const struct macsmc_hwmon_fan *fan, u32 attr) { if (fan->attrs & BIT(attr)) { - if (attr == hwmon_fan_target && fan_control && fan->mode.macsmc_key) + if (attr == hwmon_fan_target && macsmc_hwmon_fan_user_control(fan)) return 0644; return 0444; @@ -1055,15 +815,10 @@ static int macsmc_hwmon_create_pressure(struct macsmc_hwmon *hwmon, ret = of_property_read_string(hwmon_node, "apple,thermal-pressure-key", &key); - if (ret == -EINVAL) { - const struct macsmc_hwmon_board *board = macsmc_hwmon_board(); - - if (!board || !board->pressure_key) - return 0; - key = board->pressure_key; - } else if (ret) { + if (ret == -EINVAL) + return 0; + if (ret) return ret; - } ret = macsmc_hwmon_parse_key(hwmon->dev, hwmon->smc, &pressure, key); if (ret) @@ -1237,6 +992,14 @@ static int macsmc_hwmon_create_fan(struct device *dev, struct apple_smc *smc, return ret; } + if (of_property_read_bool(fan_node, "apple,fan-manual-control")) { + if (fan->mode.macsmc_key && (fan->attrs & HWMON_F_TARGET)) + fan->manual_control = true; + else + dev_warn(dev, "%s: apple,fan-manual-control needs apple,fan-target and apple,fan-mode\n", + fan->label); + } + /* Initialise fan control mode to automatic */ fan->manual = false; @@ -1249,10 +1012,15 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, struct device_node *key_node __maybe_unused; struct macsmc_hwmon_sensor *sensor; u32 n_current = 0, n_fan = 0, n_power = 0, n_temperature = 0, n_voltage = 0; - const char *die_keys = macsmc_hwmon_die_key_list(), *list; - char key[5]; + const char *key; bool indexed; - int ret; + int n_die_keys, i, ret; + + n_die_keys = of_property_count_strings(hwmon_node, "apple,soc-die-keys"); + if (n_die_keys == -EINVAL) + n_die_keys = 0; + else if (n_die_keys < 0) + return n_die_keys; for_each_child_of_node_with_prefix(hwmon_node, key_node, "current-") { n_current++; @@ -1291,8 +1059,6 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, for_each_child_of_node_with_prefix(hwmon_node, key_node, "fan-") { struct macsmc_hwmon_fan *fan = &hwmon->fan.fans[hwmon->fan.count]; - fan->hwmon = hwmon; - fan->index = n_fan++; if (!macsmc_hwmon_create_fan(hwmon->dev, hwmon->smc, key_node, fan)) hwmon->fan.count++; } @@ -1324,8 +1090,7 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, for_each_child_of_node_with_prefix(hwmon_node, key_node, "temperature-") { n_temperature++; } - for (list = die_keys; list && (ret = macsmc_hwmon_next_key(&list, key));) - n_temperature++; + n_temperature += n_die_keys; if (n_temperature) { hwmon->temp.sensors = devm_kcalloc(hwmon->dev, n_temperature, @@ -1351,10 +1116,13 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, macsmc_hwmon_add_temp_channel(&hwmon->temp, ret); } - for (list = die_keys; list && (ret = macsmc_hwmon_next_key(&list, key));) { - if (ret < 0) { - dev_warn(hwmon->dev, "ignoring an entry of the SoC die key list \"%s\"\n", - die_keys); + for (i = 0; i < n_die_keys; i++) { + ret = of_property_read_string_index(hwmon_node, "apple,soc-die-keys", + i, &key); + if (ret) + return ret; + if (strlen(key) != 4) { + dev_warn(hwmon->dev, "Ignoring invalid SoC die key %s\n", key); continue; } hwmon->die_listed++; @@ -1684,7 +1452,6 @@ static int macsmc_hwmon_probe(struct platform_device *pdev) ret = devm_mutex_init(&pdev->dev, &hwmon->fan.lock); if (ret) return ret; - INIT_WORK(&hwmon->fan.cool_work, macsmc_hwmon_fans_cool_work); ret = macsmc_hwmon_populate_sensors(hwmon, hwmon->dev->of_node); if (ret) { @@ -1692,11 +1459,9 @@ static int macsmc_hwmon_probe(struct platform_device *pdev) return ret; } - if (macsmc_hwmon_is_m3()) { - ret = macsmc_hwmon_create_pressure(hwmon, hwmon->dev->of_node); - if (ret) - return ret; - } + ret = macsmc_hwmon_create_pressure(hwmon, hwmon->dev->of_node); + if (ret) + return ret; ret = macsmc_hwmon_discover_temperatures(hwmon); if (ret) @@ -1736,23 +1501,17 @@ static int macsmc_hwmon_probe(struct platform_device *pdev) hwmon->hwmon_dev = devm_hwmon_device_register_with_info(&pdev->dev, "macsmc_hwmon", hwmon, &hwmon->chip_info, - macsmc_hwmon_is_m3() ? macsmc_hwmon_extra_groups : NULL); + macsmc_hwmon_extra_groups); if (IS_ERR(hwmon->hwmon_dev)) return dev_err_probe(hwmon->dev, PTR_ERR(hwmon->hwmon_dev), "Probing SMC hwmon device failed\n"); - if (macsmc_hwmon_is_m3()) { + if (macsmc_hwmon_is_m3()) macsmc_hwmon_debugfs_init(hwmon); - ret = macsmc_hwmon_die_register(hwmon); - if (ret) - return ret; - } - /* - * Released before the restore action above, which stops the cooling - * work and hands the fans back. - */ - macsmc_hwmon_fans_register_cooling(hwmon); + ret = macsmc_hwmon_die_register(hwmon); + if (ret) + return ret; dev_dbg(hwmon->dev, "Registered SMC hwmon device. Sensors:\n"); dev_dbg(hwmon->dev, @@ -1775,27 +1534,22 @@ static void macsmc_hwmon_shutdown(struct platform_device *pdev) struct macsmc_hwmon *hwmon = platform_get_drvdata(pdev); if (hwmon) - macsmc_hwmon_fans_stop(hwmon); + macsmc_hwmon_fans_restore_action(hwmon); } static int macsmc_hwmon_suspend(struct device *dev) { - macsmc_hwmon_fans_stop(dev_get_drvdata(dev)); + macsmc_hwmon_fans_restore_action(dev_get_drvdata(dev)); return 0; } -/* Cooling states asked for before suspend apply again; the zones update them. */ +/* Manual requests end at suspend; resume never writes fan keys. */ static int macsmc_hwmon_resume(struct device *dev) { struct macsmc_hwmon *hwmon = dev_get_drvdata(dev); - scoped_guard(mutex, &hwmon->fan.lock) { - hwmon->fan.stopped = false; - /* The userspace targets were dropped when the fans went back. */ - for (u32 i = 0; i < hwmon->fan.count; i++) - hwmon->fan.fans[i].user_rpm = 0; - } - queue_work(system_freezable_wq, &hwmon->fan.cool_work); + guard(mutex)(&hwmon->fan.lock); + hwmon->fan.stopped = false; return 0; } diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index d7b89508311ff..5001cc2640718 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -1007,6 +1007,17 @@ config I2C_APPLE This driver can also be built as a module. If so, the module will be called i2c-apple. +config I2C_PASEMI_KUNIT_TEST + bool "KUnit tests for PA Semi/Apple I2C runtime PM" if !KUNIT_ALL_TESTS + depends on KUNIT && PM && I2C_APPLE + default KUNIT_ALL_TESTS + help + Test runtime PM reference handling on resume and transfer failures. + Exercise both I2C and SMBus transfer entry points with a fake PM + domain and register storage. Failed resumes must not touch MMIO, + and transfer errors must release their power references. + No controller hardware is needed. These tests are for developers. + config I2C_PCA_PLATFORM tristate "PCA9564/PCA9665 as platform device" select I2C_ALGOPCA diff --git a/drivers/i2c/busses/i2c-pasemi-core-test.c b/drivers/i2c/busses/i2c-pasemi-core-test.c new file mode 100644 index 0000000000000..940d2ce520e8c --- /dev/null +++ b/drivers/i2c/busses/i2c-pasemi-core-test.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by i2c-pasemi-core.c to exercise the real transfer entry points. */ + +#include +#include +#include + +struct pasemi_pm_test { + struct pasemi_smbus smbus; + int resume_error; + int resumes; +}; + +static int pasemi_test_resume(struct device *dev) +{ + struct pasemi_pm_test *ctx = dev_get_drvdata(dev); + + ctx->resumes++; + return ctx->resume_error; +} + +static struct dev_pm_domain pasemi_test_domain = { + .ops.runtime_resume = pasemi_test_resume, +}; + +static void pasemi_test_disable_pm(void *arg) +{ + struct device *dev = arg; + + pm_runtime_disable(dev); + pm_runtime_dont_use_autosuspend(dev); + dev->pm_domain = NULL; +} + +static int pasemi_pm_test_init(struct kunit *test) +{ + struct pasemi_pm_test *ctx; + struct device *dev; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + dev = kunit_device_register(test, "pasemi-pm"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + dev_set_drvdata(dev, ctx); + dev->pm_domain = &pasemi_test_domain; + ctx->smbus.dev = dev; + ctx->smbus.runtime_pm = true; + ctx->smbus.adapter.algo_data = &ctx->smbus; + /* A baseline reference of one prevents autosuspend racing assertions. */ + pm_runtime_set_autosuspend_delay(dev, -1); + pm_runtime_use_autosuspend(dev); + pm_runtime_enable(dev); + test->priv = ctx; + return kunit_add_action_or_reset(test, pasemi_test_disable_pm, dev); +} + +static void pasemi_i2c_resume_failure_test(struct kunit *test) +{ + struct pasemi_pm_test *ctx = test->priv; + struct i2c_msg msg = { .addr = 0x10 }; + + ctx->resume_error = -EIO; + /* ioaddr is NULL: neither path may access MMIO after a failed resume. */ + KUNIT_EXPECT_EQ(test, pasemi_i2c_xfer(&ctx->smbus.adapter, &msg, 1), -EIO); + KUNIT_EXPECT_EQ(test, ctx->resumes, 1); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->smbus.dev->power.usage_count), 1); +} + +static void pasemi_smb_resume_failure_test(struct kunit *test) +{ + struct pasemi_pm_test *ctx = test->priv; + union i2c_smbus_data data; + int ret; + + ctx->resume_error = -EIO; + ret = pasemi_smb_xfer(&ctx->smbus.adapter, 0x10, 0, I2C_SMBUS_READ, + 0, I2C_SMBUS_BYTE_DATA, &data); + KUNIT_EXPECT_EQ(test, ret, -EIO); + KUNIT_EXPECT_EQ(test, ctx->resumes, 1); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->smbus.dev->power.usage_count), 1); +} + +static void pasemi_test_mmio(struct kunit *test, u32 status) +{ + struct pasemi_pm_test *ctx = test->priv; + void *regs = kunit_kzalloc(test, 0x30, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, regs); + ctx->smbus.ioaddr = (void __iomem __force *)regs; + iowrite32(status, ctx->smbus.ioaddr + REG_SMSTA); +} + +static void pasemi_i2c_busy_failure_test(struct kunit *test) +{ + struct pasemi_pm_test *ctx = test->priv; + struct i2c_msg msg = { .addr = 0x10 }; + + pasemi_test_mmio(test, SMSTA_XIP); + KUNIT_EXPECT_EQ(test, pasemi_i2c_xfer(&ctx->smbus.adapter, &msg, 1), -EIO); + KUNIT_EXPECT_EQ(test, ctx->resumes, 1); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->smbus.dev->power.usage_count), 1); +} + +static void pasemi_smb_invalid_failure_test(struct kunit *test) +{ + struct pasemi_pm_test *ctx = test->priv; + union i2c_smbus_data data; + int ret; + + pasemi_test_mmio(test, SMSTA_MTE); + ret = pasemi_smb_xfer(&ctx->smbus.adapter, 0x10, 0, I2C_SMBUS_READ, + 0, -1, &data); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, ctx->resumes, 1); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->smbus.dev->power.usage_count), 1); +} + +static struct kunit_case pasemi_pm_test_cases[] = { + KUNIT_CASE(pasemi_i2c_resume_failure_test), + KUNIT_CASE(pasemi_smb_resume_failure_test), + KUNIT_CASE(pasemi_i2c_busy_failure_test), + KUNIT_CASE(pasemi_smb_invalid_failure_test), + {} +}; + +static struct kunit_suite pasemi_pm_test_suite = { + .name = "i2c-pasemi-runtime-pm", + .init = pasemi_pm_test_init, + .test_cases = pasemi_pm_test_cases, +}; + +kunit_test_suite(pasemi_pm_test_suite); diff --git a/drivers/i2c/busses/i2c-pasemi-core.c b/drivers/i2c/busses/i2c-pasemi-core.c index f4eca44ed1839..90d73a338bd27 100644 --- a/drivers/i2c/busses/i2c-pasemi-core.c +++ b/drivers/i2c/busses/i2c-pasemi-core.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -244,19 +245,29 @@ static int pasemi_i2c_xfer(struct i2c_adapter *adapter, struct pasemi_smbus *smbus = adapter->algo_data; int ret, i; + if (smbus->runtime_pm) { + ret = pm_runtime_resume_and_get(smbus->dev); + if (ret < 0) + return ret; + } + ret = pasemi_smb_clear(smbus); if (ret) - return ret; + goto out; for (i = 0; i < num && !ret; i++) ret = pasemi_i2c_xfer_msg(adapter, &msgs[i], (i == (num - 1))); +out: + if (smbus->runtime_pm) + pm_runtime_put_autosuspend(smbus->dev); return ret ? ret : num; } -static int pasemi_smb_xfer(struct i2c_adapter *adapter, - u16 addr, unsigned short flags, char read_write, u8 command, - int size, union i2c_smbus_data *data) +static int pasemi_smb_xfer_unlocked(struct i2c_adapter *adapter, u16 addr, + unsigned short flags, char read_write, + u8 command, int size, + union i2c_smbus_data *data) { struct pasemi_smbus *smbus = adapter->algo_data; unsigned int rd; @@ -414,16 +425,47 @@ static u32 pasemi_smb_func(struct i2c_adapter *adapter) I2C_FUNC_SMBUS_BLOCK_PROC_CALL | I2C_FUNC_I2C; } +static int pasemi_smb_xfer(struct i2c_adapter *adapter, + u16 addr, unsigned short flags, char read_write, + u8 command, int size, union i2c_smbus_data *data) +{ + struct pasemi_smbus *smbus = adapter->algo_data; + int ret; + + if (smbus->runtime_pm) { + ret = pm_runtime_resume_and_get(smbus->dev); + if (ret < 0) + return ret; + } + + ret = pasemi_smb_xfer_unlocked(adapter, addr, flags, read_write, command, + size, data); + if (smbus->runtime_pm) + pm_runtime_put_autosuspend(smbus->dev); + + return ret; +} + static const struct i2c_algorithm smbus_algorithm = { .xfer = pasemi_i2c_xfer, .smbus_xfer = pasemi_smb_xfer, .functionality = pasemi_smb_func, }; -int pasemi_i2c_common_probe(struct pasemi_smbus *smbus) +void pasemi_i2c_init(struct pasemi_smbus *smbus) { - int error; + init_completion(&smbus->irq_completion); + + if (smbus->hw_rev != PASEMI_HW_REV_PCI) + smbus->hw_rev = reg_read(smbus, REG_REV); + + reg_write(smbus, REG_IMASK, 0); + pasemi_reset(smbus); +} +EXPORT_SYMBOL_GPL(pasemi_i2c_init); +int pasemi_i2c_register_adapter(struct pasemi_smbus *smbus) +{ smbus->adapter.owner = THIS_MODULE; snprintf(smbus->adapter.name, sizeof(smbus->adapter.name), "PA Semi SMBus adapter (%s)", dev_name(smbus->dev)); @@ -432,21 +474,14 @@ int pasemi_i2c_common_probe(struct pasemi_smbus *smbus) /* set up the sysfs linkage to our parent device */ smbus->adapter.dev.parent = smbus->dev; - smbus->use_irq = 0; - init_completion(&smbus->irq_completion); - - if (smbus->hw_rev != PASEMI_HW_REV_PCI) - smbus->hw_rev = reg_read(smbus, REG_REV); - - reg_write(smbus, REG_IMASK, 0); - - pasemi_reset(smbus); - - error = devm_i2c_add_adapter(smbus->dev, &smbus->adapter); - if (error) - return error; + return devm_i2c_add_adapter(smbus->dev, &smbus->adapter); +} +EXPORT_SYMBOL_GPL(pasemi_i2c_register_adapter); - return 0; +int pasemi_i2c_common_probe(struct pasemi_smbus *smbus) +{ + pasemi_i2c_init(smbus); + return pasemi_i2c_register_adapter(smbus); } EXPORT_SYMBOL_GPL(pasemi_i2c_common_probe); @@ -460,6 +495,10 @@ irqreturn_t pasemi_irq_handler(int irq, void *dev_id) } EXPORT_SYMBOL_GPL(pasemi_irq_handler); +#if IS_ENABLED(CONFIG_I2C_PASEMI_KUNIT_TEST) +#include "i2c-pasemi-core-test.c" +#endif + MODULE_LICENSE("GPL"); MODULE_AUTHOR("Olof Johansson "); MODULE_DESCRIPTION("PA Semi PWRficient SMBus driver"); diff --git a/drivers/i2c/busses/i2c-pasemi-core.h b/drivers/i2c/busses/i2c-pasemi-core.h index 88821f4e8a9f8..f32300648319d 100644 --- a/drivers/i2c/busses/i2c-pasemi-core.h +++ b/drivers/i2c/busses/i2c-pasemi-core.h @@ -18,9 +18,12 @@ struct pasemi_smbus { unsigned int clk_div; int hw_rev; int use_irq; + bool runtime_pm; struct completion irq_completion; }; int pasemi_i2c_common_probe(struct pasemi_smbus *smbus); +void pasemi_i2c_init(struct pasemi_smbus *smbus); +int pasemi_i2c_register_adapter(struct pasemi_smbus *smbus); irqreturn_t pasemi_irq_handler(int irq, void *dev_id); diff --git a/drivers/i2c/busses/i2c-pasemi-platform.c b/drivers/i2c/busses/i2c-pasemi-platform.c index a486a37d38634..efeaeb43c16d6 100644 --- a/drivers/i2c/busses/i2c-pasemi-platform.c +++ b/drivers/i2c/busses/i2c-pasemi-platform.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "i2c-pasemi-core.h" @@ -18,8 +19,53 @@ struct pasemi_platform_i2c_data { struct pasemi_smbus smbus; struct clk *clk_ref; + int irq; }; +static int pasemi_platform_i2c_runtime_suspend(struct device *dev) +{ + struct pasemi_platform_i2c_data *data = dev_get_drvdata(dev); + + if (data->smbus.use_irq) + disable_irq(data->irq); + clk_disable_unprepare(data->clk_ref); + + return 0; +} + +static int pasemi_platform_i2c_runtime_resume(struct device *dev) +{ + struct pasemi_platform_i2c_data *data = dev_get_drvdata(dev); + int error; + + error = clk_prepare_enable(data->clk_ref); + if (error) + return error; + + /* Restore the divider, enable bit and FIFOs after domain power loss. */ + pasemi_i2c_init(&data->smbus); + if (data->smbus.use_irq) + enable_irq(data->irq); + + return 0; +} + +static DEFINE_RUNTIME_DEV_PM_OPS(pasemi_platform_i2c_pm_ops, + pasemi_platform_i2c_runtime_suspend, + pasemi_platform_i2c_runtime_resume, NULL); + +static void pasemi_platform_i2c_disable_pm(void *arg) +{ + struct device *dev = arg; + + /* Registered before the adapter, after the IRQ: clients are gone now. */ + pm_runtime_disable(dev); + pm_runtime_dont_use_autosuspend(dev); + if (!pm_runtime_status_suspended(dev)) + pasemi_platform_i2c_runtime_suspend(dev); + pm_runtime_set_suspended(dev); +} + static int pasemi_platform_i2c_calc_clk_div(struct pasemi_platform_i2c_data *data, u32 frequency) @@ -66,7 +112,7 @@ static int pasemi_platform_i2c_probe(struct platform_device *pdev) if (of_property_read_u32(dev->of_node, "clock-frequency", &frequency)) frequency = I2C_MAX_STANDARD_MODE_FREQ; - data->clk_ref = devm_clk_get_enabled(dev, NULL); + data->clk_ref = devm_clk_get(dev, NULL); if (IS_ERR(data->clk_ref)) return PTR_ERR(data->clk_ref); @@ -74,23 +120,54 @@ static int pasemi_platform_i2c_probe(struct platform_device *pdev) if (error) return error; - smbus->adapter.dev.of_node = pdev->dev.of_node; - error = pasemi_i2c_common_probe(smbus); + platform_set_drvdata(pdev, data); + error = clk_prepare_enable(data->clk_ref); if (error) return error; - irq_num = platform_get_irq(pdev, 0); - error = devm_request_irq(smbus->dev, irq_num, pasemi_irq_handler, 0, "pasemi_apple_i2c", (void *)smbus); - - if (!error) + pasemi_i2c_init(smbus); + irq_num = platform_get_irq_optional(pdev, 0); + if (irq_num < 0 && irq_num != -ENXIO) { + error = irq_num; + goto disable_clk; + } + if (irq_num > 0) { + error = devm_request_irq(dev, irq_num, pasemi_irq_handler, 0, + "pasemi_apple_i2c", smbus); + if (error) + goto disable_clk; smbus->use_irq = 1; - platform_set_drvdata(pdev, data); - - return 0; + data->irq = irq_num; + } + + pm_runtime_get_noresume(dev); + error = pm_runtime_set_active(dev); + if (error) { + pm_runtime_put_noidle(dev); + goto disable_clk; + } + pm_runtime_set_autosuspend_delay(dev, 100); + pm_runtime_use_autosuspend(dev); + pm_runtime_enable(dev); + error = devm_add_action_or_reset(dev, pasemi_platform_i2c_disable_pm, dev); + if (error) { + pm_runtime_put_noidle(dev); + return error; + } + + smbus->runtime_pm = true; + smbus->adapter.dev.of_node = dev->of_node; + error = pasemi_i2c_register_adapter(smbus); + pm_runtime_put_autosuspend(dev); + return error; + +disable_clk: + if (smbus->use_irq) + devm_free_irq(dev, data->irq, smbus); + clk_disable_unprepare(data->clk_ref); + return error; } -static void pasemi_platform_i2c_remove(struct platform_device *pdev) { } - static const struct of_device_id pasemi_platform_i2c_of_match[] = { { .compatible = "apple,t8103-i2c" }, { .compatible = "apple,i2c" }, @@ -102,9 +179,9 @@ static struct platform_driver pasemi_platform_i2c_driver = { .driver = { .name = "i2c-apple", .of_match_table = pasemi_platform_i2c_of_match, + .pm = pm_ptr(&pasemi_platform_i2c_pm_ops), }, .probe = pasemi_platform_i2c_probe, - .remove = pasemi_platform_i2c_remove, }; module_platform_driver(pasemi_platform_i2c_driver); diff --git a/drivers/iio/common/aop_sensors/Kconfig b/drivers/iio/common/aop_sensors/Kconfig index b2c4397787dbf..df1277d4082af 100644 --- a/drivers/iio/common/aop_sensors/Kconfig +++ b/drivers/iio/common/aop_sensors/Kconfig @@ -20,3 +20,18 @@ config IIO_AOP_SENSOR_ALS help Module to handle the ambient light sensor attached to the AOP coprocessor on Apple laptops. + +config IIO_AOP_SENSOR_MOTION + tristate "AOP accelerometer and gyroscope" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + depends on SYSFS + select APPLE_AOP + select IIO_BUFFER + select IIO_KFIFO_BUF + help + Module to handle the accelerometer and gyroscope that the AOP + coprocessor of Apple laptops reads. Each sensor is an IIO device + with X, Y and Z channels, readable directly or through a buffer. + The bootloader has to pass the machine's sensor calibration in + the device tree. diff --git a/drivers/iio/common/aop_sensors/Makefile b/drivers/iio/common/aop_sensors/Makefile index 8da5a19efe0f0..4b769a78d44c5 100644 --- a/drivers/iio/common/aop_sensors/Makefile +++ b/drivers/iio/common/aop_sensors/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_IIO_AOP_SENSOR_LAS) += aop_las.o obj-$(CONFIG_IIO_AOP_SENSOR_ALS) += aop_als.o +obj-$(CONFIG_IIO_AOP_SENSOR_MOTION) += aop_motion.o diff --git a/drivers/iio/common/aop_sensors/aop_als.rs b/drivers/iio/common/aop_sensors/aop_als.rs index 3e262a003c9ea..5e6c5001ad2b8 100644 --- a/drivers/iio/common/aop_sensors/aop_als.rs +++ b/drivers/iio/common/aop_sensors/aop_als.rs @@ -12,6 +12,7 @@ use kernel::{ module_platform_driver, of, platform, prelude::*, soc::apple::aop::{EPICService, FakehidRegistration, AOP}, + sync::Arc, }; const EPIC_SUBTYPE_GET_AOP_PROPERTY: u16 = 0xa; @@ -39,7 +40,12 @@ const ALS_VERBOSITY: u32 = 6; /// The reporting interval in microseconds. const ALS_INTERVAL_US: u32 = 200000; const LUX_OFFSET_CT720: usize = 0x1d; +// These offsets address the fake-HID EPIC payload, not a host HID event. const LUX_OFFSET_VD6286: usize = 0x28; +// The documented 122-byte Redbird report carries firmware-computed Kelvin. +// Other report layouts must be decoded separately, even for the same sensor. +const VD6286_COLOR_REPORT_LEN: usize = 122; +const CCT_OFFSET_VD6286: usize = 0x45; /* * The CT817 sends a 32-byte report: two raw channel floats at +0x08 and * +0x0c and the lux float at +0x1c, one byte below the CT720's, so the two @@ -72,15 +78,18 @@ fn enable_als( offset: usize, ) -> Result<()> { if offset != LUX_OFFSET_CT817 { - // The calibration blob comes from macOS and is not installed on every - // system. Without it the sensor still reports, uncalibrated. - match Firmware::request(c_str!("apple/aop-als-cal.bin"), dev.as_ref()) { + // The installer collects the calibration blob from macOS, so it is + // not present on every system. Without it the VD6286 of the M4 Pro + // never reports and its channel reads ENODATA. The warning below + // reports the missing file once, so the firmware loader is asked not + // to report it too. + match Firmware::request_nowarn(c_str!("apple/aop-als-cal.bin"), dev.as_ref()) { Ok(fw) => { set_als_property(aop, svc, ALS_PROP_CALIBRATION, fw.data())?; } Err(e) => dev_warn!( dev.as_ref(), - "no calibration ({:?}); readings are uncalibrated\n", + "no apple/aop-als-cal.bin ({:?}); the sensor may not report\n", e ), } @@ -165,17 +174,62 @@ fn f32_to_micro(f: u32) -> Option { struct MsgProc(usize); +fn report_float(message: &[u8], offset: usize) -> Option { + let raw = message.get(offset..offset + 4)?; + f32_to_micro(u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]])) +} + impl MessageProcessor for MsgProc { fn process(&self, message: &[u8]) -> Option { - let raw = message.get(self.0..self.0 + 4)?; - f32_to_micro(u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]])) + report_float(message, self.0) + } + + fn process_channel(&self, message: &[u8], channel: usize) -> Option { + match channel { + 0 => self.process(message), + 1 if self.0 == LUX_OFFSET_VD6286 && message.len() == VD6286_COLOR_REPORT_LEN => { + // Zero and negative CCT do not describe a colour temperature. + report_float(message, CCT_OFFSET_VD6286).filter(|value| *value != 0) + } + _ => None, + } } } +enum AlsIio { + Light { + _registration: IIORegistration, + }, + Color { + _registration: IIORegistration, + }, +} + struct IIOAopAlsDriver { /// Declared first: the subscription ends before the IIO device goes. listener: FakehidRegistration, - _iio: IIORegistration, + _iio: AlsIio, +} + +fn register_channels( + pdev: &platform::Device, + aop: &Arc, + service: EPICService, + offset: usize, + types: [u32; N], +) -> Result<(FakehidRegistration, IIORegistration)> { + let data = AopSensorData::new_channels(pdev.as_ref().into(), types, MsgProc(offset))?; + let listener = FakehidRegistration::new(aop.clone(), service, data.clone())?; + enable_als(aop.as_ref(), pdev, &service, offset)?; + let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_PROCESSED; + let iio = IIORegistration::new_channels( + data, + c"aop-sensors-als", + types, + info_mask, + kernel::module::this_module::(), + )?; + Ok((listener, iio)) } kernel::of_device_table!( @@ -208,12 +262,29 @@ impl platform::Driver for IIOAopAlsDriver { let service = unsafe { *service }; let ty = bindings::BINDINGS_IIO_LIGHT; let offset = get_lux_offset(adata.as_ref(), pdev, &service)?; - let data = AopSensorData::new(dev.into(), ty, MsgProc(offset))?; - let listener = FakehidRegistration::new(adata.clone(), service, data.clone())?; - enable_als(adata.as_ref(), pdev, &service, offset)?; - let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_PROCESSED; - let iio = - IIORegistration::::new(data, c"aop-sensors-als", ty, info_mask, kernel::module::this_module::())?; + let (listener, iio) = if offset == LUX_OFFSET_VD6286 { + let (listener, registration) = register_channels( + pdev, + &adata, + service, + offset, + [ty, bindings::BINDINGS_IIO_CCT], + )?; + ( + listener, + AlsIio::Color { + _registration: registration, + }, + ) + } else { + let (listener, registration) = register_channels(pdev, &adata, service, offset, [ty])?; + ( + listener, + AlsIio::Light { + _registration: registration, + }, + ) + }; Ok(IIOAopAlsDriver { listener, _iio: iio, diff --git a/drivers/iio/common/aop_sensors/aop_las.rs b/drivers/iio/common/aop_sensors/aop_las.rs index 7e0f5ccc01b19..598c1b607a5d7 100644 --- a/drivers/iio/common/aop_sensors/aop_las.rs +++ b/drivers/iio/common/aop_sensors/aop_las.rs @@ -17,7 +17,21 @@ struct MsgProc; impl MessageProcessor for MsgProc { fn process(&self, message: &[u8]) -> Option { - message.get(1).map(|angle| u64::from(*angle) * MICRO) + // Report 1 contains a little-endian, 9-bit angle in degrees. + // Other report IDs carry sensor properties, not angle samples. + if *message.first()? != 1 { + return None; + } + let angle = u16::from_le_bytes([*message.get(1)?, *message.get(2)?]) & 0x1ff; + if angle > 360 { + return None; + } + Some(u64::from(angle) * MICRO) + } + + fn scale(&self) -> Option<(i32, i32)> { + // IIO_ANGL uses radians: one raw count is pi / 180 radians. + Some((0, 17_453_293)) } } @@ -59,9 +73,15 @@ impl platform::Driver for IIOAopLasDriver { let ty = bindings::BINDINGS_IIO_ANGL; let data = AopSensorData::new(dev.into(), ty, MsgProc)?; let listener = FakehidRegistration::new(adata, service, data.clone())?; - let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_RAW; - let iio = - IIORegistration::::new(data, c"aop-sensors-las", ty, info_mask, kernel::module::this_module::())?; + let info_mask = (1 << bindings::BINDINGS_IIO_CHAN_INFO_RAW) + | (1 << bindings::BINDINGS_IIO_CHAN_INFO_SCALE); + let iio = IIORegistration::::new( + data, + c"aop-sensors-las", + ty, + info_mask, + kernel::module::this_module::(), + )?; Ok(IIOAopLasDriver { listener, _iio: iio, diff --git a/drivers/iio/common/aop_sensors/aop_motion.rs b/drivers/iio/common/aop_sensors/aop_motion.rs new file mode 100644 index 0000000000000..a0f2715ee4e50 --- /dev/null +++ b/drivers/iio/common/aop_sensors/aop_motion.rs @@ -0,0 +1,672 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple AOP accelerometer and gyroscope driver +//! +//! The Always-On Processor reads the machine's inertial sensor and reports +//! acceleration and angular rate as two services, one fake-HID report per +//! sample. A service starts reporting once it has been sent the machine's +//! three calibration packets and a report interval. + +use core::{ + ffi::{c_int, c_void}, + marker::PhantomPinned, + ptr, +}; + +use kernel::{ + bindings, c_str, + device::{self, Core}, + module_platform_driver, new_spinlock, of, platform, + prelude::*, + soc::apple::aop::{EPICService, FakehidListener, FakehidRegistration, AOP}, + sync::{ + aref::ARef, + atomic::{Atomic, Relaxed}, + Arc, SpinLock, + }, + types::{ForeignOwnable, Opaque}, +}; + +/// The EPIC notification that writes a property of a sensor service. +const SUBTYPE_SET_PROPERTY: u16 = 4; +/// The report interval property, in microseconds. Zero stops the reports. +const PROP_INTERVAL: u32 = 0; +/// 50 Hz. +const INTERVAL_US: u32 = 20_000; +/// The calibration properties of a service and the device tree properties the +/// bootloader fills them from. They are written in this order, before the +/// report interval. +const CALIBRATION: [(u32, &CStr); 3] = [ + (75, c_str!("apple,range-intrinsic-cal")), + (76, c_str!("apple,nominal-extrinsic-cal")), + (77, c_str!("apple,range-extrinsic-cal")), +]; +/// A sample report is 22 bytes; X, Y and Z are signed little-endian 32-bit +/// values in 1/65536 of the sensor's unit (g or degrees per second), starting +/// at byte 6. The other bytes carry no sample. +const REPORT_LEN: usize = 22; +const XYZ_OFFSET: usize = 6; +/// The status a calibration write is answered with when the service already +/// holds a calibration from an earlier start in the same AOP boot, as after a +/// rebind or a resume. The firmware keeps that calibration, so the reports can +/// be started without it. +const STATUS_UNSUPPORTED: u32 = 0xe000_02c7; + +/// What the status of a calibration write means for the start of the reports. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CalStatus { + /// The firmware took the packet. + Taken, + /// The firmware holds a calibration from an earlier start. + Held, + /// Anything else: the reports are not started. + Refused, +} + +fn cal_status(status: u32) -> CalStatus { + match status { + 0 => CalStatus::Taken, + STATUS_UNSUPPORTED => CalStatus::Held, + _ => CalStatus::Refused, + } +} + +/// What one of the two services measures. +struct Kind { + name: &'static CStr, + chan_type: u32, + /// The size of one raw count in m/s^2 or rad/s, in billionths. + scale_nano: i32, +} + +/// 9.80665 / 65536 m/s^2 per count. +static ACCEL: Kind = Kind { + name: c"aop-sensors-accel", + chan_type: bindings::BINDINGS_IIO_ACCEL, + scale_nano: 149_638, +}; + +/// pi / (180 * 65536) rad/s per count. +static GYRO: Kind = Kind { + name: c"aop-sensors-gyro", + chan_type: bindings::BINDINGS_IIO_ANGL_VEL, + scale_nano: 266, +}; + +/// Decodes a sample report into its X, Y and Z counts. +fn decode(report: &[u8]) -> Option<[i32; 3]> { + if report.len() != REPORT_LEN { + return None; + } + let axis = |i: usize| { + let at = XYZ_OFFSET + 4 * i; + i32::from_le_bytes([report[at], report[at + 1], report[at + 2], report[at + 3]]) + }; + Some([axis(0), axis(1), axis(2)]) +} + +fn le_u32(bytes: &[u8], at: usize) -> Option { + Some(u32::from_le_bytes(bytes.get(at..at + 4)?.try_into().ok()?)) +} + +/// Returns the packet for location 0 of a calibration property. +/// +/// The property is a container: byte 3 is 1 and byte 2 counts the entries +/// that follow the 4-byte header. Each entry is a little-endian 32-bit +/// location and length, then that many bytes of packet. +fn location0_packet(prop: &[u8]) -> Option<&[u8]> { + if prop.len() < 4 || prop[3] != 1 { + return None; + } + let mut at = 4; + for _ in 0..prop[2] { + let location = le_u32(prop, at)?; + let len = usize::try_from(le_u32(prop, at + 4)?).ok()?; + let start = at + 8; + let packet = prop.get(start..start.checked_add(len)?)?; + if location == 0 { + return Some(packet); + } + at = start + len; + } + None +} + +/// Starts and stops the reports of one service. +struct Control { + dev: ARef, + aop: Arc, + service: EPICService, + packets: [KVec; 3], +} + +impl Control { + fn new(dev: &device::Device, aop: Arc, service: EPICService) -> Result> { + let node = dev.fwnode().ok_or(ENODEV)?; + let mut packets = [KVec::new(), KVec::new(), KVec::new()]; + for ((_, name), packet) in CALIBRATION.iter().zip(packets.iter_mut()) { + if !node.property_present(name) { + dev_info!(dev, "no {:?} from the bootloader, sensor left off\n", name); + return Err(ENODEV); + } + let len = node.property_count_elem::(name)?; + let prop = node + .property_read_array_vec::(name, len)? + .required_by(dev)?; + let Some(data) = location0_packet(&prop) else { + dev_err!(dev, "{:?} holds no calibration for location 0\n", name); + return Err(EINVAL); + }; + packet.extend_from_slice(data, GFP_KERNEL)?; + } + Ok(Arc::new( + Self { + dev: dev.into(), + aop, + service, + packets, + }, + GFP_KERNEL, + )?) + } + + /// Writes a property and returns the firmware's status. + fn write(&self, prop: u32, value: &[u8]) -> Result { + let mut msg = KVec::with_capacity(8 + value.len(), GFP_KERNEL)?; + msg.extend_from_slice(&0u32.to_le_bytes(), GFP_KERNEL)?; + msg.extend_from_slice(&prop.to_le_bytes(), GFP_KERNEL)?; + msg.extend_from_slice(value, GFP_KERNEL)?; + self.aop + .epic_call(&self.service, SUBTYPE_SET_PROPERTY, &msg) + } + + fn refused(&self, prop: u32, status: u32) -> Error { + dev_err!( + self.dev, + "property {} refused: status {:#x}\n", + prop, + status + ); + EIO + } + + fn set(&self, prop: u32, value: &[u8]) -> Result { + match self.write(prop, value)? { + 0 => Ok(()), + status => Err(self.refused(prop, status)), + } + } + + /// Sends the calibration, then the interval that starts the reports. + fn start(&self) -> Result { + let mut held = 0; + for ((prop, _), packet) in CALIBRATION.iter().zip(self.packets.iter()) { + let status = self.write(*prop, packet)?; + match cal_status(status) { + CalStatus::Taken => {} + CalStatus::Held => held += 1, + CalStatus::Refused => return Err(self.refused(*prop, status)), + } + } + if held > 0 { + dev_info!( + self.dev, + "{} of {} calibration packets kept from an earlier start\n", + held, + CALIBRATION.len() + ); + } + self.set(PROP_INTERVAL, &INTERVAL_US.to_le_bytes()) + } + + fn stop(&self) -> Result { + self.set(PROP_INTERVAL, &0u32.to_le_bytes()) + } +} + +/// The latest sample, read by the IIO callbacks. +#[derive(Clone, Copy, Default)] +struct Sample { + xyz: [i32; 3], + valid: bool, +} + +/// One buffer scan: X, Y and Z, then the timestamp at the next 8-byte +/// boundary, where the IIO core expects it. +#[repr(C, align(8))] +struct Scan { + xyz: [i32; 3], + pad: u32, + timestamp: i64, +} + +/// The state shared by the report listener and the IIO device. +#[pin_data] +struct State { + kind: &'static Kind, + /// The IIO device, which outlives the report listener. + iio: IioPtr, + #[pin] + sample: SpinLock, +} + +struct IioPtr(*mut bindings::iio_dev); + +// SAFETY: The IIO device may be used from any thread; the functions this +// driver calls on it from the listener are safe from any context. +unsafe impl Send for IioPtr {} +// SAFETY: See above. +unsafe impl Sync for IioPtr {} + +impl FakehidListener for State { + fn process_fakehid_report(&self, data: &[u8]) -> Result<()> { + let xyz = decode(data).ok_or(EMSGSIZE)?; + *self.sample.lock() = Sample { xyz, valid: true }; + // SAFETY: The listener is removed before the IIO device is freed. + let timestamp = unsafe { bindings::iio_get_time_ns(self.iio.0) }; + let scan = Scan { + xyz, + pad: 0, + timestamp, + }; + // SAFETY: As above; `scan` holds the full scan of all three axes, + // which is the only scan mask the device offers, followed by the + // timestamp. The core copies what the enabled buffers take. + unsafe { bindings::iio_push_to_buffers(self.iio.0, ptr::from_ref(&scan).cast()) }; + Ok(()) + } +} + +/// The only scan the device offers: X, Y and Z. +static SCAN_MASKS: [usize; 2] = [0b111, 0]; + +/// The `read_raw` callback of both devices. +/// +/// # Safety +/// +/// The IIO core calls it with a device registered by [`MotionIio`], one of its +/// channels and valid result pointers. +unsafe extern "C" fn motion_read_raw( + indio_dev: *mut bindings::iio_dev, + chan: *const bindings::iio_chan_spec, + val: *mut i32, + val2: *mut i32, + mask: isize, +) -> i32 { + // SAFETY: The IIO core calls this with the device `MotionIio::new()` + // registered, whose private data is the `Arc` stored there. + let state = unsafe { Arc::::borrow((*indio_dev).priv_.cast()) }; + // SAFETY: `chan` is one of the registered channels. + let axis = unsafe { (*chan).address }; + let (int, frac, ret) = if mask == bindings::BINDINGS_IIO_CHAN_INFO_RAW as isize { + let sample = *state.sample.lock(); + if !sample.valid { + return ENODATA.to_errno(); + } + let Some(&count) = sample.xyz.get(axis) else { + return EINVAL.to_errno(); + }; + (count, 0, bindings::IIO_VAL_INT) + } else if mask == bindings::BINDINGS_IIO_CHAN_INFO_SCALE as isize { + (0, state.kind.scale_nano, bindings::IIO_VAL_INT_PLUS_NANO) + } else if mask == bindings::BINDINGS_IIO_CHAN_INFO_SAMP_FREQ as isize { + ((1_000_000 / INTERVAL_US) as i32, 0, bindings::IIO_VAL_INT) + } else { + return EINVAL.to_errno(); + }; + // SAFETY: `val` and `val2` point to the caller's result variables. + unsafe { + *val = int; + *val2 = frac; + } + ret as i32 +} + +fn axis_channel(chan_type: u32, modifier: u32, index: i32) -> bindings::iio_chan_spec { + let mut chan = bindings::iio_chan_spec { + type_: chan_type, + channel2: modifier as i32, + address: index as usize, + scan_index: index, + __bindgen_anon_1: bindings::iio_chan_spec__bindgen_ty_1 { + scan_type: bindings::iio_scan_type { + __bindgen_anon_1: bindings::iio_scan_type__bindgen_ty_1 { sign: b's' }, + realbits: 32, + storagebits: 32, + ..Default::default() + }, + }, + info_mask_separate: 1usize << bindings::BINDINGS_IIO_CHAN_INFO_RAW, + info_mask_shared_by_type: (1usize << bindings::BINDINGS_IIO_CHAN_INFO_SCALE) + | (1usize << bindings::BINDINGS_IIO_CHAN_INFO_SAMP_FREQ), + ..Default::default() + }; + chan.set_modified(1); + chan +} + +fn timestamp_channel(index: i32) -> bindings::iio_chan_spec { + bindings::iio_chan_spec { + type_: bindings::BINDINGS_IIO_TIMESTAMP, + channel: -1, + scan_index: index, + __bindgen_anon_1: bindings::iio_chan_spec__bindgen_ty_1 { + scan_type: bindings::iio_scan_type { + __bindgen_anon_1: bindings::iio_scan_type__bindgen_ty_1 { sign: b's' }, + realbits: 64, + storagebits: 64, + ..Default::default() + }, + }, + ..Default::default() + } +} + +struct Spec { + channels: [bindings::iio_chan_spec; 4], + info: bindings::iio_info, + _p: PhantomPinned, +} + +/// The IIO device of a service, with its buffer. +struct MotionIio { + dev: *mut bindings::iio_dev, + _spec: Pin>, + registered: bool, +} + +impl MotionIio { + /// Allocates the device and its buffer; [`MotionIio::register`] makes it + /// visible. + fn new(pdev: &platform::Device>, kind: &'static Kind) -> Result<(Self, Arc)> { + let dev = pdev.as_ref(); + let spec = KBox::pin( + Spec { + channels: [ + axis_channel(kind.chan_type, bindings::BINDINGS_IIO_MOD_X, 0), + axis_channel(kind.chan_type, bindings::BINDINGS_IIO_MOD_Y, 1), + axis_channel(kind.chan_type, bindings::BINDINGS_IIO_MOD_Z, 2), + timestamp_channel(3), + ], + info: bindings::iio_info { + read_raw: Some(motion_read_raw), + ..Default::default() + }, + _p: PhantomPinned, + }, + GFP_KERNEL, + )?; + // SAFETY: `dev` is a valid device. + let indio_dev = unsafe { bindings::iio_device_alloc(dev.as_raw(), 0) }; + if indio_dev.is_null() { + return Err(ENOMEM); + } + let state = match Arc::pin_init( + pin_init!(State { + kind, + iio: IioPtr(indio_dev), + sample <- new_spinlock!(Sample::default()), + }), + GFP_KERNEL, + ) { + Ok(state) => state, + Err(e) => { + // SAFETY: Allocated above and not registered. + unsafe { bindings::iio_device_free(indio_dev) }; + return Err(e); + } + }; + // SAFETY: `indio_dev` was allocated above and is not registered yet; + // `spec` is pinned and lives as long as the device. + unsafe { + (*indio_dev).priv_ = state.clone().into_foreign().cast(); + (*indio_dev).name = kind.name.as_char_ptr(); + (*indio_dev).channels = spec.channels.as_ptr(); + (*indio_dev).num_channels = spec.channels.len() as i32; + (*indio_dev).info = &spec.info; + (*indio_dev).modes = bindings::INDIO_DIRECT_MODE as i32; + (*indio_dev).available_scan_masks = SCAN_MASKS.as_ptr(); + } + let this = Self { + dev: indio_dev, + _spec: spec, + registered: false, + }; + // SAFETY: Both devices are valid. The buffer is released with the + // platform device's resources; the IIO device holds its own + // reference to it until it is freed. + let ret = unsafe { + bindings::devm_iio_kfifo_buffer_setup_ext( + dev.as_raw(), + indio_dev, + ptr::null(), + ptr::null_mut(), + ) + }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + Ok((this, state)) + } + + fn register(&mut self) -> Result { + let module = kernel::module::this_module::(); + // SAFETY: `self.dev` is a fully set up, unregistered IIO device. + let ret = unsafe { bindings::__iio_device_register(self.dev, module.as_ptr()) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + self.registered = true; + Ok(()) + } +} + +impl Drop for MotionIio { + fn drop(&mut self) { + // SAFETY: `self.dev` was allocated in `new()` with an `Arc` + // as its private data, and nothing uses it after this. + unsafe { + if self.registered { + bindings::iio_device_unregister(self.dev); + } + drop(Arc::::from_foreign((*self.dev).priv_.cast())); + bindings::iio_device_free(self.dev); + } + } +} + +// SAFETY: The IIO core serializes registration and removal internally. +unsafe impl Send for MotionIio {} +// SAFETY: See above; `MotionIio` has no methods taking `&self`. +unsafe impl Sync for MotionIio {} + +/// Stops the reports around system sleep, so that the AOP does not wake the +/// application processor 50 times a second, and starts them again after. +#[repr(C)] +struct PmHook { + /// First, so that the callback finds the hook from it. + nb: Opaque, + control: Arc, + registered: Atomic, +} + +/// The PM notifier callback. +/// +/// # Safety +/// +/// `nb` is the notifier block of a registered [`PmHook`]. +unsafe extern "C" fn pm_event( + nb: *mut bindings::notifier_block, + action: usize, + _data: *mut c_void, +) -> c_int { + // SAFETY: `nb` is the first field of a registered `PmHook`. + let hook = unsafe { &*nb.cast::() }; + let ret = match action as u32 { + bindings::PM_SUSPEND_PREPARE | bindings::PM_HIBERNATION_PREPARE => hook.control.stop(), + bindings::PM_POST_SUSPEND | bindings::PM_POST_HIBERNATION => hook.control.start(), + _ => return bindings::NOTIFY_DONE as c_int, + }; + if let Err(e) = ret { + dev_warn!( + hook.control.dev, + "reports not switched for sleep: {:?}\n", + e + ); + } + bindings::NOTIFY_OK as c_int +} + +/// # Safety +/// +/// `nb` stays valid and pinned until [`pm_unregister`] is called on it. +#[cfg(CONFIG_PM_SLEEP)] +unsafe fn pm_register(nb: *mut bindings::notifier_block) -> c_int { + // SAFETY: By the caller's contract. + unsafe { bindings::register_pm_notifier(nb) } +} + +/// # Safety +/// +/// `nb` was registered by [`pm_register`]. +#[cfg(CONFIG_PM_SLEEP)] +unsafe fn pm_unregister(nb: *mut bindings::notifier_block) { + // SAFETY: By the caller's contract. + unsafe { bindings::unregister_pm_notifier(nb) }; +} + +/// Without system sleep there is nothing to switch. +/// +/// # Safety +/// +/// None; the signature matches the one with system sleep. +#[cfg(not(CONFIG_PM_SLEEP))] +unsafe fn pm_register(_nb: *mut bindings::notifier_block) -> c_int { + 0 +} + +/// # Safety +/// +/// None; the signature matches the one with system sleep. +#[cfg(not(CONFIG_PM_SLEEP))] +unsafe fn pm_unregister(_nb: *mut bindings::notifier_block) {} + +// SAFETY: The notifier block is only touched by the PM core, which +// serializes registration, removal and the callbacks itself. +unsafe impl Send for PmHook {} +// SAFETY: See above; the other fields are `Sync`. +unsafe impl Sync for PmHook {} + +impl PmHook { + fn new(control: Arc) -> Result>> { + let hook = KBox::pin( + Self { + nb: Opaque::new(bindings::notifier_block { + notifier_call: Some(pm_event), + next: ptr::null_mut(), + priority: 0, + }), + control, + registered: Atomic::new(false), + }, + GFP_KERNEL, + )?; + // SAFETY: The block is pinned inside the hook, which unregisters it + // before it goes. + let ret = unsafe { pm_register(hook.nb.get()) }; + if ret < 0 { + return Err(Error::from_errno(ret)); + } + hook.registered.store(true, Relaxed); + Ok(hook) + } + + /// Returns once no callback is running any more; repeated calls do + /// nothing. + fn unregister(&self) { + if self.registered.xchg(false, Relaxed) { + // SAFETY: `nb` was registered by `new()`. + unsafe { pm_unregister(self.nb.get()) }; + } + } +} + +impl Drop for PmHook { + fn drop(&mut self) { + self.unregister(); + } +} + +struct IIOAopMotionDriver { + /// Fields are declared in teardown order: sleep handling first, then the + /// subscription, which ends before the IIO device goes. + pm: Pin>, + listener: FakehidRegistration, + control: Arc, + _iio: MotionIio, +} + +kernel::of_device_table!( + OF_TABLE, + &'static Kind, + [ + (of::DeviceId::new(c_str!("apple,aop-accel")), &ACCEL), + (of::DeviceId::new(c_str!("apple,aop-gyro")), &GYRO), + ] +); + +impl platform::Driver for IIOAopMotionDriver { + type IdInfo = &'static Kind; + type Data<'bound> = IIOAopMotionDriver; + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe<'bound>( + pdev: &'bound platform::Device>, + info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound { + let kind = *info.ok_or(ENODEV)?; + let dev = pdev.as_ref(); + // SAFETY: This driver binds only to the service devices the AOP core + // registers, and `pdev` is being probed. + let aop = unsafe { ::from_child(dev) }?; + // SAFETY: `dev` is a valid device; the AOP core attached its service + // record as the platform data of every service device it registers. + let service = unsafe { (*dev.as_raw()).platform_data.cast::() }; + if service.is_null() { + return Err(ENODEV); + } + // SAFETY: The record lives as long as the device it is attached to. + let service = unsafe { *service }; + + let control = Control::new(dev, aop.clone(), service)?; + let (mut iio, state) = MotionIio::new(pdev, kind)?; + let listener = FakehidRegistration::new(aop, service, state)?; + let pm = PmHook::new(control.clone())?; + control.start()?; + if let Err(e) = iio.register() { + let _ = control.stop(); + return Err(e); + } + Ok(IIOAopMotionDriver { + pm, + listener, + control, + _iio: iio, + }) + } + + fn unbind(_dev: &platform::Device>, this: Pin<&Self>) { + this.pm.unregister(); + let _ = this.control.stop(); + this.listener.unregister(); + } +} + +module_platform_driver! { + type: IIOAopMotionDriver, + name: "iio_aop_motion", + description: "AOP accelerometer and gyroscope driver", + license: "Dual MIT/GPL", + alias: ["platform:iio_aop_motion"], +} diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 51e504321798d..5f512e18bfd50 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -940,7 +940,31 @@ static void apple_dart_test_resume_reset_retry(struct kunit *test) dart->hw->ttbr_valid | 0x1000); } +static void apple_dart_test_strict_engines(struct kunit *test) +{ + static const char * const strict[] = { + "apple,t8103-avd", "apple,t6000-avd", "apple,t8112-avd", "apple,t6041-avd", + "apple,t8132-avd", "apple,t8140-avd", "apple,t6000-ave", "apple,t8112-ave", + "apple,t6041-ave", "apple,t8103-ane", "apple,t6021-ane", "apple,t6041-ane", + "apple,t8112-isp", "apple,t6041-isp", "apple,isp", + }; + static const char * const lazy[] = { + "apple,t6021-ane-mailbox", "apple,asc-mailbox-v4", "apple,t8103-display-pipe", + "apple,nvme-ans2", "apple,t8103-dart", "apple,t6000-display-crossbar", + "brcm,bcm4378-fmac", "apple,", "avd", "foo,t8112-avd", "apple,t8112-avdx", + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(strict); i++) + KUNIT_EXPECT_TRUE_MSG(test, apple_dart_compat_is_strict_engine(strict[i]), + "%s", strict[i]); + for (i = 0; i < ARRAY_SIZE(lazy); i++) + KUNIT_EXPECT_FALSE_MSG(test, apple_dart_compat_is_strict_engine(lazy[i]), + "%s", lazy[i]); +} + static struct kunit_case apple_dart_test_cases[] = { + KUNIT_CASE(apple_dart_test_strict_engines), KUNIT_CASE(apple_dart_test_iommu_specifiers), KUNIT_CASE(apple_dart_test_independent_remap_roots), KUNIT_CASE(apple_dart_test_before_attach), diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 384e243d5d0a2..ba23e2c239cd8 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -3279,6 +3280,52 @@ static bool apple_dart_capable(struct device *dev, enum iommu_cap cap) } } +/* + * Media and ML engines (video decode and encode, neural engine, camera ISP) + * are run by their own coprocessor firmware, which can still have DMA in + * flight after the host driver has given up on a job: a timeout, a reset, + * a stream torn down mid-frame. With deferred flushing the DART keeps + * translating a freed buffer's addresses until the flush queue drains, so + * such a late write lands silently in pages that already went back to the + * allocator. Strict flushing makes it fault at the DART instead. These + * engines map their buffers once per stream or model, so the per-unmap + * invalidate is not on any hot path. + * + * The client is identified by its device tree compatible, "apple,-avd" + * and so on, so every SoC generation is covered without a per-SoC table. + */ +static const char * const apple_dart_strict_engines[] = { + "avd", "ave", "ane", "isp", +}; + +static bool apple_dart_compat_is_strict_engine(const char *compat) +{ + const char *engine; + unsigned int i; + + if (!str_has_prefix(compat, "apple,")) + return false; + engine = strrchr(compat, '-'); + engine = engine ? engine + 1 : compat + strlen("apple,"); + for (i = 0; i < ARRAY_SIZE(apple_dart_strict_engines); i++) + if (!strcmp(engine, apple_dart_strict_engines[i])) + return true; + return false; +} + +static bool apple_dart_master_needs_strict(struct device *dev) +{ + struct property *prop; + const char *compat; + + if (!dev->of_node) + return false; + of_property_for_each_string(dev->of_node, "compatible", prop, compat) + if (apple_dart_compat_is_strict_engine(compat)) + return true; + return false; +} + static int apple_dart_def_domain_type(struct device *dev) { struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); @@ -3298,6 +3345,8 @@ static int apple_dart_def_domain_type(struct device *dev) return IOMMU_DOMAIN_DMA; if (cfg->locked) return IOMMU_DOMAIN_DMA; + if (apple_dart_master_needs_strict(dev)) + return IOMMU_DOMAIN_DMA; return 0; } diff --git a/drivers/leds/leds-pwm.c b/drivers/leds/leds-pwm.c index efe5a97a1f4d6..9633626053a51 100644 --- a/drivers/leds/leds-pwm.c +++ b/drivers/leds/leds-pwm.c @@ -109,9 +109,16 @@ static int led_pwm_add(struct device *dev, struct led_pwm_priv *priv, if (led_data->pwmstate.period) break; led->default_state = LEDS_DEFSTATE_OFF; - dev_warn(dev, - "failed to read period for %s, default to off", - led->name); + /* + * A PWM the bootloader left disabled and never programmed has + * no period to read back, and keeping its state means keeping + * the LED off. Only an enabled PWM without a period is a + * failure to read it. + */ + if (led_data->pwmstate.enabled) + dev_warn(dev, + "failed to read period for %s, default to off", + led->name); fallthrough; default: pwm_init_state(led_data->pwm, &led_data->pwmstate); diff --git a/drivers/media/platform/apple/Kconfig b/drivers/media/platform/apple/Kconfig index 77588c8f4e5a5..6ee29529f4466 100644 --- a/drivers/media/platform/apple/Kconfig +++ b/drivers/media/platform/apple/Kconfig @@ -4,3 +4,5 @@ comment "Apple media platform drivers" source "drivers/media/platform/apple/isp/Kconfig" source "drivers/media/platform/apple/avd/Kconfig" +source "drivers/media/platform/apple/ave/Kconfig" +source "drivers/media/platform/apple/jpeg/Kconfig" diff --git a/drivers/media/platform/apple/Makefile b/drivers/media/platform/apple/Makefile index 72f8bb54b376d..39f5eccd5bb1c 100644 --- a/drivers/media/platform/apple/Makefile +++ b/drivers/media/platform/apple/Makefile @@ -2,3 +2,5 @@ obj-y += isp/ obj-y += avd/ +obj-y += ave/ +obj-y += jpeg/ diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index 60d69bab8a342..64b441be90ffb 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -246,6 +246,19 @@ struct avd_av1_ctx { } bufs; }; +/* + * Timestamp of the reference a ref_frame_idx entry names. The entries are + * -1 on frames that use no references (FFmpeg fills key frames that way); + * indexing reference_frame_ts with them read out of bounds. Return a + * timestamp no buffer has, so the lookup takes the missing-reference path. + */ +static u64 avd_av1_ref_ts(const struct v4l2_ctrl_av1_frame *frame, int idx) +{ + if (idx < 0 || idx >= V4L2_AV1_TOTAL_REFS_PER_FRAME) + return U64_MAX; + return frame->reference_frame_ts[idx]; +} + /* * from verisilicon/rockchip_vpu981_hw_av1_dec.c */ @@ -367,8 +380,7 @@ static void set_refs(struct avd_ctx *ctx, struct avd_av1_run *run) ref_idx = i + V4L2_AV1_REF_LAST_FRAME; ref = avd_get_ref_buf( ctx, &dst->base.vb, - frame->reference_frame_ts - [frame->ref_frame_idx[i]]); + avd_av1_ref_ts(frame, frame->ref_frame_idx[i])); } addr = vb2_dma_contig_plane_dma_addr(&ref->base.vb.vb2_buf, 0) + ref->comp.start_offset; @@ -455,9 +467,8 @@ static bool is_valid_ref_frame(struct avd_ctx *ctx, struct avd_av1_run *run, const struct v4l2_ctrl_av1_frame *frame = run->frame; struct avd_decoded_buffer *ref = avd_get_ref_buf( ctx, &dst->base.vb, - frame->reference_frame_ts - [frame->ref_frame_idx[ref_frame - - V4L2_AV1_REF_LAST_FRAME]]); + avd_av1_ref_ts(frame, frame->ref_frame_idx[ref_frame - + V4L2_AV1_REF_LAST_FRAME])); int ref_mi_cols = DIV_ROUND_UP(ref->av1.width + 1, 8); int ref_mi_rows = DIV_ROUND_UP(ref->av1.height + 1, 8); bool intra_only = ((ref->av1.frame_type == V4L2_AV1_KEY_FRAME) || @@ -485,7 +496,7 @@ static int avd_select_refs(struct avd_ctx *ctx, struct avd_av1_run *run, if (seq->flags & (V4L2_AV1_SEQUENCE_FLAG_ENABLE_ORDER_HINT | V4L2_AV1_SEQUENCE_FLAG_ENABLE_REF_FRAME_MVS)) { ref = avd_get_ref_buf(ctx, &dst->base.vb, - frame->reference_frame_ts[last_ref_idx]); + avd_av1_ref_ts(frame, last_ref_idx)); if (dst != ref) { use_last = (ref->av1.order_hints[V4L2_AV1_REF_ALTREF_FRAME] != @@ -554,7 +565,7 @@ static void set_ref_hints(struct avd_ctx *ctx, struct avd_av1_run *run, V4L2_AV1_REF_LAST_FRAME]; ref = avd_get_ref_buf( ctx, &dst->base.vb, - frame->reference_frame_ts[ref_idx]); + avd_av1_ref_ts(frame, ref_idx)); push(AV1_ORDER_HINT0(ref->av1.order_hints[3]) | AV1_ORDER_HINT1(ref->av1.order_hints[2]) | @@ -585,8 +596,8 @@ static void set_ref_hdr(struct avd_ctx *ctx, struct avd_av1_run *run) /* point to the first index where the frames equal */ for (i = 0; i < 6; i++) { for (j = i + 1; j < 7; j++) { - if (frame->reference_frame_ts[frame->ref_frame_idx[i]] == - frame->reference_frame_ts[frame->ref_frame_idx[j]]) { + if (avd_av1_ref_ts(frame, frame->ref_frame_idx[i]) == + avd_av1_ref_ts(frame, frame->ref_frame_idx[j])) { ref_slots[j] = ref_slots[i]; break; } @@ -831,7 +842,7 @@ static void set_header(struct avd_ctx *ctx, struct avd_av1_run *run) V4L2_AV1_REF_LAST_FRAME]; ref = avd_get_ref_buf( ctx, &dst->base.vb, - frame->reference_frame_ts[ref_idx]); + avd_av1_ref_ts(frame, ref_idx)); ref_addr = vb2_dma_contig_plane_dma_addr( &ref->base.vb.vb2_buf, 0) + @@ -1075,9 +1086,8 @@ static void avd_av1_set_prob(struct avd_ctx *ctx, struct avd_av1_run *run) } else { ref = avd_get_ref_buf( ctx, &dst->base.vb, - run->frame->reference_frame_ts - [run->frame->ref_frame_idx - [run->frame->primary_ref_frame]]); + avd_av1_ref_ts(run->frame, + run->frame->ref_frame_idx[run->frame->primary_ref_frame])); memcpy(av1_ctx->bufs.probs.cpu, vb2_plane_vaddr(&ref->base.vb.vb2_buf, 0) + AVD_AV1_CDFS_OFFSET( @@ -1118,8 +1128,8 @@ static void avd_av1_set_seg(struct avd_ctx *ctx, struct avd_av1_run *run) if (frame->primary_ref_frame != AV1_PRIMARY_REF_NONE) { ref = avd_get_ref_buf( ctx, &dst->base.vb, - frame->reference_frame_ts - [frame->ref_frame_idx[frame->primary_ref_frame]]); + avd_av1_ref_ts(frame, + frame->ref_frame_idx[frame->primary_ref_frame])); /* avd_get_ref_buf() returns dst for a missing reference */ /* same MiCols and MiRows, as libdav1d compares them */ if (ref == dst || @@ -1384,9 +1394,9 @@ static int avd_av1_run(struct avd_ctx *ctx) ref = dst; if (run.frame->primary_ref_frame != AV1_PRIMARY_REF_NONE) ref = avd_get_ref_buf(ctx, &dst->base.vb, - run.frame->reference_frame_ts - [run.frame->ref_frame_idx - [run.frame->primary_ref_frame]]); + avd_av1_ref_ts(run.frame, + run.frame->ref_frame_idx + [run.frame->primary_ref_frame])); if (!vb2_plane_vaddr(&dst->base.vb.vb2_buf, 0) || !vb2_plane_vaddr(&ref->base.vb.vb2_buf, 0)) { avd_run_postamble(ctx, &run.base); diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index bf5bf96f4c62a..215ac921ce671 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -222,29 +223,161 @@ int avd_submit_job(struct avd_ctx *ctx) return 0; } +/* Recover at most twice a minute, never twice within this gap. */ +#define AVD_RECOVER_MAX 2 +#define AVD_RECOVER_WINDOW msecs_to_jiffies(60000) +#define AVD_RECOVER_GAP_MS 20 + +/* + * Reset the decoder after a timeout or a VP error. Every reset is a risk + * to the whole machine (the J616s hung hard after timeouts and resets + * three times in one night, and the Neo after a burst of one reset per + * frame), so: stop the interrupt handler and the CM3 first, block the + * decoder's DMA at its DART so a transfer still in flight faults instead + * of reaching memory that may already be reused, give such transfers time + * to drain, and only then assert the pmgr reset. Recover sparingly: a + * third recovery within a minute parks the decoder (DMA blocked) until the + * minute is over, and only a failed reset retires it. + */ +/* + * Parking (resting or retired): stop the CM3 and leave the decoder's DMA + * blocked at its DART, so an engine that is still running (the frame timed + * out, after all) can only fault, never write memory whose buffers are about + * to go back to userspace. + */ +static void avd_park(struct avd_dev *avd) +{ + if (pm_runtime_resume_and_get(avd->dev) < 0) + return; + disable_irq(avd->irq); + avd->irq_parked = true; + avd_shutdown(avd); + if (avd->empty_domain && !avd->dma_parked) { + if (iommu_attach_device(avd->empty_domain, avd->dev)) + dev_err(avd->dev, "retire: could not block DMA\n"); + else + avd->dma_parked = true; + } + pm_runtime_put_autosuspend(avd->dev); +} + static int avd_reset(struct avd_dev *avd) { int ret = 0; + if (avd->retired) + return -ENODEV; + if (!avd->recoveries || + time_after(jiffies, avd->recover_window_start + AVD_RECOVER_WINDOW)) { + avd->recover_window_start = jiffies; + avd->recoveries = 0; + } + if (++avd->recoveries > AVD_RECOVER_MAX) { + /* + * Out of budget: stop the decoder with its DMA blocked and let + * it rest until the window ends. Frames fail without touching + * the hardware meanwhile; the next frame after that recovers + * it. Bad streams (or conformance vectors using features the + * hardware lacks) must not cost the user the decoder until a + * reboot. + */ + avd_park(avd); + avd->rest_until = avd->recover_window_start + AVD_RECOVER_WINDOW; + dev_warn(avd->dev, "decoder resting after repeated resets, new frames fail for up to %u s\n", + jiffies_to_msecs(avd->rest_until - jiffies) / 1000 + 1); + return -EBUSY; + } + if (avd->last_recover && + time_before(jiffies, avd->last_recover + msecs_to_jiffies(AVD_RECOVER_GAP_MS))) + msleep(AVD_RECOVER_GAP_MS); + ret = pm_runtime_resume_and_get(avd->dev); if (ret < 0) - return ret; + goto retire; + /* No handler may touch the mailbox while the block is in reset. */ + disable_irq(avd->irq); + dev_warn(avd->dev, "recover: stop firmware\n"); + avd_shutdown(avd); + if (avd->empty_domain) { + dev_warn(avd->dev, "recover: block DMA\n"); + ret = iommu_attach_device(avd->empty_domain, avd->dev); + if (ret) { + dev_err(avd->dev, "recover: DART empty attach failed: %d\n", ret); + goto put; + } + usleep_range(5000, 6000); + } + dev_warn(avd->dev, "recover: reset\n"); ret = reset_control_reset(avd->rstc); if (ret) - dev_err(avd->dev, "reset: failed: %d", ret); - - if (avd->empty_domain) { - iommu_attach_device(avd->empty_domain, avd->dev); + dev_err(avd->dev, "recover: reset failed: %d\n", ret); + if (avd->empty_domain && ret) { + /* A decoder that failed its reset keeps its DMA blocked. */ + avd->dma_parked = true; + } else if (avd->empty_domain) { + dev_warn(avd->dev, "recover: restore DMA\n"); iommu_detach_device(avd->empty_domain, avd->dev); + if (avd->domain && iommu_get_domain_for_dev(avd->dev) != avd->domain) { + dev_err(avd->dev, "recover: DART domain not restored\n"); + ret = -EIO; + } + } + if (!ret) { + dev_warn(avd->dev, "recover: boot firmware\n"); + ret = avd_boot(avd); + if (ret) + dev_err(avd->dev, "recover: boot failed: %d\n", ret); + } +put: + avd->last_recover = jiffies ?: 1; + if (!ret) { + enable_irq(avd->irq); + dev_warn(avd->dev, "recover: done\n"); } + pm_runtime_put_autosuspend(avd->dev); + if (!ret) + return 0; +retire: + avd->retired = true; + dev_err(avd->dev, "decoder retired after a failed reset (%d): reload the module to use it again\n", + ret); + return ret; +} - ret = avd_boot(avd); - if (ret) - dev_err(avd->dev, "reset: failed to boot"); +/* Bring a rested decoder back: reset with DMA still blocked, restore, boot. */ +static int avd_wake(struct avd_dev *avd) +{ + int ret; + ret = pm_runtime_resume_and_get(avd->dev); + if (ret < 0) + return ret; + dev_warn(avd->dev, "recover: wake after rest\n"); + avd_shutdown(avd); + ret = reset_control_reset(avd->rstc); + if (!ret && avd->dma_parked) { + iommu_detach_device(avd->empty_domain, avd->dev); + if (avd->domain && iommu_get_domain_for_dev(avd->dev) != avd->domain) + ret = -EIO; + else + avd->dma_parked = false; + } + if (!ret) + ret = avd_boot(avd); + if (!ret) { + avd->recoveries = 0; + avd->last_recover = jiffies ?: 1; + if (avd->irq_parked) + enable_irq(avd->irq); + avd->irq_parked = false; + dev_warn(avd->dev, "recover: done\n"); + } else { + avd->retired = true; + dev_err(avd->dev, "decoder retired after a failed wake (%d): reload the module to use it again\n", + ret); + } pm_runtime_put_autosuspend(avd->dev); - return ret; } @@ -284,6 +417,12 @@ static void avd_watchdog_func(struct work_struct *work) ret = avd_reset(avd); if (ret) dev_err(avd->dev, "failed to reset: %d", ret); + /* + * A stream whose frame failed in hardware usually fails the next one + * too (a reset per frame took the Neo down), so stop it here. + */ + ctx->broken = true; + dev_err(avd->dev, "stream stopped after a failed frame; restart streaming to retry\n"); avd_job_finish(ctx, VB2_BUF_STATE_ERROR); } @@ -300,6 +439,19 @@ static irqreturn_t avd_irq_handler(int irq, void *data) if (!ctx) return IRQ_HANDLED; + /* + * Reading RETRIEVE on an empty mailbox returns the previous message + * again. A repeated PP DONE finishes the next frame before its PP has + * run: userspace gets a wrong picture, the frame's own PP DONE never + * comes, and the watchdog resets the decoder. Give a late message + * 10 us, otherwise acknowledge and ignore the interrupt. + */ + if (readl_poll_timeout_atomic(avd->mbox + AVD_REG_MBOX1_STATUS, status, + !(status & AVD_MBOX1_STATUS_EMPTY), 1, 10)) { + writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); + dev_dbg_ratelimited(avd->dev, "spurious interrupt, mailbox empty\n"); + return IRQ_HANDLED; + } status = readl(avd->mbox + AVD_REG_MBOX1_RETRIEVE); writel(AVD_MBOX1_NOT_EMPTY, avd->mbox + AVD_REG_MBOX_IRQ_CLR); @@ -317,6 +469,10 @@ static irqreturn_t avd_irq_handler(int irq, void *data) /* a vp is done, kick the pp and hope for the best */ if(ctx->coded_fmt_desc->ops->submit) ctx->coded_fmt_desc->ops->submit(ctx); + spin_lock_irqsave(&avd->job_lock, flags); + if (avd->job_ctx == ctx) + avd->job_vp_done = true; + spin_unlock_irqrestore(&avd->job_lock, flags); goto done; } else { dev_err(avd->dev, "H%d error", status); @@ -327,7 +483,7 @@ static irqreturn_t avd_irq_handler(int irq, void *data) * timeout: it resets the AVD and fails only this frame, and it * ignores a job the hardware does not own yet. */ - mod_delayed_work(system_wq, &ctx->watchdog_work, 0); + mod_delayed_work(system_dfl_wq, &ctx->watchdog_work, 0); goto done; } @@ -338,7 +494,13 @@ static irqreturn_t avd_irq_handler(int irq, void *data) * let avd_device_run() finish the job when it returns. */ spin_lock_irqsave(&avd->job_lock, flags); - if (avd->job_ctx != ctx) { + if (avd->job_ctx != ctx || !avd->job_vp_done) { + /* + * Only the PP kicked for this job's VP can finish it. A surplus + * PP DONE of a multi-kick picture used to finish the next job + * early, so userspace reused its buffers while the decoder was + * still writing them. + */ stale = true; } else if (avd->job_state == AVD_JOB_SUBMITTED) { avd->job_state = AVD_JOB_FINISHING; @@ -371,13 +533,31 @@ static void avd_device_run(void *priv) enum vb2_buffer_state result = VB2_BUF_STATE_ERROR; const char *outcome = "already finished"; unsigned long flags; - bool finish = false; + bool finish = false, fail; u64 seq; int ret; if (WARN_ON(!desc)) return; + fail = READ_ONCE(avd->retired) || ctx->broken; + /* A rested decoder stays parked until its window ends, then wakes. */ + if (!fail && avd->rest_until) { + if (time_before(jiffies, avd->rest_until) || avd_wake(avd)) + fail = true; + else + avd->rest_until = 0; + } + if (fail) { + struct vb2_v4l2_buffer *src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + + /* Complete the request's controls too, or its fd never signals. */ + if (src && src->vb2_buf.req_obj.req) + v4l2_ctrl_request_complete(src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + avd_job_finish_no_pm(ctx, VB2_BUF_STATE_ERROR); + return; + } + /* * IRQ completion cannot wait for an already running watchdog. Drain it * here, before reusing wd_seq, so an old callback cannot time out this @@ -401,6 +581,7 @@ static void avd_device_run(void *priv) avd->job_ctx = ctx; avd->job_state = AVD_JOB_RUNNING; avd->job_end_sent = false; + avd->job_vp_done = false; avd->job_pending = false; spin_unlock_irqrestore(&avd->job_lock, flags); dev_dbg(avd->dev, "avd-job: run seq %llu\n", seq); @@ -427,7 +608,7 @@ static void avd_device_run(void *priv) } else { avd->job_state = AVD_JOB_SUBMITTED; ctx->wd_seq = seq; - schedule_delayed_work(&ctx->watchdog_work, msecs_to_jiffies(2000)); + queue_delayed_work(system_dfl_wq, &ctx->watchdog_work, msecs_to_jiffies(2000)); outcome = "submitted"; } } @@ -954,6 +1135,7 @@ static int avd_probe(struct platform_device *pdev) irq = platform_get_irq(pdev, 1); if (irq < 0) return irq; + avd->irq = irq; ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, avd_irq_handler, IRQF_ONESHOT, dev_name(&pdev->dev), avd); @@ -986,6 +1168,8 @@ static void avd_remove(struct platform_device *pdev) avd_v4l2_cleanup(avd); + if (avd->dma_parked) + iommu_detach_device(avd->empty_domain, avd->dev); if (avd->empty_domain) iommu_domain_free(avd->empty_domain); diff --git a/drivers/media/platform/apple/avd/avd-hevc.c b/drivers/media/platform/apple/avd/avd-hevc.c index 9dfa20ab71913..49d8f046ac5c9 100644 --- a/drivers/media/platform/apple/avd/avd-hevc.c +++ b/drivers/media/platform/apple/avd/avd-hevc.c @@ -1282,6 +1282,51 @@ static int avd_hevc_compute_tiles(struct avd_ctx *ctx, struct avd_hevc_run *run) return 0; } +/* + * The slice builder below expects the GStreamer layout: the slices tile the + * OUTPUT buffer as whole NALs, bit_size covers the whole NAL and + * data_byte_offset is the slice data's offset inside it. FFmpeg's request + * API backend instead gives each slice's data_byte_offset from the start of + * the buffer and a bit_size of the slice data only, so every slice was given + * a wrong address and size and the VP raised its error interrupt on every + * frame. Detect the absolute layout (complete and unambiguous) and rewrite + * the run's private copy into the equivalent NAL-relative description. + */ +static void avd_hevc_normalize_slices(struct avd_hevc_run *run, u32 payload) +{ + struct v4l2_ctrl_hevc_slice_params *sl = + (struct v4l2_ctrl_hevc_slice_params *)run->sl; + u64 nal_end = 0, data_end = 0, prev_end = 0; + bool nal = true, absolute = true; + int i; + + if (!run->num_slices || !payload) + return; + for (i = 0; i < run->num_slices; i++) { + u64 bytes = DIV_ROUND_UP((u64)sl[i].bit_size, 8); + u64 start = sl[i].data_byte_offset; + + if (!bytes || start >= bytes || nal_end + bytes > payload) + nal = false; + nal_end += bytes; + if (!bytes || start < data_end || start + bytes > payload) + absolute = false; + data_end = start + bytes; + } + nal &= nal_end == payload; + absolute &= data_end == payload; + if (!absolute || nal) + return; + for (i = 0; i < run->num_slices; i++) { + u64 bytes = DIV_ROUND_UP((u64)sl[i].bit_size, 8); + u64 start = sl[i].data_byte_offset; + + sl[i].data_byte_offset = start - prev_end; + sl[i].bit_size = (start - prev_end + bytes) * 8; + prev_end = start + bytes; + } +} + static int avd_hevc_run_preamble(struct avd_ctx *ctx, struct avd_hevc_run *run) { struct v4l2_ctrl *ctrl, *sl, *ep; @@ -1353,6 +1398,8 @@ static int avd_hevc_run(struct avd_ctx *ctx) if (ret) return ret; + avd_hevc_normalize_slices(&run, + vb2_get_plane_payload(&run.base.bufs.src->vb2_buf, 0)); dst = vb2_to_avd_decoded_buf(&run.base.bufs.dst->vb2_buf); update_dec_buf_info(dst, &run.sl[0]); diff --git a/drivers/media/platform/apple/avd/avd-regs.h b/drivers/media/platform/apple/avd/avd-regs.h index 388f3e64895f5..ace0e4c48f40a 100644 --- a/drivers/media/platform/apple/avd/avd-regs.h +++ b/drivers/media/platform/apple/avd/avd-regs.h @@ -13,6 +13,8 @@ #define AVD_MBOX1_NOT_EMPTY BIT(3) #define AVD_REG_MBOX1_STATUS 0x5c +/* Read side: bits 8-11 write pointer, bits 12-15 read pointer, bit 17 empty. */ +#define AVD_MBOX1_STATUS_EMPTY BIT(17) #define AVD_REG_MBOX1_RETRIEVE 0x64 #define AVD_MBOX_ENABLE BIT(0) diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index 4622e92055486..23917fd62bdca 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -889,6 +889,10 @@ static void avd_buf_request_complete(struct vb2_buffer *vb) static int avd_start_streaming(struct vb2_queue *q, unsigned int count) { struct avd_ctx *ctx = vb2_get_drv_priv(q); + + /* A new stream gets a fresh start after a hardware failure. */ + if (!V4L2_TYPE_IS_CAPTURE(q->type)) + ctx->broken = false; const struct avd_coded_fmt_desc *desc; struct vb2_v4l2_buffer *vbuf; int ret; diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index ad7accb6e9615..1f2fa8e312d92 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -229,10 +229,23 @@ struct avd_dev { struct avd_ctx *job_ctx; u64 job_seq; bool job_end_sent; + /* The current job's VP finished and its PP was kicked. */ + bool job_vp_done; bool job_pending; enum vb2_buffer_state job_pending_result; struct reset_control *rstc; + int irq; + /* Recovery budget; a retired decoder fails every job until reload. */ + unsigned long recover_window_start; + unsigned int recoveries; + unsigned long last_recover; + bool retired; + /* DMA blocked at the DART (retired or resting); detached at remove. */ + bool dma_parked; + /* Out of recovery budget: frames fail without hardware access until then. */ + unsigned long rest_until; + bool irq_parked; const struct avd_variant *variant; unsigned int capabilities; @@ -258,6 +271,8 @@ struct avd_buf { struct avd_ctx { struct v4l2_fh fh; struct avd_dev *dev; + /* A frame failed in hardware: fail every job until streaming restarts. */ + bool broken; struct v4l2_format coded_fmt; struct v4l2_format decoded_fmt; diff --git a/drivers/media/platform/apple/ave/.kunitconfig b/drivers/media/platform/apple/ave/.kunitconfig new file mode 100644 index 0000000000000..6cf7ed0a8090e --- /dev/null +++ b/drivers/media/platform/apple/ave/.kunitconfig @@ -0,0 +1,15 @@ +CONFIG_KUNIT=y +CONFIG_EXPERT=y +CONFIG_NO_IOMEM=n +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_SUPPORT_FILTER=n +CONFIG_MEDIA_PLATFORM_DRIVERS=y +CONFIG_VIDEO_APPLE_AVE_KUNIT_TEST=y +CONFIG_ARCH_APPLE=y +CONFIG_PM=y +CONFIG_PM_GENERIC_DOMAINS=y +CONFIG_IOMMU_SUPPORT=y +CONFIG_IOMMU_DMA=y +CONFIG_VIDEO_DEV=y +CONFIG_VIDEO_APPLE_AVE=y +CONFIG_VIDEO_APPLE_AVE_CMD_KUNIT_TEST=y diff --git a/drivers/media/platform/apple/ave/Kconfig b/drivers/media/platform/apple/ave/Kconfig new file mode 100644 index 0000000000000..de05c7aeb7b25 --- /dev/null +++ b/drivers/media/platform/apple/ave/Kconfig @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config VIDEO_APPLE_AVE + tristate "Apple AVE video encoder" + depends on ARCH_APPLE || COMPILE_TEST + depends on VIDEO_DEV && IOMMU_DMA && PM_GENERIC_DOMAINS + select CRC32 + select V4L2_MEM2MEM_DEV + select VIDEOBUF2_DMA_CONTIG + help + V4L2 memory-to-memory driver for the AVE H.264/HEVC encoder in + Apple T8140 (A18 Pro) SoCs, as in the MacBook Neo. + T6000 (M1 Pro) and T8112 (M2) support lifecycle-only validation + of firmware 6070.11.1 through debugfs; encoding stays disabled. + T6041 (M4 Pro) has probe-only support. + + The encoder firmware is loaded by the bootloader; the driver builds + every command itself. See Documentation/admin-guide/media/apple-ave.rst. + + To compile this driver as a module, choose M here: the module + will be called apple-ave. + +config VIDEO_APPLE_AVE_KUNIT_TEST + tristate "KUnit tests for Apple AVE buffer sizing and recovery" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Test the exact LFS reference descriptor sizes for pre-M4 and T8140 + encoders, including picture alignment boundaries and the reference + 1920x1080 AVC and HEVC sizes. The tests do not need an encoder or + any firmware and can run independently of the AVE driver. + Also test the firmware recovery sliding window and jiffies wrap. + +config VIDEO_APPLE_AVE_CMD_KUNIT_TEST + bool "KUnit tests for Apple AVE firmware ABIs" if !KUNIT_ALL_TESTS + depends on KUNIT && VIDEO_APPLE_AVE=y + default KUNIT_ALL_TESTS + help + Compare AVC/HEVC 9003 Start and Process images byte for byte with + the pre-refactor builder. Test 6070 lifecycle envelopes, replies, + split surface arrays, seven-channel discovery and encoding refusal. + These tests run without firmware or encoder hardware. diff --git a/drivers/media/platform/apple/ave/Makefile b/drivers/media/platform/apple/ave/Makefile new file mode 100644 index 0000000000000..072642374e1d4 --- /dev/null +++ b/drivers/media/platform/apple/ave/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_VIDEO_APPLE_AVE) += apple-ave.o +apple-ave-y := ave-drv.o ave-fw.o ave-ipc.o ave-cmd.o ave-writer.o ave-abi.o ave-layout.o ave-image.o ave-sess.o ave-v4l2.o ave-vb2.o +obj-$(CONFIG_VIDEO_APPLE_AVE_KUNIT_TEST) += ave-sizes-test.o ave-recovery-test.o +obj-$(CONFIG_VIDEO_APPLE_AVE_CMD_KUNIT_TEST) += ave-abi-test.o diff --git a/drivers/media/platform/apple/ave/abi.rst b/drivers/media/platform/apple/ave/abi.rst new file mode 100644 index 0000000000000..7717bbab48ef3 --- /dev/null +++ b/drivers/media/platform/apple/ave/abi.rst @@ -0,0 +1,65 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Apple AVE firmware ABIs +====================== + +The firmware TEXT CRC chooses the wire layout. SoC match data supplies register +addresses, boot arguments and device identity. These are separate from streaming +qualification: identifying a firmware does not qualify its encoder payloads. + +9003.78.0 keeps its existing boot request, host clock synchronization, command +IDs, contiguous surface descriptors and reply envelope. The KUnit oracle in +``tests/ave-9003-reference.h`` freezes the driver builders and layout from Linux +commit 23e431a98962. Tests compare every byte of AVC and HEVC Start and Process +at 1920x1080 with B-frames enabled/disabled, and 3840x2160 with B-frames enabled. +They compare both deterministic synthetic DART addresses and masked images. +The constant image tables in ave-image.c are unchanged by this refactor. + +6070.11.1 is selected only for T6000 TEXT CRC 460c7a7c or T8112 TEXT CRC 25bfd039, +with the corresponding SoC identity. Boot, seven-channel discovery, Config, +Open, Close and scratch-confirmed Halt are implemented from the clean derived +SPEC-6070-lifecycle.md ledger. Start and Process return EOPNOTSUPP before writing +a payload or submitting a command. Reset also remains unavailable: its request +has a codec-dependent body, rather than a scalar header. No Priority command is +sent for this ABI. The split surface writer supports separate address and size +arrays, without assuming a semantic mapping for the unqualified fields. + +Lifecycle-only validation +------------------------- + +Build with CONFIG_DEBUG_FS. On a supported 6070 device, write ``1`` (AVC) or ``2`` +(HEVC) to ``/sys/kernel/debug/apple-ave/lifecycle_test``. This synchronous operation +runs boot, Config, Open, Close and Halt, then requests runtime suspend. It does +not allocate encoding surfaces or issue Start, Process or uninit for an encoder +that was never initialized. A second write exercises a cold restart and DATA +restore. An existing session makes the operation return EBUSY. + +The platform must supply the extra-0 register resource as ``extra0`` or register +resource index 3, with at least 0x24 bytes on T6000 or 0x4000 bytes on T8112. +The driver uses the translated platform resource address, maps its containing +16 KiB page through the device DMA API and puts that dynamically allocated IOVA +in Config. Platform DT/binding changes are separate from this driver change. +No fixed firmware or register IOVA is used as an ordinary DMA allocation. + +Before every boot the driver checks the firmware's IOMMU translations. DATA must +match the loader-exported pre-start CRC at probe; subsequent cold boots restore +that pristine image and verify its CRC. Firmware reservations and translations +remain persistent. IPC and extra-0 mappings are per-boot resources. + +Halt clears scratch0, sends ID 14 without expecting a ring reply, and waits up to +five seconds for scratch0 0x08042006. Only then does the driver stop the IOP and +check control readback. Runtime PM performs the power transition; a second boot +is refused until suspend/resume has intervened. A missing Halt confirmation or +failed lifecycle leaves the firmware broken and retains DMA mappings, runtime +power and the module reference. A forced unload or in-place restart is not a +recovery procedure. + +Hardware acceptance remains independent of KUnit: check both SoC VCBAR readbacks, +exact hello, ring dimensions, Config/Open/Close completion IDs and status, Halt +scratch confirmation, actual power gating and a second cold boot. Encoding on +9003 requires the existing hardware regression lane; these software tests do not +establish hardware equivalence. + +The 9003 runtime recovery path retains its upstream quarantine and reset-witness +checks. It is deliberately unavailable for 6070: a broken lifecycle must keep +its mappings and power until scratch-confirmed Halt can establish quiescence. diff --git a/drivers/media/platform/apple/ave/ave-abi-test.c b/drivers/media/platform/apple/ave/ave-abi-test.c new file mode 100644 index 0000000000000..ec295b432a40a --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-abi-test.c @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Byte equality against the pre-ABI-refactor 9003 command builders. */ +#include +#include +#include +#include +#include + +#include "ave.h" +#include "ave-regs.h" +#include "tests/ave-9003-reference.h" + +static void ave_test_buf(struct ave_buf *b, unsigned int *index) +{ + b->dma = 0x820000000ULL + (u64)(*index) * 0x2000000; + b->size = SZ_16K * (1 + (*index)++); +} + +static void ave_test_session(struct ave_session *sess, enum ave_codec codec, + u32 width, u32 height, bool bframes) +{ + struct ave_params *p = &sess->p; + unsigned int index = 0; + + *p = (struct ave_params){ + .codec = codec, + .width = width, + .height = height, + .cw = ALIGN(width, 16), + .ch = ALIGN(height, 16), + .stride = ALIGN(width, 64), + .fps_num = 30, + .fps_den = 1, + .fps_int = 30, + .bframes = bframes, + .reorder_off = !bframes, + .slots = bframes ? 5 : 3, + .refs = bframes ? 5 : 2, + .gop = 120, + .rc_mode = 4, + .bitrate = 8000000, + .limit_bytes = 2000000, + .min_qp = -1, + .max_qp = 51, + .profile = codec == AVE_CODEC_H264 ? 6 : 1, + .level = codec == AVE_CODEC_H264 ? 13 : 7, + .high = codec == AVE_CODEC_H264, + }; + p->mbs = p->cw / 16 * (p->ch / 16); + p->coded_cap = ALIGN(3 * p->cw * p->ch / 2, SZ_16K); + sess->token = 400; + sess->client = 10; + ave_test_buf(&sess->hdr, &index); + for (unsigned int i = 0; i < AVE_MAX_REFS; i++) { + ave_test_buf(&sess->ref_a[i], &index); + ave_test_buf(&sess->ref_b[i], &index); + ave_test_buf(&sess->ref_c[i], &index); + ave_test_buf(&sess->ref_d[i], &index); + } + ave_test_buf(&sess->mb, &index); + ave_test_buf(&sess->row_a, &index); + ave_test_buf(&sess->row_b, &index); + ave_test_buf(&sess->fixed, &index); + for (unsigned int i = 0; i < AVE_TILE_BUFS; i++) { + ave_test_buf(&sess->tile[i], &index); + ave_test_buf(&sess->tile_aux[i], &index); + } + for (unsigned int i = 0; i < 2; i++) + ave_test_buf(&sess->hevc_extra[i], &index); + for (unsigned int i = 0; i < p->slots; i++) { + ave_test_buf(&sess->slot[i].coded, &index); + ave_test_buf(&sess->slot[i].meta, &index); + ave_test_buf(&sess->slot[i].aux, &index); + } +} + +/* These are synthetic DART addresses, never addresses from a device capture. */ +static void ave_mask_dart(u8 *cmd, size_t len) +{ + for (size_t off = 0; off + 8 <= len; off += 8) { + u64 val = get_unaligned_le64(cmd + off); + + if (val >= 0x800000000ULL && val < 0x1000000000ULL) + put_unaligned_le64(0, cmd + off); + } +} + +static void ave_9003_byte_equality(struct kunit *test) +{ + struct ave_dev *ave = kunit_kzalloc(test, sizeof(*ave), GFP_KERNEL); + struct ave_ctx *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + struct ave_session *sess = + kunit_kzalloc(test, sizeof(*sess), GFP_KERNEL); + u8 *actual = kunit_kmalloc(test, 0x14000, GFP_KERNEL); + u8 *expected = kunit_kmalloc(test, 0x14000, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, ave); + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, sess); + KUNIT_ASSERT_NOT_NULL(test, actual); + KUNIT_ASSERT_NOT_NULL(test, expected); + ave->layout = &ave_layout_9003_78_0; + ctx->ave = ave; + ctx->sess = sess; + for (unsigned int codec = AVE_CODEC_H264; codec <= AVE_CODEC_HEVC; + codec++) { + for (unsigned int setting = 0; setting < 3; setting++) { + size_t len = + ave->layout->bytes[codec - 1][AVE_IMAGE_START]; + + ave_test_session(sess, codec, + setting == 2 ? 3840 : 1920, + setting == 2 ? 2160 : 1080, + setting != 1); + memset(actual, 0, len); + memset(expected, 0, len); + KUNIT_ASSERT_EQ(test, ave_build_start(ctx, actual, len), + 0); + sess->token = 400; + ave_reference_start(ctx, expected, len); + ave_stamp_command(ave->layout->abi, actual, 94400000, + 0x12345678); + put_unaligned_le64(94400000, expected + 0x30); + put_unaligned_le64(0x12345678, expected + 0x38); + KUNIT_EXPECT_MEMEQ(test, actual, expected, len); + ave_mask_dart(actual, len); + ave_mask_dart(expected, len); + KUNIT_EXPECT_MEMEQ(test, actual, expected, len); + for (unsigned int frame = 0; frame < 2; frame++) { + unsigned int slot = frame ? sess->p.slots - 1 : + 0; + + len = ave->layout->bytes[codec - 1] + [AVE_IMAGE_PROCESS]; + sess->frames = frame ? 7 : 0; + sess->token = 500; + memset(actual, 0, len); + memset(expected, 0, len); + KUNIT_ASSERT_EQ( + test, + ave_build_process(ctx, actual, len, + slot, 0x8f0000000ULL, + 9000000, !!frame), + 0); + sess->token = 500; + ave_reference_process(ctx, expected, len, slot, + 0x8f0000000ULL, 9000000, + !!frame); + ave_stamp_command(ave->layout->abi, actual, + 94400000, 0x12345678); + put_unaligned_le64(94400000, expected + 0x30); + put_unaligned_le64(0x12345678, expected + 0x38); + KUNIT_EXPECT_MEMEQ(test, actual, expected, len); + ave_mask_dart(actual, len); + ave_mask_dart(expected, len); + KUNIT_EXPECT_MEMEQ(test, actual, expected, len); + } + } + } +} + +static void ave_6070_images(struct kunit *test) +{ + const struct ave_fw_abi *abi = &ave_abi_6070; + struct ave_dev *ave = kunit_kzalloc(test, sizeof(*ave), GFP_KERNEL); + struct ave_ctx *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + struct ave_session *sess = + kunit_kzalloc(test, sizeof(*sess), GFP_KERNEL); + u8 actual[0x70] = {}, expected[0x70] = {}; + + KUNIT_ASSERT_NOT_NULL(test, ave); + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, sess); + ave->layout = &ave_layout_6070_11_1; + ave->extra0_dma = 0x876540000ULL; + ctx->ave = ave; + ctx->sess = sess; + abi->write_config(ave, actual); + ave_stamp_command(abi, actual, 3000000, 0x123456); + put_unaligned_le16(1, expected); + put_unaligned_le32(U32_MAX, expected + 0x1c); + put_unaligned_le64(3000000, expected + 0x28); + put_unaligned_le64(0x123456, expected + 0x30); + expected[0x41] = 1; + put_unaligned_le64(ave->extra0_dma, expected + 0x48); + KUNIT_EXPECT_MEMEQ(test, actual, expected, sizeof(actual)); + for (unsigned int codec = AVE_CODEC_H264; codec <= AVE_CODEC_HEVC; + codec++) { + static const enum ave_opcode ops[] = { AVE_OP_OPEN, + AVE_OP_CLOSE, + AVE_OP_HALT }; + static const u16 wire[] = { 2, 12, 14 }; + static const u32 semantic[] = { 3, 4, U32_MAX }; + + for (unsigned int i = 0; i < ARRAY_SIZE(ops); i++) { + memset(actual, 0, sizeof(actual)); + memset(expected, 0, sizeof(expected)); + sess->p.codec = codec; + sess->client = 10; + sess->token = 300; + if (ops[i] == AVE_OP_HALT) + ave_write_global(ave, actual, ops[i]); + else + ave_write_header(ctx, actual, ops[i], ops[i], + 0xc8, 0x1e0); + ave_stamp_command(abi, actual, 3000000, 0x123456); + put_unaligned_le16(wire[i], expected); + if (ops[i] != AVE_OP_HALT) { + put_unaligned_le64(400, expected + 0x08); + put_unaligned_le32(10, expected + 0x10); + put_unaligned_le32(codec - 1, expected + 0x18); + put_unaligned_le32(0xc8, expected + 0x20); + } + put_unaligned_le32(semantic[i], expected + 0x1c); + put_unaligned_le64(3000000, expected + 0x28); + put_unaligned_le64(0x123456, expected + 0x30); + KUNIT_EXPECT_MEMEQ(test, actual, expected, + abi->scalar_bytes[ops[i]]); + } + } + KUNIT_EXPECT_EQ(test, abi->wire_id[0][AVE_OP_PRIORITY], 0U); + KUNIT_EXPECT_EQ(test, abi->scalar_bytes[AVE_OP_RESET], 0U); + KUNIT_EXPECT_EQ(test, abi->reply_id[AVE_OP_HALT], 0U); +} + +static void ave_6070_replies(struct kunit *test) +{ + const struct ave_fw_abi *abi = &ave_abi_6070; + struct ave_reply reply; + u8 msg[0x48] = {}; + + for (unsigned int op = 1; op < AVE_OP_COUNT; op++) { + u32 len = op == AVE_OP_PROCESS ? 0x48 : 0x40; + + if (!abi->reply_id[op]) + continue; + put_unaligned_le16(abi->reply_id[op], msg); + put_unaligned_le32(10, msg + 0x10); + put_unaligned_le32(abi->semantic[op], msg + 0x1c); + put_unaligned_le32(0x00ee0000, msg + 0x38); + KUNIT_ASSERT_EQ(test, abi->parse_reply(abi, msg, len, &reply), + 0); + KUNIT_EXPECT_EQ(test, reply.op, (enum ave_opcode)op); + KUNIT_EXPECT_EQ(test, reply.client, 10ULL); + KUNIT_EXPECT_EQ(test, reply.subtype, abi->semantic[op]); + KUNIT_EXPECT_EQ(test, reply.status, 0U); + KUNIT_EXPECT_EQ(test, + abi->parse_reply(abi, msg, len - 1, &reply), + -EPROTO); + KUNIT_EXPECT_EQ(test, + abi->parse_reply(abi, msg, len + 1, &reply), + -EPROTO); + put_unaligned_le32(0, msg + 0x38); + KUNIT_ASSERT_EQ(test, abi->parse_reply(abi, msg, len, &reply), + 0); + KUNIT_EXPECT_NE(test, reply.status, 0U); + } + put_unaligned_le16(2, msg); /* Request IDs are not completion IDs. */ + KUNIT_EXPECT_EQ(test, abi->parse_reply(abi, msg, 0x40, &reply), + -EPROTO); +} + +static void ave_6070_split_surfaces(struct kunit *test) +{ + const struct ave_surface_layout bank = { + .address = 0x4b8, + .size = 0x558, + .address_stride = 8, + .size_stride = 4, + }; + u8 *cmd = kunit_kzalloc(test, 0x600, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, cmd); + for (unsigned int i = 0; i < 20; i++) + ave_write_surface(&ave_layout_6070_11_1, cmd, &bank, i, + 0x800000000ULL + i * SZ_16K, + (i + 1) * SZ_16K); + for (unsigned int i = 0; i < 20; i++) { + KUNIT_EXPECT_EQ(test, get_unaligned_le64(cmd + 0x4b8 + i * 8), + 0x800000000ULL + i * SZ_16K); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(cmd + 0x558 + i * 4), + (i + 1) * SZ_16K); + } +} + +static void ave_6070_gate(struct kunit *test) +{ + struct ave_dev *ave = kunit_kzalloc(test, sizeof(*ave), GFP_KERNEL); + struct ave_ctx *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + struct ave_session *sess = + kunit_kzalloc(test, sizeof(*sess), GFP_KERNEL); + u8 cmd[0x70], before[0x70]; + + KUNIT_ASSERT_NOT_NULL(test, ave); + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, sess); + ave->layout = &ave_layout_6070_11_1; + ctx->ave = ave; + ctx->sess = sess; + sess->p.codec = AVE_CODEC_H264; + memset(cmd, 0xa5, sizeof(cmd)); + memcpy(before, cmd, sizeof(cmd)); + KUNIT_EXPECT_PTR_EQ(test, ave_layout_for_crc(0x460c7a7c), + &ave_layout_6070_11_1); + KUNIT_EXPECT_PTR_EQ(test, ave_layout_for_crc(0x25bfd039), + &ave_layout_6070_11_1); + KUNIT_EXPECT_PTR_EQ(test, ave_layout_for_crc(0), NULL); + KUNIT_EXPECT_EQ(test, ave_build_start(ctx, cmd, sizeof(cmd)), + -EOPNOTSUPP); + KUNIT_EXPECT_EQ( + test, ave_build_process(ctx, cmd, sizeof(cmd), 0, 0, 0, false), + -EOPNOTSUPP); + KUNIT_EXPECT_MEMEQ(test, cmd, before, sizeof(cmd)); + KUNIT_EXPECT_EQ(test, sess->token, 0ULL); +} + +static void ave_6070_channels(struct kunit *test) +{ + const struct ave_fw_transport *t = ave_abi_6070.transport; + struct ave_dev *ave = kunit_kzalloc(test, sizeof(*ave), GFP_KERNEL); + u32 hello[4] = { 7, 0x9bc0, 0x100, 0 }; + u8 *desc; + + KUNIT_ASSERT_NOT_NULL(test, ave); + ave->layout = &ave_layout_6070_11_1; + ave->ipc.cpu = kunit_kzalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ave->ipc.cpu); + ave->ipc.size = t->ipc_bytes; + ave->ipc_fw_base = 0x100000000ULL; + ave->channel_bytes = hello[1]; + KUNIT_EXPECT_EQ(test, ave_ipc_check_hello(t, hello), 0); + for (unsigned int word = 0; word < 4; word++) { + hello[word]++; + KUNIT_EXPECT_EQ(test, ave_ipc_check_hello(t, hello), -EPROTO); + hello[word]--; + } + for (unsigned int i = 0; i < 7; i++) { + int id = i == 1 ? 0 : 1; + bool io = i == 1 || i == 5; + u64 address = ave->ipc_fw_base + AVE_IPC_CHANNEL_OFF + + (io ? t->offsets[id] : 0x1000 + i * 0x400); + + desc = ave->ipc.cpu + AVE_IPC_CHANNEL_OFF + i * 0x100; + strscpy(desc, io ? (id ? "IO_T2H" : "IO") : "UNUSED", 0x40); + put_unaligned_le32(io ? id : U32_MAX, desc + 0x40); + put_unaligned_le32(io ? t->doorbells[id] : U32_MAX, + desc + 0x44); + put_unaligned_le32(io ? t->slots[id] : 4, desc + 0x48); + put_unaligned_le64(address, desc + 0x4c); + } + KUNIT_ASSERT_EQ(test, ave_ipc_parse_channels(ave), 0); + KUNIT_EXPECT_EQ(test, ave->ring[0].slots, 32U); + KUNIT_EXPECT_EQ(test, ave->ring[1].slots, 8U); + KUNIT_EXPECT_EQ(test, ave->ring[0].doorbell, 1U); + KUNIT_EXPECT_EQ(test, ave->ring[1].doorbell, 3U); + desc = ave->ipc.cpu + AVE_IPC_CHANNEL_OFF + 0x100; + put_unaligned_le32(33, desc + 0x48); + KUNIT_EXPECT_EQ(test, ave_ipc_parse_channels(ave), -EPROTO); + put_unaligned_le32(32, desc + 0x48); + desc = ave->ipc.cpu + AVE_IPC_CHANNEL_OFF; + put_unaligned_le64(ave->ipc_fw_base + ave->ipc.size, desc + 0x4c); + KUNIT_EXPECT_EQ(test, ave_ipc_parse_channels(ave), -EPROTO); +} + +static void ave_6070_variants(struct kunit *test) +{ + const struct ave_fw_transport *t = ave_abi_6070.transport; + + KUNIT_EXPECT_EQ(test, ave_t6000.boot_device_id, 11U); + KUNIT_EXPECT_EQ(test, ave_t6000.device_type, 9U); + KUNIT_EXPECT_EQ(test, ave_t6000.boot_arg_value, 0x400U); + KUNIT_EXPECT_EQ(test, ave_t6000.timebase_multiplier, 1000000U); + KUNIT_EXPECT_EQ(test, ave_t6000.text_crc, 0x460c7a7cU); + KUNIT_EXPECT_EQ(test, ave_t8112.boot_device_id, 15U); + KUNIT_EXPECT_EQ(test, ave_t8112.device_type, 13U); + KUNIT_EXPECT_EQ(test, ave_t8112.boot_arg_value, 0x10000U); + KUNIT_EXPECT_EQ(test, ave_t8112.timebase_multiplier, 1U); + KUNIT_EXPECT_EQ(test, ave_t8112.text_crc, 0x25bfd039U); + KUNIT_EXPECT_EQ(test, ave_fw_vcbar(&ave_t6000, 0x10000b28000ULL), + 0x0102000000b28000ULL); + KUNIT_EXPECT_EQ(test, ave_fw_vcbar(&ave_t8112, 0x800000000ULL), + 0x0102000800000000ULL); + KUNIT_EXPECT_EQ(test, ave_t6000.regs->ctrl, 0x400044U); + KUNIT_EXPECT_EQ(test, ave_t8112.regs->timebase, 0x178000U); + KUNIT_EXPECT_GE(test, t->command_bytes, 0x32dc8U); + KUNIT_EXPECT_LE(test, t->command_offset + t->command_bytes, + t->scalar_offset); + KUNIT_EXPECT_LE(test, + t->scalar_offset + + AVE_IPC_SCALAR_BUFS * AVE_IPC_SCALAR_BYTES, + t->process_offset); + KUNIT_EXPECT_LE( + test, t->process_offset + AVE_IPC_PROC_BUFS * t->process_bytes, + t->ipc_bytes); +} + +static struct kunit_case ave_abi_cases[] = { + KUNIT_CASE(ave_9003_byte_equality), KUNIT_CASE(ave_6070_images), + KUNIT_CASE(ave_6070_replies), KUNIT_CASE(ave_6070_split_surfaces), + KUNIT_CASE(ave_6070_gate), KUNIT_CASE(ave_6070_channels), + KUNIT_CASE(ave_6070_variants), {} +}; + +static struct kunit_suite ave_abi_suite = { + .name = "apple-ave-abi", + .test_cases = ave_abi_cases, +}; +kunit_test_suite(ave_abi_suite); + +MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/ave/ave-abi.c b/drivers/media/platform/apple/ave/ave-abi.c new file mode 100644 index 0000000000000..5aca710f0e854 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-abi.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Firmware command envelopes and transport descriptions. */ +#include +#include +#include + +#include "ave.h" +#include "ave-regs.h" + +static const struct ave_fw_transport ave_transport_9003 = { + .ipc_bytes = AVE_IPC_BYTES, + .command_offset = AVE_IPC_CMD_OFF, + .command_bytes = AVE_IPC_CMD_BYTES, + .scalar_offset = AVE_IPC_SCALAR_OFF, + .process_offset = AVE_IPC_PROC_OFF, + .process_bytes = AVE_IPC_PROC_BYTES, + .hello = { 2, 0, 0x100, 0 }, + .descriptors = 2, + .io_index = 0, + .t2h_index = 1, + .boot_config = true, + .sync_host_clock = true, +}; + +static const struct ave_fw_transport ave_transport_6070 = { + .ipc_bytes = 0x700000, + .command_offset = 0x40000, + .command_bytes = 0x34000, + .scalar_offset = 0x74000, + .process_offset = 0x78000, + .process_bytes = 0x10000, + .hello = { 7, 0x9bc0, 0x100, 0 }, + .descriptors = 7, + .io_index = 1, + .t2h_index = 5, + .slots = { 32, 8 }, + .doorbells = { 1, 3 }, + .offsets = { 0x8700, 0x9180 }, + .scratch_halt = true, +}; + +static void ave_header_9003(u8 *cmd, const struct ave_fw_abi *abi, + const struct ave_command *h) +{ + put_unaligned_le32(abi->wire_id[h->codec - 1][h->op], cmd); + put_unaligned_le64(h->token, cmd + 0x08); + put_unaligned_le64(h->client, cmd + 0x10); + if (h->op != AVE_OP_CONFIG && h->op != AVE_OP_HALT) { + put_unaligned_le32(1, cmd + 0x18); + put_unaligned_le32(h->codec, cmd + 0x1c); + put_unaligned_le32(h->arg0, cmd + 0x24); + put_unaligned_le32(0x36, cmd + 0x28); + put_unaligned_le32(h->arg2, cmd + 0x2c); + } + put_unaligned_le32(h->subtype, cmd + 0x20); +} + +static void ave_header_6070(u8 *cmd, const struct ave_fw_abi *abi, + const struct ave_command *h) +{ + bool global = h->op == AVE_OP_CONFIG || h->op == AVE_OP_HALT; + + put_unaligned_le16(abi->wire_id[h->codec - 1][h->op], cmd); + put_unaligned_le64(h->token, cmd + 0x08); + if (!global) { + put_unaligned_le32(h->client, cmd + 0x10); + put_unaligned_le32(h->codec - 1, cmd + 0x18); + } + put_unaligned_le32(h->op == AVE_OP_PROCESS ? h->subtype : + abi->semantic[h->op], + cmd + 0x1c); + put_unaligned_le32(global ? 0 : 0xc8, cmd + 0x20); +} + +static void ave_config_9003(struct ave_dev *ave, u8 *cmd) +{ + ave_write_global(ave, cmd, AVE_OP_CONFIG); + cmd[0x49] = 1; + cmd[0x4a] = 1; + put_unaligned_le32(ave->layout->mcc_dsid, cmd + 0x60); +} + +static void ave_config_6070(struct ave_dev *ave, u8 *cmd) +{ + ave_write_global(ave, cmd, AVE_OP_CONFIG); + cmd[0x41] = 1; + put_unaligned_le64(ave->extra0_dma, cmd + 0x48); + /* No MCC, host heap, shared area or optional firmware log. */ +} + +static int ave_reply_9003(const struct ave_fw_abi *abi, const u8 *msg, u32 len, + struct ave_reply *reply) +{ + u32 id; + + if (len < 0x48) + return -EPROTO; + id = get_unaligned_le16(msg); + reply->op = id < AVE_OP_COUNT ? id : 0; + reply->client = get_unaligned_le64(msg + 0x10); + reply->subtype = get_unaligned_le32(msg + 0x20); + reply->raw_status = get_unaligned_le32(msg + 0x40); + reply->status = reply->raw_status; + reply->bytes = len; + return 0; +} + +static int ave_reply_6070(const struct ave_fw_abi *abi, const u8 *msg, u32 len, + struct ave_reply *reply) +{ + u32 id; + + if (len < 0x40) + return -EPROTO; + id = get_unaligned_le16(msg); + for (unsigned int op = 1; op < AVE_OP_COUNT; op++) { + if (!abi->reply_id[op] || abi->reply_id[op] != id) + continue; + if (len != (op == AVE_OP_PROCESS ? 0x48 : 0x40)) + return -EPROTO; + reply->op = op; + reply->client = get_unaligned_le32(msg + 0x10); + reply->subtype = get_unaligned_le32(msg + 0x1c); + reply->raw_status = get_unaligned_le32(msg + 0x38); + reply->status = reply->raw_status; + reply->bytes = len; + if (reply->status == 0x00ee0000) + reply->status = 0; + else if (!reply->status) + reply->status = U32_MAX; + return 0; + } + return -EPROTO; +} + +#define AVE_9003_IDS \ + { [AVE_OP_CONFIG] = 1, [AVE_OP_HALT] = 2, [AVE_OP_OPEN] = 3, \ + [AVE_OP_CLOSE] = 4, [AVE_OP_START] = 6, [AVE_OP_STOP] = 7, \ + [AVE_OP_PROCESS] = 8, [AVE_OP_COMPLETE] = 9, [AVE_OP_PRIORITY] = 10 } + +const struct ave_fw_abi ave_abi_9003 = { + .transport = &ave_transport_9003, + .wire_id = { AVE_9003_IDS, AVE_9003_IDS }, + .reply_id = AVE_9003_IDS, + .scalar_bytes = { + [AVE_OP_CONFIG] = 0x78, + [AVE_OP_HALT] = 0x48, + [AVE_OP_OPEN] = 0x48, + [AVE_OP_CLOSE] = 0x48, + [AVE_OP_STOP] = 0x48, + [AVE_OP_COMPLETE] = 0x48, + [AVE_OP_PRIORITY] = 0x48, + }, + .header_bytes = 0x48, + .timeout_offset = 0x30, + .created_offset = 0x38, + .padded_surfaces = true, + .write_header = ave_header_9003, + .write_config = ave_config_9003, + .parse_reply = ave_reply_9003, +}; + +#define AVE_6070_IDS(start, process) \ + { [AVE_OP_CONFIG] = 1, [AVE_OP_OPEN] = 2, \ + [AVE_OP_RESET] = 3, [AVE_OP_START] = (start), \ + [AVE_OP_STOP] = 6, [AVE_OP_PROCESS] = (process), \ + [AVE_OP_FLUSH] = 11, [AVE_OP_CLOSE] = 12, \ + [AVE_OP_COMPLETE] = 13, [AVE_OP_HALT] = 14 } + +const struct ave_fw_abi ave_abi_6070 = { + .transport = &ave_transport_6070, + .wire_id = { AVE_6070_IDS(4, 7), AVE_6070_IDS(5, 8) }, + .reply_id = { + [AVE_OP_CONFIG] = 0xe01, + [AVE_OP_OPEN] = 0xe02, + [AVE_OP_RESET] = 0xe03, + [AVE_OP_START] = 0xe04, + [AVE_OP_STOP] = 0xe05, + [AVE_OP_PROCESS] = 0xe06, + [AVE_OP_FLUSH] = 0xe09, + [AVE_OP_CLOSE] = 0xe0a, + [AVE_OP_COMPLETE] = 0xe0b, + }, + .scalar_bytes = { + [AVE_OP_CONFIG] = 0x70, + [AVE_OP_OPEN] = 0x40, + [AVE_OP_STOP] = 0x40, + [AVE_OP_FLUSH] = 0x40, + [AVE_OP_CLOSE] = 0x48, + [AVE_OP_COMPLETE] = 0x40, + [AVE_OP_HALT] = 0x40, + }, + .semantic = { + [AVE_OP_CONFIG] = U32_MAX, + [AVE_OP_OPEN] = 3, + [AVE_OP_RESET] = 11, + [AVE_OP_START] = 6, + [AVE_OP_STOP] = 7, + [AVE_OP_FLUSH] = 10, + [AVE_OP_CLOSE] = 4, + [AVE_OP_COMPLETE] = 9, + [AVE_OP_HALT] = U32_MAX, + }, + .header_bytes = 0x40, + .timeout_offset = 0x28, + .created_offset = 0x30, + .write_header = ave_header_6070, + .write_config = ave_config_6070, + .parse_reply = ave_reply_6070, +}; diff --git a/drivers/media/platform/apple/ave/ave-cmd.c b/drivers/media/platform/apple/ave/ave-cmd.c new file mode 100644 index 0000000000000..a7fb9672a1490 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-cmd.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder: firmware commands + * + * Start and Process are large command images. Each is built in a zeroed + * buffer: the header, the constant words (ave-image.c), then every field + * that depends on the session or the frame. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include +#include +#include + +#include + +#include "ave.h" +#include "ave-regs.h" + +/* Firmware-wide configuration, sent once after every firmware boot. */ +int ave_cmd_config(struct ave_dev *ave) +{ + int buf = ave_ipc_scalar_buf(ave); + u8 *cmd; + int ret; + + if (buf < 0) + return buf; + cmd = ave_ipc_cmd(ave, buf, + ave->layout->abi->scalar_bytes[AVE_OP_CONFIG]); + if (!cmd) + return -EBUSY; + ave->layout->abi->write_config(ave, cmd); + ret = ave_ipc_submit(ave, buf, + ave->layout->abi->scalar_bytes[AVE_OP_CONFIG], + AVE_CMD_GLOBAL_TIMEOUT_US, true); + return ret ?: ave_ipc_settle(ave, AVE_CMD_WAIT_MS); +} + +/* + * Firmware-wide stop, sent after every session has closed. The firmware + * neither acknowledges nor answers it. + */ +int ave_cmd_halt(struct ave_dev *ave) +{ + int buf = ave_ipc_scalar_buf(ave); + u8 *cmd; + + if (buf < 0) + return buf; + cmd = ave_ipc_cmd(ave, buf, ave->layout->abi->header_bytes); + if (!cmd) + return -EBUSY; + ave_write_global(ave, cmd, AVE_OP_HALT); + return ave_ipc_submit(ave, buf, ave->layout->abi->header_bytes, + AVE_CMD_GLOBAL_TIMEOUT_US, false); +} + +static int ave_cmd_scalar(struct ave_ctx *ctx, enum ave_opcode op, u32 arg0, + u32 arg2) +{ + struct ave_dev *ave = ctx->ave; + int buf = ave_ipc_scalar_buf(ave); + u8 *cmd; + + if (buf < 0) + return buf; + if (!ave->layout->abi->scalar_bytes[op]) + return -EOPNOTSUPP; + cmd = ave_ipc_cmd(ave, buf, ave->layout->abi->scalar_bytes[op]); + if (!cmd) + return -EBUSY; + ave_write_header(ctx, cmd, op, op, arg0, arg2); + return ave_ipc_submit(ave, buf, ave->layout->abi->scalar_bytes[op], + ave->layout->abi->transport->boot_config ? + AVE_CMD_SESSION_TIMEOUT_US : + AVE_CMD_GLOBAL_TIMEOUT_US, + true); +} + +int ave_cmd_open(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + int ret; + + ave->next_client += AVE_CLIENT_STEP; + ctx->sess->client = ave->next_client; + ctx->sess->token = AVE_TOKEN_BASE; + ret = ave_cmd_scalar(ctx, AVE_OP_OPEN, AVE_SESSION_ARG0, + ave_session_setup_arg2(ctx)); + if (ave->layout->abi->transport->boot_config) { + if (!ret) + ctx->sess->fw_open = true; + return ret ?: ave_ipc_settle(ave, AVE_CMD_WAIT_MS); + } + ret = ret ?: ave_ipc_settle(ave, AVE_CMD_WAIT_MS); + if (!ret) + ctx->sess->fw_open = true; + return ret; +} + +/* Drain barrier: answered once every Process sent before it is answered. */ +int ave_cmd_complete(struct ave_ctx *ctx, unsigned int count) +{ + int ret = 0; + + for (unsigned int i = 0; i < count && !ret; i++) + ret = ave_cmd_scalar(ctx, AVE_OP_COMPLETE, 0, ave_session_arg2(ctx)); + return ret; +} + +/* Priority and Stop go out together, Close after Stop is answered. */ +int ave_cmd_close(struct ave_ctx *ctx) +{ + u32 arg2 = ave_session_arg2(ctx); + bool legacy = ctx->ave->layout->abi->transport->boot_config; + int ret = 0, err; + + if (legacy) + ret = ave_cmd_scalar(ctx, AVE_OP_PRIORITY, AVE_SESSION_ARG0, + arg2); + if (!ret && (legacy || ctx->sess->handed)) + ret = ave_cmd_scalar(ctx, AVE_OP_STOP, AVE_SESSION_ARG0, arg2); + err = ave_ipc_settle(ctx->ave, AVE_CMD_WAIT_MS); + ret = ret ?: err; + if (ctx->ave->fw_state != AVE_FW_RUNNING) + return ret; + /* A failed 6070 Stop cannot establish a safe Close/Halt boundary. */ + if (ret && !legacy) + return ret; + err = ave_cmd_scalar(ctx, AVE_OP_CLOSE, AVE_SESSION_ARG0, arg2); + if (!err) + err = ave_ipc_settle(ctx->ave, AVE_CMD_WAIT_MS); + if (legacy || !err) + ctx->sess->fw_open = false; + return ret ?: err; +} + +int ave_cmd_start(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + const struct ave_params *p = &sess->p; + const struct ave_fw_layout *fw = ave->layout; + size_t len = fw->bytes[p->codec - 1][AVE_IMAGE_START]; + u8 *cmd; + int ret; + + if (!fw->encode_ready) + return -EOPNOTSUPP; + cmd = ave_ipc_cmd(ave, AVE_BUF_CMD, len); + if (!cmd) + return -EBUSY; + ret = ave_build_start(ctx, cmd, len); + if (ret) + return ret; + ave_debugfs_save(ave, p->codec, AVE_IMAGE_START, cmd, len); + + /* A failed submission may already have exposed the Start buffers. */ + sess->handed = true; + ret = ave_ipc_submit(ave, AVE_BUF_CMD, len, AVE_CMD_SESSION_TIMEOUT_US, + true); + if (ret) + return ret; + return ave_ipc_settle(ave, AVE_START_WAIT_MS); +} + +int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, + struct vb2_v4l2_buffer *src, dma_addr_t input, u32 bitrate, + bool force_key) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + const struct ave_params *p = &sess->p; + const struct ave_fw_layout *fw = ave->layout; + size_t len = fw->bytes[p->codec - 1][AVE_IMAGE_PROCESS]; + struct ave_slot *s; + unsigned int buf = AVE_BUF_PROC(sess->index * AVE_MAX_SLOTS + slot); + u8 *cmd; + int ret; + + if (!fw->encode_ready) + return -EOPNOTSUPP; + + if (slot >= p->slots || p->slots > fw->max_slots) + return -EINVAL; + s = &sess->slot[slot]; + ret = ave_ipc_wait_ack(ave, buf); + if (ret) + return ret; + cmd = ave_ipc_cmd(ave, buf, len); + if (!cmd) + return -EBUSY; + ret = ave_build_process(ctx, cmd, len, slot, input, bitrate, force_key); + if (ret) + return ret; + if (!sess->frames) + ave_debugfs_save(ave, p->codec, AVE_IMAGE_PROCESS, cmd, len); + + /* Pin before publishing the command, including an uncertain send failure. */ + s->input_mem = ave_vb2_get(&src->vb2_buf); + ret = ave_ipc_submit(ave, buf, len, fw->process_timeout_us, false); + if (ret) + return ret; + s->src = src; + s->busy = true; + s->status = 0; + sess->progress = jiffies; + sess->last_sent = jiffies; + sess->nudged = false; + sess->busy++; + sess->frames++; + return 0; +} diff --git a/drivers/media/platform/apple/ave/ave-drv.c b/drivers/media/platform/apple/ave/ave-drv.c new file mode 100644 index 0000000000000..d96267c00bef3 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-drv.c @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder + * + * The AVE block runs an encoder firmware on its own coprocessor, which the + * bootloader loads but does not start. This driver starts it on first use, + * talks to it over two shared-memory rings and exposes it as a V4L2 + * stateful encoder. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ave.h" +#include "ave-regs.h" + +#define AVE_AUTOSUSPEND_MS 2000 + +static const struct ave_coproc_regs ave_coproc_v40 = { + .vcbar = AVE_COPROC_VCBAR, + .timebase_freq = AVE_COPROC_TIMEBASE_FREQ, + .timebase = AVE_COPROC_TIMEBASE, + .ctrl = AVE_COPROC_CTRL, + .status = AVE_COPROC_STATUS, + .boot_arg = AVE_COPROC_BOOT_ARG, + .keep_mappings = AVE_COPROC_KEEP_MAPPINGS, +}; + +/* Panda/Castor register family used by 6070.11.1. */ +static const struct ave_coproc_regs ave_coproc_6070 = { + .vcbar = 0x50000, + .timebase_freq = 0x160020, + .timebase = 0x178000, + .ctrl = 0x400044, + .status = 0x400048, + .boot_arg = 0x400400, + .keep_mappings = 0x400808, +}; + +const struct ave_variant ave_t6000 = { + .regs = &ave_coproc_6070, + .fw_text_size = 0xd0000, + .fw_data_size = 0x128000, + .boot_device_id = 11, + .device_type = 9, + .lfs_ref_scale = 1, + .text_crc = 0x460c7a7c, + .boot_arg_value = 0x400, + .timebase_multiplier = 1000000, + .vcbar_mask = 0x0000000ffffff800ULL, + .vcbar_flags = 0x0102000000000000ULL, + + .extra0_size = 0x24, + .boot_qualified = true, +}; + +const struct ave_variant ave_t8112 = { + .regs = &ave_coproc_6070, + .fw_text_size = 0xd0000, + .fw_data_size = 0x128000, + .boot_device_id = 15, + .device_type = 13, + .lfs_ref_scale = 1, + .text_crc = 0x25bfd039, + .boot_arg_value = 0x10000, + .timebase_multiplier = 1, + .vcbar_mask = 0x000003fffffff800ULL, + .vcbar_flags = 0x0102000000000000ULL, + + .extra0_size = 0x4000, + .boot_qualified = true, +}; + +/* + * The M4 Pro (T6040 in the ADT, apple,t6041 in the DT) starts its encoder + * like the A18 Pro: same coprocessor registers and boot sequence, its own + * device ID and type. + */ +static const struct ave_variant ave_t6041 = { + .regs = &ave_coproc_v40, + .fw_text_size = 0x14c000, + .fw_data_size = 0x12c000, + .boot_device_id = 27, + .boot_arg_value = AVE_COPROC_BOOT_ARG_VALUE, + .timebase_multiplier = 1, + .vcbar_mask = U64_MAX, + .vcbar_flags = AVE_COPROC_VCBAR_VALID, + .device_type = 25, + .lfs_ref_scale = 2, + .boot_qualified = true, +}; + +static const struct ave_variant ave_t8140 = { + .regs = &ave_coproc_v40, + .fw_text_size = 0x14c000, + .fw_data_size = 0x130000, + .boot_device_id = 29, + .boot_arg_value = AVE_COPROC_BOOT_ARG_VALUE, + .timebase_multiplier = 1, + .vcbar_mask = U64_MAX, + .vcbar_flags = AVE_COPROC_VCBAR_VALID, + .device_type = 27, + .lfs_ref_scale = 2, + .boot_qualified = true, +}; + +/* + * debugfs: apple-ave/last_start_{h264,hevc} hold the last Start built and + * last_process0_{h264,hevc} the first Process of the last session, so the + * generated commands can be compared with reference captures. + */ +void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, + enum ave_image kind, const void *cmd, size_t len) +{ + void *dst = ave->dump[codec - 1][kind]; + + if (ave->layout && dst && len == ave->layout->bytes[codec - 1][kind]) + memcpy(dst, cmd, len); +} + +static ssize_t ave_lifecycle_write(struct file *file, const char __user *user, + size_t count, loff_t *ppos) +{ + struct ave_dev *ave = file->private_data; + unsigned int codec; + int ret = kstrtouint_from_user(user, count, 0, &codec); + + if (ret) + return ret; + if (codec != AVE_CODEC_H264 && codec != AVE_CODEC_HEVC) + return -EINVAL; + ret = ave_fw_lifecycle_test(ave, codec); + return ret ?: count; +} + +static const struct file_operations ave_lifecycle_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = ave_lifecycle_write, +}; + +/* Root-only software timeout injection; does not send malformed firmware commands. */ +static int ave_inject_wedge(void *data, u64 value) +{ + struct ave_dev *ave = data; + int ret = 0; + + if (value != 1) + return -EINVAL; + mutex_lock(&ave->hw_lock); + if (ave->fw_state != AVE_FW_RUNNING) { + ret = -EBUSY; + } else { + ave->inject_wedge = true; + ave_fw_mark_broken(ave, -ETIMEDOUT, "debugfs injection"); + } + mutex_unlock(&ave->hw_lock); + return ret; +} +DEFINE_DEBUGFS_ATTRIBUTE(ave_inject_wedge_fops, NULL, ave_inject_wedge, "%llu\n"); + +static void ave_debugfs_init(struct ave_dev *ave) +{ + static const char *const names[2][AVE_IMAGE_COUNT] = { + { "last_start_h264", "last_process0_h264" }, + { "last_start_hevc", "last_process0_hevc" }, + }; + + if (!IS_ENABLED(CONFIG_DEBUG_FS) || !ave->layout) + return; + ave->debugfs = debugfs_create_dir("apple-ave", NULL); + if (ave->layout->abi == &ave_abi_6070) { + debugfs_create_file("lifecycle_test", 0200, ave->debugfs, ave, + &ave_lifecycle_fops); + return; + } + debugfs_create_file("inject_wedge", 0200, ave->debugfs, ave, + &ave_inject_wedge_fops); + for (unsigned int c = 0; c < 2; c++) + for (unsigned int k = 0; k < AVE_IMAGE_COUNT; k++) { + struct debugfs_blob_wrapper *blob; + size_t size = ave->layout->bytes[c][k]; + + blob = devm_kzalloc(ave->dev, sizeof(*blob), + GFP_KERNEL); + ave->dump[c][k] = + devm_kzalloc(ave->dev, size, GFP_KERNEL); + if (!blob || !ave->dump[c][k]) { + ave->dump[c][k] = NULL; + continue; + } + blob->data = ave->dump[c][k]; + blob->size = ave->layout->bytes[c][k]; + debugfs_create_blob(names[c][k], 0400, ave->debugfs, + blob); + } +} + +static const char *const ave_pd_names[] = { "dma", "pipe4", "pipe5", "me0", + "me1" }; + +static int ave_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct dev_pm_domain_attach_data pd_data = { + .pd_names = ave_pd_names, + .num_pd_names = ARRAY_SIZE(ave_pd_names), + .pd_flags = PD_FLAG_DEV_LINK_ON, + }; + struct ave_dev *ave; + int ret; + + ave = devm_kzalloc(dev, sizeof(*ave), GFP_KERNEL); + if (!ave) + return -ENOMEM; + ave->dev = dev; + ave->variant = of_device_get_match_data(dev); + if (!ave->variant) + return -ENODEV; + mutex_init(&ave->hw_lock); + platform_set_drvdata(pdev, ave); + + ave->codec = devm_platform_ioremap_resource_byname(pdev, "codec"); + if (IS_ERR(ave->codec)) + return PTR_ERR(ave->codec); + ave->mbox = devm_platform_ioremap_resource_byname(pdev, "mailbox"); + if (IS_ERR(ave->mbox)) + return PTR_ERR(ave->mbox); + ave->coproc = devm_platform_ioremap_resource_byname(pdev, "coproc"); + if (IS_ERR(ave->coproc)) + return PTR_ERR(ave->coproc); + + ave->domain = iommu_get_domain_for_dev(dev); + if (!ave->domain || !(ave->domain->type & __IOMMU_DOMAIN_PAGING)) + return dev_err_probe(dev, -ENODEV, "needs a translating IOMMU\n"); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + if (ret) + return ret; + + /* The encoder pipes, the DMA block and the motion engines. */ + ret = devm_pm_domain_attach_list(dev, &pd_data, &ave->pd_list); + if (ret < 0) + return dev_err_probe(dev, ret, "cannot attach power domains\n"); + + ret = ave_fw_probe(ave); + if (ret) + return ret; + + /* + * Domain attachment took active supplier references, but the driver + * starts firmware only on first use and begins suspended. Reconcile that + * state before enabling runtime PM, otherwise those references survive + * until the first session has resumed and suspended the encoder. + */ + ret = pm_runtime_set_suspended(dev); + if (ret) + goto err_fw; + + pm_runtime_set_autosuspend_delay(dev, AVE_AUTOSUSPEND_MS); + pm_runtime_use_autosuspend(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + goto err_pm; + + ret = ave_v4l2_register(ave); + if (ret) + goto err_pm; + ave_debugfs_init(ave); + return 0; + +err_pm: + pm_runtime_dont_use_autosuspend(dev); +err_fw: + ave_fw_remove(ave); + return ret; +} + +static void ave_remove(struct platform_device *pdev) +{ + struct ave_dev *ave = platform_get_drvdata(pdev); + + debugfs_remove_recursive(ave->debugfs); + ave_v4l2_unregister(ave); + + /* + * The coprocessor memory can only go once the firmware never ran or + * was reset by a power cycle. A halted coprocessor may keep power + * (its domain is shared with the DARTs), so its memory stays. + */ + pm_runtime_dont_use_autosuspend(ave->dev); + mutex_lock(&ave->hw_lock); + if (ave->fw_state != AVE_FW_OFF) { + mutex_unlock(&ave->hw_lock); + dev_warn(ave->dev, + "firmware was started, keeping its memory\n"); + return; + } + mutex_unlock(&ave->hw_lock); + ave_fw_remove(ave); +} + +static int ave_runtime_suspend(struct device *dev) +{ + struct ave_dev *ave = dev_get_drvdata(dev); + int ret; + + mutex_lock(&ave->hw_lock); + ret = ave_fw_runtime_suspend(ave); + if (!ret && ave->layout && ave->layout->abi->transport->scratch_halt) + ave_fw_release_boot(ave); + mutex_unlock(&ave->hw_lock); + return ret; +} + +static int ave_runtime_resume(struct device *dev) +{ + struct ave_dev *ave = dev_get_drvdata(dev); + + mutex_lock(&ave->hw_lock); + if (ave->layout && ave->layout->abi->transport->scratch_halt && + ave->fw_state == AVE_FW_HALTED) + ave_fw_lost(ave); + mutex_unlock(&ave->hw_lock); + return 0; +} + +static int ave_suspend(struct device *dev) +{ + struct ave_dev *ave = dev_get_drvdata(dev); + int ret; + + /* Let a running job finish; the m2m core holds back new ones. */ + v4l2_m2m_suspend(ave->m2m_dev); + + /* + * The firmware does not survive the power going down. Drain and close + * every open session and halt the firmware, as at the end of a stream; + * each session reopens in a cold-started firmware on its next frame. + */ + mutex_lock(&ave->hw_lock); + ave_v4l2_suspend(ave); + ret = ave_fw_shutdown(ave); + if (!ret && ave->layout && ave->layout->abi->transport->scratch_halt) + ave_fw_release_boot(ave); + mutex_unlock(&ave->hw_lock); + if (ret) { + /* + * genpd may gate the encoder and its DARTs after this callback. + * A stuck firmware may still DMA, so abort before that happens. + * PM does not resume a device whose suspend callback failed. + */ + dev_err(dev, "firmware cannot be stopped for sleep: %d\n", ret); + v4l2_m2m_resume(ave->m2m_dev); + } + return ret; +} + +static int ave_resume(struct device *dev) +{ + struct ave_dev *ave = dev_get_drvdata(dev); + + /* The next boot checks the coprocessor state itself. */ + ave_runtime_resume(dev); + v4l2_m2m_resume(ave->m2m_dev); + return 0; +} + +static const struct dev_pm_ops ave_pm_ops = { + SYSTEM_SLEEP_PM_OPS(ave_suspend, ave_resume) + RUNTIME_PM_OPS(ave_runtime_suspend, ave_runtime_resume, NULL) +}; + +static const struct of_device_id ave_of_match[] = { + { .compatible = "apple,t6000-ave", .data = &ave_t6000 }, + { .compatible = "apple,t6041-ave", .data = &ave_t6041 }, + { .compatible = "apple,t8112-ave", .data = &ave_t8112 }, + { .compatible = "apple,t8140-ave", .data = &ave_t8140 }, + {} +}; +MODULE_DEVICE_TABLE(of, ave_of_match); + +static struct platform_driver ave_driver = { + .probe = ave_probe, + .remove = ave_remove, + .driver = { + .name = "apple-ave", + .of_match_table = ave_of_match, + .pm = pm_ptr(&ave_pm_ops), + /* A started coprocessor keeps its memory; do not unbind at will. */ + .suppress_bind_attrs = true, + }, +}; +module_platform_driver(ave_driver); + +MODULE_DESCRIPTION("Apple AVE video encoder"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("DMA_BUF"); diff --git a/drivers/media/platform/apple/ave/ave-fw.c b/drivers/media/platform/apple/ave/ave-fw.c new file mode 100644 index 0000000000000..4311e8874a6c1 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-fw.c @@ -0,0 +1,822 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder: firmware lifecycle + * + * The bootloader leaves the coprocessor firmware in two reserved, DART-mapped + * segments: TEXT and DATA. Running the firmware changes DATA, so the driver + * keeps a copy of DATA taken before the first boot and puts it back before + * every later boot. That lets the coprocessor lose power (runtime PM, system + * sleep) and start again from the same state as the first time. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ave.h" +#include "ave-regs.h" + +/* + * Legacy TEXT CRCs accepted at T8140 probe. Encoding separately requires + * a layout from ave_layout_for_crc(); command formats change between builds. + */ +static const u32 ave_known_text_crc[] = { + 0x2ce7359e, /* AppleAVE2FW 9003.78.0 */ + 0x69c54454, /* AppleAVE2FW 9003.78.0 (macOS 26.0.x) */ + AVE_FW_TEXT_CRC_T6040_25G83, /* AppleAVE2FW 9003.78.0 / T6040 */ + 0x460c7a7c, /* AppleAVE2FW 6070.11.1 / T6000 */ + 0x25bfd039, /* AppleAVE2FW 6070.11.1 / T8112 */ +}; + +static int ave_fw_chosen_crc(struct ave_dev *ave, const char *name, u32 *crc) +{ + struct device_node *chosen = of_find_node_by_path("/chosen"); + const char *str; + int ret; + + if (!chosen) + return -ENOENT; + ret = of_property_read_string(chosen, name, &str); + of_node_put(chosen); + if (ret) + return ret; + return kstrtou32(str, 16, crc); +} + +static int ave_fw_segment(struct ave_dev *ave, unsigned int index, + struct ave_fw_segment *seg) +{ + struct device_node *np; + struct resource res; + const __be32 *prop; + int len, ret; + + np = of_parse_phandle(ave->dev->of_node, "memory-region", index); + if (!np) + return -ENOENT; + if (!of_node_name_eq(np->parent, "reserved-memory") || + !of_device_is_compatible(np, "apple,asc-mem") || + !of_property_read_bool(np, "no-map")) { + ret = -EINVAL; + goto out; + } + ret = of_address_to_resource(np, 0, &res); + if (ret) + goto out; + + /* iommu-addresses = <&ave iova-hi iova-lo size-hi size-lo> */ + prop = of_get_property(np, "iommu-addresses", &len); + if (!prop || len != 5 * sizeof(u32) || + be32_to_cpup(prop) != ave->dev->of_node->phandle) { + ret = -EINVAL; + goto out; + } + seg->phys = res.start; + seg->size = resource_size(&res); + seg->iova = of_read_number(prop + 1, 2); + if (!seg->size || seg->phys > PHYS_ADDR_MAX - seg->size || + seg->iova > U64_MAX - seg->size || + of_read_number(prop + 3, 2) != seg->size || + !IS_ALIGNED(seg->phys, SZ_16K) || !IS_ALIGNED(seg->iova, SZ_16K)) + ret = -EINVAL; +out: + of_node_put(np); + return ret; +} + +static int ave_fw_check_mapping(struct ave_dev *ave, + const struct ave_fw_segment *seg) +{ + for (size_t off = 0; off < seg->size; off += SZ_16K) + if (iommu_iova_to_phys(ave->domain, seg->iova + off) != + seg->phys + off) + return -EIO; + return 0; +} + +static int ave_fw_crc(const struct ave_fw_segment *seg, u32 *crc, void *copy) +{ + void *va = memremap(seg->phys, seg->size, MEMREMAP_WB); + + if (!va) + return -ENOMEM; + *crc = crc32_le(~0U, va, seg->size) ^ ~0U; + if (copy) + memcpy(copy, va, seg->size); + memunmap(va); + return 0; +} + +/* + * Find the firmware, check that it is a build we speak to and that DATA is + * still as the bootloader left it, and keep a copy of DATA for restarts. + */ +int ave_fw_probe(struct ave_dev *ave) +{ + struct device *dev = ave->dev; + u32 text_crc, data_crc, want; + bool known = !ave->variant->boot_qualified; + int ret; + + if (of_count_phandle_with_args(dev->of_node, "memory-region", NULL) != + 2) + return dev_err_probe(dev, -ENODEV, "no preloaded firmware\n"); + + ret = ave_fw_segment(ave, 0, &ave->text); + if (!ret) + ret = ave_fw_segment(ave, 1, &ave->data); + if (ret) + return dev_err_probe(dev, ret, "bad firmware reservation\n"); + + if (ave->text.size != ave->variant->fw_text_size || + ave->data.size != ave->variant->fw_data_size) + return dev_err_probe(dev, -EINVAL, + "unexpected firmware segment sizes\n"); + if ((ave->text.phys < ave->data.phys + ave->data.size && + ave->data.phys < ave->text.phys + ave->text.size) || + (ave->text.iova < ave->data.iova + ave->data.size && + ave->data.iova < ave->text.iova + ave->text.size)) + return dev_err_probe(dev, -EINVAL, + "overlapping firmware segments\n"); + + if (ave_fw_check_mapping(ave, &ave->text) || + ave_fw_check_mapping(ave, &ave->data)) + return dev_err_probe( + dev, -EIO, + "firmware is not mapped where it was loaded\n"); + + ret = ave_fw_crc(&ave->text, &text_crc, NULL); + if (ret) + return ret; + ret = ave_fw_chosen_crc(ave, "aurora,ave-text-crc32", &want); + if (ret || text_crc != want) + return dev_err_probe(dev, -ENODEV, + "firmware text differs from loader CRC\n"); + for (unsigned int i = 0; i < ARRAY_SIZE(ave_known_text_crc); i++) + known |= text_crc == ave_known_text_crc[i]; + if (!known || + (ave->variant->text_crc && text_crc != ave->variant->text_crc)) + return dev_err_probe(dev, -ENODEV, + "unsupported firmware (text CRC %08x)\n", + text_crc); + ave->text_crc = text_crc; + ave->layout = ave_layout_for_crc(text_crc); + + ave->data_pristine = vmalloc(ave->data.size); + if (!ave->data_pristine) + return -ENOMEM; + ret = ave_fw_crc(&ave->data, &data_crc, ave->data_pristine); + if (ret) + goto free_data; + + /* + * The loader records the DATA checksum before anything runs. If it + * differs, the firmware already ran and there is no clean copy left. + */ + ret = ave_fw_chosen_crc(ave, "aurora,ave-data-crc32", &want); + if (ret || data_crc != want) { + ret = dev_err_probe(dev, -ENODEV, + "firmware data is not in its boot state\n"); + goto free_data; + } + + ave->data_crc = data_crc; + ave->fw_state = AVE_FW_OFF; + if (!ave->variant->boot_qualified || !ave->layout) + dev_info( + dev, + "probe only: firmware TEXT %08x DATA %08x needs qualification\n", + text_crc, data_crc); + else if (!ave->layout->encode_ready) + dev_info( + dev, + "%s: TEXT %08x pristine DATA %08x; encoding disabled\n", + ave->layout->name, text_crc, data_crc); + return 0; + +free_data: + vfree(ave->data_pristine); + ave->data_pristine = NULL; + return ret; +} + +static int ave_fw_restore_data(struct ave_dev *ave) +{ + void *va; + + if (!ave->data_dirty) + return 0; + va = memremap(ave->data.phys, ave->data.size, MEMREMAP_WC); + if (!va) + return -ENOMEM; + memcpy(va, ave->data_pristine, ave->data.size); + /* The restored image must be in memory before the coprocessor starts. */ + wmb(); + memunmap(va); + if (ave->layout && ave->layout->abi->transport->scratch_halt) { + u32 crc; + int ret = ave_fw_crc(&ave->data, &crc, NULL); + + if (ret || crc != ave->data_crc) + return ret ?: -EIO; + dev_dbg(ave->dev, "restored DATA CRC %08x\n", ave->data_crc); + } + ave->data_dirty = false; + return 0; +} + +void ave_free(struct ave_dev *ave, struct ave_buf *b) +{ + if (b->cpu) + dma_free_attrs(ave->dev, b->alloc, b->cpu, b->dma, b->attrs); + b->cpu = NULL; +} + +/* + * Firmware-visible memory, zeroed and 16 KiB aligned in the encoder's DART. + * @size is what the firmware is told; @alloc (at least @size) is mapped. + */ +int ave_alloc_padded(struct ave_dev *ave, struct ave_buf *b, size_t size, + size_t alloc, unsigned long attrs) +{ + if (b->cpu) + return 0; + b->size = size; + b->alloc = PAGE_ALIGN(max(size, alloc)); + b->attrs = attrs; + b->cpu = + dma_alloc_attrs(ave->dev, b->alloc, &b->dma, GFP_KERNEL, attrs); + if (!b->cpu) + return -ENOMEM; + if (!IS_ALIGNED(b->dma, SZ_16K)) { + ave_free(ave, b); + return -EINVAL; + } + return 0; +} + +int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, + unsigned long attrs) +{ + return ave_alloc_padded(ave, b, size, size, attrs); +} + +/* + * Lifecycle support and streaming qualification are independent: a known + * transport is not enough to send unqualified Start/Process payloads. + */ +int ave_fw_check_boot(struct ave_dev *ave) +{ + if (ave->variant->boot_qualified && ave->variant->regs && ave->layout && + ave->layout == ave_layout_for_crc(ave->text_crc)) + return 0; + return -EOPNOTSUPP; +} + +int ave_fw_check_build(struct ave_dev *ave) +{ + if (!ave_fw_check_boot(ave) && ave->layout->encode_ready) + return 0; + dev_info( + ave->dev, + "encoding refused: firmware TEXT %08x DATA %08x has no qualified command payload\n", + ave->text_crc, ave->data_crc); + return -EOPNOTSUPP; +} + +/* Only after confirmed Halt, or before the firmware was ever started. */ +void ave_fw_release_boot(struct ave_dev *ave) +{ + if (ave->extra0_mapped) { + dma_unmap_resource(ave->dev, ave->extra0_dma, + ave->extra0_map_size, DMA_BIDIRECTIONAL, 0); + ave->extra0_mapped = false; + } + ave_free(ave, &ave->heap); + ave_free(ave, &ave->log); + ave_free(ave, &ave->boot_cfg); + ave_free(ave, &ave->ipc); + if (ave->module_pinned) { + ave->module_pinned = false; + module_put(THIS_MODULE); + } +} + +static int ave_fw_map_extra0(struct ave_dev *ave) +{ + struct platform_device *pdev = to_platform_device(ave->dev); + struct resource *res; + + if (!ave->variant->extra0_size || ave->extra0_mapped) + return 0; + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "extra0"); + if (!res) + res = platform_get_resource(pdev, IORESOURCE_MEM, 3); + if (!res || resource_size(res) < ave->variant->extra0_size || + !IS_ALIGNED(res->start, SZ_16K)) + return dev_err_probe( + ave->dev, -EINVAL, + "missing or invalid extra0 register resource\n"); + /* The platform resource supplies the translated physical address. */ + ave->extra0_map_size = ALIGN(ave->variant->extra0_size, SZ_16K); + ave->extra0_dma = dma_map_resource(ave->dev, res->start, + ave->extra0_map_size, + DMA_BIDIRECTIONAL, 0); + if (dma_mapping_error(ave->dev, ave->extra0_dma)) + return -EIO; + ave->extra0_mapped = true; + return 0; +} + +void ave_fw_remove(struct ave_dev *ave) +{ + ave_fw_release_boot(ave); + vfree(ave->data_pristine); + ave->data_pristine = NULL; +} + +u64 ave_fw_vcbar(const struct ave_variant *variant, u64 text_iova) +{ + return (text_iova & variant->vcbar_mask) | variant->vcbar_flags; +} + +static bool ave_fw_idle(struct ave_dev *ave) +{ + if (readq(ave->coproc + ave->variant->regs->vcbar) != + ave_fw_vcbar(ave->variant, ave->text.iova)) + return false; + if (readl(ave->coproc + ave->variant->regs->ctrl) || + readl(ave->mbox + AVE_MBOX_STATUS)) + return false; + for (unsigned int i = 0; i < AVE_MBOX_SCRATCH_COUNT; i++) + if (readl(ave->mbox + AVE_MBOX_SCRATCH(i))) + return false; + return true; +} + +/* + * After a Halt the firmware is restarted from a power cycle, as macOS does + * for every session: the encoder must have been runtime suspended in + * between, which puts the coprocessor back in reset. Hardware runs showed + * that a restart while it kept power gets no answer, so that is refused + * (-EAGAIN) and the state stays halted until the power really went. The + * 9003 code base register must still be set: that ABI uses the loader value. + */ +static int ave_fw_prepare_restart(struct ave_dev *ave) +{ + u32 status; + + if (!ave->layout->abi->transport->boot_config) { + if (ave->fw_state == AVE_FW_HALTED) + return -EAGAIN; /* A runtime PM suspend/resume must intervene. */ + if (readl(ave->coproc + ave->variant->regs->ctrl)) + return -EBUSY; + status = readl(ave->mbox + AVE_MBOX_STATUS); + if (status) + writel(status, ave->mbox + AVE_MBOX_STATUS); + for (unsigned int i = 0; i < AVE_MBOX_SCRATCH_COUNT; i++) + writel(0, ave->mbox + AVE_MBOX_SCRATCH(i)); + return 0; + } + if (readq(ave->coproc + ave->variant->regs->vcbar) != + ave_fw_vcbar(ave->variant, ave->text.iova)) { + dev_err(ave->dev, + "coprocessor lost its code base (vcbar %#llx)\n", + readq(ave->coproc + ave->variant->regs->vcbar)); + return -EIO; + } + if (ave->fw_state == AVE_FW_RECOVERING) { + /* + * Recovery cleared CTRL itself and re-armed BOOT_ARG as a + * reset witness with control off: require it to have lost + * power too, before reusing DMA memory or restoring DATA. DARTs + * sharing VENC power may suspend later than the consumer. + */ + if (readl(ave->coproc + ave->variant->regs->ctrl) || + readl(ave->coproc + ave->variant->regs->boot_arg)) + return -EAGAIN; + ave_sess_reap(ave); + ave->fw_state = AVE_FW_HALTED; + ave->inject_wedge = false; + dev_info(ave->dev, "firmware recovery power cycle confirmed\n"); + } + if (ave->fw_state != AVE_FW_HALTED) + return 0; + + if (readl(ave->coproc + ave->variant->regs->ctrl)) { + dev_dbg(ave->dev, + "coprocessor kept power after its halt; retry once it suspends\n"); + return -EAGAIN; + } + status = readl(ave->mbox + AVE_MBOX_STATUS); + if (status) + writel(status, ave->mbox + AVE_MBOX_STATUS); + for (unsigned int i = 0; i < AVE_MBOX_SCRATCH_COUNT; i++) + writel(0, ave->mbox + AVE_MBOX_SCRATCH(i)); + return 0; +} + +/* Boot the coprocessor and bring up its command rings. Caller holds hw_lock. */ +static int ave_fw_boot(struct ave_dev *ave) +{ + const struct ave_fw_transport *t = ave->layout->abi->transport; + __le32 *cfg, *log; + u32 status, hello[4]; + int ret; + + ret = ave_alloc(ave, &ave->ipc, t->ipc_bytes, 0); + if (!ret && t->boot_config) + ret = ave_alloc(ave, &ave->boot_cfg, SZ_16K, 0); + if (!ret && t->boot_config) + ret = ave_alloc(ave, &ave->log, AVE_LOG_BYTES, 0); + if (!ret) + ret = ave_fw_map_extra0(ave); + if (ret) + return ret; + + if (ave_fw_check_mapping(ave, &ave->text) || + ave_fw_check_mapping(ave, &ave->data)) { + dev_err(ave->dev, "firmware segments are no longer mapped\n"); + return -EIO; + } + + ret = ave_fw_prepare_restart(ave); + if (ret) + return ret; + + /* + * The bootloader leaves the coprocessor stopped with its code base + * set. Do not start it from any other state. + */ + if (t->boot_config && !ave_fw_idle(ave)) { + dev_err(ave->dev, + "coprocessor is not in its reset state (vcbar %#llx ctrl %#x)\n", + readq(ave->coproc + ave->variant->regs->vcbar), + readl(ave->coproc + ave->variant->regs->ctrl)); + return -EIO; + } + + memset(ave->ring, 0, sizeof(ave->ring)); + ave->unacked = 0; + memset(ave->want, 0, sizeof(ave->want)); + ave->reply_err = 0; + ave->scalar_next = 0; + + ret = ave_fw_restore_data(ave); + if (ret) + return ret; + + if (t->boot_config) { + cfg = ave->boot_cfg.cpu; + log = ave->log.cpu; + memset(cfg, 0, AVE_BOOT_CFG_BYTES); + memset(log, 0, ave->log.size); + cfg[AVE_BOOT_CFG_DEVICE] = + cpu_to_le32(ave->variant->boot_device_id); + cfg[AVE_BOOT_CFG_UNITS] = cpu_to_le32(1); + cfg[AVE_BOOT_CFG_GROUPS] = cpu_to_le32(1); + cfg[AVE_BOOT_CFG_SUBID_MASK] = cpu_to_le32(1); + cfg[AVE_BOOT_CFG_LOG_LO] = + cpu_to_le32(lower_32_bits(ave->log.dma)); + cfg[AVE_BOOT_CFG_LOG_HI] = + cpu_to_le32(upper_32_bits(ave->log.dma)); + cfg[AVE_BOOT_CFG_LOG_BYTES] = cpu_to_le32(AVE_LOG_BYTES); + cfg[AVE_BOOT_CFG_OPTIONS] = + cpu_to_le32(AVE_BOOT_CFG_OPTIONS_VALUE); + log[0] = cpu_to_le32(AVE_LOG_HDR_USED); + log[1] = cpu_to_le32(AVE_LOG_HDR_BYTES); + log[2] = cpu_to_le32(AVE_LOG_RING_BYTES); + dma_wmb(); + + writel(AVE_BOOT_MAGIC, ave->mbox + AVE_MBOX_SCRATCH(0)); + writel(lower_32_bits(ave->boot_cfg.dma), + ave->mbox + AVE_MBOX_SCRATCH(1)); + writel(upper_32_bits(ave->boot_cfg.dma), + ave->mbox + AVE_MBOX_SCRATCH(2)); + + } else { + writel(AVE_BOOT_MAGIC, ave->mbox + AVE_MBOX_SCRATCH(0)); + writel(0, ave->mbox + AVE_MBOX_SCRATCH(1)); + writel(ave->variant->boot_device_id, + ave->mbox + AVE_MBOX_SCRATCH(2)); + } + if (!t->boot_config) { + writeq(ave_fw_vcbar(ave->variant, ave->text.iova), + ave->coproc + ave->variant->regs->vcbar); + if (readq(ave->coproc + ave->variant->regs->vcbar) != + ave_fw_vcbar(ave->variant, ave->text.iova)) + return -EIO; + dev_dbg(ave->dev, "VCBAR %#llx boot argument %#x\n", + ave_fw_vcbar(ave->variant, ave->text.iova), + ave->variant->boot_arg_value); + } + + if (!t->boot_config && !ave->module_pinned) { + if (!try_module_get(THIS_MODULE)) + return -ENODEV; + ave->module_pinned = true; + } + ave->data_dirty = true; + writel(1, ave->coproc + ave->variant->regs->keep_mappings); + writel(0, ave->coproc + ave->variant->regs->ctrl); + writel(ave->variant->boot_arg_value, + ave->coproc + ave->variant->regs->boot_arg); + writel(AVE_COPROC_CTRL_RUN, ave->coproc + ave->variant->regs->ctrl); + + ret = readl_poll_timeout(ave->mbox + AVE_MBOX_STATUS, status, + status & AVE_MBOX_STATUS_PENDING, 50, + AVE_MBOX_TIMEOUT_US); + if (ret) { + dev_err(ave->dev, "firmware did not answer (status %#x)\n", + readl(ave->coproc + ave->variant->regs->status)); + return ret; + } + for (unsigned int i = 0; i < 4; i++) + hello[i] = readl(ave->mbox + AVE_MBOX_SCRATCH(i)); + dma_rmb(); + + ret = ave_ipc_negotiate(ave, hello); + if (ret) { + dev_err(ave->dev, "firmware start-up failed: %d\n", ret); + return ret; + } + + ave->fw_state = AVE_FW_RUNNING; + ret = ave_cmd_config(ave); + if (ret) + return ret; + + dev_dbg(ave->dev, "firmware running, %u/%u ring slots\n", + ave->ring[0].slots, ave->ring[1].slots); + return 0; +} + +/* Caller holds hw_lock. */ +int ave_fw_ensure_running(struct ave_dev *ave) +{ + enum ave_fw_state prev; + int ret; + + lockdep_assert_held(&ave->hw_lock); + ret = ave_fw_check_boot(ave); + if (ret) + return ret; + switch (ave->fw_state) { + case AVE_FW_RUNNING: + return 0; + case AVE_FW_BROKEN: + dev_err_ratelimited(ave->dev, "firmware recovery is waiting for idle runtime suspend\n"); + return -EIO; + case AVE_FW_OFF: + case AVE_FW_HALTED: + case AVE_FW_RECOVERING: + break; + } + + /* A boot that reached RUN must use the same recovery as a session fault. */ + prev = ave->fw_state; + ret = ave_fw_boot(ave); + if (ret && ret != -EAGAIN) { + if (ave->layout->abi->transport->scratch_halt && + (ave->module_pinned || + readl(ave->coproc + ave->variant->regs->ctrl))) { + if (!ave->module_pinned && try_module_get(THIS_MODULE)) + ave->module_pinned = true; + ave_fw_mark_broken(ave, ret, "boot"); + return ret; + } + if (readl(ave->coproc + ave->variant->regs->ctrl)) + ave_fw_mark_broken(ave, ret, "boot"); + else + ave->fw_state = prev; + } + return ret; +} + +/* + * Tell the firmware to stop, once every session is closed. Halt gets no + * ring reply. 6070 confirms quiescence through scratch0 before the host + * stops the IOP; 9003 retains its existing settle interval. + * Caller holds hw_lock. + */ +void ave_fw_halt(struct ave_dev *ave) +{ + lockdep_assert_held(&ave->hw_lock); + if (ave->fw_state != AVE_FW_RUNNING) + return; + if (ave->layout->abi->transport->scratch_halt) { + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i] && + ave->sessions[i]->sess->fw_open) { + ave->fw_state = AVE_FW_BROKEN; + return; + } + writel(0, ave->mbox + AVE_MBOX_SCRATCH(0)); + } + if (ave_cmd_halt(ave)) { + dev_err(ave->dev, "cannot send Halt; firmware treated as stuck\n"); + ave_fw_mark_broken(ave, -EIO, "Halt"); + return; + } + if (ave->layout->abi->transport->scratch_halt) { + u32 status; + int ret; + + ret = readl_poll_timeout(ave->mbox + AVE_MBOX_SCRATCH(0), + status, status == AVE_BOOT_MAGIC, 100, + AVE_MBOX_TIMEOUT_US); + if (ret) { + dev_err(ave->dev, + "Halt not confirmed; retaining firmware mappings and power\n"); + ave->fw_state = AVE_FW_BROKEN; + return; + } + dev_dbg(ave->dev, "Halt confirmed: scratch0 %#x\n", status); + /* Scratch confirmation establishes quiescence; stop and read back the IOP. + */ + writel(0, ave->coproc + ave->variant->regs->ctrl); + ret = readl_poll_timeout(ave->coproc + ave->variant->regs->ctrl, + status, !status, 100, + AVE_MBOX_TIMEOUT_US); + if (ret) { + ave->fw_state = AVE_FW_BROKEN; + return; + } + } else { + msleep(AVE_HALT_SETTLE_MS); + } + ave->fw_state = AVE_FW_HALTED; + memset(ave->ring, 0, sizeof(ave->ring)); + ave->ipc_fw_base = 0; +} + +/* + * Stop the firmware before its power goes away. Caller holds hw_lock. + * + * Sessions keep the device runtime active, so the firmware has normally + * been halted already. A firmware that stopped answering keeps its power: + * system sleep must not silently force-stop a potentially live DMA engine. + * Runtime recovery is separate and only runs after all owners have gone. + */ +int ave_fw_shutdown(struct ave_dev *ave) +{ + lockdep_assert_held(&ave->hw_lock); + switch (ave->fw_state) { + case AVE_FW_RUNNING: + ave_fw_halt(ave); + return ave->fw_state == AVE_FW_HALTED ? 0 : -EBUSY; + case AVE_FW_BROKEN: + case AVE_FW_RECOVERING: + return -EBUSY; + default: + return 0; + } +} + +/* Caller holds hw_lock. Wake even idle sessions so none pins a broken boot. */ +void ave_fw_mark_broken(struct ave_dev *ave, int err, const char *what) +{ + lockdep_assert_held(&ave->hw_lock); + if (ave->fw_state != AVE_FW_BROKEN) + dev_err(ave->dev, "firmware failed (%s: %d); scheduling recovery\n", what, err); + ave->fw_state = AVE_FW_BROKEN; + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i]) + mod_delayed_work(ave->wq, &ave->sessions[i]->poll_work, 0); +} + +/* + * Only runtime PM may force-stop a broken firmware, with no session owners + * and no runtime references. genpd and the IOMMU's PM device links then drop + * the VENC domains and DARTs; AVE must not reset or reprogram the DART itself. + * System sleep continues to use the strict ave_fw_shutdown() above. + */ +int ave_fw_runtime_suspend(struct ave_dev *ave) +{ + int ret; + + lockdep_assert_held(&ave->hw_lock); + /* 6070 requires scratch-confirmed Halt; never force its power gate. */ + if (ave->layout && ave->layout->abi->transport->scratch_halt) + return ave_fw_shutdown(ave); + if (ave->fw_state != AVE_FW_BROKEN && ave->fw_state != AVE_FW_RECOVERING) + return ave_fw_shutdown(ave); + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i]) + return -EBUSY; + if (ave->fw_state == AVE_FW_RECOVERING) + return 0; + if (!ave_recovery_allow(&ave->recovery, jiffies)) { + dev_err_ratelimited(ave->dev, "firmware recovery limit: 3 attempts per 10 minutes\n"); + return -EBUSY; + } + ave->recovery_count++; + dev_warn(ave->dev, "firmware recovery %u: stopping coprocessor and cycling VENC power\n", + ave->recovery_count); + + /* A ring may not exist after a failed boot. Never wait for Halt's reply. */ + ret = ave->inject_wedge ? -ETIMEDOUT : -EIO; + if (ave->ipc_fw_base && ave->ring[0].slots && !ave->inject_wedge) { + ave->fw_state = AVE_FW_RUNNING; + ret = ave_cmd_halt(ave); + } + if (ret) + dev_warn(ave->dev, "recovery Halt failed: %d; forcing control off\n", ret); + else + msleep(AVE_HALT_SETTLE_MS); + writel(0, ave->coproc + ave->variant->regs->ctrl); + if (readl(ave->coproc + ave->variant->regs->ctrl)) { + ave->fw_state = AVE_FW_BROKEN; + return -EIO; + } + /* CTRL=0 must not masquerade as power loss on the next attach. */ + writel(AVE_COPROC_BOOT_ARG_VALUE, ave->coproc + ave->variant->regs->boot_arg); + if (readl(ave->coproc + ave->variant->regs->boot_arg) != AVE_COPROC_BOOT_ARG_VALUE) { + ave->fw_state = AVE_FW_BROKEN; + return -EIO; + } + ave->fw_state = AVE_FW_RECOVERING; + return 0; +} + +/* The coprocessor lost power behind the firmware's back. Caller holds hw_lock. */ +void ave_fw_lost(struct ave_dev *ave) +{ + lockdep_assert_held(&ave->hw_lock); + ave->fw_state = AVE_FW_OFF; + memset(ave->ring, 0, sizeof(ave->ring)); + ave->ipc_fw_base = 0; +} + +/* Explicit lifecycle-only validation, without allocating encoder surfaces. */ +int ave_fw_lifecycle_test(struct ave_dev *ave, enum ave_codec codec) +{ + struct ave_ctx *ctx; + struct ave_session *sess; + int ret; + + if (!ave->layout || ave->layout->abi != &ave_abi_6070) + return -EOPNOTSUPP; + ctx = kzalloc_obj(*ctx); + sess = kzalloc_obj(*sess); + if (!ctx || !sess) { + ret = -ENOMEM; + goto free; + } + ctx->ave = ave; + ctx->sess = sess; + sess->p.codec = codec; + ret = pm_runtime_resume_and_get(ave->dev); + if (ret < 0) + goto free; + mutex_lock(&ave->hw_lock); + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i]) { + ret = -EBUSY; + goto unlock; + } + ret = ave_fw_ensure_running(ave); + if (!ret) + ret = ave_cmd_open(ctx); + if (!ret) + ret = ave_cmd_close(ctx); + if (!ret) { + ave_fw_halt(ave); + if (ave->fw_state != AVE_FW_HALTED) + ret = -EIO; + } else if (ave->fw_state == AVE_FW_RUNNING) { + /* Unconfirmed client lifetime: do not Halt or free anything it may own. */ + ave->fw_state = AVE_FW_BROKEN; + } +unlock: + mutex_unlock(&ave->hw_lock); + /* Runtime suspend refuses broken firmware, retaining its DMA and power. */ + if (ave->fw_state != AVE_FW_BROKEN) { + int err = pm_runtime_put_sync_suspend(ave->dev); + + if (!ret && err < 0) + ret = err; + } else { + dev_err(ave->dev, + "lifecycle failed (%d); retaining runtime power reference\n", + ret); + } + dev_info(ave->dev, "6070 lifecycle codec %u: %d\n", codec, ret); +free: + kfree(sess); + kfree(ctx); + return ret; +} diff --git a/drivers/media/platform/apple/ave/ave-image.c b/drivers/media/platform/apple/ave/ave-image.c new file mode 100644 index 0000000000000..853d0a5bccc84 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-image.c @@ -0,0 +1,531 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder: constant words of the session commands + * + * macOS builds Start and Process in code, from the session settings and + * its own defaults; nothing is stored. Besides the fields the driver + * computes (sizes, geometry, profile, level, rate control, GOP, buffer + * addresses, per-frame values), every captured command on firmware + * AppleAVE2FW 9003.78.0 holds the same small set of constant words. They + * are listed here as facts, grouped by where they sit; for most of them + * the meaning is not known, only that macOS always sets that value. The + * commands are built in a zeroed buffer: these words first, then the + * computed fields. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include + +#include "ave.h" +#include "ave-regs.h" + +/* @count consecutive u32 words at @off, each set to @val. */ +struct ave_word { + u32 off; + u16 count; + u32 val; +}; + +#define W(o, v) { .off = (o), .count = 1, .val = (v) } +#define RUN(o, n, v) { .off = (o), .count = (n), .val = (v) } + +/* IEEE 754 doubles, high words only (the low words are zero) */ +#define F64_HI_1_0 0x3ff00000 /* 1.0 */ +#define F64_HI_0_5 0x3fe00000 /* 0.5 */ +#define F64_HI_M1_0 0xbff00000 /* -1.0 */ + +/* Session configuration words common to both codecs (Start +0x68..) */ +static const struct ave_word ave_start_common[] = { + /* First surface size: always the 0x12c000 header surface */ + W(0x50, 0x12c000), + W(0x94, 0x36), + W(0x214, 0x39d), + W(0x218, 0x18), + /* Rate-control doubles: 1.0, 0.5 (the 0.5 s VBV buffer), -1, -1 */ + W(0x28c, F64_HI_1_0), + W(0x294, F64_HI_0_5), + W(0x2c4, F64_HI_M1_0), + W(0x2cc, F64_HI_M1_0), + W(0x2d0, 1), + W(0x2e4, 1), + RUN(0x2f4, 3, 1), + RUN(0x338, 2, 0xffffffff), + /* Buffer set: metadata sizes of slots 0..2, +0x13d0 surface size */ + W(0xf58, 0xc000), + W(0xf68, 0xc000), + W(0xf78, 0xc000), + W(0x13d8, 0x4000), +}; + +static const struct ave_word ave_h264_start[] = { + /* Codec (1) and four flags after it */ + RUN(0xc0, 5, 1), + W(0x370, 2), + /* Codec settings block (+0x2580..): see also ave_start_codec_bytes() */ + W(0x258c, 3), + W(0x2594, 1), + W(0x2598, 8), /* bit depth */ + W(0x25a4, 0x100), + W(0x25cc, 1), + W(0x25d4, 1), + W(0x26dc, 4), + W(0x26f8, 0x15), + RUN(0x2704, 2, 1), + W(0x2734, 1), + W(0x2738, 0x10), + W(0x27c0, 8), + /* Sequence and picture parameter block (+0x2900..) */ + W(0x2940, 1), + W(0x2970, 0x100000), + RUN(0x2974, 239, 0x100010), + W(0x2d30, 0x10), + W(0x2d34, 1), + W(0x2d3c, 2), + RUN(0x2d54, 2, 1), + W(0x2d5c, 0x1000000), + W(0x2d60, 5), + W(0x2d68, 1), + RUN(0x2d6c, 3, 2), + W(0x2dc8, 1), + /* Picture parameters: byte +0x3020 is 1; +0x3023 (8x8 transform) is computed */ + W(0x3020, 1), + /* Two runs of -13 */ + RUN(0x3004, 7, 0xfffffff3), + RUN(0x302c, 7, 0xfffffff3), + W(0x3048, 1), + /* Execution-unit info */ + W(0x3150, 1), + W(0x3158, 1), +}; + +static const struct ave_word ave_hevc_start[] = { + /* Codec (2) and four flags after it */ + W(0xc0, 2), + RUN(0xc4, 4, 1), + W(0x370, 0x2000002), + W(0x1358, 0x4000), /* +0x1350 surface size, fixed for HEVC */ + /* Codec settings block (+0x2580..): see also ave_start_codec_bytes() */ + W(0x2588, 2), + W(0x2594, 1), + W(0x2598, 8), /* bit depth */ + W(0x25a4, 0x100), + W(0x25cc, 1), + W(0x25d4, 1), + W(0x26dc, 4), + RUN(0x26f8, 2, 0x15), + W(0x2700, 2), + RUN(0x2704, 2, 1), + W(0x2710, 1), + W(0x2734, 1), + W(0x2738, 0x10), + W(0x27c0, 8), + /* Profile, tier and level block, first copy (+0x2920..) */ + W(0x2920, 0x101), + W(0x292c, 1), + W(0x2934, 1), + W(0x295c, 0x1010001), + /* Constraint flag bytes; +0x2962 (max 8 bit) is computed */ + W(0x2960, 0x01000101), + W(0x2964, 1), + RUN(0x2a8c, 7, 0x1010101), + W(0x2aa8, 0x101), + W(0x2abc, 0x1010101), + W(0x2ac0, 0x101), + RUN(0x2e38, 7, 4), + /* Timing pair 1001 / 24000, also at +0x74f0 */ + W(0x3298, 1001), + W(0x329c, 24000), + /* Sequence parameter block, second copy (+0x58f0..) */ + W(0x58f8, 1), + W(0x5900, 1), + W(0x5928, 0x1010001), + /* Constraint flag bytes, second copy; +0x592e (max 8 bit) is computed */ + W(0x592c, 0x01000000), + W(0x5930, 1), + W(0x5aac, 1), + W(0x5ae0, 7), /* log2_max_pic_order_cnt_lsb_minus4: 11-bit POC */ + RUN(0x5ae8, 7, 4), /* ends with sps_max_dec_pic_buffering_minus1 */ + W(0x5b40, 2), + W(0x5b48, 3), + W(0x5b4c, 1), + W(0x5b54, 1), + /* Picture parameter and slice defaults (+0x7400..) */ + W(0x7400, 0x100), + RUN(0x7404, 2, 7), + W(0x740c, 1), + W(0x7414, 1), + W(0x749c, 1), + W(0x74b0, 0x100), + W(0x74b4, 5), + W(0x74bc, 1), + RUN(0x74c0, 3, 2), + W(0x74f0, 1001), + W(0x74f4, 24000), + W(0x7500, 0x101), + W(0x7504, 1), + W(0x7508, 3), + W(0x750c, 5), + RUN(0x7510, 3, 0x17), + RUN(0x7524, 2, 0xbebb), + W(0x7558, 0x100), + /* Rate-control block (+0x9600..), with two runs of -13 */ + W(0x960c, 2), + RUN(0x9614, 7, 0xfffffff3), + RUN(0x9634, 7, 0xfffffff3), + W(0x9654, 0x100), + W(0x9e64, 0x10000), + W(0xb734, 2), + W(0xb774, 1), + /* Execution-unit info */ + RUN(0x13ef4, 2, 1), + W(0x13f00, 1), +}; + +/* + * HEVC with frame reordering off: a low-delay P reference structure (the + * reference picture sets hold the delta POCs -1 .. -4). All zero with + * B-frames. + */ +static const struct ave_word ave_hevc_start_low_delay[] = { + W(0xe120, 4), W(0xe154, 4), W(0xe17c, 0x01010101), + W(0xe1dc, 4), W(0xe1e4, 0x01010101), + W(0xe204, -1), W(0xe208, -2), W(0xe20c, -3), W(0xe210, -4), + W(0xe244, 1), W(0xe248, 2), W(0xe24c, 3), W(0xe250, 4), + W(0xe284, 4), W(0xe2b8, 1), W(0xe2e0, 1), W(0xe340, 1), W(0xe348, 1), + W(0xe368, -1), W(0xe3a8, 1), W(0xe3e8, 1), + W(0xe41c, 2), W(0xe444, 0x101), W(0xe4a4, 2), W(0xe4ac, 0x101), + W(0xe4cc, -1), W(0xe4d0, -2), + W(0xe50c, 1), W(0xe510, 2), W(0xe54c, 2), + W(0xe580, 3), W(0xe5a8, 0x10101), W(0xe608, 3), W(0xe610, 0x10101), + W(0xe630, -1), W(0xe634, -2), W(0xe638, -3), + W(0xe670, 1), W(0xe674, 2), W(0xe678, 3), W(0xe6b0, 3), +}; + +/* Process: the first-frame flag and per-frame values are computed. */ +static const struct ave_word ave_h264_process[] = { + /* Codec block */ + W(0x99c, 1), + W(0x9a8, 2), + W(0x9bc, 1), + W(0x9d4, 1), + W(0xeb4, 1), + W(0x12bc, 0xffffffff), + /* Picture-management block */ + W(0x29fc, 0xffffffff), + W(0x6238, 5), + W(0x6278, 14400000), /* presentation timescale, low word */ +}; + +static const struct ave_word ave_hevc_process[] = { + /* Codec block */ + W(0x9a8, 1), + W(0x9b8, 2), + W(0xab8, 0x10100), + W(0xb44, 0x10000), + RUN(0xb4c, 2, 5), + RUN(0xd70, 2, 0xffffffff), + W(0xd7c, 1), + /* Picture-management block */ + W(0x74cc, 0xffffffff), + W(0x74d0, 0x100), + W(0x74e0, 1), + W(0xad08, 5), + W(0xad3c, 0x403e0000), /* frame rate 30.0, rewritten per session */ + W(0xad48, 14400000), /* presentation timescale, low word */ + W(0xad68, 0xffffffff), +}; + +/* + * 26A428: independently derived invariant words from the field ledger. + * Computed fields overwrite any invariant default; no command images or + * captured addresses are stored here. This layout has no qualified CRC yet. + */ +static const struct ave_word ave_26a428_h264_start[] = { + /* Invariant bytes of a word whose crop flag is computed. */ + W(0x3ac4, 0x10001), + /* Byte zero stays set while byte three selects the 8x8 transform. */ + W(0x3820, 1), + W(0x50, 0xec000), + W(0x90, 0x2), + W(0xb10, 0x10000), + W(0xb20, 0x10000), + W(0xb30, 0x10000), + W(0xdb8, 0x2), + W(0xfe0, 0x2), + W(0x19b8, 0x1), + W(0x1a00, 0x1), + W(0x1a48, 0x1), + W(0x1a58, 0x4000), + W(0x1bb0, 0x4), + W(0x1bb4, 0x8), + W(0x31f0, 0x1000), + W(0x320c, 0x1e), + W(0x321c, 0x36), + W(0x3234, 0x8), + RUN(0x3248, 4, 0x1), + W(0x3258, 0x1), + W(0x325c, 0xffffffff), + W(0x3298, 0x1), + W(0x35f4, 0x39d), + W(0x35f8, 0x18), + W(0x3600, 0x1e), + W(0x366c, 0x3ff00000), + W(0x3674, 0x3fe00000), + W(0x36f4, 0xbff00000), + W(0x36fc, 0xbff00000), + W(0x3700, 0x1), + RUN(0x3714, 3, 0x1), + W(0x3728, 0x1), + RUN(0x372c, 2, 0x1), + W(0x373c, 0x7), + RUN(0x3780, 2, 0xffffffff), + W(0x37c0, 0x1), + W(0x37c4, 0x80800), + W(0x37cc, 0x60005), + W(0x37d8, 0x1000000), + W(0x37dc, 0x5), + W(0x37e0, 0x20100), + W(0x37e4, 0x20002), + W(0x3810, 0x1010100), + W(0x3814, 0x1a1a0000), + W(0x3818, 0xf3f3f300), + W(0x381c, 0xf3f3f3f3), + W(0x3824, 0xf3f30000), + W(0x3828, 0xf3f3f3f3), + W(0x382c, 0x1f3), + W(0x3834, 0x100), + W(0x3838, 0x1), + W(0x383c, 0x8), + W(0x3848, 0x100), + W(0x3860, 0x1), + W(0x3864, 0x4), + W(0x3874, 0x15), + W(0x3884, 0x1), + W(0x393c, 0x8), + W(0x3aa4, 0x1), + W(0x3ab0, 0x20001), + W(0x3ad0, 0x1000000), + W(0x3ad4, 0x5), + W(0x3ad8, 0x20100), + W(0x3adc, 0x20002), + W(0x3b00, 0xf3000000), + W(0x3b04, 0xf3f3f3f3), + W(0x3b08, 0x1f3f3), + W(0x3b10, 0xf3f3f3f3), + W(0x3b14, 0x1f3f3f3), + W(0x3b1c, 0x1), +}; + +static const struct ave_word ave_26a428_hevc_start[] = { + /* Constraint bytes: the max-8-bit byte is computed. */ + W(0x3a60, 0x01000101), + W(0x6530, 0x01000000), + W(0x50, 0xec000), + W(0x68, 0x2), + W(0x90, 0x2), + W(0x94, 0x5), + W(0xb10, 0x10000), + W(0xb20, 0x10000), + W(0xb30, 0x10000), + W(0xb40, 0x10000), + W(0xb50, 0x10000), + W(0xdb8, 0x2), + W(0xdbc, 0x5), + W(0xfe0, 0x2), + W(0xfe4, 0x5), + W(0x19b8, 0x1), + W(0x19c8, 0x4000), + W(0x1a00, 0x1), + W(0x1a48, 0x1), + W(0x1a58, 0x4000), + RUN(0x1bb0, 2, 0x4), + W(0x31f0, 0x1000), + W(0x320c, 0x1e), + W(0x321c, 0x36), + W(0x3248, 0x2), + RUN(0x324c, 4, 0x1), + RUN(0x325c, 2, 0xffffffff), + W(0x3298, 0x1), + W(0x35f0, 0xa), + W(0x35f4, 0x39d), + W(0x35f8, 0x18), + W(0x3600, 0x1e), + W(0x366c, 0x3ff00000), + W(0x3674, 0x3fe00000), + W(0x36dc, 0x1), + RUN(0x36e0, 2, 0x3), + W(0x36e8, 0x1e), + W(0x36f4, 0xbff00000), + W(0x36fc, 0xbff00000), + W(0x3700, 0x1), + RUN(0x3714, 3, 0x1), + W(0x3728, 0x1), + RUN(0x372c, 2, 0x1), + W(0x373c, 0x7), + W(0x3764, 0x4), + RUN(0x3780, 2, 0xffffffff), + W(0x37b8, 0x100), + W(0x37bc, 0x1), + W(0x37c4, 0x4), + W(0x37cc, 0x100), + W(0x37e4, 0x1), + W(0x37e8, 0x4), + RUN(0x37f8, 2, 0x15), + W(0x3800, 0x2), + W(0x3804, 0x2), + W(0x3808, 0x1), + W(0x38c0, 0x8), + W(0x3a20, 0x10100), + W(0x3a2c, 0x1), + W(0x3a34, 0x1), + W(0x3a5c, 0x1010001), + W(0x3a64, 0x1), + RUN(0x3b8c, 7, 0x1010101), + W(0x3ba8, 0x101), + W(0x3bbc, 0x1010101), + W(0x3bc0, 0x101), + RUN(0x4940, 7, 0x4), + W(0x495c, 0x2), + W(0x4da0, 0x3e9), + W(0x4da4, 0x5dc0), + W(0x64fc, 0x1), + W(0x6504, 0x1), + W(0x652c, 0x1010001), + W(0x6534, 0x1), + W(0x66b0, 0x1), + W(0x66e4, 0x7), + RUN(0x66ec, 7, 0x4), + W(0x6708, 0x2), + W(0x6744, 0x2), + W(0x674c, 0x3), + W(0x6750, 0x1), + W(0x6758, 0x1), + W(0x8004, 0x100), + RUN(0x8008, 2, 0x7), + W(0x8010, 0x1), + W(0x8018, 0x1), + W(0x80a0, 0x1), + W(0x80b4, 0x100), + W(0x80b8, 0x5), + W(0x80c0, 0x1), + RUN(0x80c4, 3, 0x2), + W(0x80f4, 0x3e9), + W(0x80f8, 0x5dc0), + W(0x8104, 0x101), + W(0x8108, 0x1), + W(0x810c, 0x3), + W(0x8110, 0x5), + RUN(0x8114, 3, 0x17), + RUN(0x8128, 2, 0xbebb), + W(0x815c, 0x100), + W(0x9de8, 0x10000), + W(0x9df4, 0x2), + RUN(0x9dfc, 7, 0xfffffff3), + RUN(0x9e1c, 7, 0xfffffff3), + W(0x9e3c, 0x100), + W(0xa64c, 0x10000), + W(0xbf10, 0x2), + W(0xbf50, 0x1), + RUN(0x13eac, 2, 0x1), + W(0x13eb8, 0x1), +}; + +static const struct ave_word ave_26a428_h264_process[] = { + W(0xdc0, 0x1), + W(0x1328, 0x1e), + W(0x25e4, 0x1), + W(0x25ec, 0x1), + W(0x29dc, 0xffffffff), + W(0x5660, 0x5), + W(0x575c, 0x403e0000), + W(0x5768, 0xdbba00), + W(0x58b8, 0x10000), + W(0x58c8, 0x4000), +}; + +static const struct ave_word ave_26a428_hevc_process[] = { + W(0x9a4, 0x1), + W(0x9ac, 0x2), + W(0xaa8, 0x10100), + W(0xb30, 0x10000), + RUN(0xb38, 2, 0x5), + W(0x1568, 0x1), + W(0x1ad0, 0x1e), + W(0x3184, 0xffffffff), + W(0x3194, 0x1), + W(0x5e08, 0x5), + W(0x5f04, 0x403e0000), + W(0x5f10, 0xdbba00), + W(0x5f48, 0xffffffff), + W(0x6060, 0x10000), + W(0x6070, 0x4000), +}; + +static void ave_put_words(u8 *cmd, size_t len, const struct ave_word *w, size_t n) +{ + for (size_t i = 0; i < n; i++) + for (unsigned int j = 0; j < w[i].count; j++) { + size_t off = w[i].off + 4 * j; + + if (WARN_ON_ONCE(off + 4 > len)) + return; + put_unaligned_le32(w[i].val, cmd + off); + } +} + +/* + * The 8 bytes at Start +0x2580 look like a DART address but are separate + * settings: byte 0 is 4 for HEVC, byte 3 is 1, byte 5 is 1 unless the + * client turned frame reordering off (H.264 only; HEVC keeps 1). + */ +static void ave_start_codec_bytes(u8 *cmd, enum ave_codec codec, bool reorder_off) +{ + cmd[0x2580] = codec == AVE_CODEC_HEVC ? 4 : 0; + cmd[0x2583] = 1; + cmd[0x2585] = codec == AVE_CODEC_H264 && reorder_off ? 0 : 1; +} + +void ave_image_start(u8 *cmd, size_t len, const struct ave_fw_layout *fw, + enum ave_codec codec, bool reorder_off, + bool hevc_low_delay) +{ + if (fw->counted) { + if (codec == AVE_CODEC_HEVC) + ave_put_words(cmd, len, ave_26a428_hevc_start, + ARRAY_SIZE(ave_26a428_hevc_start)); + else + ave_put_words(cmd, len, ave_26a428_h264_start, + ARRAY_SIZE(ave_26a428_h264_start)); + return; + } + ave_put_words(cmd, len, ave_start_common, ARRAY_SIZE(ave_start_common)); + if (codec == AVE_CODEC_HEVC) { + ave_put_words(cmd, len, ave_hevc_start, ARRAY_SIZE(ave_hevc_start)); + if (hevc_low_delay) + ave_put_words(cmd, len, ave_hevc_start_low_delay, + ARRAY_SIZE(ave_hevc_start_low_delay)); + } else { + ave_put_words(cmd, len, ave_h264_start, ARRAY_SIZE(ave_h264_start)); + } + ave_start_codec_bytes(cmd, codec, reorder_off); +} + +void ave_image_process(u8 *cmd, size_t len, const struct ave_fw_layout *fw, + enum ave_codec codec) +{ + if (fw->counted) { + if (codec == AVE_CODEC_HEVC) + ave_put_words(cmd, len, ave_26a428_hevc_process, + ARRAY_SIZE(ave_26a428_hevc_process)); + else + ave_put_words(cmd, len, ave_26a428_h264_process, + ARRAY_SIZE(ave_26a428_h264_process)); + return; + } + + if (codec == AVE_CODEC_HEVC) + ave_put_words(cmd, len, ave_hevc_process, ARRAY_SIZE(ave_hevc_process)); + else + ave_put_words(cmd, len, ave_h264_process, ARRAY_SIZE(ave_h264_process)); +} diff --git a/drivers/media/platform/apple/ave/ave-ipc.c b/drivers/media/platform/apple/ave/ave-ipc.c new file mode 100644 index 0000000000000..8b26b6c3bbf17 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-ipc.c @@ -0,0 +1,654 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder: coprocessor mailbox and command rings + * + * The mailbox negotiation and the 64-byte ring protocol here come from + * Acelogic's J700 AVE transport (the first Linux code to drive this + * firmware); this file reshapes it into the driver core. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ave.h" +#include "ave-regs.h" + +static int ave_fw_offset(struct ave_dev *ave, u64 addr, size_t len, u32 *offset) +{ + u64 delta = addr - ave->ipc_fw_base; + + if (addr < ave->ipc_fw_base || delta > ave->ipc.size || + len > ave->ipc.size - delta) + return -ERANGE; + *offset = delta; + return 0; +} + +static void ave_mbox_send(struct ave_dev *ave, u32 a, u32 b, u32 c, u32 d) +{ + writel(a, ave->mbox + AVE_MBOX_SCRATCH(0)); + writel(b, ave->mbox + AVE_MBOX_SCRATCH(1)); + writel(c, ave->mbox + AVE_MBOX_SCRATCH(2)); + writel(d, ave->mbox + AVE_MBOX_SCRATCH(3)); + /* All four words must land before the doorbell. */ + wmb(); + writel(1, ave->mbox + AVE_MBOX_DOORBELL); +} + +static int ave_mbox_recv(struct ave_dev *ave, u32 reply[4]) +{ + u32 status; + int ret; + + ret = readl_poll_timeout(ave->mbox + AVE_MBOX_STATUS, status, + status & AVE_MBOX_STATUS_PENDING, 50, + AVE_MBOX_TIMEOUT_US); + if (ret) + return ret; + + /* Acknowledge before reading the scratch words. */ + writel(AVE_MBOX_STATUS_PENDING, ave->mbox + AVE_MBOX_STATUS); + for (unsigned int i = 0; i < 4; i++) + reply[i] = readl(ave->mbox + AVE_MBOX_SCRATCH(i)); + dma_rmb(); + return 0; +} + +int ave_ipc_parse_channels(struct ave_dev *ave) +{ + bool found[2] = { false, false }; + + for (unsigned int i = 0; i < ave->layout->abi->transport->descriptors; + i++) { + u8 *desc = ave->ipc.cpu + AVE_IPC_CHANNEL_OFF + + i * AVE_CHAN_DESC_STRIDE; + u32 dir = get_unaligned_le32(desc + AVE_CHAN_DESC_DIR); + u32 doorbell = get_unaligned_le32(desc + AVE_CHAN_DESC_IRQ); + u32 slots = get_unaligned_le32(desc + AVE_CHAN_DESC_SLOTS); + u64 addr = get_unaligned_le64(desc + AVE_CHAN_DESC_ADDR); + struct ave_ring *r; + unsigned int id; + u32 offset; + + const struct ave_fw_transport *t = ave->layout->abi->transport; + + if (!memchr(desc, 0, AVE_CHAN_DESC_NAME_LEN) || + slots > ave->channel_bytes / AVE_RING_SLOT_BYTES || + !IS_ALIGNED(addr, AVE_RING_SLOT_BYTES) || + ave_fw_offset(ave, addr, + (size_t)slots * AVE_RING_SLOT_BYTES, + &offset) || + offset < AVE_IPC_CHANNEL_OFF + + t->descriptors * AVE_CHAN_DESC_STRIDE || + offset + slots * AVE_RING_SLOT_BYTES > + AVE_IPC_CHANNEL_OFF + ave->channel_bytes) + return -EPROTO; + if (!strcmp(desc, "IO")) + id = 0; + else if (!strcmp(desc, "IO_T2H")) + id = 1; + else if (t->descriptors > 2) + continue; + else + return -EPROTO; + if (found[id] || dir != id || !doorbell || doorbell >= 32 || + !slots || + (t->slots[id] && + (slots != t->slots[id] || doorbell != t->doorbells[id] || + offset != AVE_IPC_CHANNEL_OFF + t->offsets[id] || + i != (id ? t->t2h_index : t->io_index)))) + return -EPROTO; + + found[id] = true; + r = &ave->ring[id]; + strscpy(r->name, desc, sizeof(r->name)); + /* Host-to-firmware rings start empty, firmware-to-host rings full. */ + r->mode = dir ^ 1; + r->doorbell = doorbell; + r->slots = slots; + r->offset = offset; + r->consumer = r->mode ? -1 : 0; + r->producer = r->mode ? 0 : -1; + } + + if (!found[0] || !found[1]) + return -EPROTO; + if (ave->ring[0].offset < + ave->ring[1].offset + + ave->ring[1].slots * AVE_RING_SLOT_BYTES && + ave->ring[1].offset < + ave->ring[0].offset + + ave->ring[0].slots * AVE_RING_SLOT_BYTES) + return -EPROTO; + + /* Hand every host-to-firmware slot to the host; ownership is bit 0. */ + for (unsigned int i = 0; i < ave->ring[0].slots; i++) { + __le64 *slot = ave->ipc.cpu + ave->ring[0].offset + + i * AVE_RING_SLOT_BYTES; + + slot[0] = cpu_to_le64(1); + slot[1] = 0; + slot[2] = 0; + } + dma_wmb(); + return 0; +} + +/* + * Finish the coprocessor start-up once the firmware has answered the boot + * request: hand it the heap and the IPC window, then the channel area, and + * read back the two ring descriptors it writes there. + */ +int ave_ipc_check_hello(const struct ave_fw_transport *t, const u32 hello[4]) +{ + if (hello[0] != t->hello[0] || hello[2] != t->hello[2] || + hello[1] < t->descriptors * AVE_CHAN_DESC_STRIDE || + hello[1] > AVE_IPC_HANDSHAKE_OFF - AVE_IPC_CHANNEL_OFF) + return -EPROTO; + if (t->boot_config) + return !hello[3] || hello[3] > 4 * SZ_1M ? -EPROTO : 0; + return hello[1] != t->hello[1] || hello[3] != t->hello[3] ? -EPROTO : 0; +} + +static int ave_iop_time_us(struct ave_dev *ave, u64 *us) +{ + u64 ticks = readq(ave->coproc + ave->variant->regs->timebase); + u64 freq = readl(ave->coproc + ave->variant->regs->timebase_freq); + + freq *= ave->variant->timebase_multiplier; + if (freq && freq < 1000000) + return -EPROTO; + *us = div64_u64(ticks, freq ? div_u64(freq, 1000000) : 24); + return 0; +} + +int ave_ipc_negotiate(struct ave_dev *ave, const u32 hello[4]) +{ + const struct ave_fw_transport *t = ave->layout->abi->transport; + u8 *hs = ave->ipc.cpu + AVE_IPC_HANDSHAKE_OFF; + u64 ticks, offset_us; + u32 status, reply[4], off; + int ret; + + ret = ave_ipc_check_hello(t, hello); + if (ret) { + dev_err(ave->dev, "unexpected hello {%#x, %#x, %#x, %#x}\n", + hello[0], hello[1], hello[2], hello[3]); + return ret; + } + if (t->scratch_halt) + dev_dbg(ave->dev, "hello {%#x, %#x, %#x, %#x}\n", hello[0], + hello[1], hello[2], hello[3]); + ave->channel_bytes = hello[1]; + + if (t->boot_config) { + if (ave->heap.cpu && ave->heap.size != PAGE_ALIGN(hello[3])) + return -EPROTO; + if (!ave->heap.cpu) { + ret = ave_alloc(ave, &ave->heap, PAGE_ALIGN(hello[3]), + 0); + if (ret) + return ret; + } + memset(ave->heap.cpu, 0, ave->heap.size); + } + writel(AVE_MBOX_STATUS_PENDING, ave->mbox + AVE_MBOX_STATUS); + if (t->sync_host_clock) { + ret = ave_iop_time_us(ave, &ticks); + if (ret) + return ret; + offset_us = div_u64(ktime_get_ns(), NSEC_PER_USEC) - ticks; + writel(lower_32_bits(offset_us), ave->mbox + AVE_MBOX_TIME_LO); + writel(upper_32_bits(offset_us), ave->mbox + AVE_MBOX_TIME_HI); + } + + ave_mbox_send(ave, lower_32_bits(ave->ipc.dma), + upper_32_bits(ave->ipc.dma), ave->ipc.size, 0); + ret = ave_mbox_recv(ave, reply); + if (ret) + return ret; + ave->ipc_fw_base = (u64)reply[1] << 32 | reply[0]; + if (!ave->ipc_fw_base || !IS_ALIGNED(ave->ipc_fw_base, SZ_16K) || + ave->ipc_fw_base > U64_MAX - ave->ipc.size) + return -EPROTO; + + memset(ave->ipc.cpu + AVE_IPC_CHANNEL_OFF, 0, ave->channel_bytes); + memset(hs, 0, AVE_HS_BYTES); + memset(ave->ipc.cpu + AVE_IPC_SHARED_OFF, 0, AVE_IPC_SHARED_BYTES); + put_unaligned_le64(ave->ipc_fw_base + AVE_IPC_CHANNEL_OFF, + hs + AVE_HS_CHANNELS); + if (t->boot_config) { + put_unaligned_le64(ave->ipc_fw_base + AVE_IPC_SHARED_OFF, + hs + AVE_HS_SHARED); + put_unaligned_le32(AVE_IPC_SHARED_BYTES, + hs + AVE_HS_SHARED_BYTES); + put_unaligned_le64(ave->heap.dma, hs + AVE_HS_HEAP); + put_unaligned_le32(hello[3], hs + AVE_HS_HEAP_BYTES); + } + put_unaligned_le32(ave->variant->device_type, hs + AVE_HS_DEVICE_TYPE); + dma_wmb(); + + ave_mbox_send(ave, + lower_32_bits(ave->ipc_fw_base + AVE_IPC_HANDSHAKE_OFF), + upper_32_bits(ave->ipc_fw_base + AVE_IPC_HANDSHAKE_OFF), + 0, 0); + ret = ave_mbox_recv(ave, reply); + if (ret) + return ret; + if (ave_fw_offset(ave, (u64)reply[1] << 32 | reply[0], + ave->channel_bytes, &off) || + off != AVE_IPC_CHANNEL_OFF || + (t->boot_config && reply[2] > AVE_HELLO_EXTRA_MAX)) + return -EPROTO; + + ret = ave_ipc_parse_channels(ave); + if (ret) + return ret; + + writel(AVE_BOOT_MAGIC, ave->mbox + AVE_MBOX_READY); + return readl_poll_timeout(ave->mbox + AVE_MBOX_READY, status, !status, + 50, AVE_MBOX_TIMEOUT_US); +} + +static int ave_ring_send(struct ave_dev *ave, struct ave_ring *r, u64 addr, u32 len) +{ + __le64 *slot; + int next; + + if (r->producer < 0) + return -ENOSPC; + + slot = ave->ipc.cpu + r->offset + r->producer * AVE_RING_SLOT_BYTES; + WRITE_ONCE(slot[1], cpu_to_le64(len)); + WRITE_ONCE(slot[2], 0); + dma_wmb(); + WRITE_ONCE(slot[0], cpu_to_le64(addr | (r->mode ^ 1))); + dma_wmb(); + + if (r->consumer < 0) + r->consumer = r->producer; + next = (r->producer + 1) % r->slots; + r->producer = next == r->consumer ? -1 : next; + writel(BIT(r->doorbell), ave->mbox + AVE_MBOX_DOORBELL); + return 0; +} + +static int ave_ring_recv(struct ave_dev *ave, struct ave_ring *r, u64 *addr, u32 *len) +{ + __le64 *slot; + u64 word; + int next; + + if (r->consumer < 0) + return -EAGAIN; + + slot = ave->ipc.cpu + r->offset + r->consumer * AVE_RING_SLOT_BYTES; + word = le64_to_cpu(READ_ONCE(slot[0])); + if ((word & 1) != r->mode) + return -EAGAIN; + dma_rmb(); + *addr = word & ~3ULL; + *len = le64_to_cpu(READ_ONCE(slot[1])); + + if (r->producer < 0) + r->producer = r->consumer; + next = (r->consumer + 1) % r->slots; + r->consumer = next == r->producer ? -1 : next; + return 0; +} + +static int ave_ipc_buf_offset(struct ave_dev *ave, unsigned int buf, + size_t *max) +{ + const struct ave_fw_transport *t = ave->layout->abi->transport; + + if (buf == AVE_BUF_CMD) { + *max = t->command_bytes; + return t->command_offset; + } + if (buf < AVE_BUF_PROC(0)) { + *max = AVE_IPC_SCALAR_BYTES; + return t->scalar_offset + + (buf - AVE_BUF_SCALAR(0)) * AVE_IPC_SCALAR_BYTES; + } + *max = t->process_bytes; + return t->process_offset + (buf - AVE_BUF_PROC(0)) * t->process_bytes; +} + +/* Which command buffer does a transport ack refer to? */ +static u32 ave_ipc_command_end(struct ave_dev *ave) +{ + const struct ave_fw_transport *t = ave->layout->abi->transport; + + return t->process_offset + AVE_IPC_PROC_BUFS * t->process_bytes; +} + +static int ave_ipc_buf_of(struct ave_dev *ave, u64 addr) +{ + u64 off = addr - ave->ipc_fw_base; + size_t max; + + if (addr < ave->ipc_fw_base || + off < ave->layout->abi->transport->command_offset || + off >= ave_ipc_command_end(ave)) + return -EPROTO; + for (unsigned int buf = 0; buf < AVE_BUF_COUNT; buf++) + if (ave_ipc_buf_offset(ave, buf, &max) == off) + return buf; + return -EPROTO; +} + +/* + * Get command buffer @buf, cleared for @len bytes, or NULL while the + * firmware still owns it. Caller holds hw_lock. + */ +void *ave_ipc_cmd(struct ave_dev *ave, unsigned int buf, size_t len) +{ + size_t max; + int off; + + if (buf >= AVE_BUF_COUNT || (ave->unacked & BIT(buf))) + return NULL; + off = ave_ipc_buf_offset(ave, buf, &max); + if (len > max || off > ave->ipc.size || len > ave->ipc.size - off) + return NULL; + memset(ave->ipc.cpu + off, 0, len); + return ave->ipc.cpu + off; +} + +static bool ave_ipc_acked(struct ave_dev *ave, void *arg) +{ + return !(ave->unacked & BIT(*(unsigned int *)arg)); +} + +/* Wait until the firmware has taken command buffer @buf. Caller holds hw_lock. */ +int ave_ipc_wait_ack(struct ave_dev *ave, unsigned int buf) +{ + return ave_ipc_pump(ave, AVE_CMD_WAIT_MS, ave_ipc_acked, &buf); +} + +static bool ave_ipc_scalar_free(struct ave_dev *ave, void *arg) +{ + return (ave->unacked & GENMASK(AVE_BUF_PROC(0) - 1, AVE_BUF_SCALAR(0))) != + GENMASK(AVE_BUF_PROC(0) - 1, AVE_BUF_SCALAR(0)); +} + +/* Pick a free buffer for a scalar command. Caller holds hw_lock. */ +int ave_ipc_scalar_buf(struct ave_dev *ave) +{ + int ret; + + ret = ave_ipc_pump(ave, AVE_CMD_WAIT_MS, ave_ipc_scalar_free, NULL); + if (ret) + return ret; + for (unsigned int i = 0; i < AVE_IPC_SCALAR_BUFS; i++) { + unsigned int buf = AVE_BUF_SCALAR((ave->scalar_next + i) % AVE_IPC_SCALAR_BUFS); + + if (!(ave->unacked & BIT(buf))) { + ave->scalar_next = (buf - AVE_BUF_SCALAR(0) + 1) % AVE_IPC_SCALAR_BUFS; + return buf; + } + } + return -EBUSY; +} + +/* + * Hand the command in buffer @buf to the firmware. The timeout words are + * {duration, creation}, using the host clock for 9003 and the IOP clock + * for 6070. + * With @reply, the firmware's answer is counted as expected; Halt and + * Process are answered differently. Caller holds hw_lock. + */ +int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, + u64 timeout_us, bool reply) +{ + size_t max; + int off; + u8 *cmd; + const struct ave_fw_abi *abi = ave->layout->abi; + u32 wire_id; + enum ave_opcode op = 0; + u64 created; + int ret; + + if (buf >= AVE_BUF_COUNT) + return -EINVAL; + off = ave_ipc_buf_offset(ave, buf, &max); + if (len > max || off > ave->ipc.size || len > ave->ipc.size - off) + return -EINVAL; + cmd = ave->ipc.cpu + off; + wire_id = get_unaligned_le16(cmd); + for (unsigned int i = 1; i < AVE_OP_COUNT; i++) + if (wire_id && (abi->wire_id[0][i] == wire_id || + abi->wire_id[1][i] == wire_id)) { + op = i; + break; + } + lockdep_assert_held(&ave->hw_lock); + if (ave->fw_state != AVE_FW_RUNNING) { + dev_err_ratelimited(ave->dev, + "command %u not sent: firmware state %d\n", + op, ave->fw_state); + return -EIO; + } + if (!op || len < abi->header_bytes || len > max || + !IS_ALIGNED(len, 4) || (ave->unacked & BIT(buf)) || + (reply && op >= ARRAY_SIZE(ave->want))) + return -EINVAL; + + if (!abi->transport->boot_config && reply && ave->want[op]) + return -EBUSY; + if (abi->transport->sync_host_clock) { + created = div_u64(ktime_get_ns(), NSEC_PER_USEC); + } else { + ret = ave_iop_time_us(ave, &created); + if (ret) + return ret; + } + ave_stamp_command(abi, cmd, timeout_us, created); + if (!abi->transport->boot_config && reply) { + ave->expected[op].client = get_unaligned_le32(cmd + 0x10); + ave->expected[op].subtype = get_unaligned_le32(cmd + 0x1c); + } + + ret = ave_ring_send(ave, &ave->ring[0], ave->ipc_fw_base + off, len); + if (ret) { + ave_fw_mark_broken(ave, ret, "command ring full"); + return ret; + } + ave->unacked |= BIT(buf); + if (reply) + ave->want[op]++; + return 0; +} + +static int ave_ipc_dispatch(struct ave_dev *ave) +{ + const struct ave_fw_abi *abi = ave->layout->abi; + struct ave_reply reply; + int ret = abi->parse_reply(abi, ave->msg, ave->msg_len, &reply); + + if (ret) { + if (!abi->transport->boot_config) + return ret; + dev_warn(ave->dev, "short firmware message (%u bytes)\n", + ave->msg_len); + ave->reply_err = ret; + return 0; + } + if (reply.op == AVE_OP_PROCESS) { + if (!ave->layout->encode_ready) + return -EPROTO; + ave_sess_reply(ave, &reply); + return 0; + } + if (reply.op && ave->want[reply.op]) { + if (!abi->transport->boot_config && + (reply.client != ave->expected[reply.op].client || + reply.subtype != ave->expected[reply.op].subtype)) { + dev_err(ave->dev, + "reply %#x has wrong route: client %llu subtype %#x\n", + abi->reply_id[reply.op], reply.client, + reply.subtype); + return -EPROTO; + } + if (abi->transport->scratch_halt) + dev_dbg(ave->dev, + "reply %#x client %llu subtype %#x status %#x\n", + abi->reply_id[reply.op], reply.client, + reply.subtype, reply.raw_status); + ave->want[reply.op]--; + if (reply.status) { + dev_err(ave->dev, "command %u refused: status %#x\n", + reply.op, reply.raw_status); + ave->reply_err = -EIO; + } + return 0; + } + dev_warn_ratelimited(ave->dev, + "unexpected firmware message (op %u status %#x)\n", + reply.op, reply.raw_status); + return abi->transport->boot_config ? 0 : -EPROTO; +} + +/* + * Poll both rings once. Returns 1 when an event (ack or message) was + * handled, 0 when nothing was pending. + */ +static int ave_ipc_poll_once(struct ave_dev *ave) +{ + u32 pending, len, off; + u64 addr; + int ret, buf; + + pending = readl(ave->mbox + AVE_MBOX_STATUS); + if (pending) + writel(pending, ave->mbox + AVE_MBOX_STATUS); + + ret = ave_ring_recv(ave, &ave->ring[0], &addr, &len); + if (!ret) { + /* Transport acks return the command address with no payload. */ + buf = ave_ipc_buf_of(ave, addr); + if (len || buf < 0 || !(ave->unacked & BIT(buf))) { + dev_err(ave->dev, "bad command ack %#llx/%u\n", addr, + len); + return -EPROTO; + } + ave->unacked &= ~BIT(buf); + return 1; + } + if (ret != -EAGAIN) + return ret; + + ret = ave_ring_recv(ave, &ave->ring[1], &addr, &len); + if (ret == -EAGAIN) + return 0; + if (ret) + return ret; + if (len > sizeof(ave->msg) || ave_fw_offset(ave, addr, len, &off)) + return -EPROTO; + /* Messages must not land in the buffers the host writes commands to. */ + if (off < ave_ipc_command_end(ave) && + off + len > ave->layout->abi->transport->command_offset) { + dev_err(ave->dev, + "firmware message at %#x overlaps the command area\n", + off); + return -EPROTO; + } + memcpy(ave->msg, ave->ipc.cpu + off, len); + ave->msg_len = len; + /* Return the message buffer to the firmware. */ + ret = ave_ring_send(ave, &ave->ring[1], addr, len); + if (ret) + return ret; + ret = ave_ipc_dispatch(ave); + return ret ?: 1; +} + +/* + * Handle acks and messages until @cond holds, for at most @wait_ms. With no + * @cond, handle whatever is pending and return. Running out of time means + * the firmware is stuck: it is marked broken. Caller holds hw_lock. + * + * Which interrupt line signals the rings is not known, so this polls. It + * sleeps between polls: briefly at first (commands are acknowledged within + * microseconds), then for up to a millisecond, so a long wait for a frame + * costs next to no CPU time. + */ +#define AVE_POLL_FAST_ROUNDS 20 + +int ave_ipc_pump(struct ave_dev *ave, unsigned int wait_ms, + bool (*cond)(struct ave_dev *ave, void *arg), void *arg) +{ + ktime_t deadline = ktime_add_ms(ktime_get(), wait_ms); + unsigned int idle = 0; + int ret; + + lockdep_assert_held(&ave->hw_lock); + for (;;) { + if (cond && cond(ave, arg)) + return 0; + if (cond && ktime_after(ktime_get(), deadline)) { + ave_fw_mark_broken(ave, -ETIMEDOUT, "no answer"); + return -ETIMEDOUT; + } + if (ave->fw_state != AVE_FW_RUNNING) { + dev_err_ratelimited(ave->dev, "waiting on a firmware in state %d\n", + ave->fw_state); + return -EIO; + } + ret = ave_ipc_poll_once(ave); + if (ret < 0) { + ave_fw_mark_broken(ave, ret, "ring error"); + return ret; + } + if (ret) { + idle = 0; + continue; + } + if (!cond) + return 0; + if (idle++ < AVE_POLL_FAST_ROUNDS) + usleep_range(50, 100); + else + usleep_range(500, 1000); + } +} + +static bool ave_ipc_answered(struct ave_dev *ave, void *arg) +{ + u32 mask = *(u32 *)arg; + + if (ave->unacked & mask) + return false; + for (unsigned int i = 0; i < ARRAY_SIZE(ave->want); i++) + if (ave->want[i] && i != AVE_OP_HALT) + return false; + return true; +} + +/* + * Wait until every command except Process has been acknowledged and + * answered. Returns the first refusal seen since the last call. + * Caller holds hw_lock. + */ +int ave_ipc_settle(struct ave_dev *ave, unsigned int wait_ms) +{ + u32 mask = GENMASK(AVE_BUF_PROC(0) - 1, 0); + int ret; + + ret = ave_ipc_pump(ave, wait_ms, ave_ipc_answered, &mask); + if (!ret) + ret = ave->reply_err; + ave->reply_err = 0; + return ret; +} diff --git a/drivers/media/platform/apple/ave/ave-layout.c b/drivers/media/platform/apple/ave/ave-layout.c new file mode 100644 index 0000000000000..30d4d07d3abbd --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-layout.c @@ -0,0 +1,542 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Apple AVE command layouts, identified by firmware TEXT CRC. */ +#include "ave.h" + +const struct ave_fw_layout ave_layout_9003_78_0 = { + .name = "9003.78.0 / 25G83", + .abi = &ave_abi_9003, + .encode_ready = true, + .bytes = { { 0x3180, 0x63d8 }, { 0x13f28, 0xb1c0 } }, + .hdr_bytes = 0x12c000, + .meta_bytes = 0xc000, + .process_timeout_us = 46400000, + .process_subtype = 0x15, + .max_slots = 5, + .mcc_dsid = 0x12, + .start = { + [0] = { + .header = AVE_SURFACE(0x48, 0x10), + .secondary = AVE_SURFACE(0x58, 0x10), + .ref_a = AVE_SURFACE(0x390, 0x20), + .ref_b = AVE_SURFACE(0x3a0, 0x20), + .ref_c = AVE_SURFACE(0x7d0, 0x20), + .ref_c_alias = AVE_SURFACE(0x7e0, 0x20), + .ref_d = AVE_SURFACE(0x1130, 0x10), + .mb = AVE_SURFACE(0xc10, 0x10), + .coded = AVE_SURFACE(0xd50, 0x10), + .meta = AVE_SURFACE(0xf50, 0x10), + .row_a = AVE_SURFACE(0x1350, 0x10), + .row_b = AVE_SURFACE(0x1390, 0x10), + .fixed = AVE_SURFACE(0x13d0, 0x10), + .hevc_extra = AVE_SURFACE(0x2520, 0x10), + .tiles = { AVE_SURFACE(0x1410, 0x10), + AVE_SURFACE(0x1430, 0x10), + AVE_SURFACE(0x1490, 0x10), + AVE_SURFACE(0x14b0, 0x10), + AVE_SURFACE(0x1510, 0x10), + AVE_SURFACE(0x1530, 0x10), + AVE_SURFACE(0x1590, 0x10), + AVE_SURFACE(0x15b0, 0x10) }, + .codec = 0xc0, + .flag_8bit = 0x208, + .const_214 = 0x214, + .const_218 = 0x218, + .fps = 0x220, + .rc_mode = 0x234, + .bitrate = 0x238, + .init_qp = 0x240, + .limits = 0x250, + .limit_bytes = 0x258, + .limit_secs = 0x260, + .f64_288 = 0x288, + .f64_290 = 0x290, + .min_qp = 0x298, + .max_qp = 0x29c, + .gop_flags = 0x2a0, + .p_frames = 0x2ac, + .max_b = 0x2b0, + .ref_depth = 0x2b4, + .idr_interval = 0x2b8, + .f64_2c0 = 0x2c0, + .f64_2c8 = 0x2c8, + .rc_flags = 0x300, + .min_qp2 = 0x308, + .max_qp2 = 0x30c, + .rc_310 = 0x310, + .size_class = 0x320, + .coded_w = 0x368, + .coded_h = 0x36c, + .codec_flags = 0x370, + .h264_380 = 0x380, + .h264_384 = 0x384, + .codec_2588 = 0x2588, + .bit_depth = 0x2598, + .coded_h2 = 0x25dc, + .h264_profile = 0x291c, + .h264_level = 0x2938, + .h264_max_refs = 0x2d44, + .h264_width_mbs = 0x2d4c, + .h264_height_mbs = 0x2d50, + .h264_crop = 0x2db4, + .h264_crop_left = 0x2db8, + .h264_crop_right = 0x2dbc, + .h264_crop_top = 0x2dc0, + .h264_crop_bottom = 0x2dc4, + .h264_cabac = 0x2fd8, + .h264_8x8 = 0x3023, + .hevc_pad = 0x26e0, + .hevc_rc_270c = 0x270c, + .hevc_rc_9604 = 0x9604, + .hevc_rc_9608 = 0x9608, + .hevc_profile_a = 0x2938, + .hevc_compat_a = 0x293d, + .hevc_8bit_a = 0x2962, + .hevc_level_a = 0x296c, + .hevc_profile_b = 0x5904, + .hevc_compat_b = 0x5909, + .hevc_8bit_b = 0x592e, + .hevc_level_b = 0x5938, + .hevc_width = 0x5ab4, + .hevc_height = 0x5ab8, + .hevc_width_ctb = 0x5abc, + .hevc_height_ctb = 0x5ac0, + .hevc_conf = 0x5ac4, + .hevc_conf_left = 0x5ac8, + .hevc_conf_right = 0x5acc, + .hevc_conf_top = 0x5ad0, + .hevc_conf_bottom = 0x5ad4, + .hevc_depth_luma = 0x5ad8, + .hevc_depth_chroma = 0x5adc, + .hevc_refs = 0x259c, + .hevc_reorder_copy = 0x2e54, + .hevc_dpb_minus1 = 0x5b00, + .hevc_reorder = 0x5b04, + .hevc_reorder_on = 0x95fc, + }, + [1] = { + .header = AVE_SURFACE(0x48, 0x10), + .secondary = AVE_SURFACE(0x58, 0x10), + .ref_a = AVE_SURFACE(0x390, 0x20), + .ref_b = AVE_SURFACE(0x3a0, 0x20), + .ref_c = AVE_SURFACE(0x7d0, 0x20), + .ref_c_alias = AVE_SURFACE(0x7e0, 0x20), + .ref_d = AVE_SURFACE(0x1130, 0x10), + .mb = AVE_SURFACE(0xc10, 0x10), + .coded = AVE_SURFACE(0xd50, 0x10), + .meta = AVE_SURFACE(0xf50, 0x10), + .row_a = AVE_SURFACE(0x1350, 0x10), + .row_b = AVE_SURFACE(0x1390, 0x10), + .fixed = AVE_SURFACE(0x13d0, 0x10), + .hevc_extra = AVE_SURFACE(0x2520, 0x10), + .tiles = { AVE_SURFACE(0x1410, 0x10), + AVE_SURFACE(0x1430, 0x10), + AVE_SURFACE(0x1490, 0x10), + AVE_SURFACE(0x14b0, 0x10), + AVE_SURFACE(0x1510, 0x10), + AVE_SURFACE(0x1530, 0x10), + AVE_SURFACE(0x1590, 0x10), + AVE_SURFACE(0x15b0, 0x10) }, + .codec = 0xc0, + .flag_8bit = 0x208, + .const_214 = 0x214, + .const_218 = 0x218, + .fps = 0x220, + .rc_mode = 0x234, + .bitrate = 0x238, + .init_qp = 0x240, + .limits = 0x250, + .limit_bytes = 0x258, + .limit_secs = 0x260, + .f64_288 = 0x288, + .f64_290 = 0x290, + .min_qp = 0x298, + .max_qp = 0x29c, + .gop_flags = 0x2a0, + .p_frames = 0x2ac, + .max_b = 0x2b0, + .ref_depth = 0x2b4, + .idr_interval = 0x2b8, + .f64_2c0 = 0x2c0, + .f64_2c8 = 0x2c8, + .rc_flags = 0x300, + .min_qp2 = 0x308, + .max_qp2 = 0x30c, + .rc_310 = 0x310, + .size_class = 0x320, + .coded_w = 0x368, + .coded_h = 0x36c, + .codec_flags = 0x370, + .h264_380 = 0x380, + .h264_384 = 0x384, + .codec_2588 = 0x2588, + .bit_depth = 0x2598, + .coded_h2 = 0x25dc, + .h264_profile = 0x291c, + .h264_level = 0x2938, + .h264_max_refs = 0x2d44, + .h264_width_mbs = 0x2d4c, + .h264_height_mbs = 0x2d50, + .h264_crop = 0x2db4, + .h264_crop_left = 0x2db8, + .h264_crop_right = 0x2dbc, + .h264_crop_top = 0x2dc0, + .h264_crop_bottom = 0x2dc4, + .h264_cabac = 0x2fd8, + .h264_8x8 = 0x3023, + .hevc_pad = 0x26e0, + .hevc_rc_270c = 0x270c, + .hevc_rc_9604 = 0x9604, + .hevc_rc_9608 = 0x9608, + .hevc_profile_a = 0x2938, + .hevc_compat_a = 0x293d, + .hevc_8bit_a = 0x2962, + .hevc_level_a = 0x296c, + .hevc_profile_b = 0x5904, + .hevc_compat_b = 0x5909, + .hevc_8bit_b = 0x592e, + .hevc_level_b = 0x5938, + .hevc_width = 0x5ab4, + .hevc_height = 0x5ab8, + .hevc_width_ctb = 0x5abc, + .hevc_height_ctb = 0x5ac0, + .hevc_conf = 0x5ac4, + .hevc_conf_left = 0x5ac8, + .hevc_conf_right = 0x5acc, + .hevc_conf_top = 0x5ad0, + .hevc_conf_bottom = 0x5ad4, + .hevc_depth_luma = 0x5ad8, + .hevc_depth_chroma = 0x5adc, + .hevc_refs = 0x259c, + .hevc_reorder_copy = 0x2e54, + .hevc_dpb_minus1 = 0x5b00, + .hevc_reorder = 0x5b04, + .hevc_reorder_on = 0x95fc, + }, + }, + .process = { + [0] = { + .first_frame = 0x12c0, + .y = 0x5830, + .uv = 0x5850, + .in_w = 0x58e8, + .in_h = 0x58ec, + .coded_cap = 0x61c8, + .fps_int = 0x1610, + .bitrate = 0x1620, + .limits = 0x1628, + .limit_bytes = 0x1630, + .limit_secs = 0x1638, + .const5 = 0x6238, + .fps = 0x6268, + .timescale = 0x6278, + .slot = 0x61b4, + .coded = 0x61b8, + .meta = 0x61c0, + .ordinal = 0x6228, + .pts = 0x6270, + .force_key = 0x1658, + .force_type = 0x29f8, + .frame_buf = 0x58c8, + }, + [1] = { + .first_frame = 0x5d90, + .y = 0xa300, + .uv = 0xa320, + .in_w = 0xa3b8, + .in_h = 0xa3bc, + .coded_cap = 0xac98, + .fps_int = 0x60e0, + .bitrate = 0x60f0, + .limits = 0x60f8, + .limit_bytes = 0x6100, + .limit_secs = 0x6108, + .const5 = 0xad08, + .fps = 0xad38, + .timescale = 0xad48, + .slot = 0xac84, + .coded = 0xac88, + .meta = 0xac90, + .ordinal = 0xacf8, + .pts = 0xad40, + .force_key = 0x6128, + .force_type = 0x74c8, + .shdr_table = 0xd88, + .crop = 0x74dc, + .low_delay = 0xcf8, + }, + }, +}; + +const struct ave_fw_layout ave_layout_26a428 = { + .name = "26A428 (CRC unqualified)", + .abi = &ave_abi_9003, + .encode_ready = true, + .bytes = { { 0x3b40, 0x6df8 }, { 0x13ee0, 0x78b8 } }, + .hdr_bytes = 0xec000, + .meta_bytes = 0x10000, + .process_timeout_us = 46400000, + .process_subtype = 0x13, + .max_slots = 7, + .mcc_dsid = 0x12, + .counted = true, + .start = { + [0] = { + .header = AVE_SURFACE(0x48, 0x10), + .secondary = AVE_SURFACE(0x58, 0x10), + .ref_a = AVE_SURFACE(0x98, 0x40), + .ref_b = AVE_SURFACE(0xa8, 0x40), + .ref_c = AVE_SURFACE(0xfe8, 0x40), + .ref_c_alias = AVE_SURFACE(0xff8, 0x40), + .ref_d = AVE_SURFACE(0xdc0, 0x10), + .ref_e = AVE_SURFACE(0xb8, 0x40), + .ref_f = AVE_SURFACE(0xc8, 0x40), + .mb = AVE_SURFACE(0x1870, 0x10), + .coded = AVE_SURFACE(0x920, 0x10), + .meta = AVE_SURFACE(0xb08, 0x10), + .row_a = AVE_SURFACE(0x19c0, 0x10), + .row_b = AVE_SURFACE(0x1a08, 0x10), + .fixed = AVE_SURFACE(0x1a50, 0x10), + .hevc_extra = AVE_SURFACE(0x70, 0x10), + .tiles = { AVE_SURFACE(0x1bb8, 0x10), + AVE_SURFACE(0x1bd8, 0x10), + AVE_SURFACE(0x1db8, 0x10), + AVE_SURFACE(0x1dd8, 0x10), + AVE_SURFACE(0x1bf8, 0x10), + AVE_SURFACE(0x1c18, 0x10), + AVE_SURFACE(0x1df8, 0x10), + AVE_SURFACE(0x1e18, 0x10) }, + .refs_count = 0x94, + .work_count = 0xdbc, + .lfs_count = 0xfe4, + .slots_count = 0x918, + .meta_count = 0xb00, + .mb_count = 0x1868, + .row_a_count = 0x19b8, + .row_b_count = 0x1a00, + .fixed_count = 0x1a48, + .tiles_count = 0x1bb4, + .codec = 0x3248, + .flag_8bit = 0x35e8, + .const_214 = 0x35f4, + .const_218 = 0x35f8, + .fps = 0x3600, + .rc_mode = 0x3614, + .bitrate = 0x3618, + .init_qp = 0x3620, + .limits = 0x3630, + .limit_bytes = 0x3638, + .limit_secs = 0x3640, + .f64_288 = 0x3668, + .f64_290 = 0x3670, + .min_qp = 0x3678, + .max_qp = 0x367c, + .gop_flags = 0x36d0, + .p_frames = 0x36dc, + .max_b = 0x36e0, + .ref_depth = 0x36e4, + .idr_interval = 0x36e8, + .f64_2c0 = 0x36f0, + .f64_2c8 = 0x36f8, + .rc_flags = 0x3734, + .min_qp2 = 0x3740, + .max_qp2 = 0x3744, + .rc_310 = 0x3748, + .size_class = 0x3760, + .coded_w = 0x322c, + .coded_h = 0x3230, + .coded_h2 = 0x32a0, + .bit_depth = 0x383c, + .h264_profile = 0x37b0, + .h264_level = 0x37b8, + .h264_max_refs = 0x37d0, + .h264_crop = 0x3800, + .h264_crop_right = 0x3804, + .h264_crop_bottom = 0x3808, + .h264_cabac = 0x380c, + .h264_8x8 = 0x3823, + .geometry_mode = 0x31f8, + .geometry_depth = 0x3234, + .profile_idc = 0x3a98, + .level_idc = 0x3aa0, + .max_refs_copy = 0x3abe, + .mb_size = 0x3ac0, + .mb_height = 0x37d4, + .crop_copy = 0x3ac7, + .crop_right_copy = 0x3aca, + .crop_bottom_copy = 0x3ace, + .codec_byte = 0x3830, + .cabac_copy = 0x3af9, + .transform_copy = 0x3b0d, + }, + [1] = { + .header = AVE_SURFACE(0x48, 0x10), + .secondary = AVE_SURFACE(0x58, 0x10), + .ref_a = AVE_SURFACE(0x98, 0x40), + .ref_b = AVE_SURFACE(0xa8, 0x40), + .ref_c = AVE_SURFACE(0xfe8, 0x40), + .ref_c_alias = AVE_SURFACE(0xff8, 0x40), + .ref_d = AVE_SURFACE(0xdc0, 0x10), + .ref_e = AVE_SURFACE(0xb8, 0x40), + .ref_f = AVE_SURFACE(0xc8, 0x40), + .mb = AVE_SURFACE(0x1870, 0x10), + .coded = AVE_SURFACE(0x920, 0x10), + .meta = AVE_SURFACE(0xb08, 0x10), + .row_a = AVE_SURFACE(0x19c0, 0x10), + .row_b = AVE_SURFACE(0x1a08, 0x10), + .fixed = AVE_SURFACE(0x1a50, 0x10), + .hevc_extra = AVE_SURFACE(0x70, 0x10), + .tiles = { AVE_SURFACE(0x1bb8, 0x10), + AVE_SURFACE(0x1bd8, 0x10), + AVE_SURFACE(0x1db8, 0x10), + AVE_SURFACE(0x1dd8, 0x10), + AVE_SURFACE(0x1bf8, 0x10), + AVE_SURFACE(0x1c18, 0x10), + AVE_SURFACE(0x1df8, 0x10), + AVE_SURFACE(0x1e18, 0x10) }, + .refs_count = 0x94, + .work_count = 0xdbc, + .lfs_count = 0xfe4, + .slots_count = 0x918, + .meta_count = 0xb00, + .mb_count = 0x1868, + .row_a_count = 0x19b8, + .row_b_count = 0x1a00, + .fixed_count = 0x1a48, + .tiles_count = 0x1bb4, + .codec = 0x3248, + .flag_8bit = 0x35e8, + .const_214 = 0x35f4, + .const_218 = 0x35f8, + .fps = 0x3600, + .rc_mode = 0x3614, + .bitrate = 0x3618, + .init_qp = 0x3620, + .limits = 0x3630, + .limit_bytes = 0x3638, + .limit_secs = 0x3640, + .f64_288 = 0x3668, + .f64_290 = 0x3670, + .min_qp = 0x3678, + .max_qp = 0x367c, + .gop_flags = 0x36d0, + .p_frames = 0x36dc, + .max_b = 0x36e0, + .ref_depth = 0x36e4, + .idr_interval = 0x36e8, + .f64_2c0 = 0x36f0, + .f64_2c8 = 0x36f8, + .rc_flags = 0x3734, + .min_qp2 = 0x3740, + .max_qp2 = 0x3744, + .rc_310 = 0x3748, + .size_class = 0x3760, + .coded_w = 0x322c, + .coded_h = 0x3230, + .coded_h2 = 0x32a0, + .bit_depth = 0x37c0, + .hevc_pad = 0x37ec, + .hevc_refs = 0x37c4, + .hevc_rc_9604 = 0x9dec, + .hevc_rc_9608 = 0x9df0, + .hevc_profile_a = 0x3a38, + .hevc_compat_a = 0x3a3d, + .hevc_8bit_a = 0x3a62, + .hevc_level_a = 0x3a6c, + .hevc_profile_b = 0x6508, + .hevc_compat_b = 0x650d, + .hevc_8bit_b = 0x6532, + .hevc_level_b = 0x653c, + .hevc_width = 0x66b8, + .hevc_height = 0x66bc, + .hevc_width_ctb = 0x66c0, + .hevc_height_ctb = 0x66c4, + .hevc_conf = 0x66c8, + .hevc_conf_left = 0x66cc, + .hevc_conf_right = 0x66d0, + .hevc_conf_top = 0x66d4, + .hevc_conf_bottom = 0x66d8, + .hevc_depth_luma = 0x66dc, + .hevc_depth_chroma = 0x66e0, + .hevc_reorder_copy = 0x495c, + .hevc_dpb_minus1 = 0x6704, + .hevc_reorder = 0x6708, + .geometry_mode = 0x31f8, + .geometry_depth = 0x3234, + .hevc_cq = 0x3708, + .hevc_cq_copy = 0xbefc, + .hevc_cq_codec = 0x37c8, + .hevc_cq_picture = 0x9e38, + }, + }, + .process = { + [0] = { + .first_frame = 0xdc0, + .fps_int = 0x1328, + .in_w = 0x5648, + .in_h = 0x564c, + .ordinal = 0x5650, + .const5 = 0x5660, + .fps = 0x5758, + .pts = 0x5760, + .timescale = 0x5768, + .slot = 0x57b8, + .y = 0x57c0, + .uv = 0x57e0, + .coded = 0x58a0, + .coded_cap = 0x58a8, + .meta = 0x58b0, + .meta_size = 0x58b8, + .extra = 0x58c0, + .frame_buf = 0x58d8, + .frame_size = 0x58e0, + }, + [1] = { + .first_frame = 0x1568, + .fps_int = 0x1ad0, + .in_w = 0x5df0, + .in_h = 0x5df4, + .ordinal = 0x5df8, + .const5 = 0x5e08, + .fps = 0x5f00, + .pts = 0x5f08, + .timescale = 0x5f10, + .slot = 0x5f60, + .y = 0x5f68, + .uv = 0x5f88, + .coded = 0x6048, + .coded_cap = 0x6050, + .meta = 0x6058, + .meta_size = 0x6060, + .extra = 0x6068, + .crop = 0x3190, + }, + }, +}; + +/* Lifecycle is qualified from the 6070 spec; payload field ledger is pending. + */ +const struct ave_fw_layout ave_layout_6070_11_1 = { + .name = "6070.11.1 (lifecycle only)", + .abi = &ave_abi_6070, + .bytes = { { 0x10e10, 0x1940 }, { 0x32dc8, 0x6838 } }, + .start = { + {.header = AVE_SURFACE(0x40, 0x10), + .secondary = AVE_SURFACE(0x50, 0x10) }, + {.header = AVE_SURFACE(0x40, 0x10), + .secondary = AVE_SURFACE(0x50, 0x10) }, + }, +}; + +const struct ave_fw_layout *ave_layout_for_crc(u32 crc) +{ + /* + * The M4 Pro gets the same 25G83 firmware build as the A18 Pro, built + * for its own chip, and speaks the same command layout. + */ + if (crc == AVE_FW_TEXT_CRC_9003_78_0 || crc == AVE_FW_TEXT_CRC_T6040_25G83) + return &ave_layout_9003_78_0; + if (crc == 0x460c7a7c || crc == 0x25bfd039) + return &ave_layout_6070_11_1; + /* 26A428 has no verified loader TEXT CRC: no placeholder or fallback. */ + return NULL; +} diff --git a/drivers/media/platform/apple/ave/ave-layout.h b/drivers/media/platform/apple/ave/ave-layout.h new file mode 100644 index 0000000000000..24f608fabfd9b --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-layout.h @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _APPLE_AVE_LAYOUT_H_ +#define _APPLE_AVE_LAYOUT_H_ + +struct ave_ctx; +struct ave_dev; +struct ave_fw_layout; +struct ave_fw_abi; + +struct ave_command { + enum ave_opcode op; + enum ave_codec codec; + u64 token; + u32 client, subtype, arg0, arg2; +}; + +/* Status is normalized to zero for success by the ABI parser. */ +struct ave_reply { + enum ave_opcode op; + u64 client; + u32 subtype, status, raw_status, bytes; +}; + +struct ave_fw_transport { + u32 ipc_bytes; + u32 command_offset, command_bytes, scalar_offset; + u32 process_offset, process_bytes; + /* Zero channel/heap sizes mean bounded negotiation on 9003. */ + u32 hello[4]; + u32 descriptors; + u32 io_index, t2h_index; + u32 slots[2], doorbells[2], offsets[2]; /* zero: discovered */ + bool boot_config; + bool sync_host_clock; + bool scratch_halt; +}; + +struct ave_fw_abi { + const struct ave_fw_transport *transport; + u32 wire_id[2][AVE_OP_COUNT]; + u32 reply_id[AVE_OP_COUNT]; + u32 scalar_bytes[AVE_OP_COUNT]; + u32 semantic[AVE_OP_COUNT]; + u32 header_bytes, timeout_offset, created_offset; + bool padded_surfaces; + void (*write_header)(u8 *cmd, const struct ave_fw_abi *abi, + const struct ave_command *header); + void (*write_config)(struct ave_dev *ave, u8 *cmd); + int (*parse_reply)(const struct ave_fw_abi *abi, const u8 *msg, u32 len, + struct ave_reply *reply); +}; + +/* Separate address/size offsets and strides also represent split arrays. */ +struct ave_surface_layout { + u32 address, size; + u32 address_stride, size_stride; +}; + +#define AVE_SURFACE(off, stride) \ + { .address = (off), \ + .size = (off) + 8, \ + .address_stride = (stride), \ + .size_stride = (stride) } + +/* Zero offsets denote fields absent from a firmware's wire format. */ +struct ave_start_layout { + struct ave_surface_layout header, secondary; + struct ave_surface_layout ref_a, ref_b, ref_c, ref_c_alias, ref_d; + struct ave_surface_layout ref_e, ref_f; + struct ave_surface_layout mb, coded, meta, row_a, row_b, fixed, + hevc_extra; + struct ave_surface_layout tiles[8]; + u32 refs_count, work_count, lfs_count, slots_count, meta_count; + u32 mb_count, row_a_count, row_b_count, fixed_count, tiles_count; + u32 codec; + u32 flag_8bit; + u32 const_214; + u32 const_218; + u32 fps; + u32 rc_mode; + u32 bitrate; + u32 init_qp; + u32 limits; + u32 limit_bytes; + u32 limit_secs; + u32 f64_288; + u32 f64_290; + u32 min_qp; + u32 max_qp; + u32 gop_flags; + u32 p_frames; + u32 max_b; + u32 ref_depth; + u32 idr_interval; + u32 f64_2c0; + u32 f64_2c8; + u32 rc_flags; + u32 min_qp2; + u32 max_qp2; + u32 rc_310; + u32 size_class; + u32 coded_w; + u32 coded_h; + u32 codec_flags; + u32 h264_380; + u32 h264_384; + u32 codec_2588; + u32 bit_depth; + u32 coded_h2; + u32 h264_profile; + u32 h264_level; + u32 h264_max_refs; + u32 h264_width_mbs; + u32 h264_height_mbs; + u32 h264_crop; + u32 h264_crop_left; + u32 h264_crop_right; + u32 h264_crop_top; + u32 h264_crop_bottom; + u32 h264_cabac; + u32 h264_8x8; + u32 hevc_pad; + u32 hevc_rc_270c; + u32 hevc_rc_9604; + u32 hevc_rc_9608; + u32 hevc_profile_a; + u32 hevc_compat_a; + u32 hevc_8bit_a; + u32 hevc_level_a; + u32 hevc_profile_b; + u32 hevc_compat_b; + u32 hevc_8bit_b; + u32 hevc_level_b; + u32 hevc_width; + u32 hevc_height; + u32 hevc_width_ctb; + u32 hevc_height_ctb; + u32 hevc_conf; + u32 hevc_conf_left; + u32 hevc_conf_right; + u32 hevc_conf_top; + u32 hevc_conf_bottom; + u32 hevc_depth_luma; + u32 hevc_depth_chroma; + u32 hevc_refs; + u32 hevc_reorder_copy; + u32 hevc_dpb_minus1; + u32 hevc_reorder; + u32 hevc_reorder_on; + u32 geometry_mode, geometry_depth; + u32 profile_idc, level_idc, max_refs_copy, mb_size, mb_height; + u32 crop_copy, crop_right_copy, crop_bottom_copy; + u32 codec_byte, cabac_copy, transform_copy, hevc_cq, hevc_cq_copy; + u32 hevc_cq_codec, hevc_cq_picture; +}; + +struct ave_proc_layout { + u32 first_frame; + u32 y, uv; /* {u64 address, u32 size, u32 stride} */ + u32 in_w, in_h; + u32 coded_cap; + u32 fps_int; + u32 bitrate; + u32 limits, limit_bytes, limit_secs; + u32 const5; + u32 fps; /* f64 */ + u32 timescale; /* u64 */ + u32 slot; + u32 coded, meta; /* u64 addresses */ + u32 ordinal; + u32 pts; /* u64 */ + u32 force_key; /* 1 on a forced key frame */ + u32 force_type; /* 2 on a forced key frame */ + u32 frame_buf; /* H.264 only */ + u32 shdr_table; /* HEVC only */ + u32 crop; /* HEVC only */ + u32 meta_size, extra, frame_size; + u32 low_delay; /* HEVC only: 3 with reordering off */ +}; + +struct ave_fw_layout { + const char *name; + const struct ave_fw_abi *abi; + bool encode_ready; + size_t bytes[2][AVE_IMAGE_COUNT]; + struct ave_start_layout start[2]; + struct ave_proc_layout process[2]; + u32 hdr_bytes, meta_bytes; + u32 process_timeout_us; + u8 process_subtype, max_slots, mcc_dsid; + bool counted; +}; + +extern const struct ave_fw_abi ave_abi_9003, ave_abi_6070; +extern const struct ave_fw_layout ave_layout_9003_78_0; +extern const struct ave_fw_layout ave_layout_6070_11_1; +/* Not selectable until its loader TEXT CRC is measured and qualified. */ +extern const struct ave_fw_layout ave_layout_26a428; +const struct ave_fw_layout *ave_layout_for_crc(u32 crc); + +#endif diff --git a/drivers/media/platform/apple/ave/ave-recovery-test.c b/drivers/media/platform/apple/ave/ave-recovery-test.c new file mode 100644 index 0000000000000..0dbcd1b0be219 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-recovery-test.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "ave-recovery.h" + +static void ave_recovery_limit_test(struct kunit *test) +{ + struct ave_recovery_history h = {}; + + for (unsigned int i = 0; i < AVE_RECOVERY_ATTEMPTS; i++) + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, i)); + KUNIT_EXPECT_FALSE(test, ave_recovery_allow(&h, 3)); + KUNIT_EXPECT_FALSE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW - 1)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW)); + KUNIT_EXPECT_FALSE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW + 1)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW + 2)); + KUNIT_EXPECT_FALSE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW + 3)); +} + +static void ave_recovery_sliding_test(struct kunit *test) +{ + struct ave_recovery_history h = {}; + + KUNIT_ASSERT_TRUE(test, ave_recovery_allow(&h, 0)); + KUNIT_ASSERT_TRUE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW / 2)); + KUNIT_ASSERT_TRUE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW - 1)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW)); + KUNIT_EXPECT_FALSE(test, ave_recovery_allow(&h, AVE_RECOVERY_WINDOW + 1)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, 2 * AVE_RECOVERY_WINDOW)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, 2 * AVE_RECOVERY_WINDOW)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, 2 * AVE_RECOVERY_WINDOW)); + KUNIT_EXPECT_FALSE(test, ave_recovery_allow(&h, 2 * AVE_RECOVERY_WINDOW)); +} + +static void ave_recovery_wrap_test(struct kunit *test) +{ + struct ave_recovery_history h = {}; + unsigned long start = ~0UL - AVE_RECOVERY_WINDOW / 2; + + for (unsigned int i = 0; i < AVE_RECOVERY_ATTEMPTS; i++) + KUNIT_ASSERT_TRUE(test, ave_recovery_allow(&h, start)); + KUNIT_EXPECT_FALSE(test, ave_recovery_allow(&h, start + AVE_RECOVERY_WINDOW - 1)); + KUNIT_EXPECT_TRUE(test, ave_recovery_allow(&h, start + AVE_RECOVERY_WINDOW)); +} + +static struct kunit_case ave_recovery_cases[] = { + KUNIT_CASE(ave_recovery_limit_test), + KUNIT_CASE(ave_recovery_sliding_test), + KUNIT_CASE(ave_recovery_wrap_test), + {} +}; + +static struct kunit_suite ave_recovery_suite = { + .name = "apple-ave-recovery", + .test_cases = ave_recovery_cases, +}; + +kunit_test_suite(ave_recovery_suite); + +MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/ave/ave-recovery.h b/drivers/media/platform/apple/ave/ave-recovery.h new file mode 100644 index 0000000000000..93de1e2471e75 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-recovery.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _APPLE_AVE_RECOVERY_H_ +#define _APPLE_AVE_RECOVERY_H_ + +#include +#include + +#define AVE_RECOVERY_ATTEMPTS 3 +#define AVE_RECOVERY_WINDOW (10 * 60 * HZ) + +struct ave_recovery_history { + unsigned long attempts[AVE_RECOVERY_ATTEMPTS]; + unsigned int next; + unsigned int count; +}; + +/* A sliding window, including across jiffies wrap. Caller serialises access. */ +static inline bool ave_recovery_allow(struct ave_recovery_history *h, unsigned long now) +{ + if (h->count == AVE_RECOVERY_ATTEMPTS && + time_before(now, h->attempts[h->next] + AVE_RECOVERY_WINDOW)) + return false; + h->attempts[h->next] = now; + h->next = (h->next + 1) % AVE_RECOVERY_ATTEMPTS; + if (h->count < AVE_RECOVERY_ATTEMPTS) + h->count++; + return true; +} + +#endif diff --git a/drivers/media/platform/apple/ave/ave-regs.h b/drivers/media/platform/apple/ave/ave-regs.h new file mode 100644 index 0000000000000..8c910069b4b02 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-regs.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Apple AVE video encoder: registers and firmware interface constants + * + * Copyright The Aurora Silicon Contributors + */ +#ifndef _APPLE_AVE_REGS_H_ +#define _APPLE_AVE_REGS_H_ + +#include + +/* Mailbox window */ +#define AVE_MBOX_DOORBELL 0x0c +#define AVE_MBOX_STATUS 0x10 +#define AVE_MBOX_STATUS_PENDING BIT(0) +#define AVE_MBOX_SCRATCH(n) (0x18 + (n) * 4) +#define AVE_MBOX_SCRATCH_COUNT 8 +#define AVE_MBOX_READY AVE_MBOX_SCRATCH(3) +#define AVE_MBOX_TIME_LO AVE_MBOX_SCRATCH(4) +#define AVE_MBOX_TIME_HI AVE_MBOX_SCRATCH(5) +#define AVE_MBOX_TIMEOUT_US 5000000 + +/* Coprocessor window */ +#define AVE_COPROC_VCBAR 0x50000 +#define AVE_COPROC_VCBAR_VALID BIT(0) +#define AVE_COPROC_TIMEBASE_FREQ 0x360020 +#define AVE_COPROC_TIMEBASE 0x378000 +#define AVE_COPROC_CTRL 0x600044 +#define AVE_COPROC_CTRL_RUN BIT(4) +#define AVE_COPROC_STATUS 0x600048 +#define AVE_COPROC_BOOT_ARG 0x600400 +#define AVE_COPROC_BOOT_ARG_VALUE 0x10000 +#define AVE_COPROC_KEEP_MAPPINGS 0x600808 + +/* Boot request and the configuration block it points at */ +#define AVE_BOOT_MAGIC 0x08042006 +#define AVE_BOOT_CFG_BYTES 0x40 +#define AVE_BOOT_CFG_DEVICE 1 /* word index */ +#define AVE_BOOT_CFG_UNITS 2 +#define AVE_BOOT_CFG_GROUPS 3 +#define AVE_BOOT_CFG_SUBID_MASK 4 +#define AVE_BOOT_CFG_LOG_LO 8 +#define AVE_BOOT_CFG_LOG_HI 9 +#define AVE_BOOT_CFG_LOG_BYTES 10 +#define AVE_BOOT_CFG_OPTIONS 12 +#define AVE_BOOT_CFG_OPTIONS_VALUE 0x77 + +/* Firmware log: 512-byte header, then a 128 KiB ring. Mapping is page sized. */ +#define AVE_LOG_BYTES 0x24000 +#define AVE_LOG_HDR_USED 0x20200 +#define AVE_LOG_HDR_BYTES 0x200 +#define AVE_LOG_RING_BYTES 0x20000 + +/* Hello reply: {version, channel area bytes, descriptor bytes, heap bytes} */ +#define AVE_HELLO_VERSION 2 +#define AVE_HELLO_DESC_BYTES 0x100 +#define AVE_HELLO_EXTRA_MAX 0x13c000 + +/* Handshake block in the IPC window */ +#define AVE_HS_BYTES 0x50 +#define AVE_HS_CHANNELS 0x08 +#define AVE_HS_SHARED 0x10 +#define AVE_HS_SHARED_BYTES 0x18 +#define AVE_HS_HEAP 0x1c +#define AVE_HS_HEAP_BYTES 0x24 +#define AVE_HS_DEVICE_TYPE 0x28 + +/* Channel descriptors the firmware writes into the channel area */ +#define AVE_CHAN_DESC_STRIDE 0x100 +#define AVE_CHAN_DESC_NAME_LEN 0x40 +#define AVE_CHAN_DESC_DIR 0x40 +#define AVE_CHAN_DESC_IRQ 0x44 +#define AVE_CHAN_DESC_SLOTS 0x48 +#define AVE_CHAN_DESC_ADDR 0x4c +#define AVE_RING_SLOT_BYTES 64 + +/* 9003 command header (also used by the frozen regression oracle). */ +#define AVE_CMD_OPCODE 0x00 +#define AVE_CMD_TOKEN 0x08 +#define AVE_CMD_CLIENT 0x10 +#define AVE_CMD_UNIT 0x18 +#define AVE_CMD_CODEC 0x1c +#define AVE_CMD_OPCODE_ECHO 0x20 +#define AVE_CMD_ARG0 0x24 +#define AVE_CMD_ARG1 0x28 +#define AVE_CMD_ARG2 0x2c +#define AVE_CMD_TIMEOUT 0x30 +#define AVE_CMD_CREATED 0x38 +#define AVE_CMD_HDR_BYTES 0x48 + +/* Timeouts the firmware is given, and how long the host waits */ +#define AVE_CMD_GLOBAL_TIMEOUT_US 3000000ULL +#define AVE_CMD_SESSION_TIMEOUT_US 94400000ULL +#define AVE_CMD_WAIT_MS 5000 +#define AVE_START_WAIT_MS 10000 +#define AVE_FRAME_WAIT_MS 5000 +/* Halt is never answered; give the firmware this long before power-off. */ +#define AVE_HALT_SETTLE_MS 20 + +/* 9003 firmware-to-host messages */ +#define AVE_REPLY_ID 0x00 +#define AVE_REPLY_CLIENT 0x10 +#define AVE_REPLY_SUBTYPE 0x20 +#define AVE_REPLY_STATUS 0x40 +#define AVE_REPLY_MIN_BYTES 0x48 +#define AVE_REPLY_PROCESS_BYTES 0x50 + +#define AVE_HALT_ALL 0xffffffff + +/* Session command header values */ +#define AVE_SESSION_UNIT 1 +#define AVE_SESSION_ARG0 0xc8 +#define AVE_SESSION_ARG1 0x36 +#define AVE_SESSION_ARG2_SETUP 0x1e0 /* Open and Start */ +#define AVE_SESSION_ARG2_SETUP_SHARED 0x140 /* the same, another session open */ +#define AVE_SESSION_ARG2_H264 0x40 /* every other session command */ +#define AVE_SESSION_ARG2_HEVC 0x20 +#define AVE_TOKEN_STEP 100 +#define AVE_TOKEN_BASE 300 /* the first Open carries 400 */ +#define AVE_CLIENT_STEP 10 + +/* Completes sent to drain a session */ +#define AVE_DRAIN_COMPLETES 4 + +#define AVE_CONFIG_BYTES 0x78 +#define AVE_CONFIG_ALL_UNITS 0x20 +#define AVE_CONFIG_OPT_A 0x49 +#define AVE_CONFIG_OPT_B 0x4a +#define AVE_CONFIG_MCC_DSID 0x60 +#define AVE_CONFIG_MCC_DSID_T8140 0x12 + +#endif diff --git a/drivers/media/platform/apple/ave/ave-sess.c b/drivers/media/platform/apple/ave/ave-sess.c new file mode 100644 index 0000000000000..f46f8128667bf --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-sess.c @@ -0,0 +1,849 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder: encoding sessions + * + * A session owns the encoder's working buffers (sized from the picture + * size), a ring of up to five Process slots and the coded output of each + * slot. The firmware answers frames in bitstream order, so the N-th answer + * is the N-th access unit; with B-frames that is not the input order. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "ave.h" +#include "ave-regs.h" + +#define AVE_RC_VBR 4 +#define AVE_RC_CBR 2 + +/* Filler-data NAL unit types */ +#define AVE_H264_NAL_FILLER 12 +#define AVE_HEVC_NAL_FILLER 38 + +#define AVE_H264_BASELINE 2 +#define AVE_H264_MAIN 4 +#define AVE_H264_HIGH 6 +#define AVE_HEVC_MAIN 1 + +#define AVE_META_BYTES 0xc000 +/* Metadata words: coded bytes (twice), HEVC slice-header bytes, parameter sets */ +#define AVE_META_PAYLOAD 0x120 +#define AVE_META_HEVC_SHDR 0x128 +#define AVE_META_PAYLOAD2 0x12c +#define AVE_META_PARAMS 0xb2a4 +#define AVE_META_SCAN_FROM 0x200 +#define AVE_SHDR_BYTES (256 * 0x400) +#define AVE_SHDR_ENTRY_BYTES 0x400 +#define AVE_CODED_MIN 0x3c000 + +/* Level limits: max frame size, max rate, max bit rate (kbit/s, Main). */ +struct ave_level { + u32 max_fs; + u64 max_rate; + u32 max_kbps; +}; + +/* H.264 Table A-1, in V4L2 menu order (1, 1b, 1.1 ... 5.2); macroblocks. */ +static const struct ave_level ave_h264_levels[] = { + { 99, 1485, 64 }, { 99, 1485, 128 }, + { 396, 3000, 192 }, { 396, 6000, 384 }, + { 396, 11880, 768 }, { 396, 11880, 2000 }, + { 792, 19800, 4000 }, { 1620, 20250, 4000 }, + { 1620, 40500, 10000 }, { 3600, 108000, 14000 }, + { 5120, 216000, 20000 }, { 8192, 245760, 20000 }, + { 8192, 245760, 50000 }, { 8704, 522240, 50000 }, + { 22080, 589824, 135000 }, { 36864, 983040, 240000 }, + { 36864, 2073600, 240000 }, +}; + +/* HEVC Table A-8 (Main tier), in V4L2 menu order (1, 2, 2.1 ... 5.1); samples. */ +static const struct ave_level ave_hevc_levels[] = { + { 36864, 552960, 128 }, { 122880, 3686400, 1500 }, + { 245760, 7372800, 3000 }, { 552960, 16588800, 6000 }, + { 983040, 33177600, 10000 }, { 2228224, 66846720, 12000 }, + { 2228224, 133693440, 20000 }, { 8912896, 267386880, 25000 }, + { 8912896, 534773760, 40000 }, +}; + +/* + * The lowest level that fits the picture, rate and bit rate, or the + * requested one if that is higher. Returned as the firmware's 1-based enum. + */ +static int ave_pick_level(const struct ave_params *p, u32 requested) +{ + bool hevc = p->codec == AVE_CODEC_HEVC; + const struct ave_level *t = hevc ? ave_hevc_levels : ave_h264_levels; + unsigned int n = hevc ? ARRAY_SIZE(ave_hevc_levels) : ARRAY_SIZE(ave_h264_levels); + u64 size = hevc ? (u64)p->cw * p->ch : p->mbs; + u64 side = hevc ? max(p->cw, p->ch) : p->cw / 16; + u64 rate = DIV_ROUND_UP_ULL(size * p->fps_num, p->fps_den); + u32 kbps = DIV_ROUND_UP(p->bitrate, 1000); + + for (unsigned int i = 0; i < n; i++) { + u32 max_kbps = t[i].max_kbps; + + if (p->high) + max_kbps += max_kbps / 4; + if (size <= t[i].max_fs && side * side <= 8ULL * t[i].max_fs && + rate <= t[i].max_rate && kbps <= max_kbps) + return max(i, requested) + 1; + } + return -EINVAL; +} + +static int ave_sess_params(struct ave_ctx *ctx, struct ave_params *p) +{ + bool baseline = false; + int level; + + p->codec = ave_ctx_codec(ctx); + p->width = ctx->src_fmt.width; + p->height = ctx->src_fmt.height; + p->cw = ALIGN(p->width, 16); + p->ch = ALIGN(p->height, 16); + p->stride = ctx->src_fmt.plane_fmt[0].bytesperline; + p->mbs = (p->cw / 16) * (p->ch / 16); + p->fps_num = ctx->timeperframe.denominator; + p->fps_den = ctx->timeperframe.numerator; + p->fps_int = max(DIV_ROUND_CLOSEST(p->fps_num, p->fps_den), 1U); + + if (p->codec == AVE_CODEC_H264) { + switch (ctx->h264_profile) { + case V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE: + case V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE: + p->profile = AVE_H264_BASELINE; + baseline = true; + break; + case V4L2_MPEG_VIDEO_H264_PROFILE_HIGH: + p->profile = AVE_H264_HIGH; + p->high = true; + break; + default: + p->profile = AVE_H264_MAIN; + break; + } + p->min_qp = ctx->h264_min_qp; + p->max_qp = ctx->h264_max_qp; + } else { + p->profile = AVE_HEVC_MAIN; + p->min_qp = ctx->hevc_min_qp; + p->max_qp = ctx->hevc_max_qp; + } + if (p->min_qp > p->max_qp) + p->min_qp = p->max_qp; + + /* + * Reference structures as macOS sets them up: a B-frame pyramid, + * P frames only (low delay), or (H.264) intra only. + */ + p->intra_only = p->codec == AVE_CODEC_H264 && ctx->gop_size == 1; + p->bframes = !baseline && !p->intra_only && ctx->b_frames; + p->reorder_off = !ctx->b_frames; + p->refs = p->intra_only ? 1 : p->bframes ? 5 : 2; + p->slots = p->bframes ? 5 : 3; + p->gop = ctx->gop_size ?: 120; + + p->bitrate = ctx->bitrate; + if (ctx->bitrate_mode == V4L2_MPEG_VIDEO_BITRATE_MODE_CBR) { + p->rc_mode = AVE_RC_CBR; + p->limit_bytes = 0; + } else { + /* + * The data-rate limit macOS sets is twice the average rate over + * one second; a peak equal to the average starved the rate + * control on hardware. A peak of 0 means that default. + */ + p->rc_mode = AVE_RC_VBR; + p->limit_bytes = (ctx->bitrate_peak ? max(ctx->bitrate_peak, ctx->bitrate) : + 2ULL * ctx->bitrate) / 8; + } + + level = ave_pick_level(p, p->codec == AVE_CODEC_H264 ? ctx->h264_level : + ctx->hevc_level); + if (level < 0) + return level; + p->level = level; + + /* Coded-data capacity: 1.5 bytes per (aligned) pixel, 16 KiB pages. */ + p->coded_cap = ALIGN(3 * (p->codec == AVE_CODEC_HEVC ? ALIGN(p->width, 32) : + ALIGN(p->width, 16)) * + ALIGN(p->height, 16) / 2, SZ_16K); + p->coded_cap = max_t(u32, p->coded_cap, AVE_CODED_MIN); + if (ctx->ave->layout->counted && + (p->fps_num != 30ULL * p->fps_den || + (!p->bframes && (p->codec == AVE_CODEC_HEVC || (!baseline && !p->intra_only))))) + return -EOPNOTSUPP; + return 0; +} + +/* + * Two groups hold more than their size fields say, measured on hardware + * at 1080p and 2408x1506: the +0x390 group is a 4:2:0 picture whose size + * covers luma only (the firmware writes up to 1.5 times it), and the + * +0x3a0 group is written up to 1.2 times its size (mapped at 1.25 times, + * as macOS spaces them). Every other buffer stays within its size; the + * mapping is only rounded up to whole pages. + */ + +/* One buffer to allocate: what the firmware is told and what is mapped. */ +struct ave_buf_req { + struct ave_buf *b; + size_t size; + size_t alloc; + unsigned long attrs; +}; + +#define AVE_MAX_BUF_REQS \ + (1 + 6 * AVE_MAX_REFS + 6 + 2 * AVE_TILE_BUFS + 2 + 4 * AVE_MAX_SLOTS) + +static int ave_buf_req_cmp(const void *a, const void *b) +{ + const struct ave_buf_req *x = a, *y = b; + + return x->alloc < y->alloc ? 1 : x->alloc > y->alloc ? -1 : 0; +} + +static void ave_req(struct ave_buf_req *r, unsigned int *n, struct ave_buf *b, size_t size, + size_t alloc, unsigned long attrs) +{ + r[*n] = (struct ave_buf_req){ + .b = b, .size = size, .alloc = max(size, alloc), .attrs = attrs, + }; + (*n)++; +} + +/* + * Working buffer sizes, all from the requested picture size. The DMA layer + * aligns each mapping to a power of two in the encoder's 256 MiB window, so + * the buffers are mapped largest first: the smaller ones then fill the + * alignment gaps the large ones leave. + */ +static int ave_sess_alloc(struct ave_dev *ave, struct ave_session *sess) +{ + const struct ave_params *p = &sess->p; + const struct ave_fw_layout *fw = ave->layout; + bool hevc = p->codec == AVE_CODEC_HEVC; + u32 w = p->width, h = p->height; + u32 w16 = DIV_ROUND_UP(w, 16), w32 = DIV_ROUND_UP(w, 32); + u32 h32 = DIV_ROUND_UP(h, 32), h64 = DIV_ROUND_UP(h, 64); + const unsigned long nomap = DMA_ATTR_NO_KERNEL_MAPPING; + size_t ref_a, ref_b, ref_c, ref_d, tile, mapped = 0; + struct ave_buf_req *r; + unsigned int n = 0; + int ret = 0; + + r = kcalloc(AVE_MAX_BUF_REQS, sizeof(*r), GFP_KERNEL); + if (!r) + return -ENOMEM; + + ref_a = ALIGN(w, 32) * ALIGN(h + 16, 32); + if (p->bit_depth == 10) + ref_a = ref_a * 5 / 4; + ref_b = roundup_pow_of_two(w) * roundup_pow_of_two(h) / 32; + ref_c = ave_lfs_ref_size(w, h, hevc, ave->variant->lfs_ref_scale); + ref_d = ALIGN(hevc ? 64 * w32 * h32 : 128 * p->mbs, SZ_16K); + tile = hevc ? 2304 * w32 * h64 : ALIGN(w, 16) * ALIGN(h, 64); + if (p->bit_depth == 10) + tile *= 2; + + ave_req(r, &n, &sess->hdr, fw->hdr_bytes, 0, nomap); + for (unsigned int i = 0; i < p->refs; i++) { + ave_req(r, &n, &sess->ref_a[i], ref_a, ref_a * 3 / 2, nomap); + ave_req(r, &n, &sess->ref_b[i], ref_b, ref_b * 5 / 4, nomap); + ave_req(r, &n, &sess->ref_c[i], ref_c, 0, nomap); + ave_req(r, &n, &sess->ref_d[i], ref_d, 0, nomap); + if (fw->counted) { + ave_req(r, &n, &sess->ref_e[i], ref_a / 2, 0, nomap); + ave_req(r, &n, &sess->ref_f[i], ref_b / 4, 0, nomap); + } + } + ave_req(r, &n, &sess->mb, ALIGN(82 * p->mbs, SZ_16K), 0, nomap); + if (fw->counted && p->rc_mode == 3) + for (unsigned int i = 0; i < 2; i++) + ave_req(r, &n, &sess->mb_extra[i], ALIGN(82 * p->mbs, SZ_16K), 0, nomap); + /* HEVC: one 0x180-byte neighbour record per 64 pixels, at least 16 KiB. */ + ave_req(r, &n, &sess->row_a, + ALIGN(hevc ? max_t(u32, 0x180 * DIV_ROUND_UP(w, 64), SZ_16K) : 256 * w16, SZ_16K), + 0, nomap); + ave_req(r, &n, &sess->row_b, ALIGN(hevc ? 768 * w32 : 1024 * w16, SZ_16K), 0, nomap); + ave_req(r, &n, &sess->fixed, SZ_16K, 0, nomap); + for (unsigned int i = 0; i < (hevc ? 4 : AVE_TILE_BUFS); i++) { + ave_req(r, &n, &sess->tile[i], tile, 0, nomap); + ave_req(r, &n, &sess->tile_aux[i], ALIGN(h, 128), 0, nomap); + } + if (hevc) + for (unsigned int i = 0; i < 2; i++) + ave_req(r, &n, &sess->hevc_extra[i], ALIGN(w * h32, SZ_16K), 0, nomap); + for (unsigned int s = 0; s < p->slots; s++) { + ave_req(r, &n, &sess->slot[s].coded, p->coded_cap, 0, 0); + ave_req(r, &n, &sess->slot[s].meta, fw->meta_bytes, 0, 0); + /* J700 has HEVC slice headers; 26A428 has a new small slot buffer. */ + if (fw->counted) { + ave_req(r, &n, &sess->slot[s].extra, SZ_16K, 0, 0); + if (!hevc && p->rc_mode != 3) + ave_req(r, &n, &sess->slot[s].aux, + ALIGN(432 * p->mbs, SZ_16K), 0, nomap); + } else { + ave_req(r, &n, &sess->slot[s].aux, + hevc ? AVE_SHDR_BYTES : ALIGN(432 * p->mbs, SZ_16K), 0, + hevc ? 0 : nomap); + } + } + + sort(r, n, sizeof(*r), ave_buf_req_cmp, NULL); + for (unsigned int i = 0; i < n && !ret; i++) { + ret = ave_alloc_padded(ave, r[i].b, r[i].size, r[i].alloc, r[i].attrs); + mapped += r[i].b->alloc; + } + kfree(r); + if (ret) { + dev_err(ave->dev, "out of encoder DMA space after %zu MiB of session buffers\n", + mapped >> 20); + return -ENOMEM; + } + dev_dbg(ave->dev, "session buffers: %u, %zu KiB mapped\n", n, mapped >> 10); + return 0; +} + +static void ave_sess_free_bufs(struct ave_dev *ave, struct ave_session *sess) +{ + ave_free(ave, &sess->hdr); + for (unsigned int i = 0; i < AVE_MAX_REFS; i++) { + ave_free(ave, &sess->ref_a[i]); + ave_free(ave, &sess->ref_b[i]); + ave_free(ave, &sess->ref_c[i]); + ave_free(ave, &sess->ref_d[i]); + ave_free(ave, &sess->ref_e[i]); + ave_free(ave, &sess->ref_f[i]); + } + ave_free(ave, &sess->mb); + for (unsigned int i = 0; i < 2; i++) + ave_free(ave, &sess->mb_extra[i]); + ave_free(ave, &sess->row_a); + ave_free(ave, &sess->row_b); + ave_free(ave, &sess->fixed); + for (unsigned int i = 0; i < AVE_TILE_BUFS; i++) { + ave_free(ave, &sess->tile[i]); + ave_free(ave, &sess->tile_aux[i]); + } + for (unsigned int i = 0; i < 2; i++) + ave_free(ave, &sess->hevc_extra[i]); + for (unsigned int s = 0; s < AVE_MAX_SLOTS; s++) { + ave_free(ave, &sess->slot[s].coded); + ave_free(ave, &sess->slot[s].meta); + ave_free(ave, &sess->slot[s].aux); + ave_free(ave, &sess->slot[s].extra); + ave_free(ave, &sess->slot[s].bounce); + ave_vb2_put(sess->slot[s].input_mem); + sess->slot[s].input_mem = NULL; + } +} + +static void ave_note_copy(struct ave_ctx *ctx, u32 stride, const char *how) +{ + if (ctx->sess->copy_noted) + return; + ctx->sess->copy_noted = true; + dev_dbg(ctx->ave->dev, "input has a %u byte stride; moving lines to %u %s\n", stride, + ctx->sess->p.stride, how); +} + +/* + * The input picture for @slot, as a DART address. The encoder reads NV12 + * with the 64-byte aligned stride S_FMT reported and chroma right after + * height luma lines. Some clients (ffmpeg's v4l2m2m encoder) fill the + * buffer with their own, smaller line size instead, which shows in the + * payload size; such frames are copied to a per-slot buffer in the + * encoder's layout. + */ +int ave_sess_input(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *src, + dma_addr_t *input) +{ + struct ave_session *sess = ctx->sess; + const struct ave_params *p = &sess->p; + struct vb2_buffer *vb = &src->vb2_buf; + u32 off = vb->planes[0].data_offset; + u32 used = vb2_get_plane_payload(vb, 0) - min(off, vb2_get_plane_payload(vb, 0)); + u32 crows = DIV_ROUND_UP(p->height, 2), rows = p->height + crows; + u32 width = ALIGN(p->width, 2), stride; + struct ave_buf *b = &sess->slot[slot].bounce; + const u8 *from; + u8 *to; + int ret; + + *input = vb2_dma_contig_plane_dma_addr(vb, 0) + off; + if (!used || used >= p->stride * rows || used % rows || used / rows < width) { + if (IS_ALIGNED(*input, 64)) + return 0; + dev_err(ctx->ave->dev, "OUTPUT plane at %pad is not 64-byte aligned\n", input); + return -EINVAL; + } + stride = used / rows; + + from = vb2_plane_vaddr(vb, 0); + if (!from) { + dev_err(ctx->ave->dev, "cannot map an OUTPUT buffer to copy its %u byte stride\n", + stride); + return -EFAULT; + } + from += off; + + /* + * Buffers the driver allocated (MMAP) are large enough for the + * encoder's layout: spread the lines out in place, last line first, + * so no extra DMA space is needed. Imported buffers may be shared + * with other users and get a copy instead. + */ + if (vb->memory == VB2_MEMORY_MMAP && + vb2_plane_size(vb, 0) >= off + (size_t)p->stride * rows) { + to = (u8 *)from; + for (u32 y = rows; y-- > 0;) + memmove(to + y * p->stride, from + y * stride, width); + ave_note_copy(ctx, stride, "in place"); + return 0; + } + + ret = ave_alloc(ctx->ave, b, p->stride * rows, 0); + if (ret) { + dev_err(ctx->ave->dev, "no room to copy %u byte stride input\n", stride); + return ret; + } + to = b->cpu; + for (u32 y = 0; y < rows; y++) + memcpy(to + y * p->stride, from + y * stride, width); + ave_note_copy(ctx, stride, "to a copy"); + *input = b->dma; + return 0; +} + +/* Debug map of the session buffers, to match DART faults to a buffer. */ +static void ave_log_buf(struct ave_dev *ave, const char *name, int i, const struct ave_buf *b) +{ + if (b->cpu) + dev_dbg(ave->dev, "buf %s%d %pad +%#zx (mapped %#zx)\n", name, i, &b->dma, + b->size, b->alloc); +} + +static void ave_sess_log_map(struct ave_dev *ave, struct ave_session *sess) +{ + ave_log_buf(ave, "hdr", 0, &sess->hdr); + ave_log_buf(ave, "mb", 0, &sess->mb); + ave_log_buf(ave, "row_a", 0, &sess->row_a); + ave_log_buf(ave, "row_b", 0, &sess->row_b); + ave_log_buf(ave, "fixed", 0, &sess->fixed); + for (int i = 0; i < 2; i++) + ave_log_buf(ave, "hevc_extra", i, &sess->hevc_extra[i]); + for (int i = 0; i < AVE_MAX_REFS; i++) { + ave_log_buf(ave, "ref_a", i, &sess->ref_a[i]); + ave_log_buf(ave, "ref_b", i, &sess->ref_b[i]); + ave_log_buf(ave, "ref_c", i, &sess->ref_c[i]); + ave_log_buf(ave, "ref_d", i, &sess->ref_d[i]); + } + for (int i = 0; i < AVE_TILE_BUFS; i++) { + ave_log_buf(ave, "tile", i, &sess->tile[i]); + ave_log_buf(ave, "tile_aux", i, &sess->tile_aux[i]); + } + for (int i = 0; i < AVE_MAX_SLOTS; i++) { + ave_log_buf(ave, "coded", i, &sess->slot[i].coded); + ave_log_buf(ave, "meta", i, &sess->slot[i].meta); + ave_log_buf(ave, "aux", i, &sess->slot[i].aux); + } +} + +/* Work out the parameters and allocate the buffers; no firmware access. */ +struct ave_session *ave_sess_create(struct ave_ctx *ctx) +{ + struct ave_session *sess; + int ret; + + sess = kzalloc_obj(*sess); + if (!sess) + return ERR_PTR(-ENOMEM); + ret = ave_sess_params(ctx, &sess->p); + if (!ret) + ret = ave_sess_alloc(ctx->ave, sess); + if (ret) { + ave_sess_free_bufs(ctx->ave, sess); + kfree(sess); + return ERR_PTR(ret); + } + for (unsigned int s = 0; s < sess->p.slots; s++) + ave_sess_prepare_slot(sess, s); + ave_sess_log_map(ctx->ave, sess); + dev_dbg(ctx->ave->dev, "session %ux%u codec %u profile %u level %u, %u slots, %u refs\n", + sess->p.width, sess->p.height, sess->p.codec, sess->p.profile, sess->p.level, + sess->p.slots, sess->p.refs); + dev_dbg(ctx->ave->dev, "rate control mode %u, %u bit/s, limit %u bytes/s, gop %u, %u/%u fps\n", + sess->p.rc_mode, sess->p.bitrate, sess->p.limit_bytes, sess->p.gop, + sess->p.fps_num, sess->p.fps_den); + return sess; +} + +/* Keep every firmware-visible allocation mapped until a confirmed power cycle. */ +void ave_sess_destroy(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + + if (!sess) + return; + mutex_lock(&ave->hw_lock); + if (sess->handed && (ave->fw_state == AVE_FW_BROKEN || + ave->fw_state == AVE_FW_RECOVERING)) { + sess->failed_next = ave->failed_sessions; + ave->failed_sessions = sess; + } else { + ave_sess_free_bufs(ave, sess); + kfree(sess); + } + ctx->sess = NULL; + mutex_unlock(&ave->hw_lock); +} + +/* Caller holds hw_lock and has verified that the coprocessor lost power. */ +void ave_sess_reap(struct ave_dev *ave) +{ + struct ave_session *sess; + + lockdep_assert_held(&ave->hw_lock); + while ((sess = ave->failed_sessions)) { + ave->failed_sessions = sess->failed_next; + ave_sess_free_bufs(ave, sess); + kfree(sess); + } +} + +int ave_sess_free_slot(struct ave_session *sess) +{ + for (unsigned int s = 0; s < sess->p.slots; s++) + if (!sess->slot[s].busy && !sess->slot[s].done) + return s; + return -EBUSY; +} + +/* + * Clear what the firmware reports through, so a stale value is never taken + * for a new one: the whole metadata buffer and the first slice header. + */ +void ave_sess_prepare_slot(struct ave_session *sess, unsigned int slot) +{ + struct ave_slot *s = &sess->slot[slot]; + + memset(s->meta.cpu, 0, s->meta.size); + if (sess->p.codec == AVE_CODEC_HEVC && s->aux.cpu) + memset(s->aux.cpu, 0, AVE_SHDR_ENTRY_BYTES); +} + +/* + * A frame reply: +0x10 the session's client token, +0x20 the slot's + * subtype, +0x40 status. Caller holds hw_lock. + */ +void ave_sess_reply(struct ave_dev *ave, const struct ave_reply *reply) +{ + u64 client = reply->client; + u32 subtype = reply->subtype; + u32 slot = subtype - ave->layout->process_subtype; + struct ave_session *sess = NULL; + struct ave_ctx *ctx = NULL; + struct ave_slot *s; + + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i] && + ave->sessions[i]->sess->client == client) + ctx = ave->sessions[i]; + if (ctx) + sess = ctx->sess; + if (!sess || + reply->bytes < + (ave->layout->abi->transport->boot_config ? 0x50 : 0x48) || + slot >= sess->p.slots || !sess->slot[slot].busy) { + dev_warn(ave->dev, + "stray frame reply (client %llu subtype %#x len %u)\n", + client, subtype, reply->bytes); + return; + } + s = &sess->slot[slot]; + sess->progress = jiffies; + s->status = reply->status; + if (s->status) + dev_dbg(ctx->ave->dev, "frame in slot %u: status %#x\n", slot, + s->status); + s->busy = false; + s->done = true; + sess->busy--; + sess->done_fifo[(sess->done_head + sess->done_count) % AVE_MAX_SLOTS] = + slot; + sess->done_count++; + atomic_inc(&ctx->ready); +} + +/* + * Copy Annex B data from @src to @dst until the data ends: NAL units never + * hold three zero bytes in a row, and the buffer was cleared before the + * firmware wrote it, so a run of @zero_run zero bytes marks the end. + * Returns the length or -ENOSPC. + */ +static ssize_t ave_copy_annexb(u8 *dst, size_t room, const u8 *src, size_t cap, + unsigned int zero_run) +{ + size_t pos = 0, end = 0; + unsigned int zeros = 0; + + while (pos < cap) { + size_t chunk = min3((size_t)SZ_16K, cap - pos, room - pos); + + if (!chunk) + return -ENOSPC; + memcpy(dst + pos, src + pos, chunk); + for (size_t i = pos; i < pos + chunk; i++) { + if (dst[i]) { + end = i + 1; + zeros = 0; + } else if (end && ++zeros >= zero_run) { + return end; + } + } + pos += chunk; + } + return end; +} + +static bool ave_starts_annexb(const u8 *p, size_t len) +{ + return (len >= 3 && !p[0] && !p[1] && p[2] == 1) || + (len >= 4 && !p[0] && !p[1] && !p[2] && p[3] == 1); +} + +/* NAL unit types in an Annex B buffer: key frame, parameter sets, filler? */ +static void ave_scan_nals(const u8 *p, size_t len, bool hevc, bool *key, bool *params, + bool *filler) +{ + for (size_t i = 0; i + 3 < len; i++) { + u32 type; + + if (p[i] || p[i + 1] || p[i + 2] != 1) + continue; + type = hevc ? (p[i + 3] >> 1) & 0x3f : p[i + 3] & 0x1f; + if (hevc) { + *key |= type >= 16 && type <= 21; + *params |= type >= 32 && type <= 34; + *filler |= type == AVE_HEVC_NAL_FILLER; + } else { + *key |= type == 5; + *params |= type == 7; + *filler |= type == AVE_H264_NAL_FILLER; + } + i += 2; + } +} + +/* Where in the metadata buffer do the parameter sets start, if anywhere? */ +static int ave_find_params(const u8 *meta, bool hevc) +{ + if (ave_starts_annexb(meta + AVE_META_PARAMS, AVE_META_BYTES - AVE_META_PARAMS)) + return AVE_META_PARAMS; + /* Not at the HEVC offset: look for an SPS (H.264) or VPS (HEVC). */ + for (int i = AVE_META_SCAN_FROM; i + 4 < AVE_META_BYTES; i++) { + u8 type; + + if (meta[i] || meta[i + 1] || meta[i + 2] != 1) + continue; + type = hevc ? (meta[i + 3] >> 1) & 0x3f : meta[i + 3] & 0x1f; + if (type == (hevc ? 32 : 7)) + return i > AVE_META_SCAN_FROM && !meta[i - 1] ? i - 1 : i; + } + return -ENOENT; +} + +static void ave_sess_dump(struct ave_ctx *ctx, struct ave_slot *s, const char *why) +{ + struct ave_session *sess = ctx->sess; + const u8 *meta = s->meta.cpu; + int ps; + + if (sess->warned) + return; + sess->warned = true; + dev_warn(ctx->ave->dev, "frame %u output: %s (enable dynamic debug for a dump)\n", + ctx->dst_sequence, why); + print_hex_dump_debug("ave coded: ", DUMP_PREFIX_OFFSET, 16, 1, s->coded.cpu, 128, false); + print_hex_dump_debug("ave meta: ", DUMP_PREFIX_OFFSET, 16, 4, meta, 0x200, false); + print_hex_dump_debug("ave meta b200: ", DUMP_PREFIX_OFFSET, 16, 1, meta + 0xb200, + 0x200, false); + ps = ave_find_params(meta, sess->p.codec == AVE_CODEC_HEVC); + if (ps >= 0) { + dev_dbg(ctx->ave->dev, "parameter sets at metadata +%#x\n", ps); + print_hex_dump_debug("ave meta params: ", DUMP_PREFIX_OFFSET, 16, 1, meta + ps, + min(128, AVE_META_BYTES - ps), false); + } + /* Anything else the firmware wrote past the header words. */ + for (int i = AVE_META_SCAN_FROM; i < AVE_META_BYTES; i += 16) + if (memchr_inv(meta + i, 0, 16)) + dev_dbg(ctx->ave->dev, "metadata +%#x: %16ph\n", i, meta + i); +} + +/* + * CBR padding. macOS streams in CBR mode follow a buffer model: each frame + * has a budget of bitrate / (8 x fps) bytes, and once the bytes left unused + * add up to more than 0.5 s of bit rate (the 0.5 s VBV buffer of Start + * +0x290), every frame is padded with a filler-data NAL unit so the surplus + * stays there. Whether the firmware or macOS user space writes that filler + * is not known; if the firmware's output never carries filler, the driver + * adds it here. + */ +#define AVE_FILLER_OVERHEAD 7 /* start code, NAL header, trailing bits */ + +static size_t ave_sess_cbr_fill(struct ave_ctx *ctx, u8 *out, size_t len, size_t room, + bool fw_filler) +{ + struct ave_session *sess = ctx->sess; + const struct ave_params *p = &sess->p; + bool hevc = p->codec == AVE_CODEC_HEVC; + s64 budget = div_u64((u64)p->bitrate * p->fps_den, 8ULL * p->fps_num); + s64 vbv = p->bitrate / 16; /* 0.5 s of bit rate, in bytes */ + s64 excess, pad; + u8 *f; + + if (p->rc_mode != AVE_RC_CBR) + return len; + if (fw_filler && !sess->fw_fills) { + sess->fw_fills = true; + dev_dbg(ctx->ave->dev, "CBR: the firmware writes filler data\n"); + } + excess = sess->cbr_surplus + budget - (s64)len - vbv; + if (!sess->fw_fills && excess > AVE_FILLER_OVERHEAD) { + pad = min_t(s64, excess, room - len); + if (pad > AVE_FILLER_OVERHEAD) { + f = out + len; + f[0] = 0; + f[1] = 0; + f[2] = 0; + f[3] = 1; + if (hevc) { + /* nal_unit_type 38, layer 0, temporal id 0 + 1 */ + f[4] = AVE_HEVC_NAL_FILLER << 1; + f[5] = 1; + memset(f + 6, 0xff, pad - AVE_FILLER_OVERHEAD); + } else { + f[4] = AVE_H264_NAL_FILLER; + memset(f + 5, 0xff, pad - AVE_FILLER_OVERHEAD + 1); + } + f[pad - 1] = 0x80; /* rbsp_stop_one_bit */ + len += pad; + if (!sess->host_fill_noted) { + sess->host_fill_noted = true; + dev_dbg(ctx->ave->dev, + "CBR: no filler from the firmware, the driver adds it\n"); + } + } + } + sess->cbr_surplus += budget - (s64)len; + return len; +} + +/* + * Assemble the access unit of @slot into @dst: the parameter sets the + * firmware leaves in the metadata buffer (when it emits them), for HEVC the + * slice header from the first slice-header entry, then the coded data. + * Metadata +0x120 (repeated at +0x12c) holds the coded length for both + * codecs, +0x128 the HEVC slice-header length. + */ +int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, struct vb2_v4l2_buffer *dst, + bool *keyframe) +{ + struct ave_session *sess = ctx->sess; + struct ave_slot *s = &sess->slot[slot]; + bool hevc = sess->p.codec == AVE_CODEC_HEVC; + u8 *out = vb2_plane_vaddr(&dst->vb2_buf, 0); + size_t room = vb2_plane_size(&dst->vb2_buf, 0); + const u8 *meta = s->meta.cpu; + u32 data, data2, shdr = 0; + bool params = false, filler = false; + ssize_t ps = 0; + size_t len; + int at; + + *keyframe = false; + if (!out) + return -EFAULT; + if (s->status) { + dev_warn_ratelimited(ctx->ave->dev, "frame %u: firmware status %#x\n", + ctx->dst_sequence, s->status); + return -EIO; + } + + data = get_unaligned_le32(meta + AVE_META_PAYLOAD); + data2 = get_unaligned_le32(meta + AVE_META_PAYLOAD2); + if (hevc) + shdr = get_unaligned_le32(meta + AVE_META_HEVC_SHDR); + if (data != data2) + dev_dbg(ctx->ave->dev, "coded length words differ: %u/%u\n", data, data2); + + at = ave_find_params(meta, hevc); + if (at >= 0) { + ps = ave_copy_annexb(out, room, meta + at, AVE_META_BYTES - at, 16); + if (ps < 0) + return ps; + } + if (!data || data > sess->p.coded_cap || shdr > AVE_SHDR_ENTRY_BYTES || + ps + shdr + data > room) { + dev_err(ctx->ave->dev, "bad output sizes %u/%u (capacity %u, room %zu)\n", + shdr, data, sess->p.coded_cap, room); + ave_sess_dump(ctx, s, "bad sizes"); + return -EPROTO; + } + if (hevc) + memcpy(out + ps, s->aux.cpu, shdr); + memcpy(out + ps + shdr, s->coded.cpu, data); + len = ps + shdr + data; + + if (!ave_starts_annexb(out, len)) + ave_sess_dump(ctx, s, "not Annex B"); + ave_scan_nals(out, len, hevc, keyframe, ¶ms, &filler); + if (!ctx->dst_sequence && !params) + ave_sess_dump(ctx, s, "first frame has no parameter sets"); + len = ave_sess_cbr_fill(ctx, out, len, room, filler); + vb2_set_plane_payload(&dst->vb2_buf, 0, len); + return 0; +} + +/* + * Frames are outstanding, nothing came back for a while, and the firmware + * has enough frames that it should have answered: without B-frames every + * frame is answered on its own; with B-frames, once four are inside (one + * reference plus three B) one of them must come out. + */ +bool ave_sess_stalled(struct ave_session *sess) +{ + unsigned int need = sess->p.bframes ? 4 : 1; + + return sess->busy && (sess->busy >= need || sess->nudged) && + time_after(jiffies, sess->progress + msecs_to_jiffies(AVE_FRAME_WAIT_MS)); +} + +/* + * With B-frames the firmware keeps frames until later ones arrive. A client + * that waits for each coded frame before sending the next (ffmpeg's + * v4l2m2m encoder does) would wait forever. When frames are held and no + * input came for a while, a Complete makes the firmware code what it holds, + * as macOS does after the first frame. + */ +#define AVE_STARVE_MS 50 + +bool ave_sess_starved(struct ave_session *sess) +{ + return sess->busy && !sess->nudged && sess->handed && + time_after(jiffies, sess->last_sent + msecs_to_jiffies(AVE_STARVE_MS)); +} diff --git a/drivers/media/platform/apple/ave/ave-sizes-test.c b/drivers/media/platform/apple/ave/ave-sizes-test.c new file mode 100644 index 0000000000000..72e92f67b2d5e --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-sizes-test.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Apple AVE LFS reference descriptor sizes, not DMA allocation sizes. */ +#include +#include + +#include "ave-sizes.h" + +struct ave_size_case { + u32 width; + u32 height; + u32 avc; + u32 hevc; + u32 avc_t8140; + u32 hevc_t8140; +}; + +static const struct ave_size_case ave_size_cases[] = { + { 320, 240, 0x5a00, 0x1400, 0xb400, 0x2800 }, + { 640, 480, 0x14a00, 0x5000, 0x29400, 0xa000 }, + { 642, 482, 0x17600, 0x5800, 0x2ec00, 0xb000 }, + { 1280, 720, 0x3c000, 0xf000, 0x78000, 0x1e000 }, + { 1920, 1080, 0x85200, 0x1fe00, 0x10a400, 0x3fc00 }, + { 1920, 1088, 0x85200, 0x1fe00, 0x10a400, 0x3fc00 }, + { 1922, 1090, 0x8b800, 0x22e00, 0x117000, 0x45c00 }, + { 2560, 1600, 0x101800, 0x3e800, 0x203000, 0x7d000 }, +}; + +static void ave_size_desc(const struct ave_size_case *c, char *desc) +{ + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%ux%u", c->width, c->height); +} + +KUNIT_ARRAY_PARAM(ave_size, ave_size_cases, ave_size_desc); + +static void ave_lfs_ref_test(struct kunit *test) +{ + const struct ave_size_case *c = test->param_value; + + /* T6000 and T8112 both select scale 1; T8140 selects scale 2. */ + KUNIT_EXPECT_EQ(test, ave_lfs_ref_size(c->width, c->height, false, 1), c->avc); + KUNIT_EXPECT_EQ(test, ave_lfs_ref_size(c->width, c->height, true, 1), c->hevc); + KUNIT_EXPECT_EQ(test, ave_lfs_ref_size(c->width, c->height, false, 2), c->avc_t8140); + KUNIT_EXPECT_EQ(test, ave_lfs_ref_size(c->width, c->height, true, 2), c->hevc_t8140); +} + +static struct kunit_case ave_sizes_tests[] = { + KUNIT_CASE_PARAM(ave_lfs_ref_test, ave_size_gen_params), + { } +}; + +static struct kunit_suite ave_sizes_suite = { + .name = "apple-ave-sizes", + .test_cases = ave_sizes_tests, +}; + +kunit_test_suite(ave_sizes_suite); + +MODULE_DESCRIPTION("Apple AVE buffer sizing tests"); +MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/ave/ave-sizes.h b/drivers/media/platform/apple/ave/ave-sizes.h new file mode 100644 index 0000000000000..01e29678035b5 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-sizes.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _APPLE_AVE_SIZES_H_ +#define _APPLE_AVE_SIZES_H_ + +#include +#include +#include + +/* Start +0x7d0 LFSRef: exact descriptor bytes, before DMA page rounding. */ +static inline u32 ave_lfs_ref_size(u32 width, u32 height, bool hevc, u32 scale) +{ + u32 size; + + if (hevc) + size = 256 * DIV_ROUND_UP(width, 64) * DIV_ROUND_UP(height, 64); + else + size = ALIGN(width, 64) * (ALIGN(height, 16) / 4 + 12); + return size * scale; +} + +#endif diff --git a/drivers/media/platform/apple/ave/ave-v4l2.c b/drivers/media/platform/apple/ave/ave-v4l2.c new file mode 100644 index 0000000000000..e01710ad07fe1 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-v4l2.c @@ -0,0 +1,1379 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder: V4L2 stateful encoder interface + * + * Follows Documentation/userspace-api/media/v4l/dev-encoder.rst: the CAPTURE + * queue carries the coded stream (H.264 or HEVC), the OUTPUT queue raw NV12. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "ave.h" +#include "ave-regs.h" + +#define AVE_NAME "apple-ave" + +/* + * Frame size limits and input stride alignment: the range the encoder was + * seen working at (320x240 up to the 2408x1506 panel), with a little room. + * Larger sizes need buffer sizes nobody has checked yet. + */ +#define AVE_MIN_W 320 +#define AVE_MIN_H 240 +#define AVE_MAX_W 4096 +#define AVE_MAX_H 2304 +#define AVE_STEP 16 +#define AVE_STRIDE_ALIGN 64 +#define AVE_DEFAULT_W 1920 +#define AVE_DEFAULT_H 1080 +/* + * Buffer counts are limited so the queues of a few concurrent sessions fit + * the encoder's 4 GiB DART window next to their session buffers: each queue + * may use this much, counting every buffer at the power-of-two size the DMA + * layer maps it with. That is 16 OUTPUT frames at 3840x2160. + */ +#define AVE_OUTPUT_BUDGET (256 * SZ_1M) +#define AVE_CAPTURE_BUDGET (128 * SZ_1M) +#define AVE_MAX_QUEUE_BUFS 16 +#define AVE_RESTART_TRIES 10 +#define AVE_RESTART_WAIT_MS 100 + +static const u32 ave_coded_fmts[] = { + V4L2_PIX_FMT_H264, + V4L2_PIX_FMT_HEVC, +}; + +static const u32 ave_raw_fmts[] = { + V4L2_PIX_FMT_NV12, +}; + +static const struct v4l2_event ave_eos_event = { + .type = V4L2_EVENT_EOS, +}; + +enum ave_codec ave_ctx_codec(struct ave_ctx *ctx) +{ + return ctx->dst_fmt.pixelformat == V4L2_PIX_FMT_HEVC ? AVE_CODEC_HEVC : + AVE_CODEC_H264; +} + +static bool ave_fmt_listed(const u32 *list, unsigned int n, u32 fourcc) +{ + for (unsigned int i = 0; i < n; i++) + if (list[i] == fourcc) + return true; + return false; +} + +/* HEVC Main tier levels 5 to 5.2 allow at most this many luma samples. */ +#define AVE_HEVC_MAX_SAMPLES 8912896 + +/* @coded is the coded format the size is for: HEVC caps the picture area. */ +static void ave_clamp_size(u32 coded, u32 *w, u32 *h) +{ + *w = clamp_t(u32, ALIGN(*w, 2), AVE_MIN_W, AVE_MAX_W); + *h = clamp_t(u32, ALIGN(*h, 2), AVE_MIN_H, AVE_MAX_H); + if (coded == V4L2_PIX_FMT_HEVC && *w * *h > AVE_HEVC_MAX_SAMPLES) + *h = rounddown(AVE_HEVC_MAX_SAMPLES / *w, 2); +} + +/* + * Raw NV12, one buffer: Y plane, then interleaved CbCr at half height. + * @coded is the context's coded format, which limits the size. + */ +static void ave_fill_raw(struct v4l2_pix_format_mplane *f, u32 coded) +{ + u32 stride, height; + + f->pixelformat = V4L2_PIX_FMT_NV12; + ave_clamp_size(coded, &f->width, &f->height); + /* Keep a client's stride, within reason, rounded to 64 bytes. */ + stride = ALIGN(clamp_t(u32, f->plane_fmt[0].bytesperline, f->width, 2 * AVE_MAX_W), + AVE_STRIDE_ALIGN); + height = ALIGN(f->height, AVE_STEP); + f->num_planes = 1; + f->plane_fmt[0].bytesperline = stride; + f->plane_fmt[0].sizeimage = stride * height * 3 / 2; + f->field = V4L2_FIELD_NONE; + memset(f->plane_fmt[0].reserved, 0, sizeof(f->plane_fmt[0].reserved)); + memset(f->reserved, 0, sizeof(f->reserved)); +} + +static void ave_fill_coded(struct v4l2_pix_format_mplane *f) +{ + u32 min; + + if (!ave_fmt_listed(ave_coded_fmts, ARRAY_SIZE(ave_coded_fmts), f->pixelformat)) + f->pixelformat = ave_coded_fmts[0]; + ave_clamp_size(f->pixelformat, &f->width, &f->height); + /* The firmware's coded-data capacity plus parameter sets and headers. */ + min = ALIGN(3 * ALIGN(f->width, 32) * ALIGN(f->height, 16) / 2, SZ_16K) + SZ_64K; + f->num_planes = 1; + f->plane_fmt[0].bytesperline = 0; + f->plane_fmt[0].sizeimage = clamp_t(u32, f->plane_fmt[0].sizeimage, min, SZ_32M); + f->field = V4L2_FIELD_NONE; + memset(f->plane_fmt[0].reserved, 0, sizeof(f->plane_fmt[0].reserved)); + memset(f->reserved, 0, sizeof(f->reserved)); +} + +static void ave_reset_fmts(struct ave_ctx *ctx) +{ + struct v4l2_pix_format_mplane *src = &ctx->src_fmt, *dst = &ctx->dst_fmt; + + memset(dst, 0, sizeof(*dst)); + dst->pixelformat = V4L2_PIX_FMT_H264; + dst->width = AVE_DEFAULT_W; + dst->height = AVE_DEFAULT_H; + ave_fill_coded(dst); + + memset(src, 0, sizeof(*src)); + src->width = dst->width; + src->height = dst->height; + ave_fill_raw(src, dst->pixelformat); + src->colorspace = V4L2_COLORSPACE_REC709; + src->xfer_func = V4L2_XFER_FUNC_DEFAULT; + src->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; + src->quantization = V4L2_QUANTIZATION_LIM_RANGE; + dst->colorspace = src->colorspace; + dst->xfer_func = src->xfer_func; + dst->ycbcr_enc = src->ycbcr_enc; + dst->quantization = src->quantization; + + ctx->timeperframe = (struct v4l2_fract){ 1, 30 }; +} + +/* ioctls */ + +static int ave_querycap(struct file *file, void *priv, struct v4l2_capability *cap) +{ + strscpy(cap->driver, AVE_NAME, sizeof(cap->driver)); + strscpy(cap->card, "Apple AVE video encoder", sizeof(cap->card)); + return 0; +} + +static int ave_enum_fmt_cap(struct file *file, void *priv, struct v4l2_fmtdesc *f) +{ + if (f->index >= ARRAY_SIZE(ave_coded_fmts)) + return -EINVAL; + f->pixelformat = ave_coded_fmts[f->index]; + f->flags = V4L2_FMT_FLAG_COMPRESSED; + return 0; +} + +static int ave_enum_fmt_out(struct file *file, void *priv, struct v4l2_fmtdesc *f) +{ + if (f->index >= ARRAY_SIZE(ave_raw_fmts)) + return -EINVAL; + f->pixelformat = ave_raw_fmts[f->index]; + return 0; +} + +static int ave_enum_framesizes(struct file *file, void *priv, struct v4l2_frmsizeenum *fs) +{ + if (fs->index || + (!ave_fmt_listed(ave_coded_fmts, ARRAY_SIZE(ave_coded_fmts), fs->pixel_format) && + !ave_fmt_listed(ave_raw_fmts, ARRAY_SIZE(ave_raw_fmts), fs->pixel_format))) + return -EINVAL; + fs->type = V4L2_FRMSIZE_TYPE_STEPWISE; + fs->stepwise = (struct v4l2_frmsize_stepwise){ + AVE_MIN_W, AVE_MAX_W, 2, AVE_MIN_H, AVE_MAX_H, 2, + }; + return 0; +} + +static int ave_g_fmt_cap(struct file *file, void *priv, struct v4l2_format *f) +{ + f->fmt.pix_mp = ave_file_to_ctx(file)->dst_fmt; + return 0; +} + +static int ave_g_fmt_out(struct file *file, void *priv, struct v4l2_format *f) +{ + f->fmt.pix_mp = ave_file_to_ctx(file)->src_fmt; + return 0; +} + +static int ave_try_fmt_cap(struct file *file, void *priv, struct v4l2_format *f) +{ + ave_fill_coded(&f->fmt.pix_mp); + return 0; +} + +static int ave_try_fmt_out(struct file *file, void *priv, struct v4l2_format *f) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp; + + ave_fill_raw(pix, ctx->dst_fmt.pixelformat); + if (pix->colorspace == V4L2_COLORSPACE_DEFAULT) { + pix->colorspace = ctx->src_fmt.colorspace; + pix->xfer_func = ctx->src_fmt.xfer_func; + pix->ycbcr_enc = ctx->src_fmt.ycbcr_enc; + pix->quantization = ctx->src_fmt.quantization; + } + return 0; +} + +static int ave_s_fmt_cap(struct file *file, void *priv, struct v4l2_format *f) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + struct vb2_queue *vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type); + struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp; + + if (vb2_is_busy(vq)) + return -EBUSY; + /* The coded format comes first; the raw side follows its size. */ + ave_fill_coded(pix); + pix->colorspace = ctx->src_fmt.colorspace; + pix->xfer_func = ctx->src_fmt.xfer_func; + pix->ycbcr_enc = ctx->src_fmt.ycbcr_enc; + pix->quantization = ctx->src_fmt.quantization; + ctx->dst_fmt = *pix; + return 0; +} + +static int ave_s_fmt_out(struct file *file, void *priv, struct v4l2_format *f) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + struct vb2_queue *vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type); + struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp; + int ret; + + if (vb2_is_busy(vq)) + return -EBUSY; + ret = ave_try_fmt_out(file, priv, f); + if (ret) + return ret; + ctx->src_fmt = *pix; + + /* The coded size follows the raw size (crop is the whole frame). */ + ctx->dst_fmt.width = pix->width; + ctx->dst_fmt.height = pix->height; + ctx->dst_fmt.colorspace = pix->colorspace; + ctx->dst_fmt.xfer_func = pix->xfer_func; + ctx->dst_fmt.ycbcr_enc = pix->ycbcr_enc; + ctx->dst_fmt.quantization = pix->quantization; + ave_fill_coded(&ctx->dst_fmt); + return 0; +} + +static int ave_g_selection(struct file *file, void *priv, struct v4l2_selection *s) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + + if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) + return -EINVAL; + switch (s->target) { + case V4L2_SEL_TGT_CROP: + case V4L2_SEL_TGT_CROP_DEFAULT: + case V4L2_SEL_TGT_CROP_BOUNDS: + s->r = (struct v4l2_rect){ 0, 0, ctx->src_fmt.width, ctx->src_fmt.height }; + return 0; + default: + return -EINVAL; + } +} + +static int ave_s_selection(struct file *file, void *priv, struct v4l2_selection *s) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + + if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT || s->target != V4L2_SEL_TGT_CROP) + return -EINVAL; + /* Only full-frame encoding for now. */ + s->r = (struct v4l2_rect){ 0, 0, ctx->src_fmt.width, ctx->src_fmt.height }; + return 0; +} + +static int ave_g_parm(struct file *file, void *priv, struct v4l2_streamparm *a) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + + if (!V4L2_TYPE_IS_OUTPUT(a->type)) + return -EINVAL; + a->parm.output.capability = V4L2_CAP_TIMEPERFRAME; + a->parm.output.timeperframe = ctx->timeperframe; + return 0; +} + +static int ave_s_parm(struct file *file, void *priv, struct v4l2_streamparm *a) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + struct v4l2_fract *t = &a->parm.output.timeperframe; + + if (!V4L2_TYPE_IS_OUTPUT(a->type)) + return -EINVAL; + if (!t->numerator || !t->denominator) + *t = ctx->timeperframe; + /* 1 to 240 frames per second. */ + if ((u64)t->denominator > (u64)t->numerator * 240) + *t = (struct v4l2_fract){ 1, 240 }; + if (t->numerator > t->denominator) + *t = (struct v4l2_fract){ 1, 1 }; + ctx->timeperframe = *t; + a->parm.output.capability = V4L2_CAP_TIMEPERFRAME; + return 0; +} + +/* + * Drain: frames still inside the firmware (B-frames) only come out after a + * flush, so the driver tracks the drain point itself instead of using the + * m2m helpers, which would mark the next CAPTURE buffer last right away. + */ +static int ave_encoder_cmd(struct file *file, void *priv, struct v4l2_encoder_cmd *ec) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + int ret; + + ret = v4l2_m2m_ioctl_try_encoder_cmd(file, priv, ec); + if (ret) + return ret; + if (!vb2_is_streaming(v4l2_m2m_get_src_vq(m2m)) || + !vb2_is_streaming(v4l2_m2m_get_dst_vq(m2m))) + return 0; + + if (ec->cmd == V4L2_ENC_CMD_STOP) { + if (ctx->draining || ctx->stopped) + return 0; + ctx->drain_src = v4l2_m2m_last_src_buf(m2m); + ctx->draining = true; + if (!ctx->drain_src) + WRITE_ONCE(ctx->flush_pending, true); + } else { + if (ctx->draining && !ctx->stopped) + return -EBUSY; + ctx->draining = false; + ctx->stopped = false; + vb2_clear_last_buffer_dequeued(v4l2_m2m_get_dst_vq(m2m)); + } + v4l2_m2m_try_schedule(m2m); + return 0; +} + +static int ave_subscribe_event(struct v4l2_fh *fh, const struct v4l2_event_subscription *sub) +{ + switch (sub->type) { + case V4L2_EVENT_EOS: + return v4l2_event_subscribe(fh, sub, 0, NULL); + default: + return v4l2_ctrl_subscribe_event(fh, sub); + } +} + +static const struct v4l2_ioctl_ops ave_ioctl_ops = { + .vidioc_querycap = ave_querycap, + .vidioc_enum_fmt_vid_cap = ave_enum_fmt_cap, + .vidioc_enum_fmt_vid_out = ave_enum_fmt_out, + .vidioc_enum_framesizes = ave_enum_framesizes, + .vidioc_g_fmt_vid_cap_mplane = ave_g_fmt_cap, + .vidioc_g_fmt_vid_out_mplane = ave_g_fmt_out, + .vidioc_try_fmt_vid_cap_mplane = ave_try_fmt_cap, + .vidioc_try_fmt_vid_out_mplane = ave_try_fmt_out, + .vidioc_s_fmt_vid_cap_mplane = ave_s_fmt_cap, + .vidioc_s_fmt_vid_out_mplane = ave_s_fmt_out, + .vidioc_g_selection = ave_g_selection, + .vidioc_s_selection = ave_s_selection, + .vidioc_g_parm = ave_g_parm, + .vidioc_s_parm = ave_s_parm, + + .vidioc_reqbufs = v4l2_m2m_ioctl_reqbufs, + .vidioc_querybuf = v4l2_m2m_ioctl_querybuf, + .vidioc_qbuf = v4l2_m2m_ioctl_qbuf, + .vidioc_dqbuf = v4l2_m2m_ioctl_dqbuf, + .vidioc_prepare_buf = v4l2_m2m_ioctl_prepare_buf, + .vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs, + .vidioc_expbuf = v4l2_m2m_ioctl_expbuf, + .vidioc_streamon = v4l2_m2m_ioctl_streamon, + .vidioc_streamoff = v4l2_m2m_ioctl_streamoff, + + .vidioc_try_encoder_cmd = v4l2_m2m_ioctl_try_encoder_cmd, + .vidioc_encoder_cmd = ave_encoder_cmd, + + .vidioc_subscribe_event = ave_subscribe_event, + .vidioc_unsubscribe_event = v4l2_event_unsubscribe, +}; + +/* Controls */ + +static int ave_s_ctrl(struct v4l2_ctrl *ctrl) +{ + struct ave_ctx *ctx = container_of(ctrl->handler, struct ave_ctx, hdl); + + switch (ctrl->id) { + case V4L2_CID_MPEG_VIDEO_BITRATE_MODE: + ctx->bitrate_mode = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_BITRATE: + WRITE_ONCE(ctx->bitrate, ctrl->val); + break; + case V4L2_CID_MPEG_VIDEO_BITRATE_PEAK: + ctx->bitrate_peak = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_GOP_SIZE: + ctx->gop_size = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME: + WRITE_ONCE(ctx->force_key_frame, true); + break; + case V4L2_CID_MPEG_VIDEO_B_FRAMES: + ctx->b_frames = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_H264_PROFILE: + ctx->h264_profile = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_H264_LEVEL: + ctx->h264_level = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_H264_MIN_QP: + ctx->h264_min_qp = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_H264_MAX_QP: + ctx->h264_max_qp = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_HEVC_LEVEL: + ctx->hevc_level = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_HEVC_MIN_QP: + ctx->hevc_min_qp = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_HEVC_MAX_QP: + ctx->hevc_max_qp = ctrl->val; + break; + case V4L2_CID_MPEG_VIDEO_HEADER_MODE: + case V4L2_CID_MPEG_VIDEO_HEVC_PROFILE: + case V4L2_CID_MIN_BUFFERS_FOR_OUTPUT: + break; + default: + return -EINVAL; + } + return 0; +} + +static const struct v4l2_ctrl_ops ave_ctrl_ops = { + .s_ctrl = ave_s_ctrl, +}; + +static int ave_init_ctrls(struct ave_ctx *ctx) +{ + struct v4l2_ctrl_handler *hdl = &ctx->hdl; + const struct v4l2_ctrl_ops *ops = &ave_ctrl_ops; + struct v4l2_ctrl *c; + + v4l2_ctrl_handler_init(hdl, 17); + + /* Constant quality needs a quality-to-QP mapping that is not known yet. */ + v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_BITRATE_MODE, + V4L2_MPEG_VIDEO_BITRATE_MODE_CBR, + BIT(V4L2_MPEG_VIDEO_BITRATE_MODE_CQ), + V4L2_MPEG_VIDEO_BITRATE_MODE_VBR); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_BITRATE, 100000, 100000000, 1, + 4000000); + /* + * In VBR mode the peak caps the data rate over any one second; + * 0 means twice the average, as macOS sets it. + */ + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_BITRATE_PEAK, 0, 200000000, 1, 0); + /* 0 means no limit; the firmware then uses 120 frames. */ + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_GOP_SIZE, 0, 65535, 1, 30); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME, 0, 0, 0, 0); + /* + * The firmware either uses up to three B-frames (reordering them + * itself) or none. Baseline and intra-only H.264 never have any. + */ + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_B_FRAMES, 0, 3, 3, 3); + v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_HEADER_MODE, + V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME, + BIT(V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE), + V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME); + + v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_H264_PROFILE, + V4L2_MPEG_VIDEO_H264_PROFILE_HIGH, + ~(BIT(V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE) | + BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE) | + BIT(V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) | + BIT(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH)), + V4L2_MPEG_VIDEO_H264_PROFILE_MAIN); + /* A minimum: the driver raises it to what size, rate and bit rate need. */ + v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_H264_LEVEL, + V4L2_MPEG_VIDEO_H264_LEVEL_5_2, 0, + V4L2_MPEG_VIDEO_H264_LEVEL_1_0); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_H264_MIN_QP, 0, 51, 1, 0); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_H264_MAX_QP, 0, 51, 1, 51); + + v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_HEVC_PROFILE, + V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN, 0, + V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN); + v4l2_ctrl_new_std_menu(hdl, ops, V4L2_CID_MPEG_VIDEO_HEVC_LEVEL, + V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1, 0, + V4L2_MPEG_VIDEO_HEVC_LEVEL_1); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_HEVC_MIN_QP, 0, 51, 1, 0); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MPEG_VIDEO_HEVC_MAX_QP, 0, 51, 1, 51); + + /* The firmware holds up to five frames; one more keeps it fed. */ + c = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_MIN_BUFFERS_FOR_OUTPUT, 1, 32, 1, + AVE_MAX_SLOTS + 1); + if (c) + c->flags |= V4L2_CTRL_FLAG_READ_ONLY; + + if (hdl->error) { + int ret = hdl->error; + + v4l2_ctrl_handler_free(hdl); + return ret; + } + ctx->fh.ctrl_handler = hdl; + return v4l2_ctrl_handler_setup(hdl); +} + +/* Queues */ + +static int ave_queue_setup(struct vb2_queue *vq, unsigned int *nbufs, unsigned int *nplanes, + unsigned int sizes[], struct device *alloc_devs[]) +{ + struct ave_ctx *ctx = vb2_get_drv_priv(vq); + const struct v4l2_pix_format_mplane *f = + V4L2_TYPE_IS_OUTPUT(vq->type) ? &ctx->src_fmt : &ctx->dst_fmt; + unsigned int have = vb2_get_num_buffers(vq); + unsigned long slot = roundup_pow_of_two(PAGE_ALIGN(f->plane_fmt[0].sizeimage)); + unsigned int max = (V4L2_TYPE_IS_OUTPUT(vq->type) ? AVE_OUTPUT_BUDGET : + AVE_CAPTURE_BUDGET) / slot; + + max = clamp(max, V4L2_TYPE_IS_OUTPUT(vq->type) ? AVE_MAX_SLOTS + 1U : 2U, + AVE_MAX_QUEUE_BUFS); + + /* Frames stay with the firmware until encoded; keep one spare. */ + if (V4L2_TYPE_IS_OUTPUT(vq->type) && have + *nbufs < AVE_MAX_SLOTS + 1) + *nbufs = AVE_MAX_SLOTS + 1 - have; + /* A full set of both queues must fit the budget for the DART window. */ + if (have >= max) { + dev_dbg(ctx->ave->dev, "%s queue already has its %u buffers\n", + V4L2_TYPE_IS_OUTPUT(vq->type) ? "OUTPUT" : "CAPTURE", max); + return -ENOMEM; + } + if (*nbufs > max - have) + dev_dbg(ctx->ave->dev, "%s queue: %u buffers asked, %u given\n", + V4L2_TYPE_IS_OUTPUT(vq->type) ? "OUTPUT" : "CAPTURE", *nbufs, max - have); + *nbufs = min(*nbufs, max - have); + if (*nplanes) + return (*nplanes != 1 || sizes[0] < f->plane_fmt[0].sizeimage) ? -EINVAL : 0; + *nplanes = 1; + sizes[0] = f->plane_fmt[0].sizeimage; + return 0; +} + +static int ave_buf_prepare(struct vb2_buffer *vb) +{ + struct ave_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + const struct v4l2_pix_format_mplane *f = + V4L2_TYPE_IS_OUTPUT(vb->type) ? &ctx->src_fmt : &ctx->dst_fmt; + + if (vb2_plane_size(vb, 0) < f->plane_fmt[0].sizeimage) + return -EINVAL; + /* Keep what the client filled in, so its layout can be checked. */ + if (V4L2_TYPE_IS_OUTPUT(vb->type) && !vb2_get_plane_payload(vb, 0)) + vb2_set_plane_payload(vb, 0, f->plane_fmt[0].sizeimage); + return 0; +} + +static void ave_buf_queue(struct vb2_buffer *vb) +{ + struct ave_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + unsigned long flags; + + spin_lock_irqsave(&ctx->queue_lock, flags); + if (ctx->failed) + v4l2_m2m_buf_done(to_vb2_v4l2_buffer(vb), VB2_BUF_STATE_ERROR); + else + v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, to_vb2_v4l2_buffer(vb)); + spin_unlock_irqrestore(&ctx->queue_lock, flags); +} + +static void ave_return_bufs(struct ave_ctx *ctx, struct vb2_queue *q, + enum vb2_buffer_state state) +{ + struct vb2_v4l2_buffer *vbuf; + + for (;;) { + vbuf = V4L2_TYPE_IS_OUTPUT(q->type) ? + v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx) : + v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + if (!vbuf) + break; + v4l2_m2m_buf_done(vbuf, state); + } +} + +/* Sessions */ + +static int ave_session_index(struct ave_dev *ave, struct ave_ctx *ctx) +{ + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i] == ctx) + return i; + return -1; +} + +static unsigned int ave_session_count(struct ave_dev *ave) +{ + unsigned int n = 0; + + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + n += !!ave->sessions[i]; + return n; +} + +/* + * Drain (with @flush), stop and close the session in the firmware. The + * last session to go halts the firmware. Caller holds hw_lock. + */ +static void ave_session_close(struct ave_ctx *ctx, bool flush) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + int idx = ave_session_index(ave, ctx); + + lockdep_assert_held(&ave->hw_lock); + if (!sess || idx < 0) + return; + if (sess->fw_open && ave->fw_state == AVE_FW_RUNNING) { + /* Complete is a barrier: answered after every frame before it. */ + if (flush && sess->frames && !ave_cmd_complete(ctx, AVE_DRAIN_COMPLETES)) + ave_ipc_settle(ave, AVE_FRAME_WAIT_MS); + if (ave->fw_state == AVE_FW_RUNNING) + ave_cmd_close(ctx); + } + sess->fw_open = false; + ave->sessions[idx] = NULL; + if (!ave_session_count(ave)) + ave_fw_halt(ave); +} + +/* + * Open and Start the session in the firmware, booting the firmware first + * if no other session runs. Returns -EAGAIN when a halted coprocessor has + * not lost power yet. Caller holds hw_lock. + */ +static int ave_session_attach(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + int idx = -1, ret; + + lockdep_assert_held(&ave->hw_lock); + for (unsigned int i = 0; i < AVE_MAX_SESSIONS && idx < 0; i++) + if (!ave->sessions[i]) + idx = i; + if (idx < 0) { + dev_err(ave->dev, "already encoding %u streams\n", AVE_MAX_SESSIONS); + return -EBUSY; + } + if (!ave_session_count(ave)) { + ret = ave_fw_ensure_running(ave); + if (ret) + return ret; + } else if (ave->fw_state != AVE_FW_RUNNING) { + dev_err(ave->dev, "firmware state %d, cannot add a session\n", ave->fw_state); + return -EIO; + } + + sess->index = idx; + sess->frames = 0; + sess->nudged = false; + ave->sessions[idx] = ctx; + ret = ave_cmd_open(ctx); + if (!ret) + ret = ave_cmd_start(ctx); + if (ret) + ave_session_close(ctx, false); + return ret; +} + +/* + * Attach with the device runtime active and one PM reference held by the + * caller. A halted coprocessor restarts only from reset, and its power can + * take a moment to go (the DARTs suspend a little later), so drop the + * reference and cycle the power a few times before giving up. On return + * the reference is still held, unless *pm_lost says otherwise. + */ +static int ave_session_attach_cycled(struct ave_ctx *ctx, bool *pm_lost) +{ + struct ave_dev *ave = ctx->ave; + int ret = -EAGAIN; + + *pm_lost = false; + for (unsigned int i = 0; i < AVE_RESTART_TRIES && ret == -EAGAIN; i++) { + if (i) { + pm_runtime_put_sync_suspend(ave->dev); + msleep(AVE_RESTART_WAIT_MS); + ret = pm_runtime_resume_and_get(ave->dev); + if (ret) { + *pm_lost = true; + return ret; + } + } + mutex_lock(&ave->hw_lock); + ret = ave_session_attach(ctx); + mutex_unlock(&ave->hw_lock); + } + if (ret == -EAGAIN) + dev_err(ave->dev, "coprocessor did not power down; cannot restart it\n"); + return ret; +} + +/* Return frames the firmware never answered; then drop the session. */ +static void ave_session_free(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + + if (!sess) + return; + for (unsigned int i = 0; i < sess->p.slots; i++) { + struct ave_slot *s = &sess->slot[i]; + + if (s->src) + v4l2_m2m_buf_done(s->src, VB2_BUF_STATE_ERROR); + s->src = NULL; + } + atomic_set(&ctx->ready, 0); + ave_sess_destroy(ctx); + pm_runtime_mark_last_busy(ave->dev); + pm_runtime_put_autosuspend(ave->dev); +} + +/* Boot the firmware if needed, then Open and Start a session. */ +static int ave_session_begin(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess; + bool pm_lost; + int ret; + + if (ctx->sess) + return 0; + ret = ave_fw_check_build(ave); + if (ret) + return ret; + if (READ_ONCE(ave->sessions[0]) && READ_ONCE(ave->sessions[1])) + return -EBUSY; + /* A halted firmware restarts only from reset: let the power go first. */ + if (READ_ONCE(ave->fw_state) == AVE_FW_HALTED || + READ_ONCE(ave->fw_state) == AVE_FW_BROKEN || + READ_ONCE(ave->fw_state) == AVE_FW_RECOVERING) + pm_runtime_suspend(ave->dev); + sess = ave_sess_create(ctx); + if (IS_ERR(sess)) { + dev_err(ave->dev, "cannot set up a session: %ld\n", PTR_ERR(sess)); + return PTR_ERR(sess); + } + + ctx->sess = sess; + ret = pm_runtime_resume_and_get(ave->dev); + if (ret) { + ave_sess_destroy(ctx); + return ret; + } + ret = ave_session_attach_cycled(ctx, &pm_lost); + if (ret) { + dev_err(ave->dev, "cannot start an encoding session: %d\n", ret); + if (pm_lost) + ave_sess_destroy(ctx); + else + ave_session_free(ctx); + } + return ret; +} + +/* + * After system sleep: the session was closed when the firmware went down. + * Open and Start it again with the same parameters; its next frame is an + * IDR with fresh parameter sets, so the stream stays decodable. + */ +static int ave_session_reopen(struct ave_ctx *ctx) +{ + bool pm_lost; + int ret; + + ret = ave_session_attach_cycled(ctx, &pm_lost); + if (ret) { + if (pm_lost) { + /* The session's PM reference is gone; do not drop it again. */ + pm_runtime_get_noresume(ctx->ave->dev); + } + return ret; + } + ctx->sess->resume = false; + dev_dbg(ctx->ave->dev, "session reopened after system sleep\n"); + return 0; +} + +/* + * System sleep: drain every open session, close it and halt the firmware, + * as at the end of a stream. Sessions reopen on their next frame. + * Caller holds hw_lock. + */ +void ave_v4l2_suspend(struct ave_dev *ave) +{ + lockdep_assert_held(&ave->hw_lock); + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) { + struct ave_ctx *ctx = ave->sessions[i]; + + if (!ctx) + continue; + cancel_delayed_work(&ctx->poll_work); + ave_session_close(ctx, true); + ctx->sess->resume = true; + } +} + +static void ave_session_end(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + + cancel_delayed_work_sync(&ctx->poll_work); + mutex_lock(&ave->hw_lock); + ave_session_close(ctx, true); + mutex_unlock(&ave->hw_lock); + ave_session_free(ctx); +} + +static void ave_reset_drain(struct ave_ctx *ctx) +{ + ctx->draining = false; + WRITE_ONCE(ctx->flush_pending, false); + ctx->stopped = false; + ctx->drain_src = NULL; +} + +static int ave_start_streaming(struct vb2_queue *q, unsigned int count) +{ + struct ave_ctx *ctx = vb2_get_drv_priv(q); + struct vb2_queue *other = V4L2_TYPE_IS_OUTPUT(q->type) ? + v4l2_m2m_get_dst_vq(ctx->fh.m2m_ctx) : + v4l2_m2m_get_src_vq(ctx->fh.m2m_ctx); + int ret = 0; + + if (V4L2_TYPE_IS_OUTPUT(q->type)) + ctx->src_sequence = 0; + else + ctx->dst_sequence = 0; + ave_reset_drain(ctx); + + /* The session starts once both queues stream; errors show up here. */ + if (vb2_is_streaming(other)) + ret = ave_session_begin(ctx); + if (ret) + ave_return_bufs(ctx, q, VB2_BUF_STATE_QUEUED); + return ret; +} + +static void ave_stop_streaming(struct vb2_queue *q) +{ + struct ave_ctx *ctx = vb2_get_drv_priv(q); + struct vb2_queue *other = V4L2_TYPE_IS_OUTPUT(q->type) ? + v4l2_m2m_get_dst_vq(ctx->fh.m2m_ctx) : + v4l2_m2m_get_src_vq(ctx->fh.m2m_ctx); + + ave_session_end(ctx); + ave_reset_drain(ctx); + ave_return_bufs(ctx, q, VB2_BUF_STATE_ERROR); + /* vb2 hands queued buffers to buf_queue before the next STREAMON callback. */ + if (!vb2_is_streaming(other)) + WRITE_ONCE(ctx->failed, false); +} + +static const struct vb2_ops ave_qops = { + .queue_setup = ave_queue_setup, + .buf_prepare = ave_buf_prepare, + .buf_queue = ave_buf_queue, + .start_streaming = ave_start_streaming, + .stop_streaming = ave_stop_streaming, +}; + +static int ave_queue_init(void *priv, struct vb2_queue *src, struct vb2_queue *dst) +{ + struct ave_ctx *ctx = priv; + int ret; + + src->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + src->io_modes = VB2_MMAP | VB2_DMABUF; + src->drv_priv = ctx; + src->buf_struct_size = sizeof(struct v4l2_m2m_buffer); + src->ops = &ave_qops; + src->mem_ops = &ave_vb2_memops; + src->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + src->lock = &ctx->ave->vdev_lock; + src->dev = ctx->ave->dev; + ret = vb2_queue_init(src); + if (ret) + return ret; + + dst->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + dst->io_modes = VB2_MMAP | VB2_DMABUF; + dst->drv_priv = ctx; + dst->buf_struct_size = sizeof(struct v4l2_m2m_buffer); + dst->ops = &ave_qops; + dst->mem_ops = &vb2_dma_contig_memops; + dst->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + dst->lock = &ctx->ave->vdev_lock; + dst->dev = ctx->ave->dev; + return vb2_queue_init(dst); +} + +/* + * Jobs. A job delivers finished frames to CAPTURE buffers and sends at most + * one new frame. Up to five frames are inside the firmware at once; their + * answers are collected by the job and, while frames are outstanding, by a + * short poller, since the firmware interrupt is not wired up. + */ + +#define AVE_POLL_MS 2 + +static int ave_job_ready(void *priv) +{ + struct ave_ctx *ctx = priv; + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + + if (READ_ONCE(ctx->failed)) + return 0; + if (atomic_read(&ctx->ready) || READ_ONCE(ctx->flush_pending)) + return 1; + if (ctx->stopped || (ctx->draining && !ctx->drain_src)) + return 0; + return v4l2_m2m_num_src_bufs_ready(m2m) > 0; +} + +/* + * Hand finished frames, in bitstream order, to CAPTURE buffers. Returns + * true when the frame that ends a drain went out (with the LAST flag). + */ +static bool ave_deliver(struct ave_ctx *ctx) +{ + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + struct ave_session *sess = ctx->sess; + bool last = false; + + while (sess && sess->done_count) { + unsigned int slot = sess->done_fifo[sess->done_head]; + struct ave_slot *s = &sess->slot[slot]; + struct vb2_v4l2_buffer *dst; + enum vb2_buffer_state state = VB2_BUF_STATE_DONE; + bool key = false; + + dst = v4l2_m2m_dst_buf_remove(m2m); + if (!dst) + break; + if (ave_sess_output(ctx, slot, dst, &key)) { + vb2_set_plane_payload(&dst->vb2_buf, 0, 0); + state = VB2_BUF_STATE_ERROR; + } + v4l2_m2m_buf_copy_metadata(s->src, dst); + dst->flags &= ~(V4L2_BUF_FLAG_KEYFRAME | V4L2_BUF_FLAG_PFRAME | + V4L2_BUF_FLAG_BFRAME | V4L2_BUF_FLAG_LAST); + dst->flags |= key ? V4L2_BUF_FLAG_KEYFRAME : V4L2_BUF_FLAG_PFRAME; + dst->sequence = ctx->dst_sequence++; + + sess->done_head = (sess->done_head + 1) % AVE_MAX_SLOTS; + sess->done_count--; + atomic_dec(&ctx->ready); + s->done = false; + ave_sess_prepare_slot(sess, slot); + + /* The last frame of a drain carries the LAST flag. */ + if (READ_ONCE(ctx->flush_pending) && !sess->fw_open && !sess->done_count) { + dst->flags |= V4L2_BUF_FLAG_LAST; + last = true; + } + ave_vb2_put(s->input_mem); + s->input_mem = NULL; + v4l2_m2m_buf_done(s->src, state); + s->src = NULL; + v4l2_m2m_buf_done(dst, state); + } + return last; +} + +static bool ave_slot_answered(struct ave_dev *ave, void *arg) +{ + struct ave_session *sess = arg; + + return sess->busy < sess->p.slots || sess->done_count; +} + +static bool ave_frame_answered(struct ave_dev *ave, void *arg) +{ + struct ave_slot *s = arg; + + return !s->busy; +} + +/* Send the next OUTPUT frame. Caller holds hw_lock. */ +static int ave_encode_one(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + struct vb2_v4l2_buffer *src; + dma_addr_t input; + bool last; + int slot, ret; + + slot = ave_sess_free_slot(sess); + if (slot < 0 && sess->busy == sess->p.slots) { + /* Every slot is inside the firmware: wait for one to come back. */ + ret = ave_ipc_pump(ave, AVE_FRAME_WAIT_MS, ave_slot_answered, sess); + if (ret) + return ret; + ave_deliver(ctx); + slot = ave_sess_free_slot(sess); + } + if (slot < 0) + return 0; /* waiting for CAPTURE buffers */ + + src = v4l2_m2m_src_buf_remove(m2m); + if (!src) + return 0; + last = src == ctx->drain_src; + src->sequence = ctx->src_sequence++; + + ret = ave_sess_input(ctx, slot, src, &input); + if (!ret) + ret = ave_cmd_process(ctx, slot, src, input, READ_ONCE(ctx->bitrate), + xchg(&ctx->force_key_frame, false)); + if (ret) { + v4l2_m2m_buf_done(src, VB2_BUF_STATE_ERROR); + return ret; + } + + /* Like macOS: the first frame goes alone, then a Complete. */ + if (sess->frames == 1) { + ret = ave_ipc_pump(ave, AVE_FRAME_WAIT_MS, ave_frame_answered, &sess->slot[slot]); + if (!ret) + ret = ave_cmd_complete(ctx, 1); + if (!ret) + ret = ave_ipc_settle(ave, AVE_FRAME_WAIT_MS); + if (ret) + return ret; + } else { + ret = ave_ipc_pump(ave, 0, NULL, NULL); + if (ret) + return ret; + } + + if (last) { + ctx->drain_src = NULL; + WRITE_ONCE(ctx->flush_pending, true); + } + return 0; +} + +/* + * End of a drain: flush and close the session, deliver what is left and + * finish with a LAST buffer. Caller holds hw_lock; returns true once the + * LAST buffer is out. + */ +static bool ave_finish_drain(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + struct vb2_v4l2_buffer *dst; + + if (sess && sess->fw_open) + ave_session_close(ctx, true); + if (sess && sess->busy) { + /* The run job fails the session. */ + dev_err(ave->dev, "%u frames never came back from the firmware\n", sess->busy); + return false; + } + if (sess && sess->done_count) + return ave_deliver(ctx); /* false: waiting for CAPTURE buffers */ + + /* Nothing left to deliver: an empty buffer carries the LAST flag. */ + dst = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + if (!dst) + return false; + vb2_set_plane_payload(&dst->vb2_buf, 0, 0); + dst->flags |= V4L2_BUF_FLAG_LAST; + dst->sequence = ctx->dst_sequence++; + v4l2_m2m_buf_done(dst, VB2_BUF_STATE_DONE); + return true; +} + +/* The firmware or the session failed: return everything, fail the queues. */ +static void ave_job_fail(struct ave_ctx *ctx, int err) +{ + struct ave_dev *ave = ctx->ave; + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + unsigned long flags; + + /* Publish the fatal error before any teardown that can wait on PM/IPC. */ + spin_lock_irqsave(&ctx->queue_lock, flags); + WRITE_ONCE(ctx->failed, true); + vb2_queue_error(v4l2_m2m_get_src_vq(m2m)); + vb2_queue_error(v4l2_m2m_get_dst_vq(m2m)); + ave_return_bufs(ctx, v4l2_m2m_get_src_vq(m2m), VB2_BUF_STATE_ERROR); + ave_return_bufs(ctx, v4l2_m2m_get_dst_vq(m2m), VB2_BUF_STATE_ERROR); + spin_unlock_irqrestore(&ctx->queue_lock, flags); + cancel_delayed_work(&ctx->poll_work); + dev_err(ave->dev, "encoding failed: %d\n", err); + mutex_lock(&ave->hw_lock); + ave_session_close(ctx, false); + mutex_unlock(&ave->hw_lock); + ave_session_free(ctx); + ave_reset_drain(ctx); +} + +static void ave_run_work(struct work_struct *work) +{ + struct ave_ctx *ctx = container_of(work, struct ave_ctx, run_work); + struct ave_dev *ave = ctx->ave; + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + bool eos = false, poll = false; + int ret = 0; + + if (READ_ONCE(ctx->failed)) + goto finish; + + /* + * After a drain the next frame starts a new session; after system + * sleep it reopens the old one in the restarted firmware. + */ + if (!READ_ONCE(ctx->flush_pending) && v4l2_m2m_num_src_bufs_ready(m2m)) { + if (!ctx->sess) + ret = ave_session_begin(ctx); + else if (ctx->sess->resume) + ret = ave_session_reopen(ctx); + } + + mutex_lock(&ave->hw_lock); + /* + * A closed session (after a drain or for sleep) may still hold frames + * for CAPTURE buffers; only an open one must still own the firmware. + */ + if (!ret && ctx->sess && ctx->sess->fw_open && ave_session_index(ave, ctx) < 0) { + dev_err(ave->dev, "session lost the firmware\n"); + ret = -EIO; + } + if (!ret) { + eos = ave_deliver(ctx); + if (!eos && !READ_ONCE(ctx->flush_pending) && ctx->sess && + v4l2_m2m_num_src_bufs_ready(m2m) && v4l2_m2m_num_dst_bufs_ready(m2m)) + ret = ave_encode_one(ctx); + } + if (!ret && !eos) { + eos = ave_deliver(ctx); + if (!eos && READ_ONCE(ctx->flush_pending)) + eos = ave_finish_drain(ctx); + /* Frames outstanding need a running firmware. */ + if (!eos && ctx->sess && ctx->sess->busy && ave->fw_state != AVE_FW_RUNNING) { + dev_err(ave->dev, "%u frames outstanding, firmware state %d\n", + ctx->sess->busy, ave->fw_state); + ret = -EIO; + } + poll = !ret && !eos && ctx->sess && ctx->sess->busy; + } + mutex_unlock(&ave->hw_lock); + + if (ret) { + ave_job_fail(ctx, ret); + } else if (eos) { + ave_session_free(ctx); + ctx->draining = false; + WRITE_ONCE(ctx->flush_pending, false); + ctx->stopped = true; + v4l2_event_queue_fh(&ctx->fh, &ave_eos_event); + } else if (poll) { + mod_delayed_work(ave->wq, &ctx->poll_work, msecs_to_jiffies(AVE_POLL_MS)); + } +finish: + pm_runtime_mark_last_busy(ave->dev); + v4l2_m2m_job_finish(ave->m2m_dev, m2m); +} + +/* Collect frame answers while frames are outstanding and no job runs. */ +static void ave_poll_work(struct work_struct *work) +{ + struct ave_ctx *ctx = container_of(to_delayed_work(work), struct ave_ctx, poll_work); + struct ave_dev *ave = ctx->ave; + bool again = false, fail = false; + + mutex_lock(&ave->hw_lock); + if (ctx->sess && ave->fw_state == AVE_FW_BROKEN) { + fail = true; + } else if (ctx->sess && ave_session_index(ave, ctx) >= 0) { + fail = ave_ipc_pump(ave, 0, NULL, NULL) != 0; + if (!fail && ave_sess_starved(ctx->sess) && + !v4l2_m2m_num_src_bufs_ready(ctx->fh.m2m_ctx)) { + dev_dbg(ave->dev, "no input, flushing %u held frames\n", ctx->sess->busy); + ctx->sess->nudged = true; + ctx->sess->progress = jiffies; + fail = ave_cmd_complete(ctx, 1) != 0; + } + if (!fail && ave_sess_stalled(ctx->sess)) { + /* + * The firmware holds frames it should have answered. + * Treat it as stuck, so the session fails now instead + * of leaving the client waiting. + */ + dev_err(ave->dev, "%u frames unanswered for %u ms; firmware is stuck\n", + ctx->sess->busy, AVE_FRAME_WAIT_MS); + ave_fw_mark_broken(ave, -ETIMEDOUT, "frame timeout"); + fail = true; + } + again = !fail && ctx->sess->busy; + } else if (ctx->sess && ctx->sess->busy) { + dev_err(ave->dev, "%u frames outstanding in a session that lost the firmware\n", + ctx->sess->busy); + fail = true; + } + mutex_unlock(&ave->hw_lock); + + /* Jobs run on the same ordered queue, so none runs right now. */ + if (fail) + ave_job_fail(ctx, -EIO); + else if (atomic_read(&ctx->ready)) + v4l2_m2m_try_schedule(ctx->fh.m2m_ctx); + if (again) + queue_delayed_work(ave->wq, &ctx->poll_work, msecs_to_jiffies(AVE_POLL_MS)); +} + +static void ave_device_run(void *priv) +{ + struct ave_ctx *ctx = priv; + + queue_work(ctx->ave->wq, &ctx->run_work); +} + +static const struct v4l2_m2m_ops ave_m2m_ops = { + .device_run = ave_device_run, + .job_ready = ave_job_ready, +}; + +/* File operations */ + +static int ave_open(struct file *file) +{ + struct ave_dev *ave = video_drvdata(file); + struct ave_ctx *ctx; + int ret; + + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + ctx->ave = ave; + spin_lock_init(&ctx->queue_lock); + INIT_WORK(&ctx->run_work, ave_run_work); + INIT_DELAYED_WORK(&ctx->poll_work, ave_poll_work); + atomic_set(&ctx->ready, 0); + ave_reset_fmts(ctx); + + v4l2_fh_init(&ctx->fh, video_devdata(file)); + ret = ave_init_ctrls(ctx); + if (ret) + goto err_fh; + + ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(ave->m2m_dev, ctx, ave_queue_init); + if (IS_ERR(ctx->fh.m2m_ctx)) { + ret = PTR_ERR(ctx->fh.m2m_ctx); + goto err_ctrls; + } + /* Finished frames can be delivered with no new OUTPUT buffer queued. */ + v4l2_m2m_set_src_buffered(ctx->fh.m2m_ctx, true); + v4l2_fh_add(&ctx->fh, file); + return 0; + +err_ctrls: + v4l2_ctrl_handler_free(&ctx->hdl); +err_fh: + v4l2_fh_exit(&ctx->fh); + kfree(ctx); + return ret; +} + +static int ave_release(struct file *file) +{ + struct ave_ctx *ctx = ave_file_to_ctx(file); + struct ave_dev *ave = ctx->ave; + + mutex_lock(&ave->vdev_lock); + v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); + mutex_unlock(&ave->vdev_lock); + cancel_delayed_work_sync(&ctx->poll_work); + cancel_work_sync(&ctx->run_work); + v4l2_ctrl_handler_free(&ctx->hdl); + v4l2_fh_del(&ctx->fh, file); + v4l2_fh_exit(&ctx->fh); + kfree(ctx); + return 0; +} + +static const struct v4l2_file_operations ave_fops = { + .owner = THIS_MODULE, + .open = ave_open, + .release = ave_release, + .poll = v4l2_m2m_fop_poll, + .unlocked_ioctl = video_ioctl2, + .mmap = v4l2_m2m_fop_mmap, +}; + +int ave_v4l2_register(struct ave_dev *ave) +{ + struct video_device *vdev = &ave->vdev; + int ret; + + mutex_init(&ave->vdev_lock); + ave->wq = alloc_ordered_workqueue("apple-ave", WQ_MEM_RECLAIM); + if (!ave->wq) + return -ENOMEM; + + ret = v4l2_device_register(ave->dev, &ave->v4l2_dev); + if (ret) + goto err_wq; + + ave->m2m_dev = v4l2_m2m_init(&ave_m2m_ops); + if (IS_ERR(ave->m2m_dev)) { + ret = PTR_ERR(ave->m2m_dev); + goto err_v4l2; + } + + strscpy(vdev->name, AVE_NAME, sizeof(vdev->name)); + vdev->fops = &ave_fops; + vdev->ioctl_ops = &ave_ioctl_ops; + vdev->release = video_device_release_empty; + vdev->lock = &ave->vdev_lock; + vdev->v4l2_dev = &ave->v4l2_dev; + vdev->vfl_dir = VFL_DIR_M2M; + vdev->device_caps = V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING; + video_set_drvdata(vdev, ave); + + ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1); + if (ret) + goto err_m2m; + return 0; + +err_m2m: + v4l2_m2m_release(ave->m2m_dev); +err_v4l2: + v4l2_device_unregister(&ave->v4l2_dev); +err_wq: + destroy_workqueue(ave->wq); + return ret; +} + +void ave_v4l2_unregister(struct ave_dev *ave) +{ + video_unregister_device(&ave->vdev); + v4l2_m2m_release(ave->m2m_dev); + v4l2_device_unregister(&ave->v4l2_dev); + destroy_workqueue(ave->wq); +} diff --git a/drivers/media/platform/apple/ave/ave-vb2.c b/drivers/media/platform/apple/ave/ave-vb2.c new file mode 100644 index 0000000000000..ba0325c9c82dc --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-vb2.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Keep OUTPUT mappings alive independently of vb2 buffer completion. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include + +#include + +#include "ave.h" + +/* + * vb2 owns one reference, each submitted Process owns another. A failed + * session keeps its references in quarantine until the coprocessor and DART + * have lost power. In particular, DQBUF/REQBUFS/close may release vb2's + * reference without unmapping an imported attachment or freeing an MMAP IOVA. + */ +struct ave_vb2_mem { + refcount_t refs; + void *priv; + struct dma_buf *dbuf; + struct device *dev; + bool mapped; +}; + +struct ave_vb2_mem *ave_vb2_get(struct vb2_buffer *vb) +{ + struct ave_vb2_mem *mem = vb->planes[0].mem_priv; + + refcount_inc(&mem->refs); + return mem; +} + +void ave_vb2_put(void *buf_priv) +{ + struct ave_vb2_mem *mem = buf_priv; + + if (!mem || !refcount_dec_and_test(&mem->refs)) + return; + if (mem->dbuf) { + if (mem->mapped) + vb2_dma_contig_memops.unmap_dmabuf(mem->priv); + vb2_dma_contig_memops.detach_dmabuf(mem->priv); + dma_buf_put(mem->dbuf); + put_device(mem->dev); + } else { + vb2_dma_contig_memops.put(mem->priv); + } + kfree(mem); +} + +static void *ave_vb2_alloc(struct vb2_buffer *vb, struct device *dev, + unsigned long size) +{ + struct ave_vb2_mem *mem; + + mem = kzalloc_obj(*mem); + if (!mem) + return ERR_PTR(-ENOMEM); + mem->priv = vb2_dma_contig_memops.alloc(vb, dev, size); + if (IS_ERR(mem->priv)) { + void *err = mem->priv; + + kfree(mem); + return err; + } + refcount_set(&mem->refs, 1); + return mem; +} + +static void *ave_vb2_attach(struct vb2_buffer *vb, struct device *dev, + struct dma_buf *dbuf, unsigned long size) +{ + struct ave_vb2_mem *mem; + + mem = kzalloc_obj(*mem); + if (!mem) + return ERR_PTR(-ENOMEM); + mem->priv = vb2_dma_contig_memops.attach_dmabuf(vb, dev, dbuf, size); + if (IS_ERR(mem->priv)) { + void *err = mem->priv; + + kfree(mem); + return err; + } + /* vb2 can drop its dma_buf and device references before quarantine ends. */ + get_dma_buf(dbuf); + mem->dbuf = dbuf; + mem->dev = get_device(dev); + refcount_set(&mem->refs, 1); + return mem; +} + +static int ave_vb2_map(void *buf_priv) +{ + struct ave_vb2_mem *mem = buf_priv; + int ret; + + if (mem->mapped) + return 0; + ret = vb2_dma_contig_memops.map_dmabuf(mem->priv); + if (!ret) + mem->mapped = true; + return ret; +} + +static void ave_vb2_unmap(void *buf_priv) +{ + /* The final put unmaps, after both vb2 and firmware have let go. */ +} + +static void *ave_vb2_cookie(struct vb2_buffer *vb, void *buf_priv) +{ + struct ave_vb2_mem *mem = buf_priv; + + return vb2_dma_contig_memops.cookie(vb, mem->priv); +} + +static void *ave_vb2_vaddr(struct vb2_buffer *vb, void *buf_priv) +{ + struct ave_vb2_mem *mem = buf_priv; + + return vb2_dma_contig_memops.vaddr(vb, mem->priv); +} + +static int ave_vb2_mmap(void *buf_priv, struct vm_area_struct *vma) +{ + struct ave_vb2_mem *mem = buf_priv; + + return vb2_dma_contig_memops.mmap(mem->priv, vma); +} + +static struct dma_buf *ave_vb2_export(struct vb2_buffer *vb, void *buf_priv, + unsigned long flags) +{ + struct ave_vb2_mem *mem = buf_priv; + + return vb2_dma_contig_memops.get_dmabuf(vb, mem->priv, flags); +} + +static unsigned int ave_vb2_num_users(void *buf_priv) +{ + struct ave_vb2_mem *mem = buf_priv; + + return vb2_dma_contig_memops.num_users(mem->priv); +} + +static void ave_vb2_prepare(void *buf_priv) +{ + struct ave_vb2_mem *mem = buf_priv; + + vb2_dma_contig_memops.prepare(mem->priv); +} + +static void ave_vb2_finish(void *buf_priv) +{ + struct ave_vb2_mem *mem = buf_priv; + + vb2_dma_contig_memops.finish(mem->priv); +} + +/* OUTPUT supports MMAP and DMABUF; CAPTURE never goes to the firmware. */ +const struct vb2_mem_ops ave_vb2_memops = { + .alloc = ave_vb2_alloc, + .put = ave_vb2_put, + .get_dmabuf = ave_vb2_export, + .cookie = ave_vb2_cookie, + .vaddr = ave_vb2_vaddr, + .mmap = ave_vb2_mmap, + .prepare = ave_vb2_prepare, + .finish = ave_vb2_finish, + .map_dmabuf = ave_vb2_map, + .unmap_dmabuf = ave_vb2_unmap, + .attach_dmabuf = ave_vb2_attach, + .detach_dmabuf = ave_vb2_put, + .num_users = ave_vb2_num_users, +}; diff --git a/drivers/media/platform/apple/ave/ave-writer.c b/drivers/media/platform/apple/ave/ave-writer.c new file mode 100644 index 0000000000000..59f7d067bfc14 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave-writer.c @@ -0,0 +1,520 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple AVE video encoder: firmware commands + * + * Start and Process are large command images. Each is built in a zeroed + * buffer: the header, the constant words (ave-image.c), then every field + * that depends on the session or the frame. + * + * Copyright The Aurora Silicon Contributors + */ +#include +#include +#include +#include +#include + +#include + +#include "ave.h" +#include "ave-regs.h" + +/* + * IEEE 754 double for the positive value @num / @den (0 for 0), built with + * integer arithmetic; the firmware takes rates and limits as doubles. + */ +static u64 ave_f64(u64 num, u64 den) +{ + u64 mant = 0, rem; + int exp = 0; + + if (!num || !den) + return 0; + while (num >= den << 1) { + den <<= 1; + exp++; + } + while (num < den) { + num <<= 1; + exp--; + } + /* 1 <= num/den < 2: take 53 significant bits by long division. */ + rem = num; + for (unsigned int i = 0; i < 53; i++) { + mant <<= 1; + if (rem >= den) { + mant |= 1; + rem -= den; + } + rem <<= 1; + } + return (u64)(exp + 1023) << 52 | (mant & GENMASK_ULL(51, 0)); +} + +#define AVE_F64_NEG BIT_ULL(63) + +static void ave_put32(u8 *cmd, u32 off, u32 val) +{ + put_unaligned_le32(val, cmd + off); +} + +static void ave_put64(u8 *cmd, u32 off, u64 val) +{ + put_unaligned_le64(val, cmd + off); +} + +void ave_write_surface(const struct ave_fw_layout *fw, u8 *cmd, + const struct ave_surface_layout *l, unsigned int index, + dma_addr_t addr, u32 size) +{ + u32 address_off = l->address + l->address_stride * index; + u32 size_off = l->size + l->size_stride * index; + + ave_put64(cmd, address_off, addr); + ave_put32(cmd, size_off, size); + if (fw->abi->padded_surfaces) + ave_put32(cmd, size_off + 4, 0); +} + +static void ave_put_buf(const struct ave_fw_layout *fw, u8 *cmd, + const struct ave_surface_layout *l, unsigned int index, + const struct ave_buf *b, u32 size) +{ + ave_write_surface(fw, cmd, l, index, b->dma, size); +} + +/* Is another session open in the firmware besides @ctx's? */ +static bool ave_cmd_others_open(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i] && ave->sessions[i] != ctx && + ave->sessions[i]->sess->fw_open) + return true; + return false; +} + +/* + * Header word +0x2c depends on the set of open sessions: 0x1e0 on Open and + * Start (0x140 when another session is open), the codec value on every + * other command of a lone session, and 0 while another one is open. + */ +u32 ave_session_arg2(struct ave_ctx *ctx) +{ + if (ctx->ave->layout->counted && ctx->sess->p.codec == AVE_CODEC_H264) + return ctx->sess->p.bframes ? AVE_SESSION_ARG2_H264 : 0x140; + if (ave_cmd_others_open(ctx)) + return 0; + return ctx->sess->p.codec == AVE_CODEC_HEVC ? AVE_SESSION_ARG2_HEVC : + AVE_SESSION_ARG2_H264; +} + +u32 ave_session_setup_arg2(struct ave_ctx *ctx) +{ + return ave_cmd_others_open(ctx) ? AVE_SESSION_ARG2_SETUP_SHARED : + AVE_SESSION_ARG2_SETUP; +} + +void ave_write_header(struct ave_ctx *ctx, u8 *cmd, enum ave_opcode op, + u32 echo, u32 arg0, u32 arg2) +{ + const struct ave_fw_abi *abi = ctx->ave->layout->abi; + struct ave_command header = { + .op = op, + .codec = ctx->sess->p.codec, + .client = ctx->sess->client, + .subtype = echo, + .arg0 = arg0, + .arg2 = arg2, + }; + + ctx->sess->token += AVE_TOKEN_STEP; + header.token = ctx->sess->token; + abi->write_header(cmd, abi, &header); +} + +void ave_write_global(struct ave_dev *ave, u8 *cmd, enum ave_opcode op) +{ + const struct ave_fw_abi *abi = ave->layout->abi; + struct ave_command header = { + .op = op, + .codec = AVE_CODEC_H264, + .subtype = U32_MAX, + }; + + abi->write_header(cmd, abi, &header); +} + +void ave_stamp_command(const struct ave_fw_abi *abi, u8 *cmd, u64 timeout, + u64 created) +{ + ave_put64(cmd, abi->timeout_offset, timeout); + ave_put64(cmd, abi->created_offset, created); +} + +#define AVE_START_ARRAY_LEN 30 +#define AVE_START_INIT_QP_VALUE 26 + +static void ave_start_surfaces(struct ave_session *sess, u8 *cmd, + const struct ave_fw_layout *fw) +{ + const struct ave_params *p = &sess->p; + bool hevc = p->codec == AVE_CODEC_HEVC; + unsigned int tiles = hevc ? 4 : AVE_TILE_BUFS; + const struct ave_start_layout *l = &fw->start[p->codec - 1]; + + /* Clear every descriptor the image may carry, then fill ours. */ + for (unsigned int i = 0; i < AVE_MAX_REFS; i++) { + ave_write_surface(fw, cmd, &l->ref_a, i, 0, 0); + ave_write_surface(fw, cmd, &l->ref_b, i, 0, 0); + ave_write_surface(fw, cmd, &l->ref_c, i, 0, 0); + ave_write_surface(fw, cmd, &l->ref_c_alias, i, 0, 0); + ave_write_surface(fw, cmd, &l->ref_d, i, 0, 0); + } + for (unsigned int s = 0; s < AVE_START_ARRAY_LEN; s++) { + ave_write_surface(fw, cmd, &l->coded, s, 0, 0); + ave_write_surface(fw, cmd, &l->meta, s, 0, 0); + } + + ave_put_buf(fw, cmd, &l->header, 0, &sess->hdr, fw->hdr_bytes); + ave_write_surface(fw, cmd, &l->secondary, 0, 0, 0); + for (unsigned int i = 0; i < p->refs; i++) { + ave_put_buf(fw, cmd, &l->ref_a, i, &sess->ref_a[i], + sess->ref_a[i].size); + ave_put_buf(fw, cmd, &l->ref_b, i, &sess->ref_b[i], + sess->ref_b[i].size); + ave_put_buf(fw, cmd, &l->ref_c, i, &sess->ref_c[i], + sess->ref_c[i].size); + ave_write_surface(fw, cmd, &l->ref_c_alias, i, + sess->ref_c[i].dma, 0); + ave_put_buf(fw, cmd, &l->ref_d, i, &sess->ref_d[i], + sess->ref_d[i].size); + } + if (fw->counted) { + ave_put32(cmd, l->refs_count, p->refs); + ave_put32(cmd, l->work_count, p->refs); + ave_put32(cmd, l->lfs_count, p->refs); + ave_put32(cmd, l->slots_count, p->slots); + ave_put32(cmd, l->meta_count, p->slots); + ave_put32(cmd, l->mb_count, p->rc_mode == 3 ? 3 : 1); + if (p->rc_mode == 3) + for (unsigned int i = 0; i < 2; i++) + ave_put_buf(fw, cmd, &l->mb, i + 1, + &sess->mb_extra[i], + sess->mb_extra[i].size); + ave_put32(cmd, l->tiles_count, tiles); + for (unsigned int i = 0; i < p->refs; i++) { + ave_put_buf(fw, cmd, &l->ref_e, i, &sess->ref_e[i], + sess->ref_e[i].size); + ave_put_buf(fw, cmd, &l->ref_f, i, &sess->ref_f[i], + sess->ref_f[i].size); + } + } + ave_put_buf(fw, cmd, &l->mb, 0, &sess->mb, sess->mb.size); + for (unsigned int s = 0; s < p->slots; s++) { + ave_put_buf(fw, cmd, &l->coded, s, &sess->slot[s].coded, + p->coded_cap); + ave_put_buf(fw, cmd, &l->meta, s, &sess->slot[s].meta, + sess->slot[s].meta.size); + } + ave_put_buf(fw, cmd, &l->row_a, 0, &sess->row_a, sess->row_a.size); + ave_put_buf(fw, cmd, &l->row_b, 0, &sess->row_b, sess->row_b.size); + ave_put_buf(fw, cmd, &l->fixed, 0, &sess->fixed, SZ_16K); + for (unsigned int i = 0; i < AVE_TILE_BUFS; i++) { + if (i < tiles) { + ave_put_buf(fw, cmd, &l->tiles[i], 0, &sess->tile[i], + sess->tile[i].size); + ave_put_buf(fw, cmd, &l->tiles[i], 1, + &sess->tile_aux[i], sess->tile_aux[i].size); + } else { + ave_write_surface(fw, cmd, &l->tiles[i], 0, 0, 0); + ave_write_surface(fw, cmd, &l->tiles[i], 1, 0, 0); + } + } + if (hevc) + for (unsigned int i = 0; i < 2; i++) + ave_put_buf(fw, cmd, &l->hevc_extra, i, + &sess->hevc_extra[i], + sess->hevc_extra[i].size); +} + +static void ave_start_put32(u8 *cmd, u32 off, u32 val) +{ + if (off) + ave_put32(cmd, off, val); +} + +static void ave_start_config(const struct ave_params *p, u8 *cmd, + const struct ave_fw_layout *fw) +{ + bool hevc = p->codec == AVE_CODEC_HEVC; + bool cq = p->rc_mode == 3; + u32 depth = p->bit_depth ?: 8; + u32 crop_r = (p->cw - p->width) / 2, crop_b = (p->ch - p->height) / 2; + bool crop = crop_r || crop_b; + const struct ave_start_layout *l = &fw->start[p->codec - 1]; + + ave_start_put32(cmd, l->codec, p->codec); + ave_start_put32( + cmd, l->flag_8bit, + fw->counted ? + 0x10000 | (hevc ? (depth == 8 ? 0x41 : 0x40) : 0) | + (cq ? 0x10 : 0) : + hevc ? 1 : + 0); + ave_start_put32(cmd, l->geometry_mode, + (hevc ? 0x40 : 0) | (cq ? 0 : 2)); + ave_start_put32(cmd, l->geometry_depth, depth); + ave_start_put32(cmd, l->const_214, 0x39d); + ave_start_put32(cmd, l->const_218, 0x18); + ave_start_put32(cmd, l->fps, p->fps_int); + + /* Rate control */ + ave_start_put32(cmd, l->rc_mode, p->rc_mode); + ave_start_put32(cmd, l->bitrate, p->bitrate); + for (unsigned int i = 0; i < 3; i++) + ave_put32(cmd, (l->init_qp + 4 * i), + cq ? p->initial_qp : AVE_START_INIT_QP_VALUE); + ave_start_put32(cmd, l->limits, p->limit_bytes ? 1 : 0); + if (l->limit_bytes && p->limit_bytes) { + ave_put64(cmd, l->limit_bytes, + ave_f64((u64)p->limit_bytes * (fw->counted ? 8 : 1), + 1)); + ave_put64(cmd, l->limit_secs, ave_f64(1, 1)); + } + ave_put64(cmd, l->f64_288, ave_f64(1, 1)); + ave_put64(cmd, l->f64_290, ave_f64(1, 2)); + ave_put64(cmd, l->f64_2c0, AVE_F64_NEG | ave_f64(1, 1)); + ave_put64(cmd, l->f64_2c8, AVE_F64_NEG | ave_f64(1, 1)); + ave_start_put32(cmd, l->min_qp, p->min_qp); + ave_start_put32(cmd, l->max_qp, p->max_qp); + ave_start_put32(cmd, l->min_qp2, p->min_qp); + ave_start_put32(cmd, l->max_qp2, p->max_qp); + ave_start_put32(cmd, l->rc_flags, cq ? 0 : hevc ? 0x736 : 0x8202); + ave_start_put32(cmd, l->rc_310, cq ? 0 : 30); + if (!hevc) + ave_start_put32(cmd, l->h264_380, 1); + + /* GOP structure: with B-frames as captured, or P-only, or intra-only. */ + ave_start_put32( + cmd, l->gop_flags, + (p->bframes ? 0x600 : 0x200) | + (fw->counted && cq ? 0x10000 | (hevc ? 0x100 : 0) : 0)); + ave_start_put32(cmd, l->p_frames, p->intra_only ? 0 : 1); + ave_start_put32(cmd, l->max_b, p->bframes ? 3 : 0); + ave_start_put32(cmd, l->ref_depth, p->bframes ? 3 : 1); + ave_start_put32(cmd, l->idr_interval, p->gop); + + /* Picture size */ + ave_start_put32(cmd, l->size_class, + p->width * p->height > 1280 * 720 ? 3 : 1); + ave_start_put32(cmd, l->coded_w, p->cw); + ave_start_put32(cmd, l->coded_h, p->ch); + ave_start_put32(cmd, l->coded_h2, p->ch); + ave_start_put32(cmd, l->codec_flags, hevc ? 0x2000002 : 2); + ave_start_put32(cmd, l->bit_depth, depth); + + if (!hevc) { + ave_start_put32(cmd, l->h264_384, + p->width * p->height <= 320 * 240 ? 0x40 : + 0x10); + ave_start_put32(cmd, l->codec_2588, p->high ? 2 : 0); + ave_start_put32(cmd, l->h264_profile, p->profile); + ave_start_put32(cmd, l->h264_level, p->level); + ave_start_put32(cmd, l->h264_max_refs, + (p->refs - 1) | + (fw->counted ? (p->cw / 16) << 16 : 0)); + if (fw->counted) { + static const u8 level_idc[] = { + 10, 9, 11, 12, 13, 20, 21, 22, 30, + 31, 32, 40, 41, 42, 50, 51, 52, + }; + + ave_put32(cmd, l->profile_idc, + p->high ? 100 : + p->profile == 2 ? 66 : + 77); + ave_put32(cmd, l->level_idc, level_idc[p->level - 1]); + put_unaligned_le16(p->refs - 1, cmd + l->max_refs_copy); + ave_put32(cmd, l->mb_size, + ((p->ch / 16 - 1) << 16) | (p->cw / 16 - 1)); + cmd[l->mb_height] = p->ch / 16; + cmd[l->mb_height + 2] = 1; + cmd[l->mb_height + 3] = 1; + cmd[l->crop_copy] = crop; + put_unaligned_le16(crop_r, cmd + l->crop_right_copy); + put_unaligned_le16(crop_b, cmd + l->crop_bottom_copy); + cmd[l->codec_byte] = !cq; + /* CABAC flag in the second, packed picture structure. */ + cmd[l->cabac_copy] = p->profile != 2; + cmd[l->transform_copy] = p->high; + } + ave_start_put32(cmd, l->h264_width_mbs, p->cw / 16 - 1); + ave_start_put32(cmd, l->h264_height_mbs, p->ch / 16 - 1); + ave_start_put32(cmd, l->h264_crop, crop); + ave_start_put32(cmd, l->h264_crop_left, 0); + ave_start_put32(cmd, l->h264_crop_right, crop_r); + ave_start_put32(cmd, l->h264_crop_top, 0); + ave_start_put32(cmd, l->h264_crop_bottom, crop_b); + ave_start_put32(cmd, l->h264_cabac, p->profile != 2); + cmd[l->h264_8x8] = p->high; + return; + } + + ave_start_put32(cmd, l->hevc_pad, + 1 | (p->cw > p->width ? BIT(16) | BIT(18) : 0) | + (p->ch > p->height ? BIT(17) | BIT(19) : 0)); + ave_start_put32(cmd, l->hevc_profile_a, p->profile); + ave_start_put32(cmd, l->hevc_profile_b, p->profile); + cmd[l->hevc_compat_a] = depth == 8; + cmd[l->hevc_compat_a + 1] = 1; + cmd[l->hevc_compat_b] = depth == 8; + cmd[l->hevc_compat_b + 1] = 1; + cmd[l->hevc_8bit_a] = depth == 8; + cmd[l->hevc_8bit_b] = depth == 8; + /* Rate-control words that only change in constant-quality mode */ + ave_start_put32(cmd, l->hevc_cq, cq); + ave_start_put32(cmd, l->hevc_cq_copy, cq); + ave_start_put32(cmd, l->hevc_cq_codec, cq); + ave_start_put32(cmd, l->hevc_cq_picture, cq ? 0x100 : 0); + ave_start_put32(cmd, l->hevc_rc_270c, 2); + ave_start_put32(cmd, l->hevc_rc_9604, cq ? 2 : 0); + ave_start_put32(cmd, l->hevc_rc_9608, cq ? 0 : 0x10000); + ave_start_put32(cmd, l->hevc_level_a, p->level); + ave_start_put32(cmd, l->hevc_level_b, p->level); + ave_start_put32(cmd, l->hevc_width, p->cw); + ave_start_put32(cmd, l->hevc_height, p->ch); + ave_start_put32(cmd, l->hevc_width_ctb, DIV_ROUND_UP(p->cw, 32)); + ave_start_put32(cmd, l->hevc_height_ctb, DIV_ROUND_UP(p->ch, 32)); + ave_start_put32(cmd, l->hevc_conf, crop); + ave_start_put32(cmd, l->hevc_conf_left, 0); + ave_start_put32(cmd, l->hevc_conf_right, crop_r); + ave_start_put32(cmd, l->hevc_conf_top, 0); + ave_start_put32(cmd, l->hevc_conf_bottom, crop_b); + ave_start_put32(cmd, l->hevc_depth_luma, depth - 8); + ave_start_put32(cmd, l->hevc_depth_chroma, depth - 8); + /* Reference structure: B-frame pyramid or low-delay P (ave-image.c) */ + ave_start_put32(cmd, l->hevc_refs, p->bframes ? 4 : 1); + ave_start_put32(cmd, l->hevc_dpb_minus1, 4); + ave_start_put32(cmd, l->hevc_reorder, p->bframes ? 2 : 0); + ave_start_put32(cmd, l->hevc_reorder_copy, p->bframes ? 2 : 0); + ave_start_put32(cmd, l->hevc_reorder_on, p->bframes); +} + +int ave_build_start(struct ave_ctx *ctx, u8 *cmd, size_t len) +{ + struct ave_dev *ave = ctx->ave; + struct ave_session *sess = ctx->sess; + const struct ave_params *p = &sess->p; + const struct ave_fw_layout *fw = ave->layout; + + if (!fw->encode_ready) + return -EOPNOTSUPP; + if (len != fw->bytes[p->codec - 1][AVE_IMAGE_START] || !p->refs || + p->refs > AVE_MAX_REFS || !p->slots || p->slots > fw->max_slots) + return -EINVAL; + ave_image_start(cmd, len, fw, p->codec, p->reorder_off, + p->codec == AVE_CODEC_HEVC && !p->bframes); + ave_write_header(ctx, cmd, AVE_OP_START, AVE_OP_START, AVE_SESSION_ARG0, + ave_session_setup_arg2(ctx)); + ave_start_surfaces(sess, cmd, fw); + ave_start_config(p, cmd, fw); + return 0; +} + +#define AVE_PROC_TIMESCALE 14400000ULL +#define AVE_PROC_SHDR_ENTRIES 256 +#define AVE_PROC_SHDR_STRIDE 0x400 +#define AVE_PROC_FORCED_TYPE 2 +#define AVE_PROC_LOW_DELAY 3 + +int ave_build_process(struct ave_ctx *ctx, u8 *cmd, size_t len, + unsigned int slot, dma_addr_t input, u32 bitrate, + bool force_key) +{ + struct ave_session *sess = ctx->sess; + const struct ave_params *p = &sess->p; + const struct ave_fw_layout *fw = ctx->ave->layout; + const struct ave_proc_layout *l = &fw->process[p->codec - 1]; + struct ave_slot *s; + dma_addr_t y = input; + u64 pts; + + if (!fw->encode_ready) + return -EOPNOTSUPP; + if (len != fw->bytes[p->codec - 1][AVE_IMAGE_PROCESS] || + slot >= p->slots || p->slots > fw->max_slots) + return -EINVAL; + if ((!l->force_key && force_key) || + (!l->bitrate && bitrate != p->bitrate)) + return -EOPNOTSUPP; + s = &sess->slot[slot]; + ave_image_process(cmd, len, fw, p->codec); + ave_write_header(ctx, cmd, AVE_OP_PROCESS, fw->process_subtype + slot, + 0, ave_session_arg2(ctx)); + /* Input picture: NV12, chroma right after p->height luma lines. */ + ave_put64(cmd, l->y, y); + ave_put32(cmd, l->y + 8, p->stride * p->height); + ave_put32(cmd, l->y + 12, p->stride); + ave_put64(cmd, l->uv, y + p->stride * p->height); + ave_put32(cmd, l->uv + 8, p->stride * p->height / 2); + ave_put32(cmd, l->uv + 12, p->stride); + ave_put32(cmd, l->in_w, p->width); + ave_put32(cmd, l->in_h, p->height); + + /* Session values repeated in every frame */ + ave_put32(cmd, l->coded_cap, p->coded_cap); + ave_put32(cmd, l->fps_int, p->fps_int); + /* The bit rate may change while streaming; it takes effect here. */ + if (l->bitrate) + ave_put32(cmd, l->bitrate, bitrate); + if (l->limits) + ave_put32(cmd, l->limits, p->limit_bytes ? 1 : 0); + if (l->limit_bytes && p->limit_bytes) { + ave_put64(cmd, l->limit_bytes, ave_f64(p->limit_bytes, 1)); + ave_put64(cmd, l->limit_secs, ave_f64(1, 1)); + } + ave_put32(cmd, l->const5, 5); + ave_put64(cmd, l->fps, ave_f64(p->fps_num, p->fps_den)); + ave_put64(cmd, l->timescale, AVE_PROC_TIMESCALE); + if (l->crop) + ave_put32(cmd, l->crop, + p->cw != p->width || p->ch != p->height); + + /* This frame */ + ave_put32(cmd, l->first_frame, sess->frames == 0); + ave_put32(cmd, l->slot, slot); + ave_put64(cmd, l->coded, s->coded.dma); + ave_put64(cmd, l->meta, s->meta.dma); + if (l->meta_size) + ave_put32(cmd, l->meta_size, s->meta.size); + if (l->extra) + ave_write_surface( + fw, cmd, + &(struct ave_surface_layout)AVE_SURFACE(l->extra, 0), 0, + s->extra.dma, SZ_16K); + if (l->frame_size) + ave_put32(cmd, l->frame_size, + p->rc_mode == 3 ? 0 : s->aux.size); + if (l->frame_buf) + ave_put64(cmd, l->frame_buf, + fw->counted && p->rc_mode == 3 ? 0 : s->aux.dma); + if (l->shdr_table) + for (unsigned int i = 0; i < AVE_PROC_SHDR_ENTRIES; i++) + ave_put64(cmd, l->shdr_table + 8 * i, + s->aux.dma + i * AVE_PROC_SHDR_STRIDE); + ave_put32(cmd, l->ordinal, sess->frames); + pts = mul_u64_u64_div_u64((u64)sess->frames * p->fps_den, + AVE_PROC_TIMESCALE, p->fps_num); + ave_put64(cmd, l->pts, pts); + if (l->force_key) + ave_put32(cmd, l->force_key, force_key); + if (l->force_type) + ave_put32(cmd, l->force_type, + force_key ? AVE_PROC_FORCED_TYPE : 0); + if (l->low_delay) + ave_put32(cmd, l->low_delay, + p->bframes ? 0 : AVE_PROC_LOW_DELAY); + return 0; +} diff --git a/drivers/media/platform/apple/ave/ave.h b/drivers/media/platform/apple/ave/ave.h new file mode 100644 index 0000000000000..9f94eb1345290 --- /dev/null +++ b/drivers/media/platform/apple/ave/ave.h @@ -0,0 +1,449 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Apple AVE video encoder + * + * Copyright The Aurora Silicon Contributors + */ +#ifndef _APPLE_AVE_H_ +#define _APPLE_AVE_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "ave-sizes.h" +#include "ave-recovery.h" + +struct debugfs_blob_wrapper; +struct dentry; +struct dev_pm_domain_list; +struct iommu_domain; +struct ave_vb2_mem; + +/* Frames the firmware can hold at once per session (Process "slots"). */ +#define AVE_MAX_SLOTS 7 +/* Sessions one firmware boot serves at once (macOS was seen running two). */ +#define AVE_MAX_SESSIONS 2 + +/* + * Coprocessor memory every firmware boot uses, all inside one allocation. + * Host commands live in three areas: one large buffer for Start, small + * buffers for the scalar commands and one buffer per Process slot, so a + * command stays untouched until the firmware acknowledges it. + */ +#define AVE_IPC_BYTES (20 * SZ_1M) +#define AVE_IPC_CHANNEL_OFF 0x4000 +#define AVE_IPC_HANDSHAKE_OFF 0x24000 +#define AVE_IPC_SHARED_OFF 0x28000 +#define AVE_IPC_SHARED_BYTES SZ_64K +#define AVE_IPC_CMD_OFF 0x40000 +#define AVE_IPC_CMD_BYTES 0x1c000 +#define AVE_IPC_SCALAR_OFF 0x5c000 +#define AVE_IPC_SCALAR_BYTES 0x100 +#define AVE_IPC_SCALAR_BUFS 16 +#define AVE_IPC_PROC_OFF 0x60000 +#define AVE_IPC_PROC_BYTES 0x10000 +#define AVE_IPC_PROC_BUFS (AVE_MAX_SESSIONS * AVE_MAX_SLOTS) +#define AVE_IPC_END (AVE_IPC_PROC_OFF + AVE_IPC_PROC_BUFS * AVE_IPC_PROC_BYTES) + +/* Command buffer ids, one bit each in ave_dev.unacked. */ +#define AVE_BUF_CMD 0 +#define AVE_BUF_SCALAR(i) (1 + (i)) +#define AVE_BUF_PROC(s) (1 + AVE_IPC_SCALAR_BUFS + (s)) +#define AVE_BUF_COUNT AVE_BUF_PROC(AVE_IPC_PROC_BUFS) + +#define AVE_MSG_MAX SZ_4K + +enum ave_codec { + AVE_CODEC_H264 = 1, + AVE_CODEC_HEVC = 2, +}; + +struct ave_buf { + void *cpu; + dma_addr_t dma; + size_t size; /* what the firmware is told */ + size_t alloc; /* what is mapped, page rounded */ + unsigned long attrs; +}; + +struct ave_fw_segment { + phys_addr_t phys; + u64 iova; + size_t size; +}; + +struct ave_ring { + char name[16]; + u32 mode; + u32 doorbell; + u32 slots; + u32 offset; + int consumer; + int producer; +}; + +enum ave_fw_state { + AVE_FW_OFF, /* coprocessor in reset */ + AVE_FW_RUNNING, + AVE_FW_HALTED, /* told to halt; restarts through the boot path */ + AVE_FW_BROKEN, + AVE_FW_RECOVERING, /* control off; awaiting a runtime power cycle */ +}; + +/* The two large session commands, and their sizes per codec */ +enum ave_image { + AVE_IMAGE_START, + AVE_IMAGE_PROCESS, + AVE_IMAGE_COUNT, +}; + +/* Logical operations; wire IDs are selected by the firmware ABI. */ +enum ave_opcode { + AVE_OP_CONFIG = 1, + AVE_OP_HALT, + AVE_OP_OPEN, + AVE_OP_CLOSE, + AVE_OP_START = 6, + AVE_OP_STOP, + AVE_OP_PROCESS, + AVE_OP_COMPLETE, + AVE_OP_PRIORITY, + AVE_OP_RESET, + AVE_OP_FLUSH, + AVE_OP_COUNT, +}; + +#include "ave-layout.h" + +/* The firmware build whose command layouts the driver generates */ +#define AVE_FW_TEXT_CRC_9003_78_0 0x69c54454 +/* The M4 Pro (T6040) build of the same 25G83 firmware. */ +#define AVE_FW_TEXT_CRC_T6040_25G83 0x68d5f9f6 + +struct ave_ctx; + +/* Register offsets for the shared 0x40 coprocessor architecture. */ +struct ave_coproc_regs { + u32 vcbar; + u32 timebase_freq; + u32 timebase; + u32 ctrl; + u32 status; + u32 boot_arg; + u32 keep_mappings; +}; + +struct ave_variant { + const struct ave_coproc_regs *regs; + u32 fw_text_size; + u32 fw_data_size; + u32 boot_device_id; + u32 device_type; + u32 lfs_ref_scale; + u32 text_crc; + u32 boot_arg_value; + u32 timebase_multiplier; + u64 vcbar_mask; + u64 vcbar_flags; + + size_t extra0_size; + /* Register sequence and boot/hello configuration have been qualified. */ + bool boot_qualified; +}; + +struct ave_dev { + struct device *dev; + const struct ave_variant *variant; + const struct ave_fw_layout *layout; + void __iomem *codec; + void __iomem *mbox; + void __iomem *coproc; + struct dev_pm_domain_list *pd_list; + struct iommu_domain *domain; + + struct ave_fw_segment text; + struct ave_fw_segment data; + u32 text_crc; + u32 data_crc; + void *data_pristine; + bool data_dirty; + + /* hw_lock protects recovery history and quarantined DMA allocations. */ + struct ave_recovery_history recovery; + unsigned int recovery_count; + struct ave_session *failed_sessions; + bool inject_wedge; + + /* debugfs: the last Start and first Process built, per codec */ + struct dentry *debugfs; + void *dump[2][AVE_IMAGE_COUNT]; + + /* Serialises every firmware and ring access. */ + struct mutex hw_lock; + enum ave_fw_state fw_state; + struct ave_buf ipc; + struct ave_buf boot_cfg; + struct ave_buf log; + struct ave_buf heap; + dma_addr_t extra0_dma; + size_t extra0_map_size; + bool extra0_mapped; + bool module_pinned; + /* A 6070 lifecycle command is serialized per logical operation. */ + struct ave_reply expected[AVE_OP_COUNT]; + u64 ipc_fw_base; + u32 channel_bytes; + struct ave_ring ring[2]; + u32 next_client; + u32 unacked; + u8 want[AVE_OP_COUNT]; + int reply_err; + unsigned int scalar_next; + struct ave_ctx *sessions[AVE_MAX_SESSIONS]; + u8 msg[AVE_MSG_MAX]; + u32 msg_len; + + struct v4l2_device v4l2_dev; + struct video_device vdev; + struct v4l2_m2m_dev *m2m_dev; + /* The video device's ioctl and queue lock. */ + struct mutex vdev_lock; + struct workqueue_struct *wq; +}; + +/* Encoding parameters, fixed for the lifetime of a session. */ +struct ave_params { + enum ave_codec codec; + u32 width, height; /* visible picture */ + u32 cw, ch; /* coded size, multiples of 16 */ + u32 stride; /* NV12 line stride, both planes */ + u32 mbs; /* 16x16 macroblocks */ + u32 fps_num, fps_den; /* frames per second = num / den */ + u32 fps_int; /* the same, rounded */ + u32 slots; + u32 refs; /* max_num_ref_frames + 1 */ + bool bframes; + bool intra_only; + bool reorder_off; /* the client asked for no B-frames */ + u32 gop; + u32 rc_mode; + u32 bitrate; + u32 limit_bytes; /* 0: no data-rate limit */ + s32 min_qp; + u32 max_qp; + u32 profile; /* H.264: 2, 4, 6; HEVC: 1 */ + u32 level; /* firmware level enum, 1-based */ + bool high; + u32 coded_cap; + u32 bit_depth; /* zero also means the existing 8-bit API */ + u32 initial_qp; /* CQ only; bit-rate modes use 26 */ +}; + +struct ave_slot { + struct ave_buf coded; + struct ave_buf meta; + struct ave_buf aux; /* H.264 per-frame buffer, HEVC slice headers */ + struct ave_buf extra; /* counted layout: per-slot 16 KiB buffer */ + struct ave_buf bounce; /* input copied to the encoder's stride */ + struct vb2_v4l2_buffer *src; + struct ave_vb2_mem *input_mem; /* independent firmware mapping reference */ + u32 status; + bool busy; /* Process sent, reply pending */ + bool done; /* reply in, output not delivered yet */ +}; + +#define AVE_MAX_REFS 5 +#define AVE_TILE_BUFS 8 + +struct ave_session { + struct ave_session *failed_next; + struct ave_params p; + struct ave_buf hdr; + struct ave_buf ref_a[AVE_MAX_REFS]; + struct ave_buf ref_b[AVE_MAX_REFS]; + struct ave_buf ref_c[AVE_MAX_REFS]; + struct ave_buf ref_d[AVE_MAX_REFS]; + struct ave_buf ref_e[AVE_MAX_REFS]; + struct ave_buf ref_f[AVE_MAX_REFS]; + struct ave_buf mb; + struct ave_buf mb_extra[2]; + struct ave_buf row_a, row_b, fixed; + struct ave_buf tile[AVE_TILE_BUFS]; + struct ave_buf tile_aux[AVE_TILE_BUFS]; + struct ave_buf hevc_extra[2]; + struct ave_slot slot[AVE_MAX_SLOTS]; + u8 done_fifo[AVE_MAX_SLOTS]; + unsigned int done_head; + unsigned int done_count; + unsigned int busy; + u32 frames; /* Process commands sent */ + u32 client; + u64 token; /* command order token, per session */ + unsigned int index; /* in ave_dev.sessions[] */ + bool fw_open; /* Open sent, Close not yet */ + bool resume; /* closed for system sleep, reopen on next frame */ + bool copy_noted; + /* CBR: bytes under budget so far, and who pads (see ave_sess_cbr_fill()) */ + s64 cbr_surplus; + bool fw_fills; + bool host_fill_noted; + bool warned; + bool checked[2]; /* Start/Process images checked for stale addresses */ + bool handed; /* Start sent: the firmware knows the buffers */ + bool nudged; /* Complete sent because input stopped */ + unsigned long progress; /* jiffies of the last frame sent or answered */ + unsigned long last_sent; /* jiffies of the last frame sent */ +}; + +struct ave_ctx { + struct v4l2_fh fh; + struct ave_dev *ave; + struct v4l2_ctrl_handler hdl; + struct v4l2_pix_format_mplane src_fmt; + struct v4l2_pix_format_mplane dst_fmt; + struct v4l2_fract timeperframe; + struct work_struct run_work; + struct delayed_work poll_work; + + u32 bitrate_mode; + u32 bitrate; + u32 bitrate_peak; + u32 gop_size; + u32 b_frames; + u32 h264_profile; + u32 h264_level; + u32 h264_min_qp; + u32 h264_max_qp; + u32 hevc_level; + u32 hevc_min_qp; + u32 hevc_max_qp; + bool force_key_frame; + + struct ave_session *sess; + atomic_t ready; /* outputs waiting for a CAPTURE buffer */ + bool draining; + bool flush_pending; + bool stopped; + /* Serialises QBUF with fatal-error publication and ready-queue draining. */ + spinlock_t queue_lock; + bool failed; + struct vb2_v4l2_buffer *drain_src; + u32 src_sequence; + u32 dst_sequence; +}; + +static inline struct ave_ctx *ave_file_to_ctx(struct file *file) +{ + return container_of(file_to_v4l2_fh(file), struct ave_ctx, fh); +} + +extern const struct ave_variant ave_t6000, ave_t8112; + +/* ave-fw.c */ +u64 ave_fw_vcbar(const struct ave_variant *variant, u64 text_iova); +int ave_fw_probe(struct ave_dev *ave); +void ave_fw_remove(struct ave_dev *ave); +int ave_fw_ensure_running(struct ave_dev *ave); +void ave_fw_halt(struct ave_dev *ave); +int ave_fw_shutdown(struct ave_dev *ave); +int ave_fw_runtime_suspend(struct ave_dev *ave); +void ave_fw_mark_broken(struct ave_dev *ave, int err, const char *what); +void ave_fw_lost(struct ave_dev *ave); +int ave_fw_check_build(struct ave_dev *ave); +int ave_fw_check_boot(struct ave_dev *ave); +void ave_fw_release_boot(struct ave_dev *ave); +int ave_fw_lifecycle_test(struct ave_dev *ave, enum ave_codec codec); +int ave_alloc(struct ave_dev *ave, struct ave_buf *b, size_t size, + unsigned long attrs); +int ave_alloc_padded(struct ave_dev *ave, struct ave_buf *b, size_t size, + size_t alloc, unsigned long attrs); +void ave_free(struct ave_dev *ave, struct ave_buf *b); + +/* ave-ipc.c */ +int ave_ipc_check_hello(const struct ave_fw_transport *t, const u32 hello[4]); +int ave_ipc_parse_channels(struct ave_dev *ave); +int ave_ipc_negotiate(struct ave_dev *ave, const u32 hello[4]); +void *ave_ipc_cmd(struct ave_dev *ave, unsigned int buf, size_t len); +int ave_ipc_scalar_buf(struct ave_dev *ave); +int ave_ipc_wait_ack(struct ave_dev *ave, unsigned int buf); +int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, + u64 timeout_us, bool reply); +int ave_ipc_pump(struct ave_dev *ave, unsigned int wait_ms, + bool (*cond)(struct ave_dev *ave, void *arg), void *arg); +int ave_ipc_settle(struct ave_dev *ave, unsigned int wait_ms); + +/* ave-writer.c: pure command builders, also exercised by KUnit. */ +u32 ave_session_arg2(struct ave_ctx *ctx); +u32 ave_session_setup_arg2(struct ave_ctx *ctx); +void ave_write_header(struct ave_ctx *ctx, u8 *cmd, enum ave_opcode op, + u32 echo, u32 arg0, u32 arg2); +void ave_write_global(struct ave_dev *ave, u8 *cmd, enum ave_opcode op); +void ave_stamp_command(const struct ave_fw_abi *abi, u8 *cmd, u64 timeout, + u64 created); +void ave_write_surface(const struct ave_fw_layout *fw, u8 *cmd, + const struct ave_surface_layout *l, unsigned int index, + dma_addr_t addr, u32 size); +int ave_build_start(struct ave_ctx *ctx, u8 *cmd, size_t len); +int ave_build_process(struct ave_ctx *ctx, u8 *cmd, size_t len, + unsigned int slot, dma_addr_t input, u32 bitrate, + bool force_key); + +/* ave-cmd.c */ +int ave_cmd_config(struct ave_dev *ave); +int ave_cmd_halt(struct ave_dev *ave); +int ave_cmd_open(struct ave_ctx *ctx); +int ave_cmd_start(struct ave_ctx *ctx); +int ave_cmd_process(struct ave_ctx *ctx, unsigned int slot, + struct vb2_v4l2_buffer *src, dma_addr_t input, u32 bitrate, + bool force_key); +int ave_cmd_complete(struct ave_ctx *ctx, unsigned int count); +int ave_cmd_close(struct ave_ctx *ctx); + +/* ave-vb2.c */ +extern const struct vb2_mem_ops ave_vb2_memops; +struct ave_vb2_mem *ave_vb2_get(struct vb2_buffer *vb); +void ave_vb2_put(void *buf_priv); + +/* ave-sess.c */ +struct ave_session *ave_sess_create(struct ave_ctx *ctx); +void ave_sess_destroy(struct ave_ctx *ctx); +void ave_sess_reap(struct ave_dev *ave); +int ave_sess_free_slot(struct ave_session *sess); +void ave_sess_reply(struct ave_dev *ave, const struct ave_reply *reply); +int ave_sess_input(struct ave_ctx *ctx, unsigned int slot, + struct vb2_v4l2_buffer *src, dma_addr_t *input); +int ave_sess_output(struct ave_ctx *ctx, unsigned int slot, + struct vb2_v4l2_buffer *dst, bool *keyframe); +void ave_sess_prepare_slot(struct ave_session *sess, unsigned int slot); +bool ave_sess_stalled(struct ave_session *sess); +bool ave_sess_starved(struct ave_session *sess); + +/* ave-image.c */ +void ave_image_start(u8 *cmd, size_t len, const struct ave_fw_layout *fw, + enum ave_codec codec, bool reorder_off, + bool hevc_low_delay); +void ave_image_process(u8 *cmd, size_t len, const struct ave_fw_layout *fw, + enum ave_codec codec); + +/* ave-drv.c */ +void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, + enum ave_image kind, const void *cmd, size_t len); + +/* ave-v4l2.c */ +void ave_v4l2_suspend(struct ave_dev *ave); +int ave_v4l2_register(struct ave_dev *ave); +void ave_v4l2_unregister(struct ave_dev *ave); +enum ave_codec ave_ctx_codec(struct ave_ctx *ctx); + +#endif diff --git a/drivers/media/platform/apple/ave/tests/README b/drivers/media/platform/apple/ave/tests/README new file mode 100644 index 0000000000000..ea59e9724e991 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/README @@ -0,0 +1,146 @@ +AVE host tests +============== + +Run python3 run.py with a C compiler providing ASan and UBSan (CC may name +its executable). No encoder, firmware or captured command image is needed. + +run.py reads the real AVE structs and compiles the unmodified bodies of +ave-image.c, ave-layout.c, ave-abi.c, ave-writer.c and ave-cmd.c, plus the real +buffer-allocation, +free, reply and firmware-build-gate functions. Only includes and unused +kernel-facing declarations are omitted. shim.h supplies host primitives; +boundary.h replaces IPC, DMA and logging. This does not validate the kernel +API integration, DART mappings, boot sequence or output extraction. + +--compare-baseline PATH builds an older AVE source directory and compares +all bytes of 288 synthetic sessions (1,440 Start/Process commands, including +headers, addresses and zero padding). It covers both codecs, geometry, +profiles, B-frame settings, rates, CBR/VBR and forced-key-frame fields. + +--build PATH retains the executable for offline capture QA. Its --case +interface emits synthetic commands; capture inputs and generated artifacts +must stay outside the public source tree. Compare only with the matching +firmware layout, mask enumerated pointer fields and host-only headers, and +report inherited differences explicitly. The 26A428 description is not +reachable through the runtime CRC lookup. + +The host shim intentionally uses stdint types, Linux's private ENOTSUPP +value and side-effect-free calls to kernel-like arithmetic macros. Those +produce checkpatch PREFER_KERNEL_TYPES, ENOTSUPP and MACRO_ARG_REUSE notices. +MAINTAINERS already covers the entire AVE directory (FILE_PATH_CHANGES). + +The ABI refactor also has in-kernel arm64 KUnit tests selected by +VIDEO_APPLE_AVE_CMD_KUNIT_TEST. The host runner understands both the older +single-builder sources and the per-ABI writer split for baseline comparison. + +Recovery and PM checks +---------------------- + +Run python3 run-recovery.py for ASan/UBSan checks of the production recovery, +firmware shutdown, suspend/resume, failed-session cleanup, queue and IPC pump +callbacks. It covers two owners (one idle), both ready queues plus an owned +OUTPUT buffer, racing QBUF, PM reference balance, DMA quarantine/reclamation, +retained power, failed control stop, the recovery limit, pristine DATA restore +failure, suspend refusal/rollback and a busy IPC stream reaching its deadline. +MMIO, DMA allocation and PM are simulated: this cannot establish hardware reset +values or that a VENC domain actually powered down. + +The recovery runner also uses the production OUTPUT memory operations and +Process submission callback. Both MMAP and imported dma-buf mappings survive +immediate vb2 release after ERROR, including loss of the importing fd and an +uncertain command submission. The fake DMA boundary rejects any release of a +command-referenced mapping before the control-off marker AND simulated shared +DART/VENC power loss. A failed stop and CTRL=0 with retained BOOT_ARG must not +reclaim it. Allocator failures and ordinary import/map/release are covered too. + +The apple-ave-recovery KUnit suite tests the actual sliding-window helper at +expiry boundaries, with staggered attempts and across jiffies wrap. Run with +../.kunitconfig using tools/testing/kunit/kunit.py. The existing +tools/asahi/ave-pm-tests.py also checks the #651 sleep rollback contract. + +Hardware procedure (requires a qualified firmware/SoC and an approved lane) +------------------------------------------------------------------------- + +1. Record the kernel revision, firmware CRCs and boot ID. Mount debugfs and + enable the normal PM/genpd tracing available in the test kernel. Confirm a + fresh AVC and HEVC encode can each complete and decode correctly. +2. Start a long encode with several OUTPUT and CAPTURE buffers queued. While + it runs, as root write 1 to /sys/kernel/debug/apple-ave/inject_wedge. + This software fault marks the current firmware broken and suppresses the + recovery Halt attempt, modelling a command timeout without malformed DMA + commands. It does not reproduce a real DART overrun or a firmware deadlock. +3. Check that the client promptly sees POLLERR/EIO and every buffer still owned + by the driver completes with ERROR, including OUTPUT slots already handed + to firmware. A V4L2 queue exerciser should count completions; ffmpeg alone + only checks the client-visible exit. QBUF after failure must fail or return + the buffer with ERROR. STREAMOFF both queues before restarting that context. +4. Immediately start a new encode without rebooting. Allow the existing bounded + PM retry loop for DARTs sharing VENC power. Require the recovery stop log, + a real domain power-off/on in PM tracing, the power-cycle-confirmed log, + fresh firmware Config and valid encoded output. The driver re-arms BOOT_ARG + with control off and requires it to read zero after resume; qualify that + reset witness on each target. A retained nonzero witness must cause EAGAIN, + with no DATA restore, allocation reuse or new firmware start. +5. Repeat with two concurrent clients, including an idle open session with no + outstanding frames. Both must fail, all PM references must drop, and a new + client must encode. Check there are no stale replies, DART faults or DMA + allocation growth over repeated recoveries. +6. Inject three faults inside ten minutes. A fourth must report the recovery + limit and return queue errors without starting another firmware boot. After + the oldest attempt expires, a new session must recover successfully. +7. In a separate approved sleep test, check normal encode/sleep/resume and a + halt-failure suspend refusal. A failed suspend must restart the m2m scheduler + (#651); system sleep must never treat the forced runtime stop as a successful + firmware Halt. Preserve logs from failed sleep attempts as well as retries. + +Run this procedure on each target requiring qualification, including both +J616s and Neo. Repeat the recovery with a genuine firmware timeout/DART fault +when a safe reproducer is available. Keep captures and machine records outside +the public source tree. Rollback is a normal revert of the recovery commits; +use the lane's existing boot fallback when testing candidate kernels. + +LEAD-RUN: Neo failed-session mapping regression +---------------------------------------------- + +Run in the approved Neo hardware lane, using the hardware procedure above. +This is a re-test plan, not a claim of hardware qualification. Preserve the +normal candidate-kernel boot fallback and the known-good module/kernel for +rollback. Record the full source commit, kernel config, module hash/vermagic, +firmware CRCs and boot ID with the results. Build and install the matching +candidate through the lane's normal procedure; do not load a module built for +a different kernel. Keep machine journals and captures outside public source. + +1. Save a complete kernel log before the test and collect a continuous kernel + journal with monotonic timestamps throughout each scenario. Establish fresh + AVC and HEVC encode/decode success. Use the same queue exerciser as the + original hardware procedure to record injection time, first POLLERR/EIO and + each buffer's completion count/state. Include MMAP OUTPUT and imported + dma-buf OUTPUT runs, with immediate DQBUF, STREAMOFF, REQBUFS(0), close and + release of the importing fd after the error. This exercises lifetime beyond + the V4L2 context. Do not require the client to wait for the stop or reset. +2. Re-run hardware steps 2-4: inject during a long encode, require prompt queue + errors (compare with the prior 5-9 ms POLLERR measurements), and exactly one + ERROR completion per buffer still owned by the driver. Require the existing + recovery stop and power-cycle-confirmed messages and a valid new encode + without rebooting. Inspect the entire interval from injection through the + subsequent encode, not just the recovery's last messages. +3. Re-run step 5 with two clients, including the idle owner. Release both + contexts promptly after ERROR. Require both clients to fail promptly, + balanced PM ownership, a confirmed recovery and a valid fresh encode. +4. Re-run step 6 with a fresh recovery budget: three injections in ten minutes, + restarting and verifying an encode after each. The next failure must retain + queue errors and report the three-attempt limit without a new boot. After + the oldest attempt expires, require successful recovery and encoding. Use + the lane's normal fresh-boot procedure or allow the earlier attempts to + expire between scenarios; do not reset the counter by reloading the driver. +5. PASS requires zero apple-dart translation faults after EVERY injection, + including the limited attempt and its expiry/retry, with no new suppressed + fault messages. Preserve complete logs and per-injection windows, completion + counts, error latency and recovery results even on failure. Any new DART + fault, missing/duplicate completion, lost prompt error or unconfirmed reset + is a failed re-test. The existing strict sleep check in step 7 remains + required in its separate approved sleep lane. + +On failure, preserve the candidate logs and restore the known-good kernel or +module through the lane's rollback procedure. Keep the draft unmerged pending +this LEAD-RUN evidence. Software injection does not qualify a real DMA overrun. diff --git a/drivers/media/platform/apple/ave/tests/ave-9003-reference.h b/drivers/media/platform/apple/ave/tests/ave-9003-reference.h new file mode 100644 index 0000000000000..8402a4a5435f8 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/ave-9003-reference.h @@ -0,0 +1,828 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Frozen 9003 builder from 23e431a98962, for byte-regression tests only. + * Copyright The Aurora Silicon Contributors + */ +struct ave_ref_start_layout { + u16 ref_a, ref_b, ref_c, ref_c_alias, ref_d; + u16 ref_e, ref_f, ref_stride, ref_c_stride; + u16 mb, coded, meta, row_a, row_b, fixed, hevc_extra; + u16 tiles[8]; + u16 refs_count, work_count, lfs_count, slots_count, meta_count; + u16 mb_count, row_a_count, row_b_count, fixed_count, tiles_count; + u16 codec; + u16 flag_8bit; + u16 const_214; + u16 const_218; + u16 fps; + u16 rc_mode; + u16 bitrate; + u16 init_qp; + u16 limits; + u16 limit_bytes; + u16 limit_secs; + u16 f64_288; + u16 f64_290; + u16 min_qp; + u16 max_qp; + u16 gop_flags; + u16 p_frames; + u16 max_b; + u16 ref_depth; + u16 idr_interval; + u16 f64_2c0; + u16 f64_2c8; + u16 rc_flags; + u16 min_qp2; + u16 max_qp2; + u16 rc_310; + u16 size_class; + u16 coded_w; + u16 coded_h; + u16 codec_flags; + u16 h264_380; + u16 h264_384; + u16 codec_2588; + u16 bit_depth; + u16 coded_h2; + u16 h264_profile; + u16 h264_level; + u16 h264_max_refs; + u16 h264_width_mbs; + u16 h264_height_mbs; + u16 h264_crop; + u16 h264_crop_left; + u16 h264_crop_right; + u16 h264_crop_top; + u16 h264_crop_bottom; + u16 h264_cabac; + u16 h264_8x8; + u16 hevc_pad; + u16 hevc_rc_270c; + u16 hevc_rc_9604; + u16 hevc_rc_9608; + u16 hevc_profile_a; + u16 hevc_compat_a; + u16 hevc_8bit_a; + u16 hevc_level_a; + u16 hevc_profile_b; + u16 hevc_compat_b; + u16 hevc_8bit_b; + u16 hevc_level_b; + u16 hevc_width; + u16 hevc_height; + u16 hevc_width_ctb; + u16 hevc_height_ctb; + u16 hevc_conf; + u16 hevc_conf_left; + u16 hevc_conf_right; + u16 hevc_conf_top; + u16 hevc_conf_bottom; + u16 hevc_depth_luma; + u16 hevc_depth_chroma; + u16 hevc_refs; + u16 hevc_reorder_copy; + u16 hevc_dpb_minus1; + u16 hevc_reorder; + u16 hevc_reorder_on; + u16 geometry_mode, geometry_depth; + u16 profile_idc, level_idc, max_refs_copy, mb_size, mb_height; + u16 crop_copy, crop_right_copy, crop_bottom_copy; + u16 codec_byte, cabac_copy, transform_copy, hevc_cq, hevc_cq_copy; + u16 hevc_cq_codec, hevc_cq_picture; +}; + +struct ave_ref_proc_layout { + u16 first_frame; + u16 y, uv; /* {u64 address, u32 size, u32 stride} */ + u16 in_w, in_h; + u16 coded_cap; + u16 fps_int; + u16 bitrate; + u16 limits, limit_bytes, limit_secs; + u16 const5; + u16 fps; /* f64 */ + u16 timescale; /* u64 */ + u16 slot; + u16 coded, meta; /* u64 addresses */ + u16 ordinal; + u16 pts; /* u64 */ + u16 force_key; /* 1 on a forced key frame */ + u16 force_type; /* 2 on a forced key frame */ + u16 frame_buf; /* H.264 only */ + u16 shdr_table; /* HEVC only */ + u16 crop; /* HEVC only */ + u16 meta_size, extra, frame_size; + u16 low_delay; /* HEVC only: 3 with reordering off */ +}; + +struct ave_ref_fw_layout { + const char *name; + size_t bytes[2][AVE_IMAGE_COUNT]; + struct ave_ref_start_layout start[2]; + struct ave_ref_proc_layout process[2]; + u32 hdr_bytes, meta_bytes; + u32 process_timeout_us; + u8 process_subtype, max_slots, mcc_dsid; + bool counted; +}; + +static const struct ave_ref_fw_layout ave_reference_layout = { + .name = "9003.78.0 / 25G83", + .bytes = { { 0x3180, 0x63d8 }, { 0x13f28, 0xb1c0 } }, + .hdr_bytes = 0x12c000, + .meta_bytes = 0xc000, + .process_timeout_us = 94400000, + .process_subtype = 0x15, + .max_slots = 5, + .mcc_dsid = 0x12, + .start = { + [0] = { + .ref_a = 0x390, + .ref_b = 0x3a0, + .ref_c = 0x7d0, + .ref_c_alias = 0x7e0, + .ref_d = 0x1130, + .ref_stride = 0x20, + .ref_c_stride = 0x20, + .mb = 0xc10, + .coded = 0xd50, + .meta = 0xf50, + .row_a = 0x1350, + .row_b = 0x1390, + .fixed = 0x13d0, + .hevc_extra = 0x2520, + .tiles = { 0x1410, 0x1430, 0x1490, 0x14b0, 0x1510, + 0x1530, 0x1590, 0x15b0 }, + .codec = 0xc0, + .flag_8bit = 0x208, + .const_214 = 0x214, + .const_218 = 0x218, + .fps = 0x220, + .rc_mode = 0x234, + .bitrate = 0x238, + .init_qp = 0x240, + .limits = 0x250, + .limit_bytes = 0x258, + .limit_secs = 0x260, + .f64_288 = 0x288, + .f64_290 = 0x290, + .min_qp = 0x298, + .max_qp = 0x29c, + .gop_flags = 0x2a0, + .p_frames = 0x2ac, + .max_b = 0x2b0, + .ref_depth = 0x2b4, + .idr_interval = 0x2b8, + .f64_2c0 = 0x2c0, + .f64_2c8 = 0x2c8, + .rc_flags = 0x300, + .min_qp2 = 0x308, + .max_qp2 = 0x30c, + .rc_310 = 0x310, + .size_class = 0x320, + .coded_w = 0x368, + .coded_h = 0x36c, + .codec_flags = 0x370, + .h264_380 = 0x380, + .h264_384 = 0x384, + .codec_2588 = 0x2588, + .bit_depth = 0x2598, + .coded_h2 = 0x25dc, + .h264_profile = 0x291c, + .h264_level = 0x2938, + .h264_max_refs = 0x2d44, + .h264_width_mbs = 0x2d4c, + .h264_height_mbs = 0x2d50, + .h264_crop = 0x2db4, + .h264_crop_left = 0x2db8, + .h264_crop_right = 0x2dbc, + .h264_crop_top = 0x2dc0, + .h264_crop_bottom = 0x2dc4, + .h264_cabac = 0x2fd8, + .h264_8x8 = 0x3023, + .hevc_pad = 0x26e0, + .hevc_rc_270c = 0x270c, + .hevc_rc_9604 = 0x9604, + .hevc_rc_9608 = 0x9608, + .hevc_profile_a = 0x2938, + .hevc_compat_a = 0x293d, + .hevc_8bit_a = 0x2962, + .hevc_level_a = 0x296c, + .hevc_profile_b = 0x5904, + .hevc_compat_b = 0x5909, + .hevc_8bit_b = 0x592e, + .hevc_level_b = 0x5938, + .hevc_width = 0x5ab4, + .hevc_height = 0x5ab8, + .hevc_width_ctb = 0x5abc, + .hevc_height_ctb = 0x5ac0, + .hevc_conf = 0x5ac4, + .hevc_conf_left = 0x5ac8, + .hevc_conf_right = 0x5acc, + .hevc_conf_top = 0x5ad0, + .hevc_conf_bottom = 0x5ad4, + .hevc_depth_luma = 0x5ad8, + .hevc_depth_chroma = 0x5adc, + .hevc_refs = 0x259c, + .hevc_reorder_copy = 0x2e54, + .hevc_dpb_minus1 = 0x5b00, + .hevc_reorder = 0x5b04, + .hevc_reorder_on = 0x95fc, + }, + [1] = { + .ref_a = 0x390, + .ref_b = 0x3a0, + .ref_c = 0x7d0, + .ref_c_alias = 0x7e0, + .ref_d = 0x1130, + .ref_stride = 0x20, + .ref_c_stride = 0x20, + .mb = 0xc10, + .coded = 0xd50, + .meta = 0xf50, + .row_a = 0x1350, + .row_b = 0x1390, + .fixed = 0x13d0, + .hevc_extra = 0x2520, + .tiles = { 0x1410, 0x1430, 0x1490, 0x14b0, 0x1510, + 0x1530, 0x1590, 0x15b0 }, + .codec = 0xc0, + .flag_8bit = 0x208, + .const_214 = 0x214, + .const_218 = 0x218, + .fps = 0x220, + .rc_mode = 0x234, + .bitrate = 0x238, + .init_qp = 0x240, + .limits = 0x250, + .limit_bytes = 0x258, + .limit_secs = 0x260, + .f64_288 = 0x288, + .f64_290 = 0x290, + .min_qp = 0x298, + .max_qp = 0x29c, + .gop_flags = 0x2a0, + .p_frames = 0x2ac, + .max_b = 0x2b0, + .ref_depth = 0x2b4, + .idr_interval = 0x2b8, + .f64_2c0 = 0x2c0, + .f64_2c8 = 0x2c8, + .rc_flags = 0x300, + .min_qp2 = 0x308, + .max_qp2 = 0x30c, + .rc_310 = 0x310, + .size_class = 0x320, + .coded_w = 0x368, + .coded_h = 0x36c, + .codec_flags = 0x370, + .h264_380 = 0x380, + .h264_384 = 0x384, + .codec_2588 = 0x2588, + .bit_depth = 0x2598, + .coded_h2 = 0x25dc, + .h264_profile = 0x291c, + .h264_level = 0x2938, + .h264_max_refs = 0x2d44, + .h264_width_mbs = 0x2d4c, + .h264_height_mbs = 0x2d50, + .h264_crop = 0x2db4, + .h264_crop_left = 0x2db8, + .h264_crop_right = 0x2dbc, + .h264_crop_top = 0x2dc0, + .h264_crop_bottom = 0x2dc4, + .h264_cabac = 0x2fd8, + .h264_8x8 = 0x3023, + .hevc_pad = 0x26e0, + .hevc_rc_270c = 0x270c, + .hevc_rc_9604 = 0x9604, + .hevc_rc_9608 = 0x9608, + .hevc_profile_a = 0x2938, + .hevc_compat_a = 0x293d, + .hevc_8bit_a = 0x2962, + .hevc_level_a = 0x296c, + .hevc_profile_b = 0x5904, + .hevc_compat_b = 0x5909, + .hevc_8bit_b = 0x592e, + .hevc_level_b = 0x5938, + .hevc_width = 0x5ab4, + .hevc_height = 0x5ab8, + .hevc_width_ctb = 0x5abc, + .hevc_height_ctb = 0x5ac0, + .hevc_conf = 0x5ac4, + .hevc_conf_left = 0x5ac8, + .hevc_conf_right = 0x5acc, + .hevc_conf_top = 0x5ad0, + .hevc_conf_bottom = 0x5ad4, + .hevc_depth_luma = 0x5ad8, + .hevc_depth_chroma = 0x5adc, + .hevc_refs = 0x259c, + .hevc_reorder_copy = 0x2e54, + .hevc_dpb_minus1 = 0x5b00, + .hevc_reorder = 0x5b04, + .hevc_reorder_on = 0x95fc, + }, + }, + .process = { + [0] = { + .first_frame = 0x12c0, + .y = 0x5830, + .uv = 0x5850, + .in_w = 0x58e8, + .in_h = 0x58ec, + .coded_cap = 0x61c8, + .fps_int = 0x1610, + .bitrate = 0x1620, + .limits = 0x1628, + .limit_bytes = 0x1630, + .limit_secs = 0x1638, + .const5 = 0x6238, + .fps = 0x6268, + .timescale = 0x6278, + .slot = 0x61b4, + .coded = 0x61b8, + .meta = 0x61c0, + .ordinal = 0x6228, + .pts = 0x6270, + .force_key = 0x1658, + .force_type = 0x29f8, + .frame_buf = 0x58c8, + }, + [1] = { + .first_frame = 0x5d90, + .y = 0xa300, + .uv = 0xa320, + .in_w = 0xa3b8, + .in_h = 0xa3bc, + .coded_cap = 0xac98, + .fps_int = 0x60e0, + .bitrate = 0x60f0, + .limits = 0x60f8, + .limit_bytes = 0x6100, + .limit_secs = 0x6108, + .const5 = 0xad08, + .fps = 0xad38, + .timescale = 0xad48, + .slot = 0xac84, + .coded = 0xac88, + .meta = 0xac90, + .ordinal = 0xacf8, + .pts = 0xad40, + .force_key = 0x6128, + .force_type = 0x74c8, + .shdr_table = 0xd88, + .crop = 0x74dc, + .low_delay = 0xcf8, + }, + }, +}; + +static u64 ave_f64(u64 num, u64 den) +{ + u64 mant = 0, rem; + int exp = 0; + + if (!num || !den) + return 0; + while (num >= den << 1) { + den <<= 1; + exp++; + } + while (num < den) { + num <<= 1; + exp--; + } + /* 1 <= num/den < 2: take 53 significant bits by long division. */ + rem = num; + for (unsigned int i = 0; i < 53; i++) { + mant <<= 1; + if (rem >= den) { + mant |= 1; + rem -= den; + } + rem <<= 1; + } + return (u64)(exp + 1023) << 52 | (mant & GENMASK_ULL(51, 0)); +} + +#define AVE_F64_NEG BIT_ULL(63) + +static void ave_put32(u8 *cmd, u32 off, u32 val) +{ + put_unaligned_le32(val, cmd + off); +} + +static void ave_put64(u8 *cmd, u32 off, u64 val) +{ + put_unaligned_le64(val, cmd + off); +} + +/* A surface descriptor: {u64 DART address, u32 size, u32 0}. */ +static void ave_put_surface(u8 *cmd, u32 off, dma_addr_t addr, u32 size) +{ + ave_put64(cmd, off, addr); + ave_put32(cmd, off + 8, size); + ave_put32(cmd, off + 12, 0); +} + +static void ave_put_buf(u8 *cmd, u32 off, const struct ave_buf *b, u32 size) +{ + ave_put_surface(cmd, off, b->dma, size); +} + +/* Is another session open in the firmware besides @ctx's? */ +static bool ave_cmd_others_open(struct ave_ctx *ctx) +{ + struct ave_dev *ave = ctx->ave; + + for (unsigned int i = 0; i < AVE_MAX_SESSIONS; i++) + if (ave->sessions[i] && ave->sessions[i] != ctx && + ave->sessions[i]->sess->fw_open) + return true; + return false; +} + +/* + * Header word +0x2c depends on the set of open sessions: 0x1e0 on Open and + * Start (0x140 when another session is open), the codec value on every + * other command of a lone session, and 0 while another one is open. + */ +static u32 ave_ref_session_arg2(struct ave_ctx *ctx) +{ + if (ctx->ave->layout->counted && ctx->sess->p.codec == AVE_CODEC_H264) + return ctx->sess->p.bframes ? AVE_SESSION_ARG2_H264 : 0x140; + if (ave_cmd_others_open(ctx)) + return 0; + return ctx->sess->p.codec == AVE_CODEC_HEVC ? AVE_SESSION_ARG2_HEVC : + AVE_SESSION_ARG2_H264; +} + +static u32 ave_ref_session_setup_arg2(struct ave_ctx *ctx) +{ + return ave_cmd_others_open(ctx) ? AVE_SESSION_ARG2_SETUP_SHARED : + AVE_SESSION_ARG2_SETUP; +} + +static void ave_cmd_header(struct ave_ctx *ctx, u8 *cmd, enum ave_opcode op, + u32 echo, u32 arg0, u32 arg2) +{ + ctx->sess->token += AVE_TOKEN_STEP; + ave_put32(cmd, AVE_CMD_OPCODE, op); + ave_put64(cmd, AVE_CMD_TOKEN, ctx->sess->token); + ave_put64(cmd, AVE_CMD_CLIENT, ctx->sess->client); + ave_put32(cmd, AVE_CMD_UNIT, AVE_SESSION_UNIT); + ave_put32(cmd, AVE_CMD_CODEC, ctx->sess->p.codec); + ave_put32(cmd, AVE_CMD_OPCODE_ECHO, echo); + ave_put32(cmd, AVE_CMD_ARG0, arg0); + ave_put32(cmd, AVE_CMD_ARG1, AVE_SESSION_ARG1); + ave_put32(cmd, AVE_CMD_ARG2, arg2); +} + +#define AVE_START_ARRAY_LEN 30 +#define AVE_START_INIT_QP_VALUE 26 + +static void ave_start_surfaces(struct ave_session *sess, u8 *cmd, + const struct ave_ref_fw_layout *fw) +{ + const struct ave_params *p = &sess->p; + bool hevc = p->codec == AVE_CODEC_HEVC; + unsigned int tiles = hevc ? 4 : AVE_TILE_BUFS; + const struct ave_ref_start_layout *l = &fw->start[p->codec - 1]; + + /* Clear every descriptor the image may carry, then fill ours. */ + for (unsigned int i = 0; i < AVE_MAX_REFS; i++) { + ave_put_surface(cmd, l->ref_a + l->ref_stride * i, 0, 0); + ave_put_surface(cmd, l->ref_b + l->ref_stride * i, 0, 0); + ave_put_surface(cmd, l->ref_c + l->ref_c_stride * i, 0, 0); + ave_put_surface(cmd, l->ref_c_alias + l->ref_c_stride * i, 0, + 0); + ave_put_surface(cmd, l->ref_d + 0x10 * i, 0, 0); + } + for (unsigned int s = 0; s < AVE_START_ARRAY_LEN; s++) { + ave_put_surface(cmd, l->coded + 0x10 * s, 0, 0); + ave_put_surface(cmd, l->meta + 0x10 * s, 0, 0); + } + + ave_put_buf(cmd, 0x48, &sess->hdr, fw->hdr_bytes); + ave_put_surface(cmd, 0x58, 0, 0); + for (unsigned int i = 0; i < p->refs; i++) { + ave_put_buf(cmd, l->ref_a + l->ref_stride * i, &sess->ref_a[i], + sess->ref_a[i].size); + ave_put_buf(cmd, l->ref_b + l->ref_stride * i, &sess->ref_b[i], + sess->ref_b[i].size); + ave_put_buf(cmd, l->ref_c + l->ref_c_stride * i, + &sess->ref_c[i], sess->ref_c[i].size); + ave_put_surface(cmd, l->ref_c_alias + l->ref_c_stride * i, + sess->ref_c[i].dma, 0); + ave_put_buf(cmd, l->ref_d + 0x10 * i, &sess->ref_d[i], + sess->ref_d[i].size); + } + if (fw->counted) { + ave_put32(cmd, l->refs_count, p->refs); + ave_put32(cmd, l->work_count, p->refs); + ave_put32(cmd, l->lfs_count, p->refs); + ave_put32(cmd, l->slots_count, p->slots); + ave_put32(cmd, l->meta_count, p->slots); + ave_put32(cmd, l->mb_count, p->rc_mode == 3 ? 3 : 1); + if (p->rc_mode == 3) + for (unsigned int i = 0; i < 2; i++) + ave_put_buf(cmd, l->mb + 0x10 * (i + 1), + &sess->mb_extra[i], + sess->mb_extra[i].size); + ave_put32(cmd, l->tiles_count, tiles); + for (unsigned int i = 0; i < p->refs; i++) { + ave_put_buf(cmd, l->ref_e + l->ref_stride * i, + &sess->ref_e[i], sess->ref_e[i].size); + ave_put_buf(cmd, l->ref_f + l->ref_stride * i, + &sess->ref_f[i], sess->ref_f[i].size); + } + } + ave_put_buf(cmd, l->mb, &sess->mb, sess->mb.size); + for (unsigned int s = 0; s < p->slots; s++) { + ave_put_buf(cmd, l->coded + 0x10 * s, &sess->slot[s].coded, + p->coded_cap); + ave_put_buf(cmd, l->meta + 0x10 * s, &sess->slot[s].meta, + sess->slot[s].meta.size); + } + ave_put_buf(cmd, l->row_a, &sess->row_a, sess->row_a.size); + ave_put_buf(cmd, l->row_b, &sess->row_b, sess->row_b.size); + ave_put_buf(cmd, l->fixed, &sess->fixed, SZ_16K); + for (unsigned int i = 0; i < AVE_TILE_BUFS; i++) { + u32 off = l->tiles[i]; + + if (i < tiles) { + ave_put_buf(cmd, off, &sess->tile[i], + sess->tile[i].size); + ave_put_buf(cmd, off + 0x10, &sess->tile_aux[i], + sess->tile_aux[i].size); + } else { + ave_put_surface(cmd, off, 0, 0); + ave_put_surface(cmd, off + 0x10, 0, 0); + } + } + if (hevc) + for (unsigned int i = 0; i < 2; i++) + ave_put_buf(cmd, l->hevc_extra + 0x10 * i, + &sess->hevc_extra[i], + sess->hevc_extra[i].size); +} + +static void ave_start_put32(u8 *cmd, u32 off, u32 val) +{ + if (off) + ave_put32(cmd, off, val); +} + +static void ave_start_config(const struct ave_params *p, u8 *cmd, + const struct ave_ref_fw_layout *fw) +{ + bool hevc = p->codec == AVE_CODEC_HEVC; + bool cq = p->rc_mode == 3; + u32 depth = p->bit_depth ?: 8; + u32 crop_r = (p->cw - p->width) / 2, crop_b = (p->ch - p->height) / 2; + bool crop = crop_r || crop_b; + const struct ave_ref_start_layout *l = &fw->start[p->codec - 1]; + + ave_start_put32(cmd, l->codec, p->codec); + ave_start_put32( + cmd, l->flag_8bit, + fw->counted ? + 0x10000 | (hevc ? (depth == 8 ? 0x41 : 0x40) : 0) | + (cq ? 0x10 : 0) : + hevc ? 1 : + 0); + ave_start_put32(cmd, l->geometry_mode, + (hevc ? 0x40 : 0) | (cq ? 0 : 2)); + ave_start_put32(cmd, l->geometry_depth, depth); + ave_start_put32(cmd, l->const_214, 0x39d); + ave_start_put32(cmd, l->const_218, 0x18); + ave_start_put32(cmd, l->fps, p->fps_int); + + /* Rate control */ + ave_start_put32(cmd, l->rc_mode, p->rc_mode); + ave_start_put32(cmd, l->bitrate, p->bitrate); + for (unsigned int i = 0; i < 3; i++) + ave_put32(cmd, (l->init_qp + 4 * i), + cq ? p->initial_qp : AVE_START_INIT_QP_VALUE); + ave_start_put32(cmd, l->limits, p->limit_bytes ? 1 : 0); + if (l->limit_bytes && p->limit_bytes) { + ave_put64(cmd, l->limit_bytes, + ave_f64((u64)p->limit_bytes * (fw->counted ? 8 : 1), + 1)); + ave_put64(cmd, l->limit_secs, ave_f64(1, 1)); + } + ave_put64(cmd, l->f64_288, ave_f64(1, 1)); + ave_put64(cmd, l->f64_290, ave_f64(1, 2)); + ave_put64(cmd, l->f64_2c0, AVE_F64_NEG | ave_f64(1, 1)); + ave_put64(cmd, l->f64_2c8, AVE_F64_NEG | ave_f64(1, 1)); + ave_start_put32(cmd, l->min_qp, p->min_qp); + ave_start_put32(cmd, l->max_qp, p->max_qp); + ave_start_put32(cmd, l->min_qp2, p->min_qp); + ave_start_put32(cmd, l->max_qp2, p->max_qp); + ave_start_put32(cmd, l->rc_flags, cq ? 0 : hevc ? 0x736 : 0x8202); + ave_start_put32(cmd, l->rc_310, cq ? 0 : 30); + if (!hevc) + ave_start_put32(cmd, l->h264_380, 1); + + /* GOP structure: with B-frames as captured, or P-only, or intra-only. */ + ave_start_put32( + cmd, l->gop_flags, + (p->bframes ? 0x600 : 0x200) | + (fw->counted && cq ? 0x10000 | (hevc ? 0x100 : 0) : 0)); + ave_start_put32(cmd, l->p_frames, p->intra_only ? 0 : 1); + ave_start_put32(cmd, l->max_b, p->bframes ? 3 : 0); + ave_start_put32(cmd, l->ref_depth, p->bframes ? 3 : 1); + ave_start_put32(cmd, l->idr_interval, p->gop); + + /* Picture size */ + ave_start_put32(cmd, l->size_class, + p->width * p->height > 1280 * 720 ? 3 : 1); + ave_start_put32(cmd, l->coded_w, p->cw); + ave_start_put32(cmd, l->coded_h, p->ch); + ave_start_put32(cmd, l->coded_h2, p->ch); + ave_start_put32(cmd, l->codec_flags, hevc ? 0x2000002 : 2); + ave_start_put32(cmd, l->bit_depth, depth); + + if (!hevc) { + ave_start_put32(cmd, l->h264_384, + p->width * p->height <= 320 * 240 ? 0x40 : + 0x10); + ave_start_put32(cmd, l->codec_2588, p->high ? 2 : 0); + ave_start_put32(cmd, l->h264_profile, p->profile); + ave_start_put32(cmd, l->h264_level, p->level); + ave_start_put32(cmd, l->h264_max_refs, + (p->refs - 1) | + (fw->counted ? (p->cw / 16) << 16 : 0)); + if (fw->counted) { + static const u8 level_idc[] = { + 10, 9, 11, 12, 13, 20, 21, 22, 30, + 31, 32, 40, 41, 42, 50, 51, 52, + }; + + ave_put32(cmd, l->profile_idc, + p->high ? 100 : + p->profile == 2 ? 66 : + 77); + ave_put32(cmd, l->level_idc, level_idc[p->level - 1]); + put_unaligned_le16(p->refs - 1, cmd + l->max_refs_copy); + ave_put32(cmd, l->mb_size, + ((p->ch / 16 - 1) << 16) | (p->cw / 16 - 1)); + cmd[l->mb_height] = p->ch / 16; + cmd[l->mb_height + 2] = 1; + cmd[l->mb_height + 3] = 1; + cmd[l->crop_copy] = crop; + put_unaligned_le16(crop_r, cmd + l->crop_right_copy); + put_unaligned_le16(crop_b, cmd + l->crop_bottom_copy); + cmd[l->codec_byte] = !cq; + /* CABAC flag in the second, packed picture structure. */ + cmd[l->cabac_copy] = p->profile != 2; + cmd[l->transform_copy] = p->high; + } + ave_start_put32(cmd, l->h264_width_mbs, p->cw / 16 - 1); + ave_start_put32(cmd, l->h264_height_mbs, p->ch / 16 - 1); + ave_start_put32(cmd, l->h264_crop, crop); + ave_start_put32(cmd, l->h264_crop_left, 0); + ave_start_put32(cmd, l->h264_crop_right, crop_r); + ave_start_put32(cmd, l->h264_crop_top, 0); + ave_start_put32(cmd, l->h264_crop_bottom, crop_b); + ave_start_put32(cmd, l->h264_cabac, p->profile != 2); + cmd[l->h264_8x8] = p->high; + return; + } + + ave_start_put32(cmd, l->hevc_pad, + 1 | (p->cw > p->width ? BIT(16) | BIT(18) : 0) | + (p->ch > p->height ? BIT(17) | BIT(19) : 0)); + ave_start_put32(cmd, l->hevc_profile_a, p->profile); + ave_start_put32(cmd, l->hevc_profile_b, p->profile); + cmd[l->hevc_compat_a] = depth == 8; + cmd[l->hevc_compat_a + 1] = 1; + cmd[l->hevc_compat_b] = depth == 8; + cmd[l->hevc_compat_b + 1] = 1; + cmd[l->hevc_8bit_a] = depth == 8; + cmd[l->hevc_8bit_b] = depth == 8; + /* Rate-control words that only change in constant-quality mode */ + ave_start_put32(cmd, l->hevc_cq, cq); + ave_start_put32(cmd, l->hevc_cq_copy, cq); + ave_start_put32(cmd, l->hevc_cq_codec, cq); + ave_start_put32(cmd, l->hevc_cq_picture, cq ? 0x100 : 0); + ave_start_put32(cmd, l->hevc_rc_270c, 2); + ave_start_put32(cmd, l->hevc_rc_9604, cq ? 2 : 0); + ave_start_put32(cmd, l->hevc_rc_9608, cq ? 0 : 0x10000); + ave_start_put32(cmd, l->hevc_level_a, p->level); + ave_start_put32(cmd, l->hevc_level_b, p->level); + ave_start_put32(cmd, l->hevc_width, p->cw); + ave_start_put32(cmd, l->hevc_height, p->ch); + ave_start_put32(cmd, l->hevc_width_ctb, DIV_ROUND_UP(p->cw, 32)); + ave_start_put32(cmd, l->hevc_height_ctb, DIV_ROUND_UP(p->ch, 32)); + ave_start_put32(cmd, l->hevc_conf, crop); + ave_start_put32(cmd, l->hevc_conf_left, 0); + ave_start_put32(cmd, l->hevc_conf_right, crop_r); + ave_start_put32(cmd, l->hevc_conf_top, 0); + ave_start_put32(cmd, l->hevc_conf_bottom, crop_b); + ave_start_put32(cmd, l->hevc_depth_luma, depth - 8); + ave_start_put32(cmd, l->hevc_depth_chroma, depth - 8); + /* Reference structure: B-frame pyramid or low-delay P (ave-image.c) */ + ave_start_put32(cmd, l->hevc_refs, p->bframes ? 4 : 1); + ave_start_put32(cmd, l->hevc_dpb_minus1, 4); + ave_start_put32(cmd, l->hevc_reorder, p->bframes ? 2 : 0); + ave_start_put32(cmd, l->hevc_reorder_copy, p->bframes ? 2 : 0); + ave_start_put32(cmd, l->hevc_reorder_on, p->bframes); +} + +static void ave_reference_start(struct ave_ctx *ctx, u8 *cmd, size_t len) +{ + const struct ave_ref_fw_layout *fw = &ave_reference_layout; + const struct ave_params *p = &ctx->sess->p; + + ave_image_start(cmd, len, ctx->ave->layout, p->codec, p->reorder_off, + p->codec == AVE_CODEC_HEVC && !p->bframes); + ave_cmd_header(ctx, cmd, AVE_OP_START, AVE_OP_START, AVE_SESSION_ARG0, + ave_ref_session_setup_arg2(ctx)); + ave_start_surfaces(ctx->sess, cmd, fw); + ave_start_config(p, cmd, fw); +} + +#define AVE_PROC_TIMESCALE 14400000ULL +#define AVE_PROC_SHDR_ENTRIES 256 +#define AVE_PROC_SHDR_STRIDE 0x400 +#define AVE_PROC_FORCED_TYPE 2 +#define AVE_PROC_LOW_DELAY 3 + +static void ave_reference_process(struct ave_ctx *ctx, u8 *cmd, size_t len, + unsigned int slot, dma_addr_t input, + u32 bitrate, bool force_key) +{ + struct ave_session *sess = ctx->sess; + const struct ave_params *p = &sess->p; + const struct ave_ref_fw_layout *fw = &ave_reference_layout; + const struct ave_ref_proc_layout *l = &fw->process[p->codec - 1]; + struct ave_slot *s = &sess->slot[slot]; + dma_addr_t y = input; + u64 pts; + + ave_image_process(cmd, len, ctx->ave->layout, p->codec); + ave_cmd_header(ctx, cmd, AVE_OP_PROCESS, fw->process_subtype + slot, 0, + ave_ref_session_arg2(ctx)); + /* Input picture: NV12, chroma right after p->height luma lines. */ + ave_put64(cmd, l->y, y); + ave_put32(cmd, l->y + 8, p->stride * p->height); + ave_put32(cmd, l->y + 12, p->stride); + ave_put64(cmd, l->uv, y + p->stride * p->height); + ave_put32(cmd, l->uv + 8, p->stride * p->height / 2); + ave_put32(cmd, l->uv + 12, p->stride); + ave_put32(cmd, l->in_w, p->width); + ave_put32(cmd, l->in_h, p->height); + + /* Session values repeated in every frame */ + ave_put32(cmd, l->coded_cap, p->coded_cap); + ave_put32(cmd, l->fps_int, p->fps_int); + /* The bit rate may change while streaming; it takes effect here. */ + if (l->bitrate) + ave_put32(cmd, l->bitrate, bitrate); + if (l->limits) + ave_put32(cmd, l->limits, p->limit_bytes ? 1 : 0); + if (l->limit_bytes && p->limit_bytes) { + ave_put64(cmd, l->limit_bytes, ave_f64(p->limit_bytes, 1)); + ave_put64(cmd, l->limit_secs, ave_f64(1, 1)); + } + ave_put32(cmd, l->const5, 5); + ave_put64(cmd, l->fps, ave_f64(p->fps_num, p->fps_den)); + ave_put64(cmd, l->timescale, AVE_PROC_TIMESCALE); + if (l->crop) + ave_put32(cmd, l->crop, + p->cw != p->width || p->ch != p->height); + + /* This frame */ + ave_put32(cmd, l->first_frame, sess->frames == 0); + ave_put32(cmd, l->slot, slot); + ave_put64(cmd, l->coded, s->coded.dma); + ave_put64(cmd, l->meta, s->meta.dma); + if (l->meta_size) + ave_put32(cmd, l->meta_size, s->meta.size); + if (l->extra) + ave_put_buf(cmd, l->extra, &s->extra, SZ_16K); + if (l->frame_size) + ave_put32(cmd, l->frame_size, + p->rc_mode == 3 ? 0 : s->aux.size); + if (l->frame_buf) + ave_put64(cmd, l->frame_buf, + fw->counted && p->rc_mode == 3 ? 0 : s->aux.dma); + if (l->shdr_table) + for (unsigned int i = 0; i < AVE_PROC_SHDR_ENTRIES; i++) + ave_put64(cmd, l->shdr_table + 8 * i, + s->aux.dma + i * AVE_PROC_SHDR_STRIDE); + ave_put32(cmd, l->ordinal, sess->frames); + pts = mul_u64_u64_div_u64((u64)sess->frames * p->fps_den, + AVE_PROC_TIMESCALE, p->fps_num); + ave_put64(cmd, l->pts, pts); + if (l->force_key) + ave_put32(cmd, l->force_key, force_key); + if (l->force_type) + ave_put32(cmd, l->force_type, + force_key ? AVE_PROC_FORCED_TYPE : 0); + if (l->low_delay) + ave_put32(cmd, l->low_delay, + p->bframes ? 0 : AVE_PROC_LOW_DELAY); +} diff --git a/drivers/media/platform/apple/ave/tests/boundary.h b/drivers/media/platform/apple/ave/tests/boundary.h new file mode 100644 index 0000000000000..fb8984dd5e079 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/boundary.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Only the hardware/IPC boundary is mocked. The test uses real session types. */ +struct ave_dev { + struct ave_ctx *sessions[AVE_MAX_SESSIONS]; + u32 next_client, text_crc, data_crc; + dma_addr_t extra0_dma; + const struct ave_fw_layout *layout; + const struct ave_variant *variant; + enum ave_fw_state fw_state; + void *dev; +}; + +struct ave_ctx { + struct ave_dev *ave; + struct ave_session *sess; + int ready; +}; + +struct vb2_v4l2_buffer { + struct vb2_buffer { int unused; } vb2_buf; +}; + +static struct ave_vb2_mem *ave_vb2_get(struct vb2_buffer *vb) { return NULL; } +static void ave_vb2_put(void *mem) { assert(!mem); } + +static u8 wire[0x1c000 + 32]; +static size_t wire_len; +static unsigned int submissions; +static u64 next_dma = 0x10002000000ULL; +static unsigned int alloc_count, free_count; +static void ave_free(struct ave_dev *ave, struct ave_buf *buf) +{ + if (buf->cpu) + free_count++; + memset(buf, 0, sizeof(*buf)); +} + +static int ave_alloc_padded(struct ave_dev *ave, struct ave_buf *buf, + size_t size, size_t alloc, unsigned long attrs) +{ + assert(size); + buf->size = size; + buf->alloc = ALIGN(max(size, alloc), SZ_16K); + buf->dma = next_dma; + next_dma += ALIGN(buf->alloc, 0x100000); + buf->attrs = attrs; + buf->cpu = (void *)1; + alloc_count++; + return 0; +} + +static void *ave_ipc_cmd(struct ave_dev *ave, unsigned int buf, size_t len) +{ + assert(len <= sizeof(wire) - 32); + memset(wire, 0, len); + memset(wire + len, 0xa5, 32); + wire_len = len; + return wire; +} + +static int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, + u64 timeout, bool reply) +{ + assert(len == wire_len); + for (unsigned int i = 0; i < 32; i++) + assert(wire[len + i] == 0xa5); +#ifdef TEST_PER_ABI + put_unaligned_le64(timeout, wire + ave->layout->abi->timeout_offset); + put_unaligned_le64(1234567, wire + ave->layout->abi->created_offset); +#else + put_unaligned_le64(timeout, wire + AVE_CMD_TIMEOUT); + put_unaligned_le64(1234567, wire + AVE_CMD_CREATED); +#endif + submissions++; + return 0; +} + +static int ave_ipc_scalar_buf(struct ave_dev *ave) +{ + return 1; +} + +static int ave_ipc_settle(struct ave_dev *ave, unsigned int ms) +{ + return 0; +} + +static int ave_ipc_wait_ack(struct ave_dev *ave, unsigned int buf) +{ + return 0; +} + +static void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, + enum ave_image kind, const void *cmd, size_t len) +{ +} diff --git a/drivers/media/platform/apple/ave/tests/command-boundary.h b/drivers/media/platform/apple/ave/tests/command-boundary.h new file mode 100644 index 0000000000000..a70fc29fc3893 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/command-boundary.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Publishing a Process must already own the mapping it references. */ +static struct ave_ctx *submit_ctx; +static unsigned int submit_slot; +static int submit_result; +static u8 command[256]; +static void *ave_ipc_cmd(struct ave_dev *ave, unsigned int buf, size_t len) +{ + return command; +} +static int ave_ipc_wait_ack(struct ave_dev *ave, unsigned int buf) +{ + return 0; +} +static int ave_build_start(struct ave_ctx *ctx, void *cmd, size_t len) +{ + return 0; +} +static int ave_build_process(struct ave_ctx *ctx, void *cmd, size_t len, + unsigned int slot, dma_addr_t input, u32 bitrate, bool force_key) +{ + return 0; +} +static void ave_debugfs_save(struct ave_dev *ave, enum ave_codec codec, + enum ave_image kind, const void *cmd, size_t len) +{ +} +static int ave_ipc_submit(struct ave_dev *ave, unsigned int buf, size_t len, + u64 timeout, bool reply) +{ + struct ave_vb2_mem *mem = submit_ctx->sess->slot[submit_slot].input_mem; + struct test_mapping *mapping; + + if (reply) { + assert(submit_ctx->sess->handed); + } else { + assert(mem && mem->refs == 2); + mapping = mem->priv; + mapping->firmware_visible = true; + } + if (submit_result) + ave_fw_mark_broken(ave, submit_result, "test submission"); + return submit_result; +} diff --git a/drivers/media/platform/apple/ave/tests/recovery-boundary.h b/drivers/media/platform/apple/ave/tests/recovery-boundary.h new file mode 100644 index 0000000000000..49ab21f3398fb --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/recovery-boundary.h @@ -0,0 +1,313 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Fake PM, MMIO and vb2 boundary for unmodified production function bodies. */ +#include +#define container_of(p, t, m) ((t *)((char *)(p) - offsetof(t, m))) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +#define dev_err_ratelimited(...) test_log(__VA_ARGS__) +#define lockdep_assert_held(p) assert((p)->held) +#define MEMREMAP_WC 0 +/* Host memory is coherent; the production write barrier is simulated. */ +#define wmb() ((void)0) +#define msecs_to_jiffies(x) (x) +#define ktime_add_ms(a, b) ((a) + (b)) +#define ktime_after(a, b) ((a) > (b)) +#define atomic_set(p, v) (*(p) = (v)) +#define atomic_read(p) (*(p)) +#define to_delayed_work(p) container_of(p, struct delayed_work, work) +struct work_struct { int unused; }; + +struct delayed_work { struct work_struct work; bool scheduled; }; + +struct mutex { bool held; }; + +static void mutex_lock(struct mutex *m) +{ + assert(!m->held); + m->held = true; +} + +static void mutex_unlock(struct mutex *m) +{ + assert(m->held); + m->held = false; +} + +#define spin_lock_irqsave(p, flags) do { (flags) = 0; mutex_lock(p); } while (0) +#define spin_unlock_irqrestore(p, flags) do { (void)(flags); mutex_unlock(p); } while (0) +struct ave_dev; +struct device { struct ave_dev *drvdata; unsigned int refs, dma_refs; }; + +static void *dev_get_drvdata(struct device *dev) +{ + return dev->drvdata; +} + +struct vb2_queue { unsigned int type; bool error, streaming; void *priv; }; + +struct vb2_buffer { + struct vb2_queue *vb2_queue; + void *priv; + struct { void *mem_priv; } planes[1]; +}; + +struct vb2_v4l2_buffer { struct vb2_buffer vb2_buf; unsigned int completions; int state; }; + +struct v4l2_m2m_ctx { + struct vb2_queue src, dst; + struct vb2_v4l2_buffer *input[4], *output[4]; + unsigned int nsrc, ndst; +}; + +struct v4l2_fh { struct v4l2_m2m_ctx *m2m_ctx; }; + +struct ave_ctx { + struct ave_dev *ave; + struct v4l2_fh fh; + struct ave_session *sess; + struct delayed_work poll_work; + struct mutex queue_lock; + int ready; + bool failed, draining, flush_pending, stopped; + struct vb2_v4l2_buffer *drain_src; +}; + +struct ave_dev { + struct device *dev; + struct mutex hw_lock; + enum ave_fw_state fw_state; + struct ave_ctx *sessions[AVE_MAX_SESSIONS]; + struct ave_session *failed_sessions; + struct ave_recovery_history recovery; + unsigned int recovery_count; + bool inject_wedge, data_dirty; + void *wq, *m2m_dev, *coproc, *mbox; + const struct ave_variant *variant; + const struct ave_fw_layout *layout; + u32 data_crc; + struct ave_fw_segment text, data; + void *data_pristine; + struct ave_ring ring[2]; + u64 ipc_fw_base; +}; + +static void *test_ctrl_addr; +static int halt_result, close_result; +static unsigned int halts, closes, freed, suspends, resumes, schedules, barriers; +static unsigned int polls, sleep_ms; +static bool map_fail, ctrl_sticky, stopped_for_sleep; +static int poll_result; +typedef unsigned long ktime_t; +static ktime_t ticks; +static ktime_t ktime_get(void) +{ + return ++ticks; +} + +static void msleep(unsigned int ms) +{ + sleep_ms += ms; +} + +static void usleep_range(unsigned int lo, unsigned int hi) { } + +static u32 readl(const void *p) +{ + return *(const u32 *)p; +} + +static u64 readq(const void *p) +{ + return *(const u64 *)p; +} + +static void writel(u32 v, void *p) +{ + if (!ctrl_sticky) { + *(u32 *)p = v; + if (p == test_ctrl_addr) + stop_seen = !v; + } +} + +static void *memremap(phys_addr_t p, size_t size, unsigned int mode) +{ + return map_fail ? NULL : (void *)p; +} + +static void memunmap(void *p) +{ + barriers++; +} + +static bool mod_delayed_work(void *wq, struct delayed_work *work, unsigned long delay) +{ + work->scheduled = true; + return true; +} + +static bool cancel_delayed_work(struct delayed_work *work) +{ + work->scheduled = false; + return true; +} + +static bool cancel_delayed_work_sync(struct delayed_work *work) +{ + return cancel_delayed_work(work); +} + +static bool queue_delayed_work(void *wq, struct delayed_work *work, unsigned long delay) +{ + return mod_delayed_work(wq, work, delay); +} + +static int ave_cmd_halt(struct ave_dev *ave) +{ + halts++; + return halt_result; +} + +static int ave_cmd_close(struct ave_ctx *ctx) +{ + closes++; + return close_result; +} + +static int ave_cmd_complete(struct ave_ctx *ctx, unsigned int n) +{ + return 0; +} + +static int ave_ipc_settle(struct ave_dev *ave, unsigned int ms) +{ + return 0; +} + +static int ave_ipc_poll_once(struct ave_dev *ave) +{ + /* A deadline bug must fail, rather than leave the test spinning. */ + assert(++polls < 100); + return poll_result; +} + +static bool ave_sess_starved(struct ave_session *sess) +{ + return false; +} + +static bool ave_sess_stalled(struct ave_session *sess) +{ + return false; +} + +static void ave_free(struct ave_dev *ave, struct ave_buf *buf) +{ + if (buf->cpu) { + assert(stop_seen && power_seen); + freed++; + buf->cpu = NULL; + } +} + +static void pm_runtime_mark_last_busy(struct device *dev) { } + +static void pm_runtime_put_autosuspend(struct device *dev) +{ + assert(dev->refs); + dev->refs--; +} + +static void v4l2_m2m_suspend(void *dev) +{ + suspends++; +} + +static void v4l2_m2m_resume(void *dev) +{ + resumes++; +} + +static void ave_v4l2_suspend(struct ave_dev *ave) +{ + stopped_for_sleep = true; +} + +static void v4l2_m2m_try_schedule(struct v4l2_m2m_ctx *ctx) +{ + schedules++; +} + +static struct vb2_queue *v4l2_m2m_get_src_vq(struct v4l2_m2m_ctx *ctx) +{ + return &ctx->src; +} + +static struct vb2_queue *v4l2_m2m_get_dst_vq(struct v4l2_m2m_ctx *ctx) +{ + return &ctx->dst; +} + +#define V4L2_TYPE_IS_OUTPUT(t) ((t) == 1) +enum vb2_buffer_state { VB2_BUF_STATE_ERROR = 1 }; + +static struct vb2_v4l2_buffer *v4l2_m2m_src_buf_remove(struct v4l2_m2m_ctx *ctx) +{ + return ctx->nsrc ? ctx->input[--ctx->nsrc] : NULL; +} + +static struct vb2_v4l2_buffer *v4l2_m2m_dst_buf_remove(struct v4l2_m2m_ctx *ctx) +{ + return ctx->ndst ? ctx->output[--ctx->ndst] : NULL; +} + +static unsigned int v4l2_m2m_num_src_bufs_ready(struct v4l2_m2m_ctx *ctx) +{ + return ctx->nsrc; +} + +static void v4l2_m2m_buf_done(struct vb2_v4l2_buffer *buf, enum vb2_buffer_state state) +{ + assert(++buf->completions == 1); + buf->state = state; +} + +static bool vb2_is_streaming(struct vb2_queue *q) +{ + return q->streaming; +} + +static void vb2_queue_error(struct vb2_queue *q) +{ + q->error = true; +} + +static void *vb2_get_drv_priv(struct vb2_queue *q) +{ + return q->priv; +} +#define to_vb2_v4l2_buffer(p) container_of(p, struct vb2_v4l2_buffer, vb2_buf) +static void v4l2_m2m_buf_queue(struct v4l2_m2m_ctx *ctx, struct vb2_v4l2_buffer *buf) +{ + assert(!ctx->src.error && ctx->nsrc < 3); + ctx->input[ctx->nsrc++] = buf; +} +/* Forward declarations between the extracted, otherwise unchanged callbacks. */ +void ave_fw_mark_broken(struct ave_dev *ave, int err, const char *what); +void ave_sess_reap(struct ave_dev *ave); +static void ave_reset_drain(struct ave_ctx *ctx); +int ave_ipc_pump(struct ave_dev *ave, unsigned int wait_ms, + bool (*cond)(struct ave_dev *, void *), void *arg); + +/* ABI-selected helpers are unreachable in the 9003 recovery fixtures. */ +#define readl_poll_timeout(addr, value, condition, delay, timeout) \ + ((value) = readl(addr), (condition) ? 0 : -ETIMEDOUT) +static int ave_fw_crc(const struct ave_fw_segment *segment, u32 *crc, void *copy) +{ + assert(!"6070 CRC readback reached a 9003 recovery fixture"); + return -EIO; +} +static void ave_fw_release_boot(struct ave_dev *ave) +{ + assert(!"6070 boot teardown reached a 9003 recovery fixture"); +} diff --git a/drivers/media/platform/apple/ave/tests/recovery-test.c b/drivers/media/platform/apple/ave/tests/recovery-test.c new file mode 100644 index 0000000000000..b39174ec9f2d2 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/recovery-test.c @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: GPL-2.0-only +static const struct ave_coproc_regs regs = { .ctrl = 0, .boot_arg = 4, .vcbar = 8 }; +static const struct ave_variant variant = { + .regs = ®s, .vcbar_mask = ~0ULL, .vcbar_flags = AVE_COPROC_VCBAR_VALID, +}; +static const struct ave_fw_transport test_transport = { .boot_config = true }; +static const struct ave_fw_abi test_abi = { .transport = &test_transport }; +static const struct ave_fw_layout test_layout = { + .abi = &test_abi, .encode_ready = true, .max_slots = 2, +}; +static u64 coproc[4]; +static u32 mailbox[16]; +static struct ave_dev ave; +static struct device dev; +static struct ave_ctx ctx[2]; +static struct v4l2_m2m_ctx m2m[2]; + +static void init(void) +{ + memset(&ave, 0, sizeof(ave)); + memset(&dev, 0, sizeof(dev)); + memset(ctx, 0, sizeof(ctx)); + memset(m2m, 0, sizeof(m2m)); + memset(coproc, 0, sizeof(coproc)); + memset(mailbox, 0, sizeof(mailbox)); + halts = closes = freed = suspends = resumes = schedules = polls = sleep_ms = 0; + halt_result = close_result = poll_result = 0; + map_fail = ctrl_sticky = stopped_for_sleep = false; + stop_seen = power_seen = dma_alloc_fail = dma_attach_fail = dma_map_fail = false; + mapping_frees = mapping_unmaps = mapping_detaches = 0; + submit_result = 0; + test_ctrl_addr = (char *)coproc + regs.ctrl; + ave.dev = &dev; + dev.drvdata = &ave; + ave.variant = &variant; + ave.layout = &test_layout; + ave.coproc = coproc; + ave.mbox = mailbox; + ave.text.iova = 0x100000000; + *(u64 *)((char *)coproc + regs.vcbar) = ave.text.iova | AVE_COPROC_VCBAR_VALID; + for (unsigned int i = 0; i < 2; i++) { + ctx[i].ave = &ave; + ctx[i].fh.m2m_ctx = &m2m[i]; + m2m[i].src.type = 1; + m2m[i].src.priv = &ctx[i]; + m2m[i].dst.priv = &ctx[i]; + } +} + +static void owners(void) +{ + for (unsigned int i = 0; i < 2; i++) { + ctx[i].sess = calloc(1, sizeof(*ctx[i].sess)); + assert(ctx[i].sess); + ctx[i].sess->handed = ctx[i].sess->fw_open = true; + ctx[i].sess->p.slots = 2; + ctx[i].sess->p.codec = AVE_CODEC_H264; + ctx[i].sess->hdr.cpu = (void *)1; + ave.sessions[i] = &ctx[i]; + dev.refs++; + } + ave.fw_state = AVE_FW_RUNNING; + *(u32 *)((char *)coproc + regs.ctrl) = AVE_COPROC_CTRL_RUN; + *(u32 *)((char *)coproc + regs.boot_arg) = AVE_COPROC_BOOT_ARG_VALUE; + ave.ring[0].slots = 4; + ave.ipc_fw_base = 0x1234; +} + +static void buffer_failure(void) +{ + struct vb2_v4l2_buffer input[2][3] = {}, output[2][2] = {}; + struct dma_buf imported = { .refs = 1 }; + + init(); + owners(); + for (unsigned int i = 0; i < 2; i++) { + struct vb2_buffer *vb = &input[i][0].vb2_buf; + + vb->planes[0].mem_priv = i ? + ave_vb2_memops.attach_dmabuf(vb, &dev, &imported, SZ_16K) : + ave_vb2_memops.alloc(vb, &dev, SZ_16K); + assert(!IS_ERR(vb->planes[0].mem_priv)); + if (i) + assert(!ave_vb2_memops.map_dmabuf(vb->planes[0].mem_priv)); + submit_ctx = &ctx[i]; + submit_slot = 0; + mutex_lock(&ave.hw_lock); + assert(!ave_cmd_process(&ctx[i], 0, &input[i][0], 0, 0, false)); + mutex_unlock(&ave.hw_lock); + ctx[i].sess->slot[0].src = &input[i][0]; + /* One idle owner and one with a firmware-owned frame. */ + ctx[i].sess->busy = i; + ctx[i].ready = i; + m2m[i].input[0] = &input[i][1]; + m2m[i].input[1] = &input[i][2]; + m2m[i].nsrc = 2; + m2m[i].output[0] = &output[i][0]; + m2m[i].output[1] = &output[i][1]; + m2m[i].ndst = 2; + } + assert(!ave_inject_wedge(&ave, 1)); + assert(ave.fw_state == AVE_FW_BROKEN); + assert(ctx[0].poll_work.scheduled && ctx[1].poll_work.scheduled); + /* No runtime recovery is permitted while either owner remains. */ + assert(ave_runtime_suspend(&dev) == -EBUSY); + assert(!ave.recovery_count && !halts); + for (unsigned int i = 0; i < 2; i++) { + ave_poll_work(&ctx[i].poll_work.work); + assert(ctx[i].failed && !ctx[i].sess && !ave.sessions[i]); + assert(m2m[i].src.error && m2m[i].dst.error); + assert(!m2m[i].nsrc && !m2m[i].ndst && !ctx[i].ready); + assert(!ctx[i].poll_work.scheduled && !ave_job_ready(&ctx[i])); + for (unsigned int n = 0; n < 3; n++) + assert(input[i][n].completions == 1 && + input[i][n].state == VB2_BUF_STATE_ERROR); + for (unsigned int n = 0; n < 2; n++) + assert(output[i][n].completions == 1 && + output[i][n].state == VB2_BUF_STATE_ERROR); + /* DQBUF/REQBUFS/close immediately after ERROR: vb2 lets go. */ + if (i) { + ave_vb2_memops.unmap_dmabuf(input[i][0].vb2_buf.planes[0].mem_priv); + ave_vb2_memops.detach_dmabuf(input[i][0].vb2_buf.planes[0].mem_priv); + } else { + ave_vb2_memops.put(input[i][0].vb2_buf.planes[0].mem_priv); + } + input[i][0].vb2_buf.planes[0].mem_priv = NULL; + assert(!mapping_frees && !mapping_unmaps && !mapping_detaches); + } + /* Even dropping the importing fd cannot destroy its retained attachment. */ + dma_buf_put(&imported); + assert(imported.refs == 1 && dev.dma_refs == 1); + assert(!dev.refs && !freed && ave.failed_sessions && !closes); + /* A QBUF racing the failure is returned instead of silently requeued. */ + { + struct vb2_v4l2_buffer extra = {}; + + extra.vb2_buf.vb2_queue = &m2m[0].src; + ave_buf_queue(&extra.vb2_buf); + assert(extra.completions == 1 && extra.state == VB2_BUF_STATE_ERROR); + } + *(u32 *)(ave.coproc + regs.boot_arg) = 0; + assert(!ave_runtime_suspend(&dev)); + assert(readl(ave.coproc + regs.boot_arg) == AVE_COPROC_BOOT_ARG_VALUE); + assert(ave.fw_state == AVE_FW_RECOVERING && ave.recovery_count == 1); + assert(!readl(ave.coproc + regs.ctrl)); + mutex_lock(&ave.hw_lock); + /* CTRL=0 alone is not proof of a power cycle. */ + assert(ave_fw_prepare_restart(&ave) == -EAGAIN && !freed); + *(u32 *)(ave.coproc + regs.boot_arg) = 0; + power_seen = true; /* VENC and its DART have now lost power. */ + assert(!ave_fw_prepare_restart(&ave)); + mutex_unlock(&ave.hw_lock); + assert(freed == 2 && !ave.failed_sessions && !ave.inject_wedge); + assert(stop_seen && mapping_frees == 2 && mapping_unmaps == 1 && mapping_detaches == 1); + assert(!imported.refs && !dev.dma_refs); + assert(ave.fw_state == AVE_FW_HALTED); + m2m[0].dst.streaming = true; + ave_stop_streaming(&m2m[0].src); + assert(ctx[0].failed); /* STREAMOFF one queue cannot clear the failure. */ + m2m[0].dst.streaming = false; + ave_stop_streaming(&m2m[0].dst); + assert(!ctx[0].failed); + m2m[0].src.error = false; /* vb2 clears queue errors on STREAMOFF. */ + { + struct vb2_v4l2_buffer extra = {}; + + extra.vb2_buf.vb2_queue = &m2m[0].src; + ave_buf_queue(&extra.vb2_buf); /* Before start_streaming on STREAMON. */ + assert(!extra.completions && m2m[0].nsrc == 1); + } +} + +static void sleep_guarantees(void) +{ + init(); + ave.fw_state = AVE_FW_RUNNING; + halt_result = -ETIMEDOUT; + assert(ave_suspend(&dev) == -EBUSY); + assert(suspends == 1 && resumes == 1 && stopped_for_sleep); + assert(ave.fw_state == AVE_FW_BROKEN && !ave.recovery_count); + assert(ave_suspend(&dev) == -EBUSY && halts == 1); + assert(suspends == 2 && resumes == 2); + ave.fw_state = AVE_FW_RECOVERING; + assert(ave_suspend(&dev) == -EBUSY); + assert(resumes == 3); + ave.fw_state = AVE_FW_RUNNING; + halt_result = 0; + assert(!ave_suspend(&dev)); + assert(ave.fw_state == AVE_FW_HALTED && resumes == 3); + assert(!ave_resume(&dev) && resumes == 4); +} + +static void recovery_failures(void) +{ + init(); + jiffies = 100; + for (unsigned int i = 0; i < AVE_RECOVERY_ATTEMPTS; i++) { + ave.fw_state = AVE_FW_BROKEN; + ave.ipc_fw_base = 1; + ave.ring[0].slots = 4; + halt_result = -ETIMEDOUT; + assert(!ave_runtime_suspend(&dev)); + assert(ave.fw_state == AVE_FW_RECOVERING); + } + assert(halts == 3 && ave.recovery_count == 3); + ave.fw_state = AVE_FW_BROKEN; + assert(ave_runtime_suspend(&dev) == -EBUSY && halts == 3); + jiffies += AVE_RECOVERY_WINDOW; + ctrl_sticky = true; + *(u32 *)(ave.coproc + regs.ctrl) = AVE_COPROC_CTRL_RUN; + assert(ave_runtime_suspend(&dev) == -EIO); + assert(ave.fw_state == AVE_FW_BROKEN && ave.recovery_count == 4); + ctrl_sticky = false; + assert(!ave_runtime_suspend(&dev)); + assert(!ave_runtime_suspend(&dev)); /* Does not count the PM retry again. */ + assert(ave.recovery_count == 5); +} + +static bool never_answered(struct ave_dev *dev, void *arg) { return false; } +static void busy_ring_deadline(void) +{ + init(); + owners(); + poll_result = 1; /* A continuous stream of unrelated acks/messages. */ + mutex_lock(&ave.hw_lock); + assert(ave_ipc_pump(&ave, 5, never_answered, NULL) == -ETIMEDOUT); + mutex_unlock(&ave.hw_lock); + assert(polls < 10 && ave.fw_state == AVE_FW_BROKEN); + assert(ctx[0].poll_work.scheduled && ctx[1].poll_work.scheduled); + ave_poll_work(&ctx[0].poll_work.work); + ave_poll_work(&ctx[1].poll_work.work); + mutex_lock(&ave.hw_lock); + stop_seen = power_seen = true; + ave_sess_reap(&ave); /* Fake reset for test cleanup only. */ + mutex_unlock(&ave.hw_lock); +} + +static void restore_data(void) +{ + u8 pristine[64], live[64]; + + init(); + memset(pristine, 0xa5, sizeof(pristine)); + memset(live, 0x3c, sizeof(live)); + ave.data_pristine = pristine; + ave.data.phys = (phys_addr_t)live; + ave.data.size = sizeof(live); + ave.data_dirty = true; + map_fail = true; + assert(ave_fw_restore_data(&ave) == -ENOMEM && ave.data_dirty); + map_fail = false; + assert(!ave_fw_restore_data(&ave) && !ave.data_dirty); + assert(!memcmp(live, pristine, sizeof(live))); + assert(!ave_fw_restore_data(&ave)); +} + +static void strict_6070_recovery(void) +{ + static const struct ave_fw_transport transport = { .scratch_halt = true }; + static const struct ave_fw_abi abi = { .transport = &transport }; + static const struct ave_fw_layout layout = { .abi = &abi }; + + init(); + ave.layout = &layout; + ave.fw_state = AVE_FW_BROKEN; + *(u32 *)(ave.coproc + regs.ctrl) = AVE_COPROC_CTRL_RUN; + assert(ave_runtime_suspend(&dev) == -EBUSY); + assert(ave.fw_state == AVE_FW_BROKEN && !halts && !ave.recovery_count); + assert(readl(ave.coproc + regs.ctrl) == AVE_COPROC_CTRL_RUN); + ave.fw_state = AVE_FW_RECOVERING; + assert(ave_runtime_suspend(&dev) == -EBUSY); + assert(readl(ave.coproc + regs.ctrl) == AVE_COPROC_CTRL_RUN); +} + +/* An uncertain send failure must keep its pin even without a slot src. */ +static void failed_submission(void) +{ + struct vb2_v4l2_buffer input = {}; + void *mem; + + init(); + owners(); + mem = ave_vb2_memops.alloc(&input.vb2_buf, &dev, SZ_16K); + assert(!IS_ERR(mem)); + input.vb2_buf.planes[0].mem_priv = mem; + submit_ctx = &ctx[0]; + submit_slot = 0; + submit_result = -EIO; + mutex_lock(&ave.hw_lock); + assert(ave_cmd_process(&ctx[0], 0, &input, 0, 0, false) == -EIO); + mutex_unlock(&ave.hw_lock); + assert(!ctx[0].sess->slot[0].src && ctx[0].sess->slot[0].input_mem); + ave_vb2_memops.put(mem); /* The caller returns this failed submission. */ + ave_poll_work(&ctx[0].poll_work.work); + ave_poll_work(&ctx[1].poll_work.work); + assert(!mapping_frees); + for (unsigned int i = 0; i < AVE_RECOVERY_ATTEMPTS; i++) + assert(ave_recovery_allow(&ave.recovery, jiffies)); + assert(ave_runtime_suspend(&dev) == -EBUSY); + assert(!stop_seen && !mapping_frees && !freed); + jiffies += AVE_RECOVERY_WINDOW; + ctrl_sticky = true; + assert(ave_runtime_suspend(&dev) == -EIO); + assert(!stop_seen && !mapping_frees && !freed); + ctrl_sticky = false; + assert(!ave_runtime_suspend(&dev)); + assert(stop_seen && !mapping_frees); + assert(ave_fw_prepare_restart(&ave) == -EAGAIN); + *(u32 *)(ave.coproc + regs.boot_arg) = 0; + power_seen = true; + mutex_lock(&ave.hw_lock); + assert(!ave_fw_prepare_restart(&ave)); + mutex_unlock(&ave.hw_lock); + assert(mapping_frees == 1 && freed == 2); +} + +/* Start can expose scratch even if submission reports failure. */ +static void failed_start(void) +{ + init(); + owners(); + submit_ctx = &ctx[0]; + ctx[0].sess->handed = false; + submit_result = -EIO; + mutex_lock(&ave.hw_lock); + assert(ave_cmd_start(&ctx[0]) == -EIO); + assert(ctx[0].sess->handed); + mutex_unlock(&ave.hw_lock); + ave_poll_work(&ctx[0].poll_work.work); + ave_poll_work(&ctx[1].poll_work.work); + assert(!freed && ave.failed_sessions); + assert(!ave_runtime_suspend(&dev)); + assert(stop_seen && !freed); + *(u32 *)(ave.coproc + regs.boot_arg) = 0; + power_seen = true; + mutex_lock(&ave.hw_lock); + assert(!ave_fw_prepare_restart(&ave)); + mutex_unlock(&ave.hw_lock); + assert(freed == 2); +} + +/* Allocator failures and normal vb2-only lifetimes do not leak references. */ +static void mapping_lifetimes(void) +{ + struct vb2_buffer vb = {}; + struct dma_buf dbuf = { .refs = 1 }; + void *mem; + + init(); + dma_alloc_fail = true; + assert(IS_ERR(ave_vb2_memops.alloc(&vb, &dev, SZ_16K))); + dma_alloc_fail = false; + dma_attach_fail = true; + assert(IS_ERR(ave_vb2_memops.attach_dmabuf(&vb, &dev, &dbuf, SZ_16K))); + assert(dbuf.refs == 1 && !dev.dma_refs); + dma_attach_fail = false; + mem = ave_vb2_memops.attach_dmabuf(&vb, &dev, &dbuf, SZ_16K); + assert(!IS_ERR(mem)); + dma_map_fail = true; + assert(ave_vb2_memops.map_dmabuf(mem) == -EIO); + dma_map_fail = false; + assert(!ave_vb2_memops.map_dmabuf(mem)); + ave_vb2_memops.unmap_dmabuf(mem); + assert(!mapping_unmaps); + assert(!ave_vb2_memops.map_dmabuf(mem)); /* Reuses the retained mapping. */ + ave_vb2_memops.unmap_dmabuf(mem); + ave_vb2_memops.detach_dmabuf(mem); + assert(mapping_unmaps == 1 && mapping_detaches == 1 && mapping_frees == 1); + assert(dbuf.refs == 1 && !dev.dma_refs); +} + +int main(void) +{ + mapping_lifetimes(); + failed_submission(); + failed_start(); + strict_6070_recovery(); + buffer_failure(); + sleep_guarantees(); + recovery_failures(); + busy_ring_deadline(); + restore_data(); + puts("ok: recovery ownership, queues, quarantine, reset proof, rate limit, " + "PM rollback, DATA, IPC deadline, MMAP/dma-buf release ordering"); + return 0; +} diff --git a/drivers/media/platform/apple/ave/tests/run-recovery.py b/drivers/media/platform/apple/ave/tests/run-recovery.py new file mode 100644 index 0000000000000..0b8bc7097d194 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/run-recovery.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise production AVE recovery, PM and queue callbacks at a fake boundary.""" +import os +from pathlib import Path +import re +import subprocess +import tempfile + +from run import declaration, function, source + + +def main(): + here = Path(__file__).resolve().parent + src = here.parent + header = (src / 'ave.h').read_text() + code = (here / 'shim.h').read_text() + code += '\ntypedef uintptr_t phys_addr_t;\n' + code += '#define HZ 1000\n#define time_before(a, b) ((long)((a) - (b)) < 0)\n' + code += source((src / 'ave-recovery.h').read_text()) + for name in ('AVE_MAX_SLOTS', 'AVE_MAX_REFS', 'AVE_TILE_BUFS', 'AVE_MAX_SESSIONS'): + code += re.search(r'^#define ' + name + r'\s+[^\n]+', header, re.M)[0] + '\n' + code += source((src / 'ave-regs.h').read_text()) + code += 'struct vb2_v4l2_buffer;\n' + for name in ('ave_codec', 'ave_buf', 'ave_fw_segment', 'ave_ring', 'ave_fw_state', + 'ave_coproc_regs', 'ave_variant', 'ave_params', 'ave_slot', 'ave_session'): + code += declaration(header, name) + code += declaration(header, 'ave_image') + code += declaration(header, 'ave_opcode') + code += source((src / 'ave-layout.h').read_text()) + boundary = (here / 'recovery-boundary.h').read_text() + marker = 'static void *test_ctrl_addr;' + boundary = boundary.replace(marker, (here / 'vb2-boundary.h').read_text() + '\n' + marker) + code += boundary + code += source((src / 'ave-vb2.c').read_text()) + code += (here / 'command-boundary.h').read_text() + groups = { + 'ave-fw.c': ('ave_fw_vcbar', 'ave_fw_lost', 'ave_fw_mark_broken', 'ave_fw_halt', 'ave_fw_shutdown', + 'ave_fw_runtime_suspend', 'ave_fw_prepare_restart', 'ave_fw_restore_data'), + 'ave-sess.c': ('ave_sess_free_bufs', 'ave_sess_destroy', 'ave_sess_reap'), + 'ave-v4l2.c': ('ave_return_bufs', 'ave_buf_queue', 'ave_session_index', + 'ave_session_count', 'ave_session_close', 'ave_session_free', + 'ave_session_end', 'ave_stop_streaming', + 'ave_reset_drain', 'ave_job_ready', 'ave_job_fail', 'ave_poll_work'), + 'ave-ipc.c': ('ave_ipc_pump',), + 'ave-cmd.c': ('ave_cmd_start', 'ave_cmd_process'), + 'ave-drv.c': ('ave_runtime_resume', 'ave_runtime_suspend', 'ave_suspend', 'ave_resume', 'ave_inject_wedge'), + } + code += '#define AVE_POLL_FAST_ROUNDS 20\n#define AVE_POLL_MS 2\n' + for filename, names in groups.items(): + text = (src / filename).read_text() + for name in names: + code += function(text, name) + code += (here / 'recovery-test.c').read_text() + with tempfile.TemporaryDirectory(prefix='ave-recovery-') as tmp: + path = Path(tmp) + (path / 'test.c').write_text(code) + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-O1', '-g', + '-Wall', '-Wextra', '-Werror', '-Wno-unused-parameter', + '-Wno-unused-function', '-fsanitize=address,undefined', + '-fno-omit-frame-pointer', str(path / 'test.c'), + '-o', str(path / 'test')], check=True) + subprocess.run([str(path / 'test')], check=True) + + +if __name__ == '__main__': + main() diff --git a/drivers/media/platform/apple/ave/tests/run.py b/drivers/media/platform/apple/ave/tests/run.py new file mode 100644 index 0000000000000..dad2b55ab7161 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/run.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Host checks of the actual AVE builders with a fake IPC/DMA boundary. + +Only includes and unused kernel-facing declarations are omitted. Function +bodies are compiled unchanged. --baseline accepts an older AVE source dir +for byte comparisons; --build retains the executable for private capture QA. +No firmware, command images, or reference captures are part of these tests. +""" +import argparse +import os +from pathlib import Path +import re +import subprocess +import tempfile + + +def declaration(text, name): + match = re.search(r'^(?:struct|enum) ' + name + r' \{', text, re.M) + end = text.index('\n};', match.start()) + 3 + return text[match.start():end] + '\n' + + +def function(text, name): + match = re.search(r'^(?:static )?(?:const )?[\w *]+\b' + name + r'\([^;]+?\n\{', text, re.M) + end = text.index('\n}', match.end()) + 2 + return text[match.start():end] + '\n' + + +def source(text): + return re.sub(r'^#include .*\n', '', text, flags=re.M) + + +def build(src, out, baseline): + header = (src / 'ave.h').read_text() + per_abi = (src / 'ave-writer.c').exists() + has_layout = (src / 'ave-layout.h').exists() + code = (Path(__file__).parent / 'shim.h').read_text() + for name in ('AVE_MAX_SLOTS', 'AVE_MAX_REFS', 'AVE_TILE_BUFS', + 'AVE_MAX_SESSIONS', 'AVE_FW_TEXT_CRC_9003_78_0'): + code += re.search(r'^#define ' + name + r'\s+[^\n]+', header, re.M)[0] + '\n' + # Later firmware builds; an older baseline does not define them. + for name in ('AVE_FW_TEXT_CRC_T6040_25G83',): + m = re.search(r'^#define ' + name + r'\s+[^\n]+', header, re.M) + if m: + code += m[0] + '\n' + for name in ('ave_codec', 'ave_buf', 'ave_fw_state', 'ave_image', + 'ave_coproc_regs', 'ave_variant', 'ave_params', 'ave_slot', 'ave_session'): + code += declaration(header, name) + if per_abi: + code = '#define TEST_PER_ABI 1\n' + code + code += declaration(header, 'ave_opcode') + for name in ('AVE_IPC_BYTES', 'AVE_IPC_CMD_OFF', 'AVE_IPC_CMD_BYTES', + 'AVE_IPC_SCALAR_OFF', 'AVE_IPC_PROC_OFF', 'AVE_IPC_PROC_BYTES'): + code += re.search(r'^#define ' + name + r'\s+[^\n]+', header, re.M)[0] + '\n' + if baseline: + code = '#define TEST_BASELINE 1\n' + code + code += '\n'.join(re.findall(r'^#define AVE_(?:START|PROCESS)_\w+.*$', header, re.M)) + '\n' + if has_layout: + code = '#define TEST_LAYOUT 1\n' + code + code += source((src / 'ave-layout.h').read_text()) + code += source((src / 'ave-layout.c').read_text()) + code += source((src / 'ave-regs.h').read_text()) + code += source((src / 'ave-sizes.h').read_text()) + code += (Path(__file__).parent / 'boundary.h').read_text() + code += source((src / 'ave-image.c').read_text()) + if per_abi: + code += source((src / 'ave-writer.c').read_text()) + code += source((src / 'ave-abi.c').read_text()) + code += source((src / 'ave-cmd.c').read_text()) + session = (src / 'ave-sess.c').read_text() + code += declaration(session, 'ave_buf_req') + lines = iter(session.splitlines()) + for line in lines: + if line.startswith('#define '): + code += line + '\n' + while line.endswith('\\'): + line = next(lines) + code += line + '\n' + for name in ('ave_buf_req_cmp', 'ave_req', 'ave_sess_alloc', 'ave_sess_free_bufs', 'ave_sess_reply'): + code += function(session, name) + if not baseline: + fw = (src / 'ave-fw.c').read_text() + if per_abi: + code += function(fw, 'ave_fw_check_boot') + code += function(fw, 'ave_fw_check_build') + code += (Path(__file__).parent / 'test.c').read_text() + out.mkdir(parents=True, exist_ok=True) + (out / 'host.c').write_text(code) + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-O1', '-g', + '-Wall', '-Wextra', '-Werror', '-Wno-unused-parameter', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', + str(out / 'host.c'), '-o', str(out / 'ave-test')], check=True) + return out / 'ave-test' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--baseline', type=Path) + parser.add_argument('--build', type=Path) + parser.add_argument('--compare-baseline', type=Path) + args = parser.parse_args() + src = args.baseline or Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory(prefix='ave-host-') as tmp: + binary = build(src, args.build or Path(tmp), bool(args.baseline)) + subprocess.run([str(binary)], check=True) + if args.compare_baseline: + old = build(args.compare_baseline, Path(tmp) / 'baseline', True) + before = subprocess.check_output([str(old), '--matrix']) + after = subprocess.check_output([str(binary), '--matrix']) + if before != after: + mismatch = next(i for i, pair in enumerate(zip(before, after)) if pair[0] != pair[1]) + raise AssertionError(f'J700 matrix differs at stream byte {mismatch}') + print(f'ok: 288 J700 sessions / 1440 commands byte-identical ({len(after)} bytes)') + + +if __name__ == '__main__': + main() diff --git a/drivers/media/platform/apple/ave/tests/shim.h b/drivers/media/platform/apple/ave/tests/shim.h new file mode 100644 index 0000000000000..5a0201ac80269 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/shim.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Host types/primitives, never an alternative command implementation. */ +#include +#include +#include +#include +#include +#include +#include + +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +typedef int32_t s32; +typedef int64_t s64; +typedef uint64_t dma_addr_t; +#define AVE_BUF_CMD 0 +#define AVE_BUF_PROC(s) (17 + (s)) +#define SZ_16K 0x4000U +#define SZ_1M 0x100000U +#define U32_MAX UINT32_MAX +#define GENMASK_ULL(h, l) ((~0ULL >> (63 - (h))) & (~0ULL << (l))) +#define BIT(n) (1U << (n)) +#define BIT_ULL(n) (1ULL << (n)) +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define DIV_ROUND_UP(x, a) (((x) + (a) - 1) / (a)) +#define ALIGN(x, a) (((x) + (a) - 1) & ~((a) - 1)) +#define max(x, y) ((x) > (y) ? (x) : (y)) +#define max_t(t, x, y) ((t)(x) > (t)(y) ? (t)(x) : (t)(y)) +#define GFP_KERNEL 0 +#define DMA_ATTR_NO_KERNEL_MAPPING 1 +static inline void *kcalloc(size_t n, size_t size, int flags) +{ + return calloc(n, size); +} + +#define kfree(p) free(p) +static inline void sort(void *p, size_t n, size_t size, + int (*cmp)(const void *, const void *), void *unused) +{ + qsort(p, n, size, cmp); +} + +#define dev_err(...) test_log(__VA_ARGS__) +#define dev_info(...) test_log(__VA_ARGS__) +#define dev_dbg(...) test_log(__VA_ARGS__) +#define dev_warn(...) test_log(__VA_ARGS__) +#define WARN_ON_ONCE(c) ((c) ? (abort(), 1) : 0) +#define ENOTSUPP 524 +static inline void test_log(void *dev, const char *fmt, ...) +{ +} + +#define atomic_inc(p) (++*(p)) +static unsigned long jiffies = 42; +static inline void put_unaligned_le16(u16 v, void *p) +{ + u8 *b = p; + + for (unsigned int i = 0; i < 2; i++) + b[i] = v >> (8 * i); +} +static inline void put_unaligned_le32(u32 v, void *p) +{ + u8 *b = p; + + for (unsigned int i = 0; i < 4; i++) + b[i] = v >> (8 * i); +} + +static inline void put_unaligned_le64(u64 v, void *p) +{ + u8 *b = p; + + for (unsigned int i = 0; i < 8; i++) + b[i] = v >> (8 * i); +} + +#ifdef TEST_PER_ABI +static inline u16 get_unaligned_le16(const void *p) +{ + const u8 *b = p; + + return (u16)b[0] | (u16)b[1] << 8; +} + +#endif +static inline u32 get_unaligned_le32(const void *p) +{ + const u8 *b = p; + + return (u32)b[0] | (u32)b[1] << 8 | (u32)b[2] << 16 | (u32)b[3] << 24; +} + +static inline u64 get_unaligned_le64(const void *p) +{ + return get_unaligned_le32(p) | + (u64)get_unaligned_le32((const u8 *)p + 4) << 32; +} + +static inline u64 mul_u64_u64_div_u64(u64 a, u64 b, u64 c) +{ + return (__uint128_t)a * b / c; +} + +static inline u32 roundup_pow_of_two(u32 v) +{ + u32 n = 1; + + while (n < v) + n *= 2; + return n; +} diff --git a/drivers/media/platform/apple/ave/tests/test.c b/drivers/media/platform/apple/ave/tests/test.c new file mode 100644 index 0000000000000..55df0584c876b --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/test.c @@ -0,0 +1,370 @@ +// SPDX-License-Identifier: GPL-2.0-only +static const struct ave_coproc_regs test_regs; +static const struct ave_variant test_variant = { + .lfs_ref_scale = 2, +#ifndef TEST_BASELINE + .boot_qualified = true, + .regs = &test_regs, +#endif +}; + +static void params(struct ave_params *p, unsigned int codec, u32 w, u32 h) +{ + memset(p, 0, sizeof(*p)); + p->codec = codec; + p->width = w; + p->height = h; + p->cw = ALIGN(w, 16); + p->ch = ALIGN(h, 16); + p->stride = ALIGN(w, 64); + p->mbs = p->cw / 16 * (p->ch / 16); + p->fps_num = 30; + p->fps_int = 30; + p->fps_den = 1; + p->bframes = true; + p->refs = 5; + p->slots = 5; + p->gop = 30; + p->rc_mode = 4; + p->bitrate = 4000000; + p->limit_bytes = 1000000; + p->max_qp = 51; + p->profile = codec == 1 ? 4 : 1; + p->level = codec == 1 ? 12 : 6; + p->coded_cap = + max(ALIGN(3 * (codec == 2 ? ALIGN(w, 32) : p->cw) * p->ch / 2, + SZ_16K), + 0x3c000U); +} + +static void start(struct ave_ctx *ctx) +{ + assert(ave_sess_alloc(ctx->ave, ctx->sess) == 0); + assert(ave_cmd_open(ctx) == 0); + assert(ave_cmd_start(ctx) == 0); +} + +static void frame(struct ave_ctx *ctx, unsigned int slot, bool force) +{ + struct vb2_v4l2_buffer src; + + assert(ave_cmd_process(ctx, slot, &src, 0x10012340000ULL, + ctx->sess->p.bitrate, force) == 0); +} + +#ifndef TEST_BASELINE +static void facts(void) +{ + struct ave_dev dev = { .variant = &test_variant, + .layout = &ave_layout_26a428 }; + struct ave_session session = {}; + struct ave_ctx ctx = { .ave = &dev, .sess = &session }; + u8 reply[0x50] = {}; + + assert(!ave_layout_for_crc(0)); + assert(!ave_layout_for_crc(0xffffffff)); + assert(!ave_layout_for_crc(0x2ce7359e)); + assert(ave_layout_for_crc(0x69c54454) == &ave_layout_9003_78_0); + assert(ave_fw_check_build(&dev) == -EOPNOTSUPP); + dev.text_crc = 0x69c54454; + assert(ave_fw_check_build(&dev) == -EOPNOTSUPP); + dev.layout = ave_layout_for_crc(dev.text_crc); + assert(ave_fw_check_build(&dev) == 0); + struct ave_variant probe_only = test_variant; + + probe_only.boot_qualified = false; + probe_only.regs = NULL; + dev.variant = &probe_only; + assert(ave_fw_check_build(&dev) == -EOPNOTSUPP); + dev.variant = &test_variant; + dev.layout = &ave_layout_26a428; + + for (unsigned int codec = 1; codec <= 2; codec++) { + params(&session.p, codec, 1920, 1080); + start(&ctx); + assert(wire_len == (codec == 1 ? 0x3b40U : 0x13ee0U)); + assert(get_unaligned_le32(wire + 0x50) == 0xec000); + assert(get_unaligned_le32(wire + 0x94) == 5); + assert(get_unaligned_le32(wire + 0x918) == 5); + assert(get_unaligned_le32(wire + 0xb00) == 5); + assert(get_unaligned_le32(wire + 0xb10) == 0x10000); + assert(get_unaligned_le32(wire + 0xdbc) == 5); + assert(get_unaligned_le32(wire + 0xfe4) == 5); + assert(get_unaligned_le32(wire + 0xc0) == 1920 * 1120 / 2); + assert(get_unaligned_le32(wire + 0xd0) == 2048 * 2048 / 128); + assert(get_unaligned_le32(wire + 0x322c) == 1920); + assert(get_unaligned_le32(wire + 0x3230) == 1088); + assert(get_unaligned_le32(wire + 0x3618) == 4000000); + assert(get_unaligned_le64(wire + 0x3638) == + 0x415e848000000000ULL); /* 8e6 */ + if (codec == 1) { + assert(get_unaligned_le32(wire + 0x37d0) == + (120U << 16 | 4)); + assert(wire[0x37d4] == 68 && wire[0x37d6] == 1); + assert(get_unaligned_le32(wire + 0x3a98) == 77); + assert(get_unaligned_le32(wire + 0x3aa0) == 40); + assert(get_unaligned_le32(wire + 0x3ac0) == + (67U << 16 | 119)); + assert(wire[0x3ac7] == 1); + assert(get_unaligned_le32(wire + 0x3acc) >> 16 == 4); + } else { + assert(get_unaligned_le32(wire + 0x68) == 2); + assert(get_unaligned_le32(wire + 0x66b8) == 1920); + assert(get_unaligned_le32(wire + 0x66bc) == 1088); + assert(get_unaligned_le32(wire + 0x6708) == 2); + } + session.frames = 0; + frame(&ctx, 0, false); + assert(wire_len == (codec == 1 ? 0x6df8U : 0x78b8U)); + assert(get_unaligned_le32(wire + 0x20) == 0x13); + assert(get_unaligned_le64(wire + 0x30) == 46400000); + assert(get_unaligned_le32(wire + + (codec == 1 ? 0xdc0 : 0x1568)) == 1); + assert(get_unaligned_le32(wire + + (codec == 1 ? 0x58b8 : 0x6060)) == + 0x10000); + assert(get_unaligned_le32(wire + + (codec == 1 ? 0x58c8 : 0x6070)) == + 0x4000); + if (codec == 1) + assert(get_unaligned_le32(wire + 0x58e0) == 0x360000); + frame(&ctx, 4, false); + assert(get_unaligned_le32(wire + 0x20) == 0x17); + assert(get_unaligned_le32(wire + + (codec == 1 ? 0xdc0 : 0x1568)) == 0); + assert(get_unaligned_le64(wire + + (codec == 1 ? 0x5760 : 0x5f08)) == + 480000); + unsigned int before = submissions; + + assert(ave_cmd_process(&ctx, session.p.slots, NULL, 0, + session.p.bitrate, false) == -EINVAL); + + assert(ave_cmd_process(&ctx, 0, NULL, 0, session.p.bitrate, + true) == -EOPNOTSUPP); + assert(ave_cmd_process(&ctx, 0, NULL, 0, 12000000, false) == + -EOPNOTSUPP); + assert(submissions == before); + dev.sessions[0] = &ctx; + put_unaligned_le64(session.client, reply + 0x10); + put_unaligned_le32(0x17, reply + 0x20); + struct ave_reply decoded; + + assert(dev.layout->abi->parse_reply(dev.layout->abi, reply, + sizeof(reply), + &decoded) == 0); + ave_sess_reply(&dev, &decoded); + assert(!session.slot[4].busy && session.slot[4].done); + assert(ctx.ready == (int)codec); + dev.sessions[0] = NULL; + ave_sess_free_bufs(&dev, &session); + memset(&session, 0, sizeof(session)); + } + for (unsigned int codec = 1; codec <= 2; codec++) { + params(&session.p, codec, 1920, 1080); + session.p.rc_mode = 3; + session.p.slots = 7; + session.p.initial_qp = 28; + session.p.max_qp = 48; + session.p.bitrate = codec == 1 ? 14100000 : 7050000; + session.p.limit_bytes = 0; + session.p.coded_cap = 0x458000; + start(&ctx); + assert(get_unaligned_le32(wire + 0x918) == 7); + assert(get_unaligned_le32(wire + 0xb00) == 7); + assert(get_unaligned_le32(wire + 0x1868) == 3); + assert(get_unaligned_le32(wire + 0x1888) == 0xa4000); + assert(get_unaligned_le32(wire + 0x1898) == 0xa4000); + assert(get_unaligned_le32(wire + 0x988) == 0x458000); + assert(get_unaligned_le32(wire + 0x3620) == 28); + assert(get_unaligned_le32(wire + 0x3734) == 0); + assert(get_unaligned_le32(wire + 0x36d0) == + (codec == 1 ? 0x10600U : 0x10700U)); + if (codec == 1) { + assert(wire[0x3830] == 0); + } else { + assert(get_unaligned_le32(wire + 0x3708) == 1); + assert(get_unaligned_le32(wire + 0x37c8) == 1); + assert(get_unaligned_le32(wire + 0x9e38) == 0x100); + } + frame(&ctx, 6, false); + assert(get_unaligned_le32(wire + 0x20) == 0x19); + if (codec == 1) { + assert(get_unaligned_le64(wire + 0x58d8) == 0); + assert(get_unaligned_le32(wire + 0x58e0) == 0); + } + ave_sess_free_bufs(&dev, &session); + memset(&session, 0, sizeof(session)); + } + params(&session.p, 2, 1920, 1080); + session.p.bit_depth = 10; + session.p.profile = 2; + session.p.min_qp = -12; + session.p.stride *= 2; + session.p.coded_cap = session.p.coded_cap * 5 / 4; + start(&ctx); + assert(get_unaligned_le32(wire + 0xa0) == 0x290400); + assert(get_unaligned_le32(wire + 0xc0) == 0x148200); + assert(get_unaligned_le32(wire + 0x1bc0) == 0x47b800); + assert(get_unaligned_le32(wire + 0x35e8) == 0x10040); + assert(get_unaligned_le32(wire + 0x3a60) == 0x01000101); + assert(get_unaligned_le32(wire + 0x6530) == 0x01000000); + assert(get_unaligned_le32(wire + 0x66dc) == 2); + ave_sess_free_bufs(&dev, &session); + memset(&session, 0, sizeof(session)); + + params(&session.p, 1, 1366, 768); + session.p.profile = 6; + session.p.high = true; + start(&ctx); + assert(get_unaligned_le32(wire + 0x3a98) == 100); + assert(wire[0x3b0d] == 1 && wire[0x3823] == 1 && wire[0x3820] == 1); + assert(get_unaligned_le32(wire + 0x3ac8) >> 16 == 5); + ave_sess_free_bufs(&dev, &session); + params(&session.p, 1, 1920, 1080); + session.p.bframes = false; + session.p.profile = 2; + session.p.refs = 2; + session.p.slots = 3; + start(&ctx); + frame(&ctx, 2, false); + assert(get_unaligned_le32(wire + 0x20) == 0x15); + assert(get_unaligned_le32(wire + 0x2c) == 0x140); + ave_sess_free_bufs(&dev, &session); + + assert(alloc_count == free_count); + puts("ok: CRC/boot gates, T6040/CQ/Main10, bounds, replies, refusal, allocations"); +} +#endif + +int main(int argc, char **argv) +{ + struct ave_dev dev = { + .variant = &test_variant, +#ifdef TEST_LAYOUT + .layout = &ave_layout_9003_78_0, +#endif + }; + struct ave_session session = {}; + struct ave_ctx ctx = { .ave = &dev, .sess = &session }; + + if (argc > 1 && !strcmp(argv[1], "--matrix")) { + /* Byte-for-byte regression input, including addresses and headers. */ + static const u32 geometry[][2] = { + { 320, 240 }, { 1280, 720 }, { 1366, 768 }, + { 1920, 1080 }, { 2408, 1506 }, { 3840, 2160 }, + }; + static const u32 rates[] = { 1000000, 4000000, 12000000 }; + + for (unsigned int i = 0; i < 288; i++) { + u32 codec = i < 216 ? 1 : 2; + u32 n = codec == 1 ? i : i - 216; + u32 mode = n % 2 ? 4 : 2; + + n /= 2; + u32 rate = rates[n % 3]; + + n /= 3; + u32 b = n % 2; + + n /= 2; + u32 profile = codec == 1 ? 2 + 2 * (n % 3) : 1; + + if (codec == 1) + n /= 3; + params(&session.p, codec, geometry[n][0], + geometry[n][1]); + session.p.profile = profile; + session.p.high = codec == 1 && profile == 6; + session.p.bframes = b; + session.p.refs = b ? 5 : 2; + session.p.slots = b ? 5 : 3; + session.p.reorder_off = !b; + session.p.bitrate = rate; + session.p.rc_mode = mode; + session.p.limit_bytes = mode == 4 ? rate / 4 : 0; + start(&ctx); + assert(fwrite(wire, 1, wire_len, stdout) == wire_len); + for (unsigned int slot = 0; slot < session.p.slots; + slot++) { + frame(&ctx, slot, slot == 1); + assert(fwrite(wire, 1, wire_len, stdout) == + wire_len); + } + ave_sess_free_bufs(&dev, &session); + memset(&session, 0, sizeof(session)); + } + return 0; + } + + if (argc > 1 && !strcmp(argv[1], "--case")) { + /* An optional synthetic case for offline, private capture comparison. */ + assert(argc == 20); +#ifndef TEST_BASELINE + if (atoi(argv[2])) + dev.layout = &ave_layout_26a428; +#endif + params(&session.p, atoi(argv[3]), atoi(argv[4]), atoi(argv[5])); + session.p.profile = atoi(argv[6]); + session.p.high = session.p.profile == 6; + session.p.level = atoi(argv[7]); + session.p.bitrate = atoi(argv[8]); + session.p.rc_mode = atoi(argv[9]); + session.p.bframes = atoi(argv[10]); + session.p.refs = atoi(argv[11]); + session.p.slots = atoi(argv[12]); + session.p.gop = atoi(argv[13]); + session.p.fps_int = atoi(argv[14]); + session.p.fps_num = session.p.fps_int; + session.p.stride = atoi(argv[15]); + session.p.reorder_off = atoi(argv[16]); + session.p.intra_only = session.p.gop == 1 && + session.p.codec == 1; + session.p.limit_bytes = + session.p.rc_mode == 4 ? session.p.bitrate / 4 : 0; +#ifndef TEST_BASELINE + session.p.initial_qp = atoi(argv[17]); + session.p.bit_depth = atoi(argv[18]); + if (session.p.bit_depth == 10) + session.p.min_qp = -12; + if (session.p.bit_depth == 10) + session.p.coded_cap = session.p.coded_cap * 5 / 4; + if (session.p.rc_mode == 3) { + session.p.max_qp = 48; + session.p.coded_cap = + session.p.initial_qp == 40 ? 0x3a4000 : + session.p.initial_qp == 28 ? 0x458000 : + 0x5fc000; + } +#endif + start(&ctx); + if (atoi(argv[19])) { + session.frames = 0; + frame(&ctx, 0, false); + } + assert(fwrite(wire, 1, wire_len, stdout) == wire_len); + return 0; + } +#ifndef TEST_BASELINE + facts(); +#else + (void)test_regs; +#endif + /* Exercise alignment, profile/GOP/rate combinations under sanitizers. */ + static const u32 sizes[][2] = { + { 320, 240 }, { 640, 480 }, { 1280, 720 }, { 1366, 768 }, + { 1920, 1080 }, { 2408, 1506 }, { 3840, 2160 }, + }; + for (unsigned int c = 1; c <= 2; c++) + for (unsigned int i = 0; i < ARRAY_SIZE(sizes); i++) { + params(&session.p, c, sizes[i][0], sizes[i][1]); + start(&ctx); + for (unsigned int s = 0; s < session.p.slots; s++) + frame(&ctx, s, true); + ave_sess_free_bufs(&dev, &session); + memset(&session, 0, sizeof(session)); + } + assert(alloc_count == free_count); + puts("ok: J700 builder and allocation alignment matrix"); + return 0; +} diff --git a/drivers/media/platform/apple/ave/tests/vb2-boundary.h b/drivers/media/platform/apple/ave/tests/vb2-boundary.h new file mode 100644 index 0000000000000..92a4f517b4e07 --- /dev/null +++ b/drivers/media/platform/apple/ave/tests/vb2-boundary.h @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* DMA-contig boundary; real AVE memory operations run unchanged. */ +typedef unsigned int refcount_t; +static void refcount_set(refcount_t *r, unsigned int n) +{ + *r = n; +} +static void refcount_inc(refcount_t *r) +{ + assert(*r); + ++*r; +} +static bool refcount_dec_and_test(refcount_t *r) +{ + assert(*r); + return !--*r; +} +struct dma_buf { unsigned int refs; }; +struct vm_area_struct { int unused; }; +struct vb2_mem_ops { + void *(*alloc)(struct vb2_buffer *vb, struct device *dev, unsigned long size); + void (*put)(void *priv); + struct dma_buf *(*get_dmabuf)(struct vb2_buffer *vb, void *priv, unsigned long flags); + void *(*cookie)(struct vb2_buffer *vb, void *priv); + void *(*vaddr)(struct vb2_buffer *vb, void *priv); + int (*mmap)(void *priv, struct vm_area_struct *vma); + void (*prepare)(void *priv); + void (*finish)(void *priv); + int (*map_dmabuf)(void *priv); + void (*unmap_dmabuf)(void *priv); + void *(*attach_dmabuf)(struct vb2_buffer *vb, struct device *dev, + struct dma_buf *dbuf, unsigned long size); + void (*detach_dmabuf)(void *priv); + unsigned int (*num_users)(void *priv); +}; +static void get_dma_buf(struct dma_buf *b) +{ + assert(b->refs); + b->refs++; +} +static void dma_buf_put(struct dma_buf *b) +{ + assert(b->refs); + b->refs--; +} +static struct device *get_device(struct device *d) +{ + d->dma_refs++; + return d; +} +static void put_device(struct device *d) +{ + assert(d->dma_refs); + d->dma_refs--; +} +#define ERR_PTR(e) ((void *)(intptr_t)(e)) +#define IS_ERR(p) ((uintptr_t)(p) >= (uintptr_t)-4095) +#define kzalloc_obj(p) calloc(1, sizeof(p)) +struct test_mapping { + bool firmware_visible, mapped; + dma_addr_t dma; +}; +static bool stop_seen, power_seen, dma_alloc_fail, dma_attach_fail, dma_map_fail; +static unsigned int mapping_frees, mapping_unmaps, mapping_detaches; +static void *test_dma_alloc(struct vb2_buffer *vb, struct device *dev, unsigned long size) +{ + struct test_mapping *m = calloc(1, sizeof(*m)); + + assert(m); + if (dma_alloc_fail) { + free(m); + return ERR_PTR(-ENOMEM); + } + m->dma = (dma_addr_t)(uintptr_t)m; + return m; +} +static void test_dma_put(void *priv) +{ + struct test_mapping *m = priv; + + /* Every command-referenced mapping survives until stop AND DART reset. */ + assert(!m->firmware_visible || (stop_seen && power_seen)); + mapping_frees++; + free(m); +} +static void *test_dma_attach(struct vb2_buffer *vb, struct device *dev, + struct dma_buf *dbuf, unsigned long size) +{ + return dma_attach_fail ? ERR_PTR(-ENOMEM) : test_dma_alloc(vb, dev, size); +} +static int test_dma_map(void *priv) +{ + struct test_mapping *m = priv; + + assert(!m->mapped); + if (dma_map_fail) + return -EIO; + m->mapped = true; + return 0; +} +static void test_dma_unmap(void *priv) +{ + struct test_mapping *m = priv; + + assert(m->mapped); + assert(!m->firmware_visible || (stop_seen && power_seen)); + mapping_unmaps++; + m->mapped = false; +} +static void test_dma_detach(void *priv) +{ + struct test_mapping *m = priv; + + assert(!m->mapped); + mapping_detaches++; + test_dma_put(priv); +} +static void *test_dma_cookie(struct vb2_buffer *vb, void *priv) +{ + return &((struct test_mapping *)priv)->dma; +} +static void *test_dma_vaddr(struct vb2_buffer *vb, void *priv) +{ + return priv; +} +static int test_dma_mmap(void *priv, struct vm_area_struct *vma) +{ + return 0; +} +static unsigned int test_dma_users(void *priv) +{ + return 1; +} +static void test_dma_sync(void *priv) { } +static struct dma_buf *test_dma_export(struct vb2_buffer *vb, void *priv, unsigned long flags) +{ + return NULL; +} +static const struct vb2_mem_ops vb2_dma_contig_memops = { + .alloc = test_dma_alloc, .put = test_dma_put, + .attach_dmabuf = test_dma_attach, .detach_dmabuf = test_dma_detach, + .map_dmabuf = test_dma_map, .unmap_dmabuf = test_dma_unmap, + .cookie = test_dma_cookie, .vaddr = test_dma_vaddr, .mmap = test_dma_mmap, + .num_users = test_dma_users, .prepare = test_dma_sync, .finish = test_dma_sync, + .get_dmabuf = test_dma_export, +}; diff --git a/drivers/media/platform/apple/isp/Kconfig b/drivers/media/platform/apple/isp/Kconfig index 64f84a2471f84..d5c6ca690476a 100644 --- a/drivers/media/platform/apple/isp/Kconfig +++ b/drivers/media/platform/apple/isp/Kconfig @@ -15,9 +15,11 @@ config VIDEO_APPLE_ISP Driver for the image signal processor that runs the built-in camera of Apple silicon machines. The ISP firmware is loaded by the bootloader. When a stream starts, the driver loads the - calibration file for the detected camera module, + sensor tuning file for the detected camera model, apple/isp__XX.dat, from the firmware search - path; if it is missing, the stream runs without calibration. + path; if it is missing, the stream runs without sensor tuning. + Invalid files and firmware upload errors fail the stream setup. + These setfiles do not contain per-module factory calibration. config VIDEO_APPLE_ISP_KUNIT_TEST bool "KUnit tests for the Apple ISP driver" if !KUNIT_ALL_TESTS diff --git a/drivers/media/platform/apple/isp/isp-cam-test.c b/drivers/media/platform/apple/isp/isp-cam-test.c index 0d699add95bfe..b97fa7b06c0b4 100644 --- a/drivers/media/platform/apple/isp/isp-cam-test.c +++ b/drivers/media/platform/apple/isp/isp-cam-test.c @@ -69,6 +69,8 @@ static const struct isp_test_command isp_test_preview_commands[] = { struct isp_test_command_trace { unsigned int count; unsigned int fail_at; + int setfile_error; + unsigned int setfile_calls; }; static int isp_test_send(struct apple_isp *isp, void *args, u32 insize, @@ -77,6 +79,7 @@ static int isp_test_send(struct apple_isp *isp, void *args, u32 insize, struct kunit *test = kunit_get_current_test(); struct isp_test_command_trace *trace = test->priv; const struct isp_test_command *expected; + u32 expected_outsize; if (trace->count >= ARRAY_SIZE(isp_test_preview_commands)) { KUNIT_FAIL(test, "unexpected extra command"); @@ -84,7 +87,9 @@ static int isp_test_send(struct apple_isp *isp, void *args, u32 insize, } expected = &isp_test_preview_commands[trace->count++]; KUNIT_EXPECT_EQ(test, insize, expected->size); - KUNIT_EXPECT_EQ(test, outsize, expected->words[1] == 0x117 ? insize : 0); + expected_outsize = expected->words[1] == 0x117 || + (expected->words[1] == 0x24 && !isp_is_p010(isp)) ? insize : 0; + KUNIT_EXPECT_EQ(test, outsize, expected_outsize); if (insize == expected->size) KUNIT_EXPECT_MEMEQ(test, args, expected->words, insize); if (trace->count == trace->fail_at) @@ -158,6 +163,53 @@ static void isp_test_preview_command_failures(struct kunit *test) } } +static int isp_test_load_setfile(struct apple_isp *isp, u32 ch) +{ + struct kunit *test = kunit_get_current_test(); + struct isp_test_command_trace *trace = test->priv; + + trace->setfile_calls++; + return trace->setfile_error; +} + +static void isp_test_setfile_failure_stops_setup(struct kunit *test) +{ + static const int errors[] = { -EINVAL, -EIO, -ETIME, -EINTR, -ENOMEM }; + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_test_command_trace trace; + + isp_get_current_format(isp)->pixelformat = V4L2_PIX_FMT_NV12; + kunit_activate_static_stub(test, isp_ch_load_setfile, isp_test_load_setfile); + test->priv = &trace; + for (unsigned int i = 0; i < ARRAY_SIZE(errors); i++) { + trace = (struct isp_test_command_trace){ .setfile_error = errors[i] }; + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), errors[i]); + /* Only the flicker command preceded the rejected tuning. */ + KUNIT_EXPECT_EQ(test, trace.count, 1); + KUNIT_EXPECT_EQ(test, trace.setfile_calls, 1); + } +} + +static void isp_test_optional_setfile_continues_setup(struct kunit *test) +{ + static const int results[] = { 0, -ENOENT }; + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_test_command_trace trace; + + isp_get_current_format(isp)->pixelformat = V4L2_PIX_FMT_NV12; + kunit_activate_static_stub(test, isp_ch_load_setfile, isp_test_load_setfile); + test->priv = &trace; + for (unsigned int i = 0; i < ARRAY_SIZE(results); i++) { + trace = (struct isp_test_command_trace){ + .setfile_error = results[i], .fail_at = 2, + }; + /* The SBS command after tuning, rather than tuning, fails. */ + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), -ETIME); + KUNIT_EXPECT_EQ(test, trace.count, 2); + KUNIT_EXPECT_EQ(test, trace.setfile_calls, 1); + } +} + static int isp_test_config_reply(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) { @@ -327,7 +379,121 @@ static void isp_test_command_ack_failures(struct kunit *test) KUNIT_EXPECT_EQ(test, state.calls, 0); } +static void isp_test_call_crop(struct kunit *test) +{ + struct isp_preset source = { + .index = 7, .input_dim = { 1936, 1096 }, + .output_dim = { 1920, 1080 }, .crop_offset = { 8, 8 }, + .crop_size = { 1920, 1080 }, .p010 = true, + }; + struct isp_preset result; + + KUNIT_ASSERT_TRUE(test, + isp_size_from_crop(&source, (struct coord){ 640, 480 }, &result)); + KUNIT_EXPECT_EQ(test, result.index, 7); + KUNIT_EXPECT_TRUE(test, result.p010); + KUNIT_EXPECT_EQ(test, result.crop_offset.x, 248); + KUNIT_EXPECT_EQ(test, result.crop_offset.y, 8); + KUNIT_EXPECT_EQ(test, result.crop_size.x, 1440); + KUNIT_EXPECT_EQ(test, result.crop_size.y, 1080); + KUNIT_EXPECT_EQ(test, result.output_dim.x, 640); + KUNIT_EXPECT_EQ(test, result.output_dim.y, 480); +} + +static void isp_test_call_crop_bounds(struct kunit *test) +{ + struct isp_preset source = { + .input_dim = { 1920, 2160 }, .output_dim = { 1920, 1080 }, + .crop_size = { 1920, 1080 }, + }; + struct isp_preset result; + static const struct coord rejected[] = { + { 1080, 1920 }, { 1760, 1328 }, { 1552, 1552 }, + { 1922, 1080 }, { 0, 480 }, { 641, 480 }, + }; + + /* Stacked input rows do not make a portrait or a larger square. */ + for (unsigned int i = 0; i < ARRAY_SIZE(rejected); i++) + KUNIT_EXPECT_FALSE(test, isp_size_from_crop(&source, rejected[i], &result)); + source.crop_offset.x = 2; + KUNIT_EXPECT_FALSE(test, + isp_size_from_crop(&source, (struct coord){ 640, 480 }, &result)); + source.crop_offset.x = U32_MAX; + KUNIT_EXPECT_FALSE(test, + isp_size_from_crop(&source, (struct coord){ 640, 480 }, &result)); +} + +static void isp_test_call_size_table(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_preset source = { + .index = 7, .input_dim = { 1920, 1920 }, + .output_dim = { 1920, 1920 }, .crop_size = { 1920, 1920 }, + }; + + isp->presets = &source; + isp->num_presets = 1; + KUNIT_ASSERT_EQ(test, isp_add_call_sizes(isp), 0); + KUNIT_EXPECT_EQ(test, isp->num_presets, ARRAY_SIZE(isp_call_sizes) + 1); + KUNIT_EXPECT_EQ(test, isp->presets[0].output_dim.x, 1920); + KUNIT_EXPECT_EQ(test, isp->presets[0].output_dim.y, 1920); + for (unsigned int i = 0; i < ARRAY_SIZE(isp_call_sizes); i++) { + struct isp_preset *preset = &isp->presets[i + 1]; + + KUNIT_EXPECT_EQ(test, preset->output_dim.x, isp_call_sizes[i].x); + KUNIT_EXPECT_EQ(test, preset->output_dim.y, isp_call_sizes[i].y); + KUNIT_EXPECT_EQ(test, preset->index, 7); + KUNIT_EXPECT_FALSE(test, preset->p010); + KUNIT_EXPECT_EQ(test, preset->crop_size.x * preset->output_dim.y, + preset->crop_size.y * preset->output_dim.x); + KUNIT_EXPECT_EQ(test, (preset->crop_offset.x | preset->crop_offset.y | + preset->crop_size.x | preset->crop_size.y) & 1, 0); + KUNIT_EXPECT_LE(test, preset->crop_offset.x + preset->crop_size.x, 1920); + KUNIT_EXPECT_LE(test, preset->crop_offset.y + preset->crop_size.y, 1920); + } + /* Rebuilding preserves order and never duplicates an existing size. */ + KUNIT_ASSERT_EQ(test, isp_add_call_sizes(isp), 0); + KUNIT_EXPECT_EQ(test, isp->num_presets, ARRAY_SIZE(isp_call_sizes) + 1); +} + +static void isp_test_call_size_capabilities(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_preset sources[] = { + { .index = 5, .input_dim = { 1920, 2160 }, + .output_dim = { 1920, 1080 }, .crop_size = { 1920, 1080 }, .p010 = true }, + { .index = 5, .input_dim = { 1920, 2160 }, + .output_dim = { 1280, 720 }, .crop_size = { 1280, 720 }, .p010 = true }, + { .index = 9, .input_dim = { 1536, 3024 }, + .output_dim = { 1536, 3024 }, .crop_size = { 1536, 3024 }, .p010 = true }, + }; + + isp->presets = sources; + isp->num_presets = ARRAY_SIZE(sources); + KUNIT_ASSERT_EQ(test, isp_add_call_sizes(isp), 0); + KUNIT_EXPECT_EQ(test, isp->num_presets, 9); + for (unsigned int i = 0; i < ARRAY_SIZE(sources); i++) + KUNIT_EXPECT_MEMEQ(test, &isp->presets[i], &sources[i], sizeof(sources[i])); + for (unsigned int i = ARRAY_SIZE(sources); i < isp->num_presets; i++) { + struct isp_preset *preset = &isp->presets[i]; + + KUNIT_EXPECT_TRUE(test, preset->p010); + KUNIT_EXPECT_FALSE(test, preset->output_dim.x == 1760 && + preset->output_dim.y == 1328); + KUNIT_EXPECT_FALSE(test, preset->output_dim.x == 1552 && + preset->output_dim.y == 1552); + } + KUNIT_EXPECT_EQ(test, isp->presets[8].output_dim.x, 1536); + KUNIT_EXPECT_EQ(test, isp->presets[8].output_dim.y, 1536); +} + static struct kunit_case isp_cam_test_cases[] = { + KUNIT_CASE(isp_test_call_crop), + KUNIT_CASE(isp_test_call_crop_bounds), + KUNIT_CASE(isp_test_call_size_table), + KUNIT_CASE(isp_test_call_size_capabilities), + KUNIT_CASE(isp_test_setfile_failure_stops_setup), + KUNIT_CASE(isp_test_optional_setfile_continues_setup), KUNIT_CASE(isp_test_command_ack_failures), KUNIT_CASE(isp_test_sensor_geometry), KUNIT_CASE(isp_test_sensor_config_list), diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index aa9dd05280255..67691cc8844ac 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -2,7 +2,9 @@ /* Copyright 2023 Eileen Yoon */ #include +#include #include +#include #include "isp-cam.h" #include "isp-cmd.h" @@ -332,6 +334,107 @@ static int isp_ch_cache_sensor_configs(struct apple_isp *isp, u32 ch, u32 count) return err; } +/* Common call sizes, offered only when a known processed crop contains them. */ +static const struct coord isp_call_sizes[] = { + { 640, 480 }, { 1280, 720 }, { 1920, 1080 }, { 1080, 1920 }, + { 1760, 1328 }, { 1328, 1760 }, { 1552, 1552 }, + { 1280, 960 }, { 960, 1280 }, +}; + +/* + * CH_CROP[_SCL1]_SET takes x/y/width/height independently of the output + * dimensions in CH_OUTPUT_CONFIG[_SCL1]_SET. Stay within an established + * processed crop, rather than the sensor input (which may contain stacked + * rows). Use the largest exact-aspect crop, centred on the chroma grid, + * with no upscaling. Neither command reports a scaler limit or rotation. + */ +static bool isp_size_from_crop(const struct isp_preset *source, + struct coord size, struct isp_preset *preset) +{ + u32 divisor, x, y, scale; + + if (!size.x || !size.y || (size.x | size.y) & 1 || + !source->crop_size.x || !source->crop_size.y || + (source->crop_offset.x | source->crop_offset.y) & 1 || + source->crop_size.x > 16384 || source->crop_size.y > 16384 || + source->crop_offset.x > source->input_dim.x || + source->crop_size.x > source->input_dim.x - source->crop_offset.x || + source->crop_offset.y > source->input_dim.y || + source->crop_size.y > source->input_dim.y - source->crop_offset.y) + return false; + + divisor = gcd(size.x, size.y); + x = size.x / divisor; + y = size.y / divisor; + scale = round_down(min(source->crop_size.x / x, + source->crop_size.y / y), 2); + if (scale < divisor) + return false; + + *preset = *source; + preset->output_dim = size; + preset->crop_size = (struct coord){ x * scale, y * scale }; + preset->crop_offset.x += round_down((source->crop_size.x - x * scale) / 2, 2); + preset->crop_offset.y += round_down((source->crop_size.y - y * scale) / 2, 2); + return true; +} + +static unsigned int isp_append_size(struct isp_preset *presets, unsigned int num, + unsigned int sources, struct coord size) +{ + struct isp_preset candidate, best; + u32 area, best_area = 0; + + for (unsigned int i = 0; i < num; i++) + if (presets[i].output_dim.x == size.x && + presets[i].output_dim.y == size.y) + return num; + + /* Never use a generated size as evidence for another generated size. */ + for (unsigned int i = 0; i < sources; i++) { + if (!isp_size_from_crop(&presets[i], size, &candidate)) + continue; + area = candidate.crop_size.x * candidate.crop_size.y; + if (area > best_area) { + best = candidate; + best_area = area; + } + } + if (best_area) + presets[num++] = best; + return num; +} + +static int isp_add_call_sizes(struct apple_isp *isp) +{ + unsigned int sources = isp->num_presets, num = sources; + struct isp_preset *presets; + struct isp_preset candidate; + u32 square = 0; + + presets = devm_kcalloc(isp->dev, sources + ARRAY_SIZE(isp_call_sizes) + 1, + sizeof(*presets), GFP_KERNEL); + if (!presets) + return -ENOMEM; + memcpy(presets, isp->presets, sources * sizeof(*presets)); + + for (unsigned int i = 0; i < ARRAY_SIZE(isp_call_sizes); i++) + num = isp_append_size(presets, num, sources, isp_call_sizes[i]); + + /* A smaller sensor can still offer a square without enlarging pixels. */ + for (unsigned int i = 0; i < sources; i++) { + u32 side = round_down(min(presets[i].crop_size.x, + presets[i].crop_size.y), 2); + + if (isp_size_from_crop(&presets[i], (struct coord){ side, side }, &candidate)) + square = max(square, side); + } + num = isp_append_size(presets, num, sources, (struct coord){ square, square }); + isp->presets = presets; + isp->num_presets = num; + return 0; +} + static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) { struct isp_format *fmt = isp_get_format(isp, ch); @@ -381,6 +484,8 @@ static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) err = isp_check_presets(isp, args->num_presets); if (!err) err = isp_ch_cache_sensor_configs(isp, ch, args->num_presets); + if (!err) + err = isp_add_call_sizes(isp); exit: kfree(args); @@ -513,14 +618,20 @@ static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) u32 magic; int err; + KUNIT_STATIC_STUB_REDIRECT(isp_ch_load_setfile, isp, ch); + if (WARN_ON_ONCE(setfile->size > isp->data_surf->size)) return -ENOSPC; err = firmware_request_nowarn(&fw, setfile->path, isp->dev); if (err) { - dev_warn_once(isp->dev, - "setfile '%s' not loaded (%d), streaming without calibration\n", - setfile->path, err); + if (err == -ENOENT) + dev_warn_once(isp->dev, + "setfile '%s' missing, streaming without sensor tuning\n", + setfile->path); + else + dev_err(isp->dev, "failed to read setfile '%s': %d\n", + setfile->path, err); return err; } @@ -546,8 +657,16 @@ static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) if (isp->hw->gen == ISP_GEN_T6040) isp_setfile_fixup_h17(isp->data_surf->virt, len); - return isp_cmd_ch_set_file_load(isp, ch, isp->data_surf->iova, - setfile->size); + err = isp_cmd_ch_set_file_load(isp, ch, isp->data_surf->iova, + setfile->size); + if (err) + dev_err(isp->dev, "ch %u: setfile '%s' load failed: %d\n", + ch, setfile->path, err); + else + dev_dbg(isp->dev, "ch %u: loaded sensor tuning '%s' (0x%zx bytes)\n", + ch, setfile->path, setfile->size); + + return err; } static int isp_ch_configure_frame_rate(struct apple_isp *isp, u32 ch) @@ -614,11 +733,12 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) goto select_config; /* - * The setfile isn't requisite but then we don't get calibration. If - * loading it was interrupted by a signal, give up on the stream. + * Missing sensor tuning is optional. A present but invalid setfile or + * a failed firmware upload must not silently select firmware defaults. + * This model-wide tuning is separate from per-module factory data. */ err = isp_ch_load_setfile(isp, ch); - if (err == -EINTR) + if (err && err != -ENOENT) return err; if (isp->hw->lpdp) { diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index f8d2334fec7e2..93a21e901ff6a 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -827,6 +827,7 @@ static const struct apple_isp_hw apple_isp_hw_t8140 = { .boot_mode = 1, .capture_meta_size = ISP_CAPTURE_META_SIZE_T8140, .resident_fw = true, + .meta_exposure = true, }; /* The same for T6040, from the J616s ADT's DART address filter. */ @@ -885,6 +886,7 @@ static const struct apple_isp_hw apple_isp_hw_t6040 = { .boot_mode = 1, .shared_size = 0xde600000, .resident_fw = true, + .meta_exposure = true, }; static const struct of_device_id apple_isp_of_match[] = { diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index a5173be862dd6..956754f5ef87f 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -201,6 +201,9 @@ struct apple_isp_hw { /* size of a capture metadata buffer, 0 for no such pool */ u32 capture_meta_size; + /* metadata buffers carry the frame's exposure report (H17 firmware) */ + bool meta_exposure; + /* * The firmware can be started only once per system boot, so it is * started at probe and kept running while the driver is bound. @@ -302,6 +305,12 @@ struct apple_isp { struct v4l2_device v4l2_dev; struct v4l2_ctrl_handler ctrl_handler; struct v4l2_ctrl *exposure_priority; + /* read-only exposure report of the last frame, NULL without one */ + struct v4l2_ctrl *gain; + struct v4l2_ctrl *exposure; + /* last reported total gain (1/256 steps) and exposure (us) */ + u32 frame_gain; + u32 frame_exposure_us; struct vb2_queue vbq; struct mutex video_lock; unsigned int sequence; diff --git a/drivers/media/platform/apple/isp/isp-v4l2-test.c b/drivers/media/platform/apple/isp/isp-v4l2-test.c index 398e8e92356ab..88f428b3b0c27 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2-test.c +++ b/drivers/media/platform/apple/isp/isp-v4l2-test.c @@ -348,7 +348,53 @@ static void isp_test_imx958_last_lease_requeue(struct kunit *test) } } +/* The exposure report is found through the section table, never past it. */ +static void isp_test_meta_exposure(struct kunit *test) +{ + static const struct apple_isp_hw hw = { .meta_exposure = true }; + static const struct apple_isp_hw hw_none = { }; + struct apple_isp *isp = isp_test_alloc(test); + struct isp_surf meta = { .size = 0x2000 }; + u8 *virt = kunit_kzalloc(test, meta.size, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, virt); + meta.virt = virt; + isp->hw = &hw; + isp->frame_gain = 256; + + /* fourth table entry: the section at 0x1870 */ + put_unaligned_le32(0x1870, virt + 0x1c); + put_unaligned_le32(32475, virt + 0x1870); + put_unaligned_le16(0x0800, virt + 0x1874); + put_unaligned_le16(3253, virt + 0x1876); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_exposure_us, 32475); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + + /* a section that would end past the buffer is ignored */ + put_unaligned_le32(meta.size - 4, virt + 0x1c); + put_unaligned_le32(1000, virt + meta.size - 4); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + + /* so is an unwritten section */ + put_unaligned_le32(0x1000, virt + 0x1c); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_exposure_us, 32475); + + /* and firmware without the report */ + put_unaligned_le32(0x1870, virt + 0x1c); + put_unaligned_le16(600, virt + 0x1876); + isp->hw = &hw_none; + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + isp->hw = &hw; + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 600); +} + static struct kunit_case isp_v4l2_test_cases[] = { + KUNIT_CASE(isp_test_meta_exposure), KUNIT_CASE(isp_test_imx958_last_lease_requeue), KUNIT_CASE(isp_test_p010_report_defers_completion), KUNIT_CASE(isp_test_p010_mode_selection), diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 87f28169e05fd..3c58b56b42aa3 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -24,6 +25,8 @@ #define ISP_BUFFER_TIMEOUT msecs_to_jiffies(1500) #define ISP_STRIDE_ALIGNMENT 64 +static void isp_vb2_set_min_buffers(struct apple_isp *isp); + /* Capture rates offered to userspace, in frames per second */ static const unsigned int isp_frame_rates[] = { 30, 25, 24, 20, 15 }; @@ -66,6 +69,45 @@ static bool isp_p010_buffer_matches(const struct isp_format *fmt, !desc->iovas[2] && !desc->iovas[3] && desc->tag == buf->tag; } +/* + * A metadata buffer starts with a table of section offsets. On H17 firmware + * the fourth section opens with the frame's exposure: the integration time in + * microseconds (le32 at 0) and the total sensor gain in 1/256 steps, 256 for + * unity (le16 at 6). Observed on T6040 and T8140 with AE converging under a + * fixed scene and with the frame rate capping the integration time. + */ +#define ISP_META_SECTIONS 0x10 +#define ISP_META_SECTION_EXPOSURE 3 +#define ISP_META_EXPOSURE_TIME 0x0 +#define ISP_META_EXPOSURE_GAIN 0x6 +#define ISP_META_EXPOSURE_SIZE 0x8 + +static void isp_meta_read_exposure(struct apple_isp *isp, + const struct isp_surf *meta) +{ + const u8 *virt = meta->virt; + u32 offset, time; + u16 gain; + + if (!isp->hw->meta_exposure || !virt) + return; + + offset = get_unaligned_le32(virt + ISP_META_SECTIONS + + 4 * ISP_META_SECTION_EXPOSURE); + if (offset < ISP_META_SECTIONS || offset > meta->size || + meta->size - offset < ISP_META_EXPOSURE_SIZE) + return; + + time = get_unaligned_le32(virt + offset + ISP_META_EXPOSURE_TIME); + gain = get_unaligned_le16(virt + offset + ISP_META_EXPOSURE_GAIN); + /* An unwritten section or one this layout does not describe */ + if (!time || !gain) + return; + + WRITE_ONCE(isp->frame_exposure_us, time); + WRITE_ONCE(isp->frame_gain, gain); +} + void ipc_bt_complete(struct apple_isp *isp) { struct isp_buffer *buf, *tmp; @@ -126,6 +168,7 @@ int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) struct isp_surf *meta = isp->meta_surfs[j]; if ((u32)bufd->iovas[0] == (u32)meta->iova) { WARN_ON(!meta->submitted); + isp_meta_read_exposure(isp, meta); if (isp_is_p010(isp)) __set_bit(j, &isp->meta_returned); else @@ -849,6 +892,7 @@ static int isp_vidioc_set_format(struct file *file, void *fh, isp_get_sp_pix_format(isp, f, fmt); isp->vbq.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + isp_vb2_set_min_buffers(isp); return 0; } @@ -1039,6 +1083,22 @@ static void isp_free_meta_surfaces(struct apple_isp *isp) } } +/* + * vb2 allocates at least min_queued_buffers + 1 buffers, and the queue may + * hold no more than the rendered pool (isp_vb2_queue_setup()). The P010 pool + * has two slots, so a P010 stream starts with one queued buffer; otherwise + * REQBUFS can never get the three buffers vb2 asks for and fails with ENOMEM. + * Only called while the queue has no buffers. + */ +static void isp_vb2_set_min_buffers(struct apple_isp *isp) +{ + struct vb2_queue *vbq = &isp->vbq; + + vbq->min_queued_buffers = min_t(unsigned int, ISP_MIN_FRAMES, + isp_num_rendered(isp) - 1); + vbq->min_reqbufs_allocation = vbq->min_queued_buffers + 1; +} + static int isp_vb2_queue_init(struct apple_isp *isp) { struct vb2_queue *vbq = &isp->vbq; @@ -1057,6 +1117,53 @@ static int isp_vb2_queue_init(struct apple_isp *isp) return vb2_queue_init(vbq); } +static int isp_g_volatile_ctrl(struct v4l2_ctrl *ctrl) +{ + struct apple_isp *isp = + container_of(ctrl->handler, struct apple_isp, ctrl_handler); + + switch (ctrl->id) { + case V4L2_CID_GAIN: + ctrl->val = READ_ONCE(isp->frame_gain); + return 0; + case V4L2_CID_EXPOSURE_ABSOLUTE: + /* in 100 us units */ + ctrl->val = DIV_ROUND_CLOSEST(READ_ONCE(isp->frame_exposure_us), + 100); + return 0; + } + + return -EINVAL; +} + +static const struct v4l2_ctrl_ops isp_exposure_ctrl_ops = { + .g_volatile_ctrl = isp_g_volatile_ctrl, +}; + +/* + * Auto exposure picks the gain and exposure of each frame. Report the last + * frame's values read-only, so that userspace can follow the scene light, for + * example to decide when to denoise. The gain is the firmware's total sensor + * gain in 1/256 steps. + */ +static void isp_init_exposure_ctrls(struct apple_isp *isp) +{ + isp->frame_gain = 256; + isp->gain = v4l2_ctrl_new_std(&isp->ctrl_handler, + &isp_exposure_ctrl_ops, V4L2_CID_GAIN, + 0, U16_MAX, 1, 256); + isp->exposure = v4l2_ctrl_new_std(&isp->ctrl_handler, + &isp_exposure_ctrl_ops, + V4L2_CID_EXPOSURE_ABSOLUTE, + 0, 10000, 1, 0); + if (isp->gain) + isp->gain->flags |= V4L2_CTRL_FLAG_READ_ONLY | + V4L2_CTRL_FLAG_VOLATILE; + if (isp->exposure) + isp->exposure->flags |= V4L2_CTRL_FLAG_READ_ONLY | + V4L2_CTRL_FLAG_VOLATILE; +} + int apple_isp_setup_video(struct apple_isp *isp) { struct video_device *vdev = &isp->vdev; @@ -1111,10 +1218,12 @@ int apple_isp_setup_video(struct apple_isp *isp) * Auto exposure is always on. By default it holds the frame rate, as * V4L2 specifies; with exposure priority it may slow down in low light. */ - v4l2_ctrl_handler_init(&isp->ctrl_handler, 1); + v4l2_ctrl_handler_init(&isp->ctrl_handler, 3); isp->exposure_priority = v4l2_ctrl_new_std(&isp->ctrl_handler, NULL, V4L2_CID_EXPOSURE_AUTO_PRIORITY, 0, 1, 1, 0); + if (isp->hw->meta_exposure) + isp_init_exposure_ctrls(isp); if (isp->ctrl_handler.error) { err = isp->ctrl_handler.error; dev_err(isp->dev, "failed to create controls: %d\n", err); diff --git a/drivers/media/platform/apple/jpeg/Kconfig b/drivers/media/platform/apple/jpeg/Kconfig new file mode 100644 index 0000000000000..f429b8970dbfc --- /dev/null +++ b/drivers/media/platform/apple/jpeg/Kconfig @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config VIDEO_APPLE_JPEG + tristate "Apple Silicon JPEG codec skeleton" + depends on VIDEO_DEV + depends on ARCH_APPLE || (COMPILE_TEST && 64BIT) + depends on OF_ADDRESS && IOMMU_API && PM + select V4L2_MEM2MEM_DEV + select VIDEOBUF2_DMA_CONTIG + help + Build the common Apple JPEG memory-to-memory driver scaffold. + Hardware revisions are not qualified yet: probe reads HWREV and + refuses all currently described hardware. No reset, interrupt + acknowledgment or DMA job is implemented, and streaming is refused. + Device tree nodes must remain disabled pending hardware qualification. + +config VIDEO_APPLE_JPEG_KUNIT_TEST + tristate "KUnit tests for Apple JPEG admission" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Test exact per-compatible HWREV matching and the empty target + allow-lists, including invalid readbacks and revisions observed + on other SoCs. Synthetic match data exercises the admission helper. + These tests do not access hardware or qualify any target revision. diff --git a/drivers/media/platform/apple/jpeg/Makefile b/drivers/media/platform/apple/jpeg/Makefile new file mode 100644 index 0000000000000..eee453f627047 --- /dev/null +++ b/drivers/media/platform/apple/jpeg/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_VIDEO_APPLE_JPEG) += apple-jpeg.o +obj-$(CONFIG_VIDEO_APPLE_JPEG_KUNIT_TEST) += apple-jpeg-test.o diff --git a/drivers/media/platform/apple/jpeg/apple-jpeg-hw.h b/drivers/media/platform/apple/jpeg/apple-jpeg-hw.h new file mode 100644 index 0000000000000..332f4313e7f34 --- /dev/null +++ b/drivers/media/platform/apple/jpeg/apple-jpeg-hw.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef APPLE_JPEG_HW_H +#define APPLE_JPEG_HW_H + +#include +#include + +/* Public MIT-licensed m1n1 register map: proxyclient/m1n1/hw/jpeg.py. */ +#define APPLE_JPEG_HWREV 0x188 +#define APPLE_JPEG_REG_SIZE 0x4000 +#define APPLE_JPEG_DMA_BASE 0x10000000000ULL +#define APPLE_JPEG_DMA_SIZE 0x100000000ULL + +struct apple_jpeg_variant { + const u32 *hwrevs; + size_t num_hwrevs; +}; + +/* + * Empty until each SoC's revision and reset/IRQ behaviour are qualified. + * Neither an ADT hw-type nor m1n1's M1 Max revision qualifies another SoC. + */ +static const struct apple_jpeg_variant apple_jpeg_t6041 = { }; +static const struct apple_jpeg_variant apple_jpeg_t8140 = { }; + +static inline bool +apple_jpeg_hwrev_supported(const struct apple_jpeg_variant *variant, u32 hwrev) +{ + size_t i; + + if (!variant || !variant->hwrevs || !hwrev || hwrev == U32_MAX) + return false; + + for (i = 0; i < variant->num_hwrevs; i++) + if (variant->hwrevs[i] == hwrev) + return true; + + return false; +} + +#endif /* APPLE_JPEG_HW_H */ diff --git a/drivers/media/platform/apple/jpeg/apple-jpeg-test.c b/drivers/media/platform/apple/jpeg/apple-jpeg-test.c new file mode 100644 index 0000000000000..651a2b182252b --- /dev/null +++ b/drivers/media/platform/apple/jpeg/apple-jpeg-test.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "apple-jpeg-hw.h" + +static void apple_jpeg_unqualified_test(struct kunit *test) +{ + const struct apple_jpeg_variant *variants[] = { + &apple_jpeg_t6041, &apple_jpeg_t8140, + }; + /* Public M1 Max revision, arbitrary revisions and invalid readbacks. */ + const u32 revisions[] = { 0xd1013, 1, 0x12345678, 0, U32_MAX }; + size_t i, j; + + for (i = 0; i < ARRAY_SIZE(variants); i++) { + KUNIT_EXPECT_EQ(test, variants[i]->num_hwrevs, (size_t)0); + for (j = 0; j < ARRAY_SIZE(revisions); j++) + KUNIT_EXPECT_FALSE(test, + apple_jpeg_hwrev_supported(variants[i], revisions[j])); + } +} + +static void apple_jpeg_exact_match_test(struct kunit *test) +{ + /* Synthetic revisions exercise the matcher without qualifying hardware. */ + const u32 revisions[] = { 0x12345678, 0x1234567a }; + const struct apple_jpeg_variant variant = { + .hwrevs = revisions, + .num_hwrevs = ARRAY_SIZE(revisions), + }; + const struct apple_jpeg_variant empty = { .hwrevs = revisions }; + + KUNIT_EXPECT_TRUE(test, apple_jpeg_hwrev_supported(&variant, revisions[0])); + KUNIT_EXPECT_TRUE(test, apple_jpeg_hwrev_supported(&variant, revisions[1])); + KUNIT_EXPECT_FALSE(test, apple_jpeg_hwrev_supported(&variant, 0x12345679)); + KUNIT_EXPECT_FALSE(test, apple_jpeg_hwrev_supported(&variant, 0)); + KUNIT_EXPECT_FALSE(test, apple_jpeg_hwrev_supported(&variant, U32_MAX)); + KUNIT_EXPECT_FALSE(test, apple_jpeg_hwrev_supported(&empty, revisions[0])); + KUNIT_EXPECT_FALSE(test, apple_jpeg_hwrev_supported(NULL, revisions[0])); +} + +static struct kunit_case apple_jpeg_cases[] = { + KUNIT_CASE(apple_jpeg_unqualified_test), + KUNIT_CASE(apple_jpeg_exact_match_test), + { } +}; + +static struct kunit_suite apple_jpeg_suite = { + .name = "apple-jpeg-admission", + .test_cases = apple_jpeg_cases, +}; + +kunit_test_suite(apple_jpeg_suite); + +MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/apple/jpeg/apple-jpeg.c b/drivers/media/platform/apple/jpeg/apple-jpeg.c new file mode 100644 index 0000000000000..0d8b637fd5a29 --- /dev/null +++ b/drivers/media/platform/apple/jpeg/apple-jpeg.c @@ -0,0 +1,543 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple Silicon JPEG mem2mem scaffold. No hardware jobs are implemented. + * Register offset provenance: Asahi Linux m1n1 (MIT), commit + * f3ac615eb1376d3bbba24cd84c414b9e41892adb, proxyclient/m1n1/hw/jpeg.py. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "apple-jpeg-hw.h" + +struct apple_jpeg { + struct device *dev; + void __iomem *regs; + const struct apple_jpeg_variant *variant; + struct mutex lock; /* Serializes video operations and queue/context state. */ + struct v4l2_device v4l2_dev; + struct video_device vdev; + struct v4l2_m2m_dev *m2m_dev; +}; + +struct apple_jpeg_ctx { + struct v4l2_fh fh; + struct apple_jpeg *jpeg; + struct v4l2_pix_format_mplane output; + struct v4l2_pix_format_mplane capture; +}; + +static struct apple_jpeg_ctx *fh_to_ctx(void *priv) +{ + return container_of(priv, struct apple_jpeg_ctx, fh); +} + +static struct v4l2_pix_format_mplane * +apple_jpeg_format(struct apple_jpeg_ctx *ctx, enum v4l2_buf_type type) +{ + if (type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) + return &ctx->output; + if (type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) + return &ctx->capture; + return NULL; +} + +/* Allocation policy for the scaffold, not a qualified hardware size limit. */ +static void apple_jpeg_normalize_format(struct v4l2_pix_format_mplane *pix) +{ + u32 width = clamp_t(u32, pix->width, 16, 8192); + u32 height = clamp_t(u32, pix->height, 16, 8192); + u32 fourcc = pix->pixelformat; + + memset(pix, 0, sizeof(*pix)); + pix->width = ALIGN(width, 16); + pix->height = ALIGN(height, 16); + pix->field = V4L2_FIELD_NONE; + pix->colorspace = V4L2_COLORSPACE_JPEG; + pix->ycbcr_enc = V4L2_YCBCR_ENC_601; + pix->quantization = V4L2_QUANTIZATION_FULL_RANGE; + pix->xfer_func = V4L2_XFER_FUNC_SRGB; + + if (fourcc == V4L2_PIX_FMT_NV12M) { + pix->pixelformat = fourcc; + pix->num_planes = 2; + pix->plane_fmt[0].bytesperline = pix->width; + pix->plane_fmt[0].sizeimage = pix->width * pix->height; + pix->plane_fmt[1].bytesperline = pix->width; + pix->plane_fmt[1].sizeimage = pix->width * pix->height / 2; + } else { + pix->pixelformat = V4L2_PIX_FMT_JPEG; + pix->num_planes = 1; + pix->plane_fmt[0].sizeimage = pix->width * pix->height * 3 + 4096; + } +} + +static int apple_jpeg_querycap(struct file *file, void *priv, + struct v4l2_capability *cap) +{ + struct apple_jpeg_ctx *ctx = fh_to_ctx(priv); + + strscpy(cap->driver, "apple-jpeg", sizeof(cap->driver)); + strscpy(cap->card, "Apple JPEG scaffold", sizeof(cap->card)); + snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s", + dev_name(ctx->jpeg->dev)); + return 0; +} + +static int apple_jpeg_enum_fmt(struct file *file, void *priv, + struct v4l2_fmtdesc *f) +{ + if (f->index > 1) + return -EINVAL; + f->pixelformat = f->index ? V4L2_PIX_FMT_NV12M : V4L2_PIX_FMT_JPEG; + return 0; +} + +static int apple_jpeg_g_fmt(struct file *file, void *priv, struct v4l2_format *f) +{ + struct v4l2_pix_format_mplane *pix = apple_jpeg_format(fh_to_ctx(priv), f->type); + + if (!pix) + return -EINVAL; + f->fmt.pix_mp = *pix; + return 0; +} + +static int apple_jpeg_try_fmt(struct file *file, void *priv, struct v4l2_format *f) +{ + if (!apple_jpeg_format(fh_to_ctx(priv), f->type)) + return -EINVAL; + apple_jpeg_normalize_format(&f->fmt.pix_mp); + return 0; +} + +static int apple_jpeg_s_fmt(struct file *file, void *priv, struct v4l2_format *f) +{ + struct apple_jpeg_ctx *ctx = fh_to_ctx(priv); + struct v4l2_pix_format_mplane *pix, *other; + struct v4l2_m2m_ctx *m2m = ctx->fh.m2m_ctx; + int ret; + + ret = apple_jpeg_try_fmt(file, priv, f); + if (ret) + return ret; + if (vb2_is_busy(v4l2_m2m_get_src_vq(m2m)) || + vb2_is_busy(v4l2_m2m_get_dst_vq(m2m))) + return -EBUSY; + + pix = apple_jpeg_format(ctx, f->type); + other = pix == &ctx->output ? &ctx->capture : &ctx->output; + *pix = f->fmt.pix_mp; + *other = *pix; + other->pixelformat = pix->pixelformat == V4L2_PIX_FMT_JPEG ? + V4L2_PIX_FMT_NV12M : V4L2_PIX_FMT_JPEG; + apple_jpeg_normalize_format(other); + return 0; +} + +static int apple_jpeg_queue_setup(struct vb2_queue *q, unsigned int *num_buffers, + unsigned int *num_planes, unsigned int sizes[], + struct device *alloc_devs[]) +{ + struct apple_jpeg_ctx *ctx = vb2_get_drv_priv(q); + struct v4l2_pix_format_mplane *pix = apple_jpeg_format(ctx, q->type); + unsigned int i; + + if (*num_planes) { + if (*num_planes != pix->num_planes) + return -EINVAL; + for (i = 0; i < *num_planes; i++) + if (sizes[i] < pix->plane_fmt[i].sizeimage) + return -EINVAL; + return 0; + } + *num_planes = pix->num_planes; + for (i = 0; i < *num_planes; i++) + sizes[i] = pix->plane_fmt[i].sizeimage; + return 0; +} + +static int apple_jpeg_buf_prepare(struct vb2_buffer *vb) +{ + struct apple_jpeg_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + struct v4l2_pix_format_mplane *pix = apple_jpeg_format(ctx, vb->type); + unsigned int i; + + for (i = 0; i < pix->num_planes; i++) { + if (vb2_plane_size(vb, i) < pix->plane_fmt[i].sizeimage) + return -EINVAL; + if (!V4L2_TYPE_IS_OUTPUT(vb->type)) + vb2_set_plane_payload(vb, i, 0); + } + return 0; +} + +static void apple_jpeg_buf_queue(struct vb2_buffer *vb) +{ + struct apple_jpeg_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + + v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, to_vb2_v4l2_buffer(vb)); +} + +static void apple_jpeg_return_buffers(struct apple_jpeg_ctx *ctx, + enum v4l2_buf_type type, + enum vb2_buffer_state state) +{ + struct vb2_v4l2_buffer *buf; + + for (;;) { + buf = V4L2_TYPE_IS_OUTPUT(type) ? + v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx) : + v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + if (!buf) + break; + v4l2_m2m_buf_done(buf, state); + } +} + +static int apple_jpeg_start_streaming(struct vb2_queue *q, unsigned int count) +{ + /* VB2 requires queued buffers back on a failed STREAMON. No MMIO here. */ + apple_jpeg_return_buffers(vb2_get_drv_priv(q), q->type, VB2_BUF_STATE_QUEUED); + return -EOPNOTSUPP; +} + +static void apple_jpeg_stop_streaming(struct vb2_queue *q) +{ + apple_jpeg_return_buffers(vb2_get_drv_priv(q), q->type, VB2_BUF_STATE_ERROR); +} + +static const struct vb2_ops apple_jpeg_queue_ops = { + .queue_setup = apple_jpeg_queue_setup, + .buf_prepare = apple_jpeg_buf_prepare, + .buf_queue = apple_jpeg_buf_queue, + .start_streaming = apple_jpeg_start_streaming, + .stop_streaming = apple_jpeg_stop_streaming, +}; + +static int apple_jpeg_init_queue(struct apple_jpeg_ctx *ctx, struct vb2_queue *q, + enum v4l2_buf_type type) +{ + q->type = type; + q->io_modes = VB2_MMAP | VB2_DMABUF; + q->drv_priv = ctx; + q->buf_struct_size = sizeof(struct v4l2_m2m_buffer); + q->ops = &apple_jpeg_queue_ops; + q->mem_ops = &vb2_dma_contig_memops; + q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + q->lock = &ctx->jpeg->lock; + q->dev = ctx->jpeg->dev; + return vb2_queue_init(q); +} + +static int apple_jpeg_queue_init(void *priv, struct vb2_queue *src, + struct vb2_queue *dst) +{ + struct apple_jpeg_ctx *ctx = priv; + int ret; + + ret = apple_jpeg_init_queue(ctx, src, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + if (ret) + return ret; + return apple_jpeg_init_queue(ctx, dst, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); +} + +static int apple_jpeg_job_ready(void *priv) +{ + return 0; +} + +static void apple_jpeg_device_run(void *priv) +{ + struct apple_jpeg_ctx *ctx = priv; + + /* Defensive fallback; STREAMON and job_ready prevent scheduling. */ + apple_jpeg_return_buffers(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, + VB2_BUF_STATE_ERROR); + apple_jpeg_return_buffers(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, + VB2_BUF_STATE_ERROR); + v4l2_m2m_job_finish(ctx->jpeg->m2m_dev, ctx->fh.m2m_ctx); +} + +static const struct v4l2_m2m_ops apple_jpeg_m2m_ops = { + .device_run = apple_jpeg_device_run, + .job_ready = apple_jpeg_job_ready, +}; + +static int apple_jpeg_open(struct file *file) +{ + struct apple_jpeg *jpeg = video_drvdata(file); + struct apple_jpeg_ctx *ctx; + int ret = 0; + + if (mutex_lock_interruptible(&jpeg->lock)) + return -ERESTARTSYS; + if (!video_is_registered(&jpeg->vdev)) { + ret = -ENODEV; + goto unlock; + } + ctx = kzalloc_obj(*ctx); + if (!ctx) { + ret = -ENOMEM; + goto unlock; + } + ctx->jpeg = jpeg; + ctx->output.width = 640; + ctx->output.height = 480; + ctx->output.pixelformat = V4L2_PIX_FMT_JPEG; + apple_jpeg_normalize_format(&ctx->output); + ctx->capture = ctx->output; + ctx->capture.pixelformat = V4L2_PIX_FMT_NV12M; + apple_jpeg_normalize_format(&ctx->capture); + v4l2_fh_init(&ctx->fh, &jpeg->vdev); + ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(jpeg->m2m_dev, ctx, apple_jpeg_queue_init); + if (IS_ERR(ctx->fh.m2m_ctx)) { + ret = PTR_ERR(ctx->fh.m2m_ctx); + v4l2_fh_exit(&ctx->fh); + kfree(ctx); + goto unlock; + } + v4l2_fh_add(&ctx->fh, file); +unlock: + mutex_unlock(&jpeg->lock); + return ret; +} + +static int apple_jpeg_release(struct file *file) +{ + struct apple_jpeg_ctx *ctx = fh_to_ctx(file->private_data); + struct apple_jpeg *jpeg = ctx->jpeg; + + mutex_lock(&jpeg->lock); + v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); + v4l2_fh_del(&ctx->fh, file); + v4l2_fh_exit(&ctx->fh); + kfree(ctx); + mutex_unlock(&jpeg->lock); + return 0; +} + +static const struct v4l2_file_operations apple_jpeg_fops = { + .owner = THIS_MODULE, + .open = apple_jpeg_open, + .release = apple_jpeg_release, + .poll = v4l2_m2m_fop_poll, + .unlocked_ioctl = video_ioctl2, + .mmap = v4l2_m2m_fop_mmap, +}; + +static const struct v4l2_ioctl_ops apple_jpeg_ioctl_ops = { + .vidioc_querycap = apple_jpeg_querycap, + .vidioc_enum_fmt_vid_cap = apple_jpeg_enum_fmt, + .vidioc_enum_fmt_vid_out = apple_jpeg_enum_fmt, + .vidioc_g_fmt_vid_cap_mplane = apple_jpeg_g_fmt, + .vidioc_g_fmt_vid_out_mplane = apple_jpeg_g_fmt, + .vidioc_try_fmt_vid_cap_mplane = apple_jpeg_try_fmt, + .vidioc_try_fmt_vid_out_mplane = apple_jpeg_try_fmt, + .vidioc_s_fmt_vid_cap_mplane = apple_jpeg_s_fmt, + .vidioc_s_fmt_vid_out_mplane = apple_jpeg_s_fmt, + .vidioc_reqbufs = v4l2_m2m_ioctl_reqbufs, + .vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs, + .vidioc_querybuf = v4l2_m2m_ioctl_querybuf, + .vidioc_qbuf = v4l2_m2m_ioctl_qbuf, + .vidioc_dqbuf = v4l2_m2m_ioctl_dqbuf, + .vidioc_expbuf = v4l2_m2m_ioctl_expbuf, + .vidioc_streamon = v4l2_m2m_ioctl_streamon, + .vidioc_streamoff = v4l2_m2m_ioctl_streamoff, +}; + +static int apple_jpeg_runtime_resume(struct device *dev) +{ + struct apple_jpeg *jpeg = dev_get_drvdata(dev); + u32 hwrev = readl(jpeg->regs + APPLE_JPEG_HWREV); + + /* Identify before any reset writes; no target reset profile is known. */ + if (!apple_jpeg_hwrev_supported(jpeg->variant, hwrev)) + return dev_err_probe(dev, -ENODEV, "unqualified HWREV %#x\n", hwrev); + return 0; +} + +static int apple_jpeg_runtime_suspend(struct device *dev) +{ + /* This scaffold never starts DMA or changes the engine's register state. */ + return 0; +} + +static const struct dev_pm_ops apple_jpeg_pm_ops = { + RUNTIME_PM_OPS(apple_jpeg_runtime_suspend, apple_jpeg_runtime_resume, NULL) + SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) +}; + +static int apple_jpeg_check_dma(struct device *dev) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct of_phandle_args args; + u64 range[2]; + int ret; + + if (of_count_phandle_with_args(dev->of_node, "iommus", "#iommu-cells") != 1) + return -EINVAL; + ret = of_parse_phandle_with_args(dev->of_node, "iommus", "#iommu-cells", 0, &args); + if (ret) + return ret; + of_node_put(args.np); + if (args.args_count != 1 || args.args[0] != 0) + return -EINVAL; + if (!domain || (domain->type != IOMMU_DOMAIN_DMA && + domain->type != IOMMU_DOMAIN_DMA_FQ)) + return -ENODEV; + if (of_property_count_u64_elems(dev->of_node, "apple,dma-range") != 2) + return -EINVAL; + ret = of_property_read_u64_array(dev->of_node, "apple,dma-range", range, 2); + if (ret) + return ret; + if (range[0] != APPLE_JPEG_DMA_BASE || range[1] != APPLE_JPEG_DMA_SIZE) + return -EINVAL; + /* BIT(40) is part of the DART alias, not a 32-bit physical DMA mask. */ + return dma_set_mask_and_coherent(dev, DMA_BIT_MASK(41)); +} + +static void apple_jpeg_v4l2_release(struct v4l2_device *v4l2_dev) +{ + struct apple_jpeg *jpeg = container_of(v4l2_dev, struct apple_jpeg, v4l2_dev); + + if (jpeg->m2m_dev) + v4l2_m2m_release(jpeg->m2m_dev); + put_device(jpeg->dev); + mutex_destroy(&jpeg->lock); + kfree(jpeg); +} + +static int apple_jpeg_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_jpeg *jpeg; + struct resource *res; + int ret; + + ret = apple_jpeg_check_dma(dev); + if (ret) + return dev_err_probe(dev, ret, "requires a dedicated SID-0 DART window\n"); + if (!dev->pm_domain) + return dev_err_probe(dev, -ENODEV, "requires a power domain\n"); + ret = platform_get_irq(pdev, 0); + if (ret < 0) + return ret; + /* Do not request an IRQ before acknowledgment semantics are qualified. */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res || resource_size(res) != APPLE_JPEG_REG_SIZE) + return -EINVAL; + + jpeg = kzalloc_obj(*jpeg); + if (!jpeg) + return -ENOMEM; + jpeg->dev = get_device(dev); + jpeg->variant = of_device_get_match_data(dev); + mutex_init(&jpeg->lock); + jpeg->regs = devm_ioremap_resource(dev, res); + if (IS_ERR(jpeg->regs)) { + ret = PTR_ERR(jpeg->regs); + goto free_jpeg; + } + platform_set_drvdata(pdev, jpeg); + pm_runtime_set_autosuspend_delay(dev, 100); + pm_runtime_use_autosuspend(dev); + pm_runtime_enable(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + goto disable_pm; + ret = pm_runtime_put_sync_suspend(dev); + if (ret < 0) + goto disable_pm; + + ret = v4l2_device_register(dev, &jpeg->v4l2_dev); + if (ret) + goto disable_pm; + jpeg->v4l2_dev.release = apple_jpeg_v4l2_release; + jpeg->m2m_dev = v4l2_m2m_init(&apple_jpeg_m2m_ops); + if (IS_ERR(jpeg->m2m_dev)) { + ret = PTR_ERR(jpeg->m2m_dev); + jpeg->m2m_dev = NULL; + goto unregister_v4l2; + } + strscpy(jpeg->vdev.name, "apple-jpeg", sizeof(jpeg->vdev.name)); + jpeg->vdev.v4l2_dev = &jpeg->v4l2_dev; + jpeg->vdev.fops = &apple_jpeg_fops; + jpeg->vdev.ioctl_ops = &apple_jpeg_ioctl_ops; + jpeg->vdev.release = video_device_release_empty; + jpeg->vdev.lock = &jpeg->lock; + jpeg->vdev.vfl_dir = VFL_DIR_M2M; + jpeg->vdev.device_caps = V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING; + video_set_drvdata(&jpeg->vdev, jpeg); + ret = video_register_device(&jpeg->vdev, VFL_TYPE_VIDEO, -1); + if (ret) + goto unregister_v4l2; + return 0; + +unregister_v4l2: + v4l2_device_unregister(&jpeg->v4l2_dev); + pm_runtime_disable(dev); + pm_runtime_dont_use_autosuspend(dev); + platform_set_drvdata(pdev, NULL); + v4l2_device_put(&jpeg->v4l2_dev); + return ret; +disable_pm: + pm_runtime_disable(dev); + pm_runtime_dont_use_autosuspend(dev); + platform_set_drvdata(pdev, NULL); +free_jpeg: + put_device(jpeg->dev); + mutex_destroy(&jpeg->lock); + kfree(jpeg); + return ret; +} + +static void apple_jpeg_remove(struct platform_device *pdev) +{ + struct apple_jpeg *jpeg = platform_get_drvdata(pdev); + + mutex_lock(&jpeg->lock); + v4l2_device_disconnect(&jpeg->v4l2_dev); + video_unregister_device(&jpeg->vdev); + mutex_unlock(&jpeg->lock); + v4l2_device_unregister(&jpeg->v4l2_dev); + pm_runtime_disable(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); + platform_set_drvdata(pdev, NULL); + /* Open files retain the video_device's V4L2 reference until last close. */ + v4l2_device_put(&jpeg->v4l2_dev); +} + +static const struct of_device_id apple_jpeg_of_match[] = { + { .compatible = "apple,t6041-jpeg", .data = &apple_jpeg_t6041 }, + { .compatible = "apple,t8140-jpeg", .data = &apple_jpeg_t8140 }, + { } +}; +MODULE_DEVICE_TABLE(of, apple_jpeg_of_match); + +static struct platform_driver apple_jpeg_driver = { + .probe = apple_jpeg_probe, + .remove = apple_jpeg_remove, + .driver = { + .name = "apple-jpeg", + .of_match_table = apple_jpeg_of_match, + .pm = pm_ptr(&apple_jpeg_pm_ops), + }, +}; +module_platform_driver(apple_jpeg_driver); + +MODULE_DESCRIPTION("Apple Silicon JPEG mem2mem scaffold"); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 989546c595291..0aac63136e979 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -3210,6 +3210,10 @@ static void brcmf_convert_ratespec_to_rateinfo(u32 ratespec, /* Fill in nothing */ break; } + if ((BRCMF_RSPEC_ISHT(ratespec) || BRCMF_RSPEC_ISVHT(ratespec)) && + BRCMF_RSPEC_ISSGI(ratespec)) + rateinfo->flags |= RATE_INFO_FLAGS_SHORT_GI; + if (BRCMF_RSPEC_ISHT(ratespec)) { rateinfo->flags |= RATE_INFO_FLAGS_MCS; rateinfo->mcs = ratespec & BRCMF_RSPEC_HT_MCS_MASK; diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index 49ff2296003c1..f16a55a528b36 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -2,7 +2,7 @@ config MT7932_FULLMAC tristate "MediaTek MT7932 Wi-Fi on the Apple MacBook Neo (J700)" depends on PCI && OF && ARM64 && IOMMU_DMA - depends on PCIE_APPLE_PIODMA_DIAG && CFG80211 + depends on PCIE_APPLE_T8140_BOOTSTRAP && CFG80211 select FW_LOADER select CRYPTO select CRYPTO_HMAC diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index cf7b169dab786..6c47a8d282753 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -2,7 +2,10 @@ obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o +mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += power-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += input-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += regulatory-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += association-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += six-test.o + +mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += pmf-lpi-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/association.h b/drivers/net/wireless/mediatek/mt7932/association.h index 2028a46558de6..011a029f20357 100644 --- a/drivers/net/wireless/mediatek/mt7932/association.h +++ b/drivers/net/wireless/mediatek/mt7932/association.h @@ -56,6 +56,12 @@ mt7932_connect_auth(const struct cfg80211_connect_params *p) (p->mfp == NL80211_MFP_NO || p->mfp == NL80211_MFP_OPTIONAL)) return WLAN_AKM_SUITE_PSK; break; + case WLAN_AKM_SUITE_PSK_SHA256: + if ((versions & NL80211_WPA_VERSION_2) && p->crypto.psk && + (p->mfp == NL80211_MFP_OPTIONAL || + p->mfp == NL80211_MFP_REQUIRED)) + return WLAN_AKM_SUITE_PSK_SHA256; + break; case WLAN_AKM_SUITE_OWE: if (p->auth_type == NL80211_AUTHTYPE_OPEN_SYSTEM && p->mfp == NL80211_MFP_REQUIRED && !p->crypto.psk && @@ -82,6 +88,117 @@ mt7932_connect_auth(const struct cfg80211_connect_params *p) return -EOPNOTSUPP; } +/* A selected AKM is not an MFP policy field. Firmware owns all handshake + * keys, including IGTK, and BIP verification. No host management-key command + * or optional/required bit has been established for EA/40. + */ +static inline int mt7932_connect_psk_akm(u8 *body, size_t length, u32 akm) +{ + if (length < 856 || body[1] != 4 || + (akm != WLAN_AKM_SUITE_PSK && akm != WLAN_AKM_SUITE_PSK_SHA256)) + return -EINVAL; + put_unaligned_le16(akm == WLAN_AKM_SUITE_PSK_SHA256 ? 0x0400 : 0x0008, + body + 0x14); + return 0; +} + +/* Validate the entire RSN payload before using capabilities for admission. + * The group management suite defaults to BIP-CMAC-128 when omitted. Reject + * unsupported explicit suites rather than letting firmware choose one. + */ +static inline int mt7932_rsn_admit(const u8 *rsn, size_t length, u32 akm, + enum nl80211_mfp mfp, bool *pmf) +{ + unsigned int count, i; + u16 caps = 0; + bool selected = false, ccmp = false; + + *pmf = false; + if (mfp > NL80211_MFP_OPTIONAL || length < 8 || + get_unaligned_le16(rsn) != 1 || + get_unaligned_be32(rsn + 2) != WLAN_CIPHER_SUITE_CCMP) + return -EINVAL; + rsn += 6; + length -= 6; + count = get_unaligned_le16(rsn); + rsn += 2; + length -= 2; + if (!count || count > length / 4) + return -EINVAL; + for (i = 0; i < count; i++) + ccmp |= get_unaligned_be32(rsn + 4 * i) == + WLAN_CIPHER_SUITE_CCMP; + rsn += 4 * count; + length -= 4 * count; + if (!ccmp || length < 2) + return -EINVAL; + count = get_unaligned_le16(rsn); + rsn += 2; + length -= 2; + if (!count || count > length / 4) + return -EINVAL; + for (i = 0; i < count; i++) + selected |= get_unaligned_be32(rsn + 4 * i) == akm; + rsn += 4 * count; + length -= 4 * count; + if (!selected) + return -EOPNOTSUPP; + if (length) { + if (length < 2) + return -EINVAL; + caps = get_unaligned_le16(rsn); + rsn += 2; + length -= 2; + } + if (length) { + if (length < 2) + return -EINVAL; + count = get_unaligned_le16(rsn); + rsn += 2; + length -= 2; + if (count > length / 16) + return -EINVAL; + rsn += 16 * count; + length -= 16 * count; + } + if (length && (length != 4 || + get_unaligned_be32(rsn) != WLAN_CIPHER_SUITE_AES_CMAC)) + return -EOPNOTSUPP; + if ((caps & BIT(6)) && !(caps & BIT(7))) + return -EINVAL; + /* Legacy PSK has no established PMF firmware selection. Optional may + * join it without PMF; required must never downgrade to that arm. + */ + if (akm == WLAN_AKM_SUITE_PSK) + return mfp == NL80211_MFP_REQUIRED || (caps & BIT(6)) ? + -EOPNOTSUPP : + 0; + if (mfp == NL80211_MFP_NO || !(caps & BIT(7))) + return -EOPNOTSUPP; + *pmf = true; + return 0; +} + +static inline int mt7932_connect_admit(const u8 *ies, size_t length, u32 akm, + enum nl80211_mfp mfp, bool *pmf) +{ + const struct element *elem, *rsn = NULL; + size_t parsed = 0; + + *pmf = false; + for_each_element(elem, ies, length) { + parsed += elem->datalen + 2; + if (elem->id != WLAN_EID_RSN) + continue; + if (rsn) + return -EINVAL; + rsn = elem; + } + if (parsed != length || !rsn) + return -EINVAL; + return mt7932_rsn_admit(rsn->data, rsn->datalen, akm, mfp, pmf); +} + /* The SAE offload needs both the password and a WPA passphrase-derived PMK. * This is PBKDF2-HMAC-SHA1(password, SSID, 4096, 32), not host SAE crypto. * Run only in sleepable context and erase every keyed intermediate. diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index b749e18a93380..553a6fe0e1193 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -79,6 +79,15 @@ void mt_bss_presence(struct mt7932 *m, const struct mt7932_event *event) if (event->length < 40 || body[0] || !READ_ONCE(m->bss_active)) return; + /* response_lock: how often, and for how long, the firmware reports + * the BSS absent. TX is held while it is absent without quota. + */ + if (body[1] && !m->bss_absent) { + m->bss_absent_events++; + m->bss_absent_since = jiffies; + } else if (!body[1] && m->bss_absent) { + m->bss_absent_ms += jiffies_to_msecs(jiffies - m->bss_absent_since); + } spin_lock_irqsave(&m->data_lock, flags); m->bss_absent = !!body[1]; m->bss_quota = body[2]; @@ -107,6 +116,7 @@ void mt_data_complete(struct mt7932 *m, const u8 *packet, size_t length) return; } spin_lock_irqsave(&m->data_lock, flags); + m->tx_free_events++; mt_data_clean_locked(m); for (i = 0; i < count; i++) { unsigned int token = tokens[i]; @@ -269,6 +279,8 @@ void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) m->rx_ethernet++; spin_lock(&m->data_lock); m->station_rx_packets++; + m->rx_band_records[frame->band == NL80211_BAND_2GHZ ? 0 : + frame->band == NL80211_BAND_5GHZ ? 1 : 2]++; m->station_rx_bytes += frame->length; signal = mt7932_rx_signal(frame, m->antenna_mask); if (frame->signal_valid && signal > -128) { diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index cfbdcd9f494d2..9e19d1a492e3b 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -165,16 +165,10 @@ static int mt_scan_submit_next(struct mt7932 *m) u8 body[0x4d4], channels[MT7932_SCAN_CHANNELS]; unsigned int count = 0, i; unsigned long flags; - bool passive = false, directed = false; + bool passive = false; int ret; - /* Hidden networks are found by a directed probe; supplicants send the - * hidden SSIDs together with a wildcard entry. - */ - for (i = 0; i < request->n_ssids; i++) - directed |= !!request->ssids[i].ssid_len; - - /* 2GHz active/passive, then 5GHz active/passive. Never put a NO_IR + /* 2/5/6 GHz active/passive pairs. Never put a NO_IR * channel in a firmware request which can emit probe requests. */ while (!count && m->scan_batch <= 6) { @@ -185,9 +179,8 @@ static int mt_scan_submit_next(struct mt7932 *m) for (i = m->scan_cursor; i < request->n_channels; i++) { struct ieee80211_channel *channel = request->channels[i]; bool radar = channel->flags & IEEE80211_CHAN_RADAR; - bool listen = !request->n_ssids || radar || - (channel->band == NL80211_BAND_6GHZ && !directed) || - (channel->flags & IEEE80211_CHAN_NO_IR); + bool listen = mt7932_scan_listen(request, channel, + m->six_legacy_channels); /* Radar channels are never joined; while associated, * a passive dwell there only delays the home channel. @@ -263,6 +256,7 @@ int mt_connect_discover(struct mt7932 *m) bool passive = attempt >= 6; band = 1 + (attempt / 2) % 3; + passive |= band == 3 && !m->six_legacy_channels; sband = band == 1 ? &m->band2 : band == 2 ? &m->band5 : &m->band6; if (band == 3 && m->connect_akm != WLAN_AKM_SUITE_SAE && m->connect_akm != WLAN_AKM_SUITE_OWE) @@ -278,6 +272,9 @@ int mt_connect_discover(struct mt7932 *m) IEEE80211_CHAN_RADAR)) continue; passive |= !!(channel->flags & IEEE80211_CHAN_NO_IR); + passive |= band == 3 && + !(m->six_legacy_channels & + BIT_ULL((channel->hw_value - 1) / 4)); if (count < ARRAY_SIZE(channels)) channels[count++] = channel->hw_value; } @@ -311,7 +308,7 @@ int mt_connect_discover(struct mt7932 *m) m->scan_seq, band, count, passive, ret); mutex_unlock(&m->command_mutex); if (!ret && !wait_for_completion_timeout(&m->discovery_done, - msecs_to_jiffies(5000))) + msecs_to_jiffies(mt7932_scan_discovery_timeout(body)))) ret = -ETIMEDOUT; mutex_lock(&m->command_mutex); /* Cancellation wakes the waiter, but is not a firmware completion. @@ -711,6 +708,27 @@ void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length) if (m->connected && !m->disconnecting && !m->stopping && frame.channel == m->connect_channel && ether_addr_equal(frame.data + 16, m->connect_bssid)) { + m->home_beacons++; + if (frame.band == NL80211_BAND_6GHZ && + m->six_ap_valid) { + struct mt7932_6g_ap ap; + + if (mt7932_6g_ap_parse(&ap, frame.data + 36, + frame.length - 36, + frame.channel, + channel->flags) || + memcmp(&ap, &m->six_ap, sizeof(ap))) { + m->connected = false; + m->disconnecting = true; + m->six_ap_valid = false; + m->disconnect_local = true; + m->disconnect_reason = + WLAN_REASON_DEAUTH_LEAVING; + netif_carrier_off(m->netdev); + netif_stop_queue(m->netdev); + schedule_work(&m->disconnect_work); + } + } mt_cqm_signal(m, mt7932_rx_signal(&frame, m->antenna_mask)); if (tim && tim->datalen >= 4 && !m->power_tim) { m->power_tim = true; @@ -724,7 +742,8 @@ void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length) mt7932_rx_signal(&frame, m->antenna_mask) * 100, GFP_ATOMIC); if (bss) { m->beacons++; - dev_dbg_ratelimited(&m->pdev->dev, "BSS_RECEIVED: channel=%u BSSID=%pM\n", frame.channel, bss->bssid); + dev_dbg_ratelimited(&m->pdev->dev, "BSS_RECEIVED: channel=%u\n", + frame.channel); cfg80211_put_bss(m->wiphy, bss); } } diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 538a1f15f0a3f..c2b45bea58879 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -48,6 +48,7 @@ void mt_rf_fail_locked(struct mt7932 *m, int error) if (error >= 0) error = -EIO; WRITE_ONCE(m->rf_ready, false); + m->six_ap_valid = false; WRITE_ONCE(m->link_failed, true); if (!m->cal_state.error) m->cal_state.error = error; @@ -296,6 +297,16 @@ static void mt_peer_release(struct mt7932 *m) m->peer_valid = false; m->peer_wtbl = 0; m->event_cipher = 0; + m->six_ap_valid = false; + m->connect_pmf = false; + /* Restore the installed VLP policy after a class/TPE connection. This + * runs only after peer/DMA retirement; scans and joins stay fenced. + */ + if (m->six_class_dirty && !m->reg_pending && !m->stopping) { + m->reg_generation++; + WRITE_ONCE(m->reg_pending, true); + schedule_work(&m->startup_work); + } } /* No lifecycle/command lock held on entry. RX/IRQ and TX-free processing stay @@ -430,6 +441,20 @@ static int mt_connect_prepare(struct mt7932 *m, bool *peer_owned) m->connect_mfp, m->connect_sae_pwe) || (m->connect_bss->channel->band == NL80211_BAND_6GHZ && !m->six_power_valid)) return -EOPNOTSUPP; + if (m->connect_akm == WLAN_AKM_SUITE_PSK || + m->connect_akm == WLAN_AKM_SUITE_PSK_SHA256) { + const struct cfg80211_bss_ies *ies; + + rcu_read_lock(); + ies = rcu_dereference(m->connect_bss->ies); + ret = ies ? mt7932_connect_admit(ies->data, ies->len, + m->connect_akm, m->connect_mfp, + &m->connect_pmf) : + -EINVAL; + rcu_read_unlock(); + if (ret) + return ret; + } if (m->connect_bss->channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_NO_20MHZ | IEEE80211_CHAN_RADAR)) { return -EINVAL; @@ -527,6 +552,40 @@ static int mt_connect_attempt(struct mt7932 *m, bool *submitted, m->connect_sae_pwd_length, m->connect_owe); if (ret < 0) goto free; + if (m->connect_akm == WLAN_AKM_SUITE_PSK || + m->connect_akm == WLAN_AKM_SUITE_PSK_SHA256) { + ret = mt7932_connect_psk_akm(body, 856, m->connect_akm); + if (ret) + goto free; + } + if (m->connect_band == NL80211_BAND_6GHZ) { + const struct cfg80211_bss_ies *ies; + struct mt7932_6g_ap ap; + + rcu_read_lock(); + ies = rcu_dereference(m->connect_bss->ies); + ret = ies ? mt7932_6g_connect_ap(&ap, ies->data, ies->len, + m->connect_channel, + m->connect_bss->channel->flags, + m->six_legacy_channels) : + -EINVAL; + rcu_read_unlock(); + if (ret < 0) + goto free; + if (ret == 1) { + m->six_legacy_assoc = true; + ret = 0; + goto identity; + } + ret = mt_6g_connect_policy(m, &ap); + if (ret) + goto free; + spin_lock_irq(&m->response_lock); + m->six_ap = ap; + m->six_ap_valid = true; + spin_unlock_irq(&m->response_lock); + } +identity: /* Reprogram the same on-air station identity for either attempt. */ memcpy(mac + 2, m->mac, ETH_ALEN); *submitted = true; @@ -547,8 +606,10 @@ static int mt_connect_attempt(struct mt7932 *m, bool *submitted, m->connect_pmk_valid = false; if (ret) goto out; - dev_dbg(&m->pdev->dev, "CONNECT_SUBMITTED: BSSID=%pM channel=%u\n", - m->connect_bssid, m->connect_channel); + dev_dbg(&m->pdev->dev, + "CONNECT_SUBMITTED: channel=%u AKM=%08x MFP=%u admitted-PMF=%u\n", + m->connect_channel, m->connect_akm, m->connect_mfp, + m->connect_pmf); *auth_submitted = true; deadline = jiffies + msecs_to_jiffies(10000); /* Genuine D7 requests must remain serviceable during asynchronous waits. */ @@ -575,7 +636,8 @@ static int mt_connect_attempt(struct mt7932 *m, bool *submitted, } mt7932_peer_queue(queue, m->peer_ids.station[0], true); ret = mt_request(m, 0x6b, true, true, false, queue, sizeof(queue)); - if (!ret && m->connect_band == NL80211_BAND_6GHZ) { + if (!ret && m->connect_band == NL80211_BAND_6GHZ && + m->six_legacy_assoc) { unsigned int i; for (i = 0; !ret && i < 2; i++) @@ -598,6 +660,20 @@ static int mt_connect_attempt(struct mt7932 *m, bool *submitted, return ret ?: READ_ONCE(m->connect_error); } +/* + * Once the request has gone to the AP, a timeout, a refusal or a reset while + * the firmware is still healthy is how the AP answered, not a driver or + * firmware fault. cfg80211 hands it to the supplicant, which retries, and + * mac80211 reports the same outcomes at info level. Keep the error level for + * failures that need recovery or point at the firmware. + */ +static bool mt_connect_failed_at_peer(struct mt7932 *m, bool auth_submitted, + int ret) +{ + return auth_submitted && !READ_ONCE(m->link_failed) && + (ret == -ETIMEDOUT || ret == -ECONNREFUSED || ret == -ECONNRESET); +} + void mt_connect_work(struct work_struct *work) { struct mt7932 *m = container_of(work, struct mt7932, connect_work); @@ -760,6 +836,9 @@ void mt_connect_work(struct work_struct *work) READ_ONCE(m->data_ready)); else if (cancelled) dev_dbg(&m->pdev->dev, "CONNECT_CANCELLED: %d\n", ret); + else if (mt_connect_failed_at_peer(m, auth_submitted, ret)) + dev_info(&m->pdev->dev, "CONNECT_FAILED: %d; recovery-required=0\n", + ret); else dev_err(&m->pdev->dev, "CONNECT_FAILED: %d; recovery-required=%u\n", ret, READ_ONCE(m->link_failed)); @@ -837,6 +916,9 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, m->connect_owe = auth == WLAN_AKM_SUITE_OWE; m->connect_akm = auth; m->connect_mfp = params->mfp; + m->connect_pmf = false; + m->six_ap_valid = false; + m->six_legacy_assoc = false; m->connect_sae_pwe = params->crypto.sae_pwe; memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); memzero_explicit(m->connect_sae_pwd, sizeof(m->connect_sae_pwd)); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index acf23bb0d31da..94388b56449e1 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -229,6 +229,7 @@ static void mt_rx_drain(struct mt7932 *m, struct mt7932_ring *q) break; dma_rmb(); length = (ctrl >> 16) & 0x3fff; + m->rx_ring_records[q - m->rx]++; if (length <= MT7932_RX_CAPACITY && (ctrl & BIT(30))) mt_receive(m, q->buffers + q->tail * RX_STRIDE, length); else @@ -274,9 +275,19 @@ static irqreturn_t mt_irq(int irq, void *data) static void mt_service(struct mt7932 *m) { - u32 causes = mt_read(m, W + 0x200) & m->irq_mask; + u32 raw = mt_read(m, W + 0x200), causes = raw & m->irq_mask; unsigned int i; + /* Status bits outside the enabled mask never raise an interrupt; if + * one of them announces ring work, that work waits for an unrelated + * cause. Record them so a test can see such a source. + */ + m->irq_passes++; + if ((raw & ~m->irq_mask) && raw != U32_MAX) { + m->irq_unmasked |= raw & ~m->irq_mask; + m->irq_unmasked_passes++; + } + /* * Acknowledge before servicing the rings. A completion that lands * while they are being drained then sets its status bit again and diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 172369bb3f159..28c339e76ed29 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -70,6 +70,8 @@ struct mt7932 { /* Hardware-free calibration transport seam; production builds omit it. */ int (*cal_test_send)(struct mt7932 *m, const void *body, size_t length); void *cal_test_context; + int (*power_test_send)(struct mt7932 *m, const void *body, size_t length); + void *power_test_context; #endif u8 cal_requests[32][16]; u8 cal_request_seq[32]; @@ -110,12 +112,15 @@ struct mt7932 { struct ieee80211_channel channels6[MT7932_CHANNELS_6G]; struct ieee80211_sband_iftype_data he6; u8 six_assoc_power[2][1020]; - bool six_power_valid; + bool six_power_valid, six_legacy_assoc, six_ap_valid, six_class_dirty; + u64 six_legacy_channels; + struct mt7932_6g_ap six_ap; u8 scan_band, scan_batch; unsigned int scan_cursor; enum nl80211_band connect_band, scan_home_band; u32 connect_akm; enum nl80211_mfp connect_mfp; + bool connect_pmf; /* admitted request, never firmware readback */ enum nl80211_sae_pwe_mechanism connect_sae_pwe; struct ieee80211_rate rates[12]; struct work_struct startup_work; @@ -199,6 +204,15 @@ struct mt7932 { u64 rx_integrity, rx_malformed, rx_unsupported; u64 rx_wrong_peer, rx_unprotected, rx_duplicate, rx_no_memory; u64 rx_retry, rx_amsdu; + /* Lifetime transport observations for ethtool. The ring and cause + * counters are written only by the interrupt thread; the BSS presence + * counters are under response_lock, the data counters under data_lock. + */ + u64 irq_passes, irq_unmasked_passes, rx_ring_records[9]; + u32 irq_unmasked; + u64 tx_free_events, bss_absent_events, bss_absent_ms; + unsigned long bss_absent_since; + u64 home_beacons, rx_band_records[3]; /* TX0 descriptors published since the last doorbell; data_lock. */ unsigned int data_unkicked; bool data_ready; @@ -211,6 +225,7 @@ struct mt7932 { bool mt_rf_allowed(struct mt7932 *m); void mt_retry_missing_policy(struct mt7932 *m); +int mt_6g_connect_policy(struct mt7932 *m, const struct mt7932_6g_ap *ap); void mt_reg_apply_deferred_locked(struct mt7932 *m); void mt_policy_debugfs_init(struct mt7932 *m); int mt_net_open(struct net_device *netdev); diff --git a/drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c b/drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c new file mode 100644 index 0000000000000..927b0e43b3a9c --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +#include "association.h" +#include "regulatory.h" + +/* Standard RSN IE: CCMP, AKM 6, MFPC, empty PMKID list, BIP-CMAC-128. */ +static const u8 pmf_rsn[] = { + 48, 26, 1, 0, 0, 15, 172, 4, 1, 0, 0, 15, 172, 4, + 1, 0, 0, 15, 172, 6, 128, 0, 0, 0, 0, 15, 172, 6, +}; + +static void mt_pmf_admission_test(struct kunit *test) +{ + u8 ies[2 * sizeof(pmf_rsn)], pmk[32] = {}; + struct cfg80211_connect_params request = { + .crypto = { + .wpa_versions = NL80211_WPA_VERSION_2, + .n_ciphers_pairwise = 1, + .ciphers_pairwise = {WLAN_CIPHER_SUITE_CCMP}, + .cipher_group = WLAN_CIPHER_SUITE_CCMP, + .n_akm_suites = 1, + .akm_suites = {WLAN_AKM_SUITE_PSK_SHA256}, + .psk = pmk, + }, + }; + bool pmf; + unsigned int i; + + for (i = NL80211_MFP_REQUIRED; i <= NL80211_MFP_OPTIONAL; i++) { + request.mfp = i; + KUNIT_EXPECT_EQ(test, mt7932_connect_auth(&request), + WLAN_AKM_SUITE_PSK_SHA256); + KUNIT_EXPECT_EQ(test, + mt7932_connect_admit(pmf_rsn, sizeof(pmf_rsn), + WLAN_AKM_SUITE_PSK_SHA256, + i, &pmf), + 0); + KUNIT_EXPECT_TRUE(test, pmf); + } + request.mfp = NL80211_MFP_NO; + KUNIT_EXPECT_LT(test, mt7932_connect_auth(&request), 0); + request.mfp = NL80211_MFP_REQUIRED; + request.crypto.wpa_versions = NL80211_WPA_VERSION_3; + KUNIT_EXPECT_LT(test, mt7932_connect_auth(&request), 0); + request.crypto.wpa_versions = NL80211_WPA_VERSION_2; + request.crypto.n_akm_suites = 2; + request.crypto.akm_suites[0] = WLAN_AKM_SUITE_PSK; + request.crypto.akm_suites[1] = WLAN_AKM_SUITE_PSK_SHA256; + KUNIT_EXPECT_LT(test, mt7932_connect_auth(&request), 0); + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(pmf_rsn, sizeof(pmf_rsn), + WLAN_AKM_SUITE_PSK_SHA256, + NL80211_MFP_NO, &pmf), + 0); + for (i = 0; i < sizeof(pmf_rsn); i++) + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(pmf_rsn, i, + WLAN_AKM_SUITE_PSK_SHA256, + NL80211_MFP_REQUIRED, &pmf), + 0); + memcpy(ies, pmf_rsn, sizeof(pmf_rsn)); + ies[20] = 0; /* no MFPC */ + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(ies, sizeof(pmf_rsn), + WLAN_AKM_SUITE_PSK_SHA256, + NL80211_MFP_REQUIRED, &pmf), + 0); + ies[20] = 64; /* MFPR without MFPC */ + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(ies, sizeof(pmf_rsn), + WLAN_AKM_SUITE_PSK_SHA256, + NL80211_MFP_OPTIONAL, &pmf), + 0); + ies[20] = 128; + ies[27] = 11; /* unqualified BIP-GMAC-128 */ + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(ies, sizeof(pmf_rsn), + WLAN_AKM_SUITE_PSK_SHA256, + NL80211_MFP_REQUIRED, &pmf), + 0); + memcpy(ies, pmf_rsn, sizeof(pmf_rsn)); + memcpy(ies + sizeof(pmf_rsn), pmf_rsn, sizeof(pmf_rsn)); + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(ies, sizeof(ies), + WLAN_AKM_SUITE_PSK_SHA256, + NL80211_MFP_REQUIRED, &pmf), + 0); + ies[1] = + 20; /* no PMKID list or explicit management cipher: CMAC default */ + KUNIT_EXPECT_EQ(test, + mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK_SHA256, + NL80211_MFP_REQUIRED, &pmf), + 0); + ies[19] = 2; /* legacy PSK cannot satisfy required PMF */ + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK, + NL80211_MFP_REQUIRED, &pmf), + 0); + KUNIT_EXPECT_EQ(test, + mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK, + NL80211_MFP_OPTIONAL, &pmf), + 0); + KUNIT_EXPECT_FALSE(test, pmf); + ies[20] |= 64; + KUNIT_EXPECT_LT(test, + mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK, + NL80211_MFP_OPTIONAL, &pmf), + 0); +} + +static void mt_pmf_body_test(struct kunit *test) +{ + static const struct mt7932_peer_ids ids = { { 1, 2 }, { 3, 4, 5, 6 } }; + static const u8 peer[] = { 2, 0, 0, 0, 0, 1 }; + u8 *body, *legacy, pmk[32] = {}; + + body = kunit_kzalloc(test, 856, GFP_KERNEL); + legacy = kunit_kzalloc(test, 856, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, body); + KUNIT_ASSERT_NOT_NULL(test, legacy); + KUNIT_ASSERT_EQ(test, + mt7932_connect_body(body, 856, "test", 4, pmk, peer, + NL80211_BAND_5GHZ, 36, 20, 2, &ids, + false, NULL, 0, false), + 856); + memcpy(legacy, body, 856); + KUNIT_ASSERT_EQ(test, + mt7932_connect_psk_akm(body, 856, + WLAN_AKM_SUITE_PSK_SHA256), + 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 0x14), 0x0400); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(body + 0x40), 32U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(body + 0x44), 6U); + KUNIT_EXPECT_MEMEQ(test, body + 0x4c, pmk, 32); + KUNIT_ASSERT_EQ(test, + mt7932_connect_psk_akm(body, 856, WLAN_AKM_SUITE_PSK), 0); + KUNIT_EXPECT_MEMEQ(test, body, legacy, + 856); /* only selected AKM changed */ + KUNIT_EXPECT_LT(test, + mt7932_connect_psk_akm(body, 855, + WLAN_AKM_SUITE_PSK_SHA256), + 0); +} + +/* HE operation + uniform default-client PSD TPE at -1 dBm/MHz. */ +static const u8 lpi_ies[] = { + 255, 12, 36, 0, 0, 2, 0, 0, 0, 5, 2, 7, 0, 6, 195, 2, 24, 254, +}; + +static void mt_6g_ap_test(struct kunit *test) +{ + struct mt7932_6g_ap ap; + u8 ies[sizeof(lpi_ies) + 14]; + unsigned int i; + + KUNIT_ASSERT_EQ(test, + mt7932_6g_ap_parse(&ap, lpi_ies, sizeof(lpi_ies), 5, 0), + 0); + KUNIT_EXPECT_EQ(test, ap.mode, MT7932_6G_LPI); + KUNIT_EXPECT_EQ(test, ap.count, 4); + KUNIT_EXPECT_EQ(test, ap.limit[0], 24); + KUNIT_EXPECT_EQ(test, ap.limit[3], 42); + for (i = 0; i < sizeof(lpi_ies); i++) + KUNIT_EXPECT_LT(test, mt7932_6g_ap_parse(&ap, lpi_ies, i, 5, 0), + 0); + memcpy(ies, lpi_ies, sizeof(lpi_ies)); + ies[10] |= 8; /* SP */ + KUNIT_EXPECT_LT(test, + mt7932_6g_ap_parse(&ap, ies, sizeof(lpi_ies), 5, 0), 0); + ies[10] = 2 | 16; /* VLP */ + KUNIT_EXPECT_EQ(test, + mt7932_6g_ap_parse(&ap, ies, sizeof(lpi_ies), 5, 0), 0); + KUNIT_EXPECT_EQ(test, ap.mode, MT7932_6G_VLP); + KUNIT_EXPECT_LT(test, + mt7932_6g_ap_parse(&ap, ies, sizeof(lpi_ies), 5, + IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT), + 0); + ies[10] = 2 | 120; /* reserved cannot authorize LPI */ + KUNIT_EXPECT_LT(test, + mt7932_6g_ap_parse(&ap, ies, sizeof(lpi_ies), 5, 0), 0); + memcpy(ies, lpi_ies, sizeof(lpi_ies)); + KUNIT_EXPECT_LT(test, + mt7932_6g_ap_parse(&ap, ies, sizeof(lpi_ies), 5, + IEEE80211_CHAN_NO_IR), + 0); + KUNIT_EXPECT_LT(test, + mt7932_6g_ap_parse(&ap, ies, sizeof(lpi_ies), 9, 0), 0); + memcpy(ies + sizeof(lpi_ies), lpi_ies, 14); + KUNIT_EXPECT_LT(test, mt7932_6g_ap_parse(&ap, ies, sizeof(ies), 5, 0), + 0); + memcpy(ies, lpi_ies, sizeof(lpi_ies)); + ies[17] = + 0x80; /* PSD forbids transmission, not a very low finite cap. */ + KUNIT_EXPECT_EQ(test, + mt7932_6g_ap_parse(&ap, ies, sizeof(lpi_ies), 5, 0), + -EACCES); + /* TPE alone cannot select LPI. */ + KUNIT_EXPECT_LT(test, mt7932_6g_ap_parse(&ap, lpi_ies + 14, 4, 5, 0), + 0); +} + +static void mt_6g_tpe_body_test(struct kunit *test) +{ + static const s8 gain[3][4] = { { 2, 4, 6, 8 }, + { 4, 6, 8, 10 }, + { 6, 8, 10, 12 } }; + struct mt7932_6g_ap ap = { + .mode = MT7932_6G_LPI, + .primary = 5, + .count = 3, + .limit = { 30, -10, 127, 0 }, + }; + u8 body[61]; + + memset(body, 0xa5, sizeof(body)); + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), 60); + KUNIT_EXPECT_EQ(test, body[60], 0xa5); + KUNIT_EXPECT_EQ(test, body[0], 0); + KUNIT_EXPECT_EQ(test, body[1], 2); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 2), 60); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(body + 4), 0x03010301U); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(body + 8, 0, 36), NULL); + KUNIT_EXPECT_EQ(test, body[44], 1); + KUNIT_EXPECT_EQ(test, body[45], 3); + KUNIT_EXPECT_EQ(test, body[46], 5); + KUNIT_EXPECT_EQ(test, body[47], 3); + KUNIT_EXPECT_EQ(test, body[48], 28); + KUNIT_EXPECT_EQ(test, (s8)body[49], -14); + KUNIT_EXPECT_EQ(test, body[52], 26); + KUNIT_EXPECT_EQ(test, body[56], 24); + KUNIT_EXPECT_EQ(test, body[50], 127); + KUNIT_EXPECT_EQ(test, body[51], 127); + KUNIT_EXPECT_EQ(test, body[58], 127); + KUNIT_EXPECT_EQ(test, body[59], 127); + KUNIT_EXPECT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, NULL), + -EOPNOTSUPP); + ap.limit[0] = -128; + KUNIT_EXPECT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), -ERANGE); + memset(ap.limit, 127, 4); + KUNIT_EXPECT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, NULL), 60); + KUNIT_EXPECT_EQ(test, mt7932_6g_tpe_body(body, 59, &ap, gain), -EINVAL); + ap.count = 5; + KUNIT_EXPECT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), -EINVAL); +} + +/* t-3573 section 3: bandwidth-specific boundary rows and all six groups. */ +static void mt_6g_j700_gain_test(struct kunit *test) +{ + static const struct { + u8 channel; + u8 group[4]; + } vectors[] = { + { 1, { 1, 1, 1, 1 } }, { 29, { 1, 1, 1, 1 } }, + { 33, { 2, 2, 2, 2 } }, { 61, { 2, 2, 2, 2 } }, + { 65, { 3, 3, 3, 3 } }, { 93, { 3, 3, 3, 3 } }, + { 97, { 4, 4, 4, 4 } }, { 109, { 4, 4, 4, 4 } }, + { 113, { 4, 4, 5, 4 } }, { 117, { 5, 4, 5, 4 } }, + { 121, { 5, 5, 5, 4 } }, { 125, { 5, 5, 5, 4 } }, + { 129, { 5, 5, 5, 5 } }, { 181, { 5, 5, 5, 5 } }, + { 185, { 5, 6, 5, 5 } }, { 189, { 6, 6, 5, 5 } }, + { 193, { 6, 6, 6, 6 } }, { 221, { 6, 6, 6, 6 } }, + { 225, { 6, 6, 0, 0 } }, { 229, { 6, 6, 0, 0 } }, + { 233, { 6, 0, 0, 0 } }, { 2, { 1, 0, 0, 0 } }, + }; + static const s8 expected[][3] = { + { 16, 16, 10 }, { 14, 14, 8 }, { 15, 16, 10 }, + { 15, 15, 9 }, { 16, 16, 10 }, { 15, 16, 10 }, + }; + s8 gain[3]; + unsigned int i, bw; + + for (i = 0; i < ARRAY_SIZE(vectors); i++) { + for (bw = 0; bw < 4; bw++) { + int ret = mt7932_j700_6g_gain(vectors[i].channel, bw, + gain); + u8 group = vectors[i].group[bw]; + + if (!group) { + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + continue; + } + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_MEMEQ(test, gain, expected[group - 1], 3); + } + } + KUNIT_EXPECT_EQ(test, mt7932_j700_6g_gain(0, 0, gain), -EINVAL); + KUNIT_EXPECT_EQ(test, mt7932_j700_6g_gain(3, 0, gain), -EINVAL); + KUNIT_EXPECT_EQ(test, mt7932_j700_6g_gain(237, 0, gain), -EINVAL); + KUNIT_EXPECT_EQ(test, mt7932_j700_6g_gain(5, 4, gain), -EINVAL); +} + +static void mt_6g_finite_projection_test(struct kunit *test) +{ + struct mt7932_6g_ap ap = { + .mode = MT7932_6G_LPI, + .primary = 117, + .count = 4, + .limit = { 40, 40, 40, 40 }, + }; + /* Channel 117 uses G5/G4/G5/G4, not a frequency-only group. */ + static const u8 expected[] = { 24, 25, 24, 25, 24, 25, + 24, 25, 30, 31, 30, 31 }; + s8 gain[3][4]; + u8 body[60], lpi[60]; + + KUNIT_ASSERT_EQ(test, mt7932_j700_6g_gains(&ap, gain), 0); + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), 60); + KUNIT_EXPECT_MEMEQ(test, body + 48, expected, sizeof(expected)); + memcpy(lpi, body, 60); + ap.mode = MT7932_6G_VLP; + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), 60); + KUNIT_EXPECT_MEMEQ(test, lpi, body, 60); + /* PSD -1 dBm/MHz => 24/30/36/42 half-dBm EIRP; G1 gains. */ + KUNIT_ASSERT_EQ(test, + mt7932_6g_ap_parse(&ap, lpi_ies, sizeof(lpi_ies), 5, 0), + 0); + KUNIT_ASSERT_EQ(test, mt7932_j700_6g_gains(&ap, gain), 0); + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), 60); + KUNIT_EXPECT_EQ(test, body[48], 8); + KUNIT_EXPECT_EQ(test, body[51], 26); + KUNIT_EXPECT_EQ(test, body[56], 14); + KUNIT_EXPECT_EQ(test, body[59], 32); + ap.primary = 233; + KUNIT_EXPECT_EQ(test, mt7932_j700_6g_gains(&ap, gain), -EOPNOTSUPP); + memset(ap.limit, 127, 4); + ap.limit[0] = 40; + KUNIT_ASSERT_EQ(test, mt7932_j700_6g_gains(&ap, gain), 0); + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), 60); + KUNIT_EXPECT_EQ(test, body[49], 127); + /* Signed subtraction, overflow in both directions, and sentinel. */ + ap.count = 1; + ap.limit[0] = 0; + gain[0][0] = -7; + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), 60); + KUNIT_EXPECT_EQ(test, body[48], 7); + ap.limit[0] = 126; + KUNIT_EXPECT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), -ERANGE); + ap.limit[0] = -128; + gain[0][0] = 1; + KUNIT_EXPECT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), -ERANGE); +} + +static void mt_6g_legacy_vlp_finite_test(struct kunit *test) +{ + struct mt7932_6g_ap ap; + u8 ies[sizeof(lpi_ies)]; + + memcpy(ies, lpi_ies, sizeof(ies)); + ies[10] |= 16; /* VLP, ordinary finite default-client PSD TPE */ + /* Legacy admission selects no new class/TPE SETs, regardless of TPE. + * The caller retains the two original post-CID6b cached pages. + */ + KUNIT_EXPECT_EQ(test, + mt7932_6g_connect_ap(&ap, ies, sizeof(ies), 5, 0, + BIT_ULL(1)), + 1); + KUNIT_EXPECT_EQ(test, + mt7932_6g_connect_ap(&ap, ies, 14, 5, 0, BIT_ULL(1)), 1); + ies[10] = 3 | 16; /* AP advertises 160 MHz; STA still negotiates <=80 */ + KUNIT_EXPECT_EQ(test, + mt7932_6g_connect_ap(&ap, ies, sizeof(ies), 5, 0, + BIT_ULL(1)), + 1); + ies[10] = 2 | 16; + KUNIT_EXPECT_EQ(test, + mt7932_6g_connect_ap(&ap, ies, sizeof(ies), 5, 0, 0), 0); + KUNIT_EXPECT_EQ(test, ap.mode, MT7932_6G_VLP); + KUNIT_EXPECT_EQ(test, ap.limit[0], 24); + ies[10] = 2; /* LPI must never take the legacy VLP bypass. */ + KUNIT_EXPECT_EQ(test, + mt7932_6g_connect_ap(&ap, ies, sizeof(ies), 5, 0, + BIT_ULL(1)), + 0); + KUNIT_EXPECT_EQ(test, ap.mode, MT7932_6G_LPI); +} + +static void mt_6g_tpe_golden_test(struct kunit *test) +{ + /* Default-client EIRP: 20/40/80 MHz caps, including a signed cap and + * the no-limit sentinel. The unrelated subordinate envelope is ignored. + */ + static const u8 eirp[] = { 18, 40, 0xf6, 127 }; + static const u8 subordinate[] = { 82, 0x80, 0x80, 0x80 }; + static const u8 expected[60] = { + [1] = 2, [2] = 60, [4] = 1, 3, 1, 3, + [44] = 1, 3, 5, 3, + [48] = 24, 0xe6, 127, 127, + 24, 0xe6, 127, 127, 30, 0xec, 127, 127, + }; + struct mt7932_6g_ap ap = { + .mode = MT7932_6G_LPI, .primary = 5, + .limit = { 127, 127, 127, 127 }, + }; + s8 gain[3][4]; + u8 body[60]; + + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe(&ap, eirp, sizeof(eirp)), 0); + KUNIT_ASSERT_EQ(test, + mt7932_6g_tpe(&ap, subordinate, sizeof(subordinate)), 0); + KUNIT_ASSERT_EQ(test, mt7932_j700_6g_gains(&ap, gain), 0); + KUNIT_ASSERT_EQ(test, mt7932_6g_tpe_body(body, 60, &ap, gain), 60); + KUNIT_EXPECT_MEMEQ(test, body, expected, sizeof(expected)); +} + +static struct kunit_case mt_pmf_lpi_cases[] = { + KUNIT_CASE(mt_pmf_admission_test), + KUNIT_CASE(mt_pmf_body_test), + KUNIT_CASE(mt_6g_ap_test), + KUNIT_CASE(mt_6g_tpe_body_test), + KUNIT_CASE(mt_6g_j700_gain_test), + KUNIT_CASE(mt_6g_finite_projection_test), + KUNIT_CASE(mt_6g_legacy_vlp_finite_test), + KUNIT_CASE(mt_6g_tpe_golden_test), + {} +}; + +static struct kunit_suite mt_pmf_lpi_suite = { + .name = "mt7932-pmf-lpi", + .test_cases = mt_pmf_lpi_cases, +}; + +kunit_test_suite(mt_pmf_lpi_suite); + +MODULE_LICENSE("GPL"); diff --git a/drivers/net/wireless/mediatek/mt7932/power-test.c b/drivers/net/wireless/mediatek/mt7932/power-test.c new file mode 100644 index 0000000000000..4588ab006aaa3 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/power-test.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +#include "mt7932.h" + +struct mt_power_test_context { + struct kunit *test; + struct mt7932 *m; + unsigned int commands; + u8 mode; + int error; +}; + +static int mt_power_test_send(struct mt7932 *m, const void *data, size_t length) +{ + struct mt_power_test_context *ctx = m->power_test_context; + const u8 *body = data; + + lockdep_assert_held(&m->command_mutex); + KUNIT_EXPECT_EQ(ctx->test, length, (size_t)4); + if (length != 4) + return -EINVAL; + KUNIT_EXPECT_EQ(ctx->test, body[0], 0); /* owned BSS0 */ + KUNIT_EXPECT_EQ(ctx->test, body[2], 0); + KUNIT_EXPECT_EQ(ctx->test, body[3], 0); + ctx->commands++; + ctx->mode = body[1]; + return ctx->error; +} + +static void mt_power_test_device_release(struct device *dev) +{ +} + +static void mt_power_test_cleanup(void *data) +{ + struct mt7932 *m = data; + + cancel_work_sync(&m->power_work); + put_device(&m->pdev->dev); +} + +static int mt_power_test_init(struct kunit *test) +{ + struct mt_power_test_context *ctx; + struct mt7932 *m; + int ret; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + m = kunit_kzalloc(test, sizeof(*m), GFP_KERNEL); + if (!m) + return -ENOMEM; + m->pdev = kunit_kzalloc(test, sizeof(*m->pdev), GFP_KERNEL); + if (!m->pdev) + return -ENOMEM; + device_initialize(&m->pdev->dev); + m->pdev->dev.release = mt_power_test_device_release; + ret = dev_set_name(&m->pdev->dev, "mt7932-power-test"); + if (ret) { + put_device(&m->pdev->dev); + return ret; + } + mutex_init(&m->command_mutex); + spin_lock_init(&m->response_lock); + INIT_WORK(&m->power_work, mt_power_work); + ret = kunit_add_action_or_reset(test, mt_power_test_cleanup, m); + if (ret) + return ret; + m->rf_ready = true; + m->peer_valid = true; + m->power_test_send = mt_power_test_send; + m->power_test_context = ctx; + ctx->m = m; + ctx->test = test; + test->priv = ctx; + return 0; +} + +static int mt_power_test_prejoin(struct mt7932 *m) +{ + int ret; + + mutex_lock(&m->command_mutex); + ret = mt_power_init(m); + mutex_unlock(&m->command_mutex); + return ret; +} + +static void mt_power_test_run(struct mt7932 *m) +{ + schedule_work(&m->power_work); + flush_work(&m->power_work); +} + +static void mt_power_awake_after_join_test(struct kunit *test) +{ + struct mt_power_test_context *ctx = test->priv; + struct mt7932 *m = ctx->m; + unsigned int join; + + /* The old cache skipped the second SET in every mode-0 association. + * Exercise reconnect too: each session must submit both phases again. + */ + for (join = 0; join < 2; join++) { + m->connected = false; + KUNIT_ASSERT_EQ(test, mt_power_test_prejoin(m), 0); + KUNIT_EXPECT_EQ(test, ctx->commands, join * 2 + 1); + KUNIT_EXPECT_EQ(test, ctx->mode, 0); + KUNIT_EXPECT_FALSE(test, m->power_applied_valid); + m->connected = true; + mt_power_test_run(m); + KUNIT_EXPECT_EQ(test, ctx->commands, join * 2 + 2); + KUNIT_EXPECT_EQ(test, ctx->mode, 0); + KUNIT_EXPECT_TRUE(test, m->power_applied_valid); + /* Deduplication remains valid within this associated session. */ + mt_power_test_run(m); + KUNIT_EXPECT_EQ(test, ctx->commands, join * 2 + 2); + } +} + +static void mt_power_tim_gates_sleep_test(struct kunit *test) +{ + struct mt_power_test_context *ctx = test->priv; + struct mt7932 *m = ctx->m; + unsigned int mode; + + for (mode = 1; mode <= 2; mode++) { + m->power_mode = mode; + KUNIT_ASSERT_EQ(test, mt_power_test_prejoin(m), 0); + KUNIT_EXPECT_FALSE(test, m->power_tim); + m->connected = true; + mt_power_test_run(m); + KUNIT_EXPECT_EQ(test, ctx->commands, mode * 2 - 1); + KUNIT_EXPECT_FALSE(test, m->power_applied_valid); + m->power_tim = true; + mt_power_test_run(m); + KUNIT_EXPECT_EQ(test, ctx->commands, mode * 2); + KUNIT_EXPECT_EQ(test, ctx->mode, mode); + KUNIT_EXPECT_EQ(test, m->power_applied, mode); + } +} + +static void mt_power_failed_submission_retry_test(struct kunit *test) +{ + struct mt_power_test_context *ctx = test->priv; + struct mt7932 *m = ctx->m; + + ctx->error = -EIO; + KUNIT_EXPECT_EQ(test, mt_power_test_prejoin(m), -EIO); + KUNIT_EXPECT_FALSE(test, m->power_applied_valid); + ctx->error = 0; + KUNIT_ASSERT_EQ(test, mt_power_test_prejoin(m), 0); + m->connected = true; + ctx->error = -EIO; + mt_power_test_run(m); + KUNIT_EXPECT_FALSE(test, m->power_applied_valid); + ctx->error = 0; + mt_power_test_run(m); + KUNIT_EXPECT_EQ(test, ctx->commands, 4U); + KUNIT_EXPECT_TRUE(test, m->power_applied_valid); +} + +static void mt_power_terminal_session_test(struct kunit *test) +{ + struct mt_power_test_context *ctx = test->priv; + struct mt7932 *m = ctx->m; + unsigned int state; + + KUNIT_ASSERT_EQ(test, mt_power_test_prejoin(m), 0); + for (state = 0; state < 6; state++) { + m->stopping = state == 0; + m->rf_ready = state != 1; + m->connected = state != 2; + m->disconnecting = state == 3; + m->peer_valid = state != 4; + m->link_failed = state == 5; + mt_power_test_run(m); + KUNIT_EXPECT_EQ(test, ctx->commands, 1U); + KUNIT_EXPECT_FALSE(test, m->power_applied_valid); + } +} + +static struct kunit_case mt_power_test_cases[] = { + KUNIT_CASE(mt_power_awake_after_join_test), + KUNIT_CASE(mt_power_tim_gates_sleep_test), + KUNIT_CASE(mt_power_failed_submission_retry_test), + KUNIT_CASE(mt_power_terminal_session_test), + {} +}; + +static struct kunit_suite mt_power_test_suite = { + .name = "mt7932-power", + .init = mt_power_test_init, + .test_cases = mt_power_test_cases, +}; + +kunit_test_suite(mt_power_test_suite); diff --git a/drivers/net/wireless/mediatek/mt7932/power.c b/drivers/net/wireless/mediatek/mt7932/power.c index 62537ae867d90..f3c1c42863c9c 100644 --- a/drivers/net/wireless/mediatek/mt7932/power.c +++ b/drivers/net/wireless/mediatek/mt7932/power.c @@ -10,7 +10,12 @@ static int mt_power_submit(struct mt7932 *m, u8 mode) u8 body[4] = { 0, mode, 0, 0 }; /* owned BSS0, not a station/WTBL ID */ int ret; - ret = mt_request(m, 5, true, true, false, body, sizeof(body)); +#if IS_ENABLED(CONFIG_MT7932_RF_KUNIT_TEST) + if (m->power_test_send) + ret = m->power_test_send(m, body, sizeof(body)); + else +#endif + ret = mt_request(m, 5, true, true, false, body, sizeof(body)); m->power_applied_valid = !ret; if (!ret) m->power_applied = mode; @@ -22,11 +27,19 @@ static int mt_power_submit(struct mt7932 *m, u8 mode) int mt_power_init(struct mt7932 *m) { unsigned long flags; + int ret; spin_lock_irqsave(&m->response_lock, flags); m->power_tim = false; spin_unlock_irqrestore(&m->response_lock, flags); - return mt_power_submit(m, 0); + ret = mt_power_submit(m, 0); + /* This SET precedes EA/40. It cannot attest to the power policy of the + * associated BSS, whose authentication and scheduling firmware owns. + * Invalidate even on success so power_work submits the desired mode + * after association, including mode 0 when userspace disabled PS. + */ + m->power_applied_valid = false; + return ret; } /* command_mutex fences BSS reuse/retirement while the SET is submitted. */ diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c index 544497108d4e9..b67b276d17979 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c +++ b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c @@ -551,6 +551,164 @@ static u8 *pkg_build_v2(struct kunit *test, const char *alpha2, u64 primaries, s return data; } +static u8 *pkg_build_v4(struct kunit *test, size_t *size) +{ + size_t base_size; + u8 *base = pkg_build(test, "AU", &base_size), *data, *directory; + unsigned int i, j; + + *size = base_size + 16 + 2 * 13476; + data = kunit_kzalloc(test, *size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, data); + memcpy(data, base, base_size); + put_unaligned_le16(4, data + 4); + put_unaligned_le32(*size, data + 12); + directory = data + base_size; + memcpy(directory, "6GCL", 4); + directory[4] = 1; + directory[5] = 2; + directory[6] = 1; + put_unaligned_le32(16 + 2 * 13476, directory + 8); + for (i = 0; i < 2; i++) { + u8 *section = directory + 16 + i * 13476; + unsigned int channel = 1; + + section[0] = i + 1; + section[1] = i; + section[2] = 1; + section[3] = 110; + put_unaligned_le32(13476, section + 4); + put_unaligned_le64(BIT_ULL(i ? 1 : 9), section + 8); + memcpy(section + 16, "AU", 2); + for (j = 0; j < 110; j++) { + u8 *record = section + 56 + j * 122; + + record[0] = j ? channel : 2; + memset(record + 1, i ? 40 : 20, 121); + if (j) { + channel += 2; + while (channel == 31 || channel == 63 || + channel == 95 || channel == 127 || + channel == 159 || channel == 191 || + channel == 223 || channel == 231) + channel += 2; + } + } + } + return data; +} + +static void mt7932_policy_v4_classes_test(struct kunit *test) +{ + struct mt7932_policy *policy = + kunit_kzalloc(test, sizeof(*policy), GFP_KERNEL); + size_t size, base_size; + u8 *data = pkg_build_v4(test, &size), *body, *ext; + unsigned int first, total = 0; + int length; + + KUNIT_ASSERT_NOT_NULL(test, policy); + body = kunit_kzalloc(test, 1021, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, body); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(policy, data, size, "AU"), 0); + KUNIT_EXPECT_EQ(test, policy->six_class[MT7932_6G_LPI].channels, + BIT_ULL(1)); + KUNIT_EXPECT_EQ(test, policy->six_class[MT7932_6G_VLP].channels, + BIT_ULL(9)); + KUNIT_EXPECT_TRUE(test, + mt7932_policy_permits(policy, NL80211_BAND_6GHZ, 5)); + KUNIT_EXPECT_TRUE(test, + mt7932_policy_permits(policy, NL80211_BAND_6GHZ, 37)); + KUNIT_EXPECT_FALSE(test, + mt7932_policy_permits(policy, NL80211_BAND_6GHZ, 1)); + KUNIT_EXPECT_LT(test, + mt7932_6g_class_page(body, 1020, policy, MT7932_6G_SP, + "AU", 0), + 0); + for (first = 0; first < 110; first += 8) { + length = mt7932_6g_class_page(body, 1020, policy, MT7932_6G_LPI, + "AU", first); + KUNIT_ASSERT_GT(test, length, 0); + KUNIT_EXPECT_EQ(test, body[0], 0); + KUNIT_EXPECT_EQ(test, body[1], 0x1a); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 2), length); + KUNIT_EXPECT_EQ(test, body[5], 3); + KUNIT_EXPECT_EQ(test, body[6], first == 104); + KUNIT_EXPECT_EQ(test, body[45], 40); + total += body[4]; + } + KUNIT_EXPECT_EQ(test, total, 110U); + KUNIT_EXPECT_EQ(test, length, 776); + KUNIT_EXPECT_EQ(test, + mt7932_6g_class_page(body, 1020, policy, MT7932_6G_VLP, + "AU", 0), + 1020); + KUNIT_EXPECT_EQ(test, body[1], 0x2a); + KUNIT_EXPECT_EQ(test, body[45], 20); + KUNIT_EXPECT_LT(test, + mt7932_6g_class_page(body, 1019, policy, MT7932_6G_LPI, + "AU", 0), + 0); + KUNIT_EXPECT_LT(test, + mt7932_6g_class_page(body, 1020, policy, MT7932_6G_LPI, + "AU", 1), + 0); + base_size = size - 16 - 2 * 13476; + ext = data + base_size; + /* Unbound v3 cannot acquire runtime authority through a path rename. */ + put_unaligned_le16(3, data + 4); + ext[6] = 0; + KUNIT_EXPECT_LT(test, mt7932_policy_parse(policy, data, size, "AU"), 0); + put_unaligned_le16(4, data + 4); + KUNIT_EXPECT_LT(test, mt7932_policy_parse(policy, data, size, "AU"), 0); + ext[6] = 1; + KUNIT_EXPECT_LT(test, mt7932_policy_parse(policy, data, size - 1, "AU"), + 0); + KUNIT_EXPECT_LT(test, mt7932_policy_parse(policy, data, size, "NZ"), 0); + ext[16 + 13476] = 3; /* never serialize SP */ + KUNIT_EXPECT_LT(test, mt7932_policy_parse(policy, data, size, "AU"), 0); + ext[16 + 13476] = 2; + ext[16 + 56] = 1; /* first record must be channel 2 */ + KUNIT_EXPECT_LT(test, mt7932_policy_parse(policy, data, size, "AU"), 0); +} + +/* The public producer's 32-byte 1SS channel pad is copied verbatim. It is + * independent of the primary permission mask and of rate-support accounting. + */ +static void mt7932_policy_6g_pad_test(struct kunit *test) +{ + static const u8 expected[44] = { + [1] = 0x1a, [2] = 0xfc, 3, 8, 3, + [8] = 'A', 'U', + [12] = 0x20, [16] = 0x20, [41] = 2, + }; + struct mt7932_policy *policy = + kunit_kzalloc(test, sizeof(*policy), GFP_KERNEL); + size_t size; + u8 *data = pkg_build_v4(test, &size), *body, *pad; + + KUNIT_ASSERT_NOT_NULL(test, policy); + body = kunit_kmalloc(test, 1020, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, body); + /* Directory plus VLP section, then LPI header and its channel pad. */ + pad = data + size - 13476 + 24; + pad[0] = 0x20; /* channel 5 */ + pad[4] = 0x20; /* channel 37, not rate index 37 */ + pad[29] = 2; /* channel 233 */ + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(policy, data, size, "AU"), 0); + KUNIT_EXPECT_EQ(test, policy->six_class[MT7932_6G_LPI].channels, + BIT_ULL(1)); + KUNIT_ASSERT_EQ(test, mt7932_6g_class_page(body, 1020, policy, + MT7932_6G_LPI, "AU", 0), 1020); + KUNIT_EXPECT_MEMEQ(test, body, expected, sizeof(expected)); + KUNIT_ASSERT_EQ(test, mt7932_6g_class_page(body, 1020, policy, + MT7932_6G_LPI, "AU", 104), 776); + KUNIT_EXPECT_MEMEQ(test, body + 12, pad, 32); + KUNIT_ASSERT_EQ(test, mt7932_6g_class_page(body, 1020, policy, + MT7932_6G_VLP, "AU", 0), 1020); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(body + 12, 0, 32), NULL); +} + #define FAKE_FILES 8 #define FAKE_LOG 16 @@ -590,8 +748,9 @@ static void fake_put(void *ctx, struct mt7932_policy_file *file) { struct fake_fs *fs = ctx; - fs->held--; + fs->held -= file->class_handle ? 2 : 1; file->handle = NULL; + file->class_handle = NULL; } static void fake_add(struct fake_fs *fs, const char *path, const u8 *data, size_t size) @@ -665,10 +824,11 @@ static void mt7932_policy_order_test(struct kunit *test) char path[MT7932_POLICY_PATH_MAX]; struct select_env *env = select_env(test); - KUNIT_ASSERT_EQ(test, mt7932_policy_candidates("AU", sources), 3); - KUNIT_EXPECT_EQ(test, sources[0], MT7932_POLICY_V2); - KUNIT_EXPECT_EQ(test, sources[1], MT7932_POLICY_V1); - KUNIT_EXPECT_EQ(test, sources[2], MT7932_POLICY_WORLD); + KUNIT_ASSERT_EQ(test, mt7932_policy_candidates("AU", sources), 4); + KUNIT_EXPECT_EQ(test, sources[0], MT7932_POLICY_V4); + KUNIT_EXPECT_EQ(test, sources[1], MT7932_POLICY_V2); + KUNIT_EXPECT_EQ(test, sources[2], MT7932_POLICY_V1); + KUNIT_EXPECT_EQ(test, sources[3], MT7932_POLICY_WORLD); KUNIT_ASSERT_EQ(test, mt7932_policy_candidates("00", sources), 1); KUNIT_EXPECT_EQ(test, sources[0], MT7932_POLICY_WORLD); mt7932_policy_path(path, sizeof(path), MT7932_POLICY_V2, "AU"); @@ -680,13 +840,18 @@ static void mt7932_policy_order_test(struct kunit *test) /* Nothing installed: every candidate is asked for, in order. */ KUNIT_EXPECT_EQ(test, select_run(env, "NZ", false), -ENOENT); - KUNIT_ASSERT_EQ(test, env->fs.requests, 3); - KUNIT_EXPECT_STREQ(test, env->fs.log[0], "mediatek/mt7932/policy-v2/NZ.bin"); - KUNIT_EXPECT_STREQ(test, env->fs.log[1], "mediatek/mt7932/policy/NZ.bin"); - KUNIT_EXPECT_STREQ(test, env->fs.log[2], "mediatek/mt7932/policy/world-XZ.bin"); + KUNIT_ASSERT_EQ(test, env->fs.requests, 4); + KUNIT_EXPECT_STREQ(test, env->fs.log[0], + "mediatek/mt7932/policy-v4/NZ.bin"); + KUNIT_EXPECT_STREQ(test, env->fs.log[1], + "mediatek/mt7932/policy-v2/NZ.bin"); + KUNIT_EXPECT_STREQ(test, env->fs.log[2], + "mediatek/mt7932/policy/NZ.bin"); + KUNIT_EXPECT_STREQ(test, env->fs.log[3], + "mediatek/mt7932/policy/world-XZ.bin"); KUNIT_EXPECT_PTR_EQ(test, env->file.handle, NULL); KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_NONE); - KUNIT_EXPECT_EQ(test, env->choice.n_tried, 3); + KUNIT_EXPECT_EQ(test, env->choice.n_tried, 4); /* cfg80211's numeric codes have no package of their own. */ KUNIT_EXPECT_EQ(test, mt7932_policy_candidates("98", sources), 1); @@ -717,7 +882,7 @@ static void mt7932_policy_v2_preferred_test(struct kunit *test) fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size); KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); - KUNIT_EXPECT_EQ(test, env->fs.requests, 1); + KUNIT_EXPECT_EQ(test, env->fs.requests, 2); KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_V2); KUNIT_EXPECT_EQ(test, env->choice.version, 2); KUNIT_EXPECT_EQ(test, env->choice.size, v2_size); @@ -738,8 +903,8 @@ static void mt7932_policy_v2_preferred_test(struct kunit *test) /* Malformed v2: v1 is used and 6 GHz leaves the firmware domain. */ put_unaligned_le32(v2_size - 1, v2 + 12); KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); - KUNIT_EXPECT_EQ(test, env->fs.requests, 2); - KUNIT_EXPECT_EQ(test, env->choice.tried[0].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->fs.requests, 3); + KUNIT_EXPECT_EQ(test, env->choice.tried[1].error, -EINVAL); KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_V1); KUNIT_EXPECT_EQ(test, env->choice.version, 1); KUNIT_EXPECT_EQ(test, env->choice.fallback, 0); @@ -752,6 +917,64 @@ static void mt7932_policy_v2_preferred_test(struct kunit *test) KUNIT_EXPECT_EQ(test, env->fs.held, 0); } +static void mt7932_policy_v4_preferred_test(struct kunit *test) +{ + struct select_env *env = select_env(test); + struct mt7932_policy *baseline = + kunit_kzalloc(test, sizeof(*baseline), GFP_KERNEL); + size_t size, v2_size; + u8 *data = pkg_build_v4(test, &size); + u8 *v2 = pkg_build_v2(test, "AU", BIT_ULL(9), &v2_size); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, baseline); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(baseline, v2, v2_size, "AU"), + 0); + fake_add(&env->fs, "mediatek/mt7932/policy-v4/AU.bin", data, size); + fake_add(&env->fs, "mediatek/mt7932/policy-v2/AU.bin", v2, v2_size); + KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_V2); + KUNIT_EXPECT_EQ(test, env->choice.version, 2); + KUNIT_EXPECT_TRUE(test, env->choice.class_valid); + KUNIT_EXPECT_EQ(test, env->choice.six_primaries, 2); + KUNIT_EXPECT_EQ(test, env->policy.legacy_channels, BIT_ULL(9)); + KUNIT_EXPECT_EQ(test, env->fs.requests, 2); + KUNIT_EXPECT_EQ(test, env->fs.held, 2); + /* Every original startup CID/body, and both post-queue association + * pages, are retained byte for byte even with the LPI sidecar. + */ + KUNIT_ASSERT_EQ(test, env->policy.six_pages, baseline->six_pages); + for (i = 0; i < baseline->six_pages; i++) { + KUNIT_EXPECT_EQ(test, env->policy.six_cid[i], + baseline->six_cid[i]); + KUNIT_EXPECT_EQ(test, env->policy.six_length[i], + baseline->six_length[i]); + KUNIT_EXPECT_MEMEQ(test, env->policy.six_page[i], + baseline->six_page[i], + baseline->six_length[i]); + } + for (i = 0; i < 2; i++) + KUNIT_EXPECT_MEMEQ(test, env->policy.assoc_power[i], + baseline->assoc_power[i], 1020); + /* Extra LP channels must not change the legacy VLP numeric clamp. */ + env->reg.power[mt7932_6g_reg_index(5)] = -1; + env->reg.power[mt7932_6g_reg_index(37)] = 14; + KUNIT_EXPECT_EQ(test, + mt7932_6g_limits_mask(&env->reg, + env->policy.legacy_channels) + .eirp, + 28); + KUNIT_EXPECT_EQ(test, mt7932_6g_limits(&env->reg).eirp, -2); + data[4] = 3; + KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_V2); + KUNIT_EXPECT_FALSE(test, env->choice.class_valid); + KUNIT_EXPECT_EQ(test, env->choice.tried[0].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->choice.six_primaries, 1); + fake_put(&env->fs, &env->file); + KUNIT_EXPECT_EQ(test, env->fs.held, 0); +} + /* Each directory accepts one version, and the size limits hold. */ static void mt7932_policy_version_test(struct kunit *test) { @@ -771,8 +994,8 @@ static void mt7932_policy_version_test(struct kunit *test) fake_add(&env->fs, "mediatek/mt7932/policy/AU.bin", v2, v2_size); fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size); KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); - KUNIT_EXPECT_EQ(test, env->choice.tried[0].error, -ENOENT); - KUNIT_EXPECT_EQ(test, env->choice.tried[1].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->choice.tried[1].error, -ENOENT); + KUNIT_EXPECT_EQ(test, env->choice.tried[2].error, -EINVAL); KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_WORLD); KUNIT_EXPECT_EQ(test, env->choice.fallback, -EINVAL); KUNIT_EXPECT_EQ(test, env->choice.six_primaries, 0); @@ -782,7 +1005,7 @@ static void mt7932_policy_version_test(struct kunit *test) fake_add(&env->fs, "mediatek/mt7932/policy-v2/AU.bin", v1, v1_size); fake_add(&env->fs, "mediatek/mt7932/policy/AU.bin", v1, v1_size); KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); - KUNIT_EXPECT_EQ(test, env->choice.tried[0].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->choice.tried[1].error, -EINVAL); KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_V1); /* Over 16 KiB as v1, over 64 KiB as v2. */ @@ -795,8 +1018,8 @@ static void mt7932_policy_version_test(struct kunit *test) fake_add(&env->fs, "mediatek/mt7932/policy/AU.bin", big, big_size); fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size); KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); - KUNIT_EXPECT_EQ(test, env->choice.tried[0].error, -EINVAL); KUNIT_EXPECT_EQ(test, env->choice.tried[1].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->choice.tried[2].error, -EINVAL); KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_WORLD); /* Another country's package under this country's name. */ @@ -932,6 +1155,29 @@ static void mt7932_policy_switch_test(struct kunit *test) KUNIT_EXPECT_EQ(test, env->fs.held, 0); } +/* Unbound class archives must never be consumed as firmware command pages. */ +static void mt7932_policy_unbound_test(struct kunit *test) +{ + struct select_env *env = select_env(test); + struct mt7932_policy policy; + size_t size, xz_size; + u8 *data, *xz; + + data = pkg_build_v2(test, "AU", BIT_ULL(1), &size); + xz = pkg_build(test, "XZ", &xz_size); + put_unaligned_le16(3, data + 4); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "AU"), + -EINVAL); + fake_add(&env->fs, "mediatek/mt7932/policy-v2/AU.bin", data, size); + fake_add(&env->fs, "mediatek/mt7932/policy/AU.bin", data, size); + fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size); + KUNIT_ASSERT_EQ(test, select_run(env, "AU", false), 0); + KUNIT_EXPECT_EQ(test, env->choice.tried[1].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->choice.tried[2].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_WORLD); + KUNIT_EXPECT_EQ(test, env->choice.six_primaries, 0); +} + static struct kunit_case mt7932_policy_test_cases[] = { KUNIT_CASE(mt7932_reg_relaxation_test), KUNIT_CASE(mt7932_policy_valid_test), @@ -947,8 +1193,12 @@ static struct kunit_case mt7932_policy_test_cases[] = { KUNIT_CASE(mt7932_6g_power_limit_test), KUNIT_CASE(mt7932_policy_corruption_test), KUNIT_CASE(mt7932_policy_order_test), + KUNIT_CASE(mt7932_policy_v4_classes_test), + KUNIT_CASE(mt7932_policy_6g_pad_test), + KUNIT_CASE(mt7932_policy_v4_preferred_test), KUNIT_CASE(mt7932_policy_v2_preferred_test), KUNIT_CASE(mt7932_policy_version_test), + KUNIT_CASE(mt7932_policy_unbound_test), KUNIT_CASE(mt7932_policy_fallback_test), KUNIT_CASE(mt7932_policy_switch_test), {} diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory.h b/drivers/net/wireless/mediatek/mt7932/regulatory.h index 4bf7a78a4e113..24dcbe6336599 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory.h +++ b/drivers/net/wireless/mediatek/mt7932/regulatory.h @@ -9,17 +9,26 @@ #include #include "channels.h" +#include "six.h" #define MT7932_POLICY_TABLES 13 #define MT7932_POLICY_MODES 3 #define MT7932_POLICY_6G_PAGES 32 +#define MT7932_6G_RECORDS 110 + +struct mt7932_6g_table { + const u8 *records, *bitmap; + u64 channels; +}; + struct mt7932_policy { const u8 *mode[MT7932_POLICY_MODES]; const u8 *table[MT7932_POLICY_TABLES]; u16 length[MT7932_POLICY_TABLES]; u8 modes, six_pages; - u64 six_channels; + struct mt7932_6g_table six_class[3]; + u64 six_channels, legacy_channels; const u8 *six_page[MT7932_POLICY_6G_PAGES]; u16 six_length[MT7932_POLICY_6G_PAGES]; u8 six_cid[MT7932_POLICY_6G_PAGES]; @@ -56,6 +65,7 @@ static inline int mt7932_policy_6g_parse(struct mt7932_policy *out, memcmp(data + 16, alpha2, 2) || data[18] != 1 || memchr_inv(data + 19, 0, 5)) return -EINVAL; out->six_channels = get_unaligned_le64(data + 8); + out->legacy_channels = out->six_channels; if (!out->six_channels || (out->six_channels & ~GENMASK_ULL(58, 0))) return -EINVAL; out->six_pages = data[6]; @@ -106,6 +116,113 @@ static inline int mt7932_policy_6g_parse(struct mt7932_policy *out, return at == size && out->assoc_power[0] && out->assoc_power[1] ? 0 : -EINVAL; } +/* Version 4 binds the unchanged v3 class data to the SPEC's CID5d ABI. + * Version 3, binding revision zero, remains an unbound archive and is refused. + */ +static inline int mt7932_policy_classes_parse(struct mt7932_policy *out, + const u8 *data, size_t size, + const u8 alpha2[2]) +{ + size_t at = 16; + unsigned int i, previous = 0; + + if (size < at || memcmp(data, "6GCL", 4) || data[4] != 1 || !data[5] || + data[5] > 2 || data[6] != 1 || data[7] || + get_unaligned_le32(data + 8) != size || + get_unaligned_le32(data + 12)) + return -EINVAL; + for (i = 0; i < data[5]; i++) { + const u8 *section = data + at, *record; + struct mt7932_6g_table *table; + unsigned int id, mode, j, channel = 1; + u64 mask; + + if (size - at < 56) + return -EINVAL; + id = section[0]; + if ((id != 1 && id != 2) || id <= previous || + section[1] != id - 1 || section[2] != 1 || + section[3] != MT7932_6G_RECORDS || + get_unaligned_le32(section + 4) != + 56 + MT7932_6G_RECORDS * 122 || + size - at < 56 + MT7932_6G_RECORDS * 122 || + memcmp(section + 16, alpha2, 2) || + memchr_inv(section + 18, 0, 6)) + return -EINVAL; + mask = get_unaligned_le64(section + 8); + if (!mask || (mask & ~GENMASK_ULL(58, 0))) + return -EINVAL; + record = section + 56; + for (j = 0; j < MT7932_6G_RECORDS; j++, record += 122) { + unsigned int k; + bool power = false; + + /* Authenticated matrix order: channel 2, then primary and + * bonded centers up to 160 MHz (excluding channel 231). + */ + if (record[0] != (j ? channel : 2)) + return -EINVAL; + if (j) { + channel += 2; + while (channel == 31 || channel == 63 || + channel == 95 || channel == 127 || + channel == 159 || channel == 191 || + channel == 223 || channel == 231) + channel += 2; + } + if (record[0] % 4 != 1 || + !(mask & BIT_ULL((record[0] - 1) / 4))) + continue; + for (k = 1; k < 122; k++) + power |= record[k] != 0xc4 && record[k] != 0x7f; + if (!power) + return -EINVAL; + } + mode = id == 1 ? MT7932_6G_VLP : MT7932_6G_LPI; + table = &out->six_class[mode]; + table->channels = mask; + table->bitmap = section + 24; + table->records = section + 56; + out->six_channels |= mask; + previous = id; + at += 56 + MT7932_6G_RECORDS * 122; + } + return at == size && out->six_class[MT7932_6G_LPI].records ? 0 : + -EINVAL; +} + +/* Normal infrastructure publication: submode zero, no invented class bits. + * Class IDs and data must agree; callers cannot relabel VLP data as LPI. + */ +static inline int mt7932_6g_class_page(u8 *body, size_t capacity, + const struct mt7932_policy *policy, + unsigned int mode, const u8 alpha2[2], + unsigned int first) +{ + const struct mt7932_6g_table *table; + unsigned int count, length; + + if (mode != MT7932_6G_LPI && mode != MT7932_6G_VLP) + return -EOPNOTSUPP; + table = &policy->six_class[mode]; + if (!table->records || first >= MT7932_6G_RECORDS || first % 8) + return -EINVAL; + count = min(8U, MT7932_6G_RECORDS - first); + length = 44 + count * 122; + if (capacity < length) + return -ENOSPC; + memset(body, 0, 44); + body[1] = 0x0a | (mode << 4); + put_unaligned_le16(length, body + 2); + body[4] = count; + body[5] = 3; + body[6] = first + count == MT7932_6G_RECORDS; + memcpy(body + 8, alpha2, 2); + memcpy(body + 12, table->bitmap, 32); + memcpy(body + 44, table->records + first * 122, count * 122); + return length; +} + static inline int mt7932_policy_parse(struct mt7932_policy *out, const u8 *data, size_t size, const u8 alpha2[2]) @@ -115,11 +232,13 @@ static inline int mt7932_policy_parse(struct mt7932_policy *out, memset(out, 0, sizeof(*out)); version = size >= 6 ? get_unaligned_le16(data + 4) : 0; - if (size < at || size > (version == 2 ? 65536 : 16384) || memcmp(data, "J7RP", 4) || - (version != 1 && version != 2) || memcmp(data + 6, alpha2, 2) || - data[8] < 2 || data[8] > MT7932_POLICY_MODES || - data[9] != MT7932_POLICY_TABLES || data[10] || data[11] || - get_unaligned_le32(data + 12) != size) + if (size < at || + size > (version == 2 || version == 4 ? 65536 : 16384) || + memcmp(data, "J7RP", 4) || + (version != 1 && version != 2 && version != 4) || + memcmp(data + 6, alpha2, 2) || data[8] < 2 || + data[8] > MT7932_POLICY_MODES || data[9] != MT7932_POLICY_TABLES || + data[10] || data[11] || get_unaligned_le32(data + 12) != size) return -EINVAL; out->modes = data[8]; for (i = 0; i < out->modes; i++, at += 328) { @@ -175,6 +294,9 @@ static inline int mt7932_policy_parse(struct mt7932_policy *out, out->length[i] = n; at += n; } + if (version == 4) + return mt7932_policy_classes_parse(out, data + at, size - at, + alpha2); return version == 2 ? mt7932_policy_6g_parse(out, data + at, size - at, alpha2) : at == size ? 0 : -EINVAL; } @@ -294,7 +416,7 @@ static inline unsigned int mt7932_6g_reg_index(unsigned int channel) } static inline struct mt7932_6g_limits -mt7932_6g_limits(const struct mt7932_reg_snapshot *reg) +mt7932_6g_limits_mask(const struct mt7932_reg_snapshot *reg, u64 channels) { struct mt7932_6g_limits limits = { .eirp = 28, .psd = 256 }; unsigned int i, begin = reg->domain[8] + reg->domain[9]; @@ -303,6 +425,8 @@ mt7932_6g_limits(const struct mt7932_reg_snapshot *reg) const u8 *entry = reg->domain + 12 + i * 8; unsigned int index = mt7932_6g_reg_index(get_unaligned_le16(entry)); + if (!(channels & BIT_ULL((get_unaligned_le16(entry) - 1) / 4))) + continue; limits.eirp = min_t(int, limits.eirp, 2 * reg->power[index]); if (get_unaligned_le32(entry + 4) & IEEE80211_CHAN_PSD) limits.psd = min_t(int, limits.psd, 2 * reg->psd[index]); @@ -310,6 +434,12 @@ mt7932_6g_limits(const struct mt7932_reg_snapshot *reg) return limits; } +static inline struct mt7932_6g_limits +mt7932_6g_limits(const struct mt7932_reg_snapshot *reg) +{ + return mt7932_6g_limits_mask(reg, ~0ULL); +} + /* Remove the channels the country package forbids from a CID0f domain. */ static inline void mt7932_policy_filter(struct mt7932_reg_snapshot *reg, const struct mt7932_policy *policy) @@ -344,18 +474,21 @@ static inline void mt7932_policy_filter(struct mt7932_reg_snapshot *reg, */ enum mt7932_policy_source { MT7932_POLICY_NONE, - MT7932_POLICY_V2, /* policy-v2/.bin */ - MT7932_POLICY_V1, /* policy/.bin */ - MT7932_POLICY_WORLD, /* policy/world-XZ.bin */ + MT7932_POLICY_V4, /* policy-v4/.bin */ + MT7932_POLICY_V2, /* policy-v2/.bin */ + MT7932_POLICY_V1, /* policy/.bin */ + MT7932_POLICY_WORLD, /* policy/world-XZ.bin */ }; -#define MT7932_POLICY_CANDIDATES 3 +#define MT7932_POLICY_CANDIDATES 4 #define MT7932_POLICY_PATH_MAX 48 struct mt7932_policy_file { const void *handle; const u8 *data; size_t size; + /* Optional v4 LPI sidecar; the primary remains the original v2. */ + const void *class_handle; }; /* get() returns 0 with @file filled in, or an error; put() releases it. */ @@ -372,6 +505,8 @@ struct mt7932_policy_choice { u16 version; u32 size; u8 sha256[SHA256_DIGEST_SIZE]; + u8 class_sha256[SHA256_DIGEST_SIZE]; + bool class_valid; /* 6 GHz primaries left in the firmware domain. */ u8 six_primaries; /* Why the country's own packages were not used: -ENOENT, -EINVAL, or diff --git a/drivers/net/wireless/mediatek/mt7932/scan.h b/drivers/net/wireless/mediatek/mt7932/scan.h index 371b829026024..3be04716da001 100644 --- a/drivers/net/wireless/mediatek/mt7932/scan.h +++ b/drivers/net/wireless/mediatek/mt7932/scan.h @@ -8,6 +8,32 @@ /* CID03 carries this many band/channel slots, from 0xa0 up to 0xe0. */ #define MT7932_SCAN_CHANNELS 32 +/* cfg80211 owns PSC selection and RNR expansion for split scans. Until the + * firmware target list can carry its BSSID/short-SSID constraints, listen on + * those channels even when the request also contains an ordinary SSID. + */ +static inline bool +mt7932_scan_listen(const struct cfg80211_scan_request *request, + const struct ieee80211_channel *channel, u64 legacy_channels) +{ + unsigned int i; + + if (!request->n_ssids || + (channel->flags & (IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_RADAR))) + return true; + if (channel->band != NL80211_BAND_6GHZ) + return false; + if ((request->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ) || + request->n_6ghz_params || + !mt7932_channel_band(channel->band, channel->hw_value) || + !(legacy_channels & BIT_ULL((channel->hw_value - 1) / 4))) + return true; + for (i = 0; i < request->n_ssids; i++) + if (request->ssids[i].ssid_len) + return false; + return true; +} + /* Selected ordinary TX17/fullmac wildcard scan, not a pre-RF gate. * Caller must own BSS0 and filter cfg80211 DISABLED channels first. NO_IR * channels require mt7932_scan_passive() before submission. The body uses @@ -85,6 +111,15 @@ static inline void mt7932_scan_disconnected_passive(u8 body[0x4d4]) put_unaligned_le16(233, body + 0x4ca); } +/* Wait for the whole disconnected passive batch, not just one channel. + * Retain the existing five-second command/completion margin and active wait. + */ +static inline unsigned int mt7932_scan_discovery_timeout(const u8 body[0x4d4]) +{ + return 5000 + (body[2] ? 0 : + body[0x9f] * get_unaligned_le16(body + 0x4ca)); +} + /* CID03 carries up to four directed SSIDs in 36-byte slots (le32 length, * 32 bytes) at offset 8; selector 8 marks a directed request and byte 4 * counts the populated slots. Wildcard entries occupy no slot. A directed diff --git a/drivers/net/wireless/mediatek/mt7932/six-test.c b/drivers/net/wireless/mediatek/mt7932/six-test.c index c7578a4985e2c..59430b8483c45 100644 --- a/drivers/net/wireless/mediatek/mt7932/six-test.c +++ b/drivers/net/wireless/mediatek/mt7932/six-test.c @@ -143,6 +143,105 @@ static void mt7932_he_6g_test(struct kunit *test) KUNIT_EXPECT_EQ(test, mt7932_channel_6g_center(225, true), 0U); } +static void mt7932_scan_6g_policy_test(struct kunit *test) +{ + struct cfg80211_ssid ssids[2] = { {}, { .ssid_len = 6 } }; + struct cfg80211_scan_request request = { .ssids = ssids }; + struct ieee80211_channel channel = { + .band = NL80211_BAND_6GHZ, .hw_value = 5, + }; + u64 legacy = BIT_ULL(1) | BIT_ULL(9); + unsigned int i; + + KUNIT_EXPECT_TRUE(test, mt7932_scan_listen(&request, &channel, legacy)); + request.n_ssids = 1; /* wildcard on a PSC still listens */ + KUNIT_EXPECT_TRUE(test, mt7932_scan_listen(&request, &channel, legacy)); + request.n_ssids = 2; + KUNIT_EXPECT_FALSE(test, mt7932_scan_listen(&request, &channel, legacy)); + KUNIT_EXPECT_TRUE(test, mt7932_scan_listen(&request, &channel, 0)); + request.flags = NL80211_SCAN_FLAG_COLOCATED_6GHZ; + /* RNR constraints must not become an unconstrained directed probe, + * including a PSC with psc_no_listen and a non-PSC target channel. + */ + for (i = 1; i <= 233; i += 4) { + channel.hw_value = i; + KUNIT_EXPECT_TRUE(test, + mt7932_scan_listen(&request, &channel, ~0ULL)); + } + request.flags = 0; + request.n_6ghz_params = 1; + channel.hw_value = 37; + KUNIT_EXPECT_TRUE(test, mt7932_scan_listen(&request, &channel, legacy)); + request.n_6ghz_params = 0; + channel.flags = IEEE80211_CHAN_NO_IR; + KUNIT_EXPECT_TRUE(test, mt7932_scan_listen(&request, &channel, legacy)); + channel.flags = IEEE80211_CHAN_RADAR; + KUNIT_EXPECT_TRUE(test, mt7932_scan_listen(&request, &channel, legacy)); + channel.flags = 0; + channel.band = NL80211_BAND_2GHZ; + request.flags = NL80211_SCAN_FLAG_COLOCATED_6GHZ; + KUNIT_EXPECT_FALSE(test, mt7932_scan_listen(&request, &channel, legacy)); +} + +static void mt7932_scan_6g_golden_test(struct kunit *test) +{ + static const u8 channels[] = { 5, 37, 149 }; + static const u8 expected[0x4d4] = { + [0] = 1, [6] = 0x12, [7] = 1, + [0x9a] = 40, [0x9e] = 4, [0x9f] = 3, + [0xa0] = 3, 5, 3, 37, 3, 149, + [0xe0] = 21, + [0xe2] = 0x7f, 0x0a, 1, 0, 8, 0x8c, 1, 0x40, 0, 0, 0, 0, + 0x6b, 7, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + [0x33c] = 40, [0x4b9] = 0x18, [0x4bc] = 45, + [0x4c2] = 220, [0x4ca] = 233, + }; + u8 *body = kunit_kmalloc(test, sizeof(expected), GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, body); + KUNIT_ASSERT_EQ(test, mt7932_scan_body(body, sizeof(expected), 1, + NL80211_BAND_6GHZ, channels, 3), + 0x4d4); + KUNIT_EXPECT_EQ(test, mt7932_scan_discovery_timeout(body), 5000U); + mt7932_scan_disconnected_passive(body); + KUNIT_EXPECT_MEMEQ(test, body, expected, sizeof(expected)); + for (i = 1; i <= MT7932_SCAN_CHANNELS; i++) { + body[0x9f] = i; + KUNIT_EXPECT_EQ(test, mt7932_scan_discovery_timeout(body), + 5000U + 233 * i); + } + KUNIT_EXPECT_EQ(test, mt7932_scan_discovery_timeout(body), 12456U); + KUNIT_EXPECT_GT(test, mt7932_scan_discovery_timeout(body), 32U * 220); +} + +/* Association-start group selection does not establish runtime D7's band-2 + * procedures. Refuse before touching the container or a partial send plan. + */ +static void mt7932_cal_6g_runtime_rejected_test(struct kunit *test) +{ + static const u32 upper[] = { 0, 0x1000, 0x1001, U32_MAX }; + struct mt7932_cal_piece pieces[7], untouched[7]; + u8 request[16] = {}; + unsigned int channel, i, tag; + int ret; + + memset(untouched, 0xa5, sizeof(untouched)); + put_unaligned_le32(2, request + 8); + for (i = 0; i < ARRAY_SIZE(upper); i++) { + put_unaligned_le32(upper[i], request + 4); + for (channel = 1; channel <= 233; channel++) { + put_unaligned_le32(channel, request + 12); + memcpy(pieces, untouched, sizeof(pieces)); + tag = 0xffff; + ret = mt7932_cal_requested(NULL, 0, request, 16, 12, 0x89, pieces, &tag); + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, tag, 0U); + KUNIT_EXPECT_MEMEQ(test, pieces, untouched, sizeof(pieces)); + } + } +} + static void mt7932_cal_6g_test(struct kunit *test) { u8 blob[52] = {}; @@ -220,6 +319,9 @@ static struct kunit_case mt7932_channel_cases[] = { KUNIT_CASE(mt7932_he_6g_test), KUNIT_CASE(mt7932_6g_security_test), KUNIT_CASE(mt7932_cal_6g_test), + KUNIT_CASE(mt7932_scan_6g_policy_test), + KUNIT_CASE(mt7932_scan_6g_golden_test), + KUNIT_CASE(mt7932_cal_6g_runtime_rejected_test), {} }; diff --git a/drivers/net/wireless/mediatek/mt7932/six.h b/drivers/net/wireless/mediatek/mt7932/six.h new file mode 100644 index 0000000000000..1d2a4f68f354e --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/six.h @@ -0,0 +1,298 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef MT7932_SIX_H +#define MT7932_SIX_H + +#include +#include + +#include "channels.h" + +/* Firmware class numbering, distinct from the J7RP archive class IDs. */ +enum mt7932_6g_class { + MT7932_6G_SP, + MT7932_6G_LPI, + MT7932_6G_VLP, +}; + +struct mt7932_6g_ap { + u8 mode, primary, center, bandwidth; + u8 count; + s8 limit[4]; /* half dBm EIRP, 0x7f means no applicable limit */ +}; + +static inline int mt7932_6g_class(u8 regulatory, u32 flags) +{ + if (flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_IR | + IEEE80211_CHAN_NO_HE | IEEE80211_CHAN_NO_20MHZ | + IEEE80211_CHAN_RADAR)) + return -EACCES; + switch (regulatory) { + case 0: + return MT7932_6G_LPI; + case 1: + /* SP requires AFC; there is no SP-to-LPI fallback. */ + return -EOPNOTSUPP; + case 2: + return flags & IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT ? + -EACCES : + MT7932_6G_VLP; + default: + /* The static mapping's catch-all is not class authorization. */ + return -EOPNOTSUPP; + } +} + +/* A conservative reduction across every advertised 20 MHz PSD value also + * bounds each narrower allocation, without assuming per-subchannel ordering. + * floor(2 * 10log10(BW_MHz)) always rounds toward the restrictive limit. + */ +static inline int mt7932_6g_tpe(struct mt7932_6g_ap *ap, const u8 *data, + u8 length) +{ + static const u8 psd_gain[] = { 26, 32, 38, 44 }; + unsigned int count, interpretation, category, n, i; + int psd = 127; + + if (!ieee80211_valid_tpe_element(data, length)) + return -EINVAL; + count = FIELD_GET(IEEE80211_TX_PWR_ENV_INFO_COUNT, data[0]); + interpretation = + FIELD_GET(IEEE80211_TX_PWR_ENV_INFO_INTERPRET, data[0]); + category = FIELD_GET(IEEE80211_TX_PWR_ENV_INFO_CATEGORY, data[0]); + /* A station is a default client, never a subordinate AP. */ + if (category != IEEE80211_TPE_CAT_6GHZ_DEFAULT) + return 0; + if (interpretation == IEEE80211_TPE_LOCAL_EIRP || + interpretation == IEEE80211_TPE_REG_CLIENT_EIRP) { + n = min(count + 1, 4U); + for (i = 0; i < n; i++) + ap->limit[i] = min_t(s8, ap->limit[i], data[i + 1]); + ap->count = max_t(u8, ap->count, n); + return 0; + } + if (interpretation != IEEE80211_TPE_LOCAL_EIRP_PSD && + interpretation != IEEE80211_TPE_REG_CLIENT_EIRP_PSD) + return -EOPNOTSUPP; + n = count ? BIT(count - 1) : 1; + /* Extended/320 MHz PSD layouts are outside the established binding. */ + if (count > 4 || length != n + 1) + return -EOPNOTSUPP; + for (i = 0; i < n; i++) { + /* A PSD byte of -128 forbids that 20 MHz subchannel. There + * is no supported puncturing path that could omit it here. + */ + if (data[i + 1] == 0x80) + return -EACCES; + psd = min(psd, (int)(s8)data[i + 1]); + } + for (i = 0; i < 4; i++) { + int limit = psd == 127 ? 127 : min(psd + psd_gain[i], 126); + + ap->limit[i] = min_t(int, ap->limit[i], limit); + } + ap->count = 4; + return 0; +} + +/* Full TLV validation prevents a truncated or duplicate HE element from + * becoming authority for LPI. TPE never selects the class. + */ +static inline int mt7932_6g_ap_parse(struct mt7932_6g_ap *ap, const u8 *ies, + size_t length, u8 primary, u32 flags) +{ + const struct element *elem; + bool operation = false; + size_t parsed = 0; + int mode, ret; + + memset(ap, 0, sizeof(*ap)); + memset(ap->limit, 127, sizeof(ap->limit)); + for_each_element(elem, ies, length) { + parsed += elem->datalen + 2; + if (elem->id == WLAN_EID_TX_POWER_ENVELOPE) { + ret = mt7932_6g_tpe(ap, elem->data, elem->datalen); + if (ret) + return ret; + } + if (elem->id == WLAN_EID_EXTENSION && elem->datalen && + elem->data[0] == WLAN_EID_EXT_HE_OPERATION) { + const struct ieee80211_he_operation *he; + const struct ieee80211_he_6ghz_oper *six; + u8 width, center, regulatory; + + if (operation || elem->datalen < 1 + sizeof(*he) || + elem->datalen != + ieee80211_he_oper_size(elem->data + 1)) + return -EINVAL; + he = (const void *)(elem->data + 1); + six = ieee80211_he_6ghz_oper(he); + if (!six || six->primary != primary || + !mt7932_channel_band(NL80211_BAND_6GHZ, primary)) + return -EINVAL; + regulatory = + FIELD_GET(IEEE80211_HE_6GHZ_OPER_CTRL_REG_INFO, + six->control); + mode = mt7932_6g_class(regulatory, flags); + if (mode < 0) + return mode; + width = FIELD_GET(IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH, + six->control); + if (width > 2 || six->ccfs1) + return -EOPNOTSUPP; + center = width ? mt7932_channel_6g_center(primary, + width == 2) : + primary; + if (!center || six->ccfs0 != center) + return -EINVAL; + ap->mode = mode; + ap->primary = primary; + ap->center = center; + ap->bandwidth = width; + operation = true; + } + } + if (parsed != length || !operation || !ap->count) + return -EINVAL; + /* Every bandwidth we may use needs an advertised bound. */ + return ap->count > ap->bandwidth ? 0 : -EOPNOTSUPP; +} + +/* Return 1 for the existing v2 VLP association sequence. Its admission and + * commands never depended on TPE, including finite TPE or AP-wide 160 MHz. + * LPI and newly available VLP channels use the strict class/TPE path. + */ +static inline int mt7932_6g_connect_ap(struct mt7932_6g_ap *ap, const u8 *ies, + size_t length, u8 primary, u32 flags, + u64 legacy_channels) +{ + const struct element *elem; + const struct ieee80211_he_operation *he; + const struct ieee80211_he_6ghz_oper *six; + u8 regulatory; + + elem = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_OPERATION, ies, length); + if (!elem || elem->datalen < 1 + sizeof(*he) || + elem->datalen != ieee80211_he_oper_size(elem->data + 1)) + return -EINVAL; + he = (const void *)(elem->data + 1); + six = ieee80211_he_6ghz_oper(he); + if (!six || six->primary != primary || + !mt7932_channel_band(NL80211_BAND_6GHZ, primary)) + return -EINVAL; + regulatory = FIELD_GET(IEEE80211_HE_6GHZ_OPER_CTRL_REG_INFO, six->control); + if (regulatory == 2 && + (legacy_channels & BIT_ULL((primary - 1) / 4))) + return 1; + return mt7932_6g_ap_parse(ap, ies, length, primary, flags); +} + +/* Immutable J700 board properties, in signed half-dB ticks, from the + * t-3573 antenna-gain SPEC sections 1 and 3. The probe is J700-only. + * The bandwidth map deliberately differs from a frequency-only lookup. + */ +static inline int mt7932_j700_6g_gain(u8 primary, unsigned int bw, s8 gain[3]) +{ + static const s8 groups[6][3] = { + { 16, 16, 10 }, { 14, 14, 8 }, { 15, 16, 10 }, + { 15, 15, 9 }, { 16, 16, 10 }, { 15, 16, 10 }, + }; + unsigned int group; + + if (bw > 3 || (primary != 2 && + (primary < 1 || primary > 233 || (primary - 1) % 4))) + return -EINVAL; + if ((primary == 2 && bw) || (primary == 233 && bw) || + (primary >= 225 && bw >= 2)) + return -EOPNOTSUPP; + if (primary == 2 || primary <= 29) + group = 0; + else if (primary <= 61) + group = 1; + else if (primary <= 93) + group = 2; + else if (primary <= 109) + group = 3; + else if (primary == 113) + group = bw == 2 ? 4 : 3; + else if (primary == 117) + group = bw == 1 || bw == 3 ? 3 : 4; + else if (primary <= 125) + group = bw == 3 ? 3 : 4; + else if (primary <= 181) + group = 4; + else if (primary == 185) + group = bw == 1 ? 5 : 4; + else if (primary == 189) + group = bw < 2 ? 5 : 4; + else + group = 5; + memcpy(gain, groups[group], 3); + return 0; +} + +static inline int mt7932_j700_6g_gains(const struct mt7932_6g_ap *ap, + s8 gain[3][4]) +{ + unsigned int bw, mode; + + memset(gain, 0, 12); + for (bw = 0; bw < ap->count && bw < 4; bw++) { + s8 values[3]; + int ret; + + if (ap->limit[bw] == 127) + continue; + ret = mt7932_j700_6g_gain(ap->primary, bw, values); + if (ret) + return ret; + for (mode = 0; mode < 3; mode++) + gain[mode][bw] = values[mode]; + } + return 0; +} + +/* Gains use the same half-dB units as the AP limit. A NULL gain source is + * usable only for unconstrained limits: zero gain must never be guessed. + */ +static inline int mt7932_6g_tpe_body(u8 *body, size_t capacity, + const struct mt7932_6g_ap *ap, + const s8 gain[3][4]) +{ + unsigned int group, i; + + if (capacity < 60 || !ap->count || ap->count > 4 || + (ap->mode != MT7932_6G_LPI && ap->mode != MT7932_6G_VLP) || + !mt7932_channel_band(NL80211_BAND_6GHZ, ap->primary)) + return -EINVAL; + for (i = 0; i < ap->count; i++) + if (ap->limit[i] != 127 && !gain) + return -EOPNOTSUPP; + memset(body, 0, 60); + body[1] = 2; + put_unaligned_le16(60, body + 2); + body[4] = 1; + body[5] = 3; + body[6] = 1; + body[7] = 3; + body[44] = 1; + body[45] = 3; + body[46] = ap->primary; + body[47] = ap->count; + memset(body + 48, 127, 12); + for (group = 0; group < 3; group++) { + for (i = 0; i < ap->count; i++) { + int limit; + + if (ap->limit[i] == 127) + continue; + limit = ap->limit[i] - gain[group][i]; + /* Saturating upward at -128 would relax an unrepresentable cap. */ + if (limit < -128 || limit > 126) + return -ERANGE; + body[48 + group * 4 + i] = limit; + } + } + return 60; +} + +#endif diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 7962a5fd95a54..9bc5db8ed8031 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -149,7 +149,8 @@ static noinline_for_stack int mt_publish_policy(struct mt7932 *m, { u8 dbdc[36] = {0,0,0,0,2,0,36}; unsigned int i; - struct mt7932_6g_limits limits = mt7932_6g_limits(reg); + struct mt7932_6g_limits limits = + mt7932_6g_limits_mask(reg, policy->legacy_channels); u8 body[1020]; unsigned int j; int ret; @@ -180,6 +181,7 @@ static noinline_for_stack int mt_publish_policy(struct mt7932 *m, return ret; } m->six_power_valid = false; + m->six_legacy_channels = policy->legacy_channels; for (i = 0; i < policy->six_pages; i++) { memcpy(body, policy->six_page[i], policy->six_length[i]); if (policy->six_cid[i] == 0x5d) @@ -192,7 +194,22 @@ static noinline_for_stack int mt_publish_policy(struct mt7932 *m, if (ret) return ret; } - if (policy->six_pages) + if (m->six_class_dirty) { + struct mt7932_6g_ap clear = m->six_ap; + + /* The established all-0x7f command removes the prior AP ceiling. + * Publish VLP's country tables first, while RF admission is fenced. + */ + memset(clear.limit, 127, sizeof(clear.limit)); + ret = mt7932_6g_tpe_body(body, sizeof(body), &clear, NULL); + if (ret < 0) + return ret; + ret = mt_request(m, 0x5d, true, true, false, body, 60); + if (ret) + return ret; + m->six_class_dirty = false; + } + if (policy->six_channels) m->six_power_valid = true; /* Native SETs have no response/atomic policy-commit event. Command * submission is not misrepresented as firmware readback or an ACK. @@ -371,7 +388,132 @@ static int mt_policy_get(void *ctx, const char *path, struct mt7932_policy_file static void mt_policy_put(void *ctx, struct mt7932_policy_file *file) { release_firmware(file->handle); + release_firmware(file->class_handle); file->handle = NULL; + file->class_handle = NULL; +} + +/* command_mutex held. Reopen and authenticate the exact installed policy, + * then publish the AP's class before EA/40. No active-class readback exists. + */ +int mt_6g_connect_policy(struct mt7932 *m, const struct mt7932_6g_ap *ap) +{ + struct mt7932_policy_file file = {}; + struct mt7932_policy *policy; + struct mt7932_6g_limits limits; + struct mt7932_reg_snapshot *reg; + u8 hash[SHA256_DIGEST_SIZE], *body; + char path[MT7932_POLICY_PATH_MAX]; + u8 tpe[60]; + s8 gain[3][4]; + bool sidecar = m->policy_choice.class_valid; + u64 channels; + unsigned long flags; + unsigned int first, channel, begin, end; + int ret, length; + bool submitted = false; + + lockdep_assert_held(&m->command_mutex); + ret = mt7932_j700_6g_gains(ap, gain); + if (ret) + return ret; + ret = mt7932_6g_tpe_body(tpe, sizeof(tpe), ap, gain); + if (ret < 0) + return ret; + policy = kzalloc_obj(*policy, GFP_KERNEL); + reg = kzalloc_obj(*reg, GFP_KERNEL); + body = kmalloc(1020, GFP_KERNEL); + if (!policy || !reg || !body) { + ret = -ENOMEM; + goto free; + } + mt7932_policy_path(path, sizeof(path), + sidecar ? MT7932_POLICY_V4 : m->policy_choice.source, + m->policy_choice.alpha2); + ret = mt_policy_get(m, path, &file); + if (ret) + goto free; + sha256(file.data, file.size, hash); + if (memcmp(hash, + sidecar ? m->policy_choice.class_sha256 : + m->policy_choice.sha256, + sizeof(hash))) { + ret = -ESTALE; + goto free; + } + ret = mt7932_policy_parse(policy, file.data, file.size, + m->policy_choice.domain); + if (ret) + goto free; + channels = policy->six_class[ap->mode].channels; + begin = ap->center - 2 * (BIT(ap->bandwidth) - 1); + end = ap->center + 2 * (BIT(ap->bandwidth) - 1); + for (channel = begin; channel <= end; channel += 4) { + int frequency = ieee80211_channel_to_frequency(channel, NL80211_BAND_6GHZ); + struct ieee80211_channel *c = ieee80211_get_channel(m->wiphy, frequency); + + if (!c || !(channels & BIT_ULL((channel - 1) / 4)) || + mt7932_6g_class(ap->mode == MT7932_6G_LPI ? 0 : 2, + c->flags) < 0) { + ret = -EACCES; + goto free; + } + } + spin_lock_irqsave(&m->response_lock, flags); + *reg = m->reg_desired; + spin_unlock_irqrestore(&m->response_lock, flags); + limits = mt7932_6g_limits(reg); + if (ap->mode == MT7932_6G_LPI) { + /* The LP table, cfg80211 and AP envelope all remain upper bounds. + * Reusing VLP's fixed 14 dBm ceiling would obscure the LP data. + */ + limits.eirp = 254; + for (channel = 0; channel < MT7932_CHANNELS_6G; channel++) { + unsigned int index = + mt7932_6g_reg_index(1 + 4 * channel); + + if (channels & BIT_ULL(channel)) + limits.eirp = min_t(int, limits.eirp, + 2 * reg->power[index]); + } + } + m->six_legacy_assoc = false; + for (first = 0; first < MT7932_6G_RECORDS; first += 8) { + length = mt7932_6g_class_page(body, 1020, policy, ap->mode, + m->policy_choice.domain, first); + if (length < 0) { + ret = length; + goto free; + } + mt7932_6g_limit_page(body, &limits); + submitted = true; + m->six_class_dirty = true; + ret = mt_request(m, 0x5d, true, true, false, body, length); + if (ret) + goto free; + } + dev_dbg(&m->pdev->dev, "CID5D_CLASS: byte1=%02x band=3 channel=%u\n", + 0x0a | (ap->mode << 4), ap->primary); + ret = mt_request(m, 0x5d, true, true, false, tpe, sizeof(tpe)); + if (!ret) + dev_dbg(&m->pdev->dev, + "CID5D_TPE: length=60 band=3 channel=%u count=%u\n", + ap->primary, ap->count); +free: + /* No atomic SET/rollback exists. A partial publication closes RF + * admission; a checked firmware reset is required before reuse. + */ + if (ret && submitted) { + spin_lock_irqsave(&m->response_lock, flags); + mt_rf_fail_locked(m, ret); + spin_unlock_irqrestore(&m->response_lock, flags); + } + if (file.handle) + mt_policy_put(m, &file); + kfree(body); + kfree(reg); + kfree(policy); + return ret; } static void mt_policy_report(struct mt7932 *m, const struct mt7932_policy_choice *choice, @@ -402,19 +544,26 @@ static void mt_startup_work(struct work_struct *work) }; struct mt7932_policy_choice choice; struct mt7932_policy_file file; - struct mt7932_reg_snapshot reg; + struct mt7932_reg_snapshot *reg; struct mt7932_policy *policy; unsigned long flags; u32 generation; bool submitted; int ret; + reg = kzalloc_obj(*reg, GFP_KERNEL); + if (!reg) { + dev_err(&m->pdev->dev, + "REGULATORY_BLOCKED: snapshot allocation failed\n"); + return; + } for (;;) { spin_lock_irqsave(&m->response_lock, flags); generation = m->reg_generation; - reg = m->reg_desired; + *reg = m->reg_desired; if (m->stopping || generation == m->reg_attempted) { spin_unlock_irqrestore(&m->response_lock, flags); + kfree(reg); return; } m->reg_attempted = generation; @@ -435,12 +584,12 @@ static void mt_startup_work(struct work_struct *work) flush_work(&m->scan_finish_work); flush_work(&m->cal_work); } - ret = reg.error; + ret = reg->error; if (READ_ONCE(m->retired_scan_seq)) ret = -EBUSY; memset(&file, 0, sizeof(file)); memset(&choice, 0, sizeof(choice)); - memcpy(choice.alpha2, reg.alpha2, 2); + memcpy(choice.alpha2, reg->alpha2, 2); policy = NULL; submitted = false; if (READ_ONCE(m->policy_failed) || READ_ONCE(m->link_failed) || @@ -448,8 +597,10 @@ static void mt_startup_work(struct work_struct *work) ret = -EIO; if (!ret) { policy = kzalloc_obj(*policy, GFP_KERNEL); - ret = policy ? mt7932_policy_select(&loader, ®, policy, &file, &choice) : - -ENOMEM; + ret = policy ? + mt7932_policy_select(&loader, reg, policy, + &file, &choice) : + -ENOMEM; mt_policy_report(m, &choice, ret); if (!ret) mt_policy_disable(m, policy, generation); @@ -459,7 +610,7 @@ static void mt_startup_work(struct work_struct *work) goto next; if (!ret) { submitted = true; - ret = mt_publish_policy(m, ®, policy); + ret = mt_publish_policy(m, reg, policy); if (!ret && !m->startup_started) ret = mt_startup_once(m); /* A partial SET sequence is not a rollback. Only missing or @@ -483,13 +634,15 @@ static void mt_startup_work(struct work_struct *work) m->policy_choice = choice; } if (ret) - dev_warn_ratelimited(&m->pdev->dev, "REGULATORY_BLOCKED: %c%c generation=%u error=%d recovery-required=%u\n", - reg.alpha2[0], reg.alpha2[1], generation, ret, - m->policy_failed || m->link_failed); + dev_warn_ratelimited(&m->pdev->dev, + "REGULATORY_BLOCKED: %c%c generation=%u error=%d recovery-required=%u\n", + reg->alpha2[0], reg->alpha2[1], generation, ret, + m->policy_failed || m->link_failed); else - dev_dbg(&m->pdev->dev, "REGULATORY_READY: %c%c firmware=%c%c generation=%u current=%u\n", - reg.alpha2[0], reg.alpha2[1], reg.domain[0], reg.domain[1], - generation, !m->reg_pending); + dev_dbg(&m->pdev->dev, + "REGULATORY_READY: %c%c firmware=%c%c generation=%u current=%u\n", + reg->alpha2[0], reg->alpha2[1], reg->domain[0], + reg->domain[1], generation, !m->reg_pending); spin_unlock_irqrestore(&m->response_lock, flags); next: mutex_unlock(&m->command_mutex); @@ -542,8 +695,6 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request rule = freq_reg_info(m->wiphy, MHZ_TO_KHZ(channel->center_freq)); if (!IS_ERR(rule) && !(rule->flags & NL80211_RRF_NO_IR)) flags &= ~IEEE80211_CHAN_NO_IR; - if (flags & IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT) - flags |= IEEE80211_CHAN_DISABLED; } reg.power[i] = channel->max_power; @@ -611,6 +762,7 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request return; } m->reg_desired = reg; + m->six_ap_valid = false; m->reg_deferral = false; m->reg_generation++; /* The next package pass records its own set. */ @@ -655,9 +807,13 @@ static const struct cfg80211_ops mt_cfg_ops = { int mt_register_regulatory_gate(struct mt7932 *m) { static const int rates[] = {10,20,55,110,60,90,120,180,240,360,480,540}; - static const u32 ciphers[] = { WLAN_CIPHER_SUITE_CCMP }; + static const u32 ciphers[] = { WLAN_CIPHER_SUITE_CCMP, + WLAN_CIPHER_SUITE_AES_CMAC }; static const u32 akms[] = { - WLAN_AKM_SUITE_PSK, WLAN_AKM_SUITE_SAE, WLAN_AKM_SUITE_OWE, + WLAN_AKM_SUITE_PSK, + WLAN_AKM_SUITE_PSK_SHA256, + WLAN_AKM_SUITE_SAE, + WLAN_AKM_SUITE_OWE, }; unsigned int i; int ret; @@ -773,6 +929,7 @@ int mt_register_regulatory_gate(struct mt7932 *m) m->band6.iftype_data = &m->he6; m->band6.n_iftype_data = 1; m->wiphy->bands[NL80211_BAND_6GHZ] = &m->band6; + m->wiphy->flags |= WIPHY_FLAG_SPLIT_SCAN_6GHZ; } } ret = wiphy_register(m->wiphy); @@ -811,9 +968,8 @@ static const char *mt7932_policy_fallback_reason(int fallback) } static const char *const mt7932_policy_names[] = { - [MT7932_POLICY_NONE] = "none", - [MT7932_POLICY_V2] = "policy-v2", - [MT7932_POLICY_V1] = "policy", + [MT7932_POLICY_NONE] = "none", [MT7932_POLICY_V4] = "policy-v4", + [MT7932_POLICY_V2] = "policy-v2", [MT7932_POLICY_V1] = "policy", [MT7932_POLICY_WORLD] = "world", }; @@ -824,6 +980,7 @@ unsigned int mt7932_policy_candidates(const u8 alpha2[2], /* 00 and the other numeric codes have no country package. */ if (mt7932_policy_country(alpha2)) { + sources[n++] = MT7932_POLICY_V4; sources[n++] = MT7932_POLICY_V2; sources[n++] = MT7932_POLICY_V1; } @@ -835,6 +992,10 @@ void mt7932_policy_path(char *buf, size_t len, enum mt7932_policy_source source, const u8 alpha2[2]) { switch (source) { + case MT7932_POLICY_V4: + scnprintf(buf, len, "mediatek/mt7932/policy-v4/%c%c.bin", + alpha2[0], alpha2[1]); + break; case MT7932_POLICY_V2: scnprintf(buf, len, "mediatek/mt7932/policy-v2/%c%c.bin", alpha2[0], alpha2[1]); break; @@ -860,8 +1021,57 @@ static int mt7932_policy_check_version(enum mt7932_policy_source source, const u8 *data, size_t size) { unsigned int version = size >= 6 ? get_unaligned_le16(data + 4) : 0; + unsigned int expected = 1; - return version == (source == MT7932_POLICY_V2 ? 2 : 1) ? 0 : -EINVAL; + if (source == MT7932_POLICY_V4) + expected = 4; + else if (source == MT7932_POLICY_V2) + expected = 2; + return version == expected ? 0 : -EINVAL; +} + +/* Prefer the original v2 VLP publication whenever it is installed. Retain + * v4's LP view as a sidecar, rather than replacing the working VLP pages or + * changing the two post-queue association SETs. Both files stay pinned until + * publication and both hashes are checked again before class selection. + */ +static int mt7932_policy_keep_vlp(const struct mt7932_policy_loader *loader, + struct mt7932_policy *policy, + struct mt7932_policy_file *file, + struct mt7932_policy_choice *choice) +{ + struct mt7932_policy_file legacy = {}; + struct mt7932_policy *old; + struct mt7932_6g_table lpi = policy->six_class[MT7932_6G_LPI]; + struct mt7932_6g_table vlp = policy->six_class[MT7932_6G_VLP]; + char path[MT7932_POLICY_PATH_MAX]; + + old = kzalloc_obj(*old, GFP_KERNEL); + if (!old) + return -ENOMEM; + mt7932_policy_path(path, sizeof(path), MT7932_POLICY_V2, + choice->alpha2); + if (loader->get(loader->ctx, path, &legacy)) + goto free; + if (mt7932_policy_check_version(MT7932_POLICY_V2, legacy.data, + legacy.size) || + mt7932_policy_parse(old, legacy.data, legacy.size, + choice->alpha2)) { + loader->put(loader->ctx, &legacy); + goto free; + } + sha256(file->data, file->size, choice->class_sha256); + choice->class_valid = true; + legacy.class_handle = file->handle; + *file = legacy; + *policy = *old; + policy->six_class[MT7932_6G_LPI] = lpi; + policy->six_class[MT7932_6G_VLP] = vlp; + policy->six_channels |= lpi.channels | vlp.channels; + choice->source = MT7932_POLICY_V2; +free: + kfree(old); + return 0; } int mt7932_policy_select(const struct mt7932_policy_loader *loader, @@ -908,6 +1118,15 @@ int mt7932_policy_select(const struct mt7932_policy_loader *loader, if (ret) return ret; + if (choice->source == MT7932_POLICY_V4) { + ret = mt7932_policy_keep_vlp(loader, policy, file, choice); + if (ret) { + loader->put(loader->ctx, file); + memset(file, 0, sizeof(*file)); + return ret; + } + } + /* * A country without a usable package of its own keeps its cfg80211 * identity and channel list; the firmware gets the world profile and @@ -965,6 +1184,9 @@ static int mt_policy_show(struct seq_file *s, void *unused) seq_printf(s, "size: %u\n", c->size); seq_printf(s, "sha256: %8phN\n", c->sha256); seq_printf(s, "6ghz_primaries: %u\n", c->six_primaries); + if (c->class_valid) + seq_printf(s, "class_source: policy-v4\nclass_sha256: %*phN\n", + SHA256_DIGEST_SIZE, c->class_sha256); } seq_printf(s, "fallback: %s\n", mt7932_policy_fallback_reason(c->fallback)); for (i = 0; i < c->n_tried; i++) { diff --git a/drivers/net/wireless/mediatek/mt7932/station.c b/drivers/net/wireless/mediatek/mt7932/station.c index d592b5029d637..d6d131a507ca2 100644 --- a/drivers/net/wireless/mediatek/mt7932/station.c +++ b/drivers/net/wireless/mediatek/mt7932/station.c @@ -11,6 +11,25 @@ static const char mt_rx_stat_names[][ETH_GSTRING_LEN] = { "host_rx_no_memory", "host_rx_retry_records", "host_rx_amsdu_records", + "irq_service_passes", + "irq_unmasked_passes", + "irq_unmasked_bits", + "rx_ring0_records", + "rx_ring1_records", + "rx_ring2_records", + "rx_ring3_records", + "rx_ring4_records", + "rx_ring5_records", + "rx_ring6_records", + "rx_ring7_records", + "rx_ring8_records", + "tx_free_events", + "bss_absent_events", + "bss_absent_ms", + "home_beacons", + "rx_2ghz_records", + "rx_5ghz_records", + "rx_6ghz_records", }; static int mt_get_sset_count(struct net_device *netdev, int set) @@ -33,8 +52,19 @@ static void mt_get_ethtool_stats(struct net_device *netdev, { struct mt7932 *m = *(struct mt7932 **)netdev_priv(netdev); unsigned long flags; + unsigned int i; + BUILD_BUG_ON(ARRAY_SIZE(mt_rx_stat_names) != 28); + /* The interrupt thread alone writes these; a torn pass is harmless. */ + data[9] = READ_ONCE(m->irq_passes); + data[10] = READ_ONCE(m->irq_unmasked_passes); + data[11] = READ_ONCE(m->irq_unmasked); + for (i = 0; i < ARRAY_SIZE(m->rx_ring_records); i++) + data[12 + i] = READ_ONCE(m->rx_ring_records[i]); spin_lock_irqsave(&m->response_lock, flags); + data[22] = m->bss_absent_events; + data[23] = m->bss_absent_ms; + data[24] = m->home_beacons; data[0] = m->rx_integrity; data[1] = m->rx_malformed; data[2] = m->rx_unsupported; @@ -45,6 +75,17 @@ static void mt_get_ethtool_stats(struct net_device *netdev, data[7] = m->rx_retry; data[8] = m->rx_amsdu; spin_unlock_irqrestore(&m->response_lock, flags); + if (READ_ONCE(m->data_ready)) { + spin_lock_irqsave(&m->data_lock, flags); + data[21] = m->tx_free_events; + for (i = 0; i < ARRAY_SIZE(m->rx_band_records); i++) + data[25 + i] = m->rx_band_records[i]; + spin_unlock_irqrestore(&m->data_lock, flags); + } else { + data[21] = 0; + for (i = 0; i < ARRAY_SIZE(m->rx_band_records); i++) + data[25 + i] = 0; + } } const struct ethtool_ops mt_ethtool_ops = { diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index b758e0682ffd8..7a72e76b08262 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -35,6 +35,32 @@ config PCIE_ALTERA_MSI Say Y here if you want PCIe MSI support for the Altera FPGA. This MSI driver supports Altera MSI to GIC controller IP. +config PCIE_APPLE_T8140_BOOTSTRAP + bool "Apple T8140 radio PCIe bootstrap" + depends on PCIE_APPLE=y && ARCH_APPLE && IOMMU_DMA + depends on PCIEASPM + help + Enable the radio PCIe bootstrap on Apple T8140 systems. A reciprocal + device tree link between the host and engine selects this provider. + One fixed SID17 config read primes native ECAM; typed reads and host + resource validation admit only the expected Wi-Fi and Bluetooth pair. + + Build the host and supplier in. Published DMA memory remains owned + until full hardware reset because engine teardown is not qualified. + Suspend-to-idle retains the arena and restores configuration without + issuing another request. Hibernation, deeper sleep and kexec remain + blocked after activation, as do removal and arbitrary downstream use. + +config PCIE_APPLE_T8140_BOOTSTRAP_KUNIT_TEST + tristate "Apple T8140 PCIe bootstrap tests" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Test bootstrap ownership transitions, sleep admission, DMA address + bounds and IRQ validation without accessing hardware. The tests cover + successful admission, pre-publication unwind and irreversible failure + ownership. They do not qualify engine reset or hardware teardown. + config PCIE_APPLE_MSI_DOORBELL_ADDR hex default 0xfffff000 @@ -57,22 +83,6 @@ config PCIE_APPLE If unsure, say Y if you have an Apple Silicon system. -config PCIE_APPLE_PIODMA_DIAG - bool "J700 PIODMA diagnostic (requires full reset after use)" - depends on PCIE_APPLE=y && ARCH_APPLE && IOMMU_DMA - depends on PCIEASPM - help - Build the experimental SID17 retained bootstrap provider. It submits - one fixed request, validates typed ECAM and then enumerates the two - Neo radio functions when the board device tree links the PCIe host - to this engine. The enumerate=0 parameter keeps the host on its - root-port-only path instead. - The host and supplier must both be built in: supplier removal cannot - release a published DMA arena safely. Suspend-to-idle sleep retains that - arena and restores the configuration without repeating a request. - Hibernation and deeper suspend are rejected after activation. - This does not qualify memory reuse or arbitrary downstream devices. - config PCI_VERSATILE bool "ARM Versatile PB PCI controller" depends on ARCH_VERSATILE || COMPILE_TEST diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 3bd9cdb5b1197..978c38a61f44c 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -67,4 +67,5 @@ obj-$(CONFIG_ARM64) += pci-xgene.o endif endif -obj-$(CONFIG_PCIE_APPLE_PIODMA_DIAG) += pcie-apple-piodma-diag.o +obj-$(CONFIG_PCIE_APPLE_T8140_BOOTSTRAP) += pcie-apple-t8140-bootstrap.o +obj-$(CONFIG_PCIE_APPLE_T8140_BOOTSTRAP_KUNIT_TEST) += pcie-apple-t8140-bootstrap-test.o diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h b/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h new file mode 100644 index 0000000000000..21627bfa2ad4a --- /dev/null +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap-state.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef PCIE_APPLE_T8140_BOOTSTRAP_STATE_H +#define PCIE_APPLE_T8140_BOOTSTRAP_STATE_H + +#include +#include +#include +#include +#include +#include + +#define PIODMA_PACKET_MASK GENMASK_ULL(35, 0) +#define PIODMA_POINTER_MASK GENMASK_ULL(41, 0) +#define PIODMA_POINTER_PREFIX BIT_ULL(40) +#define PIODMA_IRQ_COMPLETE (BIT(0) | BIT(6)) +#define PIODMA_IRQ_LIMIT 16 + +/* Only RELEASE before publication can end guard ownership. No reset event. */ +enum apple_piodma_state { + PIODMA_IDLE, + PIODMA_GUARDED, + PIODMA_ATTEMPTED, + PIODMA_FAILED, + PIODMA_REJECTED, + PIODMA_RETAINED, + PIODMA_READY, + PIODMA_ADMITTED, + PIODMA_FAILED_RETAINED, + PIODMA_STATE_COUNT, +}; + +enum apple_piodma_event { + PIODMA_GUARD, + PIODMA_RELEASE, + PIODMA_PRIME, + PIODMA_PUBLISH, + PIODMA_COMPLETE, + PIODMA_ADMIT, + PIODMA_FAIL, + PIODMA_EVENT_COUNT, +}; + +static inline bool apple_piodma_retained(enum apple_piodma_state state) +{ + return state >= PIODMA_RETAINED && state < PIODMA_STATE_COUNT; +} + +static inline bool apple_piodma_guarded(enum apple_piodma_state state) +{ + return state == PIODMA_GUARDED || state == PIODMA_ATTEMPTED || + state == PIODMA_FAILED || apple_piodma_retained(state); +} + +static inline bool apple_piodma_attempted(enum apple_piodma_state state) +{ + return state == PIODMA_ATTEMPTED || state == PIODMA_FAILED || + state == PIODMA_REJECTED || apple_piodma_retained(state); +} + +static inline bool apple_piodma_ready(enum apple_piodma_state state) +{ + return state == PIODMA_READY || state == PIODMA_ADMITTED; +} + +static inline bool apple_piodma_sleep_allowed(enum apple_piodma_state state, + bool suspend_to_idle) +{ + return !apple_piodma_guarded(state) || + (state == PIODMA_ADMITTED && suspend_to_idle); +} + +static inline int apple_piodma_next(enum apple_piodma_state state, + enum apple_piodma_event event) +{ + switch (event) { + case PIODMA_GUARD: + if (state == PIODMA_IDLE) + return PIODMA_GUARDED; + break; + case PIODMA_RELEASE: + if (state == PIODMA_GUARDED) + return PIODMA_IDLE; + if (state == PIODMA_FAILED) + return PIODMA_REJECTED; + break; + case PIODMA_PRIME: + if (state == PIODMA_GUARDED) + return PIODMA_ATTEMPTED; + break; + case PIODMA_PUBLISH: + if (state == PIODMA_ATTEMPTED) + return PIODMA_RETAINED; + break; + case PIODMA_COMPLETE: + if (state == PIODMA_RETAINED) + return PIODMA_READY; + break; + case PIODMA_ADMIT: + if (state == PIODMA_READY) + return PIODMA_ADMITTED; + break; + case PIODMA_FAIL: + if (state == PIODMA_ATTEMPTED) + return PIODMA_FAILED; + if (state == PIODMA_RETAINED) + return PIODMA_FAILED_RETAINED; + break; + default: + break; + } + return -EINVAL; +} + +static inline bool apple_piodma_arena_valid(u64 dma, u64 size, u64 prefix, + bool force_aperture, u64 start, u64 end) +{ + u64 last, pointer; + + return size && IS_ALIGNED(dma, SZ_16K) && + prefix == PIODMA_POINTER_PREFIX && + !check_add_overflow(dma, size - 1, &last) && + last <= PIODMA_PACKET_MASK && + !check_add_overflow(last, prefix, &pointer) && + pointer <= PIODMA_POINTER_MASK && force_aperture && + !start && end == PIODMA_PACKET_MASK; +} + +static inline bool apple_piodma_irq_invalid(unsigned int count, u32 status, + u32 tag, u32 active) +{ + return count >= PIODMA_IRQ_LIMIT || status & ~PIODMA_IRQ_COMPLETE || + (status && (!tag || tag > active)); +} + +static inline bool apple_piodma_irq_matches(u32 status, u32 tag, u32 active) +{ + return status & PIODMA_IRQ_COMPLETE && tag == active && active; +} + +#endif diff --git a/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c b/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c new file mode 100644 index 0000000000000..941a1786c0483 --- /dev/null +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap-test.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "pcie-apple-t8140-bootstrap-state.h" + +static void apple_piodma_lifecycle_test(struct kunit *test) +{ + static const enum apple_piodma_event events[] = { + PIODMA_GUARD, PIODMA_PRIME, PIODMA_PUBLISH, + PIODMA_COMPLETE, PIODMA_ADMIT, + }; + int state = PIODMA_IDLE; + int i; + + for (i = 0; i < ARRAY_SIZE(events); i++) { + state = apple_piodma_next(state, events[i]); + KUNIT_ASSERT_GE(test, state, 0); + KUNIT_EXPECT_TRUE(test, apple_piodma_guarded(state)); + KUNIT_EXPECT_FALSE(test, apple_piodma_sleep_allowed(state, false)); + KUNIT_EXPECT_EQ(test, apple_piodma_sleep_allowed(state, true), + i == ARRAY_SIZE(events) - 1); + } + KUNIT_EXPECT_EQ(test, state, PIODMA_ADMITTED); + KUNIT_EXPECT_TRUE(test, apple_piodma_ready(state)); + KUNIT_EXPECT_TRUE(test, apple_piodma_retained(state)); +} + +static void apple_piodma_failure_test(struct kunit *test) +{ + int state; + + /* Guard contention/unwind before PRIME does not consume the attempt. */ + state = apple_piodma_next(PIODMA_GUARDED, PIODMA_RELEASE); + KUNIT_ASSERT_EQ(test, state, PIODMA_IDLE); + KUNIT_EXPECT_FALSE(test, apple_piodma_attempted(state)); + KUNIT_EXPECT_EQ(test, apple_piodma_next(state, PIODMA_GUARD), PIODMA_GUARDED); + + /* Preflight failure consumes the attempt, but may release both vetoes. */ + state = apple_piodma_next(PIODMA_ATTEMPTED, PIODMA_FAIL); + KUNIT_ASSERT_EQ(test, state, PIODMA_FAILED); + KUNIT_EXPECT_TRUE(test, apple_piodma_guarded(state)); + KUNIT_EXPECT_FALSE(test, apple_piodma_retained(state)); + state = apple_piodma_next(state, PIODMA_RELEASE); + KUNIT_ASSERT_EQ(test, state, PIODMA_REJECTED); + KUNIT_EXPECT_FALSE(test, apple_piodma_guarded(state)); + KUNIT_EXPECT_TRUE(test, apple_piodma_attempted(state)); + KUNIT_EXPECT_TRUE(test, apple_piodma_sleep_allowed(state, false)); + KUNIT_EXPECT_LT(test, apple_piodma_next(state, PIODMA_GUARD), 0); + + /* Initialization/transfer failure must keep memory and vetoes owned. */ + state = apple_piodma_next(PIODMA_RETAINED, PIODMA_FAIL); + KUNIT_ASSERT_EQ(test, state, PIODMA_FAILED_RETAINED); + KUNIT_EXPECT_TRUE(test, apple_piodma_retained(state)); + KUNIT_EXPECT_TRUE(test, apple_piodma_guarded(state)); + KUNIT_EXPECT_FALSE(test, apple_piodma_ready(state)); + KUNIT_EXPECT_FALSE(test, apple_piodma_sleep_allowed(state, true)); +} + +static void apple_piodma_transition_guards_test(struct kunit *test) +{ + int state, event, next; + + /* Exhaust all transitions: no retained state can release or retry. */ + for (state = PIODMA_IDLE; state < PIODMA_STATE_COUNT; state++) { + for (event = PIODMA_GUARD; event < PIODMA_EVENT_COUNT; event++) { + next = apple_piodma_next(state, event); + if (apple_piodma_retained(state)) { + if (next >= 0) + KUNIT_EXPECT_TRUE(test, apple_piodma_retained(next)); + if (event == PIODMA_RELEASE || event == PIODMA_PRIME || + event == PIODMA_GUARD) + KUNIT_EXPECT_LT(test, next, 0); + } + if (event == PIODMA_ADMIT && state != PIODMA_READY) + KUNIT_EXPECT_LT(test, next, 0); + if (event == PIODMA_COMPLETE && state != PIODMA_RETAINED) + KUNIT_EXPECT_LT(test, next, 0); + } + } +} + +static void apple_piodma_arena_test(struct kunit *test) +{ + const u64 end = PIODMA_PACKET_MASK, size = 3 * SZ_16K; + const u64 top = BIT_ULL(36) - size, prefix = PIODMA_POINTER_PREFIX; + const struct { + u64 dma, size, prefix, start, end; + bool forced, valid; + } cases[] = { + { 0, size, prefix, 0, end, true, true }, + { top, size, prefix, 0, end, true, true }, + { top + SZ_16K, size, prefix, 0, end, true, false }, + { 1, size, prefix, 0, end, true, false }, + { U64_MAX - SZ_16K + 1, size, prefix, 0, end, true, false }, + { 0, 0, prefix, 0, end, true, false }, + { 0, size, 0, 0, end, true, false }, + { 0, size, prefix, 0, end, false, false }, + { 0, size, prefix, SZ_16K, end, true, false }, + { 0, size, prefix, 0, end - 1, true, false }, + }; + bool valid; + int i; + + for (i = 0; i < ARRAY_SIZE(cases); i++) { + valid = apple_piodma_arena_valid(cases[i].dma, cases[i].size, + cases[i].prefix, cases[i].forced, + cases[i].start, cases[i].end); + KUNIT_EXPECT_EQ_MSG(test, valid, cases[i].valid, "case %d", i); + } +} + +static void apple_piodma_irq_test(struct kunit *test) +{ + KUNIT_EXPECT_FALSE(test, apple_piodma_irq_invalid(1, BIT(0), 1, 1)); + KUNIT_EXPECT_FALSE(test, apple_piodma_irq_invalid(15, BIT(6), 1, 1)); + KUNIT_EXPECT_TRUE(test, apple_piodma_irq_matches(BIT(0), 1, 1)); + KUNIT_EXPECT_TRUE(test, apple_piodma_irq_matches(BIT(6), 1, 1)); + KUNIT_EXPECT_TRUE(test, apple_piodma_irq_invalid(16, BIT(0), 1, 1)); + KUNIT_EXPECT_TRUE(test, apple_piodma_irq_invalid(1, BIT(1), 1, 1)); + KUNIT_EXPECT_TRUE(test, apple_piodma_irq_invalid(1, BIT(0), 0, 1)); + KUNIT_EXPECT_TRUE(test, apple_piodma_irq_invalid(1, BIT(0), 2, 1)); + KUNIT_EXPECT_TRUE(test, apple_piodma_irq_invalid(1, BIT(0), 1, 0)); + KUNIT_EXPECT_FALSE(test, apple_piodma_irq_matches(0, 1, 1)); + KUNIT_EXPECT_FALSE(test, apple_piodma_irq_matches(BIT(0), 0, 0)); + /* A late old tag neither faults nor qualifies the active request. */ + KUNIT_EXPECT_FALSE(test, apple_piodma_irq_invalid(1, BIT(0), 1, 2)); + KUNIT_EXPECT_FALSE(test, apple_piodma_irq_matches(BIT(0), 1, 2)); +} + +static struct kunit_case apple_piodma_cases[] = { + KUNIT_CASE(apple_piodma_lifecycle_test), + KUNIT_CASE(apple_piodma_failure_test), + KUNIT_CASE(apple_piodma_transition_guards_test), + KUNIT_CASE(apple_piodma_arena_test), + KUNIT_CASE(apple_piodma_irq_test), + { } +}; + +static struct kunit_suite apple_piodma_suite = { + .name = "apple-t8140-bootstrap", + .test_cases = apple_piodma_cases, +}; + +kunit_test_suite(apple_piodma_suite); + +MODULE_DESCRIPTION("Apple T8140 PCIe bootstrap ownership and guard tests"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-t8140-bootstrap.c similarity index 50% rename from drivers/pci/controller/pcie-apple-piodma-diag.c rename to drivers/pci/controller/pcie-apple-t8140-bootstrap.c index 61b72a987d23f..a95dd5112f201 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap.c @@ -1,16 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * J700 PIODMA diagnostic scaffolding, not a production DMA driver. + * T8140 radio PCIe bootstrap: one validated config read before native ECAM. * - * Public Linux DMA/IOMMU APIs own the SID17 mappings. The register and - * address-view contract is attributed in the accompanying provenance ledger - * to the reviewed Aurora reference and fresh ADT observations. - * - * This built-in driver deliberately retains its coherent arena and device - * power reference until a coordinator-controlled full hardware reset. There - * is no unbind interface or hardware retry. Runtime guards cover host admission - * and retained hardware ownership. Native ECAM is only - * opened by the host after a successful bootstrap and typed read checks. + * The Linux DMA/IOMMU APIs own the SID17 mappings. See + * Documentation/PCI/controller/apple-t8140.rst for the public history and + * the retained-ownership contract. Completion does not establish DMA teardown. + * The arena, IRQ and power reference remain owned until full hardware reset. */ #include #include @@ -37,11 +32,9 @@ #include #include -#include "pcie-apple-piodma-diag.h" +#include "pcie-apple-t8140-bootstrap.h" +#include "pcie-apple-t8140-bootstrap-state.h" -#define PIODMA_PACKET_MASK DMA_BIT_MASK(36) -#define PIODMA_POINTER_MASK DMA_BIT_MASK(42) -#define PIODMA_POINTER_PREFIX BIT_ULL(40) #define PIODMA_SID 17 #define PIODMA_REQUEST_COUNT 1 #define PIODMA_SLOT_SIZE (3 * PAGE_SIZE) @@ -50,19 +43,13 @@ #define PIODMA_COMPLETION_OFFSET PAGE_SIZE #define PIODMA_CONFIG_BASE 0x1cb0000000ULL #define PIODMA_CONFIG_SIZE SZ_256M -#define PIODMA_IRQ_COMPLETE (BIT(0) | BIT(6)) #define PIODMA_IRQ_MASK GENMASK(9, 0) #define PIODMA_FIFO_MASK GENMASK(5, 0) -#define PIODMA_IRQ_LIMIT 16 /* * The board DT opts in by pointing the host's "apple,piodma" at this engine; - * a host without that property stays root-port-only. enumerate=0 keeps every - * host on its root-port-only path for diagnosis. + * a host without that property stays root-port-only. */ -static bool enumerate = true; -module_param(enumerate, bool, 0400); -MODULE_PARM_DESC(enumerate, "Prime once, validate native ECAM, and enumerate the two Neo functions (default: on)"); struct apple_piodma_request { u16 offset; @@ -85,28 +72,22 @@ struct apple_piodma_slot { bool observed; }; -struct apple_piodma_diag { +struct apple_t8140_bootstrap { struct device *dev; struct iommu_domain *domain; void __iomem *engine; void __iomem *fabric; struct apple_piodma_slot slots[PIODMA_REQUEST_COUNT]; atomic_t irq_count; - atomic_t attempted; + atomic_t state; u32 active_tag; u64 config_source; - u32 last_irq_status; - u32 last_irq_tag; int irq; bool irq_fault; void *arena; dma_addr_t dma; u64 pointer_prefix; u8 secondary_bus; - bool retained; - bool ready; - bool guarded; - bool radio_admitted; struct notifier_block pm_notifier; struct kexec_blocker kexec_blocker; struct delayed_work admission_retry; @@ -114,7 +95,24 @@ struct apple_piodma_diag { }; /* A successful supplier probe owns its unpublished arena until full reset. */ -static struct apple_piodma_diag *active_diag; +static struct apple_t8140_bootstrap *active_bootstrap; + +/* Host device_lock serializes transitions; IRQ faults are recorded separately. */ +static enum apple_piodma_state apple_piodma_state(struct apple_t8140_bootstrap *bootstrap) +{ + return atomic_read_acquire(&bootstrap->state); +} + +static int apple_piodma_transition(struct apple_t8140_bootstrap *bootstrap, + enum apple_piodma_event event) +{ + enum apple_piodma_state state = apple_piodma_state(bootstrap); + int next = apple_piodma_next(state, event); + + if (next < 0) + return next; + return atomic_cmpxchg(&bootstrap->state, state, next) == state ? 0 : -EALREADY; +} static bool apple_piodma_trylock_sleep(unsigned int *flags) { @@ -137,7 +135,7 @@ static bool apple_piodma_trylock_sleep(unsigned int *flags) return false; } -static int apple_piodma_lock_transition(struct apple_piodma_diag *diag, +static int apple_piodma_lock_transition(struct apple_t8140_bootstrap *bootstrap, unsigned int *sleep_flags) { unsigned int attempt; @@ -146,7 +144,7 @@ static int apple_piodma_lock_transition(struct apple_piodma_diag *diag, /* Probe cannot wait on either lock while holding its device lock. */ for (attempt = 0; attempt < 50; attempt++) { if (apple_piodma_trylock_sleep(sleep_flags)) { - ret = kexec_block(&diag->kexec_blocker); + ret = kexec_block(&bootstrap->kexec_blocker); if (!ret) return 0; unlock_system_sleep(*sleep_flags); @@ -163,8 +161,8 @@ static int apple_piodma_lock_transition(struct apple_piodma_diag *diag, static int apple_piodma_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) { - struct apple_piodma_diag *diag = - container_of(nb, struct apple_piodma_diag, pm_notifier); + struct apple_t8140_bootstrap *bootstrap = + container_of(nb, struct apple_t8140_bootstrap, pm_notifier); switch (action) { case PM_SUSPEND_PREPARE: @@ -178,7 +176,7 @@ static int apple_piodma_pm_notify(struct notifier_block *nb, return NOTIFY_DONE; case PM_HIBERNATION_PREPARE: case PM_RESTORE_PREPARE: - dev_warn_ratelimited(diag->dev, + dev_warn_ratelimited(bootstrap->dev, "sleep refused: Neo guard active during radio admission or retained DMA lifetime\n"); return notifier_from_errno(-EOPNOTSUPP); default: @@ -188,13 +186,12 @@ static int apple_piodma_pm_notify(struct notifier_block *nb, static int apple_piodma_pm_prepare(struct device *dev) { - struct apple_piodma_diag *diag = dev_get_drvdata(dev); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(dev); /* Acquire admission before either PCI children or DMA pointers escape. */ - if (!diag || !smp_load_acquire(&diag->guarded)) + if (!bootstrap || !apple_piodma_guarded(apple_piodma_state(bootstrap))) return 0; - if (pm_suspend_no_platform() && READ_ONCE(diag->ready) && - smp_load_acquire(&diag->radio_admitted)) + if (apple_piodma_sleep_allowed(apple_piodma_state(bootstrap), pm_suspend_no_platform())) return 0; /* * A /dev/snapshot file may have sent its PREPARE notifier before this @@ -206,7 +203,7 @@ static int apple_piodma_pm_prepare(struct device *dev) return -EOPNOTSUPP; } -static int apple_piodma_diag_root(struct apple_piodma_diag *diag, struct pci_dev *root) +static int apple_t8140_bootstrap_root(struct apple_t8140_bootstrap *bootstrap, struct pci_dev *root) { int ret; @@ -215,23 +212,23 @@ static int apple_piodma_diag_root(struct apple_piodma_diag *diag, struct pci_dev root->vendor != PCI_VENDOR_ID_APPLE || root->device != 0x100c || root->class != PCI_CLASS_BRIDGE_PCI << 8) return -ENODEV; - ret = pci_read_config_byte(root, PCI_SECONDARY_BUS, &diag->secondary_bus); + ret = pci_read_config_byte(root, PCI_SECONDARY_BUS, &bootstrap->secondary_bus); if (ret) return pcibios_err_to_errno(ret); - if (diag->secondary_bus != 1) + if (bootstrap->secondary_bus != 1) return -EINVAL; - diag->config_source += (u64)diag->secondary_bus << 20; + bootstrap->config_source = PIODMA_CONFIG_BASE + ((u64)bootstrap->secondary_bus << 20); return 0; } -static int apple_piodma_diag_iommu(struct apple_piodma_diag *diag) +static int apple_t8140_bootstrap_iommu(struct apple_t8140_bootstrap *bootstrap) { struct of_phandle_args args; u64 range[2]; int ret; - ret = of_parse_phandle_with_args(diag->dev->of_node, "iommus", - "#iommu-cells", 0, &args); + ret = of_parse_phandle_with_args(bootstrap->dev->of_node, "iommus", + "#iommu-cells", 0, &args); if (ret) return ret; if (args.args_count != 1 || args.args[0] != PIODMA_SID || @@ -244,66 +241,62 @@ static int apple_piodma_diag_iommu(struct apple_piodma_diag *diag) ret = -EINVAL; goto out; } - diag->pointer_prefix = PIODMA_POINTER_PREFIX; + bootstrap->pointer_prefix = PIODMA_POINTER_PREFIX; ret = of_property_read_u64_array(args.np, "apple,dma-range", range, 2); if (ret) goto out; - if (diag->pointer_prefix != PIODMA_POINTER_PREFIX || range[0] || + if (bootstrap->pointer_prefix != PIODMA_POINTER_PREFIX || range[0] || range[1] != BIT_ULL(36)) { ret = -EINVAL; goto out; } - diag->domain = iommu_get_domain_for_dev(diag->dev); - if (!diag->domain || - (diag->domain->type != IOMMU_DOMAIN_DMA && - diag->domain->type != IOMMU_DOMAIN_DMA_FQ)) { + bootstrap->domain = iommu_get_domain_for_dev(bootstrap->dev); + if (!bootstrap->domain || + (bootstrap->domain->type != IOMMU_DOMAIN_DMA && + bootstrap->domain->type != IOMMU_DOMAIN_DMA_FQ)) { ret = -ENODEV; goto out; } - ret = dma_set_mask_and_coherent(diag->dev, PIODMA_PACKET_MASK); + ret = dma_set_mask_and_coherent(bootstrap->dev, PIODMA_PACKET_MASK); out: of_node_put(args.np); return ret; } -static int apple_piodma_diag_arena(struct apple_piodma_diag *diag) +static int apple_t8140_bootstrap_arena(struct apple_t8140_bootstrap *bootstrap) { - u64 last, pointer; int i; - diag->arena = dma_alloc_coherent(diag->dev, PIODMA_ARENA_SIZE, &diag->dma, GFP_KERNEL); - if (!diag->arena) + bootstrap->arena = dma_alloc_coherent(bootstrap->dev, PIODMA_ARENA_SIZE, + &bootstrap->dma, GFP_KERNEL); + if (!bootstrap->arena) return -ENOMEM; - if (!IS_ALIGNED(diag->dma, SZ_16K) || - check_add_overflow((u64)diag->dma, PIODMA_ARENA_SIZE - 1, &last) || - last > PIODMA_PACKET_MASK || - check_add_overflow(last, diag->pointer_prefix, &pointer) || - pointer > PIODMA_POINTER_MASK || - !diag->domain->geometry.force_aperture || - diag->domain->geometry.aperture_start != 0 || - diag->domain->geometry.aperture_end != PIODMA_PACKET_MASK) + if (!apple_piodma_arena_valid(bootstrap->dma, PIODMA_ARENA_SIZE, + bootstrap->pointer_prefix, + bootstrap->domain->geometry.force_aperture, + bootstrap->domain->geometry.aperture_start, + bootstrap->domain->geometry.aperture_end)) return -ERANGE; - memset(diag->arena, 0, PIODMA_ARENA_SIZE); + memset(bootstrap->arena, 0, PIODMA_ARENA_SIZE); for (i = 0; i < PIODMA_REQUEST_COUNT; i++) { - void *slot = diag->arena + i * PIODMA_SLOT_SIZE; - u64 dma = diag->dma + i * PIODMA_SLOT_SIZE; + void *slot = bootstrap->arena + i * PIODMA_SLOT_SIZE; + u64 dma = bootstrap->dma + i * PIODMA_SLOT_SIZE; memset(slot + PIODMA_DATA_OFFSET, 0xa5, PAGE_SIZE); *(__le32 *)(slot + PIODMA_DATA_OFFSET) = cpu_to_le32(U32_MAX); - dev_info(diag->dev, - "owned SID17 slot=%d dma=%#llx descriptor=%#llx completion=%#llx data=%#llx\n", - i, dma, dma + diag->pointer_prefix, - dma + diag->pointer_prefix + PIODMA_COMPLETION_OFFSET, - dma + PIODMA_DATA_OFFSET); + dev_dbg(bootstrap->dev, + "owned SID17 slot=%d dma=%#llx descriptor=%#llx completion=%#llx data=%#llx\n", + i, dma, dma + bootstrap->pointer_prefix, + dma + bootstrap->pointer_prefix + PIODMA_COMPLETION_OFFSET, + dma + PIODMA_DATA_OFFSET); } dma_wmb(); return 0; } - -static int apple_piodma_diag_resources(struct platform_device *pdev, - struct apple_piodma_diag *diag) +static int apple_t8140_bootstrap_resources(struct platform_device *pdev, + struct apple_t8140_bootstrap *bootstrap) { static const struct { const char *name; @@ -322,7 +315,7 @@ static int apple_piodma_diag_resources(struct platform_device *pdev, if (!res || res->start != regions[i].base || resource_size(res) != SZ_16K) return -EINVAL; } - host = of_parse_phandle(diag->dev->of_node, "apple,pcie-host", 0); + host = of_parse_phandle(bootstrap->dev->of_node, "apple,pcie-host", 0); if (!host) return -EINVAL; ret = of_address_to_resource(host, 0, &ecam); @@ -333,20 +326,20 @@ static int apple_piodma_diag_resources(struct platform_device *pdev, of_node_put(host); if (ret) return ret; - diag->config_source = ecam.start; - if (diag->config_source + (1 << 12) + sizeof(u32) - 1 > ecam.end) + bootstrap->config_source = ecam.start; + if (bootstrap->config_source + (1 << 12) + sizeof(u32) - 1 > ecam.end) return -ERANGE; - diag->engine = devm_platform_ioremap_resource_byname(pdev, "engine"); - if (IS_ERR(diag->engine)) - return PTR_ERR(diag->engine); - diag->fabric = devm_platform_ioremap_resource_byname(pdev, "fabric"); - if (IS_ERR(diag->fabric)) - return PTR_ERR(diag->fabric); - diag->irq = platform_get_irq(pdev, 0); - return diag->irq < 0 ? diag->irq : 0; + bootstrap->engine = devm_platform_ioremap_resource_byname(pdev, "engine"); + if (IS_ERR(bootstrap->engine)) + return PTR_ERR(bootstrap->engine); + bootstrap->fabric = devm_platform_ioremap_resource_byname(pdev, "fabric"); + if (IS_ERR(bootstrap->fabric)) + return PTR_ERR(bootstrap->fabric); + bootstrap->irq = platform_get_irq(pdev, 0); + return bootstrap->irq < 0 ? bootstrap->irq : 0; } -static int apple_piodma_diag_idle(struct apple_piodma_diag *diag) +static int apple_t8140_bootstrap_idle(struct apple_t8140_bootstrap *bootstrap) { static const struct { u32 offset; @@ -360,45 +353,42 @@ static int apple_piodma_diag_idle(struct apple_piodma_diag *diag) u32 value; for (i = 0; i < ARRAY_SIZE(engine); i++) { - value = readl(diag->engine + engine[i].offset); + value = readl(bootstrap->engine + engine[i].offset); if (value != engine[i].expected) { - dev_err(diag->dev, "idle mismatch engine[%#x]=%#x expected=%#x\n", + dev_err(bootstrap->dev, "idle mismatch engine[%#x]=%#x expected=%#x\n", engine[i].offset, value, engine[i].expected); return -EBUSY; } } - if (readl(diag->fabric) || readl(diag->fabric + 4)) + if (readl(bootstrap->fabric) || readl(bootstrap->fabric + 4)) return -EBUSY; return 0; } -static irqreturn_t apple_piodma_diag_irq(int irq, void *data) +static irqreturn_t apple_t8140_bootstrap_irq(int irq, void *data) { - struct apple_piodma_diag *diag = data; + struct apple_t8140_bootstrap *bootstrap = data; struct apple_piodma_slot *slot; u32 fabric, status, tag, active; int count; /* Acknowledge order follows the attributed prior controller contract. */ - fabric = readl(diag->fabric); - writel(fabric, diag->fabric); - status = readl(diag->engine + 4); - writel(status, diag->engine + 4); - tag = readl(diag->engine + 0xdc) & 0xff; - active = READ_ONCE(diag->active_tag); - count = atomic_inc_return(&diag->irq_count); - WRITE_ONCE(diag->last_irq_status, status); - WRITE_ONCE(diag->last_irq_tag, tag); - dev_info(diag->dev, "irq=%d fabric=%#x status=%#x tag=%#x active=%#x\n", - count, fabric, status, tag, active); - if (count >= PIODMA_IRQ_LIMIT || status & ~PIODMA_IRQ_COMPLETE || - (status && (!tag || tag > active))) { - WRITE_ONCE(diag->irq_fault, true); + fabric = readl(bootstrap->fabric); + writel(fabric, bootstrap->fabric); + status = readl(bootstrap->engine + 4); + writel(status, bootstrap->engine + 4); + tag = readl(bootstrap->engine + 0xdc) & 0xff; + active = READ_ONCE(bootstrap->active_tag); + count = atomic_inc_return(&bootstrap->irq_count); + dev_dbg(bootstrap->dev, "irq=%d fabric=%#x status=%#x tag=%#x active=%#x\n", + count, fabric, status, tag, active); + if (apple_piodma_irq_invalid(count, status, tag, active)) { + WRITE_ONCE(bootstrap->irq_fault, true); disable_irq_nosync(irq); if (active) - complete(&diag->slots[active - 1].event); - } else if (status & PIODMA_IRQ_COMPLETE && tag == active && active) { - slot = &diag->slots[active - 1]; + complete(&bootstrap->slots[active - 1].event); + } else if (apple_piodma_irq_matches(status, tag, active)) { + slot = &bootstrap->slots[active - 1]; WRITE_ONCE(slot->matched_status, READ_ONCE(slot->matched_status) | status); complete(&slot->event); } @@ -406,7 +396,7 @@ static irqreturn_t apple_piodma_diag_irq(int irq, void *data) return IRQ_HANDLED; } -static int apple_piodma_diag_initialize(struct apple_piodma_diag *diag) +static int apple_t8140_bootstrap_initialize(struct apple_t8140_bootstrap *bootstrap) { static const struct { u32 offset; @@ -423,30 +413,28 @@ static int apple_piodma_diag_initialize(struct apple_piodma_diag *diag) int i; u32 value; - /* Publish validated root/bus fields; all later failures retain ownership. */ - smp_store_release(&diag->retained, true); - writel(PIODMA_IRQ_MASK, diag->engine + 8); - writel(0, diag->fabric + 4); + writel(PIODMA_IRQ_MASK, bootstrap->engine + 8); + writel(0, bootstrap->fabric + 4); for (i = 0; i < ARRAY_SIZE(tunables); i++) { - value = readl(diag->engine + tunables[i].offset); + value = readl(bootstrap->engine + tunables[i].offset); value = (value & ~tunables[i].mask) | tunables[i].value; - writel(value, diag->engine + tunables[i].offset); + writel(value, bootstrap->engine + tunables[i].offset); } - writel(readl(diag->engine) & ~BIT(0), diag->engine); + writel(readl(bootstrap->engine) & ~BIT(0), bootstrap->engine); /* Exact prior bank values; their architectural meaning is unqualified. */ for (i = 0; i < 8; i++) { - writel(0, diag->engine + 0x24 + 4 * i); - writel(i == 1 ? 1 : 0, diag->engine + 0xf0 + 4 * i); + writel(0, bootstrap->engine + 0x24 + 4 * i); + writel(i == 1 ? 1 : 0, bootstrap->engine + 0xf0 + 4 * i); } /* Order the bank writes before their configuration readback. */ wmb(); - if (readl(diag->engine) || readl(diag->engine + 8) != PIODMA_IRQ_MASK || - readl(diag->engine + 0xf4) != 1 || READ_ONCE(diag->irq_fault)) + if (readl(bootstrap->engine) || readl(bootstrap->engine + 8) != PIODMA_IRQ_MASK || + readl(bootstrap->engine + 0xf4) != 1 || READ_ONCE(bootstrap->irq_fault)) return -EIO; return 0; } -static bool apple_piodma_diag_canary(const u8 *memory, size_t size, u8 value) +static bool apple_t8140_bootstrap_canary(const u8 *memory, size_t size, u8 value) { size_t i; @@ -456,14 +444,14 @@ static bool apple_piodma_diag_canary(const u8 *memory, size_t size, u8 value) return true; } -static int apple_piodma_diag_check_slots(struct apple_piodma_diag *diag) +static int apple_t8140_bootstrap_check_slots(struct apple_t8140_bootstrap *bootstrap) { int i, j; dma_rmb(); for (i = 0; i < PIODMA_REQUEST_COUNT; i++) { - struct apple_piodma_slot *slot = &diag->slots[i]; - void *memory = diag->arena + i * PIODMA_SLOT_SIZE; + struct apple_piodma_slot *slot = &bootstrap->slots[i]; + void *memory = bootstrap->arena + i * PIODMA_SLOT_SIZE; __le32 *packet = memory; __le32 *completion = memory + PIODMA_COMPLETION_OFFSET; __le32 *result = memory + PIODMA_DATA_OFFSET; @@ -476,42 +464,42 @@ static int apple_piodma_diag_check_slots(struct apple_piodma_diag *diag) } if ((slot->observed && le32_to_cpu(READ_ONCE(*result)) != slot->result) || (!slot->submitted && le32_to_cpu(READ_ONCE(*result)) != U32_MAX) || - !apple_piodma_diag_canary(memory + sizeof(slot->packet), + !apple_t8140_bootstrap_canary(memory + sizeof(slot->packet), PAGE_SIZE - sizeof(slot->packet), 0) || - !apple_piodma_diag_canary(memory + PIODMA_COMPLETION_OFFSET + + !apple_t8140_bootstrap_canary(memory + PIODMA_COMPLETION_OFFSET + sizeof(slot->completion), PAGE_SIZE - sizeof(slot->completion), 0) || - !apple_piodma_diag_canary(memory + PIODMA_DATA_OFFSET + sizeof(u32), + !apple_t8140_bootstrap_canary(memory + PIODMA_DATA_OFFSET + sizeof(u32), PAGE_SIZE - sizeof(u32), 0xa5)) goto changed; } return 0; changed: - dev_err(diag->dev, "slot=%d changed outside its permitted response; stop\n", i); + dev_err(bootstrap->dev, "slot=%d changed outside its permitted response; stop\n", i); return -EIO; } -static int apple_piodma_diag_submit(struct apple_piodma_diag *diag, unsigned int index) +static int apple_t8140_bootstrap_submit(struct apple_t8140_bootstrap *bootstrap, unsigned int index) { const struct apple_piodma_request *request = &requests[index]; - struct apple_piodma_slot *slot = &diag->slots[index]; - void *memory = diag->arena + index * PIODMA_SLOT_SIZE; + struct apple_piodma_slot *slot = &bootstrap->slots[index]; + void *memory = bootstrap->arena + index * PIODMA_SLOT_SIZE; __le32 *packet = memory; __le32 *result = memory + PIODMA_DATA_OFFSET; __le32 *completion = memory + PIODMA_COMPLETION_OFFSET; - u64 dma = diag->dma + index * PIODMA_SLOT_SIZE; - u64 source = diag->config_source + ((u64)request->function << 12) + request->offset; + u64 dma = bootstrap->dma + index * PIODMA_SLOT_SIZE; + u64 source = bootstrap->config_source + ((u64)request->function << 12) + request->offset; u64 dst = dma + PIODMA_DATA_OFFSET; u64 src = source & PIODMA_PACKET_MASK; - u64 descriptor = dma + diag->pointer_prefix; + u64 descriptor = dma + bootstrap->pointer_prefix; u64 comp = descriptor + PIODMA_COMPLETION_OFFSET; unsigned long waited; u32 fifo, command, tag = index + 1; int i, ret; - ret = apple_piodma_diag_check_slots(diag); - if (ret || READ_ONCE(diag->irq_fault)) + ret = apple_t8140_bootstrap_check_slots(bootstrap); + if (ret || READ_ONCE(bootstrap->irq_fault)) return ret ?: -EIO; /* Only these fixed, validated ECAM addresses use aperture1/low36. */ slot->packet[0] = cpu_to_le32(0xc007 | BIT(18) | ((src & 0xff) << 24)); @@ -520,68 +508,61 @@ static int apple_piodma_diag_submit(struct apple_piodma_diag *diag, unsigned int slot->packet[3] = cpu_to_le32(sizeof(u32)); memcpy(packet, slot->packet, sizeof(slot->packet)); dma_wmb(); - ret = readl_poll_timeout(diag->engine + 0x0c, fifo, - (fifo & PIODMA_FIFO_MASK) == 16, 10, 50000); - if (ret || READ_ONCE(diag->irq_fault)) + ret = readl_poll_timeout(bootstrap->engine + 0x0c, fifo, + (fifo & PIODMA_FIFO_MASK) == 16, 10, 50000); + if (ret || READ_ONCE(bootstrap->irq_fault)) return ret ?: -EIO; slot->submitted = true; - WRITE_ONCE(diag->active_tag, tag); - dev_info(diag->dev, - "request=%u function=%u offset=%#x source=%#llx packet=%08x:%08x:%08x:%08x tag=%u\n", - index, request->function, request->offset, source, - le32_to_cpu(packet[0]), le32_to_cpu(packet[1]), - le32_to_cpu(packet[2]), le32_to_cpu(packet[3]), tag); - writel(tag, diag->engine + 0xd8); - writel(upper_32_bits(descriptor) & 0x3ff, diag->engine + 0x50); - writel(lower_32_bits(descriptor), diag->engine + 0x4c); - writel(upper_32_bits(comp) & 0x3ff, diag->engine + 0x48); - writel(lower_32_bits(comp), diag->engine + 0x44); - writel(0x411, diag->engine + 0x54); + WRITE_ONCE(bootstrap->active_tag, tag); + dev_dbg(bootstrap->dev, + "request=%u function=%u offset=%#x source=%#llx packet=%08x:%08x:%08x:%08x tag=%u\n", + index, request->function, request->offset, source, + le32_to_cpu(packet[0]), le32_to_cpu(packet[1]), + le32_to_cpu(packet[2]), le32_to_cpu(packet[3]), tag); + writel(tag, bootstrap->engine + 0xd8); + writel(upper_32_bits(descriptor) & 0x3ff, bootstrap->engine + 0x50); + writel(lower_32_bits(descriptor), bootstrap->engine + 0x4c); + writel(upper_32_bits(comp) & 0x3ff, bootstrap->engine + 0x48); + writel(lower_32_bits(comp), bootstrap->engine + 0x44); + writel(0x411, bootstrap->engine + 0x54); waited = wait_for_completion_timeout(&slot->event, msecs_to_jiffies(50)); dma_rmb(); slot->result = le32_to_cpu(READ_ONCE(*result)); for (i = 0; i < ARRAY_SIZE(slot->completion); i++) slot->completion[i] = READ_ONCE(completion[i]); slot->observed = true; - dev_info(diag->dev, - "response=%u waited=%lu matched_status=%#x word=%#x completion=%08x:%08x:%08x:%08x\n", - index, waited, READ_ONCE(slot->matched_status), slot->result, - le32_to_cpu(slot->completion[0]), le32_to_cpu(slot->completion[1]), - le32_to_cpu(slot->completion[2]), le32_to_cpu(slot->completion[3])); + dev_dbg(bootstrap->dev, + "response=%u waited=%lu matched_status=%#x word=%#x completion=%08x:%08x:%08x:%08x\n", + index, waited, READ_ONCE(slot->matched_status), slot->result, + le32_to_cpu(slot->completion[0]), le32_to_cpu(slot->completion[1]), + le32_to_cpu(slot->completion[2]), le32_to_cpu(slot->completion[3])); if (!waited) return -ETIMEDOUT; - if (READ_ONCE(diag->irq_fault) || !READ_ONCE(slot->matched_status) || + if (READ_ONCE(bootstrap->irq_fault) || !READ_ONCE(slot->matched_status) || slot->result == U32_MAX || (slot->result & request->mask) != request->expected) return -EIO; /* Admission observations only: all published memory remains retained. */ - ret = readl_poll_timeout(diag->engine + 0x54, command, !(command & BIT(0)), - 10, 50000); + ret = readl_poll_timeout(bootstrap->engine + 0x54, command, !(command & BIT(0)), + 10, 50000); if (ret) return ret; - ret = readl_poll_timeout(diag->engine + 0x0c, fifo, - (fifo & PIODMA_FIFO_MASK) == 16, 10, 50000); + ret = readl_poll_timeout(bootstrap->engine + 0x0c, fifo, + (fifo & PIODMA_FIFO_MASK) == 16, 10, 50000); if (ret) return ret; for (i = 0; i < 10; i++) { usleep_range(1000, 2000); - ret = apple_piodma_diag_check_slots(diag); - if (ret || READ_ONCE(diag->irq_fault)) + ret = apple_t8140_bootstrap_check_slots(bootstrap); + if (ret || READ_ONCE(bootstrap->irq_fault)) return ret ?: -EIO; } - dev_info(diag->dev, - "READ_VALIDATED request=%u word=%#x FIFO=%#x command=%#x; retained slots stable\n", - index, slot->result, fifo, command); + dev_dbg(bootstrap->dev, + "READ_VALIDATED request=%u word=%#x FIFO=%#x command=%#x; retained slots stable\n", + index, slot->result, fifo, command); return 0; } -bool apple_piodma_bootstrap_enabled(void) -{ - return enumerate && of_machine_is_compatible("apple,j700") && - of_machine_is_compatible("apple,t8140"); -} -EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_enabled); - static void apple_piodma_put_supplier(void *data) { put_device(data); @@ -590,19 +571,19 @@ static void apple_piodma_put_supplier(void *data) int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) { struct device_node *node, *host_node; - struct apple_piodma_diag *diag; + struct apple_t8140_bootstrap *bootstrap; struct platform_device *pdev; struct device_link *link; bool correct_host; int ret; - if (!apple_piodma_bootstrap_enabled() || - !of_device_is_compatible(host->of_node, "apple,t8140-pcie")) + if (!of_device_is_compatible(host->of_node, "apple,t8140-pcie")) return -ENODEV; node = of_parse_phandle(host->of_node, "apple,piodma", 0); if (!node) return -EINVAL; - if (!of_device_is_compatible(node, "apple,t8140-piodma-diagnostic")) { + if (!of_device_is_compatible(node, "apple,t8140-pcie-bootstrap") && + !of_device_is_compatible(node, "apple,t8140-piodma-diagnostic")) { of_node_put(node); return -EINVAL; } @@ -615,8 +596,8 @@ int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) return -EINVAL; if (!pdev) return -EPROBE_DEFER; - diag = platform_get_drvdata(pdev); - if (!device_is_bound(&pdev->dev) || !diag) { + bootstrap = platform_get_drvdata(pdev); + if (!device_is_bound(&pdev->dev) || !bootstrap) { put_device(&pdev->dev); return -EPROBE_DEFER; } @@ -626,7 +607,7 @@ int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) return -ESHUTDOWN; } /* Host probe holds its device lock; never rebuild a consumed hierarchy. */ - if (atomic_read(&diag->attempted)) { + if (apple_piodma_attempted(apple_piodma_state(bootstrap))) { put_device(&pdev->dev); return -EALREADY; } @@ -645,25 +626,25 @@ EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_get); static void apple_piodma_retry_admission(struct work_struct *work) { - struct apple_piodma_diag *diag = - container_of(to_delayed_work(work), struct apple_piodma_diag, + struct apple_t8140_bootstrap *bootstrap = + container_of(to_delayed_work(work), struct apple_t8140_bootstrap, admission_retry); - /* Acquire the matched host's permanent reference from the scheduler. */ - struct device *host = smp_load_acquire(&diag->retry_host); + /* Pairs with the scheduler's cmpxchg of the boot-lifetime host reference. */ + struct device *host = smp_load_acquire(&bootstrap->retry_host); int ret; if (system_state > SYSTEM_RUNNING || !host || - !device_is_registered(host) || atomic_read(&diag->attempted) || - READ_ONCE(diag->guarded)) + !device_is_registered(host) || apple_piodma_attempted(apple_piodma_state(bootstrap)) || + apple_piodma_guarded(apple_piodma_state(bootstrap))) return; /* The core rechecks registration and binding under the host device lock. */ ret = device_attach(host); if (ret < 0) - dev_dbg(diag->dev, "deferred host attachment returned %d\n", ret); + dev_dbg(bootstrap->dev, "deferred host attachment returned %d\n", ret); /* Only fresh pre-child guard contention can schedule another attempt. */ } -static void apple_piodma_schedule_admission(struct apple_piodma_diag *diag, +static void apple_piodma_schedule_admission(struct apple_t8140_bootstrap *bootstrap, struct device *host) { struct device *previous; @@ -671,18 +652,18 @@ static void apple_piodma_schedule_admission(struct apple_piodma_diag *diag, if (system_state > SYSTEM_RUNNING || !device_is_registered(host)) return; /* Publish a boot-lifetime reference; this supplier has one matched host. */ - previous = cmpxchg(&diag->retry_host, NULL, get_device(host)); + previous = cmpxchg(&bootstrap->retry_host, NULL, get_device(host)); if (previous) { put_device(host); if (WARN_ON_ONCE(previous != host)) return; } - mod_delayed_work(system_dfl_wq, &diag->admission_retry, HZ); + mod_delayed_work(system_dfl_wq, &bootstrap->admission_retry, HZ); } int apple_piodma_bootstrap_guard_begin(struct device *supplier, struct device *host) { - struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); unsigned int sleep_flags; int ret; @@ -690,24 +671,25 @@ int apple_piodma_bootstrap_guard_begin(struct device *supplier, struct device *h if (system_state > SYSTEM_RUNNING) return -ESHUTDOWN; /* A worker may have waited behind a host probe that consumed its attempt. */ - if (atomic_read(&diag->attempted) || READ_ONCE(diag->guarded)) + if (apple_piodma_attempted(apple_piodma_state(bootstrap)) || + apple_piodma_guarded(apple_piodma_state(bootstrap))) return -EALREADY; /* This is the last dependency admission before PCI children exist. */ - ret = apple_piodma_lock_transition(diag, &sleep_flags); + ret = apple_piodma_lock_transition(bootstrap, &sleep_flags); if (ret) { if (ret == -EPROBE_DEFER) - apple_piodma_schedule_admission(diag, host); + apple_piodma_schedule_admission(bootstrap, host); return ret; } /* Do not wait for a worker that may be waiting on this host device lock. */ - cancel_delayed_work(&diag->admission_retry); - diag->pm_notifier.notifier_call = apple_piodma_pm_notify; - ret = register_pm_notifier(&diag->pm_notifier); + cancel_delayed_work(&bootstrap->admission_retry); + bootstrap->pm_notifier.notifier_call = apple_piodma_pm_notify; + ret = register_pm_notifier(&bootstrap->pm_notifier); if (ret) { - kexec_unblock(&diag->kexec_blocker); + kexec_unblock(&bootstrap->kexec_blocker); } else { /* Publish both vetoes before permitting system sleep to proceed. */ - smp_store_release(&diag->guarded, true); + WARN_ON_ONCE(apple_piodma_transition(bootstrap, PIODMA_GUARD)); } unlock_system_sleep(sleep_flags); return ret; @@ -716,42 +698,47 @@ EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_guard_begin); void apple_piodma_bootstrap_guard_end(struct device *supplier) { - struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); /* Post-publication ownership is irreversible until hardware reset. */ - if (!READ_ONCE(diag->guarded) || smp_load_acquire(&diag->retained)) + if (!apple_piodma_guarded(apple_piodma_state(bootstrap)) || + apple_piodma_retained(apple_piodma_state(bootstrap))) return; /* The host has removed every child before permitting sleep again. */ - smp_store_release(&diag->guarded, false); - unregister_pm_notifier(&diag->pm_notifier); - kexec_unblock(&diag->kexec_blocker); + if (WARN_ON_ONCE(apple_piodma_transition(bootstrap, PIODMA_RELEASE))) + return; + unregister_pm_notifier(&bootstrap->pm_notifier); + kexec_unblock(&bootstrap->kexec_blocker); } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_guard_end); int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) { - struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); int ret; /* Admission was completed before pci_host_common_init published children. */ - if (!smp_load_acquire(&diag->guarded)) + if (!apple_piodma_guarded(apple_piodma_state(bootstrap))) return -EINVAL; /* One hardware attempt, including permanent preflight failures. */ - if (atomic_cmpxchg(&diag->attempted, 0, 1)) + if (apple_piodma_transition(bootstrap, PIODMA_PRIME)) return -EALREADY; - ret = apple_piodma_diag_root(diag, root); - if (!ret) - ret = apple_piodma_diag_idle(diag); + ret = apple_t8140_bootstrap_root(bootstrap, root); if (!ret) - ret = apple_piodma_diag_initialize(diag); + ret = apple_t8140_bootstrap_idle(bootstrap); + if (!ret) { + /* Ownership becomes irreversible before the first engine write. */ + ret = apple_piodma_transition(bootstrap, PIODMA_PUBLISH); + if (!ret) + ret = apple_t8140_bootstrap_initialize(bootstrap); + } if (!ret) - ret = apple_piodma_diag_submit(diag, 0); + ret = apple_t8140_bootstrap_submit(bootstrap, 0); if (ret) { - dev_err(supplier, "bootstrap failure=%d; retained ownership; no retry\n", ret); + WARN_ON_ONCE(apple_piodma_transition(bootstrap, PIODMA_FAIL)); + dev_err(supplier, "bootstrap failed: %d; full hardware reset required\n", ret); } else { - WRITE_ONCE(diag->ready, true); - dev_info(supplier, "BOOTSTRAP_VALIDATED word=%#x; one retained slot\n", - diag->slots[0].result); + WARN_ON_ONCE(apple_piodma_transition(bootstrap, PIODMA_COMPLETE)); } return ret; } @@ -759,187 +746,188 @@ EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_prime); bool apple_piodma_bootstrap_retained(struct device *supplier) { - struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); /* Acquire the prime's irreversible publication and guard ownership. */ - return smp_load_acquire(&diag->retained); + return apple_piodma_retained(apple_piodma_state(bootstrap)); } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_retained); void apple_piodma_bootstrap_admitted(struct device *supplier) { - struct apple_piodma_diag *diag = dev_get_drvdata(supplier); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(supplier); - if (WARN_ON_ONCE(!diag || !diag->retained)) + if (WARN_ON_ONCE(!bootstrap || !apple_piodma_retained(apple_piodma_state(bootstrap)))) return; /* Publish the host's completed cold, resource and ASPM admission checks. */ - smp_store_release(&diag->radio_admitted, true); + WARN_ON_ONCE(apple_piodma_transition(bootstrap, PIODMA_ADMIT)); } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_admitted); int apple_piodma_radio_check(struct pci_dev *pdev) { - /* Acquire the supplier's permanent arena and PM/kexec guard ownership. */ - struct apple_piodma_diag *diag = smp_load_acquire(&active_diag); + /* Pairs with probe's smp_store_release of the boot-lifetime supplier. */ + struct apple_t8140_bootstrap *bootstrap = smp_load_acquire(&active_bootstrap); struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->bus); struct device_node *host; bool matches; - if (!apple_piodma_bootstrap_enabled() || !diag) + if (!bootstrap) return -ENODEV; /* A retained supplier alone does not prove that host admission succeeded. */ - if (!smp_load_acquire(&diag->radio_admitted) || !bridge->dev.parent || + if (apple_piodma_state(bootstrap) != PIODMA_ADMITTED || !bridge->dev.parent || pdev->vendor != PCI_VENDOR_ID_MEDIATEK || - pci_domain_nr(pdev->bus) || pdev->bus->number != diag->secondary_bus || + pci_domain_nr(pdev->bus) || pdev->bus->number != bootstrap->secondary_bus || PCI_SLOT(pdev->devfn) || (pdev->devfn != PCI_DEVFN(0, 0) && pdev->devfn != PCI_DEVFN(0, 1)) || pdev->device != (PCI_FUNC(pdev->devfn) ? 0x793b : 0x7932)) return -ENODEV; - host = of_parse_phandle(diag->dev->of_node, "apple,pcie-host", 0); + host = of_parse_phandle(bootstrap->dev->of_node, "apple,pcie-host", 0); matches = host && host == bridge->dev.parent->of_node; of_node_put(host); return matches ? 0 : -ENODEV; } EXPORT_SYMBOL_GPL(apple_piodma_radio_check); -static int apple_piodma_diag_probe(struct platform_device *pdev) +static int apple_t8140_bootstrap_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct apple_piodma_diag *diag; + struct apple_t8140_bootstrap *bootstrap; int i, ret; - if (!apple_piodma_bootstrap_enabled() || PAGE_SIZE != SZ_16K) + if (PAGE_SIZE != SZ_16K) return -ENODEV; - diag = kzalloc_obj(*diag); - if (!diag) + bootstrap = kzalloc_obj(*bootstrap); + if (!bootstrap) return -ENOMEM; - diag->dev = dev; - INIT_DELAYED_WORK(&diag->admission_retry, apple_piodma_retry_admission); - diag->kexec_blocker.reason = + bootstrap->dev = dev; + INIT_DELAYED_WORK(&bootstrap->admission_retry, apple_piodma_retry_admission); + bootstrap->kexec_blocker.reason = "Neo radio admission or retained DMA ownership"; for (i = 0; i < PIODMA_REQUEST_COUNT; i++) - init_completion(&diag->slots[i].event); - atomic_set(&diag->irq_count, 0); - atomic_set(&diag->attempted, 0); - ret = apple_piodma_diag_iommu(diag); + init_completion(&bootstrap->slots[i].event); + atomic_set(&bootstrap->irq_count, 0); + atomic_set(&bootstrap->state, PIODMA_IDLE); + ret = apple_t8140_bootstrap_iommu(bootstrap); if (ret) goto err_free; pm_runtime_enable(dev); ret = pm_runtime_resume_and_get(dev); if (ret < 0) goto err_pm; - ret = apple_piodma_diag_arena(diag); + ret = apple_t8140_bootstrap_arena(bootstrap); if (ret) goto err_arena; - ret = apple_piodma_diag_resources(pdev, diag); + ret = apple_t8140_bootstrap_resources(pdev, bootstrap); if (ret) goto err_arena; - ret = devm_request_irq(dev, diag->irq, apple_piodma_diag_irq, 0, - dev_name(dev), diag); + ret = devm_request_irq(dev, bootstrap->irq, apple_t8140_bootstrap_irq, 0, + dev_name(dev), bootstrap); if (ret) goto err_arena; /* The supplier and unpublished arena keep their device alive. */ get_device(dev); - platform_set_drvdata(pdev, diag); - /* Publish the CPU-owned arena and permanent device reference together. */ - smp_store_release(&active_diag, diag); - dev_info(dev, "bootstrap supplier ready; no command submitted during probe\n"); + platform_set_drvdata(pdev, bootstrap); + /* Pairs with radio_check's smp_load_acquire of this boot-lifetime owner. */ + smp_store_release(&active_bootstrap, bootstrap); + dev_dbg(dev, "bootstrap supplier ready; no command submitted during probe\n"); return 0; err_arena: - if (diag->arena) - dma_free_coherent(dev, PIODMA_ARENA_SIZE, diag->arena, diag->dma); + if (bootstrap->arena) + dma_free_coherent(dev, PIODMA_ARENA_SIZE, bootstrap->arena, bootstrap->dma); pm_runtime_put_sync(dev); err_pm: pm_runtime_disable(dev); err_free: - kfree(diag); - return dev_err_probe(dev, ret, "PIODMA diagnostic preflight failed\n"); + kfree(bootstrap); + return dev_err_probe(dev, ret, "PCIe bootstrap preflight failed\n"); } -static void apple_piodma_diag_shutdown(struct platform_device *pdev) +static void apple_t8140_bootstrap_shutdown(struct platform_device *pdev) { - struct apple_piodma_diag *diag = platform_get_drvdata(pdev); + struct apple_t8140_bootstrap *bootstrap = platform_get_drvdata(pdev); /* * Shutdown holds supplier/parent locks: never join a host attachment * here. Admission rejects shutdown, and the supplier lifetime is retained. */ - cancel_delayed_work(&diag->admission_retry); - if (READ_ONCE(diag->retained)) - dev_warn(&pdev->dev, "diagnostic arena retained; require full hardware reset\n"); + cancel_delayed_work(&bootstrap->admission_retry); + if (apple_piodma_retained(apple_piodma_state(bootstrap))) + dev_warn(&pdev->dev, "bootstrap arena retained; require full hardware reset\n"); } /* No new request is permitted once the host has admitted its endpoints. */ -static int apple_piodma_diag_sleep_check(struct apple_piodma_diag *diag) +static int apple_t8140_bootstrap_sleep_check(struct apple_t8140_bootstrap *bootstrap) { - if (!READ_ONCE(diag->ready) || READ_ONCE(diag->irq_fault)) + if (!apple_piodma_ready(apple_piodma_state(bootstrap)) || READ_ONCE(bootstrap->irq_fault)) return -EIO; - if (readl(diag->engine + 0x54) & BIT(0)) + if (readl(bootstrap->engine + 0x54) & BIT(0)) return -EBUSY; - if ((readl(diag->engine + 0x0c) & PIODMA_FIFO_MASK) != 16) + if ((readl(bootstrap->engine + 0x0c) & PIODMA_FIFO_MASK) != 16) return -EBUSY; - return apple_piodma_diag_check_slots(diag); + return apple_t8140_bootstrap_check_slots(bootstrap); } -static int apple_piodma_diag_suspend_noirq(struct device *dev) +static int apple_t8140_bootstrap_suspend_noirq(struct device *dev) { - struct apple_piodma_diag *diag = dev_get_drvdata(dev); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(dev); - if (!diag->retained) + if (!apple_piodma_retained(apple_piodma_state(bootstrap))) return 0; if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) return -EOPNOTSUPP; - return apple_piodma_diag_sleep_check(diag); + return apple_t8140_bootstrap_sleep_check(bootstrap); } -static int apple_piodma_diag_resume_noirq(struct device *dev) +static int apple_t8140_bootstrap_resume_noirq(struct device *dev) { - struct apple_piodma_diag *diag = dev_get_drvdata(dev); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(dev); int ret; - if (!diag->retained) + if (!apple_piodma_retained(apple_piodma_state(bootstrap))) return 0; - if (!READ_ONCE(diag->ready) || READ_ONCE(diag->irq_fault)) + if (!apple_piodma_ready(apple_piodma_state(bootstrap)) || READ_ONCE(bootstrap->irq_fault)) return -EIO; /* Genpd may gate this island; restore configuration, never a command. */ - ret = apple_piodma_diag_initialize(diag); + ret = apple_t8140_bootstrap_initialize(bootstrap); if (ret) return ret; - return apple_piodma_diag_sleep_check(diag); + return apple_t8140_bootstrap_sleep_check(bootstrap); } -static int apple_piodma_diag_freeze(struct device *dev) +static int apple_t8140_bootstrap_freeze(struct device *dev) { - struct apple_piodma_diag *diag = dev_get_drvdata(dev); + struct apple_t8140_bootstrap *bootstrap = dev_get_drvdata(dev); - return diag->retained ? -EOPNOTSUPP : 0; + return apple_piodma_retained(apple_piodma_state(bootstrap)) ? -EOPNOTSUPP : 0; } -static const struct dev_pm_ops apple_piodma_diag_pm_ops = { +static const struct dev_pm_ops apple_t8140_bootstrap_pm_ops = { .prepare = apple_piodma_pm_prepare, - .suspend_noirq = apple_piodma_diag_suspend_noirq, - .resume_noirq = apple_piodma_diag_resume_noirq, - .freeze = apple_piodma_diag_freeze, - .poweroff = apple_piodma_diag_freeze, + .suspend_noirq = apple_t8140_bootstrap_suspend_noirq, + .resume_noirq = apple_t8140_bootstrap_resume_noirq, + .freeze = apple_t8140_bootstrap_freeze, + .poweroff = apple_t8140_bootstrap_freeze, }; -static const struct of_device_id apple_piodma_diag_match[] = { +static const struct of_device_id apple_t8140_bootstrap_match[] = { + { .compatible = "apple,t8140-pcie-bootstrap" }, { .compatible = "apple,t8140-piodma-diagnostic" }, { } }; -static struct platform_driver apple_piodma_diag_driver = { - .probe = apple_piodma_diag_probe, - .shutdown = apple_piodma_diag_shutdown, +static struct platform_driver apple_t8140_bootstrap_driver = { + .probe = apple_t8140_bootstrap_probe, + .shutdown = apple_t8140_bootstrap_shutdown, .driver = { - .name = "apple-piodma-diag", - .of_match_table = apple_piodma_diag_match, - .pm = pm_sleep_ptr(&apple_piodma_diag_pm_ops), + .name = "apple-t8140-bootstrap", + .of_match_table = apple_t8140_bootstrap_match, + .pm = pm_sleep_ptr(&apple_t8140_bootstrap_pm_ops), .suppress_bind_attrs = true, }, }; -builtin_platform_driver(apple_piodma_diag_driver); -MODULE_DESCRIPTION("T8140 retained one-read PCIe bootstrap experiment"); +builtin_platform_driver(apple_t8140_bootstrap_driver); +MODULE_DESCRIPTION("Apple T8140 radio PCIe bootstrap"); MODULE_LICENSE("GPL"); diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.h b/drivers/pci/controller/pcie-apple-t8140-bootstrap.h similarity index 84% rename from drivers/pci/controller/pcie-apple-piodma-diag.h rename to drivers/pci/controller/pcie-apple-t8140-bootstrap.h index c8d29335ed0cb..b7c4eeedb92c9 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.h +++ b/drivers/pci/controller/pcie-apple-t8140-bootstrap.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef PCIE_APPLE_PIODMA_DIAG_H -#define PCIE_APPLE_PIODMA_DIAG_H +#ifndef PCIE_APPLE_T8140_BOOTSTRAP_H +#define PCIE_APPLE_T8140_BOOTSTRAP_H #include #include @@ -9,8 +9,7 @@ struct device; struct pci_dev; -#if IS_ENABLED(CONFIG_PCIE_APPLE_PIODMA_DIAG) -bool apple_piodma_bootstrap_enabled(void); +#if IS_ENABLED(CONFIG_PCIE_APPLE_T8140_BOOTSTRAP) int apple_piodma_bootstrap_get(struct device *host, struct device **supplier); int apple_piodma_bootstrap_guard_begin(struct device *supplier, struct device *host); @@ -19,11 +18,6 @@ int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root); void apple_piodma_bootstrap_admitted(struct device *supplier); bool apple_piodma_bootstrap_retained(struct device *supplier); #else -static inline bool apple_piodma_bootstrap_enabled(void) -{ - return false; -} - static inline int apple_piodma_bootstrap_get(struct device *host, struct device **supplier) { return -ENODEV; diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index e0b6ead2526e2..d9e0955fa1fc3 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -44,7 +44,7 @@ #include "../pci.h" #include "pci-host-common.h" -#include "pcie-apple-piodma-diag.h" +#include "pcie-apple-t8140-bootstrap.h" static ATOMIC_NOTIFIER_HEAD(apple_pcie_tunnel_notifiers); @@ -68,57 +68,6 @@ static int link_up_timeout = 500; module_param(link_up_timeout, int, 0644); MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); -static bool tunnel_kernel_init = true; -module_param(tunnel_kernel_init, bool, 0444); -MODULE_PARM_DESC(tunnel_kernel_init, - "Cold-initialize T600x/T602x PCIe-C ports without an m1n1 handoff (default: on)"); - -static const char *const apple_pcie_tunnel_cold_init_machines[] = { - "apple,t6000", "apple,t6001", "apple,t6020", "apple,t6021", NULL -}; - -/** - * apple_pcie_tunnel_needs_cold_init() - whether the kernel brings up a port - * @np: PCIe-C host node - * - * No m1n1 performs the PCIe-C preinit handoff on T600x/T602x, so the kernel - * cold-initializes these ports instead; pcie_apple.tunnel_kernel_init=0 turns - * that off. It is a parameter rather than a DT property because m1n1 hands - * every installed kernel the same DT. The tunnel's DART cannot probe without - * its tunables, which m1n1 does not provide, so ports whose DART lacks them - * are refused. - * - * Unlike "apple,pciec-kernel-init" (t8103), this does not select the t8103 - * power and resume sequences. - */ -bool apple_pcie_tunnel_needs_cold_init(struct device_node *np) -{ - struct of_phandle_args iommu_spec = {}; - struct device_node *dart; - bool tunables; - u32 status; - - if (!tunnel_kernel_init || - !of_device_is_compatible(np, "apple,t6000-pciec") || - !of_machine_compatible_match(apple_pcie_tunnel_cold_init_machines)) - return false; - - if (!of_property_read_u32(np, "apple,pciec-preinit-status", &status) && - status == 1) - return false; - - if (of_map_iommu_id(np, 0, &iommu_spec) || !iommu_spec.np) - return false; - dart = iommu_spec.np; - tunables = of_property_present(dart, "apple,tunable"); - if (!tunables) - pr_warn_once("%pOF: no PCIe-C DART tunables, not cold-initializing\n", - dart); - of_node_put(dart); - return tunables; -} -EXPORT_SYMBOL_GPL(apple_pcie_tunnel_needs_cold_init); - /* T8103 (original M1) and related SoCs */ #define CORE_RC_PHYIF_CTL 0x00024 #define CORE_RC_PHYIF_CTL_RUN BIT(0) @@ -250,6 +199,7 @@ struct hw_info { bool retain_clocks; bool root_bus_only; bool tunneled; + bool tunnel_intr2axi; }; static const struct hw_info t8103_hw = { @@ -301,6 +251,16 @@ static const struct hw_info t8103_pciec_hw = { .tunneled = true, }; +/* T600x and T8112 use Intr2AXI instead of the T8103 oe-fabric. */ +static const struct hw_info t8112_pciec_hw = { + .port_msiaddr = PORT_MSIADDR, + .port_perst = PORT_PERST, + .port_rid2sid = PORT_RID2SID, + .max_rid2sid = 64, + .tunneled = true, + .tunnel_intr2axi = true, +}; + /* * t8122 PCIe-C keeps the t8103 doorbell and RID2SID registers, but MSI * delivery uses the t602x map. The map is indexed by the MSI data, and @@ -315,6 +275,46 @@ static const struct hw_info t8122_pciec_hw = { .tunneled = true, }; +static const struct of_device_id apple_pcie_of_match[]; + +/** + * apple_pcie_tunnel_needs_cold_init() - whether the kernel brings up a port + * @np: PCIe-C host node + * + * DT selects kernel initialization; match data selects the hardware sequence. + * Never reset a qualified firmware handoff. Intr2AXI controllers also need + * DART tunables before ACIO may populate their cable-powered children. + */ +bool apple_pcie_tunnel_needs_cold_init(struct device_node *np) +{ + const struct of_device_id *match = of_match_node(apple_pcie_of_match, np); + struct of_phandle_args iommu_spec = {}; + const struct hw_info *hw; + bool tunables; + u32 status; + + if (!match || !of_property_read_bool(np, "apple,pciec-kernel-init")) + return false; + hw = match->data; + if (!hw->tunneled) + return false; + if (!of_property_read_u32(np, "apple,pciec-preinit-status", &status) && + status == 1) + return false; + if (!hw->tunnel_intr2axi) + return true; + + if (of_map_iommu_id(np, 0, &iommu_spec) || !iommu_spec.np) + return false; + tunables = of_property_present(iommu_spec.np, "apple,tunable"); + if (!tunables) + pr_warn_once("%pOF: no PCIe-C DART tunables, not cold-initializing\n", + iommu_spec.np); + of_node_put(iommu_spec.np); + return tunables; +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_needs_cold_init); + struct apple_pcie { struct mutex lock; struct device *dev; @@ -991,9 +991,9 @@ static int apple_pcie_tunnel_reinitialize(struct apple_pcie_port *port) } /* - * Bring up a port that no firmware handoff left running (t8103, or - * T600x/T602x with pcie_apple.tunnel_kernel_init): tunables and the reset - * table, then PERST and APPCLK, and only then the root-port config-space + * Bring up a port selected by apple,pciec-kernel-init that no firmware + * handoff left running: tunables and the reset table, then PERST and APPCLK, + * and only then the root-port config-space * tunables. Stop before releasing the tunnel reset or starting link * training. */ @@ -1668,8 +1668,8 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, * m1n1 owns PCIe-C cold initialization on T6020. Replaying the * port reset against that live handoff raises an asynchronous * SError. t8103 boots with ATC_PCIE off and no handoff, so - * apple,pciec-kernel-init selects the in-kernel sequence, as - * pcie_apple.tunnel_kernel_init does on T600x/T602x. + * apple,pciec-kernel-init selects the in-kernel sequence on each + * supported generation. */ ret = of_property_read_u32(pcie->dev->of_node, "apple,pciec-preinit-status", @@ -1688,7 +1688,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, } } else if (!preinit_ok) { dev_err(pcie->dev, - "PCIe-C requires an m1n1 preinit handoff or pcie_apple.tunnel_kernel_init=1\n"); + "PCIe-C requires a firmware handoff or apple,pciec-kernel-init\n"); ret = -ENODEV; goto err_teardown; } else { @@ -1813,10 +1813,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (pcie->hw->tunneled) WRITE_ONCE(port->started, true); if (pcie->hw->root_bus_only) - dev_info(pcie->dev, "port %pOF RUN=%#x link=%#x; downstream config %s\n", - np, apple_pcie_port_readl(port, PORT_STATUS), - apple_pcie_port_readl(port, PORT_LINKSTS), - pcie->piodma_supplier ? "waits for the bootstrap" : "blocked"); + dev_dbg(pcie->dev, "port %pOF RUN=%#x link=%#x; downstream config %s\n", + np, apple_pcie_port_readl(port, PORT_STATUS), + apple_pcie_port_readl(port, PORT_LINKSTS), + pcie->piodma_supplier ? "waits for the bootstrap" : "blocked"); return 0; @@ -2296,7 +2296,7 @@ static int apple_pcie_init(struct pci_config_window *cfg) } /* - * The T8140 experiment starts with all downstream accesses blocked. + * The T8140 host starts with all downstream accesses blocked. * Only a successful owned bootstrap and typed ECAM checks open functions0/1. * Generic config callbacks retain their normal native widths and locking. */ @@ -2651,9 +2651,9 @@ static int apple_pcie_neo_typed_ids(struct apple_pcie *pcie, struct pci_dev *roo low = readb(cfg + PCI_VENDOR_ID); high = readb(cfg + PCI_VENDOR_ID + 1); header = readb(cfg + PCI_HEADER_TYPE); - dev_info(pcie->dev, - "typed ECAM function=%d id=%#x vendor=%#x device=%#x bytes=%02x:%02x header=%#x\n", - function, id, vendor, device, low, high, header); + dev_dbg(pcie->dev, + "typed ECAM function=%d id=%#x vendor=%#x device=%#x bytes=%02x:%02x header=%#x\n", + function, id, vendor, device, low, high, header); if (id != ids[function] || vendor != (ids[function] & 0xffff) || device != ids[function] >> 16 || low != 0xc3 || high != 0x14 || (header & PCI_HEADER_TYPE_MASK) != PCI_HEADER_TYPE_NORMAL || @@ -2676,8 +2676,8 @@ static int apple_pcie_neo_check_memory(struct pci_dev *root) ret = pci_read_config_word(root, PCI_COMMAND, &command); if (!ret) ret = pci_read_config_dword(root, PCI_MEMORY_BASE, &window); - pci_info(root, "NEO_ROOT_MEMORY_READBACK status=%d command=%#x window=%#x\n", - ret, command, window); + pci_dbg(root, "NEO_ROOT_MEMORY_READBACK status=%d command=%#x window=%#x\n", + ret, command, window); if (ret || command == U16_MAX || !(command & PCI_COMMAND_MEMORY) || window == U32_MAX) return -EIO; /* Each packed register field is12 bits; PCI_MEMORY_RANGE_MASK is unsigned long. */ @@ -2706,8 +2706,8 @@ static int apple_pcie_neo_check_memory(struct pci_dev *root) if (ret) return ret; } - pci_info(root, "NEO_ROOT_MEMORY_ENABLED command=%#x window=%#x bus=%#llx-%#llx\n", - command, window, base, limit); + pci_dbg(root, "NEO_ROOT_MEMORY_ENABLED command=%#x window=%#x bus=%#llx-%#llx\n", + command, window, base, limit); return 0; } @@ -2731,8 +2731,8 @@ static int apple_pcie_neo_check_cold(struct apple_pcie *pcie, struct pci_dev *ro if (pci_read_config_word(endpoint, PCI_COMMAND, &command)) { ret = ret ?: -EIO; } else { - dev_info(pcie->dev, "enumerated %s id=%04x:%04x command=%#x\n", - pci_name(endpoint), endpoint->vendor, endpoint->device, command); + dev_dbg(pcie->dev, "enumerated %s id=%04x:%04x command=%#x\n", + pci_name(endpoint), endpoint->vendor, endpoint->device, command); if (command & PCI_COMMAND_MASTER) { pci_clear_master(endpoint); ret = ret ?: -EIO; @@ -2803,7 +2803,7 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) } pci_unlock_rescan_remove(); if (!ret) - dev_info(pcie->dev, "NEO_RADIO_READY; functions0/1 admitted with link power states disabled\n"); + dev_info(pcie->dev, "radio PCIe bootstrap complete; Wi-Fi and Bluetooth admitted\n"); out: pci_dev_put(root); return ret; @@ -2842,7 +2842,7 @@ static int apple_pcie_probe(struct platform_device *pdev) } /* Only a host that the board links to a bootstrap engine opens bus 1. */ - if (hw->root_bus_only && apple_piodma_bootstrap_enabled() && + if (hw->root_bus_only && IS_ENABLED(CONFIG_PCIE_APPLE_T8140_BOOTSTRAP) && of_property_present(dev->of_node, "apple,piodma")) { ret = apple_piodma_bootstrap_get(dev, &piodma_supplier); if (ret) @@ -2880,7 +2880,7 @@ static int apple_pcie_probe(struct platform_device *pdev) bridge->native_shpc_hotplug = false; bridge->native_aer = false; bridge->native_dpc = false; - dev_info_once(dev, "radio host: automatic AER/DPC recovery disabled for retained DMA\n"); + dev_dbg(dev, "radio host: automatic AER/DPC recovery disabled for retained DMA\n"); } if (hw->root_bus_only && !piodma_supplier) dev_info(dev, "root-port probe only; downstream config is blocked\n"); @@ -2889,9 +2889,9 @@ static int apple_pcie_probe(struct platform_device *pdev) return PTR_ERR(pcie->base); if (pcie->hw->tunneled) { pcie->kernel_init = - of_device_is_compatible(dev->of_node, "apple,t8103-pciec") && + !hw->tunnel_intr2axi && of_property_read_bool(dev->of_node, "apple,pciec-kernel-init"); - pcie->tunnel_cold_init = + pcie->tunnel_cold_init = hw->tunnel_intr2axi && apple_pcie_tunnel_needs_cold_init(dev->of_node); ret = apple_pcie_tunnel_init_resources(pdev, pcie); if (ret) @@ -2991,7 +2991,7 @@ static int apple_pcie_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "Neo bootstrap preflight failed\n"); } if (ret) - dev_err(dev, "enumeration experiment failed=%d; host/supplier retained, no retry\n", + dev_err(dev, "radio admission failed=%d; host/supplier retained, no retry\n", ret); return 0; } @@ -3288,12 +3288,11 @@ static struct apple_tunable *apple_pcie_tunable_once(struct device_node *np, * probes immediately and its invalidate command never completes while * APPCLK is still gated. */ -static const struct of_device_id apple_pcie_of_match[]; - int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) { struct device_node *np, *port_np = NULL; const struct of_device_id *match; + const struct hw_info *hw; struct apple_pcie *pcie; struct apple_pcie_port *port; struct apple_pcie_map config = {}, debug = {}, fabric = {}, portmap = {}; @@ -3304,9 +3303,7 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) np = NULL; for_each_available_child_of_node(tunnel, np) { - kernel_init = of_device_is_compatible(np, "apple,t8103-pciec") && - of_property_read_bool(np, "apple,pciec-kernel-init"); - if (kernel_init || apple_pcie_tunnel_needs_cold_init(np)) + if (apple_pcie_tunnel_needs_cold_init(np)) break; } if (!np) @@ -3324,6 +3321,8 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) ret = -ENODEV; goto out_np; } + hw = match->data; + kernel_init = !hw->tunnel_intr2axi; ret = apple_pcie_map_named(np, "port0", &portmap); if (ret) @@ -3720,8 +3719,9 @@ static const struct dev_pm_ops apple_pcie_pm_ops = { static const struct of_device_id apple_pcie_of_match[] = { { .compatible = "apple,t8122-pciec", .data = &t8122_pciec_hw }, + { .compatible = "apple,t8112-pciec", .data = &t8112_pciec_hw }, { .compatible = "apple,t8103-pciec", .data = &t8103_pciec_hw }, - { .compatible = "apple,t6000-pciec", .data = &t8103_pciec_hw }, + { .compatible = "apple,t6000-pciec", .data = &t8112_pciec_hw }, /* T8132 currently inherits controller and PHY setup from m1n1. */ { .compatible = "apple,t8132-pcie", .data = &t602x_hw }, { .compatible = "apple,t6020-pcie", .data = &t602x_hw }, diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index 366b4b86f115e..b64bcdbee1bb6 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -676,6 +676,98 @@ static void atcphy_test_stopped_reset(struct kunit *test) KUNIT_EXPECT_FALSE(test, aon & PIPEHANDLER_AON_GEN_DWC3_RESET_N); } +static int atcphy_test_count_refused_lock(struct apple_atcphy *atcphy) +{ + unsigned int *requests = kunit_get_current_test()->priv; + + (*requests)++; + return -ETIMEDOUT; +} + +static struct apple_atcphy *atcphy_test_alloc_legacy_pipe(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_dev(test, false); + void *lpdptx = kunit_kzalloc(test, LPDPTX_AUX_CONTROL + 4, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, lpdptx); + mutex_init(&atcphy->lock); + atcphy_test_fake_regs(test, atcphy); + atcphy->regs.lpdptx = (__force void __iomem *)lpdptx; + atcphy->host_active = true; + kunit_activate_static_stub(test, atcphy_pipehandler_lock, + atcphy_test_count_refused_lock); + + return atcphy; +} + +/* Ordinary M1/M2 ports have no PIPE ACK clock once xHCI has stopped. */ +static void atcphy_test_legacy_stopped_power_off(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_legacy_pipe(test); + struct phy *phy = kunit_kzalloc(test, sizeof(*phy), GFP_KERNEL); + unsigned int requests = 0; + + KUNIT_ASSERT_NOT_NULL(test, phy); + phy_set_drvdata(phy, atcphy); + test->priv = &requests; + + for (unsigned int pending = 0; pending < 2; pending++) { + for (unsigned int dp = 0; dp < 2; dp++) { + atcphy->mode = dp ? APPLE_ATCPHY_MODE_USB3_DP : APPLE_ATCPHY_MODE_USB3; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; + atcphy->pipe_pending = pending; + atcphy->host_active = true; + KUNIT_ASSERT_EQ(test, atcphy_usb3_power_off(phy), 0); + KUNIT_EXPECT_EQ(test, requests, 0U); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); + KUNIT_EXPECT_FALSE(test, atcphy->host_active); + KUNIT_EXPECT_EQ(test, atcphy->mode, + dp ? APPLE_ATCPHY_MODE_USB3_DP : APPLE_ATCPHY_MODE_OFF); + } + } +} + +static void atcphy_test_legacy_stopped_reset(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_legacy_pipe(test); + unsigned int requests = 0; + u32 aon; + + test->priv = &requests; + atcphy->pipe_pending = true; + KUNIT_ASSERT_EQ(test, atcphy_dwc3_reset_assert(&atcphy->rcdev, 0), 0); + KUNIT_EXPECT_EQ(test, requests, 0U); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); + KUNIT_EXPECT_FALSE(test, atcphy->host_active); + aon = readl(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN); + KUNIT_EXPECT_TRUE(test, aon & PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + KUNIT_EXPECT_FALSE(test, aon & PIPEHANDLER_AON_GEN_DWC3_RESET_N); +} + +/* Neither a live pending handoff nor USB3 host setup may bypass the ACK. */ +static void atcphy_test_legacy_live_lock(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_legacy_pipe(test); + unsigned int requests = 0; + + test->priv = &requests; + atcphy->pipe_pending = true; + scoped_guard(mutex, &atcphy->lock) + KUNIT_EXPECT_EQ(test, atcphy_park_pipehandler(atcphy, false), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, requests, 1U); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_USB3); + KUNIT_EXPECT_TRUE(test, atcphy->pipe_pending); + KUNIT_EXPECT_TRUE(test, atcphy->mode_dirty); + + atcphy->mode_dirty = false; + scoped_guard(mutex, &atcphy->lock) + KUNIT_EXPECT_EQ(test, atcphy_configure_pipehandler(atcphy, true), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, requests, 2U); + KUNIT_EXPECT_TRUE(test, atcphy->pipe_pending); +} + /* A SuperSpeed problem does not keep the host behind a fixed hub from starting */ static void atcphy_test_host_start_without_usb3(struct kunit *test) { @@ -857,6 +949,33 @@ static void atcphy_test_election_held_usb(struct kunit *test) atcphy_test_election_redrive(test, atcphy); } +/* First pin-D handoff after probe: USB2 lanes and a parked PIPE. */ +static void atcphy_test_boot_pipe_retry(struct kunit *test) +{ + struct apple_atcphy *atcphy = atcphy_test_alloc_election(test); + struct atcphy_test_election *election = test->priv; + + atcphy->mode = APPLE_ATCPHY_MODE_USB2; + atcphy->typec_mode = APPLE_ATCPHY_MODE_USB2; + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + election->held = true; + KUNIT_ASSERT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_USB3_DP); + KUNIT_EXPECT_EQ(test, atcphy->pipe_state, ATCPHY_PIPEHANDLER_STATE_DUMMY); + KUNIT_EXPECT_TRUE(test, atcphy->pipe_pending); + KUNIT_EXPECT_TRUE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&atcphy->usb3_port_work)); + + /* The same request must finish the handoff, not use the mode cache. */ + election->held = false; + memset(election->steps.log, 0, sizeof(election->steps.log)); + KUNIT_ASSERT_EQ(test, atcphy_test_update_dp(atcphy, TYPEC_DP_STATE_D), 0); + KUNIT_EXPECT_STREQ(test, election->steps.log, "g01"); + KUNIT_EXPECT_TRUE(test, atcphy_usb3_port_ready(atcphy)); + KUNIT_EXPECT_FALSE(test, atcphy->usb3_port_off); + KUNIT_EXPECT_FALSE(test, atcphy->pipe_pending); +} + /* * Only the fixed-hub ports of a J700 have the Type-C driver retry a timeout * until it passes; every M1 and M2 port, and fixed-hub ports on other PHYs, @@ -1207,9 +1326,13 @@ static struct kunit_case atcphy_mux_test_cases[] = { KUNIT_CASE(atcphy_test_stopped_power_off), KUNIT_CASE(atcphy_test_stopped_power_off_dp), KUNIT_CASE(atcphy_test_stopped_reset), + KUNIT_CASE(atcphy_test_legacy_stopped_power_off), + KUNIT_CASE(atcphy_test_legacy_stopped_reset), + KUNIT_CASE(atcphy_test_legacy_live_lock), KUNIT_CASE(atcphy_test_host_start_without_usb3), KUNIT_CASE(atcphy_test_election_held_safe), KUNIT_CASE(atcphy_test_election_held_usb), + KUNIT_CASE(atcphy_test_boot_pipe_retry), KUNIT_CASE(atcphy_test_transient_timeouts), #if IS_REACHABLE(CONFIG_USB) KUNIT_CASE(atcphy_test_usb3_root), diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 84cef035e0865..b4c91cf4d373c 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1233,6 +1233,11 @@ static int atcphy_pipehandler_lock_timeout(struct apple_atcphy *atcphy, u32 time dev_dbg(atcphy->dev, "PIPE request: req=%#x mode=%u pipe=%u host=%u dirty=%u\n", requested, atcphy->mode, atcphy->pipe_state, atcphy->host_active, atcphy->mode_dirty); + dev_dbg(atcphy->dev, "PIPE before lock: ack=%#x aon=%#x mux=%#x override=%#x\n", + readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK), + readl(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN), + readl(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL), + readl(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE)); if (requested & PIPEHANDLER_LOCK_EN) { dev_dbg(atcphy->dev, "Pipehandler lock already requested\n"); } else { @@ -5082,6 +5087,14 @@ static int atcphy_usb3_port_on(struct apple_atcphy *atcphy, struct atcphy_usb3_p ret = usb_hub_port_power_on(pm->root, 1); if (ret < 0) dev_warn(atcphy->dev, "Failed to power the SuperSpeed port on: %d\n", ret); + else if (!ret && atcphy->fixed_usb2 && atcphy->hw->has_t8140_dp) { + int retrain = usb_hub_port_retrain(pm->root, 1); + + /* An absent SS receiver is normal for a USB2-only device or cable. */ + if (retrain < 0 && retrain != -ENOTCONN) + dev_warn(atcphy->dev, "SuperSpeed handover retrain failed: %d\n", retrain); + /* Power was restored even if no receiver answered the bounded retrain. */ + } #endif return ret; } @@ -6202,6 +6215,7 @@ static const struct atcphy_hw atcphy_hw_t8103 = { .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, .has_usb4 = true, + .park_pipe_unlocked = true, }; static const struct atcphy_hw atcphy_hw_t8122 = { diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index e25dd30281fc4..c442356fedde1 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -53,6 +53,15 @@ config APPLE_PMP_REPORT apple_t6031_display.enable=1. The start's state is in debugfs, in apple-pmp-report/. +config APPLE_PMP_ANE_KUNIT_TEST + tristate "KUnit tests for T600x ANE performance requests" if !KUNIT_ALL_TESTS + depends on KUNIT && PM + select PM_GENERIC_DOMAINS + default KUNIT_ALL_TESTS + help + Exercise ANE level transitions, ownership refusal and ACK-timeout + rollback against memory-backed registers without Apple hardware. + config APPLE_PMC bool "Apple PMC voter interface" depends on OF @@ -101,3 +110,15 @@ config APPLE_PMP_REPORT_KUNIT_TEST argument validation without accessing hardware. Includes malformed tables, address arithmetic and device ordering. The PMP report driver does not need to be enabled to run this suite. + +config APPLE_PMP_DRAM_KUNIT_TEST + tristate "Apple PMP DRAM description parsing tests" if !KUNIT_ALL_TESTS + depends on KUNIT && PM + select PM_GENERIC_DOMAINS + default KUNIT_ALL_TESTS + help + Parse synthetic firmware properties including unordered state IDs, + missing and duplicate states, incomplete descriptions and overflow. + Software-node fixtures exercise the parser without a DT boot image. + These tests access no hardware and do not need the PMP driver. They + are only useful for kernel developers. diff --git a/drivers/pmdomain/apple/Makefile b/drivers/pmdomain/apple/Makefile index ff7f9192b5e39..b03554cd70051 100644 --- a/drivers/pmdomain/apple/Makefile +++ b/drivers/pmdomain/apple/Makefile @@ -3,4 +3,6 @@ obj-$(CONFIG_APPLE_PMGR_PWRSTATE) += pmgr-pwrstate.o obj-$(CONFIG_APPLE_PMGR_PWRSTATE) += gfx-tvm.o obj-$(CONFIG_APPLE_PMP_REPORT) += pmp-report.o obj-$(CONFIG_APPLE_PMP_REPORT_KUNIT_TEST) += pmp-report-test.o +obj-$(CONFIG_APPLE_PMP_DRAM_KUNIT_TEST) += pmp-dram-test.o obj-$(CONFIG_APPLE_PMC) += pmc.o +obj-$(CONFIG_APPLE_PMP_ANE_KUNIT_TEST) += pmp-ane-test.o diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index bfcfb5ccbf1a4..5ebd7a51172f3 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -370,12 +371,63 @@ static int apple_pmgr_reset_xlate(struct reset_controller_dev *rcdev, return 0; } +/* + * genpd adds a power state that is on only below parents that are on. With a + * single parent the platform bus has already attached this device to it and + * powered it on before probe; with more than one it attaches to none, so a + * domain the boot loader left on (DISPEXT1_SYS on the J616s, below FAB7_SOC + * and its PMP request) could not be added. Power every parent on through a + * virtual device until the subdomain links hold them. + */ +static int apple_pmgr_ps_hold_parents(struct apple_pmgr_ps *ps, bool active, + struct dev_pm_domain_list **parents) +{ + static const struct dev_pm_domain_attach_data data = { + .pd_flags = PD_FLAG_NO_DEV_LINK, + }; + struct dev_pm_domain_list *list = NULL; + unsigned int i; + int ret; + + *parents = NULL; + if (!active || of_count_phandle_with_args(ps->dev->of_node, "power-domains", + "#power-domain-cells") < 2) + return 0; + + ret = dev_pm_domain_attach_list(ps->dev, &data, &list); + if (ret <= 0) + return ret; + for (i = 0; i < list->num_pds; i++) { + ret = pm_runtime_resume_and_get(list->pd_devs[i]); + if (ret < 0) { + while (i--) + pm_runtime_put(list->pd_devs[i]); + dev_pm_domain_detach_list(list); + return dev_err_probe(ps->dev, ret, "cannot power parent domain %u\n", i); + } + } + *parents = list; + return 0; +} + +static void apple_pmgr_ps_release_parents(struct dev_pm_domain_list *parents) +{ + unsigned int i; + + if (!parents) + return; + for (i = 0; i < parents->num_pds; i++) + pm_runtime_put(parents->pd_devs[i]); + dev_pm_domain_detach_list(parents); +} + static int apple_pmgr_ps_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *node = dev->of_node; struct apple_pmgr_ps *ps; struct regmap *regmap; + struct dev_pm_domain_list *parents = NULL; struct of_phandle_iterator it; int ret; const char *name; @@ -528,6 +580,10 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) return ret; } + ret = apple_pmgr_ps_hold_parents(ps, active, &parents); + if (ret < 0) + goto err_remove; + of_for_each_phandle(&it, ret, node, "power-domains", "#power-domain-cells", -1) { struct of_phandle_args parent, child; @@ -548,6 +604,9 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) } } + /* The subdomain links hold the parents now. */ + apple_pmgr_ps_release_parents(parents); + /* * Do not participate in regular PM; parent power domains are handled via the * genpd hierarchy. @@ -567,6 +626,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) return 0; err_remove: + apple_pmgr_ps_release_parents(parents); of_genpd_del_provider(node); pm_genpd_remove(&ps->genpd); return ret; diff --git a/drivers/pmdomain/apple/pmp-ane-test.c b/drivers/pmdomain/apple/pmp-ane-test.c new file mode 100644 index 0000000000000..66b196b178cc0 --- /dev/null +++ b/drivers/pmdomain/apple/pmp-ane-test.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "pmp-ane.h" + +struct ane_test { + struct apple_pmp_ane ane; + int requests; + int releases; + int request_error; + int release_error; + struct kunit *test; +}; + +static int ane_test_request(void *ctx, bool enable) +{ + struct ane_test *t = ctx; + + if (enable) { + KUNIT_EXPECT_EQ(t->test, readl(t->ane.dvfs + PMP_ANE_DVFS_ON), 1U); + KUNIT_EXPECT_EQ(t->test, readl(t->ane.dvfs + PMP_ANE_DVFS_CMD), 0x80000000U); + t->requests++; + t->ane.owned = true; + return t->request_error; + } + t->releases++; + if (t->release_error) + return t->release_error; + t->ane.owned = false; + return 0; +} + +static struct ane_test *ane_test_init(struct kunit *test) +{ + struct ane_test *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + + if (!t) + return NULL; + t->test = test; + t->ane.dvfs = (void __iomem *)kunit_kzalloc(test, 0x4000, GFP_KERNEL); + t->ane.report = (void __iomem *)kunit_kzalloc(test, 0x20000, GFP_KERNEL); + if (!t->ane.dvfs || !t->ane.report) + return NULL; + return t; +} + +static void ane_level_cycle(struct kunit *test) +{ + struct ane_test *t = ane_test_init(test); + + KUNIT_ASSERT_NOT_NULL(test, t); + KUNIT_ASSERT_EQ(test, apple_pmp_ane_set(&t->ane, 5, ane_test_request, t), 0); + KUNIT_EXPECT_EQ(test, t->ane.level, 5U); + KUNIT_EXPECT_EQ(test, readl(t->ane.dvfs + PMP_ANE_DVFS_CMD), 0x80000005U); + KUNIT_EXPECT_EQ(test, readq(t->ane.report + PMP_ANE_LEVEL_WRITE), PMP_ANE_LEVEL_TAG | 5); + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 5, ane_test_request, t), 0); + KUNIT_EXPECT_EQ(test, t->requests, 1); + KUNIT_ASSERT_EQ(test, apple_pmp_ane_set(&t->ane, 0, ane_test_request, t), 0); + KUNIT_EXPECT_EQ(test, t->ane.level, 0U); + KUNIT_EXPECT_FALSE(test, t->ane.owned); + KUNIT_EXPECT_EQ(test, t->releases, 1); + KUNIT_EXPECT_EQ(test, readl(t->ane.dvfs + PMP_ANE_DVFS_ON), 0U); + KUNIT_EXPECT_EQ(test, readl(t->ane.dvfs + PMP_ANE_DVFS_CMD), 0x80000000U); + KUNIT_EXPECT_EQ(test, readq(t->ane.report + PMP_ANE_LEVEL_WRITE), PMP_ANE_LEVEL_TAG); + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 0, ane_test_request, t), 0); + KUNIT_EXPECT_EQ(test, t->releases, 1); +} + +static void ane_level_busy(struct kunit *test) +{ + struct ane_test *t = ane_test_init(test); + + KUNIT_ASSERT_NOT_NULL(test, t); + writel(1, t->ane.dvfs + PMP_ANE_DVFS_ON); + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 5, ane_test_request, t), -EBUSY); + KUNIT_EXPECT_EQ(test, t->requests, 0); + writel(0, t->ane.dvfs + PMP_ANE_DVFS_ON); + writeq(0xdead, t->ane.report + PMP_ANE_LEVEL_READ); + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 5, ane_test_request, t), -EBUSY); + KUNIT_EXPECT_EQ(test, readl(t->ane.dvfs + PMP_ANE_DVFS_ON), 0U); + KUNIT_EXPECT_EQ(test, t->requests, 0); + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 6, ane_test_request, t), -EINVAL); +} + +static void ane_level_timeout(struct kunit *test) +{ + struct ane_test *t = ane_test_init(test); + int i; + + KUNIT_ASSERT_NOT_NULL(test, t); + t->request_error = -ETIMEDOUT; + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, + apple_pmp_ane_set(&t->ane, 5, ane_test_request, t), + -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, t->ane.level, 0U); + KUNIT_EXPECT_EQ(test, readl(t->ane.dvfs + PMP_ANE_DVFS_ON), 0U); + KUNIT_EXPECT_FALSE(test, t->ane.owned); + } + KUNIT_EXPECT_EQ(test, t->requests, 3); + KUNIT_EXPECT_EQ(test, t->releases, 3); +} + +static void ane_level_release_timeout(struct kunit *test) +{ + struct ane_test *t = ane_test_init(test); + + KUNIT_ASSERT_NOT_NULL(test, t); + KUNIT_ASSERT_EQ(test, apple_pmp_ane_set(&t->ane, 5, ane_test_request, t), 0); + t->release_error = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 0, ane_test_request, t), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, t->ane.level, 0U); + KUNIT_EXPECT_EQ(test, t->ane.owned, true); + KUNIT_EXPECT_EQ(test, readl(t->ane.dvfs + PMP_ANE_DVFS_ON), 0U); + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 5, ane_test_request, t), -EBUSY); + t->release_error = 0; + KUNIT_EXPECT_EQ(test, apple_pmp_ane_set(&t->ane, 0, ane_test_request, t), 0); + KUNIT_EXPECT_FALSE(test, t->ane.owned); + KUNIT_EXPECT_EQ(test, t->releases, 2); +} + +static struct kunit_case ane_level_cases[] = { + KUNIT_CASE(ane_level_cycle), + KUNIT_CASE(ane_level_busy), + KUNIT_CASE(ane_level_timeout), + KUNIT_CASE(ane_level_release_timeout), + {} +}; + +static struct kunit_suite ane_level_suite = { + .name = "apple-pmp-ane", + .test_cases = ane_level_cases, +}; + +kunit_test_suite(ane_level_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/pmdomain/apple/pmp-ane.h b/drivers/pmdomain/apple/pmp-ane.h new file mode 100644 index 0000000000000..e8e3838b042da --- /dev/null +++ b/drivers/pmdomain/apple/pmp-ane.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * T600x ANE level sequence, based on the public omarchy-ane implementation + * by Joshua Warren (ane/src/ane_dvfs.c, e20d25defb78). + * The report provider owns both windows and serializes request-word updates. + */ +#ifndef __APPLE_PMP_ANE_H__ +#define __APPLE_PMP_ANE_H__ + +#include +#include +#include + +#define PMP_ANE_DVFS_ON 0x2000 +#define PMP_ANE_DVFS_CMD 0xa00 +#define PMP_ANE_LEVEL_READ 0x10c0 +#define PMP_ANE_LEVEL_WRITE 0x10860 +#define PMP_ANE_LEVEL_TAG 0x2000000000000000ULL +#define PMP_ANE_MAX_LEVEL 5 +#define PMP_ANE_ID 10 + +struct apple_pmp_ane { + void __iomem *dvfs; + void __iomem *report; + unsigned int level; + unsigned int timeouts; + bool owned; + bool reserved; +}; + +static void apple_pmp_ane_cmd(struct apple_pmp_ane *ane, unsigned int old, + unsigned int new) +{ + writel(0x80000000U | old << 4 | new, ane->dvfs + PMP_ANE_DVFS_CMD); + usleep_range(1000, 2000); +} + +/* Caller serializes transitions and pins the ANE and report power domains. */ +static int apple_pmp_ane_set(struct apple_pmp_ane *ane, unsigned int level, + int (*request)(void *ctx, bool enable), void *ctx) +{ + u64 word; + int ret = 0; + + if (!ane->dvfs) + return -EOPNOTSUPP; + if (level > PMP_ANE_MAX_LEVEL) + return -EINVAL; + if (level == ane->level && (level || !ane->owned)) + return 0; + if (!level && !ane->level) + goto release; + if (level && !ane->level && ane->owned) + return -EBUSY; + if (level && ane->timeouts >= 3) + return -ETIMEDOUT; + + if (!ane->level) { + if (readl(ane->dvfs + PMP_ANE_DVFS_ON)) + return -EBUSY; + word = readq(ane->report + PMP_ANE_LEVEL_READ); + if (word && (word < PMP_ANE_LEVEL_TAG || + word > (PMP_ANE_LEVEL_TAG | PMP_ANE_MAX_LEVEL))) + return -EBUSY; + + writel(1, ane->dvfs + PMP_ANE_DVFS_ON); + usleep_range(1000, 2000); + apple_pmp_ane_cmd(ane, 0, 0); + ret = request(ctx, true); + if (ret) { + if (ret == -ETIMEDOUT) + ane->timeouts++; + goto release; + } + } + + apple_pmp_ane_cmd(ane, ane->level, level); + writeq(PMP_ANE_LEVEL_TAG | level, ane->report + PMP_ANE_LEVEL_WRITE); + ane->level = level; + if (level) + return 0; + usleep_range(1000, 2000); + +release: + if (ane->owned) { + int err = request(ctx, false); + + if (!ret) + ret = err; + } + apple_pmp_ane_cmd(ane, 0, 0); + writel(0, ane->dvfs + PMP_ANE_DVFS_ON); + return ret; +} + +#endif diff --git a/drivers/pmdomain/apple/pmp-dram-test.c b/drivers/pmdomain/apple/pmp-dram-test.c new file mode 100644 index 0000000000000..c662b99ff3df1 --- /dev/null +++ b/drivers/pmdomain/apple/pmp-dram-test.c @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Exercise the actual firmware-property parser without MMIO or firmware. */ +#include +#include + +#include "pmp-dram.h" + +struct pmp_dram_fixture { + struct software_node nodes[8]; + const struct software_node *group[9]; + struct software_node_ref_args ref[1]; + struct property_entry props[8]; + struct property_entry table_props[2]; + struct property_entry opp_props[6][4]; +}; + +static void pmp_dram_unregister(void *data) +{ + struct pmp_dram_fixture *f = data; + + software_node_unregister_node_group(f->group); +} + +static struct pmp_dram_fixture *pmp_dram_fixture(struct kunit *test, u32 count, + u32 width, u64 rate, u64 bandwidth_class) +{ + static const char * const names[] = { + "opp-0", "opp-1", "opp-2", "opp-3", "opp-4", "opp-5", + }; + struct pmp_dram_fixture *f; + unsigned int i; + + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + f->ref[0] = SOFTWARE_NODE_REFERENCE(&f->nodes[1]); + f->props[0] = PROPERTY_ENTRY_STRING("apple,dram-type", "lpddr5"); + f->props[1] = PROPERTY_ENTRY_U32("apple,data-bus-width", width); + f->props[2] = PROPERTY_ENTRY_U32("apple,dcs-state-count", count); + f->props[3] = PROPERTY_ENTRY_U64("apple,peak-bandwidth-bytes-per-second", + rate * (width / 8)); + f->props[4] = PROPERTY_ENTRY_U64("apple,bandwidth-class-bytes-per-second", bandwidth_class); + f->props[5] = PROPERTY_ENTRY_REF_ARRAY("operating-points-v2", f->ref); + f->nodes[0].name = "pmp-dram-test"; + f->nodes[0].properties = f->props; + f->table_props[0] = PROPERTY_ENTRY_STRING("compatible", "operating-points-v2-apple-dram"); + f->nodes[1].name = "opp-table-dram"; + f->nodes[1].properties = f->table_props; + for (i = 0; i < count; i++) { + struct software_node *opp = &f->nodes[i + 2]; + + opp->name = names[i]; + opp->parent = &f->nodes[1]; + opp->properties = f->opp_props[i]; + /* Reverse order: the highest state is not the last child. */ + f->opp_props[i][0] = PROPERTY_ENTRY_U32("opp-level", count - i - 1); + if (!i) { + f->opp_props[i][1] = PROPERTY_ENTRY_U64("apple,data-rate", rate); + f->opp_props[i][2] = PROPERTY_ENTRY_U64("opp-hz", rate / 2 + rate % 2); + } + } + for (i = 0; i < count + 2; i++) + f->group[i] = &f->nodes[i]; + return f; +} + +static int pmp_dram_parse_fixture(struct kunit *test, struct pmp_dram_fixture *f, + struct apple_pmp_dram *dram) +{ + int ret; + + ret = software_node_register_node_group(f->group); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, pmp_dram_unregister, f); + KUNIT_ASSERT_EQ(test, ret, 0); + return apple_pmp_dram_parse(software_node_fwnode(&f->nodes[0]), dram); +} + +static void pmp_dram_valid(struct kunit *test) +{ + struct pmp_dram_fixture *f = pmp_dram_fixture(test, 4, 256, 6400000000ULL, 200000000000ULL); + struct apple_pmp_dram dram = {}; + + KUNIT_ASSERT_EQ(test, pmp_dram_parse_fixture(test, f, &dram), 0); + KUNIT_EXPECT_STREQ(test, dram.type, "lpddr5"); + KUNIT_EXPECT_EQ(test, dram.max_rate, 6400000000ULL); + KUNIT_EXPECT_EQ(test, dram.peak_bandwidth, 204800000000ULL); + KUNIT_EXPECT_EQ(test, dram.bandwidth_class, 200000000000ULL); + KUNIT_EXPECT_EQ(test, dram.state_count, 4U); + KUNIT_EXPECT_EQ(test, dram.width, 256U); +} + +static void pmp_dram_m2(struct kunit *test) +{ + struct pmp_dram_fixture *f = pmp_dram_fixture(test, 5, 128, 6400000000ULL, 100000000000ULL); + struct apple_pmp_dram dram = {}; + + KUNIT_ASSERT_EQ(test, pmp_dram_parse_fixture(test, f, &dram), 0); + KUNIT_EXPECT_EQ(test, dram.peak_bandwidth, 102400000000ULL); + KUNIT_EXPECT_EQ(test, dram.state_count, 5U); +} + +static void pmp_dram_m4pro(struct kunit *test) +{ + struct pmp_dram_fixture *f = pmp_dram_fixture(test, 6, 256, 8533333333ULL, 273000000000ULL); + struct apple_pmp_dram dram = {}; + + KUNIT_ASSERT_EQ(test, pmp_dram_parse_fixture(test, f, &dram), 0); + KUNIT_EXPECT_EQ(test, dram.max_rate, 8533333333ULL); + KUNIT_EXPECT_EQ(test, dram.peak_bandwidth, 273066666656ULL); + KUNIT_EXPECT_EQ(test, dram.state_count, 6U); +} + +static void pmp_dram_a18pro(struct kunit *test) +{ + struct pmp_dram_fixture *f = pmp_dram_fixture(test, 5, 64, 7500000000ULL, 60000000000ULL); + struct apple_pmp_dram dram = {}; + + KUNIT_ASSERT_EQ(test, pmp_dram_parse_fixture(test, f, &dram), 0); + KUNIT_EXPECT_EQ(test, dram.peak_bandwidth, 60000000000ULL); + KUNIT_EXPECT_EQ(test, dram.width, 64U); +} + +enum pmp_dram_fault { + DRAM_ZERO_WIDTH, DRAM_UNALIGNED_WIDTH, DRAM_ZERO_COUNT, DRAM_LARGE_COUNT, + DRAM_DUPLICATE, DRAM_MISSING, DRAM_UNKNOWN_LEVEL, DRAM_MISSING_LEVEL, + DRAM_NO_MAX, DRAM_ZERO_RATE, DRAM_BAD_CK, DRAM_CK_ONLY, DRAM_BAD_PEAK, + DRAM_OVERFLOW, DRAM_BAD_TYPE, DRAM_NO_CLASS, DRAM_ZERO_CLASS, + DRAM_NO_TABLE, DRAM_BAD_COMPATIBLE, DRAM_LOW_ABOVE_MAX, DRAM_WRONG_TYPE, +}; + +static void pmp_dram_bad_description(struct kunit *test) +{ + enum pmp_dram_fault fault = *(enum pmp_dram_fault *)test->param_value; + struct pmp_dram_fixture *f = pmp_dram_fixture(test, 4, 256, 6400000000ULL, 200000000000ULL); + struct apple_pmp_dram dram = { .max_rate = 123 }; + + switch (fault) { + case DRAM_ZERO_WIDTH: + f->props[1] = PROPERTY_ENTRY_U32("apple,data-bus-width", 0); + break; + case DRAM_UNALIGNED_WIDTH: + f->props[1] = PROPERTY_ENTRY_U32("apple,data-bus-width", 7); + break; + case DRAM_ZERO_COUNT: + f->props[2] = PROPERTY_ENTRY_U32("apple,dcs-state-count", 0); + break; + case DRAM_LARGE_COUNT: + f->props[2] = PROPERTY_ENTRY_U32("apple,dcs-state-count", 33); + break; + case DRAM_DUPLICATE: + f->opp_props[1][0] = PROPERTY_ENTRY_U32("opp-level", 3); + break; + case DRAM_MISSING: + f->group[5] = NULL; + break; + case DRAM_UNKNOWN_LEVEL: + f->opp_props[1][0] = PROPERTY_ENTRY_U32("opp-level", 4); + break; + case DRAM_MISSING_LEVEL: + memset(&f->opp_props[1][0], 0, sizeof(struct property_entry)); + break; + case DRAM_NO_MAX: + memset(&f->opp_props[0][1], 0, 2 * sizeof(struct property_entry)); + break; + case DRAM_ZERO_RATE: + f->opp_props[0][1] = PROPERTY_ENTRY_U64("apple,data-rate", 0); + break; + case DRAM_BAD_CK: + f->opp_props[0][2] = PROPERTY_ENTRY_U64("opp-hz", 6400000000ULL); + break; + case DRAM_CK_ONLY: + f->opp_props[0][1] = f->opp_props[0][2]; + memset(&f->opp_props[0][2], 0, sizeof(struct property_entry)); + break; + case DRAM_BAD_PEAK: + f->props[3] = PROPERTY_ENTRY_U64("apple,peak-bandwidth-bytes-per-second", + 200000000000ULL); + break; + case DRAM_OVERFLOW: + f->opp_props[0][1] = PROPERTY_ENTRY_U64("apple,data-rate", U64_MAX - 1); + f->opp_props[0][2] = PROPERTY_ENTRY_U64("opp-hz", (U64_MAX - 1) / 2); + break; + case DRAM_BAD_TYPE: + f->props[0] = PROPERTY_ENTRY_STRING("apple,dram-type", "lpddr5x"); + break; + case DRAM_NO_CLASS: + f->props[4] = PROPERTY_ENTRY_U64("unrelated", 1); + break; + case DRAM_ZERO_CLASS: + f->props[4] = PROPERTY_ENTRY_U64("apple,bandwidth-class-bytes-per-second", 0); + break; + case DRAM_NO_TABLE: + memset(&f->props[5], 0, sizeof(struct property_entry)); + break; + case DRAM_BAD_COMPATIBLE: + f->table_props[0] = PROPERTY_ENTRY_STRING("compatible", "operating-points-v2"); + break; + case DRAM_LOW_ABOVE_MAX: + f->opp_props[1][1] = PROPERTY_ENTRY_U64("apple,data-rate", 7500000000ULL); + f->opp_props[1][2] = PROPERTY_ENTRY_U64("opp-hz", 3750000000ULL); + break; + case DRAM_WRONG_TYPE: + f->props[2] = PROPERTY_ENTRY_STRING("apple,dcs-state-count", "four"); + break; + } + KUNIT_EXPECT_LT(test, pmp_dram_parse_fixture(test, f, &dram), 0); + /* Never publish partially parsed values or a fallback maximum. */ + KUNIT_EXPECT_EQ(test, dram.max_rate, 123ULL); +} + +static const enum pmp_dram_fault pmp_dram_faults[] = { + DRAM_ZERO_WIDTH, DRAM_UNALIGNED_WIDTH, DRAM_ZERO_COUNT, DRAM_LARGE_COUNT, + DRAM_DUPLICATE, DRAM_MISSING, DRAM_UNKNOWN_LEVEL, DRAM_MISSING_LEVEL, + DRAM_NO_MAX, DRAM_ZERO_RATE, DRAM_BAD_CK, DRAM_CK_ONLY, DRAM_BAD_PEAK, + DRAM_OVERFLOW, DRAM_BAD_TYPE, DRAM_NO_CLASS, DRAM_ZERO_CLASS, + DRAM_NO_TABLE, DRAM_BAD_COMPATIBLE, DRAM_LOW_ABOVE_MAX, DRAM_WRONG_TYPE, +}; + +KUNIT_ARRAY_PARAM(pmp_dram_fault, pmp_dram_faults, NULL); + +static void pmp_dram_old_dt(struct kunit *test) +{ + struct pmp_dram_fixture *f = pmp_dram_fixture(test, 4, 256, 6400000000ULL, 200000000000ULL); + struct apple_pmp_dram dram = {}; + + memset(f->props, 0, sizeof(f->props)); + KUNIT_EXPECT_EQ(test, pmp_dram_parse_fixture(test, f, &dram), -ENODATA); +} + +static void pmp_dram_partial_dt(struct kunit *test) +{ + struct pmp_dram_fixture *f = pmp_dram_fixture(test, 4, 256, 6400000000ULL, 200000000000ULL); + struct apple_pmp_dram dram = {}; + + f->props[0] = PROPERTY_ENTRY_U32("apple,data-bus-width", 128); + memset(&f->props[1], 0, sizeof(struct property_entry)); + KUNIT_EXPECT_EQ(test, pmp_dram_parse_fixture(test, f, &dram), -EINVAL); +} + +static struct kunit_case pmp_dram_cases[] = { + KUNIT_CASE(pmp_dram_valid), + KUNIT_CASE(pmp_dram_m2), + KUNIT_CASE(pmp_dram_m4pro), + KUNIT_CASE(pmp_dram_a18pro), + KUNIT_CASE_PARAM(pmp_dram_bad_description, pmp_dram_fault_gen_params), + KUNIT_CASE(pmp_dram_old_dt), + KUNIT_CASE(pmp_dram_partial_dt), + {} +}; + +static struct kunit_suite pmp_dram_suite = { + .name = "apple-pmp-dram", + .test_cases = pmp_dram_cases, +}; + +kunit_test_suite(pmp_dram_suite); + +MODULE_DESCRIPTION("KUnit tests for Apple PMP DRAM property parsing"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/pmdomain/apple/pmp-dram.h b/drivers/pmdomain/apple/pmp-dram.h new file mode 100644 index 0000000000000..08a7fcf9c17c8 --- /dev/null +++ b/drivers/pmdomain/apple/pmp-dram.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _APPLE_PMP_DRAM_H +#define _APPLE_PMP_DRAM_H + +#include +#include +#include +#include +#include + +#define APPLE_PMP_DRAM_MAX_STATES 32 + +struct apple_pmp_dram { + const char *type; + u32 width; + u32 state_count; + u64 max_rate; + u64 peak_bandwidth; + u64 bandwidth_class; +}; + +/* + * Read metadata only: never register these OPPs with a clock/voltage provider. + * Unknown lower-state rates are deliberately absent. Firmware state IDs, not + * child order or frequency sorting, identify the entries. + */ +static inline int apple_pmp_dram_parse(const struct fwnode_handle *node, + struct apple_pmp_dram *dram) +{ + struct apple_pmp_dram parsed = {}; + struct fwnode_reference_args ref; + struct fwnode_handle *opp; + u64 rates[APPLE_PMP_DRAM_MAX_STATES] = {}; + u64 seen = 0, peak; + u32 level; + unsigned int i; + int ret; + + /* Old DTs with no memory description retain the existing interface. */ + if (!fwnode_property_present(node, "apple,dram-type") && + !fwnode_property_present(node, "apple,data-bus-width") && + !fwnode_property_present(node, "apple,dcs-state-count") && + !fwnode_property_present(node, "apple,peak-bandwidth-bytes-per-second") && + !fwnode_property_present(node, "apple,bandwidth-class-bytes-per-second") && + !fwnode_property_present(node, "operating-points-v2")) + return -ENODATA; + + if (fwnode_property_read_string(node, "apple,dram-type", &parsed.type) || + strcmp(parsed.type, "lpddr5") || + fwnode_property_read_u32(node, "apple,data-bus-width", &parsed.width) || + !parsed.width || parsed.width % 8 || + fwnode_property_read_u32(node, "apple,dcs-state-count", &parsed.state_count) || + !parsed.state_count || parsed.state_count > APPLE_PMP_DRAM_MAX_STATES || + fwnode_property_read_u64(node, "apple,peak-bandwidth-bytes-per-second", + &parsed.peak_bandwidth) || + fwnode_property_read_u64(node, "apple,bandwidth-class-bytes-per-second", + &parsed.bandwidth_class) || !parsed.bandwidth_class) + return -EINVAL; + + ret = fwnode_property_get_reference_args(node, "operating-points-v2", NULL, 0, 0, &ref); + if (ret) + return ret; + if (fwnode_property_match_string(ref.fwnode, "compatible", + "operating-points-v2-apple-dram") != 0) { + ret = -EINVAL; + goto out; + } + fwnode_for_each_available_child_node(ref.fwnode, opp) { + u64 rate, ck; + + if (fwnode_property_read_u32(opp, "opp-level", &level) || + level >= parsed.state_count || (seen & BIT_ULL(level))) + goto bad_opp; + seen |= BIT_ULL(level); + if (!fwnode_property_present(opp, "apple,data-rate")) { + if (fwnode_property_present(opp, "opp-hz")) + goto bad_opp; + continue; + } + if (fwnode_property_read_u64(opp, "apple,data-rate", &rate) || !rate || + fwnode_property_read_u64(opp, "opp-hz", &ck) || !ck || + ck != rate / 2 + rate % 2) + goto bad_opp; + rates[level] = rate; + } + if (seen != GENMASK_ULL(parsed.state_count - 1, 0)) { + ret = -EINVAL; + goto out; + } + parsed.max_rate = rates[parsed.state_count - 1]; + if (!parsed.max_rate || + check_mul_overflow(parsed.max_rate, (u64)parsed.width / 8, &peak) || + peak != parsed.peak_bandwidth) { + ret = -EINVAL; + goto out; + } + for (i = 0; i + 1 < parsed.state_count; i++) { + if (rates[i] >= parsed.max_rate) { + ret = -EINVAL; + goto out; + } + } + /* Publish only a complete, internally consistent description. */ + *dram = parsed; +out: + fwnode_handle_put(ref.fwnode); + return ret; +bad_opp: + fwnode_handle_put(opp); + ret = -EINVAL; + goto out; +} + +#endif diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index aaa38c06daaaf..f09b38aa310ca 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -32,6 +32,8 @@ #include #include "pmp-report-validation.h" +#include "pmp-dram.h" +#include "pmp-ane.h" #define PMP_REPORT_READY 0x1 @@ -81,6 +83,9 @@ struct apple_pmp_start; struct apple_pmp_report { struct device *dev; + struct apple_pmp_dram dram; + struct apple_pmp_ane ane; + struct device *ane_owner; const struct apple_pmp_report_offsets *offsets; void __iomem *base; /* Protects the request word, @shadow, @known and @defer. */ @@ -105,11 +110,183 @@ static bool apple_pmp_report_running(const struct apple_pmp_report *rep) return rep->offsets->running_any ? status : status & PMP_REPORT_READY; } +static ssize_t type_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct apple_pmp_report *rep = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%s\n", rep->dram.type); +} +static DEVICE_ATTR_RO(type); + +#define APPLE_PMP_DRAM_ATTR(_name, _field, _format) \ +static ssize_t _name##_show(struct device *dev, struct device_attribute *attr, char *buf) \ +{ \ + struct apple_pmp_report *rep = dev_get_drvdata(dev); \ + return sysfs_emit(buf, _format "\n", rep->dram._field); \ +} \ +static DEVICE_ATTR_RO(_name) + +APPLE_PMP_DRAM_ATTR(data_bus_width, width, "%u"); +APPLE_PMP_DRAM_ATTR(state_count, state_count, "%u"); +APPLE_PMP_DRAM_ATTR(max_data_rate, max_rate, "%llu"); +APPLE_PMP_DRAM_ATTR(peak_bandwidth, peak_bandwidth, "%llu"); +APPLE_PMP_DRAM_ATTR(bandwidth_class, bandwidth_class, "%llu"); + +static struct attribute *apple_pmp_dram_attrs[] = { + &dev_attr_type.attr, + &dev_attr_data_bus_width.attr, + &dev_attr_state_count.attr, + &dev_attr_max_data_rate.attr, + &dev_attr_peak_bandwidth.attr, + &dev_attr_bandwidth_class.attr, + NULL, +}; + +static const struct attribute_group apple_pmp_dram_group = { + .name = "dram", + .attrs = apple_pmp_dram_attrs, +}; + static const struct apple_pmp_report_offsets apple_pmp_offsets_t6030; /* Bound report regions, protected by RCU until managed teardown drains readers. */ static LIST_HEAD(apple_pmp_reports); static DEFINE_MUTEX(apple_pmp_reports_lock); + +/* The report lock also protects ANE_SYS against a genpd owner appearing. */ +static int apple_pmp_ane_request(void *ctx, bool enable) +{ + struct apple_pmp_report *rep = ctx; + struct apple_pmp_ane *ane = &rep->ane; + const u64 bit = BIT_ULL(PMP_ANE_ID); + unsigned long flags; + u64 val; + + spin_lock_irqsave(&rep->lock, flags); + val = readq(rep->base + rep->offsets->tgt_read); + if (enable) { + if ((rep->known | val) & bit) { + spin_unlock_irqrestore(&rep->lock, flags); + return -EBUSY; + } + ane->owned = true; + val |= bit; + } else { + val &= ~bit; + } + writeq(val, rep->base + rep->offsets->tgt_write); + spin_unlock_irqrestore(&rep->lock, flags); + if (!enable) { + int ret = readq_poll_timeout(rep->base + rep->offsets->actual, val, + !(val & bit), 200, 50000); + + if (!ret) { + spin_lock_irqsave(&rep->lock, flags); + ane->owned = false; + spin_unlock_irqrestore(&rep->lock, flags); + } + return ret; + } + return readq_poll_timeout(rep->base + rep->offsets->actual, val, + val & bit, 200, 50000); +} + +/* Probe-time availability check: deliberately no MMIO while the ANE is off. */ +int apple_pmp_report_ane_available(const struct device_node *np) +{ + struct apple_pmp_report *rep; + int ret = -EPROBE_DEFER; + + mutex_lock(&apple_pmp_reports_lock); + list_for_each_entry(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node != np) + continue; + ret = !rep->ane.dvfs ? -EOPNOTSUPP : + (rep->ane_owner || rep->ane.reserved) ? -EBUSY : 0; + break; + } + mutex_unlock(&apple_pmp_reports_lock); + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_ane_available); + +/** + * apple_pmp_report_ane_perf() - request a T600x ANE performance level + * @np: report provider node + * @owner: ANE device holding the request + * @level: 0 to release, 1 through 5 to request a level + * @held: whether the caller still owns a request, including failed rollback + * + * The caller must keep its engine and the report SRAM powered throughout + * the request, and link itself to the report supplier. Other generations + * and reports without an ANE DVFS resource are unsupported. The returned + * level is a firmware request, not a measured frequency. + * + * Context: Process context. May sleep. + * Return: 0 on success, or a negative errno. Ownership and ACK failures + * are returned to the submitter; no CPU frequency request is substituted. + */ +int apple_pmp_report_ane_perf(const struct device_node *np, + struct device *owner, unsigned int level, bool *held) +{ + struct apple_pmp_report *rep; + int ret = -EPROBE_DEFER; + + if (!np || !owner || !held) + return -EINVAL; + *held = false; + mutex_lock(&apple_pmp_reports_lock); + list_for_each_entry(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node != np) + continue; + if (!rep->ane.dvfs) { + ret = -EOPNOTSUPP; + break; + } + if (rep->ane_owner && rep->ane_owner != owner) { + ret = -EBUSY; + break; + } + if (level && !apple_pmp_report_running(rep)) { + ret = -EAGAIN; + break; + } + /* Claim before touching DVFS, and exclude genpd for the whole cycle. */ + if (level && !rep->ane.reserved) { + unsigned long flags; + u64 req; + + spin_lock_irqsave(&rep->lock, flags); + req = readq(rep->base + rep->offsets->tgt_read); + if ((rep->known | req) & BIT_ULL(PMP_ANE_ID)) { + spin_unlock_irqrestore(&rep->lock, flags); + ret = -EBUSY; + break; + } + rep->ane.reserved = true; + spin_unlock_irqrestore(&rep->lock, flags); + } + ret = apple_pmp_ane_set(&rep->ane, level, apple_pmp_ane_request, rep); + if (!rep->ane.level && !rep->ane.owned) { + unsigned long flags; + + spin_lock_irqsave(&rep->lock, flags); + rep->ane.reserved = false; + spin_unlock_irqrestore(&rep->lock, flags); + } + /* Failed rollback retains ownership until a successful retry. */ + if (rep->ane.level || rep->ane.owned) + rep->ane_owner = owner; + else + rep->ane_owner = NULL; + *held = rep->ane_owner == owner; + break; + } + mutex_unlock(&apple_pmp_reports_lock); + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_ane_perf); + static struct dentry *apple_pmp_report_debugfs; /** @@ -1143,10 +1320,36 @@ static int apple_pmp_report_probe(struct platform_device *pdev) if (IS_ERR(rep->base)) return PTR_ERR(rep->base); rep->offsets = of_device_get_match_data(dev); + if (of_property_match_string(np, "reg-names", "ane-dvfs") >= 0) { + struct resource *res; + + if (!of_device_is_compatible(np, "apple,t6000-pmp-v2-report")) + return -EINVAL; + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res || resource_size(res) != SZ_128K) + return -EINVAL; + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ane-dvfs"); + if (!res || resource_size(res) != SZ_16K) + return -EINVAL; + rep->ane.dvfs = devm_ioremap_resource(dev, res); + if (IS_ERR(rep->ane.dvfs)) + return PTR_ERR(rep->ane.dvfs); + rep->ane.report = rep->base; + } + rep->defer = rep->offsets->defer_requests; INIT_DELAYED_WORK(&rep->ready_work, apple_pmp_report_ready_work); platform_set_drvdata(pdev, rep); + ret = apple_pmp_dram_parse(dev_fwnode(dev), &rep->dram); + if (ret && ret != -ENODATA) + return dev_err_probe(dev, ret, "invalid DRAM description\n"); + if (!ret) { + ret = devm_device_add_group(dev, &apple_pmp_dram_group); + if (ret) + return ret; + } + INIT_LIST_HEAD(&rep->list); ret = devm_add_action_or_reset(dev, apple_pmp_report_release, rep); if (ret) @@ -1338,6 +1541,10 @@ static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id, bool ready; spin_lock_irqsave(&rep->lock, flags); + if (id == PMP_ANE_ID && rep->ane.reserved) { + spin_unlock_irqrestore(&rep->lock, flags); + return -EBUSY; + } rep->known |= bit_val; if (enable) rep->shadow |= bit_val; diff --git a/drivers/power/supply/macsmc-power-test.c b/drivers/power/supply/macsmc-power-test.c index 4a86c9bd9f155..b6f9b50173d01 100644 --- a/drivers/power/supply/macsmc-power-test.c +++ b/drivers/power/supply/macsmc-power-test.c @@ -637,41 +637,28 @@ static void macsmc_charge_serialized_test(struct kunit *test) KUNIT_EXPECT_EQ(test, t->chls, 0x013c); } -/* Probe clears force-discharge left without a limit, and only that. */ +/* A stale force-discharge bit is cleared only by an explicit auto request. */ static void macsmc_charge_clear_stray_discharge_test(struct kunit *test) { - static const struct { - u16 before, after; - unsigned int writes; - } cases[] = { - { 0x0100, 0x0000, 1 }, - { 0x0164, 0x0064, 1 }, - { 0x0105, 0x0005, 1 }, - { 0x0150, 0x0150, 0 }, /* auto-discharge with a limit stays */ - { 0x0050, 0x0050, 0 }, - { 0x0000, 0x0000, 0 }, - }; + static const u16 values[] = { 0x0100, 0x0164, 0x0105, 0x0150, 0x0050, 0x0000 }; struct macsmc_charge_test *t = test->priv; unsigned int i; t->power.has_chls = true; - for (i = 0; i < ARRAY_SIZE(cases); i++) { - t->chls = cases[i].before; + for (i = 0; i < ARRAY_SIZE(values); i++) { + t->chls = values[i]; t->nops = 0; - KUNIT_EXPECT_EQ_MSG(test, macsmc_battery_clear_stray_discharge(&t->power), 0, + macsmc_battery_get_charge_behaviour(&t->power); + KUNIT_EXPECT_EQ_MSG(test, t->chls, values[i], "read case %u", i); + KUNIT_EXPECT_EQ(test, charge_test_writes(t), 0U); + KUNIT_EXPECT_EQ_MSG(test, + charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO), + 0, "case %u", i); + KUNIT_EXPECT_EQ_MSG(test, t->chls, values[i] & ~CHLS_FORCE_DISCHARGE, "case %u", i); - KUNIT_EXPECT_EQ_MSG(test, t->chls, cases[i].after, "case %u", i); - KUNIT_EXPECT_EQ_MSG(test, charge_test_writes(t), cases[i].writes, "case %u", i); + KUNIT_EXPECT_EQ_MSG(test, charge_test_writes(t), 1U, "case %u", i); } - - t->chls = 0x0100; - t->fail_key = SMC_KEY(CHLS); - t->fail_err = -ETIMEDOUT; - KUNIT_EXPECT_EQ(test, macsmc_battery_clear_stray_discharge(&t->power), -ETIMEDOUT); - t->fail_write = true; - t->fail_err = -EIO; - KUNIT_EXPECT_EQ(test, macsmc_battery_clear_stray_discharge(&t->power), -EIO); - KUNIT_EXPECT_EQ(test, t->chls, 0x0100); } static struct kunit_case macsmc_charge_test_cases[] = { diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index 36d59e5ea315a..1bc23703654a4 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -786,25 +786,6 @@ static int macsmc_battery_set_property(struct power_supply *psy, return ret < 0 ? ret : 0; } -/* - * An earlier kernel could leave force-discharge set in CHLS without a charge limit, which this - * driver never asks of the SMC. Clear the bit once at probe; the limit byte is kept. - */ -static int macsmc_battery_clear_stray_discharge(struct macsmc_power *power) -{ - u16 chls; - int ret; - - guard(mutex)(&power->charge_lock); - ret = apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &chls); - if (ret) - return ret; - if (!(chls & CHLS_FORCE_DISCHARGE) || macsmc_chls_has_limit(chls)) - return 0; - ret = apple_smc_write_u16(power->smc, SMC_KEY(CHLS), chls & ~CHLS_FORCE_DISCHARGE); - return ret < 0 ? ret : 0; -} - static int macsmc_battery_property_is_writeable(struct power_supply *psy, enum power_supply_property psp) { @@ -1108,26 +1089,16 @@ static int macsmc_power_probe(struct platform_device *pdev) of_machine_is_compatible("apple,j700"); power->status_from_current = of_machine_is_compatible("apple,j700"); - /* Reset "Optimised Battery Charging" flags to default state */ - if (power->has_chte) - apple_smc_write_u32(smc, SMC_KEY(CHTE), 0); - else if (power->has_ch0c) - apple_smc_write_u8(smc, SMC_KEY(CH0C), 0); - - if (power->has_ch0i) - apple_smc_write_u8(smc, SMC_KEY(CH0I), 0); - - apple_smc_write_u8(smc, SMC_KEY(CH0K), 0); - apple_smc_write_u8(smc, SMC_KEY(CH0B), 0); - + /* Preserve firmware charge policy until userspace changes it. */ /* Detect charge limit method (CHWA vs CHLS) */ if (apple_smc_read_flag(power->smc, SMC_KEY(CHWA), &flag) == 0) power->has_chwa = true; else if (apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16) >= 0) power->has_chls = true; - if (power->has_chls && macsmc_battery_clear_stray_discharge(power)) - dev_warn(dev, "Failed to clear force-discharge without a charge limit\n"); + if (power->has_chls && (vu16 & CHLS_FORCE_DISCHARGE) && + !macsmc_chls_has_limit(vu16)) + dev_warn(dev, "CHLS force-discharge without a limit; select charge_behaviour=auto to clear\n"); /* * CHLS carries auto-discharge, so a CHLS-only Mac can select auto diff --git a/drivers/pwm/pwm-apple.c b/drivers/pwm/pwm-apple.c index 72e4f23d2315b..2659f777fafbe 100644 --- a/drivers/pwm/pwm-apple.c +++ b/drivers/pwm/pwm-apple.c @@ -18,6 +18,7 @@ #include #include #include +#include #define APPLE_PWM_CTRL 0x00 #define APPLE_PWM_ON_CYCLES 0x1c @@ -32,7 +33,12 @@ struct apple_pwm { void __iomem *base; + struct clk *clk; u64 clkrate; + u32 on_cycles; + u32 off_cycles; + u32 ctrl; + bool enabled; }; static inline struct apple_pwm *to_apple_pwm(struct pwm_chip *chip) @@ -44,11 +50,17 @@ static int apple_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { struct apple_pwm *fpwm; + struct device *dev = pwmchip_parent(chip); + int ret; if (state->polarity == PWM_POLARITY_INVERSED) return -EINVAL; fpwm = to_apple_pwm(chip); + ret = pm_runtime_resume_and_get(dev); + if (ret) + return ret; + if (state->enabled) { u64 on_cycles, off_cycles; @@ -69,6 +81,14 @@ static int apple_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, } else { writel(0, fpwm->base + APPLE_PWM_CTRL); } + + /* Keep the controller powered for as long as it drives an output. */ + if (state->enabled && !fpwm->enabled) + pm_runtime_get_noresume(dev); + else if (!state->enabled && fpwm->enabled) + pm_runtime_put_noidle(dev); + fpwm->enabled = state->enabled; + pm_runtime_put(dev); return 0; } @@ -77,8 +97,13 @@ static int apple_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, { struct apple_pwm *fpwm; u32 on_cycles, off_cycles, ctrl; + struct device *dev = pwmchip_parent(chip); + int ret; fpwm = to_apple_pwm(chip); + ret = pm_runtime_resume_and_get(dev); + if (ret) + return ret; ctrl = readl(fpwm->base + APPLE_PWM_CTRL); on_cycles = readl(fpwm->base + APPLE_PWM_ON_CYCLES); @@ -91,6 +116,7 @@ static int apple_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, state->period = DIV64_U64_ROUND_UP(((u64)off_cycles + (u64)on_cycles) * NSEC_PER_SEC, fpwm->clkrate); + pm_runtime_put(dev); return 0; } @@ -99,11 +125,55 @@ static const struct pwm_ops apple_pwm_ops = { .get_state = apple_pwm_get_state, }; +static int apple_pwm_runtime_suspend(struct device *dev) +{ + struct apple_pwm *fpwm = to_apple_pwm(dev_get_drvdata(dev)); + + /* A disabled PWM can lose its register state when the domain is off. */ + fpwm->on_cycles = readl(fpwm->base + APPLE_PWM_ON_CYCLES); + fpwm->off_cycles = readl(fpwm->base + APPLE_PWM_OFF_CYCLES); + fpwm->ctrl = readl(fpwm->base + APPLE_PWM_CTRL); + clk_disable_unprepare(fpwm->clk); + return 0; +} + +static int apple_pwm_runtime_resume(struct device *dev) +{ + struct apple_pwm *fpwm = to_apple_pwm(dev_get_drvdata(dev)); + int ret; + + ret = clk_prepare_enable(fpwm->clk); + if (ret) + return ret; + + writel(fpwm->on_cycles, fpwm->base + APPLE_PWM_ON_CYCLES); + writel(fpwm->off_cycles, fpwm->base + APPLE_PWM_OFF_CYCLES); + writel(fpwm->ctrl, fpwm->base + APPLE_PWM_CTRL); + return 0; +} + +static void apple_pwm_runtime_disable(void *data) +{ + struct device *dev = data; + struct apple_pwm *fpwm = to_apple_pwm(dev_get_drvdata(dev)); + + pm_runtime_disable(dev); + if (fpwm->enabled) + pm_runtime_put_noidle(dev); + if (!pm_runtime_status_suspended(dev)) + clk_disable_unprepare(fpwm->clk); + pm_runtime_set_suspended(dev); +} + +static const struct dev_pm_ops apple_pwm_pm_ops = { + RUNTIME_PM_OPS(apple_pwm_runtime_suspend, apple_pwm_runtime_resume, NULL) +}; + static int apple_pwm_probe(struct platform_device *pdev) { struct pwm_chip *chip; struct apple_pwm *fpwm; - struct clk *clk; + struct device *dev = &pdev->dev; int ret; chip = devm_pwmchip_alloc(&pdev->dev, 1, sizeof(*fpwm)); @@ -116,9 +186,9 @@ static int apple_pwm_probe(struct platform_device *pdev) if (IS_ERR(fpwm->base)) return PTR_ERR(fpwm->base); - clk = devm_clk_get_enabled(&pdev->dev, NULL); - if (IS_ERR(clk)) - return dev_err_probe(&pdev->dev, PTR_ERR(clk), "unable to get the clock"); + fpwm->clk = devm_clk_get(dev, NULL); + if (IS_ERR(fpwm->clk)) + return dev_err_probe(dev, PTR_ERR(fpwm->clk), "unable to get the clock"); /* * Uses the 24MHz system clock on all existing devices, can only @@ -126,13 +196,38 @@ static int apple_pwm_probe(struct platform_device *pdev) * * This check is done to prevent an overflow in .apply */ - fpwm->clkrate = clk_get_rate(clk); + fpwm->clkrate = clk_get_rate(fpwm->clk); if (!fpwm->clkrate || fpwm->clkrate > NSEC_PER_SEC) return dev_err_probe(&pdev->dev, -EINVAL, "pwm clock out of range"); chip->ops = &apple_pwm_ops; + platform_set_drvdata(pdev, chip); + + ret = clk_prepare_enable(fpwm->clk); + if (ret) + return ret; + ret = pm_runtime_set_active(dev); + if (ret) { + clk_disable_unprepare(fpwm->clk); + return ret; + } + + /* Preserve an output that the bootloader left running. */ + fpwm->ctrl = readl(fpwm->base + APPLE_PWM_CTRL); + fpwm->enabled = (fpwm->ctrl & APPLE_PWM_CTRL_ENABLE) && + (fpwm->ctrl & APPLE_PWM_CTRL_OUTPUT_ENABLE); + pm_runtime_get_noresume(dev); /* probe reference */ + if (fpwm->enabled) + pm_runtime_get_noresume(dev); /* output reference */ + pm_runtime_enable(dev); + ret = devm_add_action_or_reset(dev, apple_pwm_runtime_disable, dev); + if (ret) { + pm_runtime_put_noidle(dev); + return ret; + } ret = devm_pwmchip_add(&pdev->dev, chip); + pm_runtime_put(dev); if (ret < 0) return dev_err_probe(&pdev->dev, ret, "unable to add pwm chip"); @@ -150,6 +245,7 @@ static struct platform_driver apple_pwm_driver = { .driver = { .name = "apple-pwm", .of_match_table = apple_pwm_of_match, + .pm = pm_ptr(&apple_pwm_pm_ops), }, }; module_platform_driver(apple_pwm_driver); diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index c6e3bc9f81982..ba8b498786414 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -51,6 +51,7 @@ config APPLE_MCC_KUNIT_TEST config APPLE_PMGR_MISC tristate "Apple SoC PMGR miscellaneous support" depends on PM + select INTERCONNECT help The PMGR block in Apple SoCs provides high-level power state controls for SoC devices. This driver manages miscellaneous @@ -58,6 +59,16 @@ config APPLE_PMGR_MISC Say 'y' here if you have a Pro/Max/Ultra Apple SoC. +config APPLE_PMGR_DCS_KUNIT_TEST + bool "KUnit tests for Apple PMGR DCS voting" if !KUNIT_ALL_TESTS + depends on KUNIT=y && APPLE_PMGR_MISC=y + default KUNIT_ALL_TESTS + help + Test bandwidth aggregation, explicit OPP selection, ownership gates, + register updates and timeout rollback with synthetic state data. + Also verify that boot constraints and concurrent endpoints retain + their votes, and that suspend restores the last confirmed request. + config APPLE_RTKIT tristate "Apple RTKit co-processor IPC protocol" depends on APPLE_MAILBOX diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index c59719d048158..380504117e52c 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -121,6 +121,10 @@ const EPIC_CATEGORY_NOTIFY: u8 = 0x10; const EPIC_CATEGORY_REPLY: u8 = 0x20; const EPIC_SUBTYPE_STD_SERVICE: u16 = 0xc0; const EPIC_SUBTYPE_FAKEHID_REPORT: u16 = 0xc4; +/// The T8132 firmware sends its gyroscope and accelerometer samples with +/// these report subtypes; they carry the same payload as a fake-HID report. +const EPIC_SUBTYPE_GYRO_REPORT: u16 = 0xc1; +const EPIC_SUBTYPE_ACCEL_REPORT: u16 = 0xc2; const EPIC_SUBTYPE_RETCODE: u16 = 0x84; const EPIC_SUBTYPE_RETCODE_PAYLOAD: u16 = 0xa0; const EPIC_SUBTYPE_STRING: u16 = 0x8a; @@ -756,7 +760,10 @@ impl AFKEndpoint { let name = &name[..name.iter().position(|x| *x == 0).unwrap_or(name.len())]; let chan = le_u32(data, EPIC_ANNOUNCE_CHANNEL_OFFSET); return Into::>::into(client).register_service(self, chan, name); - } else if subtype == EPIC_SUBTYPE_FAKEHID_REPORT { + } else if subtype == EPIC_SUBTYPE_FAKEHID_REPORT + || subtype == EPIC_SUBTYPE_GYRO_REPORT + || subtype == EPIC_SUBTYPE_ACCEL_REPORT + { return client.process_fakehid_report(self, qhdr.channel, data); } else if client.process_report(self, qhdr.channel, subtype, data)? { return Ok(()); @@ -1471,6 +1478,8 @@ impl AopData { b"aop-audio" => c_str!("audio"), b"las" => c_str!("las"), b"als" => c_str!("als"), + b"accel" => c_str!("accel"), + b"gyro" => c_str!("gyro"), _ => { return Ok(()); } @@ -2502,6 +2511,8 @@ struct AopDriver(Arc); struct AopHwConfig { ec0p: u64, + /// The `nCal` boot argument. + ncal: u64, alig: u64, aopt: u64, /// Complete the firmware's boot arguments before starting it. @@ -2525,6 +2536,7 @@ struct AopHwConfig { const HW_CFG_T8103: AopHwConfig = AopHwConfig { ec0p: 0x020000, + ncal: 0, aopt: 1, alig: 128, patch_bootargs: true, @@ -2537,6 +2549,7 @@ const HW_CFG_T8103: AopHwConfig = AopHwConfig { }; const HW_CFG_T8112: AopHwConfig = AopHwConfig { ec0p: 0x020000, + ncal: 0, aopt: 0, alig: 128, patch_bootargs: true, @@ -2549,6 +2562,7 @@ const HW_CFG_T8112: AopHwConfig = AopHwConfig { }; const HW_CFG_T6000: AopHwConfig = AopHwConfig { ec0p: 0x020000, + ncal: 0, aopt: 0, alig: 64, patch_bootargs: true, @@ -2561,6 +2575,7 @@ const HW_CFG_T6000: AopHwConfig = AopHwConfig { }; const HW_CFG_T6020: AopHwConfig = AopHwConfig { ec0p: 0x0100_00000000, + ncal: 0, aopt: 0, alig: 64, patch_bootargs: true, @@ -2571,12 +2586,14 @@ const HW_CFG_T6020: AopHwConfig = AopHwConfig { fw_regions: false, halt_on_failed_start: false, }; -/// T6040 (J616s): macOS writes the same four boot arguments as for the -/// T6020. The firmware's external code and log live in DRAM, mapped through -/// the reserved regions the bootloader describes; the RTKit firmware is shut -/// down on unbind. +/// T6040 (J616s): the firmware's external code and log live in DRAM, mapped +/// through the reserved regions the bootloader describes; the RTKit firmware +/// is shut down on unbind. `nCal` keeps the firmware's own default of 1: +/// with 0 the accelerometer and gyroscope services accept their calibration +/// and report interval but never send a report. const HW_CFG_T6040: AopHwConfig = AopHwConfig { ec0p: 0x0100_00000000, + ncal: 1, aopt: 0, alig: 64, patch_bootargs: true, @@ -2594,6 +2611,7 @@ const HW_CFG_T6040: AopHwConfig = AopHwConfig { /// are started if they will. const HW_CFG_T8140: AopHwConfig = AopHwConfig { ec0p: 0, + ncal: 0, aopt: 0, alig: 0, patch_bootargs: false, @@ -2657,7 +2675,7 @@ impl platform::Driver for AopDriver { &aop_mmio, &[ (from_fourcc(b"EC0p"), cfg.ec0p), - (from_fourcc(b"nCal"), 0x0), + (from_fourcc(b"nCal"), cfg.ncal), (from_fourcc(b"alig"), cfg.alig), (from_fourcc(b"AOPt"), cfg.aopt), ], diff --git a/drivers/soc/apple/apple-pmgr-dcs-test.c b/drivers/soc/apple/apple-pmgr-dcs-test.c new file mode 100644 index 0000000000000..f1021b6de4f61 --- /dev/null +++ b/drivers/soc/apple/apple-pmgr-dcs-test.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Synthetic OPPs and memory-backed registers; no hardware rate claims. */ +#include +#include + +static int apple_pmgr_dcs_test_init(struct kunit *test) +{ + struct apple_pmgr_misc *misc; + void *reg; + + misc = kunit_kzalloc(test, sizeof(*misc), GFP_KERNEL); + if (!misc) + return -ENOMEM; + reg = kunit_kzalloc(test, sizeof(u32), GFP_KERNEL); + if (!reg) + return -ENOMEM; + misc->dev = kunit_device_register(test, "pmgr-dcs-test"); + if (IS_ERR(misc->dev)) + return PTR_ERR(misc->dev); + mutex_init(&misc->lock); + misc->devices[DEV_DCS].base = (void __iomem *)reg; + misc->devices[DEV_DCS].active_state = 10; + misc->devices[DEV_DCS].suspend_state = DCS_DEV_PSTATE_MIN_T600X; + misc->dcs_num_opps = 3; + misc->dcs_opps[0] = (struct apple_pmgr_dcs_opp) { 7, 1000 }; + misc->dcs_opps[1] = (struct apple_pmgr_dcs_opp) { 9, 2000 }; + misc->dcs_opps[2] = (struct apple_pmgr_dcs_opp) { 10, 3000 }; + writel(0xa50000aa, misc->devices[DEV_DCS].base); + misc->dcs_data = kunit_kzalloc(test, struct_size(misc->dcs_data, nodes, 2), GFP_KERNEL); + if (!misc->dcs_data) + return -ENOMEM; + misc->dcs_data->num_nodes = 2; + for (unsigned int i = 0; i < 2; i++) { + misc->dcs_data->nodes[i] = kunit_kzalloc(test, sizeof(struct icc_node), GFP_KERNEL); + if (!misc->dcs_data->nodes[i]) + return -ENOMEM; + misc->dcs_data->nodes[i]->data = misc; + } + dev_set_drvdata(misc->dev, misc); + test->priv = misc; + return 0; +} + +static void apple_pmgr_dcs_table_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), 0); + swap(misc->dcs_opps[0], misc->dcs_opps[2]); + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), 0); + KUNIT_EXPECT_EQ(test, misc->dcs_opps[0].state, 7U); + misc->dcs_opps[1].state = 7; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), -EINVAL); + misc->dcs_opps[1].state = 9; + misc->dcs_opps[1].bw = 1000; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), -EINVAL); + misc->dcs_opps[1].bw = 2000; + misc->dcs_opps[0].bw = 0; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), -EINVAL); + misc->dcs_opps[0].bw = 1000; + misc->dcs_opps[0].state = 6; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), -EINVAL); + misc->dcs_opps[0].state = 7; + misc->devices[DEV_DCS].active_state = 9; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), -EINVAL); + misc->dcs_num_opps = 17; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), -EINVAL); + misc->dcs_num_opps = 1; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_validate(misc), -EINVAL); +} + +static void apple_pmgr_dcs_selection_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + u32 state = 0; + + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_select(misc, 0, &state), 0); + KUNIT_EXPECT_EQ(test, state, 7U); + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_select(misc, 1000, &state), 0); + KUNIT_EXPECT_EQ(test, state, 7U); + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_select(misc, 1001, &state), 0); + KUNIT_EXPECT_EQ(test, state, 9U); + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_select(misc, 3000, &state), 0); + KUNIT_EXPECT_EQ(test, state, 10U); + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_select(misc, 3001, &state), -ERANGE); + KUNIT_EXPECT_EQ(test, state, 10U); +} + +static void apple_pmgr_dcs_aggregation_test(struct kunit *test) +{ + u32 avg = 0, peak = 0; + + apple_pmgr_dcs_aggregate(NULL, 0, 800, 1000, &avg, &peak); + apple_pmgr_dcs_aggregate(NULL, 0, 900, 1200, &avg, &peak); + KUNIT_EXPECT_EQ(test, avg, 1700U); + KUNIT_EXPECT_EQ(test, peak, 1200U); + apple_pmgr_dcs_aggregate(NULL, 0, U32_MAX, 0, &avg, &peak); + KUNIT_EXPECT_EQ(test, avg, U32_MAX); +} + +static void apple_pmgr_dcs_commit_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + struct icc_node *node = misc->dcs_data->nodes[0]; + + node->avg_bw = 1001; + + writel(0xa500009a, misc->devices[DEV_DCS].base); + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_set(node, node), 0); + KUNIT_EXPECT_EQ(test, misc->devices[DEV_DCS].active_state, 9U); + KUNIT_EXPECT_TRUE(test, misc->dcs_bw_active); + KUNIT_EXPECT_EQ(test, readl(misc->devices[DEV_DCS].base), 0xa5000099U); +} + +static void apple_pmgr_dcs_owner_sleep_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + struct icc_node *node = misc->dcs_data->nodes[0]; + + node->peak_bw = 1000; + + misc->pmp_owned = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_set(node, node), -EBUSY); + misc->pmp_owned = false; + misc->sleeping = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_set(node, node), -EBUSY); + KUNIT_EXPECT_EQ(test, readl(misc->devices[DEV_DCS].base), 0xa50000aaU); +} + +static void apple_pmgr_dcs_rollback_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + struct icc_node *node = misc->dcs_data->nodes[0]; + + node->peak_bw = 1000; + + /* Actual stays at 10: request times out, rollback to 10 succeeds. */ + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_set(node, node), -ETIMEDOUT); + KUNIT_EXPECT_FALSE(test, misc->dcs_failed); + KUNIT_EXPECT_EQ(test, misc->devices[DEV_DCS].active_state, 10U); + KUNIT_EXPECT_EQ(test, readl(misc->devices[DEV_DCS].base), 0xa50000aaU); + /* An unknown actual state makes rollback fail too. */ + writel(0xa500000a, misc->devices[DEV_DCS].base); + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_set(node, node), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, misc->dcs_failed); + KUNIT_EXPECT_TRUE(test, misc->dcs_bw_active); + KUNIT_EXPECT_EQ(test, apple_pmgr_misc_prepare(misc->dev), -EIO); + KUNIT_EXPECT_EQ(test, apple_pmgr_dcs_set(node, node), -EIO); +} + +static void apple_pmgr_dcs_sync_constraints_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + struct icc_node *src = misc->dcs_data->nodes[0]; + struct icc_node *dst = misc->dcs_data->nodes[1]; + + src->avg_bw = 1001; + dst->init_peak = 3000; + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_set(src, src), 0); + KUNIT_EXPECT_EQ(test, misc->devices[DEV_DCS].active_state, 10U); + dst->init_peak = 0; + writel(0xa500009a, misc->devices[DEV_DCS].base); + /* Releasing destination's boot constraint must retain source demand. */ + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_set(dst, dst), 0); + KUNIT_EXPECT_EQ(test, misc->devices[DEV_DCS].active_state, 9U); +} + +static void apple_pmgr_dcs_suspend_restore_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + + dev_set_drvdata(misc->dev, misc); + misc->devices[DEV_DCS].active_state = 9; + KUNIT_ASSERT_EQ(test, apple_pmgr_misc_suspend_noirq(misc->dev), 0); + KUNIT_EXPECT_EQ(test, readl(misc->devices[DEV_DCS].base), 0xa50000a7U); + KUNIT_ASSERT_EQ(test, apple_pmgr_misc_resume_noirq(misc->dev), 0); + KUNIT_EXPECT_EQ(test, readl(misc->devices[DEV_DCS].base), 0xa50000a9U); +} + +static void apple_pmgr_dcs_no_table_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_init(misc), 0); + KUNIT_EXPECT_EQ(test, readl(misc->devices[DEV_DCS].base), 0xa50000aaU); +} + +static void apple_pmgr_dcs_release_test(struct kunit *test) +{ + struct apple_pmgr_misc *misc = test->priv; + struct icc_node *node = misc->dcs_data->nodes[0]; + + misc->dcs_bw_active = true; + writel(0xa500007a, misc->devices[DEV_DCS].base); + KUNIT_ASSERT_EQ(test, apple_pmgr_dcs_set(node, node), 0); + KUNIT_EXPECT_EQ(test, misc->devices[DEV_DCS].active_state, 7U); + KUNIT_EXPECT_FALSE(test, misc->dcs_bw_active); +} + +static struct kunit_case apple_pmgr_dcs_test_cases[] = { + KUNIT_CASE(apple_pmgr_dcs_table_test), + KUNIT_CASE(apple_pmgr_dcs_selection_test), + KUNIT_CASE(apple_pmgr_dcs_aggregation_test), + KUNIT_CASE(apple_pmgr_dcs_commit_test), + KUNIT_CASE(apple_pmgr_dcs_owner_sleep_test), + KUNIT_CASE(apple_pmgr_dcs_rollback_test), + KUNIT_CASE(apple_pmgr_dcs_sync_constraints_test), + KUNIT_CASE(apple_pmgr_dcs_suspend_restore_test), + KUNIT_CASE(apple_pmgr_dcs_no_table_test), + KUNIT_CASE(apple_pmgr_dcs_release_test), + {} +}; + +static struct kunit_suite apple_pmgr_dcs_test_suite = { + .name = "apple-pmgr-dcs", + .init = apple_pmgr_dcs_test_init, + .test_cases = apple_pmgr_dcs_test_cases, +}; + +kunit_test_suite(apple_pmgr_dcs_test_suite); diff --git a/drivers/soc/apple/apple-pmgr-misc.c b/drivers/soc/apple/apple-pmgr-misc.c index 0372e44ed3446..d18f4ed07d8bb 100644 --- a/drivers/soc/apple/apple-pmgr-misc.c +++ b/drivers/soc/apple/apple-pmgr-misc.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #define APPLE_CLKGEN_PSTATE 0 @@ -59,6 +61,11 @@ struct apple_pmgr_misc_hw { u32 dev_min_ps[DEV_MAX]; }; +struct apple_pmgr_dcs_opp { + u32 state; + u32 bw; +}; + struct apple_pmgr_misc { struct device *dev; struct apple_pmgr_sys_device devices[DEV_MAX]; @@ -74,6 +81,12 @@ struct apple_pmgr_misc { bool pmp_owned; /* PMP is described: no governor, holds don't raise */ bool pmp_sleep; /* PMP runs: this sleep leaves DCS to it (see prepare) */ u32 pmp_fabric; /* PMP's desired fabric p-state, saved across suspend */ + struct icc_provider dcs_provider; + struct icc_onecell_data *dcs_data; + struct apple_pmgr_dcs_opp dcs_opps[16]; + unsigned int dcs_num_opps; + bool dcs_failed; + bool dcs_bw_active; struct { struct regmap *regmap; u32 reg; @@ -83,6 +96,221 @@ struct apple_pmgr_misc { static struct apple_pmgr_misc *apple_pmgr_misc_inst; static unsigned int idle_fabric_pstate; +static int apple_pmgr_raise_fabric(struct apple_pmgr_misc *misc); + +/* Decimal kB/s throughout, as in the OPP and interconnect bindings. */ +static int apple_pmgr_dcs_aggregate(struct icc_node *node, u32 tag, u32 avg, + u32 peak, u32 *agg_avg, u32 *agg_peak) +{ + /* Saturate rather than wrap a large group of consumers to a low vote. */ + *agg_avg = min_t(u64, (u64)*agg_avg + avg, U32_MAX); + *agg_peak = max(*agg_peak, peak); + return 0; +} + +static int apple_pmgr_dcs_select(struct apple_pmgr_misc *misc, u32 bw, u32 *state) +{ + unsigned int i; + + for (i = 0; i < misc->dcs_num_opps; i++) { + if (bw <= misc->dcs_opps[i].bw) { + *state = misc->dcs_opps[i].state; + return 0; + } + } + return -ERANGE; +} + +static int apple_pmgr_dcs_write(struct apple_pmgr_misc *misc, u32 state) +{ + void __iomem *reg = misc->devices[DEV_DCS].base + APPLE_CLKGEN_PSTATE; + u32 val = readl(reg); + + FIELD_MODIFY(APPLE_CLKGEN_PSTATE_DESIRED, &val, state); + writel(val, reg); + return readl_poll_timeout(reg, val, + FIELD_GET(APPLE_CLKGEN_PSTATE_ACTUAL, val) == state, + 10, APPLE_PMGR_RAISE_TIMEOUT_US); +} + +static int apple_pmgr_dcs_set(struct icc_node *src, struct icc_node *dst) +{ + struct apple_pmgr_misc *misc = src->data; + u32 state, old, bw = 0; + unsigned int i; + int ret; + + guard(mutex)(&misc->lock); + if (misc->pmp_owned || misc->sleeping) + return -EBUSY; + if (misc->dcs_failed) + return -EIO; + /* sync_state invokes set(node, node); keep both endpoints' constraints. */ + for (i = 0; i < misc->dcs_data->num_nodes; i++) { + struct icc_node *node = misc->dcs_data->nodes[i]; + + if (!node) + continue; + bw = max(bw, max(node->avg_bw, node->peak_bw)); + bw = max(bw, max(node->init_avg, node->init_peak)); + } + ret = apple_pmgr_dcs_select(misc, bw, &state); + if (ret) + return ret; + /* Untracked DMA clients need the fabric as well as DCS bandwidth. */ + if (bw) { + ret = apple_pmgr_raise_fabric(misc); + if (ret) + return ret; + } + old = misc->devices[DEV_DCS].active_state; + ret = apple_pmgr_dcs_write(misc, state); + if (ret) { + /* A failed request must not replace the last confirmed vote. */ + if (apple_pmgr_dcs_write(misc, old)) { + misc->dcs_failed = true; + misc->dcs_bw_active = true; + dev_err(misc->dev, "DCS vote and rollback timed out; refusing further votes\n"); + } + return ret; + } + misc->devices[DEV_DCS].active_state = state; + misc->dcs_bw_active = bw != 0; + return 0; +} + +static int apple_pmgr_dcs_get_bw(struct icc_node *node, u32 *avg, u32 *peak) +{ + struct apple_pmgr_misc *misc = node->data; + + /* Keep the boot state until all described consumers have probed. */ + *avg = 0; + *peak = misc->dcs_opps[misc->dcs_num_opps - 1].bw; + return 0; +} + +static int apple_pmgr_dcs_opp_cmp(const void *a, const void *b) +{ + const struct apple_pmgr_dcs_opp *oa = a, *ob = b; + + return (oa->state > ob->state) - (oa->state < ob->state); +} + +static int apple_pmgr_dcs_validate(struct apple_pmgr_misc *misc) +{ + unsigned int i; + + if (misc->dcs_num_opps < 2 || misc->dcs_num_opps > ARRAY_SIZE(misc->dcs_opps)) + return -EINVAL; + sort(misc->dcs_opps, misc->dcs_num_opps, sizeof(misc->dcs_opps[0]), + apple_pmgr_dcs_opp_cmp, NULL); + for (i = 0; i < misc->dcs_num_opps; i++) { + const struct apple_pmgr_dcs_opp *opp = &misc->dcs_opps[i]; + + if (!opp->bw || opp->state < DCS_DEV_PSTATE_MIN_T600X || + opp->state > FIELD_MAX(APPLE_CLKGEN_PSTATE_DESIRED)) + return -EINVAL; + if (i && (opp->state <= misc->dcs_opps[i - 1].state || + opp->bw <= misc->dcs_opps[i - 1].bw)) + return -EINVAL; + } + /* Never request more than the boot firmware's known working state. */ + if (misc->dcs_opps[misc->dcs_num_opps - 1].state != + misc->devices[DEV_DCS].active_state) + return -EINVAL; + return 0; +} + +static void apple_pmgr_dcs_cleanup(void *data) +{ + struct apple_pmgr_misc *misc = data; + + icc_provider_deregister(&misc->dcs_provider); + icc_nodes_remove(&misc->dcs_provider); +} + +static void apple_pmgr_misc_sync_state(struct device *dev) +{ + struct apple_pmgr_misc *misc = dev_get_drvdata(dev); + + if (misc->dcs_data) + icc_sync_state(dev); +} + +static int apple_pmgr_dcs_init(struct apple_pmgr_misc *misc) +{ + struct device *dev = misc->dev; + struct device_node *table __free(device_node) = NULL; + struct icc_provider *provider = &misc->dcs_provider; + struct icc_node *node; + u32 cells; + unsigned int i; + int ret; + + if (!of_property_present(dev->of_node, "operating-points-v2")) + return of_property_present(dev->of_node, "#interconnect-cells") ? -EINVAL : 0; + /* This interface is for the direct host owner, never a PMP fallback. */ + if (misc->pmp_owned || !of_device_is_compatible(dev->of_node, "apple,t6000-pmgr-misc")) + return -EBUSY; + if (of_property_count_u32_elems(dev->of_node, "operating-points-v2") != 1 || + of_property_read_u32(dev->of_node, "#interconnect-cells", &cells) || cells != 1) + return -EINVAL; + table = of_parse_phandle(dev->of_node, "operating-points-v2", 0); + if (!table || !of_device_is_available(table) || + !of_device_is_compatible(table, "operating-points-v2-apple-dcs")) + return -EINVAL; + for_each_available_child_of_node_scoped(table, child) { + struct apple_pmgr_dcs_opp *opp; + + if (misc->dcs_num_opps == ARRAY_SIZE(misc->dcs_opps)) + return -E2BIG; + opp = &misc->dcs_opps[misc->dcs_num_opps++]; + if (of_property_present(child, "opp-supported-hw") || + of_property_present(child, "turbo-mode") || + of_property_count_u32_elems(child, "opp-level") != 1 || + of_property_count_u32_elems(child, "opp-peak-kBps") != 1 || + of_property_read_u32(child, "opp-level", &opp->state) || + of_property_read_u32(child, "opp-peak-kBps", &opp->bw)) + return -EINVAL; + } + ret = apple_pmgr_dcs_validate(misc); + if (ret) + return dev_err_probe(dev, ret, "invalid measured DCS OPP table\n"); + misc->dcs_data = devm_kzalloc(dev, struct_size(misc->dcs_data, nodes, 2), GFP_KERNEL); + if (!misc->dcs_data) + return -ENOMEM; + misc->dcs_data->num_nodes = 2; + provider->dev = dev; + provider->set = apple_pmgr_dcs_set; + provider->aggregate = apple_pmgr_dcs_aggregate; + provider->get_bw = apple_pmgr_dcs_get_bw; + provider->xlate = of_icc_xlate_onecell; + provider->data = misc->dcs_data; + icc_provider_init(provider); + for (i = 0; i < 2; i++) { + node = icc_node_create_dyn(); + if (IS_ERR(node)) { + ret = PTR_ERR(node); + goto remove_nodes; + } + node->name = i ? "t600x-dcs-memory" : "t600x-dcs-clients"; + node->data = misc; + misc->dcs_data->nodes[i] = node; + icc_node_add(node, provider); + } + ret = icc_link_create(misc->dcs_data->nodes[0], misc->dcs_data->nodes[1]->id); + if (ret) + goto remove_nodes; + ret = icc_provider_register(provider); + if (ret) + goto remove_nodes; + return devm_add_action_or_reset(dev, apple_pmgr_dcs_cleanup, misc); + +remove_nodes: + icc_nodes_remove(provider); + return ret; +} + static void apple_pmgr_sys_dev_set_pstate(struct apple_pmgr_misc *misc, enum sys_device dev, bool active) { @@ -259,7 +487,7 @@ static void apple_pmgr_idle_eval(struct apple_pmgr_misc *misc) if (misc->sleeping) return; low = idle && idle < active && misc->voters_ok && !misc->pmp_owned && - !misc->holds && !apple_pmgr_voters_active(misc) && + !misc->holds && !misc->dcs_bw_active && !apple_pmgr_voters_active(misc) && apple_pmgr_cpus_at_min(); if (low && misc->fabric_now != idle) apple_pmgr_write_fabric(misc, idle); @@ -365,13 +593,19 @@ static int __maybe_unused apple_pmgr_misc_prepare(struct device *dev) { struct apple_pmgr_misc *misc = dev_get_drvdata(dev); + /* Serialize the failure check and sleep gate with in-flight votes. */ + scoped_guard(mutex, &misc->lock) { + /* Failed rollback leaves no confirmed DCS state to restore. */ + if (misc->dcs_failed) + return -EIO; + if (misc->devices[DEV_FABRIC].base) + misc->sleeping = true; + } misc->pmp_sleep = misc->pmp_owned && apple_pmgr_pmp_running(); if (misc->pmp_owned && !misc->pmp_sleep) dev_warn_once(dev, "PMP is described but not running: sleeping without it\n"); if (!misc->devices[DEV_FABRIC].base) return 0; - scoped_guard(mutex, &misc->lock) - misc->sleeping = true; apple_pmgr_idle_stop(misc); return 0; } @@ -511,6 +745,9 @@ static int apple_pmgr_misc_probe(struct platform_device *pdev) INIT_DEFERRABLE_WORK(&misc->idle_work, apple_pmgr_idle_work); INIT_DELAYED_WORK(&misc->idle_work_prompt, apple_pmgr_idle_work_prompt); apple_pmgr_find_voters(misc); + ret = apple_pmgr_dcs_init(misc); + if (ret) + return ret; /* Publish only a set-up instance; pairs with the readers' acquire. */ smp_store_release(&apple_pmgr_misc_inst, misc); if (READ_ONCE(idle_fabric_pstate) && misc->pmp_owned) { @@ -561,6 +798,7 @@ static struct platform_driver apple_pmgr_misc_driver = { .of_match_table = apple_pmgr_misc_of_match, /* The governor's workers and holders would outlive an unbind. */ .suppress_bind_attrs = true, + .sync_state = apple_pmgr_misc_sync_state, .pm = pm_ptr(&apple_pmgr_misc_pm_ops), }, }; @@ -570,3 +808,7 @@ MODULE_DESCRIPTION("PMGR misc driver for Apple SoCs"); MODULE_LICENSE("GPL"); module_platform_driver(apple_pmgr_misc_driver); + +#if IS_ENABLED(CONFIG_APPLE_PMGR_DCS_KUNIT_TEST) +#include "apple-pmgr-dcs-test.c" +#endif diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 6b4e1edbe2966..573830b7973d0 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -6,7 +6,7 @@ use kernel::ioctl::{_IO, _IOR, _IOW, _IOWR}; use kernel::prelude::*; use kernel::uaccess::{UserPtr, UserSlice}; -pub(crate) const IFACE_VERSION: u32 = 4; +pub(crate) const IFACE_VERSION: u32 = 5; pub(crate) const ENROL_USER_ID: i32 = 1000; @@ -100,10 +100,35 @@ struct List { #[repr(C)] struct EnrolStart { flags: u32, + account_uid: u32, + finger: u32, reserved: u32, label: [u8; LABEL_LEN], } +#[repr(C)] +pub(crate) struct EnclaveList { + pub(crate) count: u32, + pub(crate) reserved: u32, + pub(crate) uuid: [[u8; UUID_LEN]; MAX_IDENTITIES], +} + +impl EnclaveList { + pub(crate) fn new() -> Self { + Self { count: 0, reserved: 0, uuid: [[0; UUID_LEN]; MAX_IDENTITIES] } + } + + pub(crate) fn push(&mut self, uuid: [u8; UUID_LEN]) -> Result<()> { + let at = self.count as usize; + if at >= MAX_IDENTITIES { + return Err(EOVERFLOW); + } + self.uuid[at] = uuid; + self.count += 1; + Ok(()) + } +} + #[repr(C)] #[derive(Default)] struct EnrolPoll { @@ -136,8 +161,8 @@ struct VerifyPoll { } #[repr(C)] -struct Delete { - uuid: [u8; UUID_LEN], +pub(crate) struct Delete { + pub(crate) uuid: [u8; UUID_LEN], } pub(crate) const ATTEST_CHALLENGE_LEN: usize = 32; @@ -162,7 +187,8 @@ pub(crate) struct Attest { static_assert!(core::mem::size_of::() == 144); static_assert!(core::mem::size_of::() == 32); static_assert!(core::mem::size_of::() == 4616); -static_assert!(core::mem::size_of::() == 136); +static_assert!(core::mem::size_of::() == 144); +static_assert!(core::mem::size_of::() == 520); static_assert!(core::mem::size_of::() == 40); static_assert!(core::mem::size_of::() == 40); static_assert!(core::mem::size_of::() == 72); @@ -182,7 +208,7 @@ macro_rules! ioctl_pod { }; } ioctl_pod! { - to_user: Info, List, EnrolPoll, VerifyPoll, Attest; + to_user: Info, List, EnclaveList, EnrolPoll, VerifyPoll, Attest; from_user: EnrolStart, VerifyStart, Delete, Attest } @@ -198,6 +224,9 @@ const IOC_CANCEL: u32 = _IO(MAGIC, 0x07); const IOC_DELETE: u32 = _IOW::(MAGIC, 0x08); const IOC_DELETE_ALL: u32 = _IO(MAGIC, 0x09); pub(crate) const IOC_ATTEST: u32 = _IOWR::(MAGIC, 0x0a); +pub(crate) const IOC_LIST_ENCLAVE: u32 = _IOR::(MAGIC, 0x0b); +pub(crate) const IOC_DELETE_ORPHAN: u32 = _IOW::(MAGIC, 0x0c); +pub(crate) const IOC_PREPARE_ENROL: u32 = _IOW::(MAGIC, 0x0d); pub(crate) struct MatchEvidence { identity: [u8; UUID_LEN], @@ -338,6 +367,10 @@ impl IdentityIndex { self.save(store) } + pub(crate) fn can_persist_next(&self, store: &Store) -> bool { + store.can_write(&identity_key(), 8 + (self.entries.len() + 1) * (UUID_LEN + LABEL_LEN)) + } + pub(crate) fn identity_v1_for( &self, uuid: &[u8; UUID_LEN], @@ -481,7 +514,7 @@ pub(crate) struct Handled { pub(crate) delete_identities: KVec, } -fn ok() -> Result { +pub(crate) fn ok() -> Result { Ok(Handled { ret: 0, wake: false, @@ -492,7 +525,7 @@ fn ok() -> Result { }) } -fn require_admin() -> Result<()> { +pub(crate) fn require_admin() -> Result<()> { if crate::shim::capable_admin() { Ok(()) } else { @@ -500,6 +533,12 @@ fn require_admin() -> Result<()> { } } +pub(crate) fn orphan_delete(identity: crate::sbio::IdentityV1) -> Result { + let mut handled = ok()?; + handled.delete_identity = Some(identity); + Ok(handled) +} + fn require_sensor(present: bool) -> Result<()> { if present { Ok(()) @@ -567,22 +606,85 @@ fn list(ctx: &mut Context<'_>, user: UserPtr) -> Result { ok() } +fn validate_enrol_request(request: &EnrolStart) -> Result<()> { + // NSS account resolution occurs in libfprint. This ABI requires its + // resolved uid, the named finger and the generated user label. + if request.flags != 0 || request.reserved != 0 + || request.account_uid == u32::MAX || !(1..=10).contains(&request.finger) + || !request.label.starts_with(b"FP1-") + || request.label[12] != b'-' || request.label[14] != b'-' + || request.label[23] != b'-' || request.label[24] == 0 + || request.label[13] != b"123456789A"[request.finger as usize - 1] + || !request.label.contains(&0) + { + return Err(EINVAL); + } + Ok(()) +} + +pub(crate) fn validate_enrol_start_arg(arg: usize) -> Result<()> { + let user = UserPtr::from_addr(arg); + let request: EnrolStart = UserSlice::new(user, core::mem::size_of::()) + .reader().read()?; + validate_enrol_request(&request) +} + +#[cfg(CONFIG_APPLE_SEP_KUNIT_TEST)] +#[::kernel::macros::kunit_tests(apple_sep_enrol_request)] +mod enrol_request_tests { + use super::*; + + fn valid() -> EnrolStart { + let mut request = EnrolStart { + flags: 0, account_uid: 1000, finger: 7, reserved: 0, + label: [0; LABEL_LEN], + }; + request.label[..28].copy_from_slice(b"FP1-20261027-7-12345678-user"); + request + } + + #[test] + fn resolved_account_and_known_finger() { + assert!(validate_enrol_request(&valid()).is_ok()); + } + + #[test] + fn missing_uid_is_refused() { + let mut request = valid(); + request.account_uid = u32::MAX; + assert!(validate_enrol_request(&request).is_err()); + } + + #[test] + fn unknown_or_mismatched_finger_is_refused() { + let mut request = valid(); + request.finger = 0; + assert!(validate_enrol_request(&request).is_err()); + request.finger = 6; + assert!(validate_enrol_request(&request).is_err()); + } + + #[test] + fn missing_username_is_refused() { + let mut request = valid(); + request.label[24] = 0; + assert!(validate_enrol_request(&request).is_err()); + } +} + fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { require_admin()?; let request: EnrolStart = UserSlice::new(user, core::mem::size_of::()) .reader() .read()?; - if request.flags != 0 || request.reserved != 0 { - return Err(EINVAL); - } + validate_enrol_request(&request)?; require_sensor(ctx.sensor_present)?; if ctx.session.worker_epoch.is_some() || !ctx.session.op.may_start() { return Err(EBUSY); } - if matches!(ctx.live_identity_count, Some(count) if count >= SEP_USER_CAPACITY) - || ctx.index.total() >= MAX_IDENTITIES - { + let count = ctx.live_identity_count.ok_or(EIO)?; + if count >= SEP_USER_CAPACITY || ctx.index.total() >= MAX_IDENTITIES { return Err(ENOSPC); } @@ -960,6 +1062,10 @@ pub(crate) fn starts_capture(cmd: u32) -> bool { cmd == IOC_ENROL_START || cmd == IOC_VERIFY_START } +pub(crate) fn idle_for_mutation(session: &Session) -> bool { + session.worker_epoch.is_none() && session.op.may_start() +} + fn delete(ctx: &mut Context<'_>, user: UserPtr) -> Result { require_admin()?; if !ctx.session.op.may_start() { diff --git a/drivers/soc/apple/catacomb.rs b/drivers/soc/apple/catacomb.rs index 8b18efb4ead77..d71b415fcb321 100644 --- a/drivers/soc/apple/catacomb.rs +++ b/drivers/soc/apple/catacomb.rs @@ -28,6 +28,24 @@ pub(crate) fn is_kind(kind: u8) -> bool { path_for(kind).is_some() } +pub(crate) fn writable(kind: u8) -> bool { + path_for(kind) + .and_then(|path| shim::StoreFile::open_writable_existing(path).ok()) + .is_some() +} + +/// Reserve a missing Linux owner file before SELECT_CONTEXT changes SEP state. +/// A corrupt existing file is never truncated here: it is failure evidence. +pub(crate) fn prepare_missing_owner() -> Result<()> { + let path = path_for(crate::PRIVATE_TYPE_CATACOMB_OWNER).ok_or(EINVAL)?; + let (file, created) = shim::StoreFile::open_host(path)?; + if !created { + return Err(ENODATA); + } + file.sync()?; + file.sync_parent() +} + pub(crate) fn write(kind: u8, blob: &[u8]) -> Result<()> { let path = path_for(kind).ok_or(EINVAL)?; if blob.len() > MAX_CATACOMB { diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 5a81a370dda38..98a048443da18 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -10,6 +10,128 @@ use crate::sks::SKS_AUTH_TOKEN_LEN; use kernel::prelude::*; use kernel::soc::apple::mailbox::Message; +struct EnrolPrerequisites { + lane: bool, + live_count: Option, + bag_bound: bool, + catacomb_readable: [bool; 3], + catacomb_writable: [bool; 3], + keybag_writable: bool, + host_store_capacity: bool, +} + +fn admit_enrol(p: &EnrolPrerequisites) -> Result<()> { + if !p.lane { return Err(ENXIO); } + let count = p.live_count.ok_or(EIO)?; + if count >= 3 { return Err(ENOSPC); } + if !p.bag_bound || p.catacomb_readable.contains(&false) { + return Err(ENODATA); + } + if p.catacomb_writable.contains(&false) || !p.keybag_writable { + return Err(EROFS); + } + if !p.host_store_capacity { return Err(ENOSPC); } + Ok(()) +} + +fn catacomb_needed_for_enrol(kind: u8, live_count: Option) -> bool { + live_count != Some(0) || kind == PRIVATE_TYPE_CATACOMB_OWNER +} + +fn admissible_orphan_target( + listed_linux: &[[u8; bio::UUID_LEN]], + uuid: &[u8; bio::UUID_LEN], +) -> Result<()> { + if *uuid == [0; bio::UUID_LEN] { return Err(EINVAL); } + if !listed_linux.contains(uuid) { return Err(ENOENT); } + Ok(()) +} + +#[cfg(CONFIG_APPLE_SEP_KUNIT_TEST)] +#[::kernel::macros::kunit_tests(apple_sep_enrol_admission)] +mod admission_tests { + use super::*; + + fn ready() -> EnrolPrerequisites { + EnrolPrerequisites { + lane: true, live_count: Some(1), bag_bound: true, + catacomb_readable: [true; 3], catacomb_writable: [true; 3], + keybag_writable: true, host_store_capacity: true, + } + } + + fn refused(p: &EnrolPrerequisites, expected: Error) { + assert_eq!(admit_enrol(p).unwrap_err().to_errno(), expected.to_errno()); + } + + #[test] + fn missing_os_identity() { + let mut p = ready(); p.lane = false; refused(&p, ENXIO); + } + + #[test] + fn unavailable_enclave_list() { + let mut p = ready(); p.live_count = None; refused(&p, EIO); + } + + #[test] + fn enclave_full() { + let mut p = ready(); p.live_count = Some(3); refused(&p, ENOSPC); + } + + #[test] + fn missing_identity_bag_binding() { + let mut p = ready(); p.bag_bound = false; refused(&p, ENODATA); + } + + #[test] + fn missing_catacomb() { + for i in 0..3 { + let mut p = ready(); p.catacomb_readable[i] = false; refused(&p, ENODATA); + } + } + + #[test] + fn unwritable_catacomb_or_keybag() { + for i in 0..3 { + let mut p = ready(); p.catacomb_writable[i] = false; refused(&p, EROFS); + } + let mut p = ready(); p.keybag_writable = false; refused(&p, EROFS); + } + + #[test] + fn host_store_full() { + let mut p = ready(); p.host_store_capacity = false; refused(&p, ENOSPC); + } + + #[test] + fn ready_binding_is_admitted() { + assert!(admit_enrol(&ready()).is_ok()); + } + + #[test] + fn clean_lane_requires_prepared_owner_before_first_capture() { + assert!(catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_OWNER, Some(0))); + assert!(!catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_USER, Some(0))); + assert!(!catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_MASTER, Some(0))); + assert!(catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_USER, Some(1))); + } + + #[test] + fn orphan_delete_requires_exact_linux_identity() { + let linux = [1u8; bio::UUID_LEN]; + let other_lane = [2u8; bio::UUID_LEN]; + assert!(admissible_orphan_target(&[linux], &linux).is_ok()); + refused_orphan(&[linux], &other_lane, ENOENT); + refused_orphan(&[linux], &[0; bio::UUID_LEN], EINVAL); + } + + fn refused_orphan(list: &[[u8; bio::UUID_LEN]], uuid: &[u8; bio::UUID_LEN], expected: Error) { + assert_eq!(admissible_orphan_target(list, uuid).unwrap_err().to_errno(), + expected.to_errno()); + } +} + impl SepData { fn sbio_enrol_step(&self, op: &crate::sbio::SbioOp) -> Option> { match self.sbio_call(op) { @@ -349,6 +471,12 @@ impl SepData { return self.enrol_into_existing_context(user); }; + // PREPARE_ENROL may already have selected the empty Linux context in + // this boot. A file left from an earlier boot is not that proof. + if self.empty_context_prepared.xchg(false, Relaxed) { + return self.enrol_into_existing_context(user); + } + self.open_fresh_context(user, &proof) } @@ -1777,6 +1905,16 @@ impl SepData { }; if !parsed.matches(user) { + // The enclave compared a usable capture and did not answer the + // Linux user. Usually no template matched (a different finger). + // Log the user it did answer, so a match in another user context + // can be told apart from a plain non-match. + dev_info!( + self.dev, + "verify: no enrolled finger matched the capture (enclave answered user {}, Linux enrolments use user {})\n", + parsed.user_id(), + user.value() + ); return bio::VerifyOutcome::NoMatch; } @@ -1784,6 +1922,12 @@ impl SepData { let identity = *parsed.identity_uuid(); let known = self.bio_index.lock().contains_uuid(&identity); + dev_info!( + self.dev, + "verify: enclave matched identity {} (host index {})\n", + Hex(&identity), if known { "present" } else { "absent" } + ); + if !known { dev_err!( self.dev, @@ -3404,6 +3548,133 @@ impl SepData { Some(bytes) } + fn require_linux_bio_lane(&self) -> Result<()> { + if !self.touchid_started.load(Relaxed) || self.xarm.lock().os_uuid.is_none() { + return Err(ENXIO); + } + if !self.keybag_designated.load(Relaxed) + || self.enrol_material.lock().is_none() + || !matches!(keybag::read(keybag::Slot::Identity), + Ok(keybag::State::Present(_))) { + return Err(ENODATA); + } + Ok(()) + } + + /// Refuse before ENROL_START schedules work or sends BEGIN_ENROL. A + /// userspace passwd lookup proves the host account; these are the durable + /// Linux-lane bindings that only the kernel can inspect. + fn preflight_enrol(&self, live_count: Option) -> Result<()> { + let lane = self.touchid_started.load(Relaxed) && self.xarm.lock().os_uuid.is_some(); + let bag_bound = self.keybag_designated.load(Relaxed) && self.enrol_material.lock() + .as_ref().is_some_and(|material| material.snapshot_identity.is_some()); + let kinds = [PRIVATE_TYPE_CATACOMB_OWNER, PRIVATE_TYPE_CATACOMB_USER, + PRIVATE_TYPE_CATACOMB_MASTER]; + let readable = kinds.map(|kind| !catacomb_needed_for_enrol(kind, live_count) + || crate::catacomb::read(kind).is_some()); + let writable = kinds.map(|kind| !catacomb_needed_for_enrol(kind, live_count) + || crate::catacomb::writable(kind)); + let index = self.bio_index.lock(); + let preconditions = EnrolPrerequisites { + lane, + live_count, + bag_bound, + catacomb_readable: readable, + catacomb_writable: writable, + keybag_writable: shim::StoreFile::open_writable_existing( + keybag::Slot::Identity.path()).is_ok(), + host_store_capacity: self.with_host_store(|store| index.can_persist_next(store)) + .unwrap_or(false), + }; + let result = admit_enrol(&preconditions); + if let Err(e) = &result { + dev_warn!(self.dev, "enrol: prerequisites failed before enclave capture: {:?}\n", e); + } + result + } + + fn bio_list_enclave(&self, arg: usize) -> Result { + bio::require_admin()?; + self.require_linux_bio_lane()?; + let ids = self.enclave_identities_for(SBIO_PROBE_USER_ID).ok_or(EIO)?; + let mut out = bio::EnclaveList::new(); + for uuid in ids.iter() { + out.push(*uuid)?; + } + kernel::uaccess::UserSlice::new( + kernel::uaccess::UserPtr::from_addr(arg), core::mem::size_of::() + ).writer().write(&out)?; + bio::ok() + } + + fn bio_delete_orphan(&self, arg: usize) -> Result { + bio::require_admin()?; + self.require_linux_bio_lane()?; + if !bio::idle_for_mutation(&self.bio_session.lock()) { return Err(EBUSY); } + let request: bio::Delete = kernel::uaccess::UserSlice::new( + kernel::uaccess::UserPtr::from_addr(arg), core::mem::size_of::() + ).reader().read()?; + // The exact SEP user and built-in selector are checked by the parser. + // The host index may still hold an orphan after fprintd lost its file, + // so that index cannot decide whether this exact-ID recovery is valid. + let ids = self.enclave_identities_for(SBIO_PROBE_USER_ID).ok_or(EIO)?; + admissible_orphan_target(&ids, &request.uuid)?; + bio::orphan_delete(crate::sbio::IdentityV1::from_index( + SBIO_PROBE_USER_ID, request.uuid)) + } + + fn bio_prepare_enrol(&self, arg: usize) -> Result { + bio::require_admin()?; + bio::validate_enrol_start_arg(arg)?; + self.require_linux_bio_lane()?; + if !bio::idle_for_mutation(&self.bio_session.lock()) { return Err(EBUSY); } + if !sensor::is_bound() { return Err(ENODEV); } + if !self.xart_writable() { return Err(EROFS); } + let ids = self.enclave_identities_for(SBIO_PROBE_USER_ID).ok_or(EIO)?; + if !ids.is_empty() { + // An existing identity must already have a durable owner file; + // selecting SYSTEM here could disturb its live context. + return if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_some() { + bio::ok() + } else { Err(ENODATA) }; + } + if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_some() { + return bio::ok(); + } + if !self.keybag_designated.load(Relaxed) + || !self.enrol_material.lock().as_ref() + .is_some_and(|material| material.snapshot_identity.is_some()) { + return Err(ENODATA); + } + if shim::StoreFile::open_writable_existing(keybag::Slot::Identity.path()).is_err() { + return Err(EROFS); + } + let index = self.bio_index.lock(); + if !self.with_host_store(|store| index.can_persist_next(store)).unwrap_or(false) { + return Err(ENOSPC); + } + drop(index); + let user = crate::sbio::UserId::new(SBIO_PROBE_USER_ID).ok_or(EINVAL)?; + // The host must be able to create and sync the owner file before a + // system-context write. A pre-existing invalid file is left intact. + crate::catacomb::prepare_missing_owner()?; + loop { + self.wait_until_awake()?; + let _writer = self.catacomb_writer.lock(); + if !self.capture_begin() { continue; } + let prepared = self.open_enrolment_context(user); + self.capture_end(); + if !prepared { return Err(EIO); } + break; + } + if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() + || !crate::catacomb::writable(PRIVATE_TYPE_CATACOMB_OWNER) { + return Err(EROFS); + } + self.empty_context_prepared.store(true, Relaxed); + bio::ok() + } + pub(crate) fn bio_ioctl(&self, cmd: u32, arg: usize) -> Result { if self.shutting_down.load(Relaxed) { return Err(ENODEV); @@ -3411,6 +3682,12 @@ impl SepData { if cmd == bio::IOC_ATTEST { return self.bio_attest(arg); } + if cmd == bio::IOC_LIST_ENCLAVE { + return self.bio_list_enclave(arg); + } + if cmd == bio::IOC_PREPARE_ENROL { + return self.bio_prepare_enrol(arg); + } // With xART writes off the enclave cannot persist what an enrolment or // a match changes. It saves anti-replay state after every match, and a // refused save has left it unresponsive until reboot. Refuse the @@ -3435,14 +3712,21 @@ impl SepData { let live_identity_count = if (cmd == bio::IOC_GET_INFO || cmd == bio::IOC_ENROL_START) && self.touchid_started.load(Relaxed) { - self.enclave_identity_count() + self.enclave_identities_for(SBIO_PROBE_USER_ID).map(|ids| ids.len()) } else { None }; + if cmd == bio::IOC_ENROL_START { + bio::require_admin()?; + bio::validate_enrol_start_arg(arg)?; + self.preflight_enrol(live_identity_count)?; + } // The SPI probe may have deferred until its SMC power supplier bound. // A saved attach-time snapshot would incorrectly reject later ioctls. let sensor_present = sensor::is_bound(); - let handled = { + let handled = if cmd == bio::IOC_DELETE_ORPHAN { + self.bio_delete_orphan(arg)? + } else { let mut session = self.bio_session.lock(); if bio::starts_capture(cmd) { self.wait_for_worker_exit(&mut session, false)?; @@ -3479,7 +3763,7 @@ impl SepData { if !self.capture_begin() { continue; } - let deleted = self.delete_and_persist(&handled); + let deleted = self.delete_and_persist(&handled, cmd == bio::IOC_DELETE_ORPHAN); self.capture_end(); break deleted; }; @@ -3722,7 +4006,7 @@ impl SepData { /// Called with `catacomb_writer` held. A host index entry goes only once /// its deletion is saved: if the save fails, the entry stays, userspace /// still lists the fingerprint, and a retried deletion saves again. - fn delete_and_persist(&self, handled: &bio::Handled) -> Result<()> { + fn delete_and_persist(&self, handled: &bio::Handled, recovery: bool) -> Result<()> { let retry = self.deletion_unsaved.load(Relaxed); let wanted = Iterator::chain(handled.delete_identity.iter(), handled.delete_identities.iter()); @@ -3730,10 +4014,13 @@ impl SepData { let mut gone: KVec<[u8; bio::UUID_LEN]> = KVec::with_capacity(wanted.clone().count(), GFP_KERNEL)?; let mut delete_err: Option = None; + let mut orphan_removed = false; for identity in wanted { - match self.delete_identity(identity) { + let orphan = recovery || !self.bio_index.lock().contains_uuid(identity.uuid()); + match self.delete_identity(identity, orphan) { Ok(true) => { let _ = gone.push_within_capacity(*identity.uuid()); + orphan_removed |= orphan; } Ok(false) => {} Err(e) => delete_err = delete_err.or(Some(e)), @@ -3741,7 +4028,7 @@ impl SepData { } // Save what was removed even if another deletion failed. if !gone.is_empty() { - match self.persist_deletion(retry) { + match self.persist_deletion(retry, orphan_removed) { Ok(()) => { self.deletion_unsaved.store(false, Relaxed); for uuid in gone.iter() { @@ -3766,7 +4053,7 @@ impl SepData { /// Returns whether the enclave no longer holds the identity and the host /// entry must wait for [`Self::persist_deletion`]. - fn delete_identity(&self, identity: &crate::sbio::IdentityV1) -> Result { + fn delete_identity(&self, identity: &crate::sbio::IdentityV1, orphan: bool) -> Result { if self.enclave_lists(identity.uuid()) == Some(false) { // Unless an earlier deletion's save failed, which leaves the // enclave without an identity the saved Catacomb still holds. @@ -3777,18 +4064,44 @@ impl SepData { return Ok(false); } - // 0x57 cannot be undone. Check first that its deletion can be saved. + // 0x57 cannot be undone. Ensure the host can save the affected user + // and master Catacombs before sending it. For an explicit recovery the + // owner Catacomb provides the recovery path even when the bag snapshot + // has no binding; export that owner first if this install lacks a file. + for kind in [PRIVATE_TYPE_CATACOMB_USER, PRIVATE_TYPE_CATACOMB_MASTER] { + if crate::catacomb::read(kind).is_none() || !crate::catacomb::writable(kind) { + dev_err!( + self.dev, + "bio: not deleting: user/master Catacomb is unavailable or unwritable\n" + ); + return Err(EIO); + } + } + if orphan && crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() { + let Some(reply) = self.read_component_states() else { return Err(EIO); }; + let state = crate::sbio::ComponentStates::new(&reply) + .state_for(crate::sbio::CatacombUser::OWNER.value()); + if !state.is_some_and(|s| s & crate::sbio::COMPONENT_STATE_ACTIVE != 0) + || !self.save_catacomb(crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"orphan-recovery owner Catacomb") { + dev_err!(self.dev, "bio: not deleting: owner Catacomb export failed\n"); + return Err(EIO); + } + } if crate::catacomb::read(PRIVATE_TYPE_CATACOMB_OWNER).is_none() - || !self - .enrol_material - .lock() - .as_ref() - .is_some_and(|material| material.snapshot_identity.is_some()) + || !crate::catacomb::writable(PRIVATE_TYPE_CATACOMB_OWNER) { - dev_err!( - self.dev, - "bio: not deleting: there is no owner Catacomb file or identity-bag binding to save the deletion with; nothing was changed\n" - ); + return Err(EIO); + } + if !orphan && !self.enrol_material.lock().as_ref() + .is_some_and(|material| material.snapshot_identity.is_some()) { + dev_err!(self.dev, "bio: not deleting: indexed identity has no identity-bag binding\n"); + return Err(EIO); + } + if orphan && !matches!(keybag::read(keybag::Slot::Identity), + Ok(keybag::State::Present(_))) { + dev_err!(self.dev, "bio: not deleting: Linux identity bag file is unavailable\n"); return Err(EIO); } @@ -3811,10 +4124,11 @@ impl SepData { /// the enclave marks save-pending (a SAVE_CATACOMB of one that is not has /// been refused), the user before the master because saving the user can /// change the master, state re-read before each, then the lockout and the - /// identity bag, which must match the Catacombs. Safe to repeat after a - /// failure partway: a `retry` accepts a user Catacomb an earlier attempt - /// already saved. - fn persist_deletion(&self, retry: bool) -> Result<()> { + /// identity bag when a verified binding exists. An exact-ID orphan + /// recovery without that binding saves the Linux owner Catacomb instead. + /// Safe to repeat after a failure partway: a `retry` accepts a user + /// Catacomb an earlier attempt already saved. + fn persist_deletion(&self, retry: bool, orphan: bool) -> Result<()> { let Some(user) = crate::sbio::UserId::new(SBIO_PROBE_USER_ID) else { return Err(EIO); }; @@ -3829,6 +4143,11 @@ impl SepData { PRIVATE_TYPE_CATACOMB_MASTER, c"master Catacomb after deletion", ), + ( + crate::sbio::CatacombUser::OWNER, + PRIVATE_TYPE_CATACOMB_OWNER, + c"owner Catacomb after deletion", + ), ] { let Some(reply) = self.read_component_states() else { dev_err!(self.dev, "bio: cannot read component state before saving the {}\n", what); @@ -3844,7 +4163,8 @@ impl SepData { dev_err!(self.dev, "bio: the {} is not active; the deletion cannot be saved\n", what); return Err(EIO); } - if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING != 0 { + if state & crate::sbio::COMPONENT_STATE_SAVE_PENDING != 0 || + (orphan && who.value() == crate::sbio::CatacombUser::OWNER.value()) { if !self.save_catacomb(who, kind, what) { dev_err!(self.dev, "bio: saving the {} failed\n", what); return Err(EIO); @@ -3865,11 +4185,22 @@ impl SepData { dev_err!(self.dev, "bio: lockout persistence failed after the deletion\n"); return Err(EIO); } - if !self.resnapshot_identity_keybag() { - dev_err!( + let bag_bound = self.enrol_material.lock().as_ref() + .is_some_and(|material| material.snapshot_identity.is_some()); + if bag_bound { + if !self.resnapshot_identity_keybag() { + dev_err!( + self.dev, + "bio: Catacombs saved after the deletion but the identity-bag snapshot failed\n" + ); + return Err(EIO); + } + } else if orphan { + dev_info!( self.dev, - "bio: Catacombs saved after the deletion but the identity-bag snapshot failed\n" + "bio: orphan deletion saved via Linux user/master/owner Catacombs without a bag snapshot binding\n" ); + } else { return Err(EIO); } Ok(()) @@ -4678,6 +5009,47 @@ impl<'a> IdentityRecords<'a> { } } +#[cfg(CONFIG_APPLE_SEP_KUNIT_TEST)] +#[::kernel::macros::kunit_tests(apple_sep_enclave_identity_scope)] +mod identity_scope_tests { + use super::*; + + #[test] + fn list_excludes_other_users_and_selectors() { + let mut reply = [0u8; 3 * IDENTITY_RECORD_LEN]; + for (i, (user, selector, byte)) in [ + (SBIO_PROBE_USER_ID, IR_SELECTOR_BUILTIN, 1u8), + (501, IR_SELECTOR_BUILTIN, 2u8), + (SBIO_PROBE_USER_ID, 2, 3u8), + ].iter().enumerate() { + let at = i * IDENTITY_RECORD_LEN; + reply[at..at + 4].copy_from_slice(&user.to_le_bytes()); + reply[at + 4..at + IDENTITY_V1_LEN].fill(*byte); + reply[at + IR_SELECTOR..at + IR_SELECTOR + 4] + .copy_from_slice(&selector.to_le_bytes()); + } + let records = IdentityRecords::new(&reply).unwrap(); + let mut seen = 0; + records.each_built_in_for(SBIO_PROBE_USER_ID, |identity| { + seen += 1; + assert!(*identity.uuid() == [1u8; IDENTITY_UUID_LEN]); + }); + assert_eq!(seen, 1); + } + + #[test] + fn match_result_carries_the_exact_enclave_identity() { + let mut reply = [0u8; SBIO_MATCH_RESULT_LEN]; + reply[MR_USER_ID..MR_USER_ID + 4] + .copy_from_slice(&SBIO_PROBE_USER_ID.to_le_bytes()); + reply[MR_IDENTITY..MR_IDENTITY + IDENTITY_UUID_LEN].fill(0xa5); + let result = MatchResult::parse(&reply).unwrap(); + let user = UserId::new(SBIO_PROBE_USER_ID).unwrap(); + assert!(result.matches(user)); + assert!(*result.identity_uuid() == [0xa5; IDENTITY_UUID_LEN]); + } +} + const FIC_DEVICE_BUILTIN: u32 = 1; pub(crate) const SBIO_FREE_COUNT_LEN: usize = 4 + 20; static_assert!(SBIO_FREE_COUNT_LEN == SBIO_SAVE_SELECTOR_LEN); @@ -4780,6 +5152,10 @@ impl MatchResult { &self.identity } + pub(crate) fn user_id(&self) -> i32 { + self.user_id + } + pub(crate) fn matches(&self, user: UserId) -> bool { self.user_id == user.value() } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 7d9f4f2af1869..5fd41c5bebf60 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -784,6 +784,9 @@ struct SepData { // only under `catacomb_writer`. deletion_unsaved: Atomic, + // PREPARE_ENROL selected the empty Linux context during this boot. + empty_context_prepared: Atomic, + #[pin] rx_work: Work, @@ -983,6 +986,7 @@ impl SepData { worker_gone <- new_condvar!("SepData::worker_gone"), catacomb_writer <- new_mutex!(()), deletion_unsaved: Atomic::new(false), + empty_context_prepared: Atomic::new(false), rx_work <- new_work!("SepData::rx_work"), enrol_work <- new_work!("SepData::enrol_work"), verify_work <- new_work!("SepData::verify_work"), diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 99d2661e2cb1c..ca0d3d8a797ee 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -44,6 +44,7 @@ void sep_hwrng_unregister(void *mem); void *sep_store_open(const char *path); void *sep_store_open_host(const char *path, bool *created); void *sep_store_open_trunc(const char *path); +void *sep_store_open_writable_existing(const char *path); void *sep_store_open_ro(const char *path); void *sep_store_open_block(const char *path, int writable); void sep_store_close(void *handle); diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index 8e30c3f54b817..6d546a5603f37 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -9,6 +9,7 @@ extern "C" { fn sep_store_open(path: *const c_char) -> *mut c_void; fn sep_store_open_host(path: *const c_char, created: *mut bool) -> *mut c_void; fn sep_store_open_trunc(path: *const c_char) -> *mut c_void; + fn sep_store_open_writable_existing(path: *const c_char) -> *mut c_void; fn sep_store_open_ro(path: *const c_char) -> *mut c_void; fn sep_store_open_block(path: *const c_char, writable: c_int) -> *mut c_void; fn sep_store_close(handle: *mut c_void); @@ -80,6 +81,15 @@ impl StoreFile { Ok(StoreFile { handle }) } + pub(crate) fn open_writable_existing(path: &CStr) -> Result { + // SAFETY: `path` is NUL-terminated; this open cannot create or truncate. + let handle = unsafe { sep_store_open_writable_existing(path.as_char_ptr()) }; + if handle.is_null() { + return Err(EROFS); + } + Ok(StoreFile { handle }) + } + pub(crate) fn open_readonly(path: &CStr) -> Result { // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. let handle = unsafe { sep_store_open_ro(path.as_char_ptr()) }; diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index e3abe11c7d5c8..d9930332cab99 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -263,6 +263,9 @@ impl Store { pr_info!("created a new host-state store\n"); return Ok(store); } + if size < STORE_SIZE as u64 { + return Err(store.refuse(Damage::Size)); + } let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; @@ -359,6 +362,12 @@ impl Store { self.slots.iter().position(|s| !s.used) } + /// Admission check for the copy-on-write slot used by the next write. + /// Even replacing an existing key needs a free slot until commit. + pub(crate) fn can_write(&self, key: &Key, len: usize) -> bool { + valid_kind(key.kind) && len <= MAX_VALUE && self.find_free().is_some() + } + fn block_offset(slot: usize) -> u64 { ((slot + 1) * BLOCK_SIZE) as u64 } diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c index d50270d717be5..77319a4702164 100644 --- a/drivers/soc/apple/store_shim.c +++ b/drivers/soc/apple/store_shim.c @@ -83,6 +83,14 @@ void *sep_store_open_trunc(const char *path) return IS_ERR(f) ? NULL : f; } +/* Admission check only: never create or truncate an existing record. */ +void *sep_store_open_writable_existing(const char *path) +{ + struct file *f = open_as_kernel(path, O_WRONLY | O_LARGEFILE, 0); + + return IS_ERR(f) ? NULL : f; +} + /* * Opens an existing file read-only, NULL if absent. O_RDONLY and no O_CREAT so * the seed cannot be created, truncated or written by mistake. diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index ad875e80d8ba2..dbaf0d6949249 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -146,22 +146,23 @@ config SPI_APPLE_QSPI help This enables support for Apple's command-FIFO QSPIMC controller. Unlike the ordinary Apple SPIMC block, QSPIMC accepts packed SPI - memory descriptors and is used for the boot NOR on T8140 systems. + memory descriptors and is used for the boot NOR on M3 and later + systems. - T8122/T8132 controllers default to read-only access. Only the pinned - J613 NVRAM layout admits bounded writes after a runtime opt-in. - On T8140 only the nvram partition accepts programs and erases; the - rest of the boot NOR is read-only. + The boot NOR defaults to read-only access. On the J613, J713, J616s + and J700 a runtime opt-in admits bounded, verified page programs and + 4 KiB erases inside the nvram partition only; the rest of the boot + NOR stays read-only. config SPI_APPLE_QSPI_KUNIT_TEST - tristate "KUnit tests for the Apple QSPIMC boot NOR write window" if !KUNIT_ALL_TESTS + tristate "KUnit tests for the Apple QSPIMC NVRAM write policy" if !KUNIT_ALL_TESTS depends on KUNIT depends on SPI_APPLE_QSPI || COMPILE_TEST default KUNIT_ALL_TESTS help - Tests which T8140 boot NOR requests the QSPIMC driver admits: write - enable control, and programs and erases inside the nvram partition - only. No hardware is accessed. + Tests which boot NOR requests the QSPIMC driver admits: write + enable control, and page programs and 4 KiB erases inside the + nvram partition only. No hardware is accessed. config SPI_AR934X tristate "Qualcomm Atheros AR934X/QCA95XX SPI controller driver" diff --git a/drivers/spi/spi-apple-qspi-policy-test.c b/drivers/spi/spi-apple-qspi-policy-test.c index 9ec58d587c60f..f6dc555057443 100644 --- a/drivers/spi/spi-apple-qspi-policy-test.c +++ b/drivers/spi/spi-apple-qspi-policy-test.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* - * KUnit tests for the T8140 boot NOR write window of the Apple QSPIMC driver. + * KUnit tests for the Apple QSPIMC NVRAM mutation policy, at the J700 and + * J613/J713/J616s nvram partitions. */ #include @@ -14,59 +15,57 @@ static int nv(unsigned int opcode, unsigned int addr_bytes, unsigned long long addr, unsigned int len, int data_out) { - return apple_qspi_window_allowed(opcode, addr_bytes, addr, len, - data_out, NV_START, NV_END); + return apple_qspi_mutation_allowed(opcode, addr_bytes, addr, len, + data_out, NV_START, NV_END); } -static void qspi_window_write_enable(struct kunit *test) +static void qspi_policy_write_enable(struct kunit *test) { KUNIT_EXPECT_TRUE(test, nv(0x06, 0, 0, 0, 0)); KUNIT_EXPECT_TRUE(test, nv(0x04, 0, 0, 0, 0)); KUNIT_EXPECT_FALSE(test, nv(0x06, 3, NV_START, 0, 0)); KUNIT_EXPECT_FALSE(test, nv(0x06, 0, 0, 1, 1)); - /* Write-enable control needs no window. */ - KUNIT_EXPECT_TRUE(test, apple_qspi_window_allowed(0x06, 0, 0, 0, 0, 0, 0)); + /* Write-enable control is virtual and needs no window. */ + KUNIT_EXPECT_TRUE(test, apple_qspi_mutation_allowed(0x06, 0, 0, 0, 0, 0, 0)); } -static void qspi_window_program(struct kunit *test) +static void qspi_policy_program(struct kunit *test) { KUNIT_EXPECT_TRUE(test, nv(0x02, 3, NV_START, 256, 1)); KUNIT_EXPECT_TRUE(test, nv(0x02, 3, NV_START + 0x1234, 1, 1)); KUNIT_EXPECT_TRUE(test, nv(0x02, 3, NV_END - 256, 256, 1)); - KUNIT_EXPECT_TRUE(test, nv(0x12, 4, NV_START, 256, 1)); - /* Any byte outside, or a wrap within a page outside, is refused. */ + /* Any byte outside the partition is refused. */ KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START - 256, 256, 1)); KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START - 1, 2, 1)); KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_END, 1, 1)); KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_END - 1, 2, 1)); KUNIT_EXPECT_FALSE(test, nv(0x02, 3, 0, 256, 1)); - /* A program needs data out, and at most one 4 KiB block's worth. */ + /* A program stays within one page and needs data out. */ + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START + 0xff, 2, 1)); + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START, 257, 1)); KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START, 0, 1)); KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START, 256, 0)); - KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START, 4097, 1)); + /* Four-byte programs are not admitted. */ + KUNIT_EXPECT_FALSE(test, nv(0x12, 4, NV_START, 256, 1)); } -static void qspi_window_erase(struct kunit *test) +static void qspi_policy_erase(struct kunit *test) { KUNIT_EXPECT_TRUE(test, nv(0x20, 3, NV_START, 0, 0)); - KUNIT_EXPECT_TRUE(test, nv(0x20, 3, NV_START + 0x123, 0, 0)); - KUNIT_EXPECT_TRUE(test, nv(0x20, 3, NV_END - 1, 0, 0)); - KUNIT_EXPECT_TRUE(test, nv(0x21, 4, NV_START, 0, 0)); - KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_START - 1, 0, 0)); + KUNIT_EXPECT_TRUE(test, nv(0x20, 3, NV_END - 0x1000, 0, 0)); + /* Only aligned 4 KiB sector erases inside the partition. */ + KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_START + 0x123, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_START - 0x1000, 0, 0)); KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_END, 0, 0)); - /* 32 and 64 KiB erases cover the whole aligned block. */ - KUNIT_EXPECT_TRUE(test, nv(0x52, 3, 0x28000, 0, 0)); - KUNIT_EXPECT_FALSE(test, nv(0x52, 3, NV_START, 0, 0)); - KUNIT_EXPECT_TRUE(test, nv(0xd8, 3, 0x30000, 0, 0)); - KUNIT_EXPECT_TRUE(test, nv(0xdc, 4, 0x1e0000, 0, 0)); - KUNIT_EXPECT_FALSE(test, nv(0xd8, 3, NV_START, 0, 0)); - KUNIT_EXPECT_FALSE(test, nv(0xd8, 3, 0x1f0000, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x21, 4, NV_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x52, 3, 0x28000, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0xd8, 3, 0x30000, 0, 0)); /* An erase carries no data. */ KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_START, 1, 0)); KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_START, 0, 1)); } -static void qspi_window_refused_commands(struct kunit *test) +static void qspi_policy_refused_commands(struct kunit *test) { static const u8 refused[] = { 0x01, 0x31, 0x11, /* status register writes */ @@ -86,48 +85,114 @@ static void qspi_window_refused_commands(struct kunit *test) } } -static void qspi_window_address_width(struct kunit *test) +static void qspi_policy_address_width(struct kunit *test) { - /* Three address bytes cannot carry an address above 16 MiB. */ - KUNIT_EXPECT_FALSE(test, nv(0x02, 3, (1ULL << 24) + NV_START, 1, 1)); - KUNIT_EXPECT_FALSE(test, nv(0x20, 3, (1ULL << 24) + NV_START, 0, 0)); - KUNIT_EXPECT_FALSE(test, nv(0x12, 4, (1ULL << 32) + NV_START, 1, 1)); KUNIT_EXPECT_FALSE(test, nv(0x02, 2, NV_START, 1, 1)); + KUNIT_EXPECT_FALSE(test, nv(0x02, 4, NV_START, 1, 1)); KUNIT_EXPECT_FALSE(test, nv(0x20, 0, NV_START, 0, 0)); - KUNIT_EXPECT_FALSE(test, nv(0x20, 5, NV_START, 0, 0)); - /* A window that reaches the top of 24-bit addressing. */ - KUNIT_EXPECT_TRUE(test, apple_qspi_window_allowed(0x20, 3, 0xfff000, 0, 0, - 0xff0000, 1ULL << 24)); - /* A program that crosses 16 MiB cannot use three address bytes. */ - KUNIT_EXPECT_FALSE(test, apple_qspi_window_allowed(0x02, 3, 0xffffff, 2, 1, - 0xff0000, 1ULL << 25)); - KUNIT_EXPECT_TRUE(test, apple_qspi_window_allowed(0x12, 4, 0xffffff, 2, 1, - 0xff0000, 1ULL << 25)); + KUNIT_EXPECT_FALSE(test, nv(0x20, 4, NV_START, 0, 0)); + /* Addresses are not truncated to the wire width before the check. */ + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, (1ULL << 24) + NV_START, 1, 1)); + KUNIT_EXPECT_FALSE(test, nv(0x20, 3, (1ULL << 24) + NV_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, ~0ULL, 1, 1)); +} + +static void qspi_policy_empty(struct kunit *test) +{ + KUNIT_EXPECT_FALSE(test, apple_qspi_mutation_allowed(0x02, 3, 0, 1, 1, 0, 0)); + KUNIT_EXPECT_FALSE(test, apple_qspi_mutation_allowed(0x20, 3, 0, 0, 0, 0, 0)); + KUNIT_EXPECT_FALSE(test, apple_qspi_mutation_allowed(0x20, 3, NV_START, 0, 0, + NV_START, NV_START)); +} + +/* The J613, J713 and J616s nvram partition: 0x700000, 1 MiB. */ +#define NV7_START 0x700000ULL +#define NV7_END 0x800000ULL + +static int nv7(unsigned int opcode, unsigned int addr_bytes, + unsigned long long addr, unsigned int len, int data_out) +{ + return apple_qspi_mutation_allowed(opcode, addr_bytes, addr, len, + data_out, NV7_START, NV7_END); +} + +static void qspi_policy_nvram_700000(struct kunit *test) +{ + /* First and last page and sector of the partition. */ + KUNIT_EXPECT_TRUE(test, nv7(0x02, 3, NV7_START, 256, 1)); + KUNIT_EXPECT_TRUE(test, nv7(0x02, 3, NV7_END - 256, 256, 1)); + KUNIT_EXPECT_TRUE(test, nv7(0x20, 3, NV7_START, 0, 0)); + KUNIT_EXPECT_TRUE(test, nv7(0x20, 3, NV7_END - 0x1000, 0, 0)); + /* syscfg ends at 0x700000 and the NOR is 8 MiB: both stay out. */ + KUNIT_EXPECT_FALSE(test, nv7(0x02, 3, NV7_START - 1, 1, 1)); + KUNIT_EXPECT_FALSE(test, nv7(0x02, 3, NV7_START - 256, 256, 1)); + KUNIT_EXPECT_FALSE(test, nv7(0x20, 3, NV7_START - 0x1000, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv7(0x02, 3, NV7_END, 1, 1)); + KUNIT_EXPECT_FALSE(test, nv7(0x02, 3, NV7_END - 1, 2, 1)); + KUNIT_EXPECT_FALSE(test, nv7(0x20, 3, NV7_END, 0, 0)); + /* iBoot at 0 and the J700 window are outside this partition. */ + KUNIT_EXPECT_FALSE(test, nv7(0x02, 3, 0, 256, 1)); + KUNIT_EXPECT_FALSE(test, nv7(0x20, 3, NV_START, 0, 0)); + /* Larger erases and four-byte forms are still refused here. */ + KUNIT_EXPECT_FALSE(test, nv7(0x52, 3, NV7_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv7(0xd8, 3, NV7_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv7(0x21, 4, NV7_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv7(0x12, 4, NV7_START, 256, 1)); + /* An address above 16 MiB is not aliased onto the partition. */ + KUNIT_EXPECT_FALSE(test, nv7(0x02, 3, (1ULL << 24) + NV7_START, 1, 1)); + KUNIT_EXPECT_FALSE(test, nv7(0x20, 3, (1ULL << 24) + NV7_START, 0, 0)); +} + +static void qspi_policy_status(struct kunit *test) +{ + /* BUSY and WEL are never admitted. */ + KUNIT_EXPECT_FALSE(test, apple_qspi_status_allows_write(0x01, 0, 0, 0x1c)); + KUNIT_EXPECT_FALSE(test, apple_qspi_status_allows_write(0x02, 0, 0, 0x1c)); + /* Block protect only where the board allows it. */ + KUNIT_EXPECT_FALSE(test, apple_qspi_status_allows_write(0x04, 0, 0, 0)); + KUNIT_EXPECT_TRUE(test, apple_qspi_status_allows_write(0x04, 0x02, 0, 0x1c)); + /* CMP, SUS and WPS are refused everywhere. */ + KUNIT_EXPECT_FALSE(test, apple_qspi_status_allows_write(0, 0x40, 0, 0x1c)); + KUNIT_EXPECT_FALSE(test, apple_qspi_status_allows_write(0, 0x80, 0, 0x1c)); + KUNIT_EXPECT_FALSE(test, apple_qspi_status_allows_write(0, 0, 0x04, 0x1c)); } -static void qspi_window_empty(struct kunit *test) +static void qspi_policy_mx_status(struct kunit *test) { - KUNIT_EXPECT_FALSE(test, apple_qspi_window_allowed(0x02, 3, 0, 1, 1, 0, 0)); - KUNIT_EXPECT_FALSE(test, apple_qspi_window_allowed(0x20, 3, 0, 0, 0, 0, 0)); - KUNIT_EXPECT_FALSE(test, apple_qspi_window_allowed(0x20, 3, NV_START, 0, 0, - NV_START, NV_START)); + /* The J616s Macronix part: no block protection is tolerated. */ + KUNIT_EXPECT_TRUE(test, apple_qspi_mx_status_allows_write(0x00, 0)); + /* QE and SRWD stay as found and do not block a write. */ + KUNIT_EXPECT_TRUE(test, apple_qspi_mx_status_allows_write(0x40, 0)); + KUNIT_EXPECT_TRUE(test, apple_qspi_mx_status_allows_write(0x80, 0)); + /* WIP and WEL are never admitted. */ + KUNIT_EXPECT_FALSE(test, apple_qspi_mx_status_allows_write(0x01, 0x3c)); + KUNIT_EXPECT_FALSE(test, apple_qspi_mx_status_allows_write(0x02, 0x3c)); + /* Each of BP0 to BP3, including BP3 at bit 5, which Winbond calls TB. */ + KUNIT_EXPECT_FALSE(test, apple_qspi_mx_status_allows_write(0x04, 0)); + KUNIT_EXPECT_FALSE(test, apple_qspi_mx_status_allows_write(0x08, 0)); + KUNIT_EXPECT_FALSE(test, apple_qspi_mx_status_allows_write(0x10, 0)); + KUNIT_EXPECT_FALSE(test, apple_qspi_mx_status_allows_write(0x20, 0)); + KUNIT_EXPECT_TRUE(test, apple_qspi_mx_status_allows_write(0x3c, 0x3c)); } -static struct kunit_case apple_qspi_window_cases[] = { - KUNIT_CASE(qspi_window_write_enable), - KUNIT_CASE(qspi_window_program), - KUNIT_CASE(qspi_window_erase), - KUNIT_CASE(qspi_window_refused_commands), - KUNIT_CASE(qspi_window_address_width), - KUNIT_CASE(qspi_window_empty), +static struct kunit_case apple_qspi_policy_cases[] = { + KUNIT_CASE(qspi_policy_write_enable), + KUNIT_CASE(qspi_policy_program), + KUNIT_CASE(qspi_policy_erase), + KUNIT_CASE(qspi_policy_refused_commands), + KUNIT_CASE(qspi_policy_address_width), + KUNIT_CASE(qspi_policy_empty), + KUNIT_CASE(qspi_policy_nvram_700000), + KUNIT_CASE(qspi_policy_status), + KUNIT_CASE(qspi_policy_mx_status), {} }; -static struct kunit_suite apple_qspi_window_suite = { - .name = "apple-qspi-nvram-window", - .test_cases = apple_qspi_window_cases, +static struct kunit_suite apple_qspi_policy_suite = { + .name = "apple-qspi-nvram-policy", + .test_cases = apple_qspi_policy_cases, }; -kunit_test_suite(apple_qspi_window_suite); +kunit_test_suite(apple_qspi_policy_suite); -MODULE_DESCRIPTION("KUnit tests for the Apple QSPIMC boot NOR write window"); +MODULE_DESCRIPTION("KUnit tests for the Apple QSPIMC NVRAM mutation policy"); MODULE_LICENSE("GPL"); diff --git a/drivers/spi/spi-apple-qspi-policy.h b/drivers/spi/spi-apple-qspi-policy.h index 5d788d69f4f8b..5acd9a62d5291 100644 --- a/drivers/spi/spi-apple-qspi-policy.h +++ b/drivers/spi/spi-apple-qspi-policy.h @@ -2,21 +2,33 @@ #ifndef APPLE_QSPI_POLICY_H #define APPLE_QSPI_POLICY_H -#define APPLE_QSPI_NVRAM_START 0x700000ULL -#define APPLE_QSPI_NVRAM_END 0x800000ULL #define APPLE_QSPI_PAGE_SIZE 256U #define APPLE_QSPI_ERASE_SIZE 4096U -/* Subtraction avoids overflow; addr is not truncated to wire width first. */ -static inline int apple_qspi_range_allowed(unsigned long long addr, unsigned int len) +/* + * [start, end) is the board's nvram partition. Subtraction avoids overflow; + * addr is not truncated to wire width first. + */ +static inline int apple_qspi_range_allowed(unsigned long long addr, + unsigned int len, unsigned long long start, + unsigned long long end) { - return len && addr >= APPLE_QSPI_NVRAM_START && - addr < APPLE_QSPI_NVRAM_END && len <= APPLE_QSPI_NVRAM_END - addr; + return len && start < end && addr >= start && addr < end && + len <= end - addr; } +/* + * May a write-path request reach the boot NOR? Write enable control is + * virtual and admitted. A single-wire, three-byte page program that stays + * within one 256-byte page, or an aligned 4 KiB sector erase, is admitted + * only when every byte it can change lies inside [start, end). Everything + * else (status and configuration writes, larger erases, chip erase, + * address-mode, reset and OTP commands) is refused. + */ static inline int apple_qspi_mutation_allowed(unsigned int opcode, unsigned int addr_bytes, unsigned long long addr, - unsigned int len, int data_out) + unsigned int len, int data_out, + unsigned long long start, unsigned long long end) { switch (opcode) { case 0x06: /* Virtual WREN, never sent independently. */ @@ -24,76 +36,44 @@ static inline int apple_qspi_mutation_allowed(unsigned int opcode, return !addr_bytes && !len && !data_out; case 0x02: /* PP: prevent page wrap, even if SPI-NOR gets geometry wrong. */ return addr_bytes == 3 && data_out && len <= APPLE_QSPI_PAGE_SIZE && - apple_qspi_range_allowed(addr, len) && + apple_qspi_range_allowed(addr, len, start, end) && len <= APPLE_QSPI_PAGE_SIZE - (addr & (APPLE_QSPI_PAGE_SIZE - 1)); case 0x20: /* SE: hard-code the entire affected 4 KiB range. */ return addr_bytes == 3 && !len && !data_out && !(addr & (APPLE_QSPI_ERASE_SIZE - 1)) && - apple_qspi_range_allowed(addr, APPLE_QSPI_ERASE_SIZE); + apple_qspi_range_allowed(addr, APPLE_QSPI_ERASE_SIZE, + start, end); default: return 0; } } /* - * T8140 boot NOR: may a request that does not read data reach the flash? - * Write-enable control is admitted. A page program, or a 4, 32 or 64 KiB - * erase, is admitted only when every byte it can change lies inside - * [start, end), the nvram partition. A program counts the whole 4 KiB - * blocks it touches, which covers a write that wraps within its page. Status - * and configuration register writes, chip erase, address-mode, reset and OTP - * commands are refused. The address must fit in addr_bytes: the controller - * sends only those bytes, so a wider value would alias a lower address. + * Protection and activity bits must permit a write; JEDEC ID is checked + * separately. BUSY/WEL, CMP/SUS and WPS are always refused. bp_ok names the + * SR1 block-protect bits a board's firmware leaves set at boot: they stay + * unchanged, the flash ignores programs and erases in the range they protect, + * and the driver's readback then latches it read-only. SRP, TB, SEC, QE and + * LB stay unchanged. */ -static inline int apple_qspi_window_allowed(unsigned int opcode, - unsigned int addr_bytes, unsigned long long addr, - unsigned int len, int data_out, - unsigned long long start, unsigned long long end) +static inline int apple_qspi_status_allows_write(unsigned int sr1, + unsigned int sr2, unsigned int sr3, unsigned int bp_ok) { - unsigned long long block, first, last; - - switch (opcode) { - case 0x06: /* WREN */ - case 0x04: /* WRDI */ - return !addr_bytes && !len && !data_out; - case 0x02: /* PP */ - case 0x12: /* PP_4B */ - if (!data_out || !len || len > APPLE_QSPI_ERASE_SIZE) - return 0; - block = APPLE_QSPI_ERASE_SIZE; - break; - case 0x20: /* BE_4K */ - case 0x21: /* BE_4K_4B */ - block = 0x1000; - break; - case 0x52: /* BE_32K */ - case 0x5c: /* BE_32K_4B */ - block = 0x8000; - break; - case 0xd8: /* SE */ - case 0xdc: /* SE_4B */ - block = 0x10000; - break; - default: - return 0; - } - if (opcode != 0x02 && opcode != 0x12 && (len || data_out)) - return 0; - if ((addr_bytes != 3 && addr_bytes != 4) || addr >> (8 * addr_bytes)) - return 0; - - /* [first, last) are the blocks the flash can change. */ - first = addr & ~(block - 1); - last = ((addr + (len ? len : 1) - 1) | (block - 1)) + 1; - return last <= 1ULL << (8 * addr_bytes) && - start <= first && last <= end; + return !(sr1 & 0x03) && !(sr1 & 0x1c & ~bp_ok) && + !(sr2 & 0xc0) && !(sr3 & 0x04); } -/* Protection and activity bits must permit a write; JEDEC ID is checked separately. */ -static inline int apple_qspi_status_allows_write(unsigned int sr1, - unsigned int sr2, unsigned int sr3) +/* JEDEC manufacturer ID of Macronix parts. */ +#define APPLE_QSPI_MFR_MACRONIX 0xc2 + +/* + * The Macronix status register holds WIP, WEL and four block-protect bits, + * BP[3:0] at bits 5:2. BUSY and WEL are always refused, and block-protect + * bits only where bp_ok allows them. QE and SRWD stay unchanged. + */ +static inline int apple_qspi_mx_status_allows_write(unsigned int sr, + unsigned int bp_ok) { - /* BUSY/WEL, BP[2:0]; CMP/SUS; WPS. SRP, TB, SEC, QE, LB stay unchanged. */ - return !(sr1 & 0x1f) && !(sr2 & 0xc0) && !(sr3 & 0x04); + return !(sr & 0x03) && !(sr & 0x3c & ~bp_ok); } #endif diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index c594a241dcef8..601929b85f1fd 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -7,10 +7,10 @@ * command FIFO. This driver intentionally implements the PIO spi-mem path * only; that is sufficient for SPI NOR discovery and access. * - * On T8140 reads reach the whole boot NOR, but programs and erases are - * admitted only inside its unique, aligned nvram partition. Without that - * partition the NOR is read-only. - * + * The boot NOR starts read-only. On the qualified boards (J613, J713, + * J616s and the J700) root may opt in at runtime to bounded, verified + * programs and 4 KiB erases inside the nvram partition; everything else + * stays read-only. On T8140 data reads reach the whole NOR. */ #include @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -87,6 +88,47 @@ #define APPLE_QSPI_MAX_DATA 4096 #define APPLE_QSPI_MAX_HZ 40000000 #define APPLE_QSPI_TIMEOUT_MS 1000 +#define APPLE_QSPI_AUTOSUSPEND_MS 100 + +/* Per-SoC controller differences, from the compatible. */ +struct apple_qspi_soc { + /* T8140: data reads of any shape pass straight to the controller. */ + bool multi_io_reads; + bool runtime_pm; +}; + +/* + * A board whose NVRAM may be written: the controller compatible, the + * machine, the controller aperture, the nvram partition the device tree + * must describe, the boot NOR JEDEC ID and the SR1 block-protect bits its + * firmware leaves set at boot. + */ +struct apple_qspi_board { + const char *controller; + const char *machine; + resource_size_t base; + u32 nvram_start; + u32 nvram_size; + u8 jedec[3]; + u8 bp_ok; +}; + +#define APPLE_QSPI_JEDEC_W25Q64 { 0xef, 0x65, 0x17 } +#define APPLE_QSPI_JEDEC_W25Q16 { 0xef, 0x65, 0x15 } +#define APPLE_QSPI_JEDEC_MX_2B27 { 0xc2, 0x2b, 0x27 } + +static const struct apple_qspi_board apple_qspi_boards[] = { + { "apple,t8122-qspi", "apple,j613", 0x2a1118000ULL, + 0x700000, 0x100000, APPLE_QSPI_JEDEC_W25Q64, 0 }, + { "apple,t8132-qspi", "apple,j713", 0x3ad118000ULL, + 0x700000, 0x100000, APPLE_QSPI_JEDEC_W25Q64, 0 }, + /* Macronix; the ID macOS reports for the J616s boot NOR. */ + { "apple,t6041-qspi", "apple,j616s", 0x429118000ULL, + 0x700000, 0x100000, APPLE_QSPI_JEDEC_MX_2B27, 0 }, + /* iBoot leaves BP0 set on the J700, over the top 64 KiB. */ + { "apple,t8140-qspi", "apple,j700", 0x385110000ULL, + 0x24000, 0x1d0000, APPLE_QSPI_JEDEC_W25Q16, 0x1c }, +}; static DEFINE_MUTEX(apple_qspi_policy_lock); static bool read_only = true; @@ -125,7 +167,7 @@ static const struct kernel_param_ops apple_qspi_read_only_ops = { .get = apple_qspi_get_read_only, }; module_param_cb(read_only, &apple_qspi_read_only_ops, &read_only, 0644); -MODULE_PARM_DESC(read_only, "Default true; root runtime opt-in for bounded J613/J713 NVRAM writes"); +MODULE_PARM_DESC(read_only, "Default true; root runtime opt-in for bounded NVRAM writes on qualified boards"); static bool apple_qspi_opcode_is_read(u8 opcode) { @@ -149,12 +191,16 @@ static bool apple_qspi_opcode_is_read(u8 opcode) } struct apple_qspi { - /* T8122/T8132 use the serialized, verified NVRAM policy. */ + /* Every write-path request goes through the verified NVRAM policy. */ bool nvram_policy; - /* T8140 admits programs and erases only inside nvram. */ + /* T8140: data reads of any shape pass straight to the controller. */ bool nvram_window; + /* The nvram partition [window_start, window_end). */ u64 window_start; u64 window_end; + /* Expected JEDEC ID and the SR1 block-protect bits left by firmware. */ + u8 jedec[3]; + u8 bp_ok; bool nvram_write_candidate; bool write_fault; void __iomem *base; @@ -596,7 +642,7 @@ static int apple_qspi_transfer_op(struct spi_mem *mem, return ret; } -/* Private transfers used by the serialized J613 write policy. */ +/* Private transfers used by the serialized NVRAM write policy. */ static int apple_qspi_read_register(struct spi_mem *mem, u8 opcode, u8 *buf, u32 len) { struct spi_mem_op op = SPI_MEM_OP(SPI_MEM_OP_CMD(opcode, 1), @@ -623,12 +669,24 @@ static int apple_qspi_read_array(struct spi_mem *mem, u64 addr, u8 *buf, u32 len return apple_qspi_transfer_op(mem, &op); } -static int apple_qspi_status(struct spi_mem *mem, u8 sr[3]) +/* + * Winbond parts have three status registers (05h, 35h, 15h). On Macronix + * parts 35h is EQIO, which switches the flash to QPI mode, so only the + * status (05h) and configuration (15h) registers are read there. + */ +static int apple_qspi_status(struct spi_mem *mem, u8 mfr, u8 sr[3]) { - static const u8 opcode[] = { 0x05, 0x35, 0x15 }; - int i, ret; - - for (i = 0; i < 3; i++) { + static const u8 winbond[] = { 0x05, 0x35, 0x15 }; + static const u8 macronix[] = { 0x05, 0x15 }; + const u8 *opcode = winbond; + int i, n = ARRAY_SIZE(winbond), ret; + + if (mfr == APPLE_QSPI_MFR_MACRONIX) { + opcode = macronix; + n = ARRAY_SIZE(macronix); + } + sr[2] = 0; + for (i = 0; i < n; i++) { ret = apple_qspi_read_register(mem, opcode[i], &sr[i], 1); if (ret) return ret; @@ -652,15 +710,18 @@ static int apple_qspi_nvram_modify(struct spi_mem *mem, const struct spi_mem_op ret = apple_qspi_read_register(mem, 0x9f, id, sizeof(id)); if (ret) goto out; - if (id[0] != 0xef || id[1] != 0x65 || id[2] != 0x17) { + if (memcmp(id, qspi->jedec, sizeof(id))) { ret = -ENODEV; goto out; } - ret = apple_qspi_status(mem, before); + ret = apple_qspi_status(mem, id[0], before); if (ret) goto out; /* No unlocking. Conservatively refuse BP/CMP/WPS, BUSY/WEL or suspend. */ - if (!apple_qspi_status_allows_write(before[0], before[1], before[2])) { + if (id[0] == APPLE_QSPI_MFR_MACRONIX ? + !apple_qspi_mx_status_allows_write(before[0], qspi->bp_ok) : + !apple_qspi_status_allows_write(before[0], before[1], before[2], + qspi->bp_ok)) { ret = -EACCES; goto out; } @@ -713,7 +774,7 @@ static int apple_qspi_nvram_modify(struct spi_mem *mem, const struct spi_mem_op ret = apple_qspi_command(mem, 0x04); if (ret) goto fault; - ret = apple_qspi_status(mem, after); + ret = apple_qspi_status(mem, id[0], after); if (ret) goto fault; if (memcmp(before, after, sizeof(before))) { @@ -762,6 +823,12 @@ static int apple_qspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) if (!apple_qspi_supports_op(mem, op) || apple_qspi_shape_op(op, &shape)) return -EOPNOTSUPP; mutex_lock(&apple_qspi_policy_lock); + /* 35h is EQIO, not a register read, on Macronix parts. */ + if (op->cmd.opcode == 0x35 && + qspi->jedec[0] == APPLE_QSPI_MFR_MACRONIX) { + ret = -EOPNOTSUPP; + goto out; + } if (op->data.dir == SPI_MEM_DATA_IN && op->data.nbytes && apple_qspi_opcode_is_read(op->cmd.opcode)) { ret = apple_qspi_transfer_op(mem, op); @@ -780,7 +847,8 @@ static int apple_qspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) } if (!apple_qspi_mutation_allowed(op->cmd.opcode, op->addr.nbytes, op->addr.val, op->data.nbytes, - op->data.dir == SPI_MEM_DATA_OUT)) { + op->data.dir == SPI_MEM_DATA_OUT, + qspi->window_start, qspi->window_end)) { denial = "mutation-policy"; goto out; } @@ -802,39 +870,23 @@ static int apple_qspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) return ret; } -/* T8140: data reads pass; anything else must stay inside the nvram window. */ -static int apple_qspi_window_exec_op(struct spi_mem *mem, - const struct spi_mem_op *op) -{ - struct apple_qspi *qspi = spi_controller_get_devdata(mem->spi->controller); - - if (op->data.dir == SPI_MEM_DATA_IN && op->data.nbytes) - return apple_qspi_transfer_op(mem, op); - if (!op->dummy.nbytes && - apple_qspi_window_allowed(op->cmd.opcode, op->addr.nbytes, - op->addr.val, op->data.nbytes, - op->data.dir == SPI_MEM_DATA_OUT, - qspi->window_start, qspi->window_end)) - return apple_qspi_transfer_op(mem, op); - - dev_warn_ratelimited(&mem->spi->dev, - "boot NOR request denied: opcode=%02x addr=%#llx addr_bytes=%u len=%u dir=%u nvram=[%#llx, %#llx)\n", - op->cmd.opcode, (unsigned long long)op->addr.val, - op->addr.nbytes, op->data.nbytes, op->data.dir, - (unsigned long long)qspi->window_start, - (unsigned long long)qspi->window_end); - return -EACCES; -} - static int apple_qspi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) { - struct apple_qspi *qspi = spi_controller_get_devdata(mem->spi->controller); + struct spi_controller *ctlr = mem->spi->controller; + struct apple_qspi *qspi = spi_controller_get_devdata(ctlr); + int ret; - if (qspi->nvram_window) - return apple_qspi_window_exec_op(mem, op); - if (!qspi->nvram_policy) - return apple_qspi_transfer_op(mem, op); - return apple_qspi_exec_op(mem, op); + /* T8140 keeps its multi-line data reads; spi-mem serializes them. */ + if (!qspi->nvram_policy || (qspi->nvram_window && + op->data.dir == SPI_MEM_DATA_IN && + op->data.nbytes)) + ret = apple_qspi_transfer_op(mem, op); + else + ret = apple_qspi_exec_op(mem, op); + /* spi-mem drops its runtime reference with pm_runtime_put(). */ + if (ctlr->auto_runtime_pm) + pm_runtime_mark_last_busy(ctlr->dev.parent); + return ret; } static const struct spi_controller_mem_ops apple_qspi_mem_ops = { @@ -844,57 +896,26 @@ static const struct spi_controller_mem_ops apple_qspi_mem_ops = { }; /* Keep write qualification tied to the board and its controller aperture. */ -static bool apple_qspi_nvram_write_board(struct device_node *node, - resource_size_t base) +static const struct apple_qspi_board * +apple_qspi_nvram_write_board(struct device_node *node, resource_size_t base) { - if (of_device_is_compatible(node, "apple,t8122-qspi")) - return of_machine_is_compatible("apple,j613") && - base == 0x2a1118000ULL; - if (of_device_is_compatible(node, "apple,t8132-qspi")) - return of_machine_is_compatible("apple,j713") && - base == 0x3ad118000ULL; - return false; -} - -/* Require the pinned partition and single physical chip select. */ -static bool apple_qspi_nvram_layout(struct device_node *node) -{ - struct device_node *flash, *parts, *part; - u32 reg[2], cs; - const char *label; - bool found = false; - - /* OF name matching excludes the unit address (@0); reg pins CS0. */ - flash = of_get_child_by_name(node, "flash"); - if (!flash) - return false; - if (!of_device_is_compatible(flash, "jedec,spi-nor") || - of_property_read_u32(flash, "reg", &cs) || cs) { - of_node_put(flash); - return false; - } - parts = of_get_child_by_name(flash, "partitions"); - of_node_put(flash); - if (!parts) - return false; - if (of_device_is_compatible(parts, "fixed-partitions")) { - for_each_available_child_of_node(parts, part) { - if (!of_property_read_string(part, "label", &label) && - !strcmp(label, "nvram") && - of_property_count_u32_elems(part, "reg") == 2 && - !of_property_read_u32_array(part, "reg", reg, 2) && - reg[0] == APPLE_QSPI_NVRAM_START && - reg[1] == APPLE_QSPI_NVRAM_END - APPLE_QSPI_NVRAM_START) - found = true; - } + const struct apple_qspi_board *board; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(apple_qspi_boards); i++) { + board = &apple_qspi_boards[i]; + if (of_device_is_compatible(node, board->controller) && + of_machine_is_compatible(board->machine) && + base == board->base) + return board; } - of_node_put(parts); - return found; + return NULL; } /* * The one "nvram" partition of the flash on CS0, 4 KiB aligned and within - * 32-bit addressing. Without it, no program or erase is admitted. + * three-byte addressing, since the policy only admits three-byte programs + * and erases. Without it, no program or erase is admitted. */ static void apple_qspi_nvram_window(struct device *dev, struct apple_qspi *qspi) { @@ -933,7 +954,7 @@ static void apple_qspi_nvram_window(struct device *dev, struct apple_qspi *qspi) } } of_node_put(parts); - if (found != 1 || start >= end || end > 1ULL << 32 || + if (found != 1 || start >= end || end > 1ULL << 24 || !IS_ALIGNED(start, APPLE_QSPI_ERASE_SIZE) || !IS_ALIGNED(end, APPLE_QSPI_ERASE_SIZE)) start = end = 0; @@ -941,8 +962,7 @@ static void apple_qspi_nvram_window(struct device *dev, struct apple_qspi *qspi) qspi->window_start = start; qspi->window_end = end; if (start < end) - dev_dbg(dev, "boot NOR read-only outside nvram [%#llx, %#llx)\n", - start, end); + dev_dbg(dev, "nvram partition [%#llx, %#llx)\n", start, end); else dev_dbg(dev, "boot NOR read-only: no usable nvram partition\n"); } @@ -950,22 +970,26 @@ static void apple_qspi_nvram_window(struct device *dev, struct apple_qspi *qspi) static int apple_qspi_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + const struct apple_qspi_board *board; + const struct apple_qspi_soc *soc; struct spi_controller *ctlr; struct apple_qspi *qspi; struct resource *res; u32 depths, version; int irq, ret; + soc = of_device_get_match_data(dev); + if (!soc) + return -ENODEV; + ctlr = devm_spi_alloc_host(dev, sizeof(*qspi)); if (!ctlr) return -ENOMEM; qspi = spi_controller_get_devdata(ctlr); - qspi->nvram_window = of_device_is_compatible(dev->of_node, "apple,t8140-qspi"); - if (qspi->nvram_window) - apple_qspi_nvram_window(dev, qspi); - qspi->nvram_policy = of_device_is_compatible(dev->of_node, "apple,t8122-qspi") || - of_device_is_compatible(dev->of_node, "apple,t8132-qspi"); + qspi->nvram_policy = true; + qspi->nvram_window = soc->multi_io_reads; + apple_qspi_nvram_window(dev, qspi); if (of_device_is_compatible(dev->of_node, "apple,t8122-qspi") && !of_machine_is_compatible("apple,j613")) return -ENODEV; @@ -977,9 +1001,14 @@ static int apple_qspi_probe(struct platform_device *pdev) return dev_err_probe(dev, -EINVAL, "QSPIMC aperture is too small\n"); - qspi->nvram_write_candidate = apple_qspi_nvram_write_board(dev->of_node, - res->start) && - apple_qspi_nvram_layout(dev->of_node); + /* The device tree must describe exactly the board's nvram partition. */ + board = apple_qspi_nvram_write_board(dev->of_node, res->start); + if (board && qspi->window_start == board->nvram_start && + qspi->window_end == (u64)board->nvram_start + board->nvram_size) { + memcpy(qspi->jedec, board->jedec, sizeof(qspi->jedec)); + qspi->bp_ok = board->bp_ok; + qspi->nvram_write_candidate = true; + } qspi->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(qspi->base)) return PTR_ERR(qspi->base); @@ -1016,17 +1045,33 @@ static int apple_qspi_probe(struct platform_device *pdev) ctlr->num_chipselect = 1; ctlr->max_native_cs = 1; ctlr->mode_bits = SPI_CPHA | SPI_CPOL; - if (!qspi->nvram_policy) + if (qspi->nvram_window) ctlr->mode_bits |= SPI_RX_DUAL | SPI_RX_QUAD | SPI_TX_DUAL | SPI_TX_QUAD; ctlr->bits_per_word_mask = SPI_BPW_MASK(8); ctlr->mem_ops = &apple_qspi_mem_ops; platform_set_drvdata(pdev, ctlr); + /* Keep probe and SPI NOR discovery powered until registration finishes. */ + if (soc->runtime_pm && IS_ENABLED(CONFIG_PM)) { + pm_runtime_get_noresume(dev); + ret = devm_pm_runtime_set_active_enabled(dev); + if (ret) { + pm_runtime_put_noidle(dev); + return ret; + } + /* Coalesce NOR read chunks and userspace NVRAM access bursts. */ + pm_runtime_set_autosuspend_delay(dev, APPLE_QSPI_AUTOSUSPEND_MS); + pm_runtime_use_autosuspend(dev); + ctlr->auto_runtime_pm = true; + } ret = devm_spi_register_controller(dev, ctlr); - if (ret) + if (ret) { + if (ctlr->auto_runtime_pm) + pm_runtime_put_noidle(dev); return dev_err_probe(dev, ret, "failed to register SPI controller\n"); + } mutex_lock(&apple_qspi_policy_lock); runtime_ready = true; @@ -1035,6 +1080,8 @@ static int apple_qspi_probe(struct platform_device *pdev) dev_dbg(dev, "QSPIMC version %#x, %lu Hz clock, txq=%u rxq=%u\n", version, clk_get_rate(qspi->clk), qspi->tx_depth, qspi->rx_depth); + if (ctlr->auto_runtime_pm) + pm_runtime_put_autosuspend(dev); return 0; } @@ -1045,15 +1092,24 @@ static int apple_qspi_match_cs0(struct device *dev, const void *unused) !spi_get_chipselect(to_spi_device(dev), 0); } +static int apple_qspi_flash_read_reg(struct spi_mem *mem, u8 opcode, + u8 *buf, unsigned int len) +{ + struct spi_mem_op op = SPI_MEM_OP(SPI_MEM_OP_CMD(opcode, 1), + SPI_MEM_OP_NO_ADDR, SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_IN(len, buf, 1)); + + return spi_mem_exec_op(mem, &op); +} + static ssize_t flash_status_show(struct device *dev, struct device_attribute *attr, char *buf) { - static const u8 opcodes[] = { 0x9f, 0x05, 0x35, 0x15 }; struct spi_controller *ctlr = dev_get_drvdata(dev); struct spi_mem mem = { }; struct device *child; u8 *data; - int i, ret; + int ret; child = device_find_child(&ctlr->dev, NULL, apple_qspi_match_cs0); if (!child) @@ -1067,18 +1123,34 @@ static ssize_t flash_status_show(struct device *dev, /* spi-mem supplies queue, bus and I/O serialization for each read. * These are sequential samples, not an atomic protection snapshot. + * The JEDEC ID decides which registers exist: 35h is SR2 on Winbond + * parts but EQIO on Macronix ones, so it is sent to Winbond only. */ - for (i = 0; i < ARRAY_SIZE(opcodes); i++) { - struct spi_mem_op op = SPI_MEM_OP(SPI_MEM_OP_CMD(opcodes[i], 1), - SPI_MEM_OP_NO_ADDR, SPI_MEM_OP_NO_DUMMY, - SPI_MEM_OP_DATA_IN(i ? 1 : 3, data + (i ? i + 2 : 0), 1)); - - ret = spi_mem_exec_op(&mem, &op); - if (ret) - goto out_free; + ret = apple_qspi_flash_read_reg(&mem, 0x9f, data, 3) ?: + apple_qspi_flash_read_reg(&mem, 0x05, data + 3, 1); + if (ret) + goto out_free; + switch (data[0]) { + case 0xef: + ret = apple_qspi_flash_read_reg(&mem, 0x35, data + 4, 1) ?: + apple_qspi_flash_read_reg(&mem, 0x15, data + 5, 1); + if (!ret) + ret = sysfs_emit(buf, "jedec=%02x%02x%02x sr1=%02x sr2=%02x sr3=%02x\n", + data[0], data[1], data[2], data[3], + data[4], data[5]); + break; + case APPLE_QSPI_MFR_MACRONIX: + ret = apple_qspi_flash_read_reg(&mem, 0x15, data + 4, 1); + if (!ret) + ret = sysfs_emit(buf, "jedec=%02x%02x%02x sr=%02x cr=%02x\n", + data[0], data[1], data[2], data[3], + data[4]); + break; + default: + ret = sysfs_emit(buf, "jedec=%02x%02x%02x sr1=%02x\n", + data[0], data[1], data[2], data[3]); + break; } - ret = sysfs_emit(buf, "jedec=%02x%02x%02x sr1=%02x sr2=%02x sr3=%02x\n", - data[0], data[1], data[2], data[3], data[4], data[5]); out_free: kfree(data); out_put: @@ -1093,11 +1165,41 @@ static struct attribute *apple_qspi_attrs[] = { }; ATTRIBUTE_GROUPS(apple_qspi); +/* No IRQ handler may read the block while genpd has it powered off. */ +static int apple_qspi_runtime_suspend(struct device *dev) +{ + struct spi_controller *ctlr = dev_get_drvdata(dev); + struct apple_qspi *qspi = spi_controller_get_devdata(ctlr); + + disable_irq(qspi->irq); + return 0; +} + +static int apple_qspi_runtime_resume(struct device *dev) +{ + struct spi_controller *ctlr = dev_get_drvdata(dev); + struct apple_qspi *qspi = spi_controller_get_devdata(ctlr); + int ret; + + ret = apple_qspi_hw_reset(qspi); + if (ret) + return ret; + enable_irq(qspi->irq); + return 0; +} + static int apple_qspi_suspend(struct device *dev) { struct spi_controller *ctlr = dev_get_drvdata(dev); + int ret; - return spi_controller_suspend(ctlr); + ret = spi_controller_suspend(ctlr); + if (ret || !ctlr->auto_runtime_pm) + return ret; + ret = pm_runtime_force_suspend(dev); + if (ret) + spi_controller_resume(ctlr); + return ret; } /* @@ -1112,23 +1214,36 @@ static int apple_qspi_resume(struct device *dev) /* * Reopen the controller even if the reset failed. Left suspended, it - * would fail every transfer until reboot; a transfer that times out - * resets it again instead. + * would fail every queued transfer until reboot. Runtime PM still + * refuses MMIO if the controller could not be brought back up. */ - ret = apple_qspi_hw_reset(qspi); + if (ctlr->auto_runtime_pm) + ret = pm_runtime_force_resume(dev); + else + ret = apple_qspi_hw_reset(qspi); if (ret) dev_err(dev, "controller reset failed on resume: %d\n", ret); return spi_controller_resume(ctlr); } -static DEFINE_SIMPLE_DEV_PM_OPS(apple_qspi_pm_ops, apple_qspi_suspend, - apple_qspi_resume); +static const struct dev_pm_ops apple_qspi_pm_ops = { + SYSTEM_SLEEP_PM_OPS(apple_qspi_suspend, apple_qspi_resume) + RUNTIME_PM_OPS(apple_qspi_runtime_suspend, apple_qspi_runtime_resume, NULL) +}; + +static const struct apple_qspi_soc apple_qspi_single_io = { }; + +static const struct apple_qspi_soc apple_qspi_t8140 = { + .multi_io_reads = true, + .runtime_pm = true, +}; static const struct of_device_id apple_qspi_of_match[] = { - { .compatible = "apple,t8122-qspi" }, - { .compatible = "apple,t8132-qspi" }, - { .compatible = "apple,t8140-qspi" }, + { .compatible = "apple,t8122-qspi", .data = &apple_qspi_single_io }, + { .compatible = "apple,t8132-qspi", .data = &apple_qspi_single_io }, + { .compatible = "apple,t6041-qspi", .data = &apple_qspi_single_io }, + { .compatible = "apple,t8140-qspi", .data = &apple_qspi_t8140 }, {} }; MODULE_DEVICE_TABLE(of, apple_qspi_of_match); @@ -1139,7 +1254,7 @@ static struct platform_driver apple_qspi_driver = { .name = "apple-qspi", .dev_groups = apple_qspi_groups, .of_match_table = apple_qspi_of_match, - .pm = pm_sleep_ptr(&apple_qspi_pm_ops), + .pm = pm_ptr(&apple_qspi_pm_ops), }, }; module_platform_driver(apple_qspi_driver); diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index c6625c5cbb5d3..52353e5d9059b 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -855,11 +855,7 @@ apple_cio_pcie_tunnel_is_preinitialized(struct device_node *parent) if (!of_property_read_u32(child, "apple,pciec-preinit-status", &status) && status == 1) return true; - /* t8103 brings the port up itself. */ - if (of_device_is_compatible(child, "apple,t8103-pciec") && - of_property_read_bool(child, "apple,pciec-kernel-init")) - return true; - /* T600x/T602x without a handoff, when opted in. */ + /* Use the PCI driver's generation and DT admission checks. */ return apple_pcie_tunnel_needs_cold_init(child); } diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index c09591fe0ca68..7c5d5db928b11 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -1084,33 +1084,41 @@ enum hub_activation_type { static void hub_reconnect_ss_peer(struct usb_hub *hub) { struct usb_device *ss = hub->hdev; + struct usb_device *old_hs = NULL, *old_ss = NULL; struct usb_hub *parent, *hs_hub; struct usb_port *upstream, *peer; if (!ss->parent || !hub_is_superspeed(ss)) return; - guard(mutex)(&usb_port_peer_mutex); + mutex_lock(&usb_port_peer_mutex); parent = usb_hub_to_struct_hub(ss->parent); if (!parent || ss->portnum > ss->parent->maxchild) - return; + goto unlock; upstream = parent->ports[ss->portnum - 1]; peer = upstream->peer; /* Only firmware-described, asymmetric paths need this handover. */ if (!upstream->connector_node || !peer) - return; + goto unlock; hs_hub = usb_hub_to_struct_hub(to_usb_device(peer->dev.parent->parent)); if (!hs_hub || hs_hub->disconnected || hs_hub->hdev->level == ss->parent->level) - return; + goto unlock; /* Disconnect publishes child = NULL under device_state_lock. */ scoped_guard(spinlock_irq, &device_state_lock) { - if (!peer->child || peer->reconnect_ss || peer->ss_reconnected) - return; + if (!usb_port_may_queue_ss_reconnect(peer, ss)) + goto unlock; + /* A new SS attachment supersedes a request the USB2 worker has not read. */ + old_hs = peer->reconnect_hs; + old_ss = peer->reconnect_ss; peer->reconnect_hs = usb_get_dev(peer->child); peer->reconnect_ss = usb_get_dev(ss); } kick_hub_wq(hs_hub); +unlock: + mutex_unlock(&usb_port_peer_mutex); + usb_put_dev(old_hs); + usb_put_dev(old_ss); } static void hub_init_func2(struct work_struct *ws); @@ -2254,6 +2262,10 @@ int usb_hub_port_power_off(struct usb_device *hdev, int port1) dev_dbg(&port_dev->dev, "lane handover: power off\n"); /* The runtime PM reference is dropped once the hold ends. */ port_dev->held_off = !disabled; + if (port_dev->is_superspeed && !disabled) { + guard(mutex)(&usb_port_peer_mutex); + port_dev->ss_handover_generation++; + } /* Nothing hub_wq still has queued for the port applies to it off. */ clear_bit(port1, hub->change_bits); clear_bit(port1, hub->warm_reset_bits); @@ -3479,6 +3491,93 @@ static void usb_unlock_port(struct usb_port *port_dev) __release(&port_dev->status_lock); } +/** + * usb_hub_port_retrain - restart an unconnected SuperSpeed root port + * @hdev: SuperSpeed root hub + * @port1: port number + * + * Called immediately after usb_hub_port_power_on(), under the same hub lock, + * once the caller has finished bringing up the lanes behind the port. Consume + * the queued warm reset here while keeping the port runtime-active. If that + * reset finds no receiver, restart detection once through Disabled/RxDetect. + * Each operation has a bounded wait; no further recovery work is scheduled. + * Connected links and ports with a child, an owner or a power hold are left + * alone. Only the caller's newly restored port is eligible. + * + * Return: 0 for a usable link, 1 for a port left alone, or a negative error. + * A failure does not reclaim the power hold released by power_on(). + */ +int usb_hub_port_retrain(struct usb_device *hdev, int port1) +{ + struct usb_hub *hub = usb_hub_to_struct_hub(hdev); + struct usb_interface *intf; + struct usb_port *port_dev; + u16 status, change; + int ret; + + device_lock_assert(&hdev->dev); + if (!hub || hub->disconnected || hub->quiescing) + return -ENODEV; + if (hdev->parent || !hub_is_superspeed(hdev)) + return -EOPNOTSUPP; + if (port1 < 1 || port1 > hdev->maxchild) + return -EINVAL; + port_dev = hub->ports[port1 - 1]; + if (port_dev->disabled || port_dev->held_off || port_dev->port_owner || + port_dev->child || port_dev->ignore_event || port_dev->early_stop || + !test_bit(port1, hub->warm_reset_bits)) + return 1; + + intf = to_usb_interface(hub->intfdev); + ret = usb_autopm_get_interface(intf); + if (ret < 0) + return ret; + ret = pm_runtime_resume_and_get(&port_dev->dev); + if (ret < 0) + goto put_intf; + usb_lock_port(port_dev); + ret = usb_hub_port_status(hub, port1, &status, &change); + if (ret) + goto unlock; + if (!usb_port_is_power_on(hub, status)) { + ret = 1; + goto unlock; + } + /* The caller still holds the hub lock from power_on(): this is its reset. */ + clear_bit(port1, hub->warm_reset_bits); + if ((status & (USB_PORT_STAT_CONNECTION | USB_PORT_STAT_ENABLE)) == + (USB_PORT_STAT_CONNECTION | USB_PORT_STAT_ENABLE) && + (status & USB_PORT_STAT_LINK_STATE) <= USB_SS_PORT_LS_U3) { + ret = 0; + goto unlock; + } + + dev_dbg(&port_dev->dev, "SS handover: warm reset after lane readiness\n"); + ret = set_port_feature(hdev, port1, USB_PORT_FEAT_BH_PORT_RESET); + if (!ret) + ret = hub_port_wait_reset(hub, port1, NULL, HUB_BH_RESET_TIME, true); + if (ret != -ENOTCONN) + goto rescan; + + dev_dbg(&port_dev->dev, "SS handover: restarting receiver detection\n"); + ret = hub_set_port_link_state(hub, port1, USB_SS_PORT_LS_SS_DISABLED); + if (!ret) + ret = hub_set_port_link_state(hub, port1, USB_SS_PORT_LS_RX_DETECT); + if (!ret) + ret = hub_port_wait_reset(hub, port1, NULL, HUB_BH_RESET_TIME, true); +rescan: + dev_dbg(&port_dev->dev, "SS handover: retrain result %d\n", ret); + set_bit(port1, hub->change_bits); + kick_hub_wq(hub); +unlock: + usb_unlock_port(port_dev); + pm_runtime_put(&port_dev->dev); +put_intf: + usb_autopm_put_interface(intf); + return ret; +} +EXPORT_SYMBOL_GPL(usb_hub_port_retrain); + #ifdef CONFIG_PM /* Check if a port is suspended(USB2.0 port) or in U3 state(USB3.0 port) */ @@ -6139,6 +6238,7 @@ static void hub_port_reconnect_ss(struct usb_hub *hub, int port1) { struct usb_port *port = hub->ports[port1 - 1]; struct usb_device *hs, *ss; + u32 handover = 0; bool reconnect; int ret; @@ -6150,6 +6250,8 @@ static void hub_port_reconnect_ss(struct usb_hub *hub, int port1) if (!ss) return; reconnect = usb_port_needs_ss_reconnect(port, hs, ss); + if (reconnect) + handover = port->peer->ss_handover_generation; } if (!reconnect) goto out; @@ -6161,6 +6263,8 @@ static void hub_port_reconnect_ss(struct usb_hub *hub, int port1) /* Power loss can reconnect the SS half too; it must not start a loop. */ scoped_guard(mutex, &usb_port_peer_mutex) { port->ss_reconnected = true; + /* An older in-flight cycle cannot consume a newer lane handover. */ + port->ss_reconnect_generation = handover; port->ss_reconnect_pending = true; } diff --git a/drivers/usb/core/hub.h b/drivers/usb/core/hub.h index c095b77aaf327..6cde9aaef6740 100644 --- a/drivers/usb/core/hub.h +++ b/drivers/usb/core/hub.h @@ -94,6 +94,8 @@ struct usb_hub { * @reconnect_ss: SuperSpeed hub requesting that USB2 connection restart * @ss_reconnected: the current USB2 attachment already had its late-SS handover * @ss_reconnect_pending: consume the connection event from our own power cycle + * @ss_handover_generation: explicit SS lane handovers, under usb_port_peer_mutex + * @ss_reconnect_generation: handover serviced by the last USB2 reconnect * @connector: USB Type-C connector * @req: default pm qos request for hubs without port power control * @connect_type: port's connect type @@ -140,8 +142,24 @@ struct usb_port { bool held_off; bool ss_reconnected; bool ss_reconnect_pending; + u32 ss_handover_generation; + u32 ss_reconnect_generation; }; +/* usb_port_peer_mutex protects the peer and both handover counters. */ +static inline bool usb_port_ss_reconnect_done(const struct usb_port *port) +{ + return port->ss_reconnected && port->peer && + port->ss_reconnect_generation == port->peer->ss_handover_generation; +} + +static inline bool usb_port_may_queue_ss_reconnect(const struct usb_port *port, + const struct usb_device *ss) +{ + return port->child && !usb_port_ss_reconnect_done(port) && + (port->reconnect_hs != port->child || port->reconnect_ss != ss); +} + /* Caller holds the hub lock and usb_port_peer_mutex. */ static inline void usb_port_ss_reconnect_event(struct usb_port *port, u16 status, u16 change) { @@ -160,7 +178,7 @@ static inline bool usb_port_needs_ss_reconnect(const struct usb_port *port, const struct usb_device *hs, const struct usb_device *ss) { - return hs && ss && !port->ss_reconnected && !port->is_superspeed && + return hs && ss && !usb_port_ss_reconnect_done(port) && !port->is_superspeed && port->child == hs && port->peer && port->peer->is_superspeed && port->connector_node && port->peer->connector_node == port->connector_node && diff --git a/drivers/usb/core/port-power-test.c b/drivers/usb/core/port-power-test.c index 181fa7ff0b8a8..061eec179d8a0 100644 --- a/drivers/usb/core/port-power-test.c +++ b/drivers/usb/core/port-power-test.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,11 @@ struct port_power_test_hc { u16 change; unsigned int power_ons; bool fail_power_off; + unsigned int warm_resets; + unsigned int detect_restarts; + bool connect_on_warm; + bool connect_on_detect; + bool fail_warm_reset; }; struct port_power_test { @@ -171,8 +177,31 @@ static int port_power_test_hub_control(struct usb_hcd *hcd, u16 type_req, u16 va port_power_test_set_power(hcd, true); else if (value == USB_PORT_FEAT_RESET) hc->change |= USB_PORT_STAT_C_RESET; - else if (value == USB_PORT_FEAT_BH_PORT_RESET) + else if (value == USB_PORT_FEAT_BH_PORT_RESET) { + hc->warm_resets++; + if (hc->fail_warm_reset) + return -EIO; hc->change |= USB_PORT_STAT_C_RESET | USB_PORT_STAT_C_BH_RESET; + if (hc->connect_on_warm) { + hc->status &= ~USB_PORT_STAT_LINK_STATE; + hc->status |= USB_PORT_STAT_CONNECTION | USB_PORT_STAT_ENABLE; + } + } else if (value == USB_PORT_FEAT_LINK_STATE) { + u16 state = (index >> 3) & USB_PORT_STAT_LINK_STATE; + + hc->status &= ~USB_PORT_STAT_LINK_STATE; + hc->status |= state; + if (state == USB_SS_PORT_LS_SS_DISABLED) + hc->status &= ~(USB_PORT_STAT_CONNECTION | USB_PORT_STAT_ENABLE); + if (state == USB_SS_PORT_LS_RX_DETECT) { + hc->detect_restarts++; + if (hc->connect_on_detect) { + hc->status &= ~USB_PORT_STAT_LINK_STATE; + hc->status |= USB_PORT_STAT_CONNECTION | + USB_PORT_STAT_ENABLE; + } + } + } return 0; case ClearPortFeature: if (value == USB_PORT_FEAT_POWER && hc->fail_power_off) { @@ -409,14 +438,18 @@ static void port_power_test_hold(struct kunit *test) { struct port_power_test *t = test->priv; unsigned int hs_power_ons = port_power_test_power_ons(t->hcd); + struct usb_port *ss = to_usb_port(t->ss_port); + u32 handover = ss->ss_handover_generation; int usage = port_power_test_usage(t->ss_port); usb_lock_device(t->ss_root); KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, ss->ss_handover_generation, handover + 1); KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); /* A second change of the lanes finds the port held for it */ KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, ss->ss_handover_generation, handover + 1); KUNIT_EXPECT_FALSE(test, port_power_test_powered(t->shared_hcd)); KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); @@ -425,6 +458,10 @@ static void port_power_test_hold(struct kunit *test) /* Nothing is held any more */ KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 1); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, ss->ss_handover_generation, handover + 2); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage); KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 2), -EINVAL); usb_unlock_device(t->ss_root); @@ -700,6 +737,10 @@ static void port_power_test_late_ss_decision(struct kunit *test) port->ss_reconnected = true; KUNIT_EXPECT_FALSE(test, usb_port_needs_ss_reconnect(port, hs, ss)); + peer->ss_handover_generation++; + KUNIT_EXPECT_TRUE(test, usb_port_needs_ss_reconnect(port, hs, ss)); + port->ss_reconnect_generation = peer->ss_handover_generation; + KUNIT_EXPECT_FALSE(test, usb_port_needs_ss_reconnect(port, hs, ss)); port->ss_reconnected = false; port->disabled = true; @@ -928,7 +969,176 @@ static void port_power_test_connector_graph_legacy(struct kunit *test) } #endif +/* Keep the root locked from power-on through retraining, as the PHY does. */ +static struct port_power_test_hc * +port_power_test_retrain_begin(struct kunit *test, bool warm, bool detect) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_hc(t->shared_hcd); + + usb_lock_device(t->ss_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_off(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, usb_hub_port_power_on(t->ss_root, 1), 0); + scoped_guard(spinlock_irqsave, &hc->lock) { + hc->warm_resets = 0; + hc->detect_restarts = 0; + hc->connect_on_warm = warm; + hc->connect_on_detect = detect; + } + return hc; +} + +static void port_power_test_retrain_end(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_hc(t->shared_hcd); + + /* The synthetic controller has no device to enumerate after the check. */ + scoped_guard(spinlock_irqsave, &hc->lock) { + hc->status &= ~(USB_PORT_STAT_CONNECTION | USB_PORT_STAT_ENABLE | + USB_PORT_STAT_LINK_STATE); + hc->status |= USB_SS_PORT_LS_RX_DETECT; + } + usb_unlock_device(t->ss_root); +} + +static void port_power_test_retrain_warm(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_retrain_begin(test, true, false); + int usage = port_power_test_usage(t->ss_port); + + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, hc->warm_resets, 1U); + KUNIT_EXPECT_EQ(test, hc->detect_restarts, 0U); + KUNIT_EXPECT_EQ(test, port_power_test_usage(t->ss_port), usage); + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), 1); + KUNIT_EXPECT_EQ(test, hc->warm_resets, 1U); + port_power_test_retrain_end(test); +} + +static void port_power_test_retrain_detect(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_retrain_begin(test, false, true); + + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, hc->warm_resets, 1U); + KUNIT_EXPECT_EQ(test, hc->detect_restarts, 1U); + port_power_test_retrain_end(test); +} + +static void port_power_test_retrain_bounded(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_retrain_begin(test, false, false); + + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), -ENOTCONN); + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), 1); + KUNIT_EXPECT_EQ(test, hc->warm_resets, 1U); + KUNIT_EXPECT_EQ(test, hc->detect_restarts, 1U); + port_power_test_retrain_end(test); +} + +static void port_power_test_retrain_healthy(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_retrain_begin(test, false, false); + + scoped_guard(spinlock_irqsave, &hc->lock) { + hc->status &= ~USB_PORT_STAT_LINK_STATE; + hc->status |= USB_PORT_STAT_CONNECTION | USB_PORT_STAT_ENABLE; + } + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), 0); + KUNIT_EXPECT_EQ(test, hc->warm_resets, 0U); + KUNIT_EXPECT_EQ(test, hc->detect_restarts, 0U); + port_power_test_retrain_end(test); +} + +static void port_power_test_retrain_error(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_retrain_begin(test, false, false); + + hc->fail_warm_reset = true; + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), -EIO); + KUNIT_EXPECT_EQ(test, hc->warm_resets, 1U); + KUNIT_EXPECT_EQ(test, hc->detect_restarts, 0U); + port_power_test_retrain_end(test); +} + +static void port_power_test_retrain_owner(struct kunit *test) +{ + struct port_power_test *t = test->priv; + struct port_power_test_hc *hc = port_power_test_retrain_begin(test, false, false); + + KUNIT_EXPECT_EQ(test, usb_hub_claim_port(t->ss_root, 1, (void *)t), 0); + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->ss_root, 1), 1); + KUNIT_EXPECT_EQ(test, usb_hub_release_port(t->ss_root, 1, (void *)t), 0); + KUNIT_EXPECT_EQ(test, hc->warm_resets, 0U); + KUNIT_EXPECT_EQ(test, hc->detect_restarts, 0U); + port_power_test_retrain_end(test); + usb_lock_device(t->hs_root); + KUNIT_EXPECT_EQ(test, usb_hub_port_retrain(t->hs_root, 1), -EOPNOTSUPP); + usb_unlock_device(t->hs_root); +} + +/* A new PHY handover rearms USB2 recovery; its own power cycle never does. */ +static void port_power_test_late_ss_handover(struct kunit *test) +{ + struct usb_port *port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL); + struct usb_port *peer = kunit_kzalloc(test, sizeof(*peer), GFP_KERNEL); + struct usb_device *hs = kunit_kzalloc(test, sizeof(*hs), GFP_KERNEL); + struct usb_device *ss = kunit_kzalloc(test, sizeof(*ss), GFP_KERNEL); + struct usb_device *next = kunit_kzalloc(test, sizeof(*next), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, port); + KUNIT_ASSERT_NOT_NULL(test, peer); + KUNIT_ASSERT_NOT_NULL(test, hs); + KUNIT_ASSERT_NOT_NULL(test, ss); + KUNIT_ASSERT_NOT_NULL(test, next); + port->child = hs; + port->peer = peer; + port->ss_reconnected = true; + peer->is_superspeed = true; + peer->child = ss; + port->ss_reconnect_generation = 7; + peer->ss_handover_generation = 7; + KUNIT_EXPECT_FALSE(test, usb_port_may_queue_ss_reconnect(port, ss)); + + /* A tipd rebind changes SS even while the USB2 hub remains attached. */ + peer->ss_handover_generation++; + KUNIT_EXPECT_TRUE(test, usb_port_may_queue_ss_reconnect(port, ss)); + port->reconnect_hs = hs; + port->reconnect_ss = ss; + KUNIT_EXPECT_FALSE(test, usb_port_may_queue_ss_reconnect(port, ss)); + KUNIT_EXPECT_TRUE(test, usb_port_may_queue_ss_reconnect(port, next)); + + /* Completion of the older request must not consume the newer handover. */ + port->ss_reconnect_generation = 7; + KUNIT_EXPECT_TRUE(test, usb_port_may_queue_ss_reconnect(port, next)); + port->ss_reconnect_generation = peer->ss_handover_generation; + port->ss_reconnect_pending = true; + usb_port_ss_reconnect_event(port, USB_PORT_STAT_CONNECTION, USB_PORT_STAT_C_CONNECTION); + KUNIT_EXPECT_FALSE(test, usb_port_may_queue_ss_reconnect(port, next)); + peer->ss_handover_generation++; + KUNIT_EXPECT_TRUE(test, usb_port_may_queue_ss_reconnect(port, next)); + + peer->ss_handover_generation = U32_MAX; + port->ss_reconnect_generation = U32_MAX; + KUNIT_EXPECT_FALSE(test, usb_port_may_queue_ss_reconnect(port, next)); + peer->ss_handover_generation++; + KUNIT_EXPECT_TRUE(test, usb_port_may_queue_ss_reconnect(port, next)); +} + static struct kunit_case port_power_test_cases[] = { + KUNIT_CASE(port_power_test_late_ss_handover), + KUNIT_CASE(port_power_test_retrain_warm), + KUNIT_CASE(port_power_test_retrain_detect), + KUNIT_CASE(port_power_test_retrain_bounded), + KUNIT_CASE(port_power_test_retrain_healthy), + KUNIT_CASE(port_power_test_retrain_error), + KUNIT_CASE(port_power_test_retrain_owner), #if IS_ENABLED(CONFIG_OF_DYNAMIC) KUNIT_CASE(port_power_test_connector_graph), KUNIT_CASE(port_power_test_connector_graph_legacy), diff --git a/drivers/usb/dwc3/Kconfig b/drivers/usb/dwc3/Kconfig index c862e55ae3a37..0d5c41f1a7d5d 100644 --- a/drivers/usb/dwc3/Kconfig +++ b/drivers/usb/dwc3/Kconfig @@ -226,6 +226,7 @@ config USB_DWC3_APPLE tristate "Apple Silicon DWC3 Platform Driver" depends on OF && ARCH_APPLE default USB_DWC3 + select APPLE_TUNABLE select USB_ROLE_SWITCH help Support Apple Silicon SoCs with DesignWare Core USB3 IP. diff --git a/drivers/usb/typec/tipd/cd321x-mux-test.c b/drivers/usb/typec/tipd/cd321x-mux-test.c index 5a95956d0adb4..13fe97d64520a 100644 --- a/drivers/usb/typec/tipd/cd321x-mux-test.c +++ b/drivers/usb/typec/tipd/cd321x-mux-test.c @@ -908,7 +908,92 @@ static void cd321x_mux_held_then_other_error_test(struct kunit *test) cd321x_mux_test_expect_usb(test, t); } +/* A dock already attached when probe's first SAFE request times out. */ +static struct cd321x_mux_test *cd321x_mux_test_boot(struct kunit *test, bool transient) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, transient); + + cd321x_pm_init(&t->cd.pm); + t->status = TPS_STATUS_PLUG_PRESENT; + t->data_status = TPS_DATA_STATUS_DATA_CONNECTION | TPS_DATA_STATUS_USB2_CONNECTION | + TPS_DATA_STATUS_USB3_CONNECTION; + t->phy.held = true; + KUNIT_ASSERT_EQ(test, cd321x_init_mux(&t->cd), 0); + KUNIT_EXPECT_STREQ(test, t->phy.log, "S"); + KUNIT_EXPECT_EQ(test, t->cd.pm.phase, CD321X_PM_INITIALIZING); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.update_work)); + KUNIT_EXPECT_FALSE(test, t->cd.state_valid); + + return t; +} + +static void cd321x_mux_boot_retries_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_boot(test, true); + + KUNIT_ASSERT_NE(test, t->cd.pm.mux_retry_ms, 0U); + t->cd.tps.data->ready(&t->cd.tps); + cd321x_mux_test_expect_read_now(test, t); + while (t->now_ms < TEST_ELECTIONS_MS) + KUNIT_ASSERT_TRUE(test, cd321x_mux_test_next(t)); + KUNIT_EXPECT_NE(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + /* No interrupt or physical replug is needed after the lock is released. */ + t->phy.held = false; + memset(t->phy.log, 0, sizeof(t->phy.log)); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_STREQ(test, t->phy.log, "su"); + cd321x_mux_test_expect_usb(test, t); +} + +static void cd321x_mux_boot_ceiling_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_boot(test, true); + unsigned int i; + + t->cd.tps.data->ready(&t->cd.tps); + cd321x_mux_test_expect_read_now(test, t); + for (i = 0; i < 4 * CD321X_MUX_RETRIES && cd321x_mux_test_next(t); i++) + ; + KUNIT_EXPECT_GE(test, t->now_ms, CD321X_MUX_HELD_MS); + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0U); + KUNIT_EXPECT_FALSE(test, t->cd.state_valid); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.update_work)); +} + +/* M1/M2 providers retain their ordinary three-attempt recovery budget. */ +static void cd321x_mux_boot_nontransient_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_boot(test, false); + + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0U); + t->cd.tps.data->ready(&t->cd.tps); + cd321x_mux_test_expect_read_now(test, t); + cd321x_mux_test_settle(test, t); + KUNIT_EXPECT_EQ(test, cd321x_mux_test_phase(t), CD321X_PM_STALE); + KUNIT_EXPECT_LT(test, t->now_ms, TEST_ELECTIONS_MS); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0U); +} + +static void cd321x_mux_boot_defer_test(struct kunit *test) +{ + struct cd321x_mux_test *t = cd321x_mux_test_port(test, true); + + cd321x_pm_init(&t->cd.pm); + t->phy.held = true; + t->phy.held_ret = -EAGAIN; + KUNIT_EXPECT_EQ(test, cd321x_init_mux(&t->cd), -EPROBE_DEFER); + KUNIT_EXPECT_EQ(test, t->cd.pm.mux_retry_ms, 0U); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&t->cd.resume_work)); +} + static struct kunit_case cd321x_mux_cases[] = { + KUNIT_CASE(cd321x_mux_boot_retries_test), + KUNIT_CASE(cd321x_mux_boot_ceiling_test), + KUNIT_CASE(cd321x_mux_boot_nontransient_test), + KUNIT_CASE(cd321x_mux_boot_defer_test), KUNIT_CASE(cd321x_mux_held_past_budget_test), KUNIT_CASE(cd321x_mux_held_backs_off_test), KUNIT_CASE(cd321x_mux_held_event_takes_over_test), diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index aef346d7bb4d8..5598f65af9e06 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -1070,7 +1070,29 @@ static void cd321x_dp_send_test(struct kunit *test) KUNIT_EXPECT_EQ(test, cd321x_dp_send(irq, irq, true), CD321X_DP_SEND_EDGE); } +/* Probe joins the same timed series, without making the port usable early. */ +static void cd321x_probe_mux_held_test(struct kunit *test) +{ + struct cd321x_pm_state pm = {}; + + cd321x_pm_init(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm, 1000), CD321X_MUX_RETRY_MS); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_INITIALIZING); + KUNIT_EXPECT_EQ(test, pm.attempts_left, 0U); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); + KUNIT_EXPECT_FALSE(test, cd321x_pm_begin_read(&pm)); + KUNIT_ASSERT_TRUE(test, cd321x_pm_ready(&pm)); + KUNIT_EXPECT_TRUE(test, pm.force_reconnect); + KUNIT_EXPECT_EQ(test, pm.mux_since_ms, 1000U); + KUNIT_ASSERT_TRUE(test, cd321x_pm_begin_read(&pm)); + cd321x_pm_snapshot_ready(&pm); + KUNIT_EXPECT_EQ(test, cd321x_pm_mux_held(&pm, 1000 + CD321X_MUX_HELD_MS), 0U); + KUNIT_EXPECT_EQ(test, pm.phase, CD321X_PM_STALE); + KUNIT_EXPECT_FALSE(test, cd321x_pm_can_update(&pm)); +} + static struct kunit_case cd321x_pm_cases[] = { + KUNIT_CASE(cd321x_probe_mux_held_test), KUNIT_CASE(cd321x_reset_first_irq_before_ready_test), KUNIT_CASE(cd321x_reset_first_irq_during_prepare_test), KUNIT_CASE(cd321x_reset_setup_retry_test), diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h index 8260e559b0c87..18f586682fceb 100644 --- a/drivers/usb/typec/tipd/cd321x-pm.h +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -314,11 +314,14 @@ static inline unsigned int cd321x_pm_mux_held(struct cd321x_pm_state *pm, pm->attempts_left = 0; return 0; } - if (pm->phase == CD321X_PM_RUNNING) - pm->attempts_left = CD321X_RESUME_ATTEMPTS; - else - pm->attempts_left++; - pm->phase = CD321X_PM_REVALIDATE; + /* A probe-time SAFE timeout joins this series without enabling workers. */ + if (pm->phase != CD321X_PM_INITIALIZING) { + if (pm->phase == CD321X_PM_RUNNING) + pm->attempts_left = CD321X_RESUME_ATTEMPTS; + else + pm->attempts_left++; + pm->phase = CD321X_PM_REVALIDATE; + } pm->mux_retries++; pm->mux_retry_ms = clamp(2 * pm->mux_retry_ms, CD321X_MUX_RETRY_MS, CD321X_MUX_RETRY_MAX_MS); diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index d45d66e679733..872dc2c02c2e6 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -998,10 +998,11 @@ static void cd321x_retry_held_mux(struct cd321x *cd321x) if (!cd321x->pm.mux_retry_ms) dev_info(cd321x->tps.dev, "Type-C mux held up, retrying\n"); delay = cd321x_pm_mux_held(&cd321x->pm, now_ms); - if (delay) + /* Probe records the hold, but ready() must enable the workers first. */ + if (delay && cd321x->pm.phase != CD321X_PM_INITIALIZING) mod_delayed_work(system_freezable_wq, &cd321x->resume_work, msecs_to_jiffies(delay)); - else + else if (!delay) dev_warn(cd321x->tps.dev, "Type-C mux still held up after %u retries in %u s, waiting for the next event\n", retries, held_ms / 1000); @@ -2080,6 +2081,8 @@ static void cd321x_ready(struct tps6598x *tps) guard(mutex)(&tps->lock); /* Include failures reported while the port was still being initialized. */ if (cd321x_pm_ready(&cd321x->pm)) { + dev_dbg(tps->dev, "Type-C probe ready: held=%u\n", + !!cd321x->pm.mux_retry_ms); cd321x->state_valid = false; mod_delayed_work(system_freezable_wq, &cd321x->resume_work, 0); } @@ -2229,6 +2232,27 @@ VISIBLE_IF_KUNIT void cd321x_init_work(struct cd321x *cd321x) } EXPORT_SYMBOL_IF_KUNIT(cd321x_init_work); +/* Use the same handle as the worker: timeout provenance belongs to the handle. */ +VISIBLE_IF_KUNIT int cd321x_init_mux(struct cd321x *cd321x) +{ + bool held; + int ret; + + guard(mutex)(&cd321x->tps.lock); + cd321x->state.alt = NULL; + cd321x->state.mode = TYPEC_STATE_SAFE; + cd321x->state.data = NULL; + ret = cd321x_mux_set(cd321x, &held); + cd321x->state_valid = !ret; + if (ret == -EAGAIN) + return -EPROBE_DEFER; + if (held) + cd321x_retry_held_mux(cd321x); + + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(cd321x_init_mux); + static int cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) { @@ -2266,20 +2290,14 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) goto err_put_mux; } - cd321x->state.alt = NULL; - cd321x->state.mode = TYPEC_STATE_SAFE; - cd321x->state.data = NULL; - ret = typec_set_mode(tps->port, TYPEC_STATE_SAFE); - cd321x->state_valid = !ret; /* * A fixed host can register its SuperSpeed root after the mux provider. * Wait for that consumer at probe instead of exhausting the cable retry * budget before it exists. Host startup does not depend on this port. */ - if (ret == -EAGAIN) { - ret = -EPROBE_DEFER; + ret = cd321x_init_mux(cd321x); + if (ret) goto err_put_switch; - } cd321x->tbt_notifier.notifier_call = cd321x_tbt_notify; ret = typec_thunderbolt_switch_register_notifier(cd321x->tbt_switch, @@ -2806,6 +2824,7 @@ int tipd_init(struct tps6598x *tps) int ret, cleanup_ret; bool reset_allowed = true; + dev_dbg(tps->dev, "Type-C probe start\n"); tps->reset = devm_gpiod_get_optional(tps->dev, "reset", GPIOD_OUT_LOW); if (IS_ERR(tps->reset)) return dev_err_probe(tps->dev, PTR_ERR(tps->reset), diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 195269e1831ee..5a1b73a6d4601 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -495,6 +495,7 @@ static inline void tipd_debugfs_exit(void) { } #if IS_ENABLED(CONFIG_KUNIT) void cd321x_init_work(struct cd321x *cd321x); +int cd321x_init_mux(struct cd321x *cd321x); void cd321x_stop_work(struct cd321x *cd321x); bool cd321x_tbt_switch_keep_waiting(struct device *dev); struct typec_thunderbolt_switch * diff --git a/include/linux/sched.h b/include/linux/sched.h index a2e1894a9e13e..57ef9a060eb65 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -939,6 +939,8 @@ struct task_struct { unsigned int policy; unsigned long max_allowed_capacity; + /* Runtime checkpoint for misfit rotation, protected by the rq lock. */ + u64 misfit_runtime; int nr_cpus_allowed; const cpumask_t *cpus_ptr; cpumask_t *user_cpus_ptr; diff --git a/include/linux/soc/apple/pci-apple-piodma.h b/include/linux/soc/apple/pci-apple-piodma.h index 5cb948a60943e..3b2971b8b1db4 100644 --- a/include/linux/soc/apple/pci-apple-piodma.h +++ b/include/linux/soc/apple/pci-apple-piodma.h @@ -7,7 +7,7 @@ struct pci_dev; -#if IS_ENABLED(CONFIG_PCIE_APPLE_PIODMA_DIAG) +#if IS_ENABLED(CONFIG_PCIE_APPLE_T8140_BOOTSTRAP) /* Require completed Neo admission and permanent retained-DMA protection. */ int apple_piodma_radio_check(struct pci_dev *pdev); #else diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h index 1bfb4fc69f1f7..5e819681ba106 100644 --- a/include/linux/soc/apple/pmp-report.h +++ b/include/linux/soc/apple/pmp-report.h @@ -14,11 +14,26 @@ struct device_node; struct device; #if IS_ENABLED(CONFIG_APPLE_PMP_REPORT) +int apple_pmp_report_ane_available(const struct device_node *np); +int apple_pmp_report_ane_perf(const struct device_node *np, + struct device *owner, unsigned int level, bool *held); int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int timeout_ms); int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout); int apple_pmp_report_fast_die_effort(const struct device_node *np, unsigned int lane, u8 *effort); #else +static inline int apple_pmp_report_ane_available(const struct device_node *np) +{ + return -EOPNOTSUPP; +} + +static inline int apple_pmp_report_ane_perf(const struct device_node *np, + struct device *owner, unsigned int level, bool *held) +{ + *held = false; + return -EOPNOTSUPP; +} + static inline int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int timeout_ms) { diff --git a/include/linux/usb.h b/include/linux/usb.h index 8c14a6bd3e08b..25f6b0c2fb708 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -958,6 +958,7 @@ int usb_hub_release_port(struct usb_device *hdev, unsigned port1, /* port power switching for a driver that moves the lanes behind a port */ int usb_hub_port_power_off(struct usb_device *hdev, int port1); int usb_hub_port_power_on(struct usb_device *hdev, int port1); +int usb_hub_port_retrain(struct usb_device *hdev, int port1); /** * usb_make_path - returns stable device path in the usb tree diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index 535860581f159..ad1a74a821dd5 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -10,6 +10,38 @@ #include #include +/* + * A queued misfit rotation has result -EINPROGRESS and no destination. + * The completion reports migrate_swap()'s result, or a selection failure. + */ +TRACE_EVENT(sched_misfit_rotate, + + TP_PROTO(struct task_struct *p, struct task_struct *t, + int src_cpu, int dst_cpu, int result), + + TP_ARGS(p, t, src_cpu, dst_cpu, result), + + TP_STRUCT__entry( + __field(pid_t, src_pid) + __field(pid_t, dst_pid) + __field(int, src_cpu) + __field(int, dst_cpu) + __field(int, result) + ), + + TP_fast_assign( + __entry->src_pid = p->pid; + __entry->dst_pid = t ? t->pid : 0; + __entry->src_cpu = src_cpu; + __entry->dst_cpu = dst_cpu; + __entry->result = result; + ), + + TP_printk("src_pid=%d dst_pid=%d src_cpu=%d dst_cpu=%d result=%d", + __entry->src_pid, __entry->dst_pid, __entry->src_cpu, + __entry->dst_cpu, __entry->result) +); + /* * Tracepoint for calling kthread_stop, performed to end a kthread: */ diff --git a/include/uapi/linux/apple-sep.h b/include/uapi/linux/apple-sep.h index 394b497ff7a59..fbae982a4c0ef 100644 --- a/include/uapi/linux/apple-sep.h +++ b/include/uapi/linux/apple-sep.h @@ -14,7 +14,7 @@ #include #include -#define SEP_BIO_IFACE_VERSION 4 +#define SEP_BIO_IFACE_VERSION 5 #define SEP_BIO_UUID_LEN 16 #define SEP_BIO_LABEL_LEN 128 @@ -64,10 +64,22 @@ struct sep_bio_list { struct sep_bio_enrol_start { __u32 flags; + __u32 account_uid; /* resolved passwd uid; UINT32_MAX is invalid */ + __u32 finger; /* FP_FINGER_LEFT_THUMB..FP_FINGER_RIGHT_LITTLE (1..10) */ __u32 reserved; __u8 label[SEP_BIO_LABEL_LEN]; }; +/* Linux's built-in SEP user context only. This does not enumerate other SEP + * users or OS lanes. DELETE_ORPHAN uses an exact UUID; the kernel index may + * still hold it when fprintd has already lost its own record. + */ +struct sep_bio_enclave_list { + __u32 count; + __u32 reserved; + __u8 uuid[SEP_BIO_MAX_IDENTITIES][SEP_BIO_UUID_LEN]; +}; + #define SEP_BIO_GUIDANCE_NONE 0 #define SEP_BIO_GUIDANCE_PLACE 1 #define SEP_BIO_GUIDANCE_LIFT_AND_MOVE 2 @@ -129,5 +141,11 @@ struct sep_bio_attest { #define SEP_BIO_DELETE _IOW(SEP_BIO_IOC_MAGIC, 0x08, struct sep_bio_delete) #define SEP_BIO_DELETE_ALL _IO(SEP_BIO_IOC_MAGIC, 0x09) #define SEP_BIO_ATTEST _IOWR(SEP_BIO_IOC_MAGIC, 0x0a, struct sep_bio_attest) +#define SEP_BIO_LIST_ENCLAVE _IOR(SEP_BIO_IOC_MAGIC, 0x0b, struct sep_bio_enclave_list) +#define SEP_BIO_DELETE_ORPHAN _IOW(SEP_BIO_IOC_MAGIC, 0x0c, struct sep_bio_delete) +/* Creates the empty Linux owner context only when SEP lists zero Linux + * identities. ENROL_START itself never repairs a missing persistence binding. + */ +#define SEP_BIO_PREPARE_ENROL _IOW(SEP_BIO_IOC_MAGIC, 0x0d, struct sep_bio_enrol_start) #endif /* _UAPI_LINUX_APPLE_SEP_H */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 30aec75df0efe..c481404713aaa 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3476,7 +3476,11 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) } #endif /* CONFIG_SMP */ -#ifdef CONFIG_NUMA_BALANCING +#ifdef CONFIG_SMP +/* + * Swapping two running tasks is used by NUMA balancing and by misfit + * rotation on asymmetric CPU capacity systems. + */ static void __migrate_swap_task(struct task_struct *p, int cpu) { if (task_on_rq_queued(p)) { @@ -3574,13 +3578,15 @@ int migrate_swap(struct task_struct *cur, struct task_struct *p, if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr)) goto out; +#ifdef CONFIG_NUMA_BALANCING trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); +#endif ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); out: return ret; } -#endif /* CONFIG_NUMA_BALANCING */ +#endif /* CONFIG_SMP */ /*** * kick_process - kick a running thread to enter/exit the kernel @@ -4658,6 +4664,7 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p) p->se.on_rq = 0; p->se.exec_start = 0; p->se.sum_exec_runtime = 0; + p->misfit_runtime = 0; p->se.prev_sum_exec_runtime = 0; p->se.nr_migrations = 0; p->se.vruntime = 0; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 90ea4c950b952..3b38f35c65413 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6353,6 +6353,8 @@ static inline void update_misfit_status(struct task_struct *p, struct rq *rq) task_fits_cpu(p, cpu)) { rq->misfit_task_load = 0; + if (p) + p->misfit_runtime = p->se.sum_exec_runtime; return; } @@ -8219,6 +8221,9 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) unsigned long weight; bool curr; + if (flags & ENQUEUE_WAKEUP) + p->misfit_runtime = se->sum_exec_runtime; + if (task_is_throttled(p) && enqueue_throttled_task(p)) return; @@ -9998,6 +10003,8 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) { struct sched_entity *se = &p->se; + p->misfit_runtime = se->sum_exec_runtime; + if (!task_on_rq_migrating(p)) { remove_entity_load_avg(se); @@ -15025,6 +15032,140 @@ static __latent_entropy void sched_balance_softirq(void) sched_balance_domains(this_rq, idle); } +/* + * Misfit rotation. + * + * Load balancing moves a misfit task to a bigger CPU only when a bigger CPU + * has spare capacity. When every bigger CPU runs one task of its own, the + * misfit task stays on its small CPU for as long as those tasks run, so + * equal tasks make unequal progress. Work split evenly across all CPUs then + * ends with a tail in which the tasks that started on small CPUs finish + * alone on big ones. + * + * Once a task has executed as a misfit for a few base slices, swap it with + * the task on a busy bigger CPU. Bigger CPUs are taken round robin, so each + * task on them takes its turn on a smaller CPU: the big CPUs are time + * shared, as one runqueue would share them. It only applies when every + * allowed bigger CPU runs exactly one fair task (an idle one pulls the + * misfit task through load balancing instead), and it never moves down a + * task with a higher weight or a higher uclamp_min than the misfit task. + */ +/* Serialize destination selection through completion of the swap. */ +static DEFINE_MUTEX(misfit_rotate_mutex); +static int misfit_rotate_last_cpu; + +static bool misfit_rotate_candidate(struct task_struct *p) +{ + return p && p->sched_class == &fair_sched_class && + !(p->flags & (PF_EXITING | PF_KTHREAD)) && + p->nr_cpus_allowed > 1; +} + +static void misfit_rotate_work(struct work_struct *work) +{ + struct rq *src_rq = container_of(work, struct rq, misfit_rotate_work); + int src_cpu = cpu_of(src_rq), dst_cpu = -1, cpu, start; + unsigned long src_cap = arch_scale_cpu_capacity(src_cpu); + struct task_struct *p, *t = NULL; + bool idle = false; + int ret = -EINVAL; + + p = xchg(&src_rq->misfit_rotate_p, NULL); + if (!p) + return; + + mutex_lock(&misfit_rotate_mutex); + + if (!sched_feat(MISFIT_ROTATE)) { + ret = -ECANCELED; + goto out; + } + + /* The task may have moved, slept or shrunk since the tick. */ + if (task_cpu(p) != src_cpu || !task_on_rq_queued(p) || + !misfit_rotate_candidate(p) || task_fits_cpu(p, src_cpu)) + goto out; + + rcu_read_lock(); + start = misfit_rotate_last_cpu; + for_each_cpu_wrap(cpu, p->cpus_ptr, start + 1) { + struct rq *rq = cpu_rq(cpu); + struct task_struct *c; + + if (arch_scale_cpu_capacity(cpu) <= src_cap || !cpu_active(cpu)) + continue; + + if (!READ_ONCE(rq->nr_running)) { + idle = true; + break; + } + + if (t || READ_ONCE(rq->nr_running) != 1) + continue; + + c = rcu_dereference_all(rq->curr); + if (!misfit_rotate_candidate(c) || + !cpumask_test_cpu(src_cpu, c->cpus_ptr)) + continue; + + if (c->se.load.weight > p->se.load.weight || + uclamp_eff_value(c, UCLAMP_MIN) > uclamp_eff_value(p, UCLAMP_MIN)) + continue; + + t = c; + dst_cpu = cpu; + } + + if (idle) + t = NULL; + if (t) + get_task_struct(t); + rcu_read_unlock(); + + ret = idle ? -EAGAIN : -EBUSY; + if (t) { + ret = migrate_swap(p, t, dst_cpu, src_cpu); + if (!ret) + misfit_rotate_last_cpu = dst_cpu; + } +out: + trace_sched_misfit_rotate(p, t, src_cpu, dst_cpu, ret); + if (t) + put_task_struct(t); + mutex_unlock(&misfit_rotate_mutex); + put_task_struct(p); +} + +/* Called from the tick on this CPU, so current is rq->curr. */ +static void misfit_rotate_trigger(struct rq *rq) +{ + struct task_struct *p = current; + + if (!sched_feat(MISFIT_ROTATE) || !sched_asym_cpucap_active()) + return; + + if (!READ_ONCE(rq->misfit_task_load) || + READ_ONCE(rq->nr_running) != 1 || !misfit_rotate_candidate(p)) + return; + + /* Preemption must not restart a CPU-bound task's misfit interval. */ + if (p->se.sum_exec_runtime - p->misfit_runtime < + 4ULL * sysctl_sched_base_slice) + return; + + if (READ_ONCE(rq->misfit_rotate_p)) + return; + + get_task_struct(p); + if (cmpxchg(&rq->misfit_rotate_p, NULL, p)) { + put_task_struct(p); + return; + } + p->misfit_runtime = p->se.sum_exec_runtime; + trace_sched_misfit_rotate(p, NULL, cpu_of(rq), -1, -EINPROGRESS); + queue_work_on(cpu_of(rq), system_highpri_wq, &rq->misfit_rotate_work); +} + /* * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. */ @@ -15037,6 +15178,8 @@ void sched_balance_trigger(struct rq *rq) if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq)))) return; + misfit_rotate_trigger(rq); + if (time_after_eq(jiffies, rq->next_balance)) raise_softirq(SCHED_SOFTIRQ); @@ -15942,6 +16085,7 @@ __init void init_sched_fair_class(void) zalloc_cpumask_var_node(&per_cpu(select_rq_mask, i), GFP_KERNEL, cpu_to_node(i)); zalloc_cpumask_var_node(&per_cpu(should_we_balance_tmpmask, i), GFP_KERNEL, cpu_to_node(i)); + INIT_WORK(&cpu_rq(i)->misfit_rotate_work, misfit_rotate_work); #ifdef CONFIG_CFS_BANDWIDTH INIT_CSD(&cpu_rq(i)->cfsb_csd, __cfsb_csd_unthrottle, cpu_rq(i)); diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 8f0dee8fc4755..07132f33d1e4a 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -135,6 +135,13 @@ SCHED_FEAT(WA_BIAS, true) */ SCHED_FEAT(UTIL_EST, true) +/* + * On asymmetric CPU capacity systems, swap a task that has been stuck as a + * misfit on a smaller CPU with a task on a busy bigger CPU, so equal tasks + * get an equal share of the big CPUs. + */ +SCHED_FEAT(MISFIT_ROTATE, true) + SCHED_FEAT(LATENCY_WARN, false) /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 4c25fbe84fb54..9dd8c6e6f73b5 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1285,6 +1285,9 @@ struct rq { unsigned char idle_balance; unsigned long misfit_task_load; + /* Task reference handed to the misfit rotation work. */ + struct task_struct *misfit_rotate_p; + struct work_struct misfit_rotate_work; /* For active balancing */ int active_balance; @@ -2117,6 +2120,17 @@ static inline int sched_numa_find_closest(const struct cpumask *cpus, int cpu) #endif /* !CONFIG_NUMA */ +#ifdef CONFIG_SMP +extern int migrate_swap(struct task_struct *p, struct task_struct *t, + int cpu, int scpu); +#else +static inline int migrate_swap(struct task_struct *p, struct task_struct *t, + int cpu, int scpu) +{ + return -EINVAL; +} +#endif + #ifdef CONFIG_NUMA_BALANCING /* The regions in numa_faults array from task_struct */ @@ -2129,8 +2143,6 @@ enum numa_faults_stats { extern void sched_setnuma(struct task_struct *p, int node); extern int migrate_task_to(struct task_struct *p, int cpu); -extern int migrate_swap(struct task_struct *p, struct task_struct *t, - int cpu, int scpu); extern void init_numa_balancing(u64 clone_flags, struct task_struct *p); #else /* !CONFIG_NUMA_BALANCING: */ diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 23aa685923368..145711b2ccf00 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -68,7 +69,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -106,6 +109,7 @@ #include #include #include +#include #include #include #include @@ -162,8 +166,17 @@ const snd_ctl_elem_iface_t BINDINGS_SNDRV_CTL_ELEM_IFACE_MIXER = SNDRV_CTL_ELEM_ const u32 BINDINGS_IIO_CHAN_INFO_RAW = IIO_CHAN_INFO_RAW; const u32 BINDINGS_IIO_CHAN_INFO_PROCESSED = IIO_CHAN_INFO_PROCESSED; +const u32 BINDINGS_IIO_CHAN_INFO_SCALE = IIO_CHAN_INFO_SCALE; const u32 BINDINGS_IIO_ANGL = IIO_ANGL; const u32 BINDINGS_IIO_LIGHT = IIO_LIGHT; +const u32 BINDINGS_IIO_CCT = IIO_CCT; +const u32 BINDINGS_IIO_ACCEL = IIO_ACCEL; +const u32 BINDINGS_IIO_ANGL_VEL = IIO_ANGL_VEL; +const u32 BINDINGS_IIO_TIMESTAMP = IIO_TIMESTAMP; +const u32 BINDINGS_IIO_MOD_X = IIO_MOD_X; +const u32 BINDINGS_IIO_MOD_Y = IIO_MOD_Y; +const u32 BINDINGS_IIO_MOD_Z = IIO_MOD_Z; +const u32 BINDINGS_IIO_CHAN_INFO_SAMP_FREQ = IIO_CHAN_INFO_SAMP_FREQ; const xa_mark_t RUST_CONST_HELPER_XA_PRESENT = XA_PRESENT; diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 88709a7f9b36d..81f0a90e4a517 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -44,6 +44,11 @@ pub trait DriverFile { fn memory_stats(&self) -> Option { None } + + /// Emit completed per-engine busy time in nanoseconds for this client's fdinfo. + /// Engine names must be stable, unique and contain no whitespace or colons. + /// The default emits no engine keys. See `Documentation/gpu/drm-usage-stats.rst`. + fn engine_stats(&self, _emit: impl FnMut(&crate::str::CStr, u64)) {} } /// An open DRM File. @@ -136,6 +141,18 @@ impl File { // `driver_priv` was set by `open_callback()`. The seq_file read holds a reference to the // file, so `postclose_callback()` cannot run concurrently. let file = unsafe { File::::from_raw(raw_file) }; + file.inner().engine_stats(|name, ns| { + // SAFETY: The DRM core owns printer throughout this callback; the format and + // name are NUL-terminated, and the vararg types match %s and %llu. + unsafe { + bindings::drm_printf( + printer, + c"drm-engine-%s:\t%llu ns\n".as_char_ptr(), + name.as_char_ptr(), + ns as core::ffi::c_ulonglong, + ) + }; + }); let Some(stats) = file.inner().memory_stats() else { return; }; @@ -164,7 +181,7 @@ impl File { // SAFETY: `printer` is the printer `drm_show_fdinfo()` passes, valid for this call; `raw` // and the region name outlive the call. unsafe { - bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_ptr()) + bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_char_ptr()) }; } diff --git a/rust/kernel/iio/common/aop_sensors.rs b/rust/kernel/iio/common/aop_sensors.rs index 7c80cef56b92f..dfd42f01a582e 100644 --- a/rust/kernel/iio/common/aop_sensors.rs +++ b/rust/kernel/iio/common/aop_sensors.rs @@ -41,28 +41,54 @@ pub trait MessageProcessor: Send + Sync { /// Returns the reading carried by `message` in micro-units of the channel (millionths of a /// lux, of a degree, ...), or `None` when the message carries no reading. fn process(&self, message: &[u8]) -> Option; + + /// Returns the fixed scale of a raw channel as an integer and a fractional part in + /// billionths, or `None` for a channel without a scale attribute. + fn scale(&self) -> Option<(i32, i32)> { + None + } + + /// Decodes a channel of a multi-channel sensor. Single-channel processors need only + /// implement [`Self::process`]. A missing channel leaves its previous reading in place. + fn process_channel(&self, message: &[u8], channel: usize) -> Option { + if channel == 0 { + self.process(message) + } else { + None + } + } } /// The state shared between a sensor's report listener and its IIO device: the latest reading. -pub struct AopSensorData { +pub struct AopSensorData { dev: ARef, - ty: u32, + ty: [u32; N], /// The latest reading in micro-units, at most [`MAX_MICRO`]. - value: Atomic, + value: [Atomic; N], /// Whether a reading has arrived at all; until then reads return `ENODATA`. - valid: Atomic, + valid: [Atomic; N], msg_proc: T, } impl AopSensorData { /// Creates the state for one channel of type `ty` whose reports `msg_proc` decodes. pub fn new(dev: ARef, ty: u32, msg_proc: T) -> Result>> { + Self::new_channels(dev, [ty], msg_proc) + } +} + +impl AopSensorData { + /// Creates state for the channels in `ty`, in the order decoded by the processor. + pub fn new_channels(dev: ARef, ty: [u32; N], msg_proc: T) -> Result> { + if N == 0 { + return Err(EINVAL); + } Ok(Arc::new( AopSensorData { dev, ty, - value: Atomic::new(0), - valid: Atomic::new(false), + value: [const { Atomic::new(0) }; N], + valid: [const { Atomic::new(false) }; N], msg_proc, }, GFP_KERNEL, @@ -70,13 +96,15 @@ impl AopSensorData { } } -impl FakehidListener for AopSensorData { +impl FakehidListener for AopSensorData { fn process_fakehid_report(&self, data: &[u8]) -> Result<()> { // A report without a reading leaves the last reading in place. - if let Some(value) = self.msg_proc.process(data) { - self.value.store(value.min(MAX_MICRO), Relaxed); - // Publishes the reading above to readers that see `valid`. - self.valid.store(true, Release); + for channel in 0..N { + if let Some(value) = self.msg_proc.process_channel(data, channel) { + self.value[channel].store(value.min(MAX_MICRO), Relaxed); + // Publishes the reading above to readers that see `valid`. + self.valid[channel].store(true, Release); + } } Ok(()) } @@ -84,7 +112,7 @@ impl FakehidListener for AopSensorData { /// The `read_raw` callback: processed values are reported with their micro part, raw values as /// integers. -unsafe extern "C" fn aop_read_raw( +unsafe extern "C" fn aop_read_raw( dev: *mut bindings::iio_dev, chan: *const bindings::iio_chan_spec, val: *mut i32, @@ -93,18 +121,32 @@ unsafe extern "C" fn aop_read_raw( ) -> i32 { // SAFETY: The IIO core calls this with the device and channel that `IIORegistration::new()` // registered; `priv_` is the `Arc` it stored there. - let data = unsafe { Arc::>::borrow((*dev).priv_.cast()) }; + let data = unsafe { Arc::>::borrow((*dev).priv_.cast()) }; // SAFETY: `chan` is one of the registered channel specs. let ty = unsafe { (*chan).type_ }; - if data.ty != ty { + // SAFETY: the registration assigns each channel its index as its address. + let channel = unsafe { (*chan).address as usize }; + if data.ty.get(channel) != Some(&ty) { return EINVAL.to_errno(); } + // A fixed scale is available even before the first sensor report. + if mask == bindings::BINDINGS_IIO_CHAN_INFO_SCALE as isize { + let Some((int, frac)) = data.msg_proc.scale() else { + return EINVAL.to_errno(); + }; + // SAFETY: `val` and `val2` point to the caller's result variables. + unsafe { + *val = int; + *val2 = frac; + } + return bindings::IIO_VAL_INT_PLUS_NANO as i32; + } // A sensor that has not reported yet, for instance because it lacks its // calibration, has no reading rather than a reading of zero. - if !data.valid.load(Acquire) { + if !data.valid[channel].load(Acquire) { return ENODATA.to_errno(); } - let micro = data.value.load(Relaxed); + let micro = data.value[channel].load(Relaxed); // Both parts fit an `i32` because stored readings never exceed `MAX_MICRO`. let (int, frac) = ((micro / MICRO) as i32, (micro % MICRO) as i32); // SAFETY: `val` and `val2` point to the caller's result variables. @@ -121,18 +163,18 @@ unsafe extern "C" fn aop_read_raw( } } -struct IIOSpec { - spec: [bindings::iio_chan_spec; 1], +struct IIOSpec { + spec: [bindings::iio_chan_spec; N], vtable: bindings::iio_info, _p: PhantomPinned, } /// TODO: add documentation -pub struct IIORegistration { +pub struct IIORegistration { dev: *mut bindings::iio_dev, - spec: Pin>, + spec: Pin>>, registered: bool, - _p: PhantomData>, + _p: PhantomData>, } impl IIORegistration { @@ -144,10 +186,27 @@ impl IIORegistration { info_mask: usize, module: &ThisModule, ) -> Result { + Self::new_channels(data, name, [ty], info_mask, module) + } +} + +impl IIORegistration { + /// Registers the sensor's channels. The types must match those of `data`. + pub fn new_channels( + data: Arc>, + name: &'static CStr, + ty: [u32; N], + info_mask: usize, + module: &ThisModule, + ) -> Result { + if ty != data.ty { + return Err(EINVAL); + } let spec = KBox::pin( IIOSpec { - spec: [bindings::iio_chan_spec { - type_: ty, + spec: core::array::from_fn(|channel| bindings::iio_chan_spec { + type_: ty[channel], + address: channel as _, __bindgen_anon_1: bindings::iio_chan_spec__bindgen_ty_1 { scan_type: bindings::iio_scan_type { __bindgen_anon_1: bindings::iio_scan_type__bindgen_ty_1 { @@ -160,9 +219,9 @@ impl IIORegistration { }, info_mask_separate: info_mask, ..Default::default() - }], + }), vtable: bindings::iio_info { - read_raw: Some(aop_read_raw::), + read_raw: Some(aop_read_raw::), ..Default::default() }, _p: PhantomPinned, @@ -197,19 +256,19 @@ impl IIORegistration { } } -impl Drop for IIORegistration { +impl Drop for IIORegistration { fn drop(&mut self) { if self.dev != ptr::null_mut() { unsafe { if self.registered { bindings::iio_device_unregister(self.dev); } - Arc::>::from_foreign((*self.dev).priv_.cast()); + Arc::>::from_foreign((*self.dev).priv_.cast()); bindings::iio_device_free(self.dev); } } } } -unsafe impl Send for IIORegistration {} -unsafe impl Sync for IIORegistration {} +unsafe impl Send for IIORegistration {} +unsafe impl Sync for IIORegistration {} diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index c30983562a2f1..51e51790cb4f6 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -222,6 +222,32 @@ if [ "$bindgen_libclang_cversion" -ge 2200000 ] && fi fi +# Check the behavior, not the version: distributions can backport the repair. +# Warn rather than making CONFIG_RUST unavailable, since that would silently +# remove Rust drivers from a configuration. The kernel's layout assertions +# remain the final check for every generated type. +bindgen_alignment_output=$( \ + LC_ALL=C "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_alignment.h \ + --rust-target 1.85 2>/dev/null +) || bindgen_alignment_code=$? +if [ -n "$bindgen_alignment_code" ] || + ! echo "$bindgen_alignment_output" | grep -Eq '#\[repr\(.*align\(64\)'; then + echo >&2 "***" + if [ -n "$bindgen_alignment_code" ]; then + echo >&2 "*** Could not check empty struct alignment with '$BINDGEN'." + echo >&2 "*** Exit code: $bindgen_alignment_code" + else + echo >&2 "*** Rust bindings generator '$BINDGEN'" + echo >&2 "*** does not preserve empty struct alignment." + fi + echo >&2 "*** Default Asahi builds can fail the sched_statistics layout assertion." + echo >&2 "*** Prepare the corrected generator before configuring the kernel:" + echo >&2 "*** python3 tools/aurora-bindgen/build.py" + echo >&2 "*** See Documentation/rust/bindgen-layout-checks.rst for details." + echo >&2 "***" + warning=1 +fi + # If the C compiler is Clang, then we can also check whether its version # matches the `libclang` version used by the Rust bindings generator. # diff --git a/scripts/rust_is_available_bindgen_alignment.h b/scripts/rust_is_available_bindgen_alignment.h new file mode 100644 index 0000000000000..d27e21fa9003d --- /dev/null +++ b/scripts/rust_is_available_bindgen_alignment.h @@ -0,0 +1,3 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* A defined GNU C empty record still has alignment, despite having no storage. */ +struct rust_bindgen_aligned_empty {} __attribute__((aligned(64))); diff --git a/scripts/rust_is_available_test.py b/scripts/rust_is_available_test.py index 22bdff980c35e..36da4e576396f 100755 --- a/scripts/rust_is_available_test.py +++ b/scripts/rust_is_available_test.py @@ -20,6 +20,7 @@ class TestRustIsAvailable(unittest.TestCase): class Expected(enum.Enum): SUCCESS = enum.auto() SUCCESS_WITH_WARNINGS = enum.auto() + SUCCESS_WITH_OPTIONAL_WARNINGS = enum.auto() SUCCESS_WITH_EXTRA_OUTPUT = enum.auto() FAILURE = enum.auto() @@ -54,7 +55,8 @@ def generate_rustc(cls, stdout): """) @classmethod - def generate_bindgen(cls, version_stdout, libclang_stderr, libclang_22_patched=False): + def generate_bindgen(cls, version_stdout, libclang_stderr, libclang_22_patched=False, + empty_alignment=True): if libclang_stderr is None: libclang_case = f"raise SystemExit({cls.bindgen_default_bindgen_libclang_failure_exit_code})" else: @@ -71,6 +73,8 @@ def generate_bindgen(cls, version_stdout, libclang_stderr, libclang_22_patched=F {libclang_case} elif "rust_is_available_bindgen_libclang_22.h" in " ".join(sys.argv): {libclang_22_case} +elif "rust_is_available_bindgen_alignment.h" in " ".join(sys.argv): + print({repr('#[repr(C, align(64))]' if empty_alignment else '#[repr(C)]')}) else: print({repr(version_stdout)}) """) @@ -142,6 +146,10 @@ def run_script(self, expected, override_env): # be checked further by the test). self.assertEqual(result.returncode, 0) self.assertNotEqual(result.stderr, b"") + elif expected == self.Expected.SUCCESS_WITH_OPTIONAL_WARNINGS: + self.assertEqual(result.returncode, 0) + if result.stderr: + self.assertIn(b"Please see Documentation/rust/quick-start.rst for details", result.stderr) elif expected == self.Expected.SUCCESS_WITH_WARNINGS: # When expecting a success with warnings, the script should return 0 # and it should output at least the instructions to `stderr`. @@ -338,6 +346,33 @@ def test_success_bindgen_version(self): bindgen = self.generate_bindgen_version(bindgen_stdout) result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen }) + def test_bindgen_empty_alignment_unpatched(self): + # The same version can be patched by a distribution, so test behavior. + bindgen = self.generate_bindgen("bindgen 0.73.2", + self.bindgen_default_bindgen_libclang_stderr, + empty_alignment=False) + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) + self.assertIn("does not preserve empty struct alignment", result.stderr) + self.assertIn("python3 tools/aurora-bindgen/build.py", result.stderr) + + def test_bindgen_empty_alignment_patched(self): + bindgen = self.generate_bindgen("bindgen 0.73.2", + self.bindgen_default_bindgen_libclang_stderr) + self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen }) + + def test_bindgen_empty_alignment_probe_failure(self): + bindgen = self.generate_executable(f'''#!/usr/bin/env python3 +import os +import sys +if "rust_is_available_bindgen_alignment.h" in " ".join(sys.argv): + print("#[repr(C, align(64))]") + print("probe failed", file=sys.stderr) + raise SystemExit(1) +os.execv({str(self.default_bindgen)!r}, [{str(self.default_bindgen)!r}] + sys.argv[1:]) +''') + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) + self.assertIn("Could not check empty struct alignment", result.stderr) + def test_success_bindgen_libclang(self): for stderr in ( f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} (https://github.com/llvm/llvm-project.git 4a2c05b05ed07f1f620e94f6524a8b4b2760a0b1) [-W#pragma-messages], err: false", @@ -370,7 +405,7 @@ def test_success_clang_version(self): def test_success_real_programs(self): for cc in ["gcc", "clang"]: with self.subTest(cc=cc): - result = self.run_script(self.Expected.SUCCESS, { + result = self.run_script(self.Expected.SUCCESS_WITH_OPTIONAL_WARNINGS, { "PATH": os.environ["PATH"], "RUSTC": "rustc", "BINDGEN": "bindgen", diff --git a/sound/soc/apple/aop_spkr.rs b/sound/soc/apple/aop_spkr.rs index b5b2f5a7f6acf..64efaa83ce8d6 100644 --- a/sound/soc/apple/aop_spkr.rs +++ b/sound/soc/apple/aop_spkr.rs @@ -1542,9 +1542,16 @@ fn to_kvec( Ok(v) } +/// The first PCM device number of the back-ends' internal PCMs, above every front-end's. +const BE_PCM_BASE: i32 = 16; + fn component_driver(name: &'static CStr) -> bindings::snd_soc_component_driver { bindings::snd_soc_component_driver { name: name.as_char_ptr(), + // A front-end's PCM device is its link number, so a front-end that macaudio leaves + // unbound (one that only completes the fixed routes) does not renumber the others. + use_dai_pcm_id: true, + be_pcm_base: BE_PCM_BASE, open: Some(pcm_open), close: Some(pcm_close), hw_params: Some(pcm_hw_params), diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 1d1c8924eb2fb..569cf80387dff 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -120,6 +120,14 @@ struct macaudio_platform_cfg { * for its own card). */ bool fe_cpu_is_platform; + /* + * Front-ends (bits by link index) whose CPU DAI only completes the + * fixed DAPM routes and can never carry a stream on this machine. + * Their links are not bound, so the card has no PCM device for them + * that userspace could probe and fail to open. The CPU component + * keeps the other front-ends on their link numbers. + */ + unsigned int route_only_fes; /* * The highest headphone volume, a "Jack DAC Playback Volume" value * (CS42L84_DAC_DB()), 0 for the codec's own maximum. @@ -859,6 +867,8 @@ static int macaudio_parse_of(struct macaudio_snd_data *ma) memcpy(link_props, &macaudio_fe_link_props[i], sizeof(struct macaudio_link_props)); if (ma->cfg->fe_dai_names[i]) link->cpus[0].dai_name = ma->cfg->fe_dai_names[i]; + if (ma->cfg->route_only_fes & BIT(i)) + link->ignore = 1; if (ma->cfg->dai_fmt && !link_props->is_sense) link->dai_fmt = ma->cfg->dai_fmt; if (i == 0 && ma->cfg->primary_bclk_ratio) @@ -1876,8 +1886,18 @@ static int macaudio_slk_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_v { struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + long val = uvalue->value.integer.value[0]; + + /* + * Reads return the lock state (0 or 1), so a saved mixer state holds + * one of those. Writing it back changes nothing: only the lock owner's + * magic value renews the lease. Accept it as a no-op so that restoring + * a saved state before the daemon has taken the lock is not an error. + */ + if (val == 0 || val == 1) + return 0; - if (uvalue->value.integer.value[0] != SPEAKER_MAGIC_VALUE) + if (val != SPEAKER_MAGIC_VALUE) return -EINVAL; mutex_lock(&ma->volume_lock_mutex); @@ -2117,6 +2137,8 @@ struct macaudio_platform_cfg macaudio_j616_cfg = { SND_SOC_DAIFMT_GATED | SND_SOC_DAIFMT_NB_IF, .secondary_bclk_ratio = 250, .fe_cpu_is_platform = true, + /* No headphone codec on this card: the jack is a card of its own. */ + .route_only_fes = BIT(0), }; /* @@ -2133,6 +2155,7 @@ struct macaudio_platform_cfg macaudio_j616_jack_cfg = { .fe_dai_names = { "j616-jack-pcm-0", "j616-jack-pcm-1" }, .dai_fmt = SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_CBC_CFC | SND_SOC_DAIFMT_IB_NF, .fe_cpu_is_platform = true, + .route_only_fes = BIT(1), /* * Above 0 dB the DAC only adds digital gain, which clips a full-scale * signal. The 6 dB below that are margin: the analog full scale of diff --git a/sound/soc/codecs/tas2764.c b/sound/soc/codecs/tas2764.c index 399aa42c6414a..4bb710efe35f1 100644 --- a/sound/soc/codecs/tas2764.c +++ b/sound/soc/codecs/tas2764.c @@ -55,7 +55,7 @@ struct tas2764_priv { struct snd_soc_component *component; struct gpio_desc *reset_gpio; struct gpio_desc *sdz_gpio; - struct regulator *sdz_reg; + struct regulator *sdz_reg; /* NULL when the SDZ line is a GPIO or not described */ struct regmap *regmap; struct device *dev; int irq; @@ -106,7 +106,7 @@ static int tas2764_sdz_enable(struct tas2764_priv *tas2764) int ret; if (!tas2764->sdz_group) - return regulator_enable(tas2764->sdz_reg); + return tas2764->sdz_reg ? regulator_enable(tas2764->sdz_reg) : 0; mutex_lock(&tas2764_sdz_groups_lock); ret = READ_ONCE(tas2764->sdz_group->failed) ? -EIO : regulator_enable(tas2764->sdz_reg); @@ -141,8 +141,10 @@ static int tas2764_group_join(struct device *dev, struct tas2764_priv *tas2764) if (!tas2764->irq) return dev_err_probe(dev, -EINVAL, "SDZ group without a fault interrupt\n"); supply = of_parse_phandle(dev->of_node, "SDZ-supply", 0); - if (!supply) + if (!supply || !tas2764->sdz_reg) { + of_node_put(supply); return dev_err_probe(dev, -EINVAL, "SDZ group without an SDZ supply\n"); + } mutex_lock(&tas2764_sdz_groups_lock); list_for_each_entry(group, &tas2764_sdz_groups, node) { @@ -295,7 +297,8 @@ static int tas2764_codec_suspend(struct snd_soc_component *component) if (tas2764->sdz_gpio) gpiod_set_value_cansleep(tas2764->sdz_gpio, 0); - regulator_disable(tas2764->sdz_reg); + if (tas2764->sdz_reg) + regulator_disable(tas2764->sdz_reg); } regcache_cache_only(tas2764->regmap, true); @@ -312,7 +315,7 @@ static int tas2764_codec_resume(struct snd_soc_component *component) int ret; if (!tas2764->sdz_group) { - ret = regulator_enable(tas2764->sdz_reg); + ret = tas2764->sdz_reg ? regulator_enable(tas2764->sdz_reg) : 0; if (ret) { dev_err(tas2764->dev, "Failed to enable regulator\n"); return ret; @@ -1011,7 +1014,8 @@ static void tas2764_codec_remove(struct snd_soc_component *component) if (tas2764_group_failed(tas2764)) return; - regulator_disable(tas2764->sdz_reg); + if (tas2764->sdz_reg) + regulator_disable(tas2764->sdz_reg); } static DECLARE_TLV_DB_SCALE(tas2764_digital_tlv, 1100, 50, 0); @@ -1116,10 +1120,19 @@ static int tas2764_parse_dt(struct device *dev, struct tas2764_priv *tas2764) { int ret = 0; - tas2764->sdz_reg = devm_regulator_get(dev, "SDZ"); - if (IS_ERR(tas2764->sdz_reg)) - return dev_err_probe(dev, PTR_ERR(tas2764->sdz_reg), - "Failed to get SDZ supply\n"); + /* + * The SDZ supply is optional: most boards drive SDZ from a GPIO + * (shutdown-gpios, shared between amps through the GPIO core), and only + * boards that model the line as a regulator have one. A dummy regulator + * in its place did nothing but warn on every boot. + */ + tas2764->sdz_reg = devm_regulator_get_optional(dev, "SDZ"); + if (IS_ERR(tas2764->sdz_reg)) { + if (PTR_ERR(tas2764->sdz_reg) != -ENODEV) + return dev_err_probe(dev, PTR_ERR(tas2764->sdz_reg), + "Failed to get SDZ supply\n"); + tas2764->sdz_reg = NULL; + } tas2764->reset_gpio = devm_gpiod_get_optional(tas2764->dev, "reset", GPIOD_OUT_HIGH); diff --git a/tools/accel/ane-bench.py b/tools/accel/ane-bench.py new file mode 100755 index 0000000000000..eff0879d5eabc --- /dev/null +++ b/tools/accel/ane-bench.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Collect ANE workload output, system power and idle recovery in one receipt.""" +import argparse +import errno +import grp +import json +import os +from pathlib import Path +import platform +import statistics +import subprocess +import sys +import tempfile +import time + + +def check_device(node): + try: + fd = os.open(node, os.O_RDWR | os.O_CLOEXEC) + except OSError as exc: + if exc.errno in (errno.EACCES, errno.EPERM): + try: + group = grp.getgrgid(Path(node).stat().st_gid).gr_name + except (OSError, KeyError): + group = "the device's owning group" + raise RuntimeError( + f"Permission denied opening {node}: request access through " + f"the installed device-access rule or membership in {group}; " + "start a new login after changing group membership." + ) from exc + raise RuntimeError(f"Cannot open {node}: {exc.strerror}") from exc + else: + os.close(fd) + + +def power_sensor(): + for hwmon in Path("/sys/class/hwmon").glob("hwmon*"): + try: + if (hwmon / "name").read_text().strip() != "macsmc_hwmon": + continue + for label in hwmon.glob("power*_label"): + if label.read_text().strip().lower() == "total system power": + return label.with_name(label.name.replace("_label", "_input")) + except OSError: + continue + raise RuntimeError("Total System Power sensor unavailable in macsmc_hwmon") + + +def snapshot(device): + result = {} + for name in ("performance_level", "memory_vote", "performance_error", + "power/runtime_status", "ane_stats"): + try: + result[name] = (device / name).read_text().strip() + except OSError: + result[name] = None + return result + + +def sample_power(sensor, seconds=None, process=None): + samples = [] + start = time.monotonic() + while ((process is None and time.monotonic() - start < seconds) or + (process is not None and process.poll() is None)): + samples.append({"seconds": time.monotonic() - start, + "watts": int(sensor.read_text()) / 1_000_000}) + time.sleep(0.5) + values = [s["watts"] for s in samples] + return {"duration_s": time.monotonic() - start, "samples": samples, + "mean_W": statistics.mean(values) if values else None} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--device", default="/dev/accel/accel0") + parser.add_argument("--idle-seconds", type=float, default=10) + parser.add_argument("--check", action="store_true", help="check device access only") + parser.add_argument("command", nargs=argparse.REMAINDER, + help="-- followed by an installed ANE workload command") + args = parser.parse_args() + if args.idle_seconds <= 0: + parser.error("--idle-seconds must be positive") + try: + check_device(args.device) + if args.check: + print(f"Access to {args.device}: OK") + return 0 + command = args.command + if command and command[0] == "--": + command = command[1:] + if not command: + parser.error("supply a workload command after --, or use --check") + sensor = power_sensor() + time.sleep(3) # Let the access check's runtime-PM reference expire. + device = Path("/sys/class/accel") / Path(args.device).name / "device" + receipt = {"schema": 1, "kernel": platform.release(), "command": command, + "power_sensor": str(sensor), "before": snapshot(device)} + receipt["idle_before"] = sample_power(sensor, args.idle_seconds) + # A file avoids a full stdout pipe stalling a long-running workload. + with tempfile.TemporaryFile(mode="w+") as output: + with subprocess.Popen(command, stdout=output, stderr=subprocess.STDOUT) as proc: + try: + receipt["load"] = sample_power(sensor, process=proc) + receipt["returncode"] = proc.wait() + except BaseException: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + raise + output.seek(0) + receipt["workload_output"] = output.read() + receipt["power_comparison_valid"] = len(receipt["load"]["samples"]) >= 6 + if not receipt["power_comparison_valid"]: + receipt["power_note"] = "Fewer than 6 load samples; repeat with a longer workload." + receipt["after_load"] = snapshot(device) + time.sleep(3) # Allow the default runtime-PM delay to expire. + receipt["idle_after"] = sample_power(sensor, args.idle_seconds) + receipt["after_idle"] = snapshot(device) + print(json.dumps(receipt, indent=2)) + return 0 if receipt["returncode"] == 0 else 1 + except (RuntimeError, OSError) as exc: + print(f"ane-bench: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/asahi/aop-als-channel-tests.py b/tools/asahi/aop-als-channel-tests.py new file mode 100644 index 0000000000000..d3f5f3ed48d87 --- /dev/null +++ b/tools/asahi/aop-als-channel-tests.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production report decoder, per-channel cache and IIO callback.""" +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] + + +def source(): + common = (ROOT / 'rust/kernel/iio/common/aop_sensors.rs').read_text() + als = (ROOT / 'drivers/iio/common/aop_sensors/aop_als.rs').read_text() + common = common[common.index('pub const MICRO:'):common.index('struct IIOSpec')] + constants = '\n'.join(line for line in als.splitlines() + if line.startswith('const LUX_OFFSET_') or + line.startswith('const CCT_OFFSET_') or + line.startswith('const VD6286_COLOR_REPORT_LEN')) + conversion = als[als.index('fn f32_to_micro('):als.index('enum AlsIio')] + return STUBS + common + constants + conversion + TESTS + + +STUBS = r''' +#![allow(dead_code, non_camel_case_types)] +use std::{ops::Deref, sync::atomic::Ordering::{Acquire, Relaxed, Release}}; +#[derive(Debug)] pub struct Error(i32); +impl Error { fn to_errno(&self)->i32 { self.0 } } +const EINVAL:Error=Error(-22); const ENODATA:Error=Error(-61); +type Result = std::result::Result; +const GFP_KERNEL:u32=0; +mod device { pub struct Device; } +type ARef = std::sync::Arc; +pub struct Arc(std::sync::Arc); +impl Arc { + fn new(value:T,_flags:u32)->Result { Ok(Self(std::sync::Arc::new(value))) } + unsafe fn borrow<'a>(ptr:*const T)->&'a T { unsafe { &*ptr } } +} +impl Deref for Arc { type Target=T; fn deref(&self)->&T { &self.0 } } +struct Atomic(std::sync::Mutex); +impl Atomic { + const fn new(value:T)->Self { Self(std::sync::Mutex::new(value)) } + fn store(&self,value:T,_order:std::sync::atomic::Ordering) { *self.0.lock().unwrap()=value; } + fn load(&self,_order:std::sync::atomic::Ordering)->T { *self.0.lock().unwrap() } +} +trait FakehidListener { fn process_fakehid_report(&self, data:&[u8])->Result<()>; } +mod bindings { + pub const BINDINGS_IIO_CHAN_INFO_PROCESSED:u32=2; + pub const BINDINGS_IIO_CHAN_INFO_RAW:u32=1; + pub const BINDINGS_IIO_CHAN_INFO_SCALE:u32=3; + pub const IIO_VAL_INT_PLUS_MICRO:u32=2; + pub const IIO_VAL_INT_PLUS_NANO:u32=3; + pub const IIO_VAL_INT:u32=1; + pub struct iio_dev { pub priv_:*mut std::ffi::c_void } + pub struct iio_chan_spec { pub type_:u32, pub address:u64 } +} +''' + +TESTS = r''' +fn put(report:&mut [u8],offset:usize,value:f32) { + report[offset..offset+4].copy_from_slice(&value.to_le_bytes()); +} +fn read(data:&AopSensorData,channel:u64,ty:u32)->(i32,i32,i32) { + let mut dev=bindings::iio_dev{priv_:(data as *const AopSensorData).cast_mut().cast()}; + let chan=bindings::iio_chan_spec{type_:ty,address:channel}; + let(mut a,mut b)=(-1,-1); + let ret=unsafe { aop_read_raw::(&mut dev,&chan,&mut a,&mut b,2) }; + (ret,a,b) +} +#[test] fn redbird_firmware_channels_and_abi() { + let mut report=[0u8;122]; + // Distinct sentinels at the host event's offsets catch coordinate-system mistakes. + put(&mut report,0x14,900.0); put(&mut report,0x34,1200.0); + put(&mut report,0x28,321.25); put(&mut report,0x45,5000.5); + let data=AopSensorData::new_channels(std::sync::Arc::new(device::Device),[5,6],MsgProc(LUX_OFFSET_VD6286)).unwrap(); + assert_eq!(read(&data,0,5).0,-61); assert_eq!(read(&data,1,6).0,-61); + data.process_fakehid_report(&report).unwrap(); + assert_eq!(read(&data,0,5),(2,321,250000)); + assert_eq!(read(&data,1,6),(2,5000,500000)); + assert_eq!(read(&data,2,6).0,-22); assert_eq!(read(&data,1,5).0,-22); + // Invalid lux does not prevent a valid CCT from advancing. + put(&mut report,0x28,f32::NAN); put(&mut report,0x45,4100.75); + data.process_fakehid_report(&report).unwrap(); + assert_eq!(read(&data,0,5),(2,321,250000)); assert_eq!(read(&data,1,6),(2,4100,750000)); + // Invalid CCT does not prevent lux from advancing; preserve last valid CCT. + for bad in [0.0,-1.0,f32::NAN,f32::INFINITY,f32::NEG_INFINITY] { + put(&mut report,0x45,bad); put(&mut report,0x28,0.125); + data.process_fakehid_report(&report).unwrap(); + assert_eq!(read(&data,0,5),(2,0,125000)); assert_eq!(read(&data,1,6),(2,4100,750000)); + } + // Storage saturation protects both signed IIO output parts. + put(&mut report,0x45,f32::MAX); data.process_fakehid_report(&report).unwrap(); + assert_eq!(read(&data,1,6),(2,i32::MAX,999999)); +} +#[test] fn unknown_layout_does_not_fabricate_cct() { + let decoder=MsgProc(LUX_OFFSET_VD6286); + let mut report=vec![0u8;140]; put(&mut report,0x28,2.0); put(&mut report,0x45,6000.0); + for length in 0..140 { + let sample=&report[..length]; + assert_eq!(decoder.process_channel(sample,1),if length==122 {Some(6000000000)} else {None}); + assert_eq!(decoder.process_channel(sample,0),if length>=44 {Some(2000000)} else {None}); + } + assert_eq!(decoder.process_channel(&report,2),None); + let data=AopSensorData::new_channels(std::sync::Arc::new(device::Device),[5,6],decoder).unwrap(); + data.process_fakehid_report(&report[..121]).unwrap(); + assert_eq!(read(&data,0,5),(2,2,0)); assert_eq!(read(&data,1,6).0,-61); +} +#[test] fn single_channel_clients_and_other_sensors() { + struct Scalar; + impl MessageProcessor for Scalar { fn process(&self,_m:&[u8])->Option { Some(99000000) } } + let scalar=AopSensorData::new(std::sync::Arc::new(device::Device),7,Scalar).unwrap(); + scalar.process_fakehid_report(&[]).unwrap(); assert_eq!(read(&scalar,0,7),(2,99,0)); + assert_eq!(Scalar.process_channel(&[],1),None); + for offset in [LUX_OFFSET_CT720,LUX_OFFSET_CT817] { + let mut report=[0u8;122]; put(&mut report,offset,0.5); put(&mut report,0x45,6000.0); + let decoder=MsgProc(offset); + assert_eq!(decoder.process(&report),Some(500000)); + assert_eq!(decoder.process_channel(&report,1),None); + } + assert_eq!(f32_to_micro((-0.25f32).to_bits()),Some(0)); + assert_eq!(f32_to_micro(1u32),Some(0)); +} +''' + + +def main(): + code = source() + mutations = { + 'host-cct-offset': ('CCT_OFFSET_VD6286: usize = 0x45', 'CCT_OFFSET_VD6286: usize = 0x34'), + 'unknown-layout': ('message.len() == VD6286_COLOR_REPORT_LEN', 'message.len() >= VD6286_COLOR_REPORT_LEN - 1'), + 'cct-zero-valid': ('.filter(|value| *value != 0)', ''), + 'channel-alias': ('report_float(message, CCT_OFFSET_VD6286)', 'report_float(message, LUX_OFFSET_VD6286)'), + 'missing-saturation': ('value.min(MAX_MICRO)', 'value'), + } + with tempfile.TemporaryDirectory(prefix='aop-als-channels-') as directory: + directory = Path(directory) + for name, text in [('production', code)] + [ + (name, code.replace(old, new)) for name, (old, new) in mutations.items()]: + assert text != code or name == 'production', name + path = directory / 'channels.rs' + path.write_text(text) + binary = directory / 'channels' + subprocess.run(['rustc', '--edition=2021', '-Dwarnings', '--test', str(path), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)], text=True, capture_output=True) + if name == 'production': + print(result.stdout) + assert result.returncode == 0, result.stderr + else: + assert result.returncode == 101 and 'panicked at' in result.stdout + result.stderr, (name, result) + print('killed:', name) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/apple-pwm-pm-tests.py b/tools/asahi/apple-pwm-pm-tests.py new file mode 100644 index 0000000000000..5483b968a9d3a --- /dev/null +++ b/tools/asahi/apple-pwm-pm-tests.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Compile the real Apple PWM callbacks against a small runtime-PM/MMIO model. + +Tests reference ownership, register-loss recovery and failures. The model does +not qualify physical power gating, output levels or system sleep on hardware. +""" +import argparse +import os +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] + + +def function(source, name): + match = re.search(r'^static [^\n]*\b' + name + r'\([^;]*?\n\{', source, re.M) + if not match: + raise ValueError('missing callback: ' + name) + depth = 1 + end = match.end() + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[match.start():end] + + +PRELUDE = r''' +#include +#include +#include +#include +#include +#include +#define __iomem +#define BIT(n) (1U << (n)) +#define NSEC_PER_SEC 1000000000ULL +#define PWM_POLARITY_INVERSED 1 +#define PWM_POLARITY_NORMAL 0 +#define GFP_KERNEL 0 +#define IS_ERR(p) ((intptr_t)(p) < 0) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define DIV64_U64_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) +typedef uint32_t u32; +typedef uint64_t u64; +struct clk { int refs; }; +struct device { void *data; int refs; bool active, rpm; }; +struct platform_device { struct device dev; }; +struct pwm_chip { struct device *parent; void *data; const void *ops; }; +struct pwm_device { int unused; }; +struct pwm_state { bool enabled; int polarity; u64 duty_cycle, period; }; +static struct pwm_chip chip; +static struct platform_device pdev; +static struct clk clock_model; +static u32 registers[8]; +static unsigned char private_data[256]; +static int clock_error, active_error, action_error, add_error, writes; +static void (*cleanup)(void *); +static void *cleanup_data; +static const int apple_pwm_ops; +static int apple_pwm_runtime_suspend(struct device *dev); +static int apple_pwm_runtime_resume(struct device *dev); +static void *pwmchip_get_drvdata(struct pwm_chip *p) { return p->data; } +static struct device *pwmchip_parent(struct pwm_chip *p) { return p->parent; } +static void *dev_get_drvdata(struct device *d) { return d->data; } +static void platform_set_drvdata(struct platform_device *p, void *v) { p->dev.data = v; } +static u64 mul_u64_u64_div_u64(u64 a, u64 b, u64 c) { return (__uint128_t)a * b / c; } +static u32 readl(void *p) { assert(clock_model.refs == 1); return *(u32 *)p; } +static void writel(u32 v, void *p) { assert(clock_model.refs == 1); ++writes; *(u32 *)p = v; } +static int clk_prepare_enable(struct clk *c) { + if (clock_error) return clock_error; + assert(c->refs == 0); ++c->refs; return 0; +} +static void clk_disable_unprepare(struct clk *c) { assert(c->refs == 1); --c->refs; } +static u64 clk_get_rate(struct clk *c) { return 24000000; } +static void pm_runtime_get_noresume(struct device *d) { ++d->refs; } +static void pm_runtime_put_noidle(struct device *d) { assert(d->refs > 0); --d->refs; } +static int pm_runtime_resume_and_get(struct device *d) { + if (!d->active) { + int ret = apple_pwm_runtime_resume(d); + if (ret) return ret; + d->active = true; + } + ++d->refs; return 0; +} +static void pm_runtime_put(struct device *d) { + pm_runtime_put_noidle(d); + if (!d->refs && d->rpm && d->active) { + assert(!apple_pwm_runtime_suspend(d)); + d->active = false; + memset(registers, 0, sizeof(registers)); /* domain loses context */ + } +} +static void pm_runtime_enable(struct device *d) { d->rpm = true; } +static void pm_runtime_disable(struct device *d) { d->rpm = false; } +static bool pm_runtime_status_suspended(struct device *d) { return !d->active; } +static int pm_runtime_set_active(struct device *d) { + if (active_error) return active_error; + d->active = true; return 0; +} +static int pm_runtime_set_suspended(struct device *d) { d->active = false; return 0; } +static void *devm_pwmchip_alloc(struct device *d, unsigned n, size_t size) { + assert(size <= sizeof(private_data)); + chip = (struct pwm_chip){.parent = d, .data = private_data}; return &chip; +} +static void *devm_platform_ioremap_resource(struct platform_device *p, int n) { return registers; } +static struct clk *devm_clk_get(struct device *d, const char *name) { return &clock_model; } +static int dev_err_probe(struct device *d, int err, const char *msg) { return err; } +static int devm_add_action_or_reset(struct device *d, void (*fn)(void *), void *arg) { + if (action_error) { fn(arg); return action_error; } + cleanup = fn; cleanup_data = arg; return 0; +} +static int devm_pwmchip_add(struct device *d, struct pwm_chip *p) { return add_error; } +''' + +TESTS = r''' +static struct pwm_device pwm; +static struct pwm_state state = {.period = 40000, .duty_cycle = 10000}; +static void reset_model(bool boot_on) { + assert(!clock_model.refs && !pdev.dev.refs); + memset(&pdev, 0, sizeof(pdev)); memset(private_data, 0, sizeof(private_data)); + memset(registers, 0, sizeof(registers)); + registers[APPLE_PWM_ON_CYCLES / 4] = 240; + registers[APPLE_PWM_OFF_CYCLES / 4] = 720; + registers[APPLE_PWM_CTRL / 4] = boot_on ? APPLE_PWM_CTRL_ENABLE | APPLE_PWM_CTRL_OUTPUT_ENABLE : 0; + clock_error = active_error = action_error = add_error = writes = 0; + cleanup = NULL; state.enabled = false; state.polarity = PWM_POLARITY_NORMAL; +} +static void remove_model(void) { + assert(cleanup); cleanup(cleanup_data); cleanup = NULL; + assert(!pdev.dev.refs && !pdev.dev.active && !clock_model.refs); +} +int main(void) { + struct pwm_state got; + reset_model(false); + assert(!apple_pwm_probe(&pdev)); + assert(!pdev.dev.active && !pdev.dev.refs && !clock_model.refs); + assert(!apple_pwm_get_state(&chip, &pwm, &got)); + assert(!got.enabled && got.period == 40000 && got.duty_cycle == 10000); + assert(!pdev.dev.active && !pdev.dev.refs && !clock_model.refs); + puts("PASS disabled probe, get_state and register-loss recovery"); + state.enabled = true; + assert(!apple_pwm_apply(&chip, &pwm, &state)); + assert(pdev.dev.active && pdev.dev.refs == 1 && clock_model.refs == 1); + for (int i = 0; i < 20; ++i) { + assert(!apple_pwm_apply(&chip, &pwm, &state)); + assert(!apple_pwm_get_state(&chip, &pwm, &got)); + assert(pdev.dev.refs == 1 && got.enabled && got.period == 40000); + } + puts("PASS enabled output pins power; repeated apply/get_state do not leak"); + state.duty_cycle = 0; + assert(!apple_pwm_apply(&chip, &pwm, &state)); + assert(pdev.dev.active && pdev.dev.refs == 1); + assert(!apple_pwm_get_state(&chip, &pwm, &got) && got.enabled && !got.duty_cycle); + state.duty_cycle = 10000; + assert(!apple_pwm_apply(&chip, &pwm, &state)); + puts("PASS enabled zero-duty output remains powered"); + state.enabled = false; + assert(!apple_pwm_apply(&chip, &pwm, &state)); + assert(!pdev.dev.active && !pdev.dev.refs && !clock_model.refs); + assert(!apple_pwm_get_state(&chip, &pwm, &got) && !got.enabled); + remove_model(); + puts("PASS disable releases output reference and suspended removal is balanced"); + reset_model(true); + assert(!apple_pwm_probe(&pdev)); + assert(pdev.dev.active && pdev.dev.refs == 1 && clock_model.refs == 1 && !writes); + assert(!apple_pwm_get_state(&chip, &pwm, &got) && got.enabled); + state.enabled = true; + assert(!apple_pwm_apply(&chip, &pwm, &state) && pdev.dev.refs == 1); + remove_model(); + puts("PASS bootloader output adopted without writes and active removal balanced"); + reset_model(false); + assert(!apple_pwm_probe(&pdev)); + clock_error = -EIO; + int before = writes; + state.enabled = true; + assert(apple_pwm_apply(&chip, &pwm, &state) == -EIO); + assert(apple_pwm_get_state(&chip, &pwm, &got) == -EIO); + assert(!pdev.dev.refs && !clock_model.refs && writes == before); + state.polarity = PWM_POLARITY_INVERSED; + assert(apple_pwm_apply(&chip, &pwm, &state) == -EINVAL && writes == before); + clock_error = 0; remove_model(); + puts("PASS resume failure and unsupported polarity leave registers and refs untouched"); + reset_model(false); clock_error = -EIO; + assert(apple_pwm_probe(&pdev) == -EIO && !clock_model.refs && !pdev.dev.refs); + reset_model(false); active_error = -EBUSY; + assert(apple_pwm_probe(&pdev) == -EBUSY && !clock_model.refs && !pdev.dev.refs); + puts("PASS initial clock and PM-state failures unwind"); + for (int enabled = 0; enabled <= 1; ++enabled) { + reset_model(enabled); action_error = -ENOMEM; + assert(apple_pwm_probe(&pdev) == -ENOMEM && !clock_model.refs && !pdev.dev.refs); + reset_model(enabled); add_error = -ENOMEM; + assert(apple_pwm_probe(&pdev) == -ENOMEM); + remove_model(); + } + puts("PASS action and PWM registration failures unwind both boot states"); + reset_model(false); assert(!apple_pwm_probe(&pdev)); + assert(!pm_runtime_resume_and_get(&pdev.dev)); /* another consumer/supplier vote */ + state.enabled = true; assert(!apple_pwm_apply(&chip, &pwm, &state)); + assert(pdev.dev.refs == 2); + state.enabled = false; assert(!apple_pwm_apply(&chip, &pwm, &state)); + assert(pdev.dev.active && pdev.dev.refs == 1 && clock_model.refs == 1); + pm_runtime_put(&pdev.dev); + assert(!pdev.dev.active && !clock_model.refs); remove_model(); + puts("PASS another runtime reference prevents premature suspension"); + return 0; +} +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--negative-control', action='store_true') + args = parser.parse_args() + source = (ROOT / 'drivers/pwm/pwm-apple.c').read_text() + if args.negative_control: + old = '\tif (state->enabled && !fpwm->enabled)\n\t\tpm_runtime_get_noresume(dev);' + assert old in source + source = source.replace(old, '\tif (state->enabled && !fpwm->enabled)\n\t\t; /* missing output reference */', 1) + defines = '\n'.join(re.findall(r'^#define APPLE_PWM_.*$', source, re.M)) + struct = re.search(r'struct apple_pwm \{.*?\n\};', source, re.S).group() + callbacks = '\n'.join(function(source, name) for name in ( + 'to_apple_pwm', 'apple_pwm_apply', 'apple_pwm_get_state', + 'apple_pwm_runtime_suspend', 'apple_pwm_runtime_resume', + 'apple_pwm_runtime_disable', 'apple_pwm_probe')) + with tempfile.TemporaryDirectory(prefix='apple-pwm-pm-') as tmp: + tmp = Path(tmp) + cfile, executable = tmp / 'test.c', tmp / 'test' + cfile.write_text(PRELUDE + defines + '\n' + struct + callbacks + TESTS) + subprocess.run([os.environ.get('CC', 'cc'), '-std=gnu11', '-Wall', '-Wextra', + '-Werror', '-Wno-unused-parameter', '-Wno-unused-function', + '-fsanitize=address,undefined', str(cfile), '-o', str(executable)], check=True) + result = subprocess.run([str(executable)], capture_output=True, text=True) + if args.negative_control: + if result.returncode == 0: + raise SystemExit('FAIL: missing output reference escaped detection') + print('PASS negative control: missing output reference was detected') + else: + print(result.stdout, end='') + if result.returncode: + raise SystemExit(result.stderr) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/avd-av1-tail-tests.py b/tools/asahi/avd-av1-tail-tests.py index d602801b7c0a7..d434ab3a0b8a2 100644 --- a/tools/asahi/avd-av1-tail-tests.py +++ b/tools/asahi/avd-av1-tail-tests.py @@ -15,7 +15,7 @@ def function(source, name): - match = re.search(r'^(?:static )?(?:int|void)\s+' + name + r'\(', source, re.M) + match = re.search(r'^(?:static )?(?:int|void|u64)\s+' + name + r'\(', source, re.M) if not match: raise ValueError(name) start = match.start() @@ -47,6 +47,8 @@ def structure(source, name): typedef int8_t s8; typedef uint16_t u16; typedef uint32_t u32; +typedef uint64_t u64; +#define U64_MAX UINT64_MAX typedef uint64_t dma_addr_t; #define ALIGN(x,a) (((x)+(a)-1)&~((__typeof__(x))(a)-1)) #define ALIGN_DOWN(x,a) ((x)&~((__typeof__(x))(a)-1)) @@ -211,7 +213,7 @@ def main(): text = {name: p.read_text() for name, p in paths.items()} src = text['avd-av1.c'] macros = src[src.index('#define AVD_CDFS_SIZE'):src.index('struct avd_av1_run {')] - functions = {n: function(src, n) for n in ['avd_av1_set_prob', 'avd_av1_set_seg', 'avd_av1_seg_size', 'avd_color_size', 'avd_av1_run_preamble', 'avd_av1_adjust_decoded_fmt']} + functions = {n: function(src, n) for n in ['avd_av1_ref_ts', 'avd_av1_set_prob', 'avd_av1_set_seg', 'avd_av1_seg_size', 'avd_color_size', 'avd_av1_run_preamble', 'avd_av1_adjust_decoded_fmt']} unaligned = ('\n#define UNALIGNED_CDFS_OFFSET(dst,cl) ((dst)-ALIGN(cl,AVD_ALIGN)-ALIGN(AVD_CDFS_SIZE,AVD_ALIGN))\n' '#define UNALIGNED_SEG_OFFSET(dst,cl,sl) (UNALIGNED_CDFS_OFFSET(dst,cl)-(sl))\n') # Each mutation names the extracted text it rewrites; a stale one is refused, never run as a no-op. diff --git a/tools/asahi/ave-pm-tests.py b/tools/asahi/ave-pm-tests.py new file mode 100644 index 0000000000000..9b9f7ea87fc10 --- /dev/null +++ b/tools/asahi/ave-pm-tests.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise the AVE sleep callbacks with a firmware halt failure. + +Compile the production callbacks against a simulated m2m scheduler and firmware. +No encoder registers, firmware or media device are needed. The old callback is a +negative control: it must fail the stuck-firmware assertion after compiling. +""" +import argparse +import os +from pathlib import Path +import resource +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = 'drivers/media/platform/apple/ave/ave-drv.c' + + +def function(source, name): + start = source.index('static int ' + name + '(') + end = source.index('\n}', start) + 2 + return source[start:end] + + +PREFIX = r''' +#include +#include +#include +#include +struct scheduler { bool stopped; unsigned pauses, restarts; }; +struct ave_transport { bool scratch_halt; }; +struct ave_abi { const struct ave_transport *transport; }; +struct ave_layout { const struct ave_abi *abi; }; +struct ave_dev { struct scheduler *m2m_dev; int hw_lock; const struct ave_layout *layout; }; +struct device { struct ave_dev *data; }; +static int shutdown_error; +static unsigned closes, halts, releases, wakes; +static void *dev_get_drvdata(struct device *d) { return d->data; } +static void mutex_lock(int *l) { assert(!*l); *l = 1; } +static void mutex_unlock(int *l) { assert(*l == 1); *l = 0; } +static void v4l2_m2m_suspend(struct scheduler *s) { + assert(!s->stopped); s->stopped = true; s->pauses++; +} +static void v4l2_m2m_resume(struct scheduler *s) { + assert(s->stopped); s->stopped = false; s->restarts++; +} +static void ave_v4l2_suspend(struct ave_dev *a) { + assert(a->hw_lock && a->m2m_dev->stopped); closes++; +} +static int ave_fw_shutdown(struct ave_dev *a) { + assert(a->hw_lock && a->m2m_dev->stopped && closes > halts); + halts++; return shutdown_error; +} +/* A scratch-halt firmware (6070) gives its boot buffers back after a halt. */ +static __attribute__((unused)) void ave_fw_release_boot(struct ave_dev *a) { + assert(a->hw_lock && halts > releases); releases++; +} +static __attribute__((unused)) int ave_runtime_resume(struct device *d) { (void)d; wakes++; return 0; } +#define dev_warn(...) ((void)0) +#define dev_err(...) ((void)0) +''' +CASES = r''' +int main(void) { + struct scheduler s = {0}; + struct ave_dev a = {.m2m_dev = &s}; + struct device d = {.data = &a}; + for (unsigned n = 0; n < 3; n++) { + shutdown_error = 0; + assert(ave_suspend(&d) == 0); + assert(s.stopped && !a.hw_lock); + assert(ave_resume(&d) == 0); + assert(!s.stopped && !a.hw_lock); + } + /* PM does not call resume for a device whose suspend returned an error. */ + shutdown_error = -EBUSY; + assert(ave_suspend(&d) == -EBUSY); + assert(!s.stopped && !a.hw_lock); + /* An aborted sleep must not leave the scheduler stopped for the next one. */ + shutdown_error = -EIO; + assert(ave_suspend(&d) == -EIO); + assert(!s.stopped && !a.hw_lock); + shutdown_error = 0; + assert(ave_suspend(&d) == 0); + assert(s.stopped); + assert(ave_resume(&d) == 0); + assert(!s.stopped && !a.hw_lock); + assert(s.pauses == 6 && s.restarts == 6 && closes == 6 && halts == 6); + assert(releases == 0 && wakes == 4); + /* A scratch-halt firmware releases its boot buffers only after a clean halt. */ + static const struct ave_transport scratch = { .scratch_halt = true }; + static const struct ave_abi abi = { .transport = &scratch }; + static const struct ave_layout layout = { .abi = &abi }; + a.layout = &layout; + shutdown_error = -EBUSY; + assert(ave_suspend(&d) == -EBUSY && releases == 0); + shutdown_error = 0; + assert(ave_suspend(&d) == 0 && releases == 1); + assert(ave_resume(&d) == 0 && wakes == 5); + puts("AVE PM: normal/repeated sleep, halt refusal, error propagation, rollback, retry and boot-buffer release passed"); +} +''' + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--out') + p.add_argument('--old-ref', help='expect this revision to fail the runtime control') + args = p.parse_args() + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + source = ((ROOT / SOURCE).read_text() if not args.old_ref else subprocess.check_output( + ['git', 'show', args.old_ref + ':' + SOURCE], cwd=ROOT, text=True)) + with tempfile.TemporaryDirectory(prefix='ave-pm-') as tmp: + out = Path(args.out or tmp) + out.mkdir(parents=True, exist_ok=True) + c = out / 'ave-pm.c' + c.write_text(PREFIX + function(source, 'ave_suspend') + '\n' + + function(source, 'ave_resume') + CASES) + exe = out / 'ave-pm' + subprocess.run([os.environ.get('CC', 'cc'), '-std=c11', '-Wall', '-Wextra', + '-Werror', str(c), '-o', str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + (out / 'run.log').write_text(r.stdout + r.stderr) + if args.old_ref: + if r.returncode != -6 or 'Assertion' not in r.stderr: + raise SystemExit('old callback did not fail its runtime assertion') + print('AVE PM: old callback compiled and failed the halt-refusal assertion') + elif r.returncode: + raise SystemExit(r.stdout + r.stderr) + else: + print(r.stdout, end='') + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/ave-probe-pm-tests.py b/tools/asahi/ave-probe-pm-tests.py new file mode 100644 index 0000000000000..417eecdb575df --- /dev/null +++ b/tools/asahi/ave-probe-pm-tests.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Compile the AVE probe unchanged against a supplier-reference model. + +Exercise never-opened idle, immediate registration-time use and probe errors. +The model follows runtime.c: setting SUSPENDED drops supplier references even +when the consumer status was already SUSPENDED. This tests probe ordering and +reference ownership, not hardware power gating or firmware restart. +""" +import argparse +import os +from pathlib import Path +import resource +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = 'drivers/media/platform/apple/ave/ave-drv.c' + +PREFIX = r''' +#include +#include +#include +#include +#include +#include +struct ave_dev; +struct device { bool enabled, suspended; unsigned supplier_refs; struct ave_dev *data; }; +struct platform_device { struct device dev; }; +struct iommu_domain { unsigned type; }; +struct ave_dev { struct device *dev; const void *variant; int hw_lock; + void *codec, *mbox, *coproc, *pd_list; struct iommu_domain *domain; }; +struct dev_pm_domain_attach_data { + const char *const *pd_names; unsigned num_pd_names, pd_flags; +}; +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#define GFP_KERNEL 0 +#define __IOMMU_DOMAIN_PAGING 1 +#define PD_FLAG_DEV_LINK_ON 1 +#define AVE_AUTOSUSPEND_MS 2000 +#define DMA_BIT_MASK(x) ((UINT64_C(1) << (x)) - 1) +#define IS_ERR(x) ((uintptr_t)(x) >= (uintptr_t)-4095) +#define PTR_ERR(x) ((int)(intptr_t)(x)) +static const char *const ave_pd_names[] = {"dma", "pipe4", "pipe5", "me0", "me1"}; +static struct ave_dev context; +static struct iommu_domain domain = {.type = __IOMMU_DOMAIN_PAGING}; +static int attach_error, state_error, enable_error, fw_error, register_error; +static unsigned registrations, fw_probes, fw_removes, state_updates; +static void *devm_kzalloc(struct device *d, size_t n, int f) +{ (void)d; (void)f; assert(n == sizeof(context)); return &context; } +static const void *of_device_get_match_data(struct device *d) { return d; } +static void mutex_init(int *m) { *m = 0; } +static void platform_set_drvdata(struct platform_device *p, void *v) { p->dev.data = v; } +static void *devm_platform_ioremap_resource_byname(struct platform_device *p, const char *n) +{ (void)p; return (void *)n; } +static struct iommu_domain *iommu_get_domain_for_dev(struct device *d) +{ (void)d; return &domain; } +static int dma_set_mask_and_coherent(struct device *d, uint64_t mask) +{ (void)d; assert(mask == DMA_BIT_MASK(42)); return 0; } +#define dev_err_probe(d, e, ...) (e) +static int devm_pm_domain_attach_list(struct device *d, + const struct dev_pm_domain_attach_data *data, void **list) +{ + assert(!d->enabled && d->suspended); + assert(data->num_pd_names == 5 && data->pd_flags == PD_FLAG_DEV_LINK_ON); + assert(!*list); + if (attach_error) return attach_error; + /* Five virtual devices plus the two preactivated DART supplier links. */ + d->supplier_refs = 7; + *list = d; + return 5; +} +static int pm_runtime_set_suspended(struct device *d) +{ + assert(!d->enabled); + assert(fw_probes == 1 && !fw_error); + state_updates++; + if (state_error) return state_error; + d->suspended = true; + d->supplier_refs = 0; + return 0; +} +static void pm_runtime_set_autosuspend_delay(struct device *d, unsigned delay) +{ (void)d; assert(delay == 2000); } +static void pm_runtime_use_autosuspend(struct device *d) { (void)d; } +static void pm_runtime_dont_use_autosuspend(struct device *d) { (void)d; } +static int devm_pm_runtime_enable(struct device *d) +{ + assert(!d->enabled); + if (enable_error) return enable_error; + d->enabled = true; + return 0; +} +static int ave_fw_probe(struct ave_dev *a) +{ + assert(!a->dev->enabled && a->dev->supplier_refs == 7); + fw_probes++; + return fw_error; +} +static void ave_fw_remove(struct ave_dev *a) { (void)a; fw_removes++; } +static int ave_v4l2_register(struct ave_dev *a) +{ + struct device *d = a->dev; + registrations++; + assert(d->enabled && d->suspended && d->supplier_refs == 0); + if (register_error) return register_error; + /* A client can open immediately once its video device is published. */ + for (unsigned n = 0; n < 10; n++) { + d->suspended = false; + d->supplier_refs = 7; + assert(d->enabled && !d->suspended); + d->supplier_refs = 0; + d->suspended = true; + } + return 0; +} +static void ave_debugfs_init(struct ave_dev *a) { (void)a; } +''' + +CASES = r''' +static void reset(struct platform_device *p) +{ + *p = (struct platform_device){.dev.suspended = true}; + context = (struct ave_dev){0}; + attach_error = state_error = enable_error = fw_error = register_error = 0; + registrations = fw_probes = fw_removes = state_updates = 0; +} +int main(void) +{ + struct platform_device p; + reset(&p); + assert(ave_probe(&p) == 0); + assert(state_updates == 1 && registrations == 1 && fw_probes == 1); + assert(p.dev.suspended && !p.dev.supplier_refs); + reset(&p); + attach_error = -EIO; + assert(ave_probe(&p) == -EIO); + assert(!state_updates && !p.dev.enabled && !registrations); + reset(&p); + state_error = -EBUSY; + assert(ave_probe(&p) == -EBUSY); + assert(!p.dev.enabled && fw_probes == 1 && fw_removes == 1 && !registrations); + reset(&p); + enable_error = -ENOMEM; + assert(ave_probe(&p) == -ENOMEM); + assert(!p.dev.supplier_refs && fw_probes == 1 && fw_removes == 1 && !registrations); + reset(&p); + fw_error = -EINVAL; + assert(ave_probe(&p) == -EINVAL); + assert(p.dev.supplier_refs == 7 && !state_updates && !p.dev.enabled && !registrations); + reset(&p); + register_error = -EIO; + assert(ave_probe(&p) == -EIO); + assert(!p.dev.supplier_refs && fw_removes == 1); + puts("AVE probe PM: idle suppliers, immediate use, 10 cycles and 5 error paths passed"); +} +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source', type=Path) + parser.add_argument('--out', type=Path) + parser.add_argument('--negative-control', action='store_true') + args = parser.parse_args() + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + source = (args.source or ROOT / SOURCE).read_text() + start = source.index('static int ave_probe(') + probe = source[start:source.index('\n}', start) + 2] + if args.negative_control: + old = '\tret = pm_runtime_set_suspended(dev);' + if probe.count(old) != 1: + raise SystemExit('cannot locate the production fix for the negative control') + probe = probe.replace(old, '\tret = 0;') + with tempfile.TemporaryDirectory(prefix='ave-probe-pm-') as tmp: + out = args.out or Path(tmp) + out.mkdir(parents=True, exist_ok=True) + c = out / 'ave-probe-pm.c' + c.write_text(PREFIX + probe + CASES) + exe = out / 'ave-probe-pm' + subprocess.run([os.environ.get('CC', 'cc'), '-std=c11', '-Wall', '-Wextra', + '-Werror', '-Wno-unused-function', '-fsanitize=address,undefined', + '-fno-omit-frame-pointer', str(c), '-o', str(exe)], check=True) + run = subprocess.run([str(exe)], capture_output=True, text=True) + (out / 'run.log').write_text(run.stdout + run.stderr) + if args.negative_control: + if run.returncode != -6 or 'supplier_refs == 0' not in run.stderr: + raise SystemExit('pre-fix probe did not fail the idle-supplier assertion') + print('AVE probe PM: pre-fix probe compiled and failed the idle-supplier assertion') + elif run.returncode: + raise SystemExit(run.stdout + run.stderr) + else: + print(run.stdout, end='') + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/binding-lifetime-tests.py b/tools/asahi/binding-lifetime-tests.py new file mode 100644 index 0000000000000..dedcbcd4326e5 --- /dev/null +++ b/tools/asahi/binding-lifetime-tests.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the production bound-device gate with host locking/refcount adapters.""" +import argparse +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +HEADER = r''' +#![allow(dead_code)] +extern crate self as kernel; +mod prelude { + pub type Result = std::result::Result; + pub const ENODEV:i32=-19; + pub const EOVERFLOW:i32=-75; + pub const GFP_KERNEL:()=(); + pub struct KVec(Vec); + impl KVec { pub const fn new()->Self {Self(Vec::new())} pub fn push(&mut self,v:T,_:())->Result {self.0.push(v);Ok(())} } + impl std::ops::Deref for KVec{type Target=Vec;fn deref(&self)->&Vec{&self.0}} + impl std::ops::DerefMut for KVec{fn deref_mut(&mut self)->&mut Vec{&mut self.0}} +} +#[macro_export] +macro_rules! new_mutex {($v:expr $(,$name:expr)?)=>{$crate::sync::Mutex::new($v)};} +#[macro_export] +macro_rules! global_lock { (unsafe(uninit) static $name:ident: Mutex<$ty:ty> = $value:expr;) => { static $name:$crate::sync::Mutex<$ty>=$crate::sync::Mutex::new($value); }; } +#[macro_export] +macro_rules! new_condvar {($name:expr)=>{$crate::sync::CondVar(std::sync::Condvar::new())};} +macro_rules! pin_init {($($t:tt)*)=>{$($t)*};} +mod sync { + pub use crate::global_lock; + pub struct Arc(std::sync::Arc); + impl Clone for Arc{fn clone(&self)->Self{Self(self.0.clone())}} + impl std::ops::Deref for Arc{type Target=T;fn deref(&self)->&T{&self.0}} + impl Arc{pub fn pin_init(v:T,_:())->crate::prelude::Result{Ok(Self(std::sync::Arc::new(v)))} pub fn ptr_eq(a:&Self,b:&Self)->bool{std::sync::Arc::ptr_eq(&a.0,&b.0)}} + pub struct Mutex(std::sync::Mutex); + impl Mutex{pub const fn new(v:T)->Self{Self(std::sync::Mutex::new(v))} pub unsafe fn init(&self){} pub fn lock(&self)->Guard<'_,T>{Guard(Some(self.0.lock().unwrap()))}} + pub struct Guard<'a,T>(Option>); + impl std::ops::Deref for Guard<'_,T>{type Target=T;fn deref(&self)->&T{self.0.as_ref().unwrap()}} + impl std::ops::DerefMut for Guard<'_,T>{fn deref_mut(&mut self)->&mut T{self.0.as_mut().unwrap()}} + pub struct CondVar(pub std::sync::Condvar); + impl CondVar {pub fn wait(&self,g:&mut Guard<'_,T>){let old=g.0.take().unwrap();g.0=Some(self.0.wait(old).unwrap());} pub fn notify_all(&self){self.0.notify_all();}} + pub mod aref { + pub struct ARef(&'static T); + impl std::ops::Deref for ARef{type Target=T;fn deref(&self)->&T{self.0}} + impl From<&crate::device::Device> for ARef{fn from(d:&crate::device::Device)->Self { + // The host fixtures leak their devices for the test process. + Self(unsafe{&*(d as *const _ as *const crate::device::Device)}) + }} + } +} +mod device { + pub struct Normal; + pub struct Bound; + #[repr(C)] + pub struct Device{pub bound:std::sync::atomic::AtomicBool,pub marker:std::marker::PhantomData} + impl Device{pub fn new()->Self{Self{bound:std::sync::atomic::AtomicBool::new(true),marker:std::marker::PhantomData}}} + impl Device {pub unsafe fn as_bound(&self)->&Device{assert!(self.bound.load(std::sync::atomic::Ordering::SeqCst));unsafe{&*(self as *const _ as *const Device)}}} +} +mod driver {pub struct AsahiDrmDevice(pub u64);} +mod binding; +''' +TESTS = r''' +use {binding::*,device::*,driver::AsahiDrmDevice}; +use std::{sync::{mpsc,atomic::Ordering},time::Duration}; +fn device()->&'static Device{Box::leak(Box::new(Device::new()))} +#[test] +fn close_joins_existing_access_and_rejects_future_access() { + let dev=device();let drm=Box::new(AsahiDrmDevice(1)); + let owner=unsafe{Owner::new(dev,&drm)}.unwrap(); + let gate=lookup(&drm).unwrap();let access=gate.enter().unwrap();let _=access.device(); + let (started_rx,started)= {let (a,b)=mpsc::channel();(b,a)}; + let (done,finished)=mpsc::channel(); + let thread=std::thread::spawn(move||{started.send(()).unwrap();owner.close();done.send(()).unwrap();}); + started_rx.recv().unwrap(); + assert!(finished.recv_timeout(Duration::from_millis(50)).is_err(),"unbind returned with a live device borrow"); + drop(access);finished.recv_timeout(Duration::from_secs(2)).unwrap();thread.join().unwrap(); + dev.bound.store(false,Ordering::SeqCst); + assert!(gate.enter().is_err());assert!(lookup(&drm).is_err()); +} +#[test] +fn rebind_cannot_reopen_an_old_vm_gate() { + let dev=device();let a=Box::new(AsahiDrmDevice(2));let b=Box::new(AsahiDrmDevice(3)); + let first=unsafe{Owner::new(dev,&a)}.unwrap();let old=lookup(&a).unwrap(); + first.close();let second=unsafe{Owner::new(dev,&b)}.unwrap(); + assert!(old.enter().is_err());assert!(lookup(&a).is_err()); + let new=lookup(&b).unwrap();let access=new.enter().unwrap();let _=access.device();drop(access);second.close(); +} +#[test] +fn failed_probe_owner_drop_closes_retained_vm_handles() { + let dev=device();let drm=Box::new(AsahiDrmDevice(4)); + let owner=unsafe{Owner::new(dev,&drm)}.unwrap();let gate=lookup(&drm).unwrap();drop(owner); + dev.bound.store(false,Ordering::SeqCst); + assert!(gate.enter().is_err());assert!(lookup(&drm).is_err()); +} +''' + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('--out', type=Path) + ap.add_argument('--mutation', choices=['no-close', 'no-join']) + args = ap.parse_args() + work = (args.out or Path(tempfile.mkdtemp(prefix='asahi-binding-'))).resolve() + work.mkdir(parents=True, exist_ok=True) + source = (ROOT / 'drivers/gpu/drm/asahi/binding.rs').read_text() + source = '\n'.join(line for line in source.splitlines() + if line.strip() not in ('#[pin_data]', '#[pin]')) + source = source.replace(' <- ', ': ') + source = source.replace('// SAFETY: Module initialization initializes this lock before registering adapters.', '') + if args.mutation == 'no-close': + source = source.replace('state.live = false;', 'state.live = true;') + if args.mutation == 'no-join': + source = source.replace('while state.users != 0 {', 'while false {') + (work / 'binding.rs').write_text(source) + (work / 'test.rs').write_text(HEADER + TESTS) + subprocess.run(['rustc', '--edition=2021', '--test', str(work / 'test.rs'), '-o', str(work / 'test')], check=True) + subprocess.run([str(work / 'test')], check=True) + + +if __name__ == '__main__': + try: + main() + except subprocess.CalledProcessError as error: + raise SystemExit(error.returncode if error.returncode > 0 else 128 - error.returncode) diff --git a/tools/asahi/g15-probe-init-tests.py b/tools/asahi/g15-probe-init-tests.py index 8ab22935d9c53..65cc9c0b69c9f 100644 --- a/tools/asahi/g15-probe-init-tests.py +++ b/tools/asahi/g15-probe-init-tests.py @@ -77,6 +77,7 @@ def item(text, marker): static FAIL_REGISTRATION: Cell = const { Cell::new(0) }; static COMPLETED_REGISTRATION: Cell = const { Cell::new(0) }; static ACTIVATED: Cell = const { Cell::new(false) }; + static BINDING_READY: Cell = const { Cell::new(false) }; static ATTACH_ERROR: Cell = const { Cell::new(false) }; static PROBE_ERROR: Cell = const { Cell::new(0) }; } @@ -140,6 +141,11 @@ def item(text, marker): pub trait InPlaceModule: Sized { fn init(module: &'static ThisModule) -> impl PinInit; } +// Registry synchronization is covered by binding-lifetime-tests; this adapter +// verifies that the actual module initializer orders it before registration. +mod binding { + pub unsafe fn init() { crate::BINDING_READY.set(true); } +} mod cleanup { pub struct QueueOwner; impl QueueOwner { pub fn new() -> crate::Result { Ok(Self) } } @@ -150,11 +156,12 @@ def item(text, marker): pub struct Registration { stage: u8, marker: PhantomData } impl Registration { pub fn new(name: &'static std::ffi::CStr, _module: &'static ThisModule) -> impl PinInit { + assert!(BINDING_READY.get(), "adapter registered before binding registry init"); try_pin_init!(Self { stage: { let stage = match name.to_bytes() { - b"asahi-g15" => 1, - b"asahi-g16" => 3, + b"aurora-g15" => 1, + b"aurora-g16" => 3, _ => 2, }; // Model driver_attach and a late driver_add_groups failure. A deferred @@ -198,6 +205,7 @@ def item(text, marker): } fn reset(fail_alloc: bool, touch: bool, error: i32) { MODULE_READY.store(true, Ordering::Release); + BINDING_READY.set(false); FAIL_ALLOC.set(fail_alloc); TOUCH.set(touch); PROBE_ERROR.set(error); RETAINED.set(false); ALLOCS.set(0); PROBES.set(0); } @@ -330,7 +338,7 @@ def main(): g16_registration = aliases + "\n" + item(core, "fn register_g16(") module_struct = item(core, "#[pin_data]\nstruct DriverModule") module_init = item(core, " fn init(module:") - module_code = module_struct + "\nimpl InPlaceModule for DriverModule {\n" + module_init + "\n}\nimpl ModuleMetadata for DriverModule { const NAME: &'static std::ffi::CStr = c\"asahi\"; }\n" + module_code = module_struct + "\nimpl InPlaceModule for DriverModule {\n" + module_init + "\n}\nimpl ModuleMetadata for DriverModule { const NAME: &'static std::ffi::CStr = c\"aurora\"; }\n" kbox = item(sources["rust/kernel/alloc/kbox.rs"], " fn try_pin_init(") assert "Box::<_, A>::new_uninit(flags)?.write_pin_init(init)" in kbox reprobe = item(sources["drivers/gpu/drm/asahi/g15/glue.c"], "void asahi_g15_reprobe(void)\n{") @@ -400,7 +408,7 @@ def main(): if name == "activation-before-g16-registration" and not enable_g16: continue # Macro definitions precede the fake registration, just as kernel prelude exports them. - source = macro + "\n" + PREFIX.replace("#![allow(dead_code, unused_variables)]\n", "").replace(" // SET_DRVDATA", set_data) + "\n" + ready + "\n" + driver + "\nimpl platform::Driver for Driver<'_> {\n type IdInfo = &'static Config;\n type Data<'bound> = Driver<'bound>;\n" + method + "\n}\nmod g15 { use super::*; pub type Driver<'a> = super::Driver<'a>; " + activate + "\n}\ntype G15Registration = driver::Registration>>;\n" + "mod g16 { pub struct Driver; pub const DRIVER_NAME: &std::ffi::CStr = c\"asahi-g16\"; }\n" + g16_registration + "\n" + module_variant + "\n" + SUFFIX.replace(" // CALLBACK", callback) + source = macro + "\n" + PREFIX.replace("#![allow(dead_code, unused_variables)]\n", "").replace(" // SET_DRVDATA", set_data) + "\n" + ready + "\n" + driver + "\nimpl platform::Driver for Driver<'_> {\n type IdInfo = &'static Config;\n type Data<'bound> = Driver<'bound>;\n" + method + "\n}\nmod g15 { use super::*; pub type Driver<'a> = super::Driver<'a>; " + activate + "\n}\ntype G15Registration = driver::Registration>>;\n" + "mod g16 { pub struct Driver; pub const DRIVER_NAME: &std::ffi::CStr = c\"aurora-g16\"; }\n" + g16_registration + "\n" + module_variant + "\n" + SUFFIX.replace(" // CALLBACK", callback) source = "#![allow(dead_code, unused_variables, private_bounds, private_interfaces)]\n" + source label = ("g16-" if enable_g16 else "") + name src = out / (label + ".rs"); src.write_text(source) diff --git a/tools/asahi/g15-queue-credit-tests.py b/tools/asahi/g15-queue-credit-tests.py index 2c8d7f64fb3f3..ee615d28d234f 100644 --- a/tools/asahi/g15-queue-credit-tests.py +++ b/tools/asahi/g15-queue-credit-tests.py @@ -26,6 +26,7 @@ def extract(text, marker): CODE = r''' #![allow(dead_code)] +__USAGE__ use std::{sync::{Arc as StdArc,Mutex,Barrier,OnceLock,atomic::{AtomicBool,AtomicI32,AtomicU32,AtomicU64,Ordering}},ops::Deref,cell::Cell}; #[derive(Copy,Clone,Debug,PartialEq,Eq)]struct Error(i32); type Result = std::result::Result; @@ -70,7 +71,7 @@ def extract(text, marker): Gpu{shared:Arc::wrap(Shared{logical_queues:counter,crashed:AtomicBool::new(false),dev:Device{trace}})} } fn vm(gpu:&Gpu,status:i32)->mmu::Vm{mmu::Vm{state:Arc::wrap(VmState{status:Status(AtomicI32::new(status)),trace:gpu.shared.dev.trace.clone()})}} -fn new(gpu:&Gpu,status:i32,priority:u32,base:u64)->Result>{KBox::new(Queue::new(gpu,vm(gpu,status),priority,base)?,GFP_KERNEL)} +fn new(gpu:&Gpu,status:i32,priority:u32,base:u64)->Result>{KBox::new(Queue::new(gpu,vm(gpu,status),priority,base,Arc::wrap(usage::Client::new()))?,GFP_KERNEL)} fn count(gpu:&Gpu)->u32{gpu.shared.logical_queues.load(Ordering::SeqCst)} fn main(){ // One device shared across different VM objects/files: exactly58 live queues. @@ -115,12 +116,14 @@ def main(): parser.add_argument("--mutations", action="store_true") args = parser.parse_args(); args.output.mkdir(parents=True, exist_ok=True) queue = (ROOT / "drivers/gpu/drm/asahi/g15/queue.rs").read_text() + usage = (ROOT / "drivers/gpu/drm/asahi/usage.rs").read_text() hardware = (ROOT / "drivers/gpu/drm/asahi/g15/hw.rs").read_text() limit = next(line.strip() for line in hardware.splitlines() if "const MAX_QUEUES:" in line) - replacements = {"__LIMIT__": limit, "__STATE__": extract(queue, "pub(crate) struct QueueState {"), + replacements = {"__USAGE__": "mod usage {\n" + usage + "\n}", + "__LIMIT__": limit, "__STATE__": extract(queue, "pub(crate) struct QueueState {"), "__CREDIT__": extract(queue, "struct QueueCredit {"), "__CREDIT_IMPL__": extract(queue, "impl QueueCredit {"), "__CREDIT_DROP__": extract(queue, "impl Drop for QueueCredit {"), "__QUEUE__": extract(queue, "pub(crate) struct Queue {"), - "__QUEUE_DROP__": extract(queue, "impl Drop for Queue {"), "__NEW__": extract(queue, " pub(crate) fn new(gpu: &Gpu,")} + "__QUEUE_DROP__": extract(queue, "impl Drop for Queue {"), "__NEW__": extract(queue, " pub(crate) fn new(")} code = CODE for old, new in replacements.items(): code = code.replace(old, new) variants = {"control": code}; expected = {} @@ -143,6 +146,7 @@ def main(): variants[name] = variant; expected[name] = message variants["control-after"] = code receipt = {"scope": __doc__, "sources": {"queue.rs": hashlib.sha256(queue.encode()).hexdigest(), + "usage.rs": hashlib.sha256(usage.encode()).hexdigest(), "hw.rs": hashlib.sha256(hardware.encode()).hexdigest()}, "variants": {}} for name, source in variants.items(): path=args.output/(name+".rs");path.write_text(source);binary=args.output/name diff --git a/tools/asahi/g17-capacity-fault-tests.py b/tools/asahi/g17-capacity-fault-tests.py index 7c33e92a74812..c03764ca28d60 100644 --- a/tools/asahi/g17-capacity-fault-tests.py +++ b/tools/asahi/g17-capacity-fault-tests.py @@ -7,6 +7,7 @@ """ from pathlib import Path import os +import re import subprocess import tempfile @@ -278,13 +279,18 @@ def main(): fault_fix = 'Previous::retire_witnessed(&mut self.previous);' assert source.count(idle_fix) == 1 assert source.count(fault_fix) == 2 + def mutate(pattern, replacement): + changed, count = re.subn(pattern, replacement, source) + assert count == 1, (pattern, count) + return changed + controls = [('fixed', source, True), ('missing-idle-close', source.replace(idle_fix, ''), False), ('missing-fault-release', source.replace(fault_fix, '', 1), False), ('missing-kill-release', source.replace('Previous::retire_witnessed(&mut self.previous);\n self.exit', 'self.exit'), False), ('missing-reclaim-wake', source.replace('self.queue_events();', ''), False), - ('share-before-retirement', source.replace('&& self.owner.is_some() && self.retirement_ready', '&& self.owner.is_some()'), False), - ('share-during-kill', source.replace('&& self.exit.is_none() && self.active.len', '&& self.active.len'), False)] + ('share-before-retirement', mutate(r'(&&\s*self\.owner\.is_some\(\))\s*&&\s*self\.retirement_ready', r'\1'), False), + ('share-during-kill', mutate(r'&&\s*self\.exit\.is_none\(\)\s*(?=&&\s*self\.active\.len)', ''), False)] with tempfile.TemporaryDirectory(prefix='g17-capacity-') as tmp: for name, rust, passes in controls: path = Path(tmp) / (name + '.rs') diff --git a/tools/asahi/g17-capacity-watch.sh b/tools/asahi/g17-capacity-watch.sh index 5438413b2ec21..f3d203e506833 100644 --- a/tools/asahi/g17-capacity-watch.sh +++ b/tools/asahi/g17-capacity-watch.sh @@ -2,7 +2,7 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT # Run inside an already authorized test session. Does not mount debugfs. set -eu -capacity_file=${1:-/sys/kernel/debug/asahi-g17/capacity} +capacity_file=${1:-/sys/kernel/debug/aurora-g17/capacity} interval=${2:-5} case "$interval" in ''|*[!0-9]*|0) echo 'interval must be positive integer seconds' >&2; exit 2;; esac test -r "$capacity_file" diff --git a/tools/asahi/g17-telemetry-integration-tests.py b/tools/asahi/g17-telemetry-integration-tests.py new file mode 100644 index 0000000000000..5c36b6beed579 --- /dev/null +++ b/tools/asahi/g17-telemetry-integration-tests.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production ring transport and C formatter without hardware.""" +import argparse +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +DRIVER = ROOT / 'drivers/gpu/drm/asahi' + + +def item(source, signature): + start = source.index(signature) + end = source.index('{', start) + 1 + depth = 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +RUST_HEADER = r''' +#![allow(dead_code)] +extern crate self as kernel; +mod prelude { + pub type Result = std::result::Result; + pub const EIO: i32 = -5; + pub const EAGAIN: i32 = -11; +} +mod device { + pub struct Device; + impl AsRef for Device { fn as_ref(&self) -> &Device { self } } +} +mod fmt { pub use std::fmt::*; } +macro_rules! dev_err { ($dev:expr, $($t:tt)*) => { let _ = ($dev, format_args!($($t)*)); }; } +mod g17 { + use std::sync::atomic::{AtomicU64, Ordering}; + static NOW: AtomicU64 = AtomicU64::new(1); + fn now_ns() -> u64 { NOW.load(Ordering::Relaxed) } + #[derive(Clone, Copy, Debug)] + enum Role { Primary = 0, Secondary = 1 } + mod trace { pub fn report(_:u8, _:u8, _:&[u32;18]) {} pub fn message(_:u32,_:u64) {} pub const CONTROL:u32=0; } + mod fw { + pub mod variant { + #[derive(Clone, Copy)] + pub enum InitGraph { V25G83, G17P } + } + pub mod channels { + pub const EVENT_WORDS:usize=18; + pub const EVENT_RECORD_SIZE:usize=72; + pub const RX_PRODUCER_OFFSET:usize=0x20; + pub const TRACE_SLOTS:u32=4; + pub const TRACE_TYPES:u32=1; + pub const STATISTICS_SLOTS:u32=8; + pub const STATISTICS_TYPES:u32=0x5fff0000; + pub const DEVICE_CONTROL_SLOTS:u32=8; + pub struct ControlRecord; + #[derive(Clone, Copy)] + pub enum Opcode { Dummy } + impl Opcode { pub fn asserts_gpu_power(self)->bool { false } } + impl ControlRecord { + pub fn bytes(&self)->&[u8] { &[] } + pub fn opcode(&self)->Opcode { Opcode::Dummy } + pub fn asserts_gpu_power(&self)->bool { false } + } + } + pub mod initdata { pub mod status { + pub const TRACE_STATE:u32=0; + pub const TRACE_RING:u32=0x100; + pub const STATS_STATE:u32=0x1000; + pub const STATS_RING:u32=0x2000; + } } + } + mod initdata { + use super::{Role,fw::{channels::*,initdata::status::*}}; + use crate::prelude::*; + use std::{cell::RefCell,sync::atomic::AtomicU32}; + pub struct InitData { + pub cursors:[[AtomicU32;2];2], + pub slots:RefCell<[[[u32;18];8];2]>, + } + impl InitData { + pub fn new()->Self { Self { cursors:std::array::from_fn(|_|std::array::from_fn(|_|AtomicU32::new(0))), slots:RefCell::new([[[0;18];8];2]) } } + pub fn push(&self,role:Role,words:[u32;18]) { + use std::sync::atomic::Ordering::Relaxed; + let producer=self.cursors[role as usize][1].load(Relaxed); + self.slots.borrow_mut()[role as usize][producer as usize]=words; + self.cursors[role as usize][1].store((producer+1)%STATISTICS_SLOTS,Relaxed); + } + pub fn status_word(&self,role:Role,offset:usize)->Result<&AtomicU32> { + Ok(&self.cursors[role as usize][usize::from(offset==STATS_STATE as usize+RX_PRODUCER_OFFSET)]) + } + pub fn status_read_words(&self,role:Role,offset:usize)->Result<[u32;N]> { + let mut words=[0;N]; + words.copy_from_slice(&self.slots.borrow()[role as usize][(offset-STATS_RING as usize)/EVENT_RECORD_SIZE][..N]); + Ok(words) + } + pub fn control_consumer(&self)->Result<&AtomicU32> { Ok(&self.cursors[0][0]) } + pub fn control_producer(&self)->Result<&AtomicU32> { Ok(&self.cursors[0][1]) } + pub fn publish_control(&self,_:u32,_:&[u8],_:bool,_:u32)->Result { Ok(()) } + } + } + mod reports { + use super::Role; + pub struct Reports; + impl Reports { + pub fn new()->crate::prelude::Result { Ok(Self) } + pub fn record(&mut self,_:Role,_:[u32;18],_:u64) {} + pub fn write(&self,_:&mut crate::fmt::Formatter<'_>)->crate::fmt::Result { Ok(()) } + } + } +''' +RUST_TESTS = r''' + #[test] + fn roles_merge_and_any_statistics_quarantine_disables_both() { + use {channel::Rings,fw::variant::InitGraph,Role::*}; + let dev=crate::device::Device; + let init=initdata::InitData::new(); + let mut rings=Rings::new(InitGraph::G17P).unwrap(); + let sample=|ta,render,device|[0x13,0,0,ta,render,0xdeadbeef,device,0,0,0,0,0,0,0,0,0,0,0]; + init.push(Primary,sample(25,75,50)); + init.push(Secondary,sample(35,85,65)); + rings.drain_statistics(&dev,&init,Primary); + rings.drain_statistics(&dev,&init,Secondary); + let view=rings.read_statistics(1); + assert_eq!((view.tiler,view.renderer,view.device,view.util_samples),(30,80,57,2)); + assert_eq!(view.util_status,telemetry::VALID); + init.cursors[1][1].store(fw::channels::STATISTICS_SLOTS,Ordering::Relaxed); + rings.drain_statistics(&dev,&init,Secondary); + assert_eq!(rings.read_statistics(1).util_status,telemetry::STALE); + init.cursors[1][1].store(1,Ordering::Relaxed); + for _ in 0..3 { + init.push(Primary,sample(100,100,100)); + rings.drain_statistics(&dev,&init,Primary); + } + assert_eq!(rings.read_statistics(1).util_status,telemetry::STALE); + } + #[test] + fn suspend_gating_and_restart_discard_queued_epoch_records() { + use {channel::Rings,fw::variant::InitGraph,Role::*}; + let dev=crate::device::Device; + let init=initdata::InitData::new(); + let mut rings=Rings::new(InitGraph::V25G83).unwrap(); + let words=[0x13,0,0,25,75,0,50,0,0,0,0,0,0,0,0,0,0,0]; + init.push(Primary,words); + rings.drain_statistics(&dev,&init,Primary); + assert_eq!(rings.read_statistics(1).util_status,telemetry::VALID); + for boundary in 0..4 { + init.push(Primary,words); // Unread at the epoch boundary. + match boundary { + 0=> { rings.suspend_statistics(true); rings.drain_statistics(&dev,&init,Primary); rings.suspend_statistics(false); init.push(Primary,words); } + 1=>rings.observe_statistics_power(false,0), + 2=>rings.observe_statistics_power(true,1), // Wake observed by a fast publisher. + _=>rings.invalidate_statistics(), // Restart handshake. + } + assert_eq!(rings.read_statistics(1).util_status,telemetry::STALE); + rings.drain_statistics(&dev,&init,Primary); + assert_eq!(rings.read_statistics(1).util_status,telemetry::STALE); + init.push(Primary,words); + rings.drain_statistics(&dev,&init,Primary); + assert_eq!(rings.read_statistics(1).util_status,telemetry::VALID); + } + } + #[test] + fn pstate_uses_drain_order_without_role_deduplication() { + use {channel::Rings,fw::variant::InitGraph,Role::*}; + let dev=crate::device::Device; + let init=initdata::InitData::new(); + let mut rings=Rings::new(InitGraph::G17P).unwrap(); + init.push(Primary,[0x10,0x1000,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0]); + init.push(Secondary,[0x10,0x2500,0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0]); + rings.drain_statistics(&dev,&init,Primary); + rings.drain_statistics(&dev,&init,Secondary); + assert_eq!(rings.read_statistics(1).pstate,6); + init.push(Primary,[0x10,0x1000,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0]); + rings.drain_statistics(&dev,&init,Primary); + assert_eq!(rings.read_statistics(1).pstate_status,telemetry::UNAVAILABLE); + } +} +''' + +C_HEADER = r''' +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint64_t u64; +#define PAGE_SIZE 4096 +#define READ_ONCE(x) (x) +#define container_of(ptr,type,member) ((type*)((char*)(ptr)-offsetof(type,member))) +struct device {}; +struct device_attribute { + struct { const char *name; unsigned mode; } attr; + ssize_t (*show)(struct device*,struct device_attribute*,char*); +}; +struct notifier_block { int (*notifier_call)(struct notifier_block*,unsigned long,void*); }; +#define PM_SUSPEND_PREPARE 1 +#define PM_HIBERNATION_PREPARE 2 +#define PM_RESTORE_PREPARE 3 +#define PM_POST_SUSPEND 4 +#define PM_POST_HIBERNATION 5 +#define PM_POST_RESTORE 6 +#define NOTIFY_OK 1 +#define ENODEV 19 +#define ENOMEM 12 +#define BUILD_BUG_ON(x) _Static_assert(!(x),#x) +#define kzalloc_obj(x) calloc(1,sizeof(x)) +#define sysfs_attr_init(x) ((void)0) +#define kfree(x) free(x) +static int registered, removed, created, fail_file, fail_notifier, suspend_calls; +static int register_pm_notifier(struct notifier_block *nb) { (void)nb; if(fail_notifier)return -12; registered++; return 0; } +static int unregister_pm_notifier(struct notifier_block *nb) { (void)nb; removed++; return 0; } +static int device_create_file(struct device *dev,struct device_attribute *attr) { (void)dev;(void)attr; return ++created==fail_file?-12:0; } +static void device_remove_file(struct device *dev,struct device_attribute *attr) { (void)dev;(void)attr; } +static int emit(char *buf,size_t size,const char *fmt,...) { + va_list ap;va_start(ap,fmt);int n=vsnprintf(buf,size,fmt,ap);va_end(ap);return n<(int)size?n:(int)size-1; +} +#define scnprintf emit +#define sysfs_emit(buf,...) emit(buf,PAGE_SIZE,__VA_ARGS__) +#define sysfs_emit_at(buf,at,...) emit(buf+at,PAGE_SIZE-(at),__VA_ARGS__) +static ssize_t gpu_memory_show(struct device *dev,struct device_attribute *attr,char *buf) { (void)dev;(void)attr;(void)buf;return 0; } +static u32 *asahi_idle_word; +static struct device_attribute dev_attr_idle_off_delay_ms; +''' +C_TESTS = r''' +static struct asahi_qualified_stats current; +static void read_stats(const void *ptr,struct asahi_qualified_stats *out) { (void)ptr;*out=current; } +static void suspend_stats(const void *ptr,bool suspended) { (void)ptr;suspend_calls+=suspended?1:10; } +int main(void) { + struct asahi_stats_snapshot snap; + struct asahi_stats_attribute stats={.snapshot=&snap,.export_enabled=true}; + char old[PAGE_SIZE],now[PAGE_SIZE]; + for(unsigned seed=0;seed<512;seed++) { + snap=(struct asahi_stats_snapshot){ + .util1=seed, .util2=seed+11, .util3=~seed, .util4=seed*0x01010101u, + .pstate=seed%16, .avg_power_mw=seed*1234567u, + .temperature_raw=seed^0xff00aa11u, .temperature_scale=seed+37, + .temperature_tmin=seed+42, .temperature_tmax=seed+73, + .busy_ns=((u64)seed<<32)|(UINT32_MAX-seed), + .jobs=UINT64_MAX-(u64)seed*101, + }; + ssize_t before=old_agx_stats_show(NULL,&stats.attr,old); + ssize_t after=agx_stats_show(NULL,&stats.attr,now); + assert(before==after && !memcmp(old,now,before+1)); + } + stats.export_enabled=false; + assert(agx_stats_show(NULL,&stats.attr,now)==12); + assert(!strcmp(now,"unsupported\n")); + stats.export_enabled=true;stats.read=read_stats; + for(unsigned status=0;status<3;status++) { + current=(struct asahi_qualified_stats){.util_status=status,.pstate_status=status,.tiler=30,.renderer=80,.device=57,.pstate=6}; + agx_stats_show(NULL,&stats.attr,now); + assert(strstr(now,"busy_ns unsupported\n")); + assert(strstr(now,"power_mw unsupported\n")); + assert(strstr(now,"temperature_raw unsupported\n")); + assert(strstr(now,"requested_frequency_hz unsupported\n")); + assert(strstr(now,"achieved_frequency_hz unsupported\n")); + if(status==1) { + assert(strstr(now,"util_tiler_percent 30\nutil_renderer_percent 80\nutil_device_percent 57\n")); + assert(strstr(now,"pstate 6\n")); + } else { + assert(strstr(now,"util_device_percent unsupported\n")); + assert(strstr(now,"pstate unsupported\n")); + assert(strstr(now,status==2?"pstate_status stale\n":"pstate_status unavailable\n")); + } + } + struct device dev; + struct asahi_gpu_memory memory={0}; + void *handle=NULL; + assert(!asahi_sysfs_register(&dev,&snap,1,&memory,&handle,read_stats,suspend_stats)); + struct asahi_stats_attribute *registered_stats=handle; + assert(registered==1 && removed==0); + asahi_stats_pm_notify(®istered_stats->pm_notifier,PM_SUSPEND_PREPARE,NULL); + asahi_stats_pm_notify(®istered_stats->pm_notifier,PM_POST_SUSPEND,NULL); + assert(suspend_calls==11); + asahi_sysfs_unregister(&dev,handle); + assert(removed==1); + for(int failure=1;failure<=2;failure++) { + created=0;fail_file=failure;handle=NULL; + assert(asahi_sysfs_register(&dev,&snap,1,&memory,&handle,read_stats,suspend_stats)==-12); + assert(registered==removed); + assert(handle==NULL); + } + fail_file=0;fail_notifier=1;created=0; + assert(asahi_sysfs_register(&dev,&snap,1,&memory,&handle,read_stats,suspend_stats)==-12); + assert(created==0 && registered==removed); + puts("PASS: 512 byte-identical legacy outputs, G17 capabilities, PM lifecycle and registration rollback"); +} +''' + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('--out', type=Path) + ap.add_argument('--baseline', default='ebcf9cb777a434867e9d880892ee2b46887728bb', + help='revision containing the legacy formatter') + ap.add_argument('--mutation', choices=['fields', 'range', 'timestamp', 'age', + 'quarantine', 'epoch', 'legacy-format']) + args = ap.parse_args() + work = args.out or Path(tempfile.mkdtemp(prefix='asahi-g17-telemetry-')) + work.mkdir(parents=True, exist_ok=True) + work = work.resolve() + channel = (DRIVER / 'g17/channel.rs').read_text().split('/// One bounded event-ring pass.')[0] + # Compile the production transport and decoder; only MMIO/shared memory, + # kernel logging and allocation adapters are replaced by host fixtures. + channel = '\n'.join(line for line in channel.splitlines() if not line.startswith('//!')) + (work / 'channel.rs').write_text(channel) + telemetry = (DRIVER / 'g17/telemetry.rs').read_text() + changes = { + 'fields': ('[words[3], words[4], words[6]]', '[words[3], words[4], words[5]]'), + 'range': ('value > 100', 'value > 101'), + 'timestamp': ('u64::from(words[1]) | (u64::from(words[2]) << 32)', + 'u64::from(words[2]) | (u64::from(words[1]) << 32)'), + 'age': ('now_ns - received >= max_age_ns', 'now_ns - received > max_age_ns'), + 'quarantine': ('.any(|rings| rings[ReportRing::Statistics as usize])', '.any(|_| false)'), + 'epoch': ('self.discard_statistics = [true; 2]', 'self.discard_statistics = [false; 2]'), + } + if args.mutation in changes: + before, after = changes[args.mutation] + if args.mutation in ('quarantine', 'epoch'): + assert before in channel + channel = channel.replace(before, after, 1) + (work / 'channel.rs').write_text(channel) + else: + assert before in telemetry + telemetry = telemetry.replace(before, after, 1) + (work / 'telemetry.rs').write_text(telemetry) + policy = item((DRIVER / 'g17/stats.rs').read_text(), 'pub(super) const fn max_age_ns') + (work / 'rings.rs').write_text(RUST_HEADER + 'mod stats { use super::fw::variant::InitGraph;\n' + + policy + '\n}\nmod telemetry;\nmod channel;\n' + RUST_TESTS) + # Rust resolves child modules relative to the inline parent's directory. + (work / 'g17').mkdir(exist_ok=True) + for name in ['channel.rs', 'telemetry.rs']: + (work / 'g17' / name).write_text((work / name).read_text()) + subprocess.run(['rustc', '--edition=2021', '--test', str(work / 'rings.rs'), '-o', str(work / 'rings')], check=True) + subprocess.run([str(work / 'rings')], check=True) + source = (DRIVER / 'sysfs.c').read_text() + if args.mutation == 'legacy-format': + source = source.replace('READ_ONCE(snap->util1)', 'READ_ONCE(snap->util2)', 1) + header = (DRIVER / 'sysfs.h').read_text() + baseline = subprocess.check_output(['git', '-C', str(ROOT), 'show', args.baseline + ':drivers/gpu/drm/asahi/sysfs.c'], text=True) + pieces = [C_HEADER] + for signature in ['struct asahi_qualified_stats']: + pieces.append(item(header, signature) + ';') + pieces += [line for line in header.splitlines() if line.startswith('typedef')] + for signature in ['struct asahi_stats_snapshot', 'struct asahi_gpu_memory', 'struct asahi_stats_attribute']: + pieces.append(item(source, signature) + ';') + pieces.append(item(baseline, 'static ssize_t agx_stats_show').replace('agx_stats_show(', 'old_agx_stats_show(', 1)) + for signature in ['static const char *asahi_stats_quality', 'static ssize_t agx_qualified_stats_show', + 'static int asahi_stats_pm_notify', 'static ssize_t agx_stats_show', + 'int asahi_sysfs_register(', 'void asahi_sysfs_unregister(']: + pieces.append(item(source, signature)) + pieces.append(C_TESTS) + (work / 'formatter.c').write_text('\n'.join(pieces)) + subprocess.run(['cc', '-std=gnu11', '-Wall', '-Wextra', '-Werror', '-Wno-unused-parameter', + str(work / 'formatter.c'), '-o', str(work / 'formatter')], check=True) + subprocess.run([str(work / 'formatter')], check=True) + + +if __name__ == '__main__': + try: + main() + except subprocess.CalledProcessError as error: + print(f'child exit: {error.returncode}') + raise SystemExit(error.returncode if error.returncode > 0 else 128 - error.returncode) diff --git a/tools/asahi/g17-telemetry-tests.rs b/tools/asahi/g17-telemetry-tests.rs new file mode 100644 index 0000000000000..bfedf6ea03155 --- /dev/null +++ b/tools/asahi/g17-telemetry-tests.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Run the production decoder's synthetic vectors and epoch tests on the host. +#![allow(dead_code)] +#[path = "../../drivers/gpu/drm/asahi/g17/telemetry.rs"] +mod telemetry; diff --git a/tools/asahi/g17-unpublished-tests.py b/tools/asahi/g17-unpublished-tests.py index a162b2651a037..f06eedf7f64f8 100644 --- a/tools/asahi/g17-unpublished-tests.py +++ b/tools/asahi/g17-unpublished-tests.py @@ -4,6 +4,7 @@ from pathlib import Path import importlib.util import os +import re import subprocess import tempfile ROOT=Path(__file__).resolve().parents[2]/'drivers/gpu/drm/asahi/g17' @@ -93,9 +94,11 @@ def main(): assert compute.count('visible = true;\n self.unpublished_binding = false;')==2 backend=(ROOT/'runtime/backend.rs').read_text() assert 'if !lease.is_current() { drop(state); drop(lease); continue; }' in backend + no_kill, count = re.subn(r'\|\|\s*self\.exit\.is_some\(\)\s*(?=\|\|\s*self\.active\.len\s*!=\s*0)', '', source) + assert count == 1, count cases=[('fixed',source),('omit-first-failure-rollback',source.replace('self.rollback_unpublished_compute().is_err()','false')), ('rollback-published-binding',source.replace('|| !self.unpublished_binding','')), - ('rollback-issued-kill',source.replace('|| self.exit.is_some() || self.active.len != 0','|| self.active.len != 0'))] + ('rollback-issued-kill',no_kill)] with tempfile.TemporaryDirectory(prefix='g17-unpublished-') as tmp: for name,code in cases: path=Path(tmp)/(name+'.rs');path.write_text(code);binary=path.with_suffix('') diff --git a/tools/asahi/mt7932-association-events-tests.py b/tools/asahi/mt7932-association-events-tests.py index 436ab4d00b75f..7ac7ef7709e08 100644 --- a/tools/asahi/mt7932-association-events-tests.py +++ b/tools/asahi/mt7932-association-events-tests.py @@ -85,7 +85,7 @@ def function(source, name): bool connecting,connected,disconnecting,disconnect_seen,disconnect_event_received; bool disconnect_local,connect_open,assoc_request_seen,peer_valid,link_failed,stopping,rf_ready; u8 connect_channel,connect_center,peer_wtbl,smart_version,waiting; - enum nl80211_band connect_band; u8 six_assoc_power[2][1020]; + enum nl80211_band connect_band; u8 six_assoc_power[2][1020]; bool six_legacy_assoc; int waiting_eid; struct cfg80211_chan_def connect_chandef; struct test_band band2,band5; diff --git a/tools/asahi/mt7932-connect-result-tests.py b/tools/asahi/mt7932-connect-result-tests.py index 2f8a4bf9d4c62..87054ccfcef80 100755 --- a/tools/asahi/mt7932-connect-result-tests.py +++ b/tools/asahi/mt7932-connect-result-tests.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only -"""Compile the connect worker's report path and the complete nl80211 encoder.""" +"""Compile the connect worker's report path, its log-level helper and the +complete nl80211 encoder.""" from pathlib import Path import argparse import hashlib @@ -41,9 +42,9 @@ #define spin_unlock(l) ((void)(l)) #define mutex_lock(l) ((void)(l)) #define mutex_unlock(l) ((void)(l)) -#define dev_info(...) ((void)0) +#define dev_info(...) ((void)info_logs++) #define dev_dbg(...) ((void)0) -#define dev_err(...) ((void)0) +#define dev_err(...) ((void)err_logs++) #define for_each_valid_link(cr,l) for((l)=0;(l)<1;(l)++) struct sk_buff { unsigned values[32]; bool seen[32]; }; struct nlattr { int n; }; @@ -75,11 +76,11 @@ static int wiphy_net(int*w){return 0;} static void genlmsg_multicast_netns(int*f,int n,struct sk_buff*m,int p,int g,gfp_t t){} ENCODER -struct mt7932 {bool stopping,connect_open,assoc_request_seen,connecting,connected,disconnecting,connect_cancelled,connect_publishing,connect_lost,rf_ready,connect_pmk_valid; +struct mt7932 {bool link_failed,stopping,connect_open,assoc_request_seen,connecting,connected,disconnecting,connect_cancelled,connect_publishing,connect_lost,rf_ready,connect_pmk_valid; int connect_error,response_lock,command_mutex,data_lock,wiphy,power_work;u16 disconnect_reason;struct net_device *netdev;struct cfg80211_bss *connect_bss;u8 connect_bssid[6],connect_pmk[32],connect_sae_pwd[63],connect_sae_pwd_length,assoc_request_ies[512];unsigned assoc_request_ie_len; struct {struct {bool ht_supported;}ht_cap;}band2;bool power_tim,data_ready,station_signal_valid,station_rx_rate_valid,station_tx_rate_valid;unsigned long station_tx_bytes,station_rx_bytes,station_tx_packets,station_rx_packets,station_connected;}; static unsigned long jiffies=1; -static unsigned notifications,disconnections,retirements,releases,authorizations,carrier,work_count; +static unsigned info_logs,err_logs,notifications,disconnections,retirements,releases,authorizations,carrier,work_count; static struct cfg80211_connect_resp_params saved; static bool mt_rf_allowed(struct mt7932*m){return m->rf_ready;} static void mt_transport_snapshot(struct mt7932*m){} @@ -96,16 +97,22 @@ static void netif_carrier_on(struct net_device*n){carrier++;} static void netif_wake_queue(struct net_device*n){} static void cfg80211_port_authorized(struct net_device*n,const u8*b,const u8*p,size_t l,gfp_t f){authorizations++;} -static void report_worker(struct mt7932 *m,int ret,bool retirement_attempted){ +HELPER +static void report_worker(struct mt7932 *m,int ret,bool retirement_attempted,bool auth_submitted){ struct cfg80211_connect_resp_params response={};unsigned long flags;bool submitted=true,peer_owned=true,cancelled;u16 reason; REPORT int main(void){struct net_device n={1};struct cfg80211_bss b={}; int errors[]={0,-ETIMEDOUT,-ECONNREFUSED,-EPROTO,-ECANCELED,-ECONNRESET,-EIO}; - for(unsigned i=0;i 65536 or data[:4] != b'J7RP' or ( + struct.unpack_from(' len(data): + raise ValueError('truncated legacy table') + at += struct.unpack_from(' len(data) - at + or not 0 < mask < (1 << 59) or alpha2 != country.encode('ascii') or any(reserved)): + raise ValueError('class header') + start = at + 56 + channels = [data[start + index * 122] for index in range(records)] + expected = [2] + [ch for ch in range(1, 234, 2) + if six.power.channel_width_class(3, ch) <= 3 and ch != 231] + if channels != expected: + raise ValueError('class channel order') + for bit in range(59): + if mask & (1 << bit): + index = channels.index(1 + 4 * bit) + values = data[start + index * 122 + 1:start + (index + 1) * 122] + if all(value in (0xc4, 0x7f) for value in values): + raise ValueError('primary has no power data') + result[cid] = mask + previous = cid + at += length + if at != len(data) or 2 not in result: + raise ValueError('trailing data or missing LPI') + return result + + +def bind(archive_data, country): + """v4: unchanged class data, binding revision 1 (SPEC 2026-10-31).""" + parse_archive(archive_data, country) + data = bytearray(archive_data) + at = 16 + data[8] * 328 + for _ in range(13): + at += struct.unpack_from(' +#include +#include +#include +#include +#include +typedef uint8_t u8; typedef uint16_t u16; typedef uint32_t u32; typedef uint64_t u64; +enum nl80211_band { NL80211_BAND_2GHZ, NL80211_BAND_5GHZ, NL80211_BAND_6GHZ=3 }; +#define BIT_ULL(n) (1ULL << (n)) +#define ARRAY_SIZE(x) (sizeof(x)/sizeof(*(x))) +#define IEEE80211_CHAN_DISABLED 1 +#define IEEE80211_CHAN_NO_IR 2 +#define IEEE80211_CHAN_RADAR 4 +#define IEEE80211_CHAN_NO_20MHZ 8 +#define NL80211_SCAN_FLAG_COLOCATED_6GHZ 0x4000 +#define WLAN_AKM_SUITE_SAE 0x000fac08 +#define WLAN_AKM_SUITE_OWE 0x000fac12 +struct ieee80211_channel {enum nl80211_band band; u32 flags; u8 hw_value;}; +struct ieee80211_supported_band {enum nl80211_band band; unsigned n_channels; struct ieee80211_channel *channels;}; +struct cfg80211_ssid {u8 ssid[32]; unsigned ssid_len;}; +struct cfg80211_scan_request { + struct cfg80211_ssid *ssids; unsigned n_ssids,n_channels,flags,n_6ghz_params; + const u8 *ie; size_t ie_len; struct ieee80211_channel *channels[80]; +}; +static u16 get_unaligned_le16(const void *p){const u8 *b=p;return b[0]|b[1]<<8;} +static void put_unaligned_le16(u16 v,void *p){u8 *b=p;b[0]=v;b[1]=v>>8;} +static void put_unaligned_le32(u32 v,void *p){u8 *b=p;for(unsigned i=0;i<4;i++)b[i]=v>>(8*i);} +#define READ_ONCE(x) (x) +#define spin_lock_irqsave(l,f) ((void)(l),(f)=0) +#define spin_unlock_irqrestore(l,f) ((void)(l),(void)(f)) +#define msecs_to_jiffies(x) (x) +#define dev_dbg(...) ((void)0) +static bool command_locked=true; +#define mutex_unlock(l) do {assert(command_locked);command_locked=false;} while(0) +#define mutex_lock(l) do {assert(!command_locked);command_locked=true;} while(0) +static void reinit_completion(int *c){*c=0;} +static void schedule_delayed_work(int *w,unsigned long t){assert(t==60000);} +struct mt7932 { + struct cfg80211_scan_request *scan_request; + unsigned scan_batch,scan_cursor,scan_band,scan_seq; + int response_lock,scan_done,scan_timeout_work,command_mutex,discovery_done; + bool scan_aborted,disconnecting,connected,scan_finished,rf_ready; + bool connect_cancelled,discovering,discovery_finished,link_failed; + u8 scan_home_channel; enum nl80211_band scan_home_band; + u64 six_legacy_channels; + unsigned connect_akm; int connect_error; + struct ieee80211_supported_band band2,band5,band6; + struct ieee80211_channel *connect_channel_req; + u8 connect_ssid[32]; unsigned connect_ssid_length; + void *connect_bss,*pdev; +}; +static unsigned commands,waits; +static u8 bodies[4][0x4d4]; +static struct mt7932 *active; +static bool mt_rf_allowed(struct mt7932 *m){return m->rf_ready;} +static int mt_request(struct mt7932 *m,u8 cid,bool runtime,bool set,bool reply,const void *body,size_t n){ + assert(command_locked&&cid==3&&runtime&&set&&!reply&&n==0x4d4&&commands<4); + memcpy(bodies[commands++],body,n);return 0; +} +static unsigned long wait_for_completion_timeout(int *c,unsigned long timeout){ + assert(!command_locked&&c==&active->discovery_done&&commands); + const u8 *b=bodies[commands-1];assert(!b[2]); + assert(timeout==5000+233U*b[0x9f]); + assert(timeout>220U*b[0x9f]+100); + waits++;active->discovery_finished=true;return 1; +} +static void *mt_connect_find_bss(struct mt7932 *m){return commands==2?m:NULL;} +HEADERS +SUBMIT +DISCOVER +static void reset(struct mt7932 *m){ + memset(m,0,sizeof(*m));memset(bodies,0,sizeof(bodies)); + m->rf_ready=true;commands=waits=0;active=m;command_locked=true; +} +int main(void){ + struct mt7932 m;struct cfg80211_scan_request request={}; + struct cfg80211_ssid ssid={.ssid="hidden",.ssid_len=6}; + struct ieee80211_channel channels[59]; + for(unsigned i=0;i<59;i++){ + channels[i]=(struct ieee80211_channel){.band=NL80211_BAND_6GHZ,.hw_value=1+4*i}; + request.channels[i]=channels+i; + } + request.n_channels=59;request.ssids=&ssid;request.n_ssids=1; + /* Even legacy channels must listen when RNR constraints are not serialized. */ + for(unsigned context=0;context<2;context++){ + reset(&m);m.scan_request=&request;m.six_legacy_channels=(1ULL<<59)-1; + request.flags=context?0:NL80211_SCAN_FLAG_COLOCATED_6GHZ; + request.n_6ghz_params=context?1:0; + assert(mt_scan_submit_next(&m)==0);assert(commands==1&&bodies[0][0x9f]==32); + assert(mt_scan_submit_next(&m)==0);assert(commands==2&&bodies[1][0x9f]==27); + assert(mt_scan_submit_next(&m)==1);assert(commands==2); + for(unsigned batch=0;batch<2;batch++){ + const u8 *b=bodies[batch];assert(!b[2]&&!b[3]&&!b[4]); + assert(get_unaligned_le16(b+0x4c2)==220); + for(unsigned i=0;iflags & NL80211_SCAN_FLAG_COLOCATED_6GHZ)', 'false').replace( + 'request->n_6ghz_params ||', 'false ||') + variants['mutant-rnr-probes'] = bad + bad = dict(parts) + bad['DISCOVER'] = bad['DISCOVER'].replace('mt7932_scan_discovery_timeout(body)', '5000') + variants['mutant-short-wait'] = bad + results = {} + for name, content in variants.items(): + source = PREFIX + for key, value in content.items(): + source = source.replace('\n' + key + '\n', '\n' + value + '\n') + c = args.out / (name + '.c') + binary = args.out / name + c.write_text(source) + subprocess.run(['cc', '-std=gnu11', '-Wall', '-Werror', '-Wno-unused-function', + '-Wno-unused-variable', '-fsanitize=address,undefined', + '-fno-omit-frame-pointer', str(c), '-o', str(binary)], check=True) + run = subprocess.run([str(binary.resolve())], capture_output=True, text=True) + (args.out / (name + '.log')).write_text(run.stdout + run.stderr) + results[name] = {'exit': run.returncode, + 'source_sha256': hashlib.sha256(source.encode()).hexdigest()} + print(name, json.dumps(results[name]), run.stdout.strip()) + if name == 'production': + assert run.returncode == 0, run.stderr + else: + assert run.returncode == -6 and 'Assertion' in run.stderr, run.stderr + (args.out / 'results.json').write_text(json.dumps(results, indent=2) + '\n') + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/mtp-pm-tests.py b/tools/asahi/mtp-pm-tests.py new file mode 100644 index 0000000000000..7d5b84d662ef0 --- /dev/null +++ b/tools/asahi/mtp-pm-tests.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Run production MTP PM and magicmouse resume callbacks without a device. + +Checks notification order, suspend rollback, resume-error handling and the +actuator mode cache. The old transport must compile and fail at runtime. +""" +import argparse +import os +from pathlib import Path +import resource +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +TRANSPORT = 'drivers/hid/dockchannel-hid/dockchannel-hid.c' + + +def function(source, name): + for kind in ('int', 'void'): + prefix = 'static ' + kind + ' ' + name + '(' + if prefix in source: + start = source.index(prefix) + return source[start:source.index('\n}', start)+2] + raise ValueError(name) + + +PREFIX = r''' +#include +#include +#include +#include +#define MAX_INTERFACES 16 +#define PMSG_SUSPEND 1 +#define HID_HAPTIC_MODE_DEVICE 0 +#define HID_HAPTIC_MODE_HOST 1 +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x) = (v)) +struct haptics { int mode; }; +struct magicmouse_sc { struct haptics *haptics; bool haptic_button_down, haptic_deep_down; }; +struct hid_device { struct magicmouse_sc *data; int id; }; +struct dchid_iface { struct hid_device *hid; }; +struct dockchannel_hid { struct dchid_iface *ifaces[MAX_INTERFACES]; bool suspended; }; +struct device { struct dockchannel_hid *data; }; +static struct dockchannel_hid *transport; +static unsigned suspends[16], resumes[16], key_releases; +static int suspend_error_at = -1, resume_error_at = -1; +static void *dev_get_drvdata(struct device *d) { return d->data; } +static void *hid_get_drvdata(struct hid_device *d) { return d->data; } +static void magicmouse_force_key(struct magicmouse_sc *m, bool down) { + assert(!down); key_releases++; +} +static int hid_driver_suspend(struct hid_device *h, int state) { + assert(state == PMSG_SUSPEND && !transport->suspended); + suspends[h->id]++; + return h->id == suspend_error_at ? -EBUSY : 0; +} +''' +MIDDLE = r''' +static int hid_driver_resume(struct hid_device *h) { + assert(!transport->suspended); + resumes[h->id]++; + magicmouse_resume(h); + return h->id == resume_error_at ? -EIO : 0; +} +''' +CASES = r''' +int main(void) { + struct haptics hap = {.mode = HID_HAPTIC_MODE_HOST}; + struct magicmouse_sc m = {.haptics = &hap, .haptic_button_down = true, .haptic_deep_down = true}; + struct hid_device h0 = {.data = &m, .id = 0}, h4 = {.id = 4}, h8 = {.id = 8}; + struct dchid_iface i0 = {.hid = &h0}, i4 = {.hid = &h4}, i8 = {.hid = &h8}, empty = {0}; + struct dockchannel_hid t = {.ifaces = {[0] = &i0, [3] = &empty, [4] = &i4, [8] = &i8}}; + struct device d = {.data = &t}; transport = &t; + assert(!dockchannel_hid_suspend(&d) && t.suspended); + assert(!dockchannel_hid_resume(&d) && !t.suspended); + assert(hap.mode == HID_HAPTIC_MODE_DEVICE); + assert(!m.haptic_button_down && !m.haptic_deep_down && key_releases == 1); + assert(suspends[0] == 1 && resumes[0] == 1 && resumes[8] == 1); + /* A later HID failure rolls back only the drivers that suspended. */ + suspend_error_at = 4; + assert(dockchannel_hid_suspend(&d) == -EBUSY && !t.suspended); + assert(resumes[0] == 2 && resumes[4] == 1 && suspends[8] == 1); + suspend_error_at = 0; + assert(dockchannel_hid_suspend(&d) == -EBUSY && !t.suspended); + assert(resumes[0] == 2 && suspends[4] == 2); + suspend_error_at = -1; + assert(!dockchannel_hid_suspend(&d) && t.suspended); + /* One failed resume cannot keep the other interfaces suspended. */ + resume_error_at = 4; + assert(dockchannel_hid_resume(&d) == -EIO && !t.suspended); + assert(resumes[0] == 3 && resumes[4] == 2 && resumes[8] == 2); + resume_error_at = -1; + for (unsigned n = 0; n < 3; n++) { + hap.mode = HID_HAPTIC_MODE_HOST; + assert(!dockchannel_hid_suspend(&d)); + assert(!dockchannel_hid_resume(&d)); + assert(hap.mode == HID_HAPTIC_MODE_DEVICE); + } + puts("MTP PM: HID notifications, rollback, all-interface resume and haptic cache passed"); +} +''' + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--out') + p.add_argument('--old-transport') + args = p.parse_args() + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + source = ((ROOT / TRANSPORT).read_text() if not args.old_transport else subprocess.check_output( + ['git', 'show', args.old_transport + ':' + TRANSPORT], cwd=ROOT, text=True)) + mouse = (ROOT / 'drivers/hid/hid-magicmouse.c').read_text() + with tempfile.TemporaryDirectory(prefix='mtp-pm-') as tmp: + out = Path(args.out or tmp) + out.mkdir(parents=True, exist_ok=True) + c, exe = out/'mtp-pm.c', out/'mtp-pm' + c.write_text(PREFIX + function(mouse, 'magicmouse_haptic_forget_mode') + '\n' + + function(mouse, 'magicmouse_resume') + MIDDLE + + function(source, 'dockchannel_hid_suspend') + '\n' + + function(source, 'dockchannel_hid_resume') + CASES) + subprocess.run([os.environ.get('CC', 'cc'), '-std=c11', '-Wall', '-Wextra', + '-Wno-unused-parameter', '-Wno-unused-function', + str(c), '-o', str(exe)], check=True) + r = subprocess.run([str(exe)], capture_output=True, text=True) + (out/'run.log').write_text(r.stdout+r.stderr) + if args.old_transport: + if r.returncode != -6 or 'Assertion' not in r.stderr: + raise SystemExit('old transport did not fail its runtime assertion') + print('MTP PM: old transport compiled and failed the haptic-cache assertion') + elif r.returncode: + raise SystemExit(r.stdout+r.stderr) + else: + print(r.stdout, end='') + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/neo-stability-tests.py b/tools/asahi/neo-stability-tests.py index ff02d627e5711..16256c1843fd8 100644 --- a/tools/asahi/neo-stability-tests.py +++ b/tools/asahi/neo-stability-tests.py @@ -116,6 +116,7 @@ def function(source, name): #define for_each_set_bit(i,b,n) for((i)=0;(i)<(n);(i)++) if((b)[(i)/64]&(1ULL<<((i)%64))) #define bitmap_zero(b,n) memset((b),0,((n)+63)/64*8) #define IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT 128 +#define IEEE80211_CHAN_NO_HE 256 #define NL80211_RRF_NO_IR (1<<7) struct ieee80211_channel {unsigned flags,hw_value;int max_power;enum nl80211_band band;int center_freq;s8 psd;}; static u64 get_unaligned_le64(const u8*p){return get_unaligned_le32(p)|(u64)get_unaligned_le32(p+4)<<32;} @@ -131,7 +132,7 @@ def function(source, name): static void rtnl_unlock(void){assert(rtnl_depth--==1);} static void wiphy_lock(struct wiphy*w){assert(rtnl_depth&& !wiphy_depth++);} static void wiphy_unlock(struct wiphy*w){assert(wiphy_depth--==1);} -struct mt7932 {struct ieee80211_channel channels[MT7932_CHANNELS_2G],channels5[MT7932_CHANNELS_5G],channels6[MT7932_CHANNELS_6G];struct wiphy*wiphy;struct net_device*netdev;struct {struct device dev;}*pdev;struct mt7932_reg_snapshot reg_desired,reg_deferred;unsigned reg_generation,response_lock,startup_work;bool reg_pending,stopping,interface_registered,connecting,connected,disconnecting,reg_deferral,data_ready;int data_lock;int connect_error,assoc_start,assoc_done,discovery_done;uint64_t policy_disabled[(MT7932_CHANNELS+63)/64];}; +struct mt7932 {struct ieee80211_channel channels[MT7932_CHANNELS_2G],channels5[MT7932_CHANNELS_5G],channels6[MT7932_CHANNELS_6G];struct wiphy*wiphy;struct net_device*netdev;struct {struct device dev;}*pdev;struct mt7932_reg_snapshot reg_desired,reg_deferred;unsigned reg_generation,response_lock,startup_work;bool reg_pending,stopping,interface_registered,connecting,connected,disconnecting,reg_deferral,data_ready,six_ap_valid;int data_lock;int connect_error,assoc_start,assoc_done,discovery_done;uint64_t policy_disabled[(MT7932_CHANNELS+63)/64];}; #define spin_lock(l) ((void)(l)) #define spin_unlock(l) ((void)(l)) static void mt_data_kick_locked(struct mt7932*m){assert(0 && "unexpected TX boundary");} @@ -191,7 +192,14 @@ def function(source, name): memset(tables[t]+44+j*122,0xc4,122);tables[t][44+j*122]=c->hw_value; if(i!=10)tables[t][45+j*122]=1; } - /* 6 GHz: VLP permission bitmap, NO_IR until cfg80211's rule lifts it, PSD limits. */ + /* The VLP-client flag gates the selected AP class, not the whole + * country domain: the same channel can still admit LPI. + */ + assert(mt7932_6g_class(2,0)==MT7932_6G_VLP); + assert(mt7932_6g_class(2,IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT)==-EACCES); + assert(mt7932_6g_class(0,IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT)==MT7932_6G_LPI); + assert(mt7932_6g_class(1,0)==-EOPNOTSUPP); + /* Country bitmap, NO_IR until cfg80211's rule lifts it, PSD limits. */ p.six_channels=GENMASK_ULL(MT7932_CHANNELS_6G-1,0)&~BIT_ULL(3); for(unsigned k=0;khw_value&&!f); - else{unsigned k=i-N25+(i>=VLP);assert(get_unaligned_le16(e)==mt7932_channel_6g(k)&&f==IEEE80211_CHAN_PSD);} + else{unsigned k=i-N25;u32 expected=IEEE80211_CHAN_PSD|(i==VLP?IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT:0);assert(get_unaligned_le16(e)==mt7932_channel_6g(k)&&f==expected);} } for(unsigned k=0;k=N25-1?c!=13:true);} + assert(reg.length==12+(MT7932_CHANNELS-2)*8&®.domain[8]==12&®.domain[9]==MT7932_CHANNELS_5G&®.domain[10]==MT7932_CHANNELS_6G-1); + for(unsigned i=0;i=N25-1?c!=13:true);} /* A rule that keeps NO_IR, or no rule at all, leaves the channel passive. */ m.channels6[0].flags|=IEEE80211_CHAN_NO_IR;m.channels6[1].flags|=IEEE80211_CHAN_NO_IR; rule6.flags=NL80211_RRF_NO_IR;mt_policy_disable(&m,&p,generation);assert(m.channels6[0].flags&IEEE80211_CHAN_NO_IR); @@ -255,6 +264,7 @@ def main(): 'drivers/soc/apple/dockchannel.c': ['dockchannel_init_wakeup'], 'drivers/soc/apple/pm_shim.c': ['sep_pm_release_domains', 'sep_pm_keep_domains'], 'drivers/usb/dwc3/dwc3-apple.c': ['dwc3_apple_core_stop', 'dwc3_apple_init', 'dwc3_apple_exit'], + 'drivers/net/wireless/mediatek/mt7932/six.h': ['mt7932_6g_class'], 'drivers/net/wireless/mediatek/mt7932/startup.c': ['mt_channel', 'mt_policy_disable', 'mt_regulatory_notify'], } bodies = {} @@ -267,7 +277,13 @@ def main(): bodies[name] = function(text, name) headers = [ROOT/'drivers/net/wireless/mediatek/mt7932/channels.h', ROOT/'drivers/net/wireless/mediatek/mt7932/regulatory.h'] - header = '\n'.join(re.sub(r'^#include .*\n', '', path.read_text(), flags=re.M) + # regulatory.h's class parser/pages depend on the production enum. + # Extract only that declaration, avoiding the unrelated HE/TPE helpers. + six = (ROOT/'drivers/net/wireless/mediatek/mt7932/six.h').read_text() + power_class = re.search(r'^enum mt7932_6g_class \{[^}]*\};', six, re.M) + if not power_class: + raise ValueError('MT7932 6 GHz class enum') + header = power_class[0] + '\n' + '\n'.join(re.sub(r'^#include .*\n', '', path.read_text(), flags=re.M) for path in headers) # The receive IRQ request statement from dockchannel_init(), run against a recording stub. dock = (ROOT/'drivers/soc/apple/dockchannel.c').read_text() @@ -287,7 +303,7 @@ def main(): 'rx-suspend': ('IRQF_NO_AUTOEN | IRQF_NO_SUSPEND', 'IRQF_NO_AUTOEN'), 'stale-policy': ('generation != READ_ONCE(m->reg_generation)', 'false'), 'usb-rpm': ('pm_runtime_put_autosuspend(appledwc->dev);', '(void)0;'), - 'vlp-client': ('if (flags & IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT)', 'if (0)'), + 'vlp-client': ('return flags & IEEE80211_CHAN_NO_6GHZ_VLP_CLIENT ?', 'return false ?'), 'six-band': ('(policy->six_channels & BIT_ULL((channel - 1) / 4))', 'true'), } if args.mutation: diff --git a/tools/asahi/pciec-policy-tests.py b/tools/asahi/pciec-policy-tests.py new file mode 100644 index 0000000000000..825caefac0912 --- /dev/null +++ b/tools/asahi/pciec-policy-tests.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise PCIe-C cold-init admission and ACIO admission using production C.""" +import argparse +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] + + +def function(source, name): + start = re.search(r'^(?:static\s+)?bool\s+' + name + r'\(', source, re.M).start() + end = source.index('{', start) + 1 + depth = 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +FIXTURE = r''' +#include +#include +#include +#include +#include +typedef uint32_t u32; +#define BIT(n) (1U << (n)) +#define GENMASK(h, l) ((~0U << (l)) & (~0U >> (31 - (h)))) +struct device_node { + const char *compat[3]; + bool kernel_init, tunables; + int status, refs; + struct device_node *dart, *child; +}; +struct of_device_id { const char *compatible; const void *data; }; +struct of_phandle_args { struct device_node *np; }; +static int maps; +static bool of_device_is_compatible(struct device_node *np, const char *compat) +{ + for (int i = 0; np && i < 3 && np->compat[i]; i++) + if (!strcmp(np->compat[i], compat)) + return true; + return false; +} +static const struct of_device_id *of_match_node(const struct of_device_id *ids, + struct device_node *np) +{ + for (int i = 0; np && i < 3 && np->compat[i]; i++) + for (const struct of_device_id *id = ids; id->compatible; id++) + if (!strcmp(np->compat[i], id->compatible)) + return id; + return NULL; +} +static bool of_property_read_bool(struct device_node *np, const char *name) +{ + assert(!strcmp(name, "apple,pciec-kernel-init")); + return np->kernel_init; +} +static bool of_property_present(struct device_node *np, const char *name) +{ + assert(!strcmp(name, "apple,tunable")); + return np->tunables; +} +static int of_property_read_u32(struct device_node *np, const char *name, u32 *v) +{ + assert(!strcmp(name, "apple,pciec-preinit-status")); + if (np->status < 0) + return -1; + *v = np->status; + return 0; +} +static int of_map_iommu_id(struct device_node *np, int rid, struct of_phandle_args *a) +{ + assert(rid == 0); + maps++; + if (!np->dart) + return -1; + a->np = np->dart; + a->np->refs++; + return 0; +} +static void of_node_put(struct device_node *np) { assert(np->refs > 0); np->refs--; } +#define pr_warn_once(...) ((void)0) +#define for_each_available_child_of_node_scoped(parent, child) \ + for (struct device_node *child = (parent)->child; child; child = NULL) +/* PRODUCTION */ +static void expect(struct device_node *host, bool cold, bool admitted) +{ + struct device_node parent = { .child = host }; + maps = 0; + assert(apple_pcie_tunnel_needs_cold_init(host) == cold); + assert(apple_cio_pcie_tunnel_is_preinitialized(&parent) == admitted); + assert(!host->dart || host->dart->refs == 0); + if (!host->kernel_init || host->status == 1) + assert(maps == 0); +} +int main(void) +{ + struct device_node dart = { .tunables = true }; + const char *intr2axi[] = { "apple,t8112-pciec", "apple,t6000-pciec" }; + for (unsigned int i = 0; i < 2; i++) { + /* M2 must win over its T8103 fallback, without a board allowlist. */ + struct device_node host = { .compat = { intr2axi[i], "apple,t8103-pciec" }, + .status = -1, .dart = &dart }; + const struct of_device_id *id = of_match_node(apple_pcie_of_match, &host); + assert(id && ((const struct hw_info *)id->data)->tunnel_intr2axi); + expect(&host, false, false); /* No implicit cold reset. */ + host.kernel_init = true; + expect(&host, true, true); + dart.tunables = false; + expect(&host, false, false); /* Do not let the fallback bypass DART admission. */ + host.status = 1; + expect(&host, false, true); /* Preserve a firmware handoff. */ + host.status = 0; + expect(&host, false, false); + dart.tunables = true; + expect(&host, true, true); /* Failed/missing handoff can be initialized. */ + host.dart = NULL; + expect(&host, false, false); + } + struct device_node m1 = { .compat = { "apple,t8103-pciec" }, .status = -1 }; + expect(&m1, false, false); + m1.kernel_init = true; + expect(&m1, true, true); /* Original M1 does not need T8110 DART tunables. */ + m1.status = 1; + expect(&m1, false, true); + struct device_node regular = { .compat = { "apple,pcie" }, .kernel_init = true, + .status = -1 }; + expect(®ular, false, false); + regular.compat[0] = "apple,unknown-pciec"; + expect(®ular, false, false); + puts("PASS: M1, M1 Pro, M2, firmware handoff, missing DART and DT opt-in"); + return 0; +} +''' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--out', type=Path) + args = parser.parse_args() + pci = (ROOT / 'drivers/pci/controller/pcie-apple.c').read_text() + acio = (ROOT / 'drivers/thunderbolt/apple.c').read_text() + defines = '\n'.join(re.findall(r'^#define .*$', pci, re.M)) + info = pci[pci.index('struct hw_info {'):pci.index('static const struct of_device_id apple_pcie_of_match[];')] + matches = re.search(r'static const struct of_device_id apple_pcie_of_match\[\] = \{.*?\n\};', pci, re.S)[0] + production = '\n'.join((defines, info, matches, + function(pci, 'apple_pcie_tunnel_needs_cold_init'), + function(acio, 'apple_cio_pcie_tunnel_is_preinitialized'))) + code = FIXTURE.replace('/* PRODUCTION */', production) + with tempfile.TemporaryDirectory(prefix='pciec-policy-') as tmp: + out = args.out or Path(tmp) + out.mkdir(parents=True, exist_ok=True) + cases = {'production': code, + 'm2-fallback': code.replace('{ .compatible = "apple,t8112-pciec",\t.data = &t8112_pciec_hw },', ''), + 'dart-bypass': code.replace('tunables = of_property_present(iommu_spec.np, "apple,tunable");', 'tunables = true;')} + for name, body in cases.items(): + source = out / (name + '.c') + source.write_text(body) + binary = out / name + subprocess.run(['cc', '-std=gnu11', '-Wall', '-Werror', '-Wno-unused-function', str(source), '-o', str(binary)], check=True) + run = subprocess.run([str(binary)], capture_output=True, text=True) + (out / (name + '.log')).write_text(run.stdout + run.stderr) + if name == 'production': + assert run.returncode == 0, run.stderr + print(run.stdout, end='') + else: + assert run.returncode < 0 and 'Assertion' in run.stderr, (name, run.returncode, run.stderr) + print('PASS negative control:', name) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/qspi-nvram-tests.py b/tools/asahi/qspi-nvram-tests.py index 5b48039e8dbab..a13ce3ac46e17 100644 --- a/tools/asahi/qspi-nvram-tests.py +++ b/tools/asahi/qspi-nvram-tests.py @@ -5,6 +5,7 @@ import argparse import contextlib import json +import re import subprocess import tempfile @@ -68,8 +69,12 @@ def function(source, signature): struct apple_qspi { bool nvram_policy, nvram_write_candidate, write_fault, nvram_window; u64 window_start, window_end; + u8 jedec[3]; u8 bp_ok; + int irq; }; -struct spi_controller { struct apple_qspi *q; }; +struct spi_controller; +struct device { struct device *parent; struct spi_controller *controller; }; +struct spi_controller { struct apple_qspi *q; bool auto_runtime_pm; struct device dev; }; struct spi_device { struct spi_controller *controller; unsigned int cs; int dev; }; struct spi_mem { struct spi_device *spi; }; struct spi_mem_op { @@ -90,20 +95,25 @@ def function(source, signature): static u8 flash[0x800000], sr[3], jedec[3]={0xef,0x65,0x17}; static unsigned int wire[4096],wire_count,fail_opcode; static bool corrupt_readback,stuck_busy,programmed; +static unsigned int pm_refs,busy_calls,busy_wire; +static void pm_runtime_mark_last_busy(struct device*d){ + assert(d&&pm_refs);busy_calls++;busy_wire=wire_count; +} static int apple_qspi_transfer_op(struct spi_mem*m,const struct spi_mem_op*o){ + if(m->spi->controller->auto_runtime_pm)assert(pm_refs); assert(wire_countcmd.opcode; if(o->cmd.opcode==fail_opcode)return -EIO; switch(o->cmd.opcode){ case 0x9f:memcpy(o->data.buf.in,jedec,o->data.nbytes);break; case 0x05:*(u8*)o->data.buf.in=sr[0]|(stuck_busy&&programmed?1:0);break; - case 0x35:*(u8*)o->data.buf.in=sr[1];break; + case 0x35:assert(jedec[0]!=0xc2);*(u8*)o->data.buf.in=sr[1];break; case 0x15:*(u8*)o->data.buf.in=sr[2];break; case 0x03: assert(o->addr.val+o->data.nbytes<=sizeof(flash)); memcpy(o->data.buf.in,flash+o->addr.val,o->data.nbytes); if(corrupt_readback&&programmed)((u8*)o->data.buf.in)[0]^=1; break; - case 0x06:assert(!sr[0]);sr[0]|=2;break; + case 0x06:assert(!(sr[0]&3));sr[0]|=2;break; case 0x04:sr[0]&=~2;break; case 0x02: assert(sr[0]&2);assert(o->addr.val>=0x700000&&o->addr.val+o->data.nbytes<=0x800000); @@ -119,14 +129,17 @@ def function(source, signature): CONTROLS = r''' static struct apple_qspi q; static struct spi_controller ctl={&q}; +static struct device parent; static struct spi_device spi={&ctl}; static struct spi_mem mem={&spi}; static struct kernel_param parameter={&read_only}; static void reset(void){ - memset(flash,0xff,sizeof(flash));memset(sr,0,sizeof(sr));q=(struct apple_qspi){.nvram_policy=true,.nvram_write_candidate=true}; + memset(flash,0xff,sizeof(flash));memset(sr,0,sizeof(sr));q=(struct apple_qspi){.nvram_policy=true,.nvram_write_candidate=true,.window_start=0x700000,.window_end=0x800000,.jedec={0xef,0x65,0x17}}; read_only=true;runtime_ready=false;write_fault_latched=false;wire_count=0;fail_opcode=0;spi.cs=0; corrupt_readback=false;stuck_busy=false;programmed=false;deny_alloc=false;jiffies=0; jedec[0]=0xef;jedec[1]=0x65;jedec[2]=0x17; + ctl.auto_runtime_pm=false;ctl.dev.parent=&parent; + pm_refs=busy_calls=busy_wire=0; } static struct spi_mem_op program(u64 addr,unsigned int n,const u8*b){ struct spi_mem_op o=SPI_MEM_OP(SPI_MEM_OP_CMD(0x02,1),SPI_MEM_OP_ADDR(3,addr,1),SPI_MEM_OP_NO_DUMMY,SPI_MEM_OP_NO_DATA); @@ -136,41 +149,47 @@ def function(source, signature): static bool sent(unsigned int o){for(unsigned int i=0;invram_start==0x700000&&qb->nvram_size==0x100000&&qb->jedec[0]==0xc2&&qb->jedec[1]==0x2b&&qb->jedec[2]==0x27&&!qb->bp_ok); + machine="apple,j700";qb=apple_qspi_nvram_write_board(&t8140,0x385110000ULL); + assert(qb&&qb->nvram_start==0x24000&&qb->nvram_size==0x1d0000&&qb->jedec[2]==0x15&&qb->bp_ok==0x1c); + puts("PASS J613/J713/J700/J616s board, controller and full-width aperture qualification (140 combinations)"); u8 data[256];memset(data,0x55,sizeof(data));reset(); struct spi_mem_op wren=SPI_MEM_OP(SPI_MEM_OP_CMD(0x06,1),SPI_MEM_OP_NO_ADDR,SPI_MEM_OP_NO_DUMMY,SPI_MEM_OP_NO_DATA); - /* T8140 uses a separate nvram window while the J613 gate stays read-only. - * This synthetic window shares the flash model's range; KUnit also covers - * the J700 partition's exact bounds. */ - q.nvram_policy=false;q.nvram_window=true; - q.window_start=0x700000;q.window_end=0x800000; - assert(!apple_qspi_mem_exec_op(&mem,&wren) && wire_count==1 && sr[0]==2); + /* T8140 (J700) shares the verified policy at its DT nvram window; its data + * reads pass without the opt-in. This synthetic window shares the flash + * model's range; KUnit covers the J700 partition's exact bounds. */ + q.nvram_window=true;q.bp_ok=0x1c; + u8 rdbuf[4];struct spi_mem_op t8140_read=SPI_MEM_OP(SPI_MEM_OP_CMD(0x03,1),SPI_MEM_OP_ADDR(3,0x10,1),SPI_MEM_OP_NO_DUMMY,SPI_MEM_OP_DATA_IN(4,rdbuf,1)); + assert(!apple_qspi_mem_exec_op(&mem,&t8140_read) && wire_count==1); + wire_count=0;assert(apple_qspi_mem_exec_op(&mem,&wren)==-EACCES && !wire_count); struct spi_mem_op window_program=program(0x700000,256,data); - assert(!apple_qspi_mem_exec_op(&mem,&window_program) && wire_count==2); - assert(!memcmp(flash+0x700000,data,256) && read_only); + assert(apple_qspi_mem_exec_op(&mem,&window_program)==-EACCES && !wire_count); + arm();sr[0]=4; + assert(!apple_qspi_mem_exec_op(&mem,&wren) && !wire_count); + assert(!apple_qspi_mem_exec_op(&mem,&window_program) && sent(0x02) && sr[0]==4); + assert(!memcmp(flash+0x700000,data,256)); const u64 outside_window[]={0,0x6fffff,0x800000,0x100700000ULL}; - for(unsigned int i=0;i= APPLE_QSPI_NVRAM_START', 'addr >= 1'), + ('controller', 'of_device_is_compatible(node, board->controller) &&', ''), + ('board', 'of_machine_is_compatible(board->machine) &&', ''), + ('aperture', 'base == board->base', 'base != 0'), + ('j616s-aperture', '"apple,j616s", 0x429118000ULL', '"apple,j616s", 0x429118800ULL'), + ('j616s-partition', '"apple,j616s", 0x429118000ULL,\n\t 0x700000', '"apple,j616s", 0x429118000ULL,\n\t 0x680000'), + ('range', 'addr >= start &&', 'addr >= 1 &&'), + ('bp-ok', '!(sr1 & 0x1c & ~bp_ok)', '1'), + ('mx-bp3', '!(sr & 0x3c & ~bp_ok)', '!(sr & 0x1c & ~bp_ok)'), + ('mx-eqio', 'opcode = macronix;', 'opcode = winbond;'), + ('mx-eqio-read', 'qspi->jedec[0] == APPLE_QSPI_MFR_MACRONIX) {', '0) {'), ('read-only', 'read_only || !qspi->nvram_write_candidate || qspi->write_fault', '!qspi->nvram_write_candidate || qspi->write_fault'), - ('fault-latch', 'write_fault_latched = true;', 'write_fault_latched = false;')]: + ('fault-latch', 'write_fault_latched = true;', 'write_fault_latched = false;'), + ('pm-last-busy', 'pm_runtime_mark_last_busy(ctlr->dev.parent);', '(void)ctlr;')]: assert old in code mutant = run(code.replace(old, new), Path(directory) / (name + '.c')) if not mutant.returncode: diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index bcc42fe042c0f..318f9e892505c 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -50,6 +50,15 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'aop-als-channel-tests.py': [([], PASS)], + 'mtp-pm-tests.py': [(out(), PASS), + (out() + ['--old-transport', '4db3a5a22c43'], PASS)], + 'ave-pm-tests.py': [(out(), PASS), + (out() + ['--old-ref', '4db3a5a22c43'], PASS)], + 'ave-probe-pm-tests.py': [(out(), PASS), (out() + ['--negative-control'], PASS)], + 'binding-lifetime-tests.py': [(out(), PASS)] + + mutations('--mutation', ('no-close', 'no-join'), out()), + 'apple-pwm-pm-tests.py': [([], PASS), (['--negative-control'], PASS)], 'apple-dart-fw-tests.py': [(out(), PASS)], 'apple-dart-range-tests.py': [([], PASS)], 'atc-usb3-tests.py': [(out(), PASS), (out() + ['--mutations'], PASS)], @@ -89,6 +98,9 @@ def mutations(flag, names, base, expect=KILL): 'g15-retention-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 'g15-stats-drain-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 'g15-zero-words-tests.py': [([], PASS), (['--mutations'], PASS)], + 'g17-telemetry-integration-tests.py': [(out(), PASS)] + + mutations('--mutation', ('fields', 'range', 'timestamp', 'age', + 'quarantine', 'epoch', 'legacy-format'), out()), 'g17-admission-deadline-tests.py': [([], PASS)], 'g17-admission-order-tests.py': [([], PASS)], 'g17-capacity-fault-tests.py': [([], PASS)], @@ -108,12 +120,15 @@ def mutations(flag, names, base, expect=KILL): 'mt7932-connect-result-tests.py': [(out('--output'), PASS)], 'mt7932-owe-retry-tests.py': [(out(), PASS)], 'mt7932-policy-6ghz-tests.py': [([], PASS)], + 'mt7932-policy-classes-tests.py': [([], PASS)], + 'mt7932-scan-tests.py': [(out(), PASS)], 'neo-auxiliary-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutation'], KILL), (out('--output') + ['--old'], KILL)], 'neo-stability-tests.py': [(out('--output'), PASS)] + mutations('--mutation', ('sep-syscore', 'wake-irq', 'rx-suspend', 'stale-policy', 'usb-rpm', 'vlp-client', 'six-band'), out('--output')) + [(out('--output') + ['--old-usb'], KILL), (out('--output') + ['--old-regulatory'], KILL)], + 'pciec-policy-tests.py': [(out(), PASS)], 'pmp-j613-hwmon-tests.py': [([], PASS)], 'qspi-nvram-tests.py': [(out('--output'), PASS)], 'rtkit-oslog-tests.py': [(out(), PASS)] @@ -134,9 +149,13 @@ def mutations(flag, names, base, expect=KILL): } # Rust unit-test files built with `rustc --test`, and shell drivers. -ASAHI_RUST = ['m3-batching-tests.rs', 'm3-runtime-tests.rs', 'pmp-bootargs-tests.rs', 'pmp-power-tests.rs'] +ASAHI_RUST = ['m3-batching-tests.rs', 'm3-runtime-tests.rs', 'pmp-bootargs-tests.rs', 'pmp-power-tests.rs', + 'usage-tests.rs'] ASAHI_SHELL = ['g17-ray-query-timeout-tests.sh'] +# Production G17 decoder vectors also build directly with rustc --test. +ASAHI_RUST.append('g17-telemetry-tests.rs') + # Python that runs on the host but imports its neighbours, so it keeps the # script directory on sys.path (-E -s instead of -I). SUITES = [ diff --git a/tools/asahi/t6030-pmp-graph-tests.c b/tools/asahi/t6030-pmp-graph-tests.c index 9f018ee94d628..3d2493e57fb63 100644 --- a/tools/asahi/t6030-pmp-graph-tests.c +++ b/tools/asahi/t6030-pmp-graph-tests.c @@ -7,10 +7,11 @@ #include #include #include +#include #include #include #include -typedef uint64_t u64; +typedef unsigned long long u64; typedef uint32_t u32; typedef uint16_t u16; typedef uint8_t u8; @@ -26,6 +27,10 @@ typedef uint8_t u8; #define MAX_PHANDLE_ARGS 4 #define EPROBE_DEFER 517 #define SZ_4K 4096 +#define SZ_16K 16384 +#define SZ_128K 131072 +#define IORESOURCE_MEM 0x200 +#define PMP_ANE_ID 10 #define __iomem #define __force #define container_of(p, t, m) ((t *)((char *)(p) - offsetof(t, m))) @@ -55,6 +60,34 @@ struct device { struct device *parent; void *data; }; +#include "dram-type.h" +#if HOST_HAS_DRAM +struct fwnode_handle { int unused; }; +struct attribute { + const char *name; + unsigned int mode; +}; +struct device_attribute { + struct attribute attr; + ssize_t (*show)(struct device *, struct device_attribute *, char *); +}; +struct attribute_group { + const char *name; + struct attribute **attrs; +}; +#define DEVICE_ATTR_RO(n) struct device_attribute dev_attr_##n = { { #n, 0444 }, n##_show } +static int sysfs_emit(char *buf, const char *format, ...) + __attribute__((format(printf, 2, 3))); +static int sysfs_emit(char *buf, const char *format, ...) +{ + va_list args; + va_start(args, format); + int ret = vsnprintf(buf, 4096, format, args); + va_end(args); + return ret; +} +static int devm_device_add_group(struct device *, const struct attribute_group *); +#endif struct platform_device { struct device dev; }; @@ -104,8 +137,18 @@ struct work_struct { struct delayed_work { struct work_struct work; }; +/* The T600x ANE window; T6030 reports never name one. */ +struct apple_pmp_ane { + void __iomem *dvfs; + void __iomem *report; + unsigned int level; + unsigned int timeouts; + bool owned; + bool reserved; +}; struct apple_pmp_report { struct device *dev; + struct apple_pmp_dram dram; const struct apple_pmp_report_offsets *offsets; void *base; int lock; @@ -117,6 +160,7 @@ struct apple_pmp_report { bool defer; unsigned int polls; struct delayed_work ready_work; + struct apple_pmp_ane ane; }; struct completion { bool done; @@ -250,6 +294,33 @@ static void rcu_read_unlock(void) static struct list_head apple_pmp_starts, apple_pmp_reports; static struct device_node report_node, pmp_node, entries[4]; static struct platform_device report_dev, entry_devs[4]; +#if HOST_HAS_DRAM +/* Typed parser boundary: the real property parser is tested by pmp-dram KUnit. */ +static struct fwnode_handle report_fwnode; +static int dram_parse_error, dram_group_error; +static unsigned int dram_parse_calls, dram_group_calls; +static const struct attribute_group *registered_group; +static const struct apple_pmp_dram parsed_dram = { + .type = "lpddr5", .width = 256, .state_count = 4, + .max_rate = 6400000000ULL, .peak_bandwidth = 204800000000ULL, + .bandwidth_class = 200000000000ULL, +}; +static struct fwnode_handle *dev_fwnode(struct device *dev) +{ + assert(dev == &report_dev.dev); + return &report_fwnode; +} +static int apple_pmp_dram_parse(const struct fwnode_handle *node, + struct apple_pmp_dram *dram) +{ + struct apple_pmp_report *rep = report_dev.dev.data; + assert(node == &report_fwnode && rep && dram == &rep->dram); + assert(++dram_parse_calls == 1); + if (!dram_parse_error) + *dram = parsed_dram; + return dram_parse_error; +} +#endif static u64 regs[0x14000 / 8]; static u8 ranges[64], devices[17 * 0x7c]; static struct work_struct *pending; @@ -539,6 +610,40 @@ static int devm_add_action_or_reset(struct device *d, void (*f)(void *), actions[action_count++] = (struct action){ f, p }; return 0; } +struct resource { + u64 start, end; +}; +static int of_property_match_string(const struct device_node *n, + const char *p, const char *s) +{ + (void)n; (void)p; (void)s; + return -EINVAL; +} +static struct resource *platform_get_resource(struct platform_device *p, + unsigned int t, unsigned int i) +{ + (void)p; (void)t; (void)i; + assert(!"T6030 reports have no ANE window"); + return NULL; +} +static struct resource *platform_get_resource_byname(struct platform_device *p, + unsigned int t, + const char *n) +{ + (void)p; (void)t; (void)n; + assert(!"T6030 reports have no ANE window"); + return NULL; +} +static u64 resource_size(const struct resource *r) +{ + return r->end - r->start + 1; +} +static void *devm_ioremap_resource(struct device *d, struct resource *r) +{ + (void)d; (void)r; + assert(!"T6030 reports have no ANE window"); + return NULL; +} static void *devm_platform_ioremap_resource(struct platform_device *p, int i) { assert(p == &report_dev && !i); @@ -603,6 +708,38 @@ static const struct apple_pmp_start_ops apple_pmp_start_hw_ops = { hw_check, static const struct apple_pmp_start_timing apple_pmp_start_timing = { 2, 0, 2, 2 }; #include "production.h" +#if HOST_HAS_DRAM +static int devm_device_add_group(struct device *dev, const struct attribute_group *group) +{ + assert(dev == &report_dev.dev && group == &apple_pmp_dram_group); + assert(dram_parse_calls == 1 && !dram_parse_error); + assert(++dram_group_calls == 1 && !registered_group); + if (!dram_group_error) + registered_group = group; + return dram_group_error; +} +static void check_dram_attributes(void) +{ + static const char * const names[] = { + "type", "data_bus_width", "state_count", "max_data_rate", + "peak_bandwidth", "bandwidth_class", + }; + static const char * const values[] = { + "lpddr5\n", "256\n", "4\n", "6400000000\n", + "204800000000\n", "200000000000\n", + }; + assert(registered_group && !strcmp(registered_group->name, "dram")); + for (unsigned int i = 0; i < ARRAY_SIZE(names); i++) { + struct attribute *attr = registered_group->attrs[i]; + assert(attr && !strcmp(attr->name, names[i]) && attr->mode == 0444); + struct device_attribute *da = container_of(attr, struct device_attribute, attr); + char buf[4096]; + assert(da->show(&report_dev.dev, da, buf) == (ssize_t)strlen(values[i])); + assert(!strcmp(buf, values[i])); + } + assert(!registered_group->attrs[ARRAY_SIZE(names)]); +} +#endif static const struct apple_pmp_report_offsets * of_device_get_match_data(struct device *d) { @@ -699,6 +836,12 @@ static struct apple_pmp_report *setup(void) hw_binds = hw_ready = true; hw_ack = BIT_ULL(7); enables = commands = 0; +#if HOST_HAS_DRAM + dram_parse_error = -ENODATA; + dram_group_error = 0; + dram_parse_calls = dram_group_calls = 0; + registered_group = NULL; +#endif return NULL; } static int g15_vote(void) @@ -719,6 +862,9 @@ int main(void) setup(); assert(apple_pmp_report_probe(&report_dev) == 0); struct apple_pmp_report *r = report_dev.dev.data; +#if HOST_HAS_DRAM + assert(dram_parse_calls == 1 && !dram_group_calls && !registered_group); +#endif /* The old passive descriptor fails the real DCP entry waiter here. */ if (!r->start) assert(apple_pmp_report_wait_ready(&entries[0], 30000) == 0); @@ -846,6 +992,44 @@ int main(void) assert(g15_vote() == -EINVAL && commands == 0 && !enables); cases++; cleanup(); + +#if HOST_HAS_DRAM + /* Metadata registration leaves the original managed-start graph intact. */ + setup(); + dram_parse_error = 0; + assert(apple_pmp_report_probe(&report_dev) == 0); + assert(dram_parse_calls == 1 && dram_group_calls == 1); + /* Reads use cached values, even if a subsequent parse would fail. */ + dram_parse_error = -EIO; + check_dram_attributes(); + assert(dram_parse_calls == 1 && dram_group_calls == 1); + assert(!enables && !pmp_node.available); + assert(g15_vote() == 0 && enables == 1 && commands == 1); + cases++; + cleanup(); + + /* Errors must propagate before child seeding or startup ownership. */ + const int errors[] = { -EINVAL, -EPROBE_DEFER, -EIO }; + for (unsigned int i = 0; i < ARRAY_SIZE(errors); i++) { + setup(); + dram_parse_error = errors[i]; + assert(apple_pmp_report_probe(&report_dev) == errors[i]); + assert(dram_parse_calls == 1 && !dram_group_calls && !registered_group); + assert(!action_count && !enables && !commands && !pending); + assert(!regs[0x1180 / 8] && !pmp_node.available); + cases++; + cleanup(); + } + setup(); + dram_parse_error = 0; + dram_group_error = -ENOMEM; + assert(apple_pmp_report_probe(&report_dev) == -ENOMEM); + assert(dram_parse_calls == 1 && dram_group_calls == 1 && !registered_group); + assert(!action_count && !enables && !commands && !pending); + assert(!regs[0x1180 / 8] && !pmp_node.available); + cases++; + cleanup(); +#endif printf("PASS: %u actual overlay/probe/entry/DCP7/managed-start/supplier graph cases\n", cases); return 0; diff --git a/tools/asahi/t6030-pmp-graph-tests.py b/tools/asahi/t6030-pmp-graph-tests.py index 6ba26a09c0176..bbe4dd99ea6c0 100644 --- a/tools/asahi/t6030-pmp-graph-tests.py +++ b/tools/asahi/t6030-pmp-graph-tests.py @@ -3,7 +3,9 @@ """Execute the actual report probe -> entry seed -> DCP -> PMP -> G15 graph. The actual overlay supplies the report entries. OF storage, allocation, MMIO, -workqueue execution and the PMP hardware operations are typed host boundaries. +workqueue execution, DRAM parsing, sysfs registration and PMP hardware operations +are typed host boundaries. The DRAM parser itself has firmware-property KUnit +coverage; this suite checks its probe integration and the actual sysfs readers. The production probe, descriptor validation, startup, waiter and supplier/command bodies are extracted verbatim. This is not a firmware or hardware test. """ @@ -51,7 +53,20 @@ def main(): offsets = re.search(r'static const struct apple_pmp_report_offsets apple_pmp_offsets_t6030 = \{.*?\n\};', source, re.S)[0] functions = '\n'.join(extract.function(source, n) for n in names) validation = re.sub(r'^#include .*\n', '', (ROOT / 'drivers/pmdomain/apple/pmp-report-validation.h').read_text(), flags=re.M) - code = offsets + '\n' + validation + '\n' + functions + dram_type = re.search(r'struct apple_pmp_dram \{.*?\n\};', + (ROOT / 'drivers/pmdomain/apple/pmp-dram.h').read_text(), re.S)[0] + has_dram = 'ret = apple_pmp_dram_parse(' in source + assert has_dram or args.ref, 'DRAM probe hook missing from production source' + (args.output / 'dram-type.h').write_text(f'#define HOST_HAS_DRAM {int(has_dram)}\n' + dram_type + '\n') + # Keep the real readers, modes and group wired into the extracted probe. + dram_attrs = source[source.index('static ssize_t type_show('): + source.index('static const struct apple_pmp_report_offsets apple_pmp_offsets_t6030;')] if has_dram else '' + # Kernel sysfs callbacks intentionally leave the attribute argument unused. + # Match the kernel warning policy only around those verbatim callbacks. + dram_attrs = ('#pragma GCC diagnostic push\n' + '#pragma GCC diagnostic ignored "-Wunused-parameter"\n' + + dram_attrs + '#pragma GCC diagnostic pop\n') + code = offsets + '\n' + validation + '\n' + dram_attrs + '\n' + functions # The actual overlay must still describe a halted supplier and disabled DMA/mailbox. assert overlay.count('status = \"disabled\";') >= 3 (args.output / 'production.h').write_text(code) @@ -70,7 +85,21 @@ def main(): print(result.stdout + result.stderr) mutations = [] if args.mutations: - controls = [ + dram_controls = [ + ('no-dram-parse', 'ret = apple_pmp_dram_parse(dev_fwnode(dev), &rep->dram);', + 'ret = -ENODATA; (void)apple_pmp_dram_parse; (void)dev_fwnode;'), + ('reject-absent-dram', 'ret && ret != -ENODATA', 'ret'), + ('ignore-invalid-dram', 'ret && ret != -ENODATA', 'false'), + ('no-dram-group', 'ret = devm_device_add_group(dev, &apple_pmp_dram_group);', + 'ret = 0; (void)devm_device_add_group; (void)apple_pmp_dram_group;'), + ('ignore-dram-group-error', 'ret = devm_device_add_group(dev, &apple_pmp_dram_group);', + '(void)devm_device_add_group(dev, &apple_pmp_dram_group); ret = 0;'), + ('dram-group-without-metadata', 'if (!ret) {\n\t\tret = devm_device_add_group', + 'if (ret == -ENODATA) {\n\t\tret = devm_device_add_group'), + ('dram-wrong-rate-reader', 'APPLE_PMP_DRAM_ATTR(max_data_rate, max_rate,', + 'APPLE_PMP_DRAM_ATTR(max_data_rate, peak_bandwidth,'), + ] + controls = (dram_controls if has_dram else []) + [ ('no-managed-owner', '.starts_pmp = IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK)', '.starts_pmp = false'), ('no-managed-probe', 'ret = apple_pmp_start_init(rep);', 'ret = 0; (void)apple_pmp_start_init;'), ('no-child-seeding', 'apple_pmp_report_entry_set_state(&ent->genpd, true);', '(void)0;'), diff --git a/tools/asahi/usage-tests.rs b/tools/asahi/usage-tests.rs new file mode 100644 index 0000000000000..67bbbc58d6276 --- /dev/null +++ b/tools/asahi/usage-tests.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Run the production per-client engine ledger's tests without a kernel build. + +#[path = "../../drivers/gpu/drm/asahi/usage.rs"] +mod usage; diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index c220e3ee95a3b..15772498d4225 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -102,6 +102,26 @@ and starts a match again by itself. When the kernel ends an enrolment because too many captures in a row were unusable (`SEP_BIO_STATUS_REJECTED`), the enrolment fails with that reason in fprintd's log, and fprintd reports `enroll-unknown-error`. Kernels from before these statuses never send them. + +Interface version 5 requires a resolved Unix uid, a known finger, the selected +Linux SEP OS identity, a live identity count below three, a bound identity bag, +and readable, writable owner/user/master Catacomb files before ENROL_START can +schedule enclave capture. On an empty Linux lane, the driver prepares the owner +context after libfprint validates the account; the user/master Catacombs are +saved at the first completed enrolment. The matching aurora libfprint build +rejects a missing passwd entry before either ioctl. A missing persistence file +for an existing identity is an admission error; repair the binding rather than +retrying a capture against unbound state. + +`sep-bio-identities` is the recovery interface for a print with no fprintd +record. As root, run `sep-bio-identities` to list only the selected Linux OS +identity's built-in SEP user context. Use `sep-bio-identities --delete-orphan +UUID --keep GOOD_UUID` when another identity must remain. The kernel index may +still hold a UUID whose fprintd record was lost, so compare the IDs carefully. +The utility does not expose DELETE_ALL. Record the list and +backup of any known-good print before deletion, and confirm the deleted UUID +stays absent after a reboot. This path must be tested on the affected hardware +before treating a deletion as a successful recovery. The device-specific `apple/mesa_calibration.bin` on m2 was recovered using the bundled [`extract-mesa-calibration.py`](extract-mesa-calibration.py), copied from [Gist revision `eb079a8007985d04e75182f20994f1a1c496f12e`](https://gist.github.com/DjDeveloperr/867a1961b861c570442724f48f770158/eb079a8007985d04e75182f20994f1a1c496f12e), diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index c11412556abb0..cb658d657b121 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -7,7 +7,7 @@ pkgname=aurora-sep pkgver=1 -pkgrel=15 +pkgrel=16 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -33,7 +33,8 @@ source=(load-driver aurora-sep-policy-change README.md XART-SAFETY.md - OS-IDENTITY.md) + OS-IDENTITY.md + sep-bio-identities) sha256sums=('129225c2602290221fc38f754c486e1cba8f0b160e56c5aee2446100aa4e8a04' '2f401629733d280a1b8a5a74f5b668d42307f9885f916abf9e377a811e358b9a' 'f8d0c4b9385e6716d5ac5dd9941a57459fbc551a8238383bcde36cd929d0523f' @@ -45,9 +46,10 @@ sha256sums=('129225c2602290221fc38f754c486e1cba8f0b160e56c5aee2446100aa4e8a04' '3d1c9c0cab1b090bd6b7d3c4bacaec8719e264afba6b565ada80b1e7914f203a' 'fb58f07738b7654b15173fe0ffce2dff2f14b588d8edd24545e6c2c739605026' '6d96cdd20bc993c00eb0b2d7f77d52f81531c72e9d7141455822690ddec348bb' - 'fbb61228397e59c088cba1537ce36f09d26f0f6721e887d43f8138dbdeea2598' + '0d3cafa62973fe86284652066c2d2f94958380c20314702f45a172fd4c7d649f' '5c799f27a424162200071b4e19aba6fae6bc9d2152a0b5a518512ae12d62104a' - '52020e636e62b60f0c3e6251c3b67b8a8c409bff0486bc3463f356fd3073384b') + '52020e636e62b60f0c3e6251c3b67b8a8c409bff0486bc3463f356fd3073384b' + '6209d874a8d2950e5c6a68e2a9c19ffd4ba6e86a27bde3351f37e69290830349') # A unit and the helper it runs ($1, installed as $3), pointed at the # packaged helper. @@ -80,5 +82,6 @@ package() { >"$pkgdir/usr/lib/systemd/system/fprintd.service.d/50-aurora-sep-no-timeout.conf" install -Dm644 logind-aurora-sep.conf \ "$pkgdir/usr/lib/systemd/logind.conf.d/50-aurora-sep.conf" + install -Dm755 sep-bio-identities "$pkgdir/usr/bin/sep-bio-identities" install -Dm644 -t "$pkgdir/usr/share/doc/$pkgname" README.md XART-SAFETY.md OS-IDENTITY.md } diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/sep-bio-identities b/tools/aurora-sep/packaging/arch/aurora-sep/sep-bio-identities new file mode 120000 index 0000000000000..ddd469de602a7 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/sep-bio-identities @@ -0,0 +1 @@ +../../../sep-bio-identities \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index fc30995ee7f5c..359ec582e1877 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -8,7 +8,7 @@ Name: aurora-sep Version: 1 -Release: 10%{?dist} +Release: 11%{?dist} Summary: Loader and desktop integration for the Apple SEP fingerprint driver License: GPL-2.0-only @@ -29,6 +29,7 @@ Source15: aurora-sep-policy-change Source11: aurora_sep.te Source12: aurora_sep.fc Source13: OS-IDENTITY.md +Source16: sep-bio-identities BuildArch: noarch BuildRequires: systemd-rpm-macros @@ -97,6 +98,7 @@ sed 's|@FPRINTD@|%{_libexecdir}/fprintd|' %{SOURCE7} \ >%{buildroot}%{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf install -Dpm 0644 %{SOURCE8} \ %{buildroot}%{_prefix}/lib/systemd/logind.conf.d/50-aurora-sep.conf +install -Dpm 0755 %{SOURCE16} %{buildroot}%{_bindir}/sep-bio-identities # Load the driver at boot, so the enclave is up before the login screen asks. install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset @@ -137,6 +139,7 @@ fi %{_libexecdir}/aurora-sep-load %{_libexecdir}/aurora-sep-ready %{_libexecdir}/aurora-sep-greeter-fingerprint +%{_bindir}/sep-bio-identities %{_sbindir}/aurora-sep-policy-change %{_unitdir}/aurora-sep.service %{_unitdir}/aurora-sep-ready.service @@ -153,11 +156,14 @@ fi %ghost %verify(not md5 size mode mtime) %{_sharedstatedir}/selinux/%{selinuxtype}/active/modules/200/%{modulename} %changelog -* Fri Oct 30 2026 Ryan Murray - 1-10 +* Sun Nov 01 2026 Ryan Murray - 1-11 - aurora-sep-selinux: name the device type aurora_sep_device_t, as the distribution packages do, let fprintd read input devices for the lid follower and let the readiness helpers open the device as init_t +* Fri Oct 30 2026 Ryan Murray - 1-10 +- Add the Linux-lane orphan identity list and delete tool + * Wed Oct 28 2026 Ryan Murray - 1-9 - Start the login screen without waiting for Touch ID, and show it again once Touch ID is ready if nobody has logged in yet diff --git a/tools/aurora-sep/sep-bio-identities b/tools/aurora-sep/sep-bio-identities new file mode 100644 index 0000000000000..2231434678645 --- /dev/null +++ b/tools/aurora-sep/sep-bio-identities @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""List Linux SEP identities and remove one fprintd orphan by its UUID. + +The kernel limits both ioctls to the selected Linux SEP OS identity's built-in +user context. The kernel index may still hold a UUID after fprintd lost its +record, so the operator must identify the exact orphan. This utility has no +delete-all operation. +""" + +import argparse +import fcntl +import os +import struct +import sys +import uuid + +MAGIC = 0xB1 +LIST_SIZE = 8 + 32 * 16 +DELETE_SIZE = 16 + + +def ioctl_number(direction, number, size): + return (direction << 30) | (size << 16) | (MAGIC << 8) | number + + +LIST_ENCLAVE = ioctl_number(2, 0x0B, LIST_SIZE) +DELETE_ORPHAN = ioctl_number(1, 0x0C, DELETE_SIZE) + + +def list_identities(fd): + reply = bytearray(LIST_SIZE) + fcntl.ioctl(fd, LIST_ENCLAVE, reply, True) + count, reserved = struct.unpack_from("=II", reply) + if reserved != 0 or count > 32: + raise ValueError("invalid SEP identity-list reply") + return [uuid.UUID(bytes=bytes(reply[8 + 16 * i:24 + 16 * i])) + for i in range(count)] + + +def check_target(before, target, keep): + if target not in before: + raise ValueError("identity is absent from the Linux-lane list") + if len(before) > 1 and not keep: + raise ValueError("name the identity to preserve with --keep before deleting") + if target in keep or any(identity not in before for identity in keep): + raise ValueError("--keep must list a different identity present in the Linux lane") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--delete-orphan", type=uuid.UUID, metavar="UUID", + help="delete exactly this Linux-lane identity with no fprintd record") + parser.add_argument("--keep", type=uuid.UUID, action="append", default=[], metavar="UUID", + help="identity that must remain; required when more than one is listed") + args = parser.parse_args() + + try: + with os.fdopen(os.open("/dev/sep-bio", os.O_RDWR | os.O_CLOEXEC), "rb", closefd=True) as device: + fd = device.fileno() + before = list_identities(fd) + if args.delete_orphan is None: + for identity in before: + print(identity) + return 0 + check_target(before, args.delete_orphan, args.keep) + fcntl.ioctl(fd, DELETE_ORPHAN, args.delete_orphan.bytes) + after = list_identities(fd) + if args.delete_orphan in after: + raise RuntimeError("SEP still lists the identity after deletion") + if any(identity not in after for identity in args.keep): + raise RuntimeError("a protected identity is absent after deletion") + print("deleted orphan", args.delete_orphan) + print("Linux-lane identities:", len(before), "->", len(after)) + return 0 + except (OSError, ValueError, RuntimeError) as error: + print(f"sep-bio-identities: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-sep/tests/test_enrol_admission.py b/tools/aurora-sep/tests/test_enrol_admission.py new file mode 100644 index 0000000000000..5bb688d3b5571 --- /dev/null +++ b/tools/aurora-sep/tests/test_enrol_admission.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: MIT +"""Execute the kernel's enrol request and persistence admission functions.""" + +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + + +APPLE = Path(__file__).resolve().parents[3] / "drivers/soc/apple" + + +def function(file, name): + source = (APPLE / file).read_text() + match = re.search(r"^fn " + name + r"\(", source, re.M) + if match is None: + raise AssertionError(f"missing driver function {name}") + opening = source.index("{", match.start()) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start():end] + + +HARNESS = r''' +#![allow(dead_code)] +type Error = i32; +type Result = std::result::Result; +const EINVAL: Error = -22; +const ENXIO: Error = -6; +const EIO: Error = -5; +const ENOSPC: Error = -28; +const ENODATA: Error = -61; +const EROFS: Error = -30; +const PRIVATE_TYPE_CATACOMB_OWNER: u8 = 0xf2; +const PRIVATE_TYPE_CATACOMB_USER: u8 = 0xf3; +const PRIVATE_TYPE_CATACOMB_MASTER: u8 = 0xf4; + +struct EnrolPrerequisites { + lane: bool, + live_count: Option, + bag_bound: bool, + catacomb_readable: [bool; 3], + catacomb_writable: [bool; 3], + keybag_writable: bool, + host_store_capacity: bool, +} +const LABEL_LEN: usize = 128; +struct EnrolStart { + flags: u32, + account_uid: u32, + finger: u32, + reserved: u32, + label: [u8; LABEL_LEN], +} + +FUNCTIONS + +fn ready() -> EnrolPrerequisites { + EnrolPrerequisites { + lane: true, live_count: Some(1), bag_bound: true, + catacomb_readable: [true; 3], catacomb_writable: [true; 3], + keybag_writable: true, host_store_capacity: true, + } +} +fn request() -> EnrolStart { + let mut r = EnrolStart { + flags: 0, account_uid: 1000, finger: 7, reserved: 0, + label: [0; LABEL_LEN], + }; + r.label[..28].copy_from_slice(b"FP1-20261027-7-12345678-user"); + r +} + +fn main() { + assert!(admit_enrol(&ready()).is_ok()); + let mut p = ready(); p.lane = false; assert_eq!(admit_enrol(&p), Err(ENXIO)); + let mut p = ready(); p.live_count = None; assert_eq!(admit_enrol(&p), Err(EIO)); + let mut p = ready(); p.live_count = Some(3); assert_eq!(admit_enrol(&p), Err(ENOSPC)); + let mut p = ready(); p.bag_bound = false; assert_eq!(admit_enrol(&p), Err(ENODATA)); + for i in 0..3 { + let mut p = ready(); p.catacomb_readable[i] = false; + assert_eq!(admit_enrol(&p), Err(ENODATA)); + let mut p = ready(); p.catacomb_writable[i] = false; + assert_eq!(admit_enrol(&p), Err(EROFS)); + } + let mut p = ready(); p.keybag_writable = false; assert_eq!(admit_enrol(&p), Err(EROFS)); + let mut p = ready(); p.host_store_capacity = false; assert_eq!(admit_enrol(&p), Err(ENOSPC)); + + assert!(catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_OWNER, Some(0))); + assert!(!catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_USER, Some(0))); + assert!(!catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_MASTER, Some(0))); + assert!(catacomb_needed_for_enrol(PRIVATE_TYPE_CATACOMB_USER, Some(1))); + + assert!(validate_enrol_request(&request()).is_ok()); + let mut r = request(); r.account_uid = u32::MAX; + assert_eq!(validate_enrol_request(&r), Err(EINVAL)); + let mut r = request(); r.finger = 0; + assert_eq!(validate_enrol_request(&r), Err(EINVAL)); + let mut r = request(); r.finger = 11; + assert_eq!(validate_enrol_request(&r), Err(EINVAL)); + let mut r = request(); r.finger = 6; + assert_eq!(validate_enrol_request(&r), Err(EINVAL)); + let mut r = request(); r.label[24] = 0; + assert_eq!(validate_enrol_request(&r), Err(EINVAL)); +} +''' + + +class EnrolAdmissionTests(unittest.TestCase): + def test_refused_cases_before_capture(self): + methods = "\n".join([ + function("sbio.rs", "admit_enrol"), + function("sbio.rs", "catacomb_needed_for_enrol"), + function("bio.rs", "validate_enrol_request"), + ]) + source = HARNESS.replace("FUNCTIONS", methods) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + (path / "test.rs").write_text(source) + subprocess.run(["rustc", "--edition=2021", "test.rs", "-o", "test"], + cwd=path, check=True, capture_output=True, text=True) + subprocess.run([str(path / "test")], check=True, capture_output=True, text=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/aurora-sep/tests/test_orphan_delete.py b/tools/aurora-sep/tests/test_orphan_delete.py new file mode 100644 index 0000000000000..229f6a2c43e38 --- /dev/null +++ b/tools/aurora-sep/tests/test_orphan_delete.py @@ -0,0 +1,151 @@ +# SPDX-License-Identifier: MIT +"""Run the driver's delete-and-persist method with scripted save failures.""" + +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + + +SOURCE = Path(__file__).resolve().parents[3] / "drivers/soc/apple/sbio.rs" + + +def driver_method(name): + source = SOURCE.read_text() + match = re.search(r"^ fn " + name + r"\(", source, re.M) + if match is None: + raise AssertionError(f"missing driver method {name}") + opening = source.index("{", match.start()) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start():end] + + +HARNESS = r''' +#![allow(dead_code, unused_variables)] +use std::ffi::CStr; +use std::ops::Deref; +use std::sync::atomic::{AtomicBool, Ordering::Relaxed}; +use std::sync::{Mutex as StdMutex, MutexGuard}; + +type Error = i32; +type Result = std::result::Result; +const GFP_KERNEL: u32 = 0; +const EIO: Error = -5; + +macro_rules! dev_err { ($($arg:tt)*) => {}; } + +struct Mutex(StdMutex); +impl Mutex { + fn new(value: T) -> Self { Self(StdMutex::new(value)) } + fn lock(&self) -> MutexGuard<'_, T> { self.0.lock().unwrap() } +} + +struct KVec(Vec); +impl KVec { + fn with_capacity(count: usize, _: u32) -> Result { Ok(Self(Vec::with_capacity(count))) } + fn push_within_capacity(&mut self, value: T) -> Result<()> { self.0.push(value); Ok(()) } + fn is_empty(&self) -> bool { self.0.is_empty() } + fn iter(&self) -> std::slice::Iter<'_, T> { self.0.iter() } +} +impl Deref for KVec { + type Target = [T]; + fn deref(&self) -> &[T] { &self.0 } +} + +mod sbio { + #[derive(Clone)] + pub struct IdentityV1 { pub uuid: [u8; 16] } + impl IdentityV1 { pub fn uuid(&self) -> &[u8; 16] { &self.uuid } } +} +mod bio { + pub const UUID_LEN: usize = 16; + pub struct Handled { + pub delete_identity: Option, + pub delete_identities: Vec, + } +} + +struct Index { held: Vec<[u8; 16]> } +impl Index { + fn contains_uuid(&self, uuid: &[u8; 16]) -> bool { self.held.contains(uuid) } +} + +struct SepData { + dev: (), + deletion_unsaved: AtomicBool, + bio_index: Mutex, + delete_replies: Mutex>>, + save_replies: Mutex>>, + events: Mutex>, +} +impl SepData { + fn new(uuid: [u8; 16], saves: Vec>) -> Self { + Self { + dev: (), deletion_unsaved: AtomicBool::new(false), + bio_index: Mutex::new(Index { held: vec![uuid] }), + delete_replies: Mutex::new(vec![Ok(true), Ok(true)]), + save_replies: Mutex::new(saves), events: Mutex::new(Vec::new()), + } + } + fn delete_identity(&self, _: &sbio::IdentityV1, recovery: bool) -> Result { + self.events.lock().push(format!("delete:{recovery}")); + self.delete_replies.lock().remove(0) + } + fn persist_deletion(&self, retry: bool, recovery: bool) -> Result<()> { + self.events.lock().push(format!("save:{retry}:{recovery}")); + self.save_replies.lock().remove(0) + } + fn forget_identity(&self, uuid: &[u8; 16], _: &CStr) { + self.events.lock().push("forget".into()); + self.bio_index.lock().held.retain(|held| held != uuid); + } +METHOD +} + +fn handled(uuid: [u8; 16]) -> bio::Handled { + bio::Handled { delete_identity: Some(sbio::IdentityV1 { uuid }), delete_identities: vec![] } +} + +fn main() { + let uuid = [0xa5; 16]; + let sep = SepData::new(uuid, vec![Ok(())]); + sep.delete_and_persist(&handled(uuid), true).unwrap(); + assert_eq!(*sep.events.lock(), ["delete:true", "save:false:true", "forget"]); + assert!(!sep.bio_index.lock().contains_uuid(&uuid)); + + let sep = SepData::new(uuid, vec![Err(EIO), Ok(())]); + assert!(sep.delete_and_persist(&handled(uuid), true).is_err()); + assert!(sep.deletion_unsaved.load(Relaxed)); + assert!(sep.bio_index.lock().contains_uuid(&uuid)); + sep.delete_and_persist(&handled(uuid), true).unwrap(); + assert_eq!(*sep.events.lock(), ["delete:true", "save:false:true", + "delete:true", "save:true:true", "forget"]); + assert!(!sep.deletion_unsaved.load(Relaxed)); + assert!(!sep.bio_index.lock().contains_uuid(&uuid)); + + let sep = SepData::new(uuid, vec![]); + *sep.delete_replies.lock() = vec![Err(EIO)]; + assert!(sep.delete_and_persist(&handled(uuid), false).is_err()); + assert_eq!(*sep.events.lock(), ["delete:false"]); + assert!(sep.bio_index.lock().contains_uuid(&uuid)); +} +''' + + +class OrphanDeleteTests(unittest.TestCase): + def test_failure_keeps_index_and_retry_saves(self): + source = HARNESS.replace("METHOD", driver_method("delete_and_persist")) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + (path / "test.rs").write_text(source) + subprocess.run(["rustc", "--edition=2021", "test.rs", "-o", "test"], + cwd=path, check=True, capture_output=True, text=True) + subprocess.run([str(path / "test")], check=True, capture_output=True, text=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/aurora-sep/tests/test_owner_preparation.py b/tools/aurora-sep/tests/test_owner_preparation.py new file mode 100644 index 0000000000000..29c1217d7eb24 --- /dev/null +++ b/tools/aurora-sep/tests/test_owner_preparation.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: MIT +"""Execute the kernel's owner-file preflight against scripted file outcomes.""" + +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + + +SOURCE = Path(__file__).resolve().parents[3] / "drivers/soc/apple/catacomb.rs" + + +def method(): + source = SOURCE.read_text() + match = re.search(r"^pub\(crate\) fn prepare_missing_owner\(", source, re.M) + if match is None: + raise AssertionError("missing owner preparation function") + opening = source.index("{", match.start()) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start():end] + + +HARNESS = r''' +use std::ffi::CStr; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering::Relaxed}; +type Error = i32; +type Result = std::result::Result; +const EINVAL: Error = -22; +const ENODATA: Error = -61; +const EROFS: Error = -30; +const PRIVATE_TYPE_CATACOMB_OWNER: u8 = 0xf2; + +fn path_for(kind: u8) -> Option<&'static CStr> { + (kind == PRIVATE_TYPE_CATACOMB_OWNER).then_some(c"/var/lib/apple-sep-catacomb-owner.bin") +} + +mod shim { + use super::*; + pub static CREATE: AtomicBool = AtomicBool::new(true); + pub static FAIL_OPEN: AtomicBool = AtomicBool::new(false); + pub static SYNC: AtomicU32 = AtomicU32::new(0); + pub static PARENT: AtomicU32 = AtomicU32::new(0); + pub struct StoreFile; + impl StoreFile { + pub fn open_host(_: &CStr) -> Result<(StoreFile, bool)> { + if FAIL_OPEN.load(Relaxed) { Err(EROFS) } + else { Ok((StoreFile, CREATE.load(Relaxed))) } + } + pub fn sync(&self) -> Result<()> { SYNC.fetch_add(1, Relaxed); Ok(()) } + pub fn sync_parent(&self) -> Result<()> { PARENT.fetch_add(1, Relaxed); Ok(()) } + } +} + +METHOD + +fn main() { + assert_eq!(prepare_missing_owner(), Ok(())); + assert_eq!(shim::SYNC.load(Relaxed), 1); + assert_eq!(shim::PARENT.load(Relaxed), 1); + + shim::CREATE.store(false, Relaxed); + assert_eq!(prepare_missing_owner(), Err(ENODATA)); + assert_eq!(shim::SYNC.load(Relaxed), 1); + assert_eq!(shim::PARENT.load(Relaxed), 1); + + shim::FAIL_OPEN.store(true, Relaxed); + assert_eq!(prepare_missing_owner(), Err(EROFS)); + assert_eq!(shim::SYNC.load(Relaxed), 1); + assert_eq!(shim::PARENT.load(Relaxed), 1); +} +''' + + +class OwnerPreparationTests(unittest.TestCase): + def test_invalid_file_is_preserved_before_sep_write(self): + source = HARNESS.replace("METHOD", method()) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + (path / "test.rs").write_text(source) + subprocess.run(["rustc", "--edition=2021", "test.rs", "-o", "test"], + cwd=path, check=True, capture_output=True, text=True) + subprocess.run([str(path / "test")], check=True, capture_output=True, text=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/aurora-sep/tests/test_sep_bio_identities.py b/tools/aurora-sep/tests/test_sep_bio_identities.py new file mode 100644 index 0000000000000..23505157866f5 --- /dev/null +++ b/tools/aurora-sep/tests/test_sep_bio_identities.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0-only +"""Synthetic tests for the exact-ID Linux-lane recovery utility.""" + +import runpy +import struct +import unittest +import uuid +from pathlib import Path +from unittest.mock import patch + + +CLI = runpy.run_path(str(Path(__file__).parents[1] / "sep-bio-identities"), run_name="sep_bio_test") + + +class IdentityToolTests(unittest.TestCase): + def test_list_decodes_only_counted_uuids(self): + a = uuid.UUID(int=1) + b = uuid.UUID(int=2) + + def fake_ioctl(_fd, request, reply, _mutate): + self.assertEqual(request, CLI["LIST_ENCLAVE"]) + struct.pack_into("=II", reply, 0, 2, 0) + reply[8:24] = a.bytes + reply[24:40] = b.bytes + + with patch.object(CLI["fcntl"], "ioctl", fake_ioctl): + self.assertEqual(CLI["list_identities"](7), [a, b]) + + def test_exact_target_and_keep_are_required(self): + a = uuid.UUID(int=1) + b = uuid.UUID(int=2) + self.assertIsNone(CLI["check_target"]([a, b], a, [b])) + for before, target, keep in [ + ([a, b], a, []), + ([a, b], a, [a]), + ([a, b], uuid.UUID(int=3), [b]), + ([a, b], a, [uuid.UUID(int=3)]), + ]: + with self.assertRaises(ValueError): + CLI["check_target"](before, target, keep) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/testing/kunit/configs/apple_t8140_bootstrap.config b/tools/testing/kunit/configs/apple_t8140_bootstrap.config new file mode 100644 index 0000000000000..20903b6ec30df --- /dev/null +++ b/tools/testing/kunit/configs/apple_t8140_bootstrap.config @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_PCI=y +CONFIG_PCIE_APPLE_T8140_BOOTSTRAP_KUNIT_TEST=y diff --git a/tools/testing/kunit/configs/mt7932.config b/tools/testing/kunit/configs/mt7932.config index 323269882d301..dbd1f710ae24b 100644 --- a/tools/testing/kunit/configs/mt7932.config +++ b/tools/testing/kunit/configs/mt7932.config @@ -6,7 +6,7 @@ CONFIG_PCI_MSI=y CONFIG_PCIEPORTBUS=y CONFIG_PCIEASPM=y CONFIG_PCIE_APPLE=y -CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_PCIE_APPLE_T8140_BOOTSTRAP=y CONFIG_IOMMU_SUPPORT=y CONFIG_IOMMU_DMA=y CONFIG_NET=y diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 90ea8b9c4dd51..1f2c4b5b33fa8 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -20,6 +20,7 @@ TARGETS += dax TARGETS += devices/error_logs TARGETS += devices/probe TARGETS += dmabuf-heaps +TARGETS += drivers/ane TARGETS += drivers/apple TARGETS += drivers/dma-buf TARGETS += drivers/gpu/apple-afkv2 diff --git a/tools/testing/selftests/drivers/ane/Makefile b/tools/testing/selftests/drivers/ane/Makefile new file mode 100644 index 0000000000000..48712ae211d84 --- /dev/null +++ b/tools/testing/selftests/drivers/ane/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 +TEST_PROGS := h16-fixture.py +TEST_PROGS_EXTENDED := recovery.py make-hang-fixture.py +include ../../lib.mk diff --git a/tools/testing/selftests/drivers/ane/h16-fixture.py b/tools/testing/selftests/drivers/ane/h16-fixture.py new file mode 100755 index 0000000000000..1e74809b07732 --- /dev/null +++ b/tools/testing/selftests/drivers/ane/h16-fixture.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Prepare and verify SPEC-r2 H16G add-128 owner inputs without hardware.""" + +import argparse +import hashlib +import pathlib +import struct + +HWX_SIZE = 49152 +HWX_SHA256 = "231f54c28ea6b14a2867fac78eac1a2cb22aa3b0eeaa28113ed03c520b83a1d3" +EXPECTED_HASHES = { + "a.bin": "3a944e8c891c9e4b00d1cd12bbec5f17cde432fafc1fce0df5809b3deb513e03", + "b.bin": "779fff95d69316770cfff014c8bab4249c2f797c673a4518af80ab87b8d86a05", + "expected.bin": "d113d9ae6734b7a7e233458229f64f382da99a3f93868eef3abb06a29f7f4ce8", +} +FIRST_VALUES = ( + -2.25, -1.875, -1.5, -1.125, -0.75, -0.375, 0, 0.375, + 0.75, 1.125, 1.5, -0.875, -0.5, -0.125, 0.25, 0.625, +) + + +def buffers(): + a = [(i % 17 - 8) * 0.125 for i in range(16384)] + b = [(i % 11 - 5) * 0.25 for i in range(16384)] + expected = [x + y for x, y in zip(a, b)] + if tuple(expected[:16]) != FIRST_VALUES: + raise ValueError("unexpected CPU oracle") + result = { + "a.bin": b"".join(struct.pack(" +#include +#include +int main(int argc, char **argv) { + using namespace ane::h13; + ConvShape shape{1, 1, 2, 1, false, {128, 56, 56}, {256, 28, 28}}; + if (argc == 1) { + auto why = convGenericRefusal(shape); + std::cout << why << "\n"; + return why.find("8 interleaved lanes") == std::string::npos; + } + // Nonzero, exactly representable fp16 weights; no private compiler output. + std::vector weights(256 * 128 * 2); + for (unsigned i = 0; i < weights.size() / 2; ++i) { + const unsigned half = 0x2400 + (i * 17 % 0x400); + weights[2 * i] = half; weights[2 * i + 1] = half >> 8; + } + auto bytes = encodeANEC(encodeConvGeneric(shape, weights.data(), weights.size(), nullptr, 0)); + std::ofstream(argv[1], std::ios::binary).write((const char *)bytes.data(), bytes.size()); + return 0; +} +''' + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("compiler", type=Path, help="clean mil-hwx-compiler checkout at pinned PR 2 commit") + p.add_argument("output", type=Path) + p.add_argument("--cxx", default="c++") + args = p.parse_args() + source = args.compiler.resolve() + head = subprocess.check_output(["git", "-C", str(source), "rev-parse", "HEAD"], text=True).strip() + if head != COMMIT: + p.error(f"compiler HEAD must be {COMMIT}") + directory = source / "plugins/H13" + # Compare all encoder inputs with the licensed public commit. The bypass is + # confined to a temporary translation unit; the checkout stays unmodified. + for item in directory.iterdir(): + if item.suffix in (".cpp", ".h", ".inc"): + original = subprocess.check_output(["git", "-C", str(source), "show", f"HEAD:plugins/H13/{item.name}"]) + if item.read_bytes() != original: + p.error(f"modified encoder input: {item.name}") + program = (directory / "H13Program.cpp").read_text() + if program.count(GUARD) != 1: + p.error("expected refusal changed") + args.output = args.output.resolve() + args.output.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="ane-hang-fixture-") as temporary: + tmp = Path(temporary) + (tmp / "fixture.cpp").write_text(DRIVER) + def build(unit): + subprocess.run([args.cxx, "-std=c++17", "-O2", "-I", str(directory), + str(unit), str(directory / "H13ANEC.cpp"), str(tmp / "fixture.cpp"), + "-o", str(tmp / "fixture")], check=True) + build(directory / "H13Program.cpp") + subprocess.run([str(tmp / "fixture")], check=True) # normal compiler must refuse + (tmp / "H13Program.cpp").write_text(program.replace(GUARD, "chunks.size() != 1")) + build(tmp / "H13Program.cpp") + subprocess.run([str(tmp / "fixture"), str(args.output)], check=True) + receipt = {"compiler": COMMIT, "shape": "1x1 s2 128->256 56->28", "bias": False, + "sha256": hashlib.sha256(args.output.read_bytes()).hexdigest(), + "bytes": args.output.stat().st_size} + args.output.with_suffix(".json").write_text(json.dumps(receipt, indent=2) + "\n") + print(json.dumps(receipt)) + + +if __name__ == "__main__": + main() diff --git a/tools/testing/selftests/drivers/ane/pm.kunitconfig b/tools/testing/selftests/drivers/ane/pm.kunitconfig new file mode 100644 index 0000000000000..f98c6bc08df1f --- /dev/null +++ b/tools/testing/selftests/drivers/ane/pm.kunitconfig @@ -0,0 +1,5 @@ +CONFIG_KUNIT=y +CONFIG_PM=y +CONFIG_DRM=y +CONFIG_DRM_ACCEL=y +CONFIG_DRM_ACCEL_ANE_PM_KUNIT_TEST=y diff --git a/tools/testing/selftests/drivers/ane/recovery.py b/tools/testing/selftests/drivers/ane/recovery.py new file mode 100755 index 0000000000000..9d76725e0bb6e --- /dev/null +++ b/tools/testing/selftests/drivers/ane/recovery.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Destructive H13 hang/recovery test. Run only with exclusive device ownership.""" +import argparse +import errno +import fcntl +import hashlib +import json +import mmap +import os +from pathlib import Path +import signal +import struct +import sys +import time + +# include/uapi/drm/ane_accel.h, Linux arm64's generic ioctl encoding. +CAPS = struct.Struct("<8I") +BO_INIT = struct.Struct(" self.tsk_size or + self.size != ((self.tsk_size + 15) & ~15) + kernel or + len(self.data) - 4096 > self.size or self.size > 16 * 1024 * 1024 or + self.size > self.tiles[0] * TILE or self.tsk_size >= self.tiles[0] * TILE or + not self.tiles[4] or not self.tiles[5] or + bool(self.tiles[6]) != (self.inputs == 2) or + any(n for i, n in enumerate(self.tiles) if i not in (0, 4, 5, 6)) or + sum(self.tiles) * TILE > 64 * 1024 * 1024): + raise ValueError("unexpected canonical H13 ANEC layout") + self.payload = self.data[4096:].ljust(self.size, b"\0") + + +class Program: + def __init__(self, fixture, node): + self.fd = os.open(node, os.O_RDWR | os.O_CLOEXEC) + self.buffers = {} + self.bootstrap = None + self.fixture = fixture + try: + caps = bytearray(CAPS.size) + fcntl.ioctl(self.fd, request(0, CAPS), caps) + _, size, abi, chip, *_ = CAPS.unpack(caps) + if size != CAPS.size or abi != 1 or chip != 13: + raise RuntimeError("H13 ABI 1 required") + for channel, tiles in enumerate(fixture.tiles): + if tiles: + self.buffers[channel] = self.allocate(tiles * TILE) + self.bootstrap = self.allocate((fixture.td_size + TILE - 1) & ~(TILE - 1)) + self.buffers[0][1][:fixture.size] = fixture.payload + self.bootstrap[1][:fixture.td_size] = fixture.payload[:fixture.td_size] + header, = struct.unpack_from(" i32 { self.0 } } +const EINVAL: Error = Error(-22); +const ENODATA: Error = Error(-61); +type Result = std::result::Result; +mod bindings { + pub const BINDINGS_IIO_CHAN_INFO_RAW: u32 = 0; + pub const BINDINGS_IIO_CHAN_INFO_PROCESSED: u32 = 1; + pub const BINDINGS_IIO_CHAN_INFO_SCALE: u32 = 2; + pub const IIO_VAL_INT: u32 = 1; + pub const IIO_VAL_INT_PLUS_MICRO: u32 = 2; + pub const IIO_VAL_INT_PLUS_NANO: u32 = 3; + pub struct iio_dev { pub priv_: *const std::ffi::c_void } + pub struct iio_chan_spec { pub type_: u32 } +} +struct Arc(std::marker::PhantomData); +impl Arc { + unsafe fn borrow<'a>(p: *const std::ffi::c_void) -> &'a T { + unsafe { &*p.cast::() } + } +} +trait FakehidListener { + fn process_fakehid_report(&self, data: &[u8]) -> Result<()>; +} +struct AopSensorData { + ty: u32, value: AtomicU64, valid: AtomicBool, msg_proc: T, +} +struct MsgProc; +''', block(common, "pub trait MessageProcessor"), + block(las, "impl MessageProcessor for MsgProc"), + block(common, "impl FakehidListener"), + block(common, 'unsafe extern "C" fn aop_read_raw'), r''' +fn sensor(msg_proc: T) -> AopSensorData { + AopSensorData { + ty: 17, value: AtomicU64::new(0), valid: AtomicBool::new(false), msg_proc, + } +} +fn read(data: &AopSensorData, mask: u32) -> (i32, i32, i32) { + let mut dev = bindings::iio_dev { priv_: (data as *const AopSensorData).cast() }; + let chan = bindings::iio_chan_spec { type_: data.ty }; + let (mut val, mut val2) = (-1, -1); + let ret = unsafe { aop_read_raw::(&mut dev, &chan, &mut val, &mut val2, mask as isize) }; + (ret, val, val2) +} +#[test] +fn full_angle_and_padding() { + // Synthetic reports: cover bit 8, both limits, invalid values and every padding pattern. + for wire in 0..=u16::MAX { + let bytes = wire.to_le_bytes(); + let angle = wire & 0x1ff; + let expected = if angle <= 360 { Some(u64::from(angle) * MICRO) } else { None }; + assert_eq!(MsgProc.process(&[1, bytes[0], bytes[1]]), expected); + } + assert_eq!(MsgProc.process(&[1, 99, 0]), Some(99 * MICRO)); + assert_eq!(MsgProc.process(&[1, 0, 1]), Some(256 * MICRO)); + assert_eq!(MsgProc.process(&[1, 104, 1]), Some(360 * MICRO)); +} +#[test] +fn report_identity_and_truncation() { + for id in 0..=u8::MAX { + if id != 1 { assert_eq!(MsgProc.process(&[id, 99, 0]), None); } + } + for truncated in [&[][..], &[1][..], &[1, 99][..]] { + assert_eq!(MsgProc.process(truncated), None); + } +} +#[test] +fn raw_scale_and_last_valid_sample() { + use bindings::*; + let data = sensor(MsgProc); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_RAW).0, -61); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_SCALE), + (IIO_VAL_INT_PLUS_NANO as i32, 0, 17_453_293)); + data.process_fakehid_report(&[2, 39]).unwrap(); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_RAW).0, -61); + data.process_fakehid_report(&[1, 99, 0]).unwrap(); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_RAW), (1, 99, 0)); + for report in [&[3, 0, 0, 0, 0, 1][..], &[1, 105, 1][..], &[1, 42][..]] { + data.process_fakehid_report(report).unwrap(); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_RAW), (1, 99, 0)); + } + let (_, int, nano) = read(&data, BINDINGS_IIO_CHAN_INFO_SCALE); + let radians = 99.0 * (int as f64 + nano as f64 / 1e9); + assert!((radians - 99.0_f64.to_radians()).abs() < 1e-7); +} +struct Processed; +impl MessageProcessor for Processed { + fn process(&self, _: &[u8]) -> Option { Some(12_345_678) } +} +#[test] +fn processed_channel_unchanged() { + use bindings::*; + let data = sensor(Processed); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_PROCESSED).0, -61); + data.process_fakehid_report(&[]).unwrap(); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_PROCESSED), (2, 12, 345_678)); + assert_eq!(read(&data, BINDINGS_IIO_CHAN_INFO_SCALE).0, -22); + assert_eq!(read(&data, 999).0, -22); +} +'''] + with tempfile.TemporaryDirectory(prefix="aop-las-") as tmp: + source = Path(tmp) / "test.rs" + binary = Path(tmp) / "test" + source.write_text("\n".join(parts)) + subprocess.run(shlex.split(os.environ.get("RUSTC", "rustc")) + [ + "--edition=2021", "--test", "-Dwarnings", str(source), "-o", str(binary), + ], check=True) + subprocess.run([str(binary)], check=True) + + +if __name__ == "__main__": + main() diff --git a/tools/testing/selftests/drivers/apple/aop-motion.py b/tools/testing/selftests/drivers/apple/aop-motion.py new file mode 100755 index 0000000000000..a66d26b77810a --- /dev/null +++ b/tools/testing/selftests/drivers/apple/aop-motion.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run the production motion report decoder and calibration selector on the host.""" +import os +from pathlib import Path +import shlex +import subprocess +import tempfile + + +def block(source, marker): + start = source.index(marker) + brace = source.index("{", start) + depth = 1 + end = brace + 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + +def const(source, name): + start = source.index(f"const {name}") + return source[start:source.index(";", start) + 1] + + +def main(): + root = Path(__file__).resolve().parents[5] + motion = (root / "drivers/iio/common/aop_sensors/aop_motion.rs").read_text() + parts = [const(motion, "REPORT_LEN"), const(motion, "XYZ_OFFSET"), + block(motion, "fn decode("), block(motion, "fn le_u32("), + block(motion, "fn location0_packet("), + const(motion, "STATUS_UNSUPPORTED"), + block(motion, "#[derive(Clone, Copy, Debug, PartialEq, Eq)]\nenum CalStatus"), block(motion, "fn cal_status("), r''' +fn report(prefix: u8, xyz: [i32; 3], suffix: u8) -> Vec { + let mut r = vec![prefix; 6]; + for v in xyz { + r.extend_from_slice(&v.to_le_bytes()); + } + r.extend_from_slice(&[suffix; 4]); + r +} +#[test] +fn decodes_signed_axes() { + // Synthetic reports; the bytes around the samples carry no sample. + for (prefix, suffix) in [(0, 0), (0xff, 0xff), (0x5a, 0xa5)] { + for xyz in [[65536, -65536, 0], [i32::MIN, i32::MAX, -1]] { + assert_eq!(decode(&report(prefix, xyz, suffix)), Some(xyz)); + } + } +} +#[test] +fn rejects_other_lengths() { + let r = report(0, [1, 2, 3], 0); + for len in 0..40 { + if len == REPORT_LEN { + continue; + } + let mut v = r.clone(); + v.resize(len, 0); + assert_eq!(decode(&v), None); + } +} +fn container(entries: &[(u32, &[u8])]) -> Vec { + let mut c = vec![2, 0, entries.len() as u8, 1]; + for (location, packet) in entries { + c.extend_from_slice(&location.to_le_bytes()); + c.extend_from_slice(&(packet.len() as u32).to_le_bytes()); + c.extend_from_slice(packet); + } + c +} +#[test] +fn calibration_status() { + assert_eq!(cal_status(0), CalStatus::Taken); + // Answered once the service holds a calibration from an earlier start. + assert_eq!(cal_status(0xe000_02c7), CalStatus::Held); + for status in [1, 0xe000_02bc, 0xe000_02c2, 0xe000_02e2, 0xe000_02c6, u32::MAX] { + assert_eq!(cal_status(status), CalStatus::Refused, "{status:#x}"); + } +} +#[test] +fn selects_location_zero() { + let a = [1u8; 44]; + let b = [2u8; 40]; + let one = container(&[(0, &a)]); + assert_eq!(location0_packet(&one), Some(&a[..])); + // Trailing bytes after the entries are not part of the packet. + let mut padded = one.clone(); + padded.extend_from_slice(&[0; 4]); + assert_eq!(location0_packet(&padded), Some(&a[..])); + let two = container(&[(3, &b), (0, &a)]); + assert_eq!(location0_packet(&two), Some(&a[..])); + assert_eq!(location0_packet(&container(&[(1, &a)])), None); +} +#[test] +fn rejects_malformed_containers() { + let a = [1u8; 44]; + let good = container(&[(0, &a)]); + for len in 0..good.len() { + assert_eq!(location0_packet(&good[..len]), None, "truncated to {len}"); + } + let mut not_container = good.clone(); + not_container[3] = 0; + assert_eq!(location0_packet(¬_container), None); + let mut huge = good.clone(); + huge[8..12].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(location0_packet(&huge), None); + let mut many = good.clone(); + many[2] = 255; + assert_eq!(location0_packet(&many), Some(&a[..])); +} +'''] + with tempfile.TemporaryDirectory(prefix="aop-motion-") as tmp: + source = Path(tmp) / "test.rs" + binary = Path(tmp) / "test" + source.write_text("\n".join(parts)) + subprocess.run(shlex.split(os.environ.get("RUSTC", "rustc")) + [ + "--edition=2021", "--test", "-Dwarnings", str(source), "-o", str(binary), + ], check=True) + subprocess.run([str(binary)], check=True) + + +if __name__ == "__main__": + main() diff --git a/tools/testing/selftests/drivers/apple/j616s-memory.rst b/tools/testing/selftests/drivers/apple/j616s-memory.rst new file mode 100644 index 0000000000000..da1dfa386e092 --- /dev/null +++ b/tools/testing/selftests/drivers/apple/j616s-memory.rst @@ -0,0 +1,64 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +J616s memory policy checks +========================= + +The T6040 uses the ``apple,t6041-pmc`` and ``apple,t6041-clpc`` compatibles. +The J616s DT enables both the PMC and the PMP. ``apple,pmp`` gates PMC probe +on PMP readiness; static voters 0, 1 and 3 enable PMP, CLPC and AP requests on +the DCS and fabric rails. Do not enable the PMP voter before its owner is +running: its boot-time floor can be the maximum state. + +For CPU demand, CLPC samples every 4 ms, qualifies three consecutive samples +with at least 50 percent busy time on a CPU in a cluster, and uses the cluster +frequency quartile to request DCS states 1, 2, 5, 5 and fabric states 0, 1, 2, +2. Sustained work in the upper half of a cluster's range requests DCS state 5 +(the sixth state). On idle, the host floor is released after a 32 ms hold; +firmware and display clients may continue requesting memory states. The +host's zero floor does not imply that DRAM actually runs in state zero. + +``/sys/kernel/debug/apple-pmc/*/status`` reports requested and honoured agent +floors and rail enable masks. The CLPC device's debugfs ``status`` reports its +CPU requests, current vote and error count. These are policy inputs, not DCS +achieved-state residency. In particular, neither an honoured floor nor the +``pmc-dcs`` thermal cooling state is a current DRAM clock. + +The PMP report provider's ``dram/`` sysfs group contains static metadata:: + + find /sys/devices/platform -type d -name dram -print + +On a J616s DT carrying the DRAM description, its attributes report type +``lpddr5``, 6 states, width 256, maximum data rate 8533333333 transfers/s, +physical peak 273066666656 bytes/s, and published class 273000000000 bytes/s. +The group is conditional on valid boot DT metadata; a missing group with an +older boot DT does not imply a disabled DCS governor. It contains no current +rate or residency counters. + +For a reproducible application bandwidth workload, build on Linux:: + + cc -O2 -Wall -Wextra -Werror -pthread \ + tools/testing/selftests/drivers/apple/memcpy-bw.c -o /tmp/memcpy-bw + /tmp/memcpy-bw 8 256 10 + +Each of eight threads copies between its own two 256 MiB buffers for ten +seconds, after first-touch initialization. Output counts one source read and +one destination write per completed libc copy, in decimal GB/s, over the +wall time until the last thread finishes. It is an application estimate, not +an AMCC bus measurement: write allocation, cache effects and libc algorithm +can change the physical traffic. Record libc, CPU governor, display topology, +thermal state and the exact kernel/DT alongside the result. Compare idle, +bounded CPU load, memcpy, and recovery using the same observation method. +Do not infer lower-state rates from bandwidth response thresholds. + +The capture helper checks the board model, records available policy and IIO +attributes, then runs two seconds idle, five seconds of fourteen CPU workers, +two seconds recovery, the ten-second memcpy workload, and two seconds recovery:: + + python3 tools/testing/selftests/drivers/apple/j616s-parity.py \ + --memcpy /tmp/memcpy-bw --output /tmp/j616s-parity-NEW + +The output directory must be new. Each read failure is recorded in JSON, and +the helper terminates only its own workload processes on failure or interrupt. +The memcpy phase needs 4 GiB plus overhead. Polling at 20 Hz can itself cause +CPU wakeups; use low-overhead counter snapshots for final idle-residency and +power comparisons. Cached IIO values do not establish report arrival cadence. diff --git a/tools/testing/selftests/drivers/apple/j616s-parity.py b/tools/testing/selftests/drivers/apple/j616s-parity.py new file mode 100644 index 0000000000000..d8835771f36c8 --- /dev/null +++ b/tools/testing/selftests/drivers/apple/j616s-parity.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Collect J616s policy snapshots around bounded CPU and memcpy workloads.""" +import argparse +import json +import pathlib +import subprocess +import time + +parser = argparse.ArgumentParser() +parser.add_argument('--memcpy', required=True, help='Path to public memcpy-bw binary') +parser.add_argument('--output', required=True, help='New private evidence directory') +args = parser.parse_args() +try: + compatible = pathlib.Path('/proc/device-tree/compatible').read_bytes() +except OSError as error: + raise SystemExit(f'Cannot read device identity: {error.strerror}') +if b'apple,j616s' not in compatible.split(b'\0'): + raise SystemExit('Device identity is not apple,j616s') +output = pathlib.Path(args.output) +output.mkdir(parents=True, exist_ok=False) + +# Read only documented policy, cached IIO readings, and requested clock attributes. +paths = sorted(set( + list(pathlib.Path('/sys/kernel/debug/apple-pmc').glob('*/status')) + + list(pathlib.Path('/sys/kernel/debug').glob('*performance-controller*/status')) + + list(pathlib.Path('/sys/devices/system/cpu/cpufreq').glob('policy*/scaling_cur_freq')) +)) +als = [] +for device in pathlib.Path('/sys/bus/iio/devices').glob('iio:device*'): + try: + if (device / 'name').read_text().strip() == 'aop-sensors-als': + als += [device / 'in_illuminance_input', device / 'in_cct_input'] + except OSError: + pass +paths += als + +def read(path): + try: + return path.read_text().strip() + except OSError as error: + return {'error': error.strerror, 'errno': error.errno} + +identity = {'kernel': subprocess.check_output(['uname', '-r'], text=True).strip(), + 'compatible': compatible.decode(errors='replace').split('\0'), + 'selected_paths': [str(path) for path in paths], + 'dram': {}} +for group in pathlib.Path('/sys/devices/platform').rglob('dram'): + for name in ['type', 'state_count', 'data_bus_width', 'max_data_rate', 'peak_bandwidth', 'bandwidth_class']: + identity['dram'][str(group / name)] = read(group / name) +(output / 'identity.json').write_text(json.dumps(identity, indent=2) + '\n') +children = [] +with (output / 'snapshots.jsonl').open('x') as log: + def sample(phase): + record = {'monotonic_ns': time.monotonic_ns(), 'phase': phase, + 'values': {str(path): read(path) for path in paths}} + log.write(json.dumps(record) + '\n') + log.flush() + + def observe(phase, seconds): + end = time.monotonic() + seconds + while time.monotonic() < end: + sample(phase) + time.sleep(0.05) + + try: + observe('idle', 2) + with open('/dev/null', 'wb') as null: + for _ in range(14): + children.append(subprocess.Popen(['yes'], stdout=null)) + observe('all-core', 5) + for child in children: + child.terminate() + for child in children: + child.wait() + children = [] + observe('cpu-recovery', 2) + with (output / 'memcpy.txt').open('x') as metrics: + child = subprocess.Popen([args.memcpy, '8', '256', '10'], stdout=metrics, stderr=subprocess.STDOUT) + children = [child] + deadline = time.monotonic() + 60 + while child.poll() is None and time.monotonic() < deadline: + sample('memcpy-including-initialization') + time.sleep(0.05) + if child.poll() is None: + raise RuntimeError('memcpy did not finish within 60 seconds') + (output / 'memcpy-exit.txt').write_text(str(child.returncode) + '\n') + children = [] + observe('memory-recovery', 2) + finally: + for child in children: + if child.poll() is None: + child.terminate() + for child in children: + try: + child.wait(timeout=2) + except subprocess.TimeoutExpired: + child.kill() + child.wait() diff --git a/tools/testing/selftests/drivers/apple/memcpy-bw.c b/tools/testing/selftests/drivers/apple/memcpy-bw.c new file mode 100644 index 0000000000000..9c8961e355356 --- /dev/null +++ b/tools/testing/selftests/drivers/apple/memcpy-bw.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Bounded application memcpy throughput; this is not an AMCC bus counter. */ +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include +#include +#include + +#define MAX_THREADS 64 + +static pthread_barrier_t ready; +static uint64_t start_ns, duration_ns; +static size_t bytes; +/* A volatile function pointer keeps each libc copy observable to the compiler. */ +static void *(*volatile copy_memory)(void *, const void *, size_t) = memcpy; + +struct worker { + void *src, *dst; + uint64_t copies, end_ns; +}; + +static uint64_t now_ns(void) +{ + struct timespec ts; + + if (clock_gettime(CLOCK_MONOTONIC, &ts)) { + perror("clock_gettime"); + exit(1); + } + return (uint64_t)ts.tv_sec * 1000000000 + ts.tv_nsec; +} + +static void *copy_worker(void *arg) +{ + struct worker *worker = arg; + + memset(worker->src, 0x5a, bytes); + memset(worker->dst, 0, bytes); + pthread_barrier_wait(&ready); + pthread_barrier_wait(&ready); + do { + copy_memory(worker->dst, worker->src, bytes); + worker->copies++; + worker->end_ns = now_ns(); + } while (worker->end_ns - start_ns < duration_ns); + return NULL; +} + +static unsigned long number(const char *arg, unsigned long max) +{ + char *end; + unsigned long value; + + errno = 0; + value = strtoul(arg, &end, 10); + if (errno || !*arg || *end || !value || value > max) { + fprintf(stderr, "Invalid argument: %s (expected 1..%lu)\n", arg, max); + exit(2); + } + return value; +} + +int main(int argc, char **argv) +{ + struct worker workers[MAX_THREADS] = {}; + pthread_t threads[MAX_THREADS]; + unsigned long count = 8, mib = 256, seconds = 10; + uint64_t copies = 0, end_ns = 0; + unsigned long i; + int ret; + + if (argc > 4) { + fprintf(stderr, "Usage: %s [threads [MiB-per-buffer [seconds]]]\n", argv[0]); + return 2; + } + if (argc > 1) + count = number(argv[1], MAX_THREADS); + if (argc > 2) + mib = number(argv[2], 1024); + if (argc > 3) + seconds = number(argv[3], 60); + bytes = mib * 1024 * 1024; + duration_ns = seconds * 1000000000ULL; + for (i = 0; i < count; i++) { + ret = posix_memalign(&workers[i].src, 16384, bytes); + if (!ret) + ret = posix_memalign(&workers[i].dst, 16384, bytes); + if (ret) { + fprintf(stderr, "Allocation: %s\n", strerror(ret)); + return 1; + } + } + ret = pthread_barrier_init(&ready, NULL, count + 1); + if (ret) { + fprintf(stderr, "Barrier: %s\n", strerror(ret)); + return 1; + } + for (i = 0; i < count; i++) { + ret = pthread_create(&threads[i], NULL, copy_worker, &workers[i]); + if (ret) { + fprintf(stderr, "Thread: %s\n", strerror(ret)); + /* Terminate any workers already waiting at the barrier. */ + exit(1); + } + } + pthread_barrier_wait(&ready); + start_ns = now_ns(); + pthread_barrier_wait(&ready); + for (i = 0; i < count; i++) { + pthread_join(threads[i], NULL); + copies += workers[i].copies; + if (workers[i].end_ns > end_ns) + end_ns = workers[i].end_ns; + if (memcmp(workers[i].src, workers[i].dst, bytes)) { + fprintf(stderr, "Copy verification failed\n"); + return 1; + } + free(workers[i].src); + free(workers[i].dst); + } + pthread_barrier_destroy(&ready); + printf("threads=%lu bytes_per_buffer=%zu elapsed_s=%.6f copies=%llu " + "application_rd_wr_GBps=%.3f\n", count, bytes, + (end_ns - start_ns) / 1e9, (unsigned long long)copies, + 2.0 * bytes * copies / (end_ns - start_ns)); + return 0; +} diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile index 2c25d757aca5e..d46a4dc8c05d2 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT TEST_PROGS := run.sh m4-error-paths.py plane-layout.py -TEST_FILES := run.py common.py desktop.py display.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py boot.py quiesce.py transaction.py accounting.py m4.py m4-fixtures native-m4-kms.c run-native-m4-kms.py dwc3 README +TEST_FILES := run.py common.py audio.py desktop.py display.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py boot.py quiesce.py transaction.py accounting.py m4.py m4-fixtures native-m4-kms.c run-native-m4-kms.py dwc3 README include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index 6777d85cf3431..4934b7eb7eef2 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -51,6 +51,12 @@ concurrency and system suspend require separate kernel/hardware qualification. Kernel KUnit tests and apple-afkv2 remain separate tests in their existing standard layouts. No archived diagnostic script is a dependency of this suite. +The display group's desktop-gamma fixture executes the production A420/A421 +callbacks and composition with synthetic tables. It checks full return-status +handling, the external empty status-2 reply, internal-panel diagnostics, +malformed tables, independent RGB interpolation and restoration of the base. +It does not establish firmware table encoding or external A421 support. + The runner preserves host artifacts on failure as well as success. Existing physical-disconnect fixture warnings for unused mock boundaries and signed host/kernel ABI comparisons are handled only by those fixtures' original @@ -202,3 +208,10 @@ ensure the machine's serial cannot override the panel's. Panel EDID tests cover a missing firmware identity, real EDID priority and invalid replacement retention. Desktop backlight notification tests exercise accepted, rejected, duplicate and out-of-order completions and preserve the requested target during notification. + +The audio group executes the native AV call and OSObject reply boundaries with +synthetic transport replies, including truncated bodies, inconsistent used +sizes and transport errors. It retains the legacy reply behavior. KUnit's +apple-av-native-audio suite separately runs service replacement, Terminate and +link lifecycle checks for both external 14.7 and desktop 26.x firmware roles. +These fixtures do not establish the desktop audio firmware ABI or SIO support. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/audio.py b/tools/testing/selftests/drivers/gpu/apple-dcp/audio.py new file mode 100644 index 0000000000000..44e2bd9768089 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/audio.py @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright 2026 Ryan Murray +"""Compact audio reply bounds through production AV methods, synthetic transport.""" +from common import block + + +def reply_bounds(root): + source = (root / 'drivers/gpu/drm/apple/av.c').read_text() + prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint16_t u16; +typedef uint64_t __le64; +#define KUNIT_STATIC_STUB_REDIRECT(...) ((void)0) +#define READ_ONCE(x) (x) +#undef static_assert +#define static_assert(expr, ...) _Static_assert(expr, #expr) +#define GFP_KERNEL 0 +static void free_buffer(void *p) {free(*(void **)p);} +#define __free(x) __attribute__((cleanup(free_buffer))) +static void *kzalloc(size_t n, int flags) {(void)flags;return calloc(1,n);} +#define cpu_to_le64(x) (x) +#define le64_to_cpu(x) (x) +struct apple_dcp_afkep {bool compact;}; +struct apple_epic_service {struct apple_dcp_afkep *ep;}; +static int transport_error; +static size_t wire_received, used; +static unsigned compact_calls, legacy_calls; +static int transfer(struct apple_epic_service *srv, u16 group, u32 cmd, + const void *data, size_t len, size_t pad, void *out, + size_t room, size_t out_pad) +{ + assert(srv && cmd && !out_pad); + (void)pad; + if (transport_error) return transport_error; + if (group==1) { + assert(len==48 && room==64 && data==out); + __le64 max, count=used; + memcpy(&max,data,8);assert(max==16); + memcpy((u8 *)out+32,&count,8); + memset((u8 *)out+48,0x5a,16); + } else { + assert(group==0 && len==64 && data && room==64); + memset(out,0,room); + } + return 0; +} +static int afk_service_call_with_reply_len(struct apple_epic_service *srv, + u16 group,u32 cmd,const void *data,size_t len,size_t pad,void *out, + size_t room,size_t out_pad,size_t *received) +{ + compact_calls++;*received=wire_received; + return transfer(srv,group,cmd,data,len,pad,out,room,out_pad); +} +static int afk_service_call(struct apple_epic_service *srv,u16 group,u32 cmd, + const void *data,size_t len,size_t pad,void *out,size_t room,size_t out_pad) +{ + legacy_calls++; + return transfer(srv,group,cmd,data,len,pad,out,room,out_pad); +} +''' + return prefix + block(source, 'static int av_audiosrv_call(') + '\n' + block( + source, 'static int\ndcp_audiosrv_osobject_call(') + r''' +int main(void) +{ + struct apple_dcp_afkep ep={.compact=true}; + struct apple_epic_service srv={.ep=&ep}; + u8 body[64]={0},reply[64],elements[16]; + size_t actual=0; + wire_received=64; + assert(!av_audiosrv_call(&srv,6,body,64,0,reply,64)); + for (size_t n=0;n<66;n++) { + wire_received=n; + assert(av_audiosrv_call(&srv,6,body,64,0,reply,64)== + (n==64?0:-EMSGSIZE)); + } + transport_error=-EIO; + assert(av_audiosrv_call(&srv,6,body,64,0,reply,64)==-EIO); + transport_error=0; + used=8;wire_received=56; + memset(elements,0xff,sizeof(elements)); + assert(!dcp_audiosrv_osobject_call(&srv,1,16,elements,16,&actual)); + assert(actual==8); + for (size_t i=0;i<16;i++) assert(elements[i]==(i<8?0x5a:0)); + for (size_t n=0;n<56;n++) { + wire_received=n; + assert(dcp_audiosrv_osobject_call(&srv,1,16,elements,16,&actual)==-EMSGSIZE); + } + wire_received=64;used=17; + assert(dcp_audiosrv_osobject_call(&srv,1,16,elements,16,&actual)==-EMSGSIZE); + used=16; + assert(!dcp_audiosrv_osobject_call(&srv,1,16,elements,16,&actual) && actual==16); + assert(!dcp_audiosrv_osobject_call(&srv,1,18,NULL,16,NULL)); + transport_error=-ETIMEDOUT; + assert(dcp_audiosrv_osobject_call(&srv,1,16,elements,16,&actual)==-ETIMEDOUT); + transport_error=0; + ep.compact=false; + unsigned compact_before=compact_calls; + assert(!av_audiosrv_call(&srv,6,body,64,0,reply,64)); + used=0; + assert(!dcp_audiosrv_osobject_call(&srv,1,16,elements,16,&actual) && actual==0); + for (size_t i=0;i<16;i++) assert(elements[i]==0x5a); + assert(compact_calls==compact_before && legacy_calls==2); + return 0; +} +''' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/colorimetry.py b/tools/testing/selftests/drivers/gpu/apple-dcp/colorimetry.py new file mode 100644 index 0000000000000..f4cd307926988 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/colorimetry.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Synthetic firmware colour data through the production parser and EDID builder.""" +import copy +import struct +import desktop +import rate + + +def panel(root, output): + fixture = rate.parser_fixture(root) + unit = desktop.panel_identity(root).split('int main(int argc, char **argv)')[0] + # Standard colour-space coordinates, never a captured panel calibration. + p3 = {'Chromaticity': {'Red': {'X': 44564, 'Y': 20972}, + 'Green': {'X': 17367, 'Y': 45220}, + 'Blue': {'X': 9830, 'Y': 3932}}, + 'DefaultWhitePoint': {'X': 20493, 'Y': 21561, 'Gamma': 144179}, + 'ProductAttributes': {'ManufacturerID': 'TST', 'ProductID': 42, + 'ProductName': 'Synthetic', 'SerialNumber': 123}} + srgb = copy.deepcopy(p3) + srgb['Chromaticity']['Red'] = {'X': 41943, 'Y': 21627} + srgb['Chromaticity']['Green'] = {'X': 19661, 'Y': 39322} + absent = {'ProductAttributes': p3['ProductAttributes']} + white_only = copy.deepcopy(p3); del white_only['Chromaticity'] + chroma_only = copy.deepcopy(p3); del chroma_only['DefaultWhitePoint'] + zero = copy.deepcopy(p3); zero['Chromaticity']['Red']['X'] = 0 + gamma = copy.deepcopy(p3); del gamma['DefaultWhitePoint']['Gamma'] + bad_gamma = copy.deepcopy(p3); bad_gamma['DefaultWhitePoint']['Gamma'] = 65535 + wrong_type = copy.deepcopy(p3); wrong_type['Chromaticity'] = 'P3' + # The wire encoder supports an ordered multimap to exercise duplicate keys. + pairs = fixture['ObjectPairs'] + duplicate = pairs(list(p3.items()) + [('Chromaticity', p3['Chromaticity'])]) + duplicate_white = pairs(list(p3.items()) + [('DefaultWhitePoint', p3['DefaultWhitePoint'])]) + duplicate_bad = pairs(list(zero.items()) + [('Chromaticity', p3['Chromaticity'])]) + inputs = [p3, srgb, absent, white_only, chroma_only, zero, gamma, bad_gamma, + wrong_type, duplicate, duplicate_white, duplicate_bad] + for i, value in enumerate(inputs): + raw = bytearray(struct.pack('raw; + if(color) { + u32 expected[8]={metadata->chromaticity[0][0],metadata->chromaticity[0][1], + metadata->chromaticity[1][0],metadata->chromaticity[1][1], + metadata->chromaticity[2][0],metadata->chromaticity[2][1], + metadata->white[0],metadata->white[1]}; + for(unsigned i=0;i<8;i++) { + u8 low=raw[i<4?25:26]; + u32 coordinate=(u32)raw[27+i]<<2 | ((low>>(6-2*(i%4)))&3); + assert(coordinate==(expected[i]+32)/64); + } + assert(raw[23]==(metadata->gamma?120:255)); + } else { + assert(raw[23]==255); + for(unsigned i=27;i<33;i++)assert(!raw[i]); + assert(!raw[25] && !(raw[26]&0xf0)); + } + if(path) {FILE *file=fopen(path,"wb");assert(file);assert(fwrite(raw,128,1,file)==1);fclose(file);} + drm_edid_free(edid); +} +int main(int argc, char **argv) +{ + assert(argc==3); + struct dcp_display_metadata p3=decode(color_0,sizeof(color_0)); + struct dcp_display_metadata srgb=decode(color_1,sizeof(color_1)); + assert(p3.gamma==144179 && p3.chromaticity[0][0]==44564 && p3.white[0]==20493); + assert(srgb.chromaticity[0][0]==41943); + check(&p3,true,argv[1]);check(&srgb,true,argv[2]); +''' + for i in range(2, len(inputs)): + if i == 6: + unit += f'{{struct dcp_display_metadata m=decode(color_{i},sizeof(color_{i}));assert(!m.gamma && m.white[0] && m.chromaticity[0][0]);check(&m,true,NULL);}}\n' + continue + unit += f'{{struct dcp_display_metadata m=decode(color_{i},sizeof(color_{i}));assert(!m.gamma && !m.white[0] && !m.chromaticity[0][0]);check(&m,false,NULL);}}\n' + unit += r''' + struct dcp_display_metadata invalid=p3; + memcpy(invalid.chromaticity[1],invalid.chromaticity[0],sizeof(invalid.chromaticity[0])); + check(&invalid,false,NULL); + invalid=p3;invalid.white[0]=1;invalid.white[1]=1;check(&invalid,false,NULL); + invalid=p3;invalid.chromaticity[0][0]=65535;check(&invalid,false,NULL); + /* A ProductAttributes-only setter must preserve the established colour. */ + struct apple_dcp d={.fw_compat=5,.has_panel=true,.connector=&d}; + __typeof__(*d.desktop) state={0};d.desktop=&state;notifications=0; + dcp_panel_metadata_replace(&d,&p3);assert(notifications==1); + struct dcp_display_metadata partial=decode(color_2,sizeof(color_2)); + dcp_panel_metadata_replace(&d,&partial); + assert(notifications==1 && !memcmp(&state.panel_metadata,&p3,sizeof(p3))); + /* Newly reported primaries with unknown gamma must not inherit an old gamma. */ + struct dcp_display_metadata unknown_gamma=decode(color_6,sizeof(color_6)); + dcp_panel_metadata_replace(&d,&unknown_gamma); + assert(notifications==2 && !state.panel_metadata.gamma && state.panel_metadata.white[0]); + dcp_panel_metadata_replace(&d,&partial); + assert(notifications==2 && !state.panel_metadata.gamma && state.panel_metadata.white[0]); + /* A replacement panel must not inherit another serial's colour data. */ + partial.serial++; + dcp_panel_metadata_replace(&d,&partial); + assert(notifications==3 && !state.panel_metadata.gamma); + return 0; +} +''' + return unit, ((str(output/'panel-display-p3.edid'), str(output/'panel-srgb.edid')),) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py b/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py index 8d6a44988faa3..b7f4f7dd633dc 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py @@ -942,3 +942,157 @@ def identity_trace_budget(root): return 0; } ''' + + +def gamma(root): + base = root / 'drivers/gpu/drm/apple' + header = (base / 'iomfb.h').read_text() + template = (base / 'iomfb_template.c').read_text() + prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32, __le32; +typedef int64_t s64; +#define __packed __attribute__((packed)) +#define le32_to_cpu(v) (v) +#define cpu_to_le32(v) (v) +#define div_s64(a, b) ((a) / (b)) +''' + prefix += next(line for line in header.splitlines() + if line.startswith('#define IOMFB_GAMMA_ENTRIES')) + '\n' + for name in ('req', 'resp'): + prefix += block(header, 'struct iomfb_gamma_table_' + name + ' {') + ';\n' + prefix += r''' +struct drm_color_lut { u16 red, green, blue, reserved; }; +struct dcp_desktop_state { + struct iomfb_gamma_table_req *gamma_fw, *gamma_out; + struct drm_color_lut gamma_lut[IOMFB_GAMMA_ENTRIES]; + bool gamma_user, gamma_fw_read, gamma_unusable; +}; +struct apple_dcp { struct dcp_desktop_state *desktop; bool panel; void *dev; }; +static unsigned warnings, debug, swaps, writes; +static bool dcp_has_panel(struct apple_dcp *dcp) { return dcp->panel; } +static void *memchr_inv(const void *p, int c, size_t n) +{ + const unsigned char *bytes = p; + for (size_t i = 0; i < n; i++) + if (bytes[i] != c) return (void *)(bytes + i); + return NULL; +} +#define dev_warn(dev, ...) ((void)(dev), warnings++) +#define dev_dbg(dev, ...) ((void)(dev), debug++) +static void do_swap(struct apple_dcp *dcp, void *out, void *cookie) +{ assert(dcp && !out && !cookie); swaps++; } +static const int dcp_set_gamma_table_method; +static struct iomfb_gamma_table_req sent; +static void dcp_push(struct apple_dcp *dcp, bool oob, const int *method, + size_t in_size, size_t out_size, void *in, + void (*cb)(struct apple_dcp *, void *, void *), void *cookie) +{ + assert(!oob && method == &dcp_set_gamma_table_method); + assert(in_size == sizeof(sent)); + assert(out_size == sizeof(struct iomfb_gamma_table_resp)); + struct iomfb_gamma_table_resp reply = {0}; + memcpy(&sent, in, sizeof(sent)); + writes++; + cb(dcp, &reply, cookie); +} +''' + production = '\n'.join(block(template, 'static ' + signature) for signature in ( + 'u32 dcp_gamma_lookup(', 'void dcp_gamma_set_done(', + 'void dcp_gamma_set(', 'void dcp_gamma_read(')) + suffix = r''' +static struct iomfb_gamma_table_req fw, output; +static struct dcp_desktop_state state; +static struct apple_dcp dcp; +static struct iomfb_gamma_table_resp reply; +static void reset(bool panel) +{ + memset(&fw, 0xa5, sizeof(fw)); + memset(&output, 0, sizeof(output)); + memset(&state, 0, sizeof(state)); + memset(&reply, 0, sizeof(reply)); + state.gamma_fw = &fw; + state.gamma_out = &output; + dcp = (struct apple_dcp){ .desktop = &state, .panel = panel }; + warnings = debug = swaps = writes = 0; +} +static void increasing(void) +{ + /* Arbitrary synthetic scales: no assertion about the firmware encoding. */ + for (unsigned ch = 0; ch < 3; ch++) + for (unsigned i = 0; i < IOMFB_GAMMA_ENTRIES; i++) + reply.table[ch][i] = (ch + 1) * i * i; +} +static void rejected(bool warn) +{ + dcp_gamma_read(&dcp, &reply, NULL); + assert(state.gamma_unusable && !state.gamma_fw_read); + assert(warnings == warn && swaps == 1 && writes == 0); + assert(debug == !warn); + /* Failure must not turn the reply into a calibration base. */ + const unsigned char *bytes = (const void *)&fw; + for (size_t i = 0; i < sizeof(fw); i++) assert(bytes[i] == 0xa5); +} +int main(void) +{ + _Static_assert(sizeof(reply) == 3088, "gamma response size"); + _Static_assert(offsetof(struct iomfb_gamma_table_resp, ret) == 3084, + "gamma return status offset"); + for (unsigned panel = 0; panel < 2; panel++) { + reset(panel); reply.ret = 2; rejected(panel); + reset(panel); reply.ret = 1; rejected(true); + reset(panel); reply.ret = 0x100; increasing(); rejected(true); + reset(panel); reply.ret = 2; increasing(); rejected(true); + reset(panel); rejected(true); /* Successful but empty table. */ + for (unsigned ch = 0; ch < 3; ch++) { + reset(panel); increasing(); + reply.table[ch][10] = reply.table[ch][9] - 1; + rejected(true); + reset(panel); increasing(); + memset(reply.table[ch], 0, sizeof(reply.table[ch])); + rejected(true); + } + reset(panel); + dcp_gamma_read(&dcp, NULL, NULL); + assert(state.gamma_unusable && !state.gamma_fw_read); + assert(warnings == 1 && swaps == 1 && writes == 0); + + reset(panel); increasing(); + dcp_gamma_read(&dcp, &reply, NULL); + assert(state.gamma_fw_read && !state.gamma_unusable); + assert(!warnings && writes == 1 && swaps == 1); + assert(!sent.table_null); + assert(!memcmp(sent.table, reply.table, sizeof(sent.table))); + assert(!sent.padding[0] && !sent.padding[1] && !sent.padding[2]); + + /* Nonlinear, independent RGB channels; endpoints and interpolation. */ + state.gamma_user = true; + for (unsigned i = 0; i < IOMFB_GAMMA_ENTRIES; i++) { + state.gamma_lut[i].red = 0; + state.gamma_lut[i].green = 0x7fff; + state.gamma_lut[i].blue = 0xffff; + } + dcp_gamma_set(&dcp); + assert(writes == 2 && swaps == 2 && !warnings); + for (unsigned i = 0; i < IOMFB_GAMMA_ENTRIES; i++) { + assert(sent.table[0][i] == 0); + assert(sent.table[1][i] == 32767); + assert(sent.table[2][i] == 196608); + } + state.gamma_user = false; + dcp_gamma_set(&dcp); + assert(writes == 3 && swaps == 3 && !warnings); + assert(!memcmp(sent.table, reply.table, sizeof(sent.table))); + } + puts("gamma RPC status, admission, composition and restoration passed"); + return 0; +} +''' + return prefix + production + suffix diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/display.py b/tools/testing/selftests/drivers/gpu/apple-dcp/display.py index 83f29c1ad7be1..e8e355955052f 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/display.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/display.py @@ -381,3 +381,93 @@ def brightness_commit(root): return 0; } ''' + + +def hdr_metadata(root): + """HDR10 static metadata message for DCPAV selector 0x15, byte for byte.""" + header = (root / 'drivers/gpu/drm/apple/dcp-hdr-metadata.h').read_text() + header = '\n'.join(line for line in header.splitlines() + if not line.startswith('#include')) + header = header.replace('static inline ', 'static inline __attribute__((unused)) ') + # An independent encoding of the same message: the DCPAV IPC header, then + # the serialized { "Type": "CEA-861:Type1", "Data": <24 bytes> }. + def tag(last, kind, size): + return struct.pack(' +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; typedef uint16_t u16; typedef uint32_t u32; +typedef uint8_t __u8; typedef uint16_t __u16; typedef uint32_t __u32; +static void put_unaligned_le16(u16 v, void *p) { u8 *b = p; b[0] = v; b[1] = v >> 8; } +static void put_unaligned_le32(u32 v, void *p) { u8 *b = p; for (int i = 0; i < 4; i++) b[i] = v >> (8 * i); } +static u16 __attribute__((unused)) get_unaligned_le16(const void *p) { const u8 *b = p; return b[0] | b[1] << 8; } +static u32 get_unaligned_le32(const void *p) { const u8 *b = p; return b[0] | b[1] << 8 | b[2] << 16 | (u32)b[3] << 24; } +/* include/uapi/drm/drm_mode.h */ +struct hdr_metadata_infoframe { + __u8 eotf; __u8 metadata_type; + struct { __u16 x, y; } display_primaries[3]; + struct { __u16 x, y; } white_point; + __u16 max_display_mastering_luminance, min_display_mastering_luminance; + __u16 max_cll, max_fall; +}; +struct hdr_output_metadata { + __u32 metadata_type; + union { struct hdr_metadata_infoframe hdmi_metadata_type1; }; +}; +""" + header + r""" +static const u8 expected[] = {""" + expected + r"""}; +int main(void) +{ + struct hdr_output_metadata hdr = { .metadata_type = 0, .hdmi_metadata_type1 = { + .eotf = 2, .metadata_type = 0, + .display_primaries = { { 35400, 14600 }, { 8500, 39850 }, { 6550, 2300 } }, + .white_point = { 15635, 16450 }, + .max_display_mastering_luminance = 1000, .min_display_mastering_luminance = 50, + .max_cll = 1000, .max_fall = 400 } }; + u8 data[DCP_HDR_DATA_SIZE], msg[DCP_HDR_MSG_SIZE], reply[DCP_HDR_MSG_SIZE] = {0}; + + assert(sizeof(expected) == DCP_HDR_MSG_SIZE); + assert(!dcp_hdr_data(data, &hdr)); + dcp_hdr_message(msg, data); + for (size_t i = 0; i < sizeof(msg); i++) + if (msg[i] != expected[i]) { + fprintf(stderr, "byte %#zx: %#x, expected %#x\n", i, msg[i], expected[i]); + return 1; + } + /* Only static metadata Type 1 is sent. */ + hdr.metadata_type = 1; + assert(dcp_hdr_data(data, &hdr) == -EINVAL); + hdr.metadata_type = 0; hdr.hdmi_metadata_type1.metadata_type = 1; + assert(dcp_hdr_data(data, &hdr) == -EINVAL); + /* Transport error, AFK return code, short reply, DCP status at 0x40. */ + assert(dcp_hdr_reply(-ETIMEDOUT, 0, reply, sizeof(reply)) == -ETIMEDOUT); + assert(dcp_hdr_reply(0, 5, reply, sizeof(reply)) == -EIO); + assert(dcp_hdr_reply(0, 0, reply, 0x43) == -EPROTO); + assert(dcp_hdr_reply(0, 0, reply, 0x44) == 0); + put_unaligned_le32(0xe00002c2, reply + 0x40); + assert(dcp_hdr_reply(0, 0, reply, sizeof(reply)) == -EIO); + printf("PASS: HDR10 static metadata message (%zu bytes) and reply checks\n", sizeof(msg)); + return 0; +} +""" diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 79a0adf4a7b32..89dd1f31f954b 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -1694,7 +1694,7 @@ def vrr(root): DCP_FIRMWARE_V_14_7, DCP_FIRMWARE_V_14_7_TEMPLATE, DCP_FIRMWARE_V_26_6, DCP_FIRMWARE_V_26_6_DESKTOP }; struct apple_dcp { - bool crashed, external, external_native, quiescing; + bool crashed, external, external_native, quiescing, main_display; unsigned int nr_typec_routes; enum dcp_firmware_version fw_compat; struct { int iomfb_method_profile; } hw; @@ -1730,6 +1730,9 @@ def vrr(root): static int dcp_crtc_check_hdr(struct apple_dcp *d, struct drm_crtc *c, struct drm_atomic_commit *s) { (void)d; (void)c; (void)s; return 0; } +/* The panel of a desktop 26.6 machine takes VRR_ENABLED live. */ +static bool dcp_vrr_live(struct apple_dcp *d) +{ return d->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && d->main_display; } static bool iomfb_is_m4(struct apple_dcp *d) { (void)d; return false; } static int iomfb_m4_error(struct apple_dcp *d) { (void)d; assert(false); return -ENODEV; } static int iomfb_m4_get_mode(struct apple_dcp *d,struct drm_display_mode *m) @@ -1788,33 +1791,36 @@ def vrr(root): return 0; } """ - desktop = '' - if 'DCP_FIRMWARE_V_26_6_DESKTOP' in check: - # Desktop 26.6 applies adaptive sync with a mode set: a VRR_ENABLED - # change of an enabled CRTC must become one, nothing else may. - desktop = r""" dcp.fw_compat = DCP_FIRMWARE_V_26_6_DESKTOP; + # Desktop 26.6 takes VRR_ENABLED live on the panel (main display), with + # the next frame: compositors toggle it in plain page flips, so there a + # VRR_ENABLED change must never become a mode set. The external + # processors ignore a live minimum rate and still apply it with a mode set. + desktop = r""" dcp.fw_compat = DCP_FIRMWARE_V_26_6_DESKTOP; dcp.hw.iomfb_method_profile = 0; dcp.mode_state.valid = true; dcp.connector = &connector; connector.connected = true; state.crtc.modeset = false; + for (unsigned int panel = 0; panel < 2; panel++) for (unsigned int enable = 0; enable < 2; enable++) for (unsigned int old_vrr = 0; old_vrr < 3; old_vrr++) for (unsigned int vrr = 0; vrr < 2; vrr++) { + dcp.main_display = panel; state.crtc.enable = enable; state.crtc.vrr_enabled = vrr; state.crtc.mode_changed = false; state.no_old_crtc = old_vrr == 2; state.old_crtc.vrr_enabled = old_vrr == 1; assert(dcp_crtc_atomic_check(&crtc, &state) == 0); - bool expected = enable && old_vrr != 2 && (old_vrr == 1) != vrr; + bool expected = !panel && enable && old_vrr != 2 && (old_vrr == 1) != vrr; if (state.crtc.mode_changed != expected) { - fprintf(stderr, "desktop enable=%u old_vrr=%u vrr=%u: mode_changed=%d expected %d\n", - enable, old_vrr, vrr, state.crtc.mode_changed, expected); + fprintf(stderr, "desktop panel=%u enable=%u old_vrr=%u vrr=%u: mode_changed=%d expected %d\n", + panel, enable, old_vrr, vrr, state.crtc.mode_changed, expected); return 1; } cases++; } + dcp.main_display = false; dcp.fw_compat = DCP_FIRMWARE_UNKNOWN; """ return prefix + check + suffix.replace('DESKTOP_VRR', desktop, 1) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 3c2f2b8df2bd5..faae722b9d8e1 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -214,6 +214,13 @@ def ownership(root, usb_unreachable=False): usb3_port_down=false; return 0; } +static int usb_hub_port_retrain(struct usb_device *u, int port) +{ + assert(u==&roots[3]&&port==1&&root_locked&&pm_refs==1&&!pm_phy->lock); + assert(!usb3_port_down&&pm_phy->host_active&&!pm_phy->mode_dirty&&!pm_phy->pipe_pending); + assert(pm_phy->pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3); + return 0; +} #endif ''' suffix=ns['suffix'].replace('int main(', 'int __attribute__((unused)) fixture_main(',1) @@ -499,6 +506,7 @@ def function(text, name): }; static struct fwnode_handle connector; static int mux_error, notifier_error, switch_error; +static bool mux_transient; static int ports,altmodes,muxes,switches,notifiers,stop_calls; struct typec_thunderbolt_switch; static bool keep_waiting=true; @@ -521,7 +529,14 @@ def function(text, name): static void *fwnode_typec_thunderbolt_switch_get(struct fwnode_handle *f) { (void)f;if(switch_error)return (void *)(intptr_t)switch_error;switches++;return &switches; } -static int typec_set_mode(void *p,unsigned long m) { (void)p;assert(m==TYPEC_STATE_SAFE);return mux_error; } +static int cd321x_mux_set(struct cd321x *c,bool *held) { + assert(c->state.mode==TYPEC_STATE_SAFE); + *held=mux_error==-ETIMEDOUT&&mux_transient;return mux_error; +} +static void cd321x_retry_held_mux(struct cd321x *c) { + assert(c->pm.phase==CD321X_PM_INITIALIZING); + assert(cd321x_pm_mux_held(&c->pm,0)); +} static int cd321x_tbt_notify(struct notifier_block *n,unsigned long e,void *d) { (void)n;(void)e;(void)d;return 0; } static int typec_thunderbolt_switch_register_notifier(void *s,struct notifier_block *n) { (void)n;if(!s)return 0;if(notifier_error)return notifier_error;notifiers++;return 0; @@ -540,11 +555,13 @@ def function(text, name): production = switch_get.replace('VISIBLE_IF_KUNIT', 'static', 1) + '\n' init_work = block(core, 'VISIBLE_IF_KUNIT void cd321x_init_work(') production += init_work.replace('VISIBLE_IF_KUNIT', 'static', 1) + '\n' + init_mux = block(core, 'VISIBLE_IF_KUNIT int cd321x_init_mux(') + production += init_mux.replace('VISIBLE_IF_KUNIT', 'static', 1) + '\n' production += function(core, 'cd321x_register_port') + '\n' + function(core, 'cd321x_unregister_port') controls = r''' static void reset(struct cd321x *c) { memset(c,0,sizeof(*c));connector.refs=0; - mux_error=notifier_error=switch_error=0;stop_calls=0;keep_waiting=true; + mux_error=notifier_error=switch_error=0;stop_calls=0;keep_waiting=true;mux_transient=false; assert(!ports&&!altmodes&&!muxes&&!switches&&!notifiers); } static void balanced(void) { @@ -608,6 +625,13 @@ def function(text, name): reset(&c);mux_error=-EIO; assert(cd321x_register_port(&c.tps,&connector)==0&&!c.state_valid); cd321x_unregister_port(&c.tps);balanced(); + /* A probe-time transient timeout keeps its series until ready enables work. */ + reset(&c);mux_error=-ETIMEDOUT;mux_transient=true; + assert(cd321x_register_port(&c.tps,&connector)==0&&!c.state_valid); + assert(c.pm.phase==CD321X_PM_INITIALIZING&&c.pm.mux_retry_ms&&c.pm.force_reconnect); + assert(!c.resume_work.pending&&!c.update_work.pending&&cd321x_pm_ready(&c.pm)); + assert(c.pm.mux_retry_ms&&c.pm.force_reconnect); + cd321x_unregister_port(&c.tps);balanced(); /* Missing router defers while suppliers may still bind, then is optional. */ reset(&c);switch_error=-EPROBE_DEFER; assert(cd321x_register_port(&c.tps,&connector)==-EPROBE_DEFER);balanced(); @@ -710,7 +734,7 @@ def kconfig(root, output): def probe_dispatch(root): - """Connect real port probe -> typec_set_mode -> mux core -> PM wrapper.""" + """Connect real port probe -> init_mux -> mux core -> PM wrapper.""" unit = ownership(root) unit = unit.replace('int main(void)', 'static int __attribute__((unused)) ownership_fixture_main(void)', 1) unit = unit.replace('#define IS_ERR_OR_NULL(p) (!(p))', '#define IS_ERR(p) ((intptr_t)(p)<0)\n #define IS_ERR_OR_NULL(p) (!(p)||IS_ERR(p))') @@ -719,13 +743,20 @@ def probe_dispatch(root): # Type-C and mutex declarations already belong to the shared PM fixture. port = port.replace('struct typec_mux_state { void *alt,*data; unsigned long mode; };', '') port = port.replace('#define guard(kind) (void)', '') - port = port.replace('static int typec_set_mode(void *p,unsigned long m) { (void)p;assert(m==TYPEC_STATE_SAFE);return mux_error; }', '') + start = port.index('static int cd321x_mux_set(') + end = port.index('static void cd321x_retry_held_mux(', start) + core = (root / 'drivers/usb/typec/tipd/core.c').read_text() + dispatch = block(core, 'static bool cd321x_mux_held(') + '\n' + block(core, 'static int cd321x_mux_set(') + timeout = block((root / 'drivers/usb/typec/mux.c').read_text(), + 'bool typec_mux_timeout_transient(') + port = port[:start] + timeout + '\n' + dispatch + '\n' + port[end:] port = port.replace('(void)f;t->port=t;ports++;return 0;', '(void)f;t->port=&probe_port;ports++;return 0;') - # Real class dispatch requires the port's mux; its allocator remains a mock. + # Both handles refer to the same route; its allocator remains a mock. declarations = 'struct typec_port { struct typec_mux *mux; };\nstatic struct typec_port probe_port;\n' - dispatch = block((root / 'drivers/usb/typec/class.c').read_text(), 'int typec_set_mode(') marker = port.index('static int tps6598x_register_port(') - port = port[:marker] + declarations + dispatch + '\n' + port[marker:] + port = port[:marker] + declarations + port[marker:] + port = port.replace('(void)f;muxes++;return &muxes;', + '(void)f;muxes++;return probe_port.mux;') import re # Only helpers used at this boundary belong in this translation unit; # the separate probe group exercises the complete retry state machine. @@ -733,13 +764,14 @@ def probe_dispatch(root): 'cd321x_pm_begin_read', 'cd321x_pm_snapshot_ready', 'cd321x_pm_complete', 'cd321x_pm_can_update', # which, once retries of a held mux exist, ends them on removal - 'cd321x_pm_mux_done'} + 'cd321x_pm_mux_done', 'cd321x_pm_mux_held'} for match in list(re.finditer(r'^static inline (?:void|bool|unsigned int) (cd321x_\w+)\(', port, re.M))[::-1]: if match.group(1) not in used: body = block(port, match.group()) port = port[:match.start()] + port[match.start() + len(body):] port = port.replace('static int mux_error, notifier_error, switch_error;', 'static int notifier_error, switch_error;') + port = port.replace('static bool mux_transient;', '') existing = set(re.findall(r'^\s*#define (\w+)', unit, re.M)) port = '\n'.join(line for line in port.splitlines() if not (line.startswith('#define ') and line.split()[1].split('(')[0] in existing)) @@ -770,7 +802,13 @@ def probe_dispatch(root): memset(&c,0,sizeof(c));prepare(&p,true,true,false);resume_error=-EIO;before=writes; assert(cd321x_register_port(&c.tps,&connector)==0 && !c.state_valid && writes==before); balanced(&p);cd321x_unregister_port(&c.tps);port_balance(); - puts("PASS: actual CD321x probe -> typec_set_mode -> typec_mux_set -> bounded PM wrapper; delayed root reprobe and rollback references"); + memset(&c,0,sizeof(c));prepare(&p,true,true,false);resume_error=0; + provider.transient_timeouts=true;fail_caller="atcphy_pipehandler_lock_timeout"; + assert(cd321x_register_port(&c.tps,&connector)==0&&!c.state_valid); + assert(c.pm.phase==CD321X_PM_INITIALIZING&&c.pm.mux_retry_ms&&c.pm.force_reconnect); + assert(!c.resume_work.pending&&!c.update_work.pending); + balanced(&p);cd321x_unregister_port(&c.tps);port_balance();fail_caller=NULL; + puts("PASS: actual CD321x probe -> init_mux -> typec_mux_set -> bounded PM wrapper; delayed root, initial PIPE timeout and rollback references"); } """ return unit + '\n' + port + '\n' + controls diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index f836743f2fc8d..3235ca5145866 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -27,11 +27,13 @@ import accounting import idle import desktop +import audio import j616s import rate import boot import m4 import display +import colorimetry def parking_unit(root): @@ -43,7 +45,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--source', type=Path, default=Path(__file__).resolve().parents[6]) parser.add_argument('--output', type=Path, help='retain generated C, commands and results here') - parser.add_argument('--group', choices=('all', 'edid', 'rate', 'phy', 'lifecycle', 'boot', 'quiesce', 'm4', 'display'), default='all') + parser.add_argument('--group', choices=('all', 'edid', 'rate', 'phy', 'lifecycle', 'boot', 'quiesce', 'm4', 'display', 'audio'), default='all') args = parser.parse_args() root = args.source.resolve() output = args.output or Path(tempfile.mkdtemp(prefix='apple-dcp-selftest-')) @@ -54,6 +56,10 @@ def main(): 'orientation-faults', 'backend-change', 'safe-first', 'failed-flip-return', 'live-park')) tests = [ + ('display', 'panel-colorimetry', lambda r: colorimetry.panel(r, output), ((),), ('-Wno-unused-function', '-Wno-unused-const-variable')), + ('audio', 'compact-audio-reply-bounds', audio.reply_bounds, ((),), ()), + ('display', 'desktop-gamma', desktop.gamma, ((),), ()), + ('display', 'hdr-static-metadata', display.hdr_metadata, ((),), ()), ('lifecycle', 'desktop-modeset-unmap', desktop.modeset_unmap, ((),), ()), ('lifecycle', 'desktop-unknown-modeset-recovery', desktop.modeset_unknown, ((),), ()), ('display', 'panel-backlight-ramp', display.backlight, ((),), ()), diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/README b/tools/testing/selftests/drivers/gpu/asahi-m4/README index c88abd122428e..040a90a056ef6 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/README +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/README @@ -64,7 +64,7 @@ paired native Mesa installation: make hardware MESA=/path/to/installed/mesa RENDER_NODE=/dev/dri/renderD128 ROUNDS=32 -This builds native-gl46.c, verifies asahi-g16 owns the render node, selects the +This builds native-gl46.c, verifies aurora-g16 (asahi-g16 on older kernels) owns the render node, selects the requested Mesa libraries, clears software/translation/API-version overrides, and runs native OpenGL 4.6 triangle readback, compute and geometry-feedback checks. Results include kernel/boot identity and Mesa/binary hashes. Ordinary diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/native-ioctl-regression.c b/tools/testing/selftests/drivers/gpu/asahi-m4/native-ioctl-regression.c index 5f19e9f6eef97..aa324b7976525 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/native-ioctl-regression.c +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/native-ioctl-regression.c @@ -170,7 +170,10 @@ struct compute_stats { static int compute_stats(struct compute_stats *s) { - FILE *file = fopen("/sys/kernel/debug/asahi-g16/stats", "re"); + /* Kernels before the Aurora rename used "asahi-g16". */ + FILE *file = fopen("/sys/kernel/debug/aurora-g16/stats", "re"); + if (!file) + file = fopen("/sys/kernel/debug/asahi-g16/stats", "re"); char *line = NULL; size_t size = 0; bool compute = false, mixed = false, publications = false; @@ -410,7 +413,7 @@ static int identity(const char *release, const char *boot, const char *source, c if (n < 0) return -1; target[n] = 0; const char *owner = strrchr(target, '/'); - if (!owner || strcmp(owner + 1, "asahi-g16") || + if (!owner || (strcmp(owner + 1, "aurora-g16") && strcmp(owner + 1, "asahi-g16")) || access("/sys/module/gpu_m4_drm", F_OK) == 0 || access("/sys/module/apple_gpu_m4_power", F_OK) == 0) return -1; return 0; diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/run-hardware-tests.py b/tools/testing/selftests/drivers/gpu/asahi-m4/run-hardware-tests.py index b2f01f06cefc8..d9b09d5820a04 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/run-hardware-tests.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/run-hardware-tests.py @@ -31,8 +31,8 @@ def main(): node = args.render_node.resolve(strict=True) device = (Path('/sys/class/drm') / node.name / 'device').resolve(strict=True) owners = [p / 'driver' for p in [device, *device.parents] if (p / 'driver').is_symlink()] - if not any(p.resolve().name == 'asahi-g16' for p in owners): - parser.error('Selected render node is not owned by the Rust asahi-g16 platform driver') + if not any(p.resolve().name in ('aurora-g16', 'asahi-g16') for p in owners): + parser.error('Selected render node is not owned by the Rust G16 platform driver (aurora-g16)') output = (args.output or Path(tempfile.mkdtemp(prefix='asahi-m4-hardware-'))).resolve() output.mkdir(parents=True, exist_ok=True) environment = os.environ.copy() diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/run-native-ioctl-regression.py b/tools/testing/selftests/drivers/gpu/asahi-m4/run-native-ioctl-regression.py index 06b71eb83a90c..0932dc9f13fe5 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/run-native-ioctl-regression.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/run-native-ioctl-regression.py @@ -20,7 +20,11 @@ UNPATCHED = {'78d2519c82fca4c0403315e040d595628bc5a4d6', 'df583754ae2b3607af4155ba0442bc93077bdce8'} FAULT = re.compile(r'Kernel panic|BUG:|WARNING: CPU|SError|Oops:|Unable to handle kernel|' - r'\b(?:asahi|G16|GPU|PPL)\b.*(?:fault|fail(?:ed|ure)?|timeout|timed out|quarantin)', re.I) + r'\b(?:aurora|asahi|G16|GPU|PPL)\b.*(?:fault|fail(?:ed|ure)?|timeout|timed out|quarantin)', re.I) +# The G16 platform driver and its debugfs directory; kernels before the Aurora +# rename used the asahi-g16 name. +G16_DRIVERS = ('aurora-g16', 'asahi-g16') +G16_STATS = [Path('/sys/kernel/debug') / name / 'stats' for name in G16_DRIVERS] EXPECTED_REJECTION = re.compile(r'IOCTL: submit failed! \(submission ID: \d+ err: E2BIG\)\s*$') @@ -41,7 +45,8 @@ def self_test(): 'Kernel panic - not syncing: ' + expected, expected + ' GPU fault bank 0', expected.replace('err: E2BIG', 'err: EIO'), - 'asahi-g16: GPU fault bank 0; ' + expected] + 'asahi-g16: GPU fault bank 0; ' + expected, + 'aurora-g16: GPU fault bank 0; ' + expected] for line in harmless: assert not kernel_fault(line), 'False fault match: ' + line for line in failures: @@ -96,8 +101,9 @@ def preflight(args): if not stat.S_ISCHR(node.stat().st_mode): raise RuntimeError('Expected a DRM character device') device = Path('/sys/class/drm') / node.name / 'device' - if (device / 'driver').resolve(strict=True).name != 'asahi-g16': - raise RuntimeError('Rust asahi-g16 does not own the requested render node') + owner = (device / 'driver').resolve(strict=True).name + if owner not in G16_DRIVERS: + raise RuntimeError('The Rust G16 platform driver does not own the requested render node') if any((Path('/sys/module') / name).exists() for name in ['gpu_m4_drm', 'apple_gpu_m4_power']): raise RuntimeError('Legacy C GPU provider is loaded') active = root_json('/run/aurorasilicon/m4-asahi-rust.json') @@ -118,7 +124,7 @@ def preflight(args): raise RuntimeError('Qualification marker already exists; rollback supervisor may disarm') return {'boot_id': boot, 'source_commit': args.expected_source, 'kernel_release': os.uname().release, 'kernel_image_sha256': sha(image), 'asahi_module_sha256': sha(module), - 'source_state_sha256': sha(source_path), 'render_node': str(node), 'owner': 'asahi-g16', + 'source_state_sha256': sha(source_path), 'render_node': str(node), 'owner': owner, 'rollback_guard': guard, 'activation': active} @@ -149,7 +155,7 @@ def compile_helper(args, output): def snapshot(): - stats = Path('/sys/kernel/debug/asahi-g16/stats').read_text() + stats = next(p for p in G16_STATS if p.exists()).read_text() published = re.search(r'^publications: compute (\d+) render (\d+)\s*$', stats, re.M) if not published: raise RuntimeError('Missing actual firmware-producer publication counters; refuse live regression')