diff --git a/.github/workflows/host-tests.yml b/.github/workflows/host-tests.yml index f16fb33953f7c..fd665f948d991 100644 --- a/.github/workflows/host-tests.yml +++ b/.github/workflows/host-tests.yml @@ -57,6 +57,8 @@ jobs: command -v "$p" >/dev/null || need="$need $p" done command -v "ld.lld-$CLANG_VERSION" >/dev/null || need="$need lld-$CLANG_VERSION" + # Device-tree tests read the built trees with fdtget. + command -v fdtget >/dev/null || need="$need device-tree-compiler" dpkg -s "libclang-rt-$CLANG_VERSION-dev" >/dev/null 2>&1 || need="$need libclang-rt-$CLANG_VERSION-dev" if [ -n "$need" ]; then sudo apt-get update -q diff --git a/Documentation/ABI/testing/debugfs-driver-asahi-stats b/Documentation/ABI/testing/debugfs-driver-asahi-stats new file mode 100644 index 0000000000000..f795d25ace2ba --- /dev/null +++ b/Documentation/ABI/testing/debugfs-driver-asahi-stats @@ -0,0 +1,47 @@ +What: /sys/kernel/debug/dri//asahi_stats +Date: October 2026 +Contact: Ryan Murray +Description: + Read-only, root-only dump of the raw AGX firmware statistics + that the drm/asahi driver receives on its statistics channel. + The driver stores the latest message of each kind and prints + it unchanged. It does not compute utilization or any other + derived value. + + The file is created in the debugfs directory of the DRM + device when the device is registered. It also appears under + the other names of that directory (dri/128 and the platform + device name, for example dri/406400000.gpu). + + The format is ASCII text, one line per record: + + - line 1: asahi_stats_v1, the format tag. A reader must check it. + - line 2: device= + - line 3: now_ns= + - line 4: messages count= + - then one line per message kind, in this order: power, + power_on, power_off, utilization, avg_power, temperature, + power_state, fw_busy, pstate, temp_sensor. + + A kind line is " count=" followed, when n is not 0, + by " captured_ns=" and the fields of + that kind as key=value pairs: + + - power: power + - power_on: off_time + - power_off: on_time + - utilization: timestamp, util1, util2, util3, util4 + - avg_power: active_cs, unk2, unk3, unk4, avg_power + - temperature: raw_value, scale, tmin, tmax + - power_state: timestamp, last_busy_ts, active, poweroff, + unk1, pstate, unk2, unk3 + - fw_busy: timestamp, busy + - pstate: ps_min, unk1, ps_max, unk2 + - temp_sensor: sensor_id, raw_value, scale, tmin, tmax + + All counts are cumulative since the driver bound the device. A rate needs two reads. + The values are the raw firmware values. Their units and the + meaning of the util*, unk* fields are not established. A kind + that was never received prints count=0 and no payload. Keys + may be added in later versions of the format tag; readers + must ignore keys they do not know. diff --git a/Documentation/accel/apple-ane-cold.rst b/Documentation/accel/apple-ane-cold.rst new file mode 100644 index 0000000000000..b72b18131426a --- /dev/null +++ b/Documentation/accel/apple-ane-cold.rst @@ -0,0 +1,137 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +Cold-acquired H17 firmware backend +================================= + +This experimental backend shares the H16/H17 channel transport, LOAD and +EXECUTE wire formats and UUID completion helpers. It does not take over a +running Exclave engine. Production T8140 nodes remain disabled. The separate +``t8140-j700-ane-test.dtb`` enables only the lead's one-shot test. + +The staged observer is installed before platform driver registration. This +avoids the platform bus's automatic genpd power-on and default DMA attachment +before the read-only probe. The existing H14 ABI is never exposed for H17. +Build ``CONFIG_DRM_ACCEL_ANE=y`` and ``CONFIG_APPLE_DART=y`` for acquisition; +an ANE module supports observation only because the kexec blocker requires a +built-in caller. No module parameter opens this backend. + +Loader contract +--------------- + +The loader must publish ``/chosen/apple,ane-cold-boot`` with the exact string +``raw-no-xnu-ane-untouched-v1`` only when the reset path ran no XNU, SPTM, +ExclaveCore or other ANE owner and left the engine untouched. This is boot +evidence, not an operator opt-in. Neither the production nor test DTS supplies +it. A warm handoff, kexec, unknown loader or missing evidence must refuse. + +The loader also supplies two ``no-map``, non-reusable reserved-memory nodes, +referenced by ``memory-region`` and named ``firmware-text`` and +``firmware-data``. Their sizes are 0xc0000 and 0x2ac000. Physical addresses +come from that boot's preload records, never from compiled-in lab addresses. +The DVA pair is 0x10000000000 and 0x100000c0000. + +TEXT must match the measured 26.6.2 H17 SHA-256 pin in the driver. The loader's +``apple,preload-data-sha256`` is a digest of the full live DATA segment. The +driver copies both reserved segments into private coherent RAM and preserves +every iBoot-written DATA byte. The DATA digest establishes preservation of the +loader's supplied segment; it is not a cross-build immutable-DATA manifest. +A pinned manifest excluding only the individually documented mutable fields +remains required before production enablement. No firmware image is shipped. + +Stages and recovery +------------------- + +Use the platform device's root-writable ``firmware_stage`` attribute. Reading +it reports ``unverified``, ``probe``, ``map``, ``boot``, ``startup``, +``program`` or ``quarantined``. Write these actions in order:: + + echo probe > firmware_stage + echo map > firmware_stage + echo boot > firmware_stage + echo startup > firmware_stage + echo program > firmware_stage + +The engine block and its DARTs sit behind ANE_SYS, and an access to them +while it is off stalls the bus. ANE_SYS is no cold marker: it can read on +while genpd reports off, and generic DART runtime PM or a refused platform +probe turns it off again. Every register is therefore read only once its +block is known to be powered, in the order the U2 runs measured as safe. + +``probe`` reads only PMGR: ANE_SYS, the ANE CPU power state (low byte off) +and the sub-block power state (0x300). With the loader's raw-boot evidence it +moves to ``probe``; it writes nothing and never touches the engine or DARTs. + +``map`` repeats the PMGR check, then retains the module and power lifetime, +blocks system sleep (and kexec when built in) and acquires genpd ANE_SYS. It +reads PWGATE (request 3, acknowledge 0), turns the CPU power state on, clears +the PWGATE request and turns the sub-block on. Only then does it read and +check the engine and DARTs: CE state and error, CPU control (0) and status +(0x6a), RVBAR, IRQ pending, scratch, and unlocked, unfaulted DARTs of version +0x203 with no stream enabled. DAPF records and invalid TTBR payloads are +logged, never used as cold markers. Nothing in the engine has been written at +that point; a failed check quarantines with power retained. ``map`` then +completes CE, attaches only SID0 on three DARTs to the private paging domain, +writes and reads back the four DT-provided DAPF ranges, maps their pages as +identity MMIO, and maps preloads, command pool, IPC and heap at their +documented DVAs. + +``boot`` runs the scratch handshake, checks the exact request sizes, validates +all eight channel descriptors and arms host rings. ``startup`` sends only +PRINT_ENABLE, START, CONFIG_GET and RESOURCE_INFO_GET, servicing shared +allocations with an absolute deadline. Completion is polled. Neither IRQ 655 +nor IRQ 668 is registered or enabled. Secure-mode commands 0x27, 0x28, 0x34 +and Exclave commands 0x30, 0x31 are denied at the publication boundary. 0x2f +is shared-event initialization, not an Exclave command, and is not sent. + +Any failure after acquisition, including a normal map synchronization timeout, +quarantines the device, retains mappings and power and requires a reboot. +There is no in-place reset or retry. Suspend is refused while retained. +The ANE DART profile also refuses generic suspend/resume and never resets +foreign streams. Rollback is an ordinary reboot to the saved production +entry, never module unload after acquisition. + +Program integration and measured limits +-------------------------------------- + +``program`` creates the shared H16/H17 program session on this transport and +registers the ABI 3 accel node (``DRM_ANE_CHIP_H17``). Userspace passes the +original H17 HWX and its four host-built sections, as on H16; the kernel +validates them, places every section in private mappings, binds a linked +program's ``__DATA`` scratch to kernel-owned memory and runs the +0x2d/EXECUTE/0x2d sequence with UUID completion. Operation records with a +kernel table must bind the kernel BAR to resource index 1007; legacy tag 2 is +refused before LOAD because the measured H17 firmware asserts on it. The +firmware frees each process pool after UNLOAD, often only during a later +program; that FREE is acknowledged and its backing retained until reboot. + +Several programs may run in one firmware boot. The firmware has a direct +submission path that reuses the engine's current BAR context. If that context +belongs to a different program, a job can write through the earlier program's +output binding. This happens even after that program was terminated and +unloaded, and a full TLB invalidation does not prevent it. Bit 0 of the +EXECUTE flags word at offset 0x30 keeps a request off the direct path and +queues it for the full execution loop, which installs a fresh BAR context. +The transport sets that bit in its copy of the first EXECUTE after any change +of program or process, and after any LOAD, CREATE, TERMINATE or UNLOAD, since +firmware IDs may be reused. Later jobs of the same process keep the direct +path. Measured on the J700 with 11 of 11 program switches correct, at about +60 us extra on the switching job only. H16 shares the firmware design but has +not executed under Linux, so its requests are published unchanged. Output +mappings of earlier programs stay pinned and are never reused. +The observed FREE response is DVA|1 with zero length and handle. It is +accepted for any live exact allocation once the program stage started, since +the firmware frees a process pool after UNLOAD, often only during a later +program. Its backing and quota stay pinned and are never recycled, so the +12 MiB allocation quota bounds the program lifetimes of one firmware boot +(each process pool is 2 MiB). Duplicate, interior and unsolicited FREEs +refuse. The canonical FREE form remains unproven. + +Provenance +---------- + +Register and wire facts were taken from the supplied clean H16/H17 field +specifications and the measured H17 cold-boot and harness results. No private +implementation source or firmware bytes are included. The operation-record +geometry is also documented by the MIT-licensed public ``libane/ane_m2.c`` +in https://github.com/joshuaswarren/omarchy-ane. The H17 resource-index rule +overrides that older generation's kernel-tag convention. diff --git a/Documentation/accel/apple-ane.rst b/Documentation/accel/apple-ane.rst index 2253c72015f1c..80feec377a251 100644 --- a/Documentation/accel/apple-ane.rst +++ b/Documentation/accel/apple-ane.rst @@ -22,23 +22,28 @@ H16 preload handoff ------------------ The bootloader must reserve the personalized TEXT and DATA preload segments -as separate, non-reusable ``no-map`` reserved-memory nodes, attach them in +as separate, non-reusable ``no-map`` children of ``/reserved-memory``, attach them in that order through ``memory-region`` and name them ``firmware-text`` and ``firmware-data``. It must also supply ``apple,firmware-iovas`` as two 64-bit values: ``0x10000000000`` and ``0x100000c0000``. Their sizes must be ``0xc0000`` and ``0x2ac000``. Physical placement is boot-dependent; board DT sources must not contain captured physical addresses. The reservations must -remain intact until reboot, including after driver failure. - -H16 preflight checks resource geometry, firmware identity and these ranges -without MMIO or power access. No disk firmware is requested. Missing -reservations fail with ``-ENODATA``. Malformed reservations fail with -``-EINVAL``. Even a valid handoff currently fails with ``-EOPNOTSUPP``: -the strict preload validator, original-container HWX ABI, firmware process -resource decoder and cold boot backend are incomplete. The section-only -load ABI is explicitly refused for H16. No accel node or program-loading interface is published. - -The H16 DT nodes stay disabled. The five real PMGR providers already encode +remain intact until reboot, including after driver failure. The driver +reads each range from its node and refuses it when any part is Linux +System RAM, so a loader may either reserve the range at boot or leave it +outside the memory map and describe it with a node. + +Metadata preflight checks resource geometry and these ranges without MMIO +or power access. Board DTs carry disabled reservation placeholders; while +the loader has not filled and enabled them the device is refused with +``-ENODEV`` and one informational message. Missing reservations fail with +``-ENODATA`` and malformed reservations with ``-EINVAL``. A device that +passes is brought up through the program stage described below once per +boot, from a work item after probe. No accel node or program-loading +interface exists before the program stage. + +The J616s enables the ANE and its three DARTs; other H16 boards keep them +disabled until proven. The five real PMGR providers already encode the system/CPU/MPM/task/base dependency graph. The PMGR binding does not represent the virtual no-state gates, so no fictitious register-backed domains are created for them. SID 0 and SID 11 on all three DARTs use the @@ -84,9 +89,9 @@ It never opens an accel device or claims a Linux inference result. Remaining implementation gates ------------------------------ -SPEC-r3 supplies the custom descriptor/aperture layouts and a normalized -immutable preload digest with 42 ordered patchbay records. The complete -preload validator, original-container ABI, process-resource decoder and +The preload validator can authenticate owned snapshots of the normalized +resident image with 42 ordered patchbay records. Live snapshot acquisition, +the original-container userspace load ABI, resolved program bindings and physical boot backend still need implementation. Off-list preload mutations remain refused. @@ -97,6 +102,63 @@ traffic does not prove them. Resource and allocation ceilings stay unchanged. H17 additionally requires a Linux ownership backend before direct access. There are no module parameters to bypass these gates. +Resident image identity +----------------------- + +The preload checker requires exactly ``0xc0000`` TEXT bytes and ``0x2ac000`` +DATA bytes. It validates the raw four-byte tag and little-endian width of +each of the 42 packed patchbay records, including records at odd offsets. +It then hashes TEXT followed by DATA, substituting zero only for the 20 +reviewed mutable value cells and the DATA-relative tunables interval +``[0x17c80, 0x18320)``. The expected SHA-256 is compiled into the validator: +``1f865ed04c4c38f6fdcc4543a8814f818d3cea583de64d22ffe96e3a93ca4697``. +Tags, widths, unlisted values, all TEXT bytes and the DATA zerofill tail +remain authenticated. An off-list change is never accepted merely because +a zero word became nonzero or a value resembles an address. + +Validation never changes its input snapshots. A future boot backend must +prove ownership and snapshot stability before calling it, preserving the +resident pages, their personalized mutable values and the reservations. +The checker is not connected to preflight yet: a generic CPU resource claim +cannot distinguish an ordinary busy ``no-map`` reservation from another +owner. No live pages are read by this preparatory path. A successful identity +check does not establish RTKit state, ownership of a running firmware +instance or permission to publish new DMA mappings. + +Original-container import and load planning +------------------------------------------ + +The internal original-HWX import helper bounds the complete 64-bit byte +count before allocation, copies userspace memory once, then validates and +digests that private snapshot. It accepts no caller-provided digest or +section metadata. A failed import leaves the destination untouched, and an +occupied snapshot cannot be overwritten. No ioctl exposes this helper yet. + +Validated metadata retains whole-segment geometry and file extents, so one +TEXT allocation covers both task and constant sections. Scratch and external +surfaces have zero file bytes. The load planner reparses the authenticated +original instead of trusting cached metadata, rounds allocations to 16 KiB +and applies a 512 MiB aggregate host ceiling before any DMA allocation. + +The resource geometry helper requires one backend-owned mapping per planned +segment and one binding per external surface, resolved by the backend from +the validated program contract. It checks exact sizes, permissions, +alignment, full-width DVA overflow and overlap against every other mapping +and the resident firmware aperture. Binding VMs, classes and unique indices +must match the validated surface requirements. +Failure publishes no partial resource map. The helper checks geometry; +the backend must separately prove mapping ownership and lifetime. + +There is no H16 LOAD_PROGRAM or CREATE_PROCESS body serializer or backend +that resolves resources and installs a process here. CREATE_PROCESS reply +IDs do not supply resource bindings. The backend must resolve them from the +validated program contract and verify mapping ownership. No user ID or +surface ordinal substitutes for that authority, and no load plan installs +an executable process. The original-HWX userspace ABI remains deferred until +the backend can return a program capability backed by validated resources. +Channel-5 allocation replies, boot publication and H17 direct ownership +remain refused. + H16 macOS 27.0 runtime contract ------------------------------ @@ -126,7 +188,7 @@ EVENT_REPORT 0x0300 6 0x60 (firmware to host) These sizes bound the observed version, not future firmware. POWER_GATE has a u32 Boolean at +8. The execute-size increment of 0x18 counts custom BAR records. The channel numbers are observed table indices; -their mapping to channel names remains unproved. The parser retains each +the clean cold-boot facts identify their channel names. The parser retains each descriptor's original index rather than assuming name order is wire order. The transport prepares the H16 envelope and rejects unrecognized runtime @@ -201,8 +263,9 @@ barriers order output reads after completion. Streaming DMA sync APIs must not be applied to these coherent allocations. There is no userspace alias to code, scratch or live firmware buffers. -The resource map must come from the original HWX and a decoded firmware -process reply. A userspace index, numeric class or program/process ID cannot +The backend must resolve the resource map from the original HWX and the +reviewed program contract, rather than decode it from CREATE_PROCESS reply +IDs. A userspace index, numeric class or program/process ID cannot create that map or confer ownership. Only established input/output classes are admitted; custom BAR requests require a validated selector map and remain unavailable. The legacy 0x28-byte completion predicate remains @@ -210,7 +273,7 @@ H14-only. Channel-5 allocation/control/free operations require a validated H16 reply provider and remain unavailable; H14 allocation replies are never reused for H16. -No original-container load ABI or H16 process-resource decoder is supplied +No original-container load ABI or H16 process installer is supplied by this change. Consequently no H16 process can yet be installed, inference remains refused and no accel node is registered. The submission integration is preparatory and does not open the boot, preload, ownership, firmware @@ -221,6 +284,76 @@ zero/one/two custom BAR lengths, wide DVAs/spans, reserved-zero output, boundary and overflow refusal, all event states, interleaved identities, ISP routing and malformed reports. No test packet is copied from a capture. +Shared H16/H17 firmware transport +-------------------------------- + +``ane_fw_transport.h`` supplies device-independent ring operations over owned +coherent surfaces and caller-provided memory, DMA-barrier, MMIO, clock and +allocation accessors. Engine-relative doorbell and pending/acknowledge offsets +come from ``ane_fw_platform`` match data. The caller must establish ownership, +firmware identity, power and the private DMA domain before using it. This +helper does not change probe or enable either generation. + +Initialization copies exactly eight 0x100-byte descriptors from the IPC +surface and validates their names, types, bits, slot counts, alignment and +non-overlapping ranges, including exclusion of the table itself. IO and +DATA_CHAIN_H2T require type 0; SHAREDMALLOC and IO_T2H require type 1. Each +operation checks the immutable table snapshot. Only type-0 slots are armed, +in word-1, word-2, word-0 order, with values 0, 0, 1 and a DMA write barrier. +Descriptor geometry and doorbell bits remain authoritative. Logical channel +names are independent of their descriptor order. + +Publication copies a command into a 64-byte-aligned arena inside the pool, +after the caller's reserved boot prefix. It publishes both length words, +a DMA write barrier, the command DVA, another barrier and the descriptor's +doorbell bit. There is one outstanding transaction per sending channel; +heads advance modulo the descriptor slot count only after a successful +reply. A reply requires the exact DVA with bit 0 set, unchanged length, +zero result word, zero command prefix, unchanged u16 ID and zero u16 ack. +Bit 1 is corruption even before ownership returns. Nonzero ack is returned +as ``-EREMOTEIO`` and retained for diagnosis. + +IO_T2H is present when word 0 has bit 0 clear and either word 0 or word 1 is +nonzero. Reports must be four-byte aligned and wholly inside one owned +surface. The transport copies the report before setting word 0 bit 0 and +ringing the descriptor doorbell, without changing the other slot words. +Report dispatch and pending UUID lookup belong to the caller. In particular, +consuming a stale UUID report cannot complete an unrelated request, and the +DATA_CHAIN reply acknowledges submission rather than inference completion. + +The SHAREDMALLOC provider accepts allocation tuples only. Requests are +rounded to 16 KiB, with a 1 MiB request ceiling, the exact 2 MiB allocation +with tag 0x45585050 exception, 12 MiB in rounded allocations and 64 handles. +The allocation callback supplies a zeroed, pinned private-domain mapping; +the transport checks its size, DVA alignment, overflow and exclusion from +all other owned surfaces before publishing it. Handles are nonzero ordinals. +By default, FREE and distinguishable control tuples return ``-EOPNOTSUPP`` +through the mandatory refusal logger. The pinned H16 build enables the +proved native FREE reply described below. An all-zero slot is empty and +cannot distinguish a control request; its lookup request layout remains +unproved. No control reply is invented. + +IRQ service reads pending once, rejects bits above 7 and writes the exact +nonzero snapshot to acknowledge. The wait helper services one allocation at +a time and checks the original absolute monotonic deadline, including time +spent inside the allocation callback. Servicing never starts a new timeout. + +The caller serializes all entry points. Initialize a fresh transport only +before firmware starts or after a proved reset. The pool arena is monotonic +and returns ``-ENOSPC`` when exhausted; command storage is not recycled. +Successful reply bodies remain in the pool transaction span. A timeout, +malformed firmware tuple, allocation failure or firmware refusal latches an +error, retains the transaction and mappings and stops further publication. +No surface is freed by these helpers, including a rejected allocation result. +The backend must retain mappings, power and module references until reset. + +The field contract follows the clean H16/H17 cold-boot facts, sections 1, +7 through 11, and the SPEC-r2 channel and deadline rules. Allocation limits +are conservative host policy observed sufficient on H17, rather than +firmware-advertised authority. The KUnit suite uses synthetic memory and +accessor traces, with no MMIO or hardware access. Its configuration fragment +is ``drivers/accel/ane/transport.kunitconfig`` and runs on x86_64 and arm64. + Original HWX validation ----------------------- @@ -256,7 +389,7 @@ then exercise command, geometry, tensor, aperture, symbol, relocation and digest refusal paths. No hardware inference result is implied. H13 timeout and engine recovery -============================== +======================= The host task-manager path (``ane.ko``) gives each submission one second, including DART fault-drain retries. An expired submission returns @@ -540,3 +673,230 @@ size, and vary inputs or poison outputs so retained output cannot pass. 4. Restore pm_test and mem_sleep selections. Preserve failure evidence before using the approved known-good boot fallback. Do not force-unload retained firmware memory or treat an unproven stop as permission to remove power. + +Staged H16 cold boot +------------------- + +The boot bring-up runs every stage in order. ``/ane-h16//stage`` +(root only) reports the last completed stage and the run, refusal and +quarantine state. Before the boot bring-up has started, writing +``preflight``, ``map``, ``boot``, ``startup`` or ``program`` runs every +stage that has not run yet, up to the one written, for diagnosis. +A stage at or below the completed one is refused with ``-EALREADY``. There +is no module parameter and no H14 boot parameter changes the H16 path. H17 still refuses ownership and registration. +All build-dependent boot values and ordered masked updates belong to the +H16 27.0 (26A428) match-data profile. The existing preload validator must +accept the resident image before that profile can run. Live mutable patchbay +and firmware tunable values are preserved, with no disk firmware owner. + +``preflight`` raises the five real domains through genpd, in DT order, and +checks every requested/ACTUAL low byte twice before reading the engine. +CPU control must be zero and RVBAR must be ``0x10000000001``. It reads all +three DART fault status/address words, version/protect/lock headers and four +DAPF records per instance against ``apple,dapf-ranges``. The ANE backend +writes no engine, DART or DAPF register at this rung. Genpd owns power writes. + +``map`` claims an exclusive, singleton IOMMU group and an unmanaged domain +covering the three SID-0 DART instances. Private surfaces currently require +a DMA-coherent client; noncoherent clients refuse pool allocation before RUN. No +unexported architecture cache helper is called from the module. It checks +the finalized aperture +rather than widening geometry in the client. TEXT is read-only, DATA is +read/write, and the four range-containing 16 KiB pages use identity addresses +and MMIO attributes. Every DAPF lock is checked before any record changes; +a locked mismatch refuses. Unlocked changes preserve unrelated flag bits +and are read back. The backend does not change protect or lock registers. + +The DT must describe only SID 0 on these three instances for this backend. +The generic IOMMU attachment API attaches every stream in ``iommus``; it +cannot select a subset from the six-stream production description. Until +the test DT supplies only SID 0, map and later stages explicitly refuse +extra streams. The inherited test DTS currently enables the six-stream +node and therefore needs this integration adjustment before rung 2. The +production node stays disabled. The driver does not rewrite the live DT +or access the DART driver's private stream configuration. + +``boot`` allocates the zeroed command pool at ``0x10000384000`` and completes +the H16 CE request/ack protocol, bounded to 5001 delays of 10 us. Ordered +ANEAXI2AF, ANEXLAXI2AF and selected C1PPT updates follow before any scratch +write. A pre-existing C1PPT lock requires matching values. There is no H17 +PWGATE sequence. The host clears scratch 0 through 7 in order, prepares the +polled management client, writes scratch 6 = 1 and 7 = 0, rechecks RVBAR, +and writes CPU control 0 then 0x10. RVBAR is never rewritten. + +The host polls READY ``0x08042006`` every 1 ms, with at most 1000 polls and +one absolute deadline. Scratch 0 is a channel count below 0x21. IPC bytes +are scratch 1 plus one; heap bytes are scratch 3; scratch 4 is a Boolean. +IPC and heap allocations are rounded to 16 KiB and bounded before use. +The IPC allocation at ``0x100003d0000`` must fit below the fixed heap base +``0x100003e4000``. The heap has a 16 MiB host-policy limit and must also fit +below shared allocations. These limits are not firmware capability claims. + +The 0x174-byte init record at pool + 0x180 has a zero first qword, IPC DVA, +0x380000 configured span, the corresponding 0xfc80000 remainder, requested +heap DVA/bytes, unrounded IPC bytes and pool DVA/bytes. Reserved bytes and +the cold-start retained value are zero. Template count is 64 and record ++0x12c is 4. A DMA write barrier precedes publication to scratch 0 low, +scratch 1 high, then scratch 7 ``0xf7fbdff9``. The reply must contain both +READY and the exact IPC base; unchanged init-record addresses time out. +The shared transport validates the table before arming any type-0 ring. +Only then does the host write READY to scratch 3 and wait for zero. + +``startup`` sends only PRINT_ENABLE (0x4, 12 bytes), START (0x0, 12), +CONFIG_GET (0x3, 16) and RESOURCE_INFO_GET (0x22, 100) on IO. SHAREDMALLOC +allocation requests are serviced under the shared transport's bounded host +policy while waiting. The pinned H16 profile also acknowledges native FREE +for its own live allocations with a zero tuple. Backing storage and quota +stay pinned until reboot; retired allocations cannot authorize reports or +a second FREE. Foreign frees and control forms without a proved request +layout quarantine without an invented reply. Other firmware profiles retain +the default refusal of FREE/control. Each command keeps one absolute +deadline and requires a zero acknowledgment. The CONFIG_GET tick must be nonzero and is logged; +24 MHz is a comparison value, not a refusal predicate. All 100 resource-info +reply bytes are logged and do not grant allocation authority. + +Mailbox service uses 64-bit pop-on-read accesses and bounded, nonblocking +receive/transmit batches. Offered HELLO, EPMAP and power messages use the +shared version-12 parser. Unknown or incompatible messages are logged and +receive no invented response. A missing or stalled RTKit session never +blocks the scratch protocol. IRQ 1054 remains disabled through all stages. + +Before RUN, errors unwind mappings and genpd references with a rollback log. +Immediately before the first RUN write the backend pins the module. Every +later failure latches quarantine, dumps scratch 0 through 7 and DART fault +words, and retains mappings, pages and power. Probe stays bound on such an +error so automatic devres cleanup cannot reclaim firmware DMA. A successful +boot also stays pinned because no safe in-place shutdown/reset is proved. +Further attempts are refused until reboot. There is no writable stage knob. + +Hardware test plan +------------------ + +Use a matching kernel/module set and a DT that enables the ANE and its three +DARTs with SID 0 only, the two preload ranges and an aperture covering the +identity pages (``t6040-j616s-ane-test.dts``). Keep shipping DT nodes +disabled. Each write below is one stage; stop and preserve full dmesg at the +first unexpected value:: + + echo preflight > /sys/kernel/debug/ane-h16//stage + echo map > /sys/kernel/debug/ane-h16//stage + +Before RUN a failed stage rolls back and the module can be reloaded. +Expected preflight messages include ``H16 preload identity H16 27.0 26A428 +accepted``, ``H16 power ACTUAL`` twice for each of the five words with low +byte ff, ``H16 CPU control/RVBAR 0 10000000001``, DART status/address and +DAPF comparisons, then ``H16 stage preflight complete``. Map adds TEXT/DATA +mapping messages and four ``H16 identity MMIO`` messages, clean fault words +and ``H16 stage map complete``. A six-stream tree instead logs ``H16 map +requires only SID 0 on three DARTs; refusing extra streams``. + +The boot rung releases the firmware CPU:: + + echo boot > /sys/kernel/debug/ane-h16//stage + +Expect CE state/ack and ``H16 tunables applied`` (or locked-match) logs, +then ``H16 READY 8042006 0``, request logs (the comparator values are +channels 8, IPC-minus-one c440, heap f00000, mode 0), then +``H16 table validated 100003d0000 8``, ``H16 scratch 3 cleared 0 0`` and +``H16 stage boot complete; held until reboot``. Comparator sizing values are +not additional admission checks. From here the module stays loaded. + +In the same boot:: + + echo startup > /sys/kernel/debug/ane-h16//stage + +Expect SHAREDMALLOC allocation messages as offered, ``H16 startup ack`` for +IDs 4, 0, 3 and 22 with zero status, ``H16 CONFIG_GET tick`` with a nonzero +value, exactly 100 bytes under ``H16 RESOURCE_INFO`` and ``H16 stage startup +complete; held until reboot``. A failure after RUN must instead show the +scratch/fault dump and ``H16 quarantine``. Do not retry; collect logs and +reboot. No inference result is claimed by these diagnostic rungs. + +The ``ane-h16-boot`` KUnit suite uses a simulated engine, scratch cells, +mailbox polling and rings. It covers startup including allocation service, +missing READY, wrong/stale table publication, oversized IPC/heap requests, +channel bounds, locked DAPF differences, nonzero acks, scratch-3 timeout, +zero tick, default refusal and quarantine after RUN. It also covers matching +locked DAPF, heap/shared overlap, native FREE, foreign FREE and control refusal. + +H16 program stage +----------------- + +The ``program`` stage runs every startup step, then registers an accel node +with ABI version 3 (``DRM_ANE_CHIP_H16``). GET_CAPS, H16_LOAD and H16_EXEC +are its only ioctls. The node is absent at every other stage, so the default +refusal and the H17 refusal are unchanged. + +H16_LOAD takes the original HWX and four host-built sections: generic, +operation, procedure and text properties. The kernel imports the HWX once, +validates it with the HWX and task-stream checks, and plans whole-segment +mappings. Each region and each host-built section gets its own private +mapping in a program window above the SHAREDMALLOC range. Only the output +surface and scratch are device-writable. Userspace never supplies a DVA, +firmware ID, resource index or command. The LOAD_PROGRAM slots come from one +table: + +==== ========================== ======== ===== +Slot Source Index Class +==== ========================== ======== ===== +0 generic (host-built) 1002 4 +1 kernel section, else const 1007 3 +2 HWX task stream 1006 4 +3 operation (host-built) 1003 4 +4 procedure (host-built) 1004 4 +5 empty +6 text properties 1005 4 +7, 8 empty +==== ========================== ======== ===== + +Every populated slot has +0x10 = 1 and a full 64-bit DVA and size. The +request carries 0xffffffff at +0x1b8. The reply must echo every slot, keep a +zero acknowledgment and replace that word with the program ID. Zero is a +valid ID. CREATE_PROCESS (0x258 bytes) sends the program ID with a zero body. +Its reply must keep the body zero and returns the process ID, where zero is +also valid. A program with a kernel section must tag its operation record +with the kernel slot index 1007. The uapi header names these indices. + +Slot 1 carries the ``__KERN_0`` kernel section when the program has one and +otherwise the ``__TEXT,__const`` constants, which macOS loads in every +program. + +EXECUTE resources come from a kernel-owned map. The class follows the +resource usage mapping (inputs 0, the output 1). Each descriptor names its +buffer by the ID of the generic section that LOAD installs: generic entry +k describes surface k in HWX surface order, and the kernel refuses a generic +section whose count, direction, type or size disagrees with the program, +that repeats an ID or that uses the reserved IDs 2 and 3. The mcache tuple in the linear program +descriptor (BAR ID, bit offset and width) adds one custom-BAR record with +selector 1, value type 0 and the first pool DSID, 36. The scratch form sends +none. + +Each H16_EXEC copies inputs into the private input mappings and poisons the +output mapping. It then sends 0x002d with 0, keeps the request UUID pending +before publishing EXECUTE on channel 7, and treats the channel-7 reply as +transport acknowledgment only. Completion is a state-1 0x0300 report whose +UUID, program, process and procedure IDs all match. Then 0x002d with 1 and +a DMA read barrier precede the output copy. A state-2 report, a mismatched +ID, a malformed report or a timeout quarantines the session. Reports with +an unknown UUID never complete a request. Closing the file sends TERMINATE +then UNLOAD. Mappings stay until reboot. + +One program lifetime per firmware boot is mandatory. On this firmware +family the first EXECUTE of a newly loaded program can write into the +previous program's output buffer when the buffer counts match, and a larger +output overruns that mapping. A second H16_LOAD therefore fails with -EBUSY +and the log line ``H16 one program per firmware boot; reboot required``, +also after the first program closed. The transport never publishes the +secure-mode commands 0x27, 0x28 and 0x34 or the exclave commands 0x30 and +0x31. + +The ``ane-h16-program`` KUnit suite covers add-128 and synthetic slot +packing, LOAD/CREATE/TERMINATE/UNLOAD bodies and replies, the deny list, +the custom-BAR record, a fake-firmware session with two executions and +teardown, the second-LOAD refusal, bad LOAD/CREATE replies, mismatched +UUIDs and IDs, and state-2 quarantine. +``tools/testing/selftests/drivers/ane/h16-run.py`` loads the pinned add-128 +HWX, runs it twice with poisoned output and compares with ``expected.bin``. +======= +For the experimental T8140 cold AP-direct stages and their ownership and +loader requirements, see :doc:`apple-ane-cold`. diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 025400ef93910..9c24d63e9c673 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -28,6 +28,10 @@ properties: - apple,t8103-usb4-dart - apple,t8110-dart - apple,t6000-dart + - items: + - const: apple,t8140-ane-dart + - const: apple,t8140-dart + - const: apple,t8110-dart - items: - enum: - apple,t6000-dart diff --git a/Documentation/devicetree/bindings/media/apple,avd.yaml b/Documentation/devicetree/bindings/media/apple,avd.yaml index 450af7ffa4559..4c3c045e69764 100644 --- a/Documentation/devicetree/bindings/media/apple,avd.yaml +++ b/Documentation/devicetree/bindings/media/apple,avd.yaml @@ -54,7 +54,7 @@ properties: minItems: 1 items: - description: decoder power state - - description: performance floor (PMC rail level) held while decoding + - description: performance floor (PMC rail or PMP device level) held while decoding power-domain-names: minItems: 1 diff --git a/Documentation/devicetree/bindings/npu/apple,ane.yaml b/Documentation/devicetree/bindings/npu/apple,ane.yaml index 3167780b88884..93d16e55fffe7 100644 --- a/Documentation/devicetree/bindings/npu/apple,ane.yaml +++ b/Documentation/devicetree/bindings/npu/apple,ane.yaml @@ -75,10 +75,11 @@ properties: iommus: description: - Stream 0 of each of the three DARTs, followed on T6041 by stream 11 of - each, which the ANE's MPM uses. + Stream 0 of each of the three DARTs. On T6041 the ANE's MPM also uses + stream 11 of each DART; the firmware drives that stream itself and it + is not described here. minItems: 3 - maxItems: 6 + maxItems: 3 power-domains: description: @@ -119,7 +120,39 @@ properties: description: Device addresses of the preloaded TEXT and DATA segments. Their sizes are 0xc0000 and 0x2ac000 bytes respectively. The bootloader supplies - this property together with the physical reservations. + the physical reservations. The device addresses are fixed by the + firmware layout and may be described statically. + + apple,dapf-ranges: + $ref: /schemas/types.yaml#/definitions/uint64-matrix + minItems: 4 + maxItems: 4 + items: + items: + - description: Physical start address, inclusive + - description: Physical end address, inclusive + description: + Four DART address protection filter (DAPF) ranges, in hardware record + order. Each pair contains inclusive 64-bit physical bounds, not a + base and length. The same records apply to all three ANE DARTs. + Firmware accesses these registers through identity mappings of the + containing 16 KiB pages in its private domain. These are MMIO ranges, + not allocatable DMA memory or additional preload segments. + + apple,dapf-ranges: + $ref: /schemas/types.yaml#/definitions/uint64-array + minItems: 8 + maxItems: 8 + description: Four inclusive SID0 physical MMIO start/end pairs + + apple,preload-data-sha256: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 32 + maxItems: 32 + description: + Bootloader-produced digest of the live reserved DATA including all + personalized parameter bytes. This establishes preservation of that + boot's DATA, not a cross-build firmware identity. TEXT has a driver pin. apple,ane-type: $ref: /schemas/types.yaml#/definitions/uint32 @@ -185,19 +218,18 @@ allOf: then: properties: reg: - maxItems: 6 + minItems: 3 + maxItems: 3 reg-names: - minItems: 1 items: - const: engine - const: pmgr - - const: aux-0 - - const: aux-1 - - const: aux-2 - - const: aux-3 + - const: pwgate + required: + - apple,dapf-ranges - if: required: - - apple,firmware-iovas + - apple,preload-data-sha256 then: properties: memory-region: @@ -243,8 +275,6 @@ allOf: - const: aux-2 - const: aux-3 - const: aux-4 - iommus: - minItems: 6 power-domains: minItems: 5 maxItems: 5 @@ -253,10 +283,7 @@ allOf: required: - apple,ane-type - apple,firmware-preloaded - else: - properties: - iommus: - maxItems: 3 + - apple,dapf-ranges - if: properties: compatible: @@ -293,6 +320,7 @@ allOf: apple,ane-type: false apple,firmware-preloaded: false apple,exclave-assigned: false + apple,dapf-ranges: false required: - interrupt-names @@ -370,13 +398,19 @@ examples: ane@400000000 { compatible = "apple,t8140-ane"; - reg = <0x4 0x0 0x0 0x2000000>; - reg-names = "engine"; + reg = <0x4 0x0 0x0 0x2000000>, + <0x3 0x00700000 0x0 0x18000>, + <0x3 0x00726680 0x0 0x34>; + reg-names = "engine", "pmgr", "pwgate"; interrupt-parent = <&aic>; interrupts = , ; iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; power-domains = <&ps_ane_sys>; + apple,dapf-ranges = /bits/ 64 <0x321668000 0x321668003 + 0x30070c000 0x30070c013 + 0x300700290 0x300700293 + 0x482668000 0x482668003>; status = "disabled"; }; }; @@ -428,3 +462,40 @@ examples: memory-region = <&ane_alias_iova>; }; }; + + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + ane@484000000 { + compatible = "apple,t6041-ane"; + reg = <0x4 0x84000000 0x0 0x2000000>, + <0x5 0x02280000 0x0 0x18000>, + <0x5 0x022a0000 0x0 0x4000>, + <0x5 0x003c0000 0x0 0x30000>, + <0x2 0x12000000 0x0 0xff4000>, + <0x5 0x22000000 0x0 0x2000000>, + <0x5 0x08374000 0x0 0x4000>; + reg-names = "engine", "pmgr", "aux-0", "aux-1", "aux-2", + "aux-3", "aux-4"; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_sys>, <&ps_ane_cpu>, <&ps_ane_mpm>, + <&ps_ane_td>, <&ps_ane_base>; + apple,ane-type = <0x110>; + apple,firmware-preloaded; + apple,dapf-ranges = /bits/ 64 <0x5022803c0 0x5022803c3>, + /bits/ 64 <0x50228c000 0x50228c01b>, + /bits/ 64 <0x581668000 0x581668003>, + /bits/ 64 <0x28a668000 0x28a668003>; + memory-region = <&ane_fw_text>, <&ane_fw_data>; + memory-region-names = "firmware-text", "firmware-data"; + apple,firmware-iovas = /bits/ 64 <0x10000000000 0x100000c0000>; + status = "disabled"; + }; + }; diff --git a/Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml b/Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml index 2d468e3c302c6..f0581393dbcd9 100644 --- a/Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml +++ b/Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml @@ -20,15 +20,36 @@ properties: const: apple,smc-rtc nvmem-cells: + minItems: 1 items: - description: RTC offset. By default a 48bit value, specified in 32768 (2^15) Hz clock ticks. With apple,rtc-offset-seconds, a 64bit little-endian count of whole seconds. + - description: + PMU alarm control byte, with the alarm enable, the monitor bit and + the latched alarm event. + - description: + PMU alarm compare value, a 32-bit count of whole seconds of the RTC + counter without the offset. + - description: PMU alarm mask byte. nvmem-cell-names: - items: - - const: rtc_offset + oneOf: + - items: + - const: rtc_offset + - items: + - const: rtc_offset + - const: rtc_ctrl + - const: rtc_alarm + - const: rtc_alarm_mask + + interrupts: + maxItems: 1 + description: + The PMU's RTC alarm interrupt. Only used with the three alarm cells. + + wakeup-source: true apple,rtc-offset-seconds: type: boolean diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index a8fa08c4e3893..3fd4a1c81853e 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -30,8 +30,11 @@ bootstrap DMA ownership is active. Kernels may still support kexec elsewhere. The station uses 2.4 GHz channels 1-13 and the 5 GHz channels 36-64, 100-144 and 149-165 that the cfg80211 regulatory domain permits. Radar (DFS) channels stay passive: the firmware listens on them while the station is not -associated, but never probes or joins an access point there, and scans made -while associated skip them. A channel whose restrictions the driver cannot +associated and never probes there, and scans made while associated skip them. +An access point heard on a radar channel may be joined, since the access point +owns radar detection. A beacon from it that carries a Channel Switch +Announcement or an Extended one ends the connection, or the join in progress, +and logs RADAR_CHANNEL_SWITCH. A channel whose restrictions the driver cannot apply (reduced power, no OFDM, PSD limits) is disabled rather than used, and so is a channel the country package does not permit: every power limit of that channel in the applied country package is left undefined. @@ -122,21 +125,27 @@ permit security fallback. For SAE, userspace supplies SAE authentication, an 8-63 byte password and required or optional management frame protection (MFP). Firmware uses required MFP for the SAE arm. On a PSK+SAE transition network, the first AKM supplied -by userspace selects the arm: PSK keeps the existing binary-PMK request and -disables MFP, while SAE uses the password and required MFP. PSK with required -MFP is rejected on legacy AKM 2. WPA2 PMF uses the explicitly selected -PSK-SHA256 AKM (6), with a binary 32-byte PMK and optional or required MFP. -The only EA/40 security delta is little-endian 0x0400 at body +0x14. The -host retains the MFP policy; there is no standalone optional/required field. +by userspace selects the arm: PSK keeps the binary-PMK request, while SAE +uses the password and required MFP. On the PSK (AKM 2) arm the firmware sets +MFPC in its own RSN element and negotiates PMF with any AP that sets MFPC, so +a WPA2 network with optional or required PMF is joined with PMF on. iwd and +wpa_supplicant both ask for required MFP once they have negotiated PMF with +such an AP, and wpa_supplicant asks for optional MFP when either outcome will +do; every MFP setting is accepted on this arm. WPA2 PMF with the PSK-SHA256 AKM (6) +also takes a binary 32-byte PMK and optional or required MFP; its only EA/40 +security delta is little-endian 0x0400 at body +0x14. There is no standalone +optional/required field. Before any association command, the host validates the candidate's RSN IE, -selected AKM, CCMP and MFPC/MFPR. A required request cannot downgrade to -AKM 2 or a non-MFPC AP. Only BIP-CMAC-128 is accepted as an explicit group +selected AKM, CCMP and MFPC/MFPR. A request for PMF cannot join an AP +without MFPC, and a request with MFP disabled cannot join an AP that sets +MFPR. With MFP disabled, the AKM 2 arm still negotiates PMF when the AP +offers it, because the firmware always sets MFPC there. Only BIP-CMAC-128 +is accepted as an explicit group management suite; the standard omitted-suite default is also CMAC-128. Firmware owns IGTK extraction, install/removal and BIP verification. No -add_key or set_default_mgmt_key operation is added. iwd's BIP gate is now -satisfied, but WPA2-PMF and iwd behavior still require hardware qualification. -The admitted PMF state is host policy, not a firmware readback. Selectors +add_key or set_default_mgmt_key operation is added. The admitted PMF state +follows the AP's RSN capabilities; it is not a firmware readback. Selectors 0x40/0x42 retain their normal completion contract; neither reports PMF state. The standard SAE PWE settings (unspecified, hunting-and-pecking, @@ -153,6 +162,14 @@ transition network has confirmed, on 2.4 GHz channel 6 and 5 GHz channel 149: joins the transition network as PSK; * repeated reconnection through NetworkManager after each change of profile. +With iwd 3.12 and with NetworkManager and wpa_supplicant, against hostapd +access points, every network that offers PMF is joined with it (hostapd +lists the station with MFP): WPA2 with optional PMF and with required PMF on +AKM 2, WPA2 with required PMF on AKM 6, the WPA2/WPA3 transition network and +WPA3-only networks on 5 GHz, and WPA3 on 6 GHz. OWE with iwd needs an iwd +that knows ``NL80211_EXT_FEATURE_OWE_OFFLOAD``; iwd 3.12 runs its own OWE +exchange, which a firmware-offload device cannot serve. + The request layout, transition admission and PMK derivation also have automated tests. A pure WPA3 network, exact PWE policy behavior, WPA2 throughput regression comparisons and long-duration reliability still need diff --git a/MAINTAINERS b/MAINTAINERS index 2e0aab08c5d6f..8b47b86359ee9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2695,6 +2695,7 @@ F: drivers/dma/apple-sio-probe-test.dtso F: drivers/dma/apple-sio-t6030-test.dtso F: drivers/dma/apple-sio-test.c F: drivers/dma/apple-sio.c +F: include/linux/dma/apple-sio.h F: sound/soc/apple/* F: sound/soc/codecs/cs42l83-i2c.c F: sound/soc/codecs/cs42l84.* @@ -8190,8 +8191,10 @@ L: dri-devel@lists.freedesktop.org S: Maintained F: Documentation/ABI/testing/sysfs-class-accel-ane F: Documentation/ABI/testing/sysfs-driver-ane-performance +F: Documentation/accel/apple-ane-cold.rst F: Documentation/accel/apple-ane.rst F: Documentation/devicetree/bindings/npu/apple,ane.yaml +F: arch/arm64/boot/dts/apple/t8140-j700-ane-test.dts F: drivers/accel/ane/ F: include/uapi/drm/ane_accel.h F: tools/accel/ane-bench.py diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index 0f6e6714f7de7..f75d2692cb8e4 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -116,3 +116,6 @@ dtb-$(CONFIG_ARCH_APPLE) += t8132-j773g.dtb dtb-$(CONFIG_ARCH_APPLE) += t8140-j700.dtb dtb-$(CONFIG_ARCH_APPLE) += t8142-j813.dtb dtb-$(CONFIG_ARCH_APPLE) += t8152-j873g.dtb + +# Explicit cold-ANE lead test; never select as the production board DTB. +dtb-$(CONFIG_ARCH_APPLE) += t8140-j700-ane-test.dtb diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index ca735a7975bbc..a15e5d77a70f5 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -568,6 +568,7 @@ /* To be filled by the loader */ local-mac-address = [00 10 18 00 00 10]; apple,antenna-sku = "XX"; + brcm,ccode-map-trivial; }; bluetooth0: bluetooth@0,1 { diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index 2b16c93d410bd..07eb49ea25dfe 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -646,6 +646,7 @@ /* To be filled by the loader */ local-mac-address = [00 10 18 00 00 10]; apple,antenna-sku = "XX"; + brcm,ccode-map-trivial; }; bluetooth0: bluetooth@0,1 { diff --git a/arch/arm64/boot/dts/apple/t600x-pciec.dtsi b/arch/arm64/boot/dts/apple/t600x-pciec.dtsi index c58c66eda9bc9..2d22b78f084c8 100644 --- a/arch/arm64/boot/dts/apple/t600x-pciec.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pciec.dtsi @@ -120,7 +120,7 @@ bus-range = <0x0 0x7f>; #address-cells = <3>; #size-cells = <2>; - ranges = <0x43000000 0x8 0x0 + ranges = <0x43000000 0xc 0x0 0xc 0x0 0x2 0x0>, <0x02000000 0x0 0x00100000 0xe 0x00100000 0x0 0x3ff00000>, @@ -191,7 +191,7 @@ bus-range = <0x0 0x7f>; #address-cells = <3>; #size-cells = <2>; - ranges = <0x43000000 0x8 0x0 + ranges = <0x43000000 0x10 0x0 0x10 0x0 0x2 0x0>, <0x02000000 0x0 0x00100000 0x12 0x00100000 0x0 0x3ff00000>, @@ -261,7 +261,7 @@ bus-range = <0x0 0x7f>; #address-cells = <3>; #size-cells = <2>; - ranges = <0x43000000 0x8 0x0 + ranges = <0x43000000 0x14 0x0 0x14 0x0 0x2 0x0>, <0x02000000 0x0 0x00100000 0x16 0x00100000 0x0 0x3ff00000>, diff --git a/arch/arm64/boot/dts/apple/t6022-j180d.dts b/arch/arm64/boot/dts/apple/t6022-j180d.dts index cbfa09273c41a..91677000cc120 100644 --- a/arch/arm64/boot/dts/apple/t6022-j180d.dts +++ b/arch/arm64/boot/dts/apple/t6022-j180d.dts @@ -1004,6 +1004,7 @@ apple,antenna-sku = "XX"; /* To be filled by the loader */ local-mac-address = [00 10 18 00 00 10]; + brcm,ccode-map-trivial; }; bluetooth0: network@0,1 { diff --git a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi index 831e1dfa35dfa..cb4907704b057 100644 --- a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi @@ -838,7 +838,7 @@ bus-range = <0x0 0x7f>; #address-cells = <3>; #size-cells = <2>; - ranges = <0x43000000 0x8 0x0 + ranges = <0x43000000 0xc 0x0 0xc 0x0 0x2 0x0>, <0x02000000 0x0 0x00100000 0xe 0x00100000 0x0 0x3ff00000>, @@ -1065,7 +1065,7 @@ bus-range = <0x0 0x7f>; #address-cells = <3>; #size-cells = <2>; - ranges = <0x43000000 0x8 0x0 + ranges = <0x43000000 0x10 0x0 0x10 0x0 0x2 0x0>, <0x02000000 0x0 0x00100000 0x12 0x00100000 0x0 0x3ff00000>, @@ -1290,7 +1290,7 @@ bus-range = <0x0 0x7f>; #address-cells = <3>; #size-cells = <2>; - ranges = <0x43000000 0x8 0x0 + ranges = <0x43000000 0x14 0x0 0x14 0x0 0x2 0x0>, <0x02000000 0x0 0x00100000 0x16 0x00100000 0x0 0x3ff00000>, diff --git a/arch/arm64/boot/dts/apple/t6030.dtsi b/arch/arm64/boot/dts/apple/t6030.dtsi index 685b67606dc87..59deb4d59aa95 100644 --- a/arch/arm64/boot/dts/apple/t6030.dtsi +++ b/arch/arm64/boot/dts/apple/t6030.dtsi @@ -836,6 +836,8 @@ #clock-cells = <1>; }; + /* The display handoff gate installs PMP unless explicitly static. */ + #ifdef APPLE_USE_PMP pmp_dart: iommu@350300000 { compatible = "apple,t6030-dart", "apple,t8110-dart"; reg = <0x3 0x50300000 0x0 0x4000>; @@ -985,6 +987,8 @@ power-domains = <&ps_pmp>, <&ps_pms_sram>; }; + #endif /* APPLE_USE_PMP */ + aic: interrupt-controller@351000000 { compatible = "apple,t6030-aic3", "apple,t8122-aic3"; #interrupt-cells = <3>; @@ -1056,6 +1060,8 @@ #size-cells = <0>; interrupt-parent = <&aic>; interrupts = ; + interrupt-controller; + #interrupt-cells = <2>; pmic1: pmic@b { compatible = "apple,chapel-pmic", "apple,spmi-nvmem"; @@ -1066,10 +1072,22 @@ #address-cells = <1>; #size-cells = <1>; + rtc_ctrl: rtc-ctrl@2000 { + reg = <0x2000 0x1>; + }; + pm_setting: pm-setting@2001 { reg = <0x2001 0x1>; }; + rtc_alarm: rtc-alarm@2008 { + reg = <0x2008 0x4>; + }; + + rtc_alarm_mask: rtc-alarm-mask@200e { + reg = <0x200e 0x1>; + }; + rtc_offset: rtc-offset@2100 { reg = <0x2100 0x6>; }; @@ -1178,8 +1196,13 @@ rtc { compatible = "apple,smc-rtc"; - nvmem-cells = <&rtc_offset>; - nvmem-cell-names = "rtc_offset"; + nvmem-cells = <&rtc_offset>, <&rtc_ctrl>, <&rtc_alarm>, + <&rtc_alarm_mask>; + nvmem-cell-names = "rtc_offset", "rtc_ctrl", "rtc_alarm", + "rtc_alarm_mask"; + /* The PMIC's RTC alarm, peripheral interrupt 2 of nub-spmi0 */ + interrupts-extended = <&nub_spmi 2 IRQ_TYPE_EDGE_RISING>; + wakeup-source; }; }; diff --git a/arch/arm64/boot/dts/apple/t6031-dcp.dtsi b/arch/arm64/boot/dts/apple/t6031-dcp.dtsi index 6d3b53cd95e57..f679c57b97709 100644 --- a/arch/arm64/boot/dts/apple/t6031-dcp.dtsi +++ b/arch/arm64/boot/dts/apple/t6031-dcp.dtsi @@ -9,9 +9,10 @@ * interrupts and DART windows are those of the J516C ADT. * * Every node stays status = "disabled". The display gate enables the DCP, - * its mailbox, the two DARTs and the display subsystem only when - * apple_t6031_display.enable=1 is on the kernel command line and the boot - * loader has set apple,t6031-handoff = <1>. Compatibles are t6031-specific, + * its mailbox, the two DARTs and the display subsystem only when the boot + * loader has set apple,t6031-handoff = <1> and the native display is + * requested: by default on the J516C, with apple_t6031_display.enable=1 on + * the other boards, never with apple_t6031_display.enable=0. Compatibles are t6031-specific, * so a kernel that does not know them leaves the nodes unbound. * * Only the T6031 board files include this file: t6034.dtsi includes diff --git a/arch/arm64/boot/dts/apple/t603x-j514-j516.dtsi b/arch/arm64/boot/dts/apple/t603x-j514-j516.dtsi index 74fd80bc5af71..c94ac3fd08a26 100644 --- a/arch/arm64/boot/dts/apple/t603x-j514-j516.dtsi +++ b/arch/arm64/boot/dts/apple/t603x-j514-j516.dtsi @@ -231,6 +231,7 @@ /* To be filled by the loader */ local-mac-address = [00 10 18 00 00 10]; apple,antenna-sku = "XX"; + brcm,ccode-map-trivial; }; bluetooth0: bluetooth@0,1 { diff --git a/arch/arm64/boot/dts/apple/t603x-j514s-j516s-usb4.dtsi b/arch/arm64/boot/dts/apple/t603x-j514s-j516s-usb4.dtsi index b6c397e17c905..8aae684af663a 100644 --- a/arch/arm64/boot/dts/apple/t603x-j514s-j516s-usb4.dtsi +++ b/arch/arm64/boot/dts/apple/t603x-j514s-j516s-usb4.dtsi @@ -70,3 +70,42 @@ }; }; +/* + * DPA2: the Type-C DisplayPort processor's (dcpext1) audio, SIO channel 0x68, + * as the ADT links dcpext1 to dp-audio2. + */ +&dcpext1 { + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dcpext1_audio: endpoint { + remote-endpoint = <&dpaudio2_dcpext1>; + }; + }; + }; +}; + +/* DPA2, like DPA1, has no AP-side registers: not a child of the SoC bus. */ +/ { + dpaudio2: audio-controller-2 { + compatible = "apple,t6030-dpaudio", "apple,dpaudio"; + dmas = <&sio 0x68>; + dma-names = "tx"; + power-domains = <&ps_dpa2>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dpaudio2_dcpext1: endpoint { + remote-endpoint = <&dcpext1_audio>; + }; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index 25cded4aa83df..af3c800c577f5 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -55,6 +55,27 @@ ranges; /* The loader adds its own reservations */ + /* + * The loader must fill reg from ane0 segment-ranges, rename each + * node to match its physical address and enable it. Leave these + * disabled until then; allocating replacement memory would lose + * the personalized preload. TEXT is 0xc0000 bytes and DATA is + * 0x2ac000 bytes. Neither segment may be reclaimed before reboot. + */ + ane_fw_text: ane-firmware-text@0 { + compatible = "apple,asc-mem"; + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + ane_fw_data: ane-firmware-data@0 { + compatible = "apple,asc-mem"; + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + /* * iBoot loads the ISP firmware below the kernel's memory. Its * segments are the ADT's isp0 segment-ranges (macOS 26.6.2, @@ -4730,3 +4751,25 @@ #undef J616S_DELETE_FAN_TRIPS #undef J616S_DELETE_FAN_MAPS + +/* + * Proven on the J616s (MobileNetV1 top-1 16/16). The ANE stays idle until + * the loader fills and enables the two preload reservations above. + */ +&ane0_dart0 { + status = "okay"; +}; + +&ane0_dart1 { + status = "okay"; +}; + +&ane0_dart2 { + status = "okay"; +}; + +&ane { + memory-region = <&ane_fw_text>, <&ane_fw_data>; + memory-region-names = "firmware-text", "firmware-data"; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi index e0af3e1903cfb..735df142bd0eb 100644 --- a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi @@ -536,7 +536,6 @@ label = "fab4_soc"; power-domains = <&ps_afc>; apple,always-on; - status = "disabled"; }; ps_sio_cpu: power-controller@130 { @@ -1057,7 +1056,6 @@ #reset-cells = <0>; label = "ane_sys"; power-domains = <&ps_fab5_soc>, <&ps_fab4_soc>; - status = "disabled"; }; }; @@ -1835,7 +1833,6 @@ label = "fab5_soc"; power-domains = <&ps_afi>; apple,always-on; - status = "disabled"; }; ps_sbr: power-controller@448 { @@ -1949,7 +1946,6 @@ #reset-cells = <0>; label = "ane_mpm"; power-domains = <&ps_ane_sys>; - status = "disabled"; }; ps_ane_cpu: power-controller@8 { @@ -1959,7 +1955,6 @@ #reset-cells = <0>; label = "ane_cpu"; power-domains = <&ps_ane_sys>; - status = "disabled"; }; ps_ane_td: power-controller@10 { @@ -1969,7 +1964,6 @@ #reset-cells = <0>; label = "ane_td"; power-domains = <&ps_ane_sys>; - status = "disabled"; }; ps_ane_base: power-controller@18 { @@ -1979,7 +1973,6 @@ #reset-cells = <0>; label = "ane_base"; power-domains = <&ps_ane_td>; - status = "disabled"; }; }; diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 50ad177d4f17d..0389dfcbbeab3 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -2000,7 +2000,8 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - apple,dma-range = <0x100 0x0 0x300 0x0>; + /* Include firmware identity MMIO pages below the preload DVAs. */ + apple,dma-range = <0x0 0x0 0x400 0x0>; status = "disabled"; }; @@ -2011,7 +2012,8 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - apple,dma-range = <0x100 0x0 0x300 0x0>; + /* Include firmware identity MMIO pages below the preload DVAs. */ + apple,dma-range = <0x0 0x0 0x400 0x0>; status = "disabled"; }; @@ -2022,7 +2024,8 @@ interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - apple,dma-range = <0x100 0x0 0x300 0x0>; + /* Include firmware identity MMIO pages below the preload DVAs. */ + apple,dma-range = <0x0 0x0 0x400 0x0>; status = "disabled"; }; @@ -2251,7 +2254,12 @@ status = "disabled"; }; - /* Firmware interface support is required before enabling the ANE. */ + /* + * Disabled until the H16 firmware path is proven on hardware. The + * loader fills the preload reservations and the firmware IOVAs from + * the ADT, so enabling it then only takes status = "okay" here and on + * the three DARTs. + */ ane: ane@484000000 { compatible = "apple,t6041-ane"; reg = <0x4 0x84000000 0x0 0x2000000>, @@ -2265,13 +2273,20 @@ "aux-3", "aux-4"; interrupt-parent = <&aic>; interrupts = ; - iommus = <&ane0_dart0 0>, <&ane0_dart1 0>, <&ane0_dart2 0>, - <&ane0_dart0 11>, <&ane0_dart1 11>, <&ane0_dart2 11>; + /* Stream 0 only: the MPM stream 11 is not given to the host. */ + iommus = <&ane0_dart0 0>, <&ane0_dart1 0>, <&ane0_dart2 0>; power-domains = <&ps_ane_sys>, <&ps_ane_cpu>, <&ps_ane_mpm>, <&ps_ane_td>, <&ps_ane_base>; apple,ane-type = <0x110>; apple,firmware-preloaded; - /* TEXT/DATA reservations and IOVAs must come from iBoot. */ + /* Inclusive DAPF bounds, in hardware record order. */ + apple,dapf-ranges = /bits/ 64 <0x5022803c0 0x5022803c3>, + /bits/ 64 <0x50228c000 0x50228c01b>, + /bits/ 64 <0x581668000 0x581668003>, + /bits/ 64 <0x28a668000 0x28a668003>; + /* Default firmware layout; the loader rewrites it from the ADT. */ + apple,firmware-iovas = /bits/ 64 <0x10000000000 0x100000c0000>; + /* The bootloader supplies the physical TEXT/DATA reservations. */ status = "disabled"; }; diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi index 627c8cac8d15c..77485eb85b0b3 100644 --- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi @@ -168,7 +168,7 @@ &display { iommus = <&disp0_dart 0 0x0 0x0 0x0 0xfc000000>, - <&dispext0_dart 0 0x0 0x0 0xf 0xffff0000>; + <&dispext0_dart 0 0x0 0x0 0x0 0xfc000000>; }; &dispext0_dart { @@ -374,6 +374,7 @@ /* To be filled by the loader */ local-mac-address = [00 00 00 00 00 00]; apple,antenna-sku = "XX"; + brcm,ccode-map-trivial; }; bluetooth0: bluetooth@0,1 { diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 6a9f4f00f1aae..3ae5a351d55f3 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -1618,7 +1618,7 @@ #size-cells = <2>; piodma { - iommus = <&dispext0_dart 4 0x0 0x0 0xf 0xffff0000>; + iommus = <&dispext0_dart 4 0x0 0x0 0x0 0xfc000000>; }; ports { diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 8f6f3c6948710..86314d4ecc0ca 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -68,6 +68,7 @@ local-mac-address = [00 10 18 00 00 10]; apple,antenna-sku = "XX"; brcm,board-type = "apple,hokkaido"; + brcm,ccode-map-trivial; }; bluetooth0: bluetooth@0,1 { diff --git a/arch/arm64/boot/dts/apple/t8112-j415.dts b/arch/arm64/boot/dts/apple/t8112-j415.dts index cdcb4802396b5..3deb986fcfe98 100644 --- a/arch/arm64/boot/dts/apple/t8112-j415.dts +++ b/arch/arm64/boot/dts/apple/t8112-j415.dts @@ -68,6 +68,7 @@ local-mac-address = [00 10 18 00 00 10]; apple,antenna-sku = "XX"; brcm,board-type = "apple,snake"; + brcm,ccode-map-trivial; }; bluetooth0: bluetooth@0,1 { diff --git a/arch/arm64/boot/dts/apple/t8112-j473.dts b/arch/arm64/boot/dts/apple/t8112-j473.dts index 46992c6868411..e66816a66acad 100644 --- a/arch/arm64/boot/dts/apple/t8112-j473.dts +++ b/arch/arm64/boot/dts/apple/t8112-j473.dts @@ -175,6 +175,7 @@ local-mac-address = [00 10 18 00 00 10]; apple,antenna-sku = "XX"; brcm,board-type = "apple,miyake"; + brcm,ccode-map-trivial; }; bluetooth0: bluetooth@0,1 { diff --git a/arch/arm64/boot/dts/apple/t8112-j493.dts b/arch/arm64/boot/dts/apple/t8112-j493.dts index ca6a3ce364ec1..c8009d3e7024c 100644 --- a/arch/arm64/boot/dts/apple/t8112-j493.dts +++ b/arch/arm64/boot/dts/apple/t8112-j493.dts @@ -115,6 +115,7 @@ local-mac-address = [00 00 00 00 00 00]; apple,antenna-sku = "XX"; brcm,board-type = "apple,kyushu"; + brcm,ccode-map-trivial; }; bluetooth0: bluetooth@0,1 { diff --git a/arch/arm64/boot/dts/apple/t8122-j613.dts b/arch/arm64/boot/dts/apple/t8122-j613.dts index 3768661807e96..dab87a037b4f6 100644 --- a/arch/arm64/boot/dts/apple/t8122-j613.dts +++ b/arch/arm64/boot/dts/apple/t8122-j613.dts @@ -13,6 +13,7 @@ #include "t8122-usbpd-spmi.dtsi" #include "t8122-jxxx.dtsi" #include +#include / { compatible = "apple,j613", "apple,t8122", "apple,arm-platform"; @@ -34,6 +35,321 @@ default-state = "keep"; }; }; + + /* + * The J613 is fanless: the CPUs are the only cooling devices. As on + * the J700, every die sensor gets a zone that throttles both clusters, + * since they share one die and one heat spreader, and the power + * allocator splits the budget by the energy models, with the E cluster + * weighted three to one. The E maps stop at cooling state 3 (2004 MHz): + * the E cluster adds at most about 2 W, and the governor would + * otherwise send an idle E cluster to its lowest state. + * + * The trips are the J700's: switch-on 58 C (idle is 43-52 C), control + * 95 C, critical 110 C. sustainable-power comes from load tests with the + * gains derived from it: with the four P cores busy, the J700's 1.5 W + * held the hottest sensor at 72 C and the P cores at 2.1 GHz; 6 W held + * 90-95 C at 2.6-3.0 GHz (one 96.5 C peak); 7.5 W swung between 2.2 and + * 3.6 GHz with peaks of 98 C. With 6 W, all eight cores busy settled at + * 92 C with the P cores at 2.2-2.4 GHz and the E cores at full clock; + * the P cores with the GPU busy at 618 MHz peaked at 97 C (the GPU is + * not a cooling device here). + */ + thermal-zones { + cpu-e0-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 0>; /* Te05 */ + sustainable-power = <6000>; + + trips { + cpu_e0_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_e0_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_e0_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_e0_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; + contribution = <3072>; + }; + }; + }; + + cpu-e1-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 1>; /* Te0P */ + sustainable-power = <6000>; + + trips { + cpu_e1_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_e1_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_e1_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_e1_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; + contribution = <3072>; + }; + }; + }; + + cpu-p0-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 2>; /* Tp05 */ + sustainable-power = <6000>; + + trips { + cpu_p0_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p0_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p0_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p0_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; + contribution = <3072>; + }; + }; + }; + + cpu-p1-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 3>; /* Tp0L */ + sustainable-power = <6000>; + + trips { + cpu_p1_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p1_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p1_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p1_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; + contribution = <3072>; + }; + }; + }; + + cpu-p2-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 4>; /* Tp0b */ + sustainable-power = <6000>; + + trips { + cpu_p2_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p2_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p2_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p2_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; + contribution = <3072>; + }; + }; + }; + + cpu-p3-thermal { + polling-delay-passive = <500>; + polling-delay = <500>; + thermal-sensors = <&smc_hwmon 5>; /* Tp0r */ + sustainable-power = <6000>; + + trips { + cpu_p3_alert: trip-alert { + temperature = <58000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_p3_control: trip-control { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + trip-critical { + temperature = <110000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_p3_control>; + cooling-device = + <&cpu_p0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_p3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + contribution = <1024>; + }; + + map1 { + trip = <&cpu_p3_control>; + cooling-device = + <&cpu_e0 THERMAL_NO_LIMIT 3>, + <&cpu_e1 THERMAL_NO_LIMIT 3>, + <&cpu_e2 THERMAL_NO_LIMIT 3>, + <&cpu_e3 THERMAL_NO_LIMIT 3>; + contribution = <3072>; + }; + }; + }; + }; }; &dcp { @@ -347,3 +663,16 @@ }; }; }; + +/* + * The base M3 has no Tf keys. With only the GPU busy, the six Tg keys below + * rose by about 50 C, the most of any key, and each has a twin about 4.5 C + * lower; they read 0 C or slightly below while the GPU is powered off. The + * others are the CPU die keys of the thermal zones above (the E-cluster + * groups and one per P core), so the zone also follows a hot CPU. + */ +&smc_hwmon { + apple,soc-die-keys = "Tg0D", "Tg0P", "Tg0X", "Tg0b", "Tg0j", "Tg0v", + "Te05", "Te0P", "Tp05", "Tp0L", "Tp0b", "Tp0r"; + apple,thermal-pressure-key = "mTPL"; +}; diff --git a/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi index 5c0051a652631..b7739c2558d5a 100644 --- a/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi @@ -97,6 +97,7 @@ /* To be filled by the loader */ local-mac-address = [00 00 00 00 00 00]; apple,antenna-sku = "XX"; + brcm,ccode-map-trivial; }; bluetooth0: bluetooth@0,1 { diff --git a/arch/arm64/boot/dts/apple/t8122.dtsi b/arch/arm64/boot/dts/apple/t8122.dtsi index 3e190bd9c4317..fb3f3cca9e0d4 100644 --- a/arch/arm64/boot/dts/apple/t8122.dtsi +++ b/arch/arm64/boot/dts/apple/t8122.dtsi @@ -74,6 +74,7 @@ operating-points-v2 = <&sawtooth_opp>; capacity-dmips-mhz = <804>; performance-domains = <&cpufreq_e>; + #cooling-cells = <2>; }; cpu_e1: cpu@1 { @@ -88,6 +89,7 @@ operating-points-v2 = <&sawtooth_opp>; capacity-dmips-mhz = <804>; performance-domains = <&cpufreq_e>; + #cooling-cells = <2>; }; cpu_e2: cpu@2 { @@ -102,6 +104,7 @@ operating-points-v2 = <&sawtooth_opp>; capacity-dmips-mhz = <804>; performance-domains = <&cpufreq_e>; + #cooling-cells = <2>; }; cpu_e3: cpu@3 { @@ -116,6 +119,7 @@ operating-points-v2 = <&sawtooth_opp>; capacity-dmips-mhz = <804>; performance-domains = <&cpufreq_e>; + #cooling-cells = <2>; }; cpu_p0: cpu@10100 { @@ -130,6 +134,7 @@ operating-points-v2 = <&everest_opp>; capacity-dmips-mhz = <1024>; performance-domains = <&cpufreq_p>; + #cooling-cells = <2>; }; cpu_p1: cpu@10101 { @@ -144,6 +149,7 @@ operating-points-v2 = <&everest_opp>; capacity-dmips-mhz = <1024>; performance-domains = <&cpufreq_p>; + #cooling-cells = <2>; }; cpu_p2: cpu@10102 { @@ -158,6 +164,7 @@ operating-points-v2 = <&everest_opp>; capacity-dmips-mhz = <1024>; performance-domains = <&cpufreq_p>; + #cooling-cells = <2>; }; cpu_p3: cpu@10103 { @@ -172,6 +179,7 @@ operating-points-v2 = <&everest_opp>; capacity-dmips-mhz = <1024>; performance-domains = <&cpufreq_p>; + #cooling-cells = <2>; }; l2_cache_0: l2-cache-0 { @@ -189,6 +197,22 @@ }; }; + /* + * opp-microwatt is the power one busy CPU adds at each state, measured + * on a J613 with the SMC's cluster power keys (PPbb for the E cluster, + * PP0b for the P cluster): the cluster pinned to the state, all four of + * its CPUs busy minus the cluster idle, divided by four, starting + * below 60 C. The values are curves fitted to the measurements + * (rms deviation 31 mW for E, 22 mW for P), so that they rise + * monotonically: E P = 58 mW * f^1.842 (f in GHz), + * P ln(P/mW) = 5.394 + 1.164 ln f + 0.521 (ln f)^2. + * + * Above 3288 MHz the P cluster's power stops rising at about 11 W with + * all four CPUs busy, and the work done per clock at 4056 MHz is 8-10% + * lower than at 3420 MHz, also with one CPU busy: the hardware limits + * the top states by itself. Their values are the fitted curve + * extrapolated, the power these states would draw unlimited. + */ sawtooth_opp: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -197,36 +221,43 @@ opp-hz = /bits/ 64 <744000000>; opp-level = <1>; clock-latency-ns = <7800>; + opp-microwatt = <33600>; }; opp02 { opp-hz = /bits/ 64 <1044000000>; opp-level = <2>; clock-latency-ns = <19400>; + opp-microwatt = <62800>; }; opp03 { opp-hz = /bits/ 64 <1476000000>; opp-level = <3>; clock-latency-ns = <24800>; + opp-microwatt = <118800>; }; opp04 { opp-hz = /bits/ 64 <2004000000>; opp-level = <4>; clock-latency-ns = <43600>; + opp-microwatt = <208700>; }; opp05 { opp-hz = /bits/ 64 <2268000000>; opp-level = <5>; clock-latency-ns = <49200>; + opp-microwatt = <262100>; }; opp06 { opp-hz = /bits/ 64 <2556000000>; opp-level = <6>; clock-latency-ns = <53400>; + opp-microwatt = <326700>; }; opp07 { opp-hz = /bits/ 64 <2748000000>; opp-level = <7>; clock-latency-ns = <56800>; + opp-microwatt = <373300>; }; }; @@ -238,101 +269,121 @@ opp-hz = /bits/ 64 <696000000>; opp-level = <1>; clock-latency-ns = <9400>; + opp-microwatt = <154600>; }; opp02 { opp-hz = /bits/ 64 <1092000000>; opp-level = <2>; clock-latency-ns = <19800>; + opp-microwatt = <244800>; }; opp03 { opp-hz = /bits/ 64 <1356000000>; opp-level = <3>; clock-latency-ns = <23700>; + opp-microwatt = <329200>; }; opp04 { opp-hz = /bits/ 64 <1596000000>; opp-level = <4>; clock-latency-ns = <25000>; + opp-microwatt = <425000>; }; opp05 { opp-hz = /bits/ 64 <1884000000>; opp-level = <5>; clock-latency-ns = <28200>; + opp-microwatt = <567000>; }; opp06 { opp-hz = /bits/ 64 <2172000000>; opp-level = <6>; clock-latency-ns = <32300>; + opp-microwatt = <742700>; }; opp07 { opp-hz = /bits/ 64 <2424000000>; opp-level = <7>; clock-latency-ns = <35600>; + opp-microwatt = <928000>; }; opp08 { opp-hz = /bits/ 64 <2616000000>; opp-level = <8>; clock-latency-ns = <38200>; + opp-microwatt = <1091300>; }; opp09 { opp-hz = /bits/ 64 <2808000000>; opp-level = <9>; clock-latency-ns = <39700>; + opp-microwatt = <1275600>; }; opp10 { opp-hz = /bits/ 64 <2988000000>; opp-level = <10>; clock-latency-ns = <41700>; + opp-microwatt = <1469000>; }; opp11 { opp-hz = /bits/ 64 <3144000000>; opp-level = <11>; clock-latency-ns = <43700>; + opp-microwatt = <1654100>; }; opp12 { opp-hz = /bits/ 64 <3288000000>; opp-level = <12>; clock-latency-ns = <45000>; + opp-microwatt = <1840200>; }; opp13 { opp-hz = /bits/ 64 <3420000000>; opp-level = <13>; clock-latency-ns = <48700>; + opp-microwatt = <2024500>; }; opp14 { opp-hz = /bits/ 64 <3540000000>; opp-level = <14>; clock-latency-ns = <55700>; + opp-microwatt = <2204000>; }; opp15 { opp-hz = /bits/ 64 <3636000000>; opp-level = <15>; clock-latency-ns = <55900>; + opp-microwatt = <2356100>; }; opp16 { opp-hz = /bits/ 64 <3720000000>; opp-level = <16>; clock-latency-ns = <55900>; + opp-microwatt = <2495800>; }; opp17 { opp-hz = /bits/ 64 <3780000000>; opp-level = <17>; clock-latency-ns = <56000>; + opp-microwatt = <2599300>; }; opp18 { opp-hz = /bits/ 64 <3864000000>; opp-level = <18>; clock-latency-ns = <56000>; + opp-microwatt = <2749900>; }; opp19 { opp-hz = /bits/ 64 <3960000000>; opp-level = <19>; clock-latency-ns = <61800>; + opp-microwatt = <2930000>; }; opp20 { opp-hz = /bits/ 64 <4056000000>; opp-level = <20>; clock-latency-ns = <61900>; + opp-microwatt = <3119000>; }; }; @@ -1085,6 +1136,44 @@ smc_hwmon: hwmon { compatible = "apple,smc-hwmon"; + #thermal-sensor-cells = <1>; + + /* + * CPU die temperatures, one per E-cluster sensor + * group and one per P core (the key that rose most + * with only that core busy, on a J613). Board + * thermal zones refer to these by channel index, so + * keep them first and in this order. + */ + temperature-Te05 { + apple,key-id = "Te05"; + label = "E-cluster Temperature 1"; + }; + + temperature-Te0P { + apple,key-id = "Te0P"; + label = "E-cluster Temperature 2"; + }; + + temperature-Tp05 { + apple,key-id = "Tp05"; + label = "P-core 0 Temperature"; + }; + + temperature-Tp0L { + apple,key-id = "Tp0L"; + label = "P-core 1 Temperature"; + }; + + temperature-Tp0b { + apple,key-id = "Tp0b"; + label = "P-core 2 Temperature"; + }; + + temperature-Tp0r { + apple,key-id = "Tp0r"; + label = "P-core 3 Temperature"; + }; }; smc_reboot: reboot { diff --git a/arch/arm64/boot/dts/apple/t8140-j700-ane-test.dts b/arch/arm64/boot/dts/apple/t8140-j700-ane-test.dts new file mode 100644 index 0000000000000..efdea008a35cf --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8140-j700-ane-test.dts @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* One-shot lead test only. Loader reservations and cold evidence are required. + * This DTB deliberately does not manufacture either assertion. + */ +#include "t8140-j700.dts" + +&ane { + status = "okay"; +}; + +&ane_dart0 { + status = "okay"; +}; + +&ane_dart1 { + status = "okay"; +}; + +&ane_dart2 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index d25741151ea15..302a7939962bb 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -545,6 +545,20 @@ power-domains = <&ps_venc0_sys>, <&pmc_venc_cluster>; apple,dvfs = <0x198 2>; }; + + /* + * AVD is soc-device record 17, DVFS ordinal 10. Its VMAX + * request votes the SOC rail through the PMP's static agent. + * It has no cluster-agent bit: VENC-CLSTR-V is AVE-only. + */ + pmp_report_avd0: report@11 { + compatible = "apple,t8140-pmp-v2-report-entry"; + reg = <0x11>; + label = "pmp-avd0"; + #power-domain-cells = <0>; + power-domains = <&ps_avd_sys>; + apple,dvfs = <0x19a 2>; + }; }; pmp: pmp@300500000 { @@ -1397,7 +1411,8 @@ interrupt-parent = <&aic>; interrupts = , ; - power-domains = <&ps_avd_sys>; + power-domains = <&ps_avd_sys>, <&pmp_report_avd0>; + power-domain-names = "core", "perf"; resets = <&ps_avd_sys>; reg = <0x3 0x81080000 0x0 0x10000>, <0x3 0x81090000 0x0 0x10000>, @@ -1787,48 +1802,57 @@ }; ane_dart0: iommu@401800000 { - compatible = "apple,t8140-dart", "apple,t8110-dart"; + compatible = "apple,t8140-ane-dart", "apple,t8140-dart", + "apple,t8110-dart"; reg = <0x4 0x01800000 0x0 0xc000>; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - apple,dma-range = <0x100 0x0 0x300 0x0>; status = "disabled"; }; ane_dart1: iommu@401820000 { - compatible = "apple,t8140-dart", "apple,t8110-dart"; + compatible = "apple,t8140-ane-dart", "apple,t8140-dart", + "apple,t8110-dart"; reg = <0x4 0x01820000 0x0 0x20000>; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - apple,dma-range = <0x100 0x0 0x300 0x0>; status = "disabled"; }; ane_dart2: iommu@401840000 { - compatible = "apple,t8140-dart", "apple,t8110-dart"; + compatible = "apple,t8140-ane-dart", "apple,t8140-dart", + "apple,t8110-dart"; reg = <0x4 0x01840000 0x0 0x20000>; #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; power-domains = <&ps_ane_sys>; - apple,dma-range = <0x100 0x0 0x300 0x0>; status = "disabled"; }; - /* The T8140 firmware and child power sequence need their own backend. */ + /* Cold AP-direct is staged; production nodes remain disabled. */ ane: ane@400000000 { compatible = "apple,t8140-ane"; - reg = <0x4 0x0 0x0 0x2000000>; - reg-names = "engine"; + reg = <0x4 0x0 0x0 0x2000000>, + <0x3 0x00700000 0x0 0x18000>, + <0x3 0x00726680 0x0 0x34>; + reg-names = "engine", "pmgr", "pwgate"; interrupt-parent = <&aic>; interrupts = , ; iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; power-domains = <&ps_ane_sys>; + apple,firmware-preloaded; + apple,firmware-iovas = /bits/ 64 <0x10000000000 0x100000c0000>; + /* Inclusive SID0 identity windows from the ADT. */ + apple,dapf-ranges = /bits/ 64 <0x321668000 0x321668003 + 0x30070c000 0x30070c013 + 0x300700290 0x300700293 + 0x482668000 0x482668003>; status = "disabled"; }; }; diff --git a/drivers/accel/ane/Kconfig b/drivers/accel/ane/Kconfig index 1484a95b16a70..0f60463fc97e0 100644 --- a/drivers/accel/ane/Kconfig +++ b/drivers/accel/ane/Kconfig @@ -15,8 +15,13 @@ config DRM_ACCEL_ANE Choose this option if you have an Apple SoC with a Neural Engine: M1 (T8103), M1 Pro, Max and Ultra (T6000, T6001, T6002), M2 (T8112), or M2 Pro, Max and Ultra (T6020, T6021, T6022). - H16 (M4 Pro) and H17 are recognized for firmware preflight and - ownership refusal only; execution support is not available. + H16 (M4 Pro) and H17 (A18 Pro) are recognized for firmware preflight + and ownership refusal by default. A bound H16 device stays idle until + a bring-up stage is written to its debugfs control file. H17 has a + staged cold-boot backend that requires loader ownership evidence and + reserved preload memory, driven from its firmware_stage sysfs file. + On both, the last stage, program, adds an ABI 3 accel node that + loads original HWX programs. The interface exposed to userspace is described in include/uapi/drm/ane_accel.h. @@ -47,11 +52,14 @@ config DRM_ACCEL_ANE_FW_KUNIT_TEST tristate "KUnit tests for ANE firmware validation" if !KUNIT_ALL_TESTS depends on DRM_ACCEL && KUNIT select DRM_ACCEL_ANE_H16_VALIDATION + select DRM_KUNIT_TEST_HELPERS default KUNIT_ALL_TESTS help - Check preload ranges, 27.x channel and reply validation, RTKit - management ordering, H16 HWX/resource bounds, UUID retirement and - the H14 firmware allocation and program lifetime without hardware. + Check preload identity/ranges, 27.x channel and reply validation, + RTKit management ordering, H16 HWX/resource bounds, UUID retirement, + shared transport and bounded allocation, staged H16 cold boot, the H16 + program lifecycle, opening the H16 accel node, and the H14 firmware + allocation and program lifetime without hardware. These tests can run on architectures other than Apple arm64. If unsure, say N. diff --git a/drivers/accel/ane/Makefile b/drivers/accel/ane/Makefile index 767efba5b3e1e..4557afbf5ddc5 100644 --- a/drivers/accel/ane/Makefile +++ b/drivers/accel/ane/Makefile @@ -2,10 +2,14 @@ obj-$(CONFIG_DRM_ACCEL_ANE) += ane.o ane_t6021.o obj-$(CONFIG_DRM_ACCEL_ANE_PM_KUNIT_TEST) += ane_pm_test.o ane_t6021_pm_test.o ane_fw_ids_test.o -obj-$(CONFIG_DRM_ACCEL_ANE_H16_VALIDATION) += ane_h16_hwx.o +obj-$(CONFIG_DRM_ACCEL_ANE_H16_VALIDATION) += ane_h16_hwx.o ane_h16_preload.o ane_h16_load.o \ + ane_h16_program.o obj-$(CONFIG_DRM_ACCEL_ANE_KUNIT_TEST) += ane_drv_test.o obj-$(CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST) += ane_fw_platform_test.o \ - ane_fw_protocol_test.o ane_h16_tasks_test.o ane_h16_execute_test.o ane_h16_pending_test.o ane_h16_hwx_test.o \ + ane_fw_protocol_test.o ane_h16_tasks_test.o ane_h16_execute_test.o \ + ane_fw_transport_test.o ane_h16_boot_test.o \ + ane_h16_pending_test.o ane_h16_hwx_test.o ane_h16_preload_test.o \ + ane_h16_program_test.o ane_fw_cold_test.o ane_h16_accel_test.o \ ane_t6021_lifetime_test.o ane-y := \ @@ -17,6 +21,9 @@ ane-y := \ ane_t6021-y := \ ane_fw_platform.o \ + ane_h16_backend.o \ + ane_h16_accel.o \ + ane_fw_cold.o \ ane_t6021_rtclient_main.o \ ane_t6021_fwload.o \ ane_t6021_boot.o diff --git a/drivers/accel/ane/ane_fw_cold.c b/drivers/accel/ane/ane_fw_cold.c new file mode 100644 index 0000000000000..87de31720bff3 --- /dev/null +++ b/drivers/accel/ane/ane_fw_cold.c @@ -0,0 +1,991 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Shared firmware backend. H17 is the first measured cold power provider. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane_fw_cold.h" +#include "ane_h16_accel.h" + +#define ANE_FW_POOL_DVA 0x10000384000ULL +#define ANE_FW_IPC_DVA 0x100003d0000ULL +#define ANE_FW_HEAP_DVA 0x100003e4000ULL +#define ANE_FW_ALLOC_DVA 0x10001300000ULL +#define ANE_FW_TIMEOUT_NS (3ULL * NSEC_PER_SEC) + +enum ane_fw_stage { + ANE_STAGE_UNVERIFIED, + ANE_STAGE_PROBED, + ANE_STAGE_MAPPED, + ANE_STAGE_BOOTED, + ANE_STAGE_STARTUP, + ANE_STAGE_PROGRAM, + ANE_STAGE_QUARANTINED, +}; + +struct ane_fw_mapping { + struct list_head list; + struct ane_fw_surface surface; + dma_addr_t physical; +}; + +struct ane_fw_backend { + struct platform_device *pdev; + const struct ane_fw_platform *soc; + void __iomem *engine, *pmgr, *gate, *dart[ANE_FW_DARTS]; + /* Mapped register windows, for the MMIO trace hook. */ + struct { + void __iomem *base; + phys_addr_t pa; + resource_size_t size; + } window[3 + ANE_FW_DARTS]; + struct device *power, *memory; + struct iommu_domain *domain; + struct ane_fw_transport transport; + /* Programs and their ABI 3 node, after the program stage. */ + struct ane_h16_session session; + struct ane_fw_surface pool, ipc, heap; + struct list_head mappings; + /* Serialize stage changes, command publication and retained lifetimes. */ + struct mutex lock; + struct notifier_block pm_notifier; + struct ane_fw_cold_state cold; + u64 ranges[ANE_FW_DAPF_RANGES * 2]; + u64 next_dva; + enum ane_fw_stage stage; + bool retained; + /* ANE_SYS and the child power states are on: the engine is readable. */ + bool powered; +}; + +/* One ANE per SoC. The observer exists before platform driver registration: + * the platform bus would otherwise resume genpd and attach DMA before probe. + */ +static struct ane_fw_backend *ane_backend; + +/* + * Every register access of this backend goes through these helpers, so a + * debug build can log the physical address of each access before it is + * made. The hook is empty here. + */ +static inline void ane_fw_trace(struct ane_fw_backend *b, char op, const void __iomem *a, + u64 value) +{ +} + +static u32 ane_rd32(struct ane_fw_backend *b, const void __iomem *a) +{ + ane_fw_trace(b, 'r', a, 0); + return readl(a); +} + +static u64 ane_rd64(struct ane_fw_backend *b, const void __iomem *a) +{ + ane_fw_trace(b, 'R', a, 0); + return readq(a); +} + +static void ane_wr32(struct ane_fw_backend *b, u32 value, void __iomem *a) +{ + ane_fw_trace(b, 'w', a, value); + writel(value, a); +} + +static void ane_wr64(struct ane_fw_backend *b, u64 value, void __iomem *a) +{ + ane_fw_trace(b, 'W', a, value); + writeq(value, a); +} + +static int ane_poll32(struct ane_fw_backend *b, void __iomem *a, u32 mask, u32 want, + unsigned long sleep_us, u64 timeout_us) +{ + u32 value; + + ane_fw_trace(b, 'p', a, want); + return readl_poll_timeout(a, value, (value & mask) == want, sleep_us, timeout_us); +} +static struct kexec_blocker ane_fw_kexec = { + .users = ATOMIC_INIT(0), + .reason = "ANE firmware retains private DMA mappings until hardware reset", +}; + +static int ane_fw_quarantine(struct ane_fw_backend *b, int error) +{ + unsigned int i; + + b->stage = ANE_STAGE_QUARANTINED; + dev_err(&b->pdev->dev, "ANE quarantined (%d); mappings and power retained, reboot required\n", + error); + /* The engine and DARTs are only readable once powered. */ + if (!b->powered) + return error; + for (i = 0; i < 8; i++) + dev_info(&b->pdev->dev, "scratch%u=%#x\n", i, + ane_rd32(b, b->engine + b->soc->scratch + i * 4)); + for (i = 0; i < ANE_FW_DARTS; i++) + dev_info(&b->pdev->dev, "DART%u error=%#x address=%#x:%#x\n", i, + ane_rd32(b, b->dart[i] + 0x100), ane_rd32(b, b->dart[i] + 0x174), + ane_rd32(b, b->dart[i] + 0x170)); + return error; +} + +static bool ane_fw_raw_boot(void) +{ + struct device_node *chosen __free(device_node) = of_find_node_by_path("/chosen"); + const char *evidence; + + return chosen && !of_property_read_string(chosen, "apple,ane-cold-boot", &evidence) && + !strcmp(evidence, "raw-no-xnu-ane-untouched-v1"); +} + +/* + * Step 1: PMGR words only. These are the only reads made with ANE_SYS in an + * unknown power state; the engine block and its DARTs sit behind ANE_SYS and + * an access to them while it is off stalls the bus. + */ +static void ane_fw_snapshot_pmgr(struct ane_fw_backend *b) +{ + struct ane_fw_cold_state *s = &b->cold; + + s->sys = ane_rd32(b, b->pmgr + 0x290); + s->cpu = ane_rd32(b, b->pmgr + 0xc000); + s->subblock = ane_rd32(b, b->pmgr + 0xc008); + dev_info(&b->pdev->dev, "U2 ANE_SYS=%#x CPU_PS=%#x subblock=%#x\n", + s->sys, s->cpu, s->subblock); +} + +/* Step 2: PWGATE, with ANE_SYS on and before the child power step. */ +static void ane_fw_snapshot_gate(struct ane_fw_backend *b) +{ + struct ane_fw_cold_state *s = &b->cold; + + s->gate_request = ane_rd32(b, b->gate + 0xc); + s->gate_ack = ane_rd32(b, b->gate + 0x30); + dev_info(&b->pdev->dev, "U2 PWGATE=%#x/%#x\n", s->gate_request, s->gate_ack); +} + +/* Step 3: the engine and its DARTs, after the child power step. */ +static void ane_fw_snapshot_engine(struct ane_fw_backend *b) +{ + struct ane_fw_cold_state *s = &b->cold; + struct device *dev = &b->pdev->dev; + unsigned int i, j; + + s->ce_state = ane_rd64(b, b->engine + 0x18a4010); + s->ce_error = ane_rd64(b, b->engine + 0x18a4018); + s->rvbar = ane_rd64(b, b->engine + 0x1050000); + s->control = ane_rd32(b, b->engine + b->soc->wrapper + 0x44); + s->status = ane_rd32(b, b->engine + b->soc->wrapper + 0x48); + s->pending = ane_rd32(b, b->engine + b->soc->irq_status); + for (i = 0; i < 8; i++) + s->scratch[i] = ane_rd32(b, b->engine + b->soc->scratch + i * 4); + dev_info(dev, "U2 control=%#x status=%#x RVBAR=%#llx CE=%#llx/%#llx pending=%#x\n", + s->control, s->status, s->rvbar, s->ce_state, s->ce_error, s->pending); + for (i = 0; i < ANE_FW_DARTS; i++) { + s->dart[i].version = ane_rd32(b, b->dart[i] + 8); + s->dart[i].protect = ane_rd32(b, b->dart[i] + 0x200); + s->dart[i].unprotect = ane_rd32(b, b->dart[i] + 0x204); + s->dart[i].lock = ane_rd32(b, b->dart[i] + 0x208); + s->dart[i].error = ane_rd32(b, b->dart[i] + 0x100); + for (j = 0; j < 16; j++) + s->dart[i].enabled[j] = ane_rd32(b, b->dart[i] + 0xc00 + 4 * j); + dev_info(dev, "U2 DART%u version=%#x protect=%#x unprotect=%#x lock=%#x error=%#x\n", + i, s->dart[i].version, s->dart[i].protect, + s->dart[i].unprotect, s->dart[i].lock, s->dart[i].error); + for (j = 0; j < 16; j++) + if (s->dart[i].enabled[j]) + dev_info(dev, "U2 DART%u stream enable word %u=%#x\n", i, j, + s->dart[i].enabled[j]); + } + for (i = 0; i < ANE_FW_DAPF_RANGES; i++) { + void __iomem *r = b->engine + 0x1810000 + i * 0x40; + + dev_info(dev, "U2 DAPF%u flags=%#x start=%#llx end=%#llx enable=%#x\n", + i, ane_rd32(b, r), ane_rd64(b, r + 8), ane_rd64(b, r + 0x10), + ane_rd32(b, r + 0x20)); + } +} + +static int ane_fw_pm_event(struct notifier_block *nb, unsigned long event, void *unused) +{ + struct ane_fw_backend *b = container_of(nb, struct ane_fw_backend, pm_notifier); + + if ((event == PM_SUSPEND_PREPARE || event == PM_HIBERNATION_PREPARE || + event == PM_RESTORE_PREPARE) && READ_ONCE(b->retained)) + return NOTIFY_BAD; + return NOTIFY_DONE; +} + +static int ane_fw_retain(struct ane_fw_backend *b) +{ + int ret; + + /* + * The kexec blocker takes built-in users only. A module, like the H16 + * backend, pins itself and refuses system sleep, but cannot veto kexec. + */ + if (IS_BUILTIN(CONFIG_DRM_ACCEL_ANE)) { + ret = kexec_block(&ane_fw_kexec); + if (ret) + return ret; + } + b->pm_notifier.notifier_call = ane_fw_pm_event; + ret = register_pm_notifier(&b->pm_notifier); + if (ret) { + if (IS_BUILTIN(CONFIG_DRM_ACCEL_ANE)) + kexec_unblock(&ane_fw_kexec); + return ret; + } + __module_get(THIS_MODULE); + b->retained = true; + return 0; +} + +static int ane_fw_map(struct ane_fw_backend *b, u64 dva, phys_addr_t pa, + size_t size, int prot) +{ + int ret = iommu_map(b->domain, dva, pa, size, prot, GFP_KERNEL); + + if (ret) + return ret; + /* iommu_map includes the DART's normal range synchronization and + * propagates its timeout. Extra invalidates do not fix stale output + * BAR bindings between programs; the transport's queued first EXECUTE + * after a program change does. + */ + for (unsigned int i = 0; i < ANE_FW_DARTS; i++) + if (ane_rd32(b, b->dart[i] + 0x100)) + return -EIO; + return 0; +} + +static int ane_fw_alloc_at(struct ane_fw_backend *b, u64 dva, size_t size, + int prot, struct ane_fw_surface *s) +{ + struct ane_fw_mapping *m; + int ret; + + m = kzalloc_obj(*m); + if (!m) + return -ENOMEM; + m->surface.cpu = dma_alloc_coherent(b->memory, size, &m->physical, GFP_KERNEL); + if (!m->surface.cpu) { + kfree(m); + return -ENOMEM; + } + m->surface.dva = dva; + m->surface.size = size; + /* Record before publication, including a failed/partial mapping. */ + list_add_tail(&m->list, &b->mappings); + ret = ane_fw_map(b, dva, m->physical, size, prot); + if (!ret) + *s = m->surface; + return ret; +} + +static int ane_fw_alloc(void *ctx, size_t size, u32 tag, struct ane_fw_surface *s) +{ + struct ane_fw_backend *b = ctx; + int ret; + + if (!size || size > ANE_FW_SHARED_TOTAL || + b->next_dva > ANE_FW_ALLOC_DVA + SZ_512M - size) + return -ENOSPC; + ret = ane_fw_alloc_at(b, b->next_dva, size, IOMMU_READ | IOMMU_WRITE, s); + if (!ret) { + b->next_dva += size; + dev_info(&b->pdev->dev, "SHAREDMALLOC tag=%#x bytes=%zu\n", tag, size); + } + return ret; +} + +static u64 ane_fw_load64(void *ctx, const void *p) +{ + return le64_to_cpu(READ_ONCE(*(__le64 *)p)); +} + +static void ane_fw_store64(void *ctx, void *p, u64 value) +{ + WRITE_ONCE(*(__le64 *)p, cpu_to_le64(value)); +} + +static void ane_fw_rmb(void *ctx) +{ + dma_rmb(); +} + +static void ane_fw_wmb(void *ctx) +{ + dma_wmb(); +} + +static u32 ane_fw_read32(void *ctx, u32 reg) +{ + struct ane_fw_backend *b = ctx; + + return ane_rd32(b, b->engine + reg); +} + +static void ane_fw_write32(void *ctx, u32 reg, u32 value) +{ + struct ane_fw_backend *b = ctx; + + ane_wr32(b, value, b->engine + reg); +} + +static u64 ane_fw_now(void *ctx) +{ + return ktime_get_ns(); +} + +/* + * Every wait polls. Sleep in the shared short steps first (see + * ane_fw_poll_sleep_us()) and back off for long waits such as a stalled + * request, so a stall costs no more CPU than a 1 ms poll. + */ +static void ane_fw_relax(void *ctx, u64 waited_ns) +{ + unsigned int min_us, max_us; + + min_us = ane_fw_poll_sleep_us(waited_ns, &max_us); + usleep_range(min_us, max_us); +} + +static void ane_fw_refused(void *ctx, u64 word0, u64 word1, u64 word2, int error) +{ + struct ane_fw_backend *b = ctx; + + dev_info(&b->pdev->dev, "SHAREDMALLOC refused tuple=%#llx/%#llx/%#llx error=%d\n", + word0, word1, word2, error); +} + +static const struct ane_fw_transport_ops ane_fw_ops = { + .load64 = ane_fw_load64, .store64 = ane_fw_store64, + .read_barrier = ane_fw_rmb, .write_barrier = ane_fw_wmb, + .read32 = ane_fw_read32, .write32 = ane_fw_write32, + .now_ns = ane_fw_now, .relax = ane_fw_relax, + .alloc = ane_fw_alloc, .refused = ane_fw_refused, +}; + +/* + * Power in the order the H17 U2 runs measured: genpd ANE_SYS on, PWGATE read, + * CPU power state on, PWGATE request cleared, sub-block power state on. Only + * then are the engine and DARTs read and checked for their reset state, before + * Linux writes anything to the engine. A failed check after power-on retains + * power and mappings until reboot, like any other quarantine. + */ +static int ane_fw_power(struct ane_fw_backend *b, bool raw_boot) +{ + struct device *dev = &b->pdev->dev; + u64 ce; + int ret; + + b->power = dev_pm_domain_attach_by_id(dev, 0); + if (IS_ERR_OR_NULL(b->power)) + return b->power ? PTR_ERR(b->power) : -ENODEV; + ret = pm_runtime_resume_and_get(b->power); + if (ret < 0) + return ret; + if ((ane_rd32(b, b->pmgr + 0x290) & 0xff) != 0xff) + return -EIO; + /* The child state may not have moved since the probe stage. */ + ane_fw_snapshot_pmgr(b); + ane_fw_snapshot_gate(b); + if (!ane_fw_cold_pmgr_valid(&b->cold, raw_boot) || !ane_fw_cold_gate_valid(&b->cold)) { + dev_info(dev, "H17 ANE: child power or gate not in the cold state; refusing\n"); + return -EPERM; + } + ane_wr32(b, ane_rd32(b, b->pmgr + 0xc000) | 0xf, b->pmgr + 0xc000); + ret = ane_poll32(b, b->pmgr + 0xc000, 0xff, 0xff, 1, 100000); + if (ret) + return ret; + ane_wr32(b, 0, b->gate + 0xc); + ret = ane_poll32(b, b->gate + 0xc, 3, 0, 1, 100000); + if (ret) + return ret; + ane_wr32(b, ane_rd32(b, b->pmgr + 0xc008) | 0xf, b->pmgr + 0xc008); + ret = ane_poll32(b, b->pmgr + 0xc008, 0xff, 0xff, 1, 100000); + if (ret) + return ret; + ret = ane_poll32(b, b->gate + 0x30, U32_MAX, 0, 1, 100000); + if (ret) + return ret; + b->powered = true; + ane_fw_snapshot_engine(b); + if (!ane_fw_cold_engine_valid(&b->cold)) { + dev_info(dev, "H17 ANE: engine or DART not in its reset state; refusing\n"); + return -EPERM; + } + ane_wr64(b, 3, b->engine + 0x18a4000); + ane_fw_trace(b, 'P', b->engine + 0x18a4008, 3); + ret = readq_poll_timeout(b->engine + 0x18a4008, ce, (ce & 3) == 3, 1, 100000); + if (ret) + return ret; + ane_wr64(b, 3, b->engine + 0x18a4008); + if ((ane_rd64(b, b->engine + 0x18a4010) & 3) != 3 || + ane_rd64(b, b->engine + 0x18a4018)) + return -EIO; + dev_info(dev, "ANE power: ANE_SYS on, child/PWGATE/CE ready\n"); + return 0; +} + +static int ane_fw_dapf(struct ane_fw_backend *b) +{ + unsigned int i; + + for (i = 0; i < ANE_FW_DAPF_RANGES; i++) { + void __iomem *r = b->engine + 0x1810000 + i * 0x40; + u64 start = b->ranges[i * 2], end = b->ranges[i * 2 + 1]; + + if (ane_rd32(b, b->dart[0] + 0x208) & BIT(2)) + return -EPERM; + ane_wr32(b, 0x31, r); + ane_wr32(b, lower_32_bits(start), r + 8); + ane_wr32(b, upper_32_bits(start), r + 0xc); + ane_wr32(b, lower_32_bits(end), r + 0x10); + ane_wr32(b, upper_32_bits(end), r + 0x14); + ane_wr32(b, 1, r + 0x20); + if ((ane_rd32(b, r) & 0x333) != 0x31 || ane_rd64(b, r + 8) != start || + ane_rd64(b, r + 0x10) != end || (ane_rd32(b, r + 0x20) & 0xffff) != 1) + return -EIO; + } + return 0; +} + +static int ane_fw_preload_metadata(struct ane_fw_backend *b, u64 physical[2], + u64 size[2], u64 dva[2], u8 data_pin[SHA256_DIGEST_SIZE]) +{ + struct device_node *np = b->pdev->dev.of_node; + unsigned int i; + int ret; + + if (!of_property_read_bool(np, "apple,firmware-preloaded") || + of_property_count_u32_elems(np, "memory-region") != 2 || + of_property_read_u64_array(np, "apple,firmware-iovas", dva, 2) || + of_property_read_u8_array(np, "apple,preload-data-sha256", data_pin, + SHA256_DIGEST_SIZE)) + return -ENODATA; + /* The same no-map, outside-System-RAM rule as the H16 preload. */ + for (i = 0; i < 2; i++) { + const char *name; + + if (of_property_read_string_index(np, "memory-region-names", i, &name) || + strcmp(name, i ? "firmware-data" : "firmware-text")) + return -EINVAL; + ret = ane_fw_preload_range(np, i, &physical[i], &size[i]); + if (ret) + return ret; + } + if (!ane_fw_preload_ranges_valid(b->soc, physical, size, dva)) + return -EINVAL; + return 0; +} + +static int ane_fw_preload(struct ane_fw_backend *b) +{ + static const u8 text_pin[SHA256_DIGEST_SIZE] = { + 0xcd, 0xa0, 0x9e, 0xc1, 0xae, 0xe6, 0x8b, 0x88, + 0xb4, 0x7e, 0xdc, 0x0e, 0xeb, 0xc8, 0x10, 0x45, + 0x25, 0x32, 0xa8, 0x55, 0x56, 0x3f, 0xbd, 0x65, + 0x0b, 0x95, 0xe2, 0x84, 0x6c, 0x34, 0x53, 0x10, + }; + u64 physical[2], size[2], dva[2], tail, tail_size; + struct ane_fw_surface snapshots[2], heap_tail; + u8 digest[SHA256_DIGEST_SIZE], data_pin[SHA256_DIGEST_SIZE]; + unsigned int i; + int ret; + + ret = ane_fw_preload_metadata(b, physical, size, dva, data_pin); + if (ret) + return ret; + for (i = 0; i < 2; i++) { + void *source = memremap(physical[i], size[i], MEMREMAP_WB); + struct sha256_ctx hash; + + if (!source) + return -ENOMEM; + ret = ane_fw_alloc_at(b, dva[i], size[i], + IOMMU_READ | (i ? IOMMU_WRITE : 0), &snapshots[i]); + if (!ret) + memcpy(snapshots[i].cpu, source, size[i]); + memunmap(source); + if (ret) + return ret; + sha256_init(&hash); + sha256_update(&hash, snapshots[i].cpu, size[i]); + sha256_final(&hash, digest); + if (memcmp(digest, i ? data_pin : text_pin, sizeof(digest))) + return -EKEYREJECTED; + } + /* Preserve every personalized DATA byte, including the iBoot TLV table. */ + dma_wmb(); + ret = ane_fw_heap_tail(dva, size, ANE_FW_POOL_DVA, &tail, &tail_size); + if (ret || !tail_size) + return ret; + /* Zeroed by the coherent allocation; kept until reset like the preload. */ + ret = ane_fw_alloc_at(b, tail, tail_size, IOMMU_READ | IOMMU_WRITE, &heap_tail); + if (!ret) + dev_info(&b->pdev->dev, "ANE map: firmware heap tail at %#llx bytes %#llx\n", + tail, tail_size); + return ret; +} + +static int ane_fw_prepare(struct ane_fw_backend *b) +{ + struct device *dev = &b->pdev->dev; + unsigned int i; + int ret; + + ret = ane_fw_power(b, ane_fw_raw_boot()); + if (ret) + return ret; + ret = of_dma_configure(dev, dev->of_node, true); + if (ret) + return ret; + ret = iommu_device_claim_dma_owner(dev, b); + if (ret) + return ret; + b->domain = iommu_paging_domain_alloc(dev); + if (IS_ERR(b->domain)) + return PTR_ERR(b->domain); + ret = iommu_attach_device(b->domain, dev); + if (ret) + return ret; + if (!(b->domain->pgsize_bitmap & SZ_16K) || !b->domain->ops->iotlb_sync_map || + b->domain->geometry.aperture_start || + b->domain->geometry.aperture_end < ANE_FW_ALLOC_DVA + SZ_512M) + return -ERANGE; + b->memory = root_device_register("ane-fw-memory"); + if (IS_ERR(b->memory)) + return PTR_ERR(b->memory); + ret = dma_coerce_mask_and_coherent(b->memory, DMA_BIT_MASK(64)); + if (ret) + return ret; + ret = ane_fw_preload(b); + if (ret) + return ret; + for (i = 0; i < ANE_FW_DAPF_RANGES; i++) { + u64 page = round_down(b->ranges[i * 2], SZ_16K); + + ret = ane_fw_map(b, page, page, SZ_16K, IOMMU_READ | IOMMU_WRITE | IOMMU_MMIO); + if (ret) + return ret; + } + ret = ane_fw_dapf(b); + if (ret) + return ret; + ret = ane_fw_alloc_at(b, ANE_FW_POOL_DVA, SZ_256K, + IOMMU_READ | IOMMU_WRITE, &b->pool); + if (ret) + return ret; + ret = ane_fw_alloc_at(b, ANE_FW_IPC_DVA, SZ_64K, + IOMMU_READ | IOMMU_WRITE, &b->ipc); + if (ret) + return ret; + ret = ane_fw_alloc_at(b, ANE_FW_HEAP_DVA, 15 * SZ_1M, + IOMMU_READ | IOMMU_WRITE, &b->heap); + if (!ret) + dev_info(dev, "ANE map: reserved preload, SID0 on three DARTs, DAPF verified\n"); + return ret; +} + +static int ane_fw_scratch_wait(struct ane_fw_backend *b, unsigned int cell, u32 wanted) +{ + return ane_poll32(b, b->engine + b->soc->scratch + cell * 4, U32_MAX, wanted, + 100, 3000000); +} + +static int ane_fw_boot(struct ane_fw_backend *b) +{ + void __iomem *scratch = b->engine + b->soc->scratch; + u8 *init = b->pool.cpu + 0x180; + u64 table; + unsigned int i; + int ret; + + for (i = 0; i < 8; i++) + ane_wr32(b, 0, scratch + i * 4); + ane_wr32(b, 1, scratch + 6 * 4); + ane_wr32(b, 0, b->engine + b->soc->wrapper + 0x44); + dma_wmb(); + ane_wr32(b, 0x10, b->engine + b->soc->wrapper + 0x44); + ret = ane_fw_scratch_wait(b, 7, 0x08042006); + if (ret) + return ret; + dev_info(&b->pdev->dev, "ANE boot: READY\n"); + if (ane_rd32(b, scratch) != 8 || ane_rd32(b, scratch + 4) != 0xc440 || + ane_rd32(b, scratch + 12) != 0xf00000 || ane_rd32(b, scratch + 16)) + return -EPROTO; + put_unaligned_le64(b->ipc.dva, init + 8); + put_unaligned_le64(ANE_FW_SPAN, init + 0x10); + put_unaligned_le64(0xfc80000, init + 0x18); + put_unaligned_le64(b->heap.dva, init + 0x20); + put_unaligned_le64(b->heap.size, init + 0x28); + put_unaligned_le64(0xc441, init + 0x50); + put_unaligned_le64(b->pool.dva, init + 0x58); + put_unaligned_le64(b->pool.size, init + 0x60); + put_unaligned_le32(64, init + 0x68); + put_unaligned_le32(4, init + 0x12c); + dma_wmb(); + ane_wr32(b, lower_32_bits(b->pool.dva + 0x180), scratch); + ane_wr32(b, upper_32_bits(b->pool.dva + 0x180), scratch + 4); + ane_wr32(b, 0xf7fbdff9, scratch + 7 * 4); + ret = ane_fw_scratch_wait(b, 7, 0x08042006); + if (ret) + return ret; + table = (u64)ane_rd32(b, scratch + 4) << 32 | ane_rd32(b, scratch); + if (table != b->ipc.dva) + return -EPROTO; + ret = ane_fw_transport_init(&b->transport, b->soc, &ane_fw_ops, b, + b->ipc, b->pool, 0x4000, table, ANE_FW_TABLE_SIZE); + if (ret) + return ret; + ret = ane_fw_transport_arm(&b->transport); + if (ret) + return ret; + ane_wr32(b, 0x08042006, scratch + 3 * 4); + ret = ane_fw_scratch_wait(b, 3, 0); + if (!ret) + dev_info(&b->pdev->dev, "ANE boot: eight channels armed, scratch3 cleared (polled)\n"); + return ret; +} + +static int ane_fw_command(struct ane_fw_backend *b, unsigned int channel, + void *command, size_t size, u64 deadline) +{ + struct ane_fw_transaction *tx = &b->transport.pending[channel]; + int ret; + + ret = ane_fw_transport_publish(&b->transport, channel, command, size, deadline); + if (ret) + return ret; + ret = ane_fw_transport_wait(&b->transport, channel); + if (!ret) + memcpy(command, b->pool.cpu + tx->dva - b->pool.dva, size); + return ret; +} + +static int ane_fw_startup(struct ane_fw_backend *b) +{ + static const struct { u16 id, size; } commands[] = { + { 4, 12 }, { 0, 12 }, { 3, 16 }, { 0x22, 100 }, + }; + u8 command[100]; + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(commands); i++) { + memset(command, 0, sizeof(command)); + put_unaligned_le16(commands[i].id, command + 4); + ret = ane_fw_command(b, ANE_FW_IO, command, commands[i].size, + ktime_get_ns() + ANE_FW_TIMEOUT_NS); + if (ret) + return ret; + if (commands[i].id == 4 && get_unaligned_le32(command + 8)) + return -EPROTO; + if (commands[i].id == 3 && ane_fw_config_status(command, 16)) + return -EPROTO; + dev_info(&b->pdev->dev, "ANE startup: command %#x ack=0\n", commands[i].id); + if (commands[i].id == 3) + dev_info(&b->pdev->dev, "ANE CONFIG_GET tick=%u\n", + get_unaligned_le32(command + 8)); + if (commands[i].id == 0x22) + dev_info(&b->pdev->dev, "ANE RESOURCE_INFO: %*ph\n", 100, command); + } + return 0; +} + +/* Private program mappings come from the same window as SHAREDMALLOC. */ +static int ane_fw_program_map(void *ctx, size_t size, bool write, struct ane_fw_surface *out) +{ + struct ane_fw_backend *b = ctx; + int ret; + + if (!size || !IS_ALIGNED(size, SZ_16K) || + b->next_dva > ANE_FW_ALLOC_DVA + SZ_512M - size) + return -ENOSPC; + ret = ane_fw_alloc_at(b, b->next_dva, size, + IOMMU_READ | (write ? IOMMU_WRITE : 0), out); + if (ret) + return ret; + b->next_dva += size; + return 0; +} + +static void ane_fw_program_log(void *ctx, const char *item, u64 a, u64 c) +{ + struct ane_fw_backend *b = ctx; + + dev_info(&b->pdev->dev, "H17 %s %llx %llx\n", item, a, c); +} + +static const struct ane_h16_session_ops ane_fw_session_ops = { + .map = ane_fw_program_map, + .log = ane_fw_program_log, +}; + +/* + * After startup: the shared program session and its ABI 3 node. From here on + * the node serializes all firmware traffic. The firmware frees each process + * pool after UNLOAD, often only during a later program, so FREE stays + * admitted; backing is retained until reboot. + */ +static int ane_fw_program_stage(struct ane_fw_backend *b) +{ + int ret; + + b->transport.allow_free = true; + ret = ane_h16_session_init(&b->session, &b->transport, &ane_fw_session_ops, b, + ANE_FW_TIMEOUT_NS); + if (ret) + return ret; + ret = ane_h16_accel_register(&b->pdev->dev, &b->session); + if (!ret) + dev_info(&b->pdev->dev, "ANE program: session and ABI 3 node ready\n"); + return ret; +} + +static ssize_t firmware_stage_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + static const char * const names[] = { + "unverified", "probe", "map", "boot", "startup", "program", "quarantined", + }; + struct ane_fw_backend *b = ane_backend; + + return sysfs_emit(buf, "%s\n", names[READ_ONCE(b->stage)]); +} + +static ssize_t ane_fw_stage_action(struct device *dev, const char *buf, size_t count) +{ + struct ane_fw_backend *b = ane_backend; + enum ane_fw_stage next; + int ret = 0; + + guard(mutex)(&b->lock); + if (b->stage == ANE_STAGE_QUARANTINED) + return -EIO; + if (sysfs_streq(buf, "probe")) { + if (b->retained) + return -EBUSY; + /* PMGR only: ANE_SYS may be off, which makes the engine unreadable. */ + ane_fw_snapshot_pmgr(b); + if (!ane_fw_cold_pmgr_valid(&b->cold, ane_fw_raw_boot())) { + dev_info(dev, "H17 ANE: cold acquisition unproven (loader or child power state); refusing\n"); + return -EPERM; + } + b->stage = ANE_STAGE_PROBED; + dev_info(dev, "ANE probe: raw-boot evidence and cold child power verified; no writes\n"); + return count; + } + if (sysfs_streq(buf, "map")) + next = ANE_STAGE_MAPPED; + else if (sysfs_streq(buf, "boot")) + next = ANE_STAGE_BOOTED; + else if (sysfs_streq(buf, "startup")) + next = ANE_STAGE_STARTUP; + else if (sysfs_streq(buf, "program")) + next = ANE_STAGE_PROGRAM; + else + return -EINVAL; + if (next != b->stage + 1) + return -EINVAL; + if (next == ANE_STAGE_MAPPED) { + u64 physical[2], size[2], dva[2]; + u8 data_pin[SHA256_DIGEST_SIZE]; + + ane_fw_snapshot_pmgr(b); + if (!ane_fw_cold_pmgr_valid(&b->cold, ane_fw_raw_boot())) { + dev_info(dev, "H17 ANE: cold acquisition unproven (loader or child power state); refusing\n"); + return -EPERM; + } + ret = ane_fw_preload_metadata(b, physical, size, dva, data_pin); + if (ret) { + dev_info(dev, "H17 ANE: loader preload reservation or digest missing; refusing\n"); + return ret; + } + ret = ane_fw_retain(b); + if (ret) + return ret; + ret = ane_fw_prepare(b); + } else if (next == ANE_STAGE_BOOTED) { + ret = ane_fw_boot(b); + } else if (next == ANE_STAGE_STARTUP) { + ret = ane_fw_startup(b); + } else { + ret = ane_fw_program_stage(b); + } + if (ret) + return ane_fw_quarantine(b, ret); + b->stage = next; + return count; +} + +static ssize_t firmware_stage_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + unsigned int sleep_flags = lock_system_sleep(); + ssize_t ret = ane_fw_stage_action(dev, buf, count); + + unlock_system_sleep(sleep_flags); + return ret; +} +static DEVICE_ATTR_RW(firmware_stage); + +static void ane_fw_observer_free(struct ane_fw_backend *b) +{ + unsigned int i; + + for (i = 0; i < ANE_FW_DARTS; i++) + if (b->dart[i]) + iounmap(b->dart[i]); + if (b->engine) + iounmap(b->engine); + if (b->pmgr) + iounmap(b->pmgr); + if (b->gate) + iounmap(b->gate); + put_device(&b->pdev->dev); + kfree(b); +} + +/* Record a mapped register window; the trace hook names accesses by PA. */ +static int ane_fw_window(struct ane_fw_backend *b, unsigned int index, void __iomem *base, + struct device_node *np, unsigned int reg) +{ + struct resource res; + + if (!base) + return -ENOMEM; + if (of_address_to_resource(np, reg, &res)) + return -EINVAL; + b->window[index].base = base; + b->window[index].pa = res.start; + b->window[index].size = resource_size(&res); + return 0; +} + +int ane_fw_staged_init(struct platform_device *pdev, const struct ane_fw_platform *soc) +{ + struct device_node *np = pdev->dev.of_node; + struct ane_fw_backend *b; + unsigned int i; + u32 dart_phandles[ANE_FW_DARTS]; + int ret = -EINVAL; + + if (ane_backend || soc->soc != 0x8140) + return -EOPNOTSUPP; + b = kzalloc_obj(*b); + if (!b) + return -ENOMEM; + b->pdev = pdev; + get_device(&pdev->dev); + b->soc = soc; + b->next_dva = ANE_FW_ALLOC_DVA; + mutex_init(&b->lock); + INIT_LIST_HEAD(&b->mappings); + if (of_property_count_strings(np, "reg-names") != soc->reg_count || + of_irq_count(np) != soc->interrupt_count || + of_count_phandle_with_args(np, "power-domains", "#power-domain-cells") != 1) + goto fail; + for (i = 0; i < soc->reg_count; i++) { + struct resource res; + const char *name; + + if (of_address_to_resource(np, i, &res) || + resource_size(&res) != soc->reg_sizes[i] || + of_property_read_string_index(np, "reg-names", i, &name) || + strcmp(name, soc->reg_names[i])) + goto fail; + } + if (of_property_read_u64_array(np, "apple,dapf-ranges", b->ranges, + ARRAY_SIZE(b->ranges))) + goto fail; + for (i = 0; i < ANE_FW_DAPF_RANGES; i++) { + u64 start = b->ranges[i * 2], end = b->ranges[i * 2 + 1]; + + if (start > end || round_down(start, SZ_16K) != round_down(end, SZ_16K)) + goto fail; + for (unsigned int j = 0; j < i; j++) + if (round_down(start, SZ_16K) == round_down(b->ranges[j * 2], SZ_16K)) + goto fail; + } + if (of_count_phandle_with_args(np, "iommus", "#iommu-cells") != ANE_FW_DARTS) + goto fail; + for (i = 0; i < ANE_FW_DARTS; i++) { + struct of_phandle_args args; + + ret = of_parse_phandle_with_args(np, "iommus", "#iommu-cells", i, &args); + if (ret) + goto fail; + if (args.args_count != 1 || args.args[0] || + !of_device_is_compatible(args.np, "apple,t8140-ane-dart")) { + of_node_put(args.np); + ret = -EINVAL; + goto fail; + } + for (unsigned int j = 0; j < i; j++) { + if (dart_phandles[j] == args.np->phandle) { + of_node_put(args.np); + ret = -EINVAL; + goto fail; + } + } + dart_phandles[i] = args.np->phandle; + b->dart[i] = of_iomap(args.np, 0); + ret = ane_fw_window(b, 3 + i, b->dart[i], args.np, 0); + of_node_put(args.np); + if (ret) + goto fail; + } + b->engine = of_iomap(np, 0); + b->pmgr = of_iomap(np, 1); + b->gate = of_iomap(np, 2); + ret = ane_fw_window(b, 0, b->engine, np, 0); + if (!ret) + ret = ane_fw_window(b, 1, b->pmgr, np, 1); + if (!ret) + ret = ane_fw_window(b, 2, b->gate, np, 2); + if (ret) + goto fail; + ane_backend = b; + ret = device_create_file(&pdev->dev, &dev_attr_firmware_stage); + if (ret) { + ane_backend = NULL; + goto fail; + } + dev_info(&pdev->dev, "H17 ANE: read-only staged observer ready; IRQs 655/668 remain masked\n"); + return 0; +fail: + ane_fw_observer_free(b); + return ret; +} + +bool ane_fw_staged_exit(void) +{ + struct ane_fw_backend *b = ane_backend; + + if (!b) + return false; + /* A retained backend pins the module. No unload or in-place retry. */ + if (WARN_ON(b->retained)) + return true; + device_remove_file(&b->pdev->dev, &dev_attr_firmware_stage); + ane_backend = NULL; + ane_fw_observer_free(b); + return true; +} diff --git a/drivers/accel/ane/ane_fw_cold.h b/drivers/accel/ane/ane_fw_cold.h new file mode 100644 index 0000000000000..207044def5527 --- /dev/null +++ b/drivers/accel/ane/ane_fw_cold.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_FW_COLD_H__ +#define __ANE_FW_COLD_H__ + +#include "ane_fw_transport.h" + +#define ANE_FW_DARTS 3 +#define ANE_FW_DAPF_RANGES 4 + +struct ane_fw_cold_state { + u32 sys, cpu, subblock, gate_request, gate_ack; + u64 ce_state, ce_error, rvbar; + u32 control, status, pending, scratch[8]; + struct { + u32 version, protect, unprotect, lock, error; + u32 enabled[16]; + } dart[ANE_FW_DARTS]; +}; + +/* + * Cold acquisition is proved in three steps, each read only once its block is + * known to be powered, in the order the H17 U2 runs measured as safe: + * + * 1. PMGR, readable with ANE_SYS in either state: the ANE CPU and sub-block + * power states are off. ANE_SYS itself may read on or off (fact 5: it can + * be on while genpd reports off, and generic DART runtime PM turns it off), + * so it is no marker. + * 2. PWGATE, after ANE_SYS is on: the gate is requested and not acknowledged. + * 3. The engine and its DARTs, only after the child power step: CE, CPU and + * wrapper reset values, and unlocked, unfaulted DARTs with no stream on. + * Nothing in the engine has been written by Linux at that point. + * + * DAPF records and invalid TTBR payloads are random at reset, not markers. + */ +static inline bool ane_fw_cold_pmgr_valid(const struct ane_fw_cold_state *s, bool raw_boot) +{ + return raw_boot && !(s->cpu & 0xff) && s->subblock == 0x300; +} + +static inline bool ane_fw_cold_gate_valid(const struct ane_fw_cold_state *s) +{ + return s->gate_request == 3 && !s->gate_ack; +} + +static inline bool ane_fw_cold_engine_valid(const struct ane_fw_cold_state *s) +{ + unsigned int i; + + if ((s->ce_state & 3) || s->ce_error || s->control || s->status != 0x6a || + s->rvbar != 0x10000000001ULL || s->pending) + return false; + for (i = 0; i < ARRAY_SIZE(s->scratch); i++) + if (s->scratch[i]) + return false; + /* + * Stream enables are not judged: the Linux DART driver enables every + * stream globally when it binds and isolates them through the per-stream + * TCRs, so they read set on every boot once the DARTs are bound. + */ + for (i = 0; i < ANE_FW_DARTS; i++) { + if ((s->dart[i].version & 0xffff) != 0x203 || + (s->dart[i].protect & BIT(0)) || + (s->dart[i].lock & (BIT(0) | BIT(2))) || s->dart[i].error) + return false; + } + return true; +} + +static inline bool ane_fw_cold_valid(const struct ane_fw_cold_state *s, bool raw_boot) +{ + return ane_fw_cold_pmgr_valid(s, raw_boot) && ane_fw_cold_gate_valid(s) && + ane_fw_cold_engine_valid(s); +} + +/* Bytes of firmware memory from the TEXT DVA (boot argument +0x10). */ +#define ANE_FW_SPAN 0x380000ULL + +/* + * The firmware owns ANE_FW_SPAN bytes from its TEXT DVA, but the bootloader + * preload covers only TEXT and DATA. The firmware keeps runtime heap state in + * the rest of the span (0x14000 bytes past DATA on the 26.6.2 build), which + * the host supplies zeroed and device-writable before boot. Returns the tail + * DVA and size (0 when DATA ends the span), or -ERANGE when the layout would + * overlap DATA, is not in whole 16 KiB pages, or reaches the command pool. + */ +static inline int ane_fw_heap_tail(const u64 dva[2], const u64 size[2], u64 limit, + u64 *tail, u64 *tail_size) +{ + u64 start = dva[1] + size[1], end = dva[0] + ANE_FW_SPAN; + + if (dva[1] < dva[0] + size[0] || start < dva[1] || start > end || + !IS_ALIGNED(start, SZ_16K) || !IS_ALIGNED(end, SZ_16K) || end > limit) + return -ERANGE; + *tail = start; + *tail_size = end - start; + return 0; +} + +int ane_fw_staged_init(struct platform_device *pdev, const struct ane_fw_platform *soc); +bool ane_fw_staged_exit(void); + +#endif /* __ANE_FW_COLD_H__ */ diff --git a/drivers/accel/ane/ane_fw_cold_test.c b/drivers/accel/ane/ane_fw_cold_test.c new file mode 100644 index 0000000000000..2b04fd99e05a6 --- /dev/null +++ b/drivers/accel/ane/ane_fw_cold_test.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "ane_fw_cold.h" + +static struct ane_fw_cold_state ane_cold_fixture(void) +{ + struct ane_fw_cold_state s = { + .sys = 0x23ff, .cpu = 0x0f000300, .subblock = 0x300, + .gate_request = 3, .rvbar = 0x10000000001, .status = 0x6a, + }; + + for (unsigned int i = 0; i < ANE_FW_DARTS; i++) { + s.dart[i].version = 0x2a2a0203; + s.dart[i].protect = i == 2 ? 0 : 2; + s.dart[i].unprotect = i == 2 ? 0 : 2; + s.dart[i].lock = i == 2 ? 0 : 2; + } + return s; +} + +static void ane_cold_evidence_test(struct kunit *test) +{ + struct ane_fw_cold_state s = ane_cold_fixture(); + + KUNIT_EXPECT_TRUE(test, ane_fw_cold_valid(&s, true)); + KUNIT_EXPECT_FALSE(test, ane_fw_cold_valid(&s, false)); + /* + * ANE_SYS is no marker: it may read on while genpd reports off, and + * generic DART runtime PM powers it off again. + */ + s.sys = 0x2300; + KUNIT_EXPECT_TRUE(test, ane_fw_cold_valid(&s, true)); +} + +/* + * Each step judges only what its power state makes readable: the probe + * step only PMGR words, so a warm engine cannot fail it, and the engine + * step nothing in PMGR. + */ +static void ane_cold_step_test(struct kunit *test) +{ + struct ane_fw_cold_state s = ane_cold_fixture(); + + s.control = 0x10; + s.dart[0].enabled[0] = 1; + s.gate_request = 0; + KUNIT_EXPECT_TRUE(test, ane_fw_cold_pmgr_valid(&s, true)); + KUNIT_EXPECT_FALSE(test, ane_fw_cold_gate_valid(&s)); + KUNIT_EXPECT_FALSE(test, ane_fw_cold_engine_valid(&s)); + s = ane_cold_fixture(); + s.cpu = 0x0f0003ff; + KUNIT_EXPECT_FALSE(test, ane_fw_cold_pmgr_valid(&s, true)); + KUNIT_EXPECT_TRUE(test, ane_fw_cold_gate_valid(&s)); + KUNIT_EXPECT_TRUE(test, ane_fw_cold_engine_valid(&s)); + s = ane_cold_fixture(); + s.subblock = 0x3ff; + KUNIT_EXPECT_FALSE(test, ane_fw_cold_pmgr_valid(&s, true)); + s = ane_cold_fixture(); + s.gate_ack = 1; + KUNIT_EXPECT_TRUE(test, ane_fw_cold_pmgr_valid(&s, true)); + KUNIT_EXPECT_FALSE(test, ane_fw_cold_gate_valid(&s)); +} + +static void ane_cold_warm_test(struct kunit *test) +{ + struct ane_fw_cold_state s, original = ane_cold_fixture(); + u32 *word; + static const size_t offsets[] = { + offsetof(struct ane_fw_cold_state, cpu), + offsetof(struct ane_fw_cold_state, subblock), + offsetof(struct ane_fw_cold_state, gate_request), + offsetof(struct ane_fw_cold_state, gate_ack), + offsetof(struct ane_fw_cold_state, control), + offsetof(struct ane_fw_cold_state, status), + offsetof(struct ane_fw_cold_state, pending), + }; + + for (unsigned int i = 0; i < ARRAY_SIZE(offsets); i++) { + s = original; + word = (u32 *)((u8 *)&s + offsets[i]); + *word ^= 1; + KUNIT_EXPECT_FALSE(test, ane_fw_cold_valid(&s, true)); + } + for (unsigned int i = 0; i < 8; i++) { + s = original; + s.scratch[i] = 1; + KUNIT_EXPECT_FALSE(test, ane_fw_cold_valid(&s, true)); + } + s = original; + s.rvbar++; + KUNIT_EXPECT_FALSE(test, ane_fw_cold_valid(&s, true)); + s = original; + s.ce_state = 1; + KUNIT_EXPECT_FALSE(test, ane_fw_cold_valid(&s, true)); + s = original; + s.ce_error = BIT_ULL(63); + KUNIT_EXPECT_FALSE(test, ane_fw_cold_valid(&s, true)); +} + +static void ane_cold_dart_test(struct kunit *test) +{ + struct ane_fw_cold_state s, original = ane_cold_fixture(); + + for (unsigned int i = 0; i < ANE_FW_DARTS; i++) { + s = original; + s.dart[i].version++; + KUNIT_EXPECT_FALSE(test, ane_fw_cold_valid(&s, true)); + s = original; + s.dart[i].protect |= BIT(0); + KUNIT_EXPECT_FALSE(test, ane_fw_cold_valid(&s, true)); + s = original; + s.dart[i].lock |= BIT(2); + KUNIT_EXPECT_FALSE(test, ane_fw_cold_valid(&s, true)); + s = original; + s.dart[i].error = 1; + KUNIT_EXPECT_FALSE(test, ane_fw_cold_valid(&s, true)); + /* The bound DART driver enables all streams; that is not a refusal. */ + for (unsigned int j = 0; j < 16; j++) { + s = original; + s.dart[i].enabled[j] = U32_MAX; + KUNIT_EXPECT_TRUE(test, ane_fw_cold_valid(&s, true)); + } + } +} + +static void ane_cold_command_policy_test(struct kunit *test) +{ + static const u16 forbidden[] = { 0x27, 0x28, 0x30, 0x31, 0x34 }; + static const u16 permitted[] = { 0, 3, 4, 0x22, 0x2d, 0x2f, 0x200, 0x202, 0x20c }; + + for (unsigned int i = 0; i < ARRAY_SIZE(forbidden); i++) + KUNIT_EXPECT_TRUE(test, ane_fw_command_denied(forbidden[i])); + for (unsigned int i = 0; i < ARRAY_SIZE(permitted); i++) + KUNIT_EXPECT_FALSE(test, ane_fw_command_denied(permitted[i])); +} + +/* The measured 26.6.2 layout: DATA ends 0x14000 short of the span. */ +static void ane_cold_heap_tail_test(struct kunit *test) +{ + u64 dva[2] = { 0x10000000000ULL, 0x100000c0000ULL }; + u64 size[2] = { 0xc0000, 0x2ac000 }; + const u64 pool = 0x10000384000ULL; + u64 tail = 0, tail_size = 0; + + KUNIT_ASSERT_EQ(test, ane_fw_heap_tail(dva, size, pool, &tail, &tail_size), 0); + KUNIT_EXPECT_EQ(test, tail, 0x1000036c000ULL); + KUNIT_EXPECT_EQ(test, tail_size, 0x14000ULL); + /* The faulting writes of neo-run-0045B lie inside the tail. */ + KUNIT_EXPECT_LE(test, tail, 0x1000037c040ULL); + KUNIT_EXPECT_GT(test, tail + tail_size, 0x1000037fc00ULL); + /* DATA that ends the span leaves nothing to map. */ + size[1] = ANE_FW_SPAN - 0xc0000; + KUNIT_ASSERT_EQ(test, ane_fw_heap_tail(dva, size, pool, &tail, &tail_size), 0); + KUNIT_EXPECT_EQ(test, tail_size, 0ULL); + /* DATA past the span, a partial page, overlap, or the pool: refused. */ + size[1] = ANE_FW_SPAN; + KUNIT_EXPECT_EQ(test, ane_fw_heap_tail(dva, size, pool, &tail, &tail_size), -ERANGE); + size[1] = 0x2ab000; + KUNIT_EXPECT_EQ(test, ane_fw_heap_tail(dva, size, pool, &tail, &tail_size), -ERANGE); + size[1] = 0x2ac000; + dva[1] = 0x100000bc000ULL; + KUNIT_EXPECT_EQ(test, ane_fw_heap_tail(dva, size, pool, &tail, &tail_size), -ERANGE); + dva[1] = 0x100000c0000ULL; + KUNIT_EXPECT_EQ(test, ane_fw_heap_tail(dva, size, 0x10000370000ULL, &tail, + &tail_size), -ERANGE); +} + +static struct kunit_case ane_cold_cases[] = { + KUNIT_CASE(ane_cold_heap_tail_test), + KUNIT_CASE(ane_cold_evidence_test), + KUNIT_CASE(ane_cold_step_test), + KUNIT_CASE(ane_cold_warm_test), + KUNIT_CASE(ane_cold_dart_test), + KUNIT_CASE(ane_cold_command_policy_test), + {} +}; + +static struct kunit_suite ane_cold_suite = { + .name = "ane-fw-cold", + .test_cases = ane_cold_cases, +}; + +kunit_test_suite(ane_cold_suite); + +MODULE_DESCRIPTION("KUnit tests for cold ANE acquisition and H17 program policy"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_fw_platform.c b/drivers/accel/ane/ane_fw_platform.c index 1a8f69a329abe..c3003b6fd857c 100644 --- a/drivers/accel/ane/ane_fw_platform.c +++ b/drivers/accel/ane/ane_fw_platform.c @@ -1,15 +1,18 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* * H16/H17 facts from the 2026-10-31 H16/H17 specification, sections 3-7, - * 12 and 15. This preflight performs no MMIO, DMA mapping or power access. + * 12 and 15, and the H16 cold-boot facts, round 5. Metadata preflight + * performs no MMIO, DMA mapping or power access. */ #include +#include #include +#include #include -#include #include #include "ane_fw_platform.h" +#include "ane_h16_boot_data.h" static const char * const ane_h16_reg_names[] = { "engine", "pmgr", "aux-0", "aux-1", "aux-2", "aux-3", "aux-4", @@ -24,7 +27,18 @@ static const u32 ane_h16_power_words[] = { 0x3c0, 0xc008, 0xc000, 0xc010, 0xc018, }; +static const char * const ane_h17_reg_names[] = { "engine", "pmgr", "pwgate" }; +static const u32 ane_h17_reg_sizes[] = { 0x2000000, 0x18000, 0x34 }; + const struct ane_fw_platform ane_h16_platform = { + .h16_profiles = ane_h16_profiles, + .h16_profile_count = ARRAY_SIZE(ane_h16_profiles), + .h16_mmio = ane_h16_mmio_t6041, + .h16_mmio_count = ARRAY_SIZE(ane_h16_mmio_t6041), + .dart_params = 0x8, + .dapf_dart = 0, + .dapf_offset = 0x10000, + .dapf_size = SZ_16K, /* The DT compatible is t6041; the physical M4 Pro is T6040. */ .soc = 0x6040, .chip_family = 16, @@ -61,55 +75,84 @@ const struct ane_fw_platform ane_h17_platform = { .task_generation = 19, .interrupt_count = 2, .owner = ANE_FW_OWNER_EXCLAVE, + .reg_names = ane_h17_reg_names, + .reg_sizes = ane_h17_reg_sizes, + .reg_count = ARRAY_SIZE(ane_h17_reg_names), + .wrapper = 0x1600000, + .mailbox = 0x1608000, + .scratch = 0x1880020, + .doorbell = 0x1884000, + .irq_status = 0x188c000, + .irq_ack = 0x1890000, .preload_dva = { 0x10000000000, 0x100000c0000 }, .preload_size = { 0xc0000, 0x2ac000 }, - .stream_mask = BIT(0) | BIT(11), + .stream_mask = BIT(0), .firmware_preloaded = true, .safe_reset = false, - /* No H16 register-map inference is executable on H17. */ + /* Cold AP-direct acquisition is staged before any power or DMA write. + * The observer uses the measured H17 26.6.2 register layout. + */ }; +/* + * The iBoot preload is firmware memory. Accept it only as a no-map child of + * /reserved-memory that Linux does not manage as System RAM, whether the + * loader reserved it at boot or left it outside the memory map. The physical + * range comes from the node itself, so the check does not depend on when the + * node was added. + */ +int ane_fw_preload_range(struct device_node *np, unsigned int index, u64 *phys, u64 *size) +{ + struct device_node *reserved __free(device_node) = + of_find_node_by_path("/reserved-memory"); + struct device_node *mem __free(device_node) = + of_parse_phandle(np, "memory-region", index); + struct resource r; + + if (!reserved || !mem || mem->parent != reserved || + !of_device_is_available(mem) || + !of_property_read_bool(mem, "no-map") || + of_property_read_bool(mem, "reusable") || + of_address_to_resource(mem, 0, &r) || !resource_size(&r)) + return -EINVAL; + if (region_intersects(r.start, resource_size(&r), IORESOURCE_SYSTEM_RAM, + IORES_DESC_NONE) != REGION_DISJOINT) + return -EBUSY; + *phys = r.start; + *size = resource_size(&r); + return 0; +} +EXPORT_SYMBOL_GPL(ane_fw_preload_range); + static int ane_fw_preload_check(struct device *dev, const struct ane_fw_platform *soc) { struct device_node *np = dev->of_node; - struct device_node *reserved __free(device_node) = - of_find_node_by_path("/reserved-memory"); u64 phys[2], size[2], dva[2]; unsigned int i; + int ret; - if (!reserved || - of_property_count_u32_elems(np, "memory-region") != 2 || + if (of_property_count_u32_elems(np, "memory-region") != 2 || of_property_count_strings(np, "memory-region-names") != 2 || of_property_count_u64_elems(np, "apple,firmware-iovas") != 2 || of_property_read_u64_array(np, "apple,firmware-iovas", dva, 2)) return -ENODATA; for (i = 0; i < 2; i++) { - struct device_node *mem; - struct reserved_mem *rmem; + struct device_node *mem __free(device_node) = + of_parse_phandle(np, "memory-region", i); const char *name; - bool valid; + /* Board DTs carry disabled placeholders the loader fills in. */ + if (mem && !of_device_is_available(mem)) + return -ENODEV; if (of_property_read_string_index(np, "memory-region-names", i, &name) || strcmp(name, i ? "firmware-data" : "firmware-text")) return -EINVAL; - mem = of_parse_phandle(np, "memory-region", i); - if (!mem) - return -EINVAL; - rmem = of_reserved_mem_lookup(mem); - valid = rmem && mem->parent == reserved && - of_device_is_available(mem) && - of_property_read_bool(mem, "no-map") && - !of_property_read_bool(mem, "reusable"); - if (valid) { - phys[i] = rmem->base; - size[i] = rmem->size; - } - of_node_put(mem); - if (!valid) - return -EINVAL; + ret = ane_fw_preload_range(np, i, &phys[i], &size[i]); + if (ret) + return ret; } return ane_fw_preload_ranges_valid(soc, phys, size, dva) ? 0 : -EINVAL; @@ -129,7 +172,7 @@ int ane_fw_platform_preflight(struct platform_device *pdev, * Check before resources, firmware, power, IOMMU or engine access. */ if (soc->owner == ANE_FW_OWNER_EXCLAVE) { - dev_info(dev, "H17 ANE: Linux exclave ownership handoff unavailable; refusing probe\n"); + dev_info(dev, "H17 ANE: cold acquisition requires staged preflight; refusing direct probe\n"); return -EOPNOTSUPP; } if (of_property_read_bool(np, "apple,exclave-assigned")) { @@ -155,14 +198,30 @@ int ane_fw_platform_preflight(struct platform_device *pdev, } ret = ane_fw_preload_check(dev, soc); + if (ret == -ENODEV) { + dev_info(dev, "H16 firmware preload not handed off by the bootloader\n"); + return ret; + } if (ret) return dev_err_probe(dev, ret, "H16 requires bootloader-reserved no-map TEXT/DATA preload ranges\n"); - /* Original HWX validation does not establish firmware ownership, - * preload integrity, startup state or a decoded process resource map. - * Keep the section-only ABI and incomplete boot backend refused. - */ - dev_info(dev, "H16 ANE: preload validation, HWX ABI and boot backend incomplete; refusing before power access\n"); - return -EOPNOTSUPP; + if (!soc->h16_profile_count || !soc->h16_mmio_count || soc->dapf_dart >= 3 || + !soc->dapf_size || of_property_count_u64_elems(np, "apple,dapf-ranges") != 8) + return -ENODATA; + return 0; +} + +int ane_fw_platform_probe(struct platform_device *pdev, const struct ane_fw_platform *soc) +{ + int ret = ane_fw_platform_preflight(pdev, soc); + + if (ret) + return ret; + return ane_h16_backend_probe(pdev, soc); +} + +void ane_fw_platform_remove(struct platform_device *pdev) +{ + ane_h16_backend_remove(pdev); } diff --git a/drivers/accel/ane/ane_fw_platform.h b/drivers/accel/ane/ane_fw_platform.h index a7d4ed4f344f0..6e0e5a211415a 100644 --- a/drivers/accel/ane/ane_fw_platform.h +++ b/drivers/accel/ane/ane_fw_platform.h @@ -9,7 +9,10 @@ #include #include +struct device_node; struct platform_device; +struct ane_h16_profile; +struct ane_h16_mmio; enum ane_fw_owner { ANE_FW_OWNER_AP, @@ -23,6 +26,17 @@ enum ane_fw_irq_role { /* Later firmware generations must never enter the H14 staging path. */ struct ane_fw_platform { + const struct ane_h16_profile *h16_profiles; + unsigned int h16_profile_count; + /* DART/DAPF register tables, one per supported DART revision. */ + const struct ane_h16_mmio *h16_mmio; + unsigned int h16_mmio_count; + /* DART PARAMS3, the one DART register read before a table is chosen. */ + u32 dart_params; + /* The DAPF bank: iommus entry that owns it, offset from its reg, size. */ + u32 dapf_dart; + u32 dapf_offset; + u32 dapf_size; u32 soc; u16 chip_family; u16 ane_type; @@ -71,4 +85,11 @@ extern const struct ane_fw_platform ane_h16_platform, ane_h17_platform; int ane_fw_platform_preflight(struct platform_device *pdev, const struct ane_fw_platform *soc); +int ane_fw_platform_probe(struct platform_device *pdev, + const struct ane_fw_platform *soc); +void ane_fw_platform_remove(struct platform_device *pdev); +int ane_fw_preload_range(struct device_node *np, unsigned int index, u64 *phys, u64 *size); +int ane_h16_backend_probe(struct platform_device *pdev, const struct ane_fw_platform *soc); +void ane_h16_backend_remove(struct platform_device *pdev); + #endif /* __ANE_FW_PLATFORM_H__ */ diff --git a/drivers/accel/ane/ane_fw_protocol.h b/drivers/accel/ane/ane_fw_protocol.h index ec2aca2b35845..4910d2016ee8a 100644 --- a/drivers/accel/ane/ane_fw_protocol.h +++ b/drivers/accel/ane/ane_fw_protocol.h @@ -25,7 +25,8 @@ struct ane_fw_channel { static inline bool ane_fw_contains(u64 base, u64 bytes, u64 addr, u64 size) { - return size <= bytes && addr >= base && addr - base <= bytes - size; + return bytes <= U64_MAX - base && size <= bytes && addr >= base && + addr - base <= bytes - size; } /* Parse a host-owned descriptor snapshot; output indices are logical names. */ @@ -42,7 +43,8 @@ ane_fw_channels_parse(const void *table, u64 ipc_dva, size_t ipc_size, u32 seen = 0; unsigned int i, j; - if (!table || !ane_fw_contains(ipc_dva, ipc_size, table_dva, ANE_FW_TABLE_SIZE)) + if (!table || !channels || + !ane_fw_contains(ipc_dva, ipc_size, table_dva, ANE_FW_TABLE_SIZE)) return -EPROTO; table_off = table_dva - ipc_dva; for (i = 0; i < ANE_FW_CHANNELS; i++) { @@ -76,7 +78,8 @@ ane_fw_channels_parse(const void *table, u64 ipc_dva, size_t ipc_size, return -EPROTO; } if (seen != GENMASK(ANE_FW_CHANNELS - 1, 0) || - parsed[1].type != 0 || parsed[5].type != 1) + parsed[1].type != 0 || parsed[5].type != 1 || + parsed[6].type != 1 || parsed[7].type != 0) return -EPROTO; for (i = 0; i < ANE_FW_CHANNELS; i++) { u64 end = parsed[i].offset + parsed[i].slots * ANE_FW_SLOT_SIZE; @@ -112,6 +115,24 @@ ane_fw_channels_host_init(void *ipc, const struct ane_fw_channel *channels) } /* H14 preserves its reserved word; H16 requires bytes 0..3 to be zero. */ +/* + * Commands that change engine mode or ownership: secure mode 0x27, 0x28 and + * 0x34, exclave mode 0x30 and 0x31. Linux never publishes them. + */ +static inline bool ane_fw_command_denied(u16 id) +{ + switch (id) { + case 0x27: + case 0x28: + case 0x30: + case 0x31: + case 0x34: + return true; + default: + return false; + } +} + static inline void ane_fw_command_header(void *command, u16 opcode, bool zero_reserved) { u8 *p = command; @@ -130,6 +151,8 @@ ane_fw_reply_status(u64 header, u64 slot_length, u64 slot_result, { const u8 *p = command; + if (header & 2) + return -EPROTO; if (!(header & 1)) return -EAGAIN; if (!command || length < 8 || (command_dva & 1) || header != (command_dva | 1) || diff --git a/drivers/accel/ane/ane_fw_transport.h b/drivers/accel/ane/ane_fw_transport.h new file mode 100644 index 0000000000000..2696cdf77a162 --- /dev/null +++ b/drivers/accel/ane/ane_fw_transport.h @@ -0,0 +1,607 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_FW_TRANSPORT_H__ +#define __ANE_FW_TRANSPORT_H__ + +#include "ane_fw_platform.h" +#include "ane_fw_protocol.h" +#include "ane_h16_execute.h" + +#define ANE_FW_IO 1 +#define ANE_FW_SHAREDMALLOC 5 +#define ANE_FW_IO_T2H 6 +#define ANE_FW_DATA_CHAIN 7 +#define ANE_FW_SHARED_MAX 64 +#define ANE_FW_SHARED_LIMIT SZ_1M +#define ANE_FW_SHARED_TOTAL (12 * SZ_1M) +#define ANE_FW_SHARED_EXPP 0x45585050U +/* The firmware's text log, 0x80-byte records. */ +#define ANE_FW_SHARED_LOG 0x004c4f47U + +struct ane_fw_surface { + void *cpu; + u64 dva; + size_t size; +}; + +/* + * Sleep bounds between two polls of one wait. Command replies arrive within + * tens of microseconds and an EXECUTE completes within a few milliseconds, + * so the first 5 ms poll every 20-50 us: a reply is seen at most ~50 us late + * instead of a whole millisecond. Long waits (firmware startup, a stalled + * request) back off to 250 us and, after 50 ms, to the 1 ms cadence, so + * they cost no more CPU than a fixed 1 ms poll did. + */ +#define ANE_FW_POLL_FINE_NS (5ULL * 1000 * 1000) +#define ANE_FW_POLL_MEDIUM_NS (50ULL * 1000 * 1000) + +static inline unsigned int ane_fw_poll_sleep_us(u64 waited_ns, unsigned int *max_us) +{ + unsigned int us = waited_ns < ANE_FW_POLL_FINE_NS ? 20 : + waited_ns < ANE_FW_POLL_MEDIUM_NS ? 250 : 1000; + + *max_us = us == 20 ? 50 : us + 50; + return us; +} + +/* + * Accessors operate on coherent owned memory and engine-relative registers. + * Barriers must supply DMA read/write ordering, not just compiler ordering. + * alloc returns a zeroed, pinned, private-domain mapping of exactly size bytes. + * It and its caller retain every mapping, including a rejected result, until + * reset. refused must log the tuple and error without exposing private data. + */ +struct ane_fw_transport_ops { + u64 (*load64)(void *ctx, const void *p); + void (*store64)(void *ctx, void *p, u64 value); + void (*read_barrier)(void *ctx); + void (*write_barrier)(void *ctx); + u32 (*read32)(void *ctx, u32 reg); + void (*write32)(void *ctx, u32 reg, u32 value); + u64 (*now_ns)(void *ctx); + /* Wait before the next poll of a wait that began waited_ns ago. */ + void (*relax)(void *ctx, u64 waited_ns); + int (*alloc)(void *ctx, size_t size, u32 tag, struct ane_fw_surface *surface); + void (*refused)(void *ctx, u64 word0, u64 word1, u64 word2, int error); +}; + +struct ane_fw_transaction { + u64 dva; + u64 deadline_ns; + size_t size; + u16 id; + bool active; +}; + +/* Caller serializes all entry points. No surface is released by this object. */ +struct ane_fw_transport { + const struct ane_fw_transport_ops *ops; + const struct ane_fw_platform *soc; + void *ctx; + struct ane_fw_surface ipc; + struct ane_fw_surface pool; + struct ane_fw_surface shared[ANE_FW_SHARED_MAX]; + bool shared_retired[ANE_FW_SHARED_MAX]; + u32 shared_tags[ANE_FW_SHARED_MAX]; + /* FREE reply form: 1 (H16 26A428) or DVA|1 (H17 25G83); else refused. */ + bool native_free; + bool allow_free; + u32 free_count; + /* H17: program and process of the last published EXECUTE. */ + bool bar_known; + u32 bar_program, bar_process; + struct ane_fw_channel channels[ANE_FW_CHANNELS]; + struct ane_fw_transaction pending[ANE_FW_CHANNELS]; + u32 head[ANE_FW_CHANNELS]; + u8 table[ANE_FW_TABLE_SIZE]; + size_t table_offset; + size_t pool_used; + size_t shared_bytes; + u32 shared_count; + int error; + /* relax() calls made by every wait so far; callers diff it. */ + u64 polls; + bool armed; +}; + +/* The live shared allocation the firmware asked for under tag, or NULL. */ +static inline const struct ane_fw_surface * +ane_fw_transport_shared_tagged(const struct ane_fw_transport *t, u32 tag) +{ + u32 i; + + for (i = 0; i < t->shared_count; i++) + if (t->shared_tags[i] == tag && !t->shared_retired[i]) + return &t->shared[i]; + return NULL; +} + +static inline bool ane_fw_surface_valid(const struct ane_fw_surface *s) +{ + return s->cpu && s->size && s->size <= U64_MAX - s->dva; +} + +static inline bool +ane_fw_surfaces_overlap(const struct ane_fw_surface *a, const struct ane_fw_surface *b) +{ + return a->dva < b->dva + b->size && b->dva < a->dva + a->size; +} + +static inline int ane_fw_transport_fail(struct ane_fw_transport *t, int error) +{ + if (!t->error) + t->error = error; + return t->error; +} + +/* pool_reserved protects the init record and any other boot-owned pool bytes. */ +static inline int +ane_fw_transport_init(struct ane_fw_transport *t, const struct ane_fw_platform *soc, + const struct ane_fw_transport_ops *ops, void *ctx, + struct ane_fw_surface ipc, struct ane_fw_surface pool, + size_t pool_reserved, u64 table_dva, size_t table_size) +{ + int ret; + + if (!t || !soc || !ops || !ops->load64 || !ops->store64 || + !ops->read_barrier || !ops->write_barrier || !ops->read32 || + !ops->write32 || !ops->now_ns || !ops->relax || !ops->alloc || + !ops->refused || !ane_fw_surface_valid(&ipc) || + !ane_fw_surface_valid(&pool) || ane_fw_surfaces_overlap(&ipc, &pool) || + !IS_ALIGNED(pool.dva, ANE_FW_SLOT_SIZE) || pool_reserved > pool.size || + table_size != ANE_FW_TABLE_SIZE || + !ane_fw_contains(ipc.dva, ipc.size, table_dva, table_size)) + return -EINVAL; + memset(t, 0, sizeof(*t)); + t->ops = ops; + t->soc = soc; + t->ctx = ctx; + t->ipc = ipc; + t->pool = pool; + t->pool_used = pool_reserved; + t->table_offset = table_dva - ipc.dva; + ops->read_barrier(ctx); + memcpy(t->table, (u8 *)ipc.cpu + t->table_offset, sizeof(t->table)); + ret = ane_fw_channels_parse(t->table, ipc.dva, ipc.size, table_dva, t->channels); + return ret ? ane_fw_transport_fail(t, ret) : 0; +} + +static inline int ane_fw_transport_check(struct ane_fw_transport *t) +{ + if (t->error) + return t->error; + t->ops->read_barrier(t->ctx); + if (memcmp(t->table, (u8 *)t->ipc.cpu + t->table_offset, sizeof(t->table))) + return ane_fw_transport_fail(t, -EPROTO); + return 0; +} + +static inline void *ane_fw_transport_slot(struct ane_fw_transport *t, unsigned int ch) +{ + return (u8 *)t->ipc.cpu + t->channels[ch].offset + + t->head[ch] * ANE_FW_SLOT_SIZE; +} + +static inline void ane_fw_transport_advance(struct ane_fw_transport *t, unsigned int ch) +{ + t->head[ch] = (t->head[ch] + 1) % t->channels[ch].slots; +} + +static inline void ane_fw_transport_doorbell(struct ane_fw_transport *t, unsigned int ch) +{ + t->ops->write32(t->ctx, t->soc->doorbell, BIT(t->channels[ch].bit)); +} + +static inline int ane_fw_transport_arm(struct ane_fw_transport *t) +{ + unsigned int ch, i; + int ret = ane_fw_transport_check(t); + + if (ret) + return ret; + if (t->armed) + return -EBUSY; + for (ch = 0; ch < ANE_FW_CHANNELS; ch++) { + const struct ane_fw_channel *c = &t->channels[ch]; + + if (c->type) + continue; + for (i = 0; i < c->slots; i++) { + u8 *p = (u8 *)t->ipc.cpu + c->offset + i * ANE_FW_SLOT_SIZE; + + t->ops->store64(t->ctx, p + 8, 0); + t->ops->store64(t->ctx, p + 16, 0); + t->ops->store64(t->ctx, p, 1); + } + } + t->ops->write_barrier(t->ctx); + t->armed = true; + return 0; +} + +/* + * The H17 firmware's direct submission path reuses the engine's current BAR + * context. When that context came from a different program, a job (of a new + * or previously loaded program, even after the earlier one was destroyed) can + * write through the earlier program's output binding. Bit 0 of the request + * flags word queues the job for the full execution loop instead, which + * installs a fresh BAR context. Queue the first EXECUTE after any change of program, process or + * program lifetime; later jobs of the same process keep the direct path. This + * lifts the one-program-per-firmware-session limit. Applies to the pool copy, + * so every published EXECUTE follows the rule. H16 shares the firmware + * architecture but has not executed under Linux yet; it is left unchanged. + */ +static inline void +ane_fw_transport_bar_context(struct ane_fw_transport *t, u8 *copy, size_t size) +{ + u16 id = get_unaligned_le16(copy + 4); + u32 program, process; + + if (id != 0x20c) { + /* LOAD, UNLOAD, CREATE and TERMINATE may reuse IDs. */ + if (id >= 0x200 && id <= 0x203) + t->bar_known = false; + return; + } + if (size < ANE_H16_EXECUTE_FLAGS + 4) { + t->bar_known = false; + return; + } + program = get_unaligned_le32(copy + 8); + process = get_unaligned_le32(copy + 0xc); + if (!t->bar_known || t->bar_program != program || t->bar_process != process) + put_unaligned_le32(get_unaligned_le32(copy + ANE_H16_EXECUTE_FLAGS) | + ANE_H16_EXECUTE_FLAG_QUEUED, copy + ANE_H16_EXECUTE_FLAGS); + t->bar_known = true; + t->bar_program = program; + t->bar_process = process; +} + +/* + * Whether one firmware session may run several programs: the transport + * queues the first EXECUTE after every program change (H17, above). H16 + * shares the firmware design, but the rule is unconfirmed there. + */ +static inline bool ane_fw_transport_switch_safe(const struct ane_fw_transport *t) +{ + return t->soc->chip_family == 17; +} + +/* Commands are copied into a monotonic pool arena, never external DMA memory. */ +static inline int +ane_fw_transport_publish(struct ane_fw_transport *t, unsigned int ch, + const void *command, size_t size, u64 deadline_ns) +{ + struct ane_fw_transaction *tx; + size_t offset; + u8 *p; + u64 header; + int ret = ane_fw_transport_check(t); + + if (ret) + return ret; + if (!t->armed || (ch != ANE_FW_IO && ch != ANE_FW_DATA_CHAIN) || + !command || size < 8 || get_unaligned_le32(command) || + get_unaligned_le16((const u8 *)command + 6)) + return -EINVAL; + if (ane_fw_command_denied(get_unaligned_le16((const u8 *)command + 4))) + return -EPERM; + tx = &t->pending[ch]; + if (tx->active) + return -EBUSY; + if (t->ops->now_ns(t->ctx) >= deadline_ns) + return ane_fw_transport_fail(t, -ETIMEDOUT); + p = ane_fw_transport_slot(t, ch); + header = t->ops->load64(t->ctx, p); + if (header & 2) + return ane_fw_transport_fail(t, -EPROTO); + if (!(header & 1)) + return -EBUSY; + if (t->pool_used > SIZE_MAX - (ANE_FW_SLOT_SIZE - 1)) + return -ENOSPC; + offset = ALIGN(t->pool_used, ANE_FW_SLOT_SIZE); + if (offset > t->pool.size || size > t->pool.size - offset) + return -ENOSPC; + memcpy((u8 *)t->pool.cpu + offset, command, size); + *tx = (struct ane_fw_transaction) { + .dva = t->pool.dva + offset, + .deadline_ns = deadline_ns, + .size = size, + .id = get_unaligned_le16((const u8 *)command + 4), + .active = true, + }; + t->pool_used = offset + size; + if (ane_fw_transport_switch_safe(t)) + ane_fw_transport_bar_context(t, (u8 *)t->pool.cpu + offset, size); + t->ops->store64(t->ctx, p + 8, size); + t->ops->store64(t->ctx, p + 16, size); + t->ops->write_barrier(t->ctx); + t->ops->store64(t->ctx, p, tx->dva); + t->ops->write_barrier(t->ctx); + ane_fw_transport_doorbell(t, ch); + return 0; +} + +/* Command arena position, for ane_fw_transport_release(). */ +static inline size_t ane_fw_transport_mark(const struct ane_fw_transport *t) +{ + return t->pool_used; +} + +/* + * Give back the arena used by every command published since @mark. The caller + * guarantees that the firmware is done with all of them, not only that they + * were acknowledged: an EXECUTE is done once its completion report arrived. + * Refused while any channel still has a command in flight. + */ +static inline int ane_fw_transport_release(struct ane_fw_transport *t, size_t mark) +{ + unsigned int ch; + + if (t->error || mark > t->pool_used || mark > t->pool.size) + return -EINVAL; + for (ch = 0; ch < ANE_FW_CHANNELS; ch++) + if (t->pending[ch].active) + return -EBUSY; + t->pool_used = mark; + return 0; +} + +/* On success/refusal the reply remains in the pool for the caller to inspect. */ +static inline int ane_fw_transport_poll(struct ane_fw_transport *t, unsigned int ch) +{ + struct ane_fw_transaction *tx; + u64 header, length, result; + u8 *p; + int ret = ane_fw_transport_check(t); + + if (ret) + return ret; + if (ch != ANE_FW_IO && ch != ANE_FW_DATA_CHAIN) + return -EINVAL; + tx = &t->pending[ch]; + if (!tx->active) + return -EINVAL; + if (t->ops->now_ns(t->ctx) >= tx->deadline_ns) + return ane_fw_transport_fail(t, -ETIMEDOUT); + p = ane_fw_transport_slot(t, ch); + header = t->ops->load64(t->ctx, p); + if (header & 2) + return ane_fw_transport_fail(t, -EPROTO); + if (!(header & 1)) { + if (header != tx->dva) + return ane_fw_transport_fail(t, -EPROTO); + return -EAGAIN; + } + t->ops->read_barrier(t->ctx); + length = t->ops->load64(t->ctx, p + 8); + result = t->ops->load64(t->ctx, p + 16); + ret = ane_fw_reply_status(header, length, result, tx->dva, tx->size, + tx->id, (u8 *)t->pool.cpu + (tx->dva - t->pool.dva), true); + if (ret) + return ane_fw_transport_fail(t, ret); + tx->active = false; + ane_fw_transport_advance(t, ch); + return 0; +} + +static inline bool ane_fw_transport_present(u64 word0, u64 word1) +{ + return !(word0 & 1) && (word0 || word1); +} + +static inline const void * +ane_fw_transport_report(struct ane_fw_transport *t, u64 dva, u64 size) +{ + const struct ane_fw_surface *s; + unsigned int i; + + for (i = 0; i < t->shared_count + 2; i++) { + if (i >= 2 && t->shared_retired[i - 2]) + continue; + s = i == 0 ? &t->ipc : i == 1 ? &t->pool : &t->shared[i - 2]; + if (ane_fw_contains(s->dva, s->size, dva, size)) + return (u8 *)s->cpu + (dva - s->dva); + } + return NULL; +} + +/* Copy before acknowledging. UUID dispatch is deliberately above the transport. */ +static inline int +ane_fw_transport_consume(struct ane_fw_transport *t, void *report, size_t capacity, + size_t *size) +{ + const void *src; + u64 word0, word1, word2; + u8 *p; + int ret = ane_fw_transport_check(t); + + if (ret) + return ret; + if (!t->armed || !report || !size) + return -EINVAL; + p = ane_fw_transport_slot(t, ANE_FW_IO_T2H); + word0 = t->ops->load64(t->ctx, p); + t->ops->read_barrier(t->ctx); + word1 = t->ops->load64(t->ctx, p + 8); + if (!ane_fw_transport_present(word0, word1)) + return -EAGAIN; + t->ops->read_barrier(t->ctx); + word2 = t->ops->load64(t->ctx, p + 16); + if ((word0 & 3) || word1 < 8 || word2) + return ane_fw_transport_fail(t, -EPROTO); + src = ane_fw_transport_report(t, word0, word1); + if (!src) + return ane_fw_transport_fail(t, -EPROTO); + if (word1 > capacity) + return -ENOSPC; + memcpy(report, src, word1); + *size = word1; + /* The copy must finish before firmware is allowed to recycle the report. */ + t->ops->write_barrier(t->ctx); + t->ops->store64(t->ctx, p, word0 | 1); + t->ops->write_barrier(t->ctx); + ane_fw_transport_doorbell(t, ANE_FW_IO_T2H); + ane_fw_transport_advance(t, ANE_FW_IO_T2H); + return 0; +} + +/* Conservative default allocation policy. Native FREE requires build opt-in. */ +static inline int ane_fw_shared_size(u64 word0, u64 bytes, u64 tag, size_t *size) +{ + if (word0 || !bytes) + return -EOPNOTSUPP; + if (tag > U32_MAX) + return -E2BIG; + if (bytes > ANE_FW_SHARED_LIMIT && + (bytes != SZ_2M || tag != ANE_FW_SHARED_EXPP)) + return -E2BIG; + *size = ALIGN(bytes, SZ_16K); + return 0; +} + +/* + * SHAREDMALLOC FREE of a live allocation. Backing storage, its mapping and + * its quota stay until reset; FREE only retires the surface, so no later + * report resolves into it. Reply before revoking authority. H16 26A428 (r5 + * section 6) takes the reply 1. H17 25G83 takes the observed DVA|1, and its + * request carries zero length and handle. + */ +static inline int +ane_fw_transport_free(struct ane_fw_transport *t, void *slot, u64 dva, u64 bytes, u64 tag) +{ + unsigned int i; + + if (!t->native_free && !t->allow_free) + return -EOPNOTSUPP; + if (!t->native_free && (bytes || tag || !IS_ALIGNED(dva, SZ_16K))) + return -EPROTO; + for (i = 0; i < t->shared_count; i++) + if (!t->shared_retired[i] && dva == t->shared[i].dva) + break; + if (i == t->shared_count) + return -ENOENT; + t->ops->store64(t->ctx, (u8 *)slot + 8, 0); + t->ops->store64(t->ctx, (u8 *)slot + 16, 0); + t->ops->write_barrier(t->ctx); + t->ops->store64(t->ctx, slot, t->native_free ? 1 : dva | 1); + t->ops->write_barrier(t->ctx); + ane_fw_transport_doorbell(t, ANE_FW_SHAREDMALLOC); + t->shared_retired[i] = true; + t->free_count++; + ane_fw_transport_advance(t, ANE_FW_SHAREDMALLOC); + return 0; +} + +static inline int ane_fw_transport_shared(struct ane_fw_transport *t) +{ + struct ane_fw_surface s = {}; + u64 word0, word1, word2; + size_t size; + unsigned int i; + u8 *p; + int ret = ane_fw_transport_check(t); + + if (ret) + return ret; + if (!t->armed) + return -EINVAL; + p = ane_fw_transport_slot(t, ANE_FW_SHAREDMALLOC); + word0 = t->ops->load64(t->ctx, p); + t->ops->read_barrier(t->ctx); + word1 = t->ops->load64(t->ctx, p + 8); + if (word0 & 1) + return -EAGAIN; + t->ops->read_barrier(t->ctx); + word2 = t->ops->load64(t->ctx, p + 16); + if (!word0 && !word1 && !word2) + return -EAGAIN; + if (word0) { + ret = ane_fw_transport_free(t, p, word0, word1, word2); + if (ret) + goto refused; + return 0; + } + ret = ane_fw_shared_size(word0, word1, word2, &size); + if (ret) + goto refused; + if (t->shared_count == ANE_FW_SHARED_MAX || size > ANE_FW_SHARED_TOTAL - + t->shared_bytes) { + ret = -ENOSPC; + goto refused; + } + ret = t->ops->alloc(t->ctx, size, word2, &s); + if (ret) + goto refused; + if (!ane_fw_surface_valid(&s) || !IS_ALIGNED(s.dva, SZ_16K) || + s.size != size || ane_fw_surfaces_overlap(&s, &t->ipc) || + ane_fw_surfaces_overlap(&s, &t->pool)) { + ret = -EPROTO; + goto refused; + } + for (i = 0; i < t->shared_count; i++) { + if (ane_fw_surfaces_overlap(&s, &t->shared[i])) { + ret = -EPROTO; + goto refused; + } + } + t->shared_tags[t->shared_count] = word2; + t->shared[t->shared_count++] = s; + t->shared_bytes += size; + t->ops->store64(t->ctx, p + 8, 0); + t->ops->store64(t->ctx, p + 16, t->shared_count); + t->ops->write_barrier(t->ctx); + t->ops->store64(t->ctx, p, s.dva | 1); + t->ops->write_barrier(t->ctx); + ane_fw_transport_doorbell(t, ANE_FW_SHAREDMALLOC); + ane_fw_transport_advance(t, ANE_FW_SHAREDMALLOC); + return 0; +refused: + t->ops->refused(t->ctx, word0, word1, word2, ret); + return ane_fw_transport_fail(t, ret); +} + +/* Read once, reject unknown channels, acknowledge exactly that snapshot. */ +static inline int ane_fw_transport_irq(struct ane_fw_transport *t, u32 *pending) +{ + u32 value; + + if (t->error) + return t->error; + if (!pending) + return -EINVAL; + value = t->ops->read32(t->ctx, t->soc->irq_status); + if (value & ~GENMASK(7, 0)) + return ane_fw_transport_fail(t, -EPROTO); + *pending = value; + if (value) + t->ops->write32(t->ctx, t->soc->irq_ack, value); + return 0; +} + +/* The original absolute command deadline survives every provider/IRQ service. */ +static inline int ane_fw_transport_wait(struct ane_fw_transport *t, unsigned int ch) +{ + u64 start = t->ops->now_ns(t->ctx), now; + u32 pending; + int ret; + + for (;;) { + ret = ane_fw_transport_poll(t, ch); + if (ret != -EAGAIN) + return ret; + ret = ane_fw_transport_irq(t, &pending); + if (ret) + return ret; + ret = ane_fw_transport_shared(t); + if (ret && ret != -EAGAIN) + return ret; + /* Recheck before sleeping, including time spent in alloc(). */ + now = t->ops->now_ns(t->ctx); + if (now >= t->pending[ch].deadline_ns) + return ane_fw_transport_fail(t, -ETIMEDOUT); + t->polls++; + t->ops->relax(t->ctx, now - start); + } +} + +#endif /* __ANE_FW_TRANSPORT_H__ */ diff --git a/drivers/accel/ane/ane_fw_transport_test.c b/drivers/accel/ane/ane_fw_transport_test.c new file mode 100644 index 0000000000000..a2c7a59f1b26f --- /dev/null +++ b/drivers/accel/ane/ane_fw_transport_test.c @@ -0,0 +1,1166 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include + +#include "ane_fw_transport.h" +#include "ane_h16_execute.h" + +#define TEST_IPC_DVA 0x100003d0000ULL +#define TEST_POOL_DVA 0x10000384000ULL +#define TEST_SHARED_DVA 0x10002000000ULL +#define TEST_IPC_SIZE SZ_64K +#define TEST_POOL_SIZE SZ_256K +#define TEST_RESERVED SZ_16K +#define TEST_DEADLINE 100ULL + +struct ane_test_access { + char kind; + const void *p; + u64 value; +}; + +struct ane_transport_test { + struct kunit *test; + struct ane_fw_transport t; + struct ane_fw_platform soc; + struct ane_fw_surface ipc; + struct ane_fw_surface pool; + struct ane_test_access access[128]; + unsigned int count; + u64 now; + u64 alloc_time; + u64 waited; + u64 next_dva; + u32 irq; + u32 last_reg; + u32 last_value; + unsigned int allocs; + unsigned int refusals; + u64 refused[3]; + int refused_error; + int alloc_error; + int alloc_bad; +}; + +static void ane_test_access(struct ane_transport_test *f, char kind, const void *p, u64 value) +{ + if (f->count < ARRAY_SIZE(f->access)) + f->access[f->count++] = (struct ane_test_access) { kind, p, value }; +} + +static u64 ane_test_load(void *ctx, const void *p) +{ + struct ane_transport_test *f = ctx; + u64 value = get_unaligned_le64(p); + + ane_test_access(f, 'r', p, value); + return value; +} + +static void ane_test_store(void *ctx, void *p, u64 value) +{ + struct ane_transport_test *f = ctx; + + ane_test_access(f, 'w', p, value); + put_unaligned_le64(value, p); +} + +static void ane_test_rmb(void *ctx) +{ + ane_test_access(ctx, 'R', NULL, 0); +} + +static void ane_test_wmb(void *ctx) +{ + ane_test_access(ctx, 'W', NULL, 0); +} + +static u32 ane_test_read32(void *ctx, u32 reg) +{ + struct ane_transport_test *f = ctx; + + f->last_reg = reg; + ane_test_access(f, 'i', NULL, reg); + return f->irq; +} + +static void ane_test_write32(void *ctx, u32 reg, u32 value) +{ + struct ane_transport_test *f = ctx; + + f->last_reg = reg; + f->last_value = value; + ane_test_access(f, 'o', NULL, value); +} + +static u64 ane_test_now(void *ctx) +{ + return ((struct ane_transport_test *)ctx)->now; +} + +static void ane_test_relax(void *ctx, u64 waited_ns) +{ + struct ane_transport_test *f = ctx; + + f->waited = waited_ns; + f->now++; +} + +static int ane_test_alloc(void *ctx, size_t size, u32 tag, struct ane_fw_surface *s) +{ + struct ane_transport_test *f = ctx; + + f->allocs++; + f->now += f->alloc_time; + if (f->alloc_error) + return f->alloc_error; + *s = (struct ane_fw_surface) { + .cpu = kunit_kzalloc(f->test, size, GFP_KERNEL), + .dva = f->next_dva, + .size = size, + }; + if (!s->cpu) + return -ENOMEM; + f->next_dva += size; + switch (f->alloc_bad) { + case 1: + s->dva++; + break; + case 2: + s->size--; + break; + case 3: + s->dva = f->pool.dva; + break; + case 4: + s->dva = f->ipc.dva; + break; + case 5: + s->dva = U64_MAX - SZ_16K + 1; + break; + case 6: + s->cpu = NULL; + break; + } + return 0; +} + +static void ane_test_refused(void *ctx, u64 word0, u64 word1, u64 word2, int error) +{ + struct ane_transport_test *f = ctx; + + f->refusals++; + f->refused[0] = word0; + f->refused[1] = word1; + f->refused[2] = word2; + f->refused_error = error; +} + +static const struct ane_fw_transport_ops ane_test_ops = { + .load64 = ane_test_load, + .store64 = ane_test_store, + .read_barrier = ane_test_rmb, + .write_barrier = ane_test_wmb, + .read32 = ane_test_read32, + .write32 = ane_test_write32, + .now_ns = ane_test_now, + .relax = ane_test_relax, + .alloc = ane_test_alloc, + .refused = ane_test_refused, +}; + +static void ane_test_table(struct ane_transport_test *f) +{ + static const char * const names[] = { + "TERMINAL", "IO", "DEBUG", "BUF_H2T", "BUF_T2H", + "SHAREDMALLOC", "IO_T2H", "DATA_CHAIN_H2T", + }; + unsigned int i; + + memset(f->ipc.cpu, 0, f->ipc.size); + for (i = 0; i < ANE_FW_CHANNELS; i++) { + u8 *p = (u8 *)f->ipc.cpu + i * ANE_FW_DESCRIPTOR_SIZE; + + strscpy((char *)p, names[i], 64); + put_unaligned_le32(i == 0 ? 2 : (i >= 4 && i <= 6), p + 0x40); + put_unaligned_le32(i, p + 0x44); + put_unaligned_le64(2, p + 0x48); + put_unaligned_le64(f->ipc.dva + 0x800 + i * 0x80, p + 0x50); + } +} + +static int ane_test_init_transport(struct ane_transport_test *f) +{ + return ane_fw_transport_init(&f->t, &f->soc, &ane_test_ops, f, + f->ipc, f->pool, TEST_RESERVED, + f->ipc.dva, ANE_FW_TABLE_SIZE); +} + +static int ane_transport_test_init(struct kunit *test) +{ + struct ane_transport_test *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + + if (!f) + return -ENOMEM; + test->priv = f; + f->test = test; + f->soc.doorbell = 0x1884000; + f->soc.irq_status = 0x188c000; + f->soc.irq_ack = 0x1890000; + f->ipc = (struct ane_fw_surface) { + .cpu = kunit_kzalloc(test, TEST_IPC_SIZE, GFP_KERNEL), + .dva = TEST_IPC_DVA, + .size = TEST_IPC_SIZE, + }; + f->pool = (struct ane_fw_surface) { + .cpu = kunit_kzalloc(test, TEST_POOL_SIZE, GFP_KERNEL), + .dva = TEST_POOL_DVA, + .size = TEST_POOL_SIZE, + }; + if (!f->ipc.cpu || !f->pool.cpu) + return -ENOMEM; + f->next_dva = TEST_SHARED_DVA; + ane_test_table(f); + return ane_test_init_transport(f); +} + +static u8 *ane_test_slot(struct ane_transport_test *f, unsigned int ch) +{ + return ane_fw_transport_slot(&f->t, ch); +} + +static int ane_test_publish(struct ane_transport_test *f, unsigned int ch) +{ + u8 command[16] = {}; + + put_unaligned_le16(ch == ANE_FW_IO ? 3 : 0x2d, command + 4); + return ane_fw_transport_publish(&f->t, ch, command, sizeof(command), TEST_DEADLINE); +} + +static void ane_test_reply(struct ane_transport_test *f, unsigned int ch) +{ + const struct ane_fw_transaction *tx = &f->t.pending[ch]; + u8 *p = ane_test_slot(f, ch); + + put_unaligned_le64(tx->dva | 1, p); + put_unaligned_le64(tx->size, p + 8); + put_unaligned_le64(0, p + 16); +} + +static void ane_test_shared_request(struct ane_transport_test *f, u64 w0, u64 w1, u64 w2) +{ + u8 *p = ane_test_slot(f, ANE_FW_SHAREDMALLOC); + + put_unaligned_le64(w0, p); + put_unaligned_le64(w1, p + 8); + put_unaligned_le64(w2, p + 16); +} + +static unsigned int ane_test_writes(struct ane_transport_test *f) +{ + unsigned int i, n = 0; + + for (i = 0; i < f->count; i++) + if (f->access[i].kind == 'w' || f->access[i].kind == 'o') + n++; + return n; +} + +static void ane_transport_table_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + struct ane_fw_channel parsed[ANE_FW_CHANNELS]; + u8 *d = (u8 *)f->ipc.cpu + ANE_FW_DESCRIPTOR_SIZE; + unsigned int i; + int ret; + + for (i = 0; i < 15; i++) { + ane_test_table(f); + switch (i) { + case 0: + put_unaligned_le64(0, d + 0x48); + break; + case 1: + put_unaligned_le64(0x401, d + 0x48); + break; + case 2: + put_unaligned_le64(U64_MAX, d + 0x48); + break; + case 3: + put_unaligned_le64(f->ipc.dva, d + 0x50); + break; + case 4: + put_unaligned_le64(f->ipc.dva + 0x800, d + 0x50); + break; + case 5: + put_unaligned_le64(f->ipc.dva + f->ipc.size - 64, d + 0x50); + break; + case 6: + put_unaligned_le64(f->ipc.dva + 0x881, d + 0x50); + break; + case 7: + put_unaligned_le64(f->ipc.dva - 64, d + 0x50); + break; + case 8: + put_unaligned_le32(32, d + 0x44); + break; + case 9: + put_unaligned_le32(3, d + 0x40); + break; + case 10: + put_unaligned_le32(1, d + 0x40); + break; + case 11: + strscpy((char *)d, "TERMINAL", 64); + break; + case 12: + memset(d, 'A', 64); + break; + case 13: + strscpy((char *)d, "UNKNOWN", 64); + break; + case 14: + put_unaligned_le64(U64_MAX - 63, d + 0x50); + break; + } + memset(parsed, 0xa5, sizeof(parsed)); + ret = ane_fw_channels_parse(f->ipc.cpu, f->ipc.dva, f->ipc.size, + f->ipc.dva, parsed); + KUNIT_EXPECT_EQ_MSG(test, ret, -EPROTO, "mutation %u", i); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(parsed, 0xa5, sizeof(parsed)), NULL); + } + KUNIT_EXPECT_FALSE(test, ane_fw_contains(U64_MAX - 15, 64, U64_MAX - 15, 8)); +} + +static void ane_transport_table_roles_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + unsigned int ch; + + for (ch = 5; ch <= 7; ch++) { + u8 *d = (u8 *)f->ipc.cpu + ch * ANE_FW_DESCRIPTOR_SIZE; + + ane_test_table(f); + put_unaligned_le32(ch == 7 ? 1 : 0, d + 0x40); + KUNIT_EXPECT_EQ(test, ane_test_init_transport(f), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_fw_transport_arm(&f->t), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_test_writes(f), 0U); + } +} + +static void ane_transport_table_bounds_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + struct ane_fw_channel channels[ANE_FW_CHANNELS]; + unsigned int i; + int ret; + + for (i = 0; i < ANE_FW_TABLE_SIZE; i++) { + ret = ane_fw_transport_init(&f->t, &f->soc, &ane_test_ops, f, + f->ipc, f->pool, 0, f->ipc.dva, i); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + } + ret = ane_fw_transport_init(&f->t, &f->soc, &ane_test_ops, f, f->ipc, + f->pool, 0, f->ipc.dva, ANE_FW_TABLE_SIZE + 1); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = ane_fw_channels_parse(f->ipc.cpu, f->ipc.dva, f->ipc.size, + f->ipc.dva + f->ipc.size - 64, channels); + KUNIT_EXPECT_EQ(test, ret, -EPROTO); + ret = ane_fw_channels_parse(NULL, f->ipc.dva, f->ipc.size, f->ipc.dva, channels); + KUNIT_EXPECT_EQ(test, ret, -EPROTO); + KUNIT_EXPECT_EQ(test, ane_test_writes(f), 0U); +} + +static void ane_transport_table_edges_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + u8 *d; + + /* One 0x400-slot ring fits exactly after a relocated table and seven slots. */ + f->ipc.size = 2 * TEST_IPC_SIZE; + f->ipc.cpu = kunit_kzalloc(test, f->ipc.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f->ipc.cpu); + ane_test_table(f); + d = f->ipc.cpu; + put_unaligned_le64(0x400, d + 0x48); + put_unaligned_le64(f->ipc.dva + TEST_IPC_SIZE, d + 0x50); + put_unaligned_le32(31, d + 0x44); + KUNIT_EXPECT_EQ(test, ane_test_init_transport(f), 0); + put_unaligned_le64(1, d + 0x48); + put_unaligned_le64(f->ipc.dva + ANE_FW_TABLE_SIZE, d + 0x50); + KUNIT_EXPECT_EQ(test, ane_test_init_transport(f), 0); +} + +static void ane_transport_arm_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + unsigned int ch, slot, n = 1; + + memset((u8 *)f->ipc.cpu + 0x800, 0xa5, 0x400); + f->count = 0; + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + for (ch = 0; ch < ANE_FW_CHANNELS; ch++) { + for (slot = 0; slot < 2; slot++) { + u8 *p = ane_test_slot(f, ch) + slot * ANE_FW_SLOT_SIZE; + + if (f->t.channels[ch].type) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(p, 0xa5, 64), NULL); + continue; + } + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p), 1ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p + 8), 0ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p + 16), 0ULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(p + 24, 0xa5, 40), NULL); + KUNIT_EXPECT_PTR_EQ(test, f->access[n++].p, p + 8); + KUNIT_EXPECT_PTR_EQ(test, f->access[n++].p, p + 16); + KUNIT_EXPECT_PTR_EQ(test, f->access[n++].p, p); + } + } + KUNIT_EXPECT_EQ(test, f->access[n].kind, 'W'); + KUNIT_EXPECT_EQ(test, ane_fw_transport_arm(&f->t), -EBUSY); +} + +static void ane_transport_publish_order_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + unsigned int ch; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + for (ch = ANE_FW_IO; ch <= ANE_FW_DATA_CHAIN; ch += 6) { + u8 *p = ane_test_slot(f, ch); + u64 dva; + + f->count = 0; + KUNIT_ASSERT_EQ(test, ane_test_publish(f, ch), 0); + dva = get_unaligned_le64(p); + KUNIT_EXPECT_TRUE(test, ane_fw_contains(f->pool.dva, f->pool.size, dva, 16)); + KUNIT_EXPECT_GE(test, dva, f->pool.dva + TEST_RESERVED); + KUNIT_EXPECT_EQ(test, dva & 63, 0ULL); + KUNIT_EXPECT_EQ(test, f->access[2].kind, 'w'); + KUNIT_EXPECT_PTR_EQ(test, f->access[2].p, p + 8); + KUNIT_EXPECT_EQ(test, f->access[2].value, 16ULL); + KUNIT_EXPECT_PTR_EQ(test, f->access[3].p, p + 16); + KUNIT_EXPECT_EQ(test, f->access[3].value, 16ULL); + KUNIT_EXPECT_EQ(test, f->access[4].kind, 'W'); + KUNIT_EXPECT_PTR_EQ(test, f->access[5].p, p); + KUNIT_EXPECT_EQ(test, f->access[5].value, dva); + KUNIT_EXPECT_EQ(test, f->access[6].kind, 'W'); + KUNIT_EXPECT_EQ(test, f->access[7].kind, 'o'); + KUNIT_EXPECT_EQ(test, f->last_reg, f->soc.doorbell); + KUNIT_EXPECT_EQ(test, f->last_value, (u32)BIT(ch)); + KUNIT_EXPECT_EQ(test, ane_test_publish(f, ch), -EBUSY); + KUNIT_EXPECT_EQ(test, ane_fw_transport_poll(&f->t, ch), -EAGAIN); + ane_test_reply(f, ch); + KUNIT_EXPECT_EQ(test, ane_fw_transport_poll(&f->t, ch), 0); + KUNIT_EXPECT_EQ(test, f->t.head[ch], 1U); + } +} + +static void ane_transport_wrap_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + unsigned int i; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + for (i = 0; i < 5; i++) { + KUNIT_ASSERT_EQ(test, ane_test_publish(f, ANE_FW_DATA_CHAIN), 0); + ane_test_reply(f, ANE_FW_DATA_CHAIN); + KUNIT_ASSERT_EQ(test, ane_fw_transport_poll(&f->t, ANE_FW_DATA_CHAIN), 0); + KUNIT_EXPECT_EQ(test, f->t.head[ANE_FW_DATA_CHAIN], (i + 1) % 2); + } +} + +static void ane_transport_reply_corruption_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + unsigned int i; + + for (i = 0; i < 8; i++) { + u8 *p, *command; + u64 dva; + + KUNIT_ASSERT_EQ(test, ane_test_init_transport(f), 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + KUNIT_ASSERT_EQ(test, ane_test_publish(f, ANE_FW_IO), 0); + ane_test_reply(f, ANE_FW_IO); + p = ane_test_slot(f, ANE_FW_IO); + dva = f->t.pending[ANE_FW_IO].dva; + command = (u8 *)f->pool.cpu + (dva - f->pool.dva); + switch (i) { + case 0: + put_unaligned_le64(dva | 3, p); + break; + case 1: + put_unaligned_le64(dva | 2, p); + break; + case 2: + put_unaligned_le64((dva + 64) | 1, p); + break; + case 3: + put_unaligned_le64(15, p + 8); + break; + case 4: + put_unaligned_le64(1, p + 16); + break; + case 5: + put_unaligned_le32(0x1000000, command); + break; + case 6: + put_unaligned_le16(4, command + 4); + break; + case 7: + put_unaligned_le64(dva + 64, p); + break; + } + KUNIT_EXPECT_EQ_MSG(test, ane_fw_transport_poll(&f->t, ANE_FW_IO), + -EPROTO, "mutation %u", i); + KUNIT_EXPECT_TRUE(test, f->t.pending[ANE_FW_IO].active); + KUNIT_EXPECT_EQ(test, f->t.head[ANE_FW_IO], 0U); + f->count = 0; + KUNIT_EXPECT_EQ(test, ane_test_publish(f, ANE_FW_DATA_CHAIN), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_test_writes(f), 0U); + } +} + +static void ane_transport_refusal_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + u8 *command; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + KUNIT_ASSERT_EQ(test, ane_test_publish(f, ANE_FW_IO), 0); + command = (u8 *)f->pool.cpu + (f->t.pending[ANE_FW_IO].dva - f->pool.dva); + ane_test_reply(f, ANE_FW_IO); + put_unaligned_le16(0x100, command + 6); + KUNIT_EXPECT_EQ(test, ane_fw_transport_poll(&f->t, ANE_FW_IO), -EREMOTEIO); + KUNIT_EXPECT_TRUE(test, f->t.pending[ANE_FW_IO].active); + KUNIT_EXPECT_EQ(test, f->t.head[ANE_FW_IO], 0U); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(command + 6), (u16)0x100); +} + +static void ane_transport_pool_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + u8 command[16] = {}; + + KUNIT_EXPECT_EQ(test, ane_test_publish(f, ANE_FW_IO), -EINVAL); + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + KUNIT_EXPECT_EQ(test, ane_fw_transport_publish(&f->t, 2, command, 16, 100), -EINVAL); + KUNIT_EXPECT_EQ(test, ane_fw_transport_publish(&f->t, 1, NULL, 16, 100), -EINVAL); + KUNIT_EXPECT_EQ(test, ane_fw_transport_publish(&f->t, 1, command, 7, 100), -EINVAL); + command[0] = 1; + KUNIT_EXPECT_EQ(test, ane_fw_transport_publish(&f->t, 1, command, 16, 100), -EINVAL); + command[0] = 0; + command[6] = 1; + KUNIT_EXPECT_EQ(test, ane_fw_transport_publish(&f->t, 1, command, 16, 100), -EINVAL); + command[6] = 0; + memset(f->pool.cpu, 0xa5, TEST_RESERVED); + KUNIT_ASSERT_EQ(test, ane_test_publish(f, ANE_FW_IO), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->pool.cpu, 0xa5, TEST_RESERVED), NULL); + ane_test_reply(f, ANE_FW_IO); + KUNIT_ASSERT_EQ(test, ane_fw_transport_poll(&f->t, ANE_FW_IO), 0); + f->t.pool_used = f->pool.size - 1; + f->count = 0; + KUNIT_EXPECT_EQ(test, ane_test_publish(f, ANE_FW_IO), -ENOSPC); + KUNIT_EXPECT_EQ(test, ane_test_writes(f), 0U); + f->t.pool_used = SIZE_MAX; + KUNIT_EXPECT_EQ(test, ane_test_publish(f, ANE_FW_IO), -ENOSPC); +} + +static void ane_transport_snapshot_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + + ((u8 *)f->ipc.cpu)[0x58] = 1; + f->count = 0; + KUNIT_EXPECT_EQ(test, ane_fw_transport_arm(&f->t), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_test_writes(f), 0U); +} + +static void ane_transport_report_request(struct ane_transport_test *f, u64 dva, u64 size) +{ + u8 *p = ane_test_slot(f, ANE_FW_IO_T2H); + + put_unaligned_le64(dva, p); + put_unaligned_le64(size, p + 8); + put_unaligned_le64(0, p + 16); +} + +static void ane_transport_consume_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + struct ane_h16_request_key key = { .uuid = 9, .program = 1, .process = 2 }; + u8 report[0x60] = {}; + u8 *source = (u8 *)f->pool.cpu + 0x100; + u8 *p; + size_t size = 0; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + KUNIT_EXPECT_EQ(test, ane_fw_transport_consume(&f->t, report, sizeof(report), &size), + -EAGAIN); + put_unaligned_le16(0x300, source + 4); + put_unaligned_le64(key.uuid, source + 8); + put_unaligned_le32(key.program, source + 0x10); + put_unaligned_le32(key.process, source + 0x14); + put_unaligned_le32(1, source + 0x1c); + ane_transport_report_request(f, f->pool.dva + 0x100, sizeof(report)); + p = ane_test_slot(f, ANE_FW_IO_T2H); + KUNIT_EXPECT_EQ(test, ane_fw_transport_consume(&f->t, report, 8, &size), -ENOSPC); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p), f->pool.dva + 0x100); + f->count = 0; + KUNIT_ASSERT_EQ(test, ane_fw_transport_consume(&f->t, report, sizeof(report), &size), 0); + KUNIT_EXPECT_MEMEQ(test, report, source, sizeof(report)); + KUNIT_EXPECT_EQ(test, size, sizeof(report)); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p), (f->pool.dva + 0x100) | 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p + 8), 0x60ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p + 16), 0ULL); + KUNIT_EXPECT_EQ(test, f->access[f->count - 4].kind, 'W'); + KUNIT_EXPECT_PTR_EQ(test, f->access[f->count - 3].p, p); + KUNIT_EXPECT_EQ(test, f->access[f->count - 2].kind, 'W'); + KUNIT_EXPECT_EQ(test, f->last_value, (u32)BIT(6)); + KUNIT_EXPECT_EQ(test, ane_h16_event_result(report, size, 6, &key), + ANE_H16_EVENT_COMPLETE); + /* A stale UUID is consumed, but cannot retire the current pending request. */ + key.uuid++; + KUNIT_EXPECT_EQ(test, ane_h16_event_result(report, size, 6, &key), -EPROTO); + KUNIT_EXPECT_EQ(test, ane_fw_transport_consume(&f->t, report, sizeof(report), &size), + -EAGAIN); +} + +static void ane_transport_report_bounds_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + u8 report[0x60]; + size_t size; + unsigned int i; + int ret; + + for (i = 0; i < 7; i++) { + u64 dva = f->pool.dva, bytes = sizeof(report); + + KUNIT_ASSERT_EQ(test, ane_test_init_transport(f), 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + switch (i) { + case 0: + dva += 2; + break; + case 1: + dva = f->pool.dva + f->pool.size - 4; + break; + case 2: + dva = TEST_SHARED_DVA; + break; + case 3: + bytes = U64_MAX; + break; + case 4: + bytes = 0; + break; + case 5: + bytes = 7; + break; + case 6: + dva = 0; + break; + } + ane_transport_report_request(f, dva, bytes); + f->count = 0; + ret = ane_fw_transport_consume(&f->t, report, sizeof(report), &size); + KUNIT_EXPECT_EQ_MSG(test, ret, -EPROTO, "mutation %u", i); + KUNIT_EXPECT_EQ(test, ane_test_writes(f), 0U); + } +} + +static void ane_transport_report_word2_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + u8 report[12]; + size_t size; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + ane_transport_report_request(f, f->pool.dva, sizeof(report)); + put_unaligned_le64(1, ane_test_slot(f, ANE_FW_IO_T2H) + 16); + KUNIT_EXPECT_EQ(test, ane_fw_transport_consume(&f->t, report, sizeof(report), &size), + -EPROTO); +} + +static void ane_transport_presence_test(struct kunit *test) +{ + KUNIT_EXPECT_FALSE(test, ane_fw_transport_present(0, 0)); + KUNIT_EXPECT_FALSE(test, ane_fw_transport_present(1, 0)); + KUNIT_EXPECT_FALSE(test, ane_fw_transport_present(0x1001, 96)); + KUNIT_EXPECT_TRUE(test, ane_fw_transport_present(0x1000, 0)); + KUNIT_EXPECT_TRUE(test, ane_fw_transport_present(0, 96)); + KUNIT_EXPECT_TRUE(test, ane_fw_transport_present(0x1000, 96)); +} + +static void ane_transport_shared_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + u8 *p; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + KUNIT_EXPECT_EQ(test, ane_fw_transport_shared(&f->t), -EAGAIN); + ane_test_shared_request(f, 0, 1, 0x1234); + p = ane_test_slot(f, ANE_FW_SHAREDMALLOC); + f->count = 0; + KUNIT_ASSERT_EQ(test, ane_fw_transport_shared(&f->t), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p), TEST_SHARED_DVA | 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p + 8), 0ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p + 16), 1ULL); + KUNIT_EXPECT_EQ(test, f->t.shared_bytes, (size_t)SZ_16K); + KUNIT_EXPECT_EQ(test, f->access[f->count - 6].value, 0ULL); + KUNIT_EXPECT_EQ(test, f->access[f->count - 5].value, 1ULL); + KUNIT_EXPECT_EQ(test, f->access[f->count - 4].kind, 'W'); + KUNIT_EXPECT_PTR_EQ(test, f->access[f->count - 3].p, p); + KUNIT_EXPECT_EQ(test, f->access[f->count - 2].kind, 'W'); + KUNIT_EXPECT_EQ(test, f->last_value, (u32)BIT(5)); + ane_test_shared_request(f, 0, SZ_2M, ANE_FW_SHARED_EXPP); + KUNIT_ASSERT_EQ(test, ane_fw_transport_shared(&f->t), 0); + KUNIT_EXPECT_EQ(test, f->t.shared_bytes, (size_t)(SZ_2M + SZ_16K)); + KUNIT_EXPECT_EQ(test, f->t.shared_count, 2U); +} + +static void ane_transport_shared_rejections_test(struct kunit *test) +{ + static const u64 tuples[][3] = { + { 0x10000, SZ_16K, 0 }, + { 0, 0, 1 }, + { 0, SZ_1M + 1, 0 }, + { 0, SZ_1M + 1, ANE_FW_SHARED_EXPP }, + { 0, SZ_2M, ANE_FW_SHARED_EXPP + 1 }, + { 0, SZ_2M + 1, ANE_FW_SHARED_EXPP }, + { 0, U64_MAX, 0 }, + { 0, SZ_16K, BIT_ULL(32) }, + }; + struct ane_transport_test *f = test->priv; + size_t size; + unsigned int i; + + KUNIT_EXPECT_EQ(test, ane_fw_shared_size(0, 0, 0, &size), -EOPNOTSUPP); + for (i = 0; i < ARRAY_SIZE(tuples); i++) { + int error = i < 2 ? -EOPNOTSUPP : -E2BIG; + + KUNIT_ASSERT_EQ(test, ane_test_init_transport(f), 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + ane_test_shared_request(f, tuples[i][0], tuples[i][1], tuples[i][2]); + f->count = 0; + KUNIT_EXPECT_EQ(test, ane_fw_transport_shared(&f->t), error); + KUNIT_EXPECT_EQ(test, f->refusals, i + 1); + KUNIT_EXPECT_MEMEQ(test, f->refused, tuples[i], sizeof(f->refused)); + KUNIT_EXPECT_EQ(test, f->refused_error, error); + KUNIT_EXPECT_EQ(test, f->allocs, 0U); + KUNIT_EXPECT_EQ(test, ane_test_writes(f), 0U); + } +} + +static void ane_transport_allocator_rejections_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + unsigned int i; + + for (i = 1; i <= 6; i++) { + KUNIT_ASSERT_EQ(test, ane_test_init_transport(f), 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + f->alloc_bad = i; + ane_test_shared_request(f, 0, SZ_16K, 0); + f->count = 0; + KUNIT_EXPECT_EQ_MSG(test, ane_fw_transport_shared(&f->t), -EPROTO, + "allocator result %u", i); + KUNIT_EXPECT_EQ(test, f->t.shared_count, 0U); + KUNIT_EXPECT_EQ(test, ane_test_writes(f), 0U); + } + KUNIT_ASSERT_EQ(test, ane_test_init_transport(f), 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + f->alloc_error = -ENOMEM; + ane_test_shared_request(f, 0, SZ_16K, 0); + KUNIT_EXPECT_EQ(test, ane_fw_transport_shared(&f->t), -ENOMEM); +} + +static void ane_transport_shared_overlap_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + ane_test_shared_request(f, 0, SZ_16K, 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_shared(&f->t), 0); + f->next_dva = TEST_SHARED_DVA; + ane_test_shared_request(f, 0, SZ_16K, 0); + KUNIT_EXPECT_EQ(test, ane_fw_transport_shared(&f->t), -EPROTO); + KUNIT_EXPECT_EQ(test, f->t.shared_count, 1U); +} + +static void ane_transport_shared_count_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + unsigned int i; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + for (i = 0; i < ANE_FW_SHARED_MAX; i++) { + ane_test_shared_request(f, 0, SZ_16K, 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_shared(&f->t), 0); + } + ane_test_shared_request(f, 0, SZ_16K, 0); + KUNIT_EXPECT_EQ(test, ane_fw_transport_shared(&f->t), -ENOSPC); + KUNIT_EXPECT_EQ(test, f->allocs, (unsigned int)ANE_FW_SHARED_MAX); +} + +static void ane_transport_shared_total_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + unsigned int i; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + for (i = 0; i < 12; i++) { + ane_test_shared_request(f, 0, SZ_1M, 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_shared(&f->t), 0); + } + KUNIT_EXPECT_EQ(test, f->t.shared_bytes, (size_t)ANE_FW_SHARED_TOTAL); + ane_test_shared_request(f, 0, 1, 0); + KUNIT_EXPECT_EQ(test, ane_fw_transport_shared(&f->t), -ENOSPC); + KUNIT_EXPECT_EQ(test, f->allocs, 12U); +} + +static void ane_transport_shared_report_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + u8 report[12]; + size_t size; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + ane_test_shared_request(f, 0, SZ_16K, 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_shared(&f->t), 0); + put_unaligned_le16(0x107, (u8 *)f->t.shared[0].cpu + 4); + ane_transport_report_request(f, TEST_SHARED_DVA, sizeof(report)); + KUNIT_ASSERT_EQ(test, ane_fw_transport_consume(&f->t, report, sizeof(report), &size), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(report + 4), (u16)0x107); + KUNIT_EXPECT_EQ(test, size, sizeof(report)); +} + +static void ane_transport_irq_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + u32 pending = 0; + + f->count = 0; + KUNIT_ASSERT_EQ(test, ane_fw_transport_irq(&f->t, &pending), 0); + KUNIT_EXPECT_EQ(test, ane_test_writes(f), 0U); + f->irq = 0xa5; + KUNIT_ASSERT_EQ(test, ane_fw_transport_irq(&f->t, &pending), 0); + KUNIT_EXPECT_EQ(test, pending, 0xa5U); + KUNIT_EXPECT_EQ(test, f->last_reg, f->soc.irq_ack); + KUNIT_EXPECT_EQ(test, f->last_value, 0xa5U); + f->irq = BIT(8) | 1; + f->count = 0; + KUNIT_EXPECT_EQ(test, ane_fw_transport_irq(&f->t, &pending), -EPROTO); + KUNIT_EXPECT_EQ(test, pending, 0xa5U); + KUNIT_EXPECT_EQ(test, ane_test_writes(f), 0U); +} + +static void ane_transport_match_data_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + u8 *d = (u8 *)f->ipc.cpu + ANE_FW_DESCRIPTOR_SIZE; + u32 pending; + + f->soc.doorbell = 0x100; + f->soc.irq_status = 0x200; + f->soc.irq_ack = 0x300; + put_unaligned_le32(31, d + 0x44); + KUNIT_ASSERT_EQ(test, ane_test_init_transport(f), 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + KUNIT_ASSERT_EQ(test, ane_test_publish(f, ANE_FW_IO), 0); + KUNIT_EXPECT_EQ(test, f->last_reg, 0x100U); + KUNIT_EXPECT_EQ(test, f->last_value, (u32)BIT(31)); + f->irq = 0xff; + KUNIT_ASSERT_EQ(test, ane_fw_transport_irq(&f->t, &pending), 0); + KUNIT_EXPECT_EQ(test, f->last_reg, 0x300U); + KUNIT_EXPECT_EQ(test, f->last_value, 0xffU); +} + +static void ane_transport_deadline_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + KUNIT_ASSERT_EQ(test, ane_test_publish(f, ANE_FW_IO), 0); + f->now = TEST_DEADLINE - 1; + KUNIT_EXPECT_EQ(test, ane_fw_transport_poll(&f->t, ANE_FW_IO), -EAGAIN); + f->now++; + ane_test_reply(f, ANE_FW_IO); + KUNIT_EXPECT_EQ(test, ane_fw_transport_poll(&f->t, ANE_FW_IO), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, f->t.pending[ANE_FW_IO].active); + KUNIT_EXPECT_EQ(test, f->t.head[ANE_FW_IO], 0U); +} + +static void ane_transport_deadline_service_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + KUNIT_ASSERT_EQ(test, ane_test_publish(f, ANE_FW_DATA_CHAIN), 0); + f->now = TEST_DEADLINE - 1; + f->alloc_time = 2; + f->irq = BIT(5); + ane_test_shared_request(f, 0, SZ_16K, 0); + KUNIT_EXPECT_EQ(test, ane_fw_transport_wait(&f->t, ANE_FW_DATA_CHAIN), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, f->allocs, 1U); + KUNIT_EXPECT_EQ(test, f->t.shared_count, 1U); + KUNIT_EXPECT_EQ(test, f->t.pending[ANE_FW_DATA_CHAIN].deadline_ns, TEST_DEADLINE); + KUNIT_EXPECT_TRUE(test, f->t.pending[ANE_FW_DATA_CHAIN].active); +} + +static void ane_transport_wait_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + KUNIT_ASSERT_EQ(test, ane_test_publish(f, ANE_FW_IO), 0); + ane_test_reply(f, ANE_FW_IO); + KUNIT_EXPECT_EQ(test, ane_fw_transport_wait(&f->t, ANE_FW_IO), 0); + KUNIT_ASSERT_EQ(test, ane_test_publish(f, ANE_FW_DATA_CHAIN), 0); + KUNIT_EXPECT_EQ(test, ane_fw_transport_wait(&f->t, ANE_FW_DATA_CHAIN), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, f->now, TEST_DEADLINE); +} + +static void ane_transport_deny_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + const u16 ids[] = { 0x27, 0x28, 0x30, 0x31, 0x34 }; + u8 command[12] = {}; + int ret; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + for (unsigned int i = 0; i < ARRAY_SIZE(ids); i++) { + put_unaligned_le16(ids[i], command + 4); + f->count = 0; + ret = ane_fw_transport_publish(&f->t, ANE_FW_IO, command, + sizeof(command), TEST_DEADLINE); + KUNIT_EXPECT_EQ(test, ret, -EPERM); + KUNIT_EXPECT_EQ(test, ane_test_writes(f), 0U); + KUNIT_EXPECT_FALSE(test, f->t.pending[ANE_FW_IO].active); + KUNIT_EXPECT_EQ(test, f->t.pool_used, (size_t)TEST_RESERVED); + } +} + +static void ane_transport_free_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + u8 *slot; + u64 dva; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + ane_test_shared_request(f, 0, SZ_2M, ANE_FW_SHARED_EXPP); + KUNIT_ASSERT_EQ(test, ane_fw_transport_shared(&f->t), 0); + dva = f->t.shared[0].dva; + KUNIT_EXPECT_EQ(test, f->t.shared_tags[0], ANE_FW_SHARED_EXPP); + f->t.allow_free = true; + ane_test_shared_request(f, dva, 0, 0); + slot = ane_test_slot(f, ANE_FW_SHAREDMALLOC); + KUNIT_ASSERT_EQ(test, ane_fw_transport_shared(&f->t), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot), dva | 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot + 8), 0ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot + 16), 0ULL); + KUNIT_EXPECT_EQ(test, f->t.free_count, 1U); + KUNIT_EXPECT_TRUE(test, f->t.shared_retired[0]); + KUNIT_EXPECT_NOT_NULL(test, f->t.shared[0].cpu); + KUNIT_EXPECT_PTR_EQ(test, ane_fw_transport_report(&f->t, dva, 8), NULL); + /* Duplicate FREE cannot acknowledge or retire a different mapping. */ + ane_test_shared_request(f, dva, 0, 0); + f->count = 0; + KUNIT_EXPECT_EQ(test, ane_fw_transport_shared(&f->t), -ENOENT); + KUNIT_EXPECT_EQ(test, ane_test_writes(f), 0U); +} + +static void ane_transport_free_refusal_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + ane_test_shared_request(f, 0, SZ_16K, 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_shared(&f->t), 0); + ane_test_shared_request(f, f->t.shared[0].dva, 0, 0); + f->count = 0; + KUNIT_EXPECT_EQ(test, ane_fw_transport_shared(&f->t), -EOPNOTSUPP); + KUNIT_EXPECT_FALSE(test, f->t.shared_retired[0]); + KUNIT_EXPECT_EQ(test, ane_test_writes(f), 0U); +} + +/* Publish one EXECUTE on the data channel; return its flags word as copied. */ +static u32 ane_test_execute(struct ane_transport_test *f, u32 program, u32 process) +{ + const struct ane_fw_transaction *tx = &f->t.pending[ANE_FW_DATA_CHAIN]; + u8 *command = kunit_kzalloc(f->test, ANE_H16_EXECUTE_BASE_SIZE, GFP_KERNEL); + u32 flags; + int ret; + + KUNIT_ASSERT_NOT_NULL(f->test, command); + put_unaligned_le16(0x20c, command + 4); + put_unaligned_le32(program, command + 8); + put_unaligned_le32(process, command + 0xc); + ret = ane_fw_transport_publish(&f->t, ANE_FW_DATA_CHAIN, command, + ANE_H16_EXECUTE_BASE_SIZE, TEST_DEADLINE); + KUNIT_EXPECT_EQ(f->test, ret, 0); + flags = get_unaligned_le32((u8 *)f->t.pool.cpu + (tx->dva - f->t.pool.dva) + + ANE_H16_EXECUTE_FLAGS); + /* The caller's request is never written. */ + KUNIT_EXPECT_EQ(f->test, get_unaligned_le32(command + ANE_H16_EXECUTE_FLAGS), 0U); + ane_test_reply(f, ANE_FW_DATA_CHAIN); + KUNIT_EXPECT_EQ(f->test, ane_fw_transport_poll(&f->t, ANE_FW_DATA_CHAIN), 0); + return flags; +} + +static void ane_test_io(struct ane_transport_test *f, u16 id, u32 program) +{ + u8 command[0x1c0] = {}; + int ret; + + put_unaligned_le16(id, command + 4); + put_unaligned_le32(program, command + 8); + ret = ane_fw_transport_publish(&f->t, ANE_FW_IO, command, + id == 0x200 ? sizeof(command) : 16, TEST_DEADLINE); + KUNIT_EXPECT_EQ(f->test, ret, 0); + ane_test_reply(f, ANE_FW_IO); + KUNIT_EXPECT_EQ(f->test, ane_fw_transport_poll(&f->t, ANE_FW_IO), 0); +} + +static void ane_transport_second_load_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + + f->soc.chip_family = 17; + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + /* A later program is admitted after UNLOAD within one firmware boot. */ + ane_test_io(f, 0x200, 0); + ane_test_io(f, 0x201, 0); + ane_test_io(f, 0x200, 1); + KUNIT_EXPECT_EQ(test, f->t.error, 0); +} + +static void ane_transport_bar_context_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + const u32 q = ANE_H16_EXECUTE_FLAG_QUEUED; + + f->soc.chip_family = 17; + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + /* Unknown context after start: queue the first job. */ + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 0, 0), q); + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 0, 0), 0U); + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 0, 0), 0U); + /* Switching to another loaded program and back queues each switch. */ + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 1, 1), q); + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 1, 1), 0U); + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 0, 0), q); + /* Same program, different process. */ + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 0, 2), q); + /* Program lifetime commands forget the context: IDs may be reused. */ + ane_test_io(f, 0x203, 0); + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 0, 2), q); + ane_test_io(f, 0x201, 0); + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 0, 2), q); + ane_test_io(f, 0x200, 0); + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 0, 2), q); + ane_test_io(f, 0x202, 0); + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 0, 2), q); + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 0, 2), 0U); + /* Other IO commands keep it. */ + ane_test_io(f, 3, 0); + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 0, 2), 0U); +} + +static void ane_transport_bar_context_h16_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + + f->soc.chip_family = 16; + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + /* Not yet measured on H16: requests are published unchanged. */ + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 0, 0), 0U); + KUNIT_EXPECT_EQ(test, ane_test_execute(f, 1, 1), 0U); +} + +/* Each relax() learns how long its wait has run, and every call is counted. */ +static void ane_transport_wait_polls_test(struct kunit *test) +{ + struct ane_transport_test *f = test->priv; + + KUNIT_ASSERT_EQ(test, ane_fw_transport_arm(&f->t), 0); + KUNIT_ASSERT_EQ(test, ane_test_publish(f, ANE_FW_IO), 0); + ane_test_reply(f, ANE_FW_IO); + KUNIT_EXPECT_EQ(test, ane_fw_transport_wait(&f->t, ANE_FW_IO), 0); + /* A reply already there: no sleep at all. */ + KUNIT_EXPECT_EQ(test, f->t.polls, 0ULL); + KUNIT_ASSERT_EQ(test, ane_test_publish(f, ANE_FW_DATA_CHAIN), 0); + f->now = TEST_DEADLINE - 5; + KUNIT_EXPECT_EQ(test, ane_fw_transport_wait(&f->t, ANE_FW_DATA_CHAIN), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, f->t.polls, 5ULL); + /* The fifth sleep came four ticks after the wait began. */ + KUNIT_EXPECT_EQ(test, f->waited, 4ULL); +} + +static void ane_transport_poll_sleep_test(struct kunit *test) +{ + unsigned int max_us; + + KUNIT_EXPECT_EQ(test, ane_fw_poll_sleep_us(0, &max_us), 20U); + KUNIT_EXPECT_EQ(test, max_us, 50U); + KUNIT_EXPECT_EQ(test, ane_fw_poll_sleep_us(ANE_FW_POLL_FINE_NS - 1, &max_us), 20U); + KUNIT_EXPECT_EQ(test, max_us, 50U); + KUNIT_EXPECT_EQ(test, ane_fw_poll_sleep_us(ANE_FW_POLL_FINE_NS, &max_us), 250U); + KUNIT_EXPECT_EQ(test, max_us, 300U); + KUNIT_EXPECT_EQ(test, ane_fw_poll_sleep_us(ANE_FW_POLL_MEDIUM_NS - 1, &max_us), 250U); + KUNIT_EXPECT_EQ(test, ane_fw_poll_sleep_us(ANE_FW_POLL_MEDIUM_NS, &max_us), 1000U); + KUNIT_EXPECT_EQ(test, max_us, 1050U); + KUNIT_EXPECT_EQ(test, ane_fw_poll_sleep_us(U64_MAX, &max_us), 1000U); +} + +static struct kunit_case ane_transport_cases[] = { + KUNIT_CASE(ane_transport_second_load_test), + KUNIT_CASE(ane_transport_bar_context_test), + KUNIT_CASE(ane_transport_bar_context_h16_test), + KUNIT_CASE(ane_transport_deny_test), + KUNIT_CASE(ane_transport_free_test), + KUNIT_CASE(ane_transport_free_refusal_test), + KUNIT_CASE(ane_transport_table_test), + KUNIT_CASE(ane_transport_table_roles_test), + KUNIT_CASE(ane_transport_table_bounds_test), + KUNIT_CASE(ane_transport_table_edges_test), + KUNIT_CASE(ane_transport_arm_test), + KUNIT_CASE(ane_transport_publish_order_test), + KUNIT_CASE(ane_transport_wrap_test), + KUNIT_CASE(ane_transport_reply_corruption_test), + KUNIT_CASE(ane_transport_refusal_test), + KUNIT_CASE(ane_transport_pool_test), + KUNIT_CASE(ane_transport_snapshot_test), + KUNIT_CASE(ane_transport_consume_test), + KUNIT_CASE(ane_transport_report_bounds_test), + KUNIT_CASE(ane_transport_report_word2_test), + KUNIT_CASE(ane_transport_presence_test), + KUNIT_CASE(ane_transport_shared_test), + KUNIT_CASE(ane_transport_shared_rejections_test), + KUNIT_CASE(ane_transport_allocator_rejections_test), + KUNIT_CASE(ane_transport_shared_overlap_test), + KUNIT_CASE(ane_transport_shared_count_test), + KUNIT_CASE(ane_transport_shared_total_test), + KUNIT_CASE(ane_transport_shared_report_test), + KUNIT_CASE(ane_transport_irq_test), + KUNIT_CASE(ane_transport_match_data_test), + KUNIT_CASE(ane_transport_deadline_test), + KUNIT_CASE(ane_transport_deadline_service_test), + KUNIT_CASE(ane_transport_wait_test), + KUNIT_CASE(ane_transport_wait_polls_test), + KUNIT_CASE(ane_transport_poll_sleep_test), + {} +}; + +static struct kunit_suite ane_transport_suite = { + .name = "ane-fw-transport", + .init = ane_transport_test_init, + .test_cases = ane_transport_cases, +}; + +kunit_test_suite(ane_transport_suite); +MODULE_DESCRIPTION("KUnit tests for the shared H16/H17 firmware transport"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_accel.c b/drivers/accel/ane/ane_h16_accel.c new file mode 100644 index 0000000000000..aaeee167ea6bb --- /dev/null +++ b/drivers/accel/ane/ane_h16_accel.c @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * H16 and H17 accel node (ABI version 3). It exists only after the staged + * opt-in completed its program stage. Every firmware-visible byte lives in private + * kernel mappings; userspace passes the original HWX, the host-built sections + * and plain input/output buffers, never DVAs, firmware IDs or commands. + */ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane_h16_accel.h" + +struct ane_h16_accel { + struct drm_device drm; + struct ane_h16_session *session; + /* Serializes the session, every program and all firmware traffic. */ + struct mutex lock; +}; + +struct ane_h16_accel_file { + struct ane_h16_program *program; +}; + +static struct ane_h16_accel *to_h16_accel(struct drm_device *drm) +{ + return container_of(drm, struct ane_h16_accel, drm); +} + +static int ane_h16_get_caps_ioctl(struct drm_device *drm, void *data, + struct drm_file *file) +{ + struct ane_h16_accel *a = to_h16_accel(drm); + struct drm_ane_get_caps *args = data; + + if (args->flags || args->pad) + return -EINVAL; + *args = (struct drm_ane_get_caps) { + .size = sizeof(*args), + .abi_version = DRM_ANE_ABI_V3, + .chip_family = a->session->transport->soc->chip_family, + .section_size = sizeof(struct drm_ane_h16_section), + .exec_io_size = sizeof(struct drm_ane_h16_buffer), + }; + return 0; +} + +static int ane_h16_sections_copy(const struct drm_ane_h16_load *args, + void *aux[ANE_H16_AUX_COUNT], + size_t aux_size[ANE_H16_AUX_COUNT]) +{ + struct drm_ane_h16_section sections[DRM_ANE_H16_SECTIONS]; + unsigned int i; + + BUILD_BUG_ON(DRM_ANE_H16_SECTIONS != ANE_H16_AUX_COUNT); + BUILD_BUG_ON(DRM_ANE_H16_SECTION_MIN != ANE_H16_AUX_MIN); + BUILD_BUG_ON(DRM_ANE_H16_SECTION_MAX != ANE_H16_AUX_MAX); + if (args->section_count != DRM_ANE_H16_SECTIONS) + return -EINVAL; + if (copy_from_user(sections, u64_to_user_ptr(args->sections_ptr), sizeof(sections))) + return -EFAULT; + for (i = 0; i < DRM_ANE_H16_SECTIONS; i++) { + const struct drm_ane_h16_section *s = §ions[i]; + unsigned int kind = s->kind - DRM_ANE_H16_SECTION_GENERIC; + void *bytes; + + if (s->kind < DRM_ANE_H16_SECTION_GENERIC || + s->kind > DRM_ANE_H16_SECTION_TEXT_PROPERTIES || aux[kind] || + s->size < ANE_H16_AUX_MIN || s->size > ANE_H16_AUX_MAX || + !IS_ALIGNED(s->size, 4)) + return -EINVAL; + bytes = memdup_user(u64_to_user_ptr(s->data_ptr), s->size); + if (IS_ERR(bytes)) + return PTR_ERR(bytes); + aux[kind] = bytes; + aux_size[kind] = s->size; + } + return 0; +} + +static int ane_h16_load_ioctl(struct drm_device *drm, void *data, struct drm_file *file) +{ + struct ane_h16_accel *a = to_h16_accel(drm); + struct ane_h16_accel_file *f = file->driver_priv; + struct drm_ane_h16_load *args = data; + struct ane_h16_hwx_snapshot snapshot = {}; + void *aux[ANE_H16_AUX_COUNT] = {}; + size_t aux_size[ANE_H16_AUX_COUNT] = {}; + struct ane_h16_program *program = NULL; + unsigned int i; + int ret; + + if (args->flags || args->pad) + return -EINVAL; + ret = ane_h16_sections_copy(args, aux, aux_size); + if (!ret) + ret = ane_h16_hwx_snapshot_from_user(u64_to_user_ptr(args->hwx_ptr), + args->hwx_size, &snapshot); + if (ret) + goto out; + mutex_lock(&a->lock); + if (f->program) { + ret = -EBUSY; + } else { + ret = ane_h16_session_load(a->session, &snapshot, + (const void *const *)aux, aux_size, &program); + if (ret == -EBUSY) + drm_err(drm, "another program is still loaded; close it first\n"); + if (!ret) + f->program = program; + } + if (!ret) { + const struct ane_h16_hwx *hwx = &program->hwx; + + args->surface_count = program->resource_count; + for (i = 0; i < DRM_ANE_H16_SURFACES; i++) { + args->surface_size[i] = i < program->resource_count ? + program->resources[i].buffer.size : 0; + args->surface_direction[i] = i < program->resource_count ? + hwx->surfaces[i].direction : 0; + } + } + mutex_unlock(&a->lock); +out: + /* A consumed snapshot was moved into the program and reads as empty. */ + if (snapshot.bytes) + ane_h16_hwx_snapshot_destroy(&snapshot); + for (i = 0; i < ANE_H16_AUX_COUNT; i++) + kfree(aux[i]); + return ret; +} + +static int ane_h16_exec_ioctl(struct drm_device *drm, void *data, struct drm_file *file) +{ + struct ane_h16_accel *a = to_h16_accel(drm); + struct ane_h16_accel_file *f = file->driver_priv; + struct drm_ane_h16_exec *args = data; + struct drm_ane_h16_buffer buffers[ANE_H16_SURFACES]; + struct ane_h16_program *p; + u64 start, locked, copied, done; + unsigned int i; + int ret = 0; + + if (args->flags || !args->count || args->count > ANE_H16_SURFACES) + return -EINVAL; + if (copy_from_user(buffers, u64_to_user_ptr(args->buffers_ptr), + array_size(args->count, sizeof(*buffers)))) + return -EFAULT; + start = ktime_get_ns(); + mutex_lock(&a->lock); + locked = ktime_get_ns(); + p = f->program; + if (!p || p->closed || args->count != p->resource_count) { + ret = -EINVAL; + goto unlock; + } + for (i = 0; i < args->count; i++) { + if (buffers[i].size != p->resources[i].buffer.size) { + ret = -EINVAL; + goto unlock; + } + } + /* The firmware is idle for this program, so its inputs may change now. */ + for (i = 0; i < args->count; i++) { + void *cpu = p->surfaces[i]->cpu; + + if (p->resources[i].permissions == 1) { + if (copy_from_user(cpu, u64_to_user_ptr(buffers[i].ptr), buffers[i].size)) { + ret = -EFAULT; + goto unlock; + } + } else { + memset(cpu, 0xff, buffers[i].size); + } + } + copied = ktime_get_ns(); + ret = ane_h16_session_execute(a->session, p); + if (ret) + goto unlock; + done = ktime_get_ns(); + for (i = 0; i < args->count; i++) + if (p->resources[i].permissions == 2 && + copy_to_user(u64_to_user_ptr(buffers[i].ptr), p->surfaces[i]->cpu, + buffers[i].size)) + ret = -EFAULT; + if (!ret) { + u64 end = ktime_get_ns(); + const u64 *ns = a->session->timing.ns; + + ane_h16_session_account(a->session, locked - start, copied - locked, end - done, + end - start); + drm_dbg_driver(drm, + "EXECUTE us: lock %llu in %llu prep %llu open %llu ack %llu report %llu done %llu close %llu fin %llu out %llu fw %llu all %llu polls %llu\n", + div_u64(locked - start, 1000), + div_u64(copied - locked, 1000), + div_u64(ns[ANE_H16_T_PREPARE], 1000), + div_u64(ns[ANE_H16_T_GATE_OPEN], 1000), + div_u64(ns[ANE_H16_T_SUBMIT_ACK], 1000), + div_u64(ns[ANE_H16_T_ACK_REPORT], 1000), + div_u64(ns[ANE_H16_T_REPORT_DONE], 1000), + div_u64(ns[ANE_H16_T_GATE_CLOSE], 1000), + div_u64(ns[ANE_H16_T_FINISH], 1000), + div_u64(end - done, 1000), + div_u64(ns[ANE_H16_T_FIRMWARE], 1000), + div_u64(end - start, 1000), a->session->timing.polls); + } +unlock: + mutex_unlock(&a->lock); + return ret; +} + +static int ane_h16_open(struct drm_device *drm, struct drm_file *file) +{ + struct ane_h16_accel_file *f = kzalloc_obj(*f); + + if (!f) + return -ENOMEM; + file->driver_priv = f; + return 0; +} + +/* TERMINATE and UNLOAD on close. Firmware-visible memory stays until reboot. */ +static void ane_h16_postclose(struct drm_device *drm, struct drm_file *file) +{ + struct ane_h16_accel *a = to_h16_accel(drm); + struct ane_h16_accel_file *f = file->driver_priv; + int ret; + + mutex_lock(&a->lock); + if (f->program) { + ret = ane_h16_session_close(a->session, f->program); + if (ret) + drm_err(drm, "program teardown failed %d; session quarantined\n", ret); + } + mutex_unlock(&a->lock); + kfree(f); +} + +static const struct drm_ioctl_desc ane_h16_ioctls[] = { + DRM_IOCTL_DEF_DRV(ANE_GET_CAPS, ane_h16_get_caps_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(ANE_H16_LOAD, ane_h16_load_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(ANE_H16_EXEC, ane_h16_exec_ioctl, DRM_RENDER_ALLOW), +}; + +/* + * No GEM objects and no mmap: buffers travel by copy only, so this is + * DRM_ACCEL_FOPS without .mmap (drm_gem_mmap needs DRIVER_GEM). The DRM core + * refuses to open a file whose fops lack FOP_UNSIGNED_OFFSET. + */ +static const struct file_operations ane_h16_fops = { + .owner = THIS_MODULE, + .fop_flags = FOP_UNSIGNED_OFFSET, + .open = accel_open, + .release = drm_release, + .unlocked_ioctl = drm_ioctl, + .compat_ioctl = drm_compat_ioctl, + .poll = drm_poll, + .read = drm_read, + .llseek = noop_llseek, +}; + +static const struct drm_driver ane_h16_drm_driver = { + .driver_features = DRIVER_COMPUTE_ACCEL, + .open = ane_h16_open, + .postclose = ane_h16_postclose, + .ioctls = ane_h16_ioctls, + .num_ioctls = ARRAY_SIZE(ane_h16_ioctls), + .fops = &ane_h16_fops, + .major = DRM_ANE_ABI_V3, + .minor = 0, + .name = "ane", + .desc = "Apple Neural Engine (H16 and later)", +}; + +int ane_h16_accel_register(struct device *dev, struct ane_h16_session *session) +{ + struct ane_h16_accel *a; + int ret; + + a = devm_drm_dev_alloc(dev, &ane_h16_drm_driver, struct ane_h16_accel, drm); + if (IS_ERR(a)) + return PTR_ERR(a); + a->session = session; + ret = drmm_mutex_init(&a->drm, &a->lock); + if (ret) + return ret; + ret = drm_dev_register(&a->drm, 0); + if (ret) + return ret; + drm_info(&a->drm, "H%u accel node ready (ABI 3, %s)\n", + session->transport->soc->chip_family, + ane_fw_transport_switch_safe(session->transport) ? + "programs switch within one firmware boot" : "one loaded program at a time"); + return 0; +} diff --git a/drivers/accel/ane/ane_h16_accel.h b/drivers/accel/ane/ane_h16_accel.h new file mode 100644 index 0000000000000..ec744e13db962 --- /dev/null +++ b/drivers/accel/ane/ane_h16_accel.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_ACCEL_H__ +#define __ANE_H16_ACCEL_H__ + +#include "ane_h16_program.h" + +struct device; + +/* Register the ABI 3 node over a session whose transport is armed and started. */ +int ane_h16_accel_register(struct device *dev, struct ane_h16_session *session); + +#endif /* __ANE_H16_ACCEL_H__ */ diff --git a/drivers/accel/ane/ane_h16_accel_test.c b/drivers/accel/ane/ane_h16_accel_test.c new file mode 100644 index 0000000000000..406dcca74cf9e --- /dev/null +++ b/drivers/accel/ane/ane_h16_accel_test.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Register the real H16 accel driver on a mock device and open its node + * through its own file operations, as /dev/accel/accelN does. + */ +#include +#include +#include + +#include + +/* The node under test is this file's copy; keep its one global symbol apart. */ +#define ane_h16_accel_register ane_h16_accel_register_kunit +#include "ane_h16_accel.c" + +static void ane_h16_accel_fops_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, ane_h16_fops.fop_flags & FOP_UNSIGNED_OFFSET); + KUNIT_EXPECT_TRUE(test, ane_h16_fops.open == accel_open); + /* Without DRIVER_GEM there is no offset manager to mmap through. */ + KUNIT_EXPECT_NULL(test, ane_h16_fops.mmap); + KUNIT_EXPECT_FALSE(test, ane_h16_drm_driver.driver_features & DRIVER_GEM); +} + +static void ane_h16_accel_open_test(struct kunit *test) +{ + struct device *dev = drm_kunit_helper_alloc_device(test); + struct ane_h16_accel *a; + struct inode *inode; + struct file *filp; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + a = drm_kunit_helper_alloc_drm_device_with_driver(test, dev, struct ane_h16_accel, drm, + &ane_h16_drm_driver); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, a); + KUNIT_ASSERT_EQ(test, drmm_mutex_init(&a->drm, &a->lock), 0); + KUNIT_ASSERT_EQ(test, drm_dev_register(&a->drm, 0), 0); + KUNIT_ASSERT_NOT_NULL(test, a->drm.accel); + + inode = kunit_kzalloc(test, sizeof(*inode), GFP_KERNEL); + filp = kunit_kzalloc(test, sizeof(*filp), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, inode); + KUNIT_ASSERT_NOT_NULL(test, filp); + inode->i_rdev = MKDEV(ACCEL_MAJOR, a->drm.accel->index); + filp->f_op = &ane_h16_fops; + filp->f_flags = O_RDWR; + filp->f_mode = FMODE_READ | FMODE_WRITE; + + ret = ane_h16_fops.open(inode, filp); + KUNIT_EXPECT_EQ(test, ret, 0); + if (!ret) { + struct drm_file *file = filp->private_data; + + KUNIT_ASSERT_NOT_NULL(test, file); + KUNIT_EXPECT_NOT_NULL(test, file->driver_priv); + KUNIT_EXPECT_EQ(test, ane_h16_fops.release(inode, filp), 0); + } + drm_dev_unregister(&a->drm); +} + +static struct kunit_case ane_h16_accel_cases[] = { + KUNIT_CASE(ane_h16_accel_fops_test), + KUNIT_CASE(ane_h16_accel_open_test), + {} +}; + +static struct kunit_suite ane_h16_accel_suite = { + .name = "ane-h16-accel", + .test_cases = ane_h16_accel_cases, +}; + +kunit_test_suite(ane_h16_accel_suite); + +MODULE_DESCRIPTION("KUnit open test for the H16 ANE accel node"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_add128_fixture.h b/drivers/accel/ane/ane_h16_add128_fixture.h new file mode 100644 index 0000000000000..6bb81db4ff5cc --- /dev/null +++ b/drivers/accel/ane/ane_h16_add128_fixture.h @@ -0,0 +1,284 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_ADD128_FIXTURE_H__ +#define __ANE_H16_ADD128_FIXTURE_H__ + +/* + * The pinned H16G add-128 program (mil-hwx-compiler prepare_alu "add 128", + * 49152 bytes, SHA-256 231f54c2...) and the four LOAD sections that + * tools/testing/selftests/drivers/ane/h16-sections.py builds for it. Stored + * as runs of nonzero bytes; everything else is zero. Generated, do not edit. + */ +#include + +struct ane_h16_sparse_run { + u32 offset; + u32 length; + const u8 *bytes; +}; + +#define ADD128_HWX_SIZE 49152 +static const u8 add128_hwx_sha256[] = { + 0x23, 0x1f, 0x54, 0xc2, 0x8e, 0xa6, 0xb1, 0x4a, + 0x28, 0x67, 0xfa, 0xc7, 0x8e, 0xac, 0x1a, 0x2c, + 0xb2, 0x2a, 0xa3, 0xb0, 0xee, 0xaa, 0x28, 0x11, + 0x3e, 0xd0, 0x3c, 0x52, 0x0b, 0x83, 0xa1, 0xd3, +}; + +static const struct ane_h16_sparse_run add128_hwx_runs[] = { + { 0x0, 50, (const u8[]){ + 0xce, 0xfa, 0xef, 0xbe, 0x80, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x38, 0x34, 0x00, 0x00, + 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x50, 0x41, 0x47, 0x45, 0x5a, 0x45, + 0x52, 0x4f, + } }, + { 0x41, 1, (const u8[]){ + 0x40, + } }, + { 0x64, 38, (const u8[]){ + 0x04, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, + 0x5f, 0x5f, 0x46, 0x56, 0x4d, 0x4c, 0x49, 0x42, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, + } }, + { 0xa0, 81, (const u8[]){ + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x46, 0x56, + 0x4d, 0x4c, 0x49, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, + } }, + { 0x100, 34, (const u8[]){ + 0x19, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x46, 0x56, + 0x4d, 0x4c, 0x49, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + } }, + { 0x138, 81, (const u8[]){ + 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x46, 0x56, + 0x4d, 0x4c, 0x49, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, + } }, + { 0x198, 34, (const u8[]){ + 0x19, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x46, 0x56, + 0x4d, 0x4c, 0x49, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + } }, + { 0x1d0, 81, (const u8[]){ + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x46, 0x56, + 0x4d, 0x4c, 0x49, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, + } }, + { 0x230, 137, (const u8[]){ + 0x19, 0x00, 0x00, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x54, 0x45, + 0x58, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x01, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x5f, 0x5f, 0x74, 0x65, 0x78, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x54, 0x45, 0x58, 0x54, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x01, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x30, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x98, 0x34, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x28, + } }, + { 0x2c8, 65, (const u8[]){ + 0x5f, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x54, 0x45, 0x58, 0x54, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x81, 0x01, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x41, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x26, + } }, + { 0x318, 132, (const u8[]){ + 0x40, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x69, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x6f, 0x30, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x69, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xa0, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x22, 0x02, 0x00, 0x00, 0x00, 0x80, 0x01, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x81, 0x01, 0x30, + } }, + { 0x3eb, 33, (const u8[]){ + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x30, + } }, + { 0xb89, 57, (const u8[]){ + 0x80, 0x01, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x4c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x98, 0x08, + } }, + { 0xbd8, 121, (const u8[]){ + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x40, 0x1c, 0x46, + 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x61, 0x69, 0x6e, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x48, 0x0d, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x4a, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x38, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x3d, 0x0d, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x01, + } }, + { 0xc69, 49, (const u8[]){ + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x3f, 0x0d, 0x00, 0x00, 0x00, + 0x80, + } }, + { 0x1950, 73, (const u8[]){ + 0x6d, 0x61, 0x69, 0x6e, 0x00, 0x78, 0x00, 0x69, 0x30, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x48, 0x0d, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x4a, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x38, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x3d, 0x0d, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x01, + } }, + { 0x19b1, 49, (const u8[]){ + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x3f, 0x0d, 0x00, 0x00, 0x00, + 0x80, + } }, + { 0x2698, 73, (const u8[]){ + 0x6d, 0x61, 0x69, 0x6e, 0x00, 0x7a, 0x00, 0x69, 0x31, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x50, 0x0d, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x4a, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x38, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x3d, 0x0d, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x01, + } }, + { 0x26f9, 49, (const u8[]){ + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x46, 0x0d, 0x00, 0x00, 0x00, + 0x80, + } }, + { 0x33e0, 177, (const u8[]){ + 0x6d, 0x61, 0x69, 0x6e, 0x00, 0x79, 0x40, 0x6f, 0x75, 0x74, 0x70, 0x75, + 0x74, 0x00, 0x6f, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x41, 0x4e, 0x45, 0x43, + 0x20, 0x76, 0x31, 0x0a, 0x72, 0x65, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x5f, 0x68, 0x77, 0x78, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, + 0x72, 0x20, 0x76, 0x31, 0x0a, 0x0a, 0x20, 0x4d, 0x6f, 0x64, 0x75, 0x6c, + 0x65, 0x20, 0x41, 0x4e, 0x45, 0x43, 0x3a, 0x0a, 0x09, 0x20, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x3a, 0x20, 0x68, 0x31, 0x36, 0x67, 0x0a, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x58, 0x34, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x88, 0x34, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x0f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0f, 0x02, 0x00, 0x00, + 0x00, 0x80, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, + 0x0f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x69, 0x30, 0x00, 0x6f, 0x30, 0x00, 0x69, 0x31, + } }, + { 0x4000, 1, (const u8[]){ + 0x01, + } }, + { 0x4012, 286, (const u8[]){ + 0x48, 0x00, 0x01, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x68, 0xf8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x40, 0x15, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x05, 0x80, + 0x41, 0x95, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x17, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x93, 0x04, 0x81, 0xce, 0x00, + 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x5a, 0x93, 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x31, 0x20, + 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x72, 0x01, + 0x50, 0x00, 0x45, 0x90, 0x7f, 0x98, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0x40, 0x00, + 0x01, 0x00, 0x40, 0x11, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x44, 0x14, + 0x00, 0x81, 0x00, 0x01, 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x40, 0x96, + 0x10, 0x83, 0x12, 0x00, 0x00, 0x00, 0xa0, 0x00, 0xa0, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0x00, 0x01, 0x07, 0x01, 0x20, 0x44, 0x93, + 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x93, + 0x00, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, + 0x80, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, + 0x00, 0x22, 0x21, 0x00, 0x00, 0x00, 0x41, 0x13, 0x00, 0x22, 0x21, 0x00, + 0x00, 0x00, 0x40, 0x14, 0x00, 0x22, 0x31, 0x00, 0x00, 0x01, + } }, +}; + +#define ADD128_GENERIC_SIZE 0x298 +static const struct ane_h16_sparse_run add128_generic_runs[] = { + { 0x0, 5, (const u8[]){ + 0x01, 0x00, 0x00, 0x00, 0x10, + } }, + { 0x204, 21, (const u8[]){ + 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + } }, + { 0x229, 32, (const u8[]){ + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + } }, + { 0x259, 32, (const u8[]){ + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + } }, + { 0x289, 9, (const u8[]){ + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + } }, +}; + +#define ADD128_OPERATION_SIZE 0x410 +static const struct ane_h16_sparse_run add128_operation_runs[] = { + { 0x0, 37, (const u8[]){ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x06, + } }, +}; + +#define ADD128_PROCEDURE_SIZE 0x38 +static const struct ane_h16_sparse_run add128_procedure_runs[] = { + { 0x0, 53, (const u8[]){ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x04, + } }, +}; + +#define ADD128_PROPERTIES_SIZE 0x28 +static const struct ane_h16_sparse_run add128_properties_runs[] = { + { 0x0, 13, (const u8[]){ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, + } }, + { 0x20, 2, (const u8[]){ + 0x30, 0x01, + } }, +}; + +#endif /* __ANE_H16_ADD128_FIXTURE_H__ */ diff --git a/drivers/accel/ane/ane_h16_backend.c b/drivers/accel/ane/ane_h16_backend.c new file mode 100644 index 0000000000000..7e6bd0b69d19c --- /dev/null +++ b/drivers/accel/ane/ane_h16_backend.c @@ -0,0 +1,1030 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Staged H16 backend, using the accepted H16 cold-boot facts, round 5. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ane_fw_platform.h" +#include "ane_h16_accel.h" +#include "ane_h16_boot.h" +#include "ane_h16_preload.h" + +#define ANE_H16_TX_WORDS 272 +/* Private program window above the SHAREDMALLOC range; host policy. */ +#define ANE_H16_PROGRAM_WINDOW SZ_1G + +struct ane_h16_mapping { + struct list_head node; + u64 dva; + size_t size; + size_t mapped; + struct page **pages; + u32 blocks; + void *cpu; +}; + +struct ane_h16_device { + struct ane_h16_boot boot; + struct device *dev; + void __iomem *window[ANE_H16_WINDOWS]; + struct device *power[5]; + struct device *dart[3]; + u32 powered; + u32 darts_powered; + u64 preload_phys[2]; + struct list_head mappings; + struct iommu_domain *domain; + struct iommu_group *group; + bool claimed; + bool attached; + bool pinned; + int irq; + struct ane_fw_rtk rtk; + struct { + u64 word0; + u64 word1; + } tx[ANE_H16_TX_WORDS]; + u32 tx_read; + u32 tx_write; + bool rtk_failed; + u64 shared_next; + u64 program_next; + u64 program_end; + struct dentry *debugfs; + bool acquired; + bool refused; + bool program_ready; + struct ane_h16_session session; + struct work_struct start; +}; + +static DEFINE_MUTEX(ane_h16_lock); +static bool ane_h16_released; +/* /ane-h16, shared by bound devices; protected by ane_h16_lock. */ +static struct dentry *ane_h16_debugfs; +static unsigned int ane_h16_debugfs_users; + +static u32 ane_h16_mmio_read32(void *ctx, enum ane_h16_window w, u32 off) +{ + struct ane_h16_device *h = ctx; + + return readl(h->window[w] + off); +} + +static u64 ane_h16_mmio_read64(void *ctx, enum ane_h16_window w, u32 off) +{ + struct ane_h16_device *h = ctx; + + return readq(h->window[w] + off); +} + +static void ane_h16_mmio_write32(void *ctx, enum ane_h16_window w, u32 off, u32 value) +{ + struct ane_h16_device *h = ctx; + + writel(value, h->window[w] + off); +} + +static void ane_h16_mmio_write64(void *ctx, enum ane_h16_window w, u32 off, u64 value) +{ + struct ane_h16_device *h = ctx; + + writeq(value, h->window[w] + off); +} + +static void ane_h16_delay(void *ctx, unsigned int us) +{ + if (us < 100) + udelay(us); + else + usleep_range(us, us + 50); +} + +static void ane_h16_log(void *ctx, const char *item, u64 a, u64 b) +{ + struct ane_h16_device *h = ctx; + + dev_info(h->dev, "H16 %s %llx %llx\n", item, a, b); +} + +static void ane_h16_quarantine(void *ctx, int error) +{ + struct ane_h16_device *h = ctx; + + dev_err(h->dev, "H16 quarantine error %d; mappings, power and module held until reboot\n", + error); +} + +static int ane_h16_pin(void *ctx) +{ + struct ane_h16_device *h = ctx; + + if (!try_module_get(THIS_MODULE)) + return -ENODEV; + h->pinned = true; + ane_h16_released = true; + return 0; +} + +/* Nonblocking transmit queue. RTKit progress never gates scratch progress. */ +static int ane_h16_mailbox_send(void *ctx, u64 word0, u64 word1) +{ + struct ane_h16_device *h = ctx; + + if (h->tx_write - h->tx_read == ARRAY_SIZE(h->tx)) + return -ENOSPC; + h->tx[h->tx_write % ARRAY_SIZE(h->tx)].word0 = word0; + h->tx[h->tx_write % ARRAY_SIZE(h->tx)].word1 = word1; + h->tx_write++; + return 0; +} + +static void ane_h16_mailbox(void *ctx) +{ + struct ane_h16_device *h = ctx; + void __iomem *mb = h->window[ANE_H16_ENGINE] + h->boot.soc->mailbox; + unsigned int i; + + for (i = 0; i < 16; i++) { + u64 word0, word1; + int ret; + + if (readl(mb + 0x114) & BIT(17)) + break; + /* Two 64-bit accesses. Reading the second word pops the entry. */ + word0 = readq(mb + 0x830); + word1 = readq(mb + 0x838); + dev_info(h->dev, "H16 ASC RX %016llx %016llx\n", word0, word1); + if (h->rtk_failed) + continue; + ret = ane_fw_rtk_receive(&h->rtk, word0, word1, h, ane_h16_mailbox_send); + if (ret) { + dev_info(h->dev, + "H16 RTKit message declined %d; scratch remains independent\n", + ret); + if (ret != -EOPNOTSUPP) + h->rtk_failed = true; + } + } + for (i = 0; i < 16 && h->tx_read != h->tx_write; i++) { + u32 index = h->tx_read % ARRAY_SIZE(h->tx); + + if (readl(mb + 0x110) & BIT(16)) + break; + writeq(h->tx[index].word0, mb + 0x800); + /* Publish word 0 before the word-1 FIFO commit. */ + wmb(); + writeq(h->tx[index].word1, mb + 0x808); + dev_info(h->dev, "H16 ASC TX %016llx %016llx\n", + h->tx[index].word0, h->tx[index].word1); + h->tx_read++; + } +} + +static int ane_h16_map_range(struct ane_h16_device *h, u64 dva, u64 pa, + size_t size, int prot) +{ + struct ane_h16_mapping *m; + int ret; + + m = kzalloc_obj(*m); + if (!m) + return -ENOMEM; + m->dva = dva; + m->size = size; + list_add_tail(&m->node, &h->mappings); + ret = iommu_map(h->domain, dva, pa, size, prot, GFP_KERNEL); + if (!ret) + m->mapped = size; + return ret; +} + +static int ane_h16_count_group(struct device *dev, void *data) +{ + unsigned int *count = data; + + (*count)++; + return 0; +} + +static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, int prot, + struct ane_fw_surface *out); + +/* + * The firmware span is config_span bytes from the TEXT DVA. The preload covers + * TEXT and DATA only; the firmware keeps runtime heap state in the rest, which + * the host supplies zeroed and device-writable before boot. + */ +static int ane_h16_map_heap_tail(struct ane_h16_device *h) +{ + struct ane_h16_boot *b = &h->boot; + u64 tail = b->soc->preload_dva[1] + b->soc->preload_size[1]; + u64 end = b->soc->preload_dva[0] + b->build->config_span; + struct ane_fw_surface s; + int ret; + + if (tail == end) + return 0; + if (tail > end || !IS_ALIGNED(tail, SZ_16K) || !IS_ALIGNED(end, SZ_16K) || + end > b->build->pool_dva) + return -ERANGE; + ret = ane_h16_alloc_prot(h, tail, end - tail, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE, &s); + if (ret) + return ret; + dev_info(h->dev, "H16 firmware heap tail mapped at %llx bytes %llx writable\n", + tail, end - tail); + return 0; +} + +static int ane_h16_map(void *ctx) +{ + struct ane_h16_device *h = ctx; + struct ane_h16_boot *b = &h->boot; + unsigned int count = 0, i; + int ret; + + if (of_count_phandle_with_args(h->dev->of_node, "iommus", "#iommu-cells") != 3) { + dev_err(h->dev, "H16 map requires only SID 0 on three DARTs; refusing extra streams\n"); + return -EOPNOTSUPP; + } + h->group = iommu_group_get(h->dev); + if (!h->group) + return -ENODEV; + ret = iommu_group_for_each_dev(h->group, &count, ane_h16_count_group); + if (ret || count != 1) + return ret ?: -EBUSY; + ret = iommu_device_claim_dma_owner(h->dev, h); + if (ret) + return ret; + h->claimed = true; + h->domain = iommu_paging_domain_alloc(h->dev); + if (IS_ERR_OR_NULL(h->domain)) { + ret = h->domain ? PTR_ERR(h->domain) : -ENOMEM; + h->domain = NULL; + return ret; + } + if (h->domain->type != IOMMU_DOMAIN_UNMANAGED || + !(h->domain->pgsize_bitmap & SZ_16K)) + return -EOPNOTSUPP; + /* Never widen a finalized DART aperture by assigning geometry fields. */ + for (i = 0; i < 4; i++) { + u64 page = round_down(b->ranges[i][0], SZ_16K); + + if (!ane_fw_contains(h->domain->geometry.aperture_start, + h->domain->geometry.aperture_end - + h->domain->geometry.aperture_start + 1, + page, SZ_16K)) + return -ERANGE; + } + if (h->domain->geometry.aperture_end < b->build->shared_dva + ANE_FW_SHARED_TOTAL) + return -ERANGE; + if (h->domain->geometry.aperture_end < h->program_end - 1) + return -ERANGE; + ret = iommu_attach_device(h->domain, h->dev); + if (ret) + return ret; + h->attached = true; + for (i = 0; i < 2; i++) { + ret = ane_h16_map_range(h, b->soc->preload_dva[i], h->preload_phys[i], + b->soc->preload_size[i], + IOMMU_READ | IOMMU_CACHE | (i ? IOMMU_WRITE : 0)); + if (ret) + return ret; + dev_info(h->dev, "H16 preload %s mapped at %llx bytes %llx\n", + i ? "DATA" : "TEXT", b->soc->preload_dva[i], b->soc->preload_size[i]); + } + ret = ane_h16_map_heap_tail(h); + if (ret) + return ret; + for (i = 0; i < 4; i++) { + u64 page = round_down(b->ranges[i][0], SZ_16K); + + ret = ane_h16_map_range(h, page, page, SZ_16K, + IOMMU_READ | IOMMU_WRITE | IOMMU_MMIO); + if (ret) + return ret; + dev_info(h->dev, "H16 identity MMIO %llx mapped\n", page); + } + return 0; +} + +static int ane_h16_alloc_prot(struct ane_h16_device *h, u64 dva, size_t size, int prot, + struct ane_fw_surface *out) +{ + struct ane_h16_mapping *m; + u32 i, j, pages_per_block = SZ_16K / PAGE_SIZE; + unsigned int order = get_order(SZ_16K); + int ret; + + if (!dev_is_dma_coherent(h->dev)) { + dev_err(h->dev, "H16 private surfaces require a coherent client\n"); + return -EOPNOTSUPP; + } + if (!size || !IS_ALIGNED(size, SZ_16K) || !IS_ALIGNED(dva, SZ_16K) || + PAGE_SIZE > SZ_16K) + return -EINVAL; + m = kzalloc_obj(*m); + if (!m) + return -ENOMEM; + m->size = size; + m->dva = dva; + list_add_tail(&m->node, &h->mappings); + m->pages = kvcalloc(size / PAGE_SIZE, sizeof(*m->pages), GFP_KERNEL); + if (!m->pages) + return -ENOMEM; + for (i = 0; i < size / SZ_16K; i++) { + struct page *page = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); + + if (!page) + return -ENOMEM; + for (j = 0; j < pages_per_block; j++) + m->pages[i * pages_per_block + j] = page + j; + m->blocks++; + } + /* The coherent-client gate permits matching cached CPU/device aliases. */ + m->cpu = vmap(m->pages, size / PAGE_SIZE, VM_MAP, PAGE_KERNEL); + if (!m->cpu) + return -ENOMEM; + memset(m->cpu, 0, size); + for (i = 0; i < m->blocks; i++) { + ret = iommu_map(h->domain, dva + i * SZ_16K, + page_to_phys(m->pages[i * pages_per_block]), SZ_16K, + prot, GFP_KERNEL); + if (ret) + return ret; + m->mapped += SZ_16K; + } + *out = (struct ane_fw_surface) { .cpu = m->cpu, .dva = dva, .size = size }; + return 0; +} + +static int ane_h16_alloc_at(void *ctx, u64 dva, size_t size, struct ane_fw_surface *out) +{ + return ane_h16_alloc_prot(ctx, dva, size, IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE, out); +} + +/* Program sections and surfaces. Only outputs and scratch are device-writable. */ +static int ane_h16_program_map(void *ctx, size_t size, bool write, struct ane_fw_surface *out) +{ + struct ane_h16_device *h = ctx; + u64 dva = h->program_next; + int ret; + + if (!size || size > h->program_end - dva) + return -ENOSPC; + /* A failed mapping keeps its range; DVAs are never reused before reset. */ + h->program_next += size; + ret = ane_h16_alloc_prot(h, dva, size, + IOMMU_READ | IOMMU_CACHE | (write ? IOMMU_WRITE : 0), out); + if (!ret) + dev_info(h->dev, "H16 program mapping %llx bytes %zx%s\n", dva, size, + write ? " writable" : ""); + return ret; +} + +static u64 ane_h16_load64(void *ctx, const void *p) +{ + return le64_to_cpu(READ_ONCE(*(const __le64 *)p)); +} + +static void ane_h16_store64(void *ctx, void *p, u64 value) +{ + WRITE_ONCE(*(__le64 *)p, cpu_to_le64(value)); +} + +static void ane_h16_dma_read_barrier(void *ctx) +{ + dma_rmb(); +} + +static void ane_h16_dma_write_barrier(void *ctx) +{ + dma_wmb(); +} + +static u32 ane_h16_engine_read(void *ctx, u32 offset) +{ + return ane_h16_mmio_read32(ctx, ANE_H16_ENGINE, offset); +} + +static void ane_h16_engine_write(void *ctx, u32 offset, u32 value) +{ + struct ane_h16_device *h = ctx; + + ane_h16_mmio_write32(ctx, ANE_H16_ENGINE, offset, value); + if (offset == h->boot.soc->irq_ack) { + /* Complete the snapshot acknowledgment before ring reads (r5). */ + mb(); + } +} + +static u64 ane_h16_now(void *ctx) +{ + return ktime_get_ns(); +} + +/* + * The channel IRQ stays masked, so every wait polls. Sleep in short steps + * first (see ane_fw_poll_sleep_us()): a fixed 1 ms sleep made each of the + * four waits of an EXECUTE cost at least a millisecond. + */ +static void ane_h16_relax(void *ctx, u64 waited_ns) +{ + struct ane_h16_device *h = ctx; + unsigned int min_us, max_us; + int ret; + + ane_h16_mailbox(h); + ret = ane_h16_faults(&h->boot, false); + if (ret) + ane_fw_transport_fail(&h->boot.transport, ret); + min_us = ane_fw_poll_sleep_us(waited_ns, &max_us); + usleep_range(min_us, max_us); +} + +static int ane_h16_alloc_shared(void *ctx, size_t size, u32 tag, struct ane_fw_surface *out) +{ + struct ane_h16_device *h = ctx; + int ret = ane_h16_alloc_at(h, h->shared_next, size, out); + + if (!ret) { + h->shared_next += size; + dev_info(h->dev, "H16 SHAREDMALLOC bytes %zx tag %08x DVA %llx\n", + size, tag, out->dva); + } + return ret; +} + +static void ane_h16_refused(void *ctx, u64 w0, u64 w1, u64 w2, int error) +{ + struct ane_h16_device *h = ctx; + + dev_err(h->dev, "H16 SHAREDMALLOC refused %llx %llx %llx error %d\n", + w0, w1, w2, error); +} + +static const struct ane_h16_boot_ops ane_h16_ops = { + .read32 = ane_h16_mmio_read32, + .read64 = ane_h16_mmio_read64, + .write32 = ane_h16_mmio_write32, + .write64 = ane_h16_mmio_write64, + .delay_us = ane_h16_delay, + .map = ane_h16_map, + .alloc_at = ane_h16_alloc_at, + .pin = ane_h16_pin, + .mailbox = ane_h16_mailbox, + .log = ane_h16_log, + .quarantine = ane_h16_quarantine, +}; + +static const struct ane_h16_session_ops ane_h16_session_ops = { + .map = ane_h16_program_map, + .log = ane_h16_log, +}; + +static const struct ane_fw_transport_ops ane_h16_transport_ops = { + .load64 = ane_h16_load64, + .store64 = ane_h16_store64, + .read_barrier = ane_h16_dma_read_barrier, + .write_barrier = ane_h16_dma_write_barrier, + .read32 = ane_h16_engine_read, + .write32 = ane_h16_engine_write, + .now_ns = ane_h16_now, + .relax = ane_h16_relax, + .alloc = ane_h16_alloc_shared, + .refused = ane_h16_refused, +}; + +/* Pre-RUN rollback of everything a stage acquired. h itself stays. */ +static void ane_h16_release(struct ane_h16_device *h) +{ + struct ane_h16_mapping *m, *next; + unsigned int i, pages_per_block = SZ_16K / PAGE_SIZE; + + if (h->boot.ran || h->pinned || !h->acquired) + return; + list_for_each_entry_safe_reverse(m, next, &h->mappings, node) { + if (m->mapped) + iommu_unmap(h->domain, m->dva, m->mapped); + if (m->cpu) + vunmap(m->cpu); + for (i = 0; i < m->blocks; i++) + __free_pages(m->pages[i * pages_per_block], get_order(SZ_16K)); + kvfree(m->pages); + list_del(&m->node); + kfree(m); + } + if (h->attached) + iommu_detach_device(h->domain, h->dev); + h->attached = false; + if (h->domain) + iommu_domain_free(h->domain); + h->domain = NULL; + if (h->claimed) + iommu_device_release_dma_owner(h->dev); + h->claimed = false; + if (h->group) + iommu_group_put(h->group); + h->group = NULL; + if (h->irq >= 0) + free_irq(h->irq, h); + h->irq = -1; + for (i = ANE_H16_WINDOWS; i-- > 0; ) { + if (h->window[i]) + iounmap(h->window[i]); + h->window[i] = NULL; + } + while (h->darts_powered) + pm_runtime_put_sync(h->dart[--h->darts_powered]); + for (i = 0; i < 3; i++) { + if (h->dart[i]) + put_device(h->dart[i]); + h->dart[i] = NULL; + } + while (h->powered) { + struct device *pd = h->power[--h->powered]; + + pm_runtime_put_sync(pd); + dev_pm_domain_detach(pd, true); + } + h->acquired = false; + dev_info(h->dev, "H16 pre-RUN rollback complete; no CPU release\n"); +} + +static irqreturn_t ane_h16_masked_irq(int irq, void *data) +{ + return IRQ_NONE; +} + +static int ane_h16_acquire(struct platform_device *pdev, struct ane_h16_device *h) +{ + struct device_node *np = pdev->dev.of_node; + static const char * const names[] = { "engine", "pmgr", "aux-3" }; + unsigned int i, j; + int ret, irq; + + if (of_count_phandle_with_args(np, "power-domains", "#power-domain-cells") != 5) + return -EINVAL; + for (i = 0; i < 5; i++) { + struct device *pd = dev_pm_domain_attach_by_id(h->dev, i); + + if (IS_ERR_OR_NULL(pd)) + return pd ? PTR_ERR(pd) : -ENODEV; + ret = pm_runtime_resume_and_get(pd); + if (ret < 0) { + dev_pm_domain_detach(pd, true); + return ret; + } + h->power[h->powered++] = pd; + } + for (i = 0; i < ARRAY_SIZE(names); i++) { + struct resource *r = platform_get_resource_byname(pdev, IORESOURCE_MEM, names[i]); + + if (!r) + return -ENODEV; + h->window[i] = ioremap_np(r->start, resource_size(r)); + if (!h->window[i]) + return -ENOMEM; + } + for (i = 0; i < 3; i++) { + struct of_phandle_args args; + struct platform_device *dart; + struct resource r; + + ret = of_parse_phandle_with_args(np, "iommus", "#iommu-cells", i, &args); + if (ret) + return ret; + if (args.args_count != 1 || args.args[0] != 0 || + !of_device_is_compatible(args.np, "apple,t8110-dart") || + !of_device_is_available(args.np)) { + of_node_put(args.np); + return -EINVAL; + } + dart = of_find_device_by_node(args.np); + ret = of_address_to_resource(args.np, 0, &r); + of_node_put(args.np); + if (!dart) + return -EPROBE_DEFER; + h->dart[i] = &dart->dev; + for (j = 0; j < i; j++) + if (h->dart[j] == h->dart[i]) + return -EINVAL; + if (ret) + return ret; + ret = pm_runtime_resume_and_get(h->dart[i]); + if (ret < 0) + return ret; + h->darts_powered++; + h->window[ANE_H16_DART0 + i] = ioremap_np(r.start, resource_size(&r)); + if (!h->window[ANE_H16_DART0 + i]) + return -ENOMEM; + /* Only the instance the platform names has a DAPF bank. */ + if (i != h->boot.soc->dapf_dart) + continue; + h->window[ANE_H16_DAPF] = ioremap_np(r.start + h->boot.soc->dapf_offset, + h->boot.soc->dapf_size); + if (!h->window[ANE_H16_DAPF]) + return -ENOMEM; + } + if (!h->window[ANE_H16_DAPF]) + return -ENODEV; + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + ret = request_irq(irq, ane_h16_masked_irq, IRQF_NO_AUTOEN, "ane-h16-polled", h); + if (ret) + return ret; + h->irq = irq; + dev_info(h->dev, "H16 polled mode; channel IRQ remains masked\n"); + return 0; +} + +static int ane_h16_preload_identity(struct ane_h16_device *h) +{ + struct ane_h16_boot *b = &h->boot; + void *snap[2] = {}, *second = NULL; + const char *identity = NULL; + unsigned int i; + int ret = -ENOMEM; + + for (i = 0; i < 2; i++) { + u64 base, size; + void *live; + + ret = ane_fw_preload_range(h->dev->of_node, i, &base, &size); + if (!ret && size != b->soc->preload_size[i]) + ret = -EINVAL; + if (ret) + goto out; + ret = -ENOMEM; + h->preload_phys[i] = base; + snap[i] = vzalloc(size); + second = vzalloc(size); + if (!snap[i] || !second) + goto out; + live = memremap(base, size, MEMREMAP_WB); + if (!live) + goto out; + memcpy(snap[i], live, size); + /* Separate the two observations used to reject an unstable preload. */ + rmb(); + memcpy(second, live, size); + memunmap(live); + if (memcmp(snap[i], second, size)) { + ret = -EBUSY; + goto out; + } + vfree(second); + second = NULL; + } + ret = -ENOEXEC; + for (i = 0; i < b->soc->h16_profile_count && ret == -ENOEXEC; i++) { + const struct ane_h16_profile *p = &b->soc->h16_profiles[i]; + u8 digest[SHA256_DIGEST_SIZE]; + + ret = ane_h16_preload_digest(p->preload, snap[0], b->soc->preload_size[0], + snap[1], b->soc->preload_size[1], digest); + if (ret) + continue; + if (memcmp(digest, p->preload->digest, sizeof(digest))) { + /* The layout matched; name the image so a new build can be pinned. */ + dev_info(h->dev, "H16 preload has the %s layout but digest %*phN\n", + p->identity, (int)sizeof(digest), digest); + ret = -ENOEXEC; + continue; + } + b->build = p->build; + identity = p->identity; + } + if (ret) { + dev_err(h->dev, "H16 preload matches none of %u firmware profiles, error %d\n", + b->soc->h16_profile_count, ret); + goto out; + } + /* Firmware DVA layout belongs to the build just selected. */ + h->shared_next = b->build->shared_dva; + h->program_next = b->build->shared_dva + ANE_FW_SHARED_TOTAL; + h->program_end = h->program_next + ANE_H16_PROGRAM_WINDOW; + dev_info(h->dev, "H16 preload identity %s accepted\n", identity); +out: + vfree(second); + vfree(snap[0]); + vfree(snap[1]); + return ret; +} + +/* After startup: one program session and the ABI 3 accel node. */ +static int ane_h16_program_stage(struct ane_h16_device *h) +{ + int ret; + + ret = ane_h16_session_init(&h->session, &h->boot.transport, &ane_h16_session_ops, h, + h->boot.build->command_timeout_ns); + if (!ret) + ret = ane_h16_accel_register(h->dev, &h->session); + if (ret) { + dev_err(h->dev, "H16 program stage unavailable, error %d; held until reboot\n", + ret); + return ret; + } + h->program_ready = true; + dev_info(h->dev, "H16 stage 5 program window %llx..%llx\n", h->program_next, + h->program_end - 1); + return 0; +} + +/* + * Advance the staged bring-up to @stage. Caller holds ane_h16_lock. Before + * RUN any failure rolls back power and mappings and refuses further stages + * until the module is reloaded. From RUN on, everything stays held. + */ +static int ane_h16_advance(struct ane_h16_device *h, enum ane_h16_stage stage) +{ + enum ane_h16_stage target = stage > ANE_H16_STARTUP ? ANE_H16_STARTUP : stage; + int ret; + + if (h->refused || h->boot.failed || h->boot.quarantined) + return -EBUSY; + if (h->program_ready || (stage != ANE_H16_PROGRAM && stage <= h->boot.stage)) + return -EALREADY; + if (!h->acquired) { + h->acquired = true; + ret = ane_h16_preload_identity(h); + if (!ret) + ret = ane_h16_acquire(to_platform_device(h->dev), h); + if (ret) + goto refuse; + } + if (target > h->boot.stage) { + ret = ane_h16_run(&h->boot, target); + if (ret && !h->boot.ran) + goto refuse; + if (ret) { + dev_err(h->dev, "H16 stage %s failed after RUN, error %d; held until reboot\n", + ane_h16_stage_name(stage), ret); + return ret; + } + if (h->boot.stage == ANE_H16_STARTUP) + print_hex_dump(KERN_INFO, "H16 RESOURCE_INFO ", DUMP_PREFIX_OFFSET, 16, 1, + h->boot.resource_info, sizeof(h->boot.resource_info), + false); + } + if (stage == ANE_H16_PROGRAM) { + ret = ane_h16_program_stage(h); + if (ret) { + h->refused = true; + return ret; + } + } + dev_info(h->dev, "H16 stage %s complete%s\n", ane_h16_stage_name(stage), + h->boot.ran ? "; held until reboot" : ""); + return 0; +refuse: + dev_err(h->dev, "H16 stage %s refused before RUN, error %d\n", + ane_h16_stage_name(stage), ret); + h->refused = true; + ane_h16_release(h); + return ret; +} + +/* + * The bootloader handed off a preload that passed metadata preflight, so the + * engine is brought up to the program stage once per boot, off the probe + * path. A failure before RUN leaves the device refused; after RUN it is held + * until reboot, as for a stage written to debugfs. + */ +static void ane_h16_start_work(struct work_struct *work) +{ + struct ane_h16_device *h = container_of(work, struct ane_h16_device, start); + + mutex_lock(&ane_h16_lock); + if (!h->acquired) + ane_h16_advance(h, ANE_H16_PROGRAM); + mutex_unlock(&ane_h16_lock); +} + +/* + * Diagnostic control in debugfs: read the stage state, or, before the boot + * bring-up has run, write a stage name to /ane-h16//stage to + * run every stage up to it. + */ +static ssize_t ane_h16_stage_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct ane_h16_device *h = file->private_data; + char buf[16]; + int stage, ret; + + if (!count || count >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, count)) + return -EFAULT; + buf[count] = 0; + strim(buf); + stage = ane_h16_stage_parse(buf); + if (stage <= ANE_H16_REFUSE) + return -EINVAL; + mutex_lock(&ane_h16_lock); + ret = ane_h16_advance(h, stage); + mutex_unlock(&ane_h16_lock); + return ret ? ret : count; +} + +static ssize_t ane_h16_stage_read(struct file *file, char __user *ubuf, size_t count, + loff_t *ppos) +{ + struct ane_h16_device *h = file->private_data; + char buf[96]; + int len; + + mutex_lock(&ane_h16_lock); + len = scnprintf(buf, sizeof(buf), "%s ran=%d refused=%d quarantined=%d error=%d\n", + h->program_ready ? "program" : ane_h16_stage_name(h->boot.stage), + h->boot.ran, h->refused || h->boot.failed, h->boot.quarantined, + h->boot.error); + mutex_unlock(&ane_h16_lock); + return simple_read_from_buffer(ubuf, count, ppos, buf, len); +} + +static const struct file_operations ane_h16_stage_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = ane_h16_stage_read, + .write = ane_h16_stage_write, + .llseek = default_llseek, +}; + +static void ane_h16_us(struct seq_file *m, u64 ns) +{ + seq_printf(m, " %9llu.%u", div_u64(ns, 1000), (unsigned int)(div_u64(ns, 100) % 10)); +} + +/* + * /ane-h16//execute: where the time of the successful EXECUTEs + * since the last reset went, per phase in microseconds (last, mean, min, + * max), and the poll sleeps per request. Writing anything resets it. + */ +static int ane_h16_execute_show(struct seq_file *m, void *v) +{ + struct ane_h16_device *h = m->private; + struct ane_h16_exec_stats *st; + unsigned int i; + + mutex_lock(&ane_h16_lock); + if (!h->program_ready) { + mutex_unlock(&ane_h16_lock); + seq_puts(m, "executes 0 (program stage not reached)\n"); + return 0; + } + mutex_unlock(&ane_h16_lock); + /* program_ready never goes back, and the session outlives this file. */ + st = kmalloc_obj(*st); + if (!st) + return -ENOMEM; + ane_h16_session_stats(&h->session, st, false); + seq_printf(m, "executes %llu\n", st->count); + if (st->count) { + seq_printf(m, "%-12s %11s %11s %11s %11s\n", "phase", "last_us", "mean_us", + "min_us", "max_us"); + for (i = 0; i < ANE_H16_T_PHASES; i++) { + seq_printf(m, "%-12s", ane_h16_exec_phase_names[i]); + ane_h16_us(m, st->last.ns[i]); + ane_h16_us(m, div64_u64(st->sum.ns[i], st->count)); + ane_h16_us(m, st->min.ns[i]); + ane_h16_us(m, st->max.ns[i]); + seq_putc(m, '\n'); + } + seq_printf(m, "%-12s %11llu %9llu.%u %11llu %11llu\n", "polls", st->last.polls, + div64_u64(st->sum.polls, st->count), + (unsigned int)(div64_u64(st->sum.polls * 10, st->count) % 10), + st->min.polls, st->max.polls); + } + kfree(st); + return 0; +} + +static int ane_h16_execute_open(struct inode *inode, struct file *file) +{ + return single_open(file, ane_h16_execute_show, inode->i_private); +} + +static ssize_t ane_h16_execute_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct ane_h16_device *h = ((struct seq_file *)file->private_data)->private; + bool ready; + + mutex_lock(&ane_h16_lock); + ready = h->program_ready; + mutex_unlock(&ane_h16_lock); + if (ready) + ane_h16_session_stats(&h->session, NULL, true); + return count; +} + +static const struct file_operations ane_h16_execute_fops = { + .owner = THIS_MODULE, + .open = ane_h16_execute_open, + .read = seq_read, + .write = ane_h16_execute_write, + .llseek = seq_lseek, + .release = single_release, +}; + +int ane_h16_backend_probe(struct platform_device *pdev, const struct ane_fw_platform *soc) +{ + struct ane_h16_device *h; + unsigned int i, j; + int ret; + + mutex_lock(&ane_h16_lock); + if (ane_h16_released) { + ret = -EBUSY; + goto unlock; + } + h = kzalloc_obj(*h); + if (!h) { + ret = -ENOMEM; + goto unlock; + } + h->dev = &pdev->dev; + h->irq = -1; + INIT_LIST_HEAD(&h->mappings); + h->boot.soc = soc; + /* The build, and with it the DVA layout, is chosen from the preload. */ + h->boot.build = NULL; + h->boot.ctx = h; + h->boot.ops = &ane_h16_ops; + h->boot.transport_ops = &ane_h16_transport_ops; + ret = of_property_read_u64_array(h->dev->of_node, "apple,dapf-ranges", + &h->boot.ranges[0][0], 8); + if (ret) + goto free; + for (i = 0; i < 4; i++) { + u64 start = h->boot.ranges[i][0], end = h->boot.ranges[i][1]; + + ret = -EINVAL; + if (!start || end < start || round_down(start, SZ_16K) != round_down(end, SZ_16K)) + goto free; + for (j = 0; j < i; j++) + if (round_down(start, SZ_16K) == round_down(h->boot.ranges[j][0], SZ_16K)) + goto free; + } + if (!ane_h16_debugfs_users++) + ane_h16_debugfs = debugfs_create_dir("ane-h16", NULL); + h->debugfs = debugfs_create_dir(dev_name(h->dev), ane_h16_debugfs); + debugfs_create_file("stage", 0600, h->debugfs, h, &ane_h16_stage_fops); + debugfs_create_file("execute", 0600, h->debugfs, h, &ane_h16_execute_fops); + platform_set_drvdata(pdev, h); + INIT_WORK(&h->start, ane_h16_start_work); + schedule_work(&h->start); + ret = 0; + goto unlock; +free: + dev_err(h->dev, "H16 DT description refused, error %d\n", ret); + kfree(h); +unlock: + mutex_unlock(&ane_h16_lock); + return ret; +} + +void ane_h16_backend_remove(struct platform_device *pdev) +{ + struct ane_h16_device *h = platform_get_drvdata(pdev); + + if (!h) + return; + /* Waits for the boot bring-up and any stage write in progress. */ + cancel_work_sync(&h->start); + debugfs_remove_recursive(h->debugfs); + mutex_lock(&ane_h16_lock); + if (!--ane_h16_debugfs_users) { + debugfs_remove_recursive(ane_h16_debugfs); + ane_h16_debugfs = NULL; + } + if (h->boot.ran || h->pinned) { + /* The module reference taken before RUN makes this unreachable by rmmod. */ + dev_warn(h->dev, "H16 remove held; reboot reclaims firmware DMA\n"); + } else { + ane_h16_release(h); + kfree(h); + } + mutex_unlock(&ane_h16_lock); +} diff --git a/drivers/accel/ane/ane_h16_boot.h b/drivers/accel/ane/ane_h16_boot.h new file mode 100644 index 0000000000000..de24af2704f14 --- /dev/null +++ b/drivers/accel/ane/ane_h16_boot.h @@ -0,0 +1,661 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_BOOT_H__ +#define __ANE_H16_BOOT_H__ + +#include "ane_fw_transport.h" + +enum ane_h16_stage { + ANE_H16_REFUSE, + ANE_H16_PREFLIGHT, + ANE_H16_MAP, + ANE_H16_BOOT, + ANE_H16_STARTUP, + ANE_H16_PROGRAM, +}; + +enum ane_h16_window { + ANE_H16_ENGINE, + ANE_H16_PMGR, + ANE_H16_XL, + ANE_H16_DART0, + ANE_H16_DART1, + ANE_H16_DART2, + /* The one DAPF bank: the DART reg entry the ADT tags FPAD. */ + ANE_H16_DAPF, + ANE_H16_WINDOWS, +}; + +/* + * Every DART and DAPF register offset the stages read or write, for one DART + * revision: PARAMS3 bits 15:0 of the instance that owns the DAPF bank. The + * platform lists one entry per supported revision. PARAMS3 is read before an + * entry is chosen; with no entry nothing else in the DARTs or the bank is + * touched, and an entry names every other offset that is. + */ +struct ane_h16_mmio { + u16 dart_revision; + u32 dart_fault; + u32 dart_address_lo; + u32 dart_address_hi; + u32 dart_protect; + u32 dart_lock; + u32 dapf_records; + u32 dapf_stride; + /* Record end bits that read back; the bounds are inclusive. */ + u64 dapf_end_mask; + /* Record words other than the bounds, as the ADT dapf-instance gives them. */ + u8 dapf_r0; + u32 dapf_r4; + u16 dapf_r20; +}; + +struct ane_h16_tunable { + u32 offset; + u32 mask; + u32 value; +}; + +struct ane_h16_tunables { + enum ane_h16_window window; + u32 base; + const struct ane_h16_tunable *values; + size_t count; + bool c1ppt; + u32 lock_offset; + u32 lock_mask; +}; + +/* Selected only after the immutable resident firmware identity passes. */ +struct ane_h16_build { + u32 cpu_control; + u32 rvbar; + u32 ce; + u32 ce_delays; + u32 ce_delay_us; + u32 ready; + u32 run; + u32 scratch_polls; + u32 scratch_delay_us; + u64 command_timeout_ns; + u64 pool_dva; + u64 ipc_dva; + u64 heap_dva; + u64 shared_dva; + u64 expected_rvbar; + u32 pool_size; + u32 init_offset; + u32 init_size; + u32 config_span; + u32 config_window; + u32 template_count; + u32 template_word; + u32 template_value; + u32 max_channels; + u32 max_heap; + bool native_free; + bool c1ppt; + const struct ane_h16_tunables *tunables; + size_t tunable_count; + struct { + u16 id; + u16 size; + } startup[4]; +}; + +/* + * One supported firmware build: how to recognize its iBoot preload and the + * values to boot it with. Builds that differ only in their preload share one + * ane_h16_build. The platform lists every profile; the first whose preload + * rule matches the live preload is used. + */ +struct ane_h16_preload_rule; +struct ane_h16_profile { + const char *identity; + const struct ane_h16_preload_rule *preload; + const struct ane_h16_build *build; +}; + +struct ane_h16_boot; +struct ane_h16_boot_ops { + u32 (*read32)(void *ctx, enum ane_h16_window window, u32 offset); + u64 (*read64)(void *ctx, enum ane_h16_window window, u32 offset); + void (*write32)(void *ctx, enum ane_h16_window window, u32 offset, u32 value); + void (*write64)(void *ctx, enum ane_h16_window window, u32 offset, u64 value); + void (*delay_us)(void *ctx, unsigned int us); + int (*map)(void *ctx); + int (*alloc_at)(void *ctx, u64 dva, size_t size, struct ane_fw_surface *s); + int (*pin)(void *ctx); + void (*mailbox)(void *ctx); + void (*log)(void *ctx, const char *item, u64 a, u64 b); + void (*quarantine)(void *ctx, int error); +}; + +/* + * One caller. Stages advance in order, each at most once, and a failure ends + * the attempt. No mapping is released here, even on success. + */ +struct ane_h16_boot { + const struct ane_fw_platform *soc; + const struct ane_h16_build *build; + /* Chosen from the DART revision in preflight. */ + const struct ane_h16_mmio *mmio; + const struct ane_h16_boot_ops *ops; + const struct ane_fw_transport_ops *transport_ops; + void *ctx; + struct ane_fw_transport transport; + struct ane_fw_surface pool; + struct ane_fw_surface ipc; + struct ane_fw_surface heap; + u64 ranges[4][2]; + u32 requests[5]; + u32 tick; + u8 resource_info[100]; + int error; + enum ane_h16_stage stage; + bool attempted; + bool failed; + bool ran; + bool quarantined; +}; + +static inline int ane_h16_stage_parse(const char *name) +{ + if (!name || !*name) + return ANE_H16_REFUSE; + if (!strcmp(name, "preflight")) + return ANE_H16_PREFLIGHT; + if (!strcmp(name, "map")) + return ANE_H16_MAP; + if (!strcmp(name, "boot")) + return ANE_H16_BOOT; + if (!strcmp(name, "startup")) + return ANE_H16_STARTUP; + if (!strcmp(name, "program")) + return ANE_H16_PROGRAM; + return -EINVAL; +} + +static inline u32 ane_h16_read(struct ane_h16_boot *b, u32 cell) +{ + return b->ops->read32(b->ctx, ANE_H16_ENGINE, b->soc->scratch + cell * 4); +} + +static inline void ane_h16_write(struct ane_h16_boot *b, u32 cell, u32 value) +{ + b->ops->write32(b->ctx, ANE_H16_ENGINE, b->soc->scratch + cell * 4, value); +} + +static inline int ane_h16_faults(struct ane_h16_boot *b, bool log) +{ + const struct ane_h16_mmio *m = b->mmio; + unsigned int i; + int ret = 0; + + if (!m) + return 0; + for (i = 0; i < 3; i++) { + enum ane_h16_window w = ANE_H16_DART0 + i; + u32 status = b->ops->read32(b->ctx, w, m->dart_fault); + u64 address = b->ops->read32(b->ctx, w, m->dart_address_lo); + + address |= (u64)b->ops->read32(b->ctx, w, m->dart_address_hi) << 32; + if (log || (status & BIT(31))) { + b->ops->log(b->ctx, "DART status", i, status); + b->ops->log(b->ctx, "DART address", i, address); + } + if (status & BIT(31)) + ret = -EIO; + } + return ret; +} + +static inline int ane_h16_fail(struct ane_h16_boot *b, int error) +{ + unsigned int i; + + if (!b->error) + b->error = error; + if (b->ran && !b->quarantined) { + b->quarantined = true; + for (i = 0; i < 8; i++) + b->ops->log(b->ctx, "quarantine scratch", i, ane_h16_read(b, i)); + ane_h16_faults(b, true); + b->ops->quarantine(b->ctx, b->error); + } + return b->error; +} + +/* + * PARAMS3 of the three instances, logged; the DAPF owner's revision selects + * the register table. The other two are only reported. + */ +static inline int ane_h16_select_mmio(struct ane_h16_boot *b) +{ + const struct ane_fw_platform *soc = b->soc; + u32 params[3]; + unsigned int i; + + for (i = 0; i < 3; i++) { + params[i] = b->ops->read32(b->ctx, ANE_H16_DART0 + i, soc->dart_params); + b->ops->log(b->ctx, "DART PARAMS3", i, params[i]); + } + for (i = 0; i < soc->h16_mmio_count; i++) { + if (soc->h16_mmio[i].dart_revision == (params[soc->dapf_dart] & 0xffff)) { + b->mmio = &soc->h16_mmio[i]; + return 0; + } + } + b->ops->log(b->ctx, "DART revision has no register table", params[soc->dapf_dart] & 0xffff, + soc->h16_mmio_count); + return -ENODEV; +} + +static inline int ane_h16_preflight(struct ane_h16_boot *b) +{ + const struct ane_h16_build *f = b->build; + unsigned int pass, i; + u64 rvbar; + u32 control; + int ret; + + /* All suppliers are raised by genpd before entry. Never write PMGR. */ + for (pass = 0; pass < 2; pass++) { + for (i = 0; i < b->soc->power_count; i++) { + u32 word = b->ops->read32(b->ctx, ANE_H16_PMGR, + b->soc->power_words[i]); + + b->ops->log(b->ctx, "power ACTUAL", i, word); + if ((word & 0xff) != 0xff) + return -EHOSTDOWN; + } + } + control = b->ops->read32(b->ctx, ANE_H16_ENGINE, f->cpu_control); + rvbar = b->ops->read64(b->ctx, ANE_H16_ENGINE, f->rvbar); + b->ops->log(b->ctx, "CPU control/RVBAR", control, rvbar); + if (control || rvbar != f->expected_rvbar) + return -EBUSY; + ret = ane_h16_select_mmio(b); + if (ret) + return ret; + ret = ane_h16_faults(b, true); + if (ret) + return ret; + return 0; +} + +/* + * One record of the DAPF bank, read only. Record words as the ADT + * dapf-instance: r0 (+0x00, only its low byte reads back), r4, the inclusive + * start and end, r20 (+0x20, its low half is the enable). + */ +static inline bool ane_h16_dapf_record(struct ane_h16_boot *b, unsigned int record) +{ + const struct ane_h16_mmio *m = b->mmio; + u32 off = m->dapf_records + record * m->dapf_stride; + u32 flags = b->ops->read32(b->ctx, ANE_H16_DAPF, off); + u32 r4 = b->ops->read32(b->ctx, ANE_H16_DAPF, off + 4); + u64 start = b->ops->read64(b->ctx, ANE_H16_DAPF, off + 8); + u64 end = b->ops->read64(b->ctx, ANE_H16_DAPF, off + 0x10); + u32 enabled = b->ops->read32(b->ctx, ANE_H16_DAPF, off + 0x20); + + b->ops->log(b->ctx, "DAPF record/r4", record, r4); + b->ops->log(b->ctx, "DAPF flags/enable", flags, enabled); + b->ops->log(b->ctx, "DAPF start/end", start, end); + return (flags & 0xff) == m->dapf_r0 && r4 == m->dapf_r4 && + (enabled & 0xffff) == m->dapf_r20 && start == b->ranges[record][0] && + (end & m->dapf_end_mask) == (b->ranges[record][1] & m->dapf_end_mask); +} + +/* + * Compare the four records with apple,dapf-ranges. Preflight only reports. + * The bank loses its records whenever its power domain does, so map writes + * every record that differs, in the order the t8110 bank takes them, and + * reads it back. A bank whose lock word has bit 2 set and that differs is + * refused before any write. + */ +static inline int ane_h16_dapf(struct ane_h16_boot *b, bool program) +{ + const struct ane_h16_mmio *m = b->mmio; + enum ane_h16_window owner = ANE_H16_DART0 + b->soc->dapf_dart; + u32 protect = b->ops->read32(b->ctx, owner, m->dart_protect); + u32 lock = b->ops->read32(b->ctx, owner, m->dart_lock); + unsigned int record, matched = 0; + bool match[4]; + + b->ops->log(b->ctx, "DAPF protect/lock", protect, lock); + for (record = 0; record < 4; record++) { + match[record] = ane_h16_dapf_record(b, record); + matched += match[record]; + } + b->ops->log(b->ctx, "DAPF records matching", matched, 4); + if (!program || matched == 4) + return 0; + if (lock & BIT(2)) + return -EPERM; + for (record = 0; record < 4; record++) { + u32 off = m->dapf_records + record * m->dapf_stride; + + if (match[record]) + continue; + b->ops->write32(b->ctx, ANE_H16_DAPF, off + 4, m->dapf_r4); + b->ops->write64(b->ctx, ANE_H16_DAPF, off + 8, b->ranges[record][0]); + b->ops->write64(b->ctx, ANE_H16_DAPF, off + 0x10, b->ranges[record][1]); + b->ops->write32(b->ctx, ANE_H16_DAPF, off, m->dapf_r0); + b->ops->write32(b->ctx, ANE_H16_DAPF, off + 0x20, m->dapf_r20); + if (!ane_h16_dapf_record(b, record)) + return -EIO; + } + b->ops->log(b->ctx, "DAPF records written/read back", 4 - matched, 4); + return 0; +} + +/* r5 section 1.1. H16 has CE, with no H17 PWGATE pair. */ +static inline int ane_h16_ce(struct ane_h16_boot *b) +{ + const struct ane_h16_build *f = b->build; + u64 state = b->ops->read64(b->ctx, ANE_H16_ENGINE, f->ce + 0x10); + u64 error = b->ops->read64(b->ctx, ANE_H16_ENGINE, f->ce + 0x18); + u64 wanted = (state & 3) ^ 3; + unsigned int i; + + b->ops->log(b->ctx, "CE state/error", state, error); + if (error & wanted) + return -EIO; + if ((state & 3) == 3) + return 0; + b->ops->write64(b->ctx, ANE_H16_ENGINE, f->ce, wanted); + for (i = 0; i <= f->ce_delays; i++) { + if ((b->ops->read64(b->ctx, ANE_H16_ENGINE, f->ce + 8) & 3) == wanted) + break; + if (i == f->ce_delays) + return -ETIMEDOUT; + b->ops->delay_us(b->ctx, f->ce_delay_us); + } + error = b->ops->read64(b->ctx, ANE_H16_ENGINE, f->ce + 0x18); + if (error & wanted) + return -EIO; + b->ops->write64(b->ctx, ANE_H16_ENGINE, f->ce + 8, wanted); + b->ops->log(b->ctx, "CE ack bits/delays", wanted, i); + return 0; +} + +static inline int ane_h16_prepare(struct ane_h16_boot *b) +{ + const struct ane_h16_build *f = b->build; + unsigned int i, j; + int ret = ane_h16_ce(b); + + if (ret) + return ret; + for (i = 0; i < f->tunable_count; i++) { + const struct ane_h16_tunables *group = &f->tunables[i]; + + if (group->c1ppt && !f->c1ppt) + continue; + if (group->lock_mask && + (b->ops->read32(b->ctx, group->window, group->base + + group->lock_offset) & group->lock_mask)) { + for (j = 0; j < group->count; j++) { + const struct ane_h16_tunable *v = &group->values[j]; + u32 value = b->ops->read32(b->ctx, group->window, + group->base + v->offset); + + if ((value & v->mask) != (v->value & v->mask)) { + b->ops->log(b->ctx, "locked tunable offset/value", + group->base + v->offset, value); + return -EPERM; + } + } + b->ops->log(b->ctx, "tunables locked match", group->window, group->count); + continue; + } + for (j = 0; j < group->count; j++) { + const struct ane_h16_tunable *v = &group->values[j]; + u32 off = group->base + v->offset; + u32 old = b->ops->read32(b->ctx, group->window, off); + + b->ops->write32(b->ctx, group->window, off, + (old & ~v->mask) | (v->value & v->mask)); + } + b->ops->log(b->ctx, "tunables applied", group->window, group->count); + } + return 0; +} + +static inline int ane_h16_wait(struct ane_h16_boot *b, u32 cell, u32 value, + bool table) +{ + const struct ane_h16_build *f = b->build; + unsigned int i; + int ret; + u64 deadline = b->transport_ops->now_ns(b->ctx) + + (u64)f->scratch_polls * f->scratch_delay_us * 1000; + + for (i = 0; i < f->scratch_polls; i++) { + if (b->transport_ops->now_ns(b->ctx) >= deadline) + break; + b->ops->mailbox(b->ctx); + ret = ane_h16_faults(b, false); + if (ret) + return ret; + if (ane_h16_read(b, cell) == value) { + u64 address; + + if (!table) + return 0; + address = ane_h16_read(b, 0); + address |= (u64)ane_h16_read(b, 1) << 32; + if (address == f->ipc_dva) + return 0; + /* READY can pre-exist publication. Never accept the init DVA. */ + if (address != f->pool_dva + f->init_offset) + return -EPROTO; + } + b->ops->delay_us(b->ctx, f->scratch_delay_us); + } + return -ETIMEDOUT; +} + +static inline int ane_h16_alloc(struct ane_h16_boot *b, u64 dva, size_t size, + struct ane_fw_surface *s) +{ + int ret = b->ops->alloc_at(b->ctx, dva, size, s); + + if (ret) + return ret; + if (!ane_fw_surface_valid(s) || s->dva != dva || s->size != size) + return -EPROTO; + return 0; +} + +static inline int ane_h16_scratch_boot(struct ane_h16_boot *b) +{ + const struct ane_h16_build *f = b->build; + u64 ipc_bytes, heap_bytes; + u8 *p; + unsigned int i; + int ret; + + ret = ane_h16_alloc(b, f->pool_dva, f->pool_size, &b->pool); + if (ret) + return ret; + ret = ane_h16_prepare(b); + if (ret) + return ret; + for (i = 0; i < 8; i++) + ane_h16_write(b, i, 0); + /* Endpoint-1 management client is installed, never awaited. */ + b->ops->mailbox(b->ctx); + ane_h16_write(b, 6, 1); + ane_h16_write(b, 7, 0); + if (b->ops->read64(b->ctx, ANE_H16_ENGINE, f->rvbar) != f->expected_rvbar) + return -EBUSY; + ret = b->ops->pin(b->ctx); + if (ret) + return ret; + b->ops->write32(b->ctx, ANE_H16_ENGINE, f->cpu_control, 0); + /* Latch before the first RUN store, including a failure at that store. */ + b->ran = true; + b->ops->write32(b->ctx, ANE_H16_ENGINE, f->cpu_control, f->run); + ret = ane_h16_wait(b, 7, f->ready, false); + if (ret) + return ret; + b->ops->log(b->ctx, "READY", f->ready, 0); + for (i = 0; i < ARRAY_SIZE(b->requests); i++) + b->requests[i] = ane_h16_read(b, i); + b->ops->log(b->ctx, "requests channels/IPC-minus-one", b->requests[0], b->requests[1]); + b->ops->log(b->ctx, "requests heap/mode", b->requests[3], b->requests[4]); + ipc_bytes = (u64)b->requests[1] + 1; + heap_bytes = b->requests[3]; + /* Host policy. IPC cannot reach the fixed heap; no guessed firmware ceiling. */ + if (!b->requests[0] || b->requests[0] >= f->max_channels || + ipc_bytes < ANE_FW_TABLE_SIZE || ipc_bytes > f->heap_dva - f->ipc_dva || + heap_bytes > f->max_heap || + ALIGN(heap_bytes, SZ_16K) > f->shared_dva - f->heap_dva || + b->requests[4] > 1) + return -E2BIG; + ret = ane_h16_alloc(b, f->ipc_dva, ALIGN(ipc_bytes, SZ_16K), &b->ipc); + if (ret) + return ret; + if (heap_bytes) { + ret = ane_h16_alloc(b, f->heap_dva, ALIGN(heap_bytes, SZ_16K), &b->heap); + if (ret) + return ret; + } + p = (u8 *)b->pool.cpu + f->init_offset; + memset(p, 0, f->init_size); + put_unaligned_le64(f->ipc_dva, p + 8); + put_unaligned_le64(f->config_span, p + 0x10); + put_unaligned_le64(f->config_window - f->config_span, p + 0x18); + put_unaligned_le64(heap_bytes ? f->heap_dva : 0, p + 0x20); + put_unaligned_le64(heap_bytes, p + 0x28); + put_unaligned_le64(ipc_bytes, p + 0x50); + put_unaligned_le64(f->pool_dva, p + 0x58); + put_unaligned_le64(f->pool_size, p + 0x60); + put_unaligned_le32(f->template_count, p + 0x68); + put_unaligned_le32(f->template_value, p + f->template_word); + b->transport_ops->write_barrier(b->ctx); + ane_h16_write(b, 0, lower_32_bits(f->pool_dva + f->init_offset)); + ane_h16_write(b, 1, upper_32_bits(f->pool_dva + f->init_offset)); + ane_h16_write(b, 7, ~f->ready); + ret = ane_h16_wait(b, 7, f->ready, true); + if (ret) + return ret; + ret = ane_fw_transport_init(&b->transport, b->soc, b->transport_ops, b->ctx, + b->ipc, b->pool, f->init_offset + f->init_size, + f->ipc_dva, ANE_FW_TABLE_SIZE); + if (ret) + return ret; + b->transport.native_free = f->native_free; + b->ops->log(b->ctx, "table validated", f->ipc_dva, ANE_FW_CHANNELS); + ret = ane_fw_transport_arm(&b->transport); + if (ret) + return ret; + ane_h16_write(b, 3, f->ready); + ret = ane_h16_wait(b, 3, 0, false); + if (!ret) + b->ops->log(b->ctx, "scratch 3 cleared", 0, 0); + return ret; +} + +static inline int ane_h16_startup(struct ane_h16_boot *b) +{ + const struct ane_h16_build *f = b->build; + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(f->startup); i++) { + u8 command[100] = {}; + u8 *reply; + u64 deadline = b->transport_ops->now_ns(b->ctx) + f->command_timeout_ns; + + ane_fw_command_header(command, f->startup[i].id, true); + ret = ane_fw_transport_publish(&b->transport, ANE_FW_IO, command, + f->startup[i].size, deadline); + if (ret) + return ret; + ret = ane_fw_transport_wait(&b->transport, ANE_FW_IO); + reply = (u8 *)b->pool.cpu + + (b->transport.pending[ANE_FW_IO].dva - b->pool.dva); + if (ret) { + if (ret == -EREMOTEIO) + b->ops->log(b->ctx, "startup refusal ID/ack", + f->startup[i].id, get_unaligned_le16(reply + 6)); + return ret; + } + b->ops->log(b->ctx, "startup ack", f->startup[i].id, get_unaligned_le16(reply + 6)); + if (f->startup[i].id == 3) { + b->tick = get_unaligned_le32(reply + 8); + b->ops->log(b->ctx, "CONFIG_GET tick", b->tick, 0); + if (!b->tick) + return -EPROTO; + } + if (f->startup[i].id == 0x22) + memcpy(b->resource_info, reply, sizeof(b->resource_info)); + ret = ane_h16_faults(b, false); + if (ret) + return ret; + } + return 0; +} + +/* + * Advance from the last completed stage to @stage. The program stage belongs + * to the caller; here it runs startup. A stage at or below the completed one, + * or any call after a failure, is refused. + */ +static inline int ane_h16_run(struct ane_h16_boot *b, enum ane_h16_stage stage) +{ + int ret = 0; + + if (stage == ANE_H16_REFUSE) + return -EOPNOTSUPP; + if (stage < ANE_H16_PREFLIGHT || stage > ANE_H16_PROGRAM) + return -EINVAL; + if (stage > ANE_H16_STARTUP) + stage = ANE_H16_STARTUP; + if (b->failed || b->quarantined || stage <= b->stage) + return -EBUSY; + b->attempted = true; + if (b->stage < ANE_H16_PREFLIGHT) { + ret = ane_h16_preflight(b); + if (!ret) + ret = ane_h16_dapf(b, false); + if (!ret) + b->stage = ANE_H16_PREFLIGHT; + } + if (!ret && stage >= ANE_H16_MAP && b->stage < ANE_H16_MAP) { + ret = b->ops->map(b->ctx); + if (!ret) + ret = ane_h16_dapf(b, true); + if (!ret) + ret = ane_h16_faults(b, true); + if (!ret) + b->stage = ANE_H16_MAP; + } + if (!ret && stage >= ANE_H16_BOOT && b->stage < ANE_H16_BOOT) { + ret = ane_h16_scratch_boot(b); + if (!ret) + b->stage = ANE_H16_BOOT; + } + if (!ret && stage >= ANE_H16_STARTUP && b->stage < ANE_H16_STARTUP) { + ret = ane_h16_startup(b); + if (!ret) + b->stage = ANE_H16_STARTUP; + } + if (!ret) + return 0; + b->failed = true; + return ane_h16_fail(b, ret); +} + +static inline const char *ane_h16_stage_name(enum ane_h16_stage stage) +{ + static const char * const names[] = { + "none", "preflight", "map", "boot", "startup", "program", + }; + + return stage <= ANE_H16_PROGRAM ? names[stage] : "invalid"; +} + +#endif /* __ANE_H16_BOOT_H__ */ diff --git a/drivers/accel/ane/ane_h16_boot_data.h b/drivers/accel/ane/ane_h16_boot_data.h new file mode 100644 index 0000000000000..4875abe800bb2 --- /dev/null +++ b/drivers/accel/ane/ane_h16_boot_data.h @@ -0,0 +1,332 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_BOOT_DATA_H__ +#define __ANE_H16_BOOT_DATA_H__ + +#include "ane_h16_boot.h" +#include "ane_h16_preload.h" + +/* H16 27.0 (26A428), r5 sections 1.2-1.4. Ordered masked updates. */ +static const struct ane_h16_tunable ane_h16_axi[] = { + { 0x0000, 0x800000ff, 0x80000022 }, + { 0x0004, 0x80ff00ff, 0x80220003 }, + { 0x0008, 0x80ff00ff, 0x80220009 }, + { 0x000c, 0x80ff00ff, 0x80220003 }, + { 0x0010, 0x80ff00ff, 0x80220003 }, + { 0x0014, 0x80ff00ff, 0x80220003 }, + { 0x0018, 0x800000ff, 0x8000001b }, + { 0x001c, 0x800000ff, 0x80000022 }, + { 0x0020, 0x80ff00ff, 0x80220003 }, + { 0x0024, 0x80ff00ff, 0x80220003 }, + { 0x0028, 0x80ff00ff, 0x80220003 }, + { 0x002c, 0x80ff00ff, 0x80220003 }, + { 0x0030, 0x800000ff, 0x8000001b }, + { 0x0034, 0x800000ff, 0x8000001b }, + { 0x0038, 0x800000ff, 0x8000001b }, + { 0x0100, 0x000000ff, 0x000000ff }, + { 0x0108, 0xffff0006, 0x00100000 }, + { 0x0304, 0x3f3f0000, 0x02020000 }, + { 0x0308, 0x000fffff, 0x00001681 }, + { 0x030c, 0x000fffff, 0x00001680 }, + { 0x0310, 0x70000fff, 0x000001e0 }, + { 0x0314, 0xf0000fff, 0x800001e0 }, + { 0x0318, 0x70000fff, 0x000005a0 }, + { 0x031c, 0xf0000fff, 0x800005a0 }, + { 0x0320, 0x70000fff, 0x000005a0 }, + { 0x0324, 0xf0000fff, 0x800005a0 }, + { 0x0328, 0x70000fff, 0x000005a0 }, + { 0x032c, 0x70000fff, 0x000005a0 }, + { 0x0330, 0x70000fff, 0x000005a0 }, + { 0x0338, 0xffffffff, 0x00050020 }, + { 0x033c, 0xffffffff, 0x000a0030 }, + { 0x0408, 0xffff0020, 0x001e0020 }, + { 0x040c, 0x000fffff, 0x00001681 }, + { 0x0410, 0x000fffff, 0x00001680 }, + { 0x0414, 0x70000fff, 0x000001e0 }, + { 0x0418, 0x70000fff, 0x000001e0 }, + { 0x041c, 0x70000fff, 0x000005a0 }, + { 0x0420, 0x70000fff, 0x000005a0 }, + { 0x0424, 0x70000fff, 0x000005a0 }, + { 0x0428, 0x70000fff, 0x000005a0 }, + { 0x042c, 0x70000fff, 0x000005a0 }, + { 0x0430, 0x70000fff, 0x000005a0 }, + { 0x0434, 0x70000fff, 0x000005a0 }, + { 0x0748, 0x07ff07ff, 0x00200030 }, + { 0x0848, 0x07ff07ff, 0x00100030 }, + { 0x0948, 0x07ff07ff, 0x01880188 }, + { 0x0b00, 0xffffffff, 0x7fffffff }, + { 0x4008, 0x00000001, 0 }, + { 0x400c, 0x00000001, 1 }, + { 0x4014, 0x00000001, 0 }, + { 0x4018, 0x00000001, 0 }, + { 0x401c, 0x00000001, 1 }, +}; + +static const struct ane_h16_tunable ane_h16_xl[] = { + { 0x0000, 0x800000ff, 0x80000036 }, + { 0x0004, 0x80ff00ff, 0x80360003 }, + { 0x0008, 0x80ff00ff, 0x80360009 }, + { 0x000c, 0x80ff00ff, 0x80010000 }, + { 0x0010, 0x80ff00ff, 0x80360003 }, + { 0x0014, 0x800000ff, 0x8000002e }, + { 0x0018, 0x800000ff, 0x80000036 }, + { 0x001c, 0x80ff00ff, 0x80360003 }, + { 0x0020, 0x80ff00ff, 0x80020000 }, + { 0x0024, 0x80ff00ff, 0x80360003 }, + { 0x0028, 0x800000ff, 0x8000002e }, + { 0x002c, 0x800000ff, 0x8000002e }, + { 0x0030, 0x800000ff, 0x8000002e }, + { 0x0100, 0x000000ff, 0x000000ff }, + { 0x0108, 0xffff0006, 0x00100000 }, + { 0x0304, 0x3f3f0000, 0x02020000 }, + { 0x0308, 0x000fffff, 0x00001681 }, + { 0x030c, 0x000fffff, 0x00001680 }, + { 0x0310, 0x70000fff, 0x000005a0 }, + { 0x0314, 0xf0000fff, 0x800005a0 }, + { 0x0318, 0x70000fff, 0x000005a0 }, + { 0x031c, 0x70000fff, 0x000005a0 }, + { 0x0320, 0x70000fff, 0x000005a0 }, + { 0x0328, 0xffffffff, 0x0000ffff }, + { 0x032c, 0xffffffff, 0x0000ffff }, + { 0x0408, 0xffff0020, 0x00310020 }, + { 0x040c, 0x000fffff, 0x00001681 }, + { 0x0410, 0x000fffff, 0x00001680 }, + { 0x0414, 0x70000fff, 0x000005a0 }, + { 0x0418, 0x70000fff, 0x000005a0 }, + { 0x041c, 0x70000fff, 0x000005a0 }, + { 0x0420, 0x70000fff, 0x000005a0 }, + { 0x0424, 0x70000fff, 0x000005a0 }, + { 0x0648, 0x07ff07ff, 0x01880188 }, + { 0x0800, 0xffffffff, 0x7fffffff }, + { 0x4008, 0x00000001, 0 }, + { 0x400c, 0x00000001, 1 }, + { 0x4014, 0x00000001, 0 }, + { 0x4018, 0x00000001, 0 }, + { 0x401c, 0x00000001, 1 }, +}; + +static const struct ane_h16_tunable ane_h16_c1ppt[] = { + { 0x20, 0x0000ffff, 0x1838 }, + { 0x24, 0x0000ffff, 0x1034 }, + { 0x28, 0xffffffff, 0x07d00708 }, + { 0x2c, 0x000000ff, 4 }, + { 0x40, 0x000003ff, 0xbf }, + { 0x44, 0x000003ff, 0x19a }, + { 0x48, 0x0001ffff, 0x7800 }, + { 0x60, 0x00001f1f, 0x0103 }, + { 0x04, 0x1, 1 }, + { 0x3ff0, 0x007fffff, 0x0077ebf9 }, + { 0x3ffc, 0x1, 1 }, +}; + +static const struct ane_h16_tunables ane_h16_tunables_26a428[] = { + { .window = ANE_H16_ENGINE, .values = ane_h16_axi, + .count = ARRAY_SIZE(ane_h16_axi) }, + { .window = ANE_H16_XL, .values = ane_h16_xl, + .count = ARRAY_SIZE(ane_h16_xl) }, + { .window = ANE_H16_ENGINE, .base = 0x197c000, + .values = ane_h16_c1ppt, .count = ARRAY_SIZE(ane_h16_c1ppt), + .c1ppt = true, .lock_offset = 0x3ffc, .lock_mask = 1 }, +}; + +/* + * T6041 (J616s) ANE DARTs, revision 2.2 (PARAMS3 0x2a2a0202). The ADT + * dart-ane0 node has one DAPF bank, the reg entry its instance property tags + * FPAD "DAPFLLT" at DART0 + 0x10000; BRD and BWR have none, and their + * +0x10000 is outside every ADT reg entry. iBoot leaves the bank unprogrammed + * and it sits in the ANE_SYS domain, so the kernel writes it after power-up. + * Every dapf-instance-0 record has r0 0x31, r4 0 and r20 1. A t8110 DAPF + * record end does not read back its low two bits on the T8132; every J616s + * end has them set. + */ +static const struct ane_h16_mmio ane_h16_mmio_t6041[] __maybe_unused = { + { + .dart_revision = 0x0202, + .dart_fault = 0x100, + .dart_address_lo = 0x170, + .dart_address_hi = 0x174, + .dart_protect = 0x200, + .dart_lock = 0x208, + .dapf_records = 0, + .dapf_stride = 0x40, + .dapf_end_mask = ~3ULL, + .dapf_r0 = 0x31, + .dapf_r4 = 0, + .dapf_r20 = 1, + }, +}; + +static const struct ane_h16_build ane_h16_build_26a428 = { + .cpu_control = 0x1600044, + .rvbar = 0x1050000, + .ce = 0x18a4000, + .ce_delays = 5001, + .ce_delay_us = 10, + .ready = 0x08042006, + .run = 0x10, + .scratch_polls = 1000, + .scratch_delay_us = 1000, + .command_timeout_ns = 3000000000ULL, + .pool_dva = 0x10000384000, + .ipc_dva = 0x100003d0000, + .heap_dva = 0x100003e4000, + .shared_dva = 0x10001300000, + .expected_rvbar = 0x10000000001, + .pool_size = 0x40000, + .init_offset = 0x180, + .init_size = 0x174, + .config_span = 0x380000, + .config_window = 0x10000000, + .template_count = 64, + .template_word = 0x12c, + .template_value = 4, + .max_channels = 0x21, + /* Host allocation policy, not a recovered firmware ceiling. */ + .max_heap = SZ_16M, + .native_free = true, + .c1ppt = true, + .tunables = ane_h16_tunables_26a428, + .tunable_count = ARRAY_SIZE(ane_h16_tunables_26a428), + .startup = { { 4, 12 }, { 0, 12 }, { 3, 16 }, { 0x22, 100 } }, +}; + +/* + * 27.0 (26A428) preload: ordered aether patchbay records in DATA (whose + * firmware VM base is 0xc0000) and the range the bootloader rewrites. + * Offsets are DATA-relative. Only the mutable value bytes normalize. + */ +static const struct ane_h16_preload_cell ane_h16_patchbay_26a428[] = { + { "BECA", 0x5a20, 8, true }, + { "SECA", 0x5a30, 8, false }, + { "GKTS", 0x5a40, 8, true }, + { "otcd", 0x5a50, 8, false }, + { "oscd", 0x5a60, 8, false }, + { "oecd", 0x5a70, 8, false }, + { "DILS", 0x5a80, 8, true }, + { "SRSA", 0x5a90, 8, false }, + { "GFCM", 0x5aa0, 8, false }, + { "SSSC", 0x5ab0, 8, true }, + { "qF8v", 0x5ac0, 4, true }, + { "LRSD", 0x5acc, 1, false }, + { "SVSD", 0x5ad5, 8, false }, + { "LCSD", 0x5ae5, 8, false }, + { "ZSTR", 0x5af5, 4, false }, + { "BPTP", 0x5b01, 8, false }, + { "TNGI", 0x5b11, 1, false }, + { "oeNS", 0x5b1a, 1, false }, + { "SEBC", 0x5b23, 8, false }, + { "ECAP", 0x5b33, 8, true }, + { "BVTP", 0x5b43, 8, true }, + { "sP1T", 0x5b53, 8, true }, + { "BtpG", 0x5b63, 8, true }, + { "StpG", 0x5b73, 8, false }, + { "BlpP", 0x5b83, 8, true }, + { "SPTP", 0x5b93, 8, false }, + { "SZSD", 0x5ba3, 8, false }, + { "LZSD", 0x5bb3, 8, false }, + { "S2xG", 0x5bc3, 8, true }, + { "SNUT", 0x5bd3, 8, true }, + { "ZNUT", 0x5be3, 4, false }, + { "OTTR", 0x5bef, 8, true }, + { "LLTR", 0x5bff, 1, false }, + { "_COS", 0x5c08, 4, true }, + { "RCOS", 0x5c14, 4, true }, + { "dApC", 0x5c20, 8, true }, + { "dArW", 0x5c30, 8, true }, + { "fVED", 0x5c40, 1, true }, + { "GLFp", 0x5c49, 8, true }, + { "ABOI", 0x5c59, 8, false }, + { "ZSOI", 0x5c69, 4, false }, + { "ARcM", 0x5c75, 4, true }, +}; + +static const struct ane_h16_preload_rule ane_h16_preload_26a428 = { + .cells = ane_h16_patchbay_26a428, + .cell_count = ARRAY_SIZE(ane_h16_patchbay_26a428), + .zero_start = 0x17c80, + .zero_end = 0x18320, + .digest = { + 0x1f, 0x86, 0x5e, 0xd0, 0x4c, 0x4c, 0x38, 0xf6, + 0xfd, 0xcc, 0x45, 0x43, 0xa8, 0x81, 0x4f, 0x81, + 0x8d, 0x3c, 0xea, 0x58, 0x3d, 0xe6, 0x4d, 0x22, + 0xff, 0xe9, 0x6e, 0x3a, 0x93, 0xca, 0x46, 0x97, + }, +}; + +/* + * 26.6.2 (25G83) preload, the build the J616s Linux volume boots with: the + * same patchbay records and rewritten range as 26A428, 0x3f80 bytes later + * in DATA. Its boot values are the 26A428 ones (25G83 facts, section 2). + */ +static const struct ane_h16_preload_cell ane_h16_patchbay_25g83[] = { + { "BECA", 0x99a0, 8, true }, + { "SECA", 0x99b0, 8, false }, + { "GKTS", 0x99c0, 8, true }, + { "otcd", 0x99d0, 8, false }, + { "oscd", 0x99e0, 8, false }, + { "oecd", 0x99f0, 8, false }, + { "DILS", 0x9a00, 8, true }, + { "SRSA", 0x9a10, 8, false }, + { "GFCM", 0x9a20, 8, false }, + { "SSSC", 0x9a30, 8, true }, + { "qF8v", 0x9a40, 4, true }, + { "LRSD", 0x9a4c, 1, false }, + { "SVSD", 0x9a55, 8, false }, + { "LCSD", 0x9a65, 8, false }, + { "ZSTR", 0x9a75, 4, false }, + { "BPTP", 0x9a81, 8, false }, + { "TNGI", 0x9a91, 1, false }, + { "oeNS", 0x9a9a, 1, false }, + { "SEBC", 0x9aa3, 8, false }, + { "ECAP", 0x9ab3, 8, true }, + { "BVTP", 0x9ac3, 8, true }, + { "sP1T", 0x9ad3, 8, true }, + { "BtpG", 0x9ae3, 8, true }, + { "StpG", 0x9af3, 8, false }, + { "BlpP", 0x9b03, 8, true }, + { "SPTP", 0x9b13, 8, false }, + { "SZSD", 0x9b23, 8, false }, + { "LZSD", 0x9b33, 8, false }, + { "S2xG", 0x9b43, 8, true }, + { "SNUT", 0x9b53, 8, true }, + { "ZNUT", 0x9b63, 4, false }, + { "OTTR", 0x9b6f, 8, true }, + { "LLTR", 0x9b7f, 1, false }, + { "_COS", 0x9b88, 4, true }, + { "RCOS", 0x9b94, 4, true }, + { "dApC", 0x9ba0, 8, true }, + { "dArW", 0x9bb0, 8, true }, + { "fVED", 0x9bc0, 1, true }, + { "GLFp", 0x9bc9, 8, true }, + { "ABOI", 0x9bd9, 8, false }, + { "ZSOI", 0x9be9, 4, false }, + { "ARcM", 0x9bf5, 4, true }, +}; + +static const struct ane_h16_preload_rule ane_h16_preload_25g83 = { + .cells = ane_h16_patchbay_25g83, + .cell_count = ARRAY_SIZE(ane_h16_patchbay_25g83), + .zero_start = 0x1bc00, + .zero_end = 0x1c2a0, + .digest = { + 0x83, 0x4b, 0xf6, 0xa8, 0xf1, 0xa7, 0x5c, 0x02, + 0xf8, 0x95, 0x2d, 0xd7, 0xd4, 0x1b, 0xce, 0xe6, + 0x6d, 0x46, 0x80, 0x7e, 0xec, 0x7e, 0xa6, 0xa6, + 0x2e, 0x44, 0x8e, 0xfe, 0x18, 0x1d, 0x8b, 0xe8, + }, +}; + +/* Every supported H16 firmware build, matched against the live preload. */ +static const struct ane_h16_profile ane_h16_profiles[] __maybe_unused = { + { + .identity = "H16 27.0 26A428", + .preload = &ane_h16_preload_26a428, + .build = &ane_h16_build_26a428, + }, + { + .identity = "H16 26.6.2 25G83", + .preload = &ane_h16_preload_25g83, + .build = &ane_h16_build_26a428, + }, +}; + +#endif /* __ANE_H16_BOOT_DATA_H__ */ diff --git a/drivers/accel/ane/ane_h16_boot_test.c b/drivers/accel/ane/ane_h16_boot_test.c new file mode 100644 index 0000000000000..4e3965bdbc06b --- /dev/null +++ b/drivers/accel/ane/ane_h16_boot_test.c @@ -0,0 +1,919 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "ane_h16_boot_data.h" + +struct ane_h16_sim { + struct kunit *test; + struct ane_h16_boot boot; + struct ane_h16_build build; + struct ane_h16_mmio mmio; + struct ane_fw_platform soc; + u32 scratch[8]; + u32 dapf[SZ_16K / 4]; + u32 dart_params; + u32 dapf_reads; + u32 dapf_order[8]; + int dapf_stuck; + u64 now; + u64 ce_ack; + u32 cpu; + u32 command_count; + u32 scratch_writes; + u32 tuning_writes; + u32 dapf_writes; + u32 mailbox_polls; + u32 maps; + u32 pins; + u32 quarantines; + u32 allocations; + u32 lock; + u32 fault; + u32 channel_request; + u32 ipc_request; + u32 heap_request; + u32 tick; + struct ane_fw_rtk rtk; + u32 mailbox_tx; + bool offered_mailbox; + bool fault_after_run; + u32 lifecycle_mode; + bool lifecycle_requested; + u16 refuse_id; + bool no_ready; + bool wrong_table; + bool stale_table; + bool no_clear; + bool shared_pending; + bool shared_replied; + bool ce_timeout; + bool bad_power; + bool bad_rvbar; + bool fail_map; +}; + +static const u32 sim_power_words[] = { 0x3c0, 0xc008, 0xc000, 0xc010, 0xc018 }; + +/* Only the PARAMS3 word and the offsets of the selected table are accessed. */ +static void sim_check_dart(struct ane_h16_sim *s, enum ane_h16_window w, u32 off) +{ + const struct ane_h16_mmio *m = s->boot.mmio; + + if (off == s->soc.dart_params) + return; + KUNIT_ASSERT_NOT_NULL(s->test, m); + if (off == m->dart_fault || off == m->dart_address_lo || off == m->dart_address_hi) + return; + KUNIT_EXPECT_EQ(s->test, w, ANE_H16_DART0 + s->soc.dapf_dart); + KUNIT_EXPECT_TRUE(s->test, off == m->dart_protect || off == m->dart_lock); +} + +static void sim_check_dapf(struct ane_h16_sim *s, u32 off, u32 width) +{ + const struct ane_h16_mmio *m = s->boot.mmio; + + KUNIT_ASSERT_NOT_NULL(s->test, m); + KUNIT_EXPECT_GE(s->test, off, m->dapf_records); + KUNIT_EXPECT_LE(s->test, off + width, m->dapf_records + 4 * m->dapf_stride); + KUNIT_EXPECT_LE(s->test, off + width, s->soc.dapf_size); +} + +static u32 sim_read32(void *ctx, enum ane_h16_window w, u32 off) +{ + struct ane_h16_sim *s = ctx; + + if (w == ANE_H16_PMGR) + return s->bad_power ? 0xf : 0xff; + if (w >= ANE_H16_DART0 && w <= ANE_H16_DART2) { + sim_check_dart(s, w, off); + if (off == s->soc.dart_params) + return s->dart_params; + if (off == s->mmio.dart_protect || off == s->mmio.dart_lock) + return s->lock; + if (off == s->mmio.dart_fault) + return s->fault; + return 0; + } + if (w == ANE_H16_DAPF) { + sim_check_dapf(s, off, 4); + s->dapf_reads++; + return s->dapf[off / 4]; + } + if (w == ANE_H16_ENGINE) { + if (off == s->build.cpu_control) + return s->cpu; + if (off == 0x197fffc) + return 0; + if (off >= s->soc.scratch && off < s->soc.scratch + 32) + return s->scratch[(off - s->soc.scratch) / 4]; + if (off == s->soc.irq_status) + return 0; + } + return 0xa5a5a5a5; +} + +static u64 sim_read64(void *ctx, enum ane_h16_window w, u32 off) +{ + struct ane_h16_sim *s = ctx; + + if (w == ANE_H16_DAPF) { + sim_check_dapf(s, off, 8); + s->dapf_reads++; + return s->dapf[off / 4] | (u64)s->dapf[off / 4 + 1] << 32; + } + KUNIT_EXPECT_EQ(s->test, w, ANE_H16_ENGINE); + if (off == s->build.rvbar) + return s->bad_rvbar ? 0 : s->build.expected_rvbar; + if (off == s->build.ce + 8) + return s->ce_ack; + return 0; +} + +static void sim_reply(struct ane_h16_sim *s) +{ + struct ane_fw_transport *t = &s->boot.transport; + struct ane_fw_transaction *tx = &t->pending[ANE_FW_IO]; + u8 *slot = ane_fw_transport_slot(t, ANE_FW_IO); + u8 *command = (u8 *)t->pool.cpu + (tx->dva - t->pool.dva); + + if (tx->id == 3) + put_unaligned_le32(s->tick, command + 8); + if (tx->id == 0x22) + memset(command + 8, 0x5a, 92); + if (tx->id == s->refuse_id) + put_unaligned_le16(7, command + 6); + put_unaligned_le64(0, slot + 16); + put_unaligned_le64(tx->dva | 1, slot); +} + +static void sim_write32(void *ctx, enum ane_h16_window w, u32 off, u32 value) +{ + struct ane_h16_sim *s = ctx; + const struct ane_h16_build *f = &s->build; + + if (w == ANE_H16_DAPF) { + sim_check_dapf(s, off, 4); + if (s->dapf_writes < ARRAY_SIZE(s->dapf_order)) + s->dapf_order[s->dapf_writes] = off; + s->dapf_writes++; + /* Like the hardware, r0 keeps only its low byte. */ + if (off % s->mmio.dapf_stride == 0) + value &= 0xff; + if (off % s->mmio.dapf_stride == 0x20 && + (int)(off / s->mmio.dapf_stride) == s->dapf_stuck) + return; + s->dapf[off / 4] = value; + return; + } + KUNIT_EXPECT_TRUE(s->test, w == ANE_H16_ENGINE || w == ANE_H16_XL); + if (w == ANE_H16_ENGINE && off == f->cpu_control) { + s->cpu = value; + if (value == f->run && s->fault_after_run) + s->fault = BIT(31); + if (value == f->run && !s->no_ready) { + s->scratch[0] = s->channel_request; + s->scratch[1] = s->ipc_request; + s->scratch[3] = s->heap_request; + s->scratch[4] = 0; + s->scratch[7] = f->ready; + } + return; + } + if (w == ANE_H16_ENGINE && off >= s->soc.scratch && off < s->soc.scratch + 32) { + u32 cell = (off - s->soc.scratch) / 4; + + if (s->scratch_writes < 8) { + KUNIT_EXPECT_EQ(s->test, cell, s->scratch_writes); + KUNIT_EXPECT_EQ(s->test, value, 0U); + KUNIT_EXPECT_EQ(s->test, s->tuning_writes, 103U); + } + s->scratch_writes++; + s->scratch[cell] = value; + if (cell == 7 && value == ~f->ready) { + u64 address = f->ipc_dva + (s->wrong_table ? SZ_16K : 0); + + if (!s->stale_table) { + s->scratch[0] = lower_32_bits(address); + s->scratch[1] = upper_32_bits(address); + } + s->scratch[7] = f->ready; + } + if (cell == 3 && value == f->ready && !s->no_clear) + s->scratch[3] = 0; + return; + } + if (w == ANE_H16_ENGINE && off == s->soc.doorbell) { + struct ane_fw_transport *t = &s->boot.transport; + + if (value == BIT(1)) { + struct ane_fw_transaction *tx = &t->pending[ANE_FW_IO]; + + KUNIT_EXPECT_LT(s->test, s->command_count, 4U); + if (s->command_count < 4) + KUNIT_EXPECT_EQ(s->test, tx->id, f->startup[s->command_count].id); + s->command_count++; + if (tx->id == 3 && s->lifecycle_mode) { + u8 *slot = ane_fw_transport_slot(t, ANE_FW_SHAREDMALLOC); + u64 dva = s->lifecycle_mode == 2 ? 0 : s->build.shared_dva; + + if (s->lifecycle_mode == 3) + dva += SZ_16K; + + put_unaligned_le64(dva, slot); + put_unaligned_le64(0, slot + 8); + put_unaligned_le64(s->lifecycle_mode == 2 ? 1 : 0, slot + 16); + s->lifecycle_requested = true; + } else if (!tx->id) { + u8 *slot = ane_fw_transport_slot(t, ANE_FW_SHAREDMALLOC); + + put_unaligned_le64(0, slot); + put_unaligned_le64(SZ_16K, slot + 8); + put_unaligned_le64(0, slot + 16); + s->shared_pending = true; + } else { + sim_reply(s); + } + } + if (value == BIT(5)) { + s->shared_replied = true; + if (s->lifecycle_requested) { + u8 *slot = ane_fw_transport_slot(t, ANE_FW_SHAREDMALLOC); + + KUNIT_EXPECT_EQ(s->test, s->lifecycle_mode, 1U); + KUNIT_EXPECT_EQ(s->test, get_unaligned_le64(slot), 1ULL); + KUNIT_EXPECT_EQ(s->test, get_unaligned_le64(slot + 8), 0ULL); + KUNIT_EXPECT_EQ(s->test, get_unaligned_le64(slot + 16), 0ULL); + } + sim_reply(s); + } + return; + } + /* The selected AXI and XL tables must finish before scratch writes. */ + KUNIT_EXPECT_FALSE(s->test, s->boot.ran); + s->tuning_writes++; +} + +static void sim_write64(void *ctx, enum ane_h16_window w, u32 off, u64 value) +{ + struct ane_h16_sim *s = ctx; + + if (w == ANE_H16_DAPF) { + sim_check_dapf(s, off, 8); + /* The hardware drops the low two bits of an end. */ + if (off % s->mmio.dapf_stride == 0x10) + value &= s->mmio.dapf_end_mask; + s->dapf[off / 4] = lower_32_bits(value); + s->dapf[off / 4 + 1] = upper_32_bits(value); + if (s->dapf_writes < ARRAY_SIZE(s->dapf_order)) + s->dapf_order[s->dapf_writes] = off; + s->dapf_writes++; + return; + } + KUNIT_EXPECT_EQ(s->test, w, ANE_H16_ENGINE); + if (off == s->build.ce && !s->ce_timeout) + s->ce_ack = value; +} + +static void sim_delay(void *ctx, unsigned int us) +{ + ((struct ane_h16_sim *)ctx)->now += (u64)us * 1000; +} + +static int sim_map(void *ctx) +{ + struct ane_h16_sim *s = ctx; + + s->maps++; + return s->fail_map ? -ENOMEM : 0; +} + +static int sim_alloc_at(void *ctx, u64 dva, size_t size, struct ane_fw_surface *out) +{ + struct ane_h16_sim *s = ctx; + + /* Sparse simulated heap. The engine only accesses pool and IPC bytes. */ + out->cpu = kunit_kzalloc(s->test, min_t(size_t, size, SZ_256K), GFP_KERNEL); + if (!out->cpu) + return -ENOMEM; + out->dva = dva; + out->size = size; + s->allocations++; + if (dva == s->build.ipc_dva) { + static const char * const names[] = { + "TERMINAL", "IO", "DEBUG", "BUF_H2T", "BUF_T2H", + "SHAREDMALLOC", "IO_T2H", "DATA_CHAIN_H2T", + }; + unsigned int i; + + for (i = 0; i < 8; i++) { + u8 *p = (u8 *)out->cpu + i * ANE_FW_DESCRIPTOR_SIZE; + + strscpy(p, names[i], 64); + put_unaligned_le32(i == 0 ? 2 : i == 4 || i == 5 || i == 6, p + 0x40); + put_unaligned_le32(i, p + 0x44); + put_unaligned_le64(1, p + 0x48); + put_unaligned_le64(dva + ANE_FW_TABLE_SIZE + i * 64, p + 0x50); + } + } + return 0; +} + +static int sim_pin(void *ctx) +{ + ((struct ane_h16_sim *)ctx)->pins++; + return 0; +} + +static int sim_mailbox_send(void *ctx, u64 word0, u64 word1) +{ + struct ane_h16_sim *s = ctx; + static const u8 types[] = { 2, 8, 8, 5, 6, 0xb, 5 }; + + KUNIT_EXPECT_LT(s->test, s->mailbox_tx, ARRAY_SIZE(types)); + if (s->mailbox_tx < ARRAY_SIZE(types)) + KUNIT_EXPECT_EQ(s->test, (word0 >> 52) & 0xff, (u64)types[s->mailbox_tx]); + KUNIT_EXPECT_EQ(s->test, word1, 0ULL); + s->mailbox_tx++; + return 0; +} + +static void sim_mailbox(void *ctx) +{ + struct ane_h16_sim *s = ctx; + static const u64 offered[] = { + ANE_FW_RTK_TYPE(1) | (12 << 16) | 12, + ANE_FW_RTK_TYPE(8) | BIT(0) | BIT(1), + ANE_FW_RTK_TYPE(8) | BIT_ULL(32) | ANE_FW_RTK_LAST | BIT(0), + ANE_FW_RTK_TYPE(7) | 0x20, + ANE_FW_RTK_TYPE(0xb) | 0x20, + }; + unsigned int i; + + s->mailbox_polls++; + if (!s->offered_mailbox || !s->boot.ran || s->mailbox_tx) + return; + for (i = 0; i < ARRAY_SIZE(offered); i++) { + int ret = ane_fw_rtk_receive(&s->rtk, offered[i], 0, s, sim_mailbox_send); + + KUNIT_EXPECT_EQ(s->test, ret, 0); + } +} + +static void sim_log(void *ctx, const char *item, u64 a, u64 b) +{ +} + +static void sim_quarantine(void *ctx, int error) +{ + ((struct ane_h16_sim *)ctx)->quarantines++; +} + +static u64 sim_load64(void *ctx, const void *p) +{ + return get_unaligned_le64(p); +} + +static void sim_store64(void *ctx, void *p, u64 value) +{ + put_unaligned_le64(value, p); +} + +static void sim_barrier(void *ctx) +{ +} + +static u32 sim_engine_read(void *ctx, u32 off) +{ + return sim_read32(ctx, ANE_H16_ENGINE, off); +} + +static void sim_engine_write(void *ctx, u32 off, u32 value) +{ + sim_write32(ctx, ANE_H16_ENGINE, off, value); +} + +static u64 sim_now(void *ctx) +{ + return ((struct ane_h16_sim *)ctx)->now; +} + +static void sim_relax(void *ctx, u64 waited_ns) +{ + sim_delay(ctx, 1000); +} + +static int sim_alloc_shared(void *ctx, size_t size, u32 tag, struct ane_fw_surface *out) +{ + struct ane_h16_sim *s = ctx; + + return sim_alloc_at(ctx, s->build.shared_dva, size, out); +} + +static void sim_refused(void *ctx, u64 w0, u64 w1, u64 w2, int error) +{ +} + +static const struct ane_h16_boot_ops sim_ops = { + .read32 = sim_read32, + .read64 = sim_read64, + .write32 = sim_write32, + .write64 = sim_write64, + .delay_us = sim_delay, + .map = sim_map, + .alloc_at = sim_alloc_at, + .pin = sim_pin, + .mailbox = sim_mailbox, + .log = sim_log, + .quarantine = sim_quarantine, +}; + +static const struct ane_fw_transport_ops sim_transport_ops = { + .load64 = sim_load64, + .store64 = sim_store64, + .read_barrier = sim_barrier, + .write_barrier = sim_barrier, + .read32 = sim_engine_read, + .write32 = sim_engine_write, + .now_ns = sim_now, + .relax = sim_relax, + .alloc = sim_alloc_shared, + .refused = sim_refused, +}; + +/* A programmed bank, ends without their low two bits. */ +static void sim_program_dapf(struct ane_h16_sim *s) +{ + unsigned int i; + + for (i = 0; i < 4; i++) { + u32 *p = &s->dapf[(s->mmio.dapf_records + i * s->mmio.dapf_stride) / 4]; + u64 end = s->boot.ranges[i][1] & s->mmio.dapf_end_mask; + + p[0] = 0x31; + p[1] = 0; + p[2] = lower_32_bits(s->boot.ranges[i][0]); + p[3] = upper_32_bits(s->boot.ranges[i][0]); + p[4] = lower_32_bits(end); + p[5] = upper_32_bits(end); + p[8] = 0xabcd0001; + } +} + +static int sim_init(struct kunit *test) +{ + struct ane_h16_sim *s = kunit_kzalloc(test, sizeof(*s), GFP_KERNEL); + unsigned int i; + + if (!s) + return -ENOMEM; + test->priv = s; + s->test = test; + s->build = ane_h16_build_26a428; + s->mmio = ane_h16_mmio_t6041[0]; + s->dart_params = 0x2a2a0202; + s->soc.h16_mmio = &s->mmio; + s->soc.h16_mmio_count = 1; + /* Separate DART header and DAPF record windows. */ + s->soc.dart_params = 8; + s->soc.dapf_dart = 0; + s->soc.dapf_offset = 0x10000; + s->soc.dapf_size = SZ_16K; + s->soc.scratch = 0x1880020; + s->soc.doorbell = 0x1884000; + s->soc.irq_status = 0x188c000; + s->soc.irq_ack = 0x1890000; + s->soc.power_words = sim_power_words; + s->soc.power_count = ARRAY_SIZE(sim_power_words); + s->boot.soc = &s->soc; + s->boot.build = &s->build; + s->boot.ops = &sim_ops; + s->boot.transport_ops = &sim_transport_ops; + s->boot.ctx = s; + for (i = 0; i < 4; i++) { + s->boot.ranges[i][0] = 0x500000000ULL + i * SZ_16K; + s->boot.ranges[i][1] = s->boot.ranges[i][0] + 3; + } + /* The J616s bank after iBoot: power-on values, flags 0. */ + for (i = 0; i < ARRAY_SIZE(s->dapf); i++) + s->dapf[i] = i % 16 ? 0x5a5a0000 | i : 0; + s->dapf_stuck = -1; + s->channel_request = 8; + s->ipc_request = 0xc440; + s->heap_request = 0xf00000; + s->tick = 25000000; + s->refuse_id = 0xffff; + return 0; +} + +static void ane_h16_happy_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + u8 *p; + + KUNIT_ASSERT_EQ(test, ane_h16_run(&s->boot, ANE_H16_STARTUP), 0); + KUNIT_EXPECT_TRUE(test, s->boot.ran); + KUNIT_EXPECT_FALSE(test, s->boot.quarantined); + KUNIT_EXPECT_EQ(test, s->pins, 1U); + KUNIT_EXPECT_EQ(test, s->command_count, 4U); + KUNIT_EXPECT_TRUE(test, s->shared_replied); + KUNIT_EXPECT_GT(test, s->mailbox_polls, 0U); + KUNIT_EXPECT_EQ(test, s->boot.tick, 25000000U); + KUNIT_EXPECT_EQ(test, s->boot.resource_info[99], (u8)0x5a); + p = (u8 *)s->boot.pool.cpu + s->build.init_offset; + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p), 0ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p + 8), s->build.ipc_dva); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p + 0x10), 0x380000ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p + 0x18), 0xfc80000ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p + 0x28), 0xf00000ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p + 0x50), 0xc441ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(p + 0x12c), 4U); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(p + 0x38), 0ULL); + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_STARTUP), -EBUSY); +} + +static void ane_h16_no_ready_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->no_ready = true; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, s->boot.quarantined); + KUNIT_EXPECT_EQ(test, s->quarantines, 1U); + KUNIT_EXPECT_EQ(test, s->allocations, 1U); + KUNIT_EXPECT_EQ(test, s->now, 1000000000ULL); + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), -EBUSY); +} + +static void ane_h16_wrong_table_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->wrong_table = true; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), -EPROTO); + KUNIT_EXPECT_TRUE(test, s->boot.quarantined); + KUNIT_EXPECT_FALSE(test, s->boot.transport.armed); +} + +static void ane_h16_stale_table_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->stale_table = true; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, s->boot.quarantined); + KUNIT_EXPECT_FALSE(test, s->boot.transport.armed); +} + +static void ane_h16_oversized_ipc_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->ipc_request = U32_MAX; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), -E2BIG); + KUNIT_EXPECT_TRUE(test, s->boot.quarantined); + KUNIT_EXPECT_EQ(test, s->allocations, 1U); +} + +static void ane_h16_oversized_heap_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->heap_request = U32_MAX; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), -E2BIG); + KUNIT_EXPECT_TRUE(test, s->boot.quarantined); +} + +static void ane_h16_channel_limit_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->channel_request = 0x21; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), -E2BIG); + KUNIT_EXPECT_TRUE(test, s->boot.quarantined); +} + +static void ane_h16_locked_dapf_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->lock = BIT(2); + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_MAP), -EPERM); + KUNIT_EXPECT_FALSE(test, s->boot.ran); + KUNIT_EXPECT_EQ(test, s->dapf_writes, 0U); + KUNIT_EXPECT_EQ(test, s->pins, 0U); +} + +static void ane_h16_nonzero_ack_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->refuse_id = 0; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_STARTUP), -EREMOTEIO); + KUNIT_EXPECT_TRUE(test, s->boot.quarantined); + KUNIT_EXPECT_EQ(test, s->command_count, 2U); + KUNIT_EXPECT_EQ(test, s->pins, 1U); +} + +static void ane_h16_preflight_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_PREFLIGHT), 0); + KUNIT_EXPECT_PTR_EQ(test, s->boot.mmio, &s->mmio); + KUNIT_EXPECT_EQ(test, s->maps, 0U); + KUNIT_EXPECT_EQ(test, s->dapf_reads, 20U); + KUNIT_EXPECT_EQ(test, s->dapf_writes, 0U); + KUNIT_EXPECT_EQ(test, s->tuning_writes, 0U); + KUNIT_EXPECT_EQ(test, s->scratch_writes, 0U); +} + +static void ane_h16_default_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + KUNIT_EXPECT_EQ(test, ane_h16_stage_parse(NULL), ANE_H16_REFUSE); + KUNIT_EXPECT_EQ(test, ane_h16_stage_parse("unknown"), -EINVAL); + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_REFUSE), -EOPNOTSUPP); + KUNIT_EXPECT_FALSE(test, s->boot.attempted); + KUNIT_EXPECT_EQ(test, s->scratch_writes, 0U); +} + +static void ane_h16_ce_timeout_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->ce_timeout = true; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), -ETIMEDOUT); + KUNIT_EXPECT_FALSE(test, s->boot.ran); + KUNIT_EXPECT_EQ(test, s->scratch_writes, 0U); + KUNIT_EXPECT_EQ(test, s->now, 50010000ULL); +} + +static void ane_h16_zero_tick_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->tick = 0; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_STARTUP), -EPROTO); + KUNIT_EXPECT_TRUE(test, s->boot.quarantined); +} + +static void ane_h16_scratch3_timeout_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->no_clear = true; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), -ETIMEDOUT); + KUNIT_EXPECT_TRUE(test, s->boot.quarantined); + KUNIT_EXPECT_TRUE(test, s->boot.transport.armed); + KUNIT_EXPECT_EQ(test, s->command_count, 0U); +} + +/* Stages advance one write at a time, each at most once, in one firmware session. */ +static void ane_h16_incremental_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + KUNIT_ASSERT_EQ(test, ane_h16_run(&s->boot, ANE_H16_PREFLIGHT), 0); + KUNIT_EXPECT_EQ(test, s->maps, 0U); + KUNIT_EXPECT_EQ(test, s->scratch_writes, 0U); + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_PREFLIGHT), -EBUSY); + KUNIT_ASSERT_EQ(test, ane_h16_run(&s->boot, ANE_H16_MAP), 0); + KUNIT_EXPECT_EQ(test, s->maps, 1U); + KUNIT_EXPECT_EQ(test, s->scratch_writes, 0U); + KUNIT_EXPECT_FALSE(test, s->boot.ran); + KUNIT_ASSERT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), 0); + KUNIT_EXPECT_TRUE(test, s->boot.ran); + KUNIT_EXPECT_EQ(test, s->boot.stage, ANE_H16_BOOT); + KUNIT_EXPECT_EQ(test, s->command_count, 0U); + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_MAP), -EBUSY); + /* The program stage belongs to the backend; here it means startup. */ + KUNIT_ASSERT_EQ(test, ane_h16_run(&s->boot, ANE_H16_PROGRAM), 0); + KUNIT_EXPECT_EQ(test, s->boot.stage, ANE_H16_STARTUP); + KUNIT_EXPECT_EQ(test, s->command_count, 4U); + KUNIT_EXPECT_EQ(test, s->maps, 1U); + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_STARTUP), -EBUSY); + KUNIT_EXPECT_STREQ(test, ane_h16_stage_name(s->boot.stage), "startup"); +} + +static void ane_h16_mailbox_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->offered_mailbox = true; + KUNIT_ASSERT_EQ(test, ane_h16_run(&s->boot, ANE_H16_STARTUP), 0); + KUNIT_EXPECT_EQ(test, s->rtk.phase, ANE_FW_RTK_READY); + KUNIT_EXPECT_EQ(test, s->mailbox_tx, 7U); +} + +static void ane_h16_no_heap_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + u8 *record; + + s->heap_request = 0; + KUNIT_ASSERT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), 0); + KUNIT_EXPECT_EQ(test, s->allocations, 2U); + record = (u8 *)s->boot.pool.cpu + s->build.init_offset; + KUNIT_EXPECT_EQ(test, get_unaligned_le64(record + 0x20), 0ULL); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(record + 0x28), 0ULL); +} + +static void ane_h16_power_actual_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->bad_power = true; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_PREFLIGHT), -EHOSTDOWN); + KUNIT_EXPECT_EQ(test, s->scratch_writes, 0U); + KUNIT_EXPECT_EQ(test, s->pins, 0U); +} + +static void ane_h16_rvbar_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->bad_rvbar = true; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), -EBUSY); + KUNIT_EXPECT_EQ(test, s->scratch_writes, 0U); + KUNIT_EXPECT_EQ(test, s->maps, 0U); +} + +static void ane_h16_dart_fault_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->fault_after_run = true; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), -EIO); + KUNIT_EXPECT_TRUE(test, s->boot.quarantined); + KUNIT_EXPECT_EQ(test, s->pins, 1U); + KUNIT_EXPECT_EQ(test, s->quarantines, 1U); +} + +static void ane_h16_map_failure_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->fail_map = true; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), -ENOMEM); + KUNIT_EXPECT_FALSE(test, s->boot.ran); + KUNIT_EXPECT_FALSE(test, s->boot.quarantined); + KUNIT_EXPECT_EQ(test, s->pins, 0U); +} + +static void ane_h16_locked_matching_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + sim_program_dapf(s); + s->lock = BIT(2); + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_MAP), 0); + KUNIT_EXPECT_EQ(test, s->dapf_writes, 0U); +} + +/* Preflight only reads the unprogrammed bank; map writes it in t8110 order. */ +static void ane_h16_dapf_program_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + static const u32 order[] = { 4, 8, 0x10, 0, 0x20 }; + unsigned int i; + + KUNIT_ASSERT_EQ(test, ane_h16_run(&s->boot, ANE_H16_PREFLIGHT), 0); + KUNIT_EXPECT_EQ(test, s->dapf_writes, 0U); + KUNIT_ASSERT_EQ(test, ane_h16_run(&s->boot, ANE_H16_MAP), 0); + KUNIT_EXPECT_EQ(test, s->dapf_writes, 20U); + for (i = 0; i < ARRAY_SIZE(order); i++) + KUNIT_EXPECT_EQ(test, s->dapf_order[i], order[i]); + for (i = 0; i < 4; i++) { + u32 *p = &s->dapf[(s->mmio.dapf_records + i * s->mmio.dapf_stride) / 4]; + + KUNIT_EXPECT_TRUE(test, ane_h16_dapf_record(&s->boot, i)); + KUNIT_EXPECT_EQ(test, p[0], 0x31U); + KUNIT_EXPECT_EQ(test, p[1], 0U); + KUNIT_EXPECT_EQ(test, p[8], 1U); + } + KUNIT_EXPECT_FALSE(test, s->boot.ran); +} + +/* Only records that differ are written. */ +static void ane_h16_dapf_partial_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + sim_program_dapf(s); + s->dapf[(s->mmio.dapf_records + 2 * s->mmio.dapf_stride + 8) / 4] ^= 0x4000; + KUNIT_ASSERT_EQ(test, ane_h16_run(&s->boot, ANE_H16_MAP), 0); + KUNIT_EXPECT_EQ(test, s->dapf_writes, 5U); + KUNIT_EXPECT_EQ(test, s->dapf_order[1], s->mmio.dapf_records + 2 * s->mmio.dapf_stride + 8); +} + +/* A record that does not read back fails map before RUN. */ +static void ane_h16_dapf_readback_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->dapf_stuck = 1; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_MAP), -EIO); + KUNIT_EXPECT_EQ(test, s->dapf_writes, 10U); + KUNIT_EXPECT_FALSE(test, s->boot.ran); + KUNIT_EXPECT_EQ(test, s->pins, 0U); +} + +/* A DART revision without a table stops after the PARAMS3 reads. */ +static void ane_h16_unknown_revision_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->dart_params = 0x2a2a0203; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_PREFLIGHT), -ENODEV); + KUNIT_EXPECT_NULL(test, s->boot.mmio); + KUNIT_EXPECT_EQ(test, s->dapf_reads, 0U); + KUNIT_EXPECT_EQ(test, s->dapf_writes, 0U); + KUNIT_EXPECT_EQ(test, s->maps, 0U); +} + +static void ane_h16_heap_overlap_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->heap_request = SZ_16M; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_BOOT), -E2BIG); + KUNIT_EXPECT_TRUE(test, s->boot.quarantined); +} + +static void ane_h16_native_free_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->lifecycle_mode = 1; + KUNIT_ASSERT_EQ(test, ane_h16_run(&s->boot, ANE_H16_STARTUP), 0); + KUNIT_EXPECT_TRUE(test, s->lifecycle_requested); + KUNIT_EXPECT_TRUE(test, s->boot.transport.shared_retired[0]); + KUNIT_EXPECT_TRUE(test, s->boot.transport.shared[0].cpu); +} + +static void ane_h16_control_refusal_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->lifecycle_mode = 2; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_STARTUP), -EOPNOTSUPP); + KUNIT_EXPECT_TRUE(test, s->boot.quarantined); + KUNIT_EXPECT_FALSE(test, s->boot.transport.shared_retired[0]); +} + +static void ane_h16_foreign_free_test(struct kunit *test) +{ + struct ane_h16_sim *s = test->priv; + + s->lifecycle_mode = 3; + KUNIT_EXPECT_EQ(test, ane_h16_run(&s->boot, ANE_H16_STARTUP), -ENOENT); + KUNIT_EXPECT_TRUE(test, s->boot.quarantined); + KUNIT_EXPECT_FALSE(test, s->boot.transport.shared_retired[0]); +} + +static struct kunit_case ane_h16_boot_cases[] = { + KUNIT_CASE(ane_h16_happy_test), + KUNIT_CASE(ane_h16_locked_matching_test), + KUNIT_CASE(ane_h16_dapf_program_test), + KUNIT_CASE(ane_h16_dapf_partial_test), + KUNIT_CASE(ane_h16_dapf_readback_test), + KUNIT_CASE(ane_h16_unknown_revision_test), + KUNIT_CASE(ane_h16_heap_overlap_test), + KUNIT_CASE(ane_h16_native_free_test), + KUNIT_CASE(ane_h16_control_refusal_test), + KUNIT_CASE(ane_h16_foreign_free_test), + KUNIT_CASE(ane_h16_mailbox_test), + KUNIT_CASE(ane_h16_incremental_test), + KUNIT_CASE(ane_h16_no_heap_test), + KUNIT_CASE(ane_h16_power_actual_test), + KUNIT_CASE(ane_h16_rvbar_test), + KUNIT_CASE(ane_h16_dart_fault_test), + KUNIT_CASE(ane_h16_map_failure_test), + KUNIT_CASE(ane_h16_no_ready_test), + KUNIT_CASE(ane_h16_wrong_table_test), + KUNIT_CASE(ane_h16_stale_table_test), + KUNIT_CASE(ane_h16_oversized_ipc_test), + KUNIT_CASE(ane_h16_oversized_heap_test), + KUNIT_CASE(ane_h16_channel_limit_test), + KUNIT_CASE(ane_h16_locked_dapf_test), + KUNIT_CASE(ane_h16_nonzero_ack_test), + KUNIT_CASE(ane_h16_preflight_test), + KUNIT_CASE(ane_h16_default_test), + KUNIT_CASE(ane_h16_ce_timeout_test), + KUNIT_CASE(ane_h16_zero_tick_test), + KUNIT_CASE(ane_h16_scratch3_timeout_test), + {} +}; + +static struct kunit_suite ane_h16_boot_suite = { + .name = "ane-h16-boot", + .init = sim_init, + .test_cases = ane_h16_boot_cases, +}; + +kunit_test_suite(ane_h16_boot_suite); + +MODULE_DESCRIPTION("KUnit simulated H16 cold boot engine"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_execute.h b/drivers/accel/ane/ane_h16_execute.h index df3cb515e9e2c..43f97b7b5b4de 100644 --- a/drivers/accel/ane/ane_h16_execute.h +++ b/drivers/accel/ane/ane_h16_execute.h @@ -2,6 +2,7 @@ #ifndef __ANE_H16_EXECUTE_H__ #define __ANE_H16_EXECUTE_H__ +#include #include #include "ane_h16_protocol.h" @@ -11,6 +12,18 @@ #define ANE_H16_BUFFER_STRIDE 0x30 #define ANE_H16_MAX_BUFFERS 64 #define ANE_H16_CUSTOM_BAR_STRIDE 0x18 +/* + * Request flags word. Bit 0 keeps the request off the firmware's direct + * submission path and queues it on the full execution loop (EXELOOP), which + * installs a fresh BAR context and the per-priority L2-spill mapping. The + * direct path reuses both and never sets them up. Measured on H17 (T8140): + * without it, the first job after the engine ran a different program can + * write through that program's stale output binding. Set it on the first + * request of each program on the engine, the first one after boot included. + */ +#define ANE_H16_EXECUTE_FLAGS 0x30 +#define ANE_H16_EXECUTE_FLAG_QUEUED BIT(0) +#define ANE_H16_EXECUTE_FLAGS_KNOWN ANE_H16_EXECUTE_FLAG_QUEUED /* Conservative observed-version ceiling, not a firmware maximum. */ #define ANE_H16_MAX_CUSTOM_BARS 2 @@ -48,6 +61,7 @@ struct ane_h16_execute { unsigned int buffer_count; const struct ane_h16_custom_bar *bars; unsigned int bar_count; + u32 flags; }; /* @@ -66,7 +80,8 @@ ane_h16_execute_encode(void *command, size_t length, const struct ane_h16_execut if (!p || !call || !call->buffers || !call->buffer_count || call->buffer_count > ANE_H16_MAX_BUFFERS || call->bar_count > ANE_H16_MAX_CUSTOM_BARS || - (call->bar_count && !call->bars)) + (call->bar_count && !call->bars) || + (call->flags & ~ANE_H16_EXECUTE_FLAGS_KNOWN)) return -EINVAL; expected = ANE_H16_EXECUTE_BASE_SIZE + call->bar_count * ANE_H16_CUSTOM_BAR_STRIDE; if (length != expected) @@ -103,6 +118,7 @@ ane_h16_execute_encode(void *command, size_t length, const struct ane_h16_execut put_unaligned_le32(5, p + 0x18); put_unaligned_le64(call->key.uuid, p + 0x20); put_unaligned_le32(call->buffer_count, p + 0x28); + put_unaligned_le32(call->flags, p + ANE_H16_EXECUTE_FLAGS); for (i = 0; i < call->buffer_count; i++) { const struct ane_h16_execute_buffer *b = &call->buffers[i]; u8 *d = p + ANE_H16_BUFFER_OFFSET + i * ANE_H16_BUFFER_STRIDE; diff --git a/drivers/accel/ane/ane_h16_execute_test.c b/drivers/accel/ane/ane_h16_execute_test.c index 933011c18a77a..c11aa561768bb 100644 --- a/drivers/accel/ane/ane_h16_execute_test.c +++ b/drivers/accel/ane/ane_h16_execute_test.c @@ -296,9 +296,29 @@ static void ane_h16_event_rejections_test(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, memchr_inv(&e, 0xa5, sizeof(e)), NULL); } +static void ane_h16_execute_flags_test(struct kunit *test) +{ + struct ane_h16_execute_buffer buffer = test_buffer(); + struct ane_h16_execute call = { .key = test_key, .buffers = &buffer, .buffer_count = 1 }; + u8 *p = kunit_kmalloc(test, 0xca8, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, p); + KUNIT_ASSERT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(p + ANE_H16_EXECUTE_FLAGS), 0U); + call.flags = ANE_H16_EXECUTE_FLAG_QUEUED; + KUNIT_ASSERT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(p + ANE_H16_EXECUTE_FLAGS), 1U); + /* Unknown flag bits are refused and leave the destination untouched. */ + memset(p, 0xa5, 0xca8); + call.flags = BIT(1); + KUNIT_EXPECT_EQ(test, ane_h16_execute_encode(p, 0xca8, &call), -EINVAL); + KUNIT_EXPECT_EQ(test, p[ANE_H16_EXECUTE_FLAGS], (u8)0xa5); +} + static struct kunit_case ane_h16_execute_cases[] = { KUNIT_CASE(ane_h16_execute_layout_test), KUNIT_CASE(ane_h16_execute_counts_test), + KUNIT_CASE(ane_h16_execute_flags_test), KUNIT_CASE(ane_h16_custom_bar_test), KUNIT_CASE(ane_h16_execute_rejections_test), KUNIT_CASE(ane_h16_bar_rejections_test), diff --git a/drivers/accel/ane/ane_h16_hwx.c b/drivers/accel/ane/ane_h16_hwx.c index a29e5f81f0aa2..e970df7d1105e 100644 --- a/drivers/accel/ane/ane_h16_hwx.c +++ b/drivers/accel/ane/ane_h16_hwx.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "ane_h16_hwx.h" #include "ane_h16_tasks.h" @@ -191,6 +192,17 @@ static int hwx_segment(struct hwx_parser *p, const u8 *c, u32 length) *dest = section; p->sections[p->section_count++] = section; } + if (p->info.region_count == ANE_H16_REGIONS) + return -ENOEXEC; + p->info.regions[p->info.region_count++] = (struct ane_h16_hwx_region) { + .vm = vm, + .size = size, + .file_offset = file, + .file_size = filesz, + .permissions = prot, + .kind = surface ? ANE_H16_REGION_SURFACE : text ? ANE_H16_REGION_TEXT : + kernel ? ANE_H16_REGION_KERNEL : ANE_H16_REGION_SCRATCH, + }; return 0; } @@ -290,16 +302,49 @@ static int hwx_program(struct hwx_parser *p) memcpy(expected + (mixed ? 0x8b0 : 0x890), c + (mixed ? 0x8b0 : 0x890), 4); memcpy(expected + (mixed ? 0x8b8 : 0x898), "main", 5); if (!memcmp(c, expected, length)) - ret = ane_h16_task_stream_validate(p->bytes + info->text.offset, - info->text.size, info->tasks); + ret = ane_hx_task_stream_validate(p->bytes + info->text.offset, + info->text.size, info->tasks, + info->subtype == ANE_HWX_SUBTYPE_H17 ? 0 : 1); kfree(expected); return ret; } -static int hwx_tensor(struct hwx_parser *p, u32 index) +/* + * Buffer references follow the surface segments, but tensor descriptors list + * the inputs before the output, so the add-128 HWX describes segments x, y, z + * with tensors x, z, y. Pair each surface with the one tensor whose short name + * is the name its buffer reference gives. + */ +static int hwx_tensor_for(struct hwx_parser *p, u32 index, u32 *used) +{ + const u8 *ref = p->bytes + p->refs[index]; + u32 i, match = ANE_H16_SURFACES; + + for (i = 0; i < p->tensor_count; i++) { + const u8 *c = p->bytes + p->tensors[i]; + u32 length = get_unaligned_le32(c + 4), shortoff = get_unaligned_le32(c + 0x7c); + size_t room; + + if (shortoff >= length) + return -ENOEXEC; + room = min_t(size_t, length - shortoff, 8); + if (strnlen(c + shortoff, room) == room || + strncmp(c + shortoff, ref + 0x18, 8)) + continue; + if (match != ANE_H16_SURFACES || (*used & BIT(i))) + return -ENOEXEC; + match = i; + } + if (match == ANE_H16_SURFACES) + return -ENOEXEC; + *used |= BIT(match); + return match; +} + +static int hwx_tensor(struct hwx_parser *p, u32 index, u32 tensor) { struct ane_h16_hwx_surface *b = &p->info.surfaces[index]; - const u8 *c = p->bytes + p->tensors[index]; + const u8 *c = p->bytes + p->tensors[tensor]; const u8 *ref = p->bytes + p->refs[index]; u32 length = get_unaligned_le32(c + 4), shortoff; u64 n, channels, height, width, row, plane, batch, minimum; @@ -344,7 +389,7 @@ static int hwx_tensor(struct hwx_parser *p, u32 index) !hwx_zero(c, shortoff + short_length + 1, length)) return -ENOEXEC; memcpy(b->name, c + shortoff, short_length); - b->symbol_offset = p->tensors[index] + 0xd3d; + b->symbol_offset = p->tensors[tensor] + 0xd3d; if (get_unaligned_le64(ref + 8) != 0x18 || get_unaligned_le64(ref + 0x10) != b->section.vm || memcmp(ref + 0x18, b->name, sizeof(b->name))) @@ -445,13 +490,15 @@ int ane_h16_hwx_validate(const void *bytes, size_t size, struct ane_h16_hwx *inf const void *relocations; const u8 *c = bytes; size_t pos = 32; - u32 count, length, i; + u32 count, length, i, outputs, used; unsigned long *seen = NULL; int ret = -ENOEXEC; if (!c || !info || size < 32 || size > ANE_H16_HWX_MAX || get_unaligned_le32(c) != 0xbeefface || get_unaligned_le32(c + 4) != 0x80 || - get_unaligned_le32(c + 8) != 7 || get_unaligned_le32(c + 12) != 2 || + (get_unaligned_le32(c + 8) != ANE_HWX_SUBTYPE_H16 && + get_unaligned_le32(c + 8) != ANE_HWX_SUBTYPE_H17) || + get_unaligned_le32(c + 12) != 2 || get_unaligned_le32(c + 24) != 0x200000 || get_unaligned_le32(c + 28)) return -ENOEXEC; count = get_unaligned_le32(c + 16); @@ -464,6 +511,7 @@ int ane_h16_hwx_validate(const void *bytes, size_t size, struct ane_h16_hwx *inf p->bytes = bytes; p->size = size; p->command_end = 32 + (size_t)length; + p->info.subtype = get_unaligned_le32(p->bytes + 8); for (i = 0; i < count; i++) { u32 cmd, len; @@ -531,14 +579,18 @@ int ane_h16_hwx_validate(const void *bytes, size_t size, struct ane_h16_hwx *inf p->info.surface_count < 2 || p->ref_count != p->info.surface_count || p->tensor_count != p->info.surface_count) goto invalid; - for (i = 0; i < p->info.surface_count; i++) { - /* Inputs in segment order, exactly one output last. */ - if (p->info.surfaces[i].direction != (i == p->info.surface_count - 1 ? 2 : 1)) - goto invalid; - ret = hwx_tensor(p, i); + for (i = 0, outputs = 0, used = 0; i < p->info.surface_count; i++) { + /* Surfaces stay in segment order: inputs and exactly one output. */ + outputs += p->info.surfaces[i].direction == 2; + ret = hwx_tensor_for(p, i, &used); + if (ret < 0) + goto out; + ret = hwx_tensor(p, i, ret); if (ret) goto out; } + if (outputs != 1) + goto invalid; ret = hwx_program(p); if (ret) goto out; @@ -572,30 +624,61 @@ int ane_h16_hwx_validate(const void *bytes, size_t size, struct ane_h16_hwx *inf } EXPORT_SYMBOL_GPL(ane_h16_hwx_validate); +static int hwx_snapshot_finish(struct ane_h16_hwx_snapshot *copy, + struct ane_h16_hwx_snapshot *snapshot) +{ + int ret; + + ret = ane_h16_hwx_validate(copy->bytes, copy->size, ©->info); + if (ret) { + kvfree(copy->bytes); + return ret; + } + sha256(copy->bytes, copy->size, copy->digest); + *snapshot = *copy; + return 0; +} + int ane_h16_hwx_snapshot_create(const void *bytes, size_t size, struct ane_h16_hwx_snapshot *snapshot) { struct ane_h16_hwx_snapshot copy = {}; - int ret; if (!bytes || !snapshot || size < 32 || size > ANE_H16_HWX_MAX) return -EINVAL; + if (snapshot->bytes) + return -EBUSY; copy.bytes = kvmalloc(size, GFP_KERNEL); if (!copy.bytes) return -ENOMEM; memcpy(copy.bytes, bytes, size); copy.size = size; - ret = ane_h16_hwx_validate(copy.bytes, size, ©.info); - if (ret) { - kvfree(copy.bytes); - return ret; - } - sha256(copy.bytes, size, copy.digest); - *snapshot = copy; - return 0; + return hwx_snapshot_finish(©, snapshot); } EXPORT_SYMBOL_GPL(ane_h16_hwx_snapshot_create); +/* Import exactly once before parsing or hashing. No userspace digest, + * section list or resource IDs can substitute for this original container. + * Callers supply an empty destination and retain it until all firmware + * references end; importing alone grants no process or execution authority. + */ +int ane_h16_hwx_snapshot_from_user(const void __user *bytes, u64 size, + struct ane_h16_hwx_snapshot *snapshot) +{ + struct ane_h16_hwx_snapshot copy = {}; + + if (!bytes || !snapshot || size < 32 || size > ANE_H16_HWX_MAX) + return -EINVAL; + if (snapshot->bytes) + return -EBUSY; + copy.bytes = vmemdup_user(bytes, size); + if (IS_ERR(copy.bytes)) + return PTR_ERR(copy.bytes); + copy.size = size; + return hwx_snapshot_finish(©, snapshot); +} +EXPORT_SYMBOL_GPL(ane_h16_hwx_snapshot_from_user); + int ane_h16_hwx_snapshot_check(const struct ane_h16_hwx_snapshot *snapshot) { u8 digest[SHA256_DIGEST_SIZE]; diff --git a/drivers/accel/ane/ane_h16_hwx.h b/drivers/accel/ane/ane_h16_hwx.h index ba913c0f25d12..dedfa58741964 100644 --- a/drivers/accel/ane/ane_h16_hwx.h +++ b/drivers/accel/ane/ane_h16_hwx.h @@ -8,7 +8,11 @@ #define ANE_H16_SURFACES 4 #define ANE_H16_SECTIONS 8 +#define ANE_H16_REGIONS 7 #define ANE_H16_HWX_MAX (SZ_256M + SZ_1M) +/* Mach-O CPU subtypes of the two targets that share this container form. */ +#define ANE_HWX_SUBTYPE_H16 7 +#define ANE_HWX_SUBTYPE_H17 9 struct ane_h16_hwx_section { u64 vm; @@ -28,12 +32,33 @@ struct ane_h16_hwx_surface { char name[8]; }; +enum ane_h16_region_kind { + ANE_H16_REGION_SURFACE, + ANE_H16_REGION_TEXT, + ANE_H16_REGION_KERNEL, + ANE_H16_REGION_SCRATCH, +}; + +/* Original segment geometry, not firmware section IDs or resource indices. */ +struct ane_h16_hwx_region { + u64 vm; + u64 size; + u64 file_offset; + u64 file_size; + u32 permissions; + enum ane_h16_region_kind kind; +}; + struct ane_h16_hwx { struct ane_h16_hwx_section text, constants, kernel, scratch; + /* Target; a backend accepts only its own engine's subtype. */ + u32 subtype; struct ane_h16_hwx_surface surfaces[ANE_H16_SURFACES]; u32 surface_count; u32 tasks; u32 program_offset; + struct ane_h16_hwx_region regions[ANE_H16_REGIONS]; + u32 region_count; }; /* Input must be one immutable kernel-owned snapshot of the original file. */ @@ -49,6 +74,8 @@ struct ane_h16_hwx_snapshot { /* Snapshot before validation; callers never publish or mutate snapshot bytes. */ int ane_h16_hwx_snapshot_create(const void *bytes, size_t size, struct ane_h16_hwx_snapshot *snapshot); +int ane_h16_hwx_snapshot_from_user(const void __user *bytes, u64 size, + struct ane_h16_hwx_snapshot *snapshot); int ane_h16_hwx_snapshot_check(const struct ane_h16_hwx_snapshot *snapshot); void ane_h16_hwx_snapshot_destroy(struct ane_h16_hwx_snapshot *snapshot); diff --git a/drivers/accel/ane/ane_h16_hwx_test.c b/drivers/accel/ane/ane_h16_hwx_test.c index b7f434748b829..86af1c4779763 100644 --- a/drivers/accel/ane/ane_h16_hwx_test.c +++ b/drivers/accel/ane/ane_h16_hwx_test.c @@ -4,207 +4,9 @@ #include "ane_h16_hwx.h" #include "ane_h16_tasks.h" +#include "ane_h16_load.h" -struct hwx_fixture { - u8 *bytes; - size_t size, cursor; - u32 commands, program, tensor[2], segment[2], text_command, symtab, reloc; - u32 symbols, strings; -}; - -static u8 *fixture_command(struct hwx_fixture *f, u32 command, u32 length) -{ - u8 *c = f->bytes + f->cursor; - - put_unaligned_le32(command, c); - put_unaligned_le32(length, c + 4); - f->cursor += length; - f->commands++; - return c; -} - -static u8 *fixture_segment(struct hwx_fixture *f, const char *name, u64 vm, - u64 size, u32 offset, u32 prot, u32 flags, u32 sections) -{ - u8 *c = fixture_command(f, 0x19, 72 + sections * 80); - - strscpy(c + 8, name, 16); - put_unaligned_le64(vm, c + 24); - put_unaligned_le64(size, c + 32); - put_unaligned_le64(offset, c + 40); - put_unaligned_le64(offset ? size : 0, c + 48); - put_unaligned_le32(prot, c + 56); - put_unaligned_le32(prot, c + 60); - put_unaligned_le32(sections, c + 64); - put_unaligned_le32(flags, c + 68); - return c + 72; -} - -static void fixture_section(u8 *s, const char *segment, const char *name, - u64 vm, u64 size, u32 offset, u32 alignment, u32 flags) -{ - strscpy(s, name, 16); - strscpy(s + 16, segment, 16); - put_unaligned_le64(vm, s + 32); - put_unaligned_le64(size, s + 40); - put_unaligned_le32(offset, s + 48); - put_unaligned_le32(alignment, s + 52); - put_unaligned_le32(flags, s + 64); -} - -static struct hwx_fixture *fixture_create(struct kunit *test, bool mixed, bool kernel) -{ - struct hwx_fixture *f; - u64 surface = 0x30000000 + (mixed ? SZ_16K : 0); - u64 text = surface + 2 * SZ_16K, kern = text + SZ_32K; - u32 i, strings_length = 1; - u8 *c, *s, *text_section; - static const char * const names[] = { "in", "out" }; - static const char * const symbols[] = { "input", "output" }; - - f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); - if (!f) - return NULL; - f->size = kernel ? SZ_64K : SZ_32K + SZ_16K; - f->bytes = kunit_kzalloc(test, f->size, GFP_KERNEL); - if (!f->bytes) - return NULL; - f->cursor = 32; - fixture_segment(f, "__PAGEZERO", 0, SZ_16K, 0, 0, 4, 0); - if (mixed) { - s = fixture_segment(f, "__DATA", 0x30000000, SZ_16K, 0, 3, 4, 1); - fixture_section(s, "__DATA", "__bss", 0x30000000, SZ_16K, 0, 6, 0x25); - } - for (i = 0; i < 2; i++) { - f->segment[i] = f->cursor; - s = fixture_segment(f, "__FVMLIB", surface + i * SZ_16K, SZ_16K, - 0, i + 1, 6, 1); - fixture_section(s, "__FVMLIB", i ? "__data" : "__const", - surface + i * SZ_16K, SZ_16K, 0, 14, i ? 0x23 : 0x21); - } - f->text_command = f->cursor; - text_section = fixture_segment(f, "__TEXT", text, SZ_32K, SZ_16K, 5, - mixed ? 4 : 0, 2); - fixture_section(text_section, "__TEXT", "__text", text, 64, SZ_16K, - 14, kernel ? 0x128 : 0x28); - fixture_section(text_section + 80, "__TEXT", "__const", text + 64, - SZ_16K, SZ_16K + 64, 6, 0x26); - if (kernel) { - s = fixture_segment(f, "__KERN_0", kern, SZ_16K, SZ_32K + SZ_16K, 1, 4, 1); - fixture_section(s, "__KERN_0", "__kern_0", kern, 64, - SZ_32K + SZ_16K, 6, 0x26); - } - for (i = 0; i < 2; i++) { - c = fixture_command(f, 0x40, 32); - put_unaligned_le64(0x18, c + 8); - put_unaligned_le64(surface + i * SZ_16K, c + 0x10); - strscpy(c + 0x18, names[i], 8); - } - f->program = f->cursor; - c = fixture_command(f, 4, mixed ? 0x8c0 : 0x8a0); - put_unaligned_le32(4, c + 8); - put_unaligned_le32(mixed ? 0x22a : 0x222, c + 0xc); - put_unaligned_le64(text, c + 0x10); - put_unaligned_le64(text + 64, c + 0x20); - if (mixed) - put_unaligned_le64(0x30000000, c + 0x40); - for (i = 0; i < 2; i++) - put_unaligned_le64(surface + i * SZ_16K, c + 0x70 + i * 16); - if (kernel) - put_unaligned_le64(kern, c + 0x90); - put_unaligned_le64(text, c + 0x810); - put_unaligned_le32(4, c + 0x820); - put_unaligned_le32(16, c + 0x824); - put_unaligned_le32(1, c + 0x830); - put_unaligned_le32(U32_MAX, c + 0x838); - put_unaligned_le32(U32_MAX, c + 0x83c); - put_unaligned_le32(0xffff, c + 0x844); - put_unaligned_le32(mixed ? 0x8b8 : 0x898, c + 0x848); - put_unaligned_le32(0x10, c + 0x860); - put_unaligned_le32(4, c + 0x868); - put_unaligned_le32(mixed ? 4 : 2, c + 0x870); - put_unaligned_le32(mixed ? 2 : 7, c + 0x878); - put_unaligned_le32(8, c + 0x87c); - put_unaligned_le32(mixed ? 256 : 0x461c4000, c + 0x880); - put_unaligned_le32(mixed ? 3 : 9, c + 0x888); - put_unaligned_le32(8, c + 0x88c); - if (mixed) { - put_unaligned_le32(256, c + 0x858); - put_unaligned_le32(0x08020004, c + 0x890); - put_unaligned_le32(8, c + 0x894); - put_unaligned_le32(7, c + 0x898); - put_unaligned_le32(8, c + 0x89c); - put_unaligned_le32(0x461c4000, c + 0x8a0); - put_unaligned_le32(9, c + 0x8a8); - put_unaligned_le32(8, c + 0x8ac); - } - memcpy(c + (mixed ? 0x8b8 : 0x898), "main", 5); - for (i = 0; i < 2; i++) { - u32 shortoff = 0xd3d + strlen(symbols[i]) + 1; - - f->tensor[i] = f->cursor; - c = fixture_command(f, 4, 0xd48); - put_unaligned_le32(3, c + 8); - put_unaligned_le32(0x34a, c + 0xc); - put_unaligned_le32(3, c + 0x10); - put_unaligned_le32(i + 1, c + 0x14); - put_unaligned_le32(0xd38, c + 0x18); - put_unaligned_le32(0xd3d, c + 0x20); - put_unaligned_le32(5, c + 0x24); - put_unaligned_le32(1, c + 0x28); - put_unaligned_le32(1, c + 0x2c); - put_unaligned_le32(1, c + 0x30); - put_unaligned_le32(128, c + 0x34); - put_unaligned_le32(1, c + 0x38); - put_unaligned_le64(256, c + 0x50); - put_unaligned_le64(256, c + 0x58); - put_unaligned_le64(256, c + 0x60); - put_unaligned_le64(2, c + 0x68); - put_unaligned_le64(256, c + 0x70); - put_unaligned_le32(1, c + 0x78); - put_unaligned_le32(shortoff, c + 0x7c); - put_unaligned_le64(256, c + 0x80); - memcpy(c + 0xd38, "main", 5); - memcpy(c + 0xd3d, symbols[i], strlen(symbols[i]) + 1); - memcpy(c + shortoff, names[i], strlen(names[i]) + 1); - } - f->symtab = f->cursor; - c = fixture_command(f, 2, 24); - f->symbols = f->cursor; - f->strings = f->symbols + 2 * 16; - for (i = 0; i < 2; i++) { - u8 *symbol = f->bytes + f->symbols + i * 16; - - put_unaligned_le32(strings_length, symbol); - symbol[4] = 0xf; - symbol[5] = (mixed ? 2 : 1) + i; - put_unaligned_le64(surface + i * SZ_16K, symbol + 8); - memcpy(f->bytes + f->strings + strings_length, names[i], strlen(names[i]) + 1); - strings_length += strlen(names[i]) + 1; - } - put_unaligned_le32(f->symbols, c + 8); - put_unaligned_le32(2, c + 12); - put_unaligned_le32(f->strings, c + 16); - put_unaligned_le32(strings_length, c + 20); - f->reloc = ALIGN(f->strings + strings_length, 8); - if (kernel) { - put_unaligned_le32(f->reloc, text_section + 56); - put_unaligned_le32(1, text_section + 60); - put_unaligned_le32(56, f->bytes + f->reloc); - put_unaligned_le32(0x07000000 | (mixed ? 6 : 5), f->bytes + f->reloc + 4); - put_unaligned_le32(32, f->bytes + SZ_16K + 56); - } - put_unaligned_le32(11 << 16, f->bytes + SZ_16K + 16); - put_unaligned_le32(0x4100 / 4, f->bytes + SZ_16K + 52); - put_unaligned_le32(0xbeefface, f->bytes); - put_unaligned_le32(0x80, f->bytes + 4); - put_unaligned_le32(7, f->bytes + 8); - put_unaligned_le32(2, f->bytes + 12); - put_unaligned_le32(f->commands, f->bytes + 16); - put_unaligned_le32(f->cursor - 32, f->bytes + 20); - put_unaligned_le32(0x200000, f->bytes + 24); - return f; -} +#include "ane_h16_hwx_test_fixture.h" static void hwx_forms_test(struct kunit *test) { @@ -300,6 +102,95 @@ static void hwx_tensor_test(struct kunit *test) KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); } +static u8 *fixture_ref(struct hwx_fixture *f, unsigned int index) +{ + size_t pos = 32; + unsigned int i, seen = 0; + + for (i = 0; i < f->commands; i++) { + u8 *c = f->bytes + pos; + + if (get_unaligned_le32(c) == 0x40 && seen++ == index) + return c; + pos += get_unaligned_le32(c + 4); + } + return NULL; +} + +static void fixture_surface_kind(struct hwx_fixture *f, unsigned int index, u32 prot) +{ + u8 *c = f->bytes + f->segment[index]; + + put_unaligned_le32(prot, c + 56); + put_unaligned_le32(prot, c + 60); + memset(c + 72, 0, 16); + strscpy(c + 72, prot == 1 ? "__const" : "__data", 16); + put_unaligned_le32(prot == 1 ? 0x21 : 0x23, c + 72 + 64); +} + +/* + * Surfaces stay in segment order whatever order the tensor descriptors use: + * the add-128 HWX has segments x, y, z (in, out, in) and tensors x, z, y. + */ +static void hwx_surface_order_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx info; + u8 *tensor = kunit_kzalloc(test, 0xd48, GFP_KERNEL), *ref[2], *symbol[2]; + u64 value; + u8 ordinal; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, tensor); + /* Tensors listed out of segment order pair by name. */ + memcpy(tensor, f->bytes + f->tensor[0], 0xd48); + memmove(f->bytes + f->tensor[0], f->bytes + f->tensor[1], 0xd48); + memcpy(f->bytes + f->tensor[1], tensor, 0xd48); + KUNIT_ASSERT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), 0); + KUNIT_EXPECT_STREQ(test, info.surfaces[0].name, "in"); + KUNIT_EXPECT_EQ(test, info.surfaces[0].direction, 1U); + KUNIT_EXPECT_STREQ(test, info.surfaces[1].name, "out"); + KUNIT_EXPECT_EQ(test, info.surfaces[1].direction, 2U); + + /* The output need not be the last segment. */ + f = fixture_create(test, false, false); + KUNIT_ASSERT_NOT_NULL(test, f); + ref[0] = fixture_ref(f, 0); + ref[1] = fixture_ref(f, 1); + KUNIT_ASSERT_NOT_NULL(test, ref[0]); + KUNIT_ASSERT_NOT_NULL(test, ref[1]); + fixture_surface_kind(f, 0, 2); + fixture_surface_kind(f, 1, 1); + memset(ref[0] + 0x18, 0, 8); + memset(ref[1] + 0x18, 0, 8); + strscpy(ref[0] + 0x18, "out", 8); + strscpy(ref[1] + 0x18, "in", 8); + symbol[0] = f->bytes + f->symbols; + symbol[1] = symbol[0] + 16; + value = get_unaligned_le64(symbol[0] + 8); + put_unaligned_le64(get_unaligned_le64(symbol[1] + 8), symbol[0] + 8); + put_unaligned_le64(value, symbol[1] + 8); + ordinal = symbol[0][5]; + symbol[0][5] = symbol[1][5]; + symbol[1][5] = ordinal; + KUNIT_ASSERT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), 0); + KUNIT_EXPECT_STREQ(test, info.surfaces[0].name, "out"); + KUNIT_EXPECT_EQ(test, info.surfaces[0].direction, 2U); + KUNIT_EXPECT_STREQ(test, info.surfaces[1].name, "in"); + KUNIT_EXPECT_EQ(test, info.surfaces[1].direction, 1U); + + /* A reference naming no tensor refuses. */ + strscpy(ref[1] + 0x18, "zz", 8); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); + + /* Exactly one output: two inputs and no output refuse. */ + f = fixture_create(test, false, false); + KUNIT_ASSERT_NOT_NULL(test, f); + fixture_surface_kind(f, 1, 1); + put_unaligned_le32(1, f->bytes + f->tensor[1] + 0x14); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(f->bytes, f->size, &info), -ENOEXEC); +} + static void hwx_program_test(struct kunit *test) { static const u32 fields[] = { @@ -405,16 +296,201 @@ static void hwx_snapshot_test(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, snapshot.bytes, NULL); } +static void hwx_snapshot_free(void *snapshot) +{ + ane_h16_hwx_snapshot_destroy(snapshot); +} + +static struct ane_h16_hwx_snapshot *fixture_snapshot(struct kunit *test, + struct hwx_fixture *f) +{ + struct ane_h16_hwx_snapshot *snapshot; + + if (!f) + return NULL; + snapshot = kunit_kzalloc(test, sizeof(*snapshot), GFP_KERNEL); + if (!snapshot || ane_h16_hwx_snapshot_create(f->bytes, f->size, snapshot)) + return NULL; + if (kunit_add_action_or_reset(test, hwx_snapshot_free, snapshot)) + return NULL; + return snapshot; +} + +static void hwx_load_plan_test(struct kunit *test) +{ + unsigned int mixed, kernel; + + for (mixed = 0; mixed < 2; mixed++) { + for (kernel = 0; kernel < 2; kernel++) { + struct hwx_fixture *f = fixture_create(test, mixed, kernel); + struct ane_h16_hwx_snapshot *snapshot = fixture_snapshot(test, f); + struct ane_h16_load_plan plan; + unsigned int text = 2 + mixed; + + KUNIT_ASSERT_NOT_NULL(test, snapshot); + KUNIT_ASSERT_EQ(test, ane_h16_load_plan_create(snapshot, &plan), 0); + KUNIT_EXPECT_EQ(test, plan.count, 3 + mixed + kernel); + KUNIT_EXPECT_EQ(test, plan.total, (u64)(4 + mixed + kernel) * SZ_16K); + KUNIT_EXPECT_EQ(test, plan.regions[text].kind, ANE_H16_REGION_TEXT); + KUNIT_EXPECT_EQ(test, plan.regions[text].permissions, 5); + KUNIT_EXPECT_EQ(test, plan.regions[text].file_offset, (u64)SZ_16K); + KUNIT_EXPECT_EQ(test, plan.regions[text].file_size, (u64)SZ_32K); + /* One TEXT allocation includes task and constant sections. */ + KUNIT_EXPECT_EQ(test, plan.allocation[text], (u64)SZ_32K); + if (mixed) { + KUNIT_EXPECT_EQ(test, plan.regions[0].kind, ANE_H16_REGION_SCRATCH); + KUNIT_EXPECT_EQ(test, plan.regions[0].file_size, 0); + KUNIT_EXPECT_EQ(test, plan.regions[0].permissions, 3); + } + } + } +} + +static void hwx_load_original_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, false, false); + struct ane_h16_hwx_snapshot *snapshot = fixture_snapshot(test, f); + struct ane_h16_load_plan plan; + + KUNIT_ASSERT_NOT_NULL(test, snapshot); + /* Cached metadata cannot replace the original authenticated container. */ + memset(&snapshot->info, 0xff, sizeof(snapshot->info)); + KUNIT_ASSERT_EQ(test, ane_h16_load_plan_create(snapshot, &plan), 0); + KUNIT_EXPECT_EQ(test, plan.count, 3); + ((u8 *)snapshot->bytes)[f->program] ^= 1; + plan.count = 0x1234; + KUNIT_EXPECT_EQ(test, ane_h16_load_plan_create(snapshot, &plan), -ENOEXEC); + KUNIT_EXPECT_EQ(test, plan.count, 0x1234); +} + +static void fixture_mappings(const struct ane_h16_load_plan *plan, + struct ane_h16_load_mapping *mappings) +{ + unsigned int i; + + for (i = 0; i < plan->count; i++) { + mappings[i].dva = 0x20000000000ULL + (u64)i * SZ_1M; + mappings[i].size = plan->allocation[i]; + mappings[i].permissions = plan->regions[i].permissions; + } +} + +static void hwx_load_resources_test(struct kunit *test) +{ + struct hwx_fixture *f = fixture_create(test, true, true); + struct ane_h16_hwx_snapshot *snapshot = fixture_snapshot(test, f); + struct ane_h16_load_mapping mappings[ANE_H16_REGIONS] = {}; + struct ane_h16_process_binding bindings[2]; + struct ane_h16_resource resources[2] = {}; + struct ane_h16_load_plan plan; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, snapshot); + KUNIT_ASSERT_EQ(test, ane_h16_load_plan_create(snapshot, &plan), 0); + fixture_mappings(&plan, mappings); + for (i = 0; i < 2; i++) { + bindings[i].hwx_vm = snapshot->info.surfaces[i].section.vm; + bindings[i].index = 123 + 11 * i; + bindings[i].class = i; + } + ret = ane_h16_load_resources_create(snapshot, mappings, plan.count, + bindings, 2, resources); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, resources[0].buffer.mapping_dva, mappings[1].dva); + KUNIT_EXPECT_EQ(test, resources[1].buffer.index, 134); + KUNIT_EXPECT_EQ(test, resources[1].permissions, 2); + for (i = 0; i < 7; i++) { + struct ane_h16_load_mapping saved = mappings[1]; + + switch (i) { + case 0: + mappings[1].dva++; + break; + case 1: + mappings[1].size++; + break; + case 2: + mappings[1].permissions = 3; + break; + case 3: + mappings[1].dva = mappings[3].dva; + break; + case 4: + mappings[1].dva = 0x10000000000ULL; + break; + case 5: + mappings[1].dva = U64_MAX - SZ_16K + 1; + break; + case 6: + mappings[1].dva = 0; + break; + } + resources[0].buffer.index = 0x1234; + ret = ane_h16_load_resources_create(snapshot, mappings, plan.count, + bindings, 2, resources); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, resources[0].buffer.index, 0x1234); + mappings[1] = saved; + } + /* Indices are class-local, so input 0 and output 0 may coexist. */ + bindings[1].index = bindings[0].index; + ret = ane_h16_load_resources_create(snapshot, mappings, plan.count, + bindings, 2, resources); + KUNIT_EXPECT_EQ(test, ret, 0); + bindings[1].index++; + bindings[1].class = 8; + ret = ane_h16_load_resources_create(snapshot, mappings, plan.count, + bindings, 2, resources); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + bindings[1].class = 1; + bindings[1].hwx_vm += SZ_16K; + ret = ane_h16_load_resources_create(snapshot, mappings, plan.count, + bindings, 2, resources); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = ane_h16_load_resources_create(snapshot, mappings, plan.count - 1, + bindings, 2, resources); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = ane_h16_load_resources_create(snapshot, mappings, plan.count, + bindings, 1, resources); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void hwx_import_bounds_test(struct kunit *test) +{ + const void __user *invalid = (const void __user *)1UL; + struct ane_h16_hwx_snapshot snapshot = {}; + struct hwx_fixture *f = fixture_create(test, false, false); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, f); + ret = ane_h16_hwx_snapshot_from_user(invalid, U64_MAX, &snapshot); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_snapshot_from_user(invalid, 31, &snapshot), -EINVAL); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_snapshot_from_user(invalid, 32, &snapshot), -EFAULT); + KUNIT_EXPECT_PTR_EQ(test, snapshot.bytes, NULL); + KUNIT_ASSERT_EQ(test, ane_h16_hwx_snapshot_create(f->bytes, f->size, &snapshot), 0); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_snapshot_create(f->bytes, f->size, &snapshot), -EBUSY); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_snapshot_from_user(invalid, 32, &snapshot), -EBUSY); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_snapshot_check(&snapshot), 0); + ane_h16_hwx_snapshot_destroy(&snapshot); +} + static struct kunit_case hwx_cases[] = { KUNIT_CASE(hwx_forms_test), KUNIT_CASE(hwx_header_test), KUNIT_CASE(hwx_command_test), KUNIT_CASE(hwx_tensor_test), + KUNIT_CASE(hwx_surface_order_test), KUNIT_CASE(hwx_program_test), KUNIT_CASE(hwx_geometry_test), KUNIT_CASE(hwx_symbols_test), KUNIT_CASE(hwx_relocations_test), KUNIT_CASE(hwx_snapshot_test), + KUNIT_CASE(hwx_load_plan_test), + KUNIT_CASE(hwx_load_original_test), + KUNIT_CASE(hwx_load_resources_test), + KUNIT_CASE(hwx_import_bounds_test), {} }; diff --git a/drivers/accel/ane/ane_h16_hwx_test_fixture.h b/drivers/accel/ane/ane_h16_hwx_test_fixture.h new file mode 100644 index 0000000000000..fd839d68afc56 --- /dev/null +++ b/drivers/accel/ane/ane_h16_hwx_test_fixture.h @@ -0,0 +1,213 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_HWX_TEST_FIXTURE_H__ +#define __ANE_H16_HWX_TEST_FIXTURE_H__ + +/* Synthetic H16G containers shared by the HWX and program KUnit suites. */ +#include +#include +#include + +struct hwx_fixture { + u8 *bytes; + size_t size, cursor; + u32 commands, program, tensor[2], segment[2], text_command, symtab, reloc; + u32 symbols, strings; +}; + +static u8 *fixture_command(struct hwx_fixture *f, u32 command, u32 length) +{ + u8 *c = f->bytes + f->cursor; + + put_unaligned_le32(command, c); + put_unaligned_le32(length, c + 4); + f->cursor += length; + f->commands++; + return c; +} + +static u8 *fixture_segment(struct hwx_fixture *f, const char *name, u64 vm, + u64 size, u32 offset, u32 prot, u32 flags, u32 sections) +{ + u8 *c = fixture_command(f, 0x19, 72 + sections * 80); + + strscpy(c + 8, name, 16); + put_unaligned_le64(vm, c + 24); + put_unaligned_le64(size, c + 32); + put_unaligned_le64(offset, c + 40); + put_unaligned_le64(offset ? size : 0, c + 48); + put_unaligned_le32(prot, c + 56); + put_unaligned_le32(prot, c + 60); + put_unaligned_le32(sections, c + 64); + put_unaligned_le32(flags, c + 68); + return c + 72; +} + +static void fixture_section(u8 *s, const char *segment, const char *name, + u64 vm, u64 size, u32 offset, u32 alignment, u32 flags) +{ + strscpy(s, name, 16); + strscpy(s + 16, segment, 16); + put_unaligned_le64(vm, s + 32); + put_unaligned_le64(size, s + 40); + put_unaligned_le32(offset, s + 48); + put_unaligned_le32(alignment, s + 52); + put_unaligned_le32(flags, s + 64); +} + +static struct hwx_fixture *fixture_create(struct kunit *test, bool mixed, bool kernel) +{ + struct hwx_fixture *f; + u64 surface = 0x30000000 + (mixed ? SZ_16K : 0); + u64 text = surface + 2 * SZ_16K, kern = text + SZ_32K; + u32 i, strings_length = 1; + u8 *c, *s, *text_section; + static const char * const names[] = { "in", "out" }; + static const char * const symbols[] = { "input", "output" }; + + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + if (!f) + return NULL; + f->size = kernel ? SZ_64K : SZ_32K + SZ_16K; + f->bytes = kunit_kzalloc(test, f->size, GFP_KERNEL); + if (!f->bytes) + return NULL; + f->cursor = 32; + fixture_segment(f, "__PAGEZERO", 0, SZ_16K, 0, 0, 4, 0); + if (mixed) { + s = fixture_segment(f, "__DATA", 0x30000000, SZ_16K, 0, 3, 4, 1); + fixture_section(s, "__DATA", "__bss", 0x30000000, SZ_16K, 0, 6, 0x25); + } + for (i = 0; i < 2; i++) { + f->segment[i] = f->cursor; + s = fixture_segment(f, "__FVMLIB", surface + i * SZ_16K, SZ_16K, + 0, i + 1, 6, 1); + fixture_section(s, "__FVMLIB", i ? "__data" : "__const", + surface + i * SZ_16K, SZ_16K, 0, 14, i ? 0x23 : 0x21); + } + f->text_command = f->cursor; + text_section = fixture_segment(f, "__TEXT", text, SZ_32K, SZ_16K, 5, + mixed ? 4 : 0, 2); + fixture_section(text_section, "__TEXT", "__text", text, 64, SZ_16K, + 14, kernel ? 0x128 : 0x28); + fixture_section(text_section + 80, "__TEXT", "__const", text + 64, + SZ_16K, SZ_16K + 64, 6, 0x26); + if (kernel) { + s = fixture_segment(f, "__KERN_0", kern, SZ_16K, SZ_32K + SZ_16K, 1, 4, 1); + fixture_section(s, "__KERN_0", "__kern_0", kern, 64, + SZ_32K + SZ_16K, 6, 0x26); + } + for (i = 0; i < 2; i++) { + c = fixture_command(f, 0x40, 32); + put_unaligned_le64(0x18, c + 8); + put_unaligned_le64(surface + i * SZ_16K, c + 0x10); + strscpy(c + 0x18, names[i], 8); + } + f->program = f->cursor; + c = fixture_command(f, 4, mixed ? 0x8c0 : 0x8a0); + put_unaligned_le32(4, c + 8); + put_unaligned_le32(mixed ? 0x22a : 0x222, c + 0xc); + put_unaligned_le64(text, c + 0x10); + put_unaligned_le64(text + 64, c + 0x20); + if (mixed) + put_unaligned_le64(0x30000000, c + 0x40); + for (i = 0; i < 2; i++) + put_unaligned_le64(surface + i * SZ_16K, c + 0x70 + i * 16); + if (kernel) + put_unaligned_le64(kern, c + 0x90); + put_unaligned_le64(text, c + 0x810); + put_unaligned_le32(4, c + 0x820); + put_unaligned_le32(16, c + 0x824); + put_unaligned_le32(1, c + 0x830); + put_unaligned_le32(U32_MAX, c + 0x838); + put_unaligned_le32(U32_MAX, c + 0x83c); + put_unaligned_le32(0xffff, c + 0x844); + put_unaligned_le32(mixed ? 0x8b8 : 0x898, c + 0x848); + put_unaligned_le32(0x10, c + 0x860); + put_unaligned_le32(4, c + 0x868); + put_unaligned_le32(mixed ? 4 : 2, c + 0x870); + put_unaligned_le32(mixed ? 2 : 7, c + 0x878); + put_unaligned_le32(8, c + 0x87c); + put_unaligned_le32(mixed ? 256 : 0x461c4000, c + 0x880); + put_unaligned_le32(mixed ? 3 : 9, c + 0x888); + put_unaligned_le32(8, c + 0x88c); + if (mixed) { + put_unaligned_le32(256, c + 0x858); + put_unaligned_le32(0x08020004, c + 0x890); + put_unaligned_le32(8, c + 0x894); + put_unaligned_le32(7, c + 0x898); + put_unaligned_le32(8, c + 0x89c); + put_unaligned_le32(0x461c4000, c + 0x8a0); + put_unaligned_le32(9, c + 0x8a8); + put_unaligned_le32(8, c + 0x8ac); + } + memcpy(c + (mixed ? 0x8b8 : 0x898), "main", 5); + for (i = 0; i < 2; i++) { + u32 shortoff = 0xd3d + strlen(symbols[i]) + 1; + + f->tensor[i] = f->cursor; + c = fixture_command(f, 4, 0xd48); + put_unaligned_le32(3, c + 8); + put_unaligned_le32(0x34a, c + 0xc); + put_unaligned_le32(3, c + 0x10); + put_unaligned_le32(i + 1, c + 0x14); + put_unaligned_le32(0xd38, c + 0x18); + put_unaligned_le32(0xd3d, c + 0x20); + put_unaligned_le32(5, c + 0x24); + put_unaligned_le32(1, c + 0x28); + put_unaligned_le32(1, c + 0x2c); + put_unaligned_le32(1, c + 0x30); + put_unaligned_le32(128, c + 0x34); + put_unaligned_le32(1, c + 0x38); + put_unaligned_le64(256, c + 0x50); + put_unaligned_le64(256, c + 0x58); + put_unaligned_le64(256, c + 0x60); + put_unaligned_le64(2, c + 0x68); + put_unaligned_le64(256, c + 0x70); + put_unaligned_le32(1, c + 0x78); + put_unaligned_le32(shortoff, c + 0x7c); + put_unaligned_le64(256, c + 0x80); + memcpy(c + 0xd38, "main", 5); + memcpy(c + 0xd3d, symbols[i], strlen(symbols[i]) + 1); + memcpy(c + shortoff, names[i], strlen(names[i]) + 1); + } + f->symtab = f->cursor; + c = fixture_command(f, 2, 24); + f->symbols = f->cursor; + f->strings = f->symbols + 2 * 16; + for (i = 0; i < 2; i++) { + u8 *symbol = f->bytes + f->symbols + i * 16; + + put_unaligned_le32(strings_length, symbol); + symbol[4] = 0xf; + symbol[5] = (mixed ? 2 : 1) + i; + put_unaligned_le64(surface + i * SZ_16K, symbol + 8); + memcpy(f->bytes + f->strings + strings_length, names[i], strlen(names[i]) + 1); + strings_length += strlen(names[i]) + 1; + } + put_unaligned_le32(f->symbols, c + 8); + put_unaligned_le32(2, c + 12); + put_unaligned_le32(f->strings, c + 16); + put_unaligned_le32(strings_length, c + 20); + f->reloc = ALIGN(f->strings + strings_length, 8); + if (kernel) { + put_unaligned_le32(f->reloc, text_section + 56); + put_unaligned_le32(1, text_section + 60); + put_unaligned_le32(56, f->bytes + f->reloc); + put_unaligned_le32(0x07000000 | (mixed ? 6 : 5), f->bytes + f->reloc + 4); + put_unaligned_le32(32, f->bytes + SZ_16K + 56); + } + /* Fixed H16 task prologue: u32 1, then twelve zero bytes. */ + put_unaligned_le32(1, f->bytes + SZ_16K); + put_unaligned_le32(11 << 16, f->bytes + SZ_16K + 16); + put_unaligned_le32(0x4100 / 4, f->bytes + SZ_16K + 52); + put_unaligned_le32(0xbeefface, f->bytes); + put_unaligned_le32(0x80, f->bytes + 4); + put_unaligned_le32(7, f->bytes + 8); + put_unaligned_le32(2, f->bytes + 12); + put_unaligned_le32(f->commands, f->bytes + 16); + put_unaligned_le32(f->cursor - 32, f->bytes + 20); + put_unaligned_le32(0x200000, f->bytes + 24); + return f; +} + +#endif /* __ANE_H16_HWX_TEST_FIXTURE_H__ */ diff --git a/drivers/accel/ane/ane_h16_load.c b/drivers/accel/ane/ane_h16_load.c new file mode 100644 index 0000000000000..66a41c7eeb8e4 --- /dev/null +++ b/drivers/accel/ane/ane_h16_load.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include + +#include "ane_h16_load.h" + +static int h16_load_plan(const struct ane_h16_hwx *hwx, struct ane_h16_load_plan *plan) +{ + struct ane_h16_load_plan next = {}; + unsigned int i; + + if (!hwx->region_count || hwx->region_count > ANE_H16_REGIONS) + return -ENOEXEC; + next.count = hwx->region_count; + for (i = 0; i < next.count; i++) { + const struct ane_h16_hwx_region *r = &hwx->regions[i]; + u64 rounded; + + if (!r->size || check_add_overflow(r->size, (u64)SZ_16K - 1, &rounded)) + return -ENOEXEC; + next.allocation[i] = round_down(rounded, SZ_16K); + if (check_add_overflow(next.total, next.allocation[i], &next.total) || + next.total > ANE_H16_LOAD_BYTES_MAX) + return -E2BIG; + next.regions[i] = *r; + } + *plan = next; + return 0; +} + +int ane_h16_load_plan_create(const struct ane_h16_hwx_snapshot *snapshot, + struct ane_h16_load_plan *plan) +{ + struct ane_h16_hwx hwx; + int ret; + + if (!plan) + return -EINVAL; + ret = ane_h16_hwx_snapshot_check(snapshot); + if (ret) + return ret; + /* Reparse the authenticated original, not caller-supplied cached metadata. */ + ret = ane_h16_hwx_validate(snapshot->bytes, snapshot->size, &hwx); + if (ret) + return ret; + return h16_load_plan(&hwx, plan); +} +EXPORT_SYMBOL_GPL(ane_h16_load_plan_create); + +int ane_h16_load_resources_create(const struct ane_h16_hwx_snapshot *snapshot, + const struct ane_h16_load_mapping *mappings, + unsigned int mapping_count, + const struct ane_h16_process_binding *bindings, + unsigned int binding_count, + struct ane_h16_resource *resources) +{ + struct ane_h16_resource next[ANE_H16_SURFACES] = {}; + struct ane_h16_load_plan plan; + struct ane_h16_hwx hwx; + unsigned int i, j; + int ret; + + if (!mappings || !bindings || !resources) + return -EINVAL; + ret = ane_h16_hwx_snapshot_check(snapshot); + if (ret) + return ret; + ret = ane_h16_hwx_validate(snapshot->bytes, snapshot->size, &hwx); + if (ret) + return ret; + ret = h16_load_plan(&hwx, &plan); + if (ret) + return ret; + if (mapping_count != plan.count || binding_count != hwx.surface_count) + return -EINVAL; + for (i = 0; i < mapping_count; i++) { + const struct ane_h16_load_mapping *m = &mappings[i]; + u64 end; + + if (!m->dva || !IS_ALIGNED(m->dva, SZ_16K) || + m->size != plan.allocation[i] || + m->permissions != plan.regions[i].permissions || + check_add_overflow(m->dva, m->size, &end) || + (m->dva < 0x1000036c000ULL && end > 0x10000000000ULL)) + return -EINVAL; + for (j = 0; j < i; j++) + if (m->dva < mappings[j].dva + mappings[j].size && + mappings[j].dva < end) + return -EINVAL; + } + for (i = 0; i < binding_count; i++) { + const struct ane_h16_hwx_surface *s = &hwx.surfaces[i]; + const struct ane_h16_process_binding *b = &bindings[i]; + + if (b->hwx_vm != s->section.vm || b->class != s->direction - 1) + return -EINVAL; + for (j = 0; j < plan.count; j++) + if (plan.regions[j].kind == ANE_H16_REGION_SURFACE && + plan.regions[j].vm == b->hwx_vm) + break; + if (j == plan.count) + return -EINVAL; + next[i].hwx_vm = b->hwx_vm; + next[i].permissions = s->direction; + next[i].buffer = (struct ane_h16_execute_buffer) { + .index = b->index, + .class = b->class, + .mapping_dva = mappings[j].dva, + .mapping_size = mappings[j].size, + .size = s->span, + }; + } + ret = ane_h16_resources_validate(&hwx, next, binding_count); + if (!ret) + memcpy(resources, next, binding_count * sizeof(*resources)); + return ret; +} +EXPORT_SYMBOL_GPL(ane_h16_load_resources_create); + +MODULE_DESCRIPTION("H16 original HWX load planning and resource geometry"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_load.h b/drivers/accel/ane/ane_h16_load.h new file mode 100644 index 0000000000000..ba55b5838525b --- /dev/null +++ b/drivers/accel/ane/ane_h16_load.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_LOAD_H__ +#define __ANE_H16_LOAD_H__ + +#include "ane_h16_resources.h" + +/* Conservative host budget; firmware resource replies cannot enlarge it. */ +#define ANE_H16_LOAD_BYTES_MAX SZ_512M + +struct ane_h16_load_plan { + struct ane_h16_hwx_region regions[ANE_H16_REGIONS]; + u64 allocation[ANE_H16_REGIONS]; + u64 total; + u32 count; +}; + +struct ane_h16_load_mapping { + u64 dva; + u64 size; + u32 permissions; +}; + +/* Only the backend may resolve these bindings from the validated program + * contract. CREATE_PROCESS reply IDs do not describe resource bindings. + */ +struct ane_h16_process_binding { + u64 hwx_vm; + u32 index; + u32 class; + /* Buffer size the generic section declared at LOAD. */ + u64 bytes; +}; + +/* Geometry helpers do not prove DMA ownership or install a process. All + * mappings must be backend-owned and private, with the stated permissions. + * Wire serialization and process installation are separate backend operations. + */ +int ane_h16_load_plan_create(const struct ane_h16_hwx_snapshot *snapshot, + struct ane_h16_load_plan *plan); +int ane_h16_load_resources_create(const struct ane_h16_hwx_snapshot *snapshot, + const struct ane_h16_load_mapping *mappings, + unsigned int mapping_count, + const struct ane_h16_process_binding *bindings, + unsigned int binding_count, + struct ane_h16_resource *resources); + +#endif /* __ANE_H16_LOAD_H__ */ diff --git a/drivers/accel/ane/ane_h16_pending_test.c b/drivers/accel/ane/ane_h16_pending_test.c index 891342b9a7a37..bff31e3d9ca39 100644 --- a/drivers/accel/ane/ane_h16_pending_test.c +++ b/drivers/accel/ane/ane_h16_pending_test.c @@ -246,27 +246,30 @@ static void pending_deadline_wrap_test(struct kunit *test) static void resources_test(struct kunit *test) { - struct ane_h16_hwx hwx = { .surface_count = 2 }; - struct ane_h16_resource resources[2] = {}; + /* add-128 shape: input, output, input with class-local indices 0, 0, 1. */ + static const u32 direction[3] = { 1, 2, 1 }, index[3] = { 0, 0, 1 }; + struct ane_h16_hwx hwx = { .surface_count = 3 }; + struct ane_h16_resource resources[3] = {}; unsigned int i, variant; - for (i = 0; i < 2; i++) { + for (i = 0; i < 3; i++) { hwx.surfaces[i].section.vm = 0x30000000 + i * SZ_16K; hwx.surfaces[i].section.size = SZ_16K; hwx.surfaces[i].allocation = SZ_16K; hwx.surfaces[i].span = 256; - hwx.surfaces[i].direction = i + 1; + hwx.surfaces[i].direction = direction[i]; resources[i].hwx_vm = hwx.surfaces[i].section.vm; - resources[i].permissions = i + 1; + resources[i].permissions = direction[i]; resources[i].buffer = (struct ane_h16_execute_buffer) { - .index = i, .class = i, .mapping_dva = 0x100000000ULL + i * SZ_16K, + .index = index[i], .class = direction[i] - 1, + .mapping_dva = 0x100000000ULL + i * SZ_16K, .mapping_size = SZ_16K, .size = 256, }; } - KUNIT_EXPECT_EQ(test, ane_h16_resources_validate(&hwx, resources, 2), 0); + KUNIT_EXPECT_EQ(test, ane_h16_resources_validate(&hwx, resources, 3), 0); KUNIT_EXPECT_EQ(test, ane_h16_resources_validate(&hwx, resources, 1), -EINVAL); for (variant = 0; variant < 9; variant++) { - struct ane_h16_resource bad[2]; + struct ane_h16_resource bad[3]; memcpy(bad, resources, sizeof(bad)); switch (variant) { @@ -289,7 +292,8 @@ static void resources_test(struct kunit *test) bad[0].buffer.mapping_dva = U64_MAX - SZ_16K + 1; break; case 6: - bad[1].buffer.index = bad[0].buffer.index; + /* Same class and index: two inputs cannot share index 0. */ + bad[2].buffer.index = bad[0].buffer.index; break; case 7: bad[1].buffer.mapping_dva = bad[0].buffer.mapping_dva; @@ -298,7 +302,7 @@ static void resources_test(struct kunit *test) bad[0].buffer.mapping_size--; break; } - KUNIT_EXPECT_EQ(test, ane_h16_resources_validate(&hwx, bad, 2), -EINVAL); + KUNIT_EXPECT_EQ(test, ane_h16_resources_validate(&hwx, bad, 3), -EINVAL); } } diff --git a/drivers/accel/ane/ane_h16_preload.c b/drivers/accel/ane/ane_h16_preload.c new file mode 100644 index 0000000000000..03bfbb021e3e1 --- /dev/null +++ b/drivers/accel/ane/ane_h16_preload.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include + +#include "ane_h16_preload.h" + +/* + * A rule is data, so check its shape before trusting it: records in DATA + * order, each record header and value inside DATA and clear of the previous + * value, and the rewritten range after the last value. + */ +static bool ane_h16_preload_rule_valid(const struct ane_h16_preload_rule *rule) +{ + u32 next = 8; + unsigned int i; + + if (!rule->cells || !rule->cell_count) + return false; + for (i = 0; i < rule->cell_count; i++) { + const struct ane_h16_preload_cell *c = &rule->cells[i]; + + if (!c->width || c->value < next || + c->value + c->width > ANE_H16_PRELOAD_DATA_SIZE) + return false; + next = c->value + c->width + 8; + } + return rule->zero_start >= next - 8 && rule->zero_start <= rule->zero_end && + rule->zero_end <= ANE_H16_PRELOAD_DATA_SIZE; +} + +static void ane_h16_hash_zeros(struct sha256_ctx *ctx, size_t length) +{ + static const u8 zero[64]; + + while (length) { + size_t chunk = min(length, sizeof(zero)); + + sha256_update(ctx, zero, chunk); + length -= chunk; + } +} + +int ane_h16_preload_digest(const struct ane_h16_preload_rule *rule, + const void *text, size_t text_size, + const void *data, size_t data_size, + u8 digest[SHA256_DIGEST_SIZE]) +{ + const u8 *bytes = data; + struct sha256_ctx ctx; + size_t cursor = 0; + unsigned int i; + + if (!rule || !text || !data || !digest || text_size != ANE_H16_PRELOAD_TEXT_SIZE || + data_size != ANE_H16_PRELOAD_DATA_SIZE || !ane_h16_preload_rule_valid(rule)) + return -EINVAL; + /* Validate all framing before producing a digest. Odd offsets are valid. */ + for (i = 0; i < rule->cell_count; i++) { + const struct ane_h16_preload_cell *c = &rule->cells[i]; + + if (memcmp(bytes + c->value - 8, c->tag, 4) || + get_unaligned_le32(bytes + c->value - 4) != c->width) + return -ENOEXEC; + } + sha256_init(&ctx); + sha256_update(&ctx, text, text_size); + for (i = 0; i < rule->cell_count; i++) { + const struct ane_h16_preload_cell *c = &rule->cells[i]; + + if (!c->mutable) + continue; + sha256_update(&ctx, bytes + cursor, c->value - cursor); + ane_h16_hash_zeros(&ctx, c->width); + cursor = c->value + c->width; + } + sha256_update(&ctx, bytes + cursor, rule->zero_start - cursor); + ane_h16_hash_zeros(&ctx, rule->zero_end - rule->zero_start); + cursor = rule->zero_end; + sha256_update(&ctx, bytes + cursor, data_size - cursor); + sha256_final(&ctx, digest); + return 0; +} +EXPORT_SYMBOL_GPL(ane_h16_preload_digest); + +int ane_h16_preload_validate(const struct ane_h16_preload_rule *rule, + const void *text, size_t text_size, + const void *data, size_t data_size) +{ + u8 digest[SHA256_DIGEST_SIZE]; + int ret; + + ret = ane_h16_preload_digest(rule, text, text_size, data, data_size, digest); + if (ret) + return ret; + return memcmp(digest, rule->digest, sizeof(digest)) ? -ENOEXEC : 0; +} +EXPORT_SYMBOL_GPL(ane_h16_preload_validate); + +MODULE_DESCRIPTION("H16 resident firmware identity validation"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_preload.h b/drivers/accel/ane/ane_h16_preload.h new file mode 100644 index 0000000000000..efec3e38bbd1d --- /dev/null +++ b/drivers/accel/ane/ane_h16_preload.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_PRELOAD_H__ +#define __ANE_H16_PRELOAD_H__ + +#include + +#define ANE_H16_PRELOAD_TEXT_SIZE 0xc0000 +#define ANE_H16_PRELOAD_DATA_SIZE 0x2ac000 + +/* + * One aether patchbay record in DATA: a four-byte tag and a 32-bit width, + * followed by the value cell at @value. A mutable cell is written by the + * bootloader and is normalized to zero before hashing. + */ +struct ane_h16_preload_cell { + char tag[4]; + u32 value; + u8 width; + bool mutable; +}; + +/* + * How to recognize one firmware build from its iBoot preload: the ordered + * patchbay records, the DATA range the bootloader rewrites (@zero_start up to + * @zero_end, normalized to zero), and the SHA-256 of the normalized TEXT and + * DATA. Adding a firmware build is a new rule, not new code. + */ +struct ane_h16_preload_rule { + const struct ane_h16_preload_cell *cells; + unsigned int cell_count; + u32 zero_start; + u32 zero_end; + u8 digest[SHA256_DIGEST_SIZE]; +}; + +/* Inputs are stable, host-owned snapshots of the reserved live pages. + * Computing a digest validates the rule's framing but not the identity. + * Neither function changes the snapshots or the live bootloader values. + */ +int ane_h16_preload_digest(const struct ane_h16_preload_rule *rule, + const void *text, size_t text_size, + const void *data, size_t data_size, + u8 digest[SHA256_DIGEST_SIZE]); +/* 0 when the snapshots are the rule's build, -ENOEXEC when they are not. */ +int ane_h16_preload_validate(const struct ane_h16_preload_rule *rule, + const void *text, size_t text_size, + const void *data, size_t data_size); + +#endif /* __ANE_H16_PRELOAD_H__ */ diff --git a/drivers/accel/ane/ane_h16_preload_test.c b/drivers/accel/ane/ane_h16_preload_test.c new file mode 100644 index 0000000000000..184b444a539e1 --- /dev/null +++ b/drivers/accel/ane/ane_h16_preload_test.c @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include + +#include "ane_h16_boot_data.h" +#include "ane_h16_preload.h" + +/* Synthetic framing only, with no firmware bytes. Walk records independently + * from their widths rather than sharing the production offset manifest. + */ +static const struct { + char tag[4]; + u8 width; + bool mutable; +} records[] = { + { "BECA", 8, true }, { "SECA", 8, false }, { "GKTS", 8, true }, + { "otcd", 8, false }, { "oscd", 8, false }, { "oecd", 8, false }, + { "DILS", 8, true }, { "SRSA", 8, false }, { "GFCM", 8, false }, + { "SSSC", 8, true }, { "qF8v", 4, true }, { "LRSD", 1, false }, + { "SVSD", 8, false }, { "LCSD", 8, false }, { "ZSTR", 4, false }, + { "BPTP", 8, false }, { "TNGI", 1, false }, { "oeNS", 1, false }, + { "SEBC", 8, false }, { "ECAP", 8, true }, { "BVTP", 8, true }, + { "sP1T", 8, true }, { "BtpG", 8, true }, { "StpG", 8, false }, + { "BlpP", 8, true }, { "SPTP", 8, false }, { "SZSD", 8, false }, + { "LZSD", 8, false }, { "S2xG", 8, true }, { "SNUT", 8, true }, + { "ZNUT", 4, false }, { "OTTR", 8, true }, { "LLTR", 1, false }, + { "_COS", 4, true }, { "RCOS", 4, true }, { "dApC", 8, true }, + { "dArW", 8, true }, { "fVED", 1, true }, { "GLFp", 8, true }, + { "ABOI", 8, false }, { "ZSOI", 4, false }, { "ARcM", 4, true }, +}; + +struct preload_fixture { + u8 *text; + u8 *data; + struct ane_h16_preload_cell cells[ARRAY_SIZE(records)]; + struct ane_h16_preload_rule rule; +}; + +/* Lay the synthetic records out from @base and describe them as a rule. */ +static size_t preload_frame(struct preload_fixture *f, size_t base, bool write) +{ + size_t pos = base; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(records); i++) { + if (write) { + memcpy(f->data + pos, records[i].tag, 4); + put_unaligned_le32(records[i].width, f->data + pos + 4); + } + memcpy(f->cells[i].tag, records[i].tag, 4); + f->cells[i].value = pos + 8; + f->cells[i].width = records[i].width; + f->cells[i].mutable = records[i].mutable; + pos += 8 + records[i].width; + } + f->rule.cells = f->cells; + f->rule.cell_count = ARRAY_SIZE(records); + f->rule.zero_start = 0x17c80; + f->rule.zero_end = 0x18320; + return pos; +} + +static void preload_free(void *allocation) +{ + kvfree(allocation); +} + +static int preload_fixture_init(struct kunit *test) +{ + struct preload_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + + if (!f) + return -ENOMEM; + test->priv = f; + f->text = kvmalloc(ANE_H16_PRELOAD_TEXT_SIZE, GFP_KERNEL); + if (!f->text) + return -ENOMEM; + if (kunit_add_action_or_reset(test, preload_free, f->text)) + return -ENOMEM; + f->data = kvmalloc(ANE_H16_PRELOAD_DATA_SIZE, GFP_KERNEL); + if (!f->data) + return -ENOMEM; + if (kunit_add_action_or_reset(test, preload_free, f->data)) + return -ENOMEM; + memset(f->text, 0x5a, ANE_H16_PRELOAD_TEXT_SIZE); + memset(f->data, 0xa5, ANE_H16_PRELOAD_DATA_SIZE); + return preload_frame(f, 0x5a18, true) == 0x5c79 ? 0 : -EINVAL; +} + +static int preload_digest(struct preload_fixture *f, u8 *digest) +{ + return ane_h16_preload_digest(&f->rule, f->text, ANE_H16_PRELOAD_TEXT_SIZE, + f->data, ANE_H16_PRELOAD_DATA_SIZE, digest); +} + +static void preload_identity_test(struct kunit *test) +{ + /* Independently normalized synthetic bytes, SHA-256 from Python hashlib. */ + static const u8 expected[SHA256_DIGEST_SIZE] = { + 0x67, 0xb5, 0x33, 0xc6, 0x0b, 0x11, 0x03, 0x2b, + 0x3e, 0x75, 0x27, 0x7b, 0x2d, 0xe1, 0x82, 0xc6, + 0xca, 0x53, 0xa7, 0x18, 0x1f, 0xfc, 0xcc, 0xfe, + 0xd3, 0xb5, 0xe8, 0x22, 0xaa, 0x12, 0xd0, 0xbb, + }; + struct preload_fixture *f = test->priv; + u8 digest[SHA256_DIGEST_SIZE]; + int ret; + + KUNIT_EXPECT_EQ(test, preload_digest(f, digest), 0); + KUNIT_EXPECT_MEMEQ(test, digest, expected, sizeof(digest)); + /* Correct framing alone cannot authenticate a synthetic image. */ + ret = ane_h16_preload_validate(&f->rule, f->text, ANE_H16_PRELOAD_TEXT_SIZE, + f->data, ANE_H16_PRELOAD_DATA_SIZE); + KUNIT_EXPECT_EQ(test, ret, -ENOEXEC); + /* The rule's own digest is the whole identity. */ + memcpy(f->rule.digest, expected, sizeof(expected)); + ret = ane_h16_preload_validate(&f->rule, f->text, ANE_H16_PRELOAD_TEXT_SIZE, + f->data, ANE_H16_PRELOAD_DATA_SIZE); + KUNIT_EXPECT_EQ(test, ret, 0); +} + +static void preload_mutable_test(struct kunit *test) +{ + struct preload_fixture *f = test->priv; + u8 baseline[SHA256_DIGEST_SIZE], digest[SHA256_DIGEST_SIZE]; + size_t pos = 0x5a18; + unsigned int i, mutable = 0; + + KUNIT_ASSERT_EQ(test, preload_digest(f, baseline), 0); + for (i = 0; i < ARRAY_SIZE(records); i++) { + if (records[i].mutable) { + memset(f->data + pos + 8, 0x3c, records[i].width); + KUNIT_ASSERT_EQ(test, preload_digest(f, digest), 0); + KUNIT_EXPECT_MEMEQ(test, digest, baseline, sizeof(digest)); + mutable++; + } + pos += 8 + records[i].width; + } + KUNIT_EXPECT_EQ(test, mutable, 20); + memset(f->data + 0x17c80, 0x7e, 0x6a0); + KUNIT_ASSERT_EQ(test, preload_digest(f, digest), 0); + KUNIT_EXPECT_MEMEQ(test, digest, baseline, sizeof(digest)); + /* Validation and normalization never erase the accepted boot data. */ + KUNIT_EXPECT_EQ(test, f->data[0x5a20], 0x3c); + KUNIT_EXPECT_EQ(test, f->data[0x17c80], 0x7e); + KUNIT_EXPECT_EQ(test, f->data[0x1831f], 0x7e); +} + +static void preload_framing_test(struct kunit *test) +{ + struct preload_fixture *f = test->priv; + u8 digest[SHA256_DIGEST_SIZE]; + size_t pos = 0x5a18; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(records); i++) { + f->data[pos] ^= 1; + KUNIT_EXPECT_EQ(test, preload_digest(f, digest), -ENOEXEC); + f->data[pos] ^= 1; + put_unaligned_le32(U32_MAX, f->data + pos + 4); + KUNIT_EXPECT_EQ(test, preload_digest(f, digest), -ENOEXEC); + put_unaligned_le32(records[i].width, f->data + pos + 4); + pos += 8 + records[i].width; + } + KUNIT_EXPECT_EQ(test, preload_digest(f, digest), 0); +} + +static void preload_immutable_test(struct kunit *test) +{ + static const u32 boundaries[] = { + 0, 0x5a17, 0x5c79, 0x17c7f, 0x18320, 0xb8000, 0x2abfff, + }; + struct preload_fixture *f = test->priv; + u8 baseline[SHA256_DIGEST_SIZE], digest[SHA256_DIGEST_SIZE]; + size_t pos = 0x5a18; + unsigned int i; + + KUNIT_ASSERT_EQ(test, preload_digest(f, baseline), 0); + for (i = 0; i < ARRAY_SIZE(records); i++) { + if (!records[i].mutable) { + f->data[pos + 8] ^= 1; + KUNIT_ASSERT_EQ(test, preload_digest(f, digest), 0); + KUNIT_EXPECT_NE(test, memcmp(digest, baseline, sizeof(digest)), 0); + f->data[pos + 8] ^= 1; + } + pos += 8 + records[i].width; + } + for (i = 0; i < ARRAY_SIZE(boundaries); i++) { + f->data[boundaries[i]] ^= 1; + KUNIT_ASSERT_EQ(test, preload_digest(f, digest), 0); + KUNIT_EXPECT_NE(test, memcmp(digest, baseline, sizeof(digest)), 0); + f->data[boundaries[i]] ^= 1; + } + f->text[ANE_H16_PRELOAD_TEXT_SIZE - 1] ^= 1; + KUNIT_ASSERT_EQ(test, preload_digest(f, digest), 0); + KUNIT_EXPECT_NE(test, memcmp(digest, baseline, sizeof(digest)), 0); +} + +static void preload_bounds_test(struct kunit *test) +{ + struct preload_fixture *f = test->priv; + u8 digest[SHA256_DIGEST_SIZE]; + int delta, ret; + + for (delta = -1; delta <= 1; delta += 2) { + ret = ane_h16_preload_digest(&f->rule, f->text, ANE_H16_PRELOAD_TEXT_SIZE + delta, + f->data, ANE_H16_PRELOAD_DATA_SIZE, digest); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = ane_h16_preload_digest(&f->rule, f->text, ANE_H16_PRELOAD_TEXT_SIZE, + f->data, ANE_H16_PRELOAD_DATA_SIZE + delta, digest); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + } + ret = ane_h16_preload_digest(&f->rule, NULL, ANE_H16_PRELOAD_TEXT_SIZE, + f->data, ANE_H16_PRELOAD_DATA_SIZE, digest); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = ane_h16_preload_digest(&f->rule, f->text, ANE_H16_PRELOAD_TEXT_SIZE, + NULL, ANE_H16_PRELOAD_DATA_SIZE, digest); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = ane_h16_preload_digest(&f->rule, f->text, ANE_H16_PRELOAD_TEXT_SIZE, + f->data, ANE_H16_PRELOAD_DATA_SIZE, NULL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = ane_h16_preload_digest(NULL, f->text, ANE_H16_PRELOAD_TEXT_SIZE, + f->data, ANE_H16_PRELOAD_DATA_SIZE, digest); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +/* A malformed rule is a driver bug: refused as -EINVAL, never as a mismatch. */ +static void preload_rule_shape_test(struct kunit *test) +{ + struct preload_fixture *f = test->priv; + u8 digest[SHA256_DIGEST_SIZE]; + struct ane_h16_preload_rule rule; + + rule = f->rule; + rule.cell_count = 0; + KUNIT_EXPECT_EQ(test, ane_h16_preload_digest(&rule, f->text, ANE_H16_PRELOAD_TEXT_SIZE, + f->data, ANE_H16_PRELOAD_DATA_SIZE, digest), + -EINVAL); + rule = f->rule; + rule.zero_start = 0x5c70; /* inside the last record */ + KUNIT_EXPECT_EQ(test, ane_h16_preload_digest(&rule, f->text, ANE_H16_PRELOAD_TEXT_SIZE, + f->data, ANE_H16_PRELOAD_DATA_SIZE, digest), + -EINVAL); + rule = f->rule; + rule.zero_end = ANE_H16_PRELOAD_DATA_SIZE + 1; + KUNIT_EXPECT_EQ(test, ane_h16_preload_digest(&rule, f->text, ANE_H16_PRELOAD_TEXT_SIZE, + f->data, ANE_H16_PRELOAD_DATA_SIZE, digest), + -EINVAL); + /* Overlapping records. */ + f->cells[1].value = f->cells[0].value + 4; + KUNIT_EXPECT_EQ(test, preload_digest(f, digest), -EINVAL); +} + +/* + * A later firmware build moves the patchbay. The rule for one layout rejects + * the other as a mismatch (-ENOEXEC), so a profile table can try each rule in + * turn and the right one wins. + */ +static void preload_layout_select_test(struct kunit *test) +{ + struct preload_fixture *f = test->priv; + struct ane_h16_preload_rule old_rule; + struct ane_h16_preload_cell old_cells[ARRAY_SIZE(records)]; + u8 digest[SHA256_DIGEST_SIZE]; + + memcpy(old_cells, f->cells, sizeof(old_cells)); + old_rule = f->rule; + old_rule.cells = old_cells; + memset(f->data + 0x5a18, 0xa5, 0x5c79 - 0x5a18); + KUNIT_ASSERT_EQ(test, preload_frame(f, 0x5a18 + 0x3f80, true), 0x5c79 + 0x3f80); + f->rule.zero_start += 0x3f80; + f->rule.zero_end += 0x3f80; + KUNIT_EXPECT_EQ(test, ane_h16_preload_digest(&old_rule, f->text, + ANE_H16_PRELOAD_TEXT_SIZE, f->data, + ANE_H16_PRELOAD_DATA_SIZE, digest), + -ENOEXEC); + KUNIT_ASSERT_EQ(test, preload_digest(f, digest), 0); + memcpy(f->rule.digest, digest, sizeof(digest)); + KUNIT_EXPECT_EQ(test, ane_h16_preload_validate(&f->rule, f->text, + ANE_H16_PRELOAD_TEXT_SIZE, f->data, + ANE_H16_PRELOAD_DATA_SIZE), + 0); + /* A mutable value of the new layout still normalizes. */ + memset(f->data + f->cells[0].value, 0x3c, f->cells[0].width); + KUNIT_EXPECT_EQ(test, ane_h16_preload_validate(&f->rule, f->text, + ANE_H16_PRELOAD_TEXT_SIZE, f->data, + ANE_H16_PRELOAD_DATA_SIZE), + 0); +} + +/* + * Every shipped profile is well formed: a build to boot with, a rule whose + * records frame cleanly when written at their offsets, and an identity no + * other profile shares. Synthetic bytes never authenticate as a profile. + */ +static void preload_profiles_test(struct kunit *test) +{ + struct preload_fixture *f = test->priv; + unsigned int i, j, k; + + KUNIT_ASSERT_GT(test, ARRAY_SIZE(ane_h16_profiles), 0); + for (i = 0; i < ARRAY_SIZE(ane_h16_profiles); i++) { + const struct ane_h16_profile *p = &ane_h16_profiles[i]; + const struct ane_h16_preload_rule *r = p->preload; + u8 digest[SHA256_DIGEST_SIZE]; + + KUNIT_ASSERT_NOT_NULL(test, p->build); + KUNIT_ASSERT_NOT_NULL(test, p->identity); + KUNIT_ASSERT_NOT_NULL(test, r); + memset(f->data, 0xa5, ANE_H16_PRELOAD_DATA_SIZE); + for (k = 0; k < r->cell_count; k++) { + memcpy(f->data + r->cells[k].value - 8, r->cells[k].tag, 4); + put_unaligned_le32(r->cells[k].width, f->data + r->cells[k].value - 4); + } + KUNIT_EXPECT_EQ(test, ane_h16_preload_digest(r, f->text, + ANE_H16_PRELOAD_TEXT_SIZE, f->data, + ANE_H16_PRELOAD_DATA_SIZE, digest), + 0); + KUNIT_EXPECT_EQ(test, ane_h16_preload_validate(r, f->text, + ANE_H16_PRELOAD_TEXT_SIZE, f->data, + ANE_H16_PRELOAD_DATA_SIZE), + -ENOEXEC); + for (j = 0; j < i; j++) { + KUNIT_EXPECT_NE(test, memcmp(r->digest, ane_h16_profiles[j].preload->digest, + SHA256_DIGEST_SIZE), 0); + KUNIT_EXPECT_STRNEQ(test, p->identity, ane_h16_profiles[j].identity); + } + } +} + +static struct kunit_case preload_cases[] = { + KUNIT_CASE(preload_identity_test), + KUNIT_CASE(preload_mutable_test), + KUNIT_CASE(preload_framing_test), + KUNIT_CASE(preload_immutable_test), + KUNIT_CASE(preload_bounds_test), + KUNIT_CASE(preload_rule_shape_test), + KUNIT_CASE(preload_layout_select_test), + KUNIT_CASE(preload_profiles_test), + {} +}; + +static struct kunit_suite preload_suite = { + .name = "ane-h16-preload", + .init = preload_fixture_init, + .test_cases = preload_cases, +}; + +kunit_test_suite(preload_suite); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_program.c b/drivers/accel/ane/ane_h16_program.c new file mode 100644 index 0000000000000..c62347ff0fd9e --- /dev/null +++ b/drivers/accel/ane/ane_h16_program.c @@ -0,0 +1,1203 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* H16 program lifecycle on the shared firmware transport. */ +#include +#include +#include +#include + +#include "ane_h16_program.h" + +/* + * The one LOAD slot table. Slot 1 carries the HWX kernel section, or the + * __TEXT,__const constants when the program has no kernel section, and slot 2 + * the task stream. macOS populated slot 1 in every captured load, with at + * least 0x4000 bytes at a sub-page DVA, which is the constants section of a + * program without a kernel. Slots 1 and 2 carry a per-program pair with + * slot 1 = slot 2 + 1. A later correction to any slot is one line here. + */ +static const struct ane_h16_slot_rule ane_h16_slot_rules[ANE_H16_LOAD_SLOTS] = { + { ANE_H16_SLOT_AUX_GENERIC, ANE_H16_RESOURCE_GENERIC, 4 }, + { ANE_H16_SLOT_KERNEL, ANE_H16_RESOURCE_KERNEL, 3 }, + { ANE_H16_SLOT_TEXT, ANE_H16_RESOURCE_TEXT, 4 }, + { ANE_H16_SLOT_AUX_OPERATION, ANE_H16_RESOURCE_OPERATION, 4 }, + { ANE_H16_SLOT_AUX_PROCEDURE, ANE_H16_RESOURCE_PROCEDURE, 4 }, + { ANE_H16_SLOT_EMPTY, 0, 0 }, + { ANE_H16_SLOT_AUX_TEXT_PROPERTIES, ANE_H16_RESOURCE_TEXT_PROPERTIES, 4 }, + { ANE_H16_SLOT_EMPTY, 0, 0 }, + { ANE_H16_SLOT_EMPTY, 0, 0 }, +}; + +const struct ane_h16_slot_rule *ane_h16_slot_rule(unsigned int slot) +{ + return slot < ANE_H16_LOAD_SLOTS ? &ane_h16_slot_rules[slot] : NULL; +} +EXPORT_SYMBOL_GPL(ane_h16_slot_rule); + +/* + * Place a section that lies wholly inside one mapped region of that kind. + * With @pages the slot extends to the end of the section's last 16 KiB page + * of the region, the form every measured H17 kernel-table LOAD used; the + * tail is the zero padding of the mapped segment. + */ +static int h16_section_dva(const struct ane_h16_load_plan *plan, + const struct ane_h16_load_layout *layout, + enum ane_h16_region_kind kind, + const struct ane_h16_hwx_section *section, + struct ane_h16_load_slot *slot, bool pages) +{ + unsigned int i; + + if (!section->size) + return -ENOEXEC; + for (i = 0; i < plan->count; i++) { + const struct ane_h16_hwx_region *r = &plan->regions[i]; + u64 offset = section->vm - r->vm; + + if (r->kind != kind || section->vm < r->vm || offset >= r->size || + section->size > r->size - offset) + continue; + if (offset > layout->regions[i].size || + section->size > layout->regions[i].size - offset) + return -EINVAL; + slot->dva = layout->regions[i].dva + offset; + slot->size = section->size; + if (pages) + slot->size = min_t(u64, ALIGN(offset + section->size, SZ_16K), + layout->regions[i].size) - offset; + return 0; + } + return -ENOEXEC; +} + +int ane_h16_load_slots(const struct ane_h16_hwx *hwx, + const struct ane_h16_load_plan *plan, + const struct ane_h16_load_layout *layout, + struct ane_h16_load_slot slots[ANE_H16_LOAD_SLOTS]) +{ + struct ane_h16_load_slot next[ANE_H16_LOAD_SLOTS] = {}; + unsigned int i, aux; + int ret; + + if (!hwx || !plan || !layout || !slots || !plan->count || + layout->region_count != plan->count) + return -EINVAL; + for (i = 0; i < plan->count; i++) + if (!layout->regions[i].dva || layout->regions[i].size != plan->allocation[i]) + return -EINVAL; + for (i = 0; i < ANE_H16_LOAD_SLOTS; i++) { + const struct ane_h16_slot_rule *rule = &ane_h16_slot_rules[i]; + struct ane_h16_load_slot *slot = &next[i]; + + switch (rule->source) { + case ANE_H16_SLOT_EMPTY: + continue; + case ANE_H16_SLOT_KERNEL: + if (hwx->kernel.size) + ret = h16_section_dva(plan, layout, ANE_H16_REGION_KERNEL, + &hwx->kernel, slot, true); + else + ret = h16_section_dva(plan, layout, ANE_H16_REGION_TEXT, + &hwx->constants, slot, false); + break; + case ANE_H16_SLOT_TEXT: + ret = h16_section_dva(plan, layout, ANE_H16_REGION_TEXT, + &hwx->text, slot, false); + break; + default: + aux = rule->source == ANE_H16_SLOT_AUX_GENERIC ? ANE_H16_AUX_GENERIC : + rule->source == ANE_H16_SLOT_AUX_OPERATION ? ANE_H16_AUX_OPERATION : + rule->source == ANE_H16_SLOT_AUX_PROCEDURE ? ANE_H16_AUX_PROCEDURE : + ANE_H16_AUX_TEXT_PROPERTIES; + ret = 0; + slot->dva = layout->aux_dva[aux]; + slot->size = layout->aux_size[aux]; + if (!slot->dva || slot->size < ANE_H16_AUX_MIN || + slot->size > ANE_H16_AUX_MAX) + ret = -EINVAL; + break; + } + if (ret) + return ret; + slot->resource = rule->resource; + slot->class = rule->class; + slot->present = true; + } + memcpy(slots, next, sizeof(next)); + return 0; +} +EXPORT_SYMBOL_GPL(ane_h16_load_slots); + +int ane_h16_load_encode(void *command, size_t length, + const struct ane_h16_load_slot slots[ANE_H16_LOAD_SLOTS]) +{ + u8 *p = command; + unsigned int i; + + if (!p || !slots || length != ANE_H16_LOAD_SIZE) + return -EINVAL; + for (i = 0; i < ANE_H16_LOAD_SLOTS; i++) + if (slots[i].present && + (!slots[i].size || slots[i].size > U64_MAX - slots[i].dva)) + return -EINVAL; + memset(p, 0, length); + for (i = 0; i < ANE_H16_LOAD_SLOTS; i++) { + u8 *d = p + ANE_H16_LOAD_SLOT_OFFSET + i * ANE_H16_LOAD_SLOT_SIZE; + + if (!slots[i].present) + continue; + put_unaligned_le32(1, d); + put_unaligned_le32(slots[i].resource, d + 4); + put_unaligned_le32(slots[i].class, d + 8); + put_unaligned_le32(1, d + 0x10); + put_unaligned_le64(slots[i].dva, d + 0x18); + put_unaligned_le64(slots[i].size, d + 0x20); + } + put_unaligned_le32(ANE_H16_LOAD_ID_REQUEST, p + ANE_H16_LOAD_ID_OFFSET); + return ane_h16_command_prepare(p, length, ANE_H16_LOAD_PROGRAM, ANE_FW_IO); +} +EXPORT_SYMBOL_GPL(ane_h16_load_encode); + +static int h16_reply_header(const u8 *reply, size_t length, size_t expected, u16 id) +{ + if (!reply || length != expected || get_unaligned_le32(reply) || + get_unaligned_le16(reply + 4) != id) + return -EPROTO; + return get_unaligned_le16(reply + 6) ? -EREMOTEIO : 0; +} + +/* The reply echoes every slot and replaces the request ID with the program ID. */ +int ane_h16_load_reply(const void *reply, size_t length, const void *request, + u32 *program) +{ + const u8 *p = reply, *q = request; + u32 id; + int ret; + + if (!program || !q) + return -EINVAL; + ret = h16_reply_header(p, length, ANE_H16_LOAD_SIZE, ANE_H16_LOAD_PROGRAM); + if (ret) + return ret; + id = get_unaligned_le32(p + ANE_H16_LOAD_ID_OFFSET); + if (id == ANE_H16_LOAD_ID_REQUEST || get_unaligned_le32(p + 0x1bc) || + memcmp(p + ANE_H16_LOAD_SLOT_OFFSET, q + ANE_H16_LOAD_SLOT_OFFSET, + ANE_H16_LOAD_ID_OFFSET - ANE_H16_LOAD_SLOT_OFFSET)) + return -EPROTO; + /* ID 0 is a valid program ID. */ + *program = id; + return 0; +} +EXPORT_SYMBOL_GPL(ane_h16_load_reply); + +int ane_h16_create_encode(void *command, size_t length, u32 program) +{ + u8 *p = command; + + if (!p || length != ANE_H16_CREATE_SIZE) + return -EINVAL; + /* No process resources travel in CREATE on 27.0; the body stays zero. */ + memset(p, 0, length); + put_unaligned_le32(program, p + 8); + return ane_h16_command_prepare(p, length, ANE_H16_CREATE_PROCESS, ANE_FW_IO); +} +EXPORT_SYMBOL_GPL(ane_h16_create_encode); + +int ane_h16_create_reply(const void *reply, size_t length, u32 program, u32 *process) +{ + const u8 *p = reply; + int ret; + + if (!process) + return -EINVAL; + ret = h16_reply_header(p, length, ANE_H16_CREATE_SIZE, ANE_H16_CREATE_PROCESS); + if (ret) + return ret; + if (get_unaligned_le32(p + 8) != program || + memchr_inv(p + 0x10, 0, ANE_H16_CREATE_SIZE - 0x10)) + return -EPROTO; + /* ID 0 is a valid process ID. */ + *process = get_unaligned_le32(p + 0xc); + return 0; +} +EXPORT_SYMBOL_GPL(ane_h16_create_reply); + +int ane_h16_terminate_encode(void *command, size_t length, u32 program, u32 process) +{ + u8 *p = command; + + if (!p || length != ANE_H16_TERMINATE_SIZE) + return -EINVAL; + memset(p, 0, length); + put_unaligned_le32(program, p + 8); + put_unaligned_le32(process, p + 0xc); + return ane_h16_command_prepare(p, length, ANE_H16_TERMINATE_PROCESS, ANE_FW_IO); +} +EXPORT_SYMBOL_GPL(ane_h16_terminate_encode); + +int ane_h16_unload_encode(void *command, size_t length, u32 program) +{ + u8 *p = command; + + if (!p || length != ANE_H16_UNLOAD_SIZE) + return -EINVAL; + memset(p, 0, length); + put_unaligned_le32(program, p + 8); + return ane_h16_command_prepare(p, length, ANE_H16_UNLOAD_PROGRAM, ANE_FW_IO); +} +EXPORT_SYMBOL_GPL(ane_h16_unload_encode); + +int ane_h16_gate_encode(void *command, size_t length, u32 value) +{ + u8 *p = command; + + if (!p || length != ANE_H16_GATE_SIZE || value > 1) + return -EINVAL; + memset(p, 0, length); + put_unaligned_le32(value, p + 8); + return ane_h16_command_prepare(p, length, ANE_H16_POWER_GATE, ANE_FW_DATA_CHAIN); +} +EXPORT_SYMBOL_GPL(ane_h16_gate_encode); + +/* + * Short commands (0x002d, TERMINATE, UNLOAD) have no recorded reply body, so + * only the transport reply shape is a predicate: same length and command, + * zero first word and a zero acknowledgment. + */ +int ane_h16_id_reply(const void *reply, size_t length, const void *request) +{ + const u8 *q = request; + + if (!q || length < 8 || length > ANE_H16_TERMINATE_SIZE) + return -EINVAL; + return h16_reply_header(reply, length, length, get_unaligned_le16(q + 4)); +} +EXPORT_SYMBOL_GPL(ane_h16_id_reply); + +/* + * Kernel-owned EXECUTE resource map. EXECUTE names each buffer by its ID in + * the generic section that LOAD installs, so the ID comes from that section: + * entry k describes surface k in HWX surface order. The entry must agree with + * the surface direction and size. IDs are unique and never the reserved + * kernel IDs 2 and 3. The class follows the round 5 usage mapping (input + * usage 1, output usage 2). + * + * A program with a __DATA scratch has one more entry, last: the scratch, + * typed as an input (type 0, direction 1) of exactly the __bss size. Linked + * programs keep the DRAM edges between their tasks there; the measured H17 + * MobileNet run bound it this way. + */ +int ane_h16_process_bindings(const struct ane_h16_hwx *hwx, const void *generic, + size_t generic_size, + struct ane_h16_process_binding *bindings, + unsigned int *count, + struct ane_h16_process_binding *scratch) +{ + const u8 *g = generic; + unsigned int i, j, entries; + u32 seen[2] = {}; + + if (!hwx || !g || !bindings || !count || hwx->surface_count < 2 || + hwx->surface_count > ANE_H16_SURFACES || (hwx->scratch.size && !scratch)) + return -EINVAL; + entries = hwx->surface_count + !!hwx->scratch.size; + if (generic_size != ANE_H16_GENERIC_HEADER + entries * ANE_H16_GENERIC_ENTRY || + get_unaligned_le32(g) != ANE_H16_GENERIC_MAGIC || + !get_unaligned_le32(g + 4) || + get_unaligned_le32(g + 4) > ANE_H16_GENERIC_VERSION || + get_unaligned_le32(g + ANE_H16_GENERIC_COUNT) != entries) + return -ENOEXEC; + for (i = 0; i < entries; i++) { + const u8 *e = g + ANE_H16_GENERIC_HEADER + i * ANE_H16_GENERIC_ENTRY; + bool is_scratch = i == hwx->surface_count; + u32 direction = is_scratch ? 1 : hwx->surfaces[i].direction; + u32 id = get_unaligned_le32(e + 4); + u64 bytes = get_unaligned_le64(e + 0x20); + + if (direction < 1 || direction > 2) + return -ENOEXEC; + if (get_unaligned_le32(e) != 1 || id == 2 || id == 3 || + id > ANE_H16_GENERIC_ID_MAX || + get_unaligned_le32(e + 8) != direction - 1 || + get_unaligned_le32(e + 0x10) != direction) + return -ENOEXEC; + /* The scratch entry may round __bss up to its 16 KiB mapping. */ + if (is_scratch ? bytes < hwx->scratch.size || + bytes > ALIGN(hwx->scratch.size, SZ_16K) : + bytes < hwx->surfaces[i].span || bytes > hwx->surfaces[i].allocation) + return -ENOEXEC; + for (j = 0; j < i && j < hwx->surface_count; j++) + if (bindings[j].index == id) + return -ENOEXEC; + if (is_scratch) { + *scratch = (struct ane_h16_process_binding) { + .hwx_vm = hwx->scratch.vm, + .index = id, + .class = ane_h16_usage_class(direction), + }; + continue; + } + bindings[i] = (struct ane_h16_process_binding) { + .hwx_vm = hwx->surfaces[i].section.vm, + .index = id, + .class = ane_h16_usage_class(direction), + .bytes = bytes, + }; + seen[direction - 1]++; + } + if (!seen[0] || seen[1] != 1) + return -ENOEXEC; + if (scratch && !hwx->scratch.size) + *scratch = (struct ane_h16_process_binding) {}; + *count = hwx->surface_count; + return 0; +} +EXPORT_SYMBOL_GPL(ane_h16_process_bindings); + +/* + * The operation section binds BAR slots to resources: per record, a count at + * +8 and (slot, tag) pairs from +0xc. A tag is a generic-section ID, or the + * kernel resource index for a __KERN_0 program. BAR slot s takes its base + * from the program descriptor qword at +0x10 * (s + 1), so every pair must + * name the resource whose HWX address is that base. Kernel programs must bind + * the kernel: the legacy tag 2 instead makes the H17 firmware assert and stop + * answering (H17 harness facts, kernel-table programs). + */ +int ane_h16_operation_validate(const struct ane_h16_hwx_snapshot *snapshot, + const struct ane_h16_hwx *hwx, const void *operation, + size_t operation_size, + const struct ane_h16_process_binding *bindings, + unsigned int count, + const struct ane_h16_process_binding *scratch) +{ + const u8 *o = operation, *c; + unsigned int r, records, i, j; + bool kernel_bound = false; + int ret; + + if (!hwx || !o || !bindings || count != hwx->surface_count || + (hwx->scratch.size && !scratch)) + return -EINVAL; + ret = ane_h16_hwx_snapshot_check(snapshot); + if (ret) + return ret; + if (!hwx->program_offset || snapshot->size < 0x800 || + hwx->program_offset > snapshot->size - 0x800) + return -ENOEXEC; + c = (const u8 *)snapshot->bytes + hwx->program_offset; + if (operation_size < 4) + return -ENOEXEC; + records = get_unaligned_le32(o); + if (!records || records > ANE_H16_OPERATION_RECORDS_MAX || + operation_size != 4 + (size_t)records * ANE_H16_OPERATION_RECORD) + return -ENOEXEC; + for (r = 0; r < records; r++) { + const u8 *rec = o + 4 + r * ANE_H16_OPERATION_RECORD; + u32 refs = get_unaligned_le32(rec + 8); + u64 seen = 0; + + if (get_unaligned_le32(rec) || get_unaligned_le32(rec + 4) || + refs > ANE_H16_OPERATION_REFS_MAX) + return -ENOEXEC; + for (i = 0; i < refs; i++) { + u32 slot = get_unaligned_le32(rec + 0xc + i * 8); + u32 tag = get_unaligned_le32(rec + 0x10 + i * 8); + u64 base, want = 0; + + if (slot >= 64 || (seen & BIT_ULL(slot))) + return -ENOEXEC; + seen |= BIT_ULL(slot); + base = get_unaligned_le64(c + 0x10 * (slot + 1)); + if (tag == ANE_H16_RESOURCE_KERNEL && hwx->kernel.size) { + want = hwx->kernel.vm; + kernel_bound = true; + } else if (scratch && hwx->scratch.size && tag == scratch->index) { + want = scratch->hwx_vm; + } else { + for (j = 0; j < count; j++) + if (tag == bindings[j].index) + want = bindings[j].hwx_vm; + } + if (!want || base != want) + return -ENOEXEC; + } + } + return hwx->kernel.size && !kernel_bound ? -ENOEXEC : 0; +} +EXPORT_SYMBOL_GPL(ane_h16_operation_validate); + +/* + * The mcache custom-BAR record comes from the program descriptor relocation + * tuple (BAR ID, field, bit offset, width at +0x868, +0x870, +0x878, +0x87c of + * the linear 0x8a0 form). The validator pins those words. The scratch form's + * tuple layout is not established, so it sends no record, which is legal. + */ +int ane_h16_custom_bars(const struct ane_h16_hwx_snapshot *snapshot, + const struct ane_h16_hwx *hwx, u32 dsid, + struct ane_h16_custom_bar *bars, unsigned int *count) +{ + const u8 *c; + u32 bar, offset, width; + int ret; + + if (!hwx || !bars || !count) + return -EINVAL; + ret = ane_h16_hwx_snapshot_check(snapshot); + if (ret) + return ret; + if (!hwx->program_offset || snapshot->size < 0x8a0 || + hwx->program_offset > snapshot->size - 0x8a0) + return -ENOEXEC; + c = (const u8 *)snapshot->bytes + hwx->program_offset; + *count = 0; + if (get_unaligned_le32(c + 4) != 0x8a0) + return 0; + bar = get_unaligned_le32(c + 0x868); + offset = get_unaligned_le32(c + 0x878); + width = get_unaligned_le32(c + 0x87c); + if (!width || width > 16 || offset > 64 - width || dsid >= BIT(width)) + return -ENOEXEC; + bars[0] = (struct ane_h16_custom_bar) { + .configuration = 1, + .value_type = 0, + .bar = bar, + .bit_offset = offset, + .bit_width = width, + .value = dsid, + }; + *count = 1; + return 0; +} +EXPORT_SYMBOL_GPL(ane_h16_custom_bars); + +int ane_h16_session_init(struct ane_h16_session *s, struct ane_fw_transport *transport, + const struct ane_h16_session_ops *ops, void *ctx, + u64 timeout_ns) +{ + if (!s || !transport || !transport->armed || !ops || !ops->map || !ops->log || + !timeout_ns) + return -EINVAL; + memset(s, 0, sizeof(*s)); + s->transport = transport; + s->ops = ops; + s->ctx = ctx; + s->timeout_ns = timeout_ns; + s->dsid = ANE_H16_MCACHE_DSID; + spin_lock_init(&s->stats_lock); + return 0; +} +EXPORT_SYMBOL_GPL(ane_h16_session_init); + +static int h16_fail(struct ane_h16_session *s, int error) +{ + if (!s->error) + s->error = error; + if (!s->quarantined) { + s->quarantined = true; + s->pending.quarantined = true; + s->ops->log(s->ctx, "program quarantine", (u64)-s->error, 0); + } + return s->error; +} + +static u64 h16_now(struct ane_h16_session *s) +{ + return s->transport->ops->now_ns(s->transport->ctx); +} + +/* Publish one command and wait for its transport reply, kept in the pool. */ +static int h16_send(struct ane_h16_session *s, unsigned int ch, const void *command, + size_t length, const u8 **reply) +{ + struct ane_fw_transport *t = s->transport; + int ret; + + if (ane_fw_command_denied(get_unaligned_le16((const u8 *)command + 4))) + return -EPERM; + ret = ane_fw_transport_publish(t, ch, command, length, h16_now(s) + s->timeout_ns); + if (ret) + return ret; + ret = ane_fw_transport_wait(t, ch); + *reply = (const u8 *)t->pool.cpu + (t->pending[ch].dva - t->pool.dva); + return ret; +} + +/* + * The transport copies commands into a pool arena. LOAD and CREATE stay in + * it; each EXECUTE sequence gives its space back once the firmware reported + * the request complete. Refuse a sequence up front when it cannot fit, so + * running out of arena is a clean -ENOSPC and never a half-sent sequence. + */ +static int h16_pool_room(struct ane_h16_session *s, const size_t *lengths, + unsigned int count) +{ + const struct ane_fw_transport *t = s->transport; + size_t used = t->pool_used; + unsigned int i; + + for (i = 0; i < count; i++) { + used = ALIGN(used, ANE_FW_SLOT_SIZE); + if (used > t->pool.size || lengths[i] > t->pool.size - used) + return -ENOSPC; + used += lengths[i]; + } + return 0; +} + +static int h16_map(struct ane_h16_session *s, size_t size, bool write, + struct ane_fw_surface *out) +{ + int ret = s->ops->map(s->ctx, size, write, out); + + if (ret) + return ret; + if (!ane_fw_surface_valid(out) || out->size != size || !IS_ALIGNED(out->dva, SZ_16K)) + return -EPROTO; + return 0; +} + +static int h16_program_map(struct ane_h16_session *s, struct ane_h16_program *p, + const void *const aux[ANE_H16_AUX_COUNT], + const size_t aux_size[ANE_H16_AUX_COUNT], + struct ane_h16_load_layout *layout) +{ + struct ane_h16_process_binding bindings[ANE_H16_SURFACES], scratch = {}; + const struct ane_h16_hwx *hwx = &p->hwx; + unsigned int i, j, count; + int ret; + + ret = ane_h16_process_bindings(hwx, aux[ANE_H16_AUX_GENERIC], + aux_size[ANE_H16_AUX_GENERIC], bindings, &count, + &scratch); + if (!ret) + ret = ane_h16_operation_validate(&p->snapshot, hwx, aux[ANE_H16_AUX_OPERATION], + aux_size[ANE_H16_AUX_OPERATION], bindings, + count, &scratch); + if (ret) + return ret; + for (i = 0; i < p->plan.count; i++) { + const struct ane_h16_hwx_region *r = &p->plan.regions[i]; + + if (r->file_size > p->plan.allocation[i] || + r->file_offset > p->snapshot.size || + r->file_size > p->snapshot.size - r->file_offset) + return -ENOEXEC; + ret = h16_map(s, p->plan.allocation[i], r->permissions & 2, &p->regions[i]); + if (ret) + return ret; + memcpy(p->regions[i].cpu, (const u8 *)p->snapshot.bytes + r->file_offset, + r->file_size); + layout->regions[i] = (struct ane_h16_load_mapping) { + .dva = p->regions[i].dva, + .size = p->regions[i].size, + .permissions = r->permissions, + }; + } + layout->region_count = p->plan.count; + for (i = 0; i < ANE_H16_AUX_COUNT; i++) { + ret = h16_map(s, ALIGN(aux_size[i], SZ_16K), false, &p->aux[i]); + if (ret) + return ret; + memcpy(p->aux[i].cpu, aux[i], aux_size[i]); + layout->aux_dva[i] = p->aux[i].dva; + layout->aux_size[i] = aux_size[i]; + } + ret = ane_h16_load_resources_create(&p->snapshot, layout->regions, + layout->region_count, bindings, count, + p->resources); + if (ret) + return ret; + p->resource_count = count; + for (i = 0; i < count; i++) + p->declared[i] = bindings[i].bytes; + for (i = 0; i < count; i++) { + for (j = 0; j < p->plan.count; j++) + if (p->regions[j].dva == p->resources[i].buffer.mapping_dva) + p->surfaces[i] = &p->regions[j]; + if (!p->surfaces[i]) + return -EINVAL; + } + if (hwx->scratch.size) { + for (j = 0; j < p->plan.count; j++) + if (p->plan.regions[j].kind == ANE_H16_REGION_SCRATCH) + break; + /* The validator puts __bss at the start of its writable segment. */ + if (j == p->plan.count || p->plan.regions[j].vm != hwx->scratch.vm || + p->plan.regions[j].permissions != 3 || + hwx->scratch.size > p->regions[j].size) + return -ENOEXEC; + p->scratch = (struct ane_h16_execute_buffer) { + .index = scratch.index, + .class = scratch.class, + .mapping_dva = p->regions[j].dva, + .mapping_size = p->regions[j].size, + .size = hwx->scratch.size, + }; + p->has_scratch = true; + } + ret = ane_h16_load_slots(hwx, &p->plan, layout, p->slots); + if (ret) + return ret; + /* + * H17 (T8140, 26.6.2 firmware) asserts in EXELOOP right after logging a + * custom BAR record (CAneProgramManagerHx.cpp:416) and never reports the + * request: the mcache record of a linear relu stopped the Neo there. + * Every H17 request that ran sent no record, so H17 programs carry none + * and keep the firmware's default mcache DSID. + */ + p->bar_count = 0; + if (s->transport->soc->chip_family >= 17) + return 0; + return ane_h16_custom_bars(&p->snapshot, hwx, s->dsid, p->bars, &p->bar_count); +} + +static const size_t h16_load_sequence[] = { + ANE_H16_LOAD_SIZE, ANE_H16_CREATE_SIZE, ANE_H16_TERMINATE_SIZE, ANE_H16_UNLOAD_SIZE, +}; + +/* + * On this firmware family a direct EXECUTE reuses the BAR context of the + * program the engine ran last, so the first EXECUTE of a program writes into + * the previous program's output buffer unless it is queued. The session + * queues the first EXECUTE whenever the engine last ran another program, the + * first one after boot included (see ane_h16_session_execute()). A LOAD is + * refused while the previous program is still loaded and accepted once it + * closed; a switch-safe transport (H17) also requeues on every program or + * process change, so its session may hold several programs at once. Every + * program's firmware-visible memory stays mapped until reset. + */ +int ane_h16_session_load(struct ane_h16_session *s, + struct ane_h16_hwx_snapshot *snapshot, + const void *const aux[ANE_H16_AUX_COUNT], + const size_t aux_size[ANE_H16_AUX_COUNT], + struct ane_h16_program **program) +{ + struct ane_h16_load_layout layout = {}; + struct ane_h16_program *p; + u8 *command; + const u8 *reply; + unsigned int i; + int ret; + + if (!s || !snapshot || !aux || !aux_size || !program) + return -EINVAL; + if (s->quarantined) + return -EIO; + if (s->program && !s->program->closed && + !ane_fw_transport_switch_safe(s->transport)) + return -EBUSY; + for (i = 0; i < ANE_H16_AUX_COUNT; i++) + if (!aux[i] || aux_size[i] < ANE_H16_AUX_MIN || + aux_size[i] > ANE_H16_AUX_MAX || !IS_ALIGNED(aux_size[i], 4)) + return -EINVAL; + ret = ane_h16_hwx_snapshot_check(snapshot); + if (ret) + return ret; + ret = h16_pool_room(s, h16_load_sequence, ARRAY_SIZE(h16_load_sequence)); + if (ret) + return ret; + p = kzalloc_obj(*p); + command = kzalloc(ANE_H16_CREATE_SIZE, GFP_KERNEL); + if (!p || !command) { + ret = -ENOMEM; + goto free; + } + ret = ane_h16_load_plan_create(snapshot, &p->plan); + if (!ret) + ret = ane_h16_hwx_validate(snapshot->bytes, snapshot->size, &p->hwx); + /* An HWX runs only on the engine generation it was compiled for. */ + if (!ret && p->hwx.subtype != s->transport->soc->hwx_subtype) + ret = -ENOEXEC; + if (ret) + goto free; + p->snapshot = *snapshot; + /* + * From the first mapping on, firmware-visible memory exists that is never + * released before reset, so the program becomes the session's here and + * is never freed. + */ + p->generation = ++s->generation; + memset(snapshot, 0, sizeof(*snapshot)); + p->session = s; + s->program = p; + ret = h16_program_map(s, p, aux, aux_size, &layout); + if (!ret) + ret = ane_h16_load_encode(command, ANE_H16_LOAD_SIZE, p->slots); + if (ret) + goto out; + ret = h16_send(s, ANE_FW_IO, command, ANE_H16_LOAD_SIZE, &reply); + if (!ret) + ret = ane_h16_load_reply(reply, ANE_H16_LOAD_SIZE, command, &p->key.program); + if (ret) { + s->ops->log(s->ctx, "LOAD refused", (u64)-ret, 0); + ret = h16_fail(s, ret); + goto out; + } + s->ops->log(s->ctx, "LOAD program", p->key.program, p->bar_count); + ret = ane_h16_create_encode(command, ANE_H16_CREATE_SIZE, p->key.program); + if (!ret) + ret = h16_send(s, ANE_FW_IO, command, ANE_H16_CREATE_SIZE, &reply); + if (!ret) + ret = ane_h16_create_reply(reply, ANE_H16_CREATE_SIZE, p->key.program, + &p->key.process); + if (ret) { + s->ops->log(s->ctx, "CREATE refused", (u64)-ret, 0); + ret = h16_fail(s, ret); + goto out; + } + /* Each validated program has exactly one procedure. */ + p->key.procedure = 0; + p->created = true; + s->ops->log(s->ctx, "CREATE process", p->key.program, p->key.process); + *program = p; +out: + kfree(command); + return ret; +free: + kfree(command); + kfree(p); + return ret; +} +EXPORT_SYMBOL_GPL(ane_h16_session_load); + +static int h16_gate(struct ane_h16_session *s, u32 value) +{ + u8 command[ANE_H16_GATE_SIZE]; + const u8 *reply; + int ret = ane_h16_gate_encode(command, sizeof(command), value); + + if (!ret) + ret = h16_send(s, ANE_FW_DATA_CHAIN, command, sizeof(command), &reply); + if (!ret) + ret = ane_h16_id_reply(reply, sizeof(command), command); + return ret; +} + +/* + * One firmware log record: a 2-byte header, then NUL-terminated text in the + * rest of the 0x80 bytes. Copies the printable text (other bytes become '.') + * into line, which holds ANE_H16_FWLOG_RECORD - 1 bytes, and returns its + * length; 0 for an empty record. + */ +size_t ane_h16_fwlog_line(const u8 *record, char *line) +{ + size_t i, n = 0; + + for (i = 2; i < ANE_H16_FWLOG_RECORD && record[i]; i++) + line[n++] = isprint(record[i]) ? record[i] : record[i] == '\t' ? ' ' : '.'; + while (n && line[n - 1] == ' ') + n--; + line[n] = 0; + return n; +} +EXPORT_SYMBOL_GPL(ane_h16_fwlog_line); + +/* + * After a failed EXECUTE the session is quarantined, so this runs at most + * once per program: print every non-empty record of the firmware log in + * buffer order (at most 512 lines for the 64 KiB log). + */ +static unsigned int h16_dump_fwlog(struct ane_h16_session *s) +{ + const struct ane_fw_surface *log = + ane_fw_transport_shared_tagged(s->transport, ANE_FW_SHARED_LOG); + char line[ANE_H16_FWLOG_RECORD]; + unsigned int i, lines = 0; + + if (!log || !log->cpu) { + s->ops->log(s->ctx, "fwlog absent", 0, 0); + return 0; + } + s->transport->ops->read_barrier(s->transport->ctx); + for (i = 0; i < log->size / ANE_H16_FWLOG_RECORD; i++) { + const u8 *r = (const u8 *)log->cpu + i * ANE_H16_FWLOG_RECORD; + + if (!ane_h16_fwlog_line(r, line)) + continue; + pr_info("H16 fwlog[%03x] %04x %s\n", i, get_unaligned_le16(r), line); + lines++; + } + s->ops->log(s->ctx, "fwlog records printed", lines, log->dva); + return lines; +} + +/* Non-zero 16-byte rows of a command or reply, as two little-endian words. */ +static void h16_dump_rows(const char *what, const u8 *p, size_t length) +{ + size_t off; + + for (off = 0; off + 16 <= length; off += 16) { + u64 lo = get_unaligned_le64(p + off), hi = get_unaligned_le64(p + off + 8); + + if (lo || hi) + pr_info("H16 %s +%04zx: %016llx %016llx\n", what, off, lo, hi); + } +} + +/* What one EXECUTE saw, logged only when it fails. */ +struct h16_execute_trace { + u32 step; + u32 reports; + u32 last_state; + u32 other; + /* When the first report and the completion of this UUID were seen. */ + u64 first_ns; + u64 done_ns; +}; + +enum { + H16_STEP_GATE_OPEN = 1, + H16_STEP_SEND, + H16_STEP_REPORT, + H16_STEP_GATE_CLOSE, +}; + +static void h16_trace_report(struct h16_execute_trace *trace, const u8 *report, + size_t size, const struct ane_h16_pending *request, u64 now) +{ + struct ane_h16_event e; + + if (!ane_h16_event_decode(report, size, ANE_FW_IO_T2H, &e) && + e.key.uuid == request->key.uuid) { + if (!trace->reports++) + trace->first_ns = now; + trace->last_state = e.state; + } else { + trace->other++; + } +} + +/* Wait for the terminal report of one pending UUID, servicing allocations. */ +static int h16_wait_report(struct ane_h16_session *s, struct ane_h16_pending *request, + u64 deadline, struct h16_execute_trace *trace) +{ + struct ane_fw_transport *t = s->transport; + u64 start = h16_now(s), now; + u8 *report; + int ret; + + report = kmalloc(ANE_H16_REPORT_MAX, GFP_KERNEL); + if (!report) + return -ENOMEM; + for (;;) { + size_t size; + u32 pending; + + ret = ane_fw_transport_consume(t, report, ANE_H16_REPORT_MAX, &size); + if (!ret) { + ret = ane_h16_pending_dispatch(&s->pending, report, size, + ANE_FW_IO_T2H, + (unsigned long)div_u64(h16_now(s), + NSEC_PER_USEC)); + if (ret == ANE_H16_EVENT_COMPLETE) { + trace->done_ns = h16_now(s); + if (!trace->reports) + trace->first_ns = trace->done_ns; + ret = 0; + break; + } + if (ret == ANE_H16_EVENT_FAILED) { + ret = -EIO; + break; + } + /* Foreign or late UUIDs and ISP-routed reports never retire this call. */ + if (ret == ANE_H16_EVENT_PROGRESS) { + h16_trace_report(trace, report, size, request, h16_now(s)); + continue; + } + if (ret == -ENOENT || ret == -ENOMSG) { + trace->other++; + continue; + } + break; + } + if (ret != -EAGAIN) + break; + ret = ane_fw_transport_irq(t, &pending); + if (!ret) { + ret = ane_fw_transport_shared(t); + if (ret == -EAGAIN) + ret = 0; + } + if (ret) + break; + now = h16_now(s); + if (now >= deadline) { + ret = -ETIMEDOUT; + break; + } + t->polls++; + t->ops->relax(t->ctx, now - start); + } + kfree(report); + return ret; +} + +/* + * 0x002d with 0, EXECUTE on channel 7 with its UUID already pending, the + * matching state-1 report on channel 6, 0x002d with 1, then a DMA read + * barrier before the caller reads outputs. Any uncertain outcome quarantines + * the session and keeps every mapping. + */ +int ane_h16_session_execute(struct ane_h16_session *s, struct ane_h16_program *p) +{ + struct ane_h16_execute_buffer buffers[ANE_H16_SURFACES + 1]; + struct ane_h16_pending request = {}; + struct h16_execute_trace trace = { .last_state = U32_MAX }; + struct ane_h16_execute call = {}; + struct ane_h16_exec_sample *x = &s->timing; + size_t length, sequence[5], mark; + u8 *command; + const u8 *reply = NULL; + unsigned int i; + u64 deadline, polls, t0, gate_open, t1 = 0, t2 = 0, t3, end; + u64 entry = h16_now(s); + int ret; + + if (!s || !p || p->session != s) + return -EINVAL; + if (s->quarantined) + return -EIO; + if (!p->created || p->closed) + return -EINVAL; + /* + * The snapshot was hashed and parsed at LOAD, everything the firmware + * reads was copied out of it then, and it stays private and unchanged + * until the program is freed. Hashing it again here (8.4 MB for + * MobileNet, about 2.9 ms) protected nothing per request. + */ + ret = ane_h16_resources_validate(&p->hwx, p->resources, p->resource_count); + if (ret) + return ret; + /* + * H17 checks each descriptor against the size the generic section + * declared for its buffer at LOAD and asserts in its program manager + * when it is smaller (relu64: section 0x1000 of a declared 0x4000 + * surface), so H17 descriptors carry the declared size. H16 keeps the + * whole HWX section, which is what ran MobileNet on the J616s; the two + * agree there (0x4c000). The declared size lies between the tensor span + * and the mapped allocation. Userspace still sees and copies the span. + */ + for (i = 0; i < p->resource_count; i++) { + buffers[i] = p->resources[i].buffer; + buffers[i].size = s->transport->soc->chip_family >= 17 ? p->declared[i] : + p->hwx.surfaces[i].section.size; + } + if (p->has_scratch) + buffers[i++] = p->scratch; + length = ANE_H16_EXECUTE_BASE_SIZE + p->bar_count * ANE_H16_CUSTOM_BAR_STRIDE; + /* Keep room for the teardown pair after this request. */ + sequence[0] = ANE_H16_GATE_SIZE; + sequence[1] = length; + sequence[2] = ANE_H16_GATE_SIZE; + sequence[3] = ANE_H16_TERMINATE_SIZE; + sequence[4] = ANE_H16_UNLOAD_SIZE; + ret = h16_pool_room(s, sequence, ARRAY_SIZE(sequence)); + if (ret) + return ret; + command = kzalloc(length, GFP_KERNEL); + if (!command) + return -ENOMEM; + mark = ane_fw_transport_mark(s->transport); + /* Inputs and poisoned outputs must reach memory before any doorbell. */ + s->transport->ops->write_barrier(s->transport->ctx); + polls = s->transport->polls; + trace.step = H16_STEP_GATE_OPEN; + t0 = h16_now(s); + ret = h16_gate(s, 0); + if (ret) + goto fail; + gate_open = h16_now(s) - t0; + deadline = t0 + gate_open + s->timeout_ns; + request.key = p->key; + request.deadline = (unsigned long)div_u64(deadline, NSEC_PER_USEC); + ret = ane_h16_pending_insert(&s->pending, &request, + (unsigned long)div_u64(h16_now(s), NSEC_PER_USEC)); + if (ret) + goto fail; + call.key = request.key; + call.buffers = buffers; + call.buffer_count = p->resource_count + (p->has_scratch ? 1 : 0); + /* + * The first request of a program on the engine, the first one after boot + * included, takes the queued path once: only EXELOOP sets up the BAR + * context and the L2-spill mapping that later direct requests reuse. + */ + if (s->executed != p->generation) { + call.flags = ANE_H16_EXECUTE_FLAG_QUEUED; + s->ops->log(s->ctx, "EXECUTE first of this program, queued", p->key.program, + request.key.uuid); + } + s->executed = p->generation; + call.bars = p->bars; + call.bar_count = p->bar_count; + ret = ane_h16_execute_encode(command, length, &call); + if (!ret) + ret = ane_h16_command_prepare(command, length, ANE_H16_EXECUTE, + ANE_FW_DATA_CHAIN); + if (!ret) { + trace.step = H16_STEP_SEND; + t1 = h16_now(s); + ret = h16_send(s, ANE_FW_DATA_CHAIN, command, length, &reply); + } + /* The channel-7 reply acknowledges transport only, never completion. */ + if (!ret) { + trace.step = H16_STEP_REPORT; + t2 = h16_now(s); + ret = h16_wait_report(s, &request, deadline, &trace); + } + if (ret) { + if (request.active) + ane_h16_pending_abort(&s->pending, &request, ret); + goto fail; + } + trace.step = H16_STEP_GATE_CLOSE; + t3 = h16_now(s); + ret = h16_gate(s, 1); + if (ret) + goto fail; + end = h16_now(s); + s->transport->ops->read_barrier(s->transport->ctx); + memset(x, 0, sizeof(*x)); + x->ns[ANE_H16_T_PREPARE] = t0 - entry; + x->ns[ANE_H16_T_GATE_OPEN] = gate_open; + x->ns[ANE_H16_T_SUBMIT_ACK] = t2 - t1; + x->ns[ANE_H16_T_ACK_REPORT] = trace.first_ns - t2; + x->ns[ANE_H16_T_REPORT_DONE] = trace.done_ns - trace.first_ns; + x->ns[ANE_H16_T_GATE_CLOSE] = end - t3; + x->ns[ANE_H16_T_FIRMWARE] = end - t0; + x->polls = s->transport->polls - polls; + kfree(command); + /* + * The completion report and the closing gate reply both arrived, so the + * firmware no longer reads this sequence's commands: reuse their space. + */ + ret = ane_fw_transport_release(s->transport, mark); + x->ns[ANE_H16_T_FINISH] = h16_now(s) - end; + return ret; +fail: + s->ops->log(s->ctx, "EXECUTE failed", (u64)-ret, request.key.uuid); + /* + * Step 1 opening gate, 2 channel-7 ack, 3 report wait, 4 closing gate. + * Reports for this UUID and the last state (0 start, 3 engine start, + * ffffffff none), then the count of other reports seen while waiting. + */ + s->ops->log(s->ctx, "EXECUTE step/flags", trace.step, call.flags); + s->ops->log(s->ctx, "EXECUTE reports/last state", trace.reports, trace.last_state); + s->ops->log(s->ctx, "EXECUTE other reports", trace.other, 0); + if (trace.step >= H16_STEP_SEND) { + h16_dump_rows("EXECUTE sent", command, length); + if (reply) + h16_dump_rows("EXECUTE ack", reply, 0x40); + } + s->fwlog_lines = h16_dump_fwlog(s); + kfree(command); + return h16_fail(s, ret); +} +EXPORT_SYMBOL_GPL(ane_h16_session_execute); + +const char *const ane_h16_exec_phase_names[ANE_H16_T_PHASES] = { + [ANE_H16_T_LOCK] = "lock", + [ANE_H16_T_COPY_IN] = "copy_in", + [ANE_H16_T_PREPARE] = "prepare", + [ANE_H16_T_GATE_OPEN] = "gate_open", + [ANE_H16_T_SUBMIT_ACK] = "submit_ack", + [ANE_H16_T_ACK_REPORT] = "ack_report", + [ANE_H16_T_REPORT_DONE] = "report_done", + [ANE_H16_T_GATE_CLOSE] = "gate_close", + [ANE_H16_T_FINISH] = "finish", + [ANE_H16_T_COPY_OUT] = "copy_out", + [ANE_H16_T_FIRMWARE] = "firmware", + [ANE_H16_T_IOCTL] = "ioctl", +}; +EXPORT_SYMBOL_GPL(ane_h16_exec_phase_names); + +/* Fold one sample into last, sum, min and max. */ +void ane_h16_exec_stats_add(struct ane_h16_exec_stats *st, + const struct ane_h16_exec_sample *x) +{ + unsigned int i; + + if (!st->count++) { + st->min = *x; + st->max = *x; + memset(&st->sum, 0, sizeof(st->sum)); + } + for (i = 0; i < ANE_H16_T_PHASES; i++) { + st->sum.ns[i] += x->ns[i]; + st->min.ns[i] = min(st->min.ns[i], x->ns[i]); + st->max.ns[i] = max(st->max.ns[i], x->ns[i]); + } + st->sum.polls += x->polls; + st->min.polls = min(st->min.polls, x->polls); + st->max.polls = max(st->max.polls, x->polls); + st->last = *x; +} +EXPORT_SYMBOL_GPL(ane_h16_exec_stats_add); + +/* + * Account the last successful EXECUTE with the caller's copy and whole-request + * times. The caller serializes this with ane_h16_session_execute(). + */ +void ane_h16_session_account(struct ane_h16_session *s, u64 lock_ns, u64 copy_in_ns, + u64 copy_out_ns, u64 ioctl_ns) +{ + struct ane_h16_exec_sample x = s->timing; + + x.ns[ANE_H16_T_LOCK] = lock_ns; + x.ns[ANE_H16_T_COPY_IN] = copy_in_ns; + x.ns[ANE_H16_T_COPY_OUT] = copy_out_ns; + x.ns[ANE_H16_T_IOCTL] = ioctl_ns; + spin_lock(&s->stats_lock); + ane_h16_exec_stats_add(&s->stats, &x); + spin_unlock(&s->stats_lock); +} +EXPORT_SYMBOL_GPL(ane_h16_session_account); + +/* A consistent copy of the statistics, optionally starting them over. */ +void ane_h16_session_stats(struct ane_h16_session *s, struct ane_h16_exec_stats *out, + bool reset) +{ + spin_lock(&s->stats_lock); + if (out) + *out = s->stats; + if (reset) + memset(&s->stats, 0, sizeof(s->stats)); + spin_unlock(&s->stats_lock); +} +EXPORT_SYMBOL_GPL(ane_h16_session_stats); + +/* TERMINATE then UNLOAD. The mappings stay until reset; a new LOAD may follow. */ +int ane_h16_session_close(struct ane_h16_session *s, struct ane_h16_program *p) +{ + u8 command[ANE_H16_TERMINATE_SIZE]; + const u8 *reply; + int ret; + + if (!s || !p || p->session != s) + return -EINVAL; + if (p->closed || !p->created) + return 0; + p->closed = true; + if (s->quarantined) + return -EIO; + ret = ane_h16_terminate_encode(command, ANE_H16_TERMINATE_SIZE, p->key.program, + p->key.process); + if (!ret) + ret = h16_send(s, ANE_FW_IO, command, ANE_H16_TERMINATE_SIZE, &reply); + if (!ret) + ret = ane_h16_id_reply(reply, ANE_H16_TERMINATE_SIZE, command); + if (!ret) + ret = ane_h16_unload_encode(command, ANE_H16_UNLOAD_SIZE, p->key.program); + if (!ret) + ret = h16_send(s, ANE_FW_IO, command, ANE_H16_UNLOAD_SIZE, &reply); + if (!ret) + ret = ane_h16_id_reply(reply, ANE_H16_UNLOAD_SIZE, command); + if (ret) + return h16_fail(s, ret); + s->ops->log(s->ctx, "TERMINATE and UNLOAD", p->key.program, p->key.process); + return 0; +} +EXPORT_SYMBOL_GPL(ane_h16_session_close); + +MODULE_DESCRIPTION("H16 program load, execute and teardown on the shared transport"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_program.h b/drivers/accel/ane/ane_h16_program.h new file mode 100644 index 0000000000000..83d2c13a8fda4 --- /dev/null +++ b/drivers/accel/ane/ane_h16_program.h @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H16_PROGRAM_H__ +#define __ANE_H16_PROGRAM_H__ + +#include + +#include "ane_fw_transport.h" +#include "ane_h16_load.h" +#include "ane_h16_pending.h" + +/* + * Program lifecycle for one H16 firmware session: LOAD_PROGRAM, CREATE_PROCESS, + * the 0x002d/EXECUTE/0x002d sequence and TERMINATE/UNLOAD. Wire facts are from + * the H16 cold-boot facts (rounds 1 and 5, program load and lifecycle) and the + * SPEC-r4 EXECUTE layout. Nothing here maps memory or touches hardware; the + * backend supplies private mappings through ane_h16_session_ops. + */ + +#define ANE_H16_LOAD_SIZE 0x1c0 +#define ANE_H16_LOAD_SLOTS 9 +#define ANE_H16_LOAD_SLOT_OFFSET 0x08 +#define ANE_H16_LOAD_SLOT_SIZE 0x30 +#define ANE_H16_LOAD_ID_OFFSET 0x1b8 +#define ANE_H16_LOAD_ID_REQUEST 0xffffffffU +#define ANE_H16_CREATE_SIZE 0x258 +#define ANE_H16_TERMINATE_SIZE 0x10 +#define ANE_H16_UNLOAD_SIZE 0xc +#define ANE_H16_GATE_SIZE 0xc +#define ANE_H16_REPORT_MAX 0x100 +/* First value of the firmware mcache DSID pool (custom-BAR facts, 4). */ +#define ANE_H16_MCACHE_DSID 36 + +/* + * Host-built generic section: header, then one entry per surface in HWX + * segment order, then one entry for the __DATA scratch when the program has + * one. + */ +#define ANE_H16_GENERIC_MAGIC 1 +#define ANE_H16_GENERIC_VERSION 0x10 +#define ANE_H16_GENERIC_COUNT 0x204 +#define ANE_H16_GENERIC_HEADER 0x208 +#define ANE_H16_GENERIC_ENTRY 0x30 +#define ANE_H16_GENERIC_ID_MAX 0x3c + +/* Host-built operation section: a count, then fixed-size BAR records. */ +#define ANE_H16_OPERATION_RECORD 0x40c +#define ANE_H16_OPERATION_RECORDS_MAX 64 +#define ANE_H16_OPERATION_REFS_MAX 64 + +/* Host-built sections. Userspace packs their bytes; the kernel places them. */ +enum ane_h16_aux { + ANE_H16_AUX_GENERIC, + ANE_H16_AUX_OPERATION, + ANE_H16_AUX_PROCEDURE, + ANE_H16_AUX_TEXT_PROPERTIES, + ANE_H16_AUX_COUNT, +}; + +#define ANE_H16_AUX_MIN 0x28 +#define ANE_H16_AUX_MAX SZ_64K + +/* Source of one LOAD slot. */ +enum ane_h16_slot_source { + ANE_H16_SLOT_EMPTY, + ANE_H16_SLOT_AUX_GENERIC, + ANE_H16_SLOT_KERNEL, + ANE_H16_SLOT_TEXT, + ANE_H16_SLOT_AUX_OPERATION, + ANE_H16_SLOT_AUX_PROCEDURE, + ANE_H16_SLOT_AUX_TEXT_PROPERTIES, +}; + +struct ane_h16_slot_rule { + enum ane_h16_slot_source source; + u32 resource; + u32 class; +}; + +/* Resource indices of the six populated slots (facts 9.2, r5 7.1). The + * kernel slot index is also the operation tag of a kernel-table program. + */ +#define ANE_H16_RESOURCE_GENERIC 1002 +#define ANE_H16_RESOURCE_OPERATION 1003 +#define ANE_H16_RESOURCE_PROCEDURE 1004 +#define ANE_H16_RESOURCE_TEXT_PROPERTIES 1005 +#define ANE_H16_RESOURCE_TEXT 1006 +#define ANE_H16_RESOURCE_KERNEL 1007 + +const struct ane_h16_slot_rule *ane_h16_slot_rule(unsigned int slot); + +struct ane_h16_load_slot { + u32 resource; + u32 class; + u64 dva; + u64 size; + bool present; +}; + +/* Firmware-visible placement of every LOAD input, all kernel-owned. */ +struct ane_h16_load_layout { + struct ane_h16_load_mapping regions[ANE_H16_REGIONS]; + u32 region_count; + u64 aux_dva[ANE_H16_AUX_COUNT]; + u64 aux_size[ANE_H16_AUX_COUNT]; +}; + +/* Round 5, section 7: resource usage type to EXECUTE descriptor class. */ +static inline u32 ane_h16_usage_class(u32 usage) +{ + static const u8 classes[] = { + [1] = 0, [2] = 1, [3] = 0, [4] = 1, [5] = 6, [6] = 8, [7] = 2, + [8] = 8, [9] = 8, [10] = 4, [11] = 8, [12] = 0, [13] = 0, + }; + + if (usage < 1 || usage >= ARRAY_SIZE(classes)) + return 8; + return classes[usage]; +} + +int ane_h16_load_slots(const struct ane_h16_hwx *hwx, + const struct ane_h16_load_plan *plan, + const struct ane_h16_load_layout *layout, + struct ane_h16_load_slot slots[ANE_H16_LOAD_SLOTS]); +int ane_h16_load_encode(void *command, size_t length, + const struct ane_h16_load_slot slots[ANE_H16_LOAD_SLOTS]); +int ane_h16_load_reply(const void *reply, size_t length, const void *request, + u32 *program); +int ane_h16_create_encode(void *command, size_t length, u32 program); +int ane_h16_create_reply(const void *reply, size_t length, u32 program, u32 *process); +int ane_h16_terminate_encode(void *command, size_t length, u32 program, u32 process); +int ane_h16_unload_encode(void *command, size_t length, u32 program); +int ane_h16_id_reply(const void *reply, size_t length, const void *request); +int ane_h16_gate_encode(void *command, size_t length, u32 value); + +int ane_h16_process_bindings(const struct ane_h16_hwx *hwx, const void *generic, + size_t generic_size, + struct ane_h16_process_binding *bindings, + unsigned int *count, + struct ane_h16_process_binding *scratch); +int ane_h16_operation_validate(const struct ane_h16_hwx_snapshot *snapshot, + const struct ane_h16_hwx *hwx, const void *operation, + size_t operation_size, + const struct ane_h16_process_binding *bindings, + unsigned int count, + const struct ane_h16_process_binding *scratch); +int ane_h16_custom_bars(const struct ane_h16_hwx_snapshot *snapshot, + const struct ane_h16_hwx *hwx, u32 dsid, + struct ane_h16_custom_bar *bars, unsigned int *count); + +/* Private, pinned mappings in the firmware domain. Never released before reset. */ +struct ane_h16_session_ops { + int (*map)(void *ctx, size_t size, bool device_write, + struct ane_fw_surface *surface); + void (*log)(void *ctx, const char *item, u64 a, u64 b); +}; + +struct ane_h16_program { + struct ane_h16_hwx_snapshot snapshot; + /* Parsed again from the authenticated bytes, never the cached info. */ + struct ane_h16_hwx hwx; + struct ane_h16_load_plan plan; + struct ane_fw_surface regions[ANE_H16_REGIONS]; + struct ane_fw_surface aux[ANE_H16_AUX_COUNT]; + struct ane_h16_load_slot slots[ANE_H16_LOAD_SLOTS]; + struct ane_h16_resource resources[ANE_H16_SURFACES]; + struct ane_fw_surface *surfaces[ANE_H16_SURFACES]; + struct ane_h16_custom_bar bars[ANE_H16_MAX_CUSTOM_BARS]; + unsigned int bar_count; + unsigned int resource_count; + /* Per-surface buffer size from the generic section, sent in EXECUTE. */ + u64 declared[ANE_H16_SURFACES]; + /* + * The __DATA scratch: a kernel-owned EXECUTE input that carries the + * DRAM edges between tasks. Userspace never sees or fills it. + */ + struct ane_h16_execute_buffer scratch; + bool has_scratch; + struct ane_h16_session *session; + /* Session-local identity; firmware program IDs are reused after UNLOAD. */ + u64 generation; + struct ane_h16_request_key key; + bool created; + bool closed; +}; + +/* + * Where the time of one EXECUTE goes, in nanoseconds. The firmware phases + * are measured by ane_h16_session_execute(); a reply or report is "seen" + * when a poll finds it, so they include the poll latency. The copy and + * ioctl phases come from the caller through ane_h16_session_account(). + */ +enum ane_h16_exec_phase { + ANE_H16_T_LOCK, /* waiting for the session lock */ + ANE_H16_T_COPY_IN, /* inputs copied in, outputs poisoned */ + ANE_H16_T_PREPARE, /* checks, command buffer, until the opening gate */ + ANE_H16_T_GATE_OPEN, /* 0x002d 0 published until its reply */ + ANE_H16_T_SUBMIT_ACK, /* EXECUTE published until its channel-7 ack */ + ANE_H16_T_ACK_REPORT, /* the ack until the first report of the UUID */ + ANE_H16_T_REPORT_DONE, /* that report until the completion report */ + ANE_H16_T_GATE_CLOSE, /* 0x002d 1 published until its reply */ + ANE_H16_T_FINISH, /* read barrier, freeing the command and its arena */ + ANE_H16_T_COPY_OUT, /* outputs copied out */ + ANE_H16_T_FIRMWARE, /* opening gate published until closing gate reply */ + ANE_H16_T_IOCTL, /* the whole request after copying its buffer table */ + ANE_H16_T_PHASES +}; + +struct ane_h16_exec_sample { + u64 ns[ANE_H16_T_PHASES]; + /* Sleeps between polls over all four waits. */ + u64 polls; +}; + +struct ane_h16_exec_stats { + u64 count; + struct ane_h16_exec_sample last, sum, min, max; +}; + +extern const char *const ane_h16_exec_phase_names[ANE_H16_T_PHASES]; +void ane_h16_exec_stats_add(struct ane_h16_exec_stats *st, + const struct ane_h16_exec_sample *x); + +/* + * Programs of one firmware session: one loaded at a time, any number per + * boot; a switch-safe transport may hold several. See ane_h16_session_load(). + */ +struct ane_h16_session { + struct ane_fw_transport *transport; + const struct ane_h16_session_ops *ops; + void *ctx; + u64 timeout_ns; + struct ane_h16_pending_table pending; + /* The last loaded program. */ + struct ane_h16_program *program; + /* Generation of the last loaded program, and of the last one executed. */ + u64 generation; + u64 executed; + u32 dsid; + int error; + /* Firmware log lines printed after the last failed EXECUTE. */ + unsigned int fwlog_lines; + bool quarantined; + /* Firmware phases of the last successful EXECUTE. */ + struct ane_h16_exec_sample timing; + /* Accounted EXECUTEs; the lock lets a reader snapshot them. */ + spinlock_t stats_lock; + struct ane_h16_exec_stats stats; +}; + +int ane_h16_session_init(struct ane_h16_session *s, struct ane_fw_transport *transport, + const struct ane_h16_session_ops *ops, void *ctx, + u64 timeout_ns); +int ane_h16_session_load(struct ane_h16_session *s, + struct ane_h16_hwx_snapshot *snapshot, + const void *const aux[ANE_H16_AUX_COUNT], + const size_t aux_size[ANE_H16_AUX_COUNT], + struct ane_h16_program **program); +int ane_h16_session_execute(struct ane_h16_session *s, struct ane_h16_program *p); +int ane_h16_session_close(struct ane_h16_session *s, struct ane_h16_program *p); +void ane_h16_session_account(struct ane_h16_session *s, u64 lock_ns, u64 copy_in_ns, + u64 copy_out_ns, u64 ioctl_ns); +void ane_h16_session_stats(struct ane_h16_session *s, struct ane_h16_exec_stats *out, + bool reset); + +#define ANE_H16_FWLOG_RECORD 0x80 +size_t ane_h16_fwlog_line(const u8 *record, char *line); + +#endif /* __ANE_H16_PROGRAM_H__ */ diff --git a/drivers/accel/ane/ane_h16_program_test.c b/drivers/accel/ane/ane_h16_program_test.c new file mode 100644 index 0000000000000..8b326b886f2e0 --- /dev/null +++ b/drivers/accel/ane/ane_h16_program_test.c @@ -0,0 +1,1667 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "ane_h16_boot.h" +#include "ane_h16_program.h" + +#include "ane_h16_hwx_test_fixture.h" +#include "ane_h16_add128_fixture.h" +#include "ane_h17_mobilenet_fixture.h" + +#define FW_IPC_DVA 0x100003d0000ULL +#define FW_POOL_DVA 0x10000384000ULL +#define FW_MAP_DVA 0x10001f00000ULL +#define FW_RING_SLOTS 8 +#define FW_REPORTS 0x8000 + +enum fw_mode { + FW_OK, + FW_LOAD_ID_UNCHANGED, + FW_LOAD_NAK, + FW_CREATE_BODY, + FW_REPORT_FOREIGN_UUID, + FW_REPORT_WRONG_PROCESS, + FW_REPORT_FAILURE, + FW_REPORT_STALL, + /* State 0 at once, then states 3 and 1 after late_polls sleeps. */ + FW_REPORT_LATE, +}; + +/* A synchronous fake firmware that answers on each doorbell. */ +struct prog_fw { + struct kunit *test; + struct ane_fw_platform soc; + struct ane_fw_surface ipc, pool; + struct ane_fw_transport t; + struct ane_h16_session s; + enum fw_mode mode; + u64 now; + /* Time each doorbell takes to answer. */ + u64 doorbell_ns; + u64 late_uuid; + u32 late_program, late_process; + unsigned int late_polls; + u64 next_dva; + u32 head[ANE_FW_CHANNELS]; + u32 reports; + /* First IDs; the firmware hands out the lowest free one, reused after teardown. */ + u32 program, process; + u32 programs_live, processes_live; + u16 ids[32]; + u32 values[32]; + unsigned int count; + u8 exec[0xd00]; + size_t exec_length; + u32 exec_flags[32]; + unsigned int execs; + u8 load[ANE_H16_LOAD_SIZE]; + unsigned int maps, writable_maps; + /* The EXECUTE failure trace lines, by step. */ + u64 fail_step, fail_flags, fail_reports, fail_state, fail_other; +}; + +static u64 fw_load(void *ctx, const void *p) +{ + return get_unaligned_le64(p); +} + +static void fw_store(void *ctx, void *p, u64 value) +{ + put_unaligned_le64(value, p); +} + +static void fw_barrier(void *ctx) +{ +} + +static u32 fw_read32(void *ctx, u32 reg) +{ + return 0; +} + +static u8 *fw_slot(struct prog_fw *f, unsigned int ch) +{ + return (u8 *)f->ipc.cpu + f->t.channels[ch].offset + f->head[ch] * ANE_FW_SLOT_SIZE; +} + +static void fw_advance(struct prog_fw *f, unsigned int ch) +{ + f->head[ch] = (f->head[ch] + 1) % f->t.channels[ch].slots; +} + +static void fw_report(struct prog_fw *f, u64 uuid, u32 program, u32 process, u32 state) +{ + u32 offset = FW_REPORTS + (f->reports++ % 16) * 0x60; + u8 *r = (u8 *)f->ipc.cpu + offset, *slot = fw_slot(f, ANE_FW_IO_T2H); + + memset(r, 0, 0x60); + put_unaligned_le32(0xfe00 + (state == 1), r); + put_unaligned_le16(0x300, r + 4); + put_unaligned_le64(uuid, r + 8); + put_unaligned_le32(program, r + 0x10); + put_unaligned_le32(process, r + 0x14); + put_unaligned_le32(state, r + 0x1c); + put_unaligned_le64(0x60, slot + 8); + put_unaligned_le64(0, slot + 16); + put_unaligned_le64(f->ipc.dva + offset, slot); + fw_advance(f, ANE_FW_IO_T2H); +} + +/* Reports carry the IDs of the EXECUTE they answer. */ +static void fw_execute(struct prog_fw *f, const u8 *cmd) +{ + u64 uuid = get_unaligned_le64(cmd + 0x20); + u32 program = get_unaligned_le32(cmd + 8), process = get_unaligned_le32(cmd + 0xc); + + if (f->execs < ARRAY_SIZE(f->exec_flags)) + f->exec_flags[f->execs] = get_unaligned_le32(cmd + ANE_H16_EXECUTE_FLAGS); + f->execs++; + switch (f->mode) { + case FW_REPORT_FOREIGN_UUID: + fw_report(f, uuid + 100, program, process, 0); + fw_report(f, uuid + 100, program, process, 1); + break; + case FW_REPORT_WRONG_PROCESS: + fw_report(f, uuid, program, process, 0); + fw_report(f, uuid, program, process + 1, 1); + break; + case FW_REPORT_FAILURE: + fw_report(f, uuid, program, process, 0); + fw_report(f, uuid, program, process, 2); + break; + case FW_REPORT_LATE: + fw_report(f, uuid, program, process, 0); + f->late_uuid = uuid; + f->late_program = program; + f->late_process = process; + break; + case FW_REPORT_STALL: + /* Started on the engine, then never a terminal report. */ + fw_report(f, uuid, program, process, 0); + fw_report(f, uuid, program, process, 3); + break; + default: + fw_report(f, uuid, program, process, 0); + fw_report(f, uuid, program, process, 3); + fw_report(f, uuid, program, process, 1); + break; + } +} + +static u32 fw_take_id(u32 *live, u32 base) +{ + unsigned int i = ffz(*live); + + *live |= BIT(i); + return base + i; +} + +static void fw_command(struct prog_fw *f, unsigned int ch) +{ + u8 *slot = fw_slot(f, ch), *cmd; + u64 dva = get_unaligned_le64(slot), length = get_unaligned_le64(slot + 8); + u16 id; + + if (dva & 1) + return; + cmd = (u8 *)f->pool.cpu + (dva - f->pool.dva); + id = get_unaligned_le16(cmd + 4); + if (f->count < ARRAY_SIZE(f->ids)) { + f->ids[f->count] = id; + f->values[f->count] = length >= 12 ? get_unaligned_le32(cmd + 8) : 0; + f->count++; + } + switch (id) { + case ANE_H16_LOAD_PROGRAM: + memcpy(f->load, cmd, sizeof(f->load)); + if (f->mode == FW_LOAD_NAK) + put_unaligned_le16(1, cmd + 6); + else if (f->mode != FW_LOAD_ID_UNCHANGED) + put_unaligned_le32(fw_take_id(&f->programs_live, f->program), + cmd + ANE_H16_LOAD_ID_OFFSET); + break; + case ANE_H16_UNLOAD_PROGRAM: + if (get_unaligned_le32(cmd + 8) - f->program < 32) + f->programs_live &= ~BIT(get_unaligned_le32(cmd + 8) - f->program); + break; + case ANE_H16_TERMINATE_PROCESS: + if (get_unaligned_le32(cmd + 0xc) - f->process < 32) + f->processes_live &= ~BIT(get_unaligned_le32(cmd + 0xc) - f->process); + break; + case ANE_H16_CREATE_PROCESS: + KUNIT_EXPECT_FALSE(f->test, memchr_inv(cmd + 0xc, 0, ANE_H16_CREATE_SIZE - 0xc)); + put_unaligned_le32(fw_take_id(&f->processes_live, f->process), cmd + 0xc); + if (f->mode == FW_CREATE_BODY) + cmd[0x100] = 1; + break; + case ANE_H16_EXECUTE: + f->exec_length = min_t(size_t, length, sizeof(f->exec)); + memcpy(f->exec, cmd, f->exec_length); + break; + } + put_unaligned_le64(length, slot + 8); + put_unaligned_le64(0, slot + 16); + put_unaligned_le64(dva | 1, slot); + fw_advance(f, ch); + if (id == ANE_H16_EXECUTE) + fw_execute(f, cmd); +} + +static void fw_write32(void *ctx, u32 reg, u32 value) +{ + struct prog_fw *f = ctx; + + if (reg != f->soc.doorbell) + return; + /* Commands take doorbell_ns to answer; returning report slots is free. */ + if (value & (BIT(ANE_FW_IO) | BIT(ANE_FW_DATA_CHAIN))) + f->now += f->doorbell_ns; + if (value & BIT(ANE_FW_IO)) + fw_command(f, ANE_FW_IO); + if (value & BIT(ANE_FW_DATA_CHAIN)) + fw_command(f, ANE_FW_DATA_CHAIN); +} + +static u64 fw_now(void *ctx) +{ + return ((struct prog_fw *)ctx)->now; +} + +static void fw_relax(void *ctx, u64 waited_ns) +{ + struct prog_fw *f = ctx; + + f->now += 10; + if (f->late_uuid && !--f->late_polls) { + fw_report(f, f->late_uuid, f->late_program, f->late_process, 3); + fw_report(f, f->late_uuid, f->late_program, f->late_process, 1); + f->late_uuid = 0; + } +} + +static int fw_alloc(void *ctx, size_t size, u32 tag, struct ane_fw_surface *s) +{ + return -ENOMEM; +} + +static void fw_refused(void *ctx, u64 w0, u64 w1, u64 w2, int error) +{ +} + +static const struct ane_fw_transport_ops fw_ops = { + .load64 = fw_load, + .store64 = fw_store, + .read_barrier = fw_barrier, + .write_barrier = fw_barrier, + .read32 = fw_read32, + .write32 = fw_write32, + .now_ns = fw_now, + .relax = fw_relax, + .alloc = fw_alloc, + .refused = fw_refused, +}; + +static void fw_kvfree(void *p) +{ + kvfree(p); +} + +/* Tests own the memory; a MobileNet kernel region exceeds kmalloc's limit. */ +static void *fw_zalloc(struct kunit *test, size_t size) +{ + void *p = kvzalloc(size, GFP_KERNEL); + + if (p && kunit_add_action_or_reset(test, fw_kvfree, p)) + return NULL; + return p; +} + +static int fw_map(void *ctx, size_t size, bool write, struct ane_fw_surface *s) +{ + struct prog_fw *f = ctx; + + *s = (struct ane_fw_surface) { + .cpu = fw_zalloc(f->test, size), + .dva = f->next_dva, + .size = size, + }; + if (!s->cpu) + return -ENOMEM; + f->next_dva += size; + f->maps++; + f->writable_maps += write; + return 0; +} + +static void fw_log(void *ctx, const char *item, u64 a, u64 b) +{ + struct prog_fw *f = ctx; + + if (!strcmp(item, "EXECUTE step/flags")) { + f->fail_step = a; + f->fail_flags = b; + } else if (!strcmp(item, "EXECUTE reports/last state")) { + f->fail_reports = a; + f->fail_state = b; + } else if (!strcmp(item, "EXECUTE other reports")) { + f->fail_other = a; + } +} + +static const struct ane_h16_session_ops fw_session_ops = { + .map = fw_map, + .log = fw_log, +}; + +static struct prog_fw *fw_create_family(struct kunit *test, unsigned int family) +{ + static const char * const names[] = { + "TERMINAL", "IO", "DEBUG", "BUF_H2T", "BUF_T2H", + "SHAREDMALLOC", "IO_T2H", "DATA_CHAIN_H2T", + }; + struct prog_fw *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + unsigned int i; + + if (!f) + return NULL; + f->test = test; + f->soc.chip_family = family; + f->soc.hwx_subtype = family == 17 ? ANE_HWX_SUBTYPE_H17 : ANE_HWX_SUBTYPE_H16; + f->soc.doorbell = 0x1884000; + f->soc.irq_status = 0x188c000; + f->soc.irq_ack = 0x1890000; + f->ipc = (struct ane_fw_surface) { + .cpu = kunit_kzalloc(test, SZ_64K, GFP_KERNEL), + .dva = FW_IPC_DVA, .size = SZ_64K, + }; + f->pool = (struct ane_fw_surface) { + .cpu = kunit_kzalloc(test, SZ_256K, GFP_KERNEL), + .dva = FW_POOL_DVA, .size = SZ_256K, + }; + if (!f->ipc.cpu || !f->pool.cpu) + return NULL; + for (i = 0; i < ANE_FW_CHANNELS; i++) { + u8 *p = (u8 *)f->ipc.cpu + i * ANE_FW_DESCRIPTOR_SIZE; + + strscpy((char *)p, names[i], 64); + put_unaligned_le32(i == 0 ? 2 : (i >= 4 && i <= 6), p + 0x40); + put_unaligned_le32(i, p + 0x44); + put_unaligned_le64(FW_RING_SLOTS, p + 0x48); + put_unaligned_le64(f->ipc.dva + 0x800 + i * 0x200, p + 0x50); + } + f->next_dva = FW_MAP_DVA; + f->program = 0x11; + f->process = 0; + if (ane_fw_transport_init(&f->t, &f->soc, &fw_ops, f, f->ipc, f->pool, SZ_16K, + f->ipc.dva, ANE_FW_TABLE_SIZE) || + ane_fw_transport_arm(&f->t) || + ane_h16_session_init(&f->s, &f->t, &fw_session_ops, f, 1000)) + return NULL; + return f; +} + +static struct prog_fw *fw_create(struct kunit *test) +{ + return fw_create_family(test, 16); +} + +static void snapshot_free(void *snapshot) +{ + ane_h16_hwx_snapshot_destroy(snapshot); +} + +static struct ane_h16_hwx_snapshot *fw_snapshot(struct kunit *test, bool mixed, bool kernel) +{ + struct hwx_fixture *x = fixture_create(test, mixed, kernel); + struct ane_h16_hwx_snapshot *snapshot; + + if (!x) + return NULL; + snapshot = kunit_kzalloc(test, sizeof(*snapshot), GFP_KERNEL); + if (!snapshot || ane_h16_hwx_snapshot_create(x->bytes, x->size, snapshot) || + kunit_add_action_or_reset(test, snapshot_free, snapshot)) + return NULL; + return snapshot; +} + +/* + * Generic section with ID base + k for surface k, then base + count for the + * scratch, as a host builder writes it. + */ +static size_t generic_fill(u8 *g, const struct ane_h16_hwx *hwx, u32 base) +{ + unsigned int i, entries = hwx->surface_count + !!hwx->scratch.size; + + memset(g, 0, ANE_H16_GENERIC_HEADER + entries * ANE_H16_GENERIC_ENTRY); + put_unaligned_le32(ANE_H16_GENERIC_MAGIC, g); + put_unaligned_le32(ANE_H16_GENERIC_VERSION, g + 4); + put_unaligned_le32(entries, g + ANE_H16_GENERIC_COUNT); + for (i = 0; i < entries; i++) { + u8 *e = g + ANE_H16_GENERIC_HEADER + i * ANE_H16_GENERIC_ENTRY; + bool scratch = i == hwx->surface_count; + u32 direction = scratch ? 1 : hwx->surfaces[i].direction; + + put_unaligned_le32(1, e); + put_unaligned_le32(base + i, e + 4); + put_unaligned_le32(direction - 1, e + 8); + put_unaligned_le32(direction, e + 0x10); + put_unaligned_le64(scratch ? hwx->scratch.size : hwx->surfaces[i].span, e + 0x20); + put_unaligned_le32(0xffff, e + 0x28); + } + return ANE_H16_GENERIC_HEADER + entries * ANE_H16_GENERIC_ENTRY; +} + +/* + * Operation section with one record that binds every program-descriptor + * slot whose base is a surface, the scratch or the kernel, with the IDs that + * generic_fill() gives. + */ +static size_t operation_fill(u8 *o, const struct ane_h16_hwx_snapshot *snapshot, + const struct ane_h16_hwx *hwx, u32 base) +{ + const u8 *c = (const u8 *)snapshot->bytes + hwx->program_offset; + unsigned int slot, i, refs = 0; + + memset(o, 0, 4 + ANE_H16_OPERATION_RECORD); + put_unaligned_le32(1, o); + for (slot = 0; slot < 64; slot++) { + u64 vm = get_unaligned_le64(c + 0x10 * (slot + 1)); + u32 tag = 0; + + if (!vm) + continue; + if (hwx->kernel.size && vm == hwx->kernel.vm) + tag = ANE_H16_RESOURCE_KERNEL; + if (hwx->scratch.size && vm == hwx->scratch.vm) + tag = base + hwx->surface_count; + for (i = 0; i < hwx->surface_count; i++) + if (vm == hwx->surfaces[i].section.vm) + tag = base + i; + if (!tag) + continue; + put_unaligned_le32(slot, o + 4 + 0xc + refs * 8); + put_unaligned_le32(tag, o + 4 + 0x10 + refs * 8); + refs++; + } + put_unaligned_le32(refs, o + 4 + 8); + return 4 + ANE_H16_OPERATION_RECORD; +} + +static u8 test_aux[ANE_H16_AUX_COUNT][0x410]; +static const void *const test_aux_ptr[ANE_H16_AUX_COUNT] = { + test_aux[0], test_aux[1], test_aux[2], test_aux[3], +}; + +static size_t test_aux_size[ANE_H16_AUX_COUNT] = { 0x298, 0x410, 0x38, 0x28 }; + +#define TEST_GENERIC_BASE 8 + +static int fw_load_snapshot(struct prog_fw *f, struct ane_h16_hwx_snapshot *snapshot, + struct ane_h16_program **p) +{ + struct ane_h16_hwx hwx; + + if (!snapshot || ane_h16_hwx_validate(snapshot->bytes, snapshot->size, &hwx)) + return -ENOMEM; + test_aux_size[ANE_H16_AUX_GENERIC] = + generic_fill(test_aux[ANE_H16_AUX_GENERIC], &hwx, TEST_GENERIC_BASE); + test_aux_size[ANE_H16_AUX_OPERATION] = + operation_fill(test_aux[ANE_H16_AUX_OPERATION], snapshot, &hwx, + TEST_GENERIC_BASE); + return ane_h16_session_load(&f->s, snapshot, test_aux_ptr, test_aux_size, p); +} + +static int fw_load_program(struct prog_fw *f, struct ane_h16_program **p) +{ + return fw_load_snapshot(f, fw_snapshot(f->test, false, false), p); +} + +static void expect_slot(struct kunit *test, const u8 *load, unsigned int slot, u32 resource, + u32 class, u64 dva, u64 size) +{ + const u8 *d = load + ANE_H16_LOAD_SLOT_OFFSET + slot * ANE_H16_LOAD_SLOT_SIZE; + + KUNIT_EXPECT_EQ(test, get_unaligned_le32(d), 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(d + 4), resource); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(d + 8), class); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(d + 0xc), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(d + 0x10), 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(d + 0x14), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(d + 0x18), dva); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(d + 0x20), size); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(d + 0x28), 0); +} + +static void expect_empty(struct kunit *test, const u8 *load, unsigned int slot) +{ + const u8 *d = load + ANE_H16_LOAD_SLOT_OFFSET + slot * ANE_H16_LOAD_SLOT_SIZE; + + KUNIT_EXPECT_FALSE(test, memchr_inv(d, 0, ANE_H16_LOAD_SLOT_SIZE)); +} + +/* add-128 shape: input, output, input surfaces of 32 KiB, no kernel section. */ +static void program_add128_slots_test(struct kunit *test) +{ + static const u32 direction[3] = { 1, 2, 1 }; + struct ane_h16_process_binding bindings[ANE_H16_SURFACES], scratch; + struct ane_h16_load_slot slots[ANE_H16_LOAD_SLOTS]; + struct ane_h16_load_layout layout = {}; + struct ane_h16_load_plan *plan = kunit_kzalloc(test, sizeof(*plan), GFP_KERNEL); + struct ane_h16_hwx *hwx = kunit_kzalloc(test, sizeof(*hwx), GFP_KERNEL); + u8 load[ANE_H16_LOAD_SIZE]; + u8 *generic = kunit_kzalloc(test, ANE_H16_GENERIC_HEADER + 3 * ANE_H16_GENERIC_ENTRY, + GFP_KERNEL); + size_t generic_size; + unsigned int i, count; + + KUNIT_ASSERT_NOT_NULL(test, plan); + KUNIT_ASSERT_NOT_NULL(test, hwx); + KUNIT_ASSERT_NOT_NULL(test, generic); + + for (i = 0; i < 3; i++) { + hwx->surfaces[i].section.vm = 0x30000000 + i * SZ_32K; + hwx->surfaces[i].direction = direction[i]; + hwx->surfaces[i].span = SZ_32K; + hwx->surfaces[i].allocation = SZ_32K; + plan->regions[i] = (struct ane_h16_hwx_region) { + .vm = hwx->surfaces[i].section.vm, .size = SZ_32K, + .permissions = direction[i], .kind = ANE_H16_REGION_SURFACE, + }; + plan->allocation[i] = SZ_32K; + } + hwx->surface_count = 3; + hwx->text = (struct ane_h16_hwx_section) { .vm = 0x30018000, .size = 0x140 }; + hwx->constants = (struct ane_h16_hwx_section) { .vm = 0x30018140, .size = 0x4000 }; + plan->regions[3] = (struct ane_h16_hwx_region) { + .vm = 0x30018000, .size = 0x4140, .file_size = 0x4140, + .permissions = 5, .kind = ANE_H16_REGION_TEXT, + }; + plan->allocation[3] = SZ_32K; + plan->count = 4; + for (i = 0; i < plan->count; i++) + layout.regions[i] = (struct ane_h16_load_mapping) { + .dva = 0x10002000000ULL + i * SZ_1M, .size = plan->allocation[i], + .permissions = plan->regions[i].permissions, + }; + layout.region_count = plan->count; + for (i = 0; i < ANE_H16_AUX_COUNT; i++) { + layout.aux_dva[i] = 0x10003000000ULL + i * SZ_16K; + layout.aux_size[i] = test_aux_size[i]; + } + + /* IDs from the generic section: x = 5, y = 4, z = 6 as the add-128 builder writes. */ + generic_size = generic_fill(generic, hwx, 0); + put_unaligned_le32(5, generic + ANE_H16_GENERIC_HEADER + 4); + put_unaligned_le32(4, generic + ANE_H16_GENERIC_HEADER + ANE_H16_GENERIC_ENTRY + 4); + put_unaligned_le32(6, generic + ANE_H16_GENERIC_HEADER + 2 * ANE_H16_GENERIC_ENTRY + 4); + KUNIT_ASSERT_EQ(test, ane_h16_process_bindings(hwx, generic, generic_size, bindings, + &count, NULL), 0); + KUNIT_EXPECT_EQ(test, count, 3); + KUNIT_EXPECT_EQ(test, bindings[0].index, 5); + KUNIT_EXPECT_EQ(test, bindings[0].class, 0); + KUNIT_EXPECT_EQ(test, bindings[1].index, 4); + KUNIT_EXPECT_EQ(test, bindings[1].class, 1); + KUNIT_EXPECT_EQ(test, bindings[2].index, 6); + KUNIT_EXPECT_EQ(test, bindings[2].class, 0); + /* No scratch segment, no scratch binding. */ + KUNIT_EXPECT_EQ(test, scratch.index, 0); + + /* A generic section that disagrees with the program refuses. */ + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, generic, generic_size - 4, + bindings, &count, NULL), -ENOEXEC); + put_unaligned_le32(2, generic + ANE_H16_GENERIC_HEADER + 4); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, generic, generic_size, bindings, + &count, NULL), -ENOEXEC); + put_unaligned_le32(4, generic + ANE_H16_GENERIC_HEADER + 4); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, generic, generic_size, bindings, + &count, NULL), -ENOEXEC); + put_unaligned_le32(5, generic + ANE_H16_GENERIC_HEADER + 4); + put_unaligned_le32(1, generic + ANE_H16_GENERIC_HEADER + 8); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, generic, generic_size, bindings, + &count, NULL), -ENOEXEC); + put_unaligned_le32(0, generic + ANE_H16_GENERIC_HEADER + 8); + put_unaligned_le64(SZ_32K - 2, generic + ANE_H16_GENERIC_HEADER + 0x20); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, generic, generic_size, bindings, + &count, NULL), -ENOEXEC); + put_unaligned_le64(SZ_32K, generic + ANE_H16_GENERIC_HEADER + 0x20); + put_unaligned_le32(2, generic + ANE_H16_GENERIC_COUNT); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, generic, generic_size, bindings, + &count, NULL), -ENOEXEC); + put_unaligned_le32(3, generic + ANE_H16_GENERIC_COUNT); + put_unaligned_le32(0x11, generic + 4); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, generic, generic_size, bindings, + &count, NULL), -ENOEXEC); + put_unaligned_le32(ANE_H16_GENERIC_VERSION, generic + 4); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, generic, generic_size, bindings, + &count, NULL), 0); + + KUNIT_ASSERT_EQ(test, ane_h16_load_slots(hwx, plan, &layout, slots), 0); + KUNIT_ASSERT_EQ(test, ane_h16_load_encode(load, sizeof(load), slots), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(load), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(load + 4), ANE_H16_LOAD_PROGRAM); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(load + 6), 0); + expect_slot(test, load, 0, 1002, 4, layout.aux_dva[0], layout.aux_size[0]); + /* No kernel section: slot 1 carries the constants at their sub-page DVA. */ + expect_slot(test, load, 1, 1007, 3, layout.regions[3].dva + 0x140, 0x4000); + expect_slot(test, load, 2, 1006, 4, layout.regions[3].dva, 0x140); + expect_slot(test, load, 3, 1003, 4, layout.aux_dva[1], test_aux_size[1]); + expect_slot(test, load, 4, 1004, 4, layout.aux_dva[2], 0x38); + expect_empty(test, load, 5); + expect_slot(test, load, 6, 1005, 4, layout.aux_dva[3], 0x28); + expect_empty(test, load, 7); + expect_empty(test, load, 8); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(load + 0x1b8), 0xffffffff); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(load + 0x1bc), 0); + + /* Missing aux, short aux, mismatched layout and wrong length refuse. */ + layout.aux_dva[2] = 0; + KUNIT_EXPECT_EQ(test, ane_h16_load_slots(hwx, plan, &layout, slots), -EINVAL); + layout.aux_dva[2] = 0x10003008000ULL; + layout.aux_size[3] = ANE_H16_AUX_MIN - 4; + KUNIT_EXPECT_EQ(test, ane_h16_load_slots(hwx, plan, &layout, slots), -EINVAL); + layout.aux_size[3] = 0x28; + layout.regions[1].size = SZ_16K; + KUNIT_EXPECT_EQ(test, ane_h16_load_slots(hwx, plan, &layout, slots), -EINVAL); + layout.regions[1].size = SZ_32K; + layout.region_count = 3; + KUNIT_EXPECT_EQ(test, ane_h16_load_slots(hwx, plan, &layout, slots), -EINVAL); + KUNIT_EXPECT_EQ(test, ane_h16_load_encode(load, sizeof(load) - 8, slots), -EINVAL); + + /* A program needs an input and exactly one output. */ + hwx->surfaces[0].direction = 2; + generic_size = generic_fill(generic, hwx, 4); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, generic, generic_size, bindings, + &count, NULL), -ENOEXEC); + hwx->surfaces[0].direction = 3; + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, generic, generic_size, bindings, + &count, NULL), -ENOEXEC); +} + +static u8 *sparse_bytes(struct kunit *test, const struct ane_h16_sparse_run *runs, + unsigned int count, size_t size) +{ + u8 *bytes = kunit_kzalloc(test, size, GFP_KERNEL); + unsigned int i; + + if (!bytes) + return NULL; + for (i = 0; i < count; i++) { + if (runs[i].offset + runs[i].length > size) + return NULL; + memcpy(bytes + runs[i].offset, runs[i].bytes, runs[i].length); + } + return bytes; +} + +/* + * The pinned add-128 program and the sections h16-sections.py builds for it + * load and run against the fake firmware. Segments are x (in), y (out), z (in) + * while the tensors list x, z, y; EXECUTE names x = 5, y = 4, z = 6. + */ +static void program_add128_hwx_test(struct kunit *test) +{ + static const u32 ids[3] = { 5, 4, 6 }, classes[3] = { 0, 1, 0 }; + struct ane_h16_hwx_snapshot *snapshot; + struct ane_h16_program *p = NULL; + const void *aux[ANE_H16_AUX_COUNT]; + size_t aux_size[ANE_H16_AUX_COUNT] = { + ADD128_GENERIC_SIZE, ADD128_OPERATION_SIZE, + ADD128_PROCEDURE_SIZE, ADD128_PROPERTIES_SIZE, + }; + struct prog_fw *f = fw_create(test); + u8 *hwx; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + hwx = sparse_bytes(test, add128_hwx_runs, ARRAY_SIZE(add128_hwx_runs), ADD128_HWX_SIZE); + aux[0] = sparse_bytes(test, add128_generic_runs, ARRAY_SIZE(add128_generic_runs), + ADD128_GENERIC_SIZE); + aux[1] = sparse_bytes(test, add128_operation_runs, ARRAY_SIZE(add128_operation_runs), + ADD128_OPERATION_SIZE); + aux[2] = sparse_bytes(test, add128_procedure_runs, ARRAY_SIZE(add128_procedure_runs), + ADD128_PROCEDURE_SIZE); + aux[3] = sparse_bytes(test, add128_properties_runs, ARRAY_SIZE(add128_properties_runs), + ADD128_PROPERTIES_SIZE); + KUNIT_ASSERT_NOT_NULL(test, hwx); + for (i = 0; i < ANE_H16_AUX_COUNT; i++) + KUNIT_ASSERT_NOT_NULL(test, aux[i]); + snapshot = kunit_kzalloc(test, sizeof(*snapshot), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, snapshot); + KUNIT_ASSERT_EQ(test, ane_h16_hwx_snapshot_create(hwx, ADD128_HWX_SIZE, snapshot), 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, snapshot_free, snapshot), 0); + KUNIT_ASSERT_MEMEQ(test, snapshot->digest, add128_hwx_sha256, SHA256_DIGEST_SIZE); + + KUNIT_ASSERT_EQ(test, ane_h16_session_load(&f->s, snapshot, aux, aux_size, &p), 0); + KUNIT_ASSERT_NOT_NULL(test, p); + KUNIT_EXPECT_EQ(test, p->hwx.surface_count, 3); + KUNIT_EXPECT_EQ(test, p->hwx.surfaces[1].direction, 2); + KUNIT_EXPECT_EQ(test, p->resource_count, 3); + expect_slot(test, f->load, 0, 1002, 4, p->aux[0].dva, ADD128_GENERIC_SIZE); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->load + ANE_H16_LOAD_SLOT_OFFSET + + ANE_H16_LOAD_SLOT_SIZE + 8), 3); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(f->load + ANE_H16_LOAD_SLOT_OFFSET + + ANE_H16_LOAD_SLOT_SIZE + 0x20), 0x4000); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(f->load + ANE_H16_LOAD_SLOT_OFFSET + + 2 * ANE_H16_LOAD_SLOT_SIZE + 0x20), 0x130); + expect_slot(test, f->load, 3, 1003, 4, p->aux[1].dva, ADD128_OPERATION_SIZE); + expect_slot(test, f->load, 4, 1004, 4, p->aux[2].dva, ADD128_PROCEDURE_SIZE); + expect_empty(test, f->load, 5); + expect_slot(test, f->load, 6, 1005, 4, p->aux[3].dva, ADD128_PROPERTIES_SIZE); + expect_empty(test, f->load, 7); + expect_empty(test, f->load, 8); + /* Only the output is device-writable. */ + KUNIT_EXPECT_EQ(test, f->writable_maps, 1); + + for (i = 0; i < 2; i++) + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, p), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0x28), 3); + for (i = 0; i < 3; i++) { + const u8 *d = f->exec + 0x60 + i * 0x30; + + KUNIT_EXPECT_EQ(test, get_unaligned_le32(d + 4), ids[i]); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(d + 8), classes[i]); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(d + 0x10), 3); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(d + 0x18), p->surfaces[i]->dva); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(d + 0x20), SZ_32K); + } + KUNIT_EXPECT_FALSE(test, f->s.quarantined); + KUNIT_EXPECT_EQ(test, ane_h16_session_close(&f->s, p), 0); +} + +/* Synthetic HWX with a kernel section: slot 1 is class 3 with index 1007. */ +static void program_kernel_slots_test(struct kunit *test) +{ + struct ane_h16_hwx_snapshot *snapshot = fw_snapshot(test, false, true); + struct ane_h16_load_slot slots[ANE_H16_LOAD_SLOTS]; + struct ane_h16_load_layout layout = {}; + struct ane_h16_load_plan *plan = kunit_kzalloc(test, sizeof(*plan), GFP_KERNEL); + struct ane_h16_hwx *hwx = kunit_kzalloc(test, sizeof(*hwx), GFP_KERNEL); + unsigned int i, kernel = ANE_H16_REGIONS, text = ANE_H16_REGIONS; + + KUNIT_ASSERT_NOT_NULL(test, snapshot); + KUNIT_ASSERT_NOT_NULL(test, plan); + KUNIT_ASSERT_NOT_NULL(test, hwx); + KUNIT_ASSERT_EQ(test, ane_h16_load_plan_create(snapshot, plan), 0); + KUNIT_ASSERT_EQ(test, ane_h16_hwx_validate(snapshot->bytes, snapshot->size, hwx), 0); + for (i = 0; i < plan->count; i++) { + layout.regions[i] = (struct ane_h16_load_mapping) { + .dva = 0x10002000000ULL + i * SZ_1M, .size = plan->allocation[i], + .permissions = plan->regions[i].permissions, + }; + if (plan->regions[i].kind == ANE_H16_REGION_KERNEL) + kernel = i; + if (plan->regions[i].kind == ANE_H16_REGION_TEXT) + text = i; + } + KUNIT_ASSERT_LT(test, kernel, ANE_H16_REGIONS); + KUNIT_ASSERT_LT(test, text, ANE_H16_REGIONS); + layout.region_count = plan->count; + for (i = 0; i < ANE_H16_AUX_COUNT; i++) { + layout.aux_dva[i] = 0x10003000000ULL + i * SZ_16K; + layout.aux_size[i] = test_aux_size[i]; + } + KUNIT_ASSERT_EQ(test, ane_h16_load_slots(hwx, plan, &layout, slots), 0); + KUNIT_EXPECT_TRUE(test, slots[1].present); + KUNIT_EXPECT_EQ(test, slots[1].resource, 1007); + KUNIT_EXPECT_EQ(test, slots[1].class, 3); + KUNIT_EXPECT_EQ(test, slots[1].dva, layout.regions[kernel].dva); + /* The kernel slot spans the section's whole 16 KiB pages. */ + KUNIT_EXPECT_EQ(test, slots[1].size, + min_t(u64, ALIGN(hwx->kernel.size, SZ_16K), plan->allocation[kernel])); + KUNIT_EXPECT_EQ(test, hwx->kernel.vm, plan->regions[kernel].vm); + KUNIT_EXPECT_EQ(test, slots[2].resource, slots[1].resource - 1); + KUNIT_EXPECT_EQ(test, slots[2].dva, + layout.regions[text].dva + (hwx->text.vm - plan->regions[text].vm)); + KUNIT_EXPECT_EQ(test, slots[2].size, hwx->text.size); + for (i = 0; i < ANE_H16_LOAD_SLOTS; i++) + KUNIT_EXPECT_EQ(test, slots[i].present, i != 5 && i < 7); +} + +static void program_replies_test(struct kunit *test) +{ + struct ane_h16_load_slot slots[ANE_H16_LOAD_SLOTS] = {}; + u8 request[ANE_H16_CREATE_SIZE], reply[ANE_H16_CREATE_SIZE]; + u32 id; + + slots[0] = (struct ane_h16_load_slot) { + .present = true, .resource = 1002, .class = 4, + .dva = 0x10002000000ULL, .size = 0x298, + }; + KUNIT_ASSERT_EQ(test, ane_h16_load_encode(request, ANE_H16_LOAD_SIZE, slots), 0); + memcpy(reply, request, ANE_H16_LOAD_SIZE); + /* ID 0 is valid. */ + put_unaligned_le32(0, reply + 0x1b8); + KUNIT_EXPECT_EQ(test, ane_h16_load_reply(reply, ANE_H16_LOAD_SIZE, request, &id), 0); + KUNIT_EXPECT_EQ(test, id, 0); + /* Request ID left unchanged. */ + put_unaligned_le32(0xffffffff, reply + 0x1b8); + KUNIT_EXPECT_EQ(test, ane_h16_load_reply(reply, ANE_H16_LOAD_SIZE, request, &id), -EPROTO); + put_unaligned_le32(0x61, reply + 0x1b8); + /* Nonzero acknowledgment, wrong length, wrong command, altered slot echo. */ + put_unaligned_le16(1, reply + 6); + KUNIT_EXPECT_EQ(test, ane_h16_load_reply(reply, ANE_H16_LOAD_SIZE, request, &id), + -EREMOTEIO); + put_unaligned_le16(0, reply + 6); + KUNIT_EXPECT_EQ(test, ane_h16_load_reply(reply, ANE_H16_LOAD_SIZE - 8, request, &id), + -EPROTO); + put_unaligned_le16(ANE_H16_CREATE_PROCESS, reply + 4); + KUNIT_EXPECT_EQ(test, ane_h16_load_reply(reply, ANE_H16_LOAD_SIZE, request, &id), -EPROTO); + put_unaligned_le16(ANE_H16_LOAD_PROGRAM, reply + 4); + reply[0x20] ^= 1; + KUNIT_EXPECT_EQ(test, ane_h16_load_reply(reply, ANE_H16_LOAD_SIZE, request, &id), -EPROTO); + reply[0x20] ^= 1; + KUNIT_EXPECT_EQ(test, ane_h16_load_reply(reply, ANE_H16_LOAD_SIZE, request, &id), 0); + KUNIT_EXPECT_EQ(test, id, 0x61); + + /* CREATE: zero body, process ID 0 valid, any nonzero body refused. */ + KUNIT_ASSERT_EQ(test, ane_h16_create_encode(request, ANE_H16_CREATE_SIZE, 0x61), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(request + 4), ANE_H16_CREATE_PROCESS); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(request + 8), 0x61); + KUNIT_EXPECT_FALSE(test, memchr_inv(request + 0xc, 0, ANE_H16_CREATE_SIZE - 0xc)); + memcpy(reply, request, ANE_H16_CREATE_SIZE); + KUNIT_EXPECT_EQ(test, ane_h16_create_reply(reply, ANE_H16_CREATE_SIZE, 0x61, &id), 0); + KUNIT_EXPECT_EQ(test, id, 0); + reply[0x257] = 1; + KUNIT_EXPECT_EQ(test, ane_h16_create_reply(reply, ANE_H16_CREATE_SIZE, 0x61, &id), + -EPROTO); + reply[0x257] = 0; + KUNIT_EXPECT_EQ(test, ane_h16_create_reply(reply, ANE_H16_CREATE_SIZE, 0x60, &id), + -EPROTO); + KUNIT_EXPECT_EQ(test, ane_h16_create_reply(reply, 0x10, 0x61, &id), -EPROTO); + put_unaligned_le16(2, reply + 6); + KUNIT_EXPECT_EQ(test, ane_h16_create_reply(reply, ANE_H16_CREATE_SIZE, 0x61, &id), + -EREMOTEIO); + KUNIT_EXPECT_EQ(test, ane_h16_create_encode(request, 0x10, 0x61), -EINVAL); + + /* TERMINATE 0x10 and UNLOAD 0xc carry IDs that the reply must echo. */ + KUNIT_ASSERT_EQ(test, ane_h16_terminate_encode(request, 0x10, 0x61, 0), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(request + 4), ANE_H16_TERMINATE_PROCESS); + memcpy(reply, request, 0x10); + KUNIT_EXPECT_EQ(test, ane_h16_id_reply(reply, 0x10, request), 0); + put_unaligned_le16(1, reply + 6); + KUNIT_EXPECT_EQ(test, ane_h16_id_reply(reply, 0x10, request), -EREMOTEIO); + put_unaligned_le16(0, reply + 6); + put_unaligned_le16(ANE_H16_UNLOAD_PROGRAM, reply + 4); + KUNIT_EXPECT_EQ(test, ane_h16_id_reply(reply, 0x10, request), -EPROTO); + KUNIT_ASSERT_EQ(test, ane_h16_unload_encode(request, 0xc, 0x61), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(request + 4), ANE_H16_UNLOAD_PROGRAM); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(request + 8), 0x61); + KUNIT_EXPECT_EQ(test, ane_h16_unload_encode(request, 0x10, 0x61), -EINVAL); + KUNIT_EXPECT_EQ(test, ane_h16_gate_encode(request, 0xc, 2), -EINVAL); +} + +static void program_deny_list_test(struct kunit *test) +{ + static const u16 denied[] = { 0x27, 0x28, 0x30, 0x31, 0x34 }; + static const u16 allowed[] = { 0x0, 0x3, 0x4, 0x22, 0x2d, 0x2f, 0x200, 0x20c }; + struct prog_fw *f = fw_create(test); + u8 command[16] = {}; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < ARRAY_SIZE(denied); i++) { + KUNIT_EXPECT_TRUE(test, ane_fw_command_denied(denied[i])); + put_unaligned_le16(denied[i], command + 4); + KUNIT_EXPECT_EQ(test, ane_fw_transport_publish(&f->t, ANE_FW_IO, command, + sizeof(command), 100), -EPERM); + KUNIT_EXPECT_EQ(test, ane_fw_transport_publish(&f->t, ANE_FW_DATA_CHAIN, + command, sizeof(command), 100), + -EPERM); + } + for (i = 0; i < ARRAY_SIZE(allowed); i++) + KUNIT_EXPECT_FALSE(test, ane_fw_command_denied(allowed[i])); + /* Nothing reached a ring and the transport stays usable. */ + KUNIT_EXPECT_EQ(test, f->count, 0); + KUNIT_EXPECT_EQ(test, f->t.error, 0); + KUNIT_EXPECT_EQ(test, ane_h16_stage_parse("program"), ANE_H16_PROGRAM); +} + +static void program_custom_bar_test(struct kunit *test) +{ + struct ane_h16_hwx_snapshot *linear = fw_snapshot(test, false, false); + struct ane_h16_hwx_snapshot *mixed = fw_snapshot(test, true, false); + struct ane_h16_custom_bar bars[ANE_H16_MAX_CUSTOM_BARS]; + unsigned int count = 9; + + KUNIT_ASSERT_NOT_NULL(test, linear); + KUNIT_ASSERT_NOT_NULL(test, mixed); + KUNIT_ASSERT_EQ(test, ane_h16_custom_bars(linear, &linear->info, 36, bars, &count), 0); + KUNIT_ASSERT_EQ(test, count, 1); + KUNIT_EXPECT_EQ(test, bars[0].configuration, 1); + KUNIT_EXPECT_EQ(test, bars[0].value_type, 0); + KUNIT_EXPECT_EQ(test, bars[0].bar, 4); + KUNIT_EXPECT_EQ(test, bars[0].bit_offset, 7); + KUNIT_EXPECT_EQ(test, bars[0].bit_width, 8); + KUNIT_EXPECT_EQ(test, bars[0].flags, 0); + KUNIT_EXPECT_EQ(test, bars[0].value, 36); + /* The DSID must fit the field. */ + KUNIT_EXPECT_EQ(test, ane_h16_custom_bars(linear, &linear->info, 256, bars, &count), + -ENOEXEC); + /* Scratch form: no established tuple layout, no record. */ + KUNIT_ASSERT_EQ(test, ane_h16_custom_bars(mixed, &mixed->info, 36, bars, &count), 0); + KUNIT_EXPECT_EQ(test, count, 0); + /* Usage type to descriptor class, round 5 section 7. */ + KUNIT_EXPECT_EQ(test, ane_h16_usage_class(1), 0); + KUNIT_EXPECT_EQ(test, ane_h16_usage_class(2), 1); + KUNIT_EXPECT_EQ(test, ane_h16_usage_class(5), 6); + KUNIT_EXPECT_EQ(test, ane_h16_usage_class(7), 2); + KUNIT_EXPECT_EQ(test, ane_h16_usage_class(10), 4); + KUNIT_EXPECT_EQ(test, ane_h16_usage_class(13), 0); + KUNIT_EXPECT_EQ(test, ane_h16_usage_class(0), 8); + KUNIT_EXPECT_EQ(test, ane_h16_usage_class(99), 8); +} + +static void program_session_test(struct kunit *test) +{ + static const u16 expected[] = { + 0x200, 0x202, 0x2d, 0x20c, 0x2d, 0x2d, 0x20c, 0x2d, 0x203, 0x201, + }; + static const u32 gates[] = { 0, 0, 0, 0, 1, 0, 0, 1 }; + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL, *second = NULL, *third = NULL; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + KUNIT_ASSERT_NOT_NULL(test, p); + KUNIT_EXPECT_EQ(test, p->key.program, 0x11); + KUNIT_EXPECT_EQ(test, p->key.process, 0); + KUNIT_EXPECT_EQ(test, p->resource_count, 2); + /* Firmware saw the request ID and the synthetic slots. */ + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->load + 0x1b8), 0xffffffff); + expect_slot(test, f->load, 0, 1002, 4, p->aux[0].dva, + test_aux_size[ANE_H16_AUX_GENERIC]); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->load + ANE_H16_LOAD_SLOT_OFFSET + + ANE_H16_LOAD_SLOT_SIZE + 4), 1007); + expect_slot(test, f->load, 6, 1005, 4, p->aux[3].dva, 0x28); + /* Only the output surface is device-writable. */ + KUNIT_EXPECT_EQ(test, f->writable_maps, 1); + + KUNIT_EXPECT_EQ(test, ane_h16_session_execute(&f->s, p), 0); + /* The first request after boot sets up EXELOOP state: queued. */ + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0x30), ANE_H16_EXECUTE_FLAG_QUEUED); + KUNIT_EXPECT_EQ(test, ane_h16_session_execute(&f->s, p), 0); + /* EXECUTE: IDs, priority 5, UUID 2, two descriptors and one mcache record. */ + KUNIT_EXPECT_EQ(test, f->exec_length, 0xcc0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 8), 0x11); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0xc), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0x18), 5); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(f->exec + 0x20), 2); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0x28), 2); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0x60 + 8), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0x90 + 8), 1); + /* Descriptor IDs are the generic-section IDs. */ + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0x60 + 4), TEST_GENERIC_BASE); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0x90 + 4), TEST_GENERIC_BASE + 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(f->exec + 0x60 + 0x18), + p->surfaces[0]->dva); + /* H16: each descriptor covers its whole HWX section. */ + KUNIT_EXPECT_EQ(test, get_unaligned_le64(f->exec + 0x60 + 0x20), + p->hwx.surfaces[0].section.size); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0xca0), 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0xca8 + 8), 4); + /* Later requests of the same program take the direct path. */ + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0x30), 0); + + /* A second LOAD while the first is loaded refuses, before any traffic. */ + i = f->count; + KUNIT_EXPECT_EQ(test, fw_load_program(f, &second), -EBUSY); + KUNIT_EXPECT_EQ(test, f->count, i); + KUNIT_EXPECT_PTR_EQ(test, second, NULL); + + KUNIT_EXPECT_EQ(test, ane_h16_session_close(&f->s, p), 0); + KUNIT_EXPECT_TRUE(test, p->closed); + KUNIT_EXPECT_EQ(test, ane_h16_session_execute(&f->s, p), -EINVAL); + + KUNIT_ASSERT_EQ(test, f->count, ARRAY_SIZE(expected)); + for (i = 0; i < ARRAY_SIZE(expected); i++) + KUNIT_EXPECT_EQ(test, f->ids[i], expected[i]); + for (i = 0; i < ARRAY_SIZE(gates); i++) + if (f->ids[i] == 0x2d) + KUNIT_EXPECT_EQ(test, f->values[i], gates[i]); + + /* After TERMINATE and UNLOAD a new program loads in the same session. */ + KUNIT_ASSERT_EQ(test, fw_load_program(f, &second), 0); + KUNIT_ASSERT_NOT_NULL(test, second); + KUNIT_EXPECT_EQ(test, second->generation, 2); + /* The firmware reuses the program ID; the session tells them apart. */ + KUNIT_EXPECT_EQ(test, second->key.program, p->key.program); + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, second), 0); + /* The engine last ran the first program: this request is queued. */ + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0x30), ANE_H16_EXECUTE_FLAG_QUEUED); + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, second), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0x30), 0); + i = f->count; + KUNIT_EXPECT_EQ(test, fw_load_program(f, &third), -EBUSY); + KUNIT_EXPECT_EQ(test, f->count, i); + KUNIT_EXPECT_PTR_EQ(test, third, NULL); + KUNIT_EXPECT_EQ(test, ane_h16_session_close(&f->s, second), 0); + /* Closing the first program's file late is harmless. */ + KUNIT_EXPECT_EQ(test, ane_h16_session_close(&f->s, p), 0); + KUNIT_EXPECT_FALSE(test, f->s.quarantined); +} + +/* A linked program: the HWX scratch is a kernel buffer sent with every EXECUTE. */ +static void program_scratch_test(struct kunit *test) +{ + struct ane_h16_process_binding bindings[ANE_H16_SURFACES], scratch; + struct ane_h16_hwx_snapshot *snapshot = fw_snapshot(test, true, false); + struct ane_h16_hwx *hwx = kunit_kzalloc(test, sizeof(*hwx), GFP_KERNEL); + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + u8 *g = test_aux[ANE_H16_AUX_GENERIC], *e; + unsigned int count; + size_t size; + const u8 *d; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, hwx); + KUNIT_ASSERT_NOT_NULL(test, snapshot); + KUNIT_ASSERT_EQ(test, ane_h16_hwx_validate(snapshot->bytes, snapshot->size, hwx), 0); + KUNIT_ASSERT_EQ(test, hwx->scratch.size, SZ_16K); + size = generic_fill(g, hwx, TEST_GENERIC_BASE); + e = g + ANE_H16_GENERIC_HEADER + 2 * ANE_H16_GENERIC_ENTRY; + KUNIT_ASSERT_EQ(test, ane_h16_process_bindings(hwx, g, size, bindings, &count, + &scratch), 0); + KUNIT_EXPECT_EQ(test, count, 2); + KUNIT_EXPECT_EQ(test, scratch.index, TEST_GENERIC_BASE + 2); + KUNIT_EXPECT_EQ(test, scratch.class, 0); + KUNIT_EXPECT_EQ(test, scratch.hwx_vm, hwx->scratch.vm); + /* No scratch entry, an output-like or short one, or a reused ID refuse. */ + put_unaligned_le32(2, g + ANE_H16_GENERIC_COUNT); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, g, size - ANE_H16_GENERIC_ENTRY, + bindings, &count, &scratch), -ENOEXEC); + put_unaligned_le32(3, g + ANE_H16_GENERIC_COUNT); + put_unaligned_le32(2, e + 0x10); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, g, size, bindings, &count, + &scratch), -ENOEXEC); + put_unaligned_le32(1, e + 0x10); + put_unaligned_le64(SZ_16K - 4, e + 0x20); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, g, size, bindings, &count, + &scratch), -ENOEXEC); + put_unaligned_le64(SZ_16K, e + 0x20); + put_unaligned_le32(TEST_GENERIC_BASE, e + 4); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, g, size, bindings, &count, + &scratch), -ENOEXEC); + put_unaligned_le32(TEST_GENERIC_BASE + 2, e + 4); + + test_aux_size[ANE_H16_AUX_GENERIC] = size; + KUNIT_ASSERT_EQ(test, ane_h16_session_load(&f->s, snapshot, test_aux_ptr, test_aux_size, + &p), 0); + KUNIT_ASSERT_TRUE(test, p->has_scratch); + /* Userspace still sees two surfaces. */ + KUNIT_EXPECT_EQ(test, p->resource_count, 2); + /* The output and the scratch are the device-writable mappings. */ + KUNIT_EXPECT_EQ(test, f->writable_maps, 2); + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, p), 0); + KUNIT_EXPECT_EQ(test, f->exec_length, ANE_H16_EXECUTE_BASE_SIZE); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0x28), 3); + d = f->exec + 0x60 + 2 * 0x30; + KUNIT_EXPECT_EQ(test, get_unaligned_le32(d), 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(d + 4), TEST_GENERIC_BASE + 2); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(d + 8), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(d + 0x18), p->scratch.mapping_dva); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(d + 0x20), SZ_16K); + KUNIT_EXPECT_EQ(test, ane_h16_session_close(&f->s, p), 0); + KUNIT_EXPECT_FALSE(test, f->s.quarantined); +} + +static void program_bad_load_test(struct kunit *test) +{ + static const enum fw_mode modes[] = { FW_LOAD_ID_UNCHANGED, FW_LOAD_NAK, FW_CREATE_BODY }; + static const int errors[] = { -EPROTO, -EREMOTEIO, -EPROTO }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(modes); i++) { + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + + KUNIT_ASSERT_NOT_NULL(test, f); + f->mode = modes[i]; + KUNIT_EXPECT_EQ(test, fw_load_program(f, &p), errors[i]); + KUNIT_EXPECT_PTR_EQ(test, p, NULL); + KUNIT_EXPECT_TRUE(test, f->s.quarantined); + /* No further firmware traffic after quarantine. */ + KUNIT_EXPECT_EQ(test, f->count, modes[i] == FW_CREATE_BODY ? 2 : 1); + KUNIT_EXPECT_EQ(test, fw_load_program(f, &p), -EIO); + } +} + +static void program_bad_report_test(struct kunit *test) +{ + static const enum fw_mode modes[] = { + FW_REPORT_FOREIGN_UUID, FW_REPORT_WRONG_PROCESS, FW_REPORT_FAILURE, + }; + static const int errors[] = { -ETIMEDOUT, -EPROTO, -EIO }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(modes); i++) { + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + unsigned int sent; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + f->mode = modes[i]; + KUNIT_EXPECT_EQ(test, ane_h16_session_execute(&f->s, p), errors[i]); + /* No closing 0x002d after an uncertain or failed request. */ + KUNIT_EXPECT_EQ(test, f->ids[f->count - 1], ANE_H16_EXECUTE); + KUNIT_EXPECT_TRUE(test, f->s.quarantined); + KUNIT_EXPECT_TRUE(test, f->s.pending.quarantined); + sent = f->count; + KUNIT_EXPECT_EQ(test, ane_h16_session_execute(&f->s, p), -EIO); + /* Teardown is not attempted while quarantined; mappings stay. */ + KUNIT_EXPECT_EQ(test, ane_h16_session_close(&f->s, p), -EIO); + KUNIT_EXPECT_EQ(test, f->count, sent); + } +} + +/* A request that starts and never ends names the wait step and the last state. */ +static void program_stall_trace_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + f->mode = FW_REPORT_STALL; + KUNIT_EXPECT_EQ(test, ane_h16_session_execute(&f->s, p), -ETIMEDOUT); + /* The fake firmware asked for no log buffer: nothing to print. */ + KUNIT_EXPECT_EQ(test, f->s.fwlog_lines, 0); + KUNIT_EXPECT_TRUE(test, f->s.quarantined); + KUNIT_EXPECT_EQ(test, f->fail_step, 3); + KUNIT_EXPECT_EQ(test, f->fail_flags, ANE_H16_EXECUTE_FLAG_QUEUED); + KUNIT_EXPECT_EQ(test, f->fail_reports, 2); + KUNIT_EXPECT_EQ(test, f->fail_state, 3); + KUNIT_EXPECT_EQ(test, f->fail_other, 0); + + /* Reports for another UUID only: none seen for this request. */ + f = fw_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + f->mode = FW_REPORT_FOREIGN_UUID; + KUNIT_EXPECT_EQ(test, ane_h16_session_execute(&f->s, p), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, f->fail_step, 3); + KUNIT_EXPECT_EQ(test, f->fail_reports, 0); + KUNIT_EXPECT_EQ(test, f->fail_state, U32_MAX); + KUNIT_EXPECT_EQ(test, f->fail_other, 2); +} + +/* The phases of one EXECUTE add up, and only successful ones are accounted. */ +static void program_timing_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + const struct ane_h16_exec_sample *x; + struct ane_h16_exec_stats st; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + f->doorbell_ns = 100; + f->mode = FW_REPORT_LATE; + f->late_polls = 7; + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, p), 0); + x = &f->s.timing; + KUNIT_EXPECT_EQ(test, x->ns[ANE_H16_T_GATE_OPEN], 100ULL); + KUNIT_EXPECT_EQ(test, x->ns[ANE_H16_T_SUBMIT_ACK], 100ULL); + /* The state-0 report was there with the ack. */ + KUNIT_EXPECT_EQ(test, x->ns[ANE_H16_T_ACK_REPORT], 0ULL); + /* Seven sleeps of 10 ns until the completion. */ + KUNIT_EXPECT_EQ(test, x->ns[ANE_H16_T_REPORT_DONE], 70ULL); + KUNIT_EXPECT_EQ(test, x->ns[ANE_H16_T_GATE_CLOSE], 100ULL); + KUNIT_EXPECT_EQ(test, x->ns[ANE_H16_T_FIRMWARE], 370ULL); + KUNIT_EXPECT_EQ(test, x->polls, 7ULL); + /* Nothing outside the firmware phases takes time in the fake. */ + KUNIT_EXPECT_EQ(test, x->ns[ANE_H16_T_PREPARE], 0ULL); + KUNIT_EXPECT_EQ(test, x->ns[ANE_H16_T_FINISH], 0ULL); + + ane_h16_session_account(&f->s, 1, 5, 6, 400); + f->mode = FW_OK; + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, p), 0); + /* All three reports came with the ack: nothing between them. */ + KUNIT_EXPECT_EQ(test, f->s.timing.ns[ANE_H16_T_REPORT_DONE], 0ULL); + KUNIT_EXPECT_EQ(test, f->s.timing.polls, 0ULL); + ane_h16_session_account(&f->s, 1, 15, 8, 340); + ane_h16_session_stats(&f->s, &st, false); + KUNIT_EXPECT_EQ(test, st.count, 2ULL); + KUNIT_EXPECT_EQ(test, st.sum.ns[ANE_H16_T_LOCK], 2ULL); + KUNIT_EXPECT_EQ(test, st.last.ns[ANE_H16_T_COPY_IN], 15ULL); + KUNIT_EXPECT_EQ(test, st.sum.ns[ANE_H16_T_COPY_IN], 20ULL); + KUNIT_EXPECT_EQ(test, st.min.ns[ANE_H16_T_REPORT_DONE], 0ULL); + KUNIT_EXPECT_EQ(test, st.max.ns[ANE_H16_T_REPORT_DONE], 70ULL); + KUNIT_EXPECT_EQ(test, st.min.ns[ANE_H16_T_IOCTL], 340ULL); + KUNIT_EXPECT_EQ(test, st.max.ns[ANE_H16_T_IOCTL], 400ULL); + KUNIT_EXPECT_EQ(test, st.sum.polls, 7ULL); + KUNIT_EXPECT_EQ(test, st.min.polls, 0ULL); + KUNIT_EXPECT_EQ(test, st.max.polls, 7ULL); + + /* A reset starts over: the next sample sets min and max alone. */ + ane_h16_session_stats(&f->s, NULL, true); + ane_h16_session_account(&f->s, 1, 9, 9, 500); + ane_h16_session_stats(&f->s, &st, false); + KUNIT_EXPECT_EQ(test, st.count, 1ULL); + KUNIT_EXPECT_EQ(test, st.min.ns[ANE_H16_T_IOCTL], 500ULL); + KUNIT_EXPECT_EQ(test, st.sum.ns[ANE_H16_T_COPY_OUT], 9ULL); + + /* A stalled request times out and leaves the statistics alone. */ + f->mode = FW_REPORT_STALL; + KUNIT_EXPECT_EQ(test, ane_h16_session_execute(&f->s, p), -ETIMEDOUT); + ane_h16_session_stats(&f->s, &st, false); + KUNIT_EXPECT_EQ(test, st.count, 1ULL); +} + +static void program_fwlog_line_test(struct kunit *test) +{ + static const char text[] = "R: [0:1]: CMD = 0x020c\tok\x01 "; + u8 *r = kunit_kzalloc(test, ANE_H16_FWLOG_RECORD, GFP_KERNEL); + char line[ANE_H16_FWLOG_RECORD]; + + KUNIT_ASSERT_NOT_NULL(test, r); + KUNIT_EXPECT_EQ(test, ane_h16_fwlog_line(r, line), 0); + r[0] = 0x12; + memcpy(r + 2, text, sizeof(text)); + KUNIT_EXPECT_EQ(test, ane_h16_fwlog_line(r, line), 26); + KUNIT_EXPECT_STREQ(test, line, "R: [0:1]: CMD = 0x020c ok."); + /* An unterminated record stops at its end. */ + memset(r + 2, 'a', ANE_H16_FWLOG_RECORD - 2); + KUNIT_EXPECT_EQ(test, ane_h16_fwlog_line(r, line), ANE_H16_FWLOG_RECORD - 2); + KUNIT_EXPECT_EQ(test, line[ANE_H16_FWLOG_RECORD - 2], 0); +} + +/* The transport finds the log allocation by its tag. */ +static void program_shared_tag_test(struct kunit *test) +{ + struct ane_fw_transport *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, t); + t->shared_count = 2; + t->shared_tags[0] = ANE_FW_SHARED_LOG; + t->shared_tags[1] = ANE_FW_SHARED_EXPP; + t->shared[0].dva = 0x10001300000ULL; + t->shared[1].dva = 0x10001358000ULL; + KUNIT_EXPECT_PTR_EQ(test, ane_fw_transport_shared_tagged(t, ANE_FW_SHARED_LOG), + &t->shared[0]); + KUNIT_EXPECT_PTR_EQ(test, ane_fw_transport_shared_tagged(t, ANE_FW_SHARED_EXPP), + &t->shared[1]); + t->shared_retired[0] = true; + KUNIT_EXPECT_NULL(test, ane_fw_transport_shared_tagged(t, ANE_FW_SHARED_LOG)); + KUNIT_EXPECT_NULL(test, ane_fw_transport_shared_tagged(t, 0x43434d44)); +} + +/* + * Each completed EXECUTE gives its arena back, so a long run never fills + * the pool; LOAD and CREATE keep theirs. + */ +static void program_pool_recycle_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + size_t mark; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + mark = ane_fw_transport_mark(&f->t); + KUNIT_EXPECT_GT(test, mark, 0); + /* Far more requests than one pass through the 256 KiB arena holds. */ + for (i = 0; i < 1000; i++) { + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, p), 0); + KUNIT_ASSERT_EQ(test, ane_fw_transport_mark(&f->t), mark); + } + KUNIT_EXPECT_FALSE(test, f->s.quarantined); + /* A failed request keeps its space: nothing is reused after a fault. */ + f->mode = FW_REPORT_FAILURE; + KUNIT_EXPECT_EQ(test, ane_h16_session_execute(&f->s, p), -EIO); + KUNIT_EXPECT_GT(test, f->t.pool_used, mark); + /* Release refuses while a command is in flight or below the arena use. */ + f->t.pending[ANE_FW_IO].active = true; + KUNIT_EXPECT_EQ(test, ane_fw_transport_release(&f->t, mark), -EBUSY); + f->t.pending[ANE_FW_IO].active = false; + KUNIT_EXPECT_EQ(test, ane_fw_transport_release(&f->t, f->t.pool_used + 1), -EINVAL); +} + +/* Running out of arena refuses cleanly, before any traffic. */ +static void program_pool_test(struct kunit *test) +{ + struct prog_fw *f = fw_create(test); + struct ane_h16_program *p = NULL; + unsigned int sent; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + sent = f->count; + f->t.pool_used = f->t.pool.size - 0x200; + KUNIT_EXPECT_EQ(test, ane_h16_session_execute(&f->s, p), -ENOSPC); + KUNIT_EXPECT_EQ(test, f->count, sent); + KUNIT_EXPECT_FALSE(test, f->s.quarantined); + KUNIT_EXPECT_EQ(test, ane_h16_session_close(&f->s, p), 0); + KUNIT_EXPECT_EQ(test, f->count, sent + 2); +} + +static u8 *fw_sparse_large(struct kunit *test, const struct ane_h16_sparse_run *runs, + unsigned int count, size_t size) +{ + u8 *bytes = fw_zalloc(test, size); + unsigned int i; + + if (!bytes) + return NULL; + for (i = 0; i < count; i++) { + if (runs[i].offset + runs[i].length > size) + return NULL; + memcpy(bytes + runs[i].offset, runs[i].bytes, runs[i].length); + } + return bytes; +} + +static struct ane_h16_hwx_snapshot *mobilenet_snapshot(struct kunit *test) +{ + struct ane_h16_hwx_snapshot *snapshot = kunit_kzalloc(test, sizeof(*snapshot), + GFP_KERNEL); + u8 *hwx = fw_sparse_large(test, mobilenet_hwx_runs, ARRAY_SIZE(mobilenet_hwx_runs), + MOBILENET_HWX_SIZE); + + if (!snapshot || !hwx || + ane_h16_hwx_snapshot_create(hwx, MOBILENET_HWX_SIZE, snapshot) || + kunit_add_action_or_reset(test, snapshot_free, snapshot)) + return NULL; + return snapshot; +} + +static void mobilenet_sections(struct kunit *test, const void *aux[ANE_H16_AUX_COUNT], + size_t aux_size[ANE_H16_AUX_COUNT]) +{ + aux_size[0] = MOBILENET_GENERIC_SIZE; + aux_size[1] = MOBILENET_OPERATION_SIZE; + aux_size[2] = MOBILENET_PROCEDURE_SIZE; + aux_size[3] = MOBILENET_PROPERTIES_SIZE; + aux[0] = sparse_bytes(test, mobilenet_generic_runs, ARRAY_SIZE(mobilenet_generic_runs), + aux_size[0]); + aux[1] = sparse_bytes(test, mobilenet_operation_runs, + ARRAY_SIZE(mobilenet_operation_runs), aux_size[1]); + aux[2] = sparse_bytes(test, mobilenet_procedure_runs, + ARRAY_SIZE(mobilenet_procedure_runs), aux_size[2]); + aux[3] = sparse_bytes(test, mobilenet_properties_runs, + ARRAY_SIZE(mobilenet_properties_runs), aux_size[3]); +} + +/* + * The H17 MobileNetV1 HWX is one linked program: 28 tasks, input x and + * output y as surfaces, the 3.5 MiB __DATA scratch on BAR slot 3 and the + * kernel table on slot 8. The H16 rules hold except the subtype and the zero + * stream prologue. + */ +static void program_h17_hwx_test(struct kunit *test) +{ + struct ane_h16_hwx_snapshot *snapshot = mobilenet_snapshot(test); + struct ane_h16_hwx *hwx = kunit_kzalloc(test, sizeof(*hwx), GFP_KERNEL); + u8 *bytes; + + KUNIT_ASSERT_NOT_NULL(test, snapshot); + KUNIT_ASSERT_NOT_NULL(test, hwx); + bytes = snapshot->bytes; + *hwx = snapshot->info; + KUNIT_EXPECT_EQ(test, hwx->subtype, ANE_HWX_SUBTYPE_H17); + KUNIT_EXPECT_EQ(test, hwx->tasks, 28); + KUNIT_EXPECT_EQ(test, hwx->surface_count, 2); + KUNIT_EXPECT_EQ(test, hwx->surfaces[0].direction, 1); + KUNIT_EXPECT_EQ(test, hwx->surfaces[0].span, 301056); + KUNIT_EXPECT_EQ(test, hwx->surfaces[1].direction, 2); + KUNIT_EXPECT_EQ(test, hwx->surfaces[1].span, 458752); + KUNIT_EXPECT_EQ(test, hwx->scratch.vm, 0x30000000); + KUNIT_EXPECT_EQ(test, hwx->scratch.size, 0x380000); + KUNIT_EXPECT_EQ(test, hwx->kernel.size, 0x81cb40); + KUNIT_EXPECT_EQ(test, hwx->text.size, 0x3110); + + /* Subtype 9 opens its stream with 0; a subtype 7 file must open with 1. */ + put_unaligned_le32(ANE_HWX_SUBTYPE_H16, bytes + 8); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(bytes, snapshot->size, hwx), -ENOEXEC); + put_unaligned_le32(1, bytes + hwx->text.offset); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(bytes, snapshot->size, hwx), 0); + KUNIT_EXPECT_EQ(test, hwx->subtype, ANE_HWX_SUBTYPE_H16); + put_unaligned_le32(ANE_HWX_SUBTYPE_H17, bytes + 8); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(bytes, snapshot->size, hwx), -ENOEXEC); + put_unaligned_le32(0, bytes + hwx->text.offset); + put_unaligned_le32(8, bytes + 8); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(bytes, snapshot->size, hwx), -ENOEXEC); + put_unaligned_le32(ANE_HWX_SUBTYPE_H17, bytes + 8); + KUNIT_EXPECT_EQ(test, ane_h16_hwx_validate(bytes, snapshot->size, hwx), 0); +} + +/* Generic and operation sections of the MobileNet program, and their faults. */ +static void program_h17_sections_test(struct kunit *test) +{ + struct ane_h16_hwx_snapshot *snapshot = mobilenet_snapshot(test); + struct ane_h16_process_binding bindings[ANE_H16_SURFACES], scratch = {}; + const struct ane_h16_hwx *hwx; + const void *aux[ANE_H16_AUX_COUNT]; + size_t aux_size[ANE_H16_AUX_COUNT]; + unsigned int count; + u8 *g, *o; + + KUNIT_ASSERT_NOT_NULL(test, snapshot); + hwx = &snapshot->info; + mobilenet_sections(test, aux, aux_size); + g = (u8 *)aux[ANE_H16_AUX_GENERIC]; + o = (u8 *)aux[ANE_H16_AUX_OPERATION]; + KUNIT_ASSERT_NOT_NULL(test, g); + KUNIT_ASSERT_NOT_NULL(test, o); + + KUNIT_ASSERT_EQ(test, ane_h16_process_bindings(hwx, g, aux_size[0], bindings, &count, + &scratch), 0); + KUNIT_EXPECT_EQ(test, count, 2); + KUNIT_EXPECT_EQ(test, bindings[0].index, 5); + KUNIT_EXPECT_EQ(test, bindings[0].class, 0); + KUNIT_EXPECT_EQ(test, bindings[1].index, 4); + KUNIT_EXPECT_EQ(test, bindings[1].class, 1); + KUNIT_EXPECT_EQ(test, scratch.index, 6); + KUNIT_EXPECT_EQ(test, scratch.class, 0); + KUNIT_EXPECT_EQ(test, scratch.hwx_vm, 0x30000000); + /* A scratch program needs somewhere to put the scratch binding. */ + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, g, aux_size[0], bindings, &count, + NULL), -EINVAL); + /* The scratch entry: input type and direction, exact __bss size, own ID. */ + g[ANE_H16_GENERIC_HEADER + 2 * ANE_H16_GENERIC_ENTRY + 8] = 1; + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, g, aux_size[0], bindings, &count, + &scratch), -ENOEXEC); + g[ANE_H16_GENERIC_HEADER + 2 * ANE_H16_GENERIC_ENTRY + 8] = 0; + put_unaligned_le64(0x400000, g + ANE_H16_GENERIC_HEADER + 2 * ANE_H16_GENERIC_ENTRY + + 0x20); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, g, aux_size[0], bindings, &count, + &scratch), -ENOEXEC); + put_unaligned_le64(0x380000, g + ANE_H16_GENERIC_HEADER + 2 * ANE_H16_GENERIC_ENTRY + + 0x20); + put_unaligned_le32(5, g + ANE_H16_GENERIC_HEADER + 2 * ANE_H16_GENERIC_ENTRY + 4); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, g, aux_size[0], bindings, &count, + &scratch), -ENOEXEC); + put_unaligned_le32(6, g + ANE_H16_GENERIC_HEADER + 2 * ANE_H16_GENERIC_ENTRY + 4); + /* Without its scratch entry the section no longer describes the program. */ + put_unaligned_le32(2, g + ANE_H16_GENERIC_COUNT); + KUNIT_EXPECT_EQ(test, ane_h16_process_bindings(hwx, g, aux_size[0] - + ANE_H16_GENERIC_ENTRY, bindings, + &count, &scratch), -ENOEXEC); + put_unaligned_le32(3, g + ANE_H16_GENERIC_COUNT); + KUNIT_ASSERT_EQ(test, ane_h16_process_bindings(hwx, g, aux_size[0], bindings, &count, + &scratch), 0); + + /* Operation refs {3: scratch 6, 6: x 5, 7: y 4, 8: kernel 1007}. */ + KUNIT_EXPECT_EQ(test, get_unaligned_le32(o + 4 + 8), 4); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(o + 4 + 0xc), 3); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(o + 4 + 0x10), 6); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(o + 4 + 0x24), 8); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(o + 4 + 0x28), 1007); + KUNIT_EXPECT_EQ(test, ane_h16_operation_validate(snapshot, hwx, o, aux_size[1], + bindings, count, &scratch), 0); + /* The legacy kernel tag 2 asserts the H17 firmware. */ + put_unaligned_le32(2, o + 4 + 0x28); + KUNIT_EXPECT_EQ(test, ane_h16_operation_validate(snapshot, hwx, o, aux_size[1], + bindings, count, &scratch), -ENOEXEC); + /* A kernel program must bind its kernel. */ + put_unaligned_le32(3, o + 4 + 8); + KUNIT_EXPECT_EQ(test, ane_h16_operation_validate(snapshot, hwx, o, aux_size[1], + bindings, count, &scratch), -ENOEXEC); + put_unaligned_le32(4, o + 4 + 8); + put_unaligned_le32(1007, o + 4 + 0x28); + /* Each tag must name the resource at its slot's descriptor base. */ + put_unaligned_le32(5, o + 4 + 0x10); + KUNIT_EXPECT_EQ(test, ane_h16_operation_validate(snapshot, hwx, o, aux_size[1], + bindings, count, &scratch), -ENOEXEC); + put_unaligned_le32(6, o + 4 + 0x10); + put_unaligned_le32(4, o + 4 + 0xc); + KUNIT_EXPECT_EQ(test, ane_h16_operation_validate(snapshot, hwx, o, aux_size[1], + bindings, count, &scratch), -ENOEXEC); + put_unaligned_le32(6, o + 4 + 0xc); + KUNIT_EXPECT_EQ(test, ane_h16_operation_validate(snapshot, hwx, o, aux_size[1], + bindings, count, &scratch), -ENOEXEC); + put_unaligned_le32(3, o + 4 + 0xc); + /* Sizes follow the record count. */ + KUNIT_EXPECT_EQ(test, ane_h16_operation_validate(snapshot, hwx, o, aux_size[1] - 4, + bindings, count, &scratch), -ENOEXEC); + put_unaligned_le32(0, o); + KUNIT_EXPECT_EQ(test, ane_h16_operation_validate(snapshot, hwx, o, aux_size[1], + bindings, count, &scratch), -ENOEXEC); + put_unaligned_le32(1, o); + KUNIT_EXPECT_EQ(test, ane_h16_operation_validate(snapshot, hwx, o, aux_size[1], + bindings, count, &scratch), 0); +} + +/* + * The MobileNet program through an H17 session: LOAD places the whole + * kernel pages, EXECUTE names x, y and the kernel-owned scratch, and only + * the first EXECUTE after the program change is queued. + */ +static void program_h17_mobilenet_test(struct kunit *test) +{ + static const u32 ids[3] = { 5, 4, 6 }, classes[3] = { 0, 1, 0 }; + /* Whole HWX sections: the input is 0x49800 of tensor in a 0x4c000 section. */ + static const u64 sizes[3] = { 0x4c000, 458752, 0x380000 }; + struct ane_h16_hwx_snapshot *snapshot = mobilenet_snapshot(test); + struct prog_fw *f = fw_create_family(test, 17); + struct ane_h16_program *p = NULL; + const void *aux[ANE_H16_AUX_COUNT]; + size_t aux_size[ANE_H16_AUX_COUNT]; + const u8 *slot; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, snapshot); + mobilenet_sections(test, aux, aux_size); + for (i = 0; i < ANE_H16_AUX_COUNT; i++) + KUNIT_ASSERT_NOT_NULL(test, aux[i]); + KUNIT_ASSERT_EQ(test, ane_h16_session_load(&f->s, snapshot, aux, aux_size, &p), 0); + KUNIT_ASSERT_NOT_NULL(test, p); + KUNIT_EXPECT_EQ(test, p->resource_count, 2); + KUNIT_EXPECT_TRUE(test, p->has_scratch); + KUNIT_EXPECT_EQ(test, p->scratch.index, 6); + KUNIT_EXPECT_EQ(test, p->scratch.size, 0x380000); + KUNIT_EXPECT_EQ(test, p->bar_count, 0); + /* Output and scratch are the only device-writable mappings. */ + KUNIT_EXPECT_EQ(test, f->writable_maps, 2); + slot = f->load + ANE_H16_LOAD_SLOT_OFFSET; + KUNIT_EXPECT_EQ(test, get_unaligned_le32(slot + ANE_H16_LOAD_SLOT_SIZE + 4), 1007); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(slot + ANE_H16_LOAD_SLOT_SIZE + 8), 3); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot + ANE_H16_LOAD_SLOT_SIZE + 0x20), 0x820000); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(slot + 2 * ANE_H16_LOAD_SLOT_SIZE + 0x20), + 0x3110); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((const u8 *)p->aux[3].cpu + 0xc), 28); + + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, p), 0); + KUNIT_EXPECT_EQ(test, f->exec_length, ANE_H16_EXECUTE_BASE_SIZE); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0x28), 3); + for (i = 0; i < 3; i++) { + const u8 *d = f->exec + 0x60 + i * 0x30; + + KUNIT_EXPECT_EQ(test, get_unaligned_le32(d + 4), ids[i]); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(d + 8), classes[i]); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(d + 0x18), + i < 2 ? p->surfaces[i]->dva : p->scratch.mapping_dva); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(d + 0x20), sizes[i]); + } + KUNIT_ASSERT_EQ(test, f->execs, 3); + KUNIT_EXPECT_EQ(test, f->exec_flags[0], ANE_H16_EXECUTE_FLAG_QUEUED); + KUNIT_EXPECT_EQ(test, f->exec_flags[1], 0); + KUNIT_EXPECT_EQ(test, f->exec_flags[2], 0); + KUNIT_EXPECT_FALSE(test, f->s.quarantined); + KUNIT_EXPECT_EQ(test, ane_h16_session_close(&f->s, p), 0); +} + +/* + * An H17 session switches between programs: a second LOAD succeeds, and + * every EXECUTE that follows a different program, a LOAD or a CREATE is + * queued. H16 keeps one program lifetime, and a program built for the other + * engine is refused before any traffic. + */ +static void program_h17_switch_test(struct kunit *test) +{ + static const u32 queued[] = { 1, 0, 1, 1, 0, 1 }; + struct prog_fw *f = fw_create_family(test, 17), *h16 = fw_create(test); + struct ane_h16_program *a = NULL, *b = NULL, *c = NULL; + struct ane_h16_hwx_snapshot *snapshot; + const void *aux[ANE_H16_AUX_COUNT]; + size_t aux_size[ANE_H16_AUX_COUNT]; + unsigned int i, sent; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, h16); + /* Two H17 programs: two loads of the MobileNet program. */ + snapshot = mobilenet_snapshot(test); + KUNIT_ASSERT_NOT_NULL(test, snapshot); + mobilenet_sections(test, aux, aux_size); + KUNIT_ASSERT_EQ(test, ane_h16_session_load(&f->s, snapshot, aux, aux_size, &a), 0); + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, a), 0); + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, a), 0); + snapshot = mobilenet_snapshot(test); + KUNIT_ASSERT_NOT_NULL(test, snapshot); + KUNIT_ASSERT_EQ(test, ane_h16_session_load(&f->s, snapshot, aux, aux_size, &b), 0); + KUNIT_EXPECT_NE(test, a->key.program, b->key.program); + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, b), 0); + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, a), 0); + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, a), 0); + KUNIT_ASSERT_EQ(test, ane_h16_session_close(&f->s, a), 0); + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, b), 0); + KUNIT_ASSERT_EQ(test, f->execs, ARRAY_SIZE(queued)); + for (i = 0; i < ARRAY_SIZE(queued); i++) + KUNIT_EXPECT_EQ(test, f->exec_flags[i], queued[i]); + KUNIT_EXPECT_EQ(test, ane_h16_session_execute(&f->s, a), -EINVAL); + KUNIT_EXPECT_EQ(test, ane_h16_session_close(&f->s, b), 0); + KUNIT_EXPECT_FALSE(test, f->s.quarantined); + + /* An H17 program on the H16 session, and an H16 one on the H17 session. */ + sent = h16->count; + snapshot = mobilenet_snapshot(test); + KUNIT_ASSERT_NOT_NULL(test, snapshot); + KUNIT_EXPECT_EQ(test, ane_h16_session_load(&h16->s, snapshot, aux, aux_size, &c), + -ENOEXEC); + KUNIT_EXPECT_EQ(test, h16->count, sent); + sent = f->count; + KUNIT_EXPECT_EQ(test, fw_load_program(f, &c), -ENOEXEC); + KUNIT_EXPECT_EQ(test, f->count, sent); + KUNIT_EXPECT_PTR_EQ(test, c, NULL); + KUNIT_EXPECT_FALSE(test, f->s.quarantined); +} + +/* + * H17 sends no custom BAR record, even for a linear program whose descriptor + * names the mcache field: the H17 firmware asserts on the record. + */ +static void program_h17_no_custom_bar_test(struct kunit *test) +{ + struct prog_fw *f = fw_create_family(test, 17); + struct ane_h16_program *p = NULL; + + KUNIT_ASSERT_NOT_NULL(test, f); + /* The linear add128 fixture is an H16 HWX; let this H17 session take it. */ + f->soc.hwx_subtype = ANE_HWX_SUBTYPE_H16; + KUNIT_ASSERT_EQ(test, fw_load_program(f, &p), 0); + KUNIT_ASSERT_NOT_NULL(test, p); + KUNIT_EXPECT_EQ(test, p->bar_count, 0); + KUNIT_ASSERT_EQ(test, ane_h16_session_execute(&f->s, p), 0); + KUNIT_EXPECT_EQ(test, f->exec_length, ANE_H16_EXECUTE_BASE_SIZE); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0xca0), 0); + /* + * H17 descriptors carry the size the generic section declared (this + * fixture declares the span), not the larger HWX section. + */ + KUNIT_EXPECT_EQ(test, get_unaligned_le64(f->exec + 0x60 + 0x20), p->hwx.surfaces[0].span); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(f->exec + 0x90 + 0x20), p->hwx.surfaces[1].span); + KUNIT_EXPECT_NE(test, p->hwx.surfaces[0].span, p->hwx.surfaces[0].section.size); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(f->exec + 0x30), ANE_H16_EXECUTE_FLAG_QUEUED); + KUNIT_EXPECT_EQ(test, ane_h16_session_close(&f->s, p), 0); + KUNIT_EXPECT_FALSE(test, f->s.quarantined); +} + +static struct kunit_case program_cases[] = { + KUNIT_CASE(program_h17_no_custom_bar_test), + KUNIT_CASE(program_add128_slots_test), + KUNIT_CASE(program_kernel_slots_test), + KUNIT_CASE(program_add128_hwx_test), + KUNIT_CASE(program_replies_test), + KUNIT_CASE(program_deny_list_test), + KUNIT_CASE(program_custom_bar_test), + KUNIT_CASE(program_session_test), + KUNIT_CASE(program_scratch_test), + KUNIT_CASE(program_bad_load_test), + KUNIT_CASE(program_bad_report_test), + KUNIT_CASE(program_stall_trace_test), + KUNIT_CASE(program_timing_test), + KUNIT_CASE(program_fwlog_line_test), + KUNIT_CASE(program_shared_tag_test), + KUNIT_CASE(program_pool_test), + KUNIT_CASE(program_pool_recycle_test), + KUNIT_CASE(program_h17_hwx_test), + KUNIT_CASE(program_h17_sections_test), + KUNIT_CASE(program_h17_mobilenet_test), + KUNIT_CASE(program_h17_switch_test), + {} +}; + +static struct kunit_suite program_suite = { + .name = "ane-h16-program", + .test_cases = program_cases, +}; + +kunit_test_suite(program_suite); +MODULE_DESCRIPTION("KUnit tests for the H16 program lifecycle"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/accel/ane/ane_h16_resources.h b/drivers/accel/ane/ane_h16_resources.h index 46fa17d506e2c..e77e17f21ca04 100644 --- a/drivers/accel/ane/ane_h16_resources.h +++ b/drivers/accel/ane/ane_h16_resources.h @@ -5,9 +5,10 @@ #include "ane_h16_execute.h" #include "ane_h16_hwx.h" -/* Created only from a validated HWX and a decoded firmware process resource - * map. No userspace field may supply this authority. Each mapping is private, - * coherent DMA memory with no userspace alias, retained for process lifetime. +/* Created only by the backend matching a validated HWX with reviewed program + * resource bindings. No userspace field may supply this authority. Each + * mapping is private, coherent DMA memory with no userspace alias, retained + * for process lifetime. */ struct ane_h16_resource { u64 hwx_vm; @@ -44,7 +45,8 @@ ane_h16_resources_validate(const struct ane_h16_hwx *hwx, for (j = 0; j < i; j++) { const struct ane_h16_execute_buffer *a = &resources[j].buffer; - if (a->index == b->index || + /* Indices are local to a class (inputs and outputs). */ + if ((a->index == b->index && a->class == b->class) || (b->mapping_dva < a->mapping_dva + a->mapping_size && a->mapping_dva < end)) return -EINVAL; diff --git a/drivers/accel/ane/ane_h16_tasks.h b/drivers/accel/ane/ane_h16_tasks.h index c270d748fbf0d..e6e4ebc70694e 100644 --- a/drivers/accel/ane/ane_h16_tasks.h +++ b/drivers/accel/ane/ane_h16_tasks.h @@ -34,18 +34,21 @@ static inline bool ane_h16_register_valid(u32 word) } /* - * H16 streams begin with little-endian u32 1 and twelve reserved zero bytes. - * This structural gate does not replace container/tensor/aperture validation. + * A stream opens with a 16-byte frame: little-endian u32 @prologue, then + * twelve reserved zero bytes. H16 programs open with 1; every measured H17 + * program opens with 0. The last task may end at the end of the section; + * every other task is padded with zeros to 16 bytes. This structural gate + * does not replace container/tensor/aperture validation. */ static inline int -ane_h16_task_stream_validate(const void *text, size_t size, u32 expected_tasks) +ane_hx_task_stream_validate(const void *text, size_t size, u32 expected_tasks, u32 prologue) { const u8 *p = text; size_t pos = 16; u32 tasks = 0; if (!p || !expected_tasks || size < 16 || size > ANE_H16_TASK_MAX || - (size & 3) || get_unaligned_le32(p) != 1 || + (size & 3) || get_unaligned_le32(p) != prologue || memchr_inv(p + 4, 0, 12)) return -ENOEXEC; while (pos < size) { @@ -94,7 +97,7 @@ ane_h16_task_stream_validate(const void *text, size_t size, u32 expected_tasks) } rec += count * 4; } - padded = ALIGN(end, 16); + padded = end == size ? end : ALIGN(end, 16); if (padded > size || memchr_inv(p + end, 0, padded - end)) return -ENOEXEC; pos = padded; @@ -104,6 +107,12 @@ ane_h16_task_stream_validate(const void *text, size_t size, u32 expected_tasks) return tasks == expected_tasks ? 0 : -ENOEXEC; } +static inline int +ane_h16_task_stream_validate(const void *text, size_t size, u32 expected_tasks) +{ + return ane_hx_task_stream_validate(text, size, expected_tasks, 1); +} + /* Section ordinal is supplied only by the validated complete section table. * This checks Mach-O relocation entries, not the distinct aperture records. */ diff --git a/drivers/accel/ane/ane_h16_tasks_test.c b/drivers/accel/ane/ane_h16_tasks_test.c index f11aab0119c10..6574dcb617b45 100644 --- a/drivers/accel/ane/ane_h16_tasks_test.c +++ b/drivers/accel/ane/ane_h16_tasks_test.c @@ -19,6 +19,9 @@ static void ane_h16_dense_task_test(struct kunit *test) put_unaligned_le32(0x419c / 4 | (1 << 15), text + 16 + 9 * 4); KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); put_unaligned_le32(0x4100 / 4, text + 16 + 9 * 4); + /* The last task may end the stream without padding (16 + 44 bytes). */ + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, 16 + 11 * 4, 1), 0); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, 16 + 10 * 4, 1), -ENOEXEC); text[63] = 1; KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); } @@ -115,8 +118,26 @@ static void ane_h16_relocation_test(struct kunit *test) KUNIT_EXPECT_EQ(test, ret, -ENOEXEC); } +/* H17 streams open with 0, and the last task may end the section unpadded. */ +static void ane_h17_prologue_test(struct kunit *test) +{ + u8 text[60] = {}; + + put_unaligned_le32(11 << 16, text + 16); + put_unaligned_le32(0x4100 / 4, text + 16 + 9 * 4); + KUNIT_EXPECT_EQ(test, ane_hx_task_stream_validate(text, sizeof(text), 1, 0), 0); + KUNIT_EXPECT_EQ(test, ane_hx_task_stream_validate(text, sizeof(text), 1, 1), -ENOEXEC); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); + put_unaligned_le32(1, text); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), 0); + /* A task that overruns the section is still refused. */ + put_unaligned_le32(12 << 16, text + 16); + KUNIT_EXPECT_EQ(test, ane_h16_task_stream_validate(text, sizeof(text), 1), -ENOEXEC); +} + static struct kunit_case ane_h16_task_cases[] = { KUNIT_CASE(ane_h16_prologue_test), + KUNIT_CASE(ane_h17_prologue_test), KUNIT_CASE(ane_h16_dense_task_test), KUNIT_CASE(ane_h16_scatter_task_test), KUNIT_CASE(ane_h16_extra_header_test), diff --git a/drivers/accel/ane/ane_h17_mobilenet_fixture.h b/drivers/accel/ane/ane_h17_mobilenet_fixture.h new file mode 100644 index 0000000000000..6e87d1ba8aa54 --- /dev/null +++ b/drivers/accel/ane/ane_h17_mobilenet_fixture.h @@ -0,0 +1,1389 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __ANE_H17_MOBILENET_FIXTURE_H__ +#define __ANE_H17_MOBILENET_FIXTURE_H__ + +/* + * The H17 MobileNetV1 224 program that mil-hwx-compiler builds as one linked + * program (--target H17 --schedule linked; 8585216 bytes, SHA-256 + * 45921d2e...), with its 28 tasks, the __DATA scratch on BAR slot 3 and the + * kernel table on slot 8. Only the header page and the task stream are kept; + * weights and constants read as zero, which no validator inspects. Also the + * four LOAD sections that tools/testing/selftests/drivers/ane/h16-sections.py + * builds for it. Stored as runs of nonzero bytes. Generated, do not edit. + */ +#include "ane_h16_add128_fixture.h" + +#define MOBILENET_HWX_SIZE 8585216 +static const struct ane_h16_sparse_run mobilenet_hwx_runs[] = { + { 0x0, 50, (const u8[]){ + 0xce, 0xfa, 0xef, 0xbe, 0x80, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x90, 0x27, 0x00, 0x00, + 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x50, 0x41, 0x47, 0x45, 0x5a, 0x45, + 0x52, 0x4f, + } }, + { 0x41, 1, (const u8[]){ + 0x40, + } }, + { 0x64, 39, (const u8[]){ + 0x04, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, + 0x5f, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x38, + } }, + { 0xa0, 81, (const u8[]){ + 0x03, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x62, 0x73, 0x73, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x44, 0x41, + 0x54, 0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, + } }, + { 0x100, 35, (const u8[]){ + 0x19, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x46, 0x56, + 0x4d, 0x4c, 0x49, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x38, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x04, + } }, + { 0x138, 81, (const u8[]){ + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x46, 0x56, + 0x4d, 0x4c, 0x49, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x38, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, + } }, + { 0x198, 35, (const u8[]){ + 0x19, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x46, 0x56, + 0x4d, 0x4c, 0x49, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc0, 0x3c, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + } }, + { 0x1d0, 81, (const u8[]){ + 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x46, 0x56, + 0x4d, 0x4c, 0x49, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc0, 0x3c, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, + } }, + { 0x230, 217, (const u8[]){ + 0x19, 0x00, 0x00, 0x00, 0xe8, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x54, 0x45, + 0x58, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc0, 0x43, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x5f, 0x5f, 0x74, 0x65, 0x78, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x54, 0x45, 0x58, 0x54, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x43, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x10, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x78, 0x2b, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x28, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x5f, 0x5f, 0x54, 0x45, 0x58, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0xf1, 0x43, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0xb1, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x26, + } }, + { 0x318, 137, (const u8[]){ + 0x19, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x4b, 0x45, + 0x52, 0x4e, 0x5f, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, 0x44, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x82, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x82, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x5f, 0x5f, 0x4b, 0x45, 0x52, 0x4e, 0x5f, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x44, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x40, 0xcb, 0x81, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x26, + } }, + { 0x3b0, 100, (const u8[]){ + 0x40, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x30, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc0, 0x3c, 0x30, 0x00, 0x00, 0x00, 0x00, 0x6f, 0x75, 0x74, 0x70, + 0x75, 0x74, 0x30, 0x00, 0x04, 0x00, 0x00, 0x00, 0xc0, 0x08, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x2a, 0x02, 0x00, 0x00, 0x00, 0xc0, 0x43, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0xf1, 0x43, 0x30, + } }, + { 0x433, 1, (const u8[]){ + 0x30, + } }, + { 0x462, 34, (const u8[]){ + 0x38, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc0, 0x3c, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x44, 0x30, + } }, + { 0xc01, 57, (const u8[]){ + 0xc0, 0x43, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x44, 0x0c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0xb8, 0x08, + } }, + { 0xc4a, 159, (const u8[]){ + 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x08, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, + 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x02, 0x08, 0x08, 0x00, + 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x1c, 0x46, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x08, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x61, + 0x69, 0x6e, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x50, 0x0d, + 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x4a, 0x03, 0x00, 0x00, 0x03, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x38, 0x0d, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x3d, 0x0d, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0xe0, 0x00, + 0x00, 0x00, 0x01, + } }, + { 0xd01, 50, (const u8[]){ + 0x98, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x04, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x44, 0x0d, 0x00, 0x00, 0x00, + 0x98, 0x04, + } }, + { 0x19e8, 81, (const u8[]){ + 0x6d, 0x61, 0x69, 0x6e, 0x00, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x30, 0x00, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x50, 0x0d, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x4a, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x38, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0x0d, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x01, + } }, + { 0x1a52, 49, (const u8[]){ + 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x45, 0x0d, 0x00, 0x00, 0x00, 0x00, + 0x07, + } }, + { 0x2738, 1312, (const u8[]){ + 0x6d, 0x61, 0x69, 0x6e, 0x00, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x30, + 0x00, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x41, 0x4e, 0x45, 0x43, + 0x20, 0x76, 0x31, 0x0a, 0x72, 0x65, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x5f, 0x68, 0x77, 0x78, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, + 0x72, 0x20, 0x76, 0x31, 0x0a, 0x0a, 0x20, 0x4d, 0x6f, 0x64, 0x75, 0x6c, + 0x65, 0x20, 0x41, 0x4e, 0x45, 0x43, 0x3a, 0x0a, 0x09, 0x20, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x3a, 0x20, 0x68, 0x31, 0x37, 0x0a, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0xb0, 0x27, 0x00, 0x00, + 0x1e, 0x00, 0x00, 0x00, 0x90, 0x29, 0x00, 0x00, 0xe2, 0x01, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x0f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x38, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x0f, 0x03, 0x00, 0x00, + 0x00, 0xc0, 0x3c, 0x30, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x0f, 0x06, 0x02, 0x00, 0x00, 0x40, 0x44, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, 0x80, 0x47, 0x44, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, + 0x80, 0x4b, 0x44, 0x30, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x0f, 0x06, 0x00, 0x00, 0x80, 0x5f, 0x44, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x50, 0x00, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, 0x40, 0x67, 0x44, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, + 0x40, 0xab, 0x44, 0x30, 0x00, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, + 0x0f, 0x06, 0x00, 0x00, 0x40, 0xb7, 0x44, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x00, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, 0x40, 0x3b, 0x45, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, + 0x40, 0x4b, 0x45, 0x30, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x00, 0x00, 0x00, + 0x0f, 0x06, 0x00, 0x00, 0x40, 0x4f, 0x46, 0x30, 0x00, 0x00, 0x00, 0x00, + 0xb0, 0x00, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, 0x40, 0x63, 0x46, 0x30, + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, + 0x40, 0x67, 0x48, 0x30, 0x00, 0x00, 0x00, 0x00, 0xd2, 0x00, 0x00, 0x00, + 0x0f, 0x06, 0x00, 0x00, 0x40, 0x83, 0x48, 0x30, 0x00, 0x00, 0x00, 0x00, + 0xe3, 0x00, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, 0x40, 0x87, 0x4c, 0x30, + 0x00, 0x00, 0x00, 0x00, 0xf4, 0x00, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, + 0x40, 0xaf, 0x4c, 0x30, 0x00, 0x00, 0x00, 0x00, 0x05, 0x01, 0x00, 0x00, + 0x0f, 0x06, 0x00, 0x00, 0x40, 0xb3, 0x54, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x16, 0x01, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, 0x40, 0xdb, 0x54, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x27, 0x01, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, + 0x40, 0xdf, 0x5c, 0x30, 0x00, 0x00, 0x00, 0x00, 0x38, 0x01, 0x00, 0x00, + 0x0f, 0x06, 0x00, 0x00, 0x40, 0x07, 0x5d, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x49, 0x01, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, 0x40, 0x0b, 0x65, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x5a, 0x01, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, + 0x40, 0x33, 0x65, 0x30, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x01, 0x00, 0x00, + 0x0f, 0x06, 0x00, 0x00, 0x40, 0x37, 0x6d, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x7c, 0x01, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, 0x40, 0x5f, 0x6d, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x8d, 0x01, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, + 0x40, 0x63, 0x75, 0x30, 0x00, 0x00, 0x00, 0x00, 0x9e, 0x01, 0x00, 0x00, + 0x0f, 0x06, 0x00, 0x00, 0x40, 0xa3, 0x75, 0x30, 0x00, 0x00, 0x00, 0x00, + 0xaf, 0x01, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, 0x40, 0xab, 0x85, 0x30, + 0x00, 0x00, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0x0f, 0x06, 0x00, 0x00, + 0x40, 0xfb, 0x85, 0x30, 0x00, 0x00, 0x00, 0x00, 0xd1, 0x01, 0x00, 0x00, + 0x0f, 0x06, 0x00, 0x00, 0x40, 0x03, 0xa6, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x30, 0x00, 0x6f, 0x75, 0x74, 0x70, + 0x75, 0x74, 0x30, 0x00, 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, + 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x30, 0x00, 0x5f, 0x5f, 0x6b, 0x65, + 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x31, 0x00, + 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, + 0x65, 0x5f, 0x32, 0x00, 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, + 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x33, 0x00, 0x5f, 0x5f, 0x6b, 0x65, + 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x34, 0x00, + 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, + 0x65, 0x5f, 0x35, 0x00, 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, + 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x36, 0x00, 0x5f, 0x5f, 0x6b, 0x65, + 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x37, 0x00, + 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, + 0x65, 0x5f, 0x38, 0x00, 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, + 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x39, 0x00, 0x5f, 0x5f, 0x6b, 0x65, + 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x31, 0x30, + 0x00, 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, + 0x6c, 0x65, 0x5f, 0x31, 0x31, 0x00, 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, + 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x31, 0x32, 0x00, 0x5f, + 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, 0x65, + 0x5f, 0x31, 0x33, 0x00, 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, + 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x31, 0x34, 0x00, 0x5f, 0x5f, 0x6b, + 0x65, 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x31, + 0x35, 0x00, 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, + 0x69, 0x6c, 0x65, 0x5f, 0x31, 0x36, 0x00, 0x5f, 0x5f, 0x6b, 0x65, 0x72, + 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x31, 0x37, 0x00, + 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, + 0x65, 0x5f, 0x31, 0x38, 0x00, 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, + 0x30, 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x31, 0x39, 0x00, 0x5f, 0x5f, + 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, + 0x32, 0x30, 0x00, 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, 0x5f, + 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x32, 0x31, 0x00, 0x5f, 0x5f, 0x6b, 0x65, + 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x32, 0x32, + 0x00, 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, + 0x6c, 0x65, 0x5f, 0x32, 0x33, 0x00, 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, + 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x32, 0x34, 0x00, 0x5f, + 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, 0x65, + 0x5f, 0x32, 0x35, 0x00, 0x5f, 0x5f, 0x6b, 0x65, 0x72, 0x6e, 0x5f, 0x30, + 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x32, 0x36, 0x00, 0x5f, 0x5f, 0x6b, + 0x65, 0x72, 0x6e, 0x5f, 0x30, 0x5f, 0x74, 0x69, 0x6c, 0x65, 0x5f, 0x32, + 0x37, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x01, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x07, 0x3c, 0x03, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, + 0x04, 0x05, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, 0xcc, 0x06, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x07, 0x90, 0x08, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, + 0x4c, 0x0a, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, 0x10, 0x0c, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x07, 0xcc, 0x0d, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, + 0x84, 0x0f, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, 0x4c, 0x11, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x07, 0x04, 0x13, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, + 0xcc, 0x14, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, 0x84, 0x16, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x07, 0x4c, 0x18, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, + 0x04, 0x1a, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, 0xcc, 0x1b, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x07, 0x84, 0x1d, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, + 0x4c, 0x1f, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, 0x04, 0x21, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x07, 0xcc, 0x22, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, + 0x84, 0x24, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, 0x4c, 0x26, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x07, 0x04, 0x28, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, + 0xcc, 0x29, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, 0x84, 0x2b, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x07, 0x4c, 0x2d, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, + 0x04, 0x2f, 0x00, 0x00, 0x06, 0x00, 0x00, 0x07, 0xe0, 0x30, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x07, + } }, + { 0x8012, 12542, (const u8[]){ + 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x68, 0xf8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, + 0xc0, 0xff, 0x40, 0x02, 0x01, 0x00, 0x21, 0x00, 0x00, 0x00, 0x21, 0x00, + 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x21, 0x00, + 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x21, 0x00, + 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, + 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x21, 0x00, + 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x21, 0x00, + 0x00, 0x00, 0x59, 0x15, 0x09, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x40, 0x02, + 0x00, 0x00, 0xc0, 0x02, 0x00, 0x00, 0x40, 0x03, 0x00, 0x00, 0xc0, 0x03, + 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, 0xc0, 0x04, 0x00, 0x00, 0x40, 0x05, + 0x00, 0x00, 0x80, 0x05, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x80, 0x06, + 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x6d, 0x95, + 0x02, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x74, 0x95, 0x01, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x01, 0x00, 0xe0, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00, 0x00, 0x03, 0x00, + 0x00, 0x00, 0x05, 0x80, 0x49, 0x93, 0x70, 0x00, 0x00, 0x00, 0x70, 0x00, + 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0xc3, 0x40, 0x01, 0x50, 0x70, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0xc0, 0x01, + 0x00, 0x00, 0x00, 0x88, 0x01, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, + 0x00, 0x01, 0x41, 0x90, 0x3c, 0x80, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, + 0x50, 0x00, 0xc0, 0x01, 0x00, 0x00, 0x40, 0x05, 0x00, 0x00, 0x40, 0x05, + 0x00, 0x00, 0x40, 0x05, 0x00, 0x00, 0x52, 0x90, 0x02, 0x00, 0x72, 0x01, + 0x50, 0x00, 0x00, 0x98, 0x04, 0x00, 0xe0, 0x00, 0x00, 0x00, 0x80, 0x1c, + 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x40, 0x92, + 0x04, 0x80, 0x82, 0x01, 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, + 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x00, 0x01, 0x00, 0x00, 0x00, 0x70, + 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, 0x00, 0x23, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, + 0x00, 0x00, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x63, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, + 0x02, 0x00, 0x21, 0x00, 0x01, 0x00, 0x21, 0x00, 0x01, 0x00, 0x21, 0x00, + 0x01, 0x00, 0x21, 0x00, 0x01, 0x00, 0x21, 0x00, 0x01, 0x00, 0x21, 0x00, + 0x01, 0x00, 0x21, 0x00, 0x01, 0x00, 0x21, 0x00, 0x01, 0x00, 0x21, 0x00, + 0x01, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, 0x01, 0x00, 0x21, 0x00, + 0x01, 0x00, 0x21, 0x00, 0x01, 0x00, 0x21, 0x00, 0x01, 0x00, 0x21, 0x00, + 0x01, 0x00, 0x21, 0x00, 0x01, 0x00, 0x21, 0x00, 0x01, 0x00, 0x59, 0x15, + 0x07, 0x00, 0x40, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0xc0, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x80, 0x01, + 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x40, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0xc0, 0x02, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x40, 0x03, 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0xc0, 0x03, + 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x70, 0x00, 0x00, 0x00, 0x70, 0x00, + 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x05, 0x80, 0x69, 0x93, 0x70, 0x00, + 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0xc3, 0xa0, + 0x42, 0x50, 0x00, 0x40, 0x01, 0x00, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, + 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x70, + 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, + 0x3c, 0x80, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0xe0, 0x00, + 0x00, 0x00, 0x50, 0x1c, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x1c, + 0x00, 0x00, 0x52, 0x90, 0x02, 0x00, 0x72, 0x01, 0x50, 0x00, 0x00, 0x63, + 0x0c, 0x00, 0xe0, 0x00, 0x00, 0x00, 0x80, 0x1c, 0x00, 0x00, 0x00, 0x1c, + 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, + 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, + 0x00, 0x81, 0x00, 0x01, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x31, 0x20, + 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x80, 0x07, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, 0x01, 0x20, 0x40, 0x14, + 0x00, 0x22, 0xe1, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, + 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, + 0xc0, 0xff, 0x40, 0x02, 0x03, 0x00, 0x21, 0x00, 0x02, 0x00, 0x21, 0x00, + 0x02, 0x00, 0x21, 0x00, 0x02, 0x00, 0x21, 0x00, 0x02, 0x00, 0x21, 0x00, + 0x02, 0x00, 0x21, 0x00, 0x02, 0x00, 0x21, 0x00, 0x02, 0x00, 0x21, 0x00, + 0x02, 0x00, 0x21, 0x00, 0x02, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, + 0x02, 0x00, 0x21, 0x00, 0x02, 0x00, 0x21, 0x00, 0x02, 0x00, 0x21, 0x00, + 0x02, 0x00, 0x21, 0x00, 0x02, 0x00, 0x21, 0x00, 0x02, 0x00, 0x21, 0x00, + 0x02, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x40, 0x01, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0xc0, 0x03, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x40, 0x06, + 0x00, 0x00, 0x80, 0x07, 0x00, 0x00, 0xc0, 0x08, 0x00, 0x00, 0x00, 0x0a, + 0x00, 0x00, 0x40, 0x0b, 0x00, 0x00, 0x80, 0x0c, 0x00, 0x00, 0xc0, 0x0d, + 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x80, 0x11, + 0x00, 0x00, 0xc0, 0x12, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, + 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, + 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, + 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, + 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, + 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x70, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x20, 0x00, + 0x00, 0x00, 0x05, 0x80, 0x41, 0x93, 0x70, 0x00, 0x00, 0x00, 0x70, 0x00, + 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x00, 0x11, + 0x23, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, + 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x70, + 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, + 0x3c, 0x80, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x50, 0x10, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x10, + 0x00, 0x00, 0x52, 0x90, 0x02, 0x00, 0x72, 0x01, 0x50, 0x00, 0x00, 0x23, + 0x07, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x20, 0x00, 0x00, 0x00, 0x20, + 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, + 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, + 0x00, 0x81, 0x00, 0x01, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x31, 0x20, + 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x80, 0x0b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, 0x02, 0x20, 0x40, 0x14, + 0x00, 0x22, 0xe1, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x6f, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, + 0x04, 0x00, 0x21, 0x00, 0x03, 0x00, 0x21, 0x00, 0x03, 0x00, 0x21, 0x00, + 0x03, 0x00, 0x21, 0x00, 0x03, 0x00, 0x21, 0x00, 0x03, 0x00, 0x21, 0x00, + 0x03, 0x00, 0x21, 0x00, 0x03, 0x00, 0x21, 0x00, 0x03, 0x00, 0x21, 0x00, + 0x03, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, 0x03, 0x00, 0x21, 0x00, + 0x03, 0x00, 0x21, 0x00, 0x03, 0x00, 0x21, 0x00, 0x03, 0x00, 0x21, 0x00, + 0x03, 0x00, 0x21, 0x00, 0x03, 0x00, 0x21, 0x00, 0x03, 0x00, 0x59, 0x95, + 0x0d, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x80, 0x01, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x80, 0x04, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x80, 0x05, 0x00, 0x00, 0x00, 0x06, + 0x00, 0x00, 0x80, 0x06, 0x00, 0x00, 0xc0, 0x06, 0x00, 0x00, 0x40, 0x07, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x76, 0x95, 0x00, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x70, 0x00, + 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x05, 0x80, + 0x69, 0x93, 0x38, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x00, 0x00, 0xc3, 0x40, 0x01, 0x50, 0x00, 0x40, 0x01, 0x00, 0x38, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, + 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, + 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x38, + 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x7a, 0x01, + 0x50, 0x00, 0x00, 0x80, 0x18, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x01, + 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, + 0x00, 0x81, 0x80, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x31, 0x20, + 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x80, 0x1f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, 0x03, 0x20, 0x40, 0x14, + 0x00, 0x22, 0xe1, 0x00, 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, + 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, + 0xc0, 0xff, 0x40, 0x02, 0x05, 0x00, 0x21, 0x00, 0x04, 0x00, 0x21, 0x00, + 0x04, 0x00, 0x21, 0x00, 0x04, 0x00, 0x21, 0x00, 0x04, 0x00, 0x21, 0x00, + 0x04, 0x00, 0x21, 0x00, 0x04, 0x00, 0x21, 0x00, 0x04, 0x00, 0x21, 0x00, + 0x04, 0x00, 0x21, 0x00, 0x04, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, + 0x04, 0x00, 0x21, 0x00, 0x04, 0x00, 0x21, 0x00, 0x04, 0x00, 0x21, 0x00, + 0x04, 0x00, 0x21, 0x00, 0x04, 0x00, 0x21, 0x00, 0x04, 0x00, 0x21, 0x00, + 0x04, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x40, 0x04, 0x00, 0x00, 0x80, 0x08, + 0x00, 0x00, 0xc0, 0x0c, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x40, 0x15, + 0x00, 0x00, 0x80, 0x19, 0x00, 0x00, 0xc0, 0x1d, 0x00, 0x00, 0x00, 0x22, + 0x00, 0x00, 0x40, 0x26, 0x00, 0x00, 0x80, 0x2a, 0x00, 0x00, 0xc0, 0x2e, + 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x40, 0x37, 0x00, 0x00, 0x80, 0x3b, + 0x00, 0x00, 0xc0, 0x3f, 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, 0x40, 0x04, + 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, 0x40, 0x04, + 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, 0x40, 0x04, + 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, 0x40, 0x04, + 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, 0x40, 0x04, + 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, 0x40, 0x04, 0x00, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x38, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x00, 0x00, 0x05, 0x80, 0x41, 0x91, 0x38, 0x00, 0x00, 0x00, 0x38, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x5a, 0x13, + 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, 0x3c, 0x80, 0x72, 0x01, + 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x70, 0x00, 0x00, 0x00, 0x50, 0x1c, + 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x52, 0x90, + 0x02, 0x00, 0x72, 0x01, 0x50, 0x00, 0x80, 0x31, 0x06, 0x00, 0x70, 0x00, + 0x00, 0x00, 0x40, 0x38, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x38, + 0x00, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, 0x00, 0x10, 0x10, 0x3c, + 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x80, 0x00, + 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, + 0x00, 0x24, 0x40, 0x27, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, + 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, + 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, + 0x00, 0x22, 0x21, 0x00, 0x04, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, + 0x04, 0x00, 0x05, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, + 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, 0x06, 0x00, 0x21, 0x00, + 0x05, 0x00, 0x21, 0x00, 0x05, 0x00, 0x21, 0x00, 0x05, 0x00, 0x21, 0x00, + 0x05, 0x00, 0x21, 0x00, 0x05, 0x00, 0x21, 0x00, 0x05, 0x00, 0x21, 0x00, + 0x05, 0x00, 0x21, 0x00, 0x05, 0x00, 0x21, 0x00, 0x05, 0x00, 0x51, 0x15, + 0x03, 0x00, 0x21, 0x00, 0x05, 0x00, 0x21, 0x00, 0x05, 0x00, 0x21, 0x00, + 0x05, 0x00, 0x21, 0x00, 0x05, 0x00, 0x21, 0x00, 0x05, 0x00, 0x21, 0x00, + 0x05, 0x00, 0x21, 0x00, 0x05, 0x00, 0x59, 0x15, 0x0f, 0x00, 0xc0, 0x00, + 0x00, 0x00, 0x80, 0x01, 0x00, 0x00, 0x40, 0x02, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0xc0, 0x03, 0x00, 0x00, 0x80, 0x04, 0x00, 0x00, 0x40, 0x05, + 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0xc0, 0x06, 0x00, 0x00, 0x80, 0x07, + 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0xc0, 0x09, + 0x00, 0x00, 0x80, 0x0a, 0x00, 0x00, 0x40, 0x0b, 0x00, 0x00, 0xc0, 0x00, + 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0xc0, 0x00, + 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0xc0, 0x00, + 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0xc0, 0x00, + 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0xc0, 0x00, + 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0xc0, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x38, 0x00, 0x00, 0x00, 0x38, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x05, 0x80, 0x69, 0x93, 0x38, 0x00, + 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0xc3, 0xa0, + 0x42, 0x50, 0x00, 0x40, 0x01, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, + 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x1c, + 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, + 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, + 0x03, 0x98, 0x70, 0x00, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x38, + 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0x00, 0x40, + 0x0c, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, 0x00, 0x10, 0x10, 0x3c, + 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x80, 0x00, + 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, + 0x00, 0x24, 0x40, 0x6b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, + 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, + 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, + 0x00, 0x22, 0x21, 0x00, 0x05, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, + 0x05, 0x01, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x70, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, + 0x07, 0x00, 0x21, 0x00, 0x06, 0x00, 0x21, 0x00, 0x06, 0x00, 0x21, 0x00, + 0x06, 0x00, 0x21, 0x00, 0x06, 0x00, 0x21, 0x00, 0x06, 0x00, 0x21, 0x00, + 0x06, 0x00, 0x21, 0x00, 0x06, 0x00, 0x21, 0x00, 0x06, 0x00, 0x21, 0x00, + 0x06, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, 0x06, 0x00, 0x21, 0x00, + 0x06, 0x00, 0x21, 0x00, 0x06, 0x00, 0x21, 0x00, 0x06, 0x00, 0x21, 0x00, + 0x06, 0x00, 0x21, 0x00, 0x06, 0x00, 0x21, 0x00, 0x06, 0x00, 0x59, 0x15, + 0x0f, 0x00, 0x40, 0x08, 0x00, 0x00, 0x80, 0x10, 0x00, 0x00, 0xc0, 0x18, + 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x40, 0x29, 0x00, 0x00, 0x80, 0x31, + 0x00, 0x00, 0xc0, 0x39, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x40, 0x4a, + 0x00, 0x00, 0x80, 0x52, 0x00, 0x00, 0xc0, 0x5a, 0x00, 0x00, 0x00, 0x63, + 0x00, 0x00, 0x40, 0x6b, 0x00, 0x00, 0x80, 0x73, 0x00, 0x00, 0xc0, 0x7b, + 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x40, 0x08, + 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x40, 0x08, + 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x40, 0x08, + 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x40, 0x08, + 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x40, 0x08, + 0x00, 0x00, 0x40, 0x08, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x38, 0x00, + 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x05, 0x80, + 0x41, 0x91, 0x38, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, + 0x00, 0x01, 0x41, 0x90, 0x3c, 0x80, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, + 0x50, 0x00, 0x70, 0x00, 0x00, 0x00, 0x50, 0x38, 0x00, 0x00, 0x00, 0x38, + 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x52, 0x90, 0x02, 0x00, 0x72, 0x01, + 0x50, 0x00, 0x80, 0x51, 0x0c, 0x00, 0x70, 0x00, 0x00, 0x00, 0x40, 0x38, + 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x40, 0x92, + 0x04, 0x80, 0x82, 0x00, 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, + 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x80, 0x00, 0x00, 0x00, 0x00, 0x1c, + 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0x77, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, + 0x06, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, 0x06, 0x00, 0x07, 0x00, + 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, + 0xc0, 0xff, 0x40, 0x02, 0x08, 0x00, 0x21, 0x00, 0x07, 0x00, 0x21, 0x00, + 0x07, 0x00, 0x21, 0x00, 0x07, 0x00, 0x21, 0x00, 0x07, 0x00, 0x21, 0x00, + 0x07, 0x00, 0x21, 0x00, 0x07, 0x00, 0x21, 0x00, 0x07, 0x00, 0x21, 0x00, + 0x07, 0x00, 0x21, 0x00, 0x07, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, + 0x07, 0x00, 0x21, 0x00, 0x07, 0x00, 0x21, 0x00, 0x07, 0x00, 0x21, 0x00, + 0x07, 0x00, 0x21, 0x00, 0x07, 0x00, 0x21, 0x00, 0x07, 0x00, 0x21, 0x00, + 0x07, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x0b, + 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x0e, + 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x38, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x80, 0x00, + 0x00, 0x00, 0x05, 0x80, 0x69, 0x93, 0x1c, 0x00, 0x00, 0x00, 0x1c, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0xc3, 0x40, 0x01, 0x50, 0x00, 0x40, + 0x01, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x5a, 0x13, + 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, + 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x70, 0x00, + 0x00, 0x00, 0x30, 0x38, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x38, + 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0x80, 0x4a, 0x0c, 0x00, 0x40, 0x92, + 0x04, 0x80, 0x82, 0x01, 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, + 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0x00, 0x07, + 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0xfb, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, + 0x07, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, 0x07, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x08, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, + 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, 0x09, 0x00, 0x21, 0x00, + 0x08, 0x00, 0x21, 0x00, 0x08, 0x00, 0x21, 0x00, 0x08, 0x00, 0x21, 0x00, + 0x08, 0x00, 0x21, 0x00, 0x08, 0x00, 0x21, 0x00, 0x08, 0x00, 0x21, 0x00, + 0x08, 0x00, 0x21, 0x00, 0x08, 0x00, 0x21, 0x00, 0x08, 0x00, 0x51, 0x15, + 0x03, 0x00, 0x21, 0x00, 0x08, 0x00, 0x21, 0x00, 0x08, 0x00, 0x21, 0x00, + 0x08, 0x00, 0x21, 0x00, 0x08, 0x00, 0x21, 0x00, 0x08, 0x00, 0x21, 0x00, + 0x08, 0x00, 0x21, 0x00, 0x08, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x40, 0x10, + 0x00, 0x00, 0x80, 0x20, 0x00, 0x00, 0xc0, 0x30, 0x00, 0x00, 0x00, 0x41, + 0x00, 0x00, 0x40, 0x51, 0x00, 0x00, 0x80, 0x61, 0x00, 0x00, 0xc0, 0x71, + 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, 0x40, 0x92, 0x00, 0x00, 0x80, 0xa2, + 0x00, 0x00, 0xc0, 0xb2, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x40, 0xd3, + 0x00, 0x00, 0x80, 0xe3, 0x00, 0x00, 0xc0, 0xf3, 0x00, 0x00, 0x40, 0x10, + 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x40, 0x10, + 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x40, 0x10, + 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x40, 0x10, + 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x40, 0x10, + 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x40, 0x10, + 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x1c, 0x00, + 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x05, 0x80, 0x41, 0x93, 0x1c, 0x00, + 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x1c, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x22, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, + 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, + 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x40, 0x00, 0x00, 0x00, 0x50, 0x20, + 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x7a, 0x01, + 0x50, 0x00, 0xc0, 0x88, 0x03, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, + 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, + 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x31, 0x20, + 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0x0b, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, 0x08, 0x20, 0x40, 0x14, + 0x00, 0x22, 0xe1, 0x00, 0x08, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x6f, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, + 0x0a, 0x00, 0x21, 0x00, 0x09, 0x00, 0x21, 0x00, 0x09, 0x00, 0x21, 0x00, + 0x09, 0x00, 0x21, 0x00, 0x09, 0x00, 0x21, 0x00, 0x09, 0x00, 0x21, 0x00, + 0x09, 0x00, 0x21, 0x00, 0x09, 0x00, 0x21, 0x00, 0x09, 0x00, 0x21, 0x00, + 0x09, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, 0x09, 0x00, 0x21, 0x00, + 0x09, 0x00, 0x21, 0x00, 0x09, 0x00, 0x21, 0x00, 0x09, 0x00, 0x21, 0x00, + 0x09, 0x00, 0x21, 0x00, 0x09, 0x00, 0x21, 0x00, 0x09, 0x00, 0x59, 0x15, + 0x0f, 0x00, 0x40, 0x01, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0xc0, 0x03, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x40, 0x06, 0x00, 0x00, 0x80, 0x07, + 0x00, 0x00, 0xc0, 0x08, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x40, 0x0b, + 0x00, 0x00, 0x80, 0x0c, 0x00, 0x00, 0xc0, 0x0d, 0x00, 0x00, 0x00, 0x0f, + 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x80, 0x11, 0x00, 0x00, 0xc0, 0x12, + 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, + 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, + 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, + 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, + 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x40, 0x01, + 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x1c, 0x00, + 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x05, 0x80, + 0x69, 0x93, 0x1c, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0xc3, 0xa0, 0x42, 0x50, 0x00, 0x40, 0x01, 0x00, 0x1c, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, + 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, + 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x40, 0x00, 0x00, 0x00, 0x50, 0x40, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x7a, 0x01, + 0x50, 0x00, 0xc0, 0x08, 0x07, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, + 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, + 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x31, 0x20, + 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0x0f, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, 0x09, 0x20, 0x40, 0x14, + 0x00, 0x22, 0xe1, 0x00, 0x09, 0x01, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, + 0x6d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, + 0xc0, 0xff, 0x40, 0x02, 0x0b, 0x00, 0x21, 0x00, 0x0a, 0x00, 0x21, 0x00, + 0x0a, 0x00, 0x21, 0x00, 0x0a, 0x00, 0x21, 0x00, 0x0a, 0x00, 0x21, 0x00, + 0x0a, 0x00, 0x21, 0x00, 0x0a, 0x00, 0x21, 0x00, 0x0a, 0x00, 0x21, 0x00, + 0x0a, 0x00, 0x21, 0x00, 0x0a, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, + 0x0a, 0x00, 0x21, 0x00, 0x0a, 0x00, 0x21, 0x00, 0x0a, 0x00, 0x21, 0x00, + 0x0a, 0x00, 0x21, 0x00, 0x0a, 0x00, 0x21, 0x00, 0x0a, 0x00, 0x21, 0x00, + 0x0a, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x40, 0x20, 0x00, 0x00, 0x80, 0x40, + 0x00, 0x00, 0xc0, 0x60, 0x00, 0x00, 0x00, 0x81, 0x00, 0x00, 0x40, 0xa1, + 0x00, 0x00, 0x80, 0xc1, 0x00, 0x00, 0xc0, 0xe1, 0x00, 0x00, 0x00, 0x02, + 0x01, 0x00, 0x40, 0x22, 0x01, 0x00, 0x80, 0x42, 0x01, 0x00, 0xc0, 0x62, + 0x01, 0x00, 0x00, 0x83, 0x01, 0x00, 0x40, 0xa3, 0x01, 0x00, 0x80, 0xc3, + 0x01, 0x00, 0xc0, 0xe3, 0x01, 0x00, 0x40, 0x20, 0x00, 0x00, 0x40, 0x20, + 0x00, 0x00, 0x40, 0x20, 0x00, 0x00, 0x40, 0x20, 0x00, 0x00, 0x40, 0x20, + 0x00, 0x00, 0x40, 0x20, 0x00, 0x00, 0x40, 0x20, 0x00, 0x00, 0x40, 0x20, + 0x00, 0x00, 0x40, 0x20, 0x00, 0x00, 0x40, 0x20, 0x00, 0x00, 0x40, 0x20, + 0x00, 0x00, 0x40, 0x20, 0x00, 0x00, 0x40, 0x20, 0x00, 0x00, 0x40, 0x20, + 0x00, 0x00, 0x40, 0x20, 0x00, 0x00, 0x40, 0x20, 0x00, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x05, 0x80, 0x41, 0x93, 0x1c, 0x00, 0x00, 0x00, 0x1c, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x22, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, + 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x07, + 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, + 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, + 0x03, 0x98, 0x40, 0x00, 0x00, 0x00, 0x50, 0x40, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0xc0, 0x08, + 0x07, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, 0x00, 0x10, 0x10, 0x3c, + 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x40, 0x00, + 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, + 0x00, 0x24, 0x40, 0x23, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, + 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, + 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, + 0x00, 0x22, 0x21, 0x00, 0x0a, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, + 0x0a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x0b, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, + 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, 0x0c, 0x00, 0x21, 0x00, + 0x0b, 0x00, 0x21, 0x00, 0x0b, 0x00, 0x21, 0x00, 0x0b, 0x00, 0x21, 0x00, + 0x0b, 0x00, 0x21, 0x00, 0x0b, 0x00, 0x21, 0x00, 0x0b, 0x00, 0x21, 0x00, + 0x0b, 0x00, 0x21, 0x00, 0x0b, 0x00, 0x21, 0x00, 0x0b, 0x00, 0x51, 0x15, + 0x03, 0x00, 0x21, 0x00, 0x0b, 0x00, 0x21, 0x00, 0x0b, 0x00, 0x21, 0x00, + 0x0b, 0x00, 0x21, 0x00, 0x0b, 0x00, 0x21, 0x00, 0x0b, 0x00, 0x21, 0x00, + 0x0b, 0x00, 0x21, 0x00, 0x0b, 0x00, 0x59, 0x15, 0x0f, 0x00, 0xc0, 0x01, + 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x40, 0x05, 0x00, 0x00, 0x00, 0x07, + 0x00, 0x00, 0xc0, 0x08, 0x00, 0x00, 0x80, 0x0a, 0x00, 0x00, 0x40, 0x0c, + 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0xc0, 0x0f, 0x00, 0x00, 0x80, 0x11, + 0x00, 0x00, 0x40, 0x13, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0xc0, 0x16, + 0x00, 0x00, 0x80, 0x18, 0x00, 0x00, 0x40, 0x1a, 0x00, 0x00, 0xc0, 0x01, + 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0xc0, 0x01, + 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0xc0, 0x01, + 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0xc0, 0x01, + 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0xc0, 0x01, + 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0xc0, 0x01, + 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x1c, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x05, 0x80, 0x69, 0x93, 0x0e, 0x00, + 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0xc3, 0x40, + 0x01, 0x50, 0x00, 0x40, 0x01, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, + 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x07, + 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, + 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, + 0x03, 0x98, 0x40, 0x00, 0x00, 0x00, 0x20, 0x40, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0x80, 0x03, + 0x07, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x01, 0x00, 0x10, 0x10, 0x3c, + 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x40, 0x00, + 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, + 0x00, 0x24, 0x40, 0x27, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, + 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, + 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, + 0x00, 0x22, 0x21, 0x00, 0x0b, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, + 0x0b, 0x01, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x6d, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, + 0x0d, 0x00, 0x21, 0x00, 0x0c, 0x00, 0x21, 0x00, 0x0c, 0x00, 0x21, 0x00, + 0x0c, 0x00, 0x21, 0x00, 0x0c, 0x00, 0x21, 0x00, 0x0c, 0x00, 0x21, 0x00, + 0x0c, 0x00, 0x21, 0x00, 0x0c, 0x00, 0x21, 0x00, 0x0c, 0x00, 0x21, 0x00, + 0x0c, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, 0x0c, 0x00, 0x21, 0x00, + 0x0c, 0x00, 0x21, 0x00, 0x0c, 0x00, 0x21, 0x00, 0x0c, 0x00, 0x21, 0x00, + 0x0c, 0x00, 0x21, 0x00, 0x0c, 0x00, 0x21, 0x00, 0x0c, 0x00, 0x59, 0x15, + 0x0f, 0x00, 0x40, 0x40, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0xc0, 0xc0, + 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x40, 0x41, 0x01, 0x00, 0x80, 0x81, + 0x01, 0x00, 0xc0, 0xc1, 0x01, 0x00, 0x00, 0x02, 0x02, 0x00, 0x40, 0x42, + 0x02, 0x00, 0x80, 0x82, 0x02, 0x00, 0xc0, 0xc2, 0x02, 0x00, 0x00, 0x03, + 0x03, 0x00, 0x40, 0x43, 0x03, 0x00, 0x80, 0x83, 0x03, 0x00, 0xc0, 0xc3, + 0x03, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, + 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, + 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, + 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, + 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, + 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x05, 0x80, + 0x41, 0x93, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, + 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x5a, 0x13, + 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, + 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x20, 0x00, + 0x00, 0x00, 0x30, 0x20, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x20, + 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0xa0, 0xc2, 0x01, 0x00, 0x40, 0x92, + 0x04, 0x80, 0x82, 0x00, 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, + 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, + 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0x43, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, + 0x0c, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, 0x0c, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, + 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, + 0xc0, 0xff, 0x40, 0x02, 0x0e, 0x00, 0x21, 0x00, 0x0d, 0x00, 0x21, 0x00, + 0x0d, 0x00, 0x21, 0x00, 0x0d, 0x00, 0x21, 0x00, 0x0d, 0x00, 0x21, 0x00, + 0x0d, 0x00, 0x21, 0x00, 0x0d, 0x00, 0x21, 0x00, 0x0d, 0x00, 0x21, 0x00, + 0x0d, 0x00, 0x21, 0x00, 0x0d, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, + 0x0d, 0x00, 0x21, 0x00, 0x0d, 0x00, 0x21, 0x00, 0x0d, 0x00, 0x21, 0x00, + 0x0d, 0x00, 0x21, 0x00, 0x0d, 0x00, 0x21, 0x00, 0x0d, 0x00, 0x21, 0x00, + 0x0d, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x80, 0x02, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x80, 0x07, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x80, 0x0c, + 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x80, 0x11, 0x00, 0x00, 0x00, 0x14, + 0x00, 0x00, 0x80, 0x16, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x80, 0x1b, + 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x80, 0x20, 0x00, 0x00, 0x00, 0x23, + 0x00, 0x00, 0x80, 0x25, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x05, 0x80, 0x69, 0x93, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0xc3, 0xa0, 0x42, 0x50, 0x00, 0x40, + 0x01, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, + 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x5a, 0x13, + 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, + 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x20, 0x00, + 0x00, 0x00, 0x30, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0xa0, 0x82, 0x03, 0x00, 0x40, 0x92, + 0x04, 0x80, 0x82, 0x00, 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, + 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, + 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0x47, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, + 0x0d, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, 0x0d, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x0e, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, + 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, 0x0f, 0x00, 0x21, 0x00, + 0x0e, 0x00, 0x21, 0x00, 0x0e, 0x00, 0x21, 0x00, 0x0e, 0x00, 0x21, 0x00, + 0x0e, 0x00, 0x21, 0x00, 0x0e, 0x00, 0x21, 0x00, 0x0e, 0x00, 0x21, 0x00, + 0x0e, 0x00, 0x21, 0x00, 0x0e, 0x00, 0x21, 0x00, 0x0e, 0x00, 0x51, 0x15, + 0x03, 0x00, 0x21, 0x00, 0x0e, 0x00, 0x21, 0x00, 0x0e, 0x00, 0x21, 0x00, + 0x0e, 0x00, 0x21, 0x00, 0x0e, 0x00, 0x21, 0x00, 0x0e, 0x00, 0x21, 0x00, + 0x0e, 0x00, 0x21, 0x00, 0x0e, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x80, 0x00, 0x01, 0x00, 0xc0, 0x80, 0x01, 0x00, 0x00, 0x01, + 0x02, 0x00, 0x40, 0x81, 0x02, 0x00, 0x80, 0x01, 0x03, 0x00, 0xc0, 0x81, + 0x03, 0x00, 0x00, 0x02, 0x04, 0x00, 0x40, 0x82, 0x04, 0x00, 0x80, 0x02, + 0x05, 0x00, 0xc0, 0x82, 0x05, 0x00, 0x00, 0x03, 0x06, 0x00, 0x40, 0x83, + 0x06, 0x00, 0x80, 0x03, 0x07, 0x00, 0xc0, 0x83, 0x07, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x05, 0x80, 0x41, 0x93, 0x0e, 0x00, + 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x00, 0x33, 0x23, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, + 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, + 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x20, 0x00, 0x00, 0x00, 0x30, 0x40, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x7a, 0x01, + 0x50, 0x00, 0xa0, 0x82, 0x03, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, + 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, + 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x31, 0x20, + 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0x6f, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, 0x0e, 0x20, 0x40, 0x14, + 0x00, 0x22, 0xe1, 0x00, 0x0e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x6f, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, + 0x10, 0x00, 0x21, 0x00, 0x0f, 0x00, 0x21, 0x00, 0x0f, 0x00, 0x21, 0x00, + 0x0f, 0x00, 0x21, 0x00, 0x0f, 0x00, 0x21, 0x00, 0x0f, 0x00, 0x21, 0x00, + 0x0f, 0x00, 0x21, 0x00, 0x0f, 0x00, 0x21, 0x00, 0x0f, 0x00, 0x21, 0x00, + 0x0f, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, 0x0f, 0x00, 0x21, 0x00, + 0x0f, 0x00, 0x21, 0x00, 0x0f, 0x00, 0x21, 0x00, 0x0f, 0x00, 0x21, 0x00, + 0x0f, 0x00, 0x21, 0x00, 0x0f, 0x00, 0x21, 0x00, 0x0f, 0x00, 0x59, 0x15, + 0x0f, 0x00, 0x80, 0x02, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x80, 0x07, + 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x80, 0x0c, 0x00, 0x00, 0x00, 0x0f, + 0x00, 0x00, 0x80, 0x11, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x80, 0x16, + 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x80, 0x1b, 0x00, 0x00, 0x00, 0x1e, + 0x00, 0x00, 0x80, 0x20, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x80, 0x25, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x05, 0x80, + 0x69, 0x93, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0xc3, 0xa0, 0x42, 0x50, 0x00, 0x40, 0x01, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, + 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, + 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x20, 0x00, 0x00, 0x00, 0x30, 0x40, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x7a, 0x01, + 0x50, 0x00, 0xa0, 0x82, 0x03, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, + 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, + 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x31, 0x20, + 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0x73, 0x10, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, 0x0f, 0x20, 0x40, 0x14, + 0x00, 0x22, 0xe1, 0x00, 0x0f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, + 0x6d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, + 0xc0, 0xff, 0x40, 0x02, 0x11, 0x00, 0x21, 0x00, 0x10, 0x00, 0x21, 0x00, + 0x10, 0x00, 0x21, 0x00, 0x10, 0x00, 0x21, 0x00, 0x10, 0x00, 0x21, 0x00, + 0x10, 0x00, 0x21, 0x00, 0x10, 0x00, 0x21, 0x00, 0x10, 0x00, 0x21, 0x00, + 0x10, 0x00, 0x21, 0x00, 0x10, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, + 0x10, 0x00, 0x21, 0x00, 0x10, 0x00, 0x21, 0x00, 0x10, 0x00, 0x21, 0x00, + 0x10, 0x00, 0x21, 0x00, 0x10, 0x00, 0x21, 0x00, 0x10, 0x00, 0x21, 0x00, + 0x10, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x40, 0x80, 0x00, 0x00, 0x80, 0x00, + 0x01, 0x00, 0xc0, 0x80, 0x01, 0x00, 0x00, 0x01, 0x02, 0x00, 0x40, 0x81, + 0x02, 0x00, 0x80, 0x01, 0x03, 0x00, 0xc0, 0x81, 0x03, 0x00, 0x00, 0x02, + 0x04, 0x00, 0x40, 0x82, 0x04, 0x00, 0x80, 0x02, 0x05, 0x00, 0xc0, 0x82, + 0x05, 0x00, 0x00, 0x03, 0x06, 0x00, 0x40, 0x83, 0x06, 0x00, 0x80, 0x03, + 0x07, 0x00, 0xc0, 0x83, 0x07, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x05, 0x80, 0x41, 0x93, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x33, + 0x23, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, + 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, + 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, + 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, + 0x03, 0x98, 0x20, 0x00, 0x00, 0x00, 0x30, 0x40, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0xa0, 0x82, + 0x03, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, 0x00, 0x10, 0x10, 0x3c, + 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x40, 0x00, + 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, + 0x00, 0x24, 0x40, 0x9b, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, + 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, + 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, + 0x00, 0x22, 0x21, 0x00, 0x10, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, + 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x11, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, + 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, 0x12, 0x00, 0x21, 0x00, + 0x11, 0x00, 0x21, 0x00, 0x11, 0x00, 0x21, 0x00, 0x11, 0x00, 0x21, 0x00, + 0x11, 0x00, 0x21, 0x00, 0x11, 0x00, 0x21, 0x00, 0x11, 0x00, 0x21, 0x00, + 0x11, 0x00, 0x21, 0x00, 0x11, 0x00, 0x21, 0x00, 0x11, 0x00, 0x51, 0x15, + 0x03, 0x00, 0x21, 0x00, 0x11, 0x00, 0x21, 0x00, 0x11, 0x00, 0x21, 0x00, + 0x11, 0x00, 0x21, 0x00, 0x11, 0x00, 0x21, 0x00, 0x11, 0x00, 0x21, 0x00, + 0x11, 0x00, 0x21, 0x00, 0x11, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x80, 0x07, 0x00, 0x00, 0x00, 0x0a, + 0x00, 0x00, 0x80, 0x0c, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x80, 0x11, + 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x80, 0x16, 0x00, 0x00, 0x00, 0x19, + 0x00, 0x00, 0x80, 0x1b, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x80, 0x20, + 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x80, 0x25, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x05, 0x80, 0x69, 0x93, 0x0e, 0x00, + 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0xc3, 0xa0, + 0x42, 0x50, 0x00, 0x40, 0x01, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, + 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, + 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, + 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, + 0x03, 0x98, 0x20, 0x00, 0x00, 0x00, 0x30, 0x40, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0xa0, 0x82, + 0x03, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, 0x00, 0x10, 0x10, 0x3c, + 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x40, 0x00, + 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, + 0x00, 0x24, 0x40, 0x9f, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, + 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, + 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, + 0x00, 0x22, 0x21, 0x00, 0x11, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, + 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x6d, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, + 0x13, 0x00, 0x21, 0x00, 0x12, 0x00, 0x21, 0x00, 0x12, 0x00, 0x21, 0x00, + 0x12, 0x00, 0x21, 0x00, 0x12, 0x00, 0x21, 0x00, 0x12, 0x00, 0x21, 0x00, + 0x12, 0x00, 0x21, 0x00, 0x12, 0x00, 0x21, 0x00, 0x12, 0x00, 0x21, 0x00, + 0x12, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, 0x12, 0x00, 0x21, 0x00, + 0x12, 0x00, 0x21, 0x00, 0x12, 0x00, 0x21, 0x00, 0x12, 0x00, 0x21, 0x00, + 0x12, 0x00, 0x21, 0x00, 0x12, 0x00, 0x21, 0x00, 0x12, 0x00, 0x59, 0x15, + 0x0f, 0x00, 0x40, 0x80, 0x00, 0x00, 0x80, 0x00, 0x01, 0x00, 0xc0, 0x80, + 0x01, 0x00, 0x00, 0x01, 0x02, 0x00, 0x40, 0x81, 0x02, 0x00, 0x80, 0x01, + 0x03, 0x00, 0xc0, 0x81, 0x03, 0x00, 0x00, 0x02, 0x04, 0x00, 0x40, 0x82, + 0x04, 0x00, 0x80, 0x02, 0x05, 0x00, 0xc0, 0x82, 0x05, 0x00, 0x00, 0x03, + 0x06, 0x00, 0x40, 0x83, 0x06, 0x00, 0x80, 0x03, 0x07, 0x00, 0xc0, 0x83, + 0x07, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x05, 0x80, + 0x41, 0x93, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x33, 0x23, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, + 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x5a, 0x13, + 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, + 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x20, 0x00, + 0x00, 0x00, 0x30, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0xa0, 0x82, 0x03, 0x00, 0x40, 0x92, + 0x04, 0x80, 0x82, 0x00, 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, + 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, + 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0xc7, + 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, + 0x12, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, 0x12, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x00, + 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, + 0xc0, 0xff, 0x40, 0x02, 0x14, 0x00, 0x21, 0x00, 0x13, 0x00, 0x21, 0x00, + 0x13, 0x00, 0x21, 0x00, 0x13, 0x00, 0x21, 0x00, 0x13, 0x00, 0x21, 0x00, + 0x13, 0x00, 0x21, 0x00, 0x13, 0x00, 0x21, 0x00, 0x13, 0x00, 0x21, 0x00, + 0x13, 0x00, 0x21, 0x00, 0x13, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, + 0x13, 0x00, 0x21, 0x00, 0x13, 0x00, 0x21, 0x00, 0x13, 0x00, 0x21, 0x00, + 0x13, 0x00, 0x21, 0x00, 0x13, 0x00, 0x21, 0x00, 0x13, 0x00, 0x21, 0x00, + 0x13, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x80, 0x02, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x80, 0x07, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x80, 0x0c, + 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x80, 0x11, 0x00, 0x00, 0x00, 0x14, + 0x00, 0x00, 0x80, 0x16, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x80, 0x1b, + 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x80, 0x20, 0x00, 0x00, 0x00, 0x23, + 0x00, 0x00, 0x80, 0x25, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x05, 0x80, 0x69, 0x93, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0xc3, 0xa0, 0x42, 0x50, 0x00, 0x40, + 0x01, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, + 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x5a, 0x13, + 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, + 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x20, 0x00, + 0x00, 0x00, 0x30, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0xa0, 0x82, 0x03, 0x00, 0x40, 0x92, + 0x04, 0x80, 0x82, 0x00, 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, + 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, + 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0xcb, + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, + 0x13, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, 0x13, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x14, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, + 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, 0x15, 0x00, 0x21, 0x00, + 0x14, 0x00, 0x21, 0x00, 0x14, 0x00, 0x21, 0x00, 0x14, 0x00, 0x21, 0x00, + 0x14, 0x00, 0x21, 0x00, 0x14, 0x00, 0x21, 0x00, 0x14, 0x00, 0x21, 0x00, + 0x14, 0x00, 0x21, 0x00, 0x14, 0x00, 0x21, 0x00, 0x14, 0x00, 0x51, 0x15, + 0x03, 0x00, 0x21, 0x00, 0x14, 0x00, 0x21, 0x00, 0x14, 0x00, 0x21, 0x00, + 0x14, 0x00, 0x21, 0x00, 0x14, 0x00, 0x21, 0x00, 0x14, 0x00, 0x21, 0x00, + 0x14, 0x00, 0x21, 0x00, 0x14, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x80, 0x00, 0x01, 0x00, 0xc0, 0x80, 0x01, 0x00, 0x00, 0x01, + 0x02, 0x00, 0x40, 0x81, 0x02, 0x00, 0x80, 0x01, 0x03, 0x00, 0xc0, 0x81, + 0x03, 0x00, 0x00, 0x02, 0x04, 0x00, 0x40, 0x82, 0x04, 0x00, 0x80, 0x02, + 0x05, 0x00, 0xc0, 0x82, 0x05, 0x00, 0x00, 0x03, 0x06, 0x00, 0x40, 0x83, + 0x06, 0x00, 0x80, 0x03, 0x07, 0x00, 0xc0, 0x83, 0x07, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x05, 0x80, 0x41, 0x93, 0x0e, 0x00, + 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x00, 0x33, 0x23, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, + 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, + 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x20, 0x00, 0x00, 0x00, 0x30, 0x40, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x7a, 0x01, + 0x50, 0x00, 0xa0, 0x82, 0x03, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, + 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, + 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x31, 0x20, + 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0xf3, 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, 0x14, 0x20, 0x40, 0x14, + 0x00, 0x22, 0xe1, 0x00, 0x14, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x6f, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, + 0x16, 0x00, 0x21, 0x00, 0x15, 0x00, 0x21, 0x00, 0x15, 0x00, 0x21, 0x00, + 0x15, 0x00, 0x21, 0x00, 0x15, 0x00, 0x21, 0x00, 0x15, 0x00, 0x21, 0x00, + 0x15, 0x00, 0x21, 0x00, 0x15, 0x00, 0x21, 0x00, 0x15, 0x00, 0x21, 0x00, + 0x15, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, 0x15, 0x00, 0x21, 0x00, + 0x15, 0x00, 0x21, 0x00, 0x15, 0x00, 0x21, 0x00, 0x15, 0x00, 0x21, 0x00, + 0x15, 0x00, 0x21, 0x00, 0x15, 0x00, 0x21, 0x00, 0x15, 0x00, 0x59, 0x15, + 0x0f, 0x00, 0x80, 0x02, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x80, 0x07, + 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x80, 0x0c, 0x00, 0x00, 0x00, 0x0f, + 0x00, 0x00, 0x80, 0x11, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x80, 0x16, + 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x80, 0x1b, 0x00, 0x00, 0x00, 0x1e, + 0x00, 0x00, 0x80, 0x20, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x80, 0x25, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x80, 0x02, + 0x00, 0x00, 0x80, 0x02, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x05, 0x80, + 0x69, 0x93, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0xc3, 0xa0, 0x42, 0x50, 0x00, 0x40, 0x01, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, + 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, + 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x20, 0x00, 0x00, 0x00, 0x30, 0x40, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x7a, 0x01, + 0x50, 0x00, 0xa0, 0x82, 0x03, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, + 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, + 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x31, 0x20, + 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0xf7, 0x28, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, 0x15, 0x20, 0x40, 0x14, + 0x00, 0x22, 0xe1, 0x00, 0x15, 0x01, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, + 0x6d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, + 0xc0, 0xff, 0x40, 0x02, 0x17, 0x00, 0x21, 0x00, 0x16, 0x00, 0x21, 0x00, + 0x16, 0x00, 0x21, 0x00, 0x16, 0x00, 0x21, 0x00, 0x16, 0x00, 0x21, 0x00, + 0x16, 0x00, 0x21, 0x00, 0x16, 0x00, 0x21, 0x00, 0x16, 0x00, 0x21, 0x00, + 0x16, 0x00, 0x21, 0x00, 0x16, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, + 0x16, 0x00, 0x21, 0x00, 0x16, 0x00, 0x21, 0x00, 0x16, 0x00, 0x21, 0x00, + 0x16, 0x00, 0x21, 0x00, 0x16, 0x00, 0x21, 0x00, 0x16, 0x00, 0x21, 0x00, + 0x16, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x40, 0x80, 0x00, 0x00, 0x80, 0x00, + 0x01, 0x00, 0xc0, 0x80, 0x01, 0x00, 0x00, 0x01, 0x02, 0x00, 0x40, 0x81, + 0x02, 0x00, 0x80, 0x01, 0x03, 0x00, 0xc0, 0x81, 0x03, 0x00, 0x00, 0x02, + 0x04, 0x00, 0x40, 0x82, 0x04, 0x00, 0x80, 0x02, 0x05, 0x00, 0xc0, 0x82, + 0x05, 0x00, 0x00, 0x03, 0x06, 0x00, 0x40, 0x83, 0x06, 0x00, 0x80, 0x03, + 0x07, 0x00, 0xc0, 0x83, 0x07, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, + 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x40, 0x80, 0x00, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x05, 0x80, 0x41, 0x93, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x33, + 0x23, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, + 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, + 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, + 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, + 0x03, 0x98, 0x20, 0x00, 0x00, 0x00, 0x30, 0x40, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0xa0, 0x82, + 0x03, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, 0x00, 0x10, 0x10, 0x3c, + 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x40, 0x00, + 0x00, 0x00, 0x80, 0x03, 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, + 0x00, 0x24, 0x40, 0x1f, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, + 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, + 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, + 0x00, 0x22, 0x21, 0x00, 0x16, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, + 0x16, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x17, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, + 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, 0x18, 0x00, 0x21, 0x00, + 0x17, 0x00, 0x21, 0x00, 0x17, 0x00, 0x21, 0x00, 0x17, 0x00, 0x21, 0x00, + 0x17, 0x00, 0x21, 0x00, 0x17, 0x00, 0x21, 0x00, 0x17, 0x00, 0x21, 0x00, + 0x17, 0x00, 0x21, 0x00, 0x17, 0x00, 0x21, 0x00, 0x17, 0x00, 0x51, 0x15, + 0x03, 0x00, 0x21, 0x00, 0x17, 0x00, 0x21, 0x00, 0x17, 0x00, 0x21, 0x00, + 0x17, 0x00, 0x21, 0x00, 0x17, 0x00, 0x21, 0x00, 0x17, 0x00, 0x21, 0x00, + 0x17, 0x00, 0x21, 0x00, 0x17, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x10, + 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x1c, + 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x28, + 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x34, + 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x05, 0x80, 0x69, 0x93, 0x07, 0x00, + 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0xc3, 0x40, + 0x01, 0x50, 0x00, 0x40, 0x01, 0x00, 0x07, 0x00, 0x00, 0x00, 0x04, 0x44, + 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, + 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x80, 0x03, + 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, + 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, + 0x03, 0x98, 0x20, 0x00, 0x00, 0x00, 0x20, 0x40, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0xc0, 0x81, + 0x03, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x01, 0x00, 0x10, 0x10, 0x3c, + 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x40, 0x00, + 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, + 0x00, 0x24, 0x40, 0x23, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, + 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, + 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, + 0x00, 0x22, 0x21, 0x00, 0x17, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, + 0x17, 0x01, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x6d, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, + 0x19, 0x00, 0x21, 0x00, 0x18, 0x00, 0x21, 0x00, 0x18, 0x00, 0x21, 0x00, + 0x18, 0x00, 0x21, 0x00, 0x18, 0x00, 0x21, 0x00, 0x18, 0x00, 0x21, 0x00, + 0x18, 0x00, 0x21, 0x00, 0x18, 0x00, 0x21, 0x00, 0x18, 0x00, 0x21, 0x00, + 0x18, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, 0x18, 0x00, 0x21, 0x00, + 0x18, 0x00, 0x21, 0x00, 0x18, 0x00, 0x21, 0x00, 0x18, 0x00, 0x21, 0x00, + 0x18, 0x00, 0x21, 0x00, 0x18, 0x00, 0x21, 0x00, 0x18, 0x00, 0x59, 0x15, + 0x0f, 0x00, 0x80, 0x00, 0x01, 0x00, 0x00, 0x01, 0x02, 0x00, 0x80, 0x01, + 0x03, 0x00, 0x00, 0x02, 0x04, 0x00, 0x80, 0x02, 0x05, 0x00, 0x00, 0x03, + 0x06, 0x00, 0x80, 0x03, 0x07, 0x00, 0x00, 0x04, 0x08, 0x00, 0x80, 0x04, + 0x09, 0x00, 0x00, 0x05, 0x0a, 0x00, 0x80, 0x05, 0x0b, 0x00, 0x00, 0x06, + 0x0c, 0x00, 0x80, 0x06, 0x0d, 0x00, 0x00, 0x07, 0x0e, 0x00, 0x80, 0x07, + 0x0f, 0x00, 0x80, 0x00, 0x01, 0x00, 0x80, 0x00, 0x01, 0x00, 0x80, 0x00, + 0x01, 0x00, 0x80, 0x00, 0x01, 0x00, 0x80, 0x00, 0x01, 0x00, 0x80, 0x00, + 0x01, 0x00, 0x80, 0x00, 0x01, 0x00, 0x80, 0x00, 0x01, 0x00, 0x80, 0x00, + 0x01, 0x00, 0x80, 0x00, 0x01, 0x00, 0x80, 0x00, 0x01, 0x00, 0x80, 0x00, + 0x01, 0x00, 0x80, 0x00, 0x01, 0x00, 0x80, 0x00, 0x01, 0x00, 0x80, 0x00, + 0x01, 0x00, 0x80, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x07, 0x00, + 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x05, 0x80, + 0x41, 0x93, 0x07, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x04, 0x44, 0x24, 0x00, 0x05, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, + 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0x5a, 0x13, + 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, + 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x10, 0x00, + 0x00, 0x00, 0x30, 0x20, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x20, + 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0x50, 0xe1, 0x00, 0x00, 0x40, 0x92, + 0x04, 0x80, 0x82, 0x00, 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, + 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0xc0, 0x01, + 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0x63, + 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, + 0x18, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, 0x18, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x19, 0x00, + 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, 0x05, 0x00, 0x40, 0x15, + 0xc0, 0xff, 0x40, 0x02, 0x1a, 0x00, 0x21, 0x00, 0x19, 0x00, 0x21, 0x00, + 0x19, 0x00, 0x21, 0x00, 0x19, 0x00, 0x21, 0x00, 0x19, 0x00, 0x21, 0x00, + 0x19, 0x00, 0x21, 0x00, 0x19, 0x00, 0x21, 0x00, 0x19, 0x00, 0x21, 0x00, + 0x19, 0x00, 0x21, 0x00, 0x19, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, + 0x19, 0x00, 0x21, 0x00, 0x19, 0x00, 0x21, 0x00, 0x19, 0x00, 0x21, 0x00, + 0x19, 0x00, 0x21, 0x00, 0x19, 0x00, 0x21, 0x00, 0x19, 0x00, 0x21, 0x00, + 0x19, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x0a, + 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x19, + 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x28, + 0x00, 0x00, 0x00, 0x2d, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x37, + 0x00, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0x00, 0x46, + 0x00, 0x00, 0x00, 0x4b, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x07, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x05, 0x80, 0x69, 0x93, 0x07, 0x00, 0x00, 0x00, 0x07, 0x00, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0xc3, 0xa0, 0x42, 0x50, 0x00, 0x40, + 0x01, 0x00, 0x07, 0x00, 0x00, 0x00, 0x04, 0x44, 0x24, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, + 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0x5a, 0x13, + 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, + 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x10, 0x00, + 0x00, 0x00, 0x30, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0x50, 0xc1, 0x01, 0x00, 0x40, 0x92, + 0x04, 0x80, 0x82, 0x00, 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, + 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0xc0, 0x01, + 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0x6b, + 0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, + 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, + 0x19, 0x20, 0x40, 0x14, 0x00, 0x22, 0xe1, 0x00, 0x19, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x1a, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0xff, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x1f, 0x01, 0x00, + 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0x40, 0x02, 0x1b, 0x00, 0x21, 0x00, + 0x1a, 0x00, 0x21, 0x00, 0x1a, 0x00, 0x21, 0x00, 0x1a, 0x00, 0x21, 0x00, + 0x1a, 0x00, 0x21, 0x00, 0x1a, 0x00, 0x21, 0x00, 0x1a, 0x00, 0x21, 0x00, + 0x1a, 0x00, 0x21, 0x00, 0x1a, 0x00, 0x21, 0x00, 0x1a, 0x00, 0x51, 0x15, + 0x03, 0x00, 0x21, 0x00, 0x1a, 0x00, 0x21, 0x00, 0x1a, 0x00, 0x21, 0x00, + 0x1a, 0x00, 0x21, 0x00, 0x1a, 0x00, 0x21, 0x00, 0x1a, 0x00, 0x21, 0x00, + 0x1a, 0x00, 0x21, 0x00, 0x1a, 0x00, 0x59, 0x15, 0x0f, 0x00, 0x80, 0x00, + 0x02, 0x00, 0x00, 0x01, 0x04, 0x00, 0x80, 0x01, 0x06, 0x00, 0x00, 0x02, + 0x08, 0x00, 0x80, 0x02, 0x0a, 0x00, 0x00, 0x03, 0x0c, 0x00, 0x80, 0x03, + 0x0e, 0x00, 0x00, 0x04, 0x10, 0x00, 0x80, 0x04, 0x12, 0x00, 0x00, 0x05, + 0x14, 0x00, 0x80, 0x05, 0x16, 0x00, 0x00, 0x06, 0x18, 0x00, 0x80, 0x06, + 0x1a, 0x00, 0x00, 0x07, 0x1c, 0x00, 0x80, 0x07, 0x1e, 0x00, 0x80, 0x00, + 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, + 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, + 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, + 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, + 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, + 0x02, 0x00, 0x01, 0x00, 0x01, 0x00, 0x07, 0x00, 0x00, 0x00, 0x07, 0x00, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x05, 0x80, 0x41, 0x93, 0x07, 0x00, + 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x07, 0x00, + 0x00, 0x00, 0x04, 0x44, 0x24, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0x5a, 0x13, 0x00, 0x00, 0x31, 0x20, + 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, 0x50, 0x00, 0x30, 0x01, + 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x10, 0x00, 0x00, 0x00, 0x30, 0x40, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x7a, 0x01, + 0x50, 0x00, 0x50, 0xc1, 0x01, 0x00, 0x40, 0x92, 0x04, 0x80, 0x82, 0x00, + 0x00, 0x10, 0x10, 0x3c, 0x11, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x44, 0x94, + 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0x31, 0x20, + 0x30, 0x01, 0x44, 0x95, 0x00, 0x24, 0x40, 0xbb, 0x41, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x44, 0x93, 0x80, 0x21, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x42, 0x94, 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x13, 0x00, 0x22, 0x21, 0x00, 0x1a, 0x20, 0x40, 0x14, + 0x00, 0x22, 0xe1, 0x00, 0x1a, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1b, 0x00, 0x74, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x60, 0xf8, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x09, 0x00, 0x05, 0x00, 0x40, 0x15, 0xc0, 0xff, 0xc0, 0x02, + 0x1c, 0x00, 0x21, 0x00, 0x1b, 0x00, 0x21, 0x00, 0x1b, 0x00, 0x21, 0x00, + 0x1b, 0x00, 0x21, 0x00, 0x1b, 0x00, 0x21, 0x00, 0x1b, 0x00, 0x21, 0x00, + 0x1b, 0x00, 0x21, 0x00, 0x1b, 0x00, 0x21, 0x00, 0x1b, 0x00, 0x21, 0x00, + 0x1b, 0x00, 0x51, 0x15, 0x03, 0x00, 0x21, 0x00, 0x1b, 0x00, 0x21, 0x00, + 0x1b, 0x00, 0x21, 0x00, 0x1b, 0x00, 0x21, 0x00, 0x1b, 0x00, 0x21, 0x00, + 0x1b, 0x00, 0x21, 0x00, 0x1b, 0x00, 0x21, 0x00, 0x1b, 0x00, 0x59, 0x15, + 0x0f, 0x00, 0x80, 0x00, 0x02, 0x00, 0x00, 0x01, 0x04, 0x00, 0x80, 0x01, + 0x06, 0x00, 0x00, 0x02, 0x08, 0x00, 0x80, 0x02, 0x0a, 0x00, 0x00, 0x03, + 0x0c, 0x00, 0x80, 0x03, 0x0e, 0x00, 0x00, 0x04, 0x10, 0x00, 0x80, 0x04, + 0x12, 0x00, 0x00, 0x05, 0x14, 0x00, 0x80, 0x05, 0x16, 0x00, 0x00, 0x06, + 0x18, 0x00, 0x80, 0x06, 0x1a, 0x00, 0x00, 0x07, 0x1c, 0x00, 0x80, 0x07, + 0x1e, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, + 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, + 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, + 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, + 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x80, 0x00, + 0x02, 0x00, 0x80, 0x00, 0x02, 0x00, 0x01, 0x00, 0x01, 0x00, 0x07, 0x00, + 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x05, 0x80, + 0x41, 0x93, 0x07, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x04, 0x44, 0x24, 0x00, 0x05, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x42, 0x13, 0x0c, 0x80, 0xce, 0x00, + 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0xc0, 0x01, 0x00, 0x00, 0x5a, 0x13, + 0x00, 0x00, 0x31, 0x20, 0x00, 0x01, 0x41, 0x90, 0x00, 0x00, 0x72, 0x01, + 0x50, 0x00, 0x30, 0x01, 0x50, 0x00, 0x45, 0x90, 0x03, 0x98, 0x10, 0x00, + 0x00, 0x00, 0x30, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x7a, 0x01, 0x50, 0x00, 0x50, 0xc1, 0x01, 0x00, 0x40, 0x92, + 0x04, 0x80, 0x82, 0x00, 0x00, 0x10, 0x10, 0x3c, 0x10, 0x00, 0x00, 0x3c, + 0x00, 0x00, 0x44, 0x94, 0x00, 0x81, 0x40, 0x00, 0x00, 0x00, 0xc0, 0x01, + 0x00, 0x00, 0x31, 0x20, 0x30, 0x01, 0x40, 0x96, 0x11, 0x83, 0x12, 0x00, + 0x00, 0x00, 0xa0, 0x00, 0xa0, 0x00, 0xe0, 0x00, 0x1b, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0x00, 0x01, 0x07, 0x01, 0x20, 0x44, 0x95, + 0x00, 0x24, 0x40, 0xc3, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x93, + 0x80, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x94, + 0x80, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x13, + 0x00, 0x22, 0x21, 0x00, 0x1b, 0x20, 0x40, 0x14, 0x00, 0x22, 0x21, 0x00, + 0x1b, 0x01, + } }, +}; + +#define MOBILENET_GENERIC_SIZE 0x298 +static const struct ane_h16_sparse_run mobilenet_generic_runs[] = { + { 0x0, 5, (const u8[]){ + 0x01, 0x00, 0x00, 0x00, 0x10, + } }, + { 0x204, 21, (const u8[]){ + 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + } }, + { 0x229, 32, (const u8[]){ + 0xc0, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + } }, + { 0x25a, 31, (const u8[]){ + 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + } }, + { 0x28a, 8, (const u8[]){ + 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + } }, +}; + +#define MOBILENET_OPERATION_SIZE 0x410 +static const struct ane_h16_sparse_run mobilenet_operation_runs[] = { + { 0x0, 46, (const u8[]){ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0xef, 0x03, + } }, +}; + +#define MOBILENET_PROCEDURE_SIZE 0x38 +static const struct ane_h16_sparse_run mobilenet_procedure_runs[] = { + { 0x0, 53, (const u8[]){ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x04, + } }, +}; + +#define MOBILENET_PROPERTIES_SIZE 0x28 +static const struct ane_h16_sparse_run mobilenet_properties_runs[] = { + { 0x0, 13, (const u8[]){ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1c, + } }, + { 0x20, 2, (const u8[]){ + 0x10, 0x31, + } }, +}; + +#endif /* __ANE_H17_MOBILENET_FIXTURE_H__ */ diff --git a/drivers/accel/ane/ane_t6021_rtclient_main.c b/drivers/accel/ane/ane_t6021_rtclient_main.c index d5691ff084fca..2e30f02392675 100644 --- a/drivers/accel/ane/ane_t6021_rtclient_main.c +++ b/drivers/accel/ane/ane_t6021_rtclient_main.c @@ -94,6 +94,7 @@ #include #include "ane_fw_platform.h" +#include "ane_fw_cold.h" #include "ane_h16_protocol.h" #include "ane_h16_pending.h" #include "ane_h16_resources.h" @@ -4025,7 +4026,7 @@ static int ane_rtclient_probe(struct platform_device *pdev) return -ENODEV; /* Later generations cannot enter H14 boot or parameter paths. */ if (soc->platform) - return ane_fw_platform_preflight(pdev, soc->platform); + return ane_fw_platform_probe(pdev, soc->platform); mutex_lock(&ane_rtclient_probe_guard_lock); if (ane_rtclient_probe_failed_dirty || ane_rtclient_active) { @@ -4057,8 +4058,14 @@ static int ane_rtclient_probe(struct platform_device *pdev) static void ane_rtclient_remove(struct platform_device *pdev) { + const struct ane_t602x_soc *soc = of_device_get_match_data(&pdev->dev); struct ane_rtclient *ane = platform_get_drvdata(pdev); + if (soc->platform) { + ane_fw_platform_remove(pdev); + return; + } + if (READ_ONCE(ane_t6021_perf_ane) == ane) WRITE_ONCE(ane_t6021_perf_ane, NULL); @@ -4137,8 +4144,12 @@ static struct attribute *ane_t6021_stats_attrs[] = { static umode_t ane_t6021_stats_is_visible(struct kobject *kobj, struct attribute *attr, int n) { - struct ane_rtclient *ane = dev_get_drvdata(kobj_to_dev(kobj)); + struct device *dev = kobj_to_dev(kobj); + const struct ane_t602x_soc *soc = of_device_get_match_data(dev); + struct ane_rtclient *ane = dev_get_drvdata(dev); + if (soc->platform) + return 0; if (attr == &dev_attr_ane_stats.attr && !ane->stats_slots) return 0; return attr->mode; @@ -4184,20 +4195,23 @@ static int __init ane_rtclient_init(void) if (!soc->platform) continue; - if (soc->platform->owner == ANE_FW_OWNER_EXCLAVE) { - of_node_put(np); - pr_info("ane: H17 ANE: Linux exclave ownership handoff unavailable; refusing registration\n"); - return -EOPNOTSUPP; - } + if (!of_device_is_available(np)) + continue; pdev = of_find_device_by_node(np); of_node_put(np); if (!pdev) { pr_info("ane: H16 ANE: firmware transport unvalidated; keeping disabled\n"); return -EOPNOTSUPP; } - ret = ane_fw_platform_preflight(pdev, soc->platform); + if (soc->platform->soc == 0x8140) + ret = ane_fw_staged_init(pdev, soc->platform); + else + ret = ane_fw_platform_preflight(pdev, soc->platform); put_device(&pdev->dev); - return ret; + if (ret) + return ret; + ane_rtclient_driver.driver_managed_dma = true; + break; } WRITE_ONCE(ane_rtclient_probe_failed_dirty, false); @@ -4211,6 +4225,8 @@ module_init(ane_rtclient_init); static void __exit ane_rtclient_exit(void) { + if (ane_fw_staged_exit()) + return; platform_driver_unregister(&ane_rtclient_driver); ane_t6021_trace_free(); } diff --git a/drivers/accel/ane/boot.kunitconfig b/drivers/accel/ane/boot.kunitconfig new file mode 100644 index 0000000000000..e42cb95da6483 --- /dev/null +++ b/drivers/accel/ane/boot.kunitconfig @@ -0,0 +1,9 @@ +CONFIG_KUNIT=y +CONFIG_DRM=y +CONFIG_DRM_ACCEL=y +CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST=y +CONFIG_KASAN=y +CONFIG_KASAN_GENERIC=y +CONFIG_UBSAN=y +CONFIG_UBSAN_BOUNDS=y +CONFIG_UBSAN_SHIFT=y diff --git a/drivers/accel/ane/transport.kunitconfig b/drivers/accel/ane/transport.kunitconfig new file mode 100644 index 0000000000000..21888e4786283 --- /dev/null +++ b/drivers/accel/ane/transport.kunitconfig @@ -0,0 +1,9 @@ +CONFIG_KUNIT=y +CONFIG_DRM=y +CONFIG_DRM_ACCEL=y +CONFIG_DRM_ACCEL_ANE_FW_KUNIT_TEST=y +CONFIG_KASAN=y +CONFIG_KASAN_GENERIC=y +CONFIG_KASAN_INLINE=y +CONFIG_UBSAN=y +CONFIG_UBSAN_BOUNDS=y diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 6ba3a178c636b..46eb147157185 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -105,7 +105,7 @@ config APPLE_SIO where it provides DMA services. config APPLE_SIO_KUNIT_TEST - bool "KUnit tests for the Apple SIO probe" if !KUNIT_ALL_TESTS + bool "KUnit tests for the Apple SIO probe and boot protocol" if !KUNIT_ALL_TESTS depends on APPLE_SIO=y && KUNIT=y depends on OF_OVERLAY default KUNIT_ALL_TESTS @@ -113,6 +113,7 @@ config APPLE_SIO_KUNIT_TEST Check which SIO device-tree nodes the probe goes on with: the generic SIO of the M1 and M2, and a T6030 SIO only with validated protocol-10 boot tables. The compatible selects the firmware layout. + Also check protocol-9 table order, DMA array counts and ACK failures. config APPLE_SIO_T6030_KUNIT_TEST bool "KUnit tests for the T6030 SIO boot-table check" if !KUNIT_ALL_TESTS diff --git a/drivers/dma/apple-sio-boot-test.c b/drivers/dma/apple-sio-boot-test.c new file mode 100644 index 0000000000000..3d8f93d11d8bb --- /dev/null +++ b/drivers/dma/apple-sio-boot-test.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Included by apple-sio.c: exercise protocol-9 publication and ACK failures. */ +#include + +struct sio_boot_test { + struct device dev; + struct sio_data *sio; + u32 keys[20]; + u32 values[20]; + unsigned int count; + u32 fail_key; +}; + +/* The loader order, including its response-array parameters. */ +static const u32 sio_boot_loader_params[] = { + 0x1e, 0x30, 0x1f, 0x30, + 0x0b, 0x31, 0x0c, 0x1b80, + 0x0f, 0x34, 0x10, 0x1e000, + 0x30d, 0x54, 0x30e, 0x4000, + 0x1a, 0x58, 0x1b, 0x40, + 0x22, 0x59, 0x23, 0xa0, + 0x1c, 0x5a, 0x1d, 0x40, +}; + +/* Protocol-9 control and performance order, independent of loader order. */ +static const u32 sio_boot_expected_keys[] = { + 0x0f, 0x10, 0x1a, 0x1b, 0x1e, 0x1f, + 0x22, 0x23, 0x1c, 0x1d, 0x30d, 0x30e, +}; + +static struct sio_boot_test *sio_boot_test_alloc(struct kunit *test) +{ + struct sio_boot_test *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->sio = kunit_kzalloc(test, sizeof(*ctx->sio), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx->sio); + ctx->sio->dev = &ctx->dev; + ctx->fail_key = U32_MAX; + return ctx; +} + +static int sio_boot_set_param(struct sio_data *sio, u32 key, u32 val) +{ + struct sio_boot_test *ctx = container_of(sio->dev, struct sio_boot_test, dev); + + if (ctx->count >= ARRAY_SIZE(ctx->keys)) + return -E2BIG; + ctx->keys[ctx->count] = key; + ctx->values[ctx->count++] = val; + return key == ctx->fail_key ? -ETIME : 0; +} + +static void sio_boot_params(__be32 *params, bool reverse) +{ + unsigned int i, n = ARRAY_SIZE(sio_boot_loader_params); + + for (i = 0; i < n; i += 2) { + unsigned int src = reverse ? n - i - 2 : i; + + params[i] = cpu_to_be32(sio_boot_loader_params[src]); + params[i + 1] = cpu_to_be32(sio_boot_loader_params[src + 1]); + } +} + +static void sio_boot_order_test(struct kunit *test) +{ + struct sio_boot_test *ctx = sio_boot_test_alloc(test); + __be32 params[ARRAY_SIZE(sio_boot_loader_params)]; + unsigned int i, reverse; + int err; + + for (reverse = 0; reverse < 2; reverse++) { + ctx->count = 0; + sio_boot_params(params, reverse); + err = sio_send_v9_dt_params(ctx->sio, params, ARRAY_SIZE(params), + sio_boot_set_param); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, ctx->count, ARRAY_SIZE(sio_boot_expected_keys)); + for (i = 0; i < ctx->count; i++) + KUNIT_EXPECT_EQ(test, ctx->keys[i], sio_boot_expected_keys[i]); + /* Reordering must preserve every loader-supplied control value. */ + for (i = 0; i < ctx->count; i++) { + unsigned int j; + + for (j = 0; j < ARRAY_SIZE(sio_boot_loader_params); j += 2) + if (sio_boot_loader_params[j] == ctx->keys[i]) + break; + KUNIT_ASSERT_LT(test, j, ARRAY_SIZE(sio_boot_loader_params)); + KUNIT_EXPECT_EQ(test, ctx->values[i], sio_boot_loader_params[j + 1]); + } + KUNIT_EXPECT_EQ(test, ctx->values[7], 0xa0U); + KUNIT_EXPECT_EQ(test, ctx->values[11], 0x4000U); + } +} + +static void sio_boot_optional_tables_test(struct kunit *test) +{ + struct sio_boot_test *ctx = sio_boot_test_alloc(test); + __be32 params[ARRAY_SIZE(sio_boot_loader_params) + 8]; + unsigned int n = ARRAY_SIZE(sio_boot_loader_params); + + sio_boot_params(params, false); + params[n++] = cpu_to_be32(0x24); + params[n++] = cpu_to_be32(0x60); + params[n++] = cpu_to_be32(0x25); + params[n++] = cpu_to_be32(0x20); + params[n++] = cpu_to_be32(0x20); + params[n++] = cpu_to_be32(0x61); + params[n++] = cpu_to_be32(0x21); + params[n++] = cpu_to_be32(0x10); + KUNIT_ASSERT_EQ(test, sio_send_v9_dt_params(ctx->sio, params, n, + sio_boot_set_param), 0); + KUNIT_ASSERT_EQ(test, ctx->count, 16U); + KUNIT_EXPECT_EQ(test, ctx->keys[6], 0x20U); + KUNIT_EXPECT_EQ(test, ctx->keys[8], 0x24U); + KUNIT_EXPECT_EQ(test, ctx->keys[10], 0x22U); + KUNIT_EXPECT_EQ(test, ctx->keys[14], 0x30dU); +} + +static void sio_boot_bad_params_test(struct kunit *test) +{ + struct sio_boot_test *ctx = sio_boot_test_alloc(test); + __be32 params[ARRAY_SIZE(sio_boot_loader_params) + 2]; + unsigned int n = ARRAY_SIZE(sio_boot_loader_params); + + sio_boot_params(params, false); + KUNIT_EXPECT_EQ(test, sio_send_v9_dt_params(ctx->sio, params, n - 1, + sio_boot_set_param), -EINVAL); + /* A missing required pair, an unknown key, a duplicate, and half a table. */ + KUNIT_EXPECT_EQ(test, sio_send_v9_dt_params(ctx->sio, params, n - 2, + sio_boot_set_param), -EINVAL); + params[n] = cpu_to_be32(0x1234); + params[n + 1] = 0; + KUNIT_EXPECT_EQ(test, sio_send_v9_dt_params(ctx->sio, params, n + 2, + sio_boot_set_param), -EINVAL); + params[n] = cpu_to_be32(0x22); + KUNIT_EXPECT_EQ(test, sio_send_v9_dt_params(ctx->sio, params, n + 2, + sio_boot_set_param), -EINVAL); + params[n] = cpu_to_be32(0x20); + KUNIT_EXPECT_EQ(test, sio_send_v9_dt_params(ctx->sio, params, n + 2, + sio_boot_set_param), -EINVAL); + KUNIT_EXPECT_EQ(test, ctx->count, 0U); +} + +static void sio_boot_control_ack_failure_test(struct kunit *test) +{ + struct sio_boot_test *ctx = sio_boot_test_alloc(test); + __be32 params[ARRAY_SIZE(sio_boot_loader_params)]; + unsigned int fail; + int err; + + sio_boot_params(params, false); + for (fail = 0; fail < ARRAY_SIZE(sio_boot_expected_keys); fail++) { + ctx->count = 0; + ctx->fail_key = sio_boot_expected_keys[fail]; + err = sio_send_v9_dt_params(ctx->sio, params, ARRAY_SIZE(params), + sio_boot_set_param); + KUNIT_EXPECT_EQ(test, err, -ETIME); + /* No dependent table follows a failed ACK, including shim size. */ + KUNIT_ASSERT_EQ(test, ctx->count, fail + 1); + KUNIT_EXPECT_EQ(test, ctx->keys[fail], ctx->fail_key); + } +} + +static void sio_boot_dma_arrays_test(struct kunit *test) +{ + struct sio_boot_test *ctx = sio_boot_test_alloc(test); + const u32 keys[] = { 1, 2, 0x0b, 0x0c }; + const u32 values[] = { 0x1234, 0xaaa, 0x5678, 0x1b80 }; + unsigned int i, fail; + + ctx->sio->shmem_iova = 0x1234000; + ctx->sio->responses_iova = 0x5678000; + KUNIT_ASSERT_EQ(test, sio_send_v9_shmem(ctx->sio, sio_boot_set_param), 0); + KUNIT_ASSERT_EQ(test, ctx->count, ARRAY_SIZE(keys)); + for (i = 0; i < ctx->count; i++) { + KUNIT_EXPECT_EQ(test, ctx->keys[i], keys[i]); + KUNIT_EXPECT_EQ(test, ctx->values[i], values[i]); + } + /* Every failed ACK stops publication, including the segment count. */ + for (fail = 0; fail < ARRAY_SIZE(keys); fail++) { + ctx->count = 0; + ctx->fail_key = keys[fail]; + KUNIT_EXPECT_EQ(test, sio_send_v9_shmem(ctx->sio, sio_boot_set_param), -ETIME); + KUNIT_EXPECT_EQ(test, ctx->count, fail + 1); + } +} + +static struct kunit_case sio_boot_test_cases[] = { + KUNIT_CASE(sio_boot_order_test), + KUNIT_CASE(sio_boot_optional_tables_test), + KUNIT_CASE(sio_boot_bad_params_test), + KUNIT_CASE(sio_boot_control_ack_failure_test), + KUNIT_CASE(sio_boot_dma_arrays_test), + {} +}; + +static struct kunit_suite sio_boot_test_suite = { + .name = "apple-sio-boot", + .test_cases = sio_boot_test_cases, +}; + +kunit_test_suite(sio_boot_test_suite); diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c index 65ba4511501c7..f2d3afaaf9bc3 100644 --- a/drivers/dma/apple-sio.c +++ b/drivers/dma/apple-sio.c @@ -6,9 +6,11 @@ */ #include +#include #include #include #include +#include #include #include #include @@ -20,6 +22,7 @@ #include #include #include +#include #include "dmaengine.h" #include "virt-dma.h" @@ -51,6 +54,12 @@ #define SIO_SHMEM_SIZE 0x1000 #define SIO_NO_DESC_SLOTS 64 +/* Protocol 9 publishes entry counts, rather than allocation sizes. */ +#define SIO_V9_SEGMENTS 0xaaa +#define SIO_V9_SEGMENT_SIZE 12 +#define SIO_V9_RESPONSES 0x1b80 +#define SIO_V9_RESPONSE_SIZE 16 + /* * There are two kinds of 'transaction descriptors' in play here. * @@ -80,6 +89,8 @@ struct sio_coproc_desc { u64 pad3; } __packed; static_assert(sizeof(struct sio_coproc_desc) == 48); +static_assert(56 + SIO_NO_DESC_SLOTS * sizeof(struct sio_coproc_desc) <= + SIO_V9_SEGMENTS * SIO_V9_SEGMENT_SIZE); struct sio_shmem_chan_config { u32 datashape; @@ -102,6 +113,38 @@ struct sio_chan { struct sio_shmem_chan_config cfg; struct sio_tx *current_tx; + + /* + * Until a client sets an idle transfer, the channel keeps the old + * accounting, charging ACKs and REPORTs to the current transfer; its + * ISSUEs not yet acknowledged are only counted. + */ + unsigned int pending; + + /* + * The ring accounting, from the first idle transfer until the channel + * is quiet again (sio_ring_leave_locked()): the client's idle transfer + * (struct apple_sio_idle), issued while no other transfer is; + * transfers ended while it is set, until their issued descriptors + * completed; and the owners of the issued descriptors in issue order, + * which is the order of their reports. The last one may still wait for + * its ISSUE to be acknowledged. + */ + bool ring; + struct sio_tx *idle; + struct list_head retired; + wait_queue_head_t drained; + struct sio_tx *issued[SIO_MAX_NINFLIGHT]; + unsigned int issued_head; + unsigned int nissued; + bool issuing; + /* Nothing is issued between a TERMINATE and the ring's reset. */ + bool stopping; + /* + * Ended transfers with nothing in flight, freed once no completion + * callback can still be running (sio_free_dead()). + */ + struct list_head dead; }; #define SIO_NTAGS 16 @@ -120,7 +163,11 @@ struct sio_data { struct dma_device dma; struct device *dev; struct apple_rtkit *rtk; + u32 protocol; void *shmem; + dma_addr_t shmem_iova; + void *responses; + dma_addr_t responses_iova; struct sio_coproc_desc *shmem_desc_base; unsigned long *desc_allocated; @@ -145,6 +192,7 @@ struct sio_tx { struct completion done; bool terminated; + bool retired; size_t period_len; int nperiods; int ninflight; @@ -435,23 +483,22 @@ static void sio_tx_free(struct virt_dma_desc *vd) kfree(siotx); } -static struct dma_async_tx_descriptor *sio_prep_dma_cyclic( - struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, - size_t period_len, enum dma_transfer_direction direction, - unsigned long flags) +static struct sio_tx *sio_alloc_tx(struct sio_chan *siochan, dma_addr_t buf_addr, + size_t buf_len, size_t period_len, gfp_t gfp) { - struct sio_chan *siochan = to_sio_chan(chan); struct sio_tx *siotx = NULL; - int i, nperiods = buf_len / period_len; + int i, nperiods; - if (direction != sio_chan_direction(siochan->no)) + if (!period_len || buf_len < period_len) return NULL; + nperiods = buf_len / period_len; - siotx = kzalloc(struct_size(siotx, siodesc, nperiods), GFP_NOWAIT); + siotx = kzalloc(struct_size(siotx, siodesc, nperiods), gfp); if (!siotx) return NULL; init_completion(&siotx->done); + siotx->vd.tx.chan = &siochan->vc.chan; siotx->period_len = period_len; siotx->nperiods = nperiods; @@ -460,7 +507,6 @@ static struct dma_async_tx_descriptor *sio_prep_dma_cyclic( siotx->siodesc[i] = d = sio_alloc_desc(siochan->host); if (!d) { - siotx->vd.tx.chan = &siochan->vc.chan; sio_tx_free(&siotx->vd); return NULL; } @@ -477,6 +523,24 @@ static struct dma_async_tx_descriptor *sio_prep_dma_cyclic( } dma_wmb(); + return siotx; +} + +static struct dma_async_tx_descriptor *sio_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct sio_chan *siochan = to_sio_chan(chan); + struct sio_tx *siotx; + + if (direction != sio_chan_direction(siochan->no)) + return NULL; + + siotx = sio_alloc_tx(siochan, buf_addr, buf_len, period_len, GFP_NOWAIT); + if (!siotx) + return NULL; + return vchan_tx_prep(&siochan->vc, &siotx->vd, flags); } @@ -521,32 +585,172 @@ static enum dma_status sio_tx_status(struct dma_chan *chan, dma_cookie_t cookie, static bool sio_fill_in_locked(struct sio_chan *siochan); -static void sio_handle_issue_ack(struct sio_chan *siochan, void *cookie, bool ok) +static struct sio_tx **sio_issued_slot(struct sio_chan *siochan, unsigned int i) { - dma_cookie_t tx_cookie = (unsigned long) cookie; - unsigned long flags; - struct sio_tx *tx; + return &siochan->issued[(siochan->issued_head + i) % SIO_MAX_NINFLIGHT]; +} + +/* Neither in flight nor waiting for its ISSUE to be acknowledged */ +static bool sio_tx_quiet_locked(struct sio_chan *siochan, struct sio_tx *tx) +{ + unsigned int i; + + if (tx->ninflight) + return false; + for (i = 0; i < siochan->nissued; i++) + if (*sio_issued_slot(siochan, i) == tx) + return false; + return true; +} + +/* A transfer that is freed no longer owns issued descriptors */ +static void sio_scrub_locked(struct sio_chan *siochan, struct sio_tx *tx) +{ + unsigned int i; + + for (i = 0; i < siochan->nissued; i++) + if (*sio_issued_slot(siochan, i) == tx) + *sio_issued_slot(siochan, i) = NULL; +} + +/* Descriptors are issued from the current transfer, else from the idle one. */ +static struct sio_tx *sio_source_locked(struct sio_chan *siochan) +{ + struct sio_tx *tx = siochan->current_tx; + + if (tx) + return tx->terminated ? NULL : tx; + if (siochan->idle && !siochan->idle->terminated) + return siochan->idle; + return NULL; +} + +/* + * The first idle transfer: the ring accounting starts from what the old one + * has in flight, all of it the current transfer's. It cannot place two ISSUEs + * still waiting for their acks, nor an ended transfer's descriptors. + */ +static int sio_ring_enter_locked(struct sio_chan *siochan) +{ + struct sio_tx *tx = siochan->current_tx; + unsigned int i, n = tx ? tx->ninflight : 0; + + if (siochan->pending > 1 || (siochan->pending && !tx) || (tx && tx->terminated) || + n + siochan->pending > SIO_MAX_NINFLIGHT) + return -EBUSY; + siochan->issued_head = 0; + siochan->nissued = n + siochan->pending; + for (i = 0; i < siochan->nissued; i++) + siochan->issued[i] = tx; + siochan->issuing = siochan->pending; + siochan->pending = 0; + siochan->ring = true; + return 0; +} + +/* + * The resync point: once the channel is quiet, with no idle, current or + * retired transfer and nothing issued, the old accounting resumes. + */ +static void sio_ring_leave_locked(struct sio_chan *siochan) +{ + if (!siochan->ring || siochan->idle || siochan->current_tx || + !list_empty(&siochan->retired) || siochan->nissued || siochan->issuing) + return; + siochan->ring = false; + siochan->issued_head = 0; +} + +/* + * An ended transfer with no descriptor left in flight: a retired one goes to + * be freed, the waiter of another one is woken. + */ +static void sio_settle_locked(struct sio_chan *siochan, struct sio_tx *tx) +{ + if (!tx || !tx->terminated || !sio_tx_quiet_locked(siochan, tx)) + return; + if (tx->retired) { + list_move_tail(&tx->vd.node, &siochan->dead); + wake_up(&siochan->drained); + return; + } + /* sio_terminate_work() or sio_clear_idle() */ + complete(&tx->done); +} + +/* + * The old accounting, for channels that never had an idle transfer: an ACK + * is the current transfer's, if its cookie still is. + */ +static void sio_old_issue_ack_locked(struct sio_chan *siochan, dma_cookie_t tx_cookie, bool ok) +{ + struct sio_tx *tx = siochan->current_tx; + if (siochan->pending) + siochan->pending--; if (!ok) { dev_err(siochan->host->dev, "nacked issue on chan %d\n", siochan->no); return; } + if (!tx || tx_cookie != tx->vd.tx.cookie || tx->terminated) + return; - spin_lock_irqsave(&siochan->vc.lock, flags); - if (!siochan->current_tx || tx_cookie != siochan->current_tx->vd.tx.cookie || - siochan->current_tx->terminated) - goto out; - - tx = siochan->current_tx; tx->next = (tx->next + 1) % tx->nperiods; tx->ninflight++; sio_fill_in_locked(siochan); +} + +/* The ring accounting: an ACK is the last issued descriptor's. */ +static void sio_ring_issue_ack_locked(struct sio_chan *siochan, bool ok) +{ + struct sio_tx *tx; + + siochan->issuing = false; + if (!siochan->nissued) { + dev_err_ratelimited(siochan->host->dev, "chan %d: ack without an issued descriptor\n", + siochan->no); + return; + } + + tx = *sio_issued_slot(siochan, siochan->nissued - 1); + if (!ok) { + dev_err(siochan->host->dev, "nacked issue on chan %d\n", siochan->no); + siochan->nissued--; + if (!siochan->nissued) + wake_up(&siochan->drained); + /* + * During a handover this may have been the ended transfer's + * last ISSUE, and no report is left to start the next one. A + * transfer whose own ISSUE was refused is not issued again. + */ + if (sio_source_locked(siochan) != tx) + sio_fill_in_locked(siochan); + sio_settle_locked(siochan, tx); + return; + } -out: + /* An ended transfer's descriptor is in flight all the same. */ + if (tx) { + tx->next = (tx->next + 1) % tx->nperiods; + tx->ninflight++; + } + sio_fill_in_locked(siochan); +} + +static void sio_handle_issue_ack(struct sio_chan *siochan, void *cookie, bool ok) +{ + unsigned long flags; + + spin_lock_irqsave(&siochan->vc.lock, flags); + if (siochan->ring) + sio_ring_issue_ack_locked(siochan, ok); + else + sio_old_issue_ack_locked(siochan, (unsigned long)cookie, ok); spin_unlock_irqrestore(&siochan->vc.lock, flags); } -static bool sio_fill_in_locked(struct sio_chan *siochan) +/* The old accounting: the current transfer's next descriptor */ +static bool sio_fill_in_old_locked(struct sio_chan *siochan) { struct sio_data *sio = siochan->host; struct sio_tx *tx = siochan->current_tx; @@ -560,13 +764,50 @@ static bool sio_fill_in_locked(struct sio_chan *siochan) ret = sio_send_siomsg_atomic(sio, FIELD_PREP(SIOMSG_EP, siochan->no) | FIELD_PREP(SIOMSG_TYPE, MSG_ISSUE) | FIELD_PREP(SIOMSG_DATA, sio_coproc_desc_slot(sio, d)), - sio_handle_issue_ack, (void *) (uintptr_t) tx->vd.tx.cookie); + sio_handle_issue_ack, (void *)(uintptr_t)tx->vd.tx.cookie); if (ret < 0) dev_err_ratelimited(sio->dev, "can't issue on chan %d ninflight %d: %d\n", siochan->no, tx->ninflight, ret); + else + siochan->pending++; + return true; +} + +/* The ring accounting: one ISSUE at a time, each after the previous one's ack */ +static bool sio_fill_in_ring_locked(struct sio_chan *siochan) +{ + struct sio_data *sio = siochan->host; + struct sio_tx *tx = sio_source_locked(siochan); + struct sio_coproc_desc *d; + int ret; + + if (!tx || siochan->issuing || siochan->stopping || + siochan->nissued >= SIO_MAX_NINFLIGHT) + return false; + + d = tx->siodesc[tx->next]; + ret = sio_send_siomsg_atomic(sio, FIELD_PREP(SIOMSG_EP, siochan->no) | + FIELD_PREP(SIOMSG_TYPE, MSG_ISSUE) | + FIELD_PREP(SIOMSG_DATA, sio_coproc_desc_slot(sio, d)), + sio_handle_issue_ack, NULL); + if (ret < 0) { + dev_err_ratelimited(sio->dev, "can't issue on chan %d ninflight %d: %d\n", + siochan->no, siochan->nissued, ret); + return false; + } + /* The ack takes the channel lock, held here. */ + *sio_issued_slot(siochan, siochan->nissued++) = tx; + siochan->issuing = true; return true; } +static bool sio_fill_in_locked(struct sio_chan *siochan) +{ + if (siochan->ring) + return sio_fill_in_ring_locked(siochan); + return siochan->current_tx && sio_fill_in_old_locked(siochan); +} + static void sio_update_current_tx_locked(struct sio_chan *siochan) { struct virt_dma_desc *vd = vchan_next_desc(&siochan->vc); @@ -592,14 +833,37 @@ static void sio_issue_pending(struct dma_chan *chan) static int sio_terminate_all(struct dma_chan *chan) { struct sio_chan *siochan = to_sio_chan(chan); + struct sio_tx *tx; unsigned long flags; LIST_HEAD(to_free); spin_lock_irqsave(&siochan->vc.lock, flags); - if (siochan->current_tx && !siochan->current_tx->terminated) { - dma_cookie_complete(&siochan->current_tx->vd.tx); - siochan->current_tx->terminated = true; - schedule_work(&siochan->terminate_wq); + tx = siochan->current_tx; + if (tx && !tx->terminated) { + dma_cookie_complete(&tx->vd.tx); + tx->terminated = true; + if (siochan->ring && siochan->idle && !siochan->idle->terminated) { + /* + * No TERMINATE: the idle transfer follows the issued + * descriptors, and the peripheral runs on without a gap. + * sio_synchronize() waits for those to complete. + */ + if (siochan->vc.cyclic == &tx->vd) + siochan->vc.cyclic = NULL; + siochan->current_tx = NULL; + /* Its callback may be running: sio_free_dead() frees it. */ + if (sio_tx_quiet_locked(siochan, tx)) { + list_add_tail(&tx->vd.node, &siochan->dead); + } else { + tx->retired = true; + list_add_tail(&tx->vd.node, &siochan->retired); + } + sio_fill_in_locked(siochan); + } else { + if (siochan->ring && sio_tx_quiet_locked(siochan, tx)) + complete(&tx->done); + schedule_work(&siochan->terminate_wq); + } } vchan_get_all_descriptors(&siochan->vc, &to_free); spin_unlock_irqrestore(&siochan->vc.lock, flags); @@ -638,23 +902,223 @@ static void sio_terminate_work(struct work_struct *wq) spin_lock_irqsave(&siochan->vc.lock, flags); WARN_ON(siochan->current_tx != tx); siochan->current_tx = NULL; + sio_scrub_locked(siochan, tx); sio_update_current_tx_locked(siochan); + sio_ring_leave_locked(siochan); spin_unlock_irqrestore(&siochan->vc.lock, flags); sio_tx_free(&tx->vd); } +static bool sio_drained(struct sio_chan *siochan) +{ + unsigned long flags; + bool drained; + + spin_lock_irqsave(&siochan->vc.lock, flags); + drained = list_empty(&siochan->retired); + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + return drained; +} + +static bool sio_ring_empty(struct sio_chan *siochan) +{ + unsigned long flags; + bool empty; + + spin_lock_irqsave(&siochan->vc.lock, flags); + empty = !siochan->nissued && !siochan->issuing; + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + return empty; +} + +/* + * Issued descriptors that did not complete in time: stop the channel and wait + * for their reports, so that their buffers are no longer read. Nothing is + * issued from the TERMINATE until the channel is known stopped. A channel + * that acknowledged TERMINATE and still reports nothing is resynced: its ring + * is emptied, as nothing it holds is played any more. A channel that refused + * TERMINATE or did not answer cannot be known to have stopped: -EIO, and its + * transfers stay as they are. Then the current or idle transfer is issued + * again. + */ +static int sio_stop_quiet(struct sio_chan *siochan) +{ + struct device *dev = siochan->host->dev; + unsigned long flags; + struct sio_tx *tx; + unsigned int i; + int ret; + + dev_err(dev, "chan %d: issued descriptors did not complete, stopping it\n", siochan->no); + spin_lock_irqsave(&siochan->vc.lock, flags); + siochan->stopping = true; + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + /* 1 on ACK, 0 on NACK, negative without an answer */ + ret = sio_call(siochan->host, FIELD_PREP(SIOMSG_EP, siochan->no) | + FIELD_PREP(SIOMSG_TYPE, MSG_TERMINATE)); + if (ret == 1) + wait_event_timeout(siochan->drained, sio_ring_empty(siochan), + msecs_to_jiffies(500)); + else if (!ret) + dev_err(dev, "chan %d: TERMINATE refused, its buffers may still be read\n", + siochan->no); + else + dev_err(dev, "chan %d: not stopped (%d), its buffers may still be read\n", + siochan->no, ret); + + spin_lock_irqsave(&siochan->vc.lock, flags); + if (ret == 1 && (siochan->nissued || siochan->issuing)) { + dev_err(dev, "chan %d: stopped with %u descriptors unreported, ring reset\n", + siochan->no, siochan->nissued); + for (i = 0; i < siochan->nissued; i++) { + tx = *sio_issued_slot(siochan, i); + if (tx) + tx->ninflight = 0; + } + siochan->nissued = 0; + siochan->issuing = false; + siochan->issued_head = 0; + } + if (ret == 1) { + list_splice_tail_init(&siochan->retired, &siochan->dead); + if (siochan->idle && siochan->idle->terminated) + complete(&siochan->idle->done); + } + siochan->stopping = false; + sio_fill_in_locked(siochan); + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + return ret == 1 ? 0 : -EIO; +} + +/* Frees the ended transfers once no completion callback can still use them. */ +static void sio_free_dead(struct sio_chan *siochan) +{ + struct sio_tx *tx, *n; + unsigned long flags; + LIST_HEAD(dead); + + spin_lock_irqsave(&siochan->vc.lock, flags); + list_splice_init(&siochan->dead, &dead); + spin_unlock_irqrestore(&siochan->vc.lock, flags); + if (list_empty(&dead)) + return; + + /* vchan_complete() runs a callback with its descriptor unlocked. */ + tasklet_kill(&siochan->vc.task); + list_for_each_entry_safe(tx, n, &dead, vd.node) { + list_del(&tx->vd.node); + sio_tx_free(&tx->vd); + } +} + static void sio_synchronize(struct dma_chan *chan) { struct sio_chan *siochan = to_sio_chan(chan); flush_work(&siochan->terminate_wq); + if (READ_ONCE(siochan->ring)) { + /* The caller frees the buffer on return; nothing can stop that. */ + if (!wait_event_timeout(siochan->drained, sio_drained(siochan), + msecs_to_jiffies(500)) && + sio_stop_quiet(siochan)) + dev_WARN_ONCE(siochan->host->dev, 1, + "chan %d: synchronized without being stopped\n", + siochan->no); + /* No callback of an ended transfer runs after this. */ + tasklet_kill(&siochan->vc.task); + } + sio_free_dead(siochan); +} + +/* + * Returns once the idle transfer's descriptors completed. If the channel + * cannot be confirmed stopped, the idle transfer stays, its buffer in use. + */ +static int sio_clear_idle(struct sio_chan *siochan) +{ + struct sio_tx *idle; + unsigned long flags; + bool quiet, stop; + int ret; + + spin_lock_irqsave(&siochan->vc.lock, flags); + idle = siochan->idle; + if (!idle) { + spin_unlock_irqrestore(&siochan->vc.lock, flags); + return 0; + } + idle->terminated = true; + quiet = sio_tx_quiet_locked(siochan, idle); + /* Nothing follows the idle transfer: the channel stops. */ + stop = !quiet && !siochan->current_tx; + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + if (stop) { + ret = sio_call(siochan->host, FIELD_PREP(SIOMSG_EP, siochan->no) | + FIELD_PREP(SIOMSG_TYPE, MSG_TERMINATE)); + if (ret < 0) + dev_err(siochan->host->dev, "terminate call on chan %d failed: %d\n", + siochan->no, ret); + } + if (!quiet && !wait_for_completion_timeout(&idle->done, msecs_to_jiffies(500))) { + ret = sio_stop_quiet(siochan); + if (ret) + return ret; + } + + spin_lock_irqsave(&siochan->vc.lock, flags); + siochan->idle = NULL; + sio_scrub_locked(siochan, idle); + sio_ring_leave_locked(siochan); + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + sio_tx_free(&idle->vd); + sio_free_dead(siochan); + return 0; +} + +/* See struct apple_sio_idle */ +static int sio_set_idle(struct sio_chan *siochan, const struct apple_sio_idle *config) +{ + struct sio_tx *idle; + unsigned long flags; + int ret; + + if (!config->len) + return sio_clear_idle(siochan); + if (!config->period_len || config->len < config->period_len) + return -EINVAL; + + idle = sio_alloc_tx(siochan, config->addr, config->len, config->period_len, GFP_KERNEL); + if (!idle) + return -ENOMEM; + + spin_lock_irqsave(&siochan->vc.lock, flags); + ret = siochan->idle ? -EBUSY : 0; + if (!ret && !siochan->ring) + ret = sio_ring_enter_locked(siochan); + if (ret) { + spin_unlock_irqrestore(&siochan->vc.lock, flags); + sio_tx_free(&idle->vd); + return ret; + } + siochan->idle = idle; + sio_fill_in_locked(siochan); + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + return 0; } static void sio_free_chan_resources(struct dma_chan *chan) { sio_terminate_all(chan); sio_synchronize(chan); + sio_clear_idle(to_sio_chan(chan)); vchan_free_chan_resources(&to_sio_chan(chan)->vc); } @@ -677,20 +1141,61 @@ static void sio_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ dev_err(sio->dev, "SIO down (crashed)"); } -static void sio_process_report(struct sio_chan *siochan) +/* The old accounting: a report is the current transfer's. */ +static void sio_old_report_locked(struct sio_chan *siochan) { - unsigned long flags; + struct sio_tx *tx = siochan->current_tx; - spin_lock_irqsave(&siochan->vc.lock, flags); - if (siochan->current_tx) { - struct sio_tx *tx = siochan->current_tx; + if (!tx) + return; + if (tx->ninflight) + tx->ninflight--; + vchan_cyclic_callback(&tx->vd); + if (!sio_fill_in_locked(siochan) && !tx->ninflight) + complete(&tx->done); +} + +/* The ring accounting: the oldest issued descriptor completed. */ +static void sio_ring_report_locked(struct sio_chan *siochan) +{ + struct sio_tx *tx; + /* An ISSUE not yet acknowledged has no completed descriptor. */ + if (siochan->nissued <= (siochan->issuing ? 1 : 0)) { + dev_warn_ratelimited(siochan->host->dev, "chan %d: report without a descriptor\n", + siochan->no); + return; + } + + tx = *sio_issued_slot(siochan, 0); + siochan->issued_head = (siochan->issued_head + 1) % SIO_MAX_NINFLIGHT; + siochan->nissued--; + if (tx) { if (tx->ninflight) tx->ninflight--; - vchan_cyclic_callback(&tx->vd); - if (!sio_fill_in_locked(siochan) && !tx->ninflight) - complete(&tx->done); + else + dev_err_ratelimited(siochan->host->dev, + "chan %d: report for a descriptor not in flight\n", + siochan->no); + if (tx == siochan->current_tx && !tx->terminated) + vchan_cyclic_callback(&tx->vd); } + sio_fill_in_locked(siochan); + sio_settle_locked(siochan, tx); + if (!siochan->nissued) + wake_up(&siochan->drained); + sio_ring_leave_locked(siochan); +} + +static void sio_process_report(struct sio_chan *siochan) +{ + unsigned long flags; + + spin_lock_irqsave(&siochan->vc.lock, flags); + if (siochan->ring) + sio_ring_report_locked(siochan); + else + sio_old_report_locked(siochan); spin_unlock_irqrestore(&siochan->vc.lock, flags); } @@ -712,6 +1217,7 @@ static void sio_recv_msg(void *cookie, u8 ep, u64 msg) switch (type) { case MSG_STARTED: + sio->protocol = data; dev_info(sio->dev, "SIO protocol v%u\n", data); type = MSG_ACK; /* Pretend this is an ACK */ fallthrough; @@ -827,6 +1333,12 @@ static int sio_device_config(struct dma_chan *chan, struct sio_shmem_chan_config cfg; int ret; + if (config->peripheral_config) { + if (config->peripheral_size != sizeof(struct apple_sio_idle)) + return -EINVAL; + return sio_set_idle(siochan, config->peripheral_config); + } + switch (is_tx ? config->dst_addr_width : config->src_addr_width) { case DMA_SLAVE_BUSWIDTH_1_BYTE: cfg.datashape = 0; @@ -897,41 +1409,166 @@ static int sio_device_config(struct dma_chan *chan, return ret; } +static int sio_set_param(struct sio_data *sio, u32 key, u32 val) +{ + int err; + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_SETUP) | + FIELD_PREP(SIOMSG_PARAM, key & 0xff) | + FIELD_PREP(SIOMSG_EP, key >> 8) | + FIELD_PREP(SIOMSG_DATA, val)); + if (err == 1) + return 0; + if (!err) + err = -ENXIO; + return dev_err_probe(sio->dev, err, "sending SIO parameter %#x value %#x\n", + key, val); +} + +/* + * Control tables must precede performance and DMA state. In particular, the + * shim size (0x23) makes the firmware consume the shim table. Do not enable + * the performance endpoint or publish DMA response state before that ACK. + */ +static const u16 sio_v9_param_order[] = { + 0x0f, 0x10, /* panic buffer */ + 0x1a, 0x1b, /* map ranges */ + 0x1e, 0x1f, /* ASC-wrapper tunables */ + 0x20, 0x21, /* optional miscellaneous tunables */ + 0x24, 0x25, /* optional power-state register table */ + 0x22, 0x23, /* DMA shims */ + 0x1c, 0x1d, /* device types, after shims */ + 0x30d, 0x30e, /* performance endpoint, after control tables */ +}; + +#define SIO_V9_REQUIRED_PARAMS (GENMASK(15, 0) & ~GENMASK(9, 6)) + +static int sio_send_v9_dt_params(struct sio_data *sio, const __be32 *params, + unsigned int nparams, + int (*set_param)(struct sio_data *, u32, u32)) +{ + u32 values[ARRAY_SIZE(sio_v9_param_order)]; + u32 present = 0; + unsigned int i, j; + int err; + + if (!nparams || nparams % 2) + return -EINVAL; + + for (i = 0; i < nparams; i += 2) { + u32 key = be32_to_cpu(params[i]); + + /* Replace the loader's undersized response mapping with our own. */ + if (key == 0x0b || key == 0x0c) + continue; + for (j = 0; j < ARRAY_SIZE(sio_v9_param_order); j++) + if (key == sio_v9_param_order[j]) + break; + if (j == ARRAY_SIZE(sio_v9_param_order) || (present & BIT(j))) + return -EINVAL; + present |= BIT(j); + values[j] = be32_to_cpu(params[i + 1]); + } + if ((present & SIO_V9_REQUIRED_PARAMS) != SIO_V9_REQUIRED_PARAMS) + return -EINVAL; + /* An optional table still requires both its address and its size. */ + for (i = 0; i < ARRAY_SIZE(sio_v9_param_order); i += 2) + if (!!(present & BIT(i)) != !!(present & BIT(i + 1))) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(sio_v9_param_order); i++) { + if (!(present & BIT(i))) + continue; + err = set_param(sio, sio_v9_param_order[i], values[i]); + if (err) + return err; + } + return 0; +} + +static void sio_free_v9_shmem(void *data) +{ + struct sio_data *sio = data; + int err; + + /* An acknowledged shutdown must precede releasing firmware DMA targets. */ + err = apple_rtkit_shutdown(sio->rtk); + if (err) { + dev_err(sio->dev, "SIO shutdown failed: %d; retaining DMA buffers\n", err); + return; + } + dma_free_coherent(sio->dev, SIO_V9_RESPONSES * SIO_V9_RESPONSE_SIZE, + sio->responses, sio->responses_iova); + dma_free_coherent(sio->dev, SIO_V9_SEGMENTS * SIO_V9_SEGMENT_SIZE, + sio->shmem, sio->shmem_iova); +} + +static int sio_send_v9_shmem(struct sio_data *sio, + int (*set_param)(struct sio_data *, u32, u32)) +{ + const struct { + u32 key; + u32 value; + } params[] = { + { 1, sio->shmem_iova >> 12 }, + { 2, SIO_V9_SEGMENTS }, + { 0x0b, sio->responses_iova >> 12 }, + { 0x0c, SIO_V9_RESPONSES }, + }; + unsigned int i; + int err; + + for (i = 0; i < ARRAY_SIZE(params); i++) { + err = set_param(sio, params[i].key, params[i].value); + if (err) + return err; + } + return 0; +} + static int sio_alloc_shmem(struct sio_data *sio) { - dma_addr_t iova; + bool v9 = sio->protocol == 9; + size_t size = v9 ? SIO_V9_SEGMENTS * SIO_V9_SEGMENT_SIZE : SIO_SHMEM_SIZE; int err; - sio->shmem = dma_alloc_coherent(sio->dev, SIO_SHMEM_SIZE, - &iova, GFP_KERNEL | __GFP_ZERO); + sio->shmem = dma_alloc_coherent(sio->dev, size, + &sio->shmem_iova, GFP_KERNEL | __GFP_ZERO); if (!sio->shmem) return -ENOMEM; sio->shmem_desc_base = (struct sio_coproc_desc *) (sio->shmem + 56); - sio->desc_allocated = devm_kzalloc(sio->dev, SIO_NO_DESC_SLOTS / 32, - GFP_KERNEL); - if (!sio->desc_allocated) - return -ENOMEM; + sio->desc_allocated = devm_bitmap_zalloc(sio->dev, SIO_NO_DESC_SLOTS, + GFP_KERNEL); + if (!sio->desc_allocated) { + err = -ENOMEM; + goto free_shmem; + } - err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_SETUP) | - FIELD_PREP(SIOMSG_PARAM, 1) | - FIELD_PREP(SIOMSG_DATA, iova >> 12)); - if (err != 1) { - if (err == 0) - err = -EINVAL; - return err; + if (v9) { + sio->responses = dma_alloc_coherent(sio->dev, + SIO_V9_RESPONSES * SIO_V9_RESPONSE_SIZE, + &sio->responses_iova, GFP_KERNEL | __GFP_ZERO); + if (!sio->responses) { + err = -ENOMEM; + goto free_shmem; + } + /* Registered before publication, released before the RTKit handle. */ + err = devm_add_action_or_reset(sio->dev, sio_free_v9_shmem, sio); + if (err) + return err; + return sio_send_v9_shmem(sio, sio_set_param); } - err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_SETUP) | - FIELD_PREP(SIOMSG_PARAM, 2) | - FIELD_PREP(SIOMSG_DATA, SIO_SHMEM_SIZE)); - if (err != 1) { - if (err == 0) - err = -EINVAL; + err = sio_set_param(sio, 1, sio->shmem_iova >> 12); + if (err) return err; - } + return sio_set_param(sio, 2, SIO_SHMEM_SIZE); - return 0; +free_shmem: + /* No address has been published yet. */ + dma_free_coherent(sio->dev, size, sio->shmem, sio->shmem_iova); + return err; } static int sio_send_dt_params(struct sio_data *sio) @@ -945,6 +1582,13 @@ static int sio_send_dt_params(struct sio_data *sio) err = nparams; goto badprop; } + if (sio->protocol == 9) { + const __be32 *params = of_get_property(np, propname, NULL); + + if (!params) + return -EINVAL; + return sio_send_v9_dt_params(sio, params, nparams, sio_set_param); + } for (i = 0; i < nparams / 2; i++) { u32 key, val; @@ -956,16 +1600,9 @@ static int sio_send_dt_params(struct sio_data *sio) if (err) goto badprop; - err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_SETUP) | - FIELD_PREP(SIOMSG_PARAM, key & 0xff) | - FIELD_PREP(SIOMSG_EP, key >> 8) | - FIELD_PREP(SIOMSG_DATA, val)); - if (err < 1) { - if (err == 0) - err = -ENXIO; - return dev_err_probe(sio->dev, err, "sending SIO parameter %#x value %#x\n", - key, val); - } + err = sio_set_param(sio, key, val); + if (err) + return err; } return 0; @@ -1060,6 +1697,9 @@ static int sio_probe(struct platform_device *pdev) siochan->no = i; siochan->vc.desc_free = sio_tx_free; INIT_WORK(&siochan->terminate_wq, sio_terminate_work); + INIT_LIST_HEAD(&siochan->retired); + INIT_LIST_HEAD(&siochan->dead); + init_waitqueue_head(&siochan->drained); vchan_init(&siochan->vc, dma); } @@ -1148,4 +1788,5 @@ MODULE_LICENSE("Dual MIT/GPL"); #if IS_ENABLED(CONFIG_APPLE_SIO_KUNIT_TEST) #include "apple-sio-probe-test.c" +#include "apple-sio-boot-test.c" #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c index 08ed1eb695586..c324fabef15d9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c @@ -132,7 +132,10 @@ static bool amdgpu_read_bios_from_vram(struct amdgpu_device *adev) return false; } } else { - bios = ioremap_wc(vram_base, size); + if (pci_resource_flags(adev->pdev, 0) & IORESOURCE_MEM_NONPOSTED) + bios = ioremap_np(vram_base, size); + else + bios = ioremap_wc(vram_base, size); if (!bios) { amdgpu_bios_release(adev); return false; @@ -154,14 +157,40 @@ static bool amdgpu_read_bios_from_vram(struct amdgpu_device *adev) bool amdgpu_read_bios(struct amdgpu_device *adev) { + struct resource *res = &adev->pdev->resource[PCI_ROM_RESOURCE]; uint8_t __iomem *bios; size_t size; adev->bios = NULL; - /* XXX: some cards may return 0 for rom size? ddx has a workaround */ - bios = pci_map_rom(adev->pdev, &size); - if (!bios) - return false; + if (res->flags & IORESOURCE_MEM_NONPOSTED) { + if (!res->parent && pci_assign_resource(adev->pdev, PCI_ROM_RESOURCE)) + return false; + size = resource_size(res); + if (!size || pci_enable_rom(adev->pdev)) + return false; + bios = ioremap_np(res->start, size); + if (!bios) { + if (!(res->flags & IORESOURCE_ROM_ENABLE)) + pci_disable_rom(adev->pdev); + return false; + } + /* + * The window is larger than the image. Check the ROM signature and copy only the + * image, whose length the header gives in 512-byte units. + */ + if (readw(bios) != 0xaa55) { + iounmap(bios); + if (!(res->flags & IORESOURCE_ROM_ENABLE)) + pci_disable_rom(adev->pdev); + return false; + } + if (readb(bios + 2) && (size_t)readb(bios + 2) * 512 < size) + size = (size_t)readb(bios + 2) * 512; + } else { + bios = pci_map_rom(adev->pdev, &size); + if (!bios) + return false; + } adev->bios = kzalloc(size, GFP_KERNEL); if (adev->bios == NULL) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c index 11d70fe4f75a9..f12469b0c4210 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c @@ -46,7 +46,7 @@ int amdgpu_allocate_static_csa(struct amdgpu_device *adev, struct amdgpu_bo **bo if (!*bo) return -ENOMEM; - memset(ptr, 0, size); + amdgpu_bo_memset(*bo, ptr, 0, size); adev->virt.csa_cpu_addr = ptr; return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 9269e780feb73..4899c230dec0b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3914,7 +3914,12 @@ int amdgpu_device_init(struct amdgpu_device *adev, for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++) atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN); - adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); + if (pci_resource_flags(adev->pdev, + adev->asic_type >= CHIP_BONAIRE ? 5 : 2) & + IORESOURCE_MEM_NONPOSTED) + adev->rmmio = ioremap_np(adev->rmmio_base, adev->rmmio_size); + else + adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); if (!adev->rmmio) return -ENOMEM; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index afd8502a2bc49..934fc7d71c4bd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -175,6 +175,16 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, struct amdgpu_bo_param bp; int r; + /* + * Non-posted VRAM requires kernel I/O accessors; userspace needs GTT. + */ + if ((flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) && + (initial_domain & AMDGPU_GEM_DOMAIN_VRAM) && + (pci_resource_flags(adev->pdev, 0) & IORESOURCE_MEM_NONPOSTED)) { + flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + initial_domain = AMDGPU_GEM_DOMAIN_GTT; + } + memset(&bp, 0, sizeof(bp)); *obj = NULL; flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 7c081b338fe2e..7702a3b8bed99 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -828,10 +828,13 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) vram_gtt.vram_size = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size) - AMDGPU_VM_RESERVED_VRAM; - vram_gtt.vram_cpu_accessible_size = - min(adev->gmc.visible_vram_size - - atomic64_read(&adev->visible_pin_size), - vram_gtt.vram_size); + if (pci_resource_flags(adev->pdev, 0) & IORESOURCE_MEM_NONPOSTED) + vram_gtt.vram_cpu_accessible_size = 0; + else + vram_gtt.vram_cpu_accessible_size = + min(adev->gmc.visible_vram_size - + atomic64_read(&adev->visible_pin_size), + vram_gtt.vram_size); vram_gtt.gtt_size = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT)->size; vram_gtt.gtt_size -= atomic64_read(&adev->gart_pin_size); return copy_to_user(out, &vram_gtt, @@ -853,16 +856,19 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) ttm_resource_manager_usage(vram_man) : 0; mem.vram.max_allocation = mem.vram.usable_heap_size * 3 / 4; - mem.cpu_accessible_vram.total_heap_size = - adev->gmc.visible_vram_size; - mem.cpu_accessible_vram.usable_heap_size = - min(adev->gmc.visible_vram_size - - atomic64_read(&adev->visible_pin_size), - mem.vram.usable_heap_size); - mem.cpu_accessible_vram.heap_usage = - amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr); - mem.cpu_accessible_vram.max_allocation = - mem.cpu_accessible_vram.usable_heap_size * 3 / 4; + if (!(pci_resource_flags(adev->pdev, 0) & + IORESOURCE_MEM_NONPOSTED)) { + mem.cpu_accessible_vram.total_heap_size = + adev->gmc.visible_vram_size; + mem.cpu_accessible_vram.usable_heap_size = + min(adev->gmc.visible_vram_size - + atomic64_read(&adev->visible_pin_size), + mem.vram.usable_heap_size); + mem.cpu_accessible_vram.heap_usage = + amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr); + mem.cpu_accessible_vram.max_allocation = + mem.cpu_accessible_vram.usable_heap_size * 3 / 4; + } mem.gtt.total_heap_size = gtt_man->size; mem.gtt.usable_heap_size = mem.gtt.total_heap_size - diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c index 1a86a47406b1d..fb217583dcb11 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c @@ -77,7 +77,8 @@ static int amdgpu_mes_event_log_init(struct amdgpu_device *adev) return r; } - memset(adev->mes.event_log_cpu_addr, 0, adev->mes.event_log_size); + amdgpu_bo_memset(adev->mes.event_log_gpu_obj, + adev->mes.event_log_cpu_addr, 0, adev->mes.event_log_size); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index 1b6c32a177fb6..4fef1673e10c6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -1369,6 +1369,30 @@ vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) /* Remember that this BO was accessed by the CPU */ abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + /* + * Behind a non-posted BAR the CPU reaches VRAM as Device-nGnRnE memory, + * where memcpy() and unaligned stores in userspace take an alignment + * fault. A BO the CPU faults on moves to GTT and stays there; a pinned + * BO cannot move. + */ + if (pci_resource_flags(adev->pdev, 0) & IORESOURCE_MEM_NONPOSTED) { + if (bo->resource->mem_type != TTM_PL_VRAM) + return 0; + if (abo->tbo.pin_count > 0) + return VM_FAULT_SIGBUS; + + abo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; + abo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; + amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT); + r = ttm_bo_validate(bo, &abo->placement, &ctx); + if (unlikely(r == -EBUSY || r == -ERESTARTSYS)) + return VM_FAULT_NOPAGE; + if (unlikely(r) || bo->resource->mem_type == TTM_PL_VRAM) + return VM_FAULT_SIGBUS; + ttm_bo_move_to_lru_tail_unlocked(bo); + return 0; + } + if (amdgpu_res_cpu_visible(adev, bo->resource)) return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index ff11a09034997..77ae85b2d3cad 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -28,6 +28,7 @@ #ifndef __AMDGPU_OBJECT_H__ #define __AMDGPU_OBJECT_H__ +#include #include #include "amdgpu.h" #include "amdgpu_res_cursor.h" @@ -248,6 +249,33 @@ static inline bool amdgpu_bo_encrypted(struct amdgpu_bo *bo) return bo->flags & AMDGPU_GEM_CREATE_ENCRYPTED; } +static inline void amdgpu_bo_memset(struct amdgpu_bo *bo, void *dst, + int value, size_t count) +{ + if (bo->tbo.resource->bus.is_nonposted) + memset_io((void __iomem *)dst, value, count); + else + memset(dst, value, count); +} + +static inline void amdgpu_bo_memcpy_to(struct amdgpu_bo *bo, void *dst, + const void *src, size_t count) +{ + if (bo->tbo.resource->bus.is_nonposted) + memcpy_toio((void __iomem *)dst, src, count); + else + memcpy(dst, src, count); +} + +static inline void amdgpu_bo_memcpy_from(struct amdgpu_bo *bo, void *dst, + const void *src, size_t count) +{ + if (bo->tbo.resource->bus.is_nonposted) + memcpy_fromio(dst, (const void __iomem *)src, count); + else + memcpy(dst, src, count); +} + bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo); void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 42adc8e738d89..cb2d012296b92 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -52,6 +52,19 @@ static int psp_load_smu_fw(struct psp_context *psp); static int psp_rap_terminate(struct psp_context *psp); static int psp_securedisplay_terminate(struct psp_context *psp); +/* + * The PSP command, fence and ring buffers and the TA shared buffers are read + * and written through plain CPU pointers. Behind a non-posted BAR a VRAM + * placement turns those into Device-nGnRnE accesses that fault on plain + * stores, so place them in GTT. + */ +static u32 psp_cpu_buf_domain(struct amdgpu_device *adev) +{ + if (pci_resource_flags(adev->pdev, 0) & IORESOURCE_MEM_NONPOSTED) + return AMDGPU_GEM_DOMAIN_GTT; + return AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT; +} + static const char * const amdgpu_ptl_fmt_str[] = { [AMDGPU_PTL_FMT_I8] = "I8", [AMDGPU_PTL_FMT_F16] = "F16", @@ -74,11 +87,10 @@ static int psp_ring_init(struct psp_context *psp, ring->ring_type = ring_type; - /* allocate 4k Page of Local Frame Buffer memory for ring */ + /* allocate a 4k page for the ring (GTT instead of VRAM behind a non-posted BAR) */ ring->ring_size = 0x1000; ret = amdgpu_bo_create_kernel(adev, ring->ring_size, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM | - AMDGPU_GEM_DOMAIN_GTT, + psp_cpu_buf_domain(adev), &adev->firmware.rbuf, &ring->ring_mem_mc_addr, (void **)&ring->ring_mem); @@ -546,8 +558,7 @@ static int psp_sw_init(struct amdgpu_ip_block *ip_block) return ret; ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM | - AMDGPU_GEM_DOMAIN_GTT, + psp_cpu_buf_domain(adev), &psp->fence_buf_bo, &psp->fence_buf_mc_addr, &psp->fence_buf); @@ -555,8 +566,7 @@ static int psp_sw_init(struct amdgpu_ip_block *ip_block) goto failed1; ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM | - AMDGPU_GEM_DOMAIN_GTT, + psp_cpu_buf_domain(adev), &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr, (void **)&psp->cmd_buf_mem); if (ret) @@ -1798,8 +1808,7 @@ int psp_ta_init_shared_buf(struct psp_context *psp, * physical) for ta to host memory */ return amdgpu_bo_create_kernel(psp->adev, mem_ctx->shared_mem_size, - PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM | - AMDGPU_GEM_DOMAIN_GTT, + PAGE_SIZE, psp_cpu_buf_domain(psp->adev), &mem_ctx->shared_bo, &mem_ctx->shared_mc_addr, &mem_ctx->shared_buf); @@ -2239,21 +2248,25 @@ static int psp_ras_send_cmd(struct psp_context *psp, mutex_lock(&psp->ras_context.mutex); ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf; - memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory)); + amdgpu_bo_memset(psp->ras_context.context.mem_context.shared_bo, ras_cmd, 0, + sizeof(struct ta_ras_shared_memory)); switch (cmd) { case TA_RAS_COMMAND__ENABLE_FEATURES: case TA_RAS_COMMAND__DISABLE_FEATURES: - memcpy(&ras_cmd->ras_in_message, - in, sizeof(ras_cmd->ras_in_message)); + amdgpu_bo_memcpy_to(psp->ras_context.context.mem_context.shared_bo, + &ras_cmd->ras_in_message, + in, sizeof(ras_cmd->ras_in_message)); break; case TA_RAS_COMMAND__TRIGGER_ERROR: - memcpy(&ras_cmd->ras_in_message.trigger_error, - in, sizeof(ras_cmd->ras_in_message.trigger_error)); + amdgpu_bo_memcpy_to(psp->ras_context.context.mem_context.shared_bo, + &ras_cmd->ras_in_message.trigger_error, + in, sizeof(ras_cmd->ras_in_message.trigger_error)); break; case TA_RAS_COMMAND__QUERY_ADDRESS: - memcpy(&ras_cmd->ras_in_message.address, - in, sizeof(ras_cmd->ras_in_message.address)); + amdgpu_bo_memcpy_to(psp->ras_context.context.mem_context.shared_bo, + &ras_cmd->ras_in_message.address, + in, sizeof(ras_cmd->ras_in_message.address)); break; default: dev_err(psp->adev->dev, "Invalid ras cmd id: %u\n", cmd); @@ -2267,15 +2280,18 @@ static int psp_ras_send_cmd(struct psp_context *psp, switch (cmd) { case TA_RAS_COMMAND__TRIGGER_ERROR: if (!ret && out) - memcpy(out, &ras_cmd->ras_status, sizeof(ras_cmd->ras_status)); + amdgpu_bo_memcpy_from(psp->ras_context.context.mem_context.shared_bo, + out, &ras_cmd->ras_status, + sizeof(ras_cmd->ras_status)); break; case TA_RAS_COMMAND__QUERY_ADDRESS: if (ret || ras_cmd->ras_status || psp->cmd_buf_mem->resp.status) ret = -EINVAL; else if (out) - memcpy(out, - &ras_cmd->ras_out_message.address, - sizeof(ras_cmd->ras_out_message.address)); + amdgpu_bo_memcpy_from(psp->ras_context.context.mem_context.shared_bo, + out, + &ras_cmd->ras_out_message.address, + sizeof(ras_cmd->ras_out_message.address)); break; default: break; @@ -4667,8 +4683,8 @@ int psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size) return -EINVAL; } - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - memcpy(psp->fw_pri_buf, start_addr, bin_size); + amdgpu_bo_memset(psp->fw_pri_bo, psp->fw_pri_buf, 0, PSP_1_MEG); + amdgpu_bo_memcpy_to(psp->fw_pri_bo, psp->fw_pri_buf, start_addr, bin_size); drm_dev_exit(idx); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 83968886c29a6..90877c904bdd6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -651,6 +652,7 @@ static int amdgpu_ttm_io_mem_reserve(struct ttm_device *bdev, { struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); + mem->bus.is_nonposted = false; switch (mem->mem_type) { case TTM_PL_SYSTEM: /* system memory */ @@ -659,6 +661,8 @@ static int amdgpu_ttm_io_mem_reserve(struct ttm_device *bdev, case AMDGPU_PL_PREEMPT: break; case TTM_PL_VRAM: + mem->bus.is_nonposted = pci_resource_flags(adev->pdev, 0) & + IORESOURCE_MEM_NONPOSTED; mem->bus.offset = mem->start << PAGE_SHIFT; if (adev->mman.aper_base_kaddr && @@ -670,12 +674,17 @@ static int amdgpu_ttm_io_mem_reserve(struct ttm_device *bdev, mem->bus.is_iomem = true; break; case AMDGPU_PL_DOORBELL: + mem->bus.is_nonposted = pci_resource_flags(adev->pdev, 2) & + IORESOURCE_MEM_NONPOSTED; mem->bus.offset = mem->start << PAGE_SHIFT; mem->bus.offset += adev->doorbell.base; mem->bus.is_iomem = true; mem->bus.caching = ttm_uncached; break; case AMDGPU_PL_MMIO_REMAP: + mem->bus.is_nonposted = pci_resource_flags(adev->pdev, + adev->asic_type >= CHIP_BONAIRE ? 5 : 2) & + IORESOURCE_MEM_NONPOSTED; mem->bus.offset = mem->start << PAGE_SHIFT; mem->bus.offset += adev->rmmio_remap.bus_addr; mem->bus.is_iomem = true; @@ -1208,7 +1217,14 @@ static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo, else gtt->pool_id = abo->xcp_id; - if (abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) + /* + * Behind a non-posted BAR the host is an Apple Silicon tunnel whose PCI + * DMA is not cache coherent: dirty CPU cache lines would be invisible + * to the GPU, so map GTT pages write-combined there. + */ + if ((abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) || + ((pci_resource_flags(adev->pdev, 0) & IORESOURCE_MEM_NONPOSTED) && + !dev_is_dma_coherent(adev->dev))) caching = ttm_write_combined; else caching = ttm_cached; @@ -2152,6 +2168,10 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) /* Change the size here instead of the init above so only lpfn is affected */ amdgpu_ttm_disable_buffer_funcs(adev); #ifdef CONFIG_64BIT + if (pci_resource_flags(adev->pdev, 0) & IORESOURCE_MEM_NONPOSTED) + adev->mman.aper_base_kaddr = ioremap_np(adev->gmc.aper_base, + adev->gmc.visible_vram_size); + else #ifdef CONFIG_X86 if (adev->gmc.xgmi.connected_to_cpu) adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index 7a006b22e4be0..68d4e82c5da20 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -1307,10 +1307,13 @@ static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *ade if (fw_size > toc_fw_size) fw_size = toc_fw_size; - memcpy(ptr + toc_offset, fw_data, fw_size); + amdgpu_bo_memcpy_to(adev->gfx.rlc.rlc_autoload_bo, ptr + toc_offset, + fw_data, fw_size); if (fw_size < toc_fw_size) - memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size); + amdgpu_bo_memset(adev->gfx.rlc.rlc_autoload_bo, + ptr + toc_offset + fw_size, 0, + toc_fw_size - fw_size); if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME)) *(uint64_t *)fw_autoload_mask |= 1ULL << id; @@ -3302,8 +3305,10 @@ static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev) return r; } - memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size); - memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size); + amdgpu_bo_memcpy_to(adev->gfx.pfp.pfp_fw_obj, adev->gfx.pfp.pfp_fw_ptr, + fw_ucode, fw_ucode_size); + amdgpu_bo_memcpy_to(adev->gfx.pfp.pfp_fw_data_obj, + adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size); amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj); amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj); @@ -3520,8 +3525,10 @@ static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev) return r; } - memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size); - memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size); + amdgpu_bo_memcpy_to(adev->gfx.me.me_fw_obj, adev->gfx.me.me_fw_ptr, + fw_ucode, fw_ucode_size); + amdgpu_bo_memcpy_to(adev->gfx.me.me_fw_data_obj, + adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size); amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj); amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj); @@ -4034,8 +4041,10 @@ static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev) return r; } - memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size); - memcpy(fw_data_ptr, fw_data, fw_data_size); + amdgpu_bo_memcpy_to(adev->gfx.mec.mec_fw_obj, fw_ucode_ptr, fw_ucode, + fw_ucode_size); + amdgpu_bo_memcpy_to(adev->gfx.mec.mec_fw_data_obj, fw_data_ptr, fw_data, + fw_data_size); amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj); @@ -4263,20 +4272,40 @@ static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring) struct amdgpu_device *adev = ring->adev; struct v11_gfx_mqd *mqd = ring->mqd_ptr; int mqd_idx = ring - &adev->gfx.gfx_ring[0]; + bool np = adev->gfx.me.mqd_backup[mqd_idx] && + ring->mqd_obj->tbo.resource->bus.is_nonposted; if (!amdgpu_in_reset(adev) && !adev->in_suspend) { - memset((void *)mqd, 0, sizeof(*mqd)); + /* + * Behind a non-posted BAR the MQD BO is Device-nGnRnE memory, + * where typed stores fault: compose the MQD in the host backup + * copy and copy it to the BO. + */ + if (np) { + ring->mqd_ptr = adev->gfx.me.mqd_backup[mqd_idx]; + memset(ring->mqd_ptr, 0, sizeof(*mqd)); + } else { + memset((void *)mqd, 0, sizeof(*mqd)); + } mutex_lock(&adev->srbm_mutex); soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); amdgpu_ring_init_mqd(ring); soc21_grbm_select(adev, 0, 0, 0, 0); mutex_unlock(&adev->srbm_mutex); - if (adev->gfx.me.mqd_backup[mqd_idx]) + if (np) { + memcpy_toio(mqd, ring->mqd_ptr, sizeof(*mqd)); + amdgpu_device_flush_hdp(adev, NULL); + ring->mqd_ptr = mqd; + } else if (adev->gfx.me.mqd_backup[mqd_idx]) { memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); + } } else { /* restore mqd with the backup copy */ - if (adev->gfx.me.mqd_backup[mqd_idx]) + if (adev->gfx.me.mqd_backup[mqd_idx]) { memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd)); + if (np) + amdgpu_device_flush_hdp(adev, NULL); + } /* reset the ring */ ring->wptr = 0; atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); @@ -4593,13 +4622,18 @@ static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring) { struct amdgpu_device *adev = ring->adev; struct v11_compute_mqd *mqd = ring->mqd_ptr; + bool np = adev->gfx.kiq[0].mqd_backup && + ring->mqd_obj->tbo.resource->bus.is_nonposted; gfx_v11_0_kiq_setting(ring); if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ /* reset MQD to a clean status */ - if (adev->gfx.kiq[0].mqd_backup) + if (adev->gfx.kiq[0].mqd_backup) { memcpy_toio(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(*mqd)); + if (np) + amdgpu_device_flush_hdp(adev, NULL); + } /* reset ring buffer */ ring->wptr = 0; @@ -4611,17 +4645,32 @@ static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring) soc21_grbm_select(adev, 0, 0, 0, 0); mutex_unlock(&adev->srbm_mutex); } else { - memset((void *)mqd, 0, sizeof(*mqd)); + /* + * Behind a non-posted BAR the MQD BO is Device-nGnRnE memory, + * where typed stores fault: compose the MQD in the host backup + * copy and copy it to the BO. + */ + if (np) { + ring->mqd_ptr = adev->gfx.kiq[0].mqd_backup; + memset(ring->mqd_ptr, 0, sizeof(*mqd)); + } else { + memset((void *)mqd, 0, sizeof(*mqd)); + } if (amdgpu_sriov_vf(adev) && adev->in_suspend) amdgpu_ring_clear_ring(ring); mutex_lock(&adev->srbm_mutex); soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); amdgpu_ring_init_mqd(ring); + if (np) { + memcpy_toio(mqd, ring->mqd_ptr, sizeof(*mqd)); + amdgpu_device_flush_hdp(adev, NULL); + ring->mqd_ptr = mqd; + } gfx_v11_0_kiq_init_register(ring); soc21_grbm_select(adev, 0, 0, 0, 0); mutex_unlock(&adev->srbm_mutex); - if (adev->gfx.kiq[0].mqd_backup) + if (adev->gfx.kiq[0].mqd_backup && !np) memcpy_fromio(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(*mqd)); } @@ -4633,21 +4682,41 @@ static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring) struct amdgpu_device *adev = ring->adev; struct v11_compute_mqd *mqd = ring->mqd_ptr; int mqd_idx = ring - &adev->gfx.compute_ring[0]; + bool np = adev->gfx.mec.mqd_backup[mqd_idx] && + ring->mqd_obj->tbo.resource->bus.is_nonposted; if (!amdgpu_in_reset(adev) && !adev->in_suspend) { - memset((void *)mqd, 0, sizeof(*mqd)); + /* + * Behind a non-posted BAR the MQD BO is Device-nGnRnE memory, + * where typed stores fault: compose the MQD in the host backup + * copy and copy it to the BO. + */ + if (np) { + ring->mqd_ptr = adev->gfx.mec.mqd_backup[mqd_idx]; + memset(ring->mqd_ptr, 0, sizeof(*mqd)); + } else { + memset((void *)mqd, 0, sizeof(*mqd)); + } mutex_lock(&adev->srbm_mutex); soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); amdgpu_ring_init_mqd(ring); soc21_grbm_select(adev, 0, 0, 0, 0); mutex_unlock(&adev->srbm_mutex); - if (adev->gfx.mec.mqd_backup[mqd_idx]) + if (np) { + memcpy_toio(mqd, ring->mqd_ptr, sizeof(*mqd)); + amdgpu_device_flush_hdp(adev, NULL); + ring->mqd_ptr = mqd; + } else if (adev->gfx.mec.mqd_backup[mqd_idx]) { memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); + } } else { /* restore MQD to a clean status */ - if (adev->gfx.mec.mqd_backup[mqd_idx]) + if (adev->gfx.mec.mqd_backup[mqd_idx]) { memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd)); + if (np) + amdgpu_device_flush_hdp(adev, NULL); + } /* reset ring buffer */ ring->wptr = 0; atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index d69c43f1e194c..56834cc13fe8d 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -1130,10 +1130,13 @@ static void gfx_v12_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *ade if (fw_size > toc_fw_size) fw_size = toc_fw_size; - memcpy(ptr + toc_offset, fw_data, fw_size); + amdgpu_bo_memcpy_to(adev->gfx.rlc.rlc_autoload_bo, ptr + toc_offset, + fw_data, fw_size); if (fw_size < toc_fw_size) - memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size); + amdgpu_bo_memset(adev->gfx.rlc.rlc_autoload_bo, + ptr + toc_offset + fw_size, 0, + toc_fw_size - fw_size); } static void @@ -2429,8 +2432,10 @@ static int gfx_v12_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev) return r; } - memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size); - memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size); + amdgpu_bo_memcpy_to(adev->gfx.pfp.pfp_fw_obj, adev->gfx.pfp.pfp_fw_ptr, + fw_ucode, fw_ucode_size); + amdgpu_bo_memcpy_to(adev->gfx.pfp.pfp_fw_data_obj, + adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size); amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj); amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj); @@ -2573,8 +2578,10 @@ static int gfx_v12_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev) return r; } - memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size); - memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size); + amdgpu_bo_memcpy_to(adev->gfx.me.me_fw_obj, adev->gfx.me.me_fw_ptr, + fw_ucode, fw_ucode_size); + amdgpu_bo_memcpy_to(adev->gfx.me.me_fw_data_obj, + adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size); amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj); amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj); @@ -2890,9 +2897,12 @@ static int gfx_v12_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev) return r; } - memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size); + amdgpu_bo_memcpy_to(adev->gfx.mec.mec_fw_obj, fw_ucode_ptr, fw_ucode, + fw_ucode_size); for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { - memcpy(fw_data_ptr + i * ALIGN(fw_data_size, 64 * 1024) / 4, fw_data, fw_data_size); + amdgpu_bo_memcpy_to(adev->gfx.mec.mec_fw_data_obj, + fw_data_ptr + i * ALIGN(fw_data_size, 64 * 1024) / 4, + fw_data, fw_data_size); } amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); @@ -3104,20 +3114,40 @@ static int gfx_v12_0_kgq_init_queue(struct amdgpu_ring *ring) struct amdgpu_device *adev = ring->adev; struct v12_gfx_mqd *mqd = ring->mqd_ptr; int mqd_idx = ring - &adev->gfx.gfx_ring[0]; + bool np = adev->gfx.me.mqd_backup[mqd_idx] && + ring->mqd_obj->tbo.resource->bus.is_nonposted; if (!amdgpu_in_reset(adev) && !adev->in_suspend) { - memset((void *)mqd, 0, sizeof(*mqd)); + /* + * Behind a non-posted BAR the MQD BO is Device-nGnRnE memory, + * where typed stores fault: compose the MQD in the host backup + * copy and copy it to the BO. + */ + if (np) { + ring->mqd_ptr = adev->gfx.me.mqd_backup[mqd_idx]; + memset(ring->mqd_ptr, 0, sizeof(*mqd)); + } else { + memset((void *)mqd, 0, sizeof(*mqd)); + } mutex_lock(&adev->srbm_mutex); soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); amdgpu_ring_init_mqd(ring); soc24_grbm_select(adev, 0, 0, 0, 0); mutex_unlock(&adev->srbm_mutex); - if (adev->gfx.me.mqd_backup[mqd_idx]) + if (np) { + memcpy_toio(mqd, ring->mqd_ptr, sizeof(*mqd)); + amdgpu_device_flush_hdp(adev, NULL); + ring->mqd_ptr = mqd; + } else if (adev->gfx.me.mqd_backup[mqd_idx]) { memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); + } } else { /* restore mqd with the backup copy */ - if (adev->gfx.me.mqd_backup[mqd_idx]) + if (adev->gfx.me.mqd_backup[mqd_idx]) { memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd)); + if (np) + amdgpu_device_flush_hdp(adev, NULL); + } /* reset the ring */ ring->wptr = 0; atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); @@ -3434,13 +3464,19 @@ static int gfx_v12_0_kiq_init_queue(struct amdgpu_ring *ring) struct amdgpu_device *adev = ring->adev; struct v12_compute_mqd *mqd = ring->mqd_ptr; int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS; + bool np = adev->gfx.mec.mqd_backup[mqd_idx] && + ring->mqd_obj->tbo.resource->bus.is_nonposted; gfx_v12_0_kiq_setting(ring); if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ /* reset MQD to a clean status */ - if (adev->gfx.mec.mqd_backup[mqd_idx]) - memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd)); + if (adev->gfx.mec.mqd_backup[mqd_idx]) { + amdgpu_bo_memcpy_to(ring->mqd_obj, mqd, adev->gfx.mec.mqd_backup[mqd_idx], + sizeof(*mqd)); + if (np) + amdgpu_device_flush_hdp(adev, NULL); + } /* reset ring buffer */ ring->wptr = 0; @@ -3452,17 +3488,32 @@ static int gfx_v12_0_kiq_init_queue(struct amdgpu_ring *ring) soc24_grbm_select(adev, 0, 0, 0, 0); mutex_unlock(&adev->srbm_mutex); } else { - memset((void *)mqd, 0, sizeof(*mqd)); + /* + * Behind a non-posted BAR the MQD BO is Device-nGnRnE memory, + * where typed stores fault: compose the MQD in the host backup + * copy and copy it to the BO. + */ + if (np) { + ring->mqd_ptr = adev->gfx.mec.mqd_backup[mqd_idx]; + memset(ring->mqd_ptr, 0, sizeof(*mqd)); + } else { + memset((void *)mqd, 0, sizeof(*mqd)); + } if (amdgpu_sriov_vf(adev) && adev->in_suspend) amdgpu_ring_clear_ring(ring); mutex_lock(&adev->srbm_mutex); soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); amdgpu_ring_init_mqd(ring); + if (np) { + memcpy_toio(mqd, ring->mqd_ptr, sizeof(*mqd)); + amdgpu_device_flush_hdp(adev, NULL); + ring->mqd_ptr = mqd; + } gfx_v12_0_kiq_init_register(ring); soc24_grbm_select(adev, 0, 0, 0, 0); mutex_unlock(&adev->srbm_mutex); - if (adev->gfx.mec.mqd_backup[mqd_idx]) + if (adev->gfx.mec.mqd_backup[mqd_idx] && !np) memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); } @@ -3474,21 +3525,41 @@ static int gfx_v12_0_kcq_init_queue(struct amdgpu_ring *ring) struct amdgpu_device *adev = ring->adev; struct v12_compute_mqd *mqd = ring->mqd_ptr; int mqd_idx = ring - &adev->gfx.compute_ring[0]; + bool np = adev->gfx.mec.mqd_backup[mqd_idx] && + ring->mqd_obj->tbo.resource->bus.is_nonposted; if (!amdgpu_in_reset(adev) && !adev->in_suspend) { - memset((void *)mqd, 0, sizeof(*mqd)); + /* + * Behind a non-posted BAR the MQD BO is Device-nGnRnE memory, + * where typed stores fault: compose the MQD in the host backup + * copy and copy it to the BO. + */ + if (np) { + ring->mqd_ptr = adev->gfx.mec.mqd_backup[mqd_idx]; + memset(ring->mqd_ptr, 0, sizeof(*mqd)); + } else { + memset((void *)mqd, 0, sizeof(*mqd)); + } mutex_lock(&adev->srbm_mutex); soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); amdgpu_ring_init_mqd(ring); soc24_grbm_select(adev, 0, 0, 0, 0); mutex_unlock(&adev->srbm_mutex); - if (adev->gfx.mec.mqd_backup[mqd_idx]) + if (np) { + memcpy_toio(mqd, ring->mqd_ptr, sizeof(*mqd)); + amdgpu_device_flush_hdp(adev, NULL); + ring->mqd_ptr = mqd; + } else if (adev->gfx.mec.mqd_backup[mqd_idx]) { memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); + } } else { /* restore MQD to a clean status */ - if (adev->gfx.mec.mqd_backup[mqd_idx]) + if (adev->gfx.mec.mqd_backup[mqd_idx]) { memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd)); + if (np) + amdgpu_device_flush_hdp(adev, NULL); + } /* reset ring buffer */ ring->wptr = 0; atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c index c57bcb9a98b19..fe5a55bf49286 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c @@ -1164,7 +1164,8 @@ static int mes_v11_0_allocate_ucode_buffer(struct amdgpu_device *adev, return r; } - memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size); + amdgpu_bo_memcpy_to(adev->mes.ucode_fw_obj[pipe], + adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size); amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]); amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]); @@ -1205,7 +1206,8 @@ static int mes_v11_0_allocate_ucode_data_buffer(struct amdgpu_device *adev, return r; } - memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size); + amdgpu_bo_memcpy_to(adev->mes.data_fw_obj[pipe], + adev->mes.data_fw_ptr[pipe], fw_data, fw_size); amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]); amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]); @@ -1414,7 +1416,8 @@ static int mes_v11_0_allocate_eop_buf(struct amdgpu_device *adev, static int mes_v11_0_mqd_init(struct amdgpu_ring *ring) { - struct v11_compute_mqd *mqd = ring->mqd_ptr; + struct v11_compute_mqd *mqd = ring->mqd_obj->tbo.resource->bus.is_nonposted ? + ring->adev->mes.mqd_backup[ring->pipe] : ring->mqd_ptr; uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; uint32_t tmp; @@ -1512,13 +1515,17 @@ static int mes_v11_0_mqd_init(struct amdgpu_ring *ring) mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT; mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT; + if (ring->mqd_obj->tbo.resource->bus.is_nonposted) + amdgpu_bo_memcpy_to(ring->mqd_obj, ring->mqd_ptr, mqd, sizeof(*mqd)); + amdgpu_device_flush_hdp(ring->adev, NULL); return 0; } static void mes_v11_0_queue_init_register(struct amdgpu_ring *ring) { - struct v11_compute_mqd *mqd = ring->mqd_ptr; + struct v11_compute_mqd *mqd = ring->mqd_obj->tbo.resource->bus.is_nonposted ? + ring->adev->mes.mqd_backup[ring->pipe] : ring->mqd_ptr; struct amdgpu_device *adev = ring->adev; uint32_t data = 0; @@ -1708,7 +1715,7 @@ static int mes_v11_0_mqd_sw_init(struct amdgpu_device *adev, return r; } - memset(ring->mqd_ptr, 0, mqd_size); + amdgpu_bo_memset(ring->mqd_obj, ring->mqd_ptr, 0, mqd_size); /* prepare MQD backup */ adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL); diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c index ee72afdb0a1d3..f856b2a8828b6 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c @@ -1246,7 +1246,8 @@ static int mes_v12_0_allocate_ucode_buffer(struct amdgpu_device *adev, return r; } - memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size); + amdgpu_bo_memcpy_to(adev->mes.ucode_fw_obj[pipe], + adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size); amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]); amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]); @@ -1280,7 +1281,8 @@ static int mes_v12_0_allocate_ucode_data_buffer(struct amdgpu_device *adev, return r; } - memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size); + amdgpu_bo_memcpy_to(adev->mes.data_fw_obj[pipe], + adev->mes.data_fw_ptr[pipe], fw_data, fw_size); amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]); amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]); diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c index f1098118d5c05..f9fc4ef413ccd 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c @@ -1012,7 +1012,8 @@ static int mes_v12_1_allocate_ucode_buffer(struct amdgpu_device *adev, return r; } - memcpy(adev->mes.ucode_fw_ptr[inst], fw_data, fw_size); + amdgpu_bo_memcpy_to(adev->mes.ucode_fw_obj[inst], + adev->mes.ucode_fw_ptr[inst], fw_data, fw_size); amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[inst]); amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[inst]); @@ -1047,7 +1048,8 @@ static int mes_v12_1_allocate_ucode_data_buffer(struct amdgpu_device *adev, return r; } - memcpy(adev->mes.data_fw_ptr[inst], fw_data, fw_size); + amdgpu_bo_memcpy_to(adev->mes.data_fw_obj[inst], + adev->mes.data_fw_ptr[inst], fw_data, fw_size); amdgpu_bo_kunmap(adev->mes.data_fw_obj[inst]); amdgpu_bo_unreserve(adev->mes.data_fw_obj[inst]); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c index 69cb89298a3e0..a422f4a19f09f 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c @@ -709,7 +709,9 @@ static int sdma_v7_0_load_microcode(struct amdgpu_device *adev) return r; } - memcpy(adev->sdma.instance[i].sdma_fw_ptr, fw_data, fw_size); + amdgpu_bo_memcpy_to(adev->sdma.instance[i].sdma_fw_obj, + adev->sdma.instance[i].sdma_fw_ptr, + fw_data, fw_size); amdgpu_bo_kunmap(adev->sdma.instance[i].sdma_fw_obj); amdgpu_bo_unreserve(adev->sdma.instance[i].sdma_fw_obj); diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig index 1727d25646cc5..f2f4a80f2a1e2 100644 --- a/drivers/gpu/drm/amd/display/Kconfig +++ b/drivers/gpu/drm/amd/display/Kconfig @@ -12,7 +12,7 @@ config DRM_AMD_DC select CEC_NOTIFIER select SND_HDA_COMPONENT if SND_HDA_CORE # !CC_IS_CLANG: https://github.com/ClangBuiltLinux/linux/issues/1752 - select DRM_AMD_DC_FP if ARCH_HAS_KERNEL_FPU_SUPPORT && !(CC_IS_CLANG && (ARM64 || LOONGARCH || RISCV)) + select DRM_AMD_DC_FP if ARCH_HAS_KERNEL_FPU_SUPPORT && !(CC_IS_CLANG && ((ARM64 && CLANG_VERSION < 220000) || LOONGARCH || RISCV)) help Choose this option if you want to use the new display engine support for AMDGPU. This adds required support for Vega and diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.c index 9ff25bc8ee5ae..cafb1b069be80 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.c @@ -655,6 +655,20 @@ int dm_dmub_sw_init(struct amdgpu_device *adev) dmub_asic != DMUB_ASIC_DCN321) mem_domain |= AMDGPU_GEM_DOMAIN_VRAM; + /* + * Behind a non-posted BAR the buffer lives in GTT. The backdoor loader of a directly + * loaded firmware converts addresses as VRAM addresses and fills the buffer with plain + * CPU copies, so only PSP firmware loading is supported there. + */ + if (pci_resource_flags(adev->pdev, 0) & IORESOURCE_MEM_NONPOSTED) { + if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { + drm_err(adev_to_drm(adev), + "Direct DMUB firmware loading is not supported behind a non-posted BAR\n"); + return -EOPNOTSUPP; + } + mem_domain = AMDGPU_GEM_DOMAIN_GTT; + } + /* * Allocate a framebuffer based on the total size of all the regions. * TODO: Move this into GART. diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index 2bd3ea17e7897..d0dd7f32607e0 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -1129,7 +1129,8 @@ int smu_cmn_update_table_read_arg(struct smu_context *smu, table_size = smu_table->tables[table_index].size; if (drv2smu) { - memcpy(table->cpu_addr, table_data, table_size); + amdgpu_bo_memcpy_to(table->bo, table->cpu_addr, table_data, + table_size); /* * Flush hdp cache: to guard the content seen by * GPU is consitent with CPU. @@ -1151,7 +1152,8 @@ int smu_cmn_update_table_read_arg(struct smu_context *smu, if (!drv2smu) { amdgpu_hdp_invalidate(adev, NULL); - memcpy(table_data, table->cpu_addr, table_size); + amdgpu_bo_memcpy_from(table->bo, table_data, table->cpu_addr, + table_size); } return 0; diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 98534a725e839..e5c6276b7a830 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -64,28 +64,44 @@ struct apple_drm_private { }; /* Component readiness is later than platform probe/bind. */ -static DEFINE_MUTEX(apple_t6030_display_lock); +static DEFINE_MUTEX(apple_g15_display_lock); static struct device *apple_t6030_display; +static struct device *apple_t6031_display; -static int apple_t6030_display_publish(struct device *dev) +/* The publication slot of a G15 SoC's internal display subsystem, or NULL. */ +static struct device **apple_g15_display_slot(struct device *dev) { - if (!of_device_is_compatible(dev->of_node, "apple,t6030-display-subsystem")) + if (of_device_is_compatible(dev->of_node, "apple,t6030-display-subsystem")) + return &apple_t6030_display; + if (of_device_is_compatible(dev->of_node, "apple,t6031-display-subsystem")) + return &apple_t6031_display; + return NULL; +} + +static int apple_g15_display_publish(struct device *dev) +{ + struct device **slot = apple_g15_display_slot(dev); + + if (!slot) return 0; - guard(mutex)(&apple_t6030_display_lock); - if (apple_t6030_display) - return apple_t6030_display == dev ? 0 : -EBUSY; - apple_t6030_display = get_device(dev); + guard(mutex)(&apple_g15_display_lock); + if (*slot) + return *slot == dev ? 0 : -EBUSY; + *slot = get_device(dev); return 0; } -static void apple_t6030_display_withdraw(struct device *dev) +static void apple_g15_display_withdraw(struct device *dev) { + struct device **slot = apple_g15_display_slot(dev); struct device *old = NULL; - scoped_guard(mutex, &apple_t6030_display_lock) { - if (apple_t6030_display == dev) { - old = apple_t6030_display; - apple_t6030_display = NULL; + if (!slot) + return; + scoped_guard(mutex, &apple_g15_display_lock) { + if (*slot == dev) { + old = *slot; + *slot = NULL; } } put_device(old); @@ -96,22 +112,35 @@ static void apple_t6030_display_withdraw(struct device *dev) * publish separately. This lock protects only publication and a device link; * no display calls or firmware RPC run while it is held. */ -int apple_drm_t6030_link_device(struct device *consumer) +static int apple_g15_link_device(struct device *consumer, const char *machine, + const char *gpu, struct device **display) { struct device_link *link; - if (!of_machine_is_compatible("apple,t6030") || !consumer->of_node || - !of_device_is_compatible(consumer->of_node, "apple,agx-t6030")) + if (!of_machine_is_compatible(machine) || !consumer->of_node || + !of_device_is_compatible(consumer->of_node, gpu)) return -ENODEV; - guard(mutex)(&apple_t6030_display_lock); - if (!apple_t6030_display) + guard(mutex)(&apple_g15_display_lock); + if (!*display) return -EPROBE_DEFER; - link = device_link_add(consumer, apple_t6030_display, - DL_FLAG_AUTOREMOVE_CONSUMER); + link = device_link_add(consumer, *display, DL_FLAG_AUTOREMOVE_CONSUMER); return link ? 0 : -EINVAL; } + +int apple_drm_t6030_link_device(struct device *consumer) +{ + return apple_g15_link_device(consumer, "apple,t6030", "apple,agx-t6030", + &apple_t6030_display); +} EXPORT_SYMBOL_GPL(apple_drm_t6030_link_device); +int apple_drm_t6031_link_device(struct device *consumer) +{ + return apple_g15_link_device(consumer, "apple,t6031", "apple,agx-t6031", + &apple_t6031_display); +} +EXPORT_SYMBOL_GPL(apple_drm_t6031_link_device); + DEFINE_DRM_GEM_DMA_FOPS(apple_fops); #define DART_PAGE_SIZE 16384 @@ -1435,7 +1464,7 @@ static int apple_drm_init(struct device *dev) drm_client_setup_with_fourcc(&apple->drm, DRM_FORMAT_XRGB8888); /* A subsequent successful platform probe retries deferred GPU probes. */ - ret = apple_t6030_display_publish(dev); + ret = apple_g15_display_publish(dev); if (ret) { drm_dev_unregister(&apple->drm); goto err_unbind; @@ -1457,7 +1486,7 @@ static void apple_drm_uninit(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - apple_t6030_display_withdraw(dev); + apple_g15_display_withdraw(dev); /* An aborted takeover leaves its DCPs running until the reboot. */ if (apple->aborted) @@ -1678,3 +1707,10 @@ module_exit(appledrm_unregister); MODULE_AUTHOR("Asahi Linux contributors"); MODULE_DESCRIPTION(DRIVER_DESC); MODULE_LICENSE("Dual MIT/GPL"); +/* + * The DCPs take their HDMI and Type-C route mux controls from the display + * crossbar. Without this, initramfs generators leave it out, it loads only + * from the root filesystem, and a DCP probe that waits past the deferred + * probe timeout (an encrypted root's passphrase prompt) gives up its routes. + */ +MODULE_SOFTDEP("pre: mux-apple-display-crossbar"); diff --git a/drivers/gpu/drm/apple/audio-link-test.c b/drivers/gpu/drm/apple/audio-link-test.c index a90d85f221ead..0ddd42dce072c 100644 --- a/drivers/gpu/drm/apple/audio-link-test.c +++ b/drivers/gpu/drm/apple/audio-link-test.c @@ -1,9 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* * Included by audio.c: a Terminate's link release ends only the link of the - * instance it is for. The production dcpaud_release_link() runs; the audio - * service's answer to which instance the link is on, and its link calls, - * are stubbed. + * instance it is for, and a stopped stereo link is held between streams. + * The production link lifecycle runs; the audio service's answer to which + * instance the link is on, its link calls and the DMA channel are stubbed. */ #include #include @@ -11,13 +11,87 @@ struct dcpaud_link_test { bool link_on; + bool link_live; + /* The error removing the idle transfer returns */ + int idle_remove_err; u64 asked; unsigned int completes; unsigned int stops; + struct dma_device dma; + struct dma_chan channel; + struct dma_async_tx_descriptor desc; + char sequence[16]; + unsigned int nsequence; }; static struct dcpaud_link_test *dcpaud_link_current; +static void dcpaud_link_record(char step) +{ + struct dcpaud_link_test *t = dcpaud_link_current; + + if (t->nsequence < sizeof(t->sequence) - 1) + t->sequence[t->nsequence++] = step; +} + +static bool dcpaud_link_test_live(struct device *dev) +{ + return dcpaud_link_current->link_live; +} + +static int dcpaud_link_test_terminate(struct dma_chan *chan) +{ + dcpaud_link_record('X'); + return 0; +} + +/* G the channel configured; the silence as its idle transfer: I set, i removed */ +static int dcpaud_link_test_config(struct dma_chan *chan, struct dma_slave_config *config) +{ + const struct apple_sio_idle *idle = config->peripheral_config; + + if (!idle) { + dcpaud_link_record('G'); + return 0; + } + if (config->peripheral_size != sizeof(*idle)) + return -EINVAL; + dcpaud_link_record(idle->len ? 'I' : 'i'); + return idle->len ? 0 : dcpaud_link_current->idle_remove_err; +} + +static dma_cookie_t dcpaud_link_test_submit(struct dma_async_tx_descriptor *tx) +{ + return 1; +} + +static struct dma_async_tx_descriptor * +dcpaud_link_test_prep(struct dma_chan *chan, dma_addr_t buf, size_t len, size_t period, + enum dma_transfer_direction dir, unsigned long flags) +{ + struct dcpaud_link_test *t = dcpaud_link_current; + + dcpaud_link_record('Q'); + t->desc.chan = chan; + t->desc.tx_submit = dcpaud_link_test_submit; + return &t->desc; +} + +static void dcpaud_link_test_issue(struct dma_chan *chan) +{ +} + +static void dcpaud_link_test_drain(struct dma_chan *chan) +{ + dcpaud_link_record('D'); +} + +static enum dma_status dcpaud_link_test_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *state) +{ + return DMA_COMPLETE; +} + static bool dcpaud_link_test_on(struct device *dev, u64 gen) { dcpaud_link_current->asked = gen; @@ -27,10 +101,17 @@ static bool dcpaud_link_test_on(struct device *dev, u64 gen) static int dcpaud_link_test_call(struct device *dev, enum dcp_audio_link op, const struct dcp_sound_cookie *cookie) { - if (op == DCP_AUDIO_LINK_COMPLETE) + if (op == DCP_AUDIO_LINK_COMPLETE) { dcpaud_link_current->completes++; - else if (op == DCP_AUDIO_LINK_STOP) + dcpaud_link_record('C'); + } else if (op == DCP_AUDIO_LINK_STOP) { dcpaud_link_current->stops++; + dcpaud_link_record('S'); + } else if (op == DCP_AUDIO_LINK_PREPARE) { + dcpaud_link_record('P'); + } else if (op == DCP_AUDIO_LINK_START) { + dcpaud_link_record('T'); + } return 0; } @@ -49,7 +130,16 @@ static struct dcp_audio *dcpaud_link_setup(struct kunit *test, struct platform_d dcpaud->native_link = true; mutex_init(&dcpaud->link_lock); dcpaud_link_current = t; + t->link_live = true; + t->channel.device = &t->dma; + t->dma.device_terminate_all = dcpaud_link_test_terminate; + t->dma.device_synchronize = dcpaud_link_test_drain; + t->dma.device_tx_status = dcpaud_link_test_status; + t->dma.device_prep_dma_cyclic = dcpaud_link_test_prep; + t->dma.device_issue_pending = dcpaud_link_test_issue; + t->dma.device_config = dcpaud_link_test_config; kunit_activate_static_stub(test, dcp_audiosrv_link_on, dcpaud_link_test_on); + kunit_activate_static_stub(test, dcp_audiosrv_link_live, dcpaud_link_test_live); kunit_activate_static_stub(test, dcp_audiosrv_link, dcpaud_link_test_call); return dcpaud; } @@ -85,8 +175,287 @@ static void dcpaud_link_release_instance(struct kunit *test) KUNIT_EXPECT_EQ(test, dcpaud_link_current->completes, 2U); } +static void dcpaud_link_test_close(void *data) +{ + snd_dmaengine_pcm_close(data); +} + +static struct snd_pcm_substream *dcpaud_link_test_open(struct kunit *test, + struct dcp_audio *dcpaud) +{ + struct snd_pcm_substream *sub = kunit_kzalloc(test, sizeof(*sub), GFP_KERNEL); + struct snd_pcm_runtime *runtime = kunit_kzalloc(test, sizeof(*runtime), GFP_KERNEL); + struct snd_pcm *pcm = kunit_kzalloc(test, sizeof(*pcm), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, sub); + KUNIT_ASSERT_NOT_NULL(test, runtime); + KUNIT_ASSERT_NOT_NULL(test, pcm); + pcm->private_data = dcpaud; + pcm->nonatomic = true; + sub->pcm = pcm; + sub->runtime = runtime; + KUNIT_ASSERT_EQ(test, snd_dmaengine_pcm_open(sub, &dcpaud_link_current->channel), 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dcpaud_link_test_close, sub), 0); + return sub; +} + +static void dcpaud_disconnect_sync_completes_live_owner(struct kunit *test) +{ + struct platform_device *pdev; + struct dcp_audio *audio = dcpaud_link_setup(test, &pdev); + struct snd_pcm_substream *sub = dcpaud_link_test_open(test, audio); + + /* Disconnect cleared connected; the deferred owner release has not run. */ + audio->connected = false; + audio->link_prepared = true; + audio->link_started = true; + KUNIT_ASSERT_EQ(test, dcp_pcm_native_sync_stop(sub), 0); + KUNIT_EXPECT_STREQ(test, dcpaud_link_current->sequence, "DSC"); + KUNIT_EXPECT_FALSE(test, audio->link_prepared); + KUNIT_EXPECT_FALSE(test, audio->link_started); + dcpaud_release_link(pdev, NULL); + KUNIT_EXPECT_EQ(test, dcpaud_link_current->stops, 1U); + KUNIT_EXPECT_EQ(test, dcpaud_link_current->completes, 1U); +} + +static void dcpaud_gone_service_drops_link(struct kunit *test) +{ + struct platform_device *pdev; + struct dcp_audio *audio = dcpaud_link_setup(test, &pdev); + struct snd_pcm_substream *sub = dcpaud_link_test_open(test, audio); + + audio->link_prepared = true; + audio->link_started = true; + dcpaud_link_current->link_live = false; + KUNIT_ASSERT_EQ(test, dcp_pcm_native_sync_stop(sub), 0); + KUNIT_EXPECT_STREQ(test, dcpaud_link_current->sequence, "D"); + KUNIT_EXPECT_FALSE(test, audio->link_prepared); + KUNIT_EXPECT_FALSE(test, audio->link_started); +} + +/* A stream stopped by STOP with its DMA playing on, on the PCM's channel */ +static void dcpaud_link_test_held_stop(struct kunit *test, struct dcp_audio *audio) +{ + audio->chan = &dcpaud_link_current->channel; + audio->silence = kunit_kzalloc(test, DCPAUD_SILENCE_BYTES, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, audio->silence); + audio->link_prepared = true; + audio->link_started = true; + audio->dma_cleared = true; + /* The stream's PM reference, carried by the cleared DMA */ + pm_runtime_get_noresume(audio->dev); +} + +static void dcpaud_stop_holds_link(struct kunit *test) +{ + struct platform_device *pdev; + struct dcp_audio *audio = dcpaud_link_setup(test, &pdev); + struct snd_pcm_substream *sub = dcpaud_link_test_open(test, audio); + + dcpaud_link_test_held_stop(test, audio); + /* The silence set first, the stream's DMA ends into it; no link call */ + KUNIT_ASSERT_EQ(test, dcp_pcm_native_sync_stop(sub), 0); + KUNIT_EXPECT_STREQ(test, dcpaud_link_current->sequence, "IXD"); + KUNIT_EXPECT_TRUE(test, audio->silence_running); + KUNIT_EXPECT_TRUE(test, audio->idle_set); + KUNIT_EXPECT_TRUE(test, audio->link_started); + + KUNIT_ASSERT_EQ(test, dcp_pcm_native_hw_free(sub), 0); + KUNIT_EXPECT_TRUE(test, audio->link_prepared); + KUNIT_EXPECT_EQ(test, dcpaud_link_current->stops + dcpaud_link_current->completes, 0U); + + /* The display goes off: the silence ends, then the link */ + dcpaud_release_link(pdev, NULL); + KUNIT_EXPECT_STREQ(test, dcpaud_link_current->sequence, "IXDiDSC"); + KUNIT_EXPECT_FALSE(test, audio->silence_running); + KUNIT_EXPECT_FALSE(test, audio->idle_set); + KUNIT_EXPECT_FALSE(test, audio->link_prepared); + /* The stream's reference went with the silence */ + KUNIT_EXPECT_EQ(test, atomic_read(&audio->dev->power.usage_count), 0); +} + +static void dcpaud_stuck_silence_keeps_pm(struct kunit *test) +{ + struct platform_device *pdev; + struct dcp_audio *audio = dcpaud_link_setup(test, &pdev); + struct snd_pcm_substream *sub = dcpaud_link_test_open(test, audio); + + dcpaud_link_test_held_stop(test, audio); + KUNIT_ASSERT_EQ(test, dcp_pcm_native_sync_stop(sub), 0); + KUNIT_ASSERT_EQ(test, dcp_pcm_native_hw_free(sub), 0); + + /* The channel cannot be confirmed stopped: it may still read the silence */ + dcpaud_link_current->idle_remove_err = -EIO; + dcpaud_release_link(pdev, NULL); + KUNIT_EXPECT_FALSE(test, audio->silence_running); + KUNIT_EXPECT_FALSE(test, audio->link_prepared); + KUNIT_EXPECT_TRUE(test, audio->idle_set); + KUNIT_EXPECT_TRUE(test, audio->idle_pm); + /* The stream's reference stays with the silence */ + KUNIT_EXPECT_EQ(test, atomic_read(&audio->dev->power.usage_count), 1); + + /* Removed later, it gives the reference back */ + dcpaud_link_current->idle_remove_err = 0; + scoped_guard(mutex, &audio->link_lock) + KUNIT_EXPECT_EQ(test, dcpaud_idle(audio, false), 0); + KUNIT_EXPECT_FALSE(test, audio->idle_set); + KUNIT_EXPECT_FALSE(test, audio->idle_pm); + KUNIT_EXPECT_EQ(test, atomic_read(&audio->dev->power.usage_count), 0); +} + +static void dcpaud_held_stop_without_service(struct kunit *test) +{ + struct platform_device *pdev; + struct dcp_audio *audio = dcpaud_link_setup(test, &pdev); + struct snd_pcm_substream *sub = dcpaud_link_test_open(test, audio); + + dcpaud_link_test_held_stop(test, audio); + dcpaud_link_current->link_live = false; + /* No service to hold it for: the DMA ends and the link is dropped */ + KUNIT_ASSERT_EQ(test, dcp_pcm_native_sync_stop(sub), 0); + KUNIT_EXPECT_STREQ(test, dcpaud_link_current->sequence, "XD"); + KUNIT_EXPECT_FALSE(test, audio->silence_running); + KUNIT_EXPECT_FALSE(test, audio->link_prepared); + KUNIT_EXPECT_FALSE(test, audio->link_started); + KUNIT_EXPECT_EQ(test, atomic_read(&audio->dev->power.usage_count), 0); +} + +/* A held link and its silence, and a prepared stream to start on it */ +static struct snd_pcm_substream *dcpaud_link_test_held(struct kunit *test, struct dcp_audio *audio) +{ + struct snd_pcm_substream *sub = dcpaud_link_test_open(test, audio); + + mutex_init(&audio->data_lock); + audio->connected = true; + audio->chan = &dcpaud_link_current->channel; + audio->link_prepared = true; + audio->link_started = true; + audio->silence_running = true; + audio->idle_set = true; + /* The held link's PM reference, then the stream's from its START */ + pm_runtime_get_noresume(audio->dev); + pm_runtime_get_noresume(audio->dev); + sub->runtime->frame_bits = 64; + sub->runtime->buffer_size = 8192; + sub->runtime->period_size = 512; + return sub; +} + +static void dcpaud_start_on_held_link(struct kunit *test) +{ + struct platform_device *pdev; + struct dcp_audio *audio = dcpaud_link_setup(test, &pdev); + struct snd_pcm_substream *sub = dcpaud_link_test_held(test, audio); + + /* Its element: the stream's DMA follows the silence, no link call */ + KUNIT_ASSERT_EQ(test, dcp_pcm_native_start(sub, SNDRV_PCM_TRIGGER_START), 0); + KUNIT_EXPECT_STREQ(test, dcpaud_link_current->sequence, "Q"); + KUNIT_EXPECT_FALSE(test, audio->silence_running); + KUNIT_EXPECT_TRUE(test, audio->idle_set); + KUNIT_EXPECT_TRUE(test, audio->link_started); + pm_runtime_put_noidle(audio->dev); +} + +static void dcpaud_start_replaces_held_link(struct kunit *test) +{ + struct platform_device *pdev; + struct dcp_audio *audio = dcpaud_link_setup(test, &pdev); + struct snd_pcm_substream *sub = dcpaud_link_test_held(test, audio); + + /* Another element: the held link ends, the stream's is configured, prepared and started */ + memset(&audio->link_cookie, 1, sizeof(audio->link_cookie)); + memset(&audio->selected_cookie, 2, sizeof(audio->selected_cookie)); + KUNIT_ASSERT_EQ(test, dcp_pcm_native_start(sub, SNDRV_PCM_TRIGGER_START), 0); + KUNIT_EXPECT_STREQ(test, dcpaud_link_current->sequence, "iSCGPQT"); + KUNIT_EXPECT_FALSE(test, audio->idle_set); + KUNIT_EXPECT_EQ(test, memcmp(&audio->link_cookie, &audio->selected_cookie, + sizeof(audio->link_cookie)), 0); + KUNIT_EXPECT_TRUE(test, audio->link_started); + pm_runtime_put_noidle(audio->dev); +} + +static void dcpaud_start_after_held_link_went(struct kunit *test) +{ + struct platform_device *pdev; + struct dcp_audio *audio = dcpaud_link_setup(test, &pdev); + struct snd_pcm_substream *sub = dcpaud_link_test_held(test, audio); + + /* The held link's service went (unplug): the stream gets a link of its own */ + dcpaud_link_current->link_live = false; + KUNIT_ASSERT_EQ(test, dcp_pcm_native_start(sub, SNDRV_PCM_TRIGGER_START), 0); + KUNIT_EXPECT_STREQ(test, dcpaud_link_current->sequence, "iGPQT"); + KUNIT_EXPECT_FALSE(test, audio->silence_running); + KUNIT_EXPECT_TRUE(test, audio->link_started); + pm_runtime_put_noidle(audio->dev); + KUNIT_EXPECT_EQ(test, atomic_read(&audio->dev->power.usage_count), 0); +} + +static void dcpaud_failed_start_keeps_held_link(struct kunit *test) +{ + struct platform_device *pdev; + struct dcp_audio *audio = dcpaud_link_setup(test, &pdev); + struct snd_pcm_substream *sub = dcpaud_link_test_held(test, audio); + + /* Unplugged before the swap: START fails, its STOP leaves the silence */ + audio->connected = false; + sub->runtime->channels = 2; + sub->runtime->dma_area = kunit_kzalloc(test, 64, GFP_KERNEL); + sub->runtime->dma_bytes = 64; + KUNIT_ASSERT_EQ(test, dcp_pcm_native_start(sub, SNDRV_PCM_TRIGGER_START), -ENXIO); + KUNIT_ASSERT_EQ(test, dcp_pcm_native_trigger(sub, SNDRV_PCM_TRIGGER_STOP), 0); + KUNIT_EXPECT_STREQ(test, dcpaud_link_current->sequence, ""); + KUNIT_EXPECT_FALSE(test, audio->dma_cleared); + KUNIT_EXPECT_TRUE(test, audio->silence_running); + /* Only the held silence's reference is left */ + KUNIT_EXPECT_EQ(test, atomic_read(&audio->dev->power.usage_count), 1); + pm_runtime_put_noidle(audio->dev); +} + +static void dcpaud_close_keeps_held_silence(struct kunit *test) +{ + struct platform_device *pdev; + struct dcp_audio *audio = dcpaud_link_setup(test, &pdev); + struct snd_pcm_substream *sub = kunit_kzalloc(test, sizeof(*sub), GFP_KERNEL); + struct snd_pcm_runtime *runtime = kunit_kzalloc(test, sizeof(*runtime), GFP_KERNEL); + struct snd_pcm *pcm = kunit_kzalloc(test, sizeof(*pcm), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, sub); + KUNIT_ASSERT_NOT_NULL(test, runtime); + KUNIT_ASSERT_NOT_NULL(test, pcm); + pcm->private_data = audio; + pcm->nonatomic = true; + sub->pcm = pcm; + sub->runtime = runtime; + KUNIT_ASSERT_EQ(test, snd_dmaengine_pcm_open(sub, &dcpaud_link_current->channel), 0); + audio->chan = &dcpaud_link_current->channel; + audio->link_prepared = true; + audio->link_started = true; + audio->silence_running = true; + audio->idle_set = true; + pm_runtime_get_noresume(audio->dev); + + /* The PCM's terminate leaves the idle silence: nothing queued, no link call */ + KUNIT_ASSERT_EQ(test, dcp_pcm_native_close(sub), 0); + KUNIT_EXPECT_STREQ(test, dcpaud_link_current->sequence, "XD"); + KUNIT_EXPECT_TRUE(test, audio->silence_running); + KUNIT_EXPECT_TRUE(test, audio->idle_set); + KUNIT_EXPECT_TRUE(test, audio->link_started); + KUNIT_EXPECT_EQ(test, atomic_read(&audio->dev->power.usage_count), 1); + pm_runtime_put_noidle(audio->dev); +} + static struct kunit_case dcpaud_link_cases[] = { KUNIT_CASE(dcpaud_link_release_instance), + KUNIT_CASE(dcpaud_disconnect_sync_completes_live_owner), + KUNIT_CASE(dcpaud_gone_service_drops_link), + KUNIT_CASE(dcpaud_stop_holds_link), + KUNIT_CASE(dcpaud_stuck_silence_keeps_pm), + KUNIT_CASE(dcpaud_held_stop_without_service), + KUNIT_CASE(dcpaud_start_on_held_link), + KUNIT_CASE(dcpaud_start_replaces_held_link), + KUNIT_CASE(dcpaud_start_after_held_link_went), + KUNIT_CASE(dcpaud_failed_start_keeps_held_link), + KUNIT_CASE(dcpaud_close_keeps_held_silence), {} }; diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index ccb10dcf3768c..44c228deb2a89 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -14,7 +14,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -43,6 +46,22 @@ */ #define DCPAUD_PERIODS_MAX 16 +/* + * The silence a held link plays between streams, as the SIO channel's idle + * transfer: SIO keeps up to four periods in flight, and a stream's first + * period follows those (about 85 ms). Its four descriptor slots stay taken + * while the link is held, next to a stream's sixteen at most. + */ +#define DCPAUD_SILENCE_PERIOD SZ_4K +#define DCPAUD_SILENCE_BYTES (4 * DCPAUD_SILENCE_PERIOD) + +/* + * A stream that ends into the held link's silence leaves up to four periods + * issued, and dmaengine_synchronize() waits 500 ms for them before it stops + * the channel. Streams on a port that holds links keep to 50 ms periods. + */ +#define DCPAUD_HELD_PERIOD_MAX_US 50000 + struct dcp_audio { struct device *dev; struct device *dcp_dev; @@ -59,6 +78,8 @@ struct dcp_audio { bool dcp_connected; /// dcp status keep for delayed initialization bool connected; unsigned int connection_cookie; + /* When the sink was last connected */ + ktime_t connected_at; struct snd_pcm_chmap_elem selected_chmap; struct dcp_sound_cookie selected_cookie; @@ -81,6 +102,22 @@ struct dcp_audio { struct mutex link_lock; bool link_prepared; bool link_started; + /* The element the link was prepared with */ + struct dcp_sound_cookie link_cookie; + /* START issued the stream's DMA */ + bool stream_dma; + /* STOP left the DMA playing the cleared buffer, see dcp_pcm_native_sync_stop() */ + bool dma_cleared; + /* Drops a held link whose service went, see dcpaud_disconnect() */ + struct work_struct stale_work; + /* The held link's silence, see dcpaud_silence_start() */ + void *silence; + dma_addr_t silence_addr; + bool silence_running; + /* The silence is the channel's idle transfer, see dcpaud_idle() */ + bool idle_set; + /* It holds a PM reference: its removal failed, see dcpaud_silence_stop() */ + bool idle_pm; #endif }; @@ -291,7 +328,7 @@ static int dcpaud_init_dma(struct dcp_audio *dcpaud) { struct dma_chan *chan; if (dcpaud->chan) - return 0; + goto silence; chan = of_dma_request_slave_channel(dcpaud->dev->of_node, "tx"); /* squelch dma channel request errors, the driver will try again alter */ @@ -311,6 +348,14 @@ static int dcpaud_init_dma(struct dcp_audio *dcpaud) dcpaud->chan->device->dev, 1024 * 1024, SIZE_MAX); +silence: +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + /* Without it a stopped link is not held, see dcpaud_silence_start(). */ + if (dcpaud_native_link(dcpaud) && !dcpaud->silence) + dcpaud->silence = dma_alloc_coherent(dcpaud->chan->device->dev, + DCPAUD_SILENCE_BYTES, + &dcpaud->silence_addr, GFP_KERNEL); +#endif return 0; } @@ -599,14 +644,206 @@ struct snd_pcm_ops dcp_playback_ops = { * * prepare: the DMA channel configured, then the link prepared, once; * START: the cyclic DMA issued, then the link started; - * STOP: the DMA terminated; - * sync_stop: the DMA drained, then the link stopped and completed; + * STOP: the DMA terminated, or for a held link (below) left playing + * the cleared buffer; + * sync_stop: the DMA drained, then the link stopped and completed, or the + * held link's silence taking over from the DMA; * hw_free: a link prepared but never started, completed. * - * Every link method carries the selected element's link data. A link of an - * earlier connection went with its service: its state is dropped unsent. - * The audio service itself is open only while the PCM is. + * Every link method carries its element's link data. A link of an earlier + * connection went with its service: its state is dropped unsent. The audio + * service itself is open while the PCM is, or while its link is held. + */ +/* + * The sinks behind the J514S HDMI port (PS190 converter and TV) and its + * Type-C ports mute the audio of a link that stops and starts again with an + * unchanged format, until the DP link trains again: only a different channel + * layout brings it back. So a stereo link, once started, is held between + * streams, as macOS holds it: silence follows the stopped stream's DMA and + * the link keeps running until the format changes or the display goes off + * (dcpaud_release_link()). + * + * The link must not run dry either: a gap of 20 ms in its samples mutes the + * sink the same way. The silence is the SIO channel's idle transfer, which + * the channel issues whenever no stream's DMA does, in descriptor order: + * a stream's DMA takes over from it and hands back to it without a gap. + */ +/* + * A change of sample rate or sample size with an unchanged channel layout + * replaces the link with a new one, which this sink mutes just as it does a + * same-format restart. PipeWire changes the rate to follow its streams + * (44.1 kHz music, 48 kHz video). macOS runs the link at one nominal rate + * and format and converts the streams to it; so does this driver while it + * holds links: the PCM offers one rate (48 kHz, else 44.1 or 32 kHz) and one + * sample size (24 bits, else 16), both of one element that takes them in + * stereo, and userspace resamples. Channel layouts stay free: a layout + * change is heard. + * + * The pair covers every layout, not only the stereo that links are held in: + * userspace commonly sets the sample format before the channel count, and a + * pair applied only once a stream is stereo let it set a format that then + * ruled stereo out (PipeWire built no HDMI sink). + */ +static int dcpaud_pin_rate_format(struct dcp_audio *dcpaud, + struct snd_pcm_runtime *runtime) +{ + static const unsigned int rates[] = { 48000, 44100, 32000 }; + static const snd_pcm_format_t formats[] = { + SNDRV_PCM_FORMAT_S24_LE, SNDRV_PCM_FORMAT_S16_LE, + }; + struct dcp_sound_format_mask sieve = { + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE, + .rates = GENMASK(snd_pcm_known_rates.count - 1, 0), + .nchans = BIT(2), + }; + struct dcp_sound_format_mask hits = { 0, 0, 0 }; + struct dcp_parse_ctx elements = { + .dcp = dev_get_drvdata(dcpaud->dcp_dev), + .blob = dcpaud->elements + 4, + .len = dcpaud_elements_size(dcpaud) - 4, + .pos = 0, + .v14_7_audio = dcpaud->native_link, + }; + unsigned int rate = 0, rate_bit = 0, i, j; + int ret; + + parse_sound_constraints(&elements, &sieve, &hits); + for (i = 0; i < ARRAY_SIZE(rates) && !rate; i++) + for (j = 0; j < snd_pcm_known_rates.count; j++) + if (snd_pcm_known_rates.list[j] == rates[i] && (hits.rates & BIT(j))) { + rate = rates[i]; + rate_bit = j; + } + if (!rate) + return 0; /* no stereo LPCM element: the sink decides */ + + /* The sample size of an element that takes that rate in stereo */ + sieve.rates = BIT(rate_bit); + elements.pos = 0; + parse_sound_constraints(&elements, &sieve, &hits); + for (i = 0; i < ARRAY_SIZE(formats); i++) + if (hits.formats & pcm_format_to_bits(formats[i])) + break; + if (i == ARRAY_SIZE(formats)) + return 0; + + dev_dbg(dcpaud->dev, "held links run at %u Hz, format %d\n", rate, + (__force int)formats[i]); + ret = snd_pcm_hw_constraint_single(runtime, SNDRV_PCM_HW_PARAM_RATE, rate); + if (ret < 0) + return ret; + ret = snd_pcm_hw_constraint_mask64(runtime, SNDRV_PCM_HW_PARAM_FORMAT, + pcm_format_to_bits(formats[i])); + return ret < 0 ? ret : 0; +} + +/* + * Seen once, at boot: a link started about 0.2 s after the display came on + * was not heard, and holding it kept it silent; the next connection's first + * link, started later, was heard. While links are held, the first link of a + * connection starts once the sink has been up this long, a margin rather + * than a measured bound. One link per stream needs no wait: the next stream + * starts a new link. + */ +#define DCPAUD_SINK_SETTLE_MS 1500 + +/* + * Sets or removes the silence as the channel's idle transfer. Removed, it + * stops the channel unless a stream's DMA runs, once its periods played. */ +static int dcpaud_idle(struct dcp_audio *dcpaud, bool on) +{ + struct apple_sio_idle idle = { + .addr = dcpaud->silence_addr, + .len = on ? DCPAUD_SILENCE_BYTES : 0, + .period_len = DCPAUD_SILENCE_PERIOD, + }; + struct dma_slave_config config = { + .direction = DMA_MEM_TO_DEV, + .peripheral_config = &idle, + .peripheral_size = sizeof(idle), + }; + int ret; + + ret = dmaengine_slave_config(dcpaud->chan, &config); + /* Not removed: the channel may still read the silence. */ + if (ret) + return ret; + dcpaud->idle_set = on; + if (!on && dcpaud->idle_pm) { + dcpaud->idle_pm = false; + pm_runtime_mark_last_busy(dcpaud->dev); + pm_runtime_put_autosuspend(dcpaud->dev); + } + return 0; +} + +/* + * The held link's silence carries the PM reference of the stream. It is set + * before the stream's DMA ends, which it then follows without a gap. + */ +static int dcpaud_silence_start(struct dcp_audio *dcpaud) +{ + int ret; + + lockdep_assert_held(&dcpaud->link_lock); + if (!dcpaud->silence) + return -ENOMEM; + if (!dcpaud->idle_set) { + ret = dcpaud_idle(dcpaud, true); + if (ret) + return ret; + } + dcpaud->silence_running = true; + dev_dbg(dcpaud->dev, "link held, playing silence\n"); + return 0; +} + +/* The link ends: so does its silence, which a stream may have taken over from. */ +static void dcpaud_silence_stop(struct dcp_audio *dcpaud) +{ + lockdep_assert_held(&dcpaud->link_lock); + if (dcpaud->idle_set) + dcpaud_idle(dcpaud, false); + if (!dcpaud->silence_running) + return; + dcpaud->silence_running = false; + /* A channel that may still read the silence keeps the device up. */ + if (dcpaud->idle_set && !dcpaud->idle_pm) { + dcpaud->idle_pm = true; + return; + } + pm_runtime_mark_last_busy(dcpaud->dev); + pm_runtime_put_autosuspend(dcpaud->dev); + dev_dbg(dcpaud->dev, "held link's silence stopped\n"); +} + +static void dcpaud_link_teardown(struct dcp_audio *dcpaud); + +/* + * A held link went with its audio service (sink change, unplug): its silence + * stops and its state is dropped unsent, so that a new service's stream + * prepares and starts its own link. + */ +static void dcpaud_drop_stale(struct dcp_audio *dcpaud) +{ + lockdep_assert_held(&dcpaud->link_lock); + if ((dcpaud->silence_running || dcpaud->link_prepared || dcpaud->link_started) && + !dcp_audiosrv_link_live(dcpaud->dcp_dev)) { + dev_dbg(dcpaud->dev, "held link went with its service\n"); + dcpaud_link_teardown(dcpaud); + } +} + +static void dcpaud_stale_work(struct work_struct *work) +{ + struct dcp_audio *dcpaud = container_of(work, struct dcp_audio, stale_work); + + guard(mutex)(&dcpaud->link_lock); + if (!dcpaud->dma_cleared) + dcpaud_drop_stale(dcpaud); +} + static int dcp_pcm_native_open(struct snd_pcm_substream *substream) { struct dcp_audio *dcpaud = substream->pcm->private_data; @@ -622,8 +859,27 @@ static int dcp_pcm_native_open(struct snd_pcm_substream *substream) } ret = dcp_pcm_open(substream); - if (ret && live) - dcp_audiosrv_close(dcpaud->dcp_dev); + if (!ret && live && !dcpaud->open_unplugged) { + ret = dcpaud_pin_rate_format(dcpaud, substream->runtime); + if (!ret) + ret = snd_pcm_hw_constraint_minmax(substream->runtime, + SNDRV_PCM_HW_PARAM_PERIOD_TIME, 0, + DCPAUD_HELD_PERIOD_MAX_US); + /* Positive: the constraint narrowed the period time. */ + if (ret > 0) + ret = 0; + if (ret) + dcp_pcm_close(substream); + } + if (ret && live) { + bool held; + + /* A held link keeps its service open. */ + scoped_guard(mutex, &dcpaud->link_lock) + held = dcpaud->silence_running; + if (!held) + dcp_audiosrv_close(dcpaud->dcp_dev); + } return ret; } @@ -632,45 +888,73 @@ static int dcp_pcm_native_close(struct snd_pcm_substream *substream) { struct dcp_audio *dcpaud = substream->pcm->private_data; int ret = dcp_pcm_close(substream); + bool held; - dcp_audiosrv_close(dcpaud->dcp_dev); + /* + * A held link keeps its service open. Closing the PCM terminated the + * channel's transfer, which leaves the idle one, the silence, running. + */ + scoped_guard(mutex, &dcpaud->link_lock) { + dcpaud_drop_stale(dcpaud); + held = dcpaud->silence_running; + } + if (!held) + dcp_audiosrv_close(dcpaud->dcp_dev); return ret; } /* - * Stop and complete the link held, on the open audio service (@live), or - * drop both unsent with a service that went away. + * A PCM disconnect does not end its firmware service. Complete the link on + * that instance while it is published; a removed instance took it along. + * The service can disappear after the check, so -ENODEV during teardown is normal. */ -static void dcpaud_link_teardown(struct dcp_audio *dcpaud, bool live) +static void dcpaud_link_teardown(struct dcp_audio *dcpaud) { + bool live; int ret; lockdep_assert_held(&dcpaud->link_lock); + /* A held link's silence ends with it. */ + dcpaud_silence_stop(dcpaud); + live = (dcpaud->link_prepared || dcpaud->link_started) && + dcp_audiosrv_link_live(dcpaud->dcp_dev); if (dcpaud->link_started && live) { ret = dcp_audiosrv_link(dcpaud->dcp_dev, DCP_AUDIO_LINK_STOP, - &dcpaud->selected_cookie); - if (ret) + &dcpaud->link_cookie); + if (ret && ret != -ENODEV) dev_err(dcpaud->dev, "audio link stop failed: %d\n", ret); } dcpaud->link_started = false; if (dcpaud->link_prepared && live) { ret = dcp_audiosrv_link(dcpaud->dcp_dev, DCP_AUDIO_LINK_COMPLETE, - &dcpaud->selected_cookie); - if (ret) + &dcpaud->link_cookie); + if (ret && ret != -ENODEV) dev_err(dcpaud->dev, "audio link complete failed: %d\n", ret); } dcpaud->link_prepared = false; } -/* New parameters select a new element: the link of the old one ends first. */ +/* + * New parameters select a new element: the link of the old one ends first. + * A held link stays until a stream of another element starts (it may be + * userspace probing), and its channel stays configured. + */ static int dcp_pcm_native_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct dcp_audio *dcpaud = substream->pcm->private_data; + bool up = !dcpaud->open_unplugged && dcpaud_connection_up(dcpaud); + int ret; - scoped_guard(mutex, &dcpaud->link_lock) - dcpaud_link_teardown(dcpaud, dcpaud_connection_up(dcpaud)); + scoped_guard(mutex, &dcpaud->link_lock) { + dcpaud_drop_stale(dcpaud); + if (up && dcpaud->silence_running) { + ret = dcpaud_select_cookie(dcpaud, params); + return ret < 0 ? ret : ret ? 0 : -EINVAL; + } + dcpaud_link_teardown(dcpaud); + } return dcp_pcm_hw_params(substream, params); } @@ -680,14 +964,15 @@ static int dcp_pcm_native_hw_free(struct snd_pcm_substream *substream) struct dcp_audio *dcpaud = substream->pcm->private_data; guard(mutex)(&dcpaud->link_lock); - dcpaud_link_teardown(dcpaud, dcpaud_connection_up(dcpaud)); + if (!dcpaud->silence_running) + dcpaud_link_teardown(dcpaud); return 0; } -static int dcp_pcm_native_prepare(struct snd_pcm_substream *substream) +/* The SIO channel is configured again before every link prepare. */ +static int dcpaud_link_prepare(struct dcp_audio *dcpaud, struct snd_pcm_substream *substream) { - struct dcp_audio *dcpaud = substream->pcm->private_data; struct dma_slave_config slave_config = { .direction = DMA_MEM_TO_DEV, /* One 32-bit DPA entry per sample word, as in hw_params. */ @@ -695,6 +980,23 @@ static int dcp_pcm_native_prepare(struct snd_pcm_substream *substream) }; int ret; + lockdep_assert_held(&dcpaud->link_lock); + ret = dmaengine_slave_config(snd_dmaengine_pcm_get_chan(substream), &slave_config); + if (ret) + return ret; + ret = dcp_audiosrv_link(dcpaud->dcp_dev, DCP_AUDIO_LINK_PREPARE, + &dcpaud->selected_cookie); + if (ret) + return ret; + dcpaud->link_prepared = true; + dcpaud->link_cookie = dcpaud->selected_cookie; + return 0; +} + +static int dcp_pcm_native_prepare(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + if (dcpaud->open_unplugged) return 0; @@ -702,44 +1004,73 @@ static int dcp_pcm_native_prepare(struct snd_pcm_substream *substream) return -ENXIO; guard(mutex)(&dcpaud->link_lock); + dcpaud_drop_stale(dcpaud); if (dcpaud->link_prepared) return 0; - - /* The SIO channel is configured again before every link prepare. */ - ret = dmaengine_slave_config(snd_dmaengine_pcm_get_chan(substream), - &slave_config); - if (ret) - return ret; - - ret = dcp_audiosrv_link(dcpaud->dcp_dev, DCP_AUDIO_LINK_PREPARE, - &dcpaud->selected_cookie); - if (ret) - return ret; - - dcpaud->link_prepared = true; - return 0; + return dcpaud_link_prepare(dcpaud, substream); } static int dcp_pcm_native_start(struct snd_pcm_substream *substream, int cmd) { struct dcp_audio *dcpaud = substream->pcm->private_data; + s64 up_ms; + bool held; int ret; if (!dcpaud_connection_up(dcpaud)) return -ENXIO; + /* The first link of the connection waits for the sink to settle. */ + scoped_guard(mutex, &dcpaud->data_lock) + up_ms = ktime_ms_delta(ktime_get(), dcpaud->connected_at); + if (!READ_ONCE(dcpaud->link_started) && up_ms >= 0 && + up_ms < DCPAUD_SINK_SETTLE_MS) { + dev_dbg(dcpaud->dev, "sink up %lld ms, link waits\n", up_ms); + msleep(DCPAUD_SINK_SETTLE_MS - up_ms); + } + + guard(mutex)(&dcpaud->link_lock); + dcpaud_drop_stale(dcpaud); + /* + * The stream's DMA follows the held link's silence, which stays the + * channel's idle transfer, and takes over the PM reference. + */ + held = dcpaud->silence_running; + if (held) { + dcpaud->silence_running = false; + pm_runtime_mark_last_busy(dcpaud->dev); + pm_runtime_put_autosuspend(dcpaud->dev); + } + + /* + * A held link of another element ends, its silence with it, and the + * stream's is prepared; so is a stream's whose prepare found a link that + * has gone since. + */ + if (!dcpaud->link_prepared || + memcmp(&dcpaud->link_cookie, &dcpaud->selected_cookie, sizeof(dcpaud->link_cookie))) { + if (dcpaud->link_prepared) + dcpaud_link_teardown(dcpaud); + ret = dcpaud_link_prepare(dcpaud, substream); + if (ret) + return ret; + dev_dbg(dcpaud->dev, "link prepared at START\n"); + } else if (held) { + dev_dbg(dcpaud->dev, "stream follows the held link's silence\n"); + } + /* The first period descriptor is issued before the link starts. */ ret = snd_dmaengine_pcm_trigger(substream, cmd); if (ret) return ret; + WRITE_ONCE(dcpaud->stream_dma, true); - guard(mutex)(&dcpaud->link_lock); if (!dcpaud->link_prepared) return -EBADFD; if (dcpaud->link_started) return 0; ret = dcp_audiosrv_link(dcpaud->dcp_dev, DCP_AUDIO_LINK_START, - &dcpaud->selected_cookie); + &dcpaud->link_cookie); dcpaud->link_started = !ret; return ret; } @@ -761,8 +1092,21 @@ static int dcp_pcm_native_trigger(struct snd_pcm_substream *substream, int cmd) return dcp_pcm_native_start(substream, cmd); case SNDRV_PCM_TRIGGER_STOP: + /* + * The DMA plays the cleared buffer on, with the stream's PM + * reference, until sync_stop hands over to the held link's + * silence: the link must not run dry meanwhile. + */ + if (READ_ONCE(dcpaud->stream_dma) && substream->runtime->channels == 2 && + substream->runtime->dma_area) { + memset(substream->runtime->dma_area, 0, substream->runtime->dma_bytes); + WRITE_ONCE(dcpaud->dma_cleared, true); + return 0; + } + fallthrough; case SNDRV_PCM_TRIGGER_SUSPEND: - ret = snd_dmaengine_pcm_trigger(substream, cmd); + /* A START that issued no DMA leaves a held link's silence alone. */ + ret = READ_ONCE(dcpaud->stream_dma) ? snd_dmaengine_pcm_trigger(substream, cmd) : 0; pm_runtime_mark_last_busy(dcpaud->dev); __pm_runtime_put_autosuspend(dcpaud->dev); return ret; @@ -777,11 +1121,34 @@ static int dcp_pcm_native_sync_stop(struct snd_pcm_substream *substream) struct dcp_audio *dcpaud = substream->pcm->private_data; int ret; + WRITE_ONCE(dcpaud->stream_dma, false); + if (READ_ONCE(dcpaud->dma_cleared)) { + guard(mutex)(&dcpaud->link_lock); + WRITE_ONCE(dcpaud->dma_cleared, false); + /* + * The silence set first, the channel hands over to it in + * descriptor order when the stream's DMA ends; synchronize + * returns once the stream's issued periods have played. + */ + if (dcpaud->silence_running || !dcpaud->link_started || + !dcp_audiosrv_link_live(dcpaud->dcp_dev) || dcpaud_silence_start(dcpaud)) { + dmaengine_terminate_async(dcpaud->chan); + dmaengine_synchronize(dcpaud->chan); + dcpaud_link_teardown(dcpaud); + pm_runtime_mark_last_busy(dcpaud->dev); + pm_runtime_put_autosuspend(dcpaud->dev); + return 0; + } + dmaengine_terminate_async(dcpaud->chan); + dmaengine_synchronize(dcpaud->chan); + return 0; + } + /* SIO TERMINATE acknowledged and the issued periods reported back. */ ret = snd_dmaengine_pcm_sync_stop(substream); guard(mutex)(&dcpaud->link_lock); - dcpaud_link_teardown(dcpaud, dcpaud_connection_up(dcpaud)); + dcpaud_link_teardown(dcpaud); return ret; } @@ -808,9 +1175,13 @@ void dcpaud_release_link(struct platform_device *pdev, const u64 *gen) /* Only that instance's link: a later instance's PCM keeps its own. */ if (gen && !dcp_audiosrv_link_on(dcpaud->dcp_dev, *gen)) return; + dcpaud_silence_stop(dcpaud); + /* A stopped stream still playing its cleared buffer: sync_stop finishes it. */ + if (dcpaud->chan && READ_ONCE(dcpaud->dma_cleared)) + dmaengine_terminate_async(dcpaud->chan); if (dcpaud->chan) dmaengine_synchronize(dcpaud->chan); - dcpaud_link_teardown(dcpaud, true); + dcpaud_link_teardown(dcpaud); } static const struct snd_pcm_ops dcp_native_playback_ops = { @@ -901,8 +1272,10 @@ static void dcpaud_report_hotplug(struct dcp_audio *dcpaud, bool connected) } dcpaud->connected = connected; - if (connected) + if (connected) { dcpaud->connection_cookie++; + dcpaud->connected_at = ktime_get(); + } mutex_unlock(&dcpaud->data_lock); snd_jack_report(dcpaud->jack, connected ? SND_JACK_AVOUT : 0); @@ -1006,6 +1379,11 @@ void dcpaud_disconnect(struct platform_device *pdev) mutex_lock(&dcpaud->data_lock); dcpaud_report_hotplug(dcpaud, false); +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + /* The service may go without a release, and a held link with it. */ + if (dcpaud->native_link) + schedule_work(&dcpaud->stale_work); +#endif } static int dcpaud_comp_bind(struct device *dev, struct device *main, void *data) @@ -1108,6 +1486,30 @@ static void dcpaud_comp_unbind(struct device *dev, struct device *main, /* snd_card_free_when_closed() checks for NULL */ snd_card_free_when_closed(dcpaud->card); +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + if (dcpaud->native_link) { + bool held; + + cancel_work_sync(&dcpaud->stale_work); + /* A held link ends with the card; an open PCM's close finds it gone. */ + scoped_guard(mutex, &dcpaud->link_lock) { + held = dcpaud->silence_running; + if (held) + dcpaud_link_teardown(dcpaud); + else if (dcpaud->idle_set) + dcpaud_idle(dcpaud, false); + if (dcpaud->silence && dcpaud->idle_set) + dev_err(dcpaud->dev, "silence kept: its DMA channel did not stop\n"); + else if (dcpaud->silence) + dma_free_coherent(dcpaud->chan->device->dev, DCPAUD_SILENCE_BYTES, + dcpaud->silence, dcpaud->silence_addr); + dcpaud->silence = NULL; + } + if (held) + dcp_audiosrv_close(dcpaud->dcp_dev); + } +#endif + if (dcpaud->dma_link) device_link_del(dcpaud->dma_link); } @@ -1139,6 +1541,7 @@ static int dcpaud_probe(struct platform_device *pdev) mutex_init(&dcpaud->data_lock); #if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) mutex_init(&dcpaud->link_lock); + INIT_WORK(&dcpaud->stale_work, dcpaud_stale_work); #endif INIT_DELAYED_WORK(&dcpaud->void_work, dcpaud_void_work); platform_set_drvdata(pdev, dcpaud); diff --git a/drivers/gpu/drm/apple/audio.h b/drivers/gpu/drm/apple/audio.h index 88e9610363234..79bee44bcefbc 100644 --- a/drivers/gpu/drm/apple/audio.h +++ b/drivers/gpu/drm/apple/audio.h @@ -35,6 +35,8 @@ int dcp_audiosrv_link(struct device *dev, enum dcp_audio_link op, void dcpaud_release_link(struct platform_device *pdev, const u64 *gen); /* Whether the link prepared last is on instance @gen, still the open one */ bool dcp_audiosrv_link_on(struct device *dev, u64 gen); +/* The prepared link still belongs to the client of the published instance. */ +bool dcp_audiosrv_link_live(struct device *dev); #endif #endif /* __AUDIO_H__ */ diff --git a/drivers/gpu/drm/apple/av-v14-test.c b/drivers/gpu/drm/apple/av-v14-test.c index 362bb77db6465..433530763af7f 100644 --- a/drivers/gpu/drm/apple/av-v14-test.c +++ b/drivers/gpu/drm/apple/av-v14-test.c @@ -29,9 +29,11 @@ struct av_v14_test { bool successor_during_release; u32 link_status; u8 link_body[64]; + bool replace_during_prepare; }; static struct av_v14_test *av_v14_current; +static void av_v14_replace(struct av_v14_test *t); static int av_v14_call(struct apple_epic_service *srv, u32 command, const void *body, size_t body_len, size_t body_pad, void *reply, size_t reply_len) @@ -50,6 +52,10 @@ static int av_v14_call(struct apple_epic_service *srv, u32 command, const void * if (body_len == sizeof(t->link_body)) { memcpy(t->link_body, body, body_len); put_unaligned_le32(t->link_status, reply + 48); + if (command == t->asrv->cmds.prepare && t->replace_during_prepare) { + t->replace_during_prepare = false; + av_v14_replace(t); + } } return 0; } @@ -59,8 +65,6 @@ static void av_v14_card_disconnect(struct platform_device *pdev) av_v14_current->disconnects++; } -static void av_v14_replace(struct av_v14_test *t); - static void av_v14_release_link(struct platform_device *pdev, const u64 *gen) { struct av_v14_test *t = av_v14_current; @@ -304,7 +308,9 @@ static void av_v14_link_instance(struct kunit *test) old = t->asrv->gen; KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_PREPARE, &cookie), 0); KUNIT_EXPECT_TRUE(test, dcp_audiosrv_link_on(t->dcp->dev, old)); + KUNIT_EXPECT_TRUE(test, dcp_audiosrv_link_live(t->dcp->dev)); av_v14_replace(t); + KUNIT_EXPECT_FALSE(test, dcp_audiosrv_link_live(t->dcp->dev)); KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); KUNIT_ASSERT_EQ(test, t->ncalls, 3U); KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_STOP, &cookie), -ENODEV); @@ -315,10 +321,68 @@ static void av_v14_link_instance(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp_audiosrv_link_on(t->dcp->dev, t->asrv->gen)); KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_PREPARE, &cookie), 0); KUNIT_EXPECT_TRUE(test, dcp_audiosrv_link_on(t->dcp->dev, t->asrv->gen)); + KUNIT_EXPECT_TRUE(test, dcp_audiosrv_link_live(t->dcp->dev)); KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_COMPLETE, &cookie), 0); KUNIT_EXPECT_EQ(test, t->ncalls, 5U); } +static void av_v14_prepare_rejects_unpublished_open(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + struct dcp_sound_cookie cookie = {}; + + KUNIT_ASSERT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + av_v14_replace(t); + /* Same slot and pointer, but the PCM still owns the old instance. */ + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_PREPARE, &cookie), + -ENODEV); + KUNIT_EXPECT_EQ(test, t->ncalls, 1U); + KUNIT_EXPECT_EQ(test, t->asrv->link_gen, 0ULL); +} + +static void av_v14_prepare_replaced_during_reply(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + struct dcp_sound_cookie cookie = {}; + + KUNIT_ASSERT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + t->replace_during_prepare = true; + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_PREPARE, &cookie), + -ENODEV); + KUNIT_EXPECT_EQ(test, t->ncalls, 2U); + KUNIT_EXPECT_EQ(test, t->asrv->link_gen, 0ULL); +} + +static void av_v14_failed_prepare_does_not_own_link(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + struct dcp_sound_cookie cookie = {}; + + KUNIT_ASSERT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + t->link_status = 0xe00002ea; + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_PREPARE, &cookie), + -EIO); + KUNIT_EXPECT_EQ(test, t->ncalls, 2U); + KUNIT_EXPECT_EQ(test, t->asrv->link_gen, 0ULL); +} + +static void av_v14_failed_reprepare_drops_owner(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + struct dcp_sound_cookie cookie = {}; + u64 gen; + + KUNIT_ASSERT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + gen = t->asrv->gen; + KUNIT_ASSERT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_PREPARE, &cookie), 0); + KUNIT_ASSERT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_COMPLETE, &cookie), 0); + t->link_status = 0xe00002ea; + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_PREPARE, &cookie), + -EIO); + KUNIT_EXPECT_FALSE(test, dcp_audiosrv_link_on(t->dcp->dev, gen)); + KUNIT_EXPECT_EQ(test, t->asrv->link_gen, 0ULL); +} + /* A Terminate from an instance that is not the published one does nothing. */ static void av_v14_terminate_unpublished(struct kunit *test) { @@ -403,6 +467,10 @@ static struct kunit_case av_v14_cases[] = { KUNIT_CASE_PARAM(av_v14_successor_during_release, av_native_gen_params), KUNIT_CASE_PARAM(av_v14_terminate_after_successor, av_native_gen_params), KUNIT_CASE_PARAM(av_v14_link_instance, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_prepare_rejects_unpublished_open, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_prepare_replaced_during_reply, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_failed_prepare_does_not_own_link, av_native_gen_params), + KUNIT_CASE_PARAM(av_v14_failed_reprepare_drops_owner, av_native_gen_params), KUNIT_CASE_PARAM(av_v14_terminate_unpublished, av_native_gen_params), KUNIT_CASE_PARAM(av_native_link_payload_status, av_native_gen_params), KUNIT_CASE_PARAM(av_desktop_service_admission, av_native_gen_params), diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index afb918fb58f89..5e11c704a86ef 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -388,6 +388,29 @@ bool dcp_audiosrv_link_on(struct device *dev, u64 gen) return asrv->open_srv && asrv->open_gen == gen && asrv->link_gen == gen; } +/* open_lock pins the client; the endpoint can replace its published instance. */ +static bool av_audiosrv_open_published(struct audiosrv_data *asrv) +{ + lockdep_assert_held(&asrv->open_lock); + guard(rwsem_read)(&asrv->srv_rwsem); + return asrv->open_srv && asrv->srv == asrv->open_srv && + asrv->gen == asrv->open_gen; +} + +bool dcp_audiosrv_link_live(struct device *dev) +{ + struct apple_dcp *dcp; + struct audiosrv_data *asrv; + + KUNIT_STATIC_STUB_REDIRECT(dcp_audiosrv_link_live, dev); + dcp = dev_get_drvdata(dev); + asrv = dcp->audiosrv; + if (!asrv) + return false; + guard(mutex)(&asrv->open_lock); + return asrv->link_gen == asrv->open_gen && av_audiosrv_open_published(asrv); +} + /* * The link commands of the 14.7 external role's audio service, the native * audio driver's: prepare, start, stop and complete all take the selected @@ -433,6 +456,11 @@ int dcp_audiosrv_link(struct device *dev, enum dcp_audio_link op, scoped_guard(mutex, &asrv->open_lock) { if (!asrv->open_srv) return -ENODEV; + if (op == DCP_AUDIO_LINK_PREPARE) { + asrv->link_gen = 0; + if (!av_audiosrv_open_published(asrv)) + return -ENODEV; + } /* A link ends on the instance it was prepared on; one gone took it along. */ if (op != DCP_AUDIO_LINK_PREPARE && asrv->link_gen != asrv->open_gen) return -ENODEV; @@ -445,8 +473,11 @@ int dcp_audiosrv_link(struct device *dev, enum dcp_audio_link op, dev_err(dev, "audiosrv: link method %u answered %#x\n", command, status); return -EIO; } - if (op == DCP_AUDIO_LINK_PREPARE) + if (op == DCP_AUDIO_LINK_PREPARE) { + if (!av_audiosrv_open_published(asrv)) + return -ENODEV; asrv->link_gen = asrv->open_gen; + } } return 0; } @@ -1010,9 +1041,28 @@ int avep_init(struct apple_dcp *dcp) #endif } + /* + * Only a processor whose port@0 links an audio device has DP/HDMI + * audio to start. Look before the firmware: a processor without one + * (an internal panel's, or one whose firmware has no audio service + * here, such as H17P) stays as it is without hdmi_audio, quietly. + */ + endpoint = of_graph_get_endpoint_by_regs(dev->of_node, 0, 0); + if (endpoint) { + audio_node = of_graph_get_remote_port_parent(endpoint); + of_node_put(endpoint); + } + if (!audio_node || !of_device_is_available(audio_node)) { + of_node_put(audio_node); + dev_dbg(dev, "No audio support\n"); + return 0; + } + audiosrv_data = devm_kzalloc(dcp->dev, sizeof(*audiosrv_data), GFP_KERNEL); - if (!audiosrv_data) + if (!audiosrv_data) { + of_node_put(audio_node); return -ENOMEM; + } init_rwsem(&audiosrv_data->srv_rwsem); mutex_init(&audiosrv_data->plug_lock); @@ -1037,6 +1087,7 @@ int avep_init(struct apple_dcp *dcp) break; } #endif + of_node_put(audio_node); dev_err(dcp->dev, "Audio not supported for firmware\n"); return -ENODEV; } @@ -1049,17 +1100,6 @@ int avep_init(struct apple_dcp *dcp) audiosrv_data->works_live = true; dcp->audiosrv = audiosrv_data; - endpoint = of_graph_get_endpoint_by_regs(dev->of_node, 0, 0); - if (endpoint) { - audio_node = of_graph_get_remote_port_parent(endpoint); - of_node_put(endpoint); - } - if (!audio_node || !of_device_is_available(audio_node)) { - of_node_put(audio_node); - dev_info(dev, "No audio support\n"); - return av_audiosrv_register_cleanup(dcp, audiosrv_data, NULL); - } - audio_pdev = of_find_device_by_node(audio_node); of_node_put(audio_node); if (!audio_pdev) { diff --git a/drivers/gpu/drm/apple/connector.c b/drivers/gpu/drm/apple/connector.c index 282fc4d1295f5..2434eba0656ed 100644 --- a/drivers/gpu/drm/apple/connector.c +++ b/drivers/gpu/drm/apple/connector.c @@ -117,7 +117,14 @@ static void dcp_afk_debugfs_root(struct platform_device *pdev, int ep, struct de { #if IS_ENABLED(CONFIG_DRM_APPLE_DEBUG) struct dentry *entry = NULL; - struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct apple_dcp *dcp; + + /* Type-C connectors can be registered before a pipeline is routed. */ + if (!pdev) + return; + dcp = platform_get_drvdata(pdev); + if (!dcp) + return; switch (ep) { case AV_ENDPOINT: @@ -153,7 +160,7 @@ void apple_connector_debugfs_init(struct drm_connector *connector, struct dentry switch (connector->connector_type) { case DRM_MODE_CONNECTOR_DisplayPort: case DRM_MODE_CONNECTOR_HDMIA: - dcp_afk_debugfs_root(apple_con->dcp, AV_ENDPOINT, root); + dcp_afk_debugfs_root(READ_ONCE(apple_con->dcp), AV_ENDPOINT, root); break; default: break; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 06b3a4db8155e..f01925807d804 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -911,6 +911,7 @@ static inline void dcp_test_point(struct apple_dcp *dcp, enum dcp_test_point poi } #if IS_ENABLED(CONFIG_KUNIT) +void dcp_work_register_backlight(struct work_struct *work); void dcp_comp_unbind(struct device *dev, struct device *main, void *data); void dcp_platform_shutdown(struct platform_device *pdev); void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c3e395f2de9a1..5583688f1957d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3958,7 +3958,9 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) { - return (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) || + /* The 14.7 panel has its own known-level blank and restore hooks. */ + return (!iomfb_v14_7_panel_nits(dcp) && + dcp_backlight_active(dcp) && dcp_has_panel(dcp)) || (dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && dcp->connector_type == DRM_MODE_CONNECTOR_eDP); @@ -4266,7 +4268,7 @@ void dcp_backlight_takeover_registered(struct platform_device *pdev) schedule_work(&dcp->bl_register_wq); } -static void dcp_work_register_backlight(struct work_struct *work) +VISIBLE_IF_KUNIT void dcp_work_register_backlight(struct work_struct *work) { int ret; struct apple_dcp *dcp; @@ -4274,7 +4276,7 @@ static void dcp_work_register_backlight(struct work_struct *work) dcp = container_of(work, struct apple_dcp, bl_register_wq); mutex_lock(&dcp->bl_register_mutex); - if (dcp->brightness.bl_dev) + if (READ_ONCE(dcp->quiescing) || dcp->brightness.bl_dev) goto out_unlock; /* try to register backlight device, */ @@ -6376,6 +6378,7 @@ static int dcp_platform_suspend(struct device *dev) return -EBUSY; if (READ_ONCE(dcp->ext_dead)) return 0; + iomfb_v14_7_backlight_suspend(dcp); /* Serialize PM against explicit bring-up, including queued startup work. * Before a startup attempt the device remains suspendable. After any @@ -6459,6 +6462,7 @@ static int dcp_platform_resume(struct device *dev) dcp->link_recovery_tries = 0; dcp_release_after_sleep(dcp); + iomfb_v14_7_backlight_resume(dcp); dcp_enable_typec_work(dcp); /* The HDMI output's PHY and crossbar may have lost their setup in sleep. */ if (dcp->external_native) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 36f3ed466185d..223a462afced8 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -16,6 +16,7 @@ #include "connector.h" #include "dcp.h" #include "dcp-internal.h" +#include "iomfb_v14_7.h" #define MIN_BRIGHTNESS_PART1 2U #define MAX_BRIGHTNESS_PART1 99U @@ -93,14 +94,33 @@ static u32 brightness_part2[] = { #define DCP_BL_FADE_MS 250U #define DCP_BL_FADE_STEP_MS 16U +static bool dcp_backlight_presents_nits(struct apple_dcp *dcp) +{ + return (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) || + iomfb_v14_7_panel_nits(dcp); +} + bool dcp_backlight_active(struct apple_dcp *dcp) { - return ((dcp->fw_compat == DCP_FIRMWARE_H17P && - dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) || - iomfb_is_m4(dcp)) && + return (dcp_backlight_presents_nits(dcp) || iomfb_is_m4(dcp)) && READ_ONCE(dcp->backlight.state.ready); } +bool dcp_backlight_level_known(struct apple_dcp *dcp) +{ + unsigned long flags; + bool known; + + /* Only a panel running the policy initialised its lock and state. */ + if (!dcp_backlight_active(dcp)) + return false; + spin_lock_irqsave(&dcp->backlight.lock, flags); + known = dcp->backlight.state.ready && dcp->backlight.state.level_known; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return known; +} + static bool dcp_backlight_desktop(struct apple_dcp *dcp) { return dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP && @@ -228,11 +248,17 @@ static int dcp_get_brightness(struct backlight_device *bd) return -ENODEV; if (dcp_backlight_active(dcp) || dcp_backlight_desktop(dcp)) { + /* + * The 14.7 panel reports no actual level until a present + * carried one: before that it shows the boot loader's level, + * which Linux does not know. + */ spin_lock_irqsave(&dcp->backlight.lock, flags); if (dcp_backlight_desktop(dcp)) actual = dcp_bl_actual(&dcp->backlight.state); else - actual = dcp->backlight.state.actual; + actual = dcp_bl_reported_actual(&dcp->backlight.state, + iomfb_v14_7_panel_nits(dcp)); spin_unlock_irqrestore(&dcp->backlight.lock, flags); return actual; } @@ -249,7 +275,7 @@ int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, int ret; if ((dcp->fw_compat != DCP_FIRMWARE_H17P && - !iomfb_is_m4(dcp)) || + !iomfb_is_m4(dcp) && !iomfb_v14_7_panel_nits(dcp)) || !dcp_has_panel(dcp) || !dcp->crtc || !kick) return -EINVAL; @@ -269,7 +295,7 @@ int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, dcp_bl_jump(&dcp->backlight.state); dcp->backlight.kick = kick; } else if (!ret) { - dcp->backlight.state.fade = true; + dcp->backlight.state.fade = !iomfb_v14_7_panel_nits(dcp); dcp->backlight.kick = kick; } } @@ -655,8 +681,7 @@ int dcp_backlight_update(struct apple_dcp *dcp) dcp_backlight_kick(dcp); return 0; } - if (dcp->fw_compat == DCP_FIRMWARE_H17P && - dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + if (dcp_backlight_presents_nits(dcp)) return -ENODATA; /* @@ -744,6 +769,22 @@ static const struct backlight_ops dcp_backlight_ops = { .controls_device = dcp_backlight_controls_device, }; +static bool dcp_backlight_controls_device_v14(struct backlight_device *bd, + struct device *display_dev) +{ + struct apple_dcp *dcp = bl_get_data(bd); + + /* An fbdev unblank must not present the unrequested default level. */ + return dcp_backlight_level_known(dcp) && + dcp_backlight_controls_device(bd, display_dev); +} + +static const struct backlight_ops dcp_backlight_ops_v14 = { + .get_brightness = dcp_get_brightness, + .update_status = dcp_set_brightness, + .controls_device = dcp_backlight_controls_device_v14, +}; + int dcp_backlight_register(struct apple_dcp *dcp) { struct device *dev = dcp->dev; @@ -756,9 +797,7 @@ int dcp_backlight_register(struct apple_dcp *dcp) .scale = BACKLIGHT_SCALE_LINEAR, }; props.max_brightness = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); - if (((dcp->fw_compat == DCP_FIRMWARE_H17P && - dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) || - iomfb_is_m4(dcp)) && + if ((dcp_backlight_presents_nits(dcp) || iomfb_is_m4(dcp)) && !dcp_backlight_active(dcp)) return -ENODATA; if (dcp_backlight_active(dcp)) { @@ -804,7 +843,9 @@ int dcp_backlight_register(struct apple_dcp *dcp) } bl_dev = devm_backlight_device_register(dev, "apple-panel-bl", dev, dcp, - &dcp_backlight_ops, &props); + iomfb_v14_7_panel_nits(dcp) ? + &dcp_backlight_ops_v14 : &dcp_backlight_ops, + &props); if (IS_ERR(bl_dev)) return PTR_ERR(bl_dev); diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 0540c87d28108..cc2d321fb1396 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -309,6 +309,13 @@ static inline bool dcp_bl_complete(struct dcp_backlight_state *state, return true; } +static inline int dcp_bl_reported_actual(const struct dcp_backlight_state *state, bool strict) +{ + if (strict && !state->level_known) + return -ENODATA; + return state->actual; +} + /* Three deferred retries per user request; a permanent rejection stays dirty. */ static inline unsigned int dcp_bl_retry_delay(const struct dcp_backlight_state *state) { @@ -376,6 +383,7 @@ void dcp_backlight_desktop_finish(struct apple_dcp *dcp, u32 id, * Configure before registration and keep the callback valid until teardown. */ bool dcp_backlight_active(struct apple_dcp *dcp); +bool dcp_backlight_level_known(struct apple_dcp *dcp); int dcp_backlight_takeover(struct apple_dcp *dcp, u32 millinits); int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, bool inherited_valid, u32 inherited, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index e72c6479c1350..f6a2df3ec55a4 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -2140,6 +2140,10 @@ int dcp_get_modes(struct drm_connector *connector) if (apple_connector->drm_edid) dcp_retry_placeholder_edid(dcp, apple_connector->drm_edid); + /* EDID updates clear display_info even for panels without an EDID. */ + if (published && dcp_has_panel(dcp)) + dcp_update_connector_dimensions(dcp, connector); + /* * An internal panel has no EDID, so nothing fills in the refresh range * that userspace needs before it will drive VRR. Supply the range DCP diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 8e82e74fc1114..ba283faf6c1a3 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -512,6 +512,68 @@ DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, DCP_THUNK_INOUT(dcp_enable_disable_video_power_savings, dcpep_enable_disable_video_power_savings, u32, int); +#if !DCP_FW_DESKTOP_26(DCP_FW_VER) +static void dcp_external_hint_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + if (out) { + WRITE_ONCE(wait->status, *(u32 *)out); + WRITE_ONCE(wait->replied, true); + } + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); +} + +/* + * After every mode set on an external display macOS sends IOMFB + * parameters 3 and 6 (16.16): 3 at 1.0, and 6 at 1.0 for an SDR colour + * mode or 2.0 for a PQ one. Send the same, after the mode set. Only the + * H17P external path does so for now: the desktop 26.x external processors + * have not been checked with an HDR display yet. + */ +static void dcp_external_hdr_hints(struct apple_dcp *dcp, bool pq) +{ + const u32 hints[][2] = { + { 3, 0x10000 }, + { 6, pq ? 0x20000 : 0x10000 }, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(hints); i++) { + struct dcp_set_parameter_dcp param = { + .param = hints[i][0], + .value = { hints[i][1] }, + .count = 1, + }; + struct dcp_wait_cookie *cookie; + unsigned long left; + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + kref_get(&cookie->refcount); + dcp_set_parameter_dcp(dcp, false, ¶m, dcp_external_hint_done, + cookie); + left = wait_for_completion_timeout(&cookie->done, + msecs_to_jiffies(500)); + if (!left || !READ_ONCE(cookie->replied) || + READ_ONCE(cookie->status)) + dev_warn(dcp->dev, + "IOMFB parameter %u = %#x not taken (%s %u)\n", + param.param, param.value[0], + left ? "status" : "timeout", + READ_ONCE(cookie->status)); + else + dev_dbg(dcp->dev, "IOMFB parameter %u = %#x\n", + param.param, param.value[0]); + kref_put(&cookie->refcount, release_wait_cookie); + } +} +#endif + DCP_THUNK_OUT(dcp_is_main_display, dcpep_is_main_display, u32); /* DCP callback handlers */ @@ -3544,6 +3606,8 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, } kref_put(&cookie->refcount, release_wait_cookie); dcp->vrr_enabled = mode->vrr && crtc_state->vrr_enabled; + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) + dcp_external_hdr_hints(dcp, dcp->output_pq); return 0; } diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 474841ff8d0ae..6940145afcc47 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -40,6 +40,7 @@ #include "dcp-lifecycle.h" #include "iomfb_internal.h" #include "iomfb_v14_7.h" +#include "iomfb_v14_7_backlight.h" #include "iomfb_v14_7_board.h" #include "iomfb_v14_7_link.h" #include "iomfb_v14_7_swap.h" @@ -102,6 +103,7 @@ static const struct dcp_v14_board dcp_v14_board_t6030 = { .firmware_uuid = DCP_V14_FIRMWARE_UUID, .handoff = "apple,t6030-handoff", .promotion = true, + .panel_nits = true, }; /* @@ -118,6 +120,7 @@ static const struct dcp_v14_board dcp_v14_board_j613 = { .panel_width = 2560, .panel_height = 1664, .promotion = false, + .panel_nits = true, .ctm_set = A(421), .ctm_get = A(420), }; @@ -159,6 +162,13 @@ static const struct dcp_v14_board dcp_v14_board_j516c = { .panel_height = 2234, .promotion = true, .log_bringup = true, + .swap_timestamps = true, + /* + * The J516S panel and this image's swap carry the level block the + * T6030 record uses (present byte +0x354, binary64 nits +0x35c, 500 nits + * at most), as the e272b67 path on this J516C presented it. + */ + .panel_nits = true, }; /* @@ -287,6 +297,19 @@ struct apple_dcp_v14 { struct drm_framebuffer *active_fb; u64 swaps; u64 swap_ns_max; + + /* Immutable probe admission: only the J514S/J516S/J516C internal panel. */ + bool panel_nits; + /* Copy of the last accepted surface; active_fb pins its DMA under lock. */ + u8 shown_surface[DCP_V14_SURFACE_SIZE]; + u64 shown_iova; + u32 shown_width, shown_height, shown_dst_y; + struct delayed_work backlight_work; + /* Serializes gate transitions with queue admission. */ + spinlock_t bl_gate_lock; + enum dcp_v14_bl_gate bl_gate; + u32 nits_sent; + u64 nits_presents, brightness_presents, nits_refused; }; /* @@ -299,6 +322,135 @@ static bool dcp_v14_panel_session; static int dcp_v14_callback(void *cookie, u32 tag, const void *input, u32 in_size, void *output, u32 out_size); +static void dcp_v14_backlight_work(struct work_struct *work); + +/* The internal panel of a board whose swap level block was validated (T6030). */ +static bool dcp_v14_board_nits(const struct apple_dcp_v14 *v14) +{ + return !v14->external && v14->panel_nits && v14->board && v14->board->panel_nits; +} + +bool iomfb_v14_7_panel_nits(struct apple_dcp *dcp) +{ + return dcp_is_v14_7_panel(dcp) && dcp->v14 && dcp_v14_board_nits(dcp->v14); +} + +static enum dcp_v14_bl_gate dcp_v14_bl_gate_get(struct apple_dcp_v14 *v14) +{ + enum dcp_v14_bl_gate gate; + unsigned long flags; + + spin_lock_irqsave(&v14->bl_gate_lock, flags); + gate = v14->bl_gate; + spin_unlock_irqrestore(&v14->bl_gate_lock, flags); + return gate; +} + +/* Queues a brightness-only present, unless the gate is shut. */ +static void dcp_v14_bl_queue(struct apple_dcp_v14 *v14, unsigned long delay) +{ + unsigned long flags; + + spin_lock_irqsave(&v14->bl_gate_lock, flags); + if (dcp_v14_bl_may_queue(v14->bl_gate)) + mod_delayed_work(system_dfl_wq, &v14->backlight_work, delay); + spin_unlock_irqrestore(&v14->bl_gate_lock, flags); +} + +/* + * Shuts the gate, until resume or (@for_good, unbind and remove) for good, + * then drains the work. The RPC lock waits for a running present; the + * gate lock excludes late queueing. Cancellation runs outside the RPC lock. + */ +static void dcp_v14_bl_shut(struct apple_dcp_v14 *v14, bool for_good) +{ + unsigned long flags; + + mutex_lock(&v14->lock); + spin_lock_irqsave(&v14->bl_gate_lock, flags); + v14->bl_gate = for_good ? DCP_V14_BL_CLOSED : dcp_v14_bl_gate_suspend(v14->bl_gate); + spin_unlock_irqrestore(&v14->bl_gate_lock, flags); + mutex_unlock(&v14->lock); + cancel_delayed_work_sync(&v14->backlight_work); +} + +/* The class device may be platform devres, outside the component bind group. */ +static void dcp_v14_stop_backlight(struct apple_dcp_v14 *v14) +{ + struct apple_dcp *dcp = READ_ONCE(v14->dcp); + + if (dcp) + WRITE_ONCE(dcp->quiescing, true); + dcp_v14_bl_shut(v14, true); + if (!dcp) + return; + dcp_backlight_stop(dcp); + cancel_work_sync(&dcp->bl_register_wq); + cancel_work_sync(&dcp->bl_update_wq); + /* Never hold this mutex while waiting for its registration worker. */ + guard(mutex)(&dcp->bl_register_mutex); + if (dcp->brightness.bl_dev) { + devm_backlight_device_unregister(dcp->dev, dcp->brightness.bl_dev); + dcp->brightness.bl_dev = NULL; + } +} + +/* The policy's kick (dcp_backlight.c): only queues; the RPC runs in the work. */ +static void dcp_v14_backlight_kick(struct apple_dcp *dcp) +{ + if (iomfb_v14_7_panel_nits(dcp) && !READ_ONCE(dcp->quiescing)) + dcp_v14_bl_queue(dcp->v14, 0); +} + +/* + * The panel's level policy, set up once the panel runs: at most 500 nits + * (less if the panel's DT record says so), and no fades. No boot loader + * level is known, so none is presented until userspace sets one; until + * then the backlight reports DCP_V14_NITS_DEFAULT as its requested level + * and no actual level (dcp_backlight.c). dcp_backlight_configure() + * registers the device from bl_register_wq. + */ +static void dcp_v14_configure_backlight(struct apple_dcp *dcp) +{ + struct apple_dcp_v14 *v14 = dcp->v14; + u32 maximum = DCP_V14_NITS_MAX; + int ret; + + if (!iomfb_v14_7_panel_nits(dcp) || READ_ONCE(dcp->quiescing) || + !v14->started || dcp_backlight_active(dcp) || + !dcp_has_panel(dcp) || !dcp->crtc) + return; + /* A described panel maximum may only lower the wire bound. */ + if (dcp->brightness.maximum) + maximum = min(maximum, dcp->brightness.maximum); + ret = dcp_backlight_configure(dcp, maximum, false, 0, true, + min_t(u32, maximum, DCP_V14_NITS_DEFAULT), + dcp_v14_backlight_kick); + if (ret && ret != -EBUSY) + dev_err_once(dcp->dev, "%s backlight unavailable: %d\n", v14->board->name, ret); +} + +void iomfb_v14_7_backlight_suspend(struct apple_dcp *dcp) +{ + if (iomfb_v14_7_panel_nits(dcp)) + dcp_v14_bl_shut(dcp->v14, false); +} + +/* Pending levels ride the next normal CRTC restore present after admission opens. */ +void iomfb_v14_7_backlight_resume(struct apple_dcp *dcp) +{ + struct apple_dcp_v14 *v14 = dcp->v14; + unsigned long flags; + + if (!iomfb_v14_7_panel_nits(dcp)) + return; + mutex_lock(&v14->lock); + spin_lock_irqsave(&v14->bl_gate_lock, flags); + v14->bl_gate = dcp_v14_bl_gate_resume(v14->bl_gate); + spin_unlock_irqrestore(&v14->bl_gate_lock, flags); + mutex_unlock(&v14->lock); +} + /* * An external session stopped on a link error, not a crash: its display is * gone until reboot. Report that as the crash path does, from process @@ -336,6 +488,8 @@ static int dcp_v14_call_guarded(struct apple_dcp_v14 *v14, u32 tag, const void * bool vetoed; int ret; + KUNIT_STATIC_STUB_REDIRECT(dcp_v14_call_guarded, v14, tag, in, in_size, + out, out_size, completion, admit, admit_cookie); if (v14->failed) return -EIO; dev_dbg(v14->dev, "call %#x in %u out %u\n", tag, in_size, out_size); @@ -1455,6 +1609,8 @@ static void dcp_v14_release(void *data) { struct apple_dcp_v14 *v14 = data; + if (dcp_v14_board_nits(v14)) + dcp_v14_stop_backlight(v14); if (!v14->rtk) kfree(v14); } @@ -1536,11 +1692,24 @@ const struct dcp_v14_board *iomfb_v14_7_board(struct device *dev) return NULL; } +static bool dcp_v14_panel_nits_admit(const struct dcp_v14_board *board, + dcp_v14_compat_fn machine_is, const void *ctx) +{ + return board && board->panel_nits && + (machine_is(ctx, "apple,j514s") || machine_is(ctx, "apple,j516s") || + machine_is(ctx, "apple,j516c") || machine_is(ctx, "apple,j613")); +} + const char *iomfb_v14_7_board_name(const struct dcp_v14_board *board) { return board->name; } +bool dcp_v14_board_stamps_swaps(const struct dcp_v14_board *board, bool external) +{ + return board && !external && board->swap_timestamps; +} + int iomfb_v14_7_probe(struct apple_dcp *dcp) { struct device *dev = dcp->dev; @@ -1589,10 +1758,13 @@ int iomfb_v14_7_probe(struct apple_dcp *dcp) return -ENOMEM; v14->dev = dev; v14->board = board; + v14->panel_nits = dcp_v14_panel_nits_admit(board, dcp_v14_machine_is, np); v14->dcp = dcp; mutex_init(&v14->lock); init_waitqueue_head(&v14->described_wait); INIT_WORK(&v14->idle_work, dcp_v14_idle); + INIT_DELAYED_WORK(&v14->backlight_work, dcp_v14_backlight_work); + spin_lock_init(&v14->bl_gate_lock); ret = devm_add_action_or_reset(dev, dcp_v14_release, v14); if (ret) return ret; @@ -2045,6 +2217,8 @@ static int dcpext_boot(struct apple_dcp *dcp) mutex_init(&v14->lock); init_waitqueue_head(&v14->described_wait); INIT_WORK(&v14->idle_work, dcp_v14_idle); + INIT_DELAYED_WORK(&v14->backlight_work, dcp_v14_backlight_work); + spin_lock_init(&v14->bl_gate_lock); INIT_WORK(&v14->stopped_work, dcp_v14_stopped_work); dcp->v14 = v14; dcp_v14_link_init(&v14->link, dev, NULL); @@ -2275,6 +2449,8 @@ void iomfb_v14_7_unbind(struct apple_dcp *dcp) flush_work(&v14->idle_work); cancel_work_sync(&v14->stopped_work); } else { + if (dcp_v14_board_nits(v14)) + dcp_v14_stop_backlight(v14); WRITE_ONCE(v14->dcp, NULL); } if (v14->rtk) @@ -2314,6 +2490,8 @@ void iomfb_v14_7_remove(struct apple_dcp *dcp) * Callbacks that already loaded the old pointer finish before the * flush returns; later ones see NULL. */ + if (dcp_v14_board_nits(v14)) + dcp_v14_stop_backlight(v14); WRITE_ONCE(v14->dcp, NULL); if (v14->rtk) apple_rtkit_flush_rx(v14->rtk); @@ -2344,6 +2522,9 @@ static int dcp_v14_status_show(struct seq_file *m, void *unused) v14->stride, v14->clock_rate); seq_printf(m, "backlight factor %#x updates %llu\n", v14->backlight_factor, v14->backlight_factor_updates); + seq_printf(m, "backlight nits_enabled %d sent %u presents %llu brightness_only %llu refused %llu\n", + dcp_v14_board_nits(v14), v14->nits_sent, v14->nits_presents, + v14->brightness_presents, v14->nits_refused); seq_printf(m, "ctm valid %d disabled %d calls %llu setter %#x getter %#x\n", v14->ctm_valid, v14->ctm_disabled, v14->ctm_calls, v14->ctm_status, v14->ctm_get_status); @@ -2466,6 +2647,7 @@ int iomfb_v14_7_start(struct apple_dcp *dcp) dcp->mode_state.valid = true; dcp->active = true; complete(&dcp->start_done); + dcp_v14_configure_backlight(dcp); mutex_lock(&dcp->modes_lock); selected = dcp->modes[0].mode; @@ -2571,20 +2753,64 @@ static bool dcp_v14_swap_admit(void *cookie) return connector && READ_ONCE(connector->connected); } +static bool dcp_v14_swap_nits(struct apple_dcp_v14 *v14, u8 *swap, + struct dcp_backlight_present *present) +{ + struct apple_dcp *dcp = READ_ONCE(v14->dcp); + + lockdep_assert_held(&v14->lock); + /* The policy exists only on the T6030 panel: nothing else is asked. */ + if (!dcp || !dcp_v14_board_nits(v14) || !dcp_backlight_active(dcp)) + return false; + if (!dcp_v14_swap_takes_nits(true, v14->external, dcp_is_typec_output(dcp), true)) + return false; + /* -EALREADY: no change; -ENODATA/-EBUSY: nothing to present now. */ + if (dcp_backlight_prepare(dcp, true, present)) + return false; + /* The policy's maximum is at most the wire bound; never above it. */ + if (WARN_ON_ONCE(dcp_v14_encode_nits(swap, true, present->nits))) { + dcp_backlight_complete(dcp, present->sequence, false); + v14->nits_refused++; + return false; + } + return true; +} + +static void dcp_v14_swap_nits_done(struct apple_dcp_v14 *v14, + const struct dcp_backlight_present *present, bool accepted) +{ + struct apple_dcp *dcp = READ_ONCE(v14->dcp); + + if (!dcp) + return; + if (!dcp_backlight_complete(dcp, present->sequence, accepted)) + return; + if (accepted) { + v14->nits_sent = present->nits; + v14->nits_presents++; + return; + } + v14->nits_refused++; + /* The internal swap failure is final: swap_locked latches failed. */ +} + /* * Swap start, then the swap; returns once the firmware completed that swap. * @ctm_state, if set, carries the colour matrix to send first. */ -static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, - u32 width, u32 height, u32 dst_y, - const struct drm_crtc_state *ctm_state) +static int dcp_v14_swap_locked(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, + u32 width, u32 height, u32 dst_y, + const struct drm_crtc_state *ctm_state) { struct dcp_v14_swap_admission admission = { .v14 = v14 }; __le32 start[4] = {}, started[2] = {}, result[3]; + struct dcp_backlight_present present; + bool nits; u8 *swap; u32 id; int ret; + lockdep_assert_held(&v14->lock); if (surface && (!iova || !width || width > v14->panel_width || !height || dst_y > v14->panel_height || height > v14->panel_height - dst_y)) return -EINVAL; @@ -2592,7 +2818,6 @@ static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, if (!swap) return -ENOMEM; - mutex_lock(&v14->lock); admission.generation = v14->description_generation; /* * A colour matrix the firmware refuses costs night light, not the @@ -2609,11 +2834,16 @@ static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, if (!ret) { dcp_v14_encode_swap(swap, id, DCP_V14_BLACK, surface, iova, width, height, dst_y, DCP_V14_LAYER); + if (dcp_v14_board_stamps_swaps(v14->board, v14->external)) + dcp_v14_swap_timestamps(swap); + nits = dcp_v14_swap_nits(v14, swap, &present); ret = dcp_v14_call_guarded(v14, A(407), swap, DCP_V14_SWAP_SIZE, result, sizeof(result), id, dcp_v14_swap_admit, &admission); /* The swap status sits at byte 5 of the reply. */ if (!ret && get_unaligned_le32((u8 *)result + 5)) ret = -EIO; + if (nits) + dcp_v14_swap_nits_done(v14, &present, !ret); } /* * An external display can go away under a swap: the firmware then @@ -2622,26 +2852,50 @@ static int dcp_v14_swap(struct apple_dcp_v14 *v14, const u8 *surface, u64 iova, */ if (ret && !v14->external) v14->failed = true; - mutex_unlock(&v14->lock); kfree(swap); return ret; } +/* Publish a successful swap while the RPC lock still pins both surfaces. */ +static struct drm_framebuffer *dcp_v14_publish_surface(struct apple_dcp_v14 *v14, + struct drm_framebuffer *fb, + const u8 *surface, u64 iova, + u32 width, u32 height, u32 dst_y) +{ + struct drm_framebuffer *old; + + KUNIT_STATIC_STUB_REDIRECT(dcp_v14_publish_surface, v14, fb, surface, iova, + width, height, dst_y); + lockdep_assert_held(&v14->lock); + old = v14->active_fb; + v14->active_fb = fb; + if (fb) { + memcpy(v14->shown_surface, surface, sizeof(v14->shown_surface)); + v14->shown_iova = iova; + v14->shown_width = width; + v14->shown_height = height; + v14->shown_dst_y = dst_y; + } + return old; +} + /* * Shows the top-left @width x @height of @fb (or only the black background), * after the colour matrix of @ctm_state if that is set, * and keeps @fb until the next swap. */ -static bool dcp_v14_present(struct apple_dcp_v14 *v14, struct drm_framebuffer *fb, - u32 width, u32 height, u32 dst_y, - const struct drm_crtc_state *ctm_state) +static bool dcp_v14_present_locked(struct apple_dcp_v14 *v14, struct drm_framebuffer *fb, + u32 width, u32 height, u32 dst_y, + const struct drm_crtc_state *ctm_state, + struct drm_framebuffer **retired) { u8 surface[DCP_V14_SURFACE_SIZE]; struct drm_framebuffer *old; u64 iova = 0, start, elapsed, swaps; int ret; + lockdep_assert_held(&v14->lock); if (READ_ONCE(v14->failed)) return false; if (fb) { @@ -2651,13 +2905,12 @@ static bool dcp_v14_present(struct apple_dcp_v14 *v14, struct drm_framebuffer *f iova = drm_fb_dma_get_gem_obj(fb, 0)->dma_addr; } start = ktime_get_ns(); - ret = dcp_v14_swap(v14, fb ? surface : NULL, iova, fb ? width : 0, - fb ? height : 0, dst_y, ctm_state); + ret = dcp_v14_swap_locked(v14, fb ? surface : NULL, iova, fb ? width : 0, + fb ? height : 0, dst_y, ctm_state); elapsed = ktime_get_ns() - start; if (ret && v14->external && !READ_ONCE(v14->failed)) { /* Refused before it was taken: the old framebuffer stays on screen. */ - if (fb) - drm_framebuffer_put(fb); + *retired = fb; WRITE_ONCE(v14->mode_swaps, 0); dev_warn_ratelimited(v14->dev, "display refused a swap: %d\n", ret); return false; @@ -2667,22 +2920,91 @@ static bool dcp_v14_present(struct apple_dcp_v14 *v14, struct drm_framebuffer *f dev_err(v14->dev, "native DCP flip failed %d; buffers pinned until reboot\n", ret); return false; } - mutex_lock(&v14->lock); - old = v14->active_fb; - v14->active_fb = fb; + old = dcp_v14_publish_surface(v14, fb, surface, iova, width, height, dst_y); swaps = ++v14->swaps; if (fb) v14->mode_swaps++; v14->swap_ns_max = max(v14->swap_ns_max, elapsed); - mutex_unlock(&v14->lock); - if (old) - drm_framebuffer_put(old); + *retired = old; if (swaps <= 3) dev_info(v14->dev, "swap %llu complete: surface %d, imported %d, %llu us\n", swaps, !!fb, fb && fb->obj[0]->import_attach, elapsed / NSEC_PER_USEC); return true; } +static bool dcp_v14_present(struct apple_dcp_v14 *v14, struct drm_framebuffer *fb, + u32 width, u32 height, u32 dst_y, + const struct drm_crtc_state *ctm_state) +{ + struct drm_framebuffer *retired = NULL; + bool shown; + + mutex_lock(&v14->lock); + shown = dcp_v14_present_locked(v14, fb, width, height, dst_y, ctm_state, &retired); + mutex_unlock(&v14->lock); + if (retired) + drm_framebuffer_put(retired); + return shown; +} + +static void dcp_v14_backlight_work(struct work_struct *work) +{ + struct apple_dcp_v14 *v14 = container_of(to_delayed_work(work), struct apple_dcp_v14, + backlight_work); + struct dcp_v14_bl_facts facts; + struct drm_framebuffer *fb; + struct apple_dcp *dcp; + int ret; + + guard(mutex)(&v14->lock); + dcp = READ_ONCE(v14->dcp); + if (!dcp) + return; + fb = v14->active_fb; + facts = (struct dcp_v14_bl_facts) { + .gate = dcp_v14_bl_gate_get(v14), + .panel_nits = dcp_v14_board_nits(v14), + .external = v14->external, + .typec = dcp_is_typec_output(dcp), + .started = v14->started, + .failed = READ_ONCE(v14->failed), + .crashed = READ_ONCE(dcp->crashed), + .quiescing = READ_ONCE(dcp->quiescing), + .pending = dcp_v14_board_nits(v14) && dcp_backlight_active(dcp) && + dcp_backlight_pending(dcp), + .frame = !!fb, + .mode_valid = READ_ONCE(dcp->mode_state.valid), + }; + if (dcp_v14_bl_plan(&facts) != DCP_V14_BL_REPRESENT) + return; + ret = dcp_v14_swap_locked(v14, v14->shown_surface, v14->shown_iova, + v14->shown_width, v14->shown_height, v14->shown_dst_y, NULL); + if (!ret) + v14->brightness_presents++; + else if (ret != -EAGAIN) + dev_err(v14->dev, "brightness present failed: %d\n", ret); +} + +static bool dcp_v14_bl_dpms_applies(struct apple_dcp_v14 *v14, struct apple_dcp *dcp) +{ + if (!dcp_v14_board_nits(v14) || !dcp_backlight_active(dcp)) + return false; + return dcp_v14_bl_follow_dpms(true, true, dcp_backlight_level_known(dcp)); +} + +static void dcp_v14_blank(struct apple_dcp_v14 *v14, struct apple_dcp *dcp) +{ + struct drm_framebuffer *retired = NULL; + + mutex_lock(&v14->lock); + if (dcp_v14_bl_dpms_applies(v14, dcp)) + dcp_backlight_dpms(dcp, false); + dcp_v14_present_locked(v14, NULL, 0, 0, 0, NULL, &retired); + mutex_unlock(&v14->lock); + if (retired) + drm_framebuffer_put(retired); +} + /* Releases helper waiters without reporting a completed flip. */ static void dcp_v14_cancel_event(struct apple_dcp *dcp) { @@ -2947,6 +3269,17 @@ static int dcp_v14_external_modeset(struct apple_dcp *dcp, struct drm_crtc_state ret = dcp_v14_set_mode(v14, mode.color_mode_id, mode.timing_mode_id); if (!ret) ret = dcp_v14_set_power(v14, true); + /* + * The mode was set with the pipe off. A mode that needs DSC is then + * reset again ("pipe not enabled"), and the power-on only configures + * DSC: the firmware starts the timings in a mode set of its own, after + * the hotplug notification it sends itself, and drops a swap that + * reaches it before then without completing it, so A407 times out. + * Setting the mode once more with the pipe on starts the timings + * before the first swap. + */ + if (!ret) + ret = dcp_v14_set_mode(v14, mode.color_mode_id, mode.timing_mode_id); if (!ret) { v14->panel_width = mode.mode.hdisplay; v14->panel_height = mode.mode.vdisplay; @@ -3015,6 +3348,8 @@ void iomfb_v14_7_flush(struct apple_dcp *dcp, struct drm_crtc *crtc, dcp_drm_crtc_vblank(dcp->crtc); return; } + if (!dcp->external) + dcp_v14_configure_backlight(dcp); dcp->swap_start = ktime_get(); if (dcp->v14 && dcp_v14_present(dcp->v14, fb, fb ? (dcp->external ? p->src_w >> 16 : fb->width) : 0, @@ -3027,7 +3362,9 @@ void iomfb_v14_7_flush(struct apple_dcp *dcp, struct drm_crtc *crtc, void iomfb_v14_7_poweron(struct apple_dcp *dcp) { - /* The panel stays on; the next swap shows the plane again. */ + /* The next present restores a level only after Linux has set one. */ + if (dcp->v14 && dcp_v14_bl_dpms_applies(dcp->v14, dcp)) + dcp_backlight_dpms(dcp, true); } void iomfb_v14_7_poweroff(struct apple_dcp *dcp) @@ -3050,9 +3387,12 @@ void iomfb_v14_7_poweroff(struct apple_dcp *dcp) } /* Blank to black; the panel and the DCP stay powered. */ if (v14 && v14->started) - dcp_v14_present(v14, NULL, 0, 0, 0, NULL); + dcp_v14_blank(v14, dcp); } #if IS_ENABLED(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) #include "iomfb_v14_7_external_audio_test.c" +#include "iomfb_v14_7_external_modeset_test.c" +#include "iomfb_v14_7_backlight_test.c" +#include "iomfb_v14_7_backlight_lifetime_test.c" #endif diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index f9bef6ed0f109..8aa4eeda4be7a 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -31,6 +31,17 @@ const char *iomfb_v14_7_board_name(const struct dcp_v14_board *board); /* Platform probe: firmware identity, boot loader handoff, PMP acknowledgment. */ int iomfb_v14_7_probe(struct apple_dcp *dcp); +/* + * Whether @dcp is a 14.x internal panel whose swaps carry its level (the + * T6030 board record): its backlight is the nits policy in dcp_backlight.c. + */ +bool iomfb_v14_7_panel_nits(struct apple_dcp *dcp); +/* + * System sleep: no brightness-only present may be queued or run from + * _suspend() until _resume(). + */ +void iomfb_v14_7_backlight_suspend(struct apple_dcp *dcp); +void iomfb_v14_7_backlight_resume(struct apple_dcp *dcp); /* Component bind: the running DCP's RTKit session. */ int iomfb_v14_7_bind(struct apple_dcp *dcp); int iomfb_v14_7_external_start(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_backlight.h b/drivers/gpu/drm/apple/iomfb_v14_7_backlight.h new file mode 100644 index 0000000000000..f2b71653664eb --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v14_7_backlight.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +#ifndef __APPLE_IOMFB_V14_7_BACKLIGHT_H__ +#define __APPLE_IOMFB_V14_7_BACKLIGHT_H__ + +#include + +/* + * Decisions of the 14.7 panel's level path (iomfb_v14_7.c), kept free of + * driver state so that KUnit runs the same code. + */ + +/* + * Whether brightness-only presents may be queued and run. Set under the + * gate lock before the work is cancelled: system sleep suspends it until + * resume, unbind and remove close it for good. + */ +enum dcp_v14_bl_gate { + DCP_V14_BL_OPEN, + DCP_V14_BL_SUSPENDED, + DCP_V14_BL_CLOSED, +}; + +/* Whether a brightness-only present may be queued, or run once queued. */ +static inline bool dcp_v14_bl_may_queue(enum dcp_v14_bl_gate gate) +{ + return gate == DCP_V14_BL_OPEN; +} + +static inline enum dcp_v14_bl_gate dcp_v14_bl_gate_suspend(enum dcp_v14_bl_gate gate) +{ + return gate == DCP_V14_BL_OPEN ? DCP_V14_BL_SUSPENDED : gate; +} + +static inline enum dcp_v14_bl_gate dcp_v14_bl_gate_resume(enum dcp_v14_bl_gate gate) +{ + return gate == DCP_V14_BL_SUSPENDED ? DCP_V14_BL_OPEN : gate; +} + +/* + * Whether a swap carries the level the backlight policy reserves: only the + * internal panel of a board whose level block was validated (T6030), only + * while it shows its own panel (not a J613 internal pipe lent to a Type-C + * display), and only with an active policy. + */ +static inline bool dcp_v14_swap_takes_nits(bool panel_nits, bool external, bool typec, + bool policy_active) +{ + return panel_nits && !external && !typec && policy_active; +} + +/* + * Blanking follows DPMS only once Linux has set a level: before that the + * panel shows the boot loader's level, and a DPMS cycle must not replace it + * with the reported default. + */ +static inline bool dcp_v14_bl_follow_dpms(bool panel_nits, bool policy_active, + bool level_known) +{ + return panel_nits && policy_active && level_known; +} + +struct dcp_v14_bl_facts { + enum dcp_v14_bl_gate gate; + bool panel_nits; + bool external; + bool typec; + bool started; + bool failed; + bool crashed; + bool quiescing; + /* A level is reserved-able: dirty and not in flight. */ + bool pending; + /* The last present showed a frame (active_fb), not the background. */ + bool frame; + bool mode_valid; +}; + +enum dcp_v14_bl_action { + /* Nothing to do, or not allowed to: no RPC. */ + DCP_V14_BL_SKIP, + /* Blanked or no frame yet: the next atomic present carries the level. */ + DCP_V14_BL_WAIT, + /* Present the frame shown last again, with the level. */ + DCP_V14_BL_REPRESENT, +}; + +static inline enum dcp_v14_bl_action dcp_v14_bl_plan(const struct dcp_v14_bl_facts *f) +{ + if (!dcp_v14_bl_may_queue(f->gate) || !f->panel_nits || f->external || f->typec || + !f->started || f->failed || f->crashed || f->quiescing || !f->pending) + return DCP_V14_BL_SKIP; + if (!f->frame || !f->mode_valid) + return DCP_V14_BL_WAIT; + return DCP_V14_BL_REPRESENT; +} + +#endif /* __APPLE_IOMFB_V14_7_BACKLIGHT_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_backlight_lifetime_test.c b/drivers/gpu/drm/apple/iomfb_v14_7_backlight_lifetime_test.c new file mode 100644 index 0000000000000..29acb9d5fe9e9 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v14_7_backlight_lifetime_test.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Port-specific controls: run the real swap/present, worker, DPMS dispatch + * and class teardown. Only the RPC transport is replaced. Threads and a + * workqueue context bridge retain KUnit's per-task static-stub context. + */ +#include +#include +#include +#include + +struct v14_bl_rpc; +struct v14_bl_actor { + struct v14_bl_rpc *ctx; + struct task_struct *task; + struct completion entered, done; + unsigned int op; + bool shown; +}; + +struct v14_bl_rpc { + struct kunit *test; + struct v14_bl_fixture f; + struct drm_device *drm; + struct drm_framebuffer fb[2]; + struct drm_gem_dma_object gem[2]; + unsigned int initialized, gems_initialized; + struct v14_bl_actor actors[2]; + struct completion rpc_entered, rpc_release; + atomic_t submits; + u64 iova[5], nits[5]; + bool nits_present[5]; + struct drm_framebuffer *owner[5]; + bool block_first, refuse; +}; + +static struct v14_bl_rpc *v14_bl_rpc_current; + +static int v14_bl_rpc_call(struct apple_dcp_v14 *v14, u32 tag, const void *in, u32 in_size, + void *out, u32 out_size, u32 completion, + dcp_v14_admit_fn admit, void *admit_cookie) +{ + struct v14_bl_rpc *c = v14_bl_rpc_current; + unsigned int n; + + KUNIT_EXPECT_TRUE(c->test, mutex_is_locked(&v14->lock)); + memset(out, 0, out_size); + if (tag == A(406)) { + put_unaligned_le32(1, out); + return 0; + } + KUNIT_EXPECT_EQ(c->test, tag, A(407)); + KUNIT_EXPECT_EQ(c->test, in_size, (u32)DCP_V14_SWAP_SIZE); + n = atomic_inc_return(&c->submits) - 1; + if (n < ARRAY_SIZE(c->iova)) { + c->iova[n] = get_unaligned_le64((const u8 *)in + DCP_V14_SWAP_IOVA(0)); + c->nits_present[n] = ((const u8 *)in)[DCP_V14_SWAP_NITS_PRESENT]; + c->nits[n] = get_unaligned_le64((const u8 *)in + DCP_V14_SWAP_NITS); + c->owner[n] = v14->active_fb; + } + if (!n && c->block_first) { + complete(&c->rpc_entered); + wait_for_completion(&c->rpc_release); + } + if (c->refuse) + put_unaligned_le32(1, (u8 *)out + 5); + return 0; +} + +static void v14_bl_rpc_work(struct work_struct *work) +{ + struct kunit *previous = current->kunit_test; + + current->kunit_test = v14_bl_rpc_current->test; + dcp_v14_backlight_work(work); + current->kunit_test = previous; +} + +static int v14_bl_actor_run(void *data) +{ + struct v14_bl_actor *a = data; + struct v14_bl_rpc *c = a->ctx; + + current->kunit_test = c->test; + complete(&a->entered); + if (a->op < 2) + a->shown = dcp_v14_present(c->f.v14, &c->fb[a->op], 64, 64, 0, NULL); + else if (a->op == 2) + iomfb_v14_7_backlight_suspend(c->f.dcp); + else + iomfb_v14_7_unbind(c->f.dcp); + complete(&a->done); + current->kunit_test = NULL; + return 0; +} + +static void v14_bl_rpc_cleanup(void *data) +{ + struct v14_bl_rpc *c = data; + unsigned int i; + + complete_all(&c->rpc_release); + for (i = 0; i < ARRAY_SIZE(c->actors); i++) + if (!IS_ERR_OR_NULL(c->actors[i].task)) + kthread_stop_put(c->actors[i].task); + dcp_v14_stop_backlight(c->f.v14); + if (c->f.v14->active_fb) { + drm_framebuffer_put(c->f.v14->active_fb); + c->f.v14->active_fb = NULL; + } + for (i = 0; i < c->initialized; i++) + drm_framebuffer_put(&c->fb[i]); + for (i = 0; i < c->gems_initialized; i++) + drm_gem_object_put(&c->gem[i].base); + v14_bl_rpc_current = NULL; +} + +static void v14_bl_fb_destroy(struct drm_framebuffer *fb) +{ + drm_framebuffer_cleanup(fb); +} + +static void v14_bl_gem_free(struct drm_gem_object *obj) +{ + drm_gem_object_release(obj); +} + +static const struct drm_gem_object_funcs v14_bl_gem_funcs = { + .free = v14_bl_gem_free, +}; + +static const struct drm_framebuffer_funcs v14_bl_fb_funcs = { + .destroy = v14_bl_fb_destroy, +}; + +static struct v14_bl_rpc *v14_bl_rpc_setup(struct kunit *test) +{ + struct v14_bl_rpc *c = kunit_kzalloc(test, sizeof(*c), GFP_KERNEL); + struct device *dev; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, c); + c->test = test; + c->f = v14_bl_fixture(test, &dcp_v14_board_t6030, true); + dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + c->drm = __drm_kunit_helper_alloc_drm_device(test, dev, sizeof(*c->drm), 0, + DRIVER_MODESET | DRIVER_GEM); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, c->drm); + init_completion(&c->rpc_entered); + init_completion(&c->rpc_release); + atomic_set(&c->submits, 0); + v14_bl_rpc_current = c; + kunit_activate_static_stub(test, dcp_v14_call_guarded, v14_bl_rpc_call); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, v14_bl_rpc_cleanup, c), 0); + for (i = 0; i < ARRAY_SIZE(c->fb); i++) { + c->fb[i].dev = c->drm; + c->fb[i].format = drm_format_info(DRM_FORMAT_XRGB8888); + c->fb[i].width = 64; + c->fb[i].height = 64; + c->fb[i].pitches[0] = 256; + drm_gem_private_object_init(c->drm, &c->gem[i].base, PAGE_SIZE); + c->gem[i].base.funcs = &v14_bl_gem_funcs; + c->gems_initialized++; + c->gem[i].dma_addr = 0x100000 + i * 0x10000; + c->fb[i].obj[0] = &c->gem[i].base; + KUNIT_ASSERT_EQ(test, drm_framebuffer_init(c->drm, &c->fb[i], &v14_bl_fb_funcs), 0); + c->initialized++; + } + c->f.v14->panel_width = 64; + c->f.v14->panel_height = 64; + c->f.dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; + c->f.dcp->connector_type = DRM_MODE_CONNECTOR_eDP; + c->f.dcp->panel.width_mm = 200; + c->f.dcp->mode_state.valid = true; + c->f.dcp->crtc = devm_kzalloc(c->f.dcp->dev, sizeof(*c->f.dcp->crtc), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, c->f.dcp->crtc); + c->f.dcp->crtc->base.dev = c->drm; + INIT_DELAYED_WORK(&c->f.v14->backlight_work, v14_bl_rpc_work); + return c; +} + +static struct v14_bl_actor *v14_bl_actor_start(struct kunit *test, + struct v14_bl_rpc *c, unsigned int slot, + unsigned int op) +{ + struct v14_bl_actor *a = &c->actors[slot]; + + a->ctx = c; + a->op = op; + init_completion(&a->entered); + init_completion(&a->done); + a->task = kthread_create(v14_bl_actor_run, a, "v14-backlight-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, a->task); + get_task_struct(a->task); + wake_up_process(a->task); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&a->entered, HZ), 0UL); + return a; +} + +static struct drm_framebuffer *v14_bl_pause_publication(struct apple_dcp_v14 *v14, + struct drm_framebuffer *fb, + const u8 *surface, u64 iova, + u32 width, u32 height, u32 dst_y) +{ + struct v14_bl_rpc *c = v14_bl_rpc_current; + + /* Only the first publication pauses; its eventual call runs the real body. */ + kunit_deactivate_static_stub(c->test, dcp_v14_publish_surface); + complete(&c->rpc_entered); + wait_for_completion(&c->rpc_release); + return dcp_v14_publish_surface(v14, fb, surface, iova, width, height, dst_y); +} + +/* Swap and framebuffer publication cannot be reordered by another present. */ +static void v14_bl_present_publication_serialized(struct kunit *test) +{ + struct v14_bl_rpc *c = v14_bl_rpc_setup(test); + struct v14_bl_actor *first, *second; + + kunit_activate_static_stub(test, dcp_v14_publish_surface, v14_bl_pause_publication); + first = v14_bl_actor_start(test, c, 0, 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&c->rpc_entered, HZ), 0UL); + second = v14_bl_actor_start(test, c, 1, 1); + KUNIT_EXPECT_EQ(test, + wait_for_completion_timeout(&second->done, msecs_to_jiffies(20)), 0UL); + KUNIT_EXPECT_EQ(test, atomic_read(&c->submits), 1); + complete_all(&c->rpc_release); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&first->done, HZ), 0UL); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&second->done, HZ), 0UL); + KUNIT_EXPECT_TRUE(test, first->shown); + KUNIT_EXPECT_TRUE(test, second->shown); + KUNIT_EXPECT_PTR_EQ(test, c->owner[1], &c->fb[0]); + KUNIT_EXPECT_PTR_EQ(test, c->f.v14->active_fb, &c->fb[1]); + KUNIT_EXPECT_EQ(test, c->f.v14->shown_iova, (u64)c->gem[1].dma_addr); + KUNIT_EXPECT_EQ(test, kref_read(&c->fb[0].base.refcount), 1U); + KUNIT_EXPECT_EQ(test, kref_read(&c->fb[1].base.refcount), 2U); +} + +/* A running brightness RPC pins its copied surface until suspend drains it. */ +static void v14_bl_running_work_suspend(struct kunit *test) +{ + struct v14_bl_rpc *c = v14_bl_rpc_setup(test); + struct v14_bl_actor *stop; + + KUNIT_ASSERT_TRUE(test, dcp_v14_present(c->f.v14, &c->fb[0], 64, 64, 0, NULL)); + atomic_set(&c->submits, 0); + c->block_first = true; + v14_bl_request(test, c->f.dcp, 200); + dcp_v14_backlight_kick(c->f.dcp); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&c->rpc_entered, HZ), 0UL); + stop = v14_bl_actor_start(test, c, 0, 2); + KUNIT_EXPECT_EQ(test, wait_for_completion_timeout(&stop->done, msecs_to_jiffies(20)), 0UL); + KUNIT_EXPECT_PTR_EQ(test, c->owner[0], &c->fb[0]); + KUNIT_EXPECT_EQ(test, kref_read(&c->fb[0].base.refcount), 2U); + complete_all(&c->rpc_release); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&stop->done, HZ), 0UL); + KUNIT_EXPECT_EQ(test, c->iova[0], (u64)c->gem[0].dma_addr); + KUNIT_EXPECT_EQ(test, c->f.v14->bl_gate, DCP_V14_BL_SUSPENDED); + v14_bl_request(test, c->f.dcp, 250); + dcp_v14_backlight_kick(c->f.dcp); + KUNIT_EXPECT_FALSE(test, work_busy(&c->f.v14->backlight_work.work)); + /* A callback queued before the gate shut must also refuse the pending level. */ + dcp_v14_backlight_work(&c->f.v14->backlight_work.work); + KUNIT_EXPECT_EQ(test, atomic_read(&c->submits), 1); + iomfb_v14_7_backlight_resume(c->f.dcp); + KUNIT_ASSERT_TRUE(test, dcp_v14_present(c->f.v14, &c->fb[0], 64, 64, 0, NULL)); + KUNIT_EXPECT_EQ(test, atomic_read(&c->submits), 2); + KUNIT_EXPECT_TRUE(test, c->nits_present[1]); + KUNIT_EXPECT_EQ(test, c->nits[1], dcp_v14_nits_binary64(250)); +} + +/* Actual dcp_poweroff/on must reach the 14.7 blank/restore hooks. */ +static void v14_bl_dpms_dispatch(struct kunit *test) +{ + struct v14_bl_rpc *c = v14_bl_rpc_setup(test); + struct platform_device *pdev = kunit_kzalloc(test, sizeof(*pdev), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, pdev); + platform_set_drvdata(pdev, c->f.dcp); + KUNIT_ASSERT_TRUE(test, dcp_v14_present(c->f.v14, &c->fb[0], 64, 64, 0, NULL)); + KUNIT_EXPECT_FALSE(test, c->nits_present[0]); + KUNIT_EXPECT_FALSE(test, dcp_backlight_level_known(c->f.dcp)); + dcp_poweroff(pdev); + KUNIT_EXPECT_EQ(test, atomic_read(&c->submits), 2); + KUNIT_EXPECT_FALSE(test, c->nits_present[1]); + KUNIT_EXPECT_PTR_EQ(test, c->f.v14->active_fb, NULL); + /* Even the advertised default must be an explicit first request. */ + dcp_poweron(pdev); + v14_bl_request(test, c->f.dcp, DCP_V14_NITS_DEFAULT); + KUNIT_ASSERT_TRUE(test, dcp_v14_present(c->f.v14, &c->fb[0], 64, 64, 0, NULL)); + KUNIT_EXPECT_TRUE(test, c->nits_present[2]); + KUNIT_EXPECT_TRUE(test, dcp_backlight_level_known(c->f.dcp)); + dcp_poweroff(pdev); + KUNIT_EXPECT_EQ(test, atomic_read(&c->submits), 4); + KUNIT_EXPECT_TRUE(test, c->nits_present[3]); + KUNIT_EXPECT_EQ(test, c->nits[3], 0ULL); + KUNIT_EXPECT_PTR_EQ(test, c->f.v14->active_fb, NULL); + dcp_poweron(pdev); + KUNIT_EXPECT_TRUE(test, dcp_backlight_pending(c->f.dcp)); + KUNIT_EXPECT_FALSE(test, c->f.dcp->backlight.state.dpms_off); + KUNIT_ASSERT_TRUE(test, dcp_v14_present(c->f.v14, &c->fb[0], 64, 64, 0, NULL)); + KUNIT_EXPECT_EQ(test, atomic_read(&c->submits), 5); + KUNIT_EXPECT_TRUE(test, c->nits_present[4]); + KUNIT_EXPECT_EQ(test, c->nits[4], dcp_v14_nits_binary64(DCP_V14_NITS_DEFAULT)); + KUNIT_EXPECT_FALSE(test, dcp_backlight_pending(c->f.dcp)); +} + +/* An actual failed submit must leave the loader's level unknown. */ +static void v14_bl_failed_submit_unknown(struct kunit *test) +{ + struct v14_bl_rpc *c = v14_bl_rpc_setup(test); + + c->refuse = true; + v14_bl_request(test, c->f.dcp, 240); + mutex_lock(&c->f.v14->lock); + KUNIT_EXPECT_EQ(test, dcp_v14_swap_locked(c->f.v14, NULL, 0, 0, 0, 0, NULL), -EIO); + mutex_unlock(&c->f.v14->lock); + KUNIT_EXPECT_FALSE(test, dcp_backlight_level_known(c->f.dcp)); + KUNIT_EXPECT_EQ(test, c->f.v14->nits_presents, 0ULL); + KUNIT_EXPECT_TRUE(test, c->f.v14->failed); + KUNIT_EXPECT_FALSE(test, work_busy(&c->f.v14->backlight_work.work)); + /* Neither a later kick nor a work callback may retry the failed panel. */ + dcp_v14_backlight_kick(c->f.dcp); + flush_delayed_work(&c->f.v14->backlight_work); + dcp_v14_backlight_work(&c->f.v14->backlight_work.work); + KUNIT_EXPECT_EQ(test, atomic_read(&c->submits), 1); + KUNIT_EXPECT_FALSE(test, work_busy(&c->f.v14->backlight_work.work)); +} + +static void v14_bl_mutex_unlock(void *data) +{ + mutex_unlock(data); +} + +/* Unbind closes class admission before draining a blocked real registrar. */ +static void v14_bl_registration_unbind_race(struct kunit *test) +{ + struct v14_bl_rpc *c = v14_bl_rpc_setup(test); + struct v14_bl_actor *stop; + unsigned long deadline; + bool running = false; + + mutex_lock(&c->f.dcp->bl_register_mutex); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, v14_bl_mutex_unlock, + &c->f.dcp->bl_register_mutex), 0); + KUNIT_EXPECT_TRUE(test, schedule_work(&c->f.dcp->bl_register_wq)); + deadline = jiffies + HZ; + do { + running = work_busy(&c->f.dcp->bl_register_wq) & WORK_BUSY_RUNNING; + if (running) + break; + usleep_range(50, 100); + } while (time_before(jiffies, deadline)); + KUNIT_EXPECT_TRUE(test, running); + stop = v14_bl_actor_start(test, c, 0, 3); + KUNIT_EXPECT_EQ(test, wait_for_completion_timeout(&stop->done, msecs_to_jiffies(20)), 0UL); + KUNIT_EXPECT_TRUE(test, READ_ONCE(c->f.dcp->quiescing)); + kunit_release_action(test, v14_bl_mutex_unlock, &c->f.dcp->bl_register_mutex); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&stop->done, HZ), 0UL); + KUNIT_EXPECT_PTR_EQ(test, c->f.dcp->brightness.bl_dev, NULL); + KUNIT_EXPECT_PTR_EQ(test, c->f.v14->dcp, NULL); + KUNIT_EXPECT_EQ(test, c->f.v14->bl_gate, DCP_V14_BL_CLOSED); + /* A late shared producer is refused as well. */ + schedule_work(&c->f.dcp->bl_register_wq); + flush_work(&c->f.dcp->bl_register_wq); + KUNIT_EXPECT_PTR_EQ(test, c->f.dcp->brightness.bl_dev, NULL); +} + +static void v14_bl_class_unregistered(struct kunit *test) +{ + struct v14_bl_rpc *c = v14_bl_rpc_setup(test); + + schedule_work(&c->f.dcp->bl_register_wq); + flush_work(&c->f.dcp->bl_register_wq); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, c->f.dcp->brightness.bl_dev); + iomfb_v14_7_unbind(c->f.dcp); + KUNIT_EXPECT_PTR_EQ(test, c->f.dcp->brightness.bl_dev, NULL); + KUNIT_EXPECT_TRUE(test, c->f.dcp->quiescing); + iomfb_v14_7_backlight_resume(c->f.dcp); + KUNIT_EXPECT_EQ(test, c->f.v14->bl_gate, DCP_V14_BL_CLOSED); +} + +/* fbdev cannot turn the advertised default into the first brightness request. */ +static void v14_bl_fbdev_unknown_level(struct kunit *test) +{ + struct v14_bl_rpc *c = v14_bl_rpc_setup(test); + struct device unrelated = {}; + struct backlight_device *bd; + + schedule_work(&c->f.dcp->bl_register_wq); + flush_work(&c->f.dcp->bl_register_wq); + bd = c->f.dcp->brightness.bl_dev; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bd); + KUNIT_EXPECT_FALSE(test, bd->ops->controls_device(bd, NULL)); + KUNIT_EXPECT_FALSE(test, dcp_backlight_pending(c->f.dcp)); + KUNIT_EXPECT_EQ(test, atomic_read(&c->submits), 0); + v14_bl_request(test, c->f.dcp, 250); + KUNIT_ASSERT_TRUE(test, dcp_v14_present(c->f.v14, &c->fb[0], 64, 64, 0, NULL)); + KUNIT_EXPECT_TRUE(test, bd->ops->controls_device(bd, NULL)); + KUNIT_EXPECT_TRUE(test, bd->ops->controls_device(bd, c->f.dcp->dev)); + KUNIT_EXPECT_FALSE(test, bd->ops->controls_device(bd, &unrelated)); +} + +static struct kunit_case v14_bl_lifetime_cases[] = { + KUNIT_CASE(v14_bl_present_publication_serialized), + KUNIT_CASE(v14_bl_running_work_suspend), + KUNIT_CASE(v14_bl_dpms_dispatch), + KUNIT_CASE(v14_bl_failed_submit_unknown), + KUNIT_CASE(v14_bl_registration_unbind_race), + KUNIT_CASE(v14_bl_class_unregistered), + KUNIT_CASE(v14_bl_fbdev_unknown_level), + {} +}; + +static struct kunit_suite v14_bl_lifetime_suite = { + .name = "dcp_v14_backlight_lifetime", + .test_cases = v14_bl_lifetime_cases, +}; + +kunit_test_suite(v14_bl_lifetime_suite); diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_backlight_test.c b/drivers/gpu/drm/apple/iomfb_v14_7_backlight_test.c new file mode 100644 index 0000000000000..e2704e5f1ba5f --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v14_7_backlight_test.c @@ -0,0 +1,576 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * The 14.7 panel level path, included at the end of iomfb_v14_7.c so that + * the production functions run: the swap's level block, which boards carry + * it, the gate that keeps brightness-only presents out of system sleep and + * after unbind, and the policy as this panel uses it (no level presented + * until one is requested, no actual level reported until one was presented, + * a refused level never becomes the actual one). + * + * The objects are built here, with no DCP: nothing in a case may reach an + * RPC. A J615 or J514C panel never initialises the backlight policy; + * its state is set to answer "yes" so that a case fails if anything on + * those boards looks at it. + */ +#include +#include + +static void v14_bl_unused_work(struct work_struct *work) +{ +} + +static void v14_bl_binary64(struct kunit *test) +{ + static const struct { + u32 nits; + u64 bits; + } known[] = { + { 0, 0 }, + { 1, 0x3ff0000000000000ULL }, + { 2, 0x4000000000000000ULL }, + { 3, 0x4008000000000000ULL }, + { 140, 0x4061800000000000ULL }, + { 255, 0x406fe00000000000ULL }, + { 256, 0x4070000000000000ULL }, + { 500, 0x407f400000000000ULL }, + }; + unsigned int i; + u32 nits; + + for (i = 0; i < ARRAY_SIZE(known); i++) + KUNIT_EXPECT_EQ_MSG(test, dcp_v14_nits_binary64(known[i].nits), known[i].bits, + "%u nits", known[i].nits); + /* Every level up to the bound is an exact integral binary64. */ + for (nits = 1; nits <= DCP_V14_NITS_MAX; nits++) { + u64 bits = dcp_v14_nits_binary64(nits); + unsigned int e = (bits >> 52) - 1023; + u64 mantissa = bits & GENMASK_ULL(51, 0); + + KUNIT_ASSERT_LT(test, e, 32U); + KUNIT_EXPECT_EQ(test, mantissa & GENMASK_ULL(51 - e, 0), 0ULL); + KUNIT_EXPECT_EQ(test, (1ULL << e) + (mantissa >> (52 - e)), (u64)nits); + KUNIT_EXPECT_FALSE(test, bits >> 63); + } +} + +static void v14_bl_level_block(struct kunit *test) +{ + u8 *swap = kunit_kzalloc(test, DCP_V14_SWAP_SIZE, GFP_KERNEL); + u8 *before = kunit_kzalloc(test, DCP_V14_SWAP_SIZE, GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, swap); + KUNIT_ASSERT_NOT_NULL(test, before); + memset(swap, 0xa5, DCP_V14_SWAP_SIZE); + memcpy(before, swap, DCP_V14_SWAP_SIZE); + + /* Above the wire bound: refused, the record untouched. */ + KUNIT_EXPECT_EQ(test, dcp_v14_encode_nits(swap, true, DCP_V14_NITS_MAX + 1), -EINVAL); + KUNIT_EXPECT_EQ(test, memcmp(swap, before, DCP_V14_SWAP_SIZE), 0); + + /* A level touches only the present byte and the binary64. */ + KUNIT_EXPECT_EQ(test, dcp_v14_encode_nits(swap, true, 140), 0); + for (i = 0; i < DCP_V14_SWAP_SIZE; i++) { + if (i == DCP_V14_SWAP_NITS_PRESENT || + (i >= DCP_V14_SWAP_NITS && i < DCP_V14_SWAP_NITS + 8)) + continue; + KUNIT_EXPECT_EQ_MSG(test, swap[i], before[i], "byte %#x", i); + } + KUNIT_EXPECT_EQ(test, swap[DCP_V14_SWAP_NITS_PRESENT], 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(swap + DCP_V14_SWAP_NITS), + 0x4061800000000000ULL); + + /* No level: zero, as the record the session sent before. */ + KUNIT_EXPECT_EQ(test, dcp_v14_encode_nits(swap, false, 0), 0); + KUNIT_EXPECT_EQ(test, swap[DCP_V14_SWAP_NITS_PRESENT], 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(swap + DCP_V14_SWAP_NITS), 0ULL); +} + +/* A swap without a level is byte for byte the one encoded before this. */ +static void v14_bl_swap_unchanged_without_level(struct kunit *test) +{ + u8 *swap = kunit_kzalloc(test, DCP_V14_SWAP_SIZE, GFP_KERNEL); + u8 *plain = kunit_kzalloc(test, DCP_V14_SWAP_SIZE, GFP_KERNEL); + u8 surface[DCP_V14_SURFACE_SIZE]; + + KUNIT_ASSERT_NOT_NULL(test, swap); + KUNIT_ASSERT_NOT_NULL(test, plain); + dcp_v14_encode_surface(surface, 3456 * 4, 3456, 2160, true); + dcp_v14_encode_swap(plain, 7, DCP_V14_BLACK, surface, 0x100000000ULL, 3456, 2160, 74, + DCP_V14_LAYER); + dcp_v14_encode_swap(swap, 7, DCP_V14_BLACK, surface, 0x100000000ULL, 3456, 2160, 74, + DCP_V14_LAYER); + dcp_v14_encode_nits(swap, false, 0); + KUNIT_EXPECT_EQ(test, memcmp(swap, plain, DCP_V14_SWAP_SIZE), 0); + /* The level block lies in the zeroed prefix, clear of every field. */ + KUNIT_EXPECT_EQ(test, plain[DCP_V14_SWAP_NITS_PRESENT], 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(plain + DCP_V14_SWAP_NITS), 0ULL); + + dcp_v14_encode_swap(plain, 8, DCP_V14_BLACK, NULL, 0, 0, 0, 0, DCP_V14_LAYER); + dcp_v14_encode_swap(swap, 8, DCP_V14_BLACK, NULL, 0, 0, 0, 0, DCP_V14_LAYER); + dcp_v14_encode_nits(swap, false, 0); + KUNIT_EXPECT_EQ(test, memcmp(swap, plain, DCP_V14_SWAP_SIZE), 0); +} + +static bool v14_bl_ctx_dcp_is(const void *ctx, const char *compat) +{ + return !strcmp(((const char *const *)ctx)[0], compat); +} + +static bool v14_bl_ctx_machine_is(const void *ctx, const char *compat) +{ + return !strcmp(((const char *const *)ctx)[1], compat); +} + +/* Only admitted T6030, J516C and J613 internal panels carry levels. */ +static void v14_bl_boards(struct kunit *test) +{ + static const char *const t6030_j516s[] = { "apple,t6030-dcp", "apple,j516s" }; + static const char *const t6030_j514s[] = { "apple,t6030-dcp", "apple,j514s" }; + static const char *const j613[] = { "apple,t8122-dcp", "apple,j613" }; + static const char *const j615[] = { "apple,t8122-dcp", "apple,j615" }; + static const char *const j516c[] = { "apple,t6031-dcp", "apple,j516c" }; + static const char *const other_t6030[] = { "apple,t6030-dcp", "apple,other" }; + const struct dcp_v14_board *board; + bool known; + + board = dcp_v14_board_select(v14_bl_ctx_dcp_is, v14_bl_ctx_machine_is, t6030_j516s, &known); + KUNIT_ASSERT_NOT_NULL(test, board); + KUNIT_EXPECT_TRUE(test, board->panel_nits); + KUNIT_EXPECT_TRUE(test, dcp_v14_panel_nits_admit(board, v14_bl_ctx_machine_is, + t6030_j516s)); + board = dcp_v14_board_select(v14_bl_ctx_dcp_is, v14_bl_ctx_machine_is, t6030_j514s, &known); + KUNIT_ASSERT_NOT_NULL(test, board); + KUNIT_EXPECT_TRUE(test, board->panel_nits); + KUNIT_EXPECT_TRUE(test, dcp_v14_panel_nits_admit(board, v14_bl_ctx_machine_is, + t6030_j514s)); + board = dcp_v14_board_select(v14_bl_ctx_dcp_is, v14_bl_ctx_machine_is, other_t6030, &known); + KUNIT_ASSERT_NOT_NULL(test, board); + KUNIT_EXPECT_FALSE(test, dcp_v14_panel_nits_admit(board, v14_bl_ctx_machine_is, + other_t6030)); + board = dcp_v14_board_select(v14_bl_ctx_dcp_is, v14_bl_ctx_machine_is, j613, &known); + KUNIT_ASSERT_NOT_NULL(test, board); + KUNIT_EXPECT_TRUE(test, board->panel_nits); + KUNIT_EXPECT_TRUE(test, dcp_v14_panel_nits_admit(board, v14_bl_ctx_machine_is, j613)); + KUNIT_EXPECT_FALSE(test, dcp_v14_panel_nits_admit(board, v14_bl_ctx_machine_is, j615)); + board = dcp_v14_board_select(v14_bl_ctx_dcp_is, v14_bl_ctx_machine_is, j615, &known); + KUNIT_ASSERT_NOT_NULL(test, board); + KUNIT_EXPECT_FALSE(test, board->panel_nits); + board = dcp_v14_board_select(v14_bl_ctx_dcp_is, v14_bl_ctx_machine_is, j516c, &known); + KUNIT_ASSERT_NOT_NULL(test, board); + KUNIT_EXPECT_TRUE(test, board->panel_nits); + KUNIT_EXPECT_TRUE(test, dcp_v14_panel_nits_admit(board, v14_bl_ctx_machine_is, j516c)); +} + +static void v14_bl_gate_transitions(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, dcp_v14_bl_may_queue(DCP_V14_BL_OPEN)); + KUNIT_EXPECT_FALSE(test, dcp_v14_bl_may_queue(DCP_V14_BL_SUSPENDED)); + KUNIT_EXPECT_FALSE(test, dcp_v14_bl_may_queue(DCP_V14_BL_CLOSED)); + KUNIT_EXPECT_EQ(test, dcp_v14_bl_gate_suspend(DCP_V14_BL_OPEN), DCP_V14_BL_SUSPENDED); + KUNIT_EXPECT_EQ(test, dcp_v14_bl_gate_suspend(DCP_V14_BL_SUSPENDED), DCP_V14_BL_SUSPENDED); + KUNIT_EXPECT_EQ(test, dcp_v14_bl_gate_suspend(DCP_V14_BL_CLOSED), DCP_V14_BL_CLOSED); + KUNIT_EXPECT_EQ(test, dcp_v14_bl_gate_resume(DCP_V14_BL_SUSPENDED), DCP_V14_BL_OPEN); + KUNIT_EXPECT_EQ(test, dcp_v14_bl_gate_resume(DCP_V14_BL_OPEN), DCP_V14_BL_OPEN); + /* Unbind is for good: no resume opens it again. */ + KUNIT_EXPECT_EQ(test, dcp_v14_bl_gate_resume(DCP_V14_BL_CLOSED), DCP_V14_BL_CLOSED); +} + +static void v14_bl_plan(struct kunit *test) +{ + const struct dcp_v14_bl_facts ready = { + .gate = DCP_V14_BL_OPEN, .panel_nits = true, .started = true, + .pending = true, .frame = true, .mode_valid = true, + }; + struct dcp_v14_bl_facts f; + + KUNIT_EXPECT_EQ(test, dcp_v14_bl_plan(&ready), DCP_V14_BL_REPRESENT); +#define V14_BL_EXPECT(field, value, action) do { \ + f = ready; \ + f.field = value; \ + KUNIT_EXPECT_EQ_MSG(test, dcp_v14_bl_plan(&f), action, \ + #field " = " #value); \ + } while (0) + V14_BL_EXPECT(gate, DCP_V14_BL_SUSPENDED, DCP_V14_BL_SKIP); + V14_BL_EXPECT(gate, DCP_V14_BL_CLOSED, DCP_V14_BL_SKIP); + V14_BL_EXPECT(panel_nits, false, DCP_V14_BL_SKIP); + V14_BL_EXPECT(external, true, DCP_V14_BL_SKIP); + V14_BL_EXPECT(typec, true, DCP_V14_BL_SKIP); + V14_BL_EXPECT(started, false, DCP_V14_BL_SKIP); + V14_BL_EXPECT(failed, true, DCP_V14_BL_SKIP); + V14_BL_EXPECT(crashed, true, DCP_V14_BL_SKIP); + V14_BL_EXPECT(quiescing, true, DCP_V14_BL_SKIP); + V14_BL_EXPECT(pending, false, DCP_V14_BL_SKIP); + V14_BL_EXPECT(frame, false, DCP_V14_BL_WAIT); + V14_BL_EXPECT(mode_valid, false, DCP_V14_BL_WAIT); +#undef V14_BL_EXPECT + KUNIT_EXPECT_TRUE(test, dcp_v14_swap_takes_nits(true, false, false, true)); + KUNIT_EXPECT_FALSE(test, dcp_v14_swap_takes_nits(false, false, false, true)); + KUNIT_EXPECT_FALSE(test, dcp_v14_swap_takes_nits(true, true, false, true)); + KUNIT_EXPECT_FALSE(test, dcp_v14_swap_takes_nits(true, false, true, true)); + KUNIT_EXPECT_FALSE(test, dcp_v14_swap_takes_nits(true, false, false, false)); +} + +/* The policy as this panel configures it: a default that is never presented. */ +static void v14_bl_policy(struct kunit *test) +{ + struct dcp_backlight_state s; + struct dcp_backlight_present p; + u64 seq; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&s, DCP_V14_NITS_MAX, false, 0, true, + DCP_V14_NITS_DEFAULT), 0); + /* Nothing to present at takeover: the panel keeps the loader's level. */ + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&s, true, &p), -EALREADY); + KUNIT_EXPECT_EQ(test, dcp_bl_reported_actual(&s, true), -ENODATA); + /* The H17P path keeps reporting its default. */ + KUNIT_EXPECT_EQ(test, dcp_bl_reported_actual(&s, false), DCP_V14_NITS_DEFAULT); + KUNIT_EXPECT_FALSE(test, dcp_v14_bl_follow_dpms(true, true, s.level_known)); + + /* A request is presented; a refusal never becomes the actual level. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&s, 250, false, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&s, true, &p), 0); + KUNIT_EXPECT_EQ(test, p.nits, 250U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&s, p.sequence, false)); + KUNIT_EXPECT_EQ(test, dcp_bl_reported_actual(&s, true), -ENODATA); + KUNIT_EXPECT_TRUE(test, s.dirty); + KUNIT_EXPECT_EQ(test, dcp_bl_retry_delay(&s), 100U); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&s, true, &p), 0); + seq = p.sequence; + /* A stale completion does not complete the newer reservation. */ + KUNIT_EXPECT_FALSE(test, dcp_bl_complete(&s, seq - 1, true)); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&s, seq, true)); + KUNIT_EXPECT_EQ(test, dcp_bl_reported_actual(&s, true), 250); + KUNIT_EXPECT_TRUE(test, dcp_v14_bl_follow_dpms(true, true, s.level_known)); + + /* A later refusal keeps the last accepted level. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&s, 300, false, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&s, true, &p), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&s, p.sequence, false)); + KUNIT_EXPECT_EQ(test, dcp_bl_reported_actual(&s, true), 250); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&s, true, &p), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&s, p.sequence, true)); + + /* DPMS off blanks at 0 nits, on presents the level again. */ + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&s, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&s, true, &p), 0); + KUNIT_EXPECT_EQ(test, p.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&s, p.sequence, true)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&s, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&s, true, &p), 0); + KUNIT_EXPECT_EQ(test, p.nits, 300U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&s, p.sequence, true)); +} + +struct v14_bl_fixture { + struct apple_dcp *dcp; + struct apple_dcp_v14 *v14; +}; + +static void v14_bl_fixture_stop(void *data) +{ + struct apple_dcp_v14 *v14 = data; + + cancel_delayed_work_sync(&v14->backlight_work); +} + +/* + * An internal 14.7 panel on @board, started, with no DCP behind it. With + * @policy, the T6030 panel's policy as dcp_v14_configure_backlight() sets it + * up; without, a policy that was never initialised (other boards), whose + * state would answer "yes" if anything looked at it. + */ +static struct v14_bl_fixture v14_bl_fixture(struct kunit *test, + const struct dcp_v14_board *board, bool policy) +{ + static atomic_t serial = ATOMIC_INIT(0); + struct v14_bl_fixture f; + struct device *dev; + char name[32]; + + snprintf(name, sizeof(name), "dcp-v14-bl.%d", atomic_inc_return(&serial)); + dev = kunit_device_register(test, name); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + /* The device's backlight cleanup must run before these objects are freed. */ + f.dcp = devm_kzalloc(dev, sizeof(*f.dcp), GFP_KERNEL); + f.v14 = devm_kzalloc(dev, sizeof(*f.v14), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f.dcp); + KUNIT_ASSERT_NOT_NULL(test, f.v14); + f.dcp->dev = dev; + spin_lock_init(&f.dcp->mode_state.lock); + mutex_init(&f.dcp->bl_register_mutex); + INIT_WORK(&f.dcp->bl_register_wq, dcp_work_register_backlight); + INIT_WORK(&f.dcp->bl_update_wq, v14_bl_unused_work); + INIT_DELAYED_WORK(&f.dcp->swap_watchdog_wq, v14_bl_unused_work); + f.dcp->fw_compat = DCP_FIRMWARE_V_14_7; + f.dcp->v14 = f.v14; + f.v14->dev = dev; + f.v14->board = board; + f.v14->panel_nits = board && board->panel_nits; + f.v14->dcp = f.dcp; + f.v14->started = true; + mutex_init(&f.v14->lock); + INIT_DELAYED_WORK(&f.v14->backlight_work, dcp_v14_backlight_work); + spin_lock_init(&f.v14->bl_gate_lock); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, v14_bl_fixture_stop, f.v14), 0); + if (policy) { + KUNIT_ASSERT_EQ(test, dcp_backlight_init(f.dcp, dev), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&f.dcp->backlight.state, DCP_V14_NITS_MAX, + false, 0, true, DCP_V14_NITS_DEFAULT), 0); + f.dcp->backlight.kick = dcp_v14_backlight_kick; + } else { + f.dcp->backlight.state.ready = true; + f.dcp->backlight.state.level_known = true; + f.dcp->backlight.state.dirty = true; + } + return f; +} + +/* Through the policy's own entry point, under its lock. */ +static void v14_bl_request(struct kunit *test, struct apple_dcp *dcp, u32 nits) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + ret = dcp_bl_request(&dcp->backlight.state, nits, false, false); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + KUNIT_ASSERT_EQ(test, ret, 0); +} + +/* Whether the work is queued or running; the caller holds lock. */ +static bool v14_bl_queued(struct apple_dcp_v14 *v14) +{ + return work_busy(&v14->backlight_work.work); +} + +/* + * The production paths on J615 and J514C: no policy is consulted, no + * level is encoded, the work does nothing, sleep leaves the gate alone. + */ +static void v14_bl_other_boards_untouched(struct kunit *test) +{ + const struct dcp_v14_board *boards[] = { + &dcp_v14_board_j615, &dcp_v14_board_j514c, + }; + struct dcp_backlight_present present; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(boards); i++) { + struct v14_bl_fixture f = v14_bl_fixture(test, boards[i], false); + u8 *swap = kunit_kzalloc(test, DCP_V14_SWAP_SIZE, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, swap); + KUNIT_EXPECT_FALSE(test, iomfb_v14_7_panel_nits(f.dcp)); + KUNIT_EXPECT_FALSE(test, dcp_backlight_active(f.dcp)); + KUNIT_EXPECT_FALSE(test, dcp_backlight_level_known(f.dcp)); + KUNIT_EXPECT_FALSE(test, dcp_v14_bl_dpms_applies(f.v14, f.dcp)); + + mutex_lock(&f.v14->lock); + KUNIT_EXPECT_FALSE(test, dcp_v14_swap_nits(f.v14, swap, &present)); + mutex_unlock(&f.v14->lock); + KUNIT_EXPECT_EQ(test, swap[DCP_V14_SWAP_NITS_PRESENT], 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(swap + DCP_V14_SWAP_NITS), 0ULL); + + /* Run directly: it must return without an RPC (there is no DCP). */ + dcp_v14_backlight_work(&f.v14->backlight_work.work); + KUNIT_EXPECT_EQ(test, f.v14->brightness_presents, 0ULL); + KUNIT_EXPECT_EQ(test, f.v14->nits_refused, 0ULL); + + iomfb_v14_7_backlight_suspend(f.dcp); + KUNIT_EXPECT_EQ(test, f.v14->bl_gate, DCP_V14_BL_OPEN); + dcp_v14_configure_backlight(f.dcp); + KUNIT_EXPECT_FALSE(test, f.dcp->backlight.kick); + } +} + +/* An external T6030 processor is no panel: nothing either. */ +static void v14_bl_external_untouched(struct kunit *test) +{ + struct v14_bl_fixture f = v14_bl_fixture(test, &dcp_v14_board_t6030, false); + struct dcp_backlight_present present; + u8 *swap = kunit_kzalloc(test, DCP_V14_SWAP_SIZE, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, swap); + f.dcp->external = true; + f.v14->external = true; + KUNIT_EXPECT_FALSE(test, iomfb_v14_7_panel_nits(f.dcp)); + KUNIT_EXPECT_FALSE(test, dcp_backlight_level_known(f.dcp)); + KUNIT_EXPECT_FALSE(test, dcp_v14_bl_dpms_applies(f.v14, f.dcp)); + mutex_lock(&f.v14->lock); + KUNIT_EXPECT_FALSE(test, dcp_v14_swap_nits(f.v14, swap, &present)); + mutex_unlock(&f.v14->lock); + KUNIT_EXPECT_EQ(test, swap[DCP_V14_SWAP_NITS_PRESENT], 0); + dcp_v14_backlight_work(&f.v14->backlight_work.work); + KUNIT_EXPECT_EQ(test, f.v14->brightness_presents, 0ULL); +} + +/* + * An admitted panel: no level until one is requested; the requested level is + * encoded; a refusal keeps the actual level without a retry; an accepted one + * is the actual level, and only then does blanking follow DPMS. + */ +static void v14_bl_swap_levels(struct kunit *test, const struct dcp_v14_board *board) +{ + struct v14_bl_fixture f = v14_bl_fixture(test, board, true); + struct dcp_backlight_present present; + u8 *swap = kunit_kzalloc(test, DCP_V14_SWAP_SIZE, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, swap); + KUNIT_EXPECT_TRUE(test, iomfb_v14_7_panel_nits(f.dcp)); + KUNIT_EXPECT_TRUE(test, dcp_backlight_active(f.dcp)); + KUNIT_EXPECT_FALSE(test, dcp_backlight_level_known(f.dcp)); + KUNIT_EXPECT_FALSE(test, dcp_v14_bl_dpms_applies(f.v14, f.dcp)); + + mutex_lock(&f.v14->lock); + /* Nothing requested: the swap carries no level. */ + KUNIT_EXPECT_FALSE(test, dcp_v14_swap_nits(f.v14, swap, &present)); + KUNIT_EXPECT_EQ(test, swap[DCP_V14_SWAP_NITS_PRESENT], 0); + mutex_unlock(&f.v14->lock); + + v14_bl_request(test, f.dcp, 250); + mutex_lock(&f.v14->lock); + KUNIT_EXPECT_TRUE(test, dcp_v14_swap_nits(f.v14, swap, &present)); + KUNIT_EXPECT_EQ(test, swap[DCP_V14_SWAP_NITS_PRESENT], 1); + KUNIT_EXPECT_EQ(test, get_unaligned_le64(swap + DCP_V14_SWAP_NITS), + dcp_v14_nits_binary64(250)); + /* Refused: no actual level, and the panel must not queue a retry. */ + dcp_v14_swap_nits_done(f.v14, &present, false); + mutex_unlock(&f.v14->lock); + KUNIT_EXPECT_FALSE(test, dcp_backlight_level_known(f.dcp)); + KUNIT_EXPECT_EQ(test, f.v14->nits_refused, 1ULL); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&f.v14->backlight_work)); + + /* While a Type-C display has the pipe, no level either. */ + f.dcp->fixed_connector_type = DRM_MODE_CONNECTOR_USB; + memset(swap, 0, DCP_V14_SWAP_SIZE); + mutex_lock(&f.v14->lock); + KUNIT_EXPECT_FALSE(test, dcp_v14_swap_nits(f.v14, swap, &present)); + mutex_unlock(&f.v14->lock); + KUNIT_EXPECT_EQ(test, swap[DCP_V14_SWAP_NITS_PRESENT], 0); + f.dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; + + mutex_lock(&f.v14->lock); + KUNIT_EXPECT_TRUE(test, dcp_v14_swap_nits(f.v14, swap, &present)); + dcp_v14_swap_nits_done(f.v14, &present, true); + mutex_unlock(&f.v14->lock); + KUNIT_EXPECT_TRUE(test, dcp_backlight_level_known(f.dcp)); + KUNIT_EXPECT_EQ(test, f.v14->nits_sent, 250U); + KUNIT_EXPECT_TRUE(test, dcp_v14_bl_dpms_applies(f.v14, f.dcp)); + + /* Unchanged: not sent again. */ + mutex_lock(&f.v14->lock); + KUNIT_EXPECT_FALSE(test, dcp_v14_swap_nits(f.v14, swap, &present)); + mutex_unlock(&f.v14->lock); +} + +static void v14_bl_t6030_swap_levels(struct kunit *test) +{ + v14_bl_swap_levels(test, &dcp_v14_board_t6030); +} + +static void v14_bl_j613_swap_levels(struct kunit *test) +{ + v14_bl_swap_levels(test, &dcp_v14_board_j613); +} + +/* + * The gate, through the production queue path: open, a kick queues the work; + * suspended, nothing queues until resume; closed, nothing ever does. The + * caller holds lock, so a queued work stays queued or running. + */ +static void v14_bl_t6030_gate(struct kunit *test) +{ + struct v14_bl_fixture f = v14_bl_fixture(test, &dcp_v14_board_t6030, true); + + mutex_lock(&f.v14->lock); + dcp_v14_backlight_kick(f.dcp); + KUNIT_EXPECT_TRUE(test, v14_bl_queued(f.v14)); + mutex_unlock(&f.v14->lock); + cancel_delayed_work_sync(&f.v14->backlight_work); + + iomfb_v14_7_backlight_suspend(f.dcp); + KUNIT_EXPECT_EQ(test, f.v14->bl_gate, DCP_V14_BL_SUSPENDED); + mutex_lock(&f.v14->lock); + dcp_v14_backlight_kick(f.dcp); + dcp_v14_bl_queue(f.v14, 0); + KUNIT_EXPECT_FALSE(test, v14_bl_queued(f.v14)); + mutex_unlock(&f.v14->lock); + + iomfb_v14_7_backlight_resume(f.dcp); + KUNIT_EXPECT_EQ(test, f.v14->bl_gate, DCP_V14_BL_OPEN); + mutex_lock(&f.v14->lock); + dcp_v14_backlight_kick(f.dcp); + KUNIT_EXPECT_TRUE(test, v14_bl_queued(f.v14)); + mutex_unlock(&f.v14->lock); + cancel_delayed_work_sync(&f.v14->backlight_work); + + /* A quiescing DCP takes no kick either. */ + WRITE_ONCE(f.dcp->quiescing, true); + mutex_lock(&f.v14->lock); + dcp_v14_backlight_kick(f.dcp); + KUNIT_EXPECT_FALSE(test, v14_bl_queued(f.v14)); + mutex_unlock(&f.v14->lock); + WRITE_ONCE(f.dcp->quiescing, false); + + dcp_v14_bl_shut(f.v14, true); + KUNIT_EXPECT_EQ(test, f.v14->bl_gate, DCP_V14_BL_CLOSED); + iomfb_v14_7_backlight_resume(f.dcp); + KUNIT_EXPECT_EQ(test, f.v14->bl_gate, DCP_V14_BL_CLOSED); + mutex_lock(&f.v14->lock); + dcp_v14_backlight_kick(f.dcp); + dcp_v14_bl_queue(f.v14, 0); + KUNIT_EXPECT_FALSE(test, v14_bl_queued(f.v14)); + mutex_unlock(&f.v14->lock); +} + +/* + * The work itself on the T6030 panel, with no DCP: blanked (no frame) it + * leaves the level to the next present; with the gate shut, or while + * suspended, it does nothing. None of these may reach an RPC. + */ +static void v14_bl_t6030_work(struct kunit *test) +{ + struct v14_bl_fixture f = v14_bl_fixture(test, &dcp_v14_board_t6030, true); + + v14_bl_request(test, f.dcp, 250); + WRITE_ONCE(f.dcp->mode_state.valid, true); + /* No frame shown: the level stays pending for the next present. */ + dcp_v14_backlight_work(&f.v14->backlight_work.work); + KUNIT_EXPECT_EQ(test, f.v14->brightness_presents, 0ULL); + KUNIT_EXPECT_TRUE(test, dcp_backlight_pending(f.dcp)); + + iomfb_v14_7_backlight_suspend(f.dcp); + dcp_v14_backlight_work(&f.v14->backlight_work.work); + KUNIT_EXPECT_TRUE(test, dcp_backlight_pending(f.dcp)); + iomfb_v14_7_backlight_resume(f.dcp); + + dcp_v14_bl_shut(f.v14, true); + dcp_v14_backlight_work(&f.v14->backlight_work.work); + KUNIT_EXPECT_EQ(test, f.v14->brightness_presents, 0ULL); + KUNIT_EXPECT_TRUE(test, dcp_backlight_pending(f.dcp)); +} + +static struct kunit_case v14_backlight_cases[] = { + KUNIT_CASE(v14_bl_binary64), + KUNIT_CASE(v14_bl_level_block), + KUNIT_CASE(v14_bl_swap_unchanged_without_level), + KUNIT_CASE(v14_bl_boards), + KUNIT_CASE(v14_bl_gate_transitions), + KUNIT_CASE(v14_bl_plan), + KUNIT_CASE(v14_bl_policy), + KUNIT_CASE(v14_bl_other_boards_untouched), + KUNIT_CASE(v14_bl_external_untouched), + KUNIT_CASE(v14_bl_t6030_swap_levels), + KUNIT_CASE(v14_bl_j613_swap_levels), + KUNIT_CASE(v14_bl_t6030_gate), + KUNIT_CASE(v14_bl_t6030_work), + {} +}; + +static struct kunit_suite v14_backlight_suite = { + .name = "dcp_v14_backlight", + .test_cases = v14_backlight_cases, +}; + +kunit_test_suite(v14_backlight_suite); diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_board.h b/drivers/gpu/drm/apple/iomfb_v14_7_board.h index 368dab49c2944..6de1ae9004122 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_board.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7_board.h @@ -39,6 +39,15 @@ struct dcp_v14_board { * releases, so they are only set where tested. */ u32 ctm_set, ctm_get; + /* + * The panel's swaps carry the constant timestamp + * DCP_V14_SWAP_TIMESTAMP_VALUE in slots 0, 1 and 6. With zero + * timestamps this board's firmware completes at most one swap per + * 16.7 ms, so a 120 Hz panel presents at 60 fps. + */ + bool swap_timestamps; + /* This board has a qualified panel level block; machine admission is separate. */ + bool panel_nits; }; /* Whether the DCP node or the machine (as @ctx gives them) has @compat. */ @@ -59,4 +68,11 @@ const struct dcp_v14_board *dcp_v14_board_select(dcp_v14_compat_fn dcp_is, */ bool dcp_v14_board_admits(const struct dcp_v14_board *board, const char *uuid); +/* + * Whether a swap carries timestamps: only the internal panel of a board whose + * record asks for them. External processors (@external, no record) and every + * other board send zero timestamps. + */ +bool dcp_v14_board_stamps_swaps(const struct dcp_v14_board *board, bool external); + #endif /* __APPLE_IOMFB_V14_7_BOARD_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_external_modeset_test.c b/drivers/gpu/drm/apple/iomfb_v14_7_external_modeset_test.c new file mode 100644 index 0000000000000..170aa59fb4234 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v14_7_external_modeset_test.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * The DCP calls of an external mode set, included at the end of + * iomfb_v14_7.c so that the production dcp_v14_external_modeset() runs. + * Every call is answered as accepted; the audio service is stubbed. + */ +#include +#include +#include + +#define V14_EXT_MODESET_MAX_CALLS 8 + +struct v14_ext_modeset { + struct apple_dcp *dcp; + struct apple_dcp_v14 *v14; + struct dcp_display_mode mode; + u32 tags[V14_EXT_MODESET_MAX_CALLS]; + bool power[V14_EXT_MODESET_MAX_CALLS]; + unsigned int calls; +}; + +static struct v14_ext_modeset *v14_ext_modeset_current; + +static int v14_ext_modeset_call(struct apple_dcp_v14 *v14, u32 tag, const void *in, u32 in_size, + void *out, u32 out_size, u32 completion, dcp_v14_admit_fn admit, + void *admit_cookie) +{ + struct v14_ext_modeset *s = v14_ext_modeset_current; + + if (s->calls < V14_EXT_MODESET_MAX_CALLS) { + s->tags[s->calls] = tag; + s->power[s->calls] = tag == A(472) && in_size >= 8 && get_unaligned_le64(in); + } + s->calls++; + if (out) + memset(out, 0, out_size); + return 0; +} + +static void v14_ext_modeset_audio(struct apple_dcp *dcp) +{ +} + +static void v14_ext_modeset_retry(struct work_struct *work) +{ +} + +static void v14_ext_modeset_stop(void *data) +{ + struct apple_dcp *dcp = data; + + cancel_delayed_work_sync(&dcp->external_retry_wq); +} + +/* A started, described native external session with one mode, its display @powered. */ +static struct v14_ext_modeset *v14_ext_modeset_setup(struct kunit *test, bool powered) +{ + struct v14_ext_modeset *s = kunit_kzalloc(test, sizeof(*s), GFP_KERNEL); + struct device *dev; + + KUNIT_ASSERT_NOT_NULL(test, s); + dev = kunit_device_register(test, "dcp-v14-ext-modeset"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + s->dcp = devm_kzalloc(dev, sizeof(*s->dcp), GFP_KERNEL); + s->v14 = devm_kzalloc(dev, sizeof(*s->v14), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, s->dcp); + KUNIT_ASSERT_NOT_NULL(test, s->v14); + s->dcp->dev = dev; + s->dcp->fw_compat = DCP_FIRMWARE_V_14_7; + s->dcp->external = true; + s->dcp->external_native = true; + /* Never dereferenced: the audio service is stubbed. */ + s->dcp->avep = (struct apple_dcp_afkep *)s; + mutex_init(&s->dcp->modes_lock); + mutex_init(&s->dcp->hpd_mutex); + spin_lock_init(&s->dcp->metadata.lock); + spin_lock_init(&s->dcp->mode_state.lock); + INIT_DELAYED_WORK(&s->dcp->external_retry_wq, v14_ext_modeset_retry); + /* 3840x2160 at 100 Hz, timing 167 and color 219 as a J514S TV reports it. */ + s->mode.mode = (struct drm_display_mode){ + DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 1188000, 3840, 4896, 4984, 5280, 0, + 2160, 2168, 2178, 2250, 0, DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) + }; + s->mode.timing_mode_id = 167; + s->mode.color_mode_id = 219; + s->dcp->modes = &s->mode; + s->dcp->nr_modes = 1; + s->dcp->modes_admitted = true; + s->dcp->v14 = s->v14; + s->v14->dev = dev; + s->v14->dcp = s->dcp; + s->v14->external = true; + s->v14->started = true; + s->v14->opened = true; + s->v14->powered = powered; + s->v14->described = DCP_V14_DESC_ALL; + mutex_init(&s->v14->lock); + init_waitqueue_head(&s->v14->described_wait); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, v14_ext_modeset_stop, s->dcp), 0); + v14_ext_modeset_current = s; + kunit_activate_static_stub(test, dcp_v14_call_guarded, v14_ext_modeset_call); + kunit_activate_static_stub(test, dcp_v14_audio_off, v14_ext_modeset_audio); + kunit_activate_static_stub(test, dcp_v14_audio_on, v14_ext_modeset_audio); + return s; +} + +static int v14_ext_modeset_run(struct kunit *test, struct v14_ext_modeset *s) +{ + struct drm_crtc_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, state); + drm_mode_copy(&state->mode, &s->mode.mode); + return dcp_v14_external_modeset(s->dcp, state); +} + +/* + * A display that was off, as on attach: the mode is set, the display powered + * on, and the mode set once more with the pipe on, before any swap. A DSC + * mode set with the pipe off only takes effect then. + */ +static void v14_ext_modeset_sets_mode_again_after_power_on(struct kunit *test) +{ + struct v14_ext_modeset *s = v14_ext_modeset_setup(test, false); + + KUNIT_EXPECT_EQ(test, v14_ext_modeset_run(test, s), 0); + KUNIT_ASSERT_EQ(test, s->calls, 3U); + KUNIT_EXPECT_EQ(test, s->tags[0], A(411)); + KUNIT_EXPECT_EQ(test, s->tags[1], A(472)); + KUNIT_EXPECT_TRUE(test, s->power[1]); + KUNIT_EXPECT_EQ(test, s->tags[2], A(411)); + KUNIT_EXPECT_TRUE(test, s->v14->powered); + KUNIT_EXPECT_TRUE(test, s->v14->mode_live); + KUNIT_EXPECT_EQ(test, s->v14->mode_timing, 167U); + KUNIT_EXPECT_EQ(test, s->v14->mode_color, 219U); + KUNIT_EXPECT_FALSE(test, mutex_is_locked(&s->v14->lock)); +} + +/* A mode change on a display that is on: power down, set, power up, set again. */ +static void v14_ext_modeset_mode_change(struct kunit *test) +{ + struct v14_ext_modeset *s = v14_ext_modeset_setup(test, true); + + KUNIT_EXPECT_EQ(test, v14_ext_modeset_run(test, s), 0); + KUNIT_ASSERT_EQ(test, s->calls, 4U); + KUNIT_EXPECT_EQ(test, s->tags[0], A(472)); + KUNIT_EXPECT_FALSE(test, s->power[0]); + KUNIT_EXPECT_EQ(test, s->tags[1], A(411)); + KUNIT_EXPECT_EQ(test, s->tags[2], A(472)); + KUNIT_EXPECT_TRUE(test, s->power[2]); + KUNIT_EXPECT_EQ(test, s->tags[3], A(411)); + KUNIT_EXPECT_TRUE(test, s->v14->powered); +} + +static struct kunit_case v14_ext_modeset_cases[] = { + KUNIT_CASE(v14_ext_modeset_sets_mode_again_after_power_on), + KUNIT_CASE(v14_ext_modeset_mode_change), + {} +}; + +static struct kunit_suite v14_ext_modeset_suite = { + .name = "dcp_v14_external_modeset", + .test_cases = v14_ext_modeset_cases, +}; + +kunit_test_suite(v14_ext_modeset_suite); diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_swap.h b/drivers/gpu/drm/apple/iomfb_v14_7_swap.h index aab3351dd9eea..c361103d06525 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_swap.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7_swap.h @@ -3,6 +3,9 @@ #ifndef __APPLE_IOMFB_V14_7_SWAP_H__ #define __APPLE_IOMFB_V14_7_SWAP_H__ +#include +#include +#include #include #include #include @@ -11,6 +14,8 @@ #define DCP_V14_SURFACE_SIZE 0x22c /* Offsets in the swap record. Rectangles are x, y, w, h (u32 each). */ +#define DCP_V14_SWAP_TIMESTAMP(n) ((n) * 8) /* u64, slots 0 to 6 */ +#define DCP_V14_SWAP_TIMESTAMPS 7 #define DCP_V14_SWAP_ID 0x98 #define DCP_V14_SWAP_SURF_ID(p) (0x9c + (p) * 4) #define DCP_V14_SWAP_SRC(p) (0xac + (p) * 16) /* in the surface */ @@ -26,6 +31,54 @@ #define DCP_V14_SWAP_LAYERS 0x7 #define DCP_V14_SWAP_SET_BACKGROUND BIT(31) +/* + * The panel level block of the swap record (the 14.7 J514S/J516S panel): a + * present byte at +0x354 and the level in nits as an unaligned IEEE 754 + * binary64 at +0x35c, both zero when the level does not change. The same + * layout as SiliconForest commit 44fe00e5eb58d54baf1cc9ad607293e774bfed44 + * and omacom/linux-aurora PR148 (02524534c6b8). + */ +#define DCP_V14_SWAP_NITS_PRESENT 0x354 +#define DCP_V14_SWAP_NITS 0x35c +/* The nits field takes integral levels up to the panel maximum. */ +#define DCP_V14_NITS_MAX 500 +/* + * The level userspace is shown before Linux has set one. It is never + * presented on its own: until a level is requested the panel keeps the boot + * loader's level, which Linux does not know. + */ +#define DCP_V14_NITS_DEFAULT 140 + +static_assert(DCP_V14_SWAP_NITS_PRESENT >= DCP_V14_SWAP_BACKGROUND + 4); +static_assert(DCP_V14_SWAP_NITS >= DCP_V14_SWAP_NITS_PRESENT + 1); +static_assert(DCP_V14_SWAP_NITS + 8 <= DCP_V14_SWAP_SURFACE(0)); + +/* Integral nits as IEEE 754 binary64, without kernel floating point. */ +static inline u64 dcp_v14_nits_binary64(u32 nits) +{ + unsigned int bit; + + if (!nits) + return 0; + bit = fls(nits) - 1; + return ((u64)(1023 + bit) << 52) | + ((u64)(nits - (1U << bit)) << (52 - bit)); +} + +/* + * The level block of an encoded swap: @nits when @change, else none. A level + * above the wire bound leaves the record untouched. + */ +static inline int dcp_v14_encode_nits(u8 *swap, bool change, u32 nits) +{ + if (change && nits > DCP_V14_NITS_MAX) + return -EINVAL; + swap[DCP_V14_SWAP_NITS_PRESENT] = change; + put_unaligned_le64(change ? dcp_v14_nits_binary64(nits) : 0, + swap + DCP_V14_SWAP_NITS); + return 0; +} + /* Offsets in a surface record. */ #define DCP_V14_SURF_OPAQUE 0x02 #define DCP_V14_SURF_FORMAT 0x0b @@ -115,4 +168,19 @@ static inline void dcp_v14_encode_swap(u8 *swap, u32 id, u32 background, const u } } +/* + * The swap timestamps of a board that asks for them: the constant 120 in + * slots 0, 1 and 6, a time that is never in the future. Counter values + * (1 GHz on T6031) make the firmware time out on the first swap. Call after + * dcp_v14_encode_swap(), which zeroes the record. + */ +#define DCP_V14_SWAP_TIMESTAMP_VALUE 120 + +static inline void dcp_v14_swap_timestamps(u8 *swap) +{ + put_unaligned_le64(DCP_V14_SWAP_TIMESTAMP_VALUE, swap + DCP_V14_SWAP_TIMESTAMP(0)); + put_unaligned_le64(DCP_V14_SWAP_TIMESTAMP_VALUE, swap + DCP_V14_SWAP_TIMESTAMP(1)); + put_unaligned_le64(DCP_V14_SWAP_TIMESTAMP_VALUE, swap + DCP_V14_SWAP_TIMESTAMP(6)); +} + #endif /* __APPLE_IOMFB_V14_7_SWAP_H__ */ diff --git a/drivers/gpu/drm/apple/v14-board-test.c b/drivers/gpu/drm/apple/v14-board-test.c index 801ed2f623a4c..f9b84ab36a7cc 100644 --- a/drivers/gpu/drm/apple/v14-board-test.c +++ b/drivers/gpu/drm/apple/v14-board-test.c @@ -3,12 +3,16 @@ * The 14.x IOMFB board records: which record a DCP gets on which machine, * and which DCP firmware image a record admits. The J516C (T6031) record * admits only the T6030 image, whose IOMFB layouts were validated, and - * refuses every other; the existing records are unchanged. + * refuses every other; the existing records are unchanged. Which DCPs stamp + * their swaps: the internal panel of a T6031 record; the T6030 and T8122 + * panels and external processors keep zero timestamps, byte for byte. */ #include #include +#include #include "iomfb_v14_7_board.h" +#include "iomfb_v14_7_swap.h" #define T6030_DCP_IMAGE "DDF38191-93B3-324A-BC8F-643006F5AC82" #define J613_DCP_IMAGE "90F849E1-B422-367E-B389-50246F8DEC47" @@ -174,12 +178,83 @@ static void v14_board_unchanged(struct kunit *test) KUNIT_EXPECT_TRUE(test, known); } +/* Only the internal panel of the J516C record stamps its swaps. */ +static void v14_board_swap_timestamps(struct kunit *test) +{ + const struct dcp_v14_board *j516c_b, *j514c_b, *t6030_b, *j613_b; + bool known; + + j516c_b = board_for("apple,t6031-dcp", j516c, &known); + j514c_b = board_for("apple,t6031-dcp", j514c, &known); + t6030_b = board_for("apple,t6030-dcp", j516s, &known); + j613_b = board_for("apple,t8122-dcp", j613, &known); + KUNIT_ASSERT_TRUE(test, j516c_b && j514c_b && t6030_b && j613_b); + + KUNIT_EXPECT_TRUE(test, j516c_b->swap_timestamps); + KUNIT_EXPECT_FALSE(test, j514c_b->swap_timestamps); + KUNIT_EXPECT_FALSE(test, t6030_b->swap_timestamps); + KUNIT_EXPECT_FALSE(test, j613_b->swap_timestamps); + + KUNIT_EXPECT_TRUE(test, dcp_v14_board_stamps_swaps(j516c_b, false)); + /* The J514C, the external role, the T6030 (J514S, J516S) and T8122 panels, no record. */ + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(j514c_b, false)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(j516c_b, true)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(t6030_b, false)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(j613_b, false)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(NULL, false)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_stamps_swaps(NULL, true)); +} + +/* + * A stamped swap holds 120 in timestamp slots 0, 1 and 6 and is otherwise the + * encoded record, with or without a surface; an unstamped one keeps every + * slot zero. + */ +static void v14_swap_timestamps_encoding(struct kunit *test) +{ + const size_t end = DCP_V14_SWAP_TIMESTAMP(DCP_V14_SWAP_TIMESTAMPS); + u8 surface[DCP_V14_SURFACE_SIZE]; + unsigned int i, pass; + u8 *plain, *stamped; + + plain = kunit_kzalloc(test, DCP_V14_SWAP_SIZE, GFP_KERNEL); + stamped = kunit_kzalloc(test, DCP_V14_SWAP_SIZE, GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, plain && stamped); + dcp_v14_encode_surface(surface, 3456 * 4, 3456, 2160, true); + + for (pass = 0; pass < 2; pass++) { + const u8 *s = pass ? surface : NULL; + + dcp_v14_encode_swap(plain, 0x2a, 0xff000000, s, 0x10000, pass ? 3456 : 0, + pass ? 2160 : 0, 74, 0); + dcp_v14_encode_swap(stamped, 0x2a, 0xff000000, s, 0x10000, pass ? 3456 : 0, + pass ? 2160 : 0, 74, 0); + dcp_v14_swap_timestamps(stamped); + + for (i = 0; i < DCP_V14_SWAP_TIMESTAMPS; i++) { + size_t at = DCP_V14_SWAP_TIMESTAMP(i); + u64 want = i == 0 || i == 1 || i == 6 ? 120 : 0; + + KUNIT_EXPECT_EQ_MSG(test, get_unaligned_le64(plain + at), 0, + "surface %u: unstamped slot %u", pass, i); + KUNIT_EXPECT_EQ_MSG(test, get_unaligned_le64(stamped + at), want, + "surface %u: slot %u", pass, i); + } + /* The rest of the record, from the word after slot 6, is unchanged. */ + KUNIT_EXPECT_EQ(test, + memcmp(plain + end, stamped + end, DCP_V14_SWAP_SIZE - end), 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(stamped + DCP_V14_SWAP_ID), 0x2a); + } +} + static struct kunit_case v14_board_cases[] = { KUNIT_CASE(v14_board_j516c), KUNIT_CASE(v14_board_j514c), KUNIT_CASE(v14_board_t6031_other_boards), KUNIT_CASE(v14_board_j516c_image), KUNIT_CASE(v14_board_unchanged), + KUNIT_CASE(v14_board_swap_timestamps), + KUNIT_CASE(v14_swap_timestamps_encoding), {} }; diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 42843e3459217..decf16ea09647 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -36,6 +36,7 @@ mod queue; mod regs; mod slotalloc; mod stats; +mod stats_raw; mod sysfs_exports; mod timestamp; mod usage; @@ -170,7 +171,7 @@ module! { }, g15_pb_max_blocks: u32 { default: 3072, - description: "G15: parameter-buffer growth maximum given to the firmware, blocks of 128 KiB (1024..4095 in effect; the blocks above it, up to 4095, answer partial renders; the 512 MiB heap is allocated either way)", + description: "G15: PB growth maximum given to the firmware, 128 KiB blocks (64..4095)", }, g15_early_tiling: u32 { default: 0, diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index 249b10bb361ca..5a14982dca053 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -24,13 +24,17 @@ use crate::{ event, gpu, mem, - module_parameters, // + module_parameters, + stats_raw::RawStatsSnapshot, // }; use core::sync::atomic::Ordering; use kernel::{ c_str, prelude::*, - sync::Arc, + sync::{ + Arc, + SpinLock, // + }, time::{ delay::fsleep, Delta, @@ -588,6 +592,9 @@ pub(crate) struct StatsChannel { dev: AsahiDevRef, ch: RxChannel, snap: Arc, + /// Last raw sample of every statistics kind, shared with the export + /// through `crate::stats_raw`. + raw: Arc>, /// Rate of firmware stats-message timestamps (Hz). All stats timestamps /// run on the chip base clock (`HwConfig::base_clock_hz`, 24 MHz on every /// supported SoC), not nanoseconds. @@ -604,12 +611,14 @@ impl StatsChannel::ver { dev: AsahiDevice, alloc: &mut gpu::KernelAllocators, snap: Arc, + raw: Arc>, ts_hz: u64, ) -> Result { Ok(StatsChannel::ver { dev: dev.clone(), ch: RxChannel::::new(alloc, 0x100)?, snap, + raw, ts_hz, last_util_ts: 0, }) @@ -631,6 +640,7 @@ impl StatsChannel::ver { // accessing the enum view is valid. let msg = unsafe { msg.msg }; cls_dev_dbg!(StatsCh, self.dev, "Stats: {:?}\n", msg); + self.record_raw(&msg); if module_parameters::stats_export.value() != 0 { // Retain the latest values for /sys/class/drm/card*/ // device/agx_stats. Read-only with respect to @@ -713,4 +723,80 @@ impl StatsChannel::ver { } } } + + /// Record one statistics message in the raw per-device snapshot. + /// + /// The firmware words are stored exactly as they arrived: no unit is + /// assumed, nothing is scaled and no metric is derived from them. + fn record_raw(&mut self, msg: &StatsMsg::ver) { + let stamp = crate::stats_raw::monotonic_ns(); + let mut snap = self.raw.lock(); + snap.note_message(); + match msg { + StatsMsg::ver::Power { power, .. } => snap.set_power(power.0, stamp), + StatsMsg::ver::PowerOn { off_time, .. } => snap.set_power_on(off_time.0, stamp), + StatsMsg::ver::PowerOff { on_time, .. } => snap.set_power_off(on_time.0, stamp), + StatsMsg::ver::Utilization { + timestamp, + util1, + util2, + util3, + util4, + } => snap.set_utilization(timestamp.0, [*util1, *util2, *util3, *util4], stamp), + StatsMsg::ver::AvgPower { + active_cs, + unk2, + unk3, + unk4, + avg_power, + } => snap.set_avg_power(active_cs.0, [*unk2, *unk3, *unk4], *avg_power, stamp), + StatsMsg::ver::Temperature { + raw_value, + scale, + tmin, + tmax, + .. + } => snap.set_temperature(*raw_value, *scale, *tmin, *tmax, stamp), + StatsMsg::ver::PowerState { + timestamp, + last_busy_ts, + active, + poweroff, + unk1, + pstate, + unk2, + unk3, + } => snap.set_power_state( + timestamp.0, + last_busy_ts.0, + *active, + *poweroff, + *unk1, + *pstate, + *unk2, + *unk3, + stamp, + ), + StatsMsg::ver::FwBusy { timestamp, busy } => { + snap.set_fw_busy(timestamp.0, *busy, stamp) + } + StatsMsg::ver::PState { + ps_min, + unk1, + ps_max, + unk2, + .. + } => snap.set_pstate(*ps_min, *unk1, *ps_max, *unk2, stamp), + StatsMsg::ver::TempSensor { + sensor_id, + raw_value, + scale, + tmin, + tmax, + .. + } => snap.set_temp_sensor(*sensor_id, *raw_value, *scale, *tmin, *tmax, stamp), + // The remaining tags are not decoded by the driver. + _ => {} + } + } } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index a89728cc2f2b3..d6e461a7c3210 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -302,12 +302,20 @@ impl platform::Driver for AsahiPlatformDriver { gpu.init()?; let stats = gpu.stats_snapshot(); + let debug_stats = gpu.raw_stats_snapshot(); let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is // unbound; it is never forgotten. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + // The DRM core created this card's debugfs directory when the device was registered; + // publish the raw firmware statistics in it. drm_dev_unregister() removes the file, and + // waits for its readers, before the registration data that owns the snapshot is dropped. + if let Some(snapshot) = debug_stats.as_ref() { + crate::stats_raw::register(reg.device(), snapshot); + } + // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); diff --git a/drivers/gpu/drm/asahi/g15/adt.rs b/drivers/gpu/drm/asahi/g15/adt.rs index aece4f083c0ad..83807d985866a 100644 --- a/drivers/gpu/drm/asahi/g15/adt.rs +++ b/drivers/gpu/drm/asahi/g15/adt.rs @@ -30,10 +30,13 @@ //! one cell per cluster). The same admission applies, without the DRAM bounds, and the four UAT //! regions may lie in System RAM: they are reserved-memory nodes without `no-map`, which the //! kernel maps, so the driver reaches them through its own mapping ([`super::regs::Mapping`]). +//! The GPU's sensor rows, which the node does not carry, come from the same ADT export as on +//! T6030: `/arm-io/aon-ptd` `ptd_data_table` ([`super::board::record::sensor_rows`]). use core::ops::Range; use kernel::{c_str, device, prelude::*}; +use super::board::record; use super::boot::glue; use super::hw; @@ -57,12 +60,13 @@ pub(crate) struct GpuAdt { pub(crate) perf_mhz: [u32; PERF_STATES], pub(crate) perf_mv: [u32; PERF_STATES], pub(crate) sram_mv: [u32; PERF_STATES], - /// Per-cluster core mV of each state, rounded up from the device tree's microvolts, how - /// many voltage cells the node has (1 on the ADT path), and the state's power in mW (0 on - /// the ADT path). + /// Per-cluster core mV of each state, rounded up from the device tree's microvolts, and how + /// many voltage cells the node has (1 on the ADT path). pub(crate) cluster_mv: [[u32; CLUSTERS]; PERF_STATES], pub(crate) cluster_cells: usize, - pub(crate) perf_mw: [u32; PERF_STATES], + /// The GPU's six sensor rows of the ADT's `aon-ptd` table (T6031; zero on the T6030 path, + /// whose templates carry their own). + pub(crate) sensor_rows: [[u8; 12]; record::SENSOR_ROW_COUNT], } /// Voltage cells the layout keeps per state. @@ -328,7 +332,7 @@ fn parse(dev: &device::Device, adt: &Adt<'_>) -> Result { sram_mv: [0; PERF_STATES], cluster_mv: [[0; CLUSTERS]; PERF_STATES], cluster_cells: 1, - perf_mw: [0; PERF_STATES], + sensor_rows: [[0; 12]; record::SENSOR_ROW_COUNT], }; collect_perf(dev, adt, sgx, &mut out)?; for i in 0..PERF_STATES { @@ -437,11 +441,11 @@ fn from_device_tree(dev: &device::Device) -> Result { sram_mv: [0; PERF_STATES], cluster_mv: [[0; CLUSTERS]; PERF_STATES], cluster_cells: 0, - perf_mw: [0; PERF_STATES], + sensor_rows: [[0; 12]; record::SENSOR_ROW_COUNT], }; // Operating points: MHz, the highest cluster voltage of each and the clusters' voltages - // (rounded up) and power for the hardware record; the SRAM voltage is not in the device - // tree. + // (rounded up) for the hardware record; the SRAM voltage is not in the device tree, and + // the powers are a boot-loader stand-in. let opps = node .parse_phandle(c_str!("operating-points-v2"), 0) .ok_or(EINVAL) @@ -450,7 +454,6 @@ fn from_device_tree(dev: &device::Device) -> Result { for opp in opps.children() { let hz: u64 = opp.get_property(c_str!("opp-hz"))?; let uv: KVec = opp.get_property(c_str!("opp-microvolt"))?; - let uw: Option = opp.get_opt_property(c_str!("opp-microwatt"))?; let mv = uv.iter().max().copied().unwrap_or(0) / 1000; if uv.is_empty() || uv.len() > CLUSTERS || (count > 0 && uv.len() != out.cluster_cells) { dev_err!(dev, "GPU operating point {} has {} voltage cells\n", count, uv.len()); @@ -467,7 +470,6 @@ fn from_device_tree(dev: &device::Device) -> Result { out.cluster_mv[count][c] = v.div_ceil(1000); } out.cluster_cells = uv.len(); - out.perf_mw[count] = uw.unwrap_or(0) / 1000; } count += 1; } @@ -480,19 +482,44 @@ fn from_device_tree(dev: &device::Device) -> Result { ); return Err(EINVAL); } + out.sensor_rows = sensor_rows(dev)?; Ok(out) } +/// A private copy of the boot loader's ADT export. +fn copy() -> Result> { + let (start, size) = glue::adt_phram()?; + if size == 0 || size > 0x200000 || start & 0x3fff != 0 { + return Err(EINVAL); + } + super::identity::copy_physical(start, usize::try_from(size)?) +} + +/// The GPU's six sensor rows: `/arm-io/aon-ptd` `ptd_data_table` of the ADT export +/// ([`record::sensor_rows`]), which the device tree does not carry. +fn sensor_rows(dev: &device::Device) -> Result<[[u8; 12]; record::SENSOR_ROW_COUNT]> { + let copy = + copy().inspect_err(|_| dev_err!(dev, "no usable ADT export for the GPU sensor rows\n"))?; + let adt = Adt { data: © }; + let table = adt + .node_end(0, 0) + .and_then(|_| adt.child(0, "arm-io")) + .and_then(|armio| adt.child(armio, "aon-ptd")) + .and_then(|ptd| adt.prop(ptd, "ptd_data_table")) + .ok_or(EINVAL) + .inspect_err(|_| dev_err!(dev, "ADT has no aon-ptd ptd_data_table\n"))?; + record::sensor_rows(table).map_err(|why| { + dev_err!(dev, "ADT aon-ptd: {}\n", why); + EINVAL + }) +} + /// Reads and admits the GPU's regions, segments and performance states (read-only), from the /// source the SoC configuration names. pub(crate) fn read(dev: &device::Device, cfg: &super::Config) -> Result { if cfg.platform == hw::Platform::DeviceTree { return from_device_tree(dev); } - let (start, size) = glue::adt_phram()?; - if size == 0 || size > 0x200000 || start & 0x3fff != 0 { - return Err(EINVAL); - } - let copy = super::identity::copy_physical(start, usize::try_from(size)?)?; + let copy = copy()?; parse(dev, &Adt { data: © }) } diff --git a/drivers/gpu/drm/asahi/g15/board.rs b/drivers/gpu/drm/asahi/g15/board.rs index da0a7408575bf..f9682b2ead10c 100644 --- a/drivers/gpu/drm/asahi/g15/board.rs +++ b/drivers/gpu/drm/asahi/g15/board.rs @@ -10,11 +10,11 @@ //! against its hash at boot. //! //! A SoC whose board data comes from its device tree ([`super::hw::Platform::DeviceTree`]) gets -//! its hardware record from [`hardware_record`] ([`record`]) and its Globals and Power from -//! [`device_tree`]: the G15S templates with the GPU node's power words and the hardware -//! record's maximum power ([`words`]). +//! its hardware record from [`hardware_record`] ([`record`]: the GPU node, the ADT's sensor +//! rows and the SoC's reference words) and its Globals and Power from [`device_tree`]: the G15S +//! templates with the GPU node's power words and the hardware record's maximum power +//! ([`words`]). -pub(crate) mod blob; pub(crate) mod data; pub(crate) mod record; pub(crate) mod template; @@ -98,7 +98,6 @@ fn aux_opps( mhz, mv: [uv[0].div_ceil(1000), 0, 0, 0], cells: 1, - mw: 0, }; n += 1; } @@ -110,13 +109,13 @@ fn aux_opps( } /// The hardware record of a device-tree SoC ([`record`]): built from the GPU node's properties -/// and operating-point tables, the layout's IOMapping table and the ADT regions, with the -/// words the device tree does not carry copied from `file`, the machine's own record. +/// and operating-point tables, the layout's IOMapping table, the ADT regions and sensor rows, +/// and the SoC's `reference` words, which neither carries. pub(crate) fn hardware_record( dev: &device::Device, cfg: &super::Config, adt: &GpuAdt, - file: &[u8], + reference: &record::Reference, ) -> Result> { let layout = cfg.layout; let node = dev.of_node().ok_or(ENODEV)?; @@ -136,7 +135,6 @@ pub(crate) fn hardware_record( mhz: adt.perf_mhz[i], mv: adt.cluster_mv[i], cells: adt.cluster_cells, - mw: adt.perf_mw[i], }; } let mut cs = [record::Opp::default(); 16]; @@ -159,12 +157,6 @@ pub(crate) fn hardware_record( writable, }; } - let max_power_mw = u32::from_le_bytes( - file.get(record::MAX_POWER..record::MAX_POWER + 4) - .ok_or(EINVAL)? - .try_into() - .map_err(|_| EINVAL)?, - ); let inputs = record::Inputs { props, zones: &zones, @@ -177,15 +169,14 @@ pub(crate) fn hardware_record( usc_base: hw::client::COMPACT_WINDOW.start, user_end: hw::USER_RANGE.end, timestamps: va::TIMESTAMPS, - max_power_mw, + sensor_rows: adt.sensor_rows, + reference, }; let mut out = KVVec::from_elem(0u8, record::SIZE, GFP_KERNEL)?; - record::from_file(&mut out, file) - .and_then(|()| record::build(&mut out, &inputs)) - .map_err(|why| { - dev_err!(dev, "hardware record from the device tree: {}\n", why); - EINVAL - })?; + record::build(&mut out, &inputs).map_err(|why| { + dev_err!(dev, "hardware record from the device tree: {}\n", why); + EINVAL + })?; Ok(out) } diff --git a/drivers/gpu/drm/asahi/g15/board/blob.rs b/drivers/gpu/drm/asahi/g15/board/blob.rs deleted file mode 100644 index 04cff8bbaa3d9..0000000000000 --- a/drivers/gpu/drm/asahi/g15/board/blob.rs +++ /dev/null @@ -1,113 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Board images from a firmware file. -//! -//! A SoC whose board images are not built-in templates names a firmware file -//! ([`super::super::hw::Board::Blob`]), built into the kernel with `CONFIG_EXTRA_FIRMWARE` or -//! loaded from `/lib/firmware`, and pins its SHA-256. The file holds the three images as the -//! firmware of that machine accepted them; the build then writes the pointers and policy words -//! it owns into them (TTBAT, IOMapping VAs, the timestamp arena, the idle policy). -//! -//! Layout, little-endian: the magic `T6031BRD`, u32 format version (1), u32 image count (3), -//! then per image `u32 kind, u32 size` with kinds 0 (HardwareData), 1 (Globals) and -//! 2 (Power), then the images' bytes in that order. Sizes must match the SoC's layout. - -use kernel::{device, firmware::Firmware, prelude::*}; - -use super::super::boot::glue; -use super::super::identity::Hex; - -const MAGIC: &[u8; 8] = b"T6031BRD"; -const VERSION: u32 = 1; -const IMAGES: usize = 3; - -/// The three images. -pub(crate) struct Images { - pub(crate) hardware: KVVec, - pub(crate) globals: KVVec, - pub(crate) power: KVVec, -} - -fn le32(b: &[u8], off: usize) -> Result { - Ok(u32::from_le_bytes( - b.get(off..off + 4) - .ok_or(EINVAL)? - .try_into() - .map_err(|_| EINVAL)?, - )) -} - -fn copy(b: &[u8], range: core::ops::Range) -> Result> { - let src = b.get(range).ok_or(EINVAL)?; - let mut out = KVVec::from_elem(0u8, src.len(), GFP_KERNEL)?; - out.copy_from_slice(src); - Ok(out) -} - -/// Requests `name`, verifies its digest against `sha256` and splits it into the images, each -/// of the size `sizes` gives for its kind. -pub(crate) fn load( - dev: &device::Device, - name: &CStr, - sha256: &[u8; 32], - sizes: [usize; IMAGES], -) -> Result { - let fw = Firmware::request(name, dev).inspect_err(|e| { - dev_err!(dev, "board images {}: not available ({:?})\n", name, e) - })?; - let data = fw.data(); - let digest = glue::sha256(data); - if digest != *sha256 { - dev_err!( - dev, - "board images {}: sha256 {} is not the pinned one\n", - name, - Hex(&digest) - ); - return Err(EINVAL); - } - if data.len() < 16 || &data[..8] != MAGIC || le32(data, 8)? != VERSION { - dev_err!(dev, "board images {}: unknown header\n", name); - return Err(EINVAL); - } - if le32(data, 12)? as usize != IMAGES { - dev_err!(dev, "board images {}: not {} images\n", name, IMAGES); - return Err(EINVAL); - } - let mut at = 16 + IMAGES * 8; - let mut images: [Option>; IMAGES] = [None, None, None]; - for i in 0..IMAGES { - let kind = le32(data, 16 + i * 8)? as usize; - let size = le32(data, 16 + i * 8 + 4)? as usize; - if kind != i || size != sizes[i] { - dev_err!( - dev, - "board images {}: image {} is kind {} of {} bytes, need {}\n", - name, - i, - kind, - size, - sizes[i] - ); - return Err(EINVAL); - } - images[i] = Some(copy(data, at..at + size)?); - at += size; - } - if at != data.len() { - dev_err!(dev, "board images {}: trailing bytes\n", name); - return Err(EINVAL); - } - let [hardware, globals, power] = images; - dev_info!( - dev, - "board images {}: sha256 {} accepted\n", - name, - Hex(&digest) - ); - Ok(Images { - hardware: hardware.ok_or(EINVAL)?, - globals: globals.ok_or(EINVAL)?, - power: power.ok_or(EINVAL)?, - }) -} diff --git a/drivers/gpu/drm/asahi/g15/board/record.rs b/drivers/gpu/drm/asahi/g15/board/record.rs index 959bc6a4ab0a2..119a692064391 100644 --- a/drivers/gpu/drm/asahi/g15/board/record.rs +++ b/drivers/gpu/drm/asahi/g15/board/record.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! The J516C hardware record (0xb480 bytes) from the device tree. +//! The J516C hardware record (0xb480 bytes) from the device tree and the ADT. //! //! The record is HardwareDataB (identity, IOMapping records, the DVFS tables) at 0, HardwareDataA //! (the power, performance and thermal controllers) at [`HWDATA_A`], and the G15C tail (sensor @@ -13,12 +13,14 @@ //! state cap and base state; //! - a layout constant of the 2419 firmware's G15C record: flags, the IOMapping flag words, //! the identity words, fixed factors (65536, 95, 100, 1.02, 0.8/0.2, 625, 11000, 125, 5); -//! - a word the device tree does not carry ([`FROM_FILE`]), copied from the machine's board file -//! by [`from_file`] and listed with its rule in the project notes: the maximum power -//! (HardwareDataA +0x6a4; every other maximum-power word derives from it), the MTR sensor -//! masks, the relative-power table (the device tree's operating-point powers are a boot-loader -//! stand-in) and the tail's six sensor rows; -//! - a word the file carries and the firmware was shown not to need ([`UNNEEDED`]): zero. +//! - the tail's six sensor rows: entries of the ADT's `aon-ptd` sensor table ([`sensor_rows`]); +//! - a word neither carries ([`REFERENCE_WORDS`]): the value of the J516C reference machine +//! ([`Reference`]), as the T6030 templates carry their reference machine's: the maximum +//! power (HardwareDataA +0x6a4; every other maximum-power word derives from it), the MTR +//! sensor masks and the relative-power table (the device tree's operating-point powers are a +//! boot-loader stand-in); +//! - a word the reference machine's record carries and the firmware was shown not to need +//! ([`UNNEEDED`]): zero. //! //! HardwareDataA has the G15S shape of the T6030 templates with insertions ([`a`]): 8 bytes //! before the fast-die controller (+0x8bc), 16-wide instead of 8-wide coefficient rows (+0x200 @@ -152,14 +154,13 @@ const SE_KP1: usize = 40; const SE_RESET: usize = 41; const SE_TARGET: usize = 42; -/// One operating point: MHz, the clusters' voltages in mV (rounded up from microvolts), how -/// many voltage cells the node had, and mW. +/// One operating point: MHz, the clusters' voltages in mV (rounded up from microvolts) and how +/// many voltage cells the node had. #[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct Opp { pub(crate) mhz: u32, pub(crate) mv: [u32; CLUSTERS], pub(crate) cells: usize, - pub(crate) mw: u32, } impl Opp { @@ -198,40 +199,63 @@ pub(crate) struct Inputs<'a> { pub(crate) usc_base: u64, pub(crate) user_end: u64, pub(crate) timestamps: u64, - /// The maximum GPU power in mW ([`MAX_POWER`], a word the device tree does not carry). + /// The six sensor rows of the ADT's `aon-ptd` table ([`sensor_rows`]). + pub(crate) sensor_rows: [[u8; 12]; SENSOR_ROW_COUNT], + /// The words neither the device tree nor the ADT carries. + pub(crate) reference: &'a Reference, +} + +/// The record's words that neither the device tree nor the ADT carries, as one machine's +/// driver wrote them ([`REFERENCE_WORDS`]). +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Reference { + /// The maximum GPU power in mW ([`MAX_POWER`]). pub(crate) max_power_mw: u32, + /// The relative powers of DVFS states 1..7, percent of the top state's (state 8 is 100). + pub(crate) relative_power: [u32; RELATIVE_STATES], + /// The MTR fast-die sensor mask (written twice) and alarm mask, as the u64 at each place. + /// The device tree's `apple,fast-die0-sensor-mask` is another value. + pub(crate) fast_die_mask: u64, + pub(crate) alarm_mask: u64, } +/// DVFS states with a relative-power word of their own. +pub(crate) const RELATIVE_STATES: usize = 7; + +/// The J516C reference machine's words (its 2419 firmware driver's values on that machine). +/// Another J516C gets them as a J514S gets the T6030 templates' machine's: the operating +/// points, voltages, gains and sensor rows are its own, these are the reference machine's. +pub(crate) const J516C: Reference = Reference { + max_power_mw: 116739, + relative_power: [11, 23, 33, 44, 55, 71, 90], + fast_die_mask: 0x802a002b_802a002b, + alarm_mask: 0x902a102b_802a002b, +}; + /// HardwareDataA's maximum-power word (= `fw::initdata::hwdata::MAX_POWER_A`). pub(crate) const MAX_POWER: usize = HWDATA_A + 0x6a4; +const RELATIVE_POWER: usize = 0x1060; +const FAST_DIE_MASK: usize = HWDATA_A + 0x8ac; +const FAST_DIE_MASK_2: usize = HWDATA_A + 0x1490; +const ALARM_MASK: usize = HWDATA_A + 0x24a8; -/// Words the device tree does not carry, copied from the board file. Record offsets. +/// The record words [`Reference`] supplies. Record offsets. +#[cfg(test)] #[rustfmt::skip] -pub(crate) const FROM_FILE: [usize; 32] = [ - // The relative powers of states 1..7 (state 8 is 100): the device tree's operating-point - // powers are a boot-loader stand-in. +pub(crate) const REFERENCE_WORDS: [usize; 14] = [ 0x1060, 0x1064, 0x1068, 0x106c, 0x1070, 0x1074, 0x1078, - // HardwareDataA: the maximum power, the MTR fast-die masks (+0x8ac, +0x1490) and the alarm - // mask (+0x24a8); the device tree's apple,fast-die0-sensor-mask is another value. MAX_POWER, - HWDATA_A + 0x8ac, HWDATA_A + 0x8b0, - HWDATA_A + 0x1490, HWDATA_A + 0x1494, - HWDATA_A + 0x24a8, HWDATA_A + 0x24ac, - // The tail's six sensor rows (key, id, kind; 12 bytes each; the boot loader does not export - // the aon-ptd table). - HWDATA_A + 0x6c58, HWDATA_A + 0x6c5c, HWDATA_A + 0x6c60, - HWDATA_A + 0x6c64, HWDATA_A + 0x6c68, HWDATA_A + 0x6c6c, - HWDATA_A + 0x6c70, HWDATA_A + 0x6c74, HWDATA_A + 0x6c78, - HWDATA_A + 0x6c7c, HWDATA_A + 0x6c80, HWDATA_A + 0x6c84, - HWDATA_A + 0x6c88, HWDATA_A + 0x6c8c, HWDATA_A + 0x6c90, - HWDATA_A + 0x6c94, HWDATA_A + 0x6c98, HWDATA_A + 0x6c9c, + FAST_DIE_MASK, FAST_DIE_MASK + 4, + FAST_DIE_MASK_2, FAST_DIE_MASK_2 + 4, + ALARM_MASK, ALARM_MASK + 4, ]; -/// Words the board file carries that the record leaves zero: the firmware ran, passed every -/// check and reached its top state without them (one boot each). The cross-check accepts the -/// file's value there. 27 words in the HardwareDataB +0x38..+0x638 block and two all-ones words +/// Words the reference machine's record carries that this one leaves zero: the firmware ran, +/// passed every check and reached its top state without them (one boot each). 27 words in +/// the HardwareDataB +0x38..+0x638 block and two all-ones words /// after it (zero in the G15S templates); the tail's thirteen readings (a single at /// HardwareDataA +0x6d34, twelve singles after +0x6d80). +#[cfg(test)] #[rustfmt::skip] pub(crate) const UNNEEDED: [usize; 42] = [ 0xe0, 0xe8, 0xf4, 0xf8, 0xfc, 0x100, 0x104, 0x108, 0x10c, 0x2d8, 0x2dc, 0x2e0, 0x2e4, @@ -247,7 +271,7 @@ pub(crate) const UNNEEDED: [usize; 42] = [ /// rows whose kind byte is `g`, their presence mask, then the `r` and `f` rows' keys with a /// count of one each. const SENSOR_ROWS: usize = HWDATA_A + 0x6c58; -const SENSOR_ROW_COUNT: usize = 6; +pub(crate) const SENSOR_ROW_COUNT: usize = 6; const SENSOR_G_KEYS: usize = HWDATA_A + 0x6cdc; const SENSOR_G_MASK: usize = HWDATA_A + 0x6cfc; const SENSOR_R_KEY: usize = HWDATA_A + 0x6d04; @@ -492,15 +516,13 @@ fn hwdata_b(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { put32(out, 0x10dc + 4 * k, t.primary[k] as u32)?; put32(out, 0x111c + 4 * k, t.secondary[k] as u32)?; if k > 0 { - // Relative power (a FROM_FILE word on the J516C) and relative boost frequency. - if top.mw != 0 { - put32( - out, - 0x105c + 4 * k, - u32::try_from(100 * u64::from(ps.mw) / u64::from(top.mw)) - .map_err(|_| "a relative power does not fit its u32 word")?, - )?; - } + // Relative power (the reference machine's) and relative boost frequency. + let relative = match i.reference.relative_power.get(k - 1) { + Some(&r) if k < t.top() => r, + None if k == t.top() => 100, + _ => return Err("the reference relative powers are for another state count"), + }; + put32(out, RELATIVE_POWER - 4 + 4 * k, relative)?; if ps.mhz > base.mhz { let span = u64::from(top.mhz - base.mhz); put32( @@ -565,7 +587,7 @@ fn hwdata_a(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { |gain: usize| f32_mul(p[gain], period_s).ok_or("a gain is not a finite normal single"); let max_scaled = 100 * t.top() as u32; let base_scaled = mul(100, p[BASE_PSTATE])?; - let max_power = i.max_power_mw; + let max_power = i.reference.max_power_mw; let max_power_f = f32_from_u32(max_power); let mut w = |off: usize, v: u32| put32(out, a(off), v); w(0x4, p[AIC_CLKS])?; @@ -644,7 +666,7 @@ fn hwdata_a(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { w(0x864, max_scaled)?; w(0x86c, F_0_8)?; w(0x870, F_0_2)?; - // Fast-die controller (its sensor mask is a FROM_FILE word). + // Fast-die controller (its sensor mask follows below). w(0x8bc, 4)?; w(0x8d0, F_1)?; w(0x8d8, per_period(FD_KI)?)?; @@ -737,15 +759,46 @@ fn hwdata_a(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { w(SE + 0x11c, F_65536)?; w(SE + 0x12c, 40)?; w(SE + 0x130, max_scaled)?; - // Past the second sensor mask (FROM_FILE), the alarm mask (FROM_FILE) and the end flag. + // Past the second sensor mask, the alarm mask and the end flag. w(0x1290, 125)?; w(0x1294, 1)?; w(0x1aa0, 1)?; w(0x3aa4, F_5)?; + // The MTR sensor masks: the fast-die mask at both places, then the alarm mask. + put64(out, FAST_DIE_MASK, i.reference.fast_die_mask)?; + put64(out, FAST_DIE_MASK_2, i.reference.fast_die_mask)?; + put64(out, ALARM_MASK, i.reference.alarm_mask)?; Ok(()) } -/// The tail's sensor keys from the six rows (which [`from_file`] fills). +/// The six sensor rows of the ADT's `aon-ptd` `ptd_data_table`: 12-byte entries of a +/// little-endian four-character key (`RT0g` is stored as `"RT0g"`) and two words. The rows are +/// the `RT` entries of kind `g` (four, in table order), then the one of kind `r` and the one +/// of kind `f`; any other count is refused rather than guessed. +pub(crate) fn sensor_rows(table: &[u8]) -> R<[[u8; 12]; SENSOR_ROW_COUNT]> { + if table.is_empty() || table.len() % 12 != 0 { + return Err("the aon-ptd sensor table is not whole 12-byte entries"); + } + let mut rows = [[0u8; 12]; SENSOR_ROW_COUNT]; + let mut n = 0; + for (kind, wanted) in [(b'g', 4), (b'r', 1), (b'f', 1)] { + let mut found = 0; + for e in table.chunks_exact(12) { + if e[..2] == *b"RT" && e[3] == kind { + let row = rows.get_mut(n).ok_or("too many GPU sensor rows in aon-ptd")?; + row.copy_from_slice(e); + n += 1; + found += 1; + } + } + if found != wanted { + return Err("aon-ptd does not hold four g, one r and one f GPU sensor row"); + } + } + Ok(rows) +} + +/// The tail's sensor keys from the six rows. fn tail(out: &mut [u8]) -> R<()> { let mut g = 0usize; for row in 0..SENSOR_ROW_COUNT { @@ -775,16 +828,10 @@ fn tail(out: &mut [u8]) -> R<()> { Ok(()) } -/// Copies the [`FROM_FILE`] words of `file` (the board file's record) into `out`. -pub(crate) fn from_file(out: &mut [u8], file: &[u8]) -> R<()> { - for &at in FROM_FILE.iter() { - put32(out, at, get32(file, at)?)?; - } - Ok(()) -} - -/// The first word at which the built record differs from the file's, except the [`UNNEEDED`] -/// words, where the file's value is accepted as long as the built word is zero. +/// The first word at which a built record differs from a machine's own record (a host test's +/// fixture), except the [`UNNEEDED`] words, where the machine's value is accepted as long as +/// the built word is zero. +#[cfg(test)] pub(crate) fn cross_check(built: &[u8], file: &[u8]) -> Option<(usize, u32, u32)> { let word = |s: &[u8], at: usize| -> u32 { let mut w = [0u8; 4]; @@ -799,23 +846,20 @@ pub(crate) fn cross_check(built: &[u8], file: &[u8]) -> Option<(usize, u32, u32) .find(|&(at, b, f)| b != f && !(b == 0 && UNNEEDED.contains(&at))) } -/// Builds the record into `out` (zeroed first) from the device-tree inputs; the [`FROM_FILE`] -/// words must have been copied in first ([`from_file`]) or are left as they are. +/// Builds the record into `out` (zeroed first) from the inputs. pub(crate) fn build(out: &mut [u8], i: &Inputs<'_>) -> R<()> { if out.len() != SIZE { return Err("record of another size"); } - let mut keep = [0u32; FROM_FILE.len()]; - for (k, &at) in FROM_FILE.iter().enumerate() { - keep[k] = get32(out, at)?; - } out.fill(0); let t = tables(i.main)?; hwdata_b(out, i, &t)?; hwdata_a(out, i, &t)?; - // The kept words win over what the rules put there (the relative powers, the masks). - for (k, &at) in FROM_FILE.iter().enumerate() { - put32(out, at, keep[k])?; + for (k, row) in i.sensor_rows.iter().enumerate() { + let at = SENSOR_ROWS + 12 * k; + out.get_mut(at..at + 12) + .ok_or("record too short")? + .copy_from_slice(row); } tail(out) } @@ -828,18 +872,13 @@ mod tests { get32(b, o).unwrap() } - fn opp(mhz: u32, mv: [u32; 4], mw: u32) -> Opp { - Opp { - mhz, - mv, - cells: 4, - mw, - } + fn opp(mhz: u32, mv: [u32; 4]) -> Opp { + Opp { mhz, mv, cells: 4 } } /// Operating points shaped like the J516C's: eight voltages, the upper five with two states. fn main_opps() -> Vec { - let mut v = vec![opp(0, [125; 4], 0)]; + let mut v = vec![opp(0, [125; 4])]; let groups: [(u32, [u32; 4], &[u32]); 8] = [ (605, [605, 605, 615, 615], &[338]), (680, [680, 680, 665, 665], &[618]), @@ -852,7 +891,7 @@ mod tests { ]; for (_, mv, mhz) in groups { for &m in mhz { - v.push(opp(m, mv, m * 40)); + v.push(opp(m, mv)); } } v @@ -888,6 +927,27 @@ mod tests { p } + /// Arbitrary reference words, distinct from the J516C's. + const TEST_REFERENCE: Reference = Reference { + max_power_mw: 54321, + relative_power: [0x1000, 0x1001, 0x1002, 0x1003, 0x1004, 0x1005, 0x1006], + fast_die_mask: 0x1111_2222_3333_4444, + alarm_mask: 0x5555_6666_7777_8888, + }; + + /// A sensor row: the key's characters in storage order, then two words. + fn row(key: &[u8; 4], id: u32, kind: u32) -> [u8; 12] { + let mut r = [0u8; 12]; + r[..4].copy_from_slice(key); + r[4..8].copy_from_slice(&id.to_le_bytes()); + r[8..].copy_from_slice(&kind.to_le_bytes()); + r + } + + fn rows() -> [[u8; 12]; SENSOR_ROW_COUNT] { + [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"].map(|k| row(k, 0x10001, 0x1401)) + } + fn inputs<'a>(main: &'a [Opp], cs: &'a [Opp], slots: &'a [IoSlot]) -> Inputs<'a> { Inputs { props: props(), @@ -901,7 +961,8 @@ mod tests { usc_base: 0x10_0000_0000, user_end: 0x2ff_ffff_8000, timestamps: 0xfffffc2ffc000000, - max_power_mw: 54321, + sensor_rows: rows(), + reference: &TEST_REFERENCE, } } @@ -927,9 +988,12 @@ mod tests { #[test] fn unneeded() { - // No UNNEEDED word is a FROM_FILE word; the cross-check accepts the file's value there - // only against a zero built word, and nothing else. - assert!(UNNEEDED.iter().all(|at| !FROM_FILE.contains(at))); + // No UNNEEDED word is a reference or sensor-row word; the cross-check accepts the + // file's value there only against a zero built word, and nothing else. + let rows = SENSOR_ROWS..SENSOR_ROWS + 12 * SENSOR_ROW_COUNT; + assert!(UNNEEDED + .iter() + .all(|at| !REFERENCE_WORDS.contains(at) && !rows.contains(at))); assert!(UNNEEDED.iter().all(|&at| at % 4 == 0 && at + 4 <= SIZE)); let mut file = vec![0u8; SIZE]; for (k, &at) in UNNEEDED.iter().enumerate() { @@ -957,9 +1021,10 @@ mod tests { assert_eq!(a(0x1aa0), HWDATA_A + 0x24b8); assert_eq!(a(0x3aa4), HWDATA_A + 0x64bc); assert_eq!(MAX_POWER, a(0x6a4)); - // FROM_FILE is sorted, unique and inside the record. - assert!(FROM_FILE.windows(2).all(|w| w[0] < w[1])); - assert!(FROM_FILE.iter().all(|&o| o % 4 == 0 && o + 4 <= SIZE)); + // REFERENCE_WORDS is sorted, unique and inside the record. + assert!(REFERENCE_WORDS.windows(2).all(|w| w[0] < w[1])); + assert!(REFERENCE_WORDS.iter().all(|&o| o % 4 == 0 && o + 4 <= SIZE)); + assert!(SENSOR_ROWS + 12 * SENSOR_ROW_COUNT <= SENSOR_G_KEYS); } #[test] @@ -995,7 +1060,6 @@ mod tests { mhz, mv: [mv, 0, 0, 0], cells: 1, - mw: 0, }) .collect(); let slots = [ @@ -1017,20 +1081,23 @@ mod tests { }, ]; let i = inputs(&main, &cs, &slots); - let mut r = vec![0u8; SIZE]; - // FROM_FILE words survive the build; the sensor rows feed the tail. - for (k, &at) in FROM_FILE.iter().enumerate() { - put32(&mut r, at, 0x1000 + k as u32).unwrap(); - } - // The file's maximum power is what the input repeats. - put32(&mut r, MAX_POWER, 54321).unwrap(); - let rows = [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"]; - for (k, row) in rows.iter().enumerate() { - put32(&mut r, SENSOR_ROWS + 12 * k, u32::from_le_bytes(**row)).unwrap(); - } + // Whatever the buffer held before is not part of the record. + let mut r = vec![0x5au8; SIZE]; build(&mut r, &i).unwrap(); let good = r.clone(); - assert_eq!(get(&r, 0x1060), 0x1000); + // The reference words: relative powers of states 1..7, the maximum power, the masks. + for k in 0..RELATIVE_STATES { + assert_eq!(get(&r, 0x1060 + 4 * k), 0x1000 + k as u32); + } + assert_eq!(get(&r, FAST_DIE_MASK), 0x3333_4444); + assert_eq!(get(&r, FAST_DIE_MASK + 4), 0x1111_2222); + assert_eq!(get(&r, FAST_DIE_MASK_2), 0x3333_4444); + assert_eq!(get(&r, FAST_DIE_MASK_2 + 4), 0x1111_2222); + assert_eq!(get(&r, ALARM_MASK), 0x7777_8888); + assert_eq!(get(&r, ALARM_MASK + 4), 0x5555_6666); + // The sensor rows as given. + assert_eq!(&r[SENSOR_ROWS..SENSOR_ROWS + 12], &rows()[0]); + assert_eq!(&r[SENSOR_ROWS + 60..SENSOR_ROWS + 72], &rows()[5]); assert_eq!(get(&r, 0x30f0), 0); // Identity, IOMapping records, flags. assert_eq!(get(&r, 0xa28), 0x6031); @@ -1099,8 +1166,15 @@ mod tests { let mut j = inputs(&main, &cs, &slots); j.empty_writable = &[0]; assert!(build(&mut r, &j).is_err()); + let zero = Reference { + max_power_mw: 0, + ..TEST_REFERENCE + }; + let mut j = inputs(&main, &cs, &slots); + j.reference = &zero; + assert!(build(&mut r, &j).is_err()); let mut j = inputs(&main, &cs, &slots); - j.max_power_mw = 0; + j.sensor_rows[0] = row(b"RT0x", 1, 1); assert!(build(&mut r, &j).is_err()); let mut j = inputs(&main, &cs, &slots); j.zones = &[100, 101, 1]; @@ -1127,34 +1201,49 @@ mod tests { let mut j = inputs(&main, &cs, &slots); j.zones = &[5000, 200, 0x4000_0000]; assert_eq!(refused(&j), Err(OVERFLOW)); - let mut skewed = main.clone(); - skewed[1].mw = u32::MAX; - skewed[13].mw = 1; + // The seven relative powers fit eight states above state 0 and nothing else. + let fewer = &main[..PERF_STATES - 2]; + assert_eq!(tables(fewer).unwrap().top(), 7); assert_eq!( - refused(&inputs(&skewed, &cs, &slots)), - Err("a relative power does not fit its u32 word") + refused(&inputs(fewer, &cs, &slots)), + Err("the reference relative powers are for another state count") ); - // The admission cross-check as initdata::blob() composes it: the FROM_FILE words from - // the file, the rest built, then the first differing word against the file. A file the - // inputs reproduce passes; a rule word the file holds differently is found at its - // offset; a FROM_FILE word follows the file and is never a difference. - let first_difference = super::super::words::first_difference; - let checked = |file: &[u8]| { - let mut built = vec![0u8; SIZE]; - from_file(&mut built, file).unwrap(); - build(&mut built, &i).unwrap(); - (first_difference(&built, file), built) - }; - assert_eq!(checked(&good).0, None); - let mut file = good.clone(); - put32(&mut file, a(0x4c), 801).unwrap(); - assert_eq!(checked(&file).0, Some((a(0x4c), 800, 801))); - let mut file = good.clone(); - assert!(FROM_FILE.contains(&0x1064)); - put32(&mut file, 0x1064, get(&good, 0x1064) ^ 0x5a).unwrap(); - let (difference, built) = checked(&file); - assert_eq!(difference, None); - assert_eq!(get(&built, 0x1064), get(&good, 0x1064) ^ 0x5a); + } + + #[test] + fn sensor_rows_from_the_ptd_table() { + // A table shaped like the J516C's: other sensors around the GPU's, the f row first. + let mut table = Vec::new(); + for r in [ + row(b"YD0C", 0x5e0001, 0x1040405), + row(b"RT0a", 0x6b0001, 0x13), + row(b"RT0f", 0xcb0001, 0x1601), + row(b"RE0g", 0xaa0001, 0xff01), + row(b"RT0g", 0xab0001, 0x1401), + row(b"RT1g", 0x8d0001, 0x1401), + row(b"RT2g", 0x8b0001, 0x1401), + row(b"RT3g", 0xad0001, 0x1401), + row(b"RT0p", 0x6d0001, 0x13), + row(b"RT0r", 0xeb0001, 0x1401), + row(b"XEXE", 0x3c0001, 0xff00), + ] { + table.extend_from_slice(&r); + } + let got = sensor_rows(&table).unwrap(); + let keys: Vec<&[u8]> = got.iter().map(|r| &r[..4]).collect(); + assert_eq!(keys, [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"]); + assert_eq!(got[5], row(b"RT0f", 0xcb0001, 0x1601)); + assert_eq!(got[4], row(b"RT0r", 0xeb0001, 0x1401)); + // Refused: no table, a partial entry, a missing row, a fifth g row, a second f row. + assert!(sensor_rows(&[]).is_err()); + assert!(sensor_rows(&table[..table.len() - 1]).is_err()); + assert!(sensor_rows(&table[12..table.len() - 12]).is_ok()); + assert!(sensor_rows(&table[..9 * 12]).is_err()); + for extra in [row(b"RT4g", 1, 1), row(b"RT1f", 1, 1)] { + let mut t = table.clone(); + t.extend_from_slice(&extra); + assert!(sensor_rows(&t).is_err()); + } } // --- The fixture: a saved copy of a J516C's device tree and its private board file. --- @@ -1236,14 +1325,12 @@ mod tests { for (c, m) in mv.iter_mut().enumerate().take(cells) { *m = be32(uv, 4 * c).div_ceil(1000); } - let mw = prop(props, "opp-microwatt").map_or(0, |w| be32(w, 0) / 1000); ( p.clone(), Opp { mhz: (hz / 1_000_000) as u32, mv, cells, - mw, }, ) }) @@ -1262,16 +1349,8 @@ mod tests { mhz: 24, mv: [635, 0, 0, 0], cells: 1, - mw: 0, }]; let mut seed = vec![0u8; SIZE]; - put32(&mut seed, MAX_POWER, 54321).unwrap(); - for (k, key) in [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"] - .iter() - .enumerate() - { - put32(&mut seed, SENSOR_ROWS + 12 * k, u32::from_le_bytes(**key)).unwrap(); - } assert_eq!(build(&mut seed, &inputs(&main, &cs, &[])), Ok(())); for prop in 0..PROPS.len() { for value in [ @@ -1289,16 +1368,59 @@ mod tests { } } - /// Builder on the saved device tree + the file's FROM_FILE words == the file's hardware - /// record, byte for byte. Needs `G15_J516C_FIXTURES=` with `j516c.dtb` and - /// `t6031-j516c-board.bin` (off the tree): ignored by default, `--ignored` runs it. + /// A property of an ADT node found by its path of `name`s from the root: nodes of u32 + /// property and child counts, properties of a 32-byte name, a u32 size (top bit a flag) + /// and the value padded to 4 bytes. + fn adt_prop<'a>(adt: &'a [u8], path: &[&str], name: &str) -> Option<&'a [u8]> { + fn le(b: &[u8], o: usize) -> usize { + u32::from_le_bytes(b[o..o + 4].try_into().unwrap()) as usize + } + // The node at `at`: its properties, its children's offsets and the offset past it. + fn node(adt: &[u8], at: usize) -> (Vec<(&[u8], &[u8])>, Vec, usize) { + let (np, nc) = (le(adt, at), le(adt, at + 4)); + let mut o = at + 8; + let mut props = Vec::new(); + for _ in 0..np { + let n = &adt[o..o + 32]; + let n = &n[..n.iter().position(|&b| b == 0).unwrap_or(32)]; + let size = le(adt, o + 32) & 0x7fff_ffff; + props.push((n, &adt[o + 36..o + 36 + size])); + o += 36 + size.next_multiple_of(4); + } + let mut children = Vec::new(); + for _ in 0..nc { + children.push(o); + o = node(adt, o).2; + } + (props, children, o) + } + let mut at = 0; + for want in path { + let (_, children, _) = node(adt, at); + let named = |c: usize| { + let (props, _, _) = node(adt, c); + props.iter().any(|(n, v)| { + *n == b"name" && v.split(|&b| b == 0).next() == Some(want.as_bytes()) + }) + }; + at = *children.iter().find(|&&c| named(c))?; + } + node(adt, at).0.into_iter().find(|(n, _)| *n == name.as_bytes()).map(|(_, v)| v) + } + + /// Builder on the saved device tree and ADT with the J516C reference words == the J516C + /// reference machine's own hardware record, byte for byte except the [`UNNEEDED`] words. + /// Needs `G15_J516C_FIXTURES=` with `j516c.dtb` (`/sys/firmware/fdt`), `adt.bin` (the + /// boot loader's ADT) and `t6031-j516c-board.bin` (that machine's record), all off the + /// tree: ignored by default, `--ignored` runs it. #[test] #[ignore = "needs the off-tree J516C fixtures (G15_J516C_FIXTURES)"] fn j516c_fixture() { let dir = std::env::var_os("G15_J516C_FIXTURES") - .expect("G15_J516C_FIXTURES names the directory with j516c.dtb and the board file"); + .expect("G15_J516C_FIXTURES names the directory with j516c.dtb, adt.bin, the record"); let dir = std::path::Path::new(&dir); let dtb = std::fs::read(dir.join("j516c.dtb")).unwrap(); + let adt = std::fs::read(dir.join("adt.bin")).unwrap(); let blob = std::fs::read(dir.join("t6031-j516c-board.bin")).unwrap(); assert_eq!(&blob[..8], b"T6031BRD"); let at = 16 + 3 * 8; @@ -1389,10 +1511,13 @@ mod tests { usc_base: 0x10_0000_0000, user_end: 0x2ff_ffff_8000, timestamps: 0xfffffc2ffc000000, - max_power_mw: get(file, MAX_POWER), + sensor_rows: sensor_rows( + adt_prop(&adt, &["arm-io", "aon-ptd"], "ptd_data_table").expect("aon-ptd table"), + ) + .unwrap(), + reference: &J516C, }; let mut built = vec![0u8; SIZE]; - from_file(&mut built, file).unwrap(); build(&mut built, &i).unwrap(); assert_eq!(cross_check(&built, file), None, "(offset, built, file)"); for &at in UNNEEDED.iter() { diff --git a/drivers/gpu/drm/asahi/g15/board/words.rs b/drivers/gpu/drm/asahi/g15/board/words.rs index 4c5082465d428..f879d9b671672 100644 --- a/drivers/gpu/drm/asahi/g15/board/words.rs +++ b/drivers/gpu/drm/asahi/g15/board/words.rs @@ -235,6 +235,7 @@ pub(crate) fn power( /// The first word (4-byte aligned offset) at which `a` and `b` differ, with both values; a /// missing word reads 0. `None` when they are equal. +#[cfg(test)] pub(crate) fn first_difference(a: &[u8], b: &[u8]) -> Option<(usize, u32, u32)> { let word = |s: &[u8], at: usize| -> u32 { let mut w = [0u8; 4]; @@ -584,7 +585,7 @@ mod tests { /// Builder on the saved device tree == the private file's Globals and Power, byte for byte. /// Needs `G15_J516C_FIXTURES=` with `j516c.dtb` (`/sys/firmware/fdt` of the machine) - /// and `t6031-j516c-board.bin` (its board file); both stay off the tree, so the test is + /// and `t6031-j516c-board.bin` (its own images); both stay off the tree, so the test is /// ignored by default (`--ignored` runs it) and fails, rather than passes, without them. #[test] #[ignore = "needs the off-tree J516C fixtures (G15_J516C_FIXTURES)"] @@ -595,7 +596,7 @@ mod tests { let dtb = std::fs::read(dir.join("j516c.dtb")).unwrap(); let blob = std::fs::read(dir.join("t6031-j516c-board.bin")).unwrap(); - // The board file: magic, version, count, 3 x (kind, size), images (board/blob.rs). + // The machine's images: magic, version, count, 3 x (kind, size), then the images. assert_eq!(&blob[..8], b"T6031BRD"); assert_eq!(get32(&blob, 8), 1); assert_eq!(get32(&blob, 12), 3); diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index 7a06aa64ffbc3..70decd06fca0d 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -61,8 +61,10 @@ pub(crate) mod glue { fn asahi_g15_machine(compatible: *const core::ffi::c_char) -> bool; fn asahi_g15_hold_power(dev: *mut core::ffi::c_void); fn asahi_g15_release_power(dev: *mut core::ffi::c_void); - fn asahi_g15_module_get(); + fn asahi_g15_module_try_get() -> bool; + fn asahi_g15_module_put(); pub(crate) fn asahi_g15_reprobe(); + pub(crate) fn asahi_g15_report_loader_gpu(); fn asahi_g15_adt_phram(start: *mut u64, size: *mut u64) -> core::ffi::c_int; fn asahi_g15_ram_disjoint(start: u64, size: u64) -> bool; fn asahi_g15_ram_within(start: u64, size: u64) -> bool; @@ -70,7 +72,9 @@ pub(crate) mod glue { fn asahi_g15_ioremap_wc(start: u64, size: usize) -> *mut core::ffi::c_void; fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; pub(crate) fn asahi_g15_pmp_prepare(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + pub(crate) fn asahi_g15_display_prepare(dev: *mut core::ffi::c_void) -> core::ffi::c_int; pub(crate) fn asahi_g15_pmp_get(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + pub(crate) fn asahi_g15_pmp_ready() -> core::ffi::c_int; pub(crate) fn asahi_g15_pmp_put(dev: *mut core::ffi::c_void); fn asahi_g15_reserved_uat(index: u32, start: u64, size: u64) -> bool; } @@ -120,12 +124,17 @@ pub(crate) mod glue { RETAINED.load(Ordering::Acquire) } - /// Pins this module until reboot, once (`__module_get(THIS_MODULE)`). - pub(crate) fn retain_module() { - if !RETAINED.swap(true, Ordering::AcqRel) { - // SAFETY: The module is live while its code runs; the reference is never put. - unsafe { asahi_g15_module_get() }; - } + /// Pins this module until reboot, once (`try_module_get(THIS_MODULE)`, see `retain.rs`). + /// ENODEV, with nothing pinned or marked, once the module's removal has begun. + pub(crate) fn retain_module() -> Result { + crate::g15::retain::retain( + &RETAINED, + // SAFETY: Takes a reference to this module unless its removal has begun. + || unsafe { asahi_g15_module_try_get() }, + // SAFETY: Drops only the reference this call took; the winner's is never put. + || unsafe { asahi_g15_module_put() }, + ) + .map_err(|_| ENODEV) } /// The phram "adt" export: start and size. @@ -478,6 +487,17 @@ pub(crate) fn admit( } else if !glue::machine(cfg.machine) { dev_err!(dev, "G15: not a {} machine\n", cfg.name); return Err(ENODEV); + } else { + // No PMP vote here, but the first start follows the native display all the same. + // SAFETY: `dev` is the bound GPU device. + let ret = unsafe { glue::asahi_g15_display_prepare(dev.as_raw().cast()) }; + if ret != 0 { + let err = Error::from_errno(ret); + if err != EPROBE_DEFER { + dev_err!(dev, "native display not admitted: {:?}\n", err); + } + return Err(err); + } } let adt = adt::read(dev, cfg) .inspect_err(|e| dev_err!(dev, "GPU regions not admitted: {:?}\n", e))?; @@ -674,8 +694,16 @@ pub(crate) fn start( return Err(e); } + // Pin the module before anything firmware-visible. Once its removal has begun this fails + // and the firmware is left untouched, cleaned up as for a failed self-check: the vote is + // released and the session stays unretained. + if let Err(e) = glue::retain_module() { + dev_err!(d, "G15: the module is being removed; the firmware is not started\n"); + // Mapped with cached firmware classes: never unmapped. + core::mem::forget(objects); + return Err(e); + } // From here on the firmware is touched: nothing it can reach is freed, the vote included. - glue::retain_module(); vote.keep(); let rtkit = match Rtkit::new(&d, shared) { Ok(rtkit) => rtkit, diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index 6fc1e03fb637c..66e52e6981197 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -146,6 +146,11 @@ pub(crate) struct Context<'a> { pub(crate) usc_base: u64, } +struct CheckedContext<'a> { + vm: &'a mut mmu::VmReadGuard, + usc_base: u64, +} + /// A stream whose timestamps and ordinals are resolved ([`prepare`]), before its spans are /// checked ([`Prepared::validate`]). pub(crate) struct Prepared<'a> { @@ -377,7 +382,13 @@ impl Timestamps<'_> { /// Checks that `[start, end)` (rounded out to pages) is user memory mapped with the requested /// GPU access: a driver range ENOENT, outside the user range EINVAL, an unmapped page ENOENT, a /// page without the access EACCES. -pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) -> Result { +pub(crate) fn cover( + vm: &mut mmu::VmReadGuard, + start: u64, + end: u64, + read: bool, + write: bool, +) -> Result { let mask = mmu::UAT_PGMSK as u64; if end <= start { return Err(EINVAL); @@ -407,7 +418,11 @@ pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) } /// Attachment spans, 128-byte granular, actually mapped with `write` or read access. -fn check_attachments(vm: &mmu::Vm, list: &[uapi::drm_asahi_attachment], write: bool) -> Result { +fn check_attachments( + vm: &mut mmu::VmReadGuard, + list: &[uapi::drm_asahi_attachment], + write: bool, +) -> Result { for attachment in list { let end = attachment .pointer @@ -467,8 +482,8 @@ fn geometry_bytes(cmd: &uapi::drm_asahi_cmd_render) -> (u64, u64) { } /// Per-command render admission. -fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { - let vm = context.vm; +fn check_render(context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { + let vm = &mut *context.vm; let allowed = PROCESS_EMPTY_TILES | NO_VERTEX_CLUSTERING | RSRC_SPEC_HI @@ -552,11 +567,11 @@ fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Resu /// Per-command compute admission. fn check_compute( - context: &Context<'_>, + context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_compute, attachments: &[uapi::drm_asahi_attachment], ) -> Result { - let vm = context.vm; + let vm = &mut *context.vm; let (base, end) = (cmd.cdm_ctrl_stream_base, cmd.cdm_ctrl_stream_end); if cmd.flags != 0 || cmd.sampler_count != 0 @@ -650,6 +665,12 @@ impl Prepared<'_> { counts, timestamps, } = self; + // On error, unlock before the last timestamp mapping can drop and unmap. + let mut access = context.vm.read_lock()?; + let mut context = CheckedContext { + vm: &mut access, + usc_base: context.usc_base, + }; let mut commands = KVec::with_capacity(resolved.len(), GFP_KERNEL)?; let mut compute = KVec::new(); let mut index = 0; @@ -658,12 +679,12 @@ impl Prepared<'_> { match entry.kind { SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS => { // Validated readable, never serialized on G15. - check_attachments(context.vm, &attachment_list(entry.payload)?, false)?; + check_attachments(&mut *context.vm, &attachment_list(entry.payload)?, false)?; } SET_COMPUTE_ATTACHMENTS => compute = attachment_list(entry.payload)?, CMD_RENDER => { let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?; - check_render(context, &cmd)?; + check_render(&mut context, &cmd)?; commands.push( Command::Render(KBox::new( RenderCommand { @@ -678,7 +699,7 @@ impl Prepared<'_> { } CMD_COMPUTE => { let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?; - check_compute(context, &cmd, &compute)?; + check_compute(&mut context, &cmd, &compute)?; let slots = resolved.get(index).ok_or(EINVAL)?; commands.push( Command::Compute(KBox::new( diff --git a/drivers/gpu/drm/asahi/g15/encode.rs b/drivers/gpu/drm/asahi/g15/encode.rs index e46ecd05bdf78..a62cca8613b33 100644 --- a/drivers/gpu/drm/asahi/g15/encode.rs +++ b/drivers/gpu/drm/asahi/g15/encode.rs @@ -64,12 +64,23 @@ const fn align(n: u64, a: u64) -> u64 { } /// Firmware stamp value of ordinal `n` on the queue with this seed. -/// Stamps step by 0x100 through 0xffffff values and never read 0. +/// Stamps step by 0x100, including zero when the counter wraps. pub(crate) const fn stamp_seed(seed: u32, n: u64) -> u32 { - let base = (seed / 0x100) as u64; - ((((base - 1 + n % 0xffffff) % 0xffffff) + 1) * 0x100) as u32 + seed.wrapping_add((n as u32).wrapping_mul(0x100)) } +/// Ordinals after which [`stamp_seed`] repeats: 2^32 / 0x100. +pub(crate) const STAMP_PERIOD: u64 = 1 << 24; + +/// Effective render ordinal at which the TA stamp wraps (0xffffff00 -> 0): 0x860000 with the TA +/// seed. A wrap that skipped zero stopped the firmware ("DAG: Non Sequential Stamp Updates"); +/// the renumbering keeps the stamps away from it. The FS stamp wraps later (0xc30000). +pub(crate) const RENDER_WRAP: u64 = 0xffffff - (hw::engine::TA_SEED / 0x100) as u64 + 1; + +/// Compute ordinal at which the compute stamp wraps (0xffffff00 -> 0): 0x3f0000 (4128768 +/// computes) with the compute seed, where the firmware may stop as at the TA stamp wrap. +pub(crate) const COMPUTE_WRAP: u64 = STAMP_PERIOD - (hw::engine::COMPUTE_SEED / 0x100) as u64; + /// Compute stamp of ordinal `n`. pub(crate) const fn stamp(n: u64) -> u32 { stamp_seed(hw::engine::COMPUTE_SEED, n) @@ -102,8 +113,9 @@ pub(crate) struct ComputeRefs { pub(crate) scratch: [u64; 5], } -/// Encodes compute command `c` as ordinal `n`. The engine copies the result into the slot -/// after zeroing the slot's images. +/// Encodes compute command `c` as ordinal `n` with stamp `t` (the engine's renumbering moves +/// the stamp off [`stamp`]`(n)`). The engine copies the result into the slot after zeroing the +/// slot's images. #[allow(clippy::too_many_arguments)] pub(crate) fn encode_compute( r: &ComputeRefs, @@ -113,12 +125,12 @@ pub(crate) fn encode_compute( ts_va: &[u64; 2], ctx: u32, n: u64, + t: u32, fault_mask: u32, w: &mut [u8; compute::WORK_SIZE], q: &mut [u8; compute::SEQ_SIZE], ) { let (wv, qv, h, pre) = (r.work, r.qinfo, r.hwdata, r.preempt_gva); - let t = stamp(n); let att_count = att.len().min(hw::sizes::MAX_ATTACHMENTS); // The saved-state pointers into the preemption buffer and the two 0x1c words; a SoC // encoded "fresh" carries zero in all six, as a first command does. diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index 51b3cfd10d594..1fa5c60bd1122 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -155,10 +155,10 @@ pub(crate) struct Flight { pub(crate) fs_start: u32, /// Channel producers (TA, FS, compute) after the prefix's records. pub(crate) rec: [u32; 3], - /// Compute member j's FW stamp target, and the FS target it waits on directly (0: none): + /// Compute member j's FW stamp target, and the FS target it waits on directly: /// its predecessors in the prefix (for the compute-hang check). pub(crate) cstamp: [u32; sizes::COMPUTE_SLOTS], - pub(crate) cwait: [u32; sizes::COMPUTE_SLOTS], + pub(crate) cwait: [Option; sizes::COMPUTE_SLOTS], /// Compute slots, pass slots and clustering buffers it holds. pub(crate) claims: Claims, /// The prefix rebuilds the render allocators (priming); nothing is published behind it. @@ -189,7 +189,7 @@ impl Flight { fs_start: 0, rec: [0; 3], cstamp: [0; sizes::COMPUTE_SLOTS], - cwait: [0; sizes::COMPUTE_SLOTS], + cwait: [None; sizes::COMPUTE_SLOTS], claims: Claims::new(), rebuild: false, doomed: false, @@ -364,6 +364,12 @@ pub(crate) struct Engine { /// Host compute ordinal and draw ordinal of the last publication. pub(crate) ordinal: u64, pub(crate) draw: u64, + /// Added to the draw ordinal before computing the TA/FS stamps; moved by + /// `renumber_render_stamps()`. + pub(crate) stamp_base: u64, + /// Added to the compute ordinal before computing the compute stamp; moved by + /// `renumber_compute_stamps()`. + pub(crate) compute_stamp_base: u64, /// Newest published receipts, the tail. pub(crate) target: u32, pub(crate) head: u32, @@ -385,9 +391,11 @@ pub(crate) struct Engine { pub(crate) rebuild_live: bool, /// A ready prefix waits only for a record's consumption: repoll sooner. pub(crate) chan_wait: bool, + /// A ready job waits for an ASID held by a completing failed job. + pub(crate) slot_wait: bool, /// Progress tracking of the oldest prefix. pub(crate) progress_flight: u64, - pub(crate) progress: [u32; 3], + pub(crate) progress: [Option; 3], pub(crate) stats: Stats, /// Phase timers (debug_flags bit 49). pub(crate) prof: Prof, @@ -409,6 +417,8 @@ impl Engine { shared: None, ordinal: 0, draw: 0, + stamp_base: 0, + compute_stamp_base: 0, target: 0, head: 0, ta_target: 0, @@ -424,8 +434,9 @@ impl Engine { render_registered: false, rebuild_live: false, chan_wait: false, + slot_wait: false, progress_flight: 0, - progress: [0; 3], + progress: [None; 3], stats: Stats::default(), prof: Prof::default(), debugfs: None, @@ -515,8 +526,12 @@ impl Firmware { if self.engine.flights.capacity() < sizes::FLIGHTS { self.engine.flights.reserve(sizes::FLIGHTS, GFP_KERNEL)?; } + // A job arriving at an idle engine: the PB reclaim, if due, happens before its + // publication (idle polls are not re-armed, so this is where a long idle ends). + self.reclaim_check(); self.engine.pending.push(job.clone(), GFP_KERNEL)?; job.progress.set_pending(true); + self.note_idle(); let q = &job.queue; let owned = self.engine.queue_published(&job) + self @@ -542,6 +557,7 @@ impl Firmware { let mut result = Ok(()); self.engine.chan_wait = false; + self.engine.slot_wait = false; // Nothing is published while a hang request is armed. while self.recovery.failed().is_none() && !self.engine.frozen @@ -552,12 +568,17 @@ impl Firmware { break; }; let job = self.engine.pending[index].clone(); + // Inside a renumbering margin nothing is published behind a flight: the flights + // drain and the next publication renumbers the stamps while idle (once every ~8.7M + // renders or ~4.0M computes with margins of 100000). + let renumber_due = self.renumber_due(); if let Some(tail) = self.engine.flights.last() { if self.engine.flights.len() >= max_inflight || !Arc::ptr_eq(&job.vm, &tail.job.vm) || self.recovery.reinit || self.recovery.fs_resync || self.engine.rebuild_live + || renumber_due { self.engine.stats.hold[0] += (self.engine.flights.len() >= max_inflight) as u64; break; @@ -588,7 +609,7 @@ impl Firmware { dev_err!( self.graph.dev.as_ref(), "G15: publication of ctx {} failed after a queue WRITE ({:?})\n", - job.vm.slot, + job.slot(), e ); complete(done, &job, Err(EIO)); @@ -601,6 +622,9 @@ impl Firmware { } } } + // A publication that left nothing published and nothing pending (jobs that failed + // before any visible record) starts the idle. + self.note_idle(); result } @@ -668,6 +692,15 @@ impl Firmware { /// yet (no free slot, or a channel record still unconsumed); EIO once a queue WRITE /// advanced; any other error means nothing was published. fn publish_prefix(&mut self, job: &Arc, done: &mut Completions) -> Result { + if let Err(error) = job.bind() { + if error == ENOSPC { + // A failed job can keep its binding until scheduler references drop after + // this pass. Leave healthy work pending and retry without the device lock. + self.engine.slot_wait = true; + return Err(EAGAIN); + } + return Err(error); + } let mut f = Flight::new(job.clone()); let result = self.publish_prefix_into(job, &mut f); match result { @@ -675,7 +708,11 @@ impl Firmware { // Never published: its storage is free again, and horizon jobs taken into it // fail with it. self.slots.release(&mut f.claims); + if job.progress.next() == job.progress.sent() { + job.release_binding(); + } for j2 in f.group.iter() { + j2.release_binding(); complete(done, j2, Err(e)); } } @@ -684,6 +721,94 @@ impl Firmware { result } + /// The TA stamp is inside its renumbering margin. + fn render_renumber_due(&self) -> bool { + let margin = params::stamp_reseed_margin(); + let ordinal = (self.engine.draw + self.engine.stamp_base) % encode::STAMP_PERIOD; + margin != 0 && ordinal + margin >= encode::RENDER_WRAP + } + + /// The compute stamp is inside its renumbering margin. + fn compute_renumber_due(&self) -> bool { + let margin = params::compute_reseed_margin(); + let ordinal = (self.engine.ordinal + self.engine.compute_stamp_base) % encode::STAMP_PERIOD; + margin != 0 && ordinal + margin >= encode::COMPUTE_WRAP + } + + /// Either stamp is inside its renumbering margin. + fn renumber_due(&self) -> bool { + self.render_renumber_due() || self.compute_renumber_due() + } + + /// Computes and renders the current numbering has left before its compute and TA stamps + /// wrap (0 once wrapped), for `asahi-g15/stats`. + pub(crate) fn stamps_left(&self) -> (u64, u64) { + let period = encode::STAMP_PERIOD; + let c = (self.engine.ordinal + self.engine.compute_stamp_base) % period; + let r = (self.engine.draw + self.engine.stamp_base) % period; + ( + encode::COMPUTE_WRAP.saturating_sub(c), + encode::RENDER_WRAP.saturating_sub(r), + ) + } + + /// Renumbers the TA/FS stamps from their seeds before the TA stamp wraps, after 0x860000 + /// renders (about six hours of desktop), where the firmware may stop. Only called with + /// nothing in flight: the last flight retired with every stamp equal to its target, so both + /// queues are idle and the next render waits on exactly the seeds written here. + fn renumber_render_stamps(&mut self) -> Result { + if !self.render_renumber_due() { + return Ok(()); + } + let ordinal = (self.engine.draw + self.engine.stamp_base) % encode::STAMP_PERIOD; + let r = &self.graph.render; + for (e, seed) in [ids::TA_SEED, ids::FS_SEED].into_iter().enumerate() { + r.user_stamp[e].w32(0, seed)?; + r.fw_stamp[e].w32(0, seed)?; + } + // The next render (draw + 1) counts as ordinal 1 again: modulo the period of + // `stamp_seed`, whose ordinal 0 is the seeds just written. + let period = encode::STAMP_PERIOD; + self.engine.stamp_base = (period - self.engine.draw % period) % period; + self.engine.ta_target = ids::TA_SEED; + self.engine.fs_target = ids::FS_SEED; + self.engine.tail_fs = ids::FS_SEED; + dev_info!( + self.graph.dev.as_ref(), + "G15: TA/FS stamps renumbered from the seeds at draw {} (ordinal {:#x})\n", + self.engine.draw, + ordinal + ); + Ok(()) + } + + /// Renumbers the compute stamp from its seed before it wraps, after 0x3f0000 computes, + /// where the firmware may stop. Like the TA/FS renumbering only with nothing in flight: the + /// compute queue is idle with both stamps at its target. The stamps are written back as + /// `ComputeGraph::init()` leaves them with no stamp base, so the next compute is ordinal 1 + /// on the stamps of a fresh boot. + fn renumber_compute_stamps(&mut self) -> Result { + if !self.compute_renumber_due() { + return Ok(()); + } + let ordinal = (self.engine.ordinal + self.engine.compute_stamp_base) % encode::STAMP_PERIOD; + let g = &self.graph.compute; + g.user_stamp.w32(0, ids::COMPUTE_SEED)?; + g.fw_stamp.w32(0, 0)?; + // The next compute (ordinal + 1) counts as ordinal 1 again. + let period = encode::STAMP_PERIOD; + self.engine.compute_stamp_base = (period - self.engine.ordinal % period) % period; + self.engine.target = 0; + self.engine.tail_ct = 0; + dev_info!( + self.graph.dev.as_ref(), + "G15: compute stamp renumbered from the seed at compute {} (ordinal {:#x})\n", + self.engine.ordinal, + ordinal + ); + Ok(()) + } + fn publish_prefix_into(&mut self, job: &Arc, f: &mut Flight) -> Result { // The PB backing is mapped into the job's VM before anything of the prefix is changed. // Nothing is published when that fails, so the error must not be EIO (which reports a @@ -691,6 +816,10 @@ impl Firmware { self.map_pb(&job.vm) .map_err(|e| if e == EIO { EFAULT } else { e })?; let overlap = !self.engine.flights.is_empty(); + if !overlap { + self.renumber_render_stamps()?; + self.renumber_compute_stamps()?; + } let mut mixed = tuning::MIXED_PREFIX; let mut pf = Prefix { n: self.engine.ordinal, @@ -892,11 +1021,22 @@ impl Firmware { hz_stop = Some(1); break; } - for k in 0..n2 { - match command(&j2, k).ok_or(EINVAL)? { - Command::Compute(c) => self.prefix_compute(&j2, c, &mut pf, f, false)?, - Command::Render(_) => return Err(EINVAL), + j2.bind()?; + let encoded = (|| { + for k in 0..n2 { + match command(&j2, k).ok_or(EINVAL)? { + Command::Compute(c) => { + self.prefix_compute(&j2, c, &mut pf, f, false)? + } + Command::Render(_) => return Err(EINVAL), + } } + Ok(()) + })(); + if let Err(error) = encoded { + // Not in the horizon yet and nothing of it reached the firmware. + j2.release_binding(); + return Err(error); } computes += n2; j2.progress.set_sent(n2); @@ -951,7 +1091,7 @@ impl Firmware { self.engine.head = pf.c_head; } if renders != 0 { - f.ta_target = encode::stamp_seed(ids::TA_SEED, pf.draw); + f.ta_target = encode::stamp_seed(ids::TA_SEED, pf.draw + self.engine.stamp_base); self.engine.ta_target = f.ta_target; f.fs_target = pf.last_fs; self.engine.fs_target = pf.last_fs; @@ -1050,7 +1190,7 @@ impl Firmware { Queue, self.graph.dev, "G15: prefix ctx={} computes {} (n={} head {}) renders {} (draw {} ta/fs head {}/{}) behind {}: {:?}\n", - job.vm.slot, + job.slot(), computes, pf.n, pf.c_head, @@ -1087,8 +1227,8 @@ impl Firmware { later_render: bool, ) -> Result { let n = pf.n + 1; - let t = encode::stamp(n); - let ctx = job.vm.slot; + let t = encode::stamp(n + self.engine.compute_stamp_base); + let ctx = job.slot(); let ts = &cmd.timestamps; let mut lap = Lap::start(); // Nothing published yet on failure. @@ -1131,6 +1271,7 @@ impl Firmware { ts, ctx, n, + t, tuning::COMPUTE_FAULT_MASK, image(w)?, image(q)?, @@ -1184,11 +1325,7 @@ impl Firmware { } let j = pf.computes as usize; f.cstamp[j] = t; - f.cwait[j] = if pf.prev == Prev::Render { - pf.last_fs - } else { - 0 - }; + f.cwait[j] = (pf.prev == Prev::Render).then_some(pf.last_fs); g.ring.w64(8 * pf.c_head as usize, s.work.va)?; pf.c_head = (pf.c_head + 1) % sizes::QUEUE_CAP; lap.lap(&mut self.engine.prof.write_ns); @@ -1233,9 +1370,10 @@ impl Firmware { let k = f.claims.pass[pf.renders as usize] as usize; let n = pf.draw + 1; let usc = job.queue.usc_base; - let ctx = job.vm.slot; - let tta = encode::stamp_seed(ids::TA_SEED, n); - let tfs = encode::stamp_seed(ids::FS_SEED, n); + let ctx = job.slot(); + let sn = n + self.engine.stamp_base; + let tta = encode::stamp_seed(ids::TA_SEED, sn); + let tfs = encode::stamp_seed(ids::FS_SEED, sn); // Early tiling: a later render member after a render of the same prefix whose render // dependencies are fragment-only (FRAGMENT_BARRIERS) and that has no start timestamps // lets its TA wait for the previous TA; its FS still waits for the previous FS, then @@ -1334,7 +1472,7 @@ impl Firmware { } else if early { let b = encode::barrier( r.fw_stamp[0].va, - encode::stamp_seed(ids::TA_SEED, n - 1), + encode::stamp_seed(ids::TA_SEED, sn - 1), 0, tta, RENDER_WAIT_UUID, @@ -1354,10 +1492,9 @@ impl Firmware { } else { let mut prev_fs = if pf.prev != Prev::None { pf.last_fs - } else if n == 1 { - ids::FS_SEED } else { - encode::stamp_seed(ids::FS_SEED, n - 1) + // With no offset, ordinal 0 is the FS seed itself (the first render). + encode::stamp_seed(ids::FS_SEED, sn - 1) }; // Restricted type 0: predecessor render in this prefix, no InitBM/resync. let internal = pf.prev == Prev::Render diff --git a/drivers/gpu/drm/asahi/g15/gate.rs b/drivers/gpu/drm/asahi/g15/gate.rs index f41b2e3f75f57..590f5e6f0a5c1 100644 --- a/drivers/gpu/drm/asahi/g15/gate.rs +++ b/drivers/gpu/drm/asahi/g15/gate.rs @@ -3,11 +3,13 @@ //! Stream admission gates (safety nets for userspace encoders). //! //! VDM: the root stream's first word is 0x21000960. CDM walk from `cdm_ctrl_stream_base` through -//! links until TERMINATE (0x40000000), at most 8192 records and 64 links (a longer or cyclic -//! stream is rejected): barriers 0x60000160, 0x60001560, 0x600215fe, 0x600201fe; +//! links until TERMINATE (0x40000000), at most 64 links and [`CDM_MAX_BYTES`] of records (a +//! longer or cyclic stream is rejected): barriers 0x60000160, 0x60001560, 0x600215fe, 0x600201fe; //! link 0x2000XXXX; direct launch (40 B, the G15 layout), indirect launches 36 B //! (`w0 & 0xff7fffff == 0x08080000`) and 24 B (0x10080000); every program VA inside the queue's -//! 4 GiB USC window, 64-aligned, first 64 bytes readable. Client memory is read through the VM's +//! 4 GiB USC window, 64-aligned, first 64 bytes readable; an indirect launch's 12 argument bytes +//! readable. A program or argument page found good is not checked again in the same walk, and a +//! walk checks at most [`CDM_MAX_PAGE_CHECKS`] pages. Client memory is read through the VM's //! page tables with a point-of-coherency clean+invalidate of the lines ([`mmu::Vm::read_user`]: //! the GPU may have written it). `g15_stream_gate=0` disables both gates (encoder bring-up only). @@ -24,8 +26,19 @@ const CDM_LINK: u32 = 0x20000000; const CDM_DIRECT: u32 = 0x00080000; const CDM_INDIRECT_SIZES: u32 = 0x08080000; const CDM_INDIRECT: u32 = 0x10080000; -const CDM_MAX_RECORDS: u32 = 8192; const CDM_MAX_LINKS: u32 = 64; +/// The stream block the encoder links to when one fills (Mesa's 64 KiB control-stream blocks). +const CDM_BLOCK_BYTES: u64 = 0x10000; +/// Stream bytes walked at most: the root block and one block per admitted link. A stream of +/// that size holds every record the encoder can write before the link budget (a record count +/// does not: the 1024 tessellated draws of one multi-draw write more than 8192). +const CDM_MAX_BYTES: u64 = (CDM_MAX_LINKS as u64 + 1) * CDM_BLOCK_BYTES; +/// Page checks (a program's or an indirect launch's arguments, each under the VM lock) a walk +/// makes at most. The last page found good is not checked again, but the encoder's indirect +/// arguments move to a new page every few launches: a stream at the link budget makes about +/// 8100. Twice that leaves room; without a bound, a stream of distinct pages would make one per +/// 24-byte launch, about 177000 in the byte budget. +const CDM_MAX_PAGE_CHECKS: u32 = 16384; /// Rejections logged per boot. const MAX_REPORTS: u32 = 32; @@ -58,7 +71,7 @@ struct Window { impl Window { /// The `count` words at `va`, reading whole spans of up to 256 bytes (not past the page /// unless the record crosses it). Unreadable memory: EFAULT. - fn words(&mut self, vm: &mmu::Vm, va: u64, count: usize) -> Result<[u32; 10]> { + fn words(&mut self, vm: &mut mmu::VmReadGuard, va: u64, count: usize) -> Result<[u32; 10]> { let n = count * 4; let cached = self.len != 0 && va >= self.va @@ -86,24 +99,57 @@ impl Window { } } +/// The pages a walk has found good and the page checks it has made. +struct Checked { + program: Option, + args: Option, + count: u32, +} + +/// One page check of `[start, end)` under the walk's page budget: ENOSPC once the budget is +/// spent. +fn check(vm: &mut mmu::VmReadGuard, start: u64, end: u64, count: &mut u32) -> Result { + if *count >= CDM_MAX_PAGE_CHECKS { + return Err(ENOSPC); + } + *count += 1; + cover(vm, start, end, true, false) +} + /// A launch's program: inside the queue's 4 GiB USC window, 64-aligned, first block readable. -/// `checked` caches the last proven program page. -fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Option) -> Result { +/// The last proven program page is not checked again. +fn program(vm: &mut mmu::VmReadGuard, usc: u64, address: u64, checked: &mut Checked) -> Result { if address < usc || address - usc >= USC_WINDOW || address & 63 != 0 { return Err(EINVAL); } // 64-aligned: the block never crosses its page. let page = address & !(mmu::UAT_PGMSK as u64); - if *checked == Some(page) { + if checked.program == Some(page) { return Ok(()); } - cover(vm, address, address + 64, true, false)?; - *checked = Some(page); + check(vm, address, address + 64, &mut checked.count)?; + checked.program = Some(page); + Ok(()) +} + +/// An indirect launch's 12 argument bytes: readable. The last proven argument page is not +/// checked again; arguments that cross a page are checked each time. +fn arguments(vm: &mut mmu::VmReadGuard, address: u64, checked: &mut Checked) -> Result { + let end = address.checked_add(12).ok_or(EINVAL)?; + let page = address & !(mmu::UAT_PGMSK as u64); + let within = (end - 1) & !(mmu::UAT_PGMSK as u64) == page; + if within && checked.args == Some(page) { + return Ok(()); + } + check(vm, address, end, &mut checked.count)?; + if within { + checked.args = Some(page); + } Ok(()) } /// VDM admission: a normal root stream opens with exactly the G15 barrier. -pub(crate) fn vdm(vm: &mmu::Vm, va: u64) -> Result { +pub(crate) fn vdm(vm: &mut mmu::VmReadGuard, va: u64) -> Result { if !params::stream_gate() { return Ok(()); } @@ -119,7 +165,7 @@ pub(crate) fn vdm(vm: &mmu::Vm, va: u64) -> Result { /// CDM admission: walks the stream from its base through no-return links until TERMINATE, /// admitting only the known G15S records: direct launch (40), indirect launch (36/24), barriers /// (4), link (8). -pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { +pub(crate) fn cdm(vm: &mut mmu::VmReadGuard, usc: u64, base: u64) -> Result { if !params::stream_gate() { return Ok(()); } @@ -128,15 +174,19 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { len: 0, bytes: [0; 256], }; - let mut checked = None; + let mut checked = Checked { + program: None, + args: None, + count: 0, + }; let mut va = base; let mut links = 0; - let mut records = 0; - while records < CDM_MAX_RECORDS { - records += 1; + let mut walked = 0; + while walked < CDM_MAX_BYTES { let w = window.words(vm, va, 1)?; if CDM_BARRIERS.contains(&w[0]) { va += 4; + walked += 4; continue; } if w[0] == CDM_TERMINATE { @@ -152,6 +202,7 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { if va & 3 != 0 { return Err(reject("CDM link", va, &w)); } + walked += 8; continue; } if w[0] == CDM_DIRECT { @@ -162,9 +213,13 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { } match program(vm, usc, address, &mut checked) { Err(error) if error == EINVAL => return Err(reject("CDM program", va, &w)), + Err(error) if error == ENOSPC => { + return Err(reject("CDM page check budget", va, &w)) + } result => result?, } va += 40; + walked += 40; continue; } let kind = w[0] & 0xff7fffff; @@ -182,15 +237,20 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { return Err(reject("CDM indirect launch", va, &w)); } let result = program(vm, usc, address, &mut checked) - .and_then(|()| cover(vm, indirect, indirect + 12, true, false)); + .and_then(|()| arguments(vm, indirect, &mut checked)); match result { Err(error) if error == EINVAL => return Err(reject("CDM indirect", va, &w)), + Err(error) if error == ENOSPC => { + return Err(reject("CDM page check budget", va, &w)) + } result => result?, } - va += if sizes { 36 } else { 24 }; + let size = if sizes { 36 } else { 24 }; + va += size; + walked += size; continue; } return Err(reject("CDM", va, &w)); } - Err(reject("CDM record budget", va, &[])) + Err(reject("CDM byte budget", va, &[])) } diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index b949a4b9b61ed..095b9cb96d51f 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -26,8 +26,10 @@ bool asahi_g15_board(void); bool asahi_g15_machine(const char *compatible); void asahi_g15_hold_power(struct device *dev); void asahi_g15_release_power(struct device *dev); -void asahi_g15_module_get(void); +bool asahi_g15_module_try_get(void); +void asahi_g15_module_put(void); void asahi_g15_reprobe(void); +void asahi_g15_report_loader_gpu(void); int asahi_g15_adt_phram(u64 *start, u64 *size); bool asahi_g15_ram_disjoint(u64 start, u64 size); bool asahi_g15_ram_within(u64 start, u64 size); @@ -35,10 +37,53 @@ unsigned int asahi_g15_memory_mode(u64 start, u64 size); void __iomem *asahi_g15_ioremap_wc(u64 start, size_t size); bool asahi_g15_dma_coherent(struct device *dev); int asahi_g15_pmp_prepare(struct device *dev); +int asahi_g15_display_prepare(struct device *dev); int asahi_g15_pmp_get(struct device *dev); +int asahi_g15_pmp_ready(void); void asahi_g15_pmp_put(struct device *dev); bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size); +/* + * The boot loader enables the T6030/T6031 GPU node only when its + * chosen.asahi,t603x-gpu setting is on; otherwise the node stays disabled, + * the adapter never probes and nothing says why native G15 is absent. Name + * the setting once, at module activation, on a machine whose /chosen has it + * and whose GPU node is disabled. Nothing else changes. + */ +void asahi_g15_report_loader_gpu(void) +{ + static const struct { + const char *prop; + const char *compatible; + } socs[] = { + { "asahi,t6031-gpu", "apple,agx-t6031" }, + { "asahi,t6030-gpu", "apple,agx-t6030" }, + }; + struct device_node *chosen, *np; + const char *value; + unsigned int i; + u32 cell; + + chosen = of_find_node_by_path("/chosen"); + if (!chosen) + return; + for (i = 0; i < ARRAY_SIZE(socs); i++) { + if (!of_property_present(chosen, socs[i].prop)) + continue; + np = of_find_compatible_node(NULL, NULL, socs[i].compatible); + if (np && !of_device_is_available(np)) { + if (!of_property_read_string(chosen, socs[i].prop, &value)) + pr_info("asahi: G15: native G15 not started: the boot loader left the GPU node %pOF disabled (/chosen/%s = \"%s\")\n", + np, socs[i].prop, value); + else if (!of_property_read_u32(chosen, socs[i].prop, &cell)) + pr_info("asahi: G15: native G15 not started: the boot loader left the GPU node %pOF disabled (/chosen/%s = %u)\n", + np, socs[i].prop, cell); + } + of_node_put(np); + } + of_node_put(chosen); +} + /* * The maintained m1n1 handoff keeps these four UAT regions in the linear map. * Require the exact enabled reserved-memory owner, in addition to the ADT @@ -46,20 +91,31 @@ bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size); */ bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size) { - static const char *const paths[] = { - "/reserved-memory/uat-ttbs", - "/reserved-memory/uat-pagetables", - "/reserved-memory/uat-handoff", - "/reserved-memory/uat-pagetables-l2", + static const char *const names[] = { + "uat-ttbs", + "uat-pagetables", + "uat-handoff", + "uat-pagetables-l2", }; - struct device_node *np; + struct device_node *rmem, *child, *np = NULL; struct resource res; + unsigned int count = 0; bool match; - if (index >= ARRAY_SIZE(paths)) + if (index >= ARRAY_SIZE(names)) return false; - np = of_find_node_by_path(paths[index]); - match = np && of_device_is_available(np) && + + /* Unit addresses belong to the DT layout, not the reservation name. */ + rmem = of_find_node_by_path("/reserved-memory"); + for_each_child_of_node(rmem, child) { + if (!of_node_name_eq(child, names[index])) + continue; + count++; + if (!np) + np = of_node_get(child); + } + of_node_put(rmem); + match = count == 1 && np && of_device_is_available(np) && !of_property_read_bool(np, "reusable") && !of_get_address(np, 1, NULL, NULL) && !of_address_to_resource(np, 0, &res) && @@ -90,6 +146,33 @@ int asahi_g15_pmp_prepare(struct device *dev) return apple_pmp_link_device(dev); } +/* + * T6031 has no PMP vote, but its first firmware start follows the native + * display as T6030's does: defer until that display is up. The display can + * be off for this boot (no handoff, or not requested), so say once why. An + * enabled display that is not published yet is said once too: if it never + * comes up (its DCP refused or failed to bind), that line is why the GPU + * stays off. + */ +int asahi_g15_display_prepare(struct device *dev) +{ + struct device_node *np; + int ret; + + if (!dev->of_node || !of_device_is_compatible(dev->of_node, "apple,agx-t6031")) + return -ENODEV; + ret = apple_drm_t6031_link_device(dev); + if (ret != -EPROBE_DEFER) + return ret; + np = of_find_compatible_node(NULL, NULL, "apple,t6031-display-subsystem"); + if (!np || !of_device_is_available(np)) + dev_info_once(dev, "G15: waiting for the T6031 native display, which this boot did not enable (it needs the boot loader's display handoff, and apple_t6031_display.enable=1 except on the J516C)\n"); + else + dev_info_once(dev, "G15: waiting for the T6031 native display to come up: the GPU firmware starts only after the display is published\n"); + of_node_put(np); + return ret; +} + int asahi_g15_pmp_get(struct device *dev) { if (!asahi_g15_board() || !dev->of_node || @@ -99,6 +182,11 @@ int asahi_g15_pmp_get(struct device *dev) } +int asahi_g15_pmp_ready(void) +{ + return apple_pmp_t6030_gpu_ready(); +} + /* * Give back the vote of asahi_g15_pmp_get() after a start that failed before * the firmware was touched. The full M3 stack's J514S bridge has no release: @@ -159,10 +247,20 @@ void asahi_g15_release_power(struct device *dev) } -/* Pin this module until reboot. */ -void asahi_g15_module_get(void) +/* + * Pin this module until reboot, unless its removal has begun: a deferred or + * asynchronous probe can run after an ordinary rmmod has passed its reference + * check, which __module_get() would not stop. + */ +bool asahi_g15_module_try_get(void) +{ + return try_module_get(THIS_MODULE); +} + +/* Drop a reference asahi_g15_module_try_get() took. */ +void asahi_g15_module_put(void) { - __module_get(THIS_MODULE); + module_put(THIS_MODULE); } /* diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs index d566a54953e07..b211088310210 100644 --- a/drivers/gpu/drm/asahi/g15/hw.rs +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -46,15 +46,12 @@ pub(crate) enum Platform { pub(crate) enum Board { /// The hash-checked T6030 templates ([`super::board`]). Templates, - /// The three images built from the device tree ([`super::board::record`], - /// [`super::board::device_tree`]) and cross-checked against a firmware file holding the - /// machine's own ([`super::board::blob`], pinned by its SHA-256), which also supplies the - /// record's words the device tree does not carry ([`super::board::record::FROM_FILE`]); a - /// difference refuses admission, before any firmware is touched. - Blob { - name: &'static kernel::str::CStr, - sha256: [u8; 32], - }, + /// The three images built from the device tree and the ADT's sensor rows + /// ([`super::board::record`], [`super::board::device_tree`]), with the SoC's reference + /// words for what neither carries ([`super::board::record::Reference`]). A value the + /// builder refuses stops admission, before any firmware is touched. + #[cfg(CONFIG_DRM_ASAHI_G15)] + DeviceTree(&'static super::board::record::Reference), /// No board data: admission refuses the SoC before anything firmware-visible. None, } @@ -163,6 +160,21 @@ pub(crate) mod t6030 { 0xe7, 0x9b, 0xb9, 0xb0, 0x22, 0x09, 0xdb, 0x53, ], }, + // The J514S (14-inch) boot chain loads the same B0 image: its + // measured TEXT digest, sizes and marker match the J516S row above. + FirmwareBuild { + name: "J514S B0 RTKit-2419.140.12 (maintained 14.x handoff)", + machine: Some(kernel::c_str!("apple,j514s")), + text_size: 0x5c000, + data_size: 0x114000, + marker: 0x58e88, + text_sha256: [ + 0xda, 0x26, 0xec, 0xca, 0xb1, 0xf8, 0x59, 0x8d, + 0xa8, 0x0c, 0x85, 0x5e, 0xca, 0x91, 0x65, 0x20, + 0x40, 0x6d, 0x1d, 0x06, 0x70, 0xf1, 0xd4, 0x70, + 0xe7, 0x9b, 0xb9, 0xb0, 0x22, 0x09, 0xdb, 0x53, + ], + }, ], machine: kernel::c_str!("apple,t6030"), platform: super::Platform::AdtWithPmp, @@ -516,8 +528,6 @@ pub(crate) mod sizes { pub(crate) const AHEAD_RECORDS: u32 = 64; /// Policy bound on logical queues reported in QUEUE_LIMITS (not a firmware limit). pub(crate) const MAX_QUEUES: u32 = 58; - /// TTBAT user slots: user VMs hold one each for their lifetime (ASIDs 1..63). - pub(crate) const MAX_VMS: u32 = 63; } /// Firmware channels, doorbells, seeds and identifiers. @@ -571,11 +581,25 @@ pub(crate) mod tuning { pub(crate) const SLOT_ROTATE: bool = true; /// Flights per VM. pub(crate) const MAX_INFLIGHT: u32 = 2; - /// PB growth through event 7 / DEVCTL 7, initial and maximum blocks. + /// PB growth through event 7 / DEVCTL 7, from the initial posting of [`PB_BLOCKS`]. pub(crate) const PB_GROW: bool = true; - /// Initial PB blocks, and the most that may be posted: one below the mapped heap's block - /// count, so the block-list producer (modulo K = 4096) never returns to 0. - pub(crate) const PB_BLOCKS: u32 = 1024; + /// PB blocks posted at (re)initialization: 64 blocks of 128 KiB (8 MiB). + pub(crate) const PB_BLOCKS: u32 = 64; + /// PB reclaim when the engine is idle: 0 never, 1 re-initialize, 2 also free the backing + /// above the initial posting and the reserve. + pub(crate) const PB_RECLAIM: u32 = 2; + /// Idle time before a PB reclaim, ms. + pub(crate) const PB_IDLE_MS: u32 = 1000; + /// Posted PB blocks above which an idle engine reclaims the PB. + pub(crate) const PB_RECLAIM_BLOCKS: u32 = 1024; + /// Renders before the TA stamp wrap within which the TA/FS stamps are renumbered while + /// nothing is in flight (the wrap is at about 16.7 million renders). + pub(crate) const STAMP_RESEED_MARGIN: u64 = 100_000; + /// Computes before the compute stamp wrap within which the compute stamp is renumbered + /// while nothing is in flight (the wrap is at 4128768 computes). + pub(crate) const COMPUTE_RESEED_MARGIN: u64 = 100_000; + /// The most PB blocks that may be posted: one below the mapped heap's block count, so the + /// block-list producer (modulo K = 4096) never returns to 0. pub(crate) const PB_MAX_BLOCKS: u32 = super::sizes::PB_BLOCKS - 1; /// At the host deadline, arm the firmware kill. pub(crate) const HANG_PREEMPT: bool = true; diff --git a/drivers/gpu/drm/asahi/g15/hw/t6031.rs b/drivers/gpu/drm/asahi/g15/hw/t6031.rs index 4af32f6068cd7..8bac33989db64 100644 --- a/drivers/gpu/drm/asahi/g15/hw/t6031.rs +++ b/drivers/gpu/drm/asahi/g15/hw/t6031.rs @@ -8,9 +8,8 @@ //! device tree node the boot loader fills, not an ADT export), the GPU power (its power domain; //! there is no PMP vote), the initdata root version, a longer HardwareData record (0xb480 //! bytes, its fourth view at +0xb400) placed aligned and ASC-cached, the IOMapping table (17 -//! slots at the T6031 register blocks) and the board images, built from the device tree and -//! checked against this machine's firmware file, which supplies the record's non-device-tree -//! words. Every +//! slots at the T6031 register blocks) and the board images, built from the device tree, the +//! ADT's sensor rows and the J516C reference words. Every //! value was measured on a J516C: the SGX identification //! registers, the loaded firmware segment, the device tree the boot loader hands over and the //! initdata of that machine. @@ -117,18 +116,6 @@ pub(crate) const LAYOUT: Layout = Layout { record_empty_writable: &RECORD_EMPTY_WRITABLE, }; -/// The J516C board file (`asahi/t6031-j516c-board.bin`, built into the kernel): the hardware -/// record, Globals and the power block that this machine's firmware accepted, as the earlier -/// J516C driver uploaded them (its device-tree-built records with the per-board words of this -/// machine applied). Kept out of the tree; pinned by its SHA-256. The three images are built -/// from the device tree and must match the file's; the file supplies the record's words the -/// device tree does not carry (`board::record::FROM_FILE`). -const BOARD_SHA256: [u8; 32] = [ - 0x4e, 0x03, 0xf2, 0x9f, 0xa4, 0x47, 0xdc, 0xeb, 0xfd, 0x4d, 0x1c, 0x22, 0x5a, 0x64, 0x7b, - 0xfa, 0x31, 0xb3, 0x63, 0x45, 0x8f, 0x28, 0x33, 0x82, 0x3a, 0x5c, 0xa8, 0xa1, 0x0e, 0xdf, - 0xf3, 0xfc, -]; - pub(crate) const CONFIG: Config = Config { name: "T6031 G15C", chip_id: 0x6031, @@ -172,15 +159,12 @@ pub(crate) const CONFIG: Config = Config { ], }, ], - // The pinned board images and measured layout belong to the J516C, not + // The reference words and measured layout belong to the J516C, not // every T6031 board (in particular, not the 14-inch J514C). machine: kernel::c_str!("apple,j516c"), platform: super::Platform::DeviceTree, layout: &LAYOUT, - board: super::Board::Blob { - name: kernel::c_str!("asahi/t6031-j516c-board.bin"), - sha256: BOARD_SHA256, - }, + board: super::Board::DeviceTree(&super::super::board::record::J516C), // Four clusters: the saved-state pointers at +0x2980 of the preemption buffer (0xd80 plus // 0x700 per cluster), UMA PTC 0x6000400020, DUPM 0x3c000000140 (32 N | 96 N << 32 for // N = 10 cores per cluster) and 0x1a0e9 = 0, with the saved-state pointers and the diff --git a/drivers/gpu/drm/asahi/g15/initdata.rs b/drivers/gpu/drm/asahi/g15/initdata.rs index f3f5232e92a28..e53cb44eff43a 100644 --- a/drivers/gpu/drm/asahi/g15/initdata.rs +++ b/drivers/gpu/drm/asahi/g15/initdata.rs @@ -160,61 +160,27 @@ fn templates( /// HardwareDataA word holding the system counter at initdata creation. const HWDATA_A_INIT_TIMESTAMP: usize = 0x424c; -/// Board images of a SoC with a firmware file. The hardware record ([`board::hardware_record`]), -/// Globals and Power ([`board::device_tree`]) are built from the device tree; the file, this -/// machine's own, supplies the record's words the device tree does not carry and is otherwise -/// a cross-check: admission ([`prepare_board`], before any firmware is touched) is refused at -/// the first differing word. Returns the images and the top operating-point frequency in kHz. -fn blob( +/// Board images of a device-tree SoC: the hardware record ([`board::hardware_record`]) from the +/// GPU node, the ADT's sensor rows and the SoC's `reference` words, Globals and Power +/// ([`board::device_tree`]) from the GPU node and that record. Returns the images and the top +/// operating-point frequency in kHz. +fn device_tree_board( dev: &device::Device, cfg: &Config, adt: &GpuAdt, - name: &CStr, - sha256: &[u8; 32], + reference: &board::record::Reference, ) -> Result<(KVVec, KVVec, KVVec, u32)> { - let layout = cfg.layout; - let images = board::blob::load( - dev, - name, - sha256, - [layout.hwdata_size, globals::SIZE, power::SIZE], - )?; - let hw_img = board::hardware_record(dev, cfg, adt, &images.hardware)?; + let hw_img = board::hardware_record(dev, cfg, adt, reference)?; let (glob, pwr, max_power_mw) = board::device_tree(dev, &hw_img)?; - for (what, built, file) in [ - ("hardware record", &hw_img, &images.hardware), - ("Globals", &glob, &images.globals), - ("Power", &pwr, &images.power), - ] { - let diff = if what == "hardware record" { - board::record::cross_check(built, file) - } else { - board::words::first_difference(built, file) - }; - if let Some((at, b, f)) = diff { - dev_err!( - dev, - "board data: device-tree {} differs from {} at +{:#x}: built {:#x}, file {:#x}\n", - what, - name, - at, - b, - f - ); - return Err(EINVAL); - } - } let max_mhz = adt.perf_mhz.iter().copied().max().unwrap_or(0); dev_info!( dev, - "board data: {} images from the device tree match {} ({} record words from the file, {} file words left zero), {} mW maximum power, {} MHz top operating point\n", + "board data: {} images from the device tree and the ADT sensor rows, reference maximum power {} mW and MTR masks, {} MHz top operating point\n", cfg.name, - name, - board::record::FROM_FILE.len(), - board::record::UNNEEDED.len(), max_power_mw, max_mhz ); + dev_info!(dev, "board data sha256 {}\n", Hex(&glue::sha256(&hw_img))); let max_khz = max_mhz.checked_mul(1000).ok_or_else(|| { dev_err!(dev, "board data: top operating point {} MHz does not fit in kHz\n", max_mhz); EINVAL @@ -225,8 +191,9 @@ fn blob( /// Verified board bytes, prepared before any GPU firmware is started. pub(crate) type BoardImages = (KVVec, KVVec, KVVec, u32); -/// Check availability, digest, layout and machine-derived tables before firmware start. -/// Keep the verified bytes so a later firmware request cannot change the admitted data. +/// Build and check the board images before any GPU firmware is started: the templates against +/// their digests and layout, or the device-tree record from the GPU node, the ADT and the SoC's +/// reference words. pub(crate) fn prepare_board( dev: &device::Device, cfg: &Config, @@ -234,7 +201,7 @@ pub(crate) fn prepare_board( ) -> Result { match cfg.board { hw::Board::Templates => templates(dev, adt), - hw::Board::Blob { name, sha256 } => blob(dev, cfg, adt, name, &sha256), + hw::Board::DeviceTree(reference) => device_tree_board(dev, cfg, adt, reference), hw::Board::None => Err(ENODEV), } } @@ -296,7 +263,7 @@ pub(crate) fn build( // Board images, verified, then the owned replacements. let (mut hw_img, mut glob, pwr, max_khz) = board_images; - if matches!(cfg.board, hw::Board::Blob { .. }) { + if matches!(cfg.board, hw::Board::DeviceTree(_)) { // This timestamp belongs to initdata creation, not board-data preflight. let now: u64; // SAFETY: Reading the architectural counter has no side effects. @@ -318,10 +285,10 @@ pub(crate) fn build( } put64(&mut hw_img, hwdata::TIMESTAMPS, va::TIMESTAMPS)?; put32(&mut glob, globals::UNK_980, 100)?; - // The device tree's standby time (a machine with a firmware file; the templates' machine + // The device tree's standby time (a device-tree SoC; the templates' machine // has none). let dt_standby = match cfg.board { - hw::Board::Blob { .. } => { + hw::Board::DeviceTree(_) => { let prop = board::words::PROPS[board::words::STANDBY_TIMER]; match dev .of_node() @@ -369,8 +336,8 @@ pub(crate) fn build( globals::IDLE_PREDICTIVE, hw::tuning::IDLE_PREDICTIVE as u32, )?; - // HardwareDataA at +0x4580: maintained S8 request (the templates' machine; a board image - // from a file carries its own machine's words). + // HardwareDataA at +0x4580: maintained S8 request (the templates' machine; a record built + // from the device tree carries its own state words). if cfg.board == hw::Board::Templates { for (offset, value) in [ (0x2c, 8), diff --git a/drivers/gpu/drm/asahi/g15/job.rs b/drivers/gpu/drm/asahi/g15/job.rs index 5f6669f06d1b0..a93b6039cc466 100644 --- a/drivers/gpu/drm/asahi/g15/job.rs +++ b/drivers/gpu/drm/asahi/g15/job.rs @@ -21,30 +21,36 @@ //! The engine owns an `Arc` from [`super::Firmware::enqueue`] on. It completes the job //! exactly once with [`Job::complete`] (later calls are ignored), through [`Completions`] //! collected under the device lock and [`complete_all`] after it is dropped. A job whose storage -//! the firmware may still reference stays in `Engine::retained` (failed jobs until reboot): its -//! [`mmu::VmJobGuard`] keeps the VM's mappings and its VM state keeps the VM's TTBAT slot until +//! the firmware may still reference after a fail-stop stays in `Engine::retained` until reboot: its +//! [`mmu::VmJobGuard`] keeps the VM's mappings and its binding keeps the VM's TTBAT slot until //! the last `Arc` drops. -use core::sync::atomic::{AtomicBool, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use kernel::{ bindings, c_str, device, dma_fence::{Fence, RawDmaFence}, drm::sched, + new_mutex, prelude::*, - sync::Arc, + sync::{Arc, Mutex}, }; use super::{command::Command, fence::JobFence, hw, queue::QueueState, vm::VmState, Shared}; use crate::mmu; /// A job as the engine sees it. +#[pin_data] pub(crate) struct Job { /// Submission ID (debugging). pub(crate) id: u64, /// The submitting queue (priority, service turn, horizon history). pub(crate) queue: Arc, - /// The job's VM (TTBAT slot, status). + /// The job's VM and status. pub(crate) vm: Arc, + /// Hardware context, assigned immediately before the engine publishes the job. + context: AtomicU32, + #[pin] + binding: Mutex>, /// Validated commands, in submission order (at most 64). pub(crate) commands: KVec, /// The job's hardware fence. @@ -73,11 +79,13 @@ impl Job { guard: mmu::VmJobGuard, ) -> Result> { let progress = super::engine::Progress::new(commands.len()); - Ok(Arc::new( - Self { + Arc::pin_init( + pin_init!(Self { id, queue, vm, + context: AtomicU32::new(0), + binding <- new_mutex!(None, "g15::Job::binding"), commands, fence, progress, @@ -85,9 +93,29 @@ impl Job { _guard: guard, accepted: AtomicBool::new(false), started: AtomicBool::new(false), - }, + }), GFP_KERNEL, - )?) + ) + } + + pub(crate) fn slot(&self) -> u32 { + self.context.load(Ordering::Relaxed) + } + + /// Called under the device lock immediately before publication. + pub(crate) fn bind(&self) -> Result { + let mut slot = self.binding.lock(); + if slot.is_none() { + let binding = self.vm.try_bind()?; + self.context.store(binding.slot(), Ordering::Relaxed); + *slot = Some(binding); + } + Ok(()) + } + + /// All commands retired successfully, or nothing of the job was published. + pub(crate) fn release_binding(&self) { + drop(self.binding.lock().take()); } /// Completes the job: latches the VM status on failure, then signals the fence (after every @@ -120,6 +148,9 @@ impl Job { self.vm.status.record(error); } } + if result.is_ok() || !engine { + self.release_binding(); + } self.fence.signal(result); } diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 713a1360b5687..9ffd858598b1c 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -14,9 +14,9 @@ //! //! # Plugging into the shared driver //! -//! - [`Driver`] is a separate platform driver. T6030 has a module alias; its probe must wait -//! for the native display and PMP supplier before booting firmware. T6031 retains explicit -//! module loading. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. +//! - [`Driver`] is a separate platform driver with a module alias. Its probe must wait for +//! the native display (T6030 and T6031) and the PMP supplier (T6030) before booting +//! firmware. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. //! - [`Gpu`] implements [`crate::gpu::Gpu`]: GET_PARAMS identity, VM creation (`new_vm`), //! queues (`new_queue`, see [`queue`]) and timestamp buffers. //! - User VMs are shared-layer [`mmu::Vm`]s on a [`mmu::Uat::new_adopted`] UAT with tracked @@ -45,6 +45,7 @@ mod adt; mod board; mod boot; +mod retain; // One admission rule for the generation mappings and shared adopted UAT. pub(crate) use boot::glue::memory_mode; @@ -151,13 +152,15 @@ const OF_TABLE: kernel::device_id::IdArray = ), ]); -// T6030 probe admission requires the native display and PMP supplier before -// firmware starts. T6031 retains explicit loading until it has an equivalent -// display-order admission gate. +// Probe admission requires the native display before firmware starts (T6030 +// also its PMP supplier), so both may be loaded at device creation. kernel::of_device_table!( AUTOLOAD_OF_TABLE, @none, - [of::DeviceId::new(c_str!("apple,agx-t6030"))] + [ + of::DeviceId::new(c_str!("apple,agx-t6030")), + of::DeviceId::new(c_str!("apple,agx-t6031")), + ] ); /// The G15 platform adapter: the shared DRM registration probed through @@ -226,6 +229,8 @@ static MODULE_READY: AtomicBool = AtomicBool::new(false); /// No fallible module initialization may follow this call. pub(crate) fn activate() { MODULE_READY.store(true, Ordering::Release); + // SAFETY: The helper only reads the device tree. + unsafe { boot::glue::asahi_g15_report_loader_gpu() }; // SAFETY: Both pinned platform registrations are fully initialized and remain owned // by DriverModule. The C helper borrows their live driver pointer only for this call. unsafe { boot::glue::asahi_g15_reprobe() }; @@ -376,7 +381,7 @@ pub(crate) struct Firmware { pub(crate) recovery: recovery::Recovery, /// Render UMA/PB allocators, their spare set and parameter-buffer growth. pub(crate) pools: pools::Pools, - /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM. + /// Live VMs and the client aliases of every user VM. pub(crate) vms: Arc, } @@ -454,17 +459,18 @@ impl Gpu { }, GFP_KERNEL, )?; - gpu.retain(); + gpu.retain()?; Ok(gpu) } /// Keeps the firmware session and this module alive until reboot. /// - /// Takes the module reference once. Nothing firmware-reachable is freed on any path: the + /// Takes the module reference once; `boot::start` already took it before touching the + /// firmware, so this only confirms it. Nothing firmware-reachable is freed on any path: the /// RTKit callbacks hold [`Shared`], which holds the [`Firmware`] (or the [`boot::Orphan`] of a /// failed start) until reboot. - pub(crate) fn retain(&self) { - boot::glue::retain_module(); + pub(crate) fn retain(&self) -> Result { + boot::glue::retain_module() } } @@ -539,7 +545,21 @@ impl Firmware { vms, }, ); - if let Err(e) = firmware.init_graphs() { + let result = firmware.init_graphs().and_then(|_| { + if cfg.platform != hw::Platform::AdtWithPmp { + return Ok(()); + } + // A report before firmware startup is ACKed without enabling bandwidth scaling. + // Reassert it after setup, including the backend's final device control. + // SAFETY: This session retains its acknowledged T6030 AGX power vote. + let ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; + if ret == 0 { + Ok(()) + } else { + Err(Error::from_errno(ret)) + } + }); + if let Err(e) = result { let Firmware { session, init, diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs index b1e3218fafc8d..c2c3486475061 100644 --- a/drivers/gpu/drm/asahi/g15/params.rs +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -18,6 +18,30 @@ use crate::module_parameters; /// updates); this bounds the kernel's staging allocation. pub(crate) const MAX_BINDS: u32 = 1 << 20; +// At least one prefix of renders, so that the last prefix published outside the margin cannot +// cross the wrap; at most half the stamp range. +static_assert!( + hw::tuning::STAMP_RESEED_MARGIN >= hw::tuning::PREFIX_RENDERS as u64 + && hw::tuning::STAMP_RESEED_MARGIN <= super::encode::RENDER_WRAP / 2 +); +// At least one prefix of computes; at most half the stamp range. +static_assert!( + hw::tuning::COMPUTE_RESEED_MARGIN >= hw::sizes::COMPUTE_SLOTS as u64 + && hw::tuning::COMPUTE_RESEED_MARGIN <= super::encode::COMPUTE_WRAP / 2 +); + +/// Renders before the TA stamp wrap within which the TA/FS stamps are renumbered from their seeds, +/// at the first moment with nothing in flight ([`hw::tuning::STAMP_RESEED_MARGIN`]). +pub(crate) fn stamp_reseed_margin() -> u64 { + hw::tuning::STAMP_RESEED_MARGIN +} + +/// Computes before the compute stamp wrap within which the compute stamp is renumbered from its +/// seed, at the first moment with nothing in flight ([`hw::tuning::COMPUTE_RESEED_MARGIN`]). +pub(crate) fn compute_reseed_margin() -> u64 { + hw::tuning::COMPUTE_RESEED_MARGIN +} + /// Host deadline of a published prefix without progress, ms (at least 100). pub(crate) fn deadline_ms() -> u32 { module_parameters::g15_deadline_ms.value().max(100) @@ -64,13 +88,33 @@ pub(crate) fn accept_text_prefix() -> u64 { } /// The parameter for the PB growth maximum the firmware is given, in blocks, clamped to -/// 64..=4095. [`super::pools::Pools::soft_max`] raises it to at least the initial 1024 blocks; -/// the blocks above it, up to the 4095 posted at most, answer the synchronous requests of -/// partial renders. It is read-only after boot and does not change the 512 MiB heap. +/// 64..=4095. [`super::pools::Pools::soft_max`] raises it to at least the initial posting; the +/// blocks above it, up to the 4095 posted at most, answer the synchronous requests of partial +/// renders. It is read-only after boot and does not change the heap's backing. pub(crate) fn pb_max_blocks() -> u32 { module_parameters::g15_pb_max_blocks.value().clamp(64, super::hw::tuning::PB_MAX_BLOCKS) } +/// The initial PB posting, in blocks: a fixed driver value, not a module parameter. +pub(crate) fn pb_blocks() -> u32 { + super::hw::tuning::PB_BLOCKS +} + +/// PB reclaim at idle: 0 never, 1 re-initialize, 2 re-initialize and free the backing. +pub(crate) fn pb_reclaim() -> u32 { + super::hw::tuning::PB_RECLAIM +} + +/// Idle time before a PB reclaim, ns. +pub(crate) fn pb_idle_ns() -> u64 { + super::hw::tuning::PB_IDLE_MS as u64 * 1_000_000 +} + +/// Posted blocks above which an idle engine reclaims the PB. +pub(crate) fn pb_reclaim_blocks() -> u32 { + super::hw::tuning::PB_RECLAIM_BLOCKS +} + /// Whether FRAGMENT_BARRIERS is advertised and mapped to early tiling. pub(crate) fn early_tiling() -> bool { module_parameters::g15_early_tiling.value() != 0 diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 566995eb342c9..b0356376ecf96 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -23,7 +23,7 @@ use kernel::{ hrtimer::{ HrTimerCallback, HrTimerCallbackContext, HrTimerPointer, HrTimerRestart, RelativeMode, }, - Delta, Jiffies, Monotonic, + msecs_to_jiffies, Delta, Jiffies, Monotonic, }, workqueue::RawWorkItem, }; @@ -85,7 +85,15 @@ impl Shared { let evented = self.evented.swap(false, Ordering::AcqRel); match firmware.poll(evented, &mut done) { Ok(true) => self.rearm(firmware.engine.chan_wait), - Ok(false) => {} + // Idle: nothing polls until new work arrives, so a PB reclaim that comes + // due with more idle time gets one more pass then. A job's arrival brings + // the poll work forward (`rearm`), which replaces this expiry. + Ok(false) => { + if let Some(ns) = firmware.reclaim_delay_ns() { + let ms = ns.div_ceil(1_000_000).min(u32::MAX as u64) as u32; + self.mod_poll(msecs_to_jiffies(ms) + 1); + } + } Err(error) => self.fail_stop(error), } } @@ -109,7 +117,7 @@ impl Shared { if let Err(error) = firmware.publish(&mut done) { self.fail_stop(error); } - if !firmware.engine.flights.is_empty() { + if !firmware.engine.flights.is_empty() || firmware.engine.slot_wait { self.rearm(firmware.engine.chan_wait); } } @@ -181,10 +189,7 @@ impl Shared { } } -/// Stamp cell `obs` reached its own target, bounded by the newest published target `tail`. -pub(crate) fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool { - obs != 0 && obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) -} +pub(crate) use super::recovery::logic::stamp_reached; /// Cursor `c` of a ring of `m` entries passed end `h` of entries from `b`, within tail `t`. pub(crate) fn ring_passed(c: u32, b: u32, h: u32, t: u32, m: u32) -> bool { @@ -207,7 +212,10 @@ impl Firmware { // prefixes published the retirement loop below drains first. if self.engine.flights.is_empty() { self.drain_rings()?; - if self.recovery_idle() { + // Idle bookkeeping of the PB: when the idle began, and the reclaim once it is due. + self.note_idle(); + self.reclaim_check(); + if self.recovery_idle() && self.engine.pending.is_empty() { return Ok(false); } } @@ -228,8 +236,14 @@ impl Firmware { self.engine.prof.polls += 1; self.engine.prof.poll_ns += now_ns().wrapping_sub(start); } + // The pass that retired the last flight is not re-armed: note the idle here, not only + // at the start of a pass that finds the flights empty. + self.note_idle(); published?; - Ok(!self.engine.flights.is_empty() || self.recovery.needs_repoll() || self.engine.frozen) + Ok(!self.engine.flights.is_empty() + || self.engine.slot_wait + || self.recovery.needs_repoll() + || self.engine.frozen) } /// Drains the rings and retires finished flights, with up to four trailing 5 us polls @@ -448,6 +462,10 @@ impl Firmware { p.count.store(p.sent(), Ordering::Relaxed); } } + if result.is_ok() && p.next() == p.sent() { + // All published prefixes retired; another VM may run before this job resumes. + job.release_binding(); + } if p.next() < p.sent() || p.sent() < p.count() { return; // more of it published or still to publish } @@ -465,6 +483,7 @@ impl Firmware { match result { Err(error) => complete(done, j2, Err(error)), Ok(()) => { + j2.release_binding(); s.jobs += 1; s.job_cmds += p2.count() as u64; s.job_max = s.job_max.max(p2.count() as u64); @@ -484,23 +503,25 @@ impl Firmware { return; }; let (g, r, e) = (&self.graph.compute, &self.graph.render, &self.engine); - let (mut c, mut ta, mut fs) = (0, 0, 0); + let pending = super::recovery::logic::pending_stamp; + let (mut c, mut ta, mut fs) = (None, None, None); if f.computes != 0 { - c = g.fw_stamp.r32(0).unwrap_or(0); - if stamp_reached(c, f.target, e.target) { - c = 0; // its computes are done: later work does not count - } + c = g + .fw_stamp + .r32(0) + .ok() + .and_then(|v| pending(v, f.target, e.target)); } if f.renders != 0 { - ta = r.fw_stamp[0].r32(0).unwrap_or(0); - fs = r.fw_stamp[1].r32(0).unwrap_or(0); - if stamp_reached(ta, f.ta_target, e.ta_target) { - ta = 0; - } - if stamp_reached(fs, f.fs_target, e.fs_target) { - fs = 0; - } + ta = r.fw_stamp[0] + .r32(0) + .ok() + .and_then(|v| pending(v, f.ta_target, e.ta_target)); + fs = r.fw_stamp[1] + .r32(0) + .ok() + .and_then(|v| pending(v, f.fs_target, e.fs_target)); } let moved = e.progress_flight == f.published_ns && [c, ta, fs] != e.progress; let published = f.published_ns; @@ -606,7 +627,7 @@ impl Firmware { self.graph.dev.as_ref(), "G15: {}: ctx={} prefixes={} n={} target={:#x} user={:#x} fw={:#x} done={} read={} write={} head={} ch={}/{} events={}/{} errors={} busy={} fault={:#x} {}ms\n", what, - f.map_or(0, |f| f.job.vm.slot), + f.map_or(0, |f| f.job.slot()), e.flights.len(), e.ordinal, e.target, @@ -716,6 +737,16 @@ impl Firmware { s.hz_ahead )?; self.pools.write_stats(f)?; + // What is left before the stamps wrap, and the renumbering margins. + let (compute_left, render_left) = self.stamps_left(); + writeln!( + f, + "stamps: compute_left {} compute_margin {} render_left {} render_margin {}", + compute_left, + params::compute_reseed_margin(), + render_left, + params::stamp_reseed_margin() + )?; writeln!( f, "stage: ta_us {} fs_us {} cdm_us {} span_us {}", diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index c635ed11c4a0c..668d7b408dcc4 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -6,16 +6,30 @@ //! (host-tested). //! //! - One 512 MiB PB heap at [`super::hw::client::PB_HEAP`] (4096 blocks of 128 KiB), backed on -//! demand: the initial blocks when the GPU starts, then chunks of at most -//! [`growth::CHUNK_BLOCKS`] (4 MiB) as growth needs them ([`Firmware::back_pb`]), never freed -//! or shrunk. The backing ([`PbChunk`]) is mapped only into user VMs, at the fixed client GVA, -//! each VM's before its prefixes are published ([`Firmware::map_pb`]); nothing names its pages -//! through the host window or context 0 (the records hold client GVAs only). Initial -//! [`super::hw::tuning::PB_BLOCKS`] (1024), at most [`super::hw::tuning::PB_MAX_BLOCKS`] -//! (4095) posted, so the producer never returns to 0; the firmware is given -//! [`Pools::soft_max`] (default 3072) as its growth maximum. Truthful K = 4096 in every +//! demand in chunks of [`growth::CHUNK_BLOCKS`] blocks (16 MiB) from the heap base: when the +//! GPU starts, the chunks of the initial posting and one reserve chunk +//! ([`growth::Ledger::chunks_for`]). The backing ([`PbChunk`]) is mapped only into user VMs, at +//! the fixed client GVA: each VM's before its prefixes are published ([`Firmware::map_pb`]), +//! and a chunk backed later also into the VM of the flights; nothing names its pages through +//! the host window or context 0 (the records hold client GVAs only). Initial +//! `tuning::PB_BLOCKS` (64), at most [`super::hw::tuning::PB_MAX_BLOCKS`] (4095) posted, so the +//! producer never returns to 0; the firmware is given [`Pools::soft_max`] (default 3072) as +//! its growth maximum. Truthful K = 4096 in every //! profile (PB reconstruction, also in recovery, synchronizes 8 * K bytes of the block list: //! K = 0x26cd overran the 0x8000-byte list and crashed the second recovery). +//! - Backing: a request is granted from the backed blocks above the posted count; after the +//! replies of a pass, while fewer than a reserve chunk remain, [`Firmware::refill_backing`] +//! backs the next chunk, so a reply never waits for the allocator. Only a reserve that ran dry +//! makes a request wait: it stays queued, the pass backs a chunk and the next pass answers it. +//! - Reclaim ([`Firmware::reclaim_check`], `tuning::PB_RECLAIM`): posted blocks never come back +//! while a PB lives (the firmware recycles pages inside its own pool; the host has no view of +//! it), so an engine idle for `tuning::PB_IDLE_MS` with more than `tuning::PB_RECLAIM_BLOCKS` posted +//! re-initializes the PB at its next render through the re-initialization path of a recovered +//! abort (fresh lists and manager, InitBM of the initial blocks, no halt) and frees the backing +//! above the kept chunks (also with fewer posted, when more is backed than they need); an +//! engine idle for 50 ms (a client boundary) with more than half the maximum posted +//! re-initializes it the same way but keeps the backing; if no render follows, with +//! `tuning::PB_RECLAIM` 2 the backing above the kept chunks is freed after `tuning::PB_IDLE_MS`. //! - Growth: the event drain queues type-7 requests; [`Firmware::answer_growth`] answers each //! one in the poll pass: type 7, PB slot 0, the context of a live published render prefix, //! counter >= next expected, epoch == current halt count; write records `(G - 0x1000000000) @@ -41,7 +55,6 @@ mod growth; use core::fmt; -use core::sync::atomic::Ordering; use kernel::{ device, prelude::*, @@ -58,7 +71,7 @@ use super::Firmware; use crate::driver::AsahiDrmDevice as AsahiDevice; use crate::{gem, mmu}; -pub(crate) use growth::Ledger; +pub(crate) use growth::{Answer, IdleClock, Ledger}; static_assert!(buffer::MAX_BLOCKS == sizes::PB_BLOCKS); static_assert!(buffer::MAX_PAGES == sizes::PB_PAGES); @@ -68,54 +81,43 @@ static_assert!(buffer::MAX_BLOCKS as u64 * buffer::BLOCK_BYTES == client::PB_HEA static_assert!(buffer::PAGE_LIST_SIZE as u64 == client::PB_PAGE_LIST_SIZE); static_assert!(uma_manager::SEEDS == sizes::FREELIST_SEED); static_assert!(uma_pools::POOL_SIZE == sizes::POOL_SIZE); -// The ledger keeps the driver's PB sizes: neither is capped by [`growth::MAX_POSTED`]. +// The ledger keeps the driver's PB maximum: it is not capped by [`growth::MAX_POSTED`]. The +// initial posting (`tuning::PB_BLOCKS`) is clamped to MIN_BLOCKS..=PB_MAX_BLOCKS. static_assert!(tuning::PB_MAX_BLOCKS <= growth::MAX_POSTED); -static_assert!(buffer::MIN_BLOCKS <= tuning::PB_BLOCKS); -static_assert!(tuning::PB_BLOCKS <= tuning::PB_MAX_BLOCKS); +static_assert!(buffer::MIN_BLOCKS <= tuning::PB_MAX_BLOCKS); // The PB alias is GPU-only (not cached and firmware-visible): dropping one is a plain unmap, // with no firmware cache flush (`KernelMapping::drop`). static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent()); +static_assert!(growth::MAX_CHUNKS * growth::CHUNK_BLOCKS == buffer::MAX_BLOCKS); -/// Physical backing of PB blocks `[first, first + blocks)`: a zeroed GEM kernel object, pinned -/// and DMA-mapped ([`object::gpu_backing`]), aliased only into user VMs at the blocks' fixed -/// client GVA ([`Firmware::map_pb`]). Retained until reboot, like every firmware object; each -/// alias also holds the object. +/// Bytes of one backing chunk. +pub(crate) const CHUNK_BYTES: usize = growth::CHUNK_BLOCKS as usize * buffer::BLOCK_BYTES as usize; + +/// Client GVA of backing chunk `i`. +pub(crate) const fn chunk_gva(i: usize) -> u64 { + client::PB_HEAP + (i * CHUNK_BYTES) as u64 +} + +/// Physical backing of one chunk of PB blocks: a zeroed GEM kernel object of [`CHUNK_BYTES`], +/// pinned and DMA-mapped ([`object::gpu_backing`]), aliased only into user VMs at its fixed +/// client GVA ([`chunk_gva`], [`VmState::map_heap`]). Each alias also holds the object. struct PbChunk { - first: u32, - blocks: u32, _pages: HeldPages, backing: gem::ObjectRef, } impl PbChunk { - fn new(dev: &AsahiDevice, first: u32, blocks: u32) -> Result { - if blocks == 0 || first.checked_add(blocks).ok_or(EINVAL)? > buffer::MAX_BLOCKS { - return Err(EINVAL); - } - let (backing, pages) = - object::gpu_backing(dev, blocks as usize * buffer::BLOCK_BYTES as usize)?; + fn new(dev: &AsahiDevice) -> Result { + let (backing, pages) = object::gpu_backing(dev, CHUNK_BYTES)?; Ok(Self { - first, - blocks, _pages: pages, backing, }) } - /// The block after the chunk's last. - fn end(&self) -> u32 { - self.first + self.blocks - } - - /// Maps the whole chunk, exactly its object's size, at its fixed client GVA in `vm`. - fn map(&self, vm: &mmu::Vm) -> Result { - vm.map_at( - client::PB_HEAP + self.first as u64 * buffer::BLOCK_BYTES, - self.blocks as usize * buffer::BLOCK_BYTES as usize, - &self.backing.gem, - mmu::Prot::from_pte(pte::LEAF_CACHED), - false, - ) + /// The GEM object, for the VM mappings. + fn gem(&self) -> ARef { + self.backing.gem.clone() } } @@ -161,6 +163,13 @@ pub(crate) struct PbStats { pub(crate) grown: u64, pub(crate) refused: u64, pub(crate) bad: u64, + /// Requests that waited a pass for backing. + pub(crate) waits: u64, + /// Chunks backed after the GPU started. + pub(crate) refills: u64, + /// Idle re-initializations, and the chunks they freed. + pub(crate) reclaims: u64, + pub(crate) freed: u64, } /// Render allocator and PB state: the allocators, the PB ledger and the spare set. @@ -179,9 +188,8 @@ pub(crate) struct Pools { spare_ready: bool, pb_pages: Arc, pb_blocks: Arc, - /// The PB backing, contiguous from block 0, in block order: the initial blocks, then the - /// growth chunks. Its capacity, reserved up front, is the most the ledger can need - /// ([`growth::max_chunks`]). + /// The PB backing chunks, from the heap base (chunk `i` at [`chunk_gva`]`(i)`). Its + /// capacity, reserved up front, is the whole heap's [`growth::MAX_CHUNKS`]. chunks: KVec, /// The PB growth ledger since the last (re)initialization. pub(crate) ledger: Ledger, @@ -189,6 +197,11 @@ pub(crate) struct Pools { c1_written: bool, /// The unanswered-at-the-maximum warning was given since the last initialization. exhausted_logged: bool, + /// The engine's idle clock (nothing published, nothing pending), for the reclaim. + idle: IdleClock, + /// The pending re-initialization was set by a reclaim ([`Firmware::reclaim_check`]), not by + /// a recovery; cleared by [`Self::init`]. + reclaim_pending: bool, pub(crate) stats: PbStats, } @@ -376,11 +389,24 @@ impl Pools { 0, true, )?; - // Backing for the initial blocks; growth adds the rest. - let ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); - let mut chunks = - KVec::with_capacity(growth::max_chunks(ledger.initial, ledger.max), GFP_KERNEL)?; - chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?; + // Backing for the initial blocks and the reserve; growth adds the rest. + let mut ledger = Ledger::new( + tuning::PB_GROW, + super::params::pb_blocks(), + tuning::PB_MAX_BLOCKS, + ); + let mut chunks = KVec::with_capacity(growth::MAX_CHUNKS as usize, GFP_KERNEL)?; + for _ in 0..Ledger::chunks_for(ledger.initial) { + chunks.push(PbChunk::new(dev)?, GFP_KERNEL)?; + ledger.back_chunk(); + } + dev_info!( + dev.as_ref(), + "G15: PB heap: {} blocks posted at start, {} chunks of {} MiB backed\n", + ledger.initial, + chunks.len(), + CHUNK_BYTES >> 20 + ); Ok(Self { dev: dev.as_ref().as_ref().into(), drm: dev.into(), @@ -394,6 +420,8 @@ impl Pools { ledger, c1_written: false, exhausted_logged: false, + idle: IdleClock::default(), + reclaim_pending: false, stats: PbStats::default(), }) } @@ -451,9 +479,11 @@ impl Pools { // PB: 16384 pages of 0x8000 in 4096 blocks of 0x20000; with growth the first initial // blocks of them, at most 4095 posted, and page limits at the firmware's growth maximum. - // In every profile the block list declares its real 4096-record capacity. - self.ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + // The backing is kept. In every profile the block list declares its real 4096-record + // capacity. + self.ledger.reinit(); self.exhausted_logged = false; + self.reclaim_pending = false; if tuning::PB_GROW { // C1 names the initial pages again. self.c1_written = false; @@ -470,7 +500,7 @@ impl Pools { } let heap = client::PB_HEAP; let initial = self.ledger.initial; - if initial > self.backed() { + if initial > self.ledger.backed { return Err(EINVAL); } write_records( @@ -524,133 +554,248 @@ impl Pools { super::params::pb_max_blocks().clamp(self.ledger.initial, self.ledger.max) } - /// The blocks with backing, from block 0. - pub(crate) fn backed(&self) -> u32 { - self.chunks.last().map_or(0, PbChunk::end) + /// Maps the backed chunks `vm` does not hold yet ([`VmState::map_heap`]). + fn map_heap(&self, vm: &VmState) -> Result { + vm.map_heap(self.chunks.iter().map(PbChunk::gem)) + } + + /// Frees the backing chunks from index `keep` on (their VM mappings are dropped by the + /// caller); the RAM returns when the last mapping of a chunk is gone. + fn free_chunks(&mut self, keep: usize) -> usize { + let freed = self.chunks.len().saturating_sub(keep); + self.chunks.truncate(keep); + self.ledger.unback_to(self.chunks.len() as u32); + freed + } + + /// Backs one more chunk; the caller maps it where needed before the ledger posts from it. + fn back_chunk(&mut self) -> Result> { + if self.chunks.len() >= growth::MAX_CHUNKS as usize { + return Err(ENOSPC); + } + let chunk = PbChunk::new(&self.drm)?; + let gem = chunk.gem(); + self.chunks.push(chunk, GFP_KERNEL)?; + Ok(gem) } /// The debugfs line `pb: blocks initial/posted/firmware-max/max backed blocks/chunks - /// requests grown refused bad`. + /// requests grown refused bad waits refills reclaims freed` (new keys are appended). pub(crate) fn write_stats(&self, f: &mut impl fmt::Write) -> fmt::Result { writeln!( f, - "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {}", + "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {} waits {} refills {} reclaims {} freed {}", self.ledger.initial, self.ledger.posted, self.soft_max(), self.ledger.max, - self.backed(), + self.ledger.backed, self.chunks.len(), self.stats.requests, self.stats.grown, self.stats.refused, - self.stats.bad + self.stats.bad, + self.stats.waits, + self.stats.refills, + self.stats.reclaims, + self.stats.freed ) } } impl Firmware { - /// Maps the PB backing into `vm` up to its last chunk, before anything of a prefix of the - /// VM is published (the start of `publish_prefix_into`). Render work runs in one VM at a - /// time (`Engine::publish`), so the VM about to run is the only one through which the GPU - /// can reach a posted block next; a VM created or idle while the backing grew catches up - /// here, also a closed VM whose accepted jobs still run. + /// Maps the PB backing into `vm`, before anything of a prefix of the VM is published (the + /// start of `publish_prefix_into`). Render work runs in one VM at a time + /// (`Engine::publish`), so the VM about to run is the only one through which the GPU can + /// reach a posted block next; a VM created or idle while the backing grew catches up here, + /// also a closed VM whose accepted jobs still run. A chunk is mapped only once + /// ([`VmState::map_heap`]); one that fails leaves the chunks mapped before it, and the + /// prefix is not published. The leaf is GPU-only, so dropping an alias is a plain unmap. /// - /// Each chunk's alias joins the VM's driver mappings, held until the VM is torn down, and - /// the VM's count moves past the chunk only then. When that fails, the alias just made is - /// dropped again: nothing of the VM runs before its count covers the chunk, and the leaf is - /// GPU-only, so the drop is a plain unmap. The chunks mapped before stay counted. - /// - /// The one-VM rule is checked first: a flight of another VM, or a count past the chunks, - /// fails with EPROTO before anything is mapped. + /// The one-VM rule is checked first: a flight of another VM fails with EPROTO before + /// anything is mapped. pub(crate) fn map_pb(&mut self, vm: &VmState) -> Result { - let mut mapped = vm.pb_chunks.load(Ordering::Relaxed); - if mapped > self.pools.chunks.len() - || self - .engine - .flights - .iter() - .any(|f| !core::ptr::eq(&*f.job.vm, vm)) - { + let others = self + .engine + .flights + .iter() + .filter(|f| !core::ptr::eq(&*f.job.vm, vm)) + .count(); + if others != 0 { dev_err!( self.dev(), - "PB backing: ctx {} maps {} of {} chunks with {} flights of another VM\n", - vm.slot, - mapped, + "PB backing: VM {} maps {} chunks with {} flights of another VM\n", + vm.id, self.pools.chunks.len(), - self.engine - .flights - .iter() - .filter(|f| !core::ptr::eq(&*f.job.vm, vm)) - .count() + others ); return Err(super::boot::eproto()); } - while let Some(chunk) = self.pools.chunks.get(mapped) { - let mut aliases = KVec::with_capacity(1, GFP_KERNEL)?; - aliases.push(chunk.map(&vm.vm)?, GFP_KERNEL)?; - vm.vm.append_driver_mappings(aliases)?; - mapped += 1; - vm.pb_chunks.store(mapped, Ordering::Relaxed); - } - Ok(()) + self.pools.map_heap(vm) } - /// The VM whose render work runs: every flight's (`Engine::publish` puts a flight behind - /// another only for the same VM), and it must be the asking context's. EPROTO otherwise. - fn running_vm(&self, ctx: u32) -> Result> { - let flights = &self.engine.flights; - let vm = flights - .first() - .ok_or_else(super::boot::eproto)? - .job - .vm - .clone(); - if vm.slot != ctx || flights.iter().any(|f| !Arc::ptr_eq(&f.job.vm, &vm)) { - dev_err!( - self.dev(), - "PB growth: ctx {} is not the one VM in flight (ctx {}, {} flights)\n", - ctx, - vm.slot, - flights.len() - ); - return Err(super::boot::eproto()); + /// Backs chunks until the reserve is whole, mapping each into the VMs of the flights (by + /// the one-VM rule, the VM that runs) before the ledger may post from it; any other VM + /// maps it at its next publication ([`Self::map_pb`]). A failure is logged and leaves the + /// reserve short: requests then wait, and the next pass tries again. + pub(crate) fn refill_backing(&mut self) { + // A pending re-initialization is about to drop the posted count: nothing to back for, + // unless a render published before it still runs (the flights behind a fragment kill + // keep asking for growth until they retire). + let render = (0..self.engine.flights.len()).any(|i| self.receipts(i).renders != 0); + if self.recovery.reinit && !render { + return; } - Ok(vm) + while self.pools.ledger.refill_due() { + let result = (|| -> Result { + let gem = self.pools.back_chunk()?; + let i = self.pools.chunks.len() - 1; + for flight in self.engine.flights.iter() { + flight.job.vm.map_chunk(i, gem.clone())?; + } + Ok(()) + })(); + match result { + Ok(()) => { + self.pools.ledger.back_chunk(); + self.pools.stats.refills += 1; + dev_dbg!( + self.dev(), + "PB heap: chunk {} backed ({} of {} blocks posted)\n", + self.pools.chunks.len() - 1, + self.pools.ledger.posted, + self.pools.ledger.backed + ); + } + Err(e) => { + // A chunk that could not be mapped into a flight's VM is not counted: drop + // it (and its aliases in the VMs) again. + let backed = (self.pools.ledger.backed / growth::CHUNK_BLOCKS) as usize; + if self.pools.chunks.len() > backed { + for flight in self.engine.flights.iter() { + flight.job.vm.unmap_heap_above(backed); + } + self.pools.chunks.truncate(backed); + } + dev_warn!( + self.dev(), + "PB heap: backing chunk {} failed: {} ({} of {} blocks posted)\n", + backed, + e.to_errno(), + self.pools.ledger.posted, + self.pools.ledger.backed + ); + return; + } + } + } + } + + /// Notes the engine idle (nothing published, nothing pending) or busy; the idle time is + /// what [`Self::reclaim_check`] measures. Called at the end of every poll pass and + /// publication (the pass that retires the last flight is not re-armed) and at a job's + /// arrival. + pub(crate) fn note_idle(&mut self) { + let idle = self.engine.flights.is_empty() && self.engine.pending.is_empty(); + self.pools.idle.note(idle, super::engine::now_ns()); } - /// Backs the PB up to block `end` in the VM that runs, the asking context `ctx`, before an - /// answer posts those blocks (nothing new when they are backed already): chunks of at most - /// [`growth::CHUNK_BLOCKS`] blocks are allocated ([`object::gpu_backing`]), mapped into that - /// VM and kept, one at a time. The VM first catches up ([`Self::map_pb`], which also checks - /// the one-VM rule), so its count equals the chunks. A chunk is kept - /// only once its alias in the VM is held, and the VM's count moves with it; when the alias - /// cannot be held, the alias and the chunk are dropped, neither ever reachable (the leaf is - /// GPU-only: a plain unmap). An error leaves the chunks backed so far for the next request, - /// and nothing is posted. The allocation sleeps (shmem pages with `__GFP_RETRY_MAYFAIL`) - /// while the firmware waits for the answer; how long is not bounded here. - fn back_pb(&mut self, end: u32, ctx: u32) -> Result { - let vm = self.running_vm(ctx)?; - self.map_pb(&vm)?; - while self.pools.backed() < end { - let (first, blocks) = - growth::next_chunk(self.pools.backed(), self.pools.ledger.max).ok_or(ENOSPC)?; - if self.pools.chunks.len() == self.pools.chunks.capacity() { - return Err(ENOSPC); + /// Reclaims the PB when the engine has been idle for `tuning::PB_IDLE_MS` with more than + /// `tuning::PB_RECLAIM_BLOCKS` posted or more backing than the posted blocks need, or for + /// [`growth::DRAIN_IDLE_NS`] with more than half the maximum posted + /// ([`growth::Ledger::drain_reinit_due`]; re-initialization only): the next render prefix + /// re-initializes the render allocators and the PB (InitBM of the initial blocks; + /// `recovery.reinit`, as after a recovered abort but with no halt), and with + /// `tuning::PB_RECLAIM` 2 the backing chunks above those of the initial posting and the reserve + /// are freed now, unmapped from every VM of the registry (a VM that publishes later drops + /// its stale mappings at [`VmState::map_heap`]). While a reclaim's re-initialization is + /// pending (a client-boundary one kept the backing), `tuning::PB_RECLAIM` 2 frees the backing + /// above those chunks once the engine has been idle for `tuning::PB_IDLE_MS` + /// ([`growth::Ledger::pending_free_due`]). Called under the device lock when a job arrives + /// at an idle engine and in idle poll passes. + pub(crate) fn reclaim_check(&mut self) { + let level = super::params::pb_reclaim(); + if level == 0 + || !self.engine.flights.is_empty() + || !self.engine.pending.is_empty() + || (self.recovery.reinit && !self.pools.reclaim_pending) + || self.recovery.armed.is_some() + || self.engine.frozen + || self.recovery.failed().is_some() + { + return; + } + let idle_ns = self.pools.idle.idle_ns(super::engine::now_ns()); + let floor = super::params::pb_reclaim_blocks(); + let min_idle = super::params::pb_idle_ns(); + let ledger = &self.pools.ledger; + if self.recovery.reinit { + if level >= 2 && ledger.pending_free_due(idle_ns, min_idle) { + let posted = ledger.posted; + let freed = self.free_backing(); + dev_info!( + self.dev(), + "PB heap: idle {} ms with {} blocks posted and the re-initialization pending: {} chunks freed, {} backed\n", + idle_ns / 1_000_000, + posted, + freed, + self.pools.chunks.len() + ); } - let chunk = PbChunk::new(&self.pools.drm, first, blocks)?; - let mut aliases = KVec::with_capacity(1, GFP_KERNEL)?; - aliases.push(chunk.map(&vm.vm)?, GFP_KERNEL)?; - vm.vm.append_driver_mappings(aliases)?; - // Within the capacity checked above. - self.pools - .chunks - .push_within_capacity(chunk) - .map_err(|_| EIO)?; - vm.pb_chunks - .store(self.pools.chunks.len(), Ordering::Relaxed); + return; } - Ok(()) + let idle = ledger.reclaim_due(floor, idle_ns, min_idle) + || (level >= 2 && ledger.free_due(idle_ns, min_idle)); + if !idle && !ledger.drain_reinit_due(idle_ns) { + return; + } + self.recovery.reinit = true; + self.pools.reclaim_pending = true; + self.pools.stats.reclaims += 1; + let posted = self.pools.ledger.posted; + let mut freed = 0; + // A short idle keeps the backing: the next scene would allocate it again. + if level >= 2 && idle { + freed = self.free_backing(); + } + dev_info!( + self.dev(), + "PB heap: idle {} ms with {} blocks posted: re-initialization to {} at the next render, {} chunks freed, {} backed\n", + idle_ns / 1_000_000, + posted, + self.pools.ledger.initial, + freed, + self.pools.chunks.len() + ); + } + + /// Frees the backing chunks above those a re-initialized heap keeps + /// ([`growth::Ledger::keep_chunks`]) and drops their mappings in every VM that could + /// publish; returns the chunks freed. If the VMs cannot be walked, their mappings keep the + /// pages until each VM drops its stale aliases ([`VmState::map_heap`]): nothing is counted + /// freed. + fn free_backing(&mut self) -> usize { + let keep = self.pools.ledger.keep_chunks() as usize; + let dropped = self.pools.free_chunks(keep); + // The mappings hold the chunks' pages: drop them in every VM that could publish. + let freed = match self.vms.open() { + Ok(states) => { + for state in states.iter() { + state.unmap_heap_above(keep); + } + dropped + } + Err(e) => { + dev_warn!( + self.dev(), + "PB heap: reclaim could not walk the VMs: {}\n", + e.to_errno() + ); + 0 + } + }; + self.pools.stats.freed += freed as u64; + freed } /// (Re)initializes the render UMA managers, PB lists and PB manager. `rebuild`: the priming @@ -683,9 +828,12 @@ impl Firmware { /// DEVCTL 7 reports success; at the ledger's maximum the reply is the refusal (+4 = 0), /// which gives the firmware no new pages, and after two refusals requests are left /// unanswered. The firmware imports the blocks itself: P is never rewritten. Only the - /// requests queued before the pass are taken. A failed reply is logged; a request that - /// cannot be processed at all (a firmware object access failed) is logged and left - /// unanswered like an invalid one. Never fails: the poll pass continues. + /// requests queued before the pass are taken. A request with no backed block left waits: + /// it goes back to the head of the queue for the next pass, and this pass ends its answers + /// there. After the answers the reserve is refilled ([`Self::refill_backing`]). A failed + /// reply is logged; a request that cannot be processed at all (a firmware object access + /// failed) is logged and left unanswered like an invalid one. Never fails: the poll pass + /// continues. pub(crate) fn answer_growth(&mut self) -> Result { // Only the requests queued before this pass: a reply's wait drains the rings and may // queue the next request, which the next pass answers (the device lock is released in @@ -695,23 +843,67 @@ impl Firmware { let Ok(request) = self.events.grow.remove(0) else { break; }; - self.pools.stats.requests += 1; - if let Err(e) = self.answer_request(&request) { - self.pools.stats.bad += 1; - dev_err!( - self.dev(), - "PB growth request {} ctx {} not answered: {}\n", - request[buffer::grow_request::COUNTER], - request[buffer::grow_request::CONTEXT], - e.to_errno() - ); + match self.answer_request(&request) { + Ok(true) => self.pools.stats.requests += 1, + Ok(false) => { + // Waits for backing: back at the head, counted once answered. + self.pools.stats.waits += 1; + // The queue has room: this request was just taken from it. + if self.events.grow.insert_within_capacity(0, request).is_err() { + self.pools.stats.bad += 1; + } + break; + } + Err(e) => { + self.pools.stats.requests += 1; + self.pools.stats.bad += 1; + dev_err!( + self.dev(), + "PB growth request {} ctx {} not answered: {}\n", + request[buffer::grow_request::COUNTER], + request[buffer::grow_request::CONTEXT], + e.to_errno() + ); + } } } + self.refill_backing(); Ok(()) } - /// Processes one growth request. - fn answer_request(&mut self, request: &[u32; 14]) -> Result { + /// How long the idle engine must stay idle before [`Self::reclaim_check`] acts, if it can + /// act without new work ([`growth::Ledger::reclaim_delay`], or with a reclaim's + /// re-initialization pending [`growth::Ledger::pending_free_delay`]); `None` while busy, + /// while a recovery or its re-initialization is pending, or when nothing would become due. + /// An idle engine runs no poll pass by itself, so the poll work is armed for this delay. + pub(crate) fn reclaim_delay_ns(&self) -> Option { + if !self.engine.flights.is_empty() + || !self.engine.pending.is_empty() + || (self.recovery.reinit && !self.pools.reclaim_pending) + || self.recovery.armed.is_some() + || self.engine.frozen + || self.recovery.failed().is_some() + { + return None; + } + let idle_ns = self.pools.idle.idle_ns(super::engine::now_ns()); + if self.recovery.reinit { + return self.pools.ledger.pending_free_delay( + super::params::pb_reclaim(), + idle_ns, + super::params::pb_idle_ns(), + ); + } + self.pools.ledger.reclaim_delay( + super::params::pb_reclaim(), + super::params::pb_reclaim_blocks(), + idle_ns, + super::params::pb_idle_ns(), + ) + } + + /// Processes one growth request: `Ok(false)` when it waits for backing. + fn answer_request(&mut self, request: &[u32; 14]) -> Result { let ctl = self.init.control.clone().ok_or(ENODEV)?; let epoch = ctl.r32(control::HALT_COUNT)? as u64 | (ctl.r32(control::HALT_COUNT + 4)? as u64) << 32; @@ -721,7 +913,7 @@ impl Firmware { .flights .iter() .enumerate() - .any(|(i, f)| self.receipts(i).renders != 0 && f.job.vm.slot == ctx); + .any(|(i, f)| self.receipts(i).renders != 0 && f.job.slot() == ctx); // At the maximum, after the refusals, the request is left unanswered: the poll pass // returns, and the host deadline and its recovery path apply to a stalled TA. if self.pools.ledger.exhausted() { @@ -737,7 +929,7 @@ impl Firmware { self.pools.ledger.max ); } - return Ok(()); + return Ok(true); } // A request from before a halt has no authority over the PB now. if !self @@ -758,15 +950,15 @@ impl Firmware { live as u32, tuning::PB_GROW as u32 ); - return Ok(()); + return Ok(true); } - // Every block the answer posts is backed in the VM that runs before anything of the - // answer is written; the one-VM rule is checked for every answer. If that fails, nothing - // is posted or answered (answer_growth logs it; the deadline applies), and the backing - // made so far stays for the next request. - let end = self.pools.ledger.posted + self.pools.ledger.next_count(); - self.back_pb(end, ctx)?; - let (first, n) = self.pools.ledger.accept(request); + // A grant posts only backed blocks, and every backed chunk is mapped in the VM of the + // flights (at its publication, or at the refill that backed it). + let (first, n) = match self.pools.ledger.accept(request) { + Answer::Grant { first, count } => (first, count), + Answer::Refuse => (self.pools.ledger.posted, 0), + Answer::Wait => return Ok(false), + }; let heap = client::PB_HEAP; for b in first..first + n { self.pools.pb_blocks.w64( @@ -831,6 +1023,6 @@ impl Firmware { ret ); } - Ok(()) + Ok(true) } } diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index 20b298e641459..e9eaee242ac51 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -12,6 +12,15 @@ //! its recovery path then apply, without a guarantee that they end the stall. A request from //! before a halt (epoch) or not matching a published render prefix is left unanswered. //! +//! The heap's VA range covers [`MAX_BLOCKS`] blocks; its RAM comes in chunks of +//! [`CHUNK_BLOCKS`] blocks from the heap base, which the ledger counts as `backed`. A request is +//! granted from the backed blocks above `posted`. A chunk is due ([`Ledger::refill_due`]) +//! whenever fewer than [`RESERVE_BLOCKS`] backed blocks remain above the posted count, so that +//! it can be backed after a reply; a request that finds no backed block waits +//! ([`Answer::Wait`]) and keeps its counter. A re-initialization ([`Ledger::reinit`]) returns +//! the posted count to the initial size and keeps the backing; [`Ledger::keep_chunks`] gives the +//! chunks that the initial posting and the reserve need. +//! //! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. use crate::g15::fw::buffer::{grow_request, BLOCK_LIST_RECORDS, GROW_STEP, MAX_BLOCKS, MIN_BLOCKS}; @@ -26,6 +35,8 @@ pub(crate) struct Ledger { pub(crate) max: u32, /// Blocks posted to the block list, cumulative (the block_ctl+4 producer is this modulo K). pub(crate) posted: u32, + /// Blocks with RAM behind them, from the heap base. + pub(crate) backed: u32, /// The next expected request counter. pub(crate) next_request: u32, /// A refusal at the maximum was logged since the last initialization. @@ -41,26 +52,30 @@ pub(crate) struct Ledger { /// Refusals answered at the maximum before requests are left to the deadline. pub(crate) const MAX_REFUSALS: u32 = 2; -/// Blocks of each PB backing chunk after the initial one (4 MiB): growth backs at most this -/// much at a time while the firmware waits for its answer. -pub(crate) const CHUNK_BLOCKS: u32 = 32; - -/// The backing chunk that follows `backed` blocks of a ledger with maximum `max`: its first block -/// and size, at most [`CHUNK_BLOCKS`] and never past `max`; none from `max` on. -pub(crate) const fn next_chunk(backed: u32, max: u32) -> Option<(u32, u32)> { - if backed >= max { - None - } else if max - backed < CHUNK_BLOCKS { - Some((backed, max - backed)) - } else { - Some((backed, CHUNK_BLOCKS)) - } -} +/// Blocks of each PB backing chunk (16 MiB). +pub(crate) const CHUNK_BLOCKS: u32 = 128; + +/// Backed blocks kept above the posted count: at 1-2 ms per request of ten blocks, 13 requests +/// of lead for the allocation of the next chunk. +pub(crate) const RESERVE_BLOCKS: u32 = CHUNK_BLOCKS; -/// The chunks that back a ledger with `initial` and `max` blocks at most: the initial one and -/// [`next_chunk`]'s from `initial` to `max`. -pub(crate) const fn max_chunks(initial: u32, max: u32) -> usize { - 1 + max.saturating_sub(initial).div_ceil(CHUNK_BLOCKS) as usize +/// Chunks of the whole heap. +pub(crate) const MAX_CHUNKS: u32 = MAX_BLOCKS / CHUNK_BLOCKS; + +/// The idle time before a re-initialization past half the maximum (a client boundary, not a +/// gap between one client's renders). +pub(crate) const DRAIN_IDLE_NS: u64 = 50_000_000; + +/// The answer to an admitted growth request. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Answer { + /// Post blocks `[first, first + count)` (`count` at most [`GROW_STEP`]) and reply grown. + Grant { first: u32, count: u32 }, + /// Reply refused: the maximum is posted. + Refuse, + /// No backed block above the posted count: leave the request queued until a chunk is + /// backed. + Wait, } /// The most blocks a ledger posts: the mapped heap, and below K, so that the block_ctl+4 @@ -85,17 +100,141 @@ impl Ledger { initial, max, posted: initial, + backed: 0, next_request: 0, refusal_logged: false, refusals: 0, } } + /// A re-initialization: the initial blocks are posted again, the request counter and the + /// refusals start over, the backing stays. + pub(crate) const fn reinit(&mut self) { + self.posted = self.initial; + self.next_request = 0; + self.refusal_logged = false; + self.refusals = 0; + } + /// The live block_ctl+4 producer. pub(crate) const fn producer(&self) -> u32 { self.posted % BLOCK_LIST_RECORDS } + /// Records one more backed chunk. + pub(crate) const fn back_chunk(&mut self) { + self.backed += CHUNK_BLOCKS; + } + + /// Drops the backing to `chunks` chunks. + pub(crate) const fn unback_to(&mut self, chunks: u32) { + self.backed = chunks * CHUNK_BLOCKS; + } + + /// Backed blocks above the posted count. + pub(crate) const fn spare(&self) -> u32 { + self.backed.saturating_sub(self.posted) + } + + /// Whether another chunk is due: fewer than the reserve above the posted count and the heap + /// not yet backed whole. + pub(crate) const fn refill_due(&self) -> bool { + self.backed < MAX_BLOCKS && self.spare() < RESERVE_BLOCKS + } + + /// Chunks that `blocks` posted blocks and the reserve need. + pub(crate) const fn chunks_for(blocks: u32) -> u32 { + let n = (blocks + RESERVE_BLOCKS).div_ceil(CHUNK_BLOCKS); + if n > MAX_CHUNKS { + MAX_CHUNKS + } else { + n + } + } + + /// Chunks a re-initialized heap keeps: those of the initial posting and the reserve. + pub(crate) const fn keep_chunks(&self) -> u32 { + Self::chunks_for(self.initial) + } + + /// Whether the heap is to be re-initialized: idle for `idle_ns` with more than `floor` + /// blocks posted, and grown since the initialization (with nothing grown a + /// re-initialization posts the same blocks again). + pub(crate) const fn reclaim_due(&self, floor: u32, idle_ns: u64, min_idle_ns: u64) -> bool { + self.posted > self.initial && self.posted > floor && idle_ns >= min_idle_ns + } + + /// Whether the backing is to be freed: idle for `idle_ns` with more chunks backed than the + /// posted blocks and the reserve need. A client-boundary re-initialization keeps the + /// backing of a heavy scene; if the next client posts below the reclaim floor, this frees it. + pub(crate) const fn free_due(&self, idle_ns: u64, min_idle_ns: u64) -> bool { + idle_ns >= min_idle_ns && self.backed > Self::chunks_for(self.posted) * CHUNK_BLOCKS + } + + /// While a reclaim's re-initialization is pending (the posted count returns to the initial + /// size at the next render): whether the backing above [`Self::keep_chunks`] is to be + /// freed, idle for `min_idle_ns`. A client-boundary re-initialization keeps the backing; + /// with no render after it, this frees it. + pub(crate) const fn pending_free_due(&self, idle_ns: u64, min_idle_ns: u64) -> bool { + idle_ns >= min_idle_ns && self.backed > self.keep_chunks() * CHUNK_BLOCKS + } + + /// How much longer the engine must stay idle, from `idle_ns`, before + /// [`Self::pending_free_due`] with `level`; `None` when it cannot become due (level below + /// 2, or nothing above the kept chunks). 0 when it is due now. + pub(crate) const fn pending_free_delay( + &self, + level: u32, + idle_ns: u64, + min_idle_ns: u64, + ) -> Option { + if level >= 2 && self.backed > self.keep_chunks() * CHUNK_BLOCKS { + Some(min_idle_ns.saturating_sub(idle_ns)) + } else { + None + } + } + + /// Whether a short idle re-initializes the heap: more than half the maximum posted and + /// idle for [`DRAIN_IDLE_NS`]. Back-to-back heavy scenes (a new client every few hundred + /// ms) never reach `tuning::PB_IDLE_MS`, and posted blocks never come back while a PB lives, so + /// without this they run the heap to the maximum. A re-initialization with no idle time + /// faulted at the heap base (at a client switch, the second in a row with no recovery + /// between); with 50 ms of idle none did, also between two renders of one client. + pub(crate) const fn drain_reinit_due(&self, idle_ns: u64) -> bool { + self.posted > self.initial && self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS + } + + /// How much longer the engine must stay idle, from `idle_ns`, before a reclaim with + /// `level` is due ([`Self::reclaim_due`] with `floor` and `min_idle_ns`, [`Self::free_due`] + /// at level 2, or [`Self::drain_reinit_due`]); `None` when none can become due while the + /// ledger stays as it is. 0 when one is due now. + pub(crate) const fn reclaim_delay( + &self, + level: u32, + floor: u32, + idle_ns: u64, + min_idle_ns: u64, + ) -> Option { + if level == 0 { + return None; + } + let grown = self.posted > self.initial; + let mut due = u64::MAX; + if grown && self.posted > self.max / 2 { + due = DRAIN_IDLE_NS; + } + let free = level >= 2 && self.backed > Self::chunks_for(self.posted) * CHUNK_BLOCKS; + if ((grown && self.posted > floor) || free) && min_idle_ns < due { + due = min_idle_ns; + } + if due == u64::MAX { + None + } else { + Some(due.saturating_sub(idle_ns)) + } + } + /// Whether the refusals at the maximum are used up: further requests go unanswered. pub(crate) const fn exhausted(&self) -> bool { self.refusals >= MAX_REFUSALS @@ -114,21 +253,24 @@ impl Ledger { && request_epoch(request) == epoch } - /// The blocks the next accepted request posts: at most [`GROW_STEP`], none at the maximum. - pub(crate) fn next_count(&self) -> u32 { - GROW_STEP.min(self.max - self.posted) - } - - /// Accepts an admitted request: the next expected counter follows it, and the returned - /// blocks `[first, first + count)` are to be written to the block list, `count` 0 meaning a - /// refusal at the maximum. - pub(crate) fn accept(&mut self, request: &[u32; 14]) -> (u32, u32) { + /// Accepts an admitted request. Granted (at most [`GROW_STEP`] blocks, and no more than are + /// backed) or refused at the maximum, the next expected counter follows it. A request that + /// finds no backed block waits: its counter is not consumed, and it is accepted again once a + /// chunk is backed. + pub(crate) fn accept(&mut self, request: &[u32; 14]) -> Answer { + let room = self.max - self.posted; + if room != 0 && self.spare() == 0 { + return Answer::Wait; + } self.next_request = request[grow_request::COUNTER].wrapping_add(1); - let count = self.next_count(); - if count == 0 { + if room == 0 { self.refusals += 1; + return Answer::Refuse; + } + Answer::Grant { + first: self.posted, + count: GROW_STEP.min(room).min(self.spare()), } - (self.posted, count) } /// Records `count` blocks as posted after their records were written and returns the new @@ -154,6 +296,37 @@ impl Ledger { } } +/// The engine's idle clock for the reclaim policy: when nothing was published and nothing +/// pending, from the pass that retired the last flight on. Noted at the end of every poll pass +/// and publication (a pass that empties the flights is not re-armed, so a note only at the +/// start of a pass with empty flights depends on a backup pass happening to run) and, busy, +/// when a job is enqueued. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct IdleClock { + /// When the idle began; 0 while busy. + since_ns: u64, +} + +impl IdleClock { + /// Notes the engine idle or busy at `now_ns`; an idle already begun keeps its start. + pub(crate) fn note(&mut self, idle: bool, now_ns: u64) { + if !idle { + self.since_ns = 0; + } else if self.since_ns == 0 { + self.since_ns = now_ns.max(1); + } + } + + /// The idle time at `now_ns`; 0 while busy. + pub(crate) fn idle_ns(&self, now_ns: u64) -> u64 { + if self.since_ns == 0 { + 0 + } else { + now_ns.wrapping_sub(self.since_ns) + } + } +} + /// The halt epoch a request names (u64 at words 5..6). pub(crate) const fn request_epoch(request: &[u32; 14]) -> u64 { request[grow_request::EPOCH] as u64 | (request[grow_request::EPOCH + 1] as u64) << 32 @@ -185,12 +358,19 @@ mod tests { q } + /// A ledger with the heap backed whole. + fn backed(grow: bool, blocks: u32, max_blocks: u32) -> Ledger { + let mut l = Ledger::new(grow, blocks, max_blocks); + l.backed = MAX_BLOCKS; + l + } + #[test] fn clamps() { - let l = Ledger::new(true, 1024, 2048); + let l = backed(true, 1024, 2048); assert_eq!( - (l.initial, l.max, l.posted, l.next_request), - (1024, 2048, 1024, 0) + (l.initial, l.max, l.posted, l.backed, l.next_request), + (1024, 2048, 1024, 4096, 0) ); assert_eq!(l.producer(), 1024); assert_eq!(Ledger::new(true, 10, 20).initial, 64); @@ -198,13 +378,14 @@ mod tests { assert_eq!(Ledger::new(true, 5000, 100).initial, 4095); assert_eq!(Ledger::new(true, 512, 1024).max, 1024); assert_eq!(Ledger::new(true, 512, 100).max, 512); + assert_eq!(Ledger::new(true, 512, 100).backed, 0); let fixed = Ledger::new(false, 512, 1024); assert_eq!((fixed.initial, fixed.max, fixed.posted), (4095, 4095, 4095)); - // The values the driver passes (tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS) are kept. - let l = Ledger::new(true, 1024, 4095); + // The values the driver passes (tuning::PB_BLOCKS 64, tuning::PB_MAX_BLOCKS) are kept. + let l = Ledger::new(true, 64, 4095); assert_eq!( (l.initial, l.max, l.posted, l.producer()), - (1024, 4095, 1024, 1024) + (64, 4095, 64, 64) ); } @@ -235,18 +416,19 @@ mod tests { for grow in [true, false] { for blocks in values { for max_blocks in values { - let mut l = Ledger::new(grow, blocks, max_blocks); + let mut l = backed(grow, blocks, max_blocks); assert!(MIN_BLOCKS <= l.initial && l.initial <= l.max && l.max <= MAX_POSTED); assert_eq!(l.posted, l.initial); assert!((1..=MAX_POSTED).contains(&l.producer())); // Every producer of a growth to the maximum stays in 1..=4095. let mut counter = 0; loop { - let (first, n) = l.accept(&request(5, counter, 1)); + let (first, n) = match l.accept(&request(5, counter, 1)) { + Answer::Grant { first, count } => (first, count), + Answer::Refuse => break, + Answer::Wait => panic!("backed whole"), + }; assert_eq!(first, l.posted); - if n == 0 { - break; - } let producer = l.post(n); assert!((1..=MAX_POSTED).contains(&producer)); assert_eq!(producer, l.posted); @@ -261,7 +443,7 @@ mod tests { #[test] fn admission() { - let l = Ledger::new(true, 1024, 2048); + let l = backed(true, 1024, 2048); let q = request(5, 0, 3); assert!(l.admits(&q, true, true, 3)); assert!(!l.admits(&q, false, true, 3)); @@ -283,17 +465,20 @@ mod tests { #[test] fn growth_to_the_maximum() { // 512 initial and 1024 maximum blocks. - let mut l = Ledger::new(true, 512, 1024); + let mut l = backed(true, 512, 1024); let mut grown = 0; let mut counter = 0; loop { let q = request(5, counter, 1); assert!(l.admits(&q, true, true, 1)); - let (first, n) = l.accept(&q); - assert_eq!(first, 512 + grown); - if n == 0 { - break; - } + let n = match l.accept(&q) { + Answer::Grant { first, count } => { + assert_eq!(first, 512 + grown); + count + } + Answer::Refuse => break, + Answer::Wait => panic!("backed whole"), + }; assert!(n <= 10); assert_eq!(l.post(n), (512 + grown + n) % 4096); grown += n; @@ -310,26 +495,97 @@ mod tests { assert_eq!(l.refusals, 1); assert!(!l.exhausted()); let q = request(5, counter + 3, 1); - assert_eq!(l.accept(&q), (1024, 0)); + assert_eq!(l.accept(&q), Answer::Refuse); assert!(l.exhausted()); assert!(!l.admits(&request(5, counter + 4, 1), true, true, 1)); - // A fresh ledger starts over. + // A fresh ledger starts over, and so does a re-initialized one (backing kept). assert!(!Ledger::new(true, 512, 1024).exhausted()); + l.reinit(); + assert_eq!( + (l.posted, l.next_request, l.refusals, l.backed), + (512, 0, 0, 4096) + ); + assert!(!l.exhausted() && !l.refusal_logged); } - /// Backing for every ledger the constructor makes, the driver's included: the chunks from - /// the initial one are contiguous, at most [`CHUNK_BLOCKS`] each, end at the maximum and - /// number at most [`max_chunks`]; growth with the backing made first never posts a block - /// past it. #[test] - fn backing_chunks() { - assert_eq!(max_chunks(1024, 4095), 1 + 96); - assert_eq!(max_chunks(MIN_BLOCKS, MAX_POSTED), 1 + 126); - assert_eq!(max_chunks(MAX_POSTED, MAX_POSTED), 1); - assert_eq!(next_chunk(1024, 4095), Some((1024, 32))); - assert_eq!(next_chunk(4064, 4095), Some((4064, 31))); - assert_eq!(next_chunk(4095, 4095), None); - assert_eq!(next_chunk(4096, 4095), None); + fn producer_wraps_at_k() { + let mut l = backed(true, 2048, 2048); + assert_eq!(l.producer(), 2048); + l.posted = 4095; + assert_eq!(l.post(1), 0); + assert_eq!(l.post(10), 10); + } + + #[test] + fn backing() { + // 64 initial blocks: two chunks (the initial posting and the reserve). + assert_eq!(Ledger::chunks_for(64), 2); + assert_eq!(Ledger::chunks_for(1024), 9); + assert_eq!(Ledger::chunks_for(4095), MAX_CHUNKS); + assert_eq!(MAX_CHUNKS, 32); + let mut l = Ledger::new(true, 64, 4095); + assert_eq!(l.keep_chunks(), 2); + assert_eq!(l.spare(), 0); + assert!(l.refill_due()); + // Nothing backed: a request waits and the counter does not advance. + let q = request(3, 0, 1); + assert_eq!(l.accept(&q), Answer::Wait); + assert_eq!(l.next_request, 0); + l.back_chunk(); + l.back_chunk(); + assert_eq!((l.backed, l.spare()), (256, 192)); + assert!(!l.refill_due()); + // Granting until the reserve line, then a refill is due; at the end of the backing the + // grant is clipped and the next request waits. + assert_eq!( + l.accept(&q), + Answer::Grant { + first: 64, + count: 10 + } + ); + assert_eq!(l.post(10), 74); + assert_eq!(l.next_request, 1); + l.posted = 128; + assert!(!l.refill_due()); + l.posted = 129; + assert!(l.refill_due()); + l.posted = 250; + assert_eq!( + l.accept(&request(3, 1, 1)), + Answer::Grant { + first: 250, + count: 6 + } + ); + l.post(6); + assert_eq!(l.accept(&request(3, 2, 1)), Answer::Wait); + // The heap end: no refill past the whole heap, the maximum refuses. + l.backed = MAX_BLOCKS; + l.posted = 4090; + assert!(!l.refill_due()); + assert_eq!( + l.accept(&request(3, 2, 1)), + Answer::Grant { + first: 4090, + count: 5 + } + ); + l.post(5); + assert_eq!(l.accept(&request(3, 3, 1)), Answer::Refuse); + // Re-initialization keeps the backing; dropping it to the kept chunks. + l.reinit(); + assert_eq!((l.posted, l.backed), (64, 4096)); + l.unback_to(l.keep_chunks()); + assert_eq!(l.backed, 256); + } + + /// Every ledger the constructor makes, the driver's included, backed a chunk at a time + /// whenever the reserve is short: no grant posts a block past the backing, no request waits, + /// and the backing ends within the heap. + #[test] + fn backing_ahead_of_every_grant() { let values = [ 0, 1, MIN_BLOCKS, 65, 1023, 1024, 1025, 3072, 4094, MAX_POSTED, 5000, ]; @@ -337,40 +593,308 @@ mod tests { for blocks in values { for max_blocks in values { let mut l = Ledger::new(grow, blocks, max_blocks); - let (mut backed, mut chunks) = (l.initial, 1); + for _ in 0..Ledger::chunks_for(l.initial) { + l.back_chunk(); + } let mut counter = 0; loop { - let end = l.posted + l.next_count(); - while backed < end { - let (first, n) = next_chunk(backed, l.max).unwrap(); - assert_eq!(first, backed); - assert!(n >= 1 && n <= CHUNK_BLOCKS); - backed += n; - chunks += 1; - } - let (first, n) = l.accept(&request(5, counter, 1)); - assert!(first + n <= backed); - if n == 0 { - break; + match l.accept(&request(5, counter, 1)) { + Answer::Grant { first, count } => { + assert_eq!(first, l.posted); + assert!(count >= 1 && first + count <= l.backed); + l.post(count); + } + Answer::Refuse => break, + Answer::Wait => panic!("waited at {} of {}", l.posted, l.backed), } - l.post(n); counter += 1; + while l.refill_due() { + l.back_chunk(); + } } - assert_eq!((l.posted, backed), (l.max, l.max)); - assert!(chunks <= max_chunks(l.initial, l.max)); - assert_eq!(next_chunk(backed, l.max), None); + assert_eq!(l.posted, l.max); + assert!(l.max <= l.backed && l.backed <= MAX_BLOCKS); } } } } #[test] - fn producer_wraps_at_k() { - let mut l = Ledger::new(true, 2048, 2048); - assert_eq!(l.producer(), 2048); - l.posted = 4095; - assert_eq!(l.post(1), 0); - assert_eq!(l.post(10), 10); + fn reclaim() { + let mut l = Ledger::new(true, 64, 4095); + let s = 1_000_000_000; + assert!(!l.reclaim_due(1024, 5 * s, s)); + l.posted = 1024; + assert!(!l.reclaim_due(1024, 5 * s, s)); + l.posted = 1025; + assert!(l.reclaim_due(1024, s, s)); + assert!(!l.reclaim_due(1024, s - 1, s)); + // The clock: busy is 0, an idle keeps its start, busy ends it. + let mut c = IdleClock::default(); + assert_eq!(c.idle_ns(10 * s), 0); + c.note(true, 10 * s); + c.note(true, 12 * s); + assert_eq!(c.idle_ns(15 * s), 5 * s); + c.note(false, 16 * s); + assert_eq!(c.idle_ns(20 * s), 0); + // A clock started at 0 ns still counts as idle. + c.note(true, 0); + assert_eq!(c.idle_ns(s), s - 1); + // Nothing grown since the initialization: no re-initialization, whatever the floor. + let l = Ledger::new(true, 64, 4095); + assert!( + !l.reclaim_due(32, 5 * s, s), + "re-initialization with nothing grown" + ); + } + + /// Heavy scenes of ~1420 blocks each, new clients back to + /// back. The engine re-initializes at a client boundary once past half the maximum, so the + /// heap never reaches it; the backing stays (no refill on the next scene). + #[test] + fn back_to_back_heavy_scenes() { + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS)); + let mut counter = 0; + for _ in 0..10 { + if l.drain_reinit_due(DRAIN_IDLE_NS) { + l.reinit(); + counter = l.next_request; + } + let start = l.posted; + while l.posted < start + 1420 { + while l.refill_due() { + l.back_chunk(); + } + match l.accept(&request(1, counter, 0)) { + Answer::Grant { count, .. } => { + l.post(count); + } + other => panic!("{other:?} at {}", l.posted), + } + counter += 1; + } + assert!(l.posted < l.max); + } + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + assert_eq!(l.backed, 24 * CHUNK_BLOCKS); + l.posted = 2047; + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS)); + l.posted = 2048; + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + // Not between two renders of one client. + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS - 1)); + assert!(!l.drain_reinit_due(0)); + // An initial posting above half the maximum: nothing grew, nothing to drain. + let l = Ledger::new(true, 3000, 4095); + assert!( + !l.drain_reinit_due(DRAIN_IDLE_NS), + "drain with nothing grown" + ); + } + + /// A client-boundary re-initialization keeps the backing + /// of a heavy scene, the next client posts little; once idle, the backing above what the + /// posted blocks need is due to be freed although the posted count is below the floor. + #[test] + fn kept_backing_freed_at_idle() { + let s = 1_000_000_000; + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 2800; + assert!(!l.free_due(5 * s, s)); + l.reinit(); + l.posted = 264; + assert!(!l.reclaim_due(1024, 5 * s, s)); + assert!(!l.free_due(s - 1, s)); + assert!(l.free_due(s, s)); + // After the re-initialization and the free, nothing more is due. + l.reinit(); + l.unback_to(l.keep_chunks()); + assert!(!l.free_due(5 * s, s)); + // A heap backed for what it posts plus the reserve is not freed. + l.posted = 1000; + l.unback_to(Ledger::chunks_for(1000)); + assert!(!l.free_due(5 * s, s)); + } + + /// A heavy scene, a reclaim that frees the backing, refills during the next heavy scene, + /// an idle gap with a small job in it, and the next render must reclaim again. + #[test] + fn reclaim_after_free_then_refill() { + let s = 1_000_000_000; + let (floor, min_idle) = (1024, s); + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + let mut clock = IdleClock::default(); + // A heavy scene grows the PB to 1484 blocks, refilling as it goes. + let heavy = |l: &mut Ledger, clock: &mut IdleClock, t: u64| { + clock.note(false, t); + let mut counter = l.next_request; + while l.posted < 1484 { + while l.refill_due() { + l.back_chunk(); + } + match l.accept(&request(1, counter, 0)) { + Answer::Grant { count, .. } => { + l.post(count); + } + other => panic!("{other:?} at {}", l.posted), + } + counter += 1; + } + // The pass that retires the last flight notes the idle; no later pass may run. + clock.note(true, t + s); + }; + heavy(&mut l, &mut clock, 100 * s); + assert_eq!((l.posted, l.backed), (1484, 13 * CHUNK_BLOCKS)); + // 5 s later a small job arrives: the reclaim is due; it frees the backing above the + // kept chunks; the re-initialization happens at the next render. + let t = 106 * s; + assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); + l.unback_to(l.keep_chunks()); + clock.note(false, t); + assert_eq!(l.backed, 2 * CHUNK_BLOCKS); + clock.note(true, t + s / 2); + // The next render re-initializes, then the second heavy scene refills to 1484 again. + l.reinit(); + assert_eq!((l.posted, l.backed), (64, 256)); + heavy(&mut l, &mut clock, 110 * s); + assert_eq!((l.posted, l.backed), (1484, 13 * CHUNK_BLOCKS)); + // The same gap again: the reclaim must be due at the small job's arrival. + let t = 116 * s; + assert_eq!(clock.idle_ns(t), 5 * s); + assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); + } + + /// The delay names exactly the idle time at which the reclaim check first acts: nothing is + /// due one nanosecond before it, something is due at it, and nothing ever becomes due + /// when it is `None`. + #[test] + fn reclaim_delay_matches_the_checks() { + let s = 1_000_000_000; + let due = |l: &Ledger, level: u32, floor: u32, idle: u64, min_idle: u64| { + level != 0 + && (l.reclaim_due(floor, idle, min_idle) + || (level >= 2 && l.free_due(idle, min_idle)) + || l.drain_reinit_due(idle)) + }; + for level in 0..=2 { + for floor in [0, 64, 1024, 4095] { + for min_idle in [DRAIN_IDLE_NS / 2, 100_000_000, s] { + for posted in [64, 200, 1024, 1025, 2047, 2048, 2900, 4095] { + for chunks in [2, 3, 9, 23, MAX_CHUNKS] { + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(chunks); + l.posted = posted; + match l.reclaim_delay(level, floor, 0, min_idle) { + None => { + for idle in [0, DRAIN_IDLE_NS, min_idle, 100 * s] { + assert!(!due(&l, level, floor, idle, min_idle)); + } + } + Some(d) => { + assert!(due(&l, level, floor, d, min_idle)); + assert!(d == 0 || !due(&l, level, floor, d - 1, min_idle)); + // Counted from an idle already begun. + assert_eq!( + l.reclaim_delay(level, floor, d / 2, min_idle), + Some(d - d / 2) + ); + assert_eq!( + l.reclaim_delay(level, floor, d + s, min_idle), + Some(0) + ); + } + } + } + } + } + } + } + // The case of the J516C run: a client-boundary re-initialization left 23 chunks for + // 264 posted blocks; the free is due after tuning::PB_IDLE_MS of idle. + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 264; + assert_eq!(l.reclaim_delay(2, 1024, 0, s), Some(s)); + assert_eq!(l.reclaim_delay(1, 1024, 0, s), None); + assert_eq!(l.reclaim_delay(0, 1024, 0, s), None); + // A desktop at the initial posting with the kept chunks: nothing to wait for. + l.unback_to(l.keep_chunks()); + l.posted = 64; + assert_eq!(l.reclaim_delay(2, 1024, 0, s), None); + // Nothing grown, with an initial posting above half the maximum or above the floor: + // nothing armed. + let mut l = Ledger::new(true, 3000, 4095); + l.unback_to(Ledger::chunks_for(3000)); + assert_eq!( + l.reclaim_delay(2, 1024, 0, s), + None, + "armed with nothing grown" + ); + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + assert_eq!( + l.reclaim_delay(2, 32, 0, s), + None, + "armed with nothing grown" + ); + } + + /// A client-boundary re-initialization with 23 chunks backed and no render after it: the + /// kept backing is due to be freed after the long idle, measured from the start of the idle, + /// and once freed nothing more is due. + #[test] + fn pending_reinit_frees_kept_backing() { + let s = 1_000_000_000; + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 2800; + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + // The poll after the 50 ms re-initialization waits for the rest of the long idle. + assert_eq!( + l.pending_free_delay(2, DRAIN_IDLE_NS, s), + Some(s - DRAIN_IDLE_NS) + ); + assert!(!l.pending_free_due(s - 1, s)); + assert!(l.pending_free_due(s, s)); + assert_eq!(l.pending_free_delay(2, s + 1, s), Some(0)); + assert_eq!(l.pending_free_delay(1, 0, s), None); + assert_eq!(l.pending_free_delay(0, 0, s), None); + l.unback_to(l.keep_chunks()); + assert!(!l.pending_free_due(100 * s, s)); + assert_eq!(l.pending_free_delay(2, 0, s), None); + } + + #[test] + fn grants_only_backed_blocks() { + let mut l = Ledger::new(true, 64, 4095); + // Nothing backed above the initial posting: a request waits and keeps its counter. + l.backed = 64; + let q = request(3, 0, 1); + assert_eq!(l.accept(&q), Answer::Wait); + assert_eq!(l.next_request, 0); + // A grant is clipped to the backed blocks; the next request waits again. + l.backed = 70; + assert_eq!( + l.accept(&q), + Answer::Grant { + first: 64, + count: 6 + } + ); + l.post(6); + assert_eq!(l.next_request, 1); + assert_eq!(l.accept(&request(3, 1, 1)), Answer::Wait); + // At the maximum the refusal does not wait for backing. + l.posted = l.max; + l.backed = l.max; + assert_eq!(l.accept(&request(3, 1, 1)), Answer::Refuse); + // A re-initialization keeps the backing. + l.reinit(); + assert_eq!((l.posted, l.backed, l.next_request), (64, 4095, 0)); } #[test] diff --git a/drivers/gpu/drm/asahi/g15/recovery.rs b/drivers/gpu/drm/asahi/g15/recovery.rs index b81734b1d5f2d..e03b0d6661f93 100644 --- a/drivers/gpu/drm/asahi/g15/recovery.rs +++ b/drivers/gpu/drm/asahi/g15/recovery.rs @@ -224,7 +224,7 @@ impl Firmware { for flight in self.engine.flights.iter_mut().skip(1) { flight.doomed = true; } - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); let kind = if offset == logic::FRAGMENT_KILL { "fragment" } else { @@ -281,7 +281,7 @@ impl Firmware { if decision.compute_hang { self.report("compute hang"); } - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); if decision.class0_hang { dev_warn!( self.dev(), @@ -323,7 +323,7 @@ impl Firmware { /// anything is published; if it cannot be, the GPU stops. Anything else fail-stops. fn fail_or_recover(&mut self, recover: bool, fault: bool, done: &mut Completions) -> Result { let flight = !self.engine.flights.is_empty(); - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); self.report(if flight { "prefix FAILED" } else { diff --git a/drivers/gpu/drm/asahi/g15/recovery/logic.rs b/drivers/gpu/drm/asahi/g15/recovery/logic.rs index 6d36d1d6550d5..767fb7c952937 100644 --- a/drivers/gpu/drm/asahi/g15/recovery/logic.rs +++ b/drivers/gpu/drm/asahi/g15/recovery/logic.rs @@ -37,9 +37,18 @@ pub(crate) const DRAIN_POLLS: u32 = 1000; pub(crate) const FURTHER_HALTS: u32 = 4; /// Stamp cell `obs` reached the own target `own`, bounded by the newest published target -/// `tail`. Stamps are seeded so that a written stamp is never 0 and has a zero low byte. +/// `tail`. Written stamps have a zero low byte, including at counter wrap. pub(crate) const fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool { - obs != 0 && obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) + obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) +} + +/// The position of unfinished work; zero is a valid position after stamp wrap. +pub(crate) const fn pending_stamp(obs: u32, own: u32, tail: u32) -> Option { + if stamp_reached(obs, own, tail) { + None + } else { + Some(obs) + } } /// Cursor `c` of a ring of `m` entries passed the end `h` of entries starting at `b`, within @@ -73,9 +82,9 @@ pub(crate) struct Receipts { pub(crate) fs_start: u32, /// Host channel producers (TA, FS, compute) after the prefix's records. pub(crate) rec: [u32; 3], - /// Compute member j's FW stamp target, and the FS target it waits on directly (0: none). + /// Compute member j's FW stamp target, and the FS target it waits on directly. pub(crate) cstamp: [u32; 16], - pub(crate) cwait: [u32; 16], + pub(crate) cwait: [Option; 16], } /// The newest published receipts. @@ -136,8 +145,10 @@ pub(crate) fn compute_hang(f: &Receipts, t: &Tail, o: &Observed) -> bool { if j == computes { return false; } - if f.cwait[j] != 0 && !stamp_reached(o.fs_stamp, f.cwait[j], t.fs_target) { - return false; // still waiting on a render of the prefix + if let Some(target) = f.cwait[j] { + if !stamp_reached(o.fs_stamp, target, t.fs_target) { + return false; // still waiting on a render of the prefix + } } if o.consumer[COMPUTE] != f.rec[COMPUTE] { return false; // its record not consumed yet @@ -332,10 +343,20 @@ mod tests { assert!(!stamp_reached(stamp(4), stamp(1), stamp(3))); // Across the u32 wrap. let own = 0xffffff00u32; + assert!(stamp_reached(0, own, 0x00000200)); + assert!(stamp_reached(0, 0, 0)); + assert!(!stamp_reached(0xffffff00, 0, 0x100)); assert!(stamp_reached(0x00000100, own, 0x00000200)); assert!(!stamp_reached(0xfffffe00, own, 0x00000200)); } + #[test] + fn progress_at_zero_wrap() { + assert_eq!(pending_stamp(0, 0x100, 0x200), Some(0)); + assert_eq!(pending_stamp(0, 0, 0x100), None); + assert_eq!(pending_stamp(0xffffff00, 0, 0x100), Some(0xffffff00)); + } + #[test] fn rings() { // Entries 10..14 of a 0x500 ring, tail at 20. @@ -401,13 +422,19 @@ mod tests { assert!(!compute_hang(&f, &t, &o)); o.compute_read = 15; // The running compute waits on an FS of the prefix that has not completed. - f.cwait[1] = 0x3d000300; + f.cwait[1] = Some(0x3d000300); o.fs_stamp = 0x3d000200; let mut t2 = t; t2.fs_target = 0x3d000400; assert!(!compute_hang(&f, &t2, &o)); o.fs_stamp = 0x3d000300; assert!(compute_hang(&f, &t2, &o)); + f.cwait[1] = Some(0); + t2.fs_target = 0x100; + o.fs_stamp = 0xffffff00; + assert!(!compute_hang(&f, &t2, &o)); + o.fs_stamp = 0; + assert!(compute_hang(&f, &t2, &o)); // No compute in the prefix. f.computes = 0; assert!(!compute_hang(&f, &t, &o)); diff --git a/drivers/gpu/drm/asahi/g15/retain.rs b/drivers/gpu/drm/asahi/g15/retain.rs new file mode 100644 index 0000000000000..30fd79d510869 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/retain.rs @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The once-per-boot module pin taken before the firmware is first touched. +//! +//! A started G15 firmware session cannot be stopped, so this module must stay loaded until +//! reboot from the first firmware-visible write on. `__module_get` alone does not ensure that: +//! a deferred or asynchronous probe can run while an ordinary rmmod is already past its +//! reference check, and the module is then unloaded under a running firmware. +//! `try_module_get` fails once removal has begun, so the pin is taken with it, before the +//! session is marked retained and before anything firmware-visible happens. + +use core::sync::atomic::{AtomicBool, Ordering}; + +/// The module is being removed: nothing was pinned or marked, and nothing may be started. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Removing; + +/// Pins the module once for the session `retained` tracks. `get` is +/// `try_module_get(THIS_MODULE)`, `put` is `module_put(THIS_MODULE)`. +/// +/// Already retained: nothing more is taken. Otherwise the reference comes first; `retained` is +/// set only once it is held. A caller that loses the race to set it drops only its own +/// reference, so exactly one reference stays and it is never put. +pub(crate) fn retain( + retained: &AtomicBool, + get: impl FnOnce() -> bool, + put: impl FnOnce(), +) -> Result<(), Removing> { + if retained.load(Ordering::Acquire) { + return Ok(()); + } + if !get() { + return Err(Removing); + } + if retained.swap(true, Ordering::AcqRel) { + put(); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use core::sync::atomic::AtomicI32; + use std::sync::Arc; + + #[test] + fn removal_fails_closed_with_nothing_pinned_or_marked() { + let retained = AtomicBool::new(false); + let refs = AtomicI32::new(1); // base reference only: rmmod has released it + let put = || panic!("nothing to put"); + assert_eq!(retain(&retained, || false, put), Err(Removing)); + assert!(!retained.load(Ordering::Acquire), "a refused pin must not mark the session"); + assert_eq!(refs.load(Ordering::Acquire), 1); + // A later live start can still pin: the refusal left no state behind. + let get = || { + refs.fetch_add(1, Ordering::AcqRel); + true + }; + assert_eq!(retain(&retained, get, || panic!("winner keeps its reference")), Ok(())); + assert!(retained.load(Ordering::Acquire)); + assert_eq!(refs.load(Ordering::Acquire), 2); + } + + #[test] + fn first_pin_is_kept_and_later_calls_take_nothing() { + let retained = AtomicBool::new(false); + let refs = AtomicI32::new(1); + let get = || { + refs.fetch_add(1, Ordering::AcqRel); + true + }; + assert_eq!(retain(&retained, get, || panic!("winner keeps its reference")), Ok(())); + // The post-start call in Gpu::retain and any later one: no new reference, no put. + for _ in 0..3 { + assert_eq!( + retain(&retained, || panic!("already retained"), || panic!("nothing to put")), + Ok(()) + ); + } + assert_eq!(refs.load(Ordering::Acquire), 2); + } + + #[test] + fn a_lost_race_drops_only_its_own_reference() { + let retained = AtomicBool::new(false); + let refs = AtomicI32::new(2); // base + the reference the winner already holds + let puts = AtomicI32::new(0); + // The winner marks the session between this caller's check and its swap. + let get = || { + refs.fetch_add(1, Ordering::AcqRel); + retained.store(true, Ordering::Release); + true + }; + let put = || { + refs.fetch_sub(1, Ordering::AcqRel); + puts.fetch_add(1, Ordering::AcqRel); + }; + assert_eq!(retain(&retained, get, put), Ok(())); + assert_eq!(puts.load(Ordering::Acquire), 1); + assert_eq!(refs.load(Ordering::Acquire), 2); + } + + #[test] + fn concurrent_pins_leave_exactly_one_reference() { + for _ in 0..500 { + let retained = Arc::new(AtomicBool::new(false)); + let refs = Arc::new(AtomicI32::new(1)); + let start = Arc::new(std::sync::Barrier::new(8)); + let threads: Vec<_> = (0..8) + .map(|_| { + let (retained, refs, start) = (retained.clone(), refs.clone(), start.clone()); + std::thread::spawn(move || { + let (a, b) = (refs.clone(), refs.clone()); + start.wait(); + retain( + &retained, + move || { + a.fetch_add(1, Ordering::AcqRel); + true + }, + move || { + b.fetch_sub(1, Ordering::AcqRel); + }, + ) + }) + }) + .collect(); + for t in threads { + assert_eq!(t.join().unwrap(), Ok(())); + } + assert!(retained.load(Ordering::Acquire)); + assert_eq!(refs.load(Ordering::Acquire), 2, "exactly one kept reference"); + } + } +} diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index d6971489ee80f..53e99f5a772b2 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -25,8 +25,10 @@ //! [`clean_lower`] for the context-0 lower tree. (The cleaning in `pgtable.rs` makes these //! redundant but harmless.) //! -//! User VMs are bound to exclusive TTBAT slots for their lifetime through `mmu::Uat::bind` -//! (`vm.rs`); the shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | +//! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) +//! and return them after successful retirement. Failed published jobs retain their slots until +//! their last reference drops (until reboot for jobs retained after a fail-stop). +//! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. use core::ops::Range; diff --git a/drivers/gpu/drm/asahi/g15/vm.rs b/drivers/gpu/drm/asahi/g15/vm.rs index ccb4f77921e0c..24a912d5968bb 100644 --- a/drivers/gpu/drm/asahi/g15/vm.rs +++ b/drivers/gpu/drm/asahi/g15/vm.rs @@ -10,18 +10,15 @@ //! ([`object::Objects::clients`]) as driver mappings at their fixed GVAs, and reserve //! [`super::hw::client::CLIENT_APERTURE`], the pass-slot range and the PB page list //! (`vm.install_driver_mappings`): user binds there fail (EINVAL). The aperture holds the -//! whole PB range, whose backing is mapped later, before the VM's first prefix is published -//! (`Firmware::map_pb`, counted in [`VmState::pb_chunks`]); -//! - bind it to an exclusive TTBAT slot ([`mmu::Uat::bind`]) for as long as its [`VmState`] -//! lives; at most [`super::hw::sizes::MAX_VMS`] such VMs (ENOSPC beyond, checked before -//! binding so the slot allocator never blocks). +//! whole PB range, whose backing chunks are mapped later, before the VM's prefixes are +//! published (`Firmware::map_pb`) and, for the VM of the flights, when a chunk is backed +//! ([`VmState::map_heap`], [`VmState::map_chunk`]); +//! - leave it unbound until a runnable job needs a TTBAT slot. Each published job holds +//! its binding through completion; failed published jobs retain theirs until their last +//! reference drops (until reboot for jobs retained after a fail-stop). //! -//! The [`VmState`] of a VM is shared by the [`Registry`] (while the client's VM handle exists), -//! the VM's queues and its jobs. It cannot be owned by the [`mmu::Vm`] itself: the slot binding -//! holds a `Vm` reference. The registry drops its reference once the shared lifetime reports the -//! client's handle closed ([`mmu::Vm::is_closed`]), at the next VM creation; the slot returns when -//! the last queue or job of the VM is gone (failed jobs are retained until reboot, and their -//! VM's slot with them). +//! The registry keeps a VM's state while its client handle is open. Queues and jobs retain +//! closed VMs until their last reference drops. //! //! User page tables are cleaned to the point of coherency by the shared `pgtable.rs` (enabled by //! the adopted UAT); user objects are prepared at creation (`gpu::Gpu::prepare_new_objects`). @@ -29,7 +26,7 @@ //! alike. use core::ops::Range; -use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; +use core::sync::atomic::Ordering; use kernel::{ drm::gem::BaseObject, new_mutex, @@ -38,7 +35,7 @@ use kernel::{ }; use super::{hw, object, Gpu}; -use crate::{debug::*, g17::status::VmStatus, gem, mmu, module_parameters}; +use crate::{debug::*, g17::status::VmStatus, gem, mmu}; const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; @@ -104,64 +101,107 @@ impl ClientAlias { } } +/// A PB heap chunk mapped in a VM at its fixed GVA ([`super::pools::chunk_gva`]). +struct HeapMapping { + gem: ARef, + _mapping: mmu::KernelMapping, +} + +/// The PB heap chunks mapped in a VM, in heap order (chunk `i` at `chunk_gva(i)`). Changed +/// under the device lock (refill, publication). +#[pin_data] +struct HeapMappings { + #[pin] + list: Mutex>, +} + /// G15 state of one user VM, shared by the registry, its queues and its jobs. pub(crate) struct VmState { /// The `mmu::Vm` identifier (`Gpu::ids.vm`). pub(crate) id: u64, - /// The VM's TTBAT slot (ASID), for the work images' context. - pub(crate) slot: u32, /// The sticky status (shared with `file.rs`). pub(crate) status: Arc, /// The VM: span checks, stream reads and job guards. pub(crate) vm: mmu::Vm, - /// The exclusive slot binding, released before the live count drops. - bind: Option, + /// The adopted UAT, used to bind published jobs without waiting for a slot. + uat: Arc, registry: Arc, - /// PB backing chunks mapped into the VM, from the first (`Firmware::map_pb`); changed only - /// under the device lock. - pub(crate) pb_chunks: AtomicUsize, + /// The PB heap chunks mapped in this VM. + heap: Arc, } impl VmState { + pub(crate) fn try_bind(&self) -> Result { + self.uat.try_bind(&self.vm) + } + /// Drops the registry's references to closed VMs (the queues call this when destroyed). pub(crate) fn prune_registry(&self) { self.registry.prune(); } -} -impl Drop for VmState { - fn drop(&mut self) { - mod_pr_debug!("G15: VM {} releases context {}\n", self.id, self.slot); - drop(self.bind.take()); - self.registry.live.fetch_sub(1, Ordering::AcqRel); + /// Maps PB heap chunk `i`, the next one this VM lacks, at its GVA. The alias is held only + /// once it is in the list. + fn map_one(&self, heap: &mut KVec, i: usize, gem: ARef) -> Result { + if i != heap.len() { + return Err(EINVAL); + } + heap.reserve(1, GFP_KERNEL)?; + let mapping = self.vm.map_at( + super::pools::chunk_gva(i), + super::pools::CHUNK_BYTES, + &gem, + mmu::Prot::from_pte(hw::pte::LEAF_CACHED), + false, + )?; + heap.push_within_capacity(HeapMapping { + gem, + _mapping: mapping, + }) + .map_err(|_| ENOMEM) } -} -/// Live-VM accounting and the VMs whose client handle is open. -#[pin_data] -pub(crate) struct Registry { - /// VMs whose client handle may still be open, oldest first. - #[pin] - vms: Mutex>>, - /// Live [`VmState`]s (each holds a TTBAT slot) plus reservations being created. - live: AtomicU32, -} + /// Maps the backed chunks (`gems`, in heap order) this VM does not hold yet. A chunk whose + /// object changed since (freed and backed again) is unmapped first, with everything above + /// it, and so are chunks past the backing. + pub(crate) fn map_heap(&self, gems: impl Iterator>) -> Result { + let mut heap = self.heap.list.lock(); + let mut n = 0; + for (i, gem) in gems.enumerate() { + n = i + 1; + if let Some(have) = heap.get(i) { + if core::ptr::eq(&*have.gem, &*gem) { + continue; + } + heap.truncate(i); + } + self.map_one(&mut heap, i, gem)?; + } + heap.truncate(n); + Ok(()) + } -/// A reserved place among the live VMs, released unless a [`VmState`] takes it over. -struct Reservation<'a>(Option<&'a Registry>); + /// Maps the newly backed chunk `i` into this VM (the VM of a flight, at a refill). + pub(crate) fn map_chunk(&self, i: usize, gem: ARef) -> Result { + let mut heap = self.heap.list.lock(); + if i < heap.len() { + // Already mapped (a flight of the same VM came first). + return Ok(()); + } + self.map_one(&mut heap, i, gem) + } -impl Reservation<'_> { - fn into_state(mut self) { - self.0 = None; + /// Unmaps the heap chunks from index `keep` on. + pub(crate) fn unmap_heap_above(&self, keep: usize) { + self.heap.list.lock().truncate(keep); } } -impl Drop for Reservation<'_> { - fn drop(&mut self) { - if let Some(registry) = self.0 { - registry.live.fetch_sub(1, Ordering::AcqRel); - } - } +/// The VMs whose client handle is open. +#[pin_data] +pub(crate) struct Registry { + #[pin] + vms: Mutex>>, } impl Registry { @@ -169,32 +209,14 @@ impl Registry { Arc::pin_init( pin_init!(Self { vms <- new_mutex!(KVec::new(), "g15::Registry::vms"), - live: AtomicU32::new(0), }), GFP_KERNEL, ) } - /// Reserves a place for a new VM after dropping the registry's references to closed VMs. - fn reserve(&self) -> Result> { - self.prune(); - // The shared slot allocator hands out a single slot with robust isolation. - let limit = if module_parameters::robust_isolation.value() != 0 { - 1 - } else { - hw::sizes::MAX_VMS - }; - self.live - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |live| { - (live < limit).then_some(live + 1) - }) - .map_err(|_| ENOSPC)?; - Ok(Reservation(Some(self))) - } - /// Drops the registry's references to VMs whose client handle is closed. Called on VM close /// ([`closed`]), on queue destruction and before a new VM is created: a closed VM's page - /// tables, bound objects and TTBAT slot go as soon as its last queue or job is gone. + /// tables and bound objects go as soon as its last queue or job is gone. fn prune(&self) { let mut closed = KVec::new(); { @@ -251,6 +273,17 @@ impl Registry { .ok_or(ENOENT) } + /// The VMs whose client handle is open (and the closed ones not pruned yet), for a walk + /// that takes VM locks: not under the registry lock. + pub(crate) fn open(&self) -> Result>> { + let vms = self.vms.lock(); + let mut out = KVec::with_capacity(vms.len(), GFP_KERNEL)?; + for state in vms.iter() { + out.push(state.clone(), GFP_KERNEL)?; + } + Ok(out) + } + /// Latches `error` on every VM whose client handle is open. The fail-stop path calls it under /// the device lock. pub(crate) fn fail_all(&self, error: Error) { @@ -286,31 +319,32 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { ClientAlias::collect(&firmware.objects)?, ) }; - let reservation = registry.reserve()?; + registry.prune(); let id = gpu.ids.vm.next(); let vm = gpu.uat.new_vm(id, kernel_range)?.with_tracked_status()?; let status = vm.status().ok_or(EINVAL)?.clone(); let (mappings, reserved) = Registry::driver_mappings(&vm, aliases)?; vm.install_driver_mappings(mappings, reserved)?; - // The context names the root only now: everything above is already visible to the walker. - let bind = gpu.uat.bind(&vm)?; - let slot = bind.slot(); - // From here on the state (even one that fails to allocate) gives the place back. - reservation.into_state(); + // No PB chunk yet: they are mapped before the VM's first prefix (`Firmware::map_pb`). + let heap = Arc::pin_init( + pin_init!(HeapMappings { + list <- new_mutex!(KVec::new(), "g15::HeapMappings::list"), + }), + GFP_KERNEL, + )?; let state = Arc::new( VmState { id, - slot, status, vm: vm.clone(), - bind: Some(bind), + uat: gpu.uat.clone(), registry: registry.clone(), - pb_chunks: AtomicUsize::new(0), + heap, }, GFP_KERNEL, )?; registry.vms.lock().push(state, GFP_KERNEL)?; - mod_pr_debug!("G15: VM {} context {}\n", id, slot); + mod_pr_debug!("G15: VM {} created\n", id); Ok(vm) } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index e9f1008fc5a72..a91f267f9e1e0 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -35,6 +35,7 @@ use kernel::{ }, macros::versions, new_mutex, + new_spinlock, prelude::*, soc::apple::rtkit, sync::{ @@ -44,6 +45,7 @@ use kernel::{ }, Arc, Mutex, + SpinLock, UniqueArc, // }, time::{ @@ -84,6 +86,7 @@ use crate::{ queue, regs, stats, + stats_raw, workqueue, // }; @@ -258,6 +261,9 @@ pub(crate) struct GpuManager { pipes: PipeChannels::ver, event_manager: Arc, stats: Arc, + /// Last raw firmware statistics of this device, exported read-only + /// through `stats_raw`. + raw_stats: Arc>, buffer_mgr: buffer::BufferManager::ver, ids: SequenceIDs, #[pin] @@ -274,6 +280,11 @@ pub(crate) trait Gpu: Send + Sync { /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; + /// The raw firmware statistics snapshot of this GPU, if its firmware reports any. + fn raw_stats_snapshot(&self) -> Option>> { + None + } + /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -827,6 +838,12 @@ impl GpuManager::ver { let buffer_mgr_clone = buffer_mgr.clone(); let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; let stats_snap_clone = stats_snap.clone(); + // Per-device raw statistics snapshot, shared with the debugfs export. + let raw_stats_snap = Arc::pin_init( + new_spinlock!(stats_raw::RawStatsSnapshot::default(), "asahi_stats_raw"), + GFP_KERNEL, + )?; + let raw_stats_snap_clone = raw_stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( try_init!(RxChannels::ver { @@ -842,6 +859,7 @@ impl GpuManager::ver { dev.into(), alloc_ref, stats_snap_clone, + raw_stats_snap_clone, u64::from(cfg.base_clock_hz), )?, }), @@ -869,6 +887,7 @@ impl GpuManager::ver { crashed: AtomicBool::new(false), event_manager, stats: stats_snap.clone(), + raw_stats: raw_stats_snap.clone(), alloc <- new_mutex!(alloc, "alloc"), fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), @@ -1324,6 +1343,10 @@ impl Gpu for GpuManager::ver { Some(self.stats.clone()) } + fn raw_stats_snapshot(&self) -> Option>> { + Some(self.raw_stats.clone()) + } + fn as_any(&self) -> &dyn Any { self } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 6baaa97977f5a..f146b2762d812 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -366,6 +366,7 @@ struct VmInner { va_range: Range, page_table: UatPageTable, mapping_epoch: Option>, + bind_sequence: Arc, epoch: Option, mm: mm::Allocator<(), KernelMappingInner>, uat_inner: Arc, @@ -411,6 +412,21 @@ fn lock_vm( Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? })) } +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +pub(crate) struct VmReadGuard(VmExecGuard); + +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +impl VmReadGuard { + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool { + Vm::covers_range_locked(&mut self.0, address, size, read, write) + } + + pub(crate) fn read_user(&mut self, va: u64, buf: &mut [u8]) -> Result { + Vm::read_user_locked(&mut self.0, va, buf) + } +} + /// Slot binding-related inner data for a Vm instance. struct VmBinding { active_users: usize, @@ -424,6 +440,11 @@ struct VmBinding { /// Data associated with a VM <=> BO pairing #[pin_data] struct VmBoData { + /// Highest sequence assigned to a GPUVA linked to this VM/BO pair. + /// + /// Map steps update this monotonically under the VM reservation before + /// linking the GPUVA. Remap pieces keep their original sequence. + newest_bind: AtomicU64, #[pin] segments: Mutex>, } @@ -479,6 +500,9 @@ struct StepContext<'ctx> { /// Preallocated resources that can be used when executing the request. resources: &'ctx mut VmOpResources, + + /// Sequence assigned by the successful map for its related metadata. + bind_sequence: u64, } impl<'ctx> StepContext<'ctx> { @@ -491,6 +515,7 @@ impl<'ctx> StepContext<'ctx> { dev, prot, resources, + bind_sequence: 0, }) } @@ -521,6 +546,7 @@ impl<'ctx> StepContext<'ctx> { #[derive(Default, Clone)] struct VmGpuVa { prot: Option, + bind_sequence: u64, } impl gpuvm::DriverGpuVm for VmInner { @@ -539,6 +565,11 @@ impl gpuvm::DriverGpuVm for VmInner { op: OpMap<'op, Self>, ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { + let bind_sequence = self.bind_sequence.fetch_update( + Ordering::AcqRel, + Ordering::Acquire, + |value| value.checked_add(1), + ).map_err(|_| EOVERFLOW)? + 1; let _mutation = self.mapping_mutation(); let (map_addr, map_length) = (op.addr(), op.length() as usize); let mut iova = op.addr(); @@ -611,12 +642,18 @@ impl gpuvm::DriverGpuVm for VmInner { return Err(error); } }; + op.vm_bo() + .data() + .newest_bind + .fetch_max(bind_sequence, Ordering::Relaxed); let op = op.insert( gpuva, VmGpuVa { prot: Some(ctx.prot), + bind_sequence, }, ); + ctx.bind_sequence = bind_sequence; self.tlbi_contexts_naming_root(map_addr, map_length); Ok(op) @@ -642,6 +679,7 @@ impl gpuvm::DriverGpuVm for VmInner { ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { let orig_prot = op.va_to_unmap().data().prot.ok_or(EINVAL)?; + let orig_bind_sequence = op.va_to_unmap().data().bind_sequence; let orig_invalidated = op.va_to_unmap().flags().contains(GpuVaFlag::INVALIDATED); // Only unmap the hole between prev/next, if they exist @@ -707,9 +745,11 @@ impl gpuvm::DriverGpuVm for VmInner { [prev_gpuva, next_gpuva], VmGpuVa { prot: Some(orig_prot), + bind_sequence: orig_bind_sequence, }, VmGpuVa { prot: Some(orig_prot), + bind_sequence: orig_bind_sequence, }, ); @@ -994,6 +1034,7 @@ pub(crate) struct Vm { dev: ARef, id: u64, mapping_epoch: Option>, + bind_sequence: Arc, lifetime: Option>, residency: Option>, context_bindings: Option>, @@ -2112,6 +2153,7 @@ impl Vm { } else { None }; + let bind_sequence = Arc::new(AtomicU64::new(0), GFP_KERNEL)?; let unique_gpuvm = gpuvm::GpuVm::new::( c"Asahi::GpuVm", &dev, @@ -2124,6 +2166,7 @@ impl Vm { va_range: va_range.clone(), page_table, mapping_epoch: mapping_epoch.clone(), + bind_sequence: bind_sequence.clone(), epoch: Some(0), mm, uat_inner, @@ -2140,6 +2183,7 @@ impl Vm { id, gpuvm, mapping_epoch, + bind_sequence, lifetime: None, residency: None, context_bindings: None, @@ -2191,7 +2235,16 @@ impl Vm { /// mappings cannot change during the copy (GPUVM exec lock). #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result { - let mut inner = lock_vm(&self.gpuvm, None, false)?; + self.read_lock()?.read_user(va, buf) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + pub(crate) fn read_lock(&self) -> Result { + Ok(VmReadGuard(lock_vm(&self.gpuvm, None, false)?)) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + fn read_user_locked(inner: &mut VmInner, va: u64, buf: &mut [u8]) -> Result { let mut done = 0usize; while done < buf.len() { let address = va.checked_add(done as u64).ok_or(EFAULT)?; @@ -2304,6 +2357,10 @@ impl Vm { self.mapping_epoch.as_ref()?.snapshot() } + fn current_bind_sequence(&self) -> u64 { + self.bind_sequence.load(Ordering::Acquire) + } + /// Get the translation table base for this Vm fn ttb(&self) -> u64 { self.binding.lock().ttb @@ -2404,7 +2461,8 @@ impl Vm { let bo = self.gpuvm.obtain( gem, pin_init!(VmBoData { - segments <- new_mutex!(None) + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), }), )?; prepare_sgt_segments(dev, gem, &bo, &lease)?; @@ -2492,7 +2550,8 @@ impl Vm { let bo = self.gpuvm.obtain( gem, pin_init!(VmBoData { - segments <- new_mutex!(None) + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), }), )?; prepare_sgt_segments(dev, gem, &bo, &lease)?; @@ -2575,7 +2634,10 @@ impl Vm { let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm - .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; + .obtain(gem, pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }))?; prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; let mut resources = VmOpResources { @@ -2829,6 +2891,66 @@ impl Vm { Ok(()) } + fn drop_mappings_before(&self, gem: &gem::Object, sequence: u64) -> Result { + let _residency = self.enter_retirement_residency(); + // Removing whole mappings only does unmaps, so no preallocated VAs + // are needed for the indexed path. + let mut resources = VmOpResources { + preallocated_gpuvas: [None, None, None], + }; + let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; + + // The BO's newest sequence is checked under the same lock as binds. If + // it is not newer than the close, every mapping belongs to that close + // and the per-object list is safe and substantially cheaper to walk. + // SAFETY: See `lock_vm()`. + let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { + Err(error) if error == ENOMEM => { + return self.drop_mappings_filtered(gem, sequence); + } + result => result?, + }; + let Some(bo) = vm_exec.unique_gpuvm.find_bo(gem) else { + return Ok(()); + }; + if bo.data().newest_bind.load(Ordering::Relaxed) > sequence { + drop(vm_exec); + drop(bo); + return self.drop_mappings_filtered(gem, sequence); + } + + mod_dev_dbg!(self.dev, "MMU: deferred bo_unmap\n"); + let result = vm_exec.unique_gpuvm.bo_unmap(&mut ctx, &bo); + if result.is_ok() { + self.untrack_context_object_before(gem, sequence); + self.untrack_shared_object_before(gem, sequence); + } + drop(vm_exec); + drop(bo); + if result == Err(ENOMEM) { + return self.drop_mappings_filtered(gem, sequence); + } + result + } + + /// A newer binding shares the BO list, so select the closed generation by + /// walking the VM without allocating. This is also the ENOMEM fallback. + fn drop_mappings_filtered(&self, gem: &gem::Object, sequence: u64) -> Result { + let mut inner = self.gpuvm.lock_inner(); + let result = inner.unmap_object_where( + gem, + |va| va.data().bind_sequence <= sequence, + VmInner::unmap_gpuva, + ); + if result.is_ok() { + self.untrack_context_object_before(gem, sequence); + self.untrack_shared_object_before(gem, sequence); + } + drop(inner); + self.bo_deferred_cleanup(); + result + } + /// Preserve the indexed fast path normally; memory pressure must not lose /// accepted object cleanup just because an exec or operation list failed. fn drop_mappings_low_memory(&self, gem: &gem::Object) -> Result { @@ -3219,6 +3341,16 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, true) + } + + /// Binds a VM without waiting for another slot owner to retire. + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn try_bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, false) + } + + fn bind_inner(&self, vm: &Vm, wait: bool) -> Result { self.inner.check_m4()?; let mut binding = vm.binding.lock(); self.inner.check_m4()?; @@ -3234,7 +3366,11 @@ impl Uat { None }); - let slot = self.slots.get(binding.bind_token)?; + let slot = if wait { + self.slots.get(binding.bind_token)? + } else { + self.slots.try_get(binding.bind_token)? + }; if slot.changed() { mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); let idx = (slot.slot() as usize) + UAT_USER_CTX_START; diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 0fa3f304af1d4..acce5677b9f92 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -84,7 +84,10 @@ impl Vm { let sg_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm - .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; + .obtain(gem, pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }))?; prepare_sgt_segments(dev, gem, &vm_bo, &sg_lease)?; if (addr | size | offset) & UAT_PGMSK as u64 != 0 { return Err(EINVAL); @@ -153,13 +156,18 @@ impl Vm { flags, }); let retired = result.is_ok().then(|| { - let retired = self.commit_context_binding(&map.gem, &map.context); + let retired = self.commit_context_binding( + &map.gem, + &map.context, + ctx.bind_sequence, + ); self.track_shared_binding( &map.gem, map.addr, map.size, map.offset, map.single_page, + ctx.bind_sequence, ); retired }); diff --git a/drivers/gpu/drm/asahi/mmu/context.rs b/drivers/gpu/drm/asahi/mmu/context.rs index d459a3a91afce..0b11394aa408c 100644 --- a/drivers/gpu/drm/asahi/mmu/context.rs +++ b/drivers/gpu/drm/asahi/mmu/context.rs @@ -14,6 +14,7 @@ use kernel::drm::gem::BaseObject; struct Binding { gem: ARef, offset: usize, + bind_sequence: u64, } struct Bindings { generation: u64, @@ -115,6 +116,7 @@ impl Vm { &self, gem: &ARef, update: &PreparedBinding, + bind_sequence: u64, ) -> RetiredBindings { let mut retired = RetiredBindings::new(); if update.touched == 0 { @@ -136,6 +138,7 @@ impl Vm { *slot = Some(Binding { gem: gem.clone(), offset, + bind_sequence, }); } } @@ -173,6 +176,10 @@ impl Vm { } pub(crate) fn untrack_context_object(&self, gem: &gem::Object) { + self.untrack_context_object_before(gem, u64::MAX); + } + + pub(super) fn untrack_context_object_before(&self, gem: &gem::Object, sequence: u64) { let Some(context) = self.context_bindings.as_ref() else { return; }; @@ -181,7 +188,8 @@ impl Vm { for slot in &mut bindings.slots { if slot .as_ref() - .is_some_and(|binding| core::ptr::eq(&*binding.gem, gem)) + .is_some_and(|binding| core::ptr::eq(&*binding.gem, gem) + && binding.bind_sequence <= sequence) { *slot = None; touched = true; diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 8cf37b37e2e0a..66d507c2d7694 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -47,6 +47,12 @@ impl DriverMappings { } } +struct ClosedObject { + gem: ARef, + /// Highest binding sequence covered by the close or merged closes. + bind_sequence: u64, +} + struct State { active: usize, commits: usize, @@ -57,7 +63,7 @@ struct State { cleanup_failed: bool, close_ranges: Option<(Range, Range)>, unmaps: KVec, - objects: KVec>, + objects: KVec, } #[pin_data] @@ -578,18 +584,22 @@ impl Vm { return self.drop_mappings_now(gem); }; let mut state = lifetime.state.lock(); - if !state - .objects - .iter() - .any(|object| core::ptr::eq(&**object, gem)) - { - state.objects.push(gem.into(), GFP_KERNEL)?; + let bind_sequence = self.current_bind_sequence(); + if let Some(object) = state.objects.iter_mut().find(|object| { + core::ptr::eq(&*object.gem, gem) + }) { + object.bind_sequence = object.bind_sequence.max(bind_sequence); + } else { + state.objects.push(ClosedObject { + gem: gem.into(), + bind_sequence, + }, GFP_KERNEL)?; } if state.active != 0 || state.draining { // Stop new jobs from selecting this accepted cleanup candidate. // The queued object retains its GEM while the candidate is removed. - // Actual unmap repeats this under exec to order any intervening bind. - self.untrack_context_object(gem); + // Preserve a context view installed by a later bind. + self.untrack_context_object_before(gem, bind_sequence); return Ok(()); } state.draining = true; @@ -674,7 +684,7 @@ impl Vm { } } for object in objects { - if let Err(error) = self.drop_mappings_now(&object) { + if let Err(error) = self.drop_mappings_before(&object.gem, object.bind_sequence) { failure.get_or_insert(error); pr_err!("MMU: deferred object unmap failed\n"); } diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs index a0c05e3d5c303..fec7a6b61ad8f 100644 --- a/drivers/gpu/drm/asahi/mmu/shared.rs +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -14,6 +14,7 @@ struct Binding { end: u64, object: usize, offset: u64, + bind_sequence: u64, } #[pin_data] @@ -43,6 +44,7 @@ impl Vm { size: u64, offset: u64, single_page: bool, + bind_sequence: u64, ) { let Some(shared) = &self.shared_bindings else { return; @@ -63,6 +65,7 @@ impl Vm { end, object: core::ptr::from_ref(gem) as usize, offset, + bind_sequence, }, GFP_KERNEL, ); @@ -86,6 +89,10 @@ impl Vm { } pub(crate) fn untrack_shared_object(&self, gem: &gem::Object) { + self.untrack_shared_object_before(gem, u64::MAX); + } + + pub(super) fn untrack_shared_object_before(&self, gem: &gem::Object, sequence: u64) { let Some(shared) = &self.shared_bindings else { return; }; @@ -94,7 +101,7 @@ impl Vm { } let object = core::ptr::from_ref(gem) as usize; let mut bindings = shared.bindings.lock(); - bindings.retain(|entry| entry.object != object); + bindings.retain(|entry| entry.object != object || entry.bind_sequence > sequence); shared.count.store(bindings.len() as u32, Ordering::Relaxed); } diff --git a/drivers/gpu/drm/asahi/slotalloc.rs b/drivers/gpu/drm/asahi/slotalloc.rs index fde7470fe5779..a504e7491bfda 100644 --- a/drivers/gpu/drm/asahi/slotalloc.rs +++ b/drivers/gpu/drm/asahi/slotalloc.rs @@ -216,6 +216,20 @@ impl SlotAllocator { &self, token: Option, cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, + ) -> Result> { + self.get_inner_impl(token, cb, true) + } + + /// Gets a slot without waiting; returns ENOSPC when every slot is held. + pub(crate) fn try_get(&self, token: Option) -> Result> { + self.get_inner_impl(token, |_a, _b| Ok(()), false) + } + + fn get_inner_impl( + &self, + token: Option, + cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, + wait: bool, ) -> Result> { let mut inner = self.0.inner.lock(); @@ -256,6 +270,9 @@ impl SlotAllocator { } if oldest_time == u64::MAX { + if !wait { + return Err(ENOSPC); + } if first && inner.slot_limit == usize::MAX { pr_warn!( "{}: out of slots, blocking\n", diff --git a/drivers/gpu/drm/asahi/stats_raw.rs b/drivers/gpu/drm/asahi/stats_raw.rs new file mode 100644 index 0000000000000..3d158337f66b7 --- /dev/null +++ b/drivers/gpu/drm/asahi/stats_raw.rs @@ -0,0 +1,501 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Raw firmware statistics snapshot and its read-only debugfs export. +//! +//! The AGX firmware sends statistics messages on the stats channel +//! (`channel::StatsChannel`). They are recorded here exactly as they arrive: +//! every field keeps the value the firmware sent, no unit is assumed, nothing +//! is scaled and no metric is derived from it. `FwBusy` is the firmware's own +//! busy flag and is *not* a GPU utilisation figure; the `Utilization` counters +//! are raw per-subqueue counters whose unit this driver does not assert. +//! +//! This complements the interpreted `stats` snapshot behind `agx_stats`: the +//! snapshot here keeps every message kind the firmware sends, together with +//! per-kind sample counters and capture times, and it never derives a value +//! from the firmware words. +//! +//! Every message kind keeps its own sample counter and the monotonic time at +//! which the sample was captured, so a reader can tell a missing sample +//! (count 0) from a stale one (old capture time), and can compute rates and +//! deltas between two reads. +//! +//! The snapshot is fixed size: recording a sample never allocates. It is +//! guarded by a spin lock because the critical section is short: a sample +//! is stored under the lock and a reader copies the snapshot out, so the lock +//! is never held across formatting. + +use core::fmt::Write; + +use kernel::{ + bindings, + prelude::*, + seq_file::SeqFile, + sync::{ + Arc, + SpinLock, // + }, +}; + +use crate::driver::AsahiDrmDevice; + +/// Format tag of the export. Bumped whenever the field set changes. +pub(crate) const STATS_FORMAT: &str = "asahi_stats_v1"; + +/// The firmware statistics message kinds that are recorded. +/// +/// The discriminants index the per-kind counters of [`RawStatsSnapshot`]; the +/// firmware message tag of each kind is given in its documentation, so every +/// exported line can be mapped back to the message it came from. +#[derive(Copy, Clone, PartialEq, Eq)] +#[repr(usize)] +pub(crate) enum StatsKind { + /// 0x00 `Power`. + Power = 0, + /// 0x02 `PowerOn`. + PowerOn = 1, + /// 0x03 `PowerOff`. + PowerOff = 2, + /// 0x04 `Utilization`. + Utilization = 3, + /// 0x09 `AvgPower`. + AvgPower = 4, + /// 0x0a `Temperature`. + Temperature = 5, + /// 0x0b `PowerState`. + PowerState = 6, + /// 0x0c `FwBusy`. + FwBusy = 7, + /// 0x0d `PState`. + PState = 8, + /// 0x0e `TempSensor`. + TempSensor = 9, +} + +impl StatsKind { + /// Number of recorded kinds. + pub(crate) const COUNT: usize = 10; + + /// All kinds, in export order. + const ALL: [StatsKind; StatsKind::COUNT] = [ + StatsKind::Power, + StatsKind::PowerOn, + StatsKind::PowerOff, + StatsKind::Utilization, + StatsKind::AvgPower, + StatsKind::Temperature, + StatsKind::PowerState, + StatsKind::FwBusy, + StatsKind::PState, + StatsKind::TempSensor, + ]; + + /// Stable name of the kind in the export. + fn name(self) -> &'static str { + match self { + StatsKind::Power => "power", + StatsKind::PowerOn => "power_on", + StatsKind::PowerOff => "power_off", + StatsKind::Utilization => "utilization", + StatsKind::AvgPower => "avg_power", + StatsKind::Temperature => "temperature", + StatsKind::PowerState => "power_state", + StatsKind::FwBusy => "fw_busy", + StatsKind::PState => "pstate", + StatsKind::TempSensor => "temp_sensor", + } + } +} + +/// Last received firmware statistics of one device. +/// +/// All values are the raw firmware words. The driver knows the unit of none +/// of them and derives nothing from them. +#[derive(Copy, Clone, Default)] +pub(crate) struct RawStatsSnapshot { + /// Number of samples received per kind (0 means: never received). + count: [u64; StatsKind::COUNT], + /// Monotonic capture time (ns) of the last sample per kind. + captured_ns: [u64; StatsKind::COUNT], + /// Number of statistics messages received on the channel. + messages: u64, + + /// 0x00 `Power`: raw power word, unit unknown. + power: u64, + /// 0x02 `PowerOn`: raw off-time word, unit unknown. + power_off_time: u64, + /// 0x03 `PowerOff`: raw on-time word, unit unknown. + power_on_time: u64, + /// 0x04 `Utilization`: raw firmware timestamp and four raw counters. + util_timestamp: u64, + /// 0x04 `Utilization`: the four raw counters. + util: [u32; 4], + /// 0x09 `AvgPower`: raw active time word, unit unknown. + avg_power_active_cs: u64, + /// 0x09 `AvgPower`: raw undecoded word. + avg_power_unk2: u32, + /// 0x09 `AvgPower`: raw undecoded word. + avg_power_unk3: u32, + /// 0x09 `AvgPower`: raw undecoded word. + avg_power_unk4: u32, + /// 0x09 `AvgPower`: raw average power word, unit unknown. + avg_power: u32, + /// 0x0a `Temperature`: raw sensor word. + temp_raw_value: u32, + /// 0x0a `Temperature`: raw scale word, travels with the sample. + temp_scale: u32, + /// 0x0a `Temperature`: raw minimum word. + temp_tmin: u32, + /// 0x0a `Temperature`: raw maximum word. + temp_tmax: u32, + /// 0x0b `PowerState`: raw firmware timestamp. + power_state_timestamp: u64, + /// 0x0b `PowerState`: raw firmware timestamp of the last busy period. + power_state_last_busy_ts: u64, + /// 0x0b `PowerState`: raw active word. + power_state_active: u32, + /// 0x0b `PowerState`: raw poweroff word. + power_state_poweroff: u32, + /// 0x0b `PowerState`: raw undecoded word. + power_state_unk1: u32, + /// 0x0b `PowerState`: raw p-state index. + power_state_pstate: u32, + /// 0x0b `PowerState`: raw undecoded word. + power_state_unk2: u32, + /// 0x0b `PowerState`: raw undecoded word. + power_state_unk3: u32, + /// 0x0c `FwBusy`: raw firmware timestamp. + fw_busy_timestamp: u64, + /// 0x0c `FwBusy`: raw firmware busy flag. + fw_busy: u32, + /// 0x0d `PState`: raw lowest p-state bound. + pstate_min: u32, + /// 0x0d `PState`: raw undecoded word. + pstate_unk1: u32, + /// 0x0d `PState`: raw highest p-state bound. + pstate_max: u32, + /// 0x0d `PState`: raw undecoded word. + pstate_unk2: u32, + /// 0x0e `TempSensor`: raw sensor identifier. + sensor_id: u32, + /// 0x0e `TempSensor`: raw sensor word. + sensor_raw_value: u32, + /// 0x0e `TempSensor`: raw scale word, travels with the sample. + sensor_scale: u32, + /// 0x0e `TempSensor`: raw minimum word. + sensor_tmin: u32, + /// 0x0e `TempSensor`: raw maximum word. + sensor_tmax: u32, +} + +impl RawStatsSnapshot { + /// Note one statistics message on the channel. + pub(crate) fn note_message(&mut self) { + self.messages += 1; + } + + /// Record one sample of `kind` captured at `stamp` (monotonic ns). + fn record(&mut self, kind: StatsKind, stamp: u64) { + self.count[kind as usize] += 1; + self.captured_ns[kind as usize] = stamp; + } + + /// Record the 0x00 `Power` sample. + pub(crate) fn set_power(&mut self, power: u64, stamp: u64) { + self.power = power; + self.record(StatsKind::Power, stamp); + } + + /// Record the 0x02 `PowerOn` sample. + pub(crate) fn set_power_on(&mut self, off_time: u64, stamp: u64) { + self.power_off_time = off_time; + self.record(StatsKind::PowerOn, stamp); + } + + /// Record the 0x03 `PowerOff` sample. + pub(crate) fn set_power_off(&mut self, on_time: u64, stamp: u64) { + self.power_on_time = on_time; + self.record(StatsKind::PowerOff, stamp); + } + + /// Record the 0x04 `Utilization` sample. + pub(crate) fn set_utilization(&mut self, timestamp: u64, util: [u32; 4], stamp: u64) { + self.util_timestamp = timestamp; + self.util = util; + self.record(StatsKind::Utilization, stamp); + } + + /// Record the 0x09 `AvgPower` sample. + pub(crate) fn set_avg_power( + &mut self, + active_cs: u64, + unk: [u32; 3], + avg_power: u32, + stamp: u64, + ) { + self.avg_power_active_cs = active_cs; + self.avg_power_unk2 = unk[0]; + self.avg_power_unk3 = unk[1]; + self.avg_power_unk4 = unk[2]; + self.avg_power = avg_power; + self.record(StatsKind::AvgPower, stamp); + } + + /// Record the 0x0a `Temperature` sample. + pub(crate) fn set_temperature( + &mut self, + raw_value: u32, + scale: u32, + tmin: u32, + tmax: u32, + stamp: u64, + ) { + self.temp_raw_value = raw_value; + self.temp_scale = scale; + self.temp_tmin = tmin; + self.temp_tmax = tmax; + self.record(StatsKind::Temperature, stamp); + } + + /// Record the 0x0b `PowerState` sample. + #[allow(clippy::too_many_arguments)] + pub(crate) fn set_power_state( + &mut self, + timestamp: u64, + last_busy_ts: u64, + active: u32, + poweroff: u32, + unk1: u32, + pstate: u32, + unk2: u32, + unk3: u32, + stamp: u64, + ) { + self.power_state_timestamp = timestamp; + self.power_state_last_busy_ts = last_busy_ts; + self.power_state_active = active; + self.power_state_poweroff = poweroff; + self.power_state_unk1 = unk1; + self.power_state_pstate = pstate; + self.power_state_unk2 = unk2; + self.power_state_unk3 = unk3; + self.record(StatsKind::PowerState, stamp); + } + + /// Record the 0x0c `FwBusy` sample. + pub(crate) fn set_fw_busy(&mut self, timestamp: u64, busy: u32, stamp: u64) { + self.fw_busy_timestamp = timestamp; + self.fw_busy = busy; + self.record(StatsKind::FwBusy, stamp); + } + + /// Record the 0x0d `PState` sample. + pub(crate) fn set_pstate( + &mut self, + ps_min: u32, + unk1: u32, + ps_max: u32, + unk2: u32, + stamp: u64, + ) { + self.pstate_min = ps_min; + self.pstate_unk1 = unk1; + self.pstate_max = ps_max; + self.pstate_unk2 = unk2; + self.record(StatsKind::PState, stamp); + } + + /// Record the 0x0e `TempSensor` sample. + pub(crate) fn set_temp_sensor( + &mut self, + sensor_id: u32, + raw_value: u32, + scale: u32, + tmin: u32, + tmax: u32, + stamp: u64, + ) { + self.sensor_id = sensor_id; + self.sensor_raw_value = raw_value; + self.sensor_scale = scale; + self.sensor_tmin = tmin; + self.sensor_tmax = tmax; + self.record(StatsKind::TempSensor, stamp); + } + + /// Write the raw payload fields of one kind. + fn write_payload(&self, kind: StatsKind, out: &mut impl Write) -> core::fmt::Result { + match kind { + StatsKind::Power => write!(out, " power={}", self.power), + StatsKind::PowerOn => write!(out, " off_time={}", self.power_off_time), + StatsKind::PowerOff => write!(out, " on_time={}", self.power_on_time), + StatsKind::Utilization => write!( + out, + " timestamp={} util1={} util2={} util3={} util4={}", + self.util_timestamp, self.util[0], self.util[1], self.util[2], self.util[3] + ), + StatsKind::AvgPower => write!( + out, + " active_cs={} unk2={} unk3={} unk4={} avg_power={}", + self.avg_power_active_cs, + self.avg_power_unk2, + self.avg_power_unk3, + self.avg_power_unk4, + self.avg_power + ), + StatsKind::Temperature => write!( + out, + " raw_value={} scale={} tmin={} tmax={}", + self.temp_raw_value, self.temp_scale, self.temp_tmin, self.temp_tmax + ), + StatsKind::PowerState => write!( + out, + " timestamp={} last_busy_ts={} active={} poweroff={} unk1={} pstate={} unk2={} unk3={}", + self.power_state_timestamp, + self.power_state_last_busy_ts, + self.power_state_active, + self.power_state_poweroff, + self.power_state_unk1, + self.power_state_pstate, + self.power_state_unk2, + self.power_state_unk3 + ), + StatsKind::FwBusy => { + write!(out, " timestamp={} busy={}", self.fw_busy_timestamp, self.fw_busy) + } + StatsKind::PState => write!( + out, + " ps_min={} unk1={} ps_max={} unk2={}", + self.pstate_min, self.pstate_unk1, self.pstate_max, self.pstate_unk2 + ), + StatsKind::TempSensor => write!( + out, + " sensor_id={} raw_value={} scale={} tmin={} tmax={}", + self.sensor_id, self.sensor_raw_value, self.sensor_scale, self.sensor_tmin, self.sensor_tmax + ), + } + } +} + +/// Write the snapshot in the documented text format. +/// +/// One line per record, `key=value` pairs, LF terminated. A kind that was +/// never received is printed with `count=0` and without payload, so a missing +/// sample is never confused with a zero value. `now_ns` and `captured_ns` are +/// monotonic (CLOCK_MONOTONIC) nanoseconds taken by the driver; every other +/// value is a raw firmware word whose unit this driver does not assert. +pub(crate) fn format_snapshot( + snap: &RawStatsSnapshot, + now_ns: u64, + device: &str, + out: &mut impl Write, +) -> core::fmt::Result { + writeln!(out, "{}", STATS_FORMAT)?; + writeln!(out, "device={}", device)?; + writeln!(out, "now_ns={}", now_ns)?; + writeln!(out, "messages count={}", snap.messages)?; + for kind in StatsKind::ALL { + let count = snap.count[kind as usize]; + write!(out, "{} count={}", kind.name(), count)?; + if count != 0 { + write!(out, " captured_ns={}", snap.captured_ns[kind as usize])?; + snap.write_payload(kind, out)?; + } + writeln!(out)?; + } + Ok(()) +} + +/// Current monotonic time in nanoseconds. +pub(crate) fn monotonic_ns() -> u64 { + // SAFETY: `ktime_get()` may be called from any context. + unsafe { bindings::ktime_get() as u64 } +} + +/// Publish the firmware statistics of one device as a read-only debugfs file. +/// +/// The file is created in the DRM device's own debugfs directory +/// (`/sys/kernel/debug/dri//asahi_stats`), so a card only ever exposes +/// its own statistics. The DRM core removes it when the device is +/// unregistered, and the caller keeps the snapshot alive until after that +/// registration is dropped, so the pointer stays valid for as long as the +/// file exists. +pub(crate) fn register(drm: &AsahiDrmDevice, stats: &Arc>) { + // `drm::Device` is `repr(C)` with the `drm_device` as its first field. + let raw = (drm as *const AsahiDrmDevice) + .cast_mut() + .cast::(); + // `drm_debugfs_add_file()` is a static inline stub without CONFIG_DEBUG_FS, so + // bindgen has no binding for it and there is no file to create. + #[cfg(CONFIG_DEBUG_FS)] + // SAFETY: `raw` is the live DRM device. It is registered at this point, + // so its debugfs directory exists and the file is created in it. + unsafe { + bindings::drm_debugfs_add_file( + raw, + c"asahi_stats".as_char_ptr(), + Some(show), + Arc::as_ptr(stats).cast_mut().cast::(), + ); + } + #[cfg(not(CONFIG_DEBUG_FS))] + let _ = (raw, stats); +} + +/// `drm_debugfs_add_file()` show callback. +/// +/// The DRM core opens the file with `single_open(file, node->show, entry)`, so +/// the seq file's private data is the `struct drm_debugfs_entry` and the +/// per-device payload is in `entry->file.data` (see `drm_debugfs_entry_open()` +/// in `drivers/gpu/drm/drm_debugfs.c`). The `data` argument of the callback is +/// the seq file start token, not the payload. +#[cfg_attr(not(CONFIG_DEBUG_FS), allow(dead_code))] +unsafe extern "C" fn show( + m: *mut bindings::seq_file, + _data: *mut core::ffi::c_void, +) -> core::ffi::c_int { + // SAFETY: The DRM core passes a valid seq file to the show callback. + let seq = unsafe { SeqFile::from_raw(m) }; + // SAFETY: The seq file was opened with the DRM entry as private data. + let entry = unsafe { (*m).private.cast::() }; + // SAFETY: `entry->file.data` is the payload pointer we registered. + let stats = unsafe { (*entry).file.data.cast::>() }; + + let mut out = SeqWriter { seq }; + // SAFETY: The device-owned snapshot outlives its debugfs entry. + let snapshot = *unsafe { (*stats).lock() }; + let device = device_name(entry.cast_const()); + let _ = format_snapshot(&snapshot, monotonic_ns(), device, &mut out); + 0 +} + +/// Name of the DRM device the statistics belong to. +/// +/// The name is owned by the DRM device, which outlives the read. +fn device_name<'a>(entry: *const bindings::drm_debugfs_entry) -> &'a str { + // SAFETY: The DRM core passes the entry it allocated for this file. + let drm = unsafe { (*entry).dev }; + if drm.is_null() { + return "unknown"; + } + // SAFETY: `drm` is the live DRM device of this entry. + let unique = unsafe { (*drm).unique }; + if unique.is_null() { + return "unknown"; + } + // SAFETY: `unique` is a NUL terminated string owned by the DRM device, + // which outlives this read. + let name = unsafe { core::ffi::CStr::from_ptr(unique) }; + name.to_str().unwrap_or("unknown") +} + +/// [`Write`] adapter that forwards to a seq file. +struct SeqWriter<'a> { + seq: &'a SeqFile, +} + +impl Write for SeqWriter<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + self.seq.call_printf(fmt!("{}", value)); + Ok(()) + } +} diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c index b6aa52428a4e6..3b4f9c5e50221 100644 --- a/drivers/gpu/drm/drm_gpuvm.c +++ b/drivers/gpu/drm/drm_gpuvm.c @@ -3258,7 +3258,7 @@ drm_gpuvm_bo_unmap(struct drm_gpuvm_bo *vm_bo, void *priv) { struct drm_gpuva_ops *ops; struct drm_gpuva_op *op; - int ret; + int ret = 0; if (unlikely(!vm_bo->vm)) return -EINVAL; diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index bd0b23ac2cc4b..271506734eb9c 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -298,6 +298,9 @@ pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res, struct ttm_resource_manager *man; enum ttm_caching caching; + if (res->bus.is_nonposted) + return pgprot_noncached(tmp); + man = ttm_manager_type(bo->bdev, res->mem_type); if (man->use_tt) { caching = bo->ttm->caching; @@ -325,7 +328,9 @@ static int ttm_bo_ioremap(struct ttm_buffer_object *bo, resource_size_t res = mem->bus.offset + offset; map->bo_kmap_type = ttm_bo_map_iomap; - if (mem->bus.caching == ttm_write_combined) + if (mem->bus.is_nonposted) + map->virtual = ioremap_np(res, size); + else if (mem->bus.caching == ttm_write_combined) map->virtual = ioremap_wc(res, size); #ifdef CONFIG_X86 else if (mem->bus.caching == ttm_cached) @@ -511,6 +516,8 @@ int ttm_bo_vmap(struct ttm_buffer_object *bo, struct iosys_map *map) if (mem->bus.addr) vaddr_iomem = (void __iomem *)mem->bus.addr; + else if (mem->bus.is_nonposted) + vaddr_iomem = ioremap_np(mem->bus.offset, bo->base.size); else if (mem->bus.caching == ttm_write_combined) vaddr_iomem = ioremap_wc(mem->bus.offset, bo->base.size); diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index 4e6d3d658d858..6be681c9430a3 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -350,6 +350,7 @@ void ttm_resource_init(struct ttm_buffer_object *bo, res->bus.addr = NULL; res->bus.offset = 0; res->bus.is_iomem = false; + res->bus.is_nonposted = false; res->bus.caching = ttm_cached; res->bo = bo; @@ -878,12 +879,20 @@ ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, } if (mem->bus.addr) { - iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr); + if (mem->bus.is_nonposted) + iosys_map_set_vaddr_iomem(&iter_io->dmap, + (void __iomem *)mem->bus.addr); + else + iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr); iter_io->needs_unmap = false; } else { iter_io->needs_unmap = true; memset(&iter_io->dmap, 0, sizeof(iter_io->dmap)); - if (mem->bus.caching == ttm_write_combined) + if (mem->bus.is_nonposted) + iosys_map_set_vaddr_iomem(&iter_io->dmap, + ioremap_np(mem->bus.offset, + mem->size)); + else if (mem->bus.caching == ttm_write_combined) iosys_map_set_vaddr_iomem(&iter_io->dmap, ioremap_wc(mem->bus.offset, mem->size)); @@ -895,7 +904,7 @@ ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, MEMREMAP_WC)); /* If uncached requested or if mapping cached or wc failed */ - if (iosys_map_is_null(&iter_io->dmap)) + if (!mem->bus.is_nonposted && iosys_map_is_null(&iter_io->dmap)) iosys_map_set_vaddr_iomem(&iter_io->dmap, ioremap(mem->bus.offset, mem->size)); diff --git a/drivers/hid/hid-apple-mtp-haptic.c b/drivers/hid/hid-apple-mtp-haptic.c index b8e3ee2f605f2..aa1e02e1f6c95 100644 --- a/drivers/hid/hid-apple-mtp-haptic.c +++ b/drivers/hid/hid-apple-mtp-haptic.c @@ -45,11 +45,33 @@ int apple_taptic_send(struct hid_device *haptic_hdev, u16 effect_type, u8 streng } EXPORT_SYMBOL_GPL(apple_taptic_send); +static bool apple_taptic_has_report(struct hid_device *hdev, unsigned int type, + unsigned int id) +{ + return hdev->report_enum[type].report_id_hash[id] != NULL; +} + int apple_taptic_switch_modes(struct hid_device *haptic_hdev, bool enable_host_controlled) { u8 msg[] = { 0x21, enable_host_controlled }; int ret; + /* + * Report 0x21 is the MTP host actuator mode switch. The J313 + * SPI actuator descriptor has input 0x3f and output 0x53 only. + * Sending 0x21 makes the controller return status 9c 17 e5 f8 + * instead of the write-ok status. The part powers up + * device-controlled, which is what a false argument asks for. + * Host control cannot be taken without the report, so refuse it + * and let the caller keep the firmware's click. The MTP actuators + * take 0x21 whatever their descriptor says, so only an SPI one + * is checked. + */ + if (haptic_hdev->bus == BUS_SPI && + !apple_taptic_has_report(haptic_hdev, HID_FEATURE_REPORT, 0x21) && + !apple_taptic_has_report(haptic_hdev, HID_OUTPUT_REPORT, 0x21)) + return enable_host_controlled ? -EOPNOTSUPP : 0; + ret = hid_hw_raw_request(haptic_hdev, msg[0], msg, sizeof(msg), HID_FEATURE_REPORT, HID_REQ_SET_REPORT); diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c index 0f8b13994f349..f2a12d1fe56af 100644 --- a/drivers/hid/spi-hid/spi-hid-apple-core.c +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -16,6 +16,7 @@ //#define DEBUG 2 +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include "spi-hid-apple.h" @@ -57,6 +59,11 @@ static const u8 spi_hid_apple_booted[4] = { 0xa0, 0x80, 0x00, 0x00 }; static const u8 spi_hid_apple_status_ok[4] = { 0xac, 0x27, 0x68, 0xd5 }; +/* Writers wait out SPIHID_PWR_BOOT and are refused while OFF. */ +#define SPIHID_PWR_OFF 0 +#define SPIHID_PWR_BOOT 1 +#define SPIHID_PWR_ON 2 + struct spihid_interface { struct hid_device *hid; u8 *hid_desc; @@ -120,6 +127,8 @@ struct spihid_apple { /* state tracking flags */ bool status_booted; + /* devm_kzalloc leaves this SPIHID_PWR_OFF. */ + atomic_t power; #ifdef IRQ_WAKE_SUPPORT bool irq_wake_enabled; @@ -194,6 +203,53 @@ struct spihid_transfer_packet { __le16 crc16; }; +static void spihid_set_power(struct spihid_apple *spihid, int power) +{ + mutex_lock(&spihid->tx_lock); + atomic_set(&spihid->power, power); + mutex_unlock(&spihid->tx_lock); + wake_up_interruptible(&spihid->wait); +} + +/* Returns with tx_lock held, or a negative errno with the lock released. */ +static int spihid_wait_ready(struct spihid_apple *spihid) +{ + int err; + + for (;;) { + err = mutex_lock_interruptible(&spihid->tx_lock); + if (err < 0) + return err; + + if (atomic_read(&spihid->power) != SPIHID_PWR_BOOT) + return 0; + + mutex_unlock(&spihid->tx_lock); + err = wait_event_interruptible(spihid->wait, + atomic_read(&spihid->power) != SPIHID_PWR_BOOT); + if (err) + return err; + } +} + +static void spihid_flush_one_led(struct hid_device *hid) +{ + /* hidraw-only devices never call hidinput_connect(). */ + if (hid && (hid->claimed & HID_CONNECT_HIDINPUT)) + flush_work(&hid->led_work); +} + +static void spihid_flush_led_reports(struct spihid_apple *spihid) +{ + /* + * Input suspend turns lit EV_LEDs off by queueing hid->led_work + * on system_wq, which is not frozen. Flush that report while the + * controller is still powered. The actuator has no input device. + */ + spihid_flush_one_led(spihid->kbd.hid); + spihid_flush_one_led(spihid->tp.hid); +} + /* * how HID is mapped onto the protocol is not fully clear. This are the known * reports/request: @@ -230,10 +286,15 @@ static int spihid_apple_request(struct spihid_apple *spihid, u8 target, u8 unk0, if (len > sizeof(pkt->data) - sizeof(*hdr) - sizeof(__le16)) return -EINVAL; - err = mutex_lock_interruptible(&spihid->tx_lock); + err = spihid_wait_ready(spihid); if (err < 0) return err; + if (atomic_read(&spihid->power) != SPIHID_PWR_ON) { + mutex_unlock(&spihid->tx_lock); + return -ENODEV; + } + pkt = (struct spihid_transfer_packet *)spihid->tx_buf; memset(pkt, 0, sizeof(*pkt)); @@ -1042,6 +1103,8 @@ int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops goto error; } + spihid_set_power(spihid, SPIHID_PWR_ON); + /* request device information */ dev_dbg(dev, "request device info"); spihid_apple_request(spihid, 0xd0, 0x20, 0x01, 0xd0, 0, NULL, 0); @@ -1089,6 +1152,13 @@ void spihid_apple_core_remove(struct spi_device *spi) { struct spihid_apple *spihid = spi_get_drvdata(spi); + /* + * hid_destroy_device() cancels led_work, which calls back into + * spihid_apple_request(). Refuse that transfer. Power stays on + * until power_off() below. + */ + spihid_set_power(spihid, SPIHID_PWR_OFF); + /* destroy input devices */ spihid_destroy_hid_device(&spihid->aux); @@ -1107,6 +1177,8 @@ void spihid_apple_core_shutdown(struct spi_device *spi) { struct spihid_apple *spihid = spi_get_drvdata(spi); + spihid_set_power(spihid, SPIHID_PWR_OFF); + /* disable irq */ spihid->ops->disable_irq(spihid->ops); @@ -1139,6 +1211,14 @@ static int spihid_apple_core_suspend(struct device *dev) } } + /* + * Children suspend first. Input core has queued a NumLock-off + * report on the non-freezable system workqueue. Deliver it + * before the controller is powered down, or the transfer reads + * status 00 00 00 00. + */ + spihid_flush_led_reports(spihid); + /* Save some power */ spihid->ops->disable_irq(spihid->ops); @@ -1151,36 +1231,71 @@ static int spihid_apple_core_suspend(struct device *dev) dev_warn(dev, "Failed to enable irq wake: %d\n", wake_status); } else { + spihid_set_power(spihid, SPIHID_PWR_OFF); spihid->ops->power_off(spihid->ops); } #else + spihid_set_power(spihid, SPIHID_PWR_OFF); spihid->ops->power_off(spihid->ops); #endif return 0; } +static int spihid_wake_controller(struct spihid_apple *spihid) +{ + int err; + + /* + * Same order as probe. The boot packet (a0 80 00 00) is + * level-triggered and stays pending while the irq is disabled, + * so drop the flag before the reset and enable the irq after. + */ + spihid_set_power(spihid, SPIHID_PWR_BOOT); + spihid->status_booted = false; + + spihid->ops->power_on(spihid->ops); + spihid->ops->enable_irq(spihid->ops); + + err = wait_event_interruptible_timeout(spihid->wait, + spihid->status_booted, + msecs_to_jiffies(1000)); + if (err == 0) + err = -ETIMEDOUT; + if (err < 0) + dev_err(&spihid->spidev->dev, + "waiting for device boot failed: %d", err); + + /* ON even on timeout, so waiters are not stuck in BOOT. */ + spihid_set_power(spihid, SPIHID_PWR_ON); + return err < 0 ? err : 0; +} + static int spihid_apple_core_resume(struct device *dev) { int ret_tp = 0, ret_kbd = 0; + int err = 0; struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); #ifdef IRQ_WAKE_SUPPORT int wake_status; - if (!device_may_wakeup(dev)) { - spihid->ops->power_on(spihid->ops); - } else if (spihid->irq_wake_enabled) { + if (device_may_wakeup(dev) && spihid->irq_wake_enabled) { wake_status = spihid->ops->disable_irq_wake(spihid->ops); if (!wake_status) spihid->irq_wake_enabled = false; else dev_warn(dev, "Failed to disable irq wake: %d\n", wake_status); + spihid->ops->enable_irq(spihid->ops); + } else { + err = spihid_wake_controller(spihid); } +#else + err = spihid_wake_controller(spihid); #endif - - spihid->ops->enable_irq(spihid->ops); - spihid->ops->power_on(spihid->ops); + if (err) + dev_warn(dev, "SPI HID controller resume incomplete: %d\n", + err); if (spihid->tp.hid) ret_tp = hid_driver_reset_resume(spihid->tp.hid); diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index ba23e2c239cd8..420cd9c3d2367 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -314,6 +314,8 @@ struct apple_dart { /* Resources reachable by a retained domain outlive device removal. */ bool retained; bool retained_mtp; + /* Cold ANE owns SID0 only; all other streams remain inherited. */ + bool cold_ane; u16 managed_sids; bool managed_failed; /* Serialize owned TCR publication against IRQ quarantine. */ @@ -1928,7 +1930,7 @@ static int apple_dart_hw_reset(struct apple_dart *dart) struct apple_dart_stream_map stream_map; int i; - if (dart->managed_sids) + if (dart->managed_sids || dart->cold_ane) return -EBUSY; stream_map.dart = dart; bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); @@ -2285,7 +2287,13 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, struct io_pgtable_cfg *pgtbl_cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; - if (dart->managed_sids) { + if (dart->cold_ane) { + if (bitmap_weight(stream_map->sidmap, dart->num_streams) != 1 || + !test_bit(0, stream_map->sidmap)) + return -EPERM; + apple_dart_writel(dart, apple_dart_readl(dart, dart->hw->enable_streams) | BIT(0), + dart->hw->enable_streams); + } else if (dart->managed_sids) { /* Admission owns only ordinary SIDs 1/2, both in the first word. */ apple_dart_writel(dart, stream_map->sidmap[0] & dart->managed_sids, dart->hw->enable_streams); @@ -2585,9 +2593,11 @@ static int apple_dart_attach_dev_paging(struct iommu_domain *domain, spin_unlock_irqrestore(&dart->managed_lock, flags); if (ret) goto err; - } else if (!stream_map->dart->locked) - apple_dart_setup_translation(dart_domain, stream_map); - else { + } else if (!stream_map->dart->locked) { + ret = apple_dart_setup_translation(dart_domain, stream_map); + if (ret) + goto err; + } else { ret = apple_dart_setup_translation_locked(dart_domain, stream_map); if (ret) goto err; @@ -2880,6 +2890,8 @@ static int apple_dart_of_xlate(struct device *dev, return i; sid = args->args[0]; + if (dart->cold_ane && sid != 0) + return -EPERM; if (dart->managed_sids && (sid <= 0 || sid >= 16 || !(dart->managed_sids & BIT(sid)))) return -EINVAL; @@ -2919,7 +2931,7 @@ static int apple_dart_of_xlate(struct device *dev, return -EINVAL; } - cfg->supports_bypass &= dart->supports_bypass && !dart->managed_sids; + cfg->supports_bypass &= dart->supports_bypass && !dart->managed_sids && !dart->cold_ane; cfg->locked |= dart->locked; for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { @@ -3996,6 +4008,9 @@ static int apple_dart_probe(struct platform_device *pdev) dart->dev = dev; dart->hw = of_device_get_match_data(dev); dart->tunneled = tunneled; + dart->cold_ane = of_device_is_compatible(dev->of_node, "apple,t8140-ane-dart"); + if (dart->cold_ane && !IS_BUILTIN(CONFIG_APPLE_DART)) + return -EOPNOTSUPP; dart->per_stream = of_device_is_compatible(dev->of_node, "apple,t8140-dart") || of_device_is_compatible(dev->of_node, "apple,t8142-dart"); if (of_property_present(dev->of_node, "apple,managed-sids")) { @@ -4036,7 +4051,7 @@ static int apple_dart_probe(struct platform_device *pdev) if (tunneled) dart->regs = apple_dart_platform_ioremap_np_resource(pdev, 0, &res); - else if (dart->managed_sids || dart->preserve_display) + else if (dart->managed_sids || dart->preserve_display || dart->cold_ane) dart->regs = apple_dart_retained_ioremap(dart, res); else dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, @@ -4283,7 +4298,21 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) goto err_clk_disable; } - if (dart->managed_sids) { + if (dart->cold_ane) { + /* No reset, TCR/TTBR clear, global enable, remap or IRQ ACK. + * Cold acquisition is checked by the staged ANE backend before + * it attaches a private SID0 domain. Even bypass SIDs stay intact. + */ + if (dart->locked || dart->version != 0x203 || dart->num_streams != 16 || + !bitmap_empty(dart->remapped, DART_MAX_STREAMS)) { + ret = -EPERM; + goto err_clk_disable; + } + dart->irq = -1; + dart->retained = true; + get_device(dev); + dev_info(dev, "cold ANE DART: inherited streams preserved; SID0 only\n"); + } else if (dart->managed_sids) { bool init_bypass = false; ret = apple_dart_fault_pending(dart) ? -EBUSY : @@ -4377,7 +4406,8 @@ static int apple_dart_probe(struct platform_device *pdev) goto err_sysfs_remove; /* Retain supplier references; generic suspend/resume is not applicable. */ - if (!dart->retained_mtp && !dart->managed_sids && !apple_dart_has_borrowed(dart)) + if (!dart->retained_mtp && !dart->managed_sids && !dart->cold_ane && + !apple_dart_has_borrowed(dart)) pm_runtime_put(dev); dev_dbg( @@ -4513,6 +4543,10 @@ static void apple_dart_remove(struct platform_device *pdev) { struct apple_dart *dart = platform_get_drvdata(pdev); + if (dart->cold_ane) { + dev_warn(&pdev->dev, "cold ANE DART retained until reset\n"); + return; + } if (dart->managed_sids) { apple_dart_managed_quarantine(dart); if (dart->irq >= 0) @@ -4693,7 +4727,8 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) if (READ_ONCE(dart->commands_gated)) return 0; /* Neither retained radio nor display has a qualified suspend boundary. */ - if (dart->managed_sids || dart->preserve_display || apple_dart_has_borrowed(dart)) + if (dart->managed_sids || dart->cold_ane || dart->preserve_display || + apple_dart_has_borrowed(dart)) return -EBUSY; /* The tunneled DART has no state to save when its domain stays on. */ if (dart->power_retained || dart->retained_mtp) @@ -4781,7 +4816,8 @@ static __maybe_unused int apple_dart_resume(struct device *dev) unsigned int sid, idx; int ret; - if (dart->managed_sids || dart->preserve_display || apple_dart_has_borrowed(dart)) + if (dart->managed_sids || dart->cold_ane || dart->preserve_display || + apple_dart_has_borrowed(dart)) return -EBUSY; /* PCIe-C restored these mappings before starting its downstream link. */ diff --git a/drivers/media/platform/apple/avd/avd-av1.c b/drivers/media/platform/apple/avd/avd-av1.c index ba7beea61c842..d5d3207deaa0b 100644 --- a/drivers/media/platform/apple/avd/avd-av1.c +++ b/drivers/media/platform/apple/avd/avd-av1.c @@ -225,12 +225,16 @@ struct avd_av1_run { }; struct avd_av1_ctx { + struct avd_av1_stream *stream; u8 submit_num; struct { struct avd_buf inst; struct avd_buf pipe_state; struct avd_buf probs; + struct avd_buf probs_out; struct avd_buf no_prev_seg; + struct avd_buf saved_seg; + struct avd_buf prev_seg; struct avd_buf above_info; struct avd_buf rf_above_info; struct avd_buf az_above; @@ -246,6 +250,25 @@ struct avd_av1_ctx { } bufs; }; +#define AVD_AV1_BANKS AVD_MAX_FRAMES + +struct avd_av1_bank { + struct avd_av1_ctx codec; + bool busy; + bool prepared; + bool vp_done; + struct avd_decoded_buffer *dst; + u64 timestamp; + u32 flags; + u8 seg_flags; +}; + +struct avd_av1_stream { + unsigned int nbanks; + struct avd_av1_bank bank[AVD_AV1_BANKS]; + struct avd_av1_bank *b; +}; + /* * Timestamp of the reference a ref_frame_idx entry names. The entries are * -1 on frames that use no references (FFmpeg fills key frames that way); @@ -259,6 +282,109 @@ static u64 avd_av1_ref_ts(const struct v4l2_ctrl_av1_frame *frame, int idx) return frame->reference_frame_ts[idx]; } +static bool avd_av1_default_cdf(const struct v4l2_ctrl_av1_frame *frame) +{ + return frame->primary_ref_frame == AV1_PRIMARY_REF_NONE || + frame->frame_type == V4L2_AV1_KEY_FRAME || + frame->frame_type == V4L2_AV1_INTRA_ONLY_FRAME || + (frame->flags & V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE); +} + +static bool avd_av1_writes_seg(u8 flags) +{ + return (flags & V4L2_AV1_SEGMENTATION_FLAG_ENABLED) && + (flags & V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP); +} + +/* + * Queued CDF output is private coherent VP storage, published by the tagged + * VP callback after the queue's dma_rmb(). CAPTURE maps and motion fields + * retain their PP boundary. Slot timestamps, not refresh order, select inputs. + */ +static bool avd_av1_depends(const struct v4l2_ctrl_av1_frame *frame, + const struct avd_av1_bank *bank) +{ + /* Preserve same-VP ordering for named references when engines differ. */ + for (unsigned int i = 0; i < V4L2_AV1_REFS_PER_FRAME; i++) + if (avd_av1_ref_ts(frame, frame->ref_frame_idx[i]) == bank->timestamp && + (!smp_load_acquire(&bank->vp_done) || + (frame->flags & V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS))) + return true; + + if (frame->primary_ref_frame >= AV1_PRIMARY_REF_NONE || + avd_av1_ref_ts(frame, frame->ref_frame_idx[frame->primary_ref_frame]) != + bank->timestamp) + return false; + return (frame->segmentation.flags & V4L2_AV1_SEGMENTATION_FLAG_ENABLED) && + avd_av1_writes_seg(bank->seg_flags) && + DIV_ROUND_UP(bank->dst->av1.width + 1, 8) == + DIV_ROUND_UP(frame->frame_width_minus_1 + 1, 8) && + DIV_ROUND_UP(bank->dst->av1.height + 1, 8) == + DIV_ROUND_UP(frame->frame_height_minus_1 + 1, 8); +} + +static bool avd_av1_ready(struct avd_ctx *ctx) +{ + struct avd_av1_ctx *codec = smp_load_acquire(&ctx->priv); + struct avd_av1_stream *av1_ctx; + struct vb2_v4l2_buffer *src; + struct v4l2_ctrl_handler *hdl; + struct v4l2_ctrl_ref *ref; + const struct v4l2_ctrl_av1_frame *frame = NULL; + bool ready = false, free = false, busy = false; + + if (!codec) + return false; + av1_ctx = codec->stream; + for (unsigned int i = 0; i < av1_ctx->nbanks; i++) { + if (!READ_ONCE(av1_ctx->bank[i].busy)) + free = true; + else if (!smp_load_acquire(&av1_ctx->bank[i].prepared)) + return false; + else + busy = true; + } + if (!free || !busy) + return free; + src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + if (!src || !src->vb2_buf.req_obj.req) + return false; + hdl = v4l2_ctrl_request_hdl_find(src->vb2_buf.req_obj.req, &ctx->ctrl_hdl); + if (!hdl) + return false; + /* + * The queued request is immutable and cannot complete while its OUTPUT + * is still on the ready queue. Read p_req, never the previous picture's + * shared p_cur, without taking a sleeping control mutex under queue_lock. + */ + list_for_each_entry(ref, &hdl->ctrl_refs, node) + if (ref->ctrl->id == V4L2_CID_STATELESS_AV1_FRAME && ref->p_req_valid) { + frame = ref->p_req.p; + break; + } + if (frame) { + ready = true; + for (unsigned int i = 0; i < av1_ctx->nbanks; i++) + if (READ_ONCE(av1_ctx->bank[i].busy) && + avd_av1_depends(frame, &av1_ctx->bank[i])) + ready = false; + } + v4l2_ctrl_request_hdl_put(hdl); + return ready; +} + +static struct avd_av1_bank *avd_av1_ref_bank(struct avd_av1_stream *av1_ctx, + struct avd_decoded_buffer *ref) +{ + if (av1_ctx->nbanks > 1 && ref->bank && ref->bank <= av1_ctx->nbanks) { + struct avd_av1_bank *bank = &av1_ctx->bank[ref->bank - 1]; + + if (bank->busy && bank->prepared && bank->dst == ref) + return bank; + } + return NULL; +} + /* * from verisilicon/rockchip_vpu981_hw_av1_dec.c */ @@ -1160,6 +1286,77 @@ static void avd_av1_set_seg(struct avd_ctx *ctx, struct avd_av1_run *run) memcpy(seg, prev_seg, dst->av1.seg_size); } +/* A live producer owns coherent CDF output until PP retirement, even after VP. */ +static void avd_av1_prepare_prob(struct avd_ctx *ctx, struct avd_av1_run *run) +{ + struct avd_av1_ctx *codec = ctx->priv; + struct avd_av1_stream *stream = codec->stream; + struct avd_decoded_buffer *dst, *ref; + struct avd_av1_bank *bank = NULL; + + dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); + if (stream->nbanks > 1 && !avd_av1_default_cdf(run->frame)) { + ref = avd_get_ref_buf(ctx, &dst->base.vb, + avd_av1_ref_ts(run->frame, + run->frame->ref_frame_idx[run->frame->primary_ref_frame])); + bank = avd_av1_ref_bank(stream, ref); + } + if (!bank) { + avd_av1_set_prob(ctx, run); + } else { + memcpy(codec->bufs.probs.cpu, bank->codec.bufs.probs_out.cpu, + AVD_CDFS_SIZE); + memcpy(vb2_plane_vaddr(&dst->base.vb.vb2_buf, 0) + + AVD_AV1_CDFS_OFFSET(dst->base.vb.vb2_buf.planes[0].length, + dst->av1.color_size), + codec->bufs.probs.cpu, AVD_CDFS_SIZE); + } + if (stream->nbanks > 1) { + /* No VP write when frame-end update is disabled: preserve its input. */ + memcpy(codec->bufs.probs_out.cpu, codec->bufs.probs.cpu, AVD_CDFS_SIZE); + run->addresses.probs_out = codec->bufs.probs_out.addr; + } +} + +static void avd_av1_prepare_seg(struct avd_ctx *ctx, struct avd_av1_run *run) +{ + struct avd_av1_ctx *codec = ctx->priv; + struct avd_av1_stream *stream = codec->stream; + const struct v4l2_ctrl_av1_frame *frame = run->frame; + struct avd_decoded_buffer *dst, *ref; + struct avd_av1_bank *bank = NULL; + void *seg; + + dst = vb2_to_avd_decoded_buf(&run->base.bufs.dst->vb2_buf); + seg = vb2_plane_vaddr(&dst->base.vb.vb2_buf, 0) + + AVD_AV1_SEG_OFFSET(dst->base.vb.vb2_buf.planes[0].length, + dst->av1.color_size, dst->av1.seg_size); + if (stream->nbanks > 1 && frame->primary_ref_frame != AV1_PRIMARY_REF_NONE) { + ref = avd_get_ref_buf(ctx, &dst->base.vb, + avd_av1_ref_ts(frame, frame->ref_frame_idx[frame->primary_ref_frame])); + if (ref != dst && + DIV_ROUND_UP(ref->av1.width + 1, 8) == + DIV_ROUND_UP(frame->frame_width_minus_1 + 1, 8) && + DIV_ROUND_UP(ref->av1.height + 1, 8) == + DIV_ROUND_UP(frame->frame_height_minus_1 + 1, 8)) + bank = avd_av1_ref_bank(stream, ref); + } + if (bank && !avd_av1_writes_seg(bank->seg_flags)) { + /* Copy into our bank: producer PP can retire during our VP. */ + memcpy(codec->bufs.prev_seg.cpu, bank->codec.bufs.saved_seg.cpu, + dst->av1.seg_size); + run->addresses.prev_seg = codec->bufs.prev_seg.addr; + if (!(frame->segmentation.flags & V4L2_AV1_SEGMENTATION_FLAG_ENABLED)) + memset(seg, 0, dst->av1.seg_size); + else if (!(frame->segmentation.flags & V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP)) + memcpy(seg, codec->bufs.prev_seg.cpu, dst->av1.seg_size); + } else { + avd_av1_set_seg(ctx, run); + } + if (stream->nbanks > 1 && !avd_av1_writes_seg(frame->segmentation.flags)) + memcpy(codec->bufs.saved_seg.cpu, seg, dst->av1.seg_size); +} + static int avd_av1_seg_size(int w, int h) { /* 3 bits per 4x4 block, in rows of 128x128 superblocks (64x64 fit too) */ @@ -1264,9 +1461,22 @@ static int avd_av1_run_preamble(struct avd_ctx *ctx, struct avd_av1_run *run) return 0; } +/* run() has selected a free bank; retain its mappings for unchanged geometry. */ +static int avd_av1_alloc_work_buf(struct avd_ctx *ctx, struct avd_buf *buf, + size_t size) +{ + struct avd_av1_ctx *codec = ctx->priv; + + if (codec->stream->nbanks > 1 && buf->cpu && buf->size == size) { + /* Preserve fresh-allocation contents, including hardware scratch. */ + memset(buf->cpu, 0, size); + return 0; + } + return avd_buf_alloc(ctx->dev, buf, size); +} + static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) { - struct avd_dev *avd = ctx->dev; struct avd_av1_ctx *av1_ctx = ctx->priv; const struct v4l2_ctrl_av1_frame *frame = run->frame; const struct v4l2_ctrl_av1_sequence *seq = run->seq; @@ -1280,6 +1490,17 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) int h = frame->frame_height_minus_1 + 1; int bit_depth = seq->bit_depth; + if (av1_ctx->stream->nbanks > 1) { + ret = avd_av1_alloc_work_buf(ctx, &av1_ctx->bufs.saved_seg, + avd_av1_seg_size(w, h)); + if (ret) + return ret; + ret = avd_av1_alloc_work_buf(ctx, &av1_ctx->bufs.prev_seg, + avd_av1_seg_size(w, h)); + if (ret) + return ret; + } + for (i = 0; i < tile_info->tile_cols; i++) { sb = tile_info->width_in_sbs_minus_1[i] + 1; max_sb_col = sb > max_sb_col ? sb : max_sb_col; @@ -1290,38 +1511,38 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) max_sb_col <<= sb_shift; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.mv_above_info, max_sb_col * 80); + ret = avd_av1_alloc_work_buf(ctx, &av1_ctx->bufs.mv_above_info, max_sb_col * 80); if (ret) return ret; /* first frame this is always much smaller */ - ret = avd_buf_alloc(avd, &av1_ctx->bufs.above_info, max_sb_col * 400); + ret = avd_av1_alloc_work_buf(ctx, &av1_ctx->bufs.above_info, max_sb_col * 400); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.lf_above_info, sb_cols1); + ret = avd_av1_alloc_work_buf(ctx, &av1_ctx->bufs.lf_above_info, sb_cols1); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.ip_above, + ret = avd_av1_alloc_work_buf(ctx, &av1_ctx->bufs.ip_above, max_sb_col * bit_depth * 22); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.lf_above, + ret = avd_av1_alloc_work_buf(ctx, &av1_ctx->bufs.lf_above, ALIGN(ALIGN(w, 8) * bit_depth * 2 + tile_info->tile_cols * bit_depth * 16, 128)); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.rf_above_info, sb_cols2); + ret = avd_av1_alloc_work_buf(ctx, &av1_ctx->bufs.rf_above_info, sb_cols2); if (ret) return ret; /* zeroes, read as the previous segment ids when no reference has them */ if (av1_ctx->bufs.no_prev_seg.size != avd_av1_seg_size(w, h)) { - ret = avd_buf_alloc(avd, &av1_ctx->bufs.no_prev_seg, + ret = avd_av1_alloc_work_buf(ctx, &av1_ctx->bufs.no_prev_seg, avd_av1_seg_size(w, h)); if (ret) return ret; @@ -1336,7 +1557,7 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) max_sb_row = max_sb_row << sb_shift; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.lf_left_info, + ret = avd_av1_alloc_work_buf(ctx, &av1_ctx->bufs.lf_left_info, max_sb_row * 72); if (ret) return ret; @@ -1351,24 +1572,24 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) * I counted blocks as luma or chroma with padding until it * changes to the other */ - ret = avd_buf_alloc(avd, &av1_ctx->bufs.lf_left, + ret = avd_av1_alloc_work_buf(ctx, &av1_ctx->bufs.lf_left, (max_sb_row * 72 + 1) * 2 * bit_depth); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.sr_left, + ret = avd_av1_alloc_work_buf(ctx, &av1_ctx->bufs.sr_left, max_sb_row * bit_depth * (53 + 52)); if (ret) return ret; - ret = avd_buf_alloc( - avd, &av1_ctx->bufs.rf_left, + ret = avd_av1_alloc_work_buf( + ctx, &av1_ctx->bufs.rf_left, max_sb_row * bit_depth * (192 + 96 + (bit_depth > 8 ? 12 : 0))); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.rf_left_info, + ret = avd_av1_alloc_work_buf(ctx, &av1_ctx->bufs.rf_left_info, max_sb_row * 24); if (ret) return ret; @@ -1380,6 +1601,7 @@ static int avd_av1_alloc_work_bufs(struct avd_ctx *ctx, struct avd_av1_run *run) static int avd_av1_run(struct avd_ctx *ctx) { struct avd_av1_run run; + struct avd_av1_stream *av1_ctx = ((struct avd_av1_ctx *)ctx->priv)->stream; struct avd_decoded_buffer *dst, *ref; int ret; @@ -1402,6 +1624,29 @@ static int avd_av1_run(struct avd_ctx *ctx) avd_run_postamble(ctx, &run.base); return -ENOMEM; } + if (av1_ctx->nbanks > 1) { + struct avd_av1_bank *bank = NULL; + + for (unsigned int i = 0; i < av1_ctx->nbanks; i++) + if (!av1_ctx->bank[i].busy) { + bank = &av1_ctx->bank[i]; + break; + } + if (!bank) { + avd_run_postamble(ctx, &run.base); + return -EBUSY; + } + av1_ctx->b = bank; + smp_store_release(&ctx->priv, &bank->codec); + WRITE_ONCE(bank->prepared, false); + WRITE_ONCE(bank->vp_done, false); + WRITE_ONCE(bank->busy, true); + bank->dst = dst; + bank->timestamp = run.base.bufs.src->vb2_buf.timestamp; + bank->flags = run.frame->flags; + bank->seg_flags = run.frame->segmentation.flags; + dst->bank = bank - av1_ctx->bank + 1; + } ret = avd_init_job(ctx, AVD_CODEC_AV1, run.frame->tile_info.tile_cols * @@ -1420,85 +1665,151 @@ static int avd_av1_run(struct avd_ctx *ctx) update_dec_buf_info(dst, run.seq, run.frame); - avd_av1_set_prob(ctx, &run); - avd_av1_set_seg(ctx, &run); + avd_av1_prepare_prob(ctx, &run); + avd_av1_prepare_seg(ctx, &run); set_header(ctx, &run); set_tiles(ctx, &run); + if (av1_ctx->nbanks > 1) { + ctx->job_fifo = av1_ctx->b->codec.bufs.inst.addr; + smp_store_release(&av1_ctx->b->prepared, true); + } avd_run_postamble(ctx, &run.base); return avd_submit_job(ctx); } -static int avd_av1_alloc_bufs(struct avd_ctx *ctx) + +/* + * VP writes probs_out, independently of PP reconstruction. The queue executes + * dma_rmb() before this callback. Publish coherent output, never read a live + * CAPTURE tail. PP still owns the bank and all buffer/request lifetimes. + */ +static void avd_av1_vp_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *dst) +{ + struct avd_av1_stream *stream = ((struct avd_av1_ctx *)ctx->priv)->stream; + struct avd_decoded_buffer *buf = vb2_to_avd_decoded_buf(&dst->vb2_buf); + struct avd_av1_bank *bank = avd_av1_ref_bank(stream, buf); + + if (bank) + smp_store_release(&bank->vp_done, true); +} + +static void avd_av1_done(struct avd_ctx *ctx, struct vb2_v4l2_buffer *src, + struct vb2_v4l2_buffer *dst, enum vb2_buffer_state result) +{ + struct avd_av1_stream *av1_ctx = ((struct avd_av1_ctx *)ctx->priv)->stream; + struct avd_decoded_buffer *buf = vb2_to_avd_decoded_buf(&dst->vb2_buf); + + if (buf->bank && buf->bank <= av1_ctx->nbanks) { + struct avd_av1_bank *bank = &av1_ctx->bank[buf->bank - 1]; + + if (bank->dst == buf) { + if (av1_ctx->nbanks > 1 && result == VB2_BUF_STATE_DONE) { + /* Retired references keep the same CAPTURE-tail ABI. */ + memcpy(vb2_plane_vaddr(&dst->vb2_buf, 0) + + AVD_AV1_CDFS_OFFSET(dst->vb2_buf.planes[0].length, + buf->av1.color_size), + bank->codec.bufs.probs_out.cpu, AVD_CDFS_SIZE); + } + WRITE_ONCE(bank->busy, false); + } + buf->bank = 0; + } +} + +static int avd_av1_alloc_bufs(struct avd_ctx *ctx, struct avd_av1_bank *bank) { struct avd_dev *avd = ctx->dev; - struct avd_av1_ctx *av1_ctx = ctx->priv; int ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.inst, fifo_size()); + ret = avd_buf_alloc(avd, &bank->codec.bufs.inst, fifo_size()); if (ret) return ret; /* no m1 to worry about :) */ - ret = avd_buf_alloc(avd, &av1_ctx->bufs.pipe_state, 0x200); + ret = avd_buf_alloc(avd, &bank->codec.bufs.pipe_state, 0x200); if (ret) return ret; - ret = avd_buf_alloc(avd, &av1_ctx->bufs.probs, + ret = avd_buf_alloc(avd, &bank->codec.bufs.probs, sizeof(struct avd_av1_cdfs)); if (ret) return ret; + if (bank->codec.stream->nbanks > 1) { + ret = avd_buf_alloc(avd, &bank->codec.bufs.probs_out, AVD_CDFS_SIZE); + if (ret) + return ret; + } + return 0; } -static void avd_av1_stop(struct avd_ctx *ctx) +static void avd_av1_free(struct avd_ctx *ctx, struct avd_av1_stream *av1_ctx) { - struct avd_av1_ctx *av1_ctx = ctx->priv; struct avd_dev *avd = ctx->dev; - avd_buf_free(avd, &av1_ctx->bufs.above_info); - avd_buf_free(avd, &av1_ctx->bufs.rf_above_info); - avd_buf_free(avd, &av1_ctx->bufs.az_above); - avd_buf_free(avd, &av1_ctx->bufs.ip_above); - avd_buf_free(avd, &av1_ctx->bufs.lf_above); - avd_buf_free(avd, &av1_ctx->bufs.lf_above_info); - avd_buf_free(avd, &av1_ctx->bufs.lf_left); - avd_buf_free(avd, &av1_ctx->bufs.lf_left_info); - avd_buf_free(avd, &av1_ctx->bufs.sr_left); - avd_buf_free(avd, &av1_ctx->bufs.rf_left); - avd_buf_free(avd, &av1_ctx->bufs.rf_left_info); - avd_buf_free(avd, &av1_ctx->bufs.no_prev_seg); - avd_buf_free(avd, &av1_ctx->bufs.mv_above_info); - - avd_buf_free(avd, &av1_ctx->bufs.pipe_state); - avd_buf_free(avd, &av1_ctx->bufs.inst); - avd_buf_free(avd, &av1_ctx->bufs.probs); - + for (unsigned int i = 0; i < av1_ctx->nbanks; i++) { + struct avd_av1_bank *bank = &av1_ctx->bank[i]; + + avd_buf_free(avd, &bank->codec.bufs.above_info); + avd_buf_free(avd, &bank->codec.bufs.rf_above_info); + avd_buf_free(avd, &bank->codec.bufs.az_above); + avd_buf_free(avd, &bank->codec.bufs.ip_above); + avd_buf_free(avd, &bank->codec.bufs.lf_above); + avd_buf_free(avd, &bank->codec.bufs.lf_above_info); + avd_buf_free(avd, &bank->codec.bufs.lf_left); + avd_buf_free(avd, &bank->codec.bufs.lf_left_info); + avd_buf_free(avd, &bank->codec.bufs.sr_left); + avd_buf_free(avd, &bank->codec.bufs.rf_left); + avd_buf_free(avd, &bank->codec.bufs.rf_left_info); + avd_buf_free(avd, &bank->codec.bufs.no_prev_seg); + avd_buf_free(avd, &bank->codec.bufs.mv_above_info); + + avd_buf_free(avd, &bank->codec.bufs.pipe_state); + avd_buf_free(avd, &bank->codec.bufs.inst); + avd_buf_free(avd, &bank->codec.bufs.probs); + avd_buf_free(avd, &bank->codec.bufs.probs_out); + avd_buf_free(avd, &bank->codec.bufs.saved_seg); + avd_buf_free(avd, &bank->codec.bufs.prev_seg); + } kfree(av1_ctx); +} + +static void avd_av1_stop(struct avd_ctx *ctx) +{ + avd_av1_free(ctx, ((struct avd_av1_ctx *)ctx->priv)->stream); ctx->priv = NULL; + ctx->max_frames = 1; } static int avd_av1_start(struct avd_ctx *ctx) { - struct avd_av1_ctx *av1_ctx; + struct avd_av1_stream *av1_ctx; int ret; av1_ctx = kzalloc(sizeof(*av1_ctx), GFP_KERNEL); if (!av1_ctx) return -ENOMEM; - ctx->priv = av1_ctx; - ret = avd_av1_alloc_bufs(ctx); - if (ret) - goto err_free_ctx; - + av1_ctx->nbanks = avd_qfw_has_vp_done(ctx->dev) ? + avd_qfw_picture_banks(ctx->dev, AVD_CODEC_AV1) : 1; + av1_ctx->b = &av1_ctx->bank[0]; + for (unsigned int i = 0; i < av1_ctx->nbanks; i++) { + av1_ctx->bank[i].codec.stream = av1_ctx; + ret = avd_av1_alloc_bufs(ctx, &av1_ctx->bank[i]); + if (ret) + goto err_free_ctx; + } + ctx->max_frames = av1_ctx->nbanks; + /* Readiness must never see partially allocated bank storage. */ + smp_store_release(&ctx->priv, &av1_ctx->bank[0].codec); return 0; err_free_ctx: - avd_av1_stop(ctx); + avd_av1_free(ctx, av1_ctx); return ret; } @@ -1586,6 +1897,9 @@ const struct avd_coded_fmt_ops avd_av1_fmt_ops = { .start = avd_av1_start, .stop = avd_av1_stop, .run = avd_av1_run, + .done = avd_av1_done, + .vp_done = avd_av1_vp_done, + .ready = avd_av1_ready, .submit = avd_av1_submit, .pp_kicks = avd_av1_pp_kicks, .try_ctrl = avd_av1_try_ctrl, diff --git a/drivers/media/platform/apple/avd/avd-backend-queue.c b/drivers/media/platform/apple/avd/avd-backend-queue.c index 5853d9d050a93..178572cb2e5d5 100644 --- a/drivers/media/platform/apple/avd/avd-backend-queue.c +++ b/drivers/media/platform/apple/avd/avd-backend-queue.c @@ -117,8 +117,8 @@ static unsigned int avd_qfw_bank_limit(struct avd_qfw *q, unsigned int codec) unsigned int banks = 1; u32 pool; - /* k4m keeps two banks with old queue firmware. */ - if (!q->multi_vp || codec > AVD_CODEC_H264) + /* Old queue firmware and VP9 retain the two-bank overlap path. */ + if (!q->multi_vp || codec == AVD_CODEC_VP9 || codec > AVD_CODEC_AV1) return 2; /* One bank per VP plus one retained by PP. No SoC counts in the host. */ for (pool = q->vp_pool[codec]; pool; pool &= pool - 1) diff --git a/drivers/media/platform/apple/avd/avd-protocol.rst b/drivers/media/platform/apple/avd/avd-protocol.rst index 138b5d81cdafd..54452958436f6 100644 --- a/drivers/media/platform/apple/avd/avd-protocol.rst +++ b/drivers/media/platform/apple/avd/avd-protocol.rst @@ -129,9 +129,10 @@ probability dependency is removed by choosing another engine. VP9 retains its two banks and producer-side backward-adaptation barrier from the VP-DONE implementation. Its tagged VP callback remains separate from full -PP retirement. AV1 retains its single-picture CDF/segmentation dependency path. -The advertised pools may select different engines across independent contexts; -neither codec's dependency rules are relaxed. +PP retirement. AV1 publishes per-bank coherent CDF output after its tagged VP callback, +while updated segmentation maps and reference motion fields retain their PP +barriers. Only PP retirement releases banks and saves CDFs into CAPTURE tails. +See tests/README.rst for the dependency matrix and hardware qualification limit. Pool counts live in the firmware image, not an additional host SoC table. The avd-fw v5-t2 T6040 counts (HEVC 4, H.264 4, VP9 1, AV1 3) are inferred from diff --git a/drivers/media/platform/apple/avd/tests/README.rst b/drivers/media/platform/apple/avd/tests/README.rst index 1d4f3917f63c4..1309e9d5b4dde 100644 --- a/drivers/media/platform/apple/avd/tests/README.rst +++ b/drivers/media/platform/apple/avd/tests/README.rst @@ -200,37 +200,105 @@ This is a backwards-compatible optional ABI 2 feature: bank reuse. Only the original tagged PP completion does that. A missing VP event remains bounded by the original per-picture watchdog. -AV1 analysis -~~~~~~~~~~~~ - -AV1's current ``avd_av1_set_prob()`` reads the primary reference's CDF tail on -the CPU, and seeds the current frame's output CDF because hardware leaves it -untouched when DISABLE_FRAME_END_UPDATE_CDF is set. ``avd_av1_set_seg()`` can -also copy the primary reference's segmentation map on the CPU. These are -separate dependencies; removing only the CDF wait is insufficient. - -=================================== ========================================= -Case CDF source dependency -=================================== ========================================= -Intra/error-resilient/PRIMARY_NONE Defaults; no primary-reference CDF read -Primary ref with frame-end disabled Its initialized CDF, available at prepare -Primary ref with frame-end enabled Its hardware-written CDF tail -=================================== ========================================= - -``disable_frame_end_update_cdf`` on the *consumer* does not remove its primary -reference input dependency. It controls that consumer's future CDF output. -Likewise, a default CDF does not necessarily remove the segmentation-map input -selected by primary_ref_frame. - -The available implementation does not establish whether the CDF tail *and* -segmentation map are fully visible at VP DONE, or which CPU DMA synchronization -is required for an in-flight CAPTURE allocation while PP writes another part. -This differs from VP9's separate coherent count buffer. AV1 therefore retains -its one-picture path. Enabling it needs evidence for both tail writes and a -safe visibility/synchronization rule, then two scratch banks and a dependency -on the actual primary reference (which need not be the preceding picture). -No AV1 or VP9 pixel correctness or throughput claim follows from these host -and emulator tests; device qualification remains necessary. +AV1 picture overlap and codec VP pools +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``av1-test.py`` compiles the production AV1 codec, entropy defaults, open +program builder and queued descriptor writer with ASan/UBSan. It is also +called by the registered ``avd-codec-tail-tests.py`` host check. Run:: + + AVD_AV1_BASELINE_REF=d62b4894 python3 tests/av1-test.py /tmp/avd-av1 + python3 tests/unicorn-av1-test.py /path/to/avd-fw \ + /path/to/avd-fw/build/avd-fw-v5-t2.bin /tmp/avd-av1 + +The independent legacy comparison builds before/after executables and compares +complete emitted programs plus initialized CDF/map tails for twelve header cases. +The flag table covers primary-reference/default CDF cases, both producer and +consumer disable_frame_end_update_cdf, segmentation enabled/update_map on both +pictures, and all 256 refresh masks. Pending request p_req values are used for +readiness; p_cur still describes an earlier picture and must not be used. + +Coherent CDF output and completion boundary +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The public AV1 specification section 7.7 updates frame CDFs from the saved +context-update tile; sections 7.19 and 7.20 save motion fields and segmentation: +https://aomediacodec.github.io/av1-spec/ . CDF adaptation depends on entropy +parsing, independently of pixel reconstruction. The driver supplies probs_out +to the VP program, while the open firmware starts ordered PP reconstruction +after VP DONE. These are the implementation grounds for using VP completion +for CDF output; they are not hardware measurements of memory visibility. + +Queued AV1 now directs probs_out to a separate dma_alloc_coherent allocation +in each bank. Input probabilities and output probabilities remain distinct. +Preparation seeds output with input, preserving frames with frame-end update +disabled. A named live producer blocks until its full-tag VP callback, invoked +by the queue after dma_rmb(). The consumer copies the coherent output into its +own input and seeded output. It never samples a PP-owned CAPTURE CDF tail. +This follows the existing coherent VP9 count-output completion boundary. +The firmware descriptor DMB alone is not a substitute for DMA coherence or +proof of the silicon's completion semantics. Physical CDF completion timing, +pixel hashes and performance must still be qualified on hardware. + +Only successful PP retirement copies coherent CDF output into the producer's +CAPTURE tail, retaining the saved-reference ABI for subsequent pictures after +bank reuse. Error/reset retirement discards output. Banks, FIFOs, CAPTURE and +requests remain owned through PP; VP completion only publishes readiness. +The existing CPU-mappable CAPTURE requirement is retained. No CPU DMA sync is +attempted on a CAPTURE allocation still owned by PP. + +Queue firmware advertising VP-DONE feature bit 2 enables two scratch banks; +with a validated codec pool (feature bit 3), AV1 uses one bank per advertised +AV1 VP plus one retained by PP, bounded by physical FIFO slots. Legacy +transport and queue firmware without tagged VP DONE retain one bank and +identical programs and initialized tails. No firmware change is required. + +Each pending request uses exact ref_frame_idx/primary_ref_frame timestamps, +independently of refresh order. The consumer's disable_frame_end_update_cdf +does not remove its input dependency. refresh_frame_flags only changes which +slots userspace subsequently names, not the dependency of this picture. + +=============================== ============================================== +Consumer / producer condition Prepare dependency while producer is in flight +=============================== ============================================== +Named live reference Wait for its tagged VP DONE on any engine +PRIMARY_REF_NONE No primary CDF or map dependency +Intra or error resilient Default CDF; map dependency still checked +Primary slot names retired ref Read saved CAPTURE CDF +Primary CDF, producer end update Enabled: coherent output after tagged VP DONE +Primary CDF, producer end update Disabled: seeded coherent output after VP DONE +Consumer end update disabled Still consumes selected primary CDF +Consumer segmentation disabled No previous-map read +Consumer segmentation enabled Updated producer map: wait for PP (same Mi size) +Producer map not updated Copy immutable saved map into consumer bank +Different reference Mi size Zero map; no map dependency +USE_REF_FRAME_MVS Wait for PP of any named in-flight reference +refresh_frame_flags 0..255 No additional dependency; named slot decides +=============================== ============================================== + +Segmentation and motion-field outputs remain in CAPTURE and retain their PP +barriers. Both consumer update_map values retain previous-map input when +segmentation is enabled. Saved maps and CDFs are copied into consumer-owned +storage, so producer bank reuse cannot invalidate a live consumer's input. +Busy banks publish fully prepared metadata with release/acquire; allocation +failures and PP error/reset retirement select the saved CAPTURE bank. + +For unchanged geometry, a free queued bank reuses scratch allocations and +zeroes their contents, preserving fresh-allocation semantics without repeated +DMA mapping and deferred unmapping in prepare. Size changes reallocate; the +single-bank legacy path retains its original allocation behavior. The zero +previous-segment map retains its existing size-based reuse. + +The Unicorn run boots the v5-t2 CM3 reset vector, routes AV1 and PP IRQs, +and checks production AV1 program words/descriptors/configuration. It shows +VP(1) issued during PP(0), concurrent VPs 9/10/11, a fourth picture waiting for +VP9 DONE, reverse engine completion, full-tag events, ordered PP and blocked +mailboxes. Routed errors on each AV1 engine drain every owner and reset. +Host tests cover all producer VP/map/MV barriers, all refresh masks, adaptive +CDF output distinct from input, poisoned CAPTURE tails, PP CDF persistence, +error discard, private consumer copies after producer bank reuse, duplicate +callbacks, allocation failures and scratch reuse/zeroing/resize. These tests +model DMA completion; they do not decode pixels or prove physical visibility. H.264 conformance fixes ---------------------- @@ -281,3 +349,12 @@ preamble. Across all four revision/address variants, weight modes and first/continuation slices, its complete program must equal the baseline IDR program exactly. Pictures with references retain their original header. The baseline comparison retains all 32 before/after empty-DPB cases. + +Film grain controls, frame flags and the legacy syntax/program emitters are +unchanged. The independent baseline comparison includes film grain control +values and SHOW_FRAME/SHOWABLE_FRAME combinations at 8 and 10 bits. It checks +program/tail parity, not grain synthesis. The stateless AV1 frame UAPI has no +show_existing_frame decode control: userspace presents the existing CAPTURE +buffer without a new tile decode. No VP event retires that CAPTURE buffer or +its media request early. Hardware qualification must include those presentation +paths as well as pixel hashes; this change makes no new grain-support claim. diff --git a/drivers/media/platform/apple/avd/tests/av1-test.py b/drivers/media/platform/apple/avd/tests/av1-test.py new file mode 100644 index 0000000000000..661eb982f30a5 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/av1-test.py @@ -0,0 +1,430 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Production AV1 bank/dependency/emitters, with explicit host DMA boundaries. + +Run in a controller build slot: av1-test.py OUT [AVD_DIR [KERNEL_ROOT]]. +AVD_AV1_BASELINE_REF optionally compares complete legacy programs/CDF/map bytes. +No decoder arithmetic or DMA visibility is modeled by these host fixtures. +""" +import hashlib +import importlib.util +import os +from pathlib import Path +import re +import subprocess +import sys + +from host_source import function + +HERE = Path(__file__).resolve().parent +AVD = Path(sys.argv[2]).resolve() if len(sys.argv) > 2 else HERE.parent +KERNEL = Path(sys.argv[3]).resolve() if len(sys.argv) > 3 else AVD.parents[4] +OUT = Path(sys.argv[1]).resolve() +OUT.mkdir(parents=True, exist_ok=True) +spec = importlib.util.spec_from_file_location('program_fixture', HERE / 'program-test.py') +fixture = importlib.util.module_from_spec(spec) +spec.loader.exec_module(fixture) +pre = fixture.PRELUDE +pre = pre.replace('typedef uint8_t u8;', 'typedef uint8_t u8; typedef uint16_t u16; typedef int8_t s8; typedef int16_t s16; typedef int64_t s64;') +pre = pre.replace('#define AVD_MAX_INST 512', '#define AVD_MAX_INST 512\n#define AVD_CODEC_HEVC 0') +pre = pre.replace('struct vb2_buffer { struct { size_t length; } planes[1]; dma_addr_t dma; };', + 'struct vb2_buffer { struct { size_t length; } planes[1]; dma_addr_t dma; void *cpu; size_t payload; u64 timestamp; struct {void *req;} req_obj; };') +header = (AVD / 'avd.h').read_text() +a = header.index('struct avd_av1_decoded_buffer_info {') +b = header.index('\n};', a) + 3 +pre = pre.replace('struct avd_decoded_buffer {', header[a:b] + '\nstruct avd_decoded_buffer {') +pre = pre.replace('struct avd_comp comp; unsigned int bank;', 'struct avd_comp comp; unsigned int bank; struct avd_av1_decoded_buffer_info av1;') +pre = pre.replace('struct vb2_v4l2_buffer *dst; } bufs;', 'struct vb2_v4l2_buffer *src,*dst; } bufs;') +pre = pre.replace('unsigned int fifo_idx, pp_total;', 'unsigned int fifo_idx, pp_total, max_frames; int ctrl_hdl;') +pre = pre.replace('dma_addr_t comp_out, y_out, uv_out;', 'dma_addr_t comp_out, y_out, uv_out, coded_in;') +pre = pre.replace('struct { u32 bytesperline; } plane_fmt[1];', 'u32 width,height; struct { u32 bytesperline; } plane_fmt[1];') +pre = pre.replace('static struct avd_decoded_buffer reference;', 'static struct avd_decoded_buffer reference; static struct avd_decoded_buffer *refs[5];') +pre = pre.replace('(void)c; (void)b; (void)t; return &reference;', '(void)c;for(unsigned int i=0;i<5;i++)if(refs[i]&&refs[i]->base.vb.vb2_buf.timestamp==t)return refs[i];return (void *)b;') +pre = pre.replace('static int avd_buf_alloc(struct avd_dev *d, struct avd_buf *b, size_t n)', + 'static struct avd_ctx *allocating_ctx;\nstatic unsigned int allocations;\nstatic int fail_alloc_after;\nstatic bool avd_av1_ready(struct avd_ctx *);\nstatic int avd_buf_alloc(struct avd_dev *d, struct avd_buf *b, size_t n)') +pre = pre.replace('{ (void)d; free(b->cpu); b->cpu = calloc(1,n);', + '{ allocations++; if(allocating_ctx){assert(!avd_av1_ready(allocating_ctx));if(fail_alloc_after&&!--fail_alloc_after)return -ENOMEM;} (void)d; free(b->cpu); b->cpu = calloc(1,n);') +pre += r''' +#define AVD_ALIGN 256 +#define S16_MIN INT16_MIN +#define S16_MAX INT16_MAX +#define clamp_val(x,l,h) clamp(x,l,h) +#define U64_MAX UINT64_MAX +#define BIT_ULL(n) (1ULL<<(n)) +#define ALIGN(x,a) (((x)+(a)-1)&~((__typeof__(x))(a)-1)) +#define ALIGN_DOWN(x,a) ((x)&~((__typeof__(x))(a)-1)) +#define max_t(t,a,b) ((t)(a)>(t)(b)?(t)(a):(t)(b)) +#define min(a,b) ((a)<(b)?(a):(b)) +#define max(a,b) ((a)>(b)?(a):(b)) +#define clamp(x,l,h) ((x)<(l)?(l):(x)>(h)?(h):(x)) +#define WARN_ON(v) (v) +#define GFP_KERNEL 0 +#define smp_load_acquire(p) (*(p)) +#define smp_store_release(p,v) (*(p)=(v)) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define static_assert _Static_assert +#define VB2_BUF_STATE_DONE 0 +#define VB2_BUF_STATE_ERROR 1 +enum vb2_buffer_state { STATE_DUMMY }; +#define kzalloc(n,f) calloc(1,n) +#define kfree free +static unsigned int avd_qfw_picture_banks(const struct avd_dev *, unsigned int); +static bool vp_feature=true; +static bool avd_qfw_has_vp_done(const struct avd_dev *d){return d->queued&&vp_feature;} +static void *vb2_plane_vaddr(struct vb2_buffer *b,int p){(void)p;return b->cpu;} +static struct vb2_v4l2_buffer *current_src,*current_dst; +static struct vb2_v4l2_buffer *v4l2_m2m_next_src_buf(void *m){(void)m;return current_src;} +struct v4l2_ctrl {unsigned int id; union {void *p;} p_cur;}; +struct v4l2_ctrl_ref {struct v4l2_ctrl_ref *next; struct v4l2_ctrl *ctrl; bool p_req_valid; union {void *p;} p_req;}; +struct v4l2_ctrl_handler {struct v4l2_ctrl_ref *ctrl_refs;}; +#define list_for_each_entry(ref,head,node) for((ref)=*(head);(ref);(ref)=(ref)->next) +static struct v4l2_ctrl_av1_frame params,request_params; +static struct v4l2_ctrl_av1_sequence seq; +static struct v4l2_ctrl_av1_tile_group_entry tile; +static struct v4l2_ctrl_av1_film_grain grain; +static struct v4l2_ctrl controls[4]; +static struct v4l2_ctrl_ref request_ref; +static struct v4l2_ctrl_handler request_hdl; +static unsigned int request_gets,request_puts; +static struct v4l2_ctrl_handler *v4l2_ctrl_request_hdl_find(void *req,void *h){(void)h;assert(req==&request_hdl);request_gets++;return &request_hdl;} +static void v4l2_ctrl_request_hdl_put(struct v4l2_ctrl_handler *h){assert(h==&request_hdl);request_puts++;} +static struct v4l2_ctrl *v4l2_ctrl_find(void *h,u32 id){ + (void)h;controls[0].p_cur.p=&seq;controls[1].p_cur.p=¶ms;controls[2].p_cur.p=&tile;controls[3].p_cur.p=&grain; + if(id==V4L2_CID_STATELESS_AV1_SEQUENCE)return &controls[0]; + if(id==V4L2_CID_STATELESS_AV1_FRAME)return &controls[1]; + if(id==V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY)return &controls[2]; + return &controls[3]; +} +static void avd_run_preamble(struct avd_ctx *ctx,struct avd_run *run){ + (void)ctx;memset(run,0,sizeof(*run));run->bufs.src=current_src;run->bufs.dst=current_dst; + run->coded_in=current_src->vb2_buf.dma;run->comp_out=current_dst->vb2_buf.dma; + run->y_out=run->comp_out+0x100000;run->uv_out=run->comp_out+0x300000; +} +static void avd_run_postamble(struct avd_ctx *ctx,struct avd_run *run){(void)ctx;(void)run;} +static int avd_init_job(struct avd_ctx *ctx,int codec,size_t segments){ + free(ctx->job.segments);ctx->job.codec=codec;ctx->job.num=0; + ctx->job.segments=calloc(segments,sizeof(*ctx->job.segments));return ctx->job.segments?0:-ENOMEM; +} +static int avd_submit_job(struct avd_ctx *ctx); +''' + + +def without_includes(s): + return re.sub(r'^#include[^\n]*\n', '', s, flags=re.M) + + +av1 = (AVD / 'avd-av1.c').read_text() +common = pre + without_includes((AVD / 'avd-av1-entropymode.h').read_text()) +common += without_includes((AVD / 'avd-av1-entropymode.c').read_text()) +common += without_includes((AVD / 'avd-inst.h').read_text()) +qfw = (AVD / 'avd-backend-queue.c').read_text() +hw = (AVD / 'avd-hw.c').read_text() +post = function((AVD / 'avd-backend-open.c').read_text(), 'avd_open_build_program') +post += qfw[qfw.index('#define AVDQ_MAGIC'):qfw.index('/* Arguments of dev_dbg()')] +post += hw[hw.index('#define AVD_V3_'):hw.index('int avd_boot(')] +post += '#define w32(r,v) writel((v),avd->ctrl+(r))\n#define m32(r,v) w32(r,(v)|readl(avd->ctrl+(r)))\n' +post += function(hw, 't8112_configure_stream') + function(hw, 't8122_configure_stream') +post += function(qfw, 'avd_qfw_picture_banks') +post += function(qfw, 'avd_qfw_stream') + function(qfw, 'avd_qfw_submit') +post += '\nstatic int avd_submit_job(struct avd_ctx *ctx){return avd_open_build_program(ctx);}\n' + + +def codec(s): + return without_includes(s[s.index('#define V4L2_AV1_SEG_LVL_ALT_LF_Y_H'):s.index('static enum avd_image_fmt avd_av1_get_image_fmt')]) + function(s, 'avd_av1_pp_kicks') + + +MAIN = r''' +static void write_program(const char *dir,const char *name,struct avd_frame *f){ + char path[4096];snprintf(path,sizeof(path),"%s/%s.bin",dir,name);FILE *out=fopen(path,"wb");assert(out); + assert(fwrite(f->prog->cpu,4,f->prog_words,out)==f->prog_words);fclose(out); + struct avd_dev *dev=f->ctx->dev;struct avd_qfw *q=dev->qfw; + q->wp=0;memset(q->sram,0,0x10000);f->tag=0x10|f->idx;assert(!avd_qfw_submit(dev,f)); + snprintf(path,sizeof(path),"%s/%s-desc.bin",dir,name);out=fopen(path,"wb");assert(out); + assert(fwrite(q->sram+0x8000+f->idx*64,1,64,out)==64);fclose(out); + snprintf(path,sizeof(path),"%s/%s-mmio.bin",dir,name);out=fopen(path,"wb");assert(out); + assert(fwrite(dev->ctrl,1,0x10000,out)==0x10000);fclose(out); +} +static void queue_request(struct vb2_v4l2_buffer *src){ + request_params=params;controls[1].id=V4L2_CID_STATELESS_AV1_FRAME; + request_ref=(struct v4l2_ctrl_ref){.ctrl=&controls[1],.p_req_valid=true,.p_req.p=&request_params}; + request_hdl.ctrl_refs=&request_ref;src->vb2_buf.req_obj.req=&request_hdl; +} +static void flag_table(struct avd_av1_bank *b){ + b->vp_done=true; + /* Actual producer timestamp is slot 5, not the immediately prior slot 0. */ + struct v4l2_ctrl_av1_frame f={.frame_type=V4L2_AV1_INTER_FRAME,.primary_ref_frame=2, + .frame_width_minus_1=1919,.frame_height_minus_1=1087}; + for(unsigned int i=0;i<7;i++)f.ref_frame_idx[i]=i; + f.ref_frame_idx[2]=5;f.reference_frame_ts[5]=b->timestamp; + for(unsigned int defaults=0;defaults<4;defaults++) + for(unsigned int output_disabled=0;output_disabled<2;output_disabled++) + for(unsigned int input_disabled=0;input_disabled<2;input_disabled++) + for(unsigned int producer_seg=0;producer_seg<4;producer_seg++) + for(unsigned int consumer_seg=0;consumer_seg<4;consumer_seg++) + for(unsigned int refresh=0;refresh<256;refresh++){ + f.primary_ref_frame=defaults==1?7:2; + f.frame_type=defaults==2?V4L2_AV1_INTRA_ONLY_FRAME:V4L2_AV1_INTER_FRAME; + f.flags=(defaults==3?V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE:0)| + (output_disabled?V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF:0); + f.refresh_frame_flags=refresh; + f.segmentation.flags=(consumer_seg&1?V4L2_AV1_SEGMENTATION_FLAG_ENABLED:0)| + (consumer_seg&2?V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP:0); + b->flags=input_disabled?V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF:0; + b->seg_flags=(producer_seg&1?V4L2_AV1_SEGMENTATION_FLAG_ENABLED:0)| + (producer_seg&2?V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP:0); + bool expect=defaults!=1&&(consumer_seg&1)&&producer_seg==3; + assert(avd_av1_depends(&f,b)==expect); + } + for(unsigned int primary=0;primary<7;primary++){ + f.primary_ref_frame=primary;f.flags=0;f.frame_type=V4L2_AV1_INTER_FRAME;f.segmentation.flags=0; + b->flags=0;f.ref_frame_idx[primary]=5;assert(!avd_av1_depends(&f,b)); + b->vp_done=false;assert(avd_av1_depends(&f,b));b->vp_done=true; + f.reference_frame_ts[5]++;assert(!avd_av1_depends(&f,b));f.reference_frame_ts[5]--; + } + f.primary_ref_frame=2;f.flags=V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS;f.reference_frame_ts[5]=b->timestamp; + assert(avd_av1_depends(&f,b));f.primary_ref_frame=7;assert(avd_av1_depends(&f,b)); + f.reference_frame_ts[5]++;assert(!avd_av1_depends(&f,b)); + puts("PASS: AV1 flag table (primary, consumer/producer CDF flags, segmentation enabled/update_map, every refresh mask), actual slot and MV dependencies"); +} +int main(int argc,char **argv){ + assert(argc==2); + struct avd_variant variant={.revision=4,.quirks=0,.fifo_slots=15,.vp_slots={4,4,1,3}}; + struct avd_dev dev={.variant=&variant,.queued=true,.capabilities=15}; + struct avd_qfw queue={.sram=calloc(1,0x10000),.aperture=0x1094000,.ring_words=0x1800,.fifo_slots=15,.vp_done=true}; + dev.qfw=&queue;dev.ctrl=calloc(1,0x10000);dev.mbox=calloc(1,0x100);variant.configure_stream=t8122_configure_stream; + writel(0x20001,dev.mbox+0x50); + static const struct avd_coded_fmt_ops ops={.pp_kicks=avd_av1_pp_kicks,.vp_done=avd_av1_vp_done}; + static const struct fmt_desc fmt={.ops=&ops}; + struct avd_ctx ctx={.dev=&dev,.coded_fmt_desc=&fmt,.pp_total=1}; + struct avd_buf program={};struct avd_frame frame={.prog=&program,.ctx=&ctx};dev.building=&frame; + struct avd_decoded_buffer dst[5]={};struct vb2_v4l2_buffer src={};u8 coded[512]={}; + src.vb2_buf=(struct vb2_buffer){.dma=0x220000000ULL,.cpu=coded,.payload=512,.timestamp=1000};current_src=&src; + for(unsigned int i=0;i<5;i++){ + dst[i].base.vb.vb2_buf=(struct vb2_buffer){.dma=0x230000000ULL+i*0x1000000ULL,.cpu=calloc(1,0x800000),.planes[0].length=0x800000,.timestamp=1000+i};refs[i]=&dst[i]; + } + ctx.decoded_fmt.fmt.pix_mp.width=1920;ctx.decoded_fmt.fmt.pix_mp.height=1088; + ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; + assert(!avd_av1_ready(&ctx));allocating_ctx=&ctx;assert(!avd_av1_start(&ctx));allocating_ctx=NULL; + struct avd_av1_stream *p=((struct avd_av1_ctx *)ctx.priv)->stream;assert(p->nbanks==2&&ctx.max_frames==2); + assert(p->bank[0].codec.bufs.probs.addr!=p->bank[1].codec.bufs.probs.addr); + /* Equal-sized idle-bank scratch reuses its mapping but starts zeroed. */ + struct avd_buf scratch={}; + assert(!avd_av1_alloc_work_buf(&ctx,&scratch,256)); + unsigned int before=allocations;dma_addr_t saved_addr=scratch.addr; + memset(scratch.cpu,0x5a,256); + assert(!avd_av1_alloc_work_buf(&ctx,&scratch,256)); + assert(allocations==before&&scratch.addr==saved_addr&&scratch.size==256); + for(unsigned int i=0;i<256;i++)assert(!((u8 *)scratch.cpu)[i]); + assert(!avd_av1_alloc_work_buf(&ctx,&scratch,512));assert(allocations==before+1&&scratch.size==512); + avd_buf_free(&dev,&scratch); + seq=(struct v4l2_ctrl_av1_sequence){.bit_depth=8,.flags=V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_X|V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_Y}; + params=(struct v4l2_ctrl_av1_frame){.frame_type=V4L2_AV1_KEY_FRAME,.primary_ref_frame=7, + .flags=0,.frame_width_minus_1=1919,.frame_height_minus_1=1087,.upscaled_width=1920}; + params.tile_info.tile_cols=params.tile_info.tile_rows=1;params.tile_info.width_in_sbs_minus_1[0]=29;params.tile_info.height_in_sbs_minus_1[0]=16; + tile.tile_size=512;tile.tile_col=tile.tile_row=0; + current_dst=&dst[0].base.vb;frame.idx=0;assert(avd_av1_ready(&ctx));assert(!avd_av1_run(&ctx)); + assert(dst[0].bank==1&&p->bank[0].busy&&p->bank[0].prepared); + assert(frame.fifo_iova==p->bank[0].codec.bufs.inst.addr); + struct avd_av1_run prob_probe={.frame=¶ms};prob_probe.base.bufs.dst=current_dst; + avd_av1_prepare_prob(&ctx,&prob_probe); + assert(prob_probe.addresses.probs_out==p->bank[0].codec.bufs.probs_out.addr); + assert(!memcmp(p->bank[0].codec.bufs.probs.cpu,p->bank[0].codec.bufs.probs_out.cpu,AVD_CDFS_SIZE)); + write_program(argv[1],"av1-overlap-0",&frame);flag_table(&p->bank[0]); + p->bank[0].flags=params.flags;p->bank[0].seg_flags=0; + params.frame_type=V4L2_AV1_INTER_FRAME;params.primary_ref_frame=0;params.ref_frame_idx[0]=0;params.reference_frame_ts[0]=1000; + params.segmentation.flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED; + queue_request(&src); + /* p_cur is deliberately stale and different from the pending request. */ + params.flags=0;request_params.flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; + p->bank[0].vp_done=false;assert(!avd_av1_ready(&ctx)); + avd_av1_vp_done(&ctx,&dst[0].base.vb);assert(p->bank[0].busy&&avd_av1_ready(&ctx)); + avd_av1_vp_done(&ctx,&dst[0].base.vb);assert(p->bank[0].busy); + p->bank[0].flags=0;assert(avd_av1_ready(&ctx));p->bank[0].flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; + p->bank[0].seg_flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED|V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP; + assert(!avd_av1_ready(&ctx));p->bank[0].seg_flags=0; + request_params.flags|=V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS;assert(!avd_av1_ready(&ctx));request_params.flags&=~V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS; + assert(avd_av1_ready(&ctx));params=request_params; + /* Model adaptive VP output distinct from input and poisoned CAPTURE. */ + assert(p->bank[0].codec.bufs.probs_out.addr!=p->bank[0].codec.bufs.probs.addr); + memset(p->bank[0].codec.bufs.probs_out.cpu,0x6c,AVD_CDFS_SIZE); + size_t tail=AVD_AV1_CDFS_OFFSET(0x800000,dst[0].av1.color_size); + memset((u8 *)dst[0].base.vb.vb2_buf.cpu+tail,0xee,sizeof(struct avd_av1_cdfs)); + memset(p->bank[0].codec.bufs.saved_seg.cpu,0x35,dst[0].av1.seg_size); + current_dst=&dst[1].base.vb;src.vb2_buf.timestamp=1001;frame.idx=1; + assert(!avd_av1_run(&ctx));assert(dst[1].bank==2&&p->bank[0].busy&&p->bank[1].busy); + assert(!memcmp(p->bank[0].codec.bufs.probs_out.cpu,p->bank[1].codec.bufs.probs.cpu,AVD_CDFS_SIZE)); + assert(!memcmp(p->bank[1].codec.bufs.probs.cpu,p->bank[1].codec.bufs.probs_out.cpu,AVD_CDFS_SIZE)); + assert(((u8 *)p->bank[1].codec.bufs.probs.cpu)[0]==0x6c); + assert(!memcmp(p->bank[0].codec.bufs.saved_seg.cpu,p->bank[1].codec.bufs.prev_seg.cpu,dst[1].av1.seg_size)); + assert(p->bank[1].codec.bufs.prev_seg.addr!=p->bank[0].codec.bufs.saved_seg.addr); + write_program(argv[1],"av1-overlap-1",&frame);assert(!avd_av1_ready(&ctx)); + current_dst=&dst[2].base.vb;assert(avd_av1_run(&ctx)==-EBUSY);assert(!dst[2].bank); + avd_av1_done(&ctx,&src,&dst[0].base.vb,VB2_BUF_STATE_DONE); + assert(!p->bank[0].busy&&p->bank[1].busy&&dst[1].bank==2); + assert(!memcmp((u8 *)dst[0].base.vb.vb2_buf.cpu+tail,p->bank[0].codec.bufs.probs_out.cpu,AVD_CDFS_SIZE)); + /* Producer bank reuse must not invalidate the consumer's VP map input. */ + memset(p->bank[0].codec.bufs.saved_seg.cpu,0x98,dst[0].av1.seg_size); + memset(p->bank[0].codec.bufs.probs_out.cpu,0x98,AVD_CDFS_SIZE); + assert(((u8 *)p->bank[1].codec.bufs.probs.cpu)[0]==0x6c); + assert(((u8 *)p->bank[1].codec.bufs.prev_seg.cpu)[0]==0x35); + avd_av1_done(&ctx,&src,&dst[0].base.vb,VB2_BUF_STATE_DONE);assert(p->bank[1].busy); + size_t tail1=AVD_AV1_CDFS_OFFSET(0x800000,dst[1].av1.color_size); + memset((u8 *)dst[1].base.vb.vb2_buf.cpu+tail1,0xa7,AVD_CDFS_SIZE); + avd_av1_done(&ctx,&src,&dst[1].base.vb,VB2_BUF_STATE_ERROR);assert(!p->bank[1].busy&&avd_av1_ready(&ctx)); + assert(((u8 *)dst[1].base.vb.vb2_buf.cpu)[tail1]==0xa7); + assert(request_gets==request_puts); + avd_av1_stop(&ctx);assert(!avd_av1_ready(&ctx)); + for(unsigned int mode=0;mode<2;mode++){ + dev.queued=mode;vp_feature=false;assert(!avd_av1_start(&ctx));p=((struct avd_av1_ctx *)ctx.priv)->stream; + assert(p->nbanks==1&&ctx.max_frames==1);avd_av1_stop(&ctx); + } + dev.queued=true;vp_feature=true; + for(int n=1;n<=8;n++){ + fail_alloc_after=n;allocating_ctx=&ctx;assert(avd_av1_start(&ctx)==-ENOMEM);allocating_ctx=NULL; + assert(!ctx.priv&&!avd_av1_ready(&ctx)); + } + + /* AV1 pool size is firmware-owned, including sparse masks and FIFO bounds. */ + for(unsigned int count=1;count<=3;count++){ + queue.multi_vp=true;queue.vp_pool[3]=((1u<stream; + assert(p->nbanks==count+1&&ctx.max_frames==count+1); + avd_av1_stop(&ctx);assert(ctx.max_frames==1); + } + queue.vp_pool[3]=BIT(9)|BIT(11);queue.fifo_slots=2; + assert(!avd_av1_start(&ctx));p=((struct avd_av1_ctx *)ctx.priv)->stream; + assert(p->nbanks==2);avd_av1_stop(&ctx); + queue.fifo_slots=15;queue.vp_pool[3]=BIT(9)|BIT(10)|BIT(11);queue.next_vp[3]=0; + assert(!avd_av1_start(&ctx));p=((struct avd_av1_ctx *)ctx.priv)->stream; + assert(p->nbanks==4&&ctx.max_frames==4); + params.primary_ref_frame=7;params.flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; + params.segmentation.flags=0; + for(unsigned int i=0;i<7;i++)params.ref_frame_idx[i]=-1; + for(unsigned int n=0;n<4;n++){ + src.vb2_buf.timestamp=1000+n;current_dst=&dst[n].base.vb;frame.idx=n; + queue_request(&src);assert(avd_av1_ready(&ctx));assert(!avd_av1_run(&ctx)); + char stem[64];snprintf(stem,sizeof(stem),"av1-multi-%u",n);write_program(argv[1],stem,&frame); + assert(dst[n].bank==n+1&&p->bank[n].busy&&p->bank[n].prepared); + assert(readl(queue.sram+0x8000+n*64+D_VP)==9+n%3); + for(unsigned int prev=0;prevbank[prev].codec.bufs.inst.addr!=p->bank[n].codec.bufs.inst.addr); + assert(p->bank[prev].codec.bufs.probs.addr!=p->bank[n].codec.bufs.probs.addr); + } + } + assert(!avd_av1_ready(&ctx));current_dst=&dst[4].base.vb; + assert(avd_av1_run(&ctx)==-EBUSY&&!dst[4].bank); + /* Free a bank, then reference each of the other engines' live producers. */ + avd_av1_done(&ctx,&src,&dst[3].base.vb,VB2_BUF_STATE_DONE); + for(unsigned int n=0;n<3;n++){ + request_params.primary_ref_frame=0;request_params.ref_frame_idx[0]=n; + request_params.reference_frame_ts[n]=1000+n; + p->bank[n].flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; + assert(!avd_av1_ready(&ctx));avd_av1_vp_done(&ctx,&dst[n].base.vb); + assert(p->bank[n].busy&&p->bank[n].vp_done&&avd_av1_ready(&ctx)); + p->bank[n].flags=0;assert(avd_av1_ready(&ctx)); + p->bank[n].flags=V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF; + assert(avd_av1_ready(&ctx)); + p->bank[n].seg_flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED|V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP; + request_params.segmentation.flags=V4L2_AV1_SEGMENTATION_FLAG_ENABLED; + assert(!avd_av1_ready(&ctx));request_params.segmentation.flags=0;p->bank[n].seg_flags=0; + request_params.flags|=V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS; + assert(!avd_av1_ready(&ctx));request_params.flags&=~V4L2_AV1_FRAME_FLAG_USE_REF_FRAME_MVS; + } + /* Reset/error retirement must release the saved bank, irrespective of builder. */ + for(unsigned int n=3;n>0;n--){ + avd_av1_done(&ctx,&src,&dst[n-1].base.vb,VB2_BUF_STATE_ERROR); + assert(!p->bank[n-1].busy&&!dst[n-1].bank); + } + assert(avd_av1_ready(&ctx)); + params.primary_ref_frame=7;params.segmentation.flags=0; + current_dst=&dst[0].base.vb;frame.idx=0;assert(!avd_av1_run(&ctx)); + assert(dst[0].bank==1&&p->bank[0].busy); + avd_av1_done(&ctx,&src,&dst[0].base.vb,VB2_BUF_STATE_DONE);avd_av1_stop(&ctx); + /* Fail allocation in every bank, including after earlier banks succeeded. */ + for(int n=1;n<=16;n++){ + fail_alloc_after=n;allocating_ctx=&ctx;assert(avd_av1_start(&ctx)==-ENOMEM);allocating_ctx=NULL; + assert(!ctx.priv&&!avd_av1_ready(&ctx)); + } + puts("PASS: AV1 firmware pool sizes, sparse masks, FIFO bounds, per-engine dispatch, all producer barriers, exhaustion, reset/error and reuse"); + for(unsigned int i=0;i<5;i++)free(dst[i].base.vb.vb2_buf.cpu); + free(program.cpu);free(queue.sram);free(dev.ctrl);free(dev.mbox); + puts("PASS: AV1 coherent adaptive CDF inheritance, PP CDF persistence, error discard, poisoned CAPTURE, request lookahead, banks, PP retirement, exhaustion, failures and legacy fallback"); +} +''' + + +def build_run(name, source): + (OUT / f'{name}.c').write_text(source) + cmd = [os.environ.get('CC', 'cc'), '-std=gnu11', '-g', '-O1', '-Wall', '-Wextra', '-Werror', + '-Wno-unused-function', '-Wno-unused-variable', '-Wno-unused-parameter', '-Wno-sign-compare', '-Wno-pointer-sign', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', str(OUT / f'{name}.c'), '-o', str(OUT / name)] + subprocess.run(cmd, check=True) + subprocess.run([str(OUT / name), str(OUT)], check=True) + + +build_run('av1-host', common + codec(av1) + post + MAIN) +(OUT / 'source.sha256').write_text('\n'.join(hashlib.sha256(p.read_bytes()).hexdigest() + ' ' + str(p) for p in ( + AVD / 'avd-av1.c', AVD / 'avd-av1-entropymode.c', AVD / 'avd-backend-open.c', AVD / 'avd-backend-queue.c')) + '\n') + +base_ref = os.environ.get('AVD_AV1_BASELINE_REF') +if base_ref: + old = subprocess.check_output(['git', '-C', str(KERNEL), 'show', + base_ref + ':drivers/media/platform/apple/avd/avd-av1.c'], text=True) + legacy = r''' +int main(int argc,char **argv){ + assert(argc==2); + struct avd_variant variant={.revision=4,.fifo_slots=15}; + struct avd_dev dev={.variant=&variant,.queued=false}; + struct avd_coded_fmt_ops ops={};struct fmt_desc fmt={.ops=&ops}; + struct avd_ctx ctx={.dev=&dev,.coded_fmt_desc=&fmt}; + struct avd_buf program={};struct avd_frame f={.prog=&program,.ctx=&ctx};dev.building=&f; + struct avd_decoded_buffer dst[2]={};struct vb2_v4l2_buffer src={};u8 coded[512]={}; + current_src=&src;src.vb2_buf=(struct vb2_buffer){.dma=0x220000000ULL,.cpu=coded,.payload=512}; + for(unsigned int i=0;i<2;i++){ + dst[i].base.vb.vb2_buf=(struct vb2_buffer){.dma=0x230000000ULL+i*0x1000000ULL,.cpu=calloc(1,0x800000),.planes[0].length=0x800000,.timestamp=1000+i};refs[i]=&dst[i]; + } + ctx.inst.addr=0x205000000ULL; + ctx.decoded_fmt.fmt.pix_mp.width=1920;ctx.decoded_fmt.fmt.pix_mp.height=1088; + ctx.decoded_fmt.fmt.pix_mp.plane_fmt[0].bytesperline=1920; + seq=(struct v4l2_ctrl_av1_sequence){.bit_depth=8,.flags=V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_X|V4L2_AV1_SEQUENCE_FLAG_SUBSAMPLING_Y}; + params=(struct v4l2_ctrl_av1_frame){.frame_type=V4L2_AV1_KEY_FRAME,.primary_ref_frame=7, + .frame_width_minus_1=1919,.frame_height_minus_1=1087,.upscaled_width=1920}; + params.tile_info.tile_cols=params.tile_info.tile_rows=1;params.tile_info.width_in_sbs_minus_1[0]=29;params.tile_info.height_in_sbs_minus_1[0]=16;tile.tile_size=512; + assert(!avd_av1_start(&ctx)); + for(unsigned int mode=0;mode<12;mode++){ + current_dst=&dst[!!mode].base.vb; + params.frame_type=mode==0?V4L2_AV1_KEY_FRAME:mode==2?V4L2_AV1_INTRA_ONLY_FRAME:V4L2_AV1_INTER_FRAME; + params.primary_ref_frame=mode==0||mode==4?7:0; + params.ref_frame_idx[0]=0;params.reference_frame_ts[0]=1000; + params.flags=(mode==3?V4L2_AV1_FRAME_FLAG_ERROR_RESILIENT_MODE:0)|(mode==5?V4L2_AV1_FRAME_FLAG_DISABLE_FRAME_END_UPDATE_CDF:0); + seq.bit_depth=mode>=10?10:8; + if(mode>=8){ + seq.flags|=V4L2_AV1_SEQUENCE_FLAG_FILM_GRAIN_PARAMS_PRESENT; + grain.flags=V4L2_AV1_FILM_GRAIN_FLAG_APPLY_GRAIN|V4L2_AV1_FILM_GRAIN_FLAG_UPDATE_GRAIN; + grain.grain_seed=1234+mode; + params.flags|=mode&1?V4L2_AV1_FRAME_FLAG_SHOW_FRAME:V4L2_AV1_FRAME_FLAG_SHOWABLE_FRAME; + } + params.segmentation.flags=mode>=6?V4L2_AV1_SEGMENTATION_FLAG_ENABLED:0; + if(mode==7)params.segmentation.flags|=V4L2_AV1_SEGMENTATION_FLAG_UPDATE_MAP; + assert(!avd_av1_run(&ctx)); + char path[4096];snprintf(path,sizeof(path),"%s/@NAME@-%u.bin",argv[1],mode);FILE *out=fopen(path,"wb");assert(out); + assert(fwrite(program.cpu,4,f.prog_words,out)==f.prog_words); + struct avd_decoded_buffer *d=&dst[!!mode];size_t off=AVD_AV1_SEG_OFFSET(0x800000,d->av1.color_size,d->av1.seg_size); + assert(fwrite((u8 *)d->base.vb.vb2_buf.cpu+off,1,d->av1.seg_size+ALIGN(AVD_CDFS_SIZE,AVD_ALIGN),out)==d->av1.seg_size+ALIGN(AVD_CDFS_SIZE,AVD_ALIGN));fclose(out); + if(mode==0){ + size_t cdf=AVD_AV1_CDFS_OFFSET(0x800000,dst[0].av1.color_size); + memset((u8 *)dst[0].base.vb.vb2_buf.cpu+cdf,0x57,AVD_CDFS_SIZE); + memset((u8 *)dst[0].base.vb.vb2_buf.cpu+cdf-dst[0].av1.seg_size,0x36,dst[0].av1.seg_size); + } + } + avd_av1_stop(&ctx);free(program.cpu); + for(unsigned int i=0;i<2;i++)free(dst[i].base.vb.vb2_buf.cpu); +} +''' + for name, body in [('legacy-before', old), ('legacy-after', av1)]: + shim = 'static bool avd_av1_ready(struct avd_ctx *ctx){(void)ctx;return false;}\n' if name == 'legacy-before' and 'static bool avd_av1_ready(' not in old else '' + build_run(name, common + codec(body) + shim + post + legacy.replace('@NAME@', name)) + for mode in range(12): + assert (OUT / f'legacy-before-{mode}.bin').read_bytes() == (OUT / f'legacy-after-{mode}.bin').read_bytes(), mode + print('PASS: separate baseline/current executables emit byte-identical legacy AV1 programs and CDF/map tails for twelve header cases including film grain/show flags at 8/10 bits') diff --git a/drivers/media/platform/apple/avd/tests/unicorn-av1-test.py b/drivers/media/platform/apple/avd/tests/unicorn-av1-test.py new file mode 100644 index 0000000000000..c635e3c2452e7 --- /dev/null +++ b/drivers/media/platform/apple/avd/tests/unicorn-av1-test.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""AV1 production programs: VP(N+1) starts after VP(N) DONE, during PP(N). + +Usage: unicorn-av1-test.py FIRMWARE_TREE v5-t2_IMAGE AV1_HOST_OUTPUT +CDF/map input snapshots are host fixtures. CM3 code, descriptors, program words, +PP commands and configure_stream writes are production. No pixels are decoded. +""" +import importlib.util +from pathlib import Path +import struct +import sys +from unicorn.arm_const import UC_ARM_REG_SP + +fw, image, programs = map(lambda p: Path(p).resolve(), sys.argv[1:4]) +spec = importlib.util.spec_from_file_location('transport', fw / 'tests/transport-test.py') +transport = importlib.util.module_from_spec(spec) +spec.loader.exec_module(transport) + + +class AV1(transport.Transport): + def __init__(self, multi=False): + self.multi = multi + self.engines = {} + self.live_engines = {} + self.booting = False + self.expected = [] + self.events = [] + self.live = self.pp_owner = None + self.feed = self.word = 0 + super().__init__(image) + assert self.version == 5 and self.tier == 2 + self.booting = True + self.uc.reg_write(UC_ARM_REG_SP, self.u32(0)) + self.uc.emu_start(self.u32(4), 0, count=500000) + assert not self.booting + + def drain(self, n): + # Simulate a host consuming each available token; self.busy separately + # models a deliberately blocked outbox in the poll-only sections. + for _ in range(n): + self.busy = False + self.poll() + + def route(self, irq): + if irq < 14: + router = self.u32(0x50010010) & (1 << irq) + else: + off = irq - 14 + router = self.u32(0x50010014 + off // 32 * 4) & (1 << (off % 32)) + nvic = self.u32(0xe000e100 + irq // 32 * 4) & (1 << (irq % 32)) + assert router and nvic, f'IRQ {irq} masked' + self.call(f'irq{irq}') + + def write(self, uc, access, addr, size, value, user): + if self.booting and addr == 0x50010090 and value == 1: + self.booting = False + uc.emu_stop() + if self.decode + self.fifo <= addr < self.decode + self.fifo + self.vps * 4: + idx, words = self.expected[self.feed] + assert value == words[self.word], (idx, self.word, value) + vp = self.engines[idx] + assert addr == self.decode + self.fifo + vp * 4 + if not self.word and self.multi: + assert vp not in self.live_engines, "VP reused before its DONE" + self.live_engines[vp] = idx + if not self.word and not self.multi: + assert self.live is None, 'successor issued before VP DONE' + self.live = idx + self.events.append(f'AV1 VP START {idx}; PP owner={self.pp_owner}') + self.word += 1 + if self.word == len(words): + self.feed += 1 + self.word = 0 + if addr == self.decode + self.pp: + idx = (value >> 4) & 15 + assert self.pp_owner is None, 'successor PP issued before predecessor PP DONE' + assert value == 0x2b00020f | idx << 4 + self.pp_owner = idx + self.events.append(f'PP START {idx}') + if addr == 0x50010060: + self.events.append(f'notification {value:#x}') + super().write(uc, access, addr, size, value, user) + + def submit_host(self, idx, start): + stem = f"av1-{'multi' if self.multi else 'overlap'}-{idx}" + data = (programs / f'{stem}.bin').read_bytes() + words = list(struct.unpack('<' + 'I' * (len(data) // 4), data)) + desc = list(struct.unpack('<16I', (programs / f'{stem}-desc.bin').read_bytes())) + vp = 9 + idx % 3 if self.multi else 9 + assert self.pool[3] & (1 << vp) + assert desc[:4] == [0x41565131, 0x10 | idx, 3, vp] + self.engines[idx] = vp + assert desc[4:6] == [0, 0] and desc[6] == len(words) + assert desc[8:16] == [1, 0, 0, 3, 0, 0, 0, 0] + assert words[0] == 0x2b000200 | idx << 4 + assert words[-1] == 0x2b000400 | idx << 4 + mmio = (programs / f'{stem}-mmio.bin').read_bytes() + for offset in (0x1d0, 0x20c, 0x248, 0x284, 0x2c0): + at = offset + idx * 4 + self.put(self.decode + at, struct.unpack_from('> 16 == 0x5453] + + +for blocked in (False, True): + t = AV1() + wp = t.submit_host(0, 0) + t.drain(20) + assert t.live == 0 and t.pp_owner is None and not pp_tokens(t) + t.finish_vp(0) + t.drain(20) + assert t.pp_owner == 0 and t.live is None and not pp_tokens(t) + # The host consumes the tagged VP barrier before dispatching this named + # reference. Its CDF/map inputs come from immutable preparation snapshots. + wp = t.submit_host(1, wp) + t.drain(20) + assert t.live == 1 and t.pp_owner == 0, 'no AV1 VP/PP overlap' + t.finish_vp(1) + t.drain(20) + assert t.pp_owner == 0 and not pp_tokens(t) + t.busy = blocked + t.finish_pp(0) + t.poll(20) + if blocked: + assert not pp_tokens(t) and t.result(0) == [1, 0, 0] + t.busy = False + t.drain(20) + assert t.result(0) == [2, 0, 0x10] and pp_tokens(t) == [0x54530010] + assert t.pp_owner == 1 + t.busy = False + t.finish_pp(1) + t.drain(20) + assert t.result(1) == [2, 0, 0x11] + assert t.notifications == [0x54540010, 0x54540011, 0x54530010, 0x54530011] + assert t.u32(0x10007ff8) == t.u32(0x10007ffc) == wp + assert not t.fetches and not t.ring_writes + assert t.u32(0x1000803c) == t.u32(0x1000807c) == 1 + print('\n'.join('TRACE ' + e for e in t.events)) + print(f'PASS: AV1 snapshot picture overlap, blocked PP mailbox={blocked}; VP(1) issued before PP(0) DONE') + + +for blocked in (False, True): + t = AV1(multi=True) + wp = 0 + for idx in range(4): + wp = t.submit_host(idx, wp) + t.drain(30) + assert t.live_engines == {9: 0, 10: 1, 11: 2} + assert t.feed == 3 and t.pp_owner is None + # DONE on a later engine never starts its PP before predecessor VP DONE. + for idx in (2, 1): + t.finish_vp(idx) + t.drain(10) + assert t.pp_owner is None and not pp_tokens(t) + t.finish_vp(0) + t.drain(20) + assert t.pp_owner == 0 and t.live_engines == {9: 3} + t.finish_vp(3) + t.drain(10) + assert not t.live_engines and t.pp_owner == 0 + for idx in range(4): + assert t.pp_owner == idx + t.busy = blocked + t.finish_pp(idx) + t.poll(10) + if blocked: + assert t.result(idx) == [1, 0, 0] + t.busy = False + t.drain(10) + assert t.result(idx) == [2, 0, 0x10 | idx] + assert pp_tokens(t) == [0x54530010 | idx for idx in range(4)] + assert [word for word in t.notifications if word >> 16 == 0x5454] == [0x54540012, 0x54540011, 0x54540010, 0x54540013] + assert t.u32(0x10007ff8) == t.u32(0x10007ffc) == wp + assert not t.fetches and not t.ring_writes + print(f'PASS: AV1 three-engine dispatch, fourth waits for VP9, reverse VP DONE, ordered PP, blocked mailbox={blocked}') + +# A routed error on each advertised AV1 engine must fault every owner; restart +# from reset then proves the next set of production descriptors can run again. +for failed_vp in (9, 10, 11): + t = AV1(multi=True) + wp = 0 + for idx in range(3): + wp = t.submit_host(idx, wp) + t.drain(20) + assert len(t.live_engines) == 3 + t.live_engines.clear() + t.route(79 + failed_vp * 5) # engine-specific error, not DONE + t.drain(30) + assert t.pp_owner is None + assert all(t.result(idx) == [2, 2, 0x10 | idx] for idx in range(3)) + assert t.notifications == [0x54530010 | idx for idx in range(3)] + # A fresh CM3 reset boots all three engines after the halt. + restarted = AV1(multi=True) + wp = 0 + for idx in range(3): + wp = restarted.submit_host(idx, wp) + restarted.drain(20) + assert restarted.live_engines == {9: 0, 10: 1, 11: 2} + print(f'PASS: AV1 VP{failed_vp} routed error drains all owners; reset restores all engines') diff --git a/drivers/media/platform/apple/isp/isp-cam-test.c b/drivers/media/platform/apple/isp/isp-cam-test.c index b97fa7b06c0b4..191d8ada0d265 100644 --- a/drivers/media/platform/apple/isp/isp-cam-test.c +++ b/drivers/media/platform/apple/isp/isp-cam-test.c @@ -71,6 +71,7 @@ struct isp_test_command_trace { unsigned int fail_at; int setfile_error; unsigned int setfile_calls; + u32 last_opcode; }; static int isp_test_send(struct apple_isp *isp, void *args, u32 insize, @@ -190,6 +191,19 @@ static void isp_test_setfile_failure_stops_setup(struct kunit *test) } } +/* Count NV12 setup commands and fail the chosen one, without a fixture. */ +static int isp_test_count_send(struct apple_isp *isp, void *args, u32 insize, + u32 outsize, int timeout) +{ + struct kunit *test = kunit_get_current_test(); + struct isp_test_command_trace *trace = test->priv; + + trace->last_opcode = ((const u32 *)args)[1]; + if (++trace->count == trace->fail_at) + return -ETIME; + return 0; +} + static void isp_test_optional_setfile_continues_setup(struct kunit *test) { static const int results[] = { 0, -ENOENT }; @@ -198,18 +212,170 @@ static void isp_test_optional_setfile_continues_setup(struct kunit *test) isp_get_current_format(isp)->pixelformat = V4L2_PIX_FMT_NV12; kunit_activate_static_stub(test, isp_ch_load_setfile, isp_test_load_setfile); + kunit_activate_static_stub(test, isp_cmd_send, isp_test_count_send); test->priv = &trace; for (unsigned int i = 0; i < ARRAY_SIZE(results); i++) { trace = (struct isp_test_command_trace){ .setfile_error = results[i], .fail_at = 2, }; - /* The SBS command after tuning, rather than tuning, fails. */ + /* The mains frequency after tuning, rather than tuning, fails. */ KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), -ETIME); KUNIT_EXPECT_EQ(test, trace.count, 2); + KUNIT_EXPECT_EQ(test, trace.last_opcode, 0x8212); KUNIT_EXPECT_EQ(test, trace.setfile_calls, 1); } } +struct isp_test_nv12_trace { + unsigned int count; + u32 opcode[64]; + u32 size[64]; + u32 word[64][3]; +}; + +/* Accept every NV12 setup command and record its opcode and first fields. */ +static int isp_test_record(struct apple_isp *isp, void *args, u32 insize, + u32 outsize, int timeout) +{ + struct kunit *test = kunit_get_current_test(); + struct isp_test_nv12_trace *trace = test->priv; + const u32 *words = args; + unsigned int i = trace->count; + + if (i >= ARRAY_SIZE(trace->opcode)) { + KUNIT_FAIL(test, "unexpected extra command"); + return -EIO; + } + trace->opcode[i] = words[1]; + trace->size[i] = insize; + for (unsigned int j = 0; j < 3 && 12 + 4 * j < insize; j++) + trace->word[i][j] = words[3 + j]; + trace->count++; + return 0; +} + +static int isp_test_find(const struct isp_test_nv12_trace *trace, u32 opcode) +{ + for (unsigned int i = 0; i < trace->count; i++) + if (trace->opcode[i] == opcode) + return i; + return -1; +} + +static int isp_test_no_setfile(struct apple_isp *isp, u32 ch) +{ + return 0; +} + +static struct apple_isp *isp_test_nv12_alloc(struct kunit *test, + enum isp_sensor_id id) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_format *fmt = isp_get_current_format(isp); + + fmt->pixelformat = V4L2_PIX_FMT_NV12; + fmt->id = id; + fmt->version = isp_setfiles[id].version; + isp->temporal_filter = 1; + kunit_activate_static_stub(test, isp_ch_load_setfile, isp_test_no_setfile); + kunit_activate_static_stub(test, isp_cmd_send, isp_test_record); + return isp; +} + +/* Expect exactly @n commands, in this order, from the first of them on. */ +static void isp_test_expect_run(struct kunit *test, + const struct isp_test_nv12_trace *trace, + const u32 (*run)[2], unsigned int n) +{ + int first = isp_test_find(trace, run[0][0]); + + KUNIT_ASSERT_GE(test, first, 0); + KUNIT_ASSERT_LE(test, first + n, trace->count); + for (unsigned int i = 0; i < n; i++) { + KUNIT_EXPECT_EQ(test, trace->opcode[first + i], run[i][0]); + KUNIT_EXPECT_EQ(test, trace->size[first + i], run[i][1]); + } +} + +static void isp_test_temporal_filter_band0(struct kunit *test) +{ + static const u32 run[][2] = { + { 0xc100, 0x18 }, { 0xc120, 0x10 }, { 0xc11c, 0x10 }, + { 0xc11d, 0x14 }, { 0xc102, 0x0c }, { 0xc113, 0x0c }, + }; + struct apple_isp *isp = isp_test_nv12_alloc(test, ISP_IMX958_2224_01); + struct isp_test_nv12_trace *trace; + int i; + + trace = kunit_kzalloc(test, sizeof(*trace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, trace); + test->priv = trace; + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), 0); + isp_test_expect_run(test, trace, run, ARRAY_SIZE(run)); + /* The start record is (1, 0, 0) whatever the DT property says. */ + i = isp_test_find(trace, 0xc100); + KUNIT_EXPECT_EQ(test, trace->word[i][0], 1); + KUNIT_EXPECT_EQ(test, trace->word[i][1], 0); + KUNIT_EXPECT_EQ(test, trace->word[i][2], 0); + i = isp_test_find(trace, 0xc11d); + KUNIT_EXPECT_EQ(test, trace->word[i][0], 0x7800); + KUNIT_EXPECT_EQ(test, trace->word[i][1], 0x6400); + KUNIT_EXPECT_LT(test, isp_test_find(trace, 0xc114), 0); +} + +static void isp_test_temporal_filter_off(struct kunit *test) +{ + static const u32 run[][2] = { + { 0xc114, 0x0c }, { 0xc103, 0x0c }, { 0xc101, 0x0c }, + }; + struct apple_isp *isp = isp_test_nv12_alloc(test, ISP_IMX558_1925_03); + struct isp_test_nv12_trace *trace; + + trace = kunit_kzalloc(test, sizeof(*trace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, trace); + test->priv = trace; + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), 0); + isp_test_expect_run(test, trace, run, ARRAY_SIZE(run)); + KUNIT_EXPECT_LT(test, isp_test_find(trace, 0xc100), 0); + KUNIT_EXPECT_LT(test, isp_test_find(trace, 0xc102), 0); + KUNIT_EXPECT_LT(test, isp_test_find(trace, 0xc113), 0); +} + +static void isp_test_power_line_frequency(struct kunit *test) +{ + struct apple_isp *isp = isp_test_nv12_alloc(test, ISP_IMX958_2224_01); + struct isp_test_nv12_trace *trace; + int i; + + trace = kunit_kzalloc(test, sizeof(*trace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, trace); + test->priv = trace; + + /* No control: 50 Hz, in Hz 16.16 */ + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), 0); + i = isp_test_find(trace, 0x8212); + KUNIT_ASSERT_GE(test, i, 0); + KUNIT_EXPECT_EQ(test, trace->size[i], 0x10); + KUNIT_EXPECT_EQ(test, trace->word[i][0], 0x00320000); + /* After the tuning, before the sensor configuration is selected */ + KUNIT_EXPECT_LT(test, i, isp_test_find(trace, 0x107)); + + isp->power_line_frequency = + v4l2_ctrl_new_std_menu(&isp->ctrl_handler, NULL, + V4L2_CID_POWER_LINE_FREQUENCY, + V4L2_CID_POWER_LINE_FREQUENCY_60HZ, + BIT(V4L2_CID_POWER_LINE_FREQUENCY_DISABLED), + V4L2_CID_POWER_LINE_FREQUENCY_50HZ); + KUNIT_ASSERT_NOT_NULL(test, isp->power_line_frequency); + KUNIT_ASSERT_EQ(test, v4l2_ctrl_s_ctrl(isp->power_line_frequency, + V4L2_CID_POWER_LINE_FREQUENCY_60HZ), 0); + memset(trace, 0, sizeof(*trace)); + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), 0); + i = isp_test_find(trace, 0x8212); + KUNIT_ASSERT_GE(test, i, 0); + KUNIT_EXPECT_EQ(test, trace->word[i][0], 0x003c0000); +} + static int isp_test_config_reply(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) { @@ -500,6 +666,9 @@ static struct kunit_case isp_cam_test_cases[] = { KUNIT_CASE(isp_test_sensor_config_failure), KUNIT_CASE(isp_test_preview_stream), KUNIT_CASE(isp_test_preview_command_failures), + KUNIT_CASE(isp_test_temporal_filter_band0), + KUNIT_CASE(isp_test_temporal_filter_off), + KUNIT_CASE(isp_test_power_line_frequency), {} }; diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 67691cc8844ac..004a7fa13d019 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -13,6 +13,18 @@ #include "isp-ipc.h" #include "isp-regs.h" +/* + * How the ISP's own temporal noise filter is set up for a sensor. macOS + * picks this per sensor on the H17 interface: the IMX558 (J700) runs + * without it and the IMX958 (J616s) runs it with explicit strength + * controls. Everything else keeps the original start sequence. + */ +enum isp_temporal_filter { + ISP_TEMPORAL_FILTER_START, + ISP_TEMPORAL_FILTER_OFF, + ISP_TEMPORAL_FILTER_BAND0, +}; + struct isp_setfile { u32 version; u32 magic; @@ -20,6 +32,7 @@ struct isp_setfile { size_t size; /* bytes present in the file if fewer than size; the rest is zero */ size_t file_size; + enum isp_temporal_filter temporal_filter; }; // clang-format off @@ -49,7 +62,8 @@ static const struct isp_setfile isp_setfiles[] = { [ISP_IMX514_2820_04] = {0x514, 0x28200405, "apple/isp_2820_04XX.dat", 0xa198}, [ISP_IMX558_1921_01] = {0x558, 0x19210106, "apple/isp_1921_01XX.dat", 0xad40}, [ISP_IMX558_1922_02] = {0x558, 0x19220201, "apple/isp_1922_02XX.dat", 0xad40}, - [ISP_IMX558_1925_03] = {0x558, 0x19250306, "apple/isp_1925_03XX.dat", 0xcbc0, 0xcba6}, + [ISP_IMX558_1925_03] = {0x558, 0x19250306, "apple/isp_1925_03XX.dat", 0xcbc0, 0xcba6, + ISP_TEMPORAL_FILTER_OFF}, [ISP_IMX603_7920_01] = {0x603, 0x79200109, "apple/isp_7920_01XX.dat", 0xad2c}, [ISP_IMX603_7920_02] = {0x603, 0x79200205, "apple/isp_7920_02XX.dat", 0xad2c}, [ISP_IMX603_7921_01] = {0x603, 0x79210104, "apple/isp_7921_01XX.dat", 0xad90}, @@ -71,7 +85,8 @@ static const struct isp_setfile isp_setfiles[] = { [ISP_IMX814_2123_01] = {0x814, 0x21230101, "apple/isp_2123_01XX.dat", 0xed54}, [ISP_IMX853_7622_01] = {0x853, 0x76220112, "apple/isp_7622_01XX.dat", 0x247f8}, [ISP_IMX913_7523_01] = {0x913, 0x75230107, "apple/isp_7523_01XX.dat", 0x247f8}, - [ISP_IMX958_2224_01] = {0x958, 0x22240106, "apple/isp_2224_01XX.dat", 0x6ff4}, + [ISP_IMX958_2224_01] = {0x958, 0x22240106, "apple/isp_2224_01XX.dat", 0x6ff4, 0, + ISP_TEMPORAL_FILTER_BAND0}, [ISP_VD56G0_6221_01] = {0xd56, 0x62210102, "apple/isp_6221_01XX.dat", 0x1b80}, [ISP_VD56G0_6222_01] = {0xd56, 0x62220102, "apple/isp_6222_01XX.dat", 0x1b80}, }; @@ -689,6 +704,80 @@ static int isp_ch_configure_frame_rate(struct apple_isp *isp, u32 ch) return isp_cmd_ch_ae_frame_rate_min_set(isp, ch, min_rate); } +/* + * Tell auto exposure the mains frequency, so that it picks exposure times + * that do not beat with lights on that supply. The firmware takes it in Hz, + * 16.16 fixed point. macOS sends it on the H17 interface for every camera. + */ +static int isp_ch_configure_power_line(struct apple_isp *isp, u32 ch) +{ + u32 hz = 50; + + if (isp->hw->fw_abi != ISP_FW_ABI_H17) + return 0; + + if (isp->power_line_frequency && + v4l2_ctrl_g_ctrl(isp->power_line_frequency) == + V4L2_CID_POWER_LINE_FREQUENCY_60HZ) + hz = 60; + + return isp_cmd_apple_ch_ae_flicker_freq_update_current_set(isp, ch, + hz << 16); +} + +static int isp_ch_configure_temporal_filter(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + int err; + + switch (isp_setfiles[fmt->id].temporal_filter) { + case ISP_TEMPORAL_FILTER_OFF: + /* The firmware keeps its state between streams; stop it. */ + err = isp_cmd_apple_ch_temporal_filter_disable(isp, ch); + if (err) + return err; + + err = isp_cmd_apple_ch_motion_history_stop(isp, ch); + if (err) + return err; + + return isp_cmd_apple_ch_temporal_filter_stop(isp, ch); + + case ISP_TEMPORAL_FILTER_BAND0: + err = isp_cmd_apple_ch_tnr_start_h17(isp, ch); + if (err) + return err; + + err = isp_cmd_apple_ch_tnr_high_strength_set(isp, ch, 0); + if (err) + return err; + + err = isp_cmd_apple_ch_tnr_band0_mod_set(isp, ch, 0); + if (err) + return err; + + /* 120 and 100, in 1/256 steps */ + err = isp_cmd_apple_ch_tnr_band0_threshold_set(isp, ch, 120 << 8, + 100 << 8); + if (err) + return err; + break; + + default: + err = isp_cmd_apple_ch_temporal_filter_start(isp, ch, + isp->temporal_filter); + if (err) + return err; + break; + } + + err = isp_cmd_apple_ch_motion_history_start(isp, ch); + if (err) + return err; + + return isp_cmd_apple_ch_temporal_filter_enable(isp, ch); +} + static int isp_ch_configure_pools(struct apple_isp *isp, u32 ch) { struct isp_format *fmt = isp_get_format(isp, ch); @@ -747,6 +836,10 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) return err; } + err = isp_ch_configure_power_line(isp, ch); + if (err) + return err; + select_config: err = isp_cmd_ch_sbs_enable(isp, ch, 1); if (err) @@ -845,15 +938,7 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) return err; } - err = isp_cmd_apple_ch_temporal_filter_start(isp, ch, isp->temporal_filter); - if (err) - return err; - - err = isp_cmd_apple_ch_motion_history_start(isp, ch); - if (err) - return err; - - err = isp_cmd_apple_ch_temporal_filter_enable(isp, ch); + err = isp_ch_configure_temporal_filter(isp, ch); if (err) return err; diff --git a/drivers/media/platform/apple/isp/isp-cmd.c b/drivers/media/platform/apple/isp/isp-cmd.c index e1670aea793b3..75b8704ccedcc 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.c +++ b/drivers/media/platform/apple/isp/isp-cmd.c @@ -607,6 +607,49 @@ int isp_cmd_apple_ch_temporal_filter_disable(struct apple_isp *isp, u32 chan) return CISP_SEND_IN(isp, args); } +int isp_cmd_apple_ch_tnr_start_h17(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_temporal_filter_start_h17 args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_START), + .chan = chan, + .unk_c = 1, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_tnr_high_strength_set(struct apple_isp *isp, u32 chan, + u32 enable) +{ + struct cmd_apple_ch_temporal_filter_enable_set args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_HIGH_STRENGTH), + .chan = chan, + .enable = enable, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_tnr_band0_mod_set(struct apple_isp *isp, u32 chan, + u32 enable) +{ + struct cmd_apple_ch_temporal_filter_enable_set args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_BAND0_MODULATION), + .chan = chan, + .enable = enable, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_tnr_band0_threshold_set(struct apple_isp *isp, u32 chan, + u32 first, u32 second) +{ + struct cmd_apple_ch_temporal_filter_band0_threshold args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_BAND0_THRESHOLD), + .chan = chan, + .threshold = { first, second }, + }; + return CISP_SEND_IN(isp, args); +} + int isp_cmd_ch_ae_stability_set(struct apple_isp *isp, u32 chan, u32 stability) { struct cmd_ch_ae_stability_set args = { diff --git a/drivers/media/platform/apple/isp/isp-cmd.h b/drivers/media/platform/apple/isp/isp-cmd.h index a3167108b128e..9e2678bd1610c 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.h +++ b/drivers/media/platform/apple/isp/isp-cmd.h @@ -97,6 +97,9 @@ #define CISP_CMD_APPLE_CH_MOTION_HISTORY_STOP 0xc103 #define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_ENABLE 0xc113 #define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_DISABLE 0xc114 +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_BAND0_MODULATION 0xc11c +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_BAND0_THRESHOLD 0xc11d +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_HIGH_STRENGTH 0xc120 #define CISP_POOL_TYPE_META 0x0 #define CISP_POOL_TYPE_RENDERED 0x1 @@ -643,6 +646,32 @@ struct cmd_apple_ch_temporal_filter_start { } __packed; static_assert(sizeof(struct cmd_apple_ch_temporal_filter_start) == 0x14); +/* The H17 firmware's start record carries one more word, sent as zero. */ +struct cmd_apple_ch_temporal_filter_start_h17 { + u64 opcode; + u32 chan; + u32 unk_c; + u32 unk_10; + u32 unk_14; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_start_h17) == 0x18); + +/* High-strength filtering (0xc120) and band 0 modulation (0xc11c) */ +struct cmd_apple_ch_temporal_filter_enable_set { + u64 opcode; + u32 chan; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_enable_set) == 0x10); + +/* Band 0 strength modulation thresholds, in 1/256 steps */ +struct cmd_apple_ch_temporal_filter_band0_threshold { + u64 opcode; + u32 chan; + u32 threshold[2]; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_band0_threshold) == 0x14); + struct cmd_apple_ch_temporal_filter_stop { u64 opcode; u32 chan; @@ -679,6 +708,13 @@ int isp_cmd_apple_ch_motion_history_start(struct apple_isp *isp, u32 chan); int isp_cmd_apple_ch_motion_history_stop(struct apple_isp *isp, u32 chan); int isp_cmd_apple_ch_temporal_filter_enable(struct apple_isp *isp, u32 chan); int isp_cmd_apple_ch_temporal_filter_disable(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_tnr_start_h17(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_tnr_high_strength_set(struct apple_isp *isp, u32 chan, + u32 enable); +int isp_cmd_apple_ch_tnr_band0_mod_set(struct apple_isp *isp, u32 chan, + u32 enable); +int isp_cmd_apple_ch_tnr_band0_threshold_set(struct apple_isp *isp, u32 chan, + u32 first, u32 second); struct cmd_ch_ae_stability_set { u64 opcode; diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 956754f5ef87f..30425f6cf483d 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -305,6 +305,7 @@ struct apple_isp { struct v4l2_device v4l2_dev; struct v4l2_ctrl_handler ctrl_handler; struct v4l2_ctrl *exposure_priority; + struct v4l2_ctrl *power_line_frequency; /* read-only exposure report of the last frame, NULL without one */ struct v4l2_ctrl *gain; struct v4l2_ctrl *exposure; diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 3c58b56b42aa3..24506ed526a34 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -592,12 +592,14 @@ static int isp_vb2_start_streaming(struct vb2_queue *q, unsigned int count) isp->sequence = 0; - /* The exposure priority is applied when the stream starts. */ + /* The exposure controls are applied when the stream starts. */ v4l2_ctrl_grab(isp->exposure_priority, true); + v4l2_ctrl_grab(isp->power_line_frequency, true); err = apple_isp_start_streaming(isp); if (err) { v4l2_ctrl_grab(isp->exposure_priority, false); + v4l2_ctrl_grab(isp->power_line_frequency, false); isp_vb2_release_buffers(isp, VB2_BUF_STATE_QUEUED); } @@ -611,6 +613,7 @@ static void isp_vb2_stop_streaming(struct vb2_queue *q) apple_isp_stop_streaming(isp); isp_vb2_release_buffers(isp, VB2_BUF_STATE_ERROR); v4l2_ctrl_grab(isp->exposure_priority, false); + v4l2_ctrl_grab(isp->power_line_frequency, false); } int apple_isp_video_suspend(struct apple_isp *isp) @@ -1218,10 +1221,22 @@ int apple_isp_setup_video(struct apple_isp *isp) * Auto exposure is always on. By default it holds the frame rate, as * V4L2 specifies; with exposure priority it may slow down in low light. */ - v4l2_ctrl_handler_init(&isp->ctrl_handler, 3); + v4l2_ctrl_handler_init(&isp->ctrl_handler, 4); isp->exposure_priority = v4l2_ctrl_new_std(&isp->ctrl_handler, NULL, V4L2_CID_EXPOSURE_AUTO_PRIORITY, 0, 1, 1, 0); + /* + * The H17 firmware's auto exposure avoids mains flicker at the given + * frequency. 50 Hz is the default, as on macOS when it does not know + * the country; userspace sets 60 Hz where the mains runs at 60 Hz. + */ + if (isp->hw->fw_abi == ISP_FW_ABI_H17) + isp->power_line_frequency = + v4l2_ctrl_new_std_menu(&isp->ctrl_handler, NULL, + V4L2_CID_POWER_LINE_FREQUENCY, + V4L2_CID_POWER_LINE_FREQUENCY_60HZ, + BIT(V4L2_CID_POWER_LINE_FREQUENCY_DISABLED), + V4L2_CID_POWER_LINE_FREQUENCY_50HZ); if (isp->hw->meta_exposure) isp_init_exposure_ctrls(isp); if (isp->ctrl_handler.error) { diff --git a/drivers/media/usb/uvc/uvc_ctrl.c b/drivers/media/usb/uvc/uvc_ctrl.c index 3ca108b83f1da..2bac164e2a30e 100644 --- a/drivers/media/usb/uvc/uvc_ctrl.c +++ b/drivers/media/usb/uvc/uvc_ctrl.c @@ -3126,6 +3126,16 @@ int uvc_xu_ctrl_query(struct uvc_video_chain *chain, goto done; } + /* + * A control whose GET_LEN is 0 has no value to get or set. Querying + * it would issue a zero-length IN transfer and leave uvc_query_ctrl() + * with a ZERO_SIZE_PTR buffer. + */ + if (size == 0) { + ret = -ENODATA; + goto done; + } + if (size != xqry->size) { ret = -ENOBUFS; goto done; diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c index fc3536a4399f4..e1cdbb6183327 100644 --- a/drivers/media/usb/uvc/uvc_video.c +++ b/drivers/media/usb/uvc/uvc_video.c @@ -108,6 +108,10 @@ int uvc_query_ctrl(struct uvc_device *dev, u8 query, u8 unit, return ret < 0 ? ret : -EPIPE; } + /* No buffer to borrow for the error code request. */ + if (size == 0) + return -EPIPE; + /* Reuse data[0] to request the error code. */ tmp = *(u8 *)data; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c index 03efae36a0b2d..1d8a85c6d2443 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c @@ -121,13 +121,19 @@ int brcmf_of_probe(struct device *dev, enum brcmf_bus_type bus_type, brcmf_dbg(INFO, "%s LPO clock\n", clk ? "enable" : "no"); - if (!np || !of_device_is_compatible(np, "brcm,bcm4329-fmac")) + if (!np) return 0; + /* The country code map applies to every bus, not only SDIO: Apple + * ARM64 platforms describe their PCIe chips with "pci14e4,..." only. + */ err = brcmf_of_get_country_codes(dev, settings); if (err) brcmf_err("failed to get OF country code map (err=%d)\n", err); + if (!of_device_is_compatible(np, "brcm,bcm4329-fmac")) + return 0; + err = of_get_mac_address(np, settings->mac); if (err == -EPROBE_DEFER) return err; diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index f16a55a528b36..1e53abffb83f7 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -14,7 +14,8 @@ config MT7932_FULLMAC WPA2-PSK, WPA3-SAE or OWE handshake. The driver is a cfg80211 station on 2.4 GHz, 5 GHz UNII-1/2/2e/3 and country-permitted 6 GHz UNII-5..8. 6 GHz requires SAE-H2E or OWE, MFP and country VLP power data. - Radar (DFS) channels are scanned passively but not joined. Other + Radar (DFS) channels are scanned passively; an AP heard there may be + joined, and is left when it announces a channel switch. Other MediaTek devices and other machines are not matched. The driver needs the original firmware, the unit's own calibration @@ -47,4 +48,5 @@ config MT7932_KUNIT_TEST without radio hardware: receive descriptors, including frames the hardware did not decrypt, command replies, regulatory snapshots and country power packages. Also check connection admission, SAE - request encoding and WPA passphrase derivation. + request encoding, WPA passphrase derivation and what is reported + when a move to another AP lands, fails or is cancelled. diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index 6c47a8d282753..99aa39372b49b 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -7,5 +7,6 @@ mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += input-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += regulatory-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += association-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += six-test.o +mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += move-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += pmf-lpi-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/association-test.c b/drivers/net/wireless/mediatek/mt7932/association-test.c index 81ef3f6a5470b..2d959e173624b 100644 --- a/drivers/net/wireless/mediatek/mt7932/association-test.c +++ b/drivers/net/wireless/mediatek/mt7932/association-test.c @@ -188,7 +188,7 @@ static void mt_connect_auth_test(struct kunit *test) } cases[] = { { "pure PSK", MT_AUTH_PSK, 0, WLAN_AKM_SUITE_PSK }, { "PSK missing PMK", MT_AUTH_PSK, 1, -EOPNOTSUPP }, - { "PSK required MFP", MT_AUTH_PSK, 2, -EOPNOTSUPP }, + { "PSK required MFP", MT_AUTH_PSK, 2, WLAN_AKM_SUITE_PSK }, { "pure SAE", MT_AUTH_NONE, 0, WLAN_AKM_SUITE_SAE }, { "optional MFP", MT_AUTH_MFP, NL80211_MFP_OPTIONAL, WLAN_AKM_SUITE_SAE }, { "no MFP", MT_AUTH_MFP, NL80211_MFP_NO, -EOPNOTSUPP }, @@ -221,7 +221,7 @@ static void mt_connect_auth_test(struct kunit *test) { "SAE first missing password", MT_AUTH_SAE_FIRST, 1, -EOPNOTSUPP }, { "PSK first optional MFP", MT_AUTH_PSK_FIRST, 0, WLAN_AKM_SUITE_PSK }, { "PSK first missing PMK", MT_AUTH_PSK_FIRST, 1, -EOPNOTSUPP }, - { "PSK first required MFP", MT_AUTH_PSK_FIRST, 2, -EOPNOTSUPP }, + { "PSK first required MFP", MT_AUTH_PSK_FIRST, 2, WLAN_AKM_SUITE_PSK }, { "PSK first no MFP", MT_AUTH_PSK_FIRST, 3, WLAN_AKM_SUITE_PSK }, { "PSK first mixed versions", MT_AUTH_PSK_FIRST, 4, WLAN_AKM_SUITE_PSK }, { "PSK first VERSION_3 only", MT_AUTH_PSK_FIRST, 5, -EOPNOTSUPP }, diff --git a/drivers/net/wireless/mediatek/mt7932/association.h b/drivers/net/wireless/mediatek/mt7932/association.h index 011a029f20357..26c3dfbcf7e92 100644 --- a/drivers/net/wireless/mediatek/mt7932/association.h +++ b/drivers/net/wireless/mediatek/mt7932/association.h @@ -52,8 +52,12 @@ mt7932_connect_auth(const struct cfg80211_connect_params *p) switch (p->crypto.akm_suites[0]) { case WLAN_AKM_SUITE_PSK: - if ((versions & NL80211_WPA_VERSION_2) && p->crypto.psk && - (p->mfp == NL80211_MFP_NO || p->mfp == NL80211_MFP_OPTIONAL)) + /* iwd and wpa_supplicant both send REQUIRED once they have + * negotiated PMF with an MFPC AP; wpa_supplicant sends OPTIONAL + * when either outcome will do. Admission against the AP's RSN + * capabilities decides. + */ + if ((versions & NL80211_WPA_VERSION_2) && p->crypto.psk) return WLAN_AKM_SUITE_PSK; break; case WLAN_AKM_SUITE_PSK_SHA256: @@ -166,13 +170,18 @@ static inline int mt7932_rsn_admit(const u8 *rsn, size_t length, u32 akm, return -EOPNOTSUPP; if ((caps & BIT(6)) && !(caps & BIT(7))) return -EINVAL; - /* Legacy PSK has no established PMF firmware selection. Optional may - * join it without PMF; required must never downgrade to that arm. + /* On the AKM 2 arm the firmware sets MFPC itself and negotiates PMF + * with any AP that sets MFPC, so the AP's capability is the outcome. + * A request for PMF needs an MFPC AP, and refusing PMF cannot join + * an AP that sets MFPR. */ - if (akm == WLAN_AKM_SUITE_PSK) - return mfp == NL80211_MFP_REQUIRED || (caps & BIT(6)) ? - -EOPNOTSUPP : - 0; + if (akm == WLAN_AKM_SUITE_PSK) { + if ((mfp == NL80211_MFP_REQUIRED && !(caps & BIT(7))) || + (mfp == NL80211_MFP_NO && (caps & BIT(6)))) + return -EOPNOTSUPP; + *pmf = !!(caps & BIT(7)); + return 0; + } if (mfp == NL80211_MFP_NO || !(caps & BIT(7))) return -EOPNOTSUPP; *pmf = true; diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 9e19d1a492e3b..3a3ab2ac47973 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -141,6 +141,25 @@ static void mt_scan_retire_wait(struct mt7932 *m, unsigned long timeout) } } +/* No lock held. A supplicant cancels its scan just before it connects, and + * the firmware ends the cancelled batch a moment later. Wait for that end, + * bounded, instead of refusing the connect: wpa_supplicant answers a refused + * connect by skipping the BSS for 10 s. A scan that is still running belongs + * to its requester and is left alone. + */ +void mt_scan_settle_retired(struct mt7932 *m) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(2000); + long left; + + while (READ_ONCE(m->retired_scan_seq) && !READ_ONCE(m->scan_request) && + !READ_ONCE(m->stopping)) { + left = (long)(deadline - jiffies); + if (left <= 0 || !wait_for_completion_timeout(&m->scan_done, left)) + break; + } +} + /* No command/response lock held. The caller closes new scan admission first. * Natural EID0d completion fences the scan; a local cancel alone does not. * The batch awaited here may already have been followed by the next one, @@ -182,8 +201,8 @@ static int mt_scan_submit_next(struct mt7932 *m) bool listen = mt7932_scan_listen(request, channel, m->six_legacy_channels); - /* Radar channels are never joined; while associated, - * a passive dwell there only delays the home channel. + /* While associated, a passive dwell on a radar channel + * only delays the home channel. */ if (mt7932_channel_band(channel->band, channel->hw_value) != m->scan_band || listen != passive || (radar && m->scan_home_channel)) @@ -268,8 +287,12 @@ int mt_connect_discover(struct mt7932 *m) if (m->connect_channel_req && channel != m->connect_channel_req) continue; - if (channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_20MHZ | - IEEE80211_CHAN_RADAR)) + if (channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_20MHZ)) + continue; + /* Probes are forbidden on radar channels: leave them to the + * passive attempts, so the rest of the band keeps probing. + */ + if (!passive && (channel->flags & IEEE80211_CHAN_RADAR)) continue; passive |= !!(channel->flags & IEEE80211_CHAN_NO_IR); passive |= band == 3 && @@ -667,6 +690,24 @@ int mt_enable_scan(struct mt7932 *m) return 0; } +/* A station on a radar channel must stop transmitting when its AP announces a + * switch away; the firmware's CSA handling is unknown. The beacon's IEs are + * already bounds-checked by mt7932_rx_beacon(). + */ +static bool mt_radar_switch_announced(const struct ieee80211_channel *channel, + const struct mt7932_rx_frame *frame) +{ + const u8 *ies = frame->data + 36; + size_t length = frame->length - 36; + const struct element *csa, *ecsa; + + if (!(channel->flags & IEEE80211_CHAN_RADAR)) + return false; + csa = cfg80211_find_elem(WLAN_EID_CHANNEL_SWITCH, ies, length); + ecsa = cfg80211_find_elem(WLAN_EID_EXT_CHANSWITCH_ANN, ies, length); + return (csa && csa->datalen >= 3) || (ecsa && ecsa->datalen >= 4); +} + void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length) { struct mt7932_rx_frame frame; @@ -705,7 +746,18 @@ void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length) unsigned long flags; spin_lock_irqsave(&m->response_lock, flags); - if (m->connected && !m->disconnecting && !m->stopping && + if (m->connecting && !m->disconnecting && !m->stopping && !m->connect_error && + frame.channel == m->connect_channel && + ether_addr_equal(frame.data + 16, m->connect_bssid) && + mt_radar_switch_announced(channel, &frame)) { + /* The connect worker owns teardown: fail its attempt and wake it. */ + m->connect_error = -ECONNRESET; + complete(&m->assoc_start); + complete(&m->assoc_done); + complete(&m->discovery_done); + dev_info(&m->pdev->dev, "RADAR_CHANNEL_SWITCH: AP announced a switch; abandoning the join on channel %u\n", + m->connect_channel); + } else if (m->connected && !m->disconnecting && !m->stopping && frame.channel == m->connect_channel && ether_addr_equal(frame.data + 16, m->connect_bssid)) { m->home_beacons++; @@ -730,7 +782,20 @@ void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length) } } mt_cqm_signal(m, mt7932_rx_signal(&frame, m->antenna_mask)); - if (tim && tim->datalen >= 4 && !m->power_tim) { + if (m->connected && mt_radar_switch_announced(channel, &frame)) { + m->connected = false; + m->disconnecting = true; + m->disconnect_seen = false; + m->disconnect_event_received = false; + m->disconnect_local = true; + m->disconnect_reason = WLAN_REASON_DISASSOC_STA_HAS_LEFT; + netif_carrier_off(m->netdev); + netif_stop_queue(m->netdev); + reinit_completion(&m->disconnect_done); + schedule_work(&m->disconnect_work); + dev_info(&m->pdev->dev, "RADAR_CHANNEL_SWITCH: AP announced a switch; leaving channel %u\n", + m->connect_channel); + } else if (tim && tim->datalen >= 4 && !m->power_tim) { m->power_tim = true; schedule_work(&m->power_work); } diff --git a/drivers/net/wireless/mediatek/mt7932/lifecycle.c b/drivers/net/wireless/mediatek/mt7932/lifecycle.c index 9a05219172bde..e0ed45dccdae8 100644 --- a/drivers/net/wireless/mediatek/mt7932/lifecycle.c +++ b/drivers/net/wireless/mediatek/mt7932/lifecycle.c @@ -62,8 +62,7 @@ void mt_stop_host(struct mt7932 *m) } if (m->connected) { m->connected = false; - cfg80211_disconnected(m->netdev, WLAN_REASON_DEAUTH_LEAVING, - NULL, 0, true, GFP_KERNEL); + mt_report_disconnected(m, WLAN_REASON_DEAUTH_LEAVING, true, GFP_KERNEL); } memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); memzero_explicit(m->connect_sae_pwd, sizeof(m->connect_sae_pwd)); diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index c2b45bea58879..7824d6f6d9b10 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -28,7 +28,7 @@ static bool mt_channel_event(struct mt7932 *m, const u8 *body, size_t length) NL80211_CHAN_WIDTH_40 : NL80211_CHAN_WIDTH_20; def.center_freq1 = ieee80211_channel_to_frequency(center, m->connect_band); if (!def.chan || !cfg80211_chandef_valid(&def) || - !cfg80211_chandef_usable(m->wiphy, &def, IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_RADAR)) + !cfg80211_chandef_usable(m->wiphy, &def, IEEE80211_CHAN_DISABLED)) return false; if (m->connect_center && (m->connect_center != center || m->connect_chandef.width != def.width)) return false; @@ -37,6 +37,26 @@ static bool mt_channel_event(struct mt7932 *m, const u8 *body, size_t length) return true; } +/* iwd keeps its reassociation mark for the life of a connection made by one, + * and meanwhile drops any disconnect that isn't from the AP. Nothing else would + * tell it the connection has gone, so report these as from the AP. iwd ignores + * the report of a disconnect it asked for either way. + * + * This works around iwd, not cfg80211. As of iwd 3.12, netdev_reassociate() + * sets in_reassoc and only netdev_associate_event() clears it, on + * NL80211_CMD_ASSOCIATE, which cfg80211 never sends for a driver that reports + * through cfg80211_connect_done(). Every nl80211 listener sees the changed + * flag, and wpa_supplicant treats a disconnect from the AP differently from a + * local one. Drop roam_landed once iwd clears in_reassoc on a successful + * connect event. + */ +void mt_report_disconnected(struct mt7932 *m, u16 reason, bool local, gfp_t gfp) +{ + cfg80211_disconnected(m->netdev, reason, NULL, 0, + mt7932_disconnect_local(local, READ_ONCE(m->roam_landed)), + gfp); +} + void mt_rf_fail_locked(struct mt7932 *m, int error) { bool first = !m->link_failed; @@ -77,15 +97,17 @@ void mt_rf_fail_locked(struct mt7932 *m, int error) } else if (m->connected) { /* Hardware ownership is uncertain: report failure, retain the peer. * A connect worker still publishing the connection reports this - * after it, so that cfg80211 sees the two in order. + * after it, so that cfg80211 sees the two in order. A move that + * mt_connect() has just started leaves it to that: its admission + * check sees link_failed and reports the end of the connection, + * after joining any such worker. */ m->connected = false; m->disconnecting = true; - if (m->connect_publishing) + if (m->connect_publishing && !m->roam_moving) m->connect_lost = true; - else - cfg80211_disconnected(m->netdev, WLAN_REASON_UNSPECIFIED, - NULL, 0, true, GFP_ATOMIC); + else if (!m->roam_moving) + mt_report_disconnected(m, WLAN_REASON_UNSPECIFIED, true, GFP_ATOMIC); } if (m->scan_request) { m->scan_aborted = true; @@ -322,7 +344,11 @@ static int mt_retire_connection(struct mt7932 *m) if (READ_ONCE(m->stopping)) return -ESHUTDOWN; - netif_carrier_off(m->netdev); + /* A move keeps the carrier, or NetworkManager would restart DHCP and the + * address could change under open connections. Its failure drops it. + */ + if (!READ_ONCE(m->roam_moving)) + netif_carrier_off(m->netdev); netif_tx_disable(m->netdev); /* ndo_start_xmit rechecks the closed admission gate under data_lock. * The queue may have stopped in the middle of a burst; hand the device @@ -455,8 +481,14 @@ static int mt_connect_prepare(struct mt7932 *m, bool *peer_owned) if (ret) return ret; } - if (m->connect_bss->channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_IR | - IEEE80211_CHAN_NO_20MHZ | IEEE80211_CHAN_RADAR)) { + /* A station joins an AP it has heard on a radar or no-IR channel: the AP + * owns radar detection. Leaving on its switch announcement is in RX. + * 6 GHz keeps its no-IR rule, which its power class depends on. + */ + if ((m->connect_bss->channel->flags & (IEEE80211_CHAN_DISABLED | + IEEE80211_CHAN_NO_20MHZ)) || + (m->connect_bss->channel->band == NL80211_BAND_6GHZ && + (m->connect_bss->channel->flags & IEEE80211_CHAN_NO_IR))) { return -EINVAL; } spin_lock_irqsave(&m->response_lock, flags); @@ -678,9 +710,10 @@ void mt_connect_work(struct work_struct *work) { struct mt7932 *m = container_of(work, struct mt7932, connect_work); struct cfg80211_connect_resp_params response = {}; + struct mt7932_connect_report outcome; unsigned long flags; bool submitted = false, peer_owned = false, cancelled, retry = false; - bool retirement_attempted = false, auth_submitted = false; + bool retirement_attempted = false, auth_submitted = false, roam; u16 reason; int ret; @@ -753,10 +786,15 @@ void mt_connect_work(struct work_struct *work) } /* Serialize the final outcome with cancellation. In particular, a late * firmware success must not publish carrier/authorization after .deauth. - * The cfg80211 calls may take bottom-half locks, so they are made after - * response_lock is dropped. command_mutex keeps the disconnect worker, - * which retires the connection before reporting its end, behind them; - * an RF failure in the meantime leaves its report to this worker. + * The connect result, the port authorization and the disconnect that + * follows a failed or cancelled move are reported after response_lock is + * dropped, since cfg80211_connect_done() may sleep. Only the connect_lost + * report below calls cfg80211_disconnected() under the lock, with + * GFP_ATOMIC, as the RF failure path, the disconnect worker and a move + * that mt_connect() refuses do; that call only queues an event. + * command_mutex keeps the disconnect worker, which retires the + * connection before reporting its end, behind them; an RF failure in + * the meantime leaves its report to this worker. */ spin_lock_irqsave(&m->response_lock, flags); if (!ret && (m->connect_error || !mt_rf_allowed(m))) { @@ -796,8 +834,16 @@ void mt_connect_work(struct work_struct *work) } m->connected = !ret; m->disconnecting = false; + roam = m->roam_moving; cancelled = m->connect_cancelled; reason = m->disconnect_reason; + outcome = mt7932_connect_report(roam, cancelled, ret, reason); + WRITE_ONCE(m->roam_moving, false); + WRITE_ONCE(m->roam_landed, outcome.landed); + if (roam && ret) { + netif_carrier_off(m->netdev); + mt_reg_apply_deferred_locked(m); + } m->connect_publishing = true; if (!ret) { schedule_work(&m->power_work); @@ -808,13 +854,12 @@ void mt_connect_work(struct work_struct *work) } spin_unlock_irqrestore(&m->response_lock, flags); - if (cancelled) { - if (response.links[0].bss) - cfg80211_put_bss(m->wiphy, response.links[0].bss); - cfg80211_disconnected(m->netdev, reason, NULL, 0, true, GFP_KERNEL); - } else { + if (outcome.result) cfg80211_connect_done(m->netdev, &response, GFP_KERNEL); - } + else if (response.links[0].bss) + cfg80211_put_bss(m->wiphy, response.links[0].bss); + if (outcome.disconnect) + mt_report_disconnected(m, outcome.reason, true, GFP_KERNEL); if (!ret && !m->connect_open) cfg80211_port_authorized(m->netdev, m->connect_bssid, NULL, 0, GFP_KERNEL); @@ -822,8 +867,7 @@ void mt_connect_work(struct work_struct *work) m->connect_publishing = false; if (m->connect_lost) { m->connect_lost = false; - cfg80211_disconnected(m->netdev, WLAN_REASON_UNSPECIFIED, - NULL, 0, true, GFP_ATOMIC); + mt_report_disconnected(m, WLAN_REASON_UNSPECIFIED, true, GFP_ATOMIC); } spin_unlock_irqrestore(&m->response_lock, flags); if (!ret && m->connect_open) @@ -871,29 +915,64 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, { struct mt7932 *m = mt_from_wiphy(wiphy); unsigned long flags; - bool open = mt_open_request(params); + bool open = mt_open_request(params), roam; int auth, ret = 0; if (netdev == m->netdev) mt_retry_missing_policy(m); - if (!mt_rf_allowed(m) || !m->bss_active || netdev != m->netdev) + if (!mt_rf_allowed(m) || !m->bss_active || netdev != m->netdev) { + if (netdev == m->netdev && params->prev_bssid) + mt_disconnect(wiphy, netdev, WLAN_REASON_DEAUTH_LEAVING); return -EAGAIN; + } dev_dbg(&m->pdev->dev, "CONNECT_REQUEST: WPA=%u pairwise-count=%d group=%08x AKM-count=%d PMK-present=%u MFP=%u IE-bytes=%zu\n", params->crypto.wpa_versions, params->crypto.n_ciphers_pairwise, params->crypto.cipher_group, params->crypto.n_akm_suites, !!params->crypto.psk, params->mfp, params->ie_len); - if (!params->ssid_len || params->ssid_len > 32) - return -EINVAL; auth = open ? 0 : mt7932_connect_auth(params); - if (auth < 0) - return auth; - if (params->bssid && !is_valid_ether_addr(params->bssid)) - return -EINVAL; + if (!params->ssid_len || params->ssid_len > 32 || + (params->bssid && !is_valid_ether_addr(params->bssid))) + ret = -EINVAL; + else if (auth < 0) + ret = auth; + /* The driver can't reassociate. Honour iwd's roam by leaving the current + * AP and joining the requested one: the worker reports the end of this + * move through the connect result, so cfg80211 and iwd see one roam. A + * roam we can't start still ends the connection, or cfg80211 and iwd + * would disagree about it. + */ + spin_lock_irqsave(&m->response_lock, flags); + roam = params->prev_bssid && m->connected; + if (roam && !ret) + WRITE_ONCE(m->roam_moving, true); + /* The link went first and cfg80211 hasn't processed its end: joining now + * would be overtaken by that report, and iwd would miss the connection. + */ + if (params->prev_bssid && !m->connected && !ret) + ret = -EBUSY; + spin_unlock_irqrestore(&m->response_lock, flags); + /* Leaving the old AP waits for a scan in progress to finish + * (mt_scan_quiesce(), about 7 s at most). A scan that is still running + * after that makes the admission check below refuse the join, so the + * move ends as a disconnect and iwd connects again. + */ + if (roam) { + dev_info(&m->pdev->dev, "ROAM_MOVE: %d; from %pM to %pM\n", ret, + params->prev_bssid, params->bssid ?: params->bssid_hint); + mt_disconnect(wiphy, netdev, WLAN_REASON_DEAUTH_LEAVING); + } + if (ret) + return ret; + mt_scan_settle_retired(m); mutex_lock(&m->command_mutex); spin_lock_irqsave(&m->response_lock, flags); if (!mt_rf_allowed(m) || m->link_failed || m->connecting || m->connected || m->disconnecting || m->scan_request || m->retired_scan_seq) { ret = m->link_failed ? -EIO : -EBUSY; + dev_info_ratelimited(&m->pdev->dev, + "CONNECT_REFUSED: %d; rf=%u connecting=%u connected=%u disconnecting=%u scan=%u retired=%u\n", + ret, mt_rf_allowed(m), m->connecting, m->connected, m->disconnecting, + !!m->scan_request, !!m->retired_scan_seq); goto unlock; } reinit_completion(&m->assoc_start); @@ -944,6 +1023,18 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, m->peer_valid = false; m->connecting = true; unlock: + if (ret && roam) { + /* mt_disconnect() does nothing once the host is stopping, so the + * old connection may still be marked here. Clear it, or + * mt_stop_host() would report its end a second time. + */ + m->connected = false; + WRITE_ONCE(m->roam_moving, false); + netif_carrier_off(m->netdev); + cfg80211_disconnected(m->netdev, WLAN_REASON_DEAUTH_LEAVING, NULL, 0, true, + GFP_ATOMIC); + mt_reg_apply_deferred_locked(m); + } spin_unlock_irqrestore(&m->response_lock, flags); if (!ret) { m->connect_bss = mt_connect_find_bss(m); @@ -971,8 +1062,15 @@ void mt_disconnect_work(struct work_struct *work) */ spin_lock_irqsave(&m->response_lock, flags); m->disconnecting = false; - cfg80211_disconnected(m->netdev, reason, NULL, 0, local, GFP_ATOMIC); - mt_reg_apply_deferred_locked(m); + /* A roam reports its end through the connect that follows. A deferred + * regulatory pass would close RF admission under it, so the connect + * worker applies that once the move has failed, and a move that lands + * keeps it for the end of the new connection. + */ + if (!m->roam_moving) { + mt_report_disconnected(m, reason, local, GFP_ATOMIC); + mt_reg_apply_deferred_locked(m); + } spin_unlock_irqrestore(&m->response_lock, flags); dev_dbg(&m->pdev->dev, "DISCONNECTED: teardown=%d local=%u\n", ret, local); } @@ -1005,7 +1103,8 @@ int mt_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 reason) m->disconnect_event_received = false; m->disconnect_local = true; m->disconnect_reason = reason; - netif_carrier_off(m->netdev); + if (!m->roam_moving) + netif_carrier_off(m->netdev); netif_stop_queue(m->netdev); reinit_completion(&m->disconnect_done); schedule_work(&m->disconnect_work); diff --git a/drivers/net/wireless/mediatek/mt7932/move-test.c b/drivers/net/wireless/mediatek/mt7932/move-test.c new file mode 100644 index 0000000000000..64f0bf8b1dd3d --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/move-test.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +#include "mt7932.h" + +static void mt_move_report_expect(struct kunit *test, bool move, bool cancelled, + int ret, bool result, bool disconnect, + bool landed, u16 reason) +{ + struct mt7932_connect_report report = + mt7932_connect_report(move, cancelled, ret, + WLAN_REASON_DISASSOC_STA_HAS_LEFT); + + KUNIT_EXPECT_EQ(test, report.result, result); + KUNIT_EXPECT_EQ(test, report.disconnect, disconnect); + KUNIT_EXPECT_EQ(test, report.landed, landed); + if (disconnect) + KUNIT_EXPECT_EQ(test, report.reason, reason); +} + +/* An ordinary connect: one connect result, or only a disconnect if cancelled. */ +static void mt_connect_report_plain_test(struct kunit *test) +{ + mt_move_report_expect(test, false, false, 0, true, false, false, 0); + mt_move_report_expect(test, false, false, -ETIMEDOUT, true, false, false, 0); + mt_move_report_expect(test, false, false, -ECONNREFUSED, true, false, false, 0); + mt_move_report_expect(test, false, false, -EIO, true, false, false, 0); + mt_move_report_expect(test, false, true, -ECANCELED, false, true, false, + WLAN_REASON_DISASSOC_STA_HAS_LEFT); +} + +/* A move that lands takes over the connection through its connect result. + * One that fails or is cancelled reports the failed result, which is what + * iwd waits for, and then ends the connection it replaced. + */ +static void mt_connect_report_move_test(struct kunit *test) +{ + mt_move_report_expect(test, true, false, 0, true, false, true, 0); + mt_move_report_expect(test, true, false, -ETIMEDOUT, true, true, false, + WLAN_REASON_DEAUTH_LEAVING); + mt_move_report_expect(test, true, false, -ECONNRESET, true, true, false, + WLAN_REASON_DEAUTH_LEAVING); + mt_move_report_expect(test, true, false, -EIO, true, true, false, + WLAN_REASON_DEAUTH_LEAVING); + /* A regulatory pass cancels a join without marking it cancelled. */ + mt_move_report_expect(test, true, false, -ECANCELED, true, true, false, + WLAN_REASON_DEAUTH_LEAVING); + mt_move_report_expect(test, true, true, -ECANCELED, true, true, false, + WLAN_REASON_DISASSOC_STA_HAS_LEFT); +} + +/* Whatever the outcome, cfg80211 hears of it, at most one disconnect follows + * and only a successful move marks the new connection. + */ +static void mt_connect_report_complete_test(struct kunit *test) +{ + static const int results[] = { + 0, -ETIMEDOUT, -ECONNREFUSED, -ECONNRESET, -EPROTO, -ECANCELED, + -EIO, -ESHUTDOWN, + }; + struct mt7932_connect_report report; + unsigned int i, flags; + + for (i = 0; i < ARRAY_SIZE(results); i++) { + for (flags = 0; flags < 4; flags++) { + bool move = flags & 1, cancelled = flags & 2; + + /* A cancel always leaves an error. */ + if (cancelled && !results[i]) + continue; + report = mt7932_connect_report(move, cancelled, results[i], + WLAN_REASON_UNSPECIFIED); + KUNIT_EXPECT_TRUE(test, report.result || report.disconnect); + KUNIT_EXPECT_EQ(test, report.landed, move && !results[i]); + if (report.landed) + KUNIT_EXPECT_FALSE(test, report.disconnect); + /* A connect result for a move, whether it failed or not. */ + if (move) + KUNIT_EXPECT_TRUE(test, report.result); + /* Only a successful connect leaves a connection behind. */ + if (report.result && results[i]) + KUNIT_EXPECT_EQ(test, report.disconnect, move); + } + } +} + +/* A connection a move made reports a local end as from the AP. */ +static void mt_disconnect_local_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, mt7932_disconnect_local(true, false)); + KUNIT_EXPECT_FALSE(test, mt7932_disconnect_local(true, true)); + KUNIT_EXPECT_FALSE(test, mt7932_disconnect_local(false, false)); + KUNIT_EXPECT_FALSE(test, mt7932_disconnect_local(false, true)); +} + +static struct kunit_case mt_move_test_cases[] = { + KUNIT_CASE(mt_connect_report_plain_test), + KUNIT_CASE(mt_connect_report_move_test), + KUNIT_CASE(mt_connect_report_complete_test), + KUNIT_CASE(mt_disconnect_local_test), + {} +}; + +static struct kunit_suite mt_move_test_suite = { + .name = "mt7932-move", + .test_cases = mt_move_test_cases, +}; + +kunit_test_suite(mt_move_test_suite); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 28c339e76ed29..a0d5a307f6cba 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -176,6 +176,12 @@ struct mt7932 { * sets connect_lost for the worker to report. Under response_lock. */ bool connect_publishing, connect_lost; + /* A connect with the previous BSSID is moving the connection to another + * AP (roam_moving), or the current connection is the result of one + * (roam_landed). Written under response_lock with WRITE_ONCE(); read + * under it, or with READ_ONCE() where the lock is not held. + */ + bool roam_moving, roam_landed; u16 disconnect_reason; int connect_error; u32 event_cipher; @@ -223,6 +229,45 @@ struct mt7932 { dma_addr_t ipc_dma, aux_dma; }; +/* What the connect worker reports once an attempt has its result. A move, a + * connect with the previous BSSID while connected, ends in a connect result, + * which is what iwd's reassociation waits for; cfg80211_roamed() is never + * used. A failed connect result leaves cfg80211 holding the connection the + * move replaced, so a move that fails or is cancelled then reports a + * disconnect, for the reason the cancel gave if it was cancelled. A + * cancelled connect that was not a move reports only the disconnect. A + * cancel always leaves an error in ret. + */ +struct mt7932_connect_report { + bool result; /* cfg80211_connect_done() */ + bool disconnect; /* then a disconnect, for reason */ + bool landed; /* the new connection is the result of a move */ + u16 reason; +}; + +static inline struct mt7932_connect_report +mt7932_connect_report(bool move, bool cancelled, int ret, u16 cancel_reason) +{ + struct mt7932_connect_report report = { + .result = move || !cancelled, + .disconnect = move ? ret != 0 : cancelled, + .landed = move && !ret, + }; + + if (report.disconnect) + report.reason = cancelled ? cancel_reason : WLAN_REASON_DEAUTH_LEAVING; + return report; +} + +/* Whether a disconnect the driver ends is reported as local. See + * mt_report_disconnected() for why a connection a move made reports it as + * from the AP. + */ +static inline bool mt7932_disconnect_local(bool local, bool landed) +{ + return local && !landed; +} + bool mt_rf_allowed(struct mt7932 *m); void mt_retry_missing_policy(struct mt7932 *m); int mt_6g_connect_policy(struct mt7932 *m, const struct mt7932_6g_ap *ap); @@ -262,6 +307,7 @@ struct mt7932 *mt_from_wiphy(struct wiphy *wiphy); void mt_scan_event(struct mt7932 *m, const struct mt7932_event *event); void mt_abort_scan(struct wiphy *wiphy, struct wireless_dev *wdev); int mt_scan_quiesce(struct mt7932 *m); +void mt_scan_settle_retired(struct mt7932 *m); int mt_connect_discover(struct mt7932 *m); int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request); int mt_register_interface(struct mt7932 *m); @@ -273,6 +319,7 @@ void mt_connect_work(struct work_struct *work); int mt_connect(struct wiphy *wiphy, struct net_device *netdev, struct cfg80211_connect_params *params); void mt_disconnect_work(struct work_struct *work); +void mt_report_disconnected(struct mt7932 *m, u16 reason, bool local, gfp_t gfp); int mt_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 reason); int mt_deauth(struct wiphy *wiphy, struct net_device *netdev, struct cfg80211_deauth_request *request); diff --git a/drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c b/drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c index 927b0e43b3a9c..59c9633c954df 100644 --- a/drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c +++ b/drivers/net/wireless/mediatek/mt7932/pmf-lpi-test.c @@ -47,7 +47,16 @@ static void mt_pmf_admission_test(struct kunit *test) request.crypto.n_akm_suites = 2; request.crypto.akm_suites[0] = WLAN_AKM_SUITE_PSK; request.crypto.akm_suites[1] = WLAN_AKM_SUITE_PSK_SHA256; + /* AKM 2 takes every MFP request; the AP's RSN decides at admission. */ + for (i = NL80211_MFP_NO; i <= NL80211_MFP_OPTIONAL; i++) { + request.mfp = i; + KUNIT_EXPECT_EQ(test, mt7932_connect_auth(&request), + WLAN_AKM_SUITE_PSK); + } + request.mfp = NL80211_MFP_REQUIRED; + request.crypto.psk = NULL; KUNIT_EXPECT_LT(test, mt7932_connect_auth(&request), 0); + request.crypto.psk = pmk; KUNIT_EXPECT_LT(test, mt7932_connect_admit(pmf_rsn, sizeof(pmf_rsn), WLAN_AKM_SUITE_PSK_SHA256, @@ -92,17 +101,45 @@ static void mt_pmf_admission_test(struct kunit *test) mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK_SHA256, NL80211_MFP_REQUIRED, &pmf), 0); - ies[19] = 2; /* legacy PSK cannot satisfy required PMF */ + /* AKM 2 on an MFPC AP: firmware negotiates PMF for every request. */ + ies[19] = 2; + for (i = NL80211_MFP_NO; i <= NL80211_MFP_OPTIONAL; i++) { + pmf = false; + KUNIT_EXPECT_EQ(test, + mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK, + i, &pmf), + 0); + KUNIT_EXPECT_TRUE(test, pmf); + } + /* AKM 2 on an MFPR AP: PMF is mandatory, so it cannot be refused. */ + ies[20] |= 64; + for (i = NL80211_MFP_REQUIRED; i <= NL80211_MFP_OPTIONAL; i++) { + KUNIT_EXPECT_EQ(test, + mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK, + i, &pmf), + 0); + KUNIT_EXPECT_TRUE(test, pmf); + } KUNIT_EXPECT_LT(test, mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK, - NL80211_MFP_REQUIRED, &pmf), + NL80211_MFP_NO, &pmf), 0); - KUNIT_EXPECT_EQ(test, + /* AKM 2 on an AP without MFPC: no PMF, and required cannot join. */ + ies[20] = 0; + KUNIT_EXPECT_LT(test, mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK, - NL80211_MFP_OPTIONAL, &pmf), + NL80211_MFP_REQUIRED, &pmf), 0); - KUNIT_EXPECT_FALSE(test, pmf); - ies[20] |= 64; + for (i = NL80211_MFP_NO; i <= NL80211_MFP_OPTIONAL; + i += NL80211_MFP_OPTIONAL) { + pmf = true; + KUNIT_EXPECT_EQ(test, + mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK, + i, &pmf), + 0); + KUNIT_EXPECT_FALSE(test, pmf); + } + ies[20] = 64; /* MFPR without MFPC stays malformed */ KUNIT_EXPECT_LT(test, mt7932_connect_admit(ies, 22, WLAN_AKM_SUITE_PSK, NL80211_MFP_OPTIONAL, &pmf), diff --git a/drivers/net/wireless/mediatek/mt7932/rf-test.c b/drivers/net/wireless/mediatek/mt7932/rf-test.c index 3a6d9959b13a1..1d901861fb69f 100644 --- a/drivers/net/wireless/mediatek/mt7932/rf-test.c +++ b/drivers/net/wireless/mediatek/mt7932/rf-test.c @@ -340,6 +340,48 @@ static void mt_scan_retire_test_finish(struct work_struct *work) schedule_delayed_work(&ctx->retire, msecs_to_jiffies(300)); } +/* A connect right after a cancelled scan waits for the firmware to end it. */ +static void mt_scan_connect_settle_test(struct kunit *test) +{ + struct mt_scan_retire_test_context ctx = {}; + struct cfg80211_scan_request *scan; + struct mt7932 *m = test->priv; + unsigned long start; + + scan = kunit_kzalloc(test, sizeof(*scan), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, scan); + init_completion(&m->scan_done); + INIT_DELAYED_WORK(&ctx.retire, mt_scan_retire_test_late); + ctx.m = m; + + /* The cancelled batch ends 300 ms later: the connect goes ahead then. */ + m->scan_request = NULL; + m->retired_scan_seq = 8; + schedule_delayed_work(&ctx.retire, msecs_to_jiffies(300)); + start = jiffies; + mt_scan_settle_retired(m); + KUNIT_EXPECT_EQ(test, m->retired_scan_seq, 0); + KUNIT_EXPECT_LT(test, jiffies - start, msecs_to_jiffies(1500)); + cancel_delayed_work_sync(&ctx.retire); + + /* A batch the firmware never ends is waited for only up to the bound. */ + reinit_completion(&m->scan_done); + m->retired_scan_seq = 8; + start = jiffies; + mt_scan_settle_retired(m); + KUNIT_EXPECT_EQ(test, m->retired_scan_seq, 8); + KUNIT_EXPECT_GE(test, jiffies - start, msecs_to_jiffies(1900)); + KUNIT_EXPECT_LE(test, jiffies - start, msecs_to_jiffies(3000)); + + /* A running scan is not waited for: the connect refuses it as before. */ + m->scan_request = scan; + start = jiffies; + mt_scan_settle_retired(m); + KUNIT_EXPECT_LT(test, jiffies - start, msecs_to_jiffies(100)); + m->scan_request = NULL; + m->retired_scan_seq = 0; +} + static void mt_scan_quiesce_retirement_test(struct kunit *test) { static const struct cfg80211_ops ops = {}; @@ -412,6 +454,40 @@ static void mt_rf_failure_during_publication_test(struct kunit *test) m->connect_publishing = false; } +/* mt_connect() has marked a move but not yet left the old AP. It reports the + * end of the connection itself once its admission check sees link_failed, so + * the RF failure must not report it too, nor queue it behind a publishing + * worker, which mt_connect() joins first. The synthetic netdev has no wdev: + * a cfg80211 report here would not survive. + */ +static void mt_rf_failure_at_move_start_test(struct kunit *test) +{ + struct mt7932 *m = test->priv; + unsigned int publishing; + + for (publishing = 0; publishing < 2; publishing++) { + m->connecting = false; + m->connected = true; + m->disconnecting = false; + m->link_failed = false; + m->rf_ready = true; + m->roam_moving = true; + m->connect_publishing = publishing; + netif_carrier_on(m->netdev); + mt_rf_test_fail(m, -EIO); + KUNIT_EXPECT_FALSE(test, m->connected); + KUNIT_EXPECT_TRUE(test, m->disconnecting); + KUNIT_EXPECT_TRUE(test, m->link_failed); + KUNIT_EXPECT_FALSE(test, m->connect_lost); + KUNIT_EXPECT_TRUE(test, m->roam_moving); + KUNIT_EXPECT_FALSE(test, mt_rf_allowed(m)); + KUNIT_EXPECT_FALSE(test, netif_carrier_ok(m->netdev)); + KUNIT_EXPECT_TRUE(test, m->peer_valid); + } + m->roam_moving = false; + m->connect_publishing = false; +} + static void mt_invalid_association_channel_wakes_both_test(struct kunit *test) { struct mt7932 *m = test->priv; @@ -537,7 +613,9 @@ static struct kunit_case mt_rf_test_cases[] = { KUNIT_CASE(mt_rf_failure_blocks_startup_test), KUNIT_CASE(mt_rf_failure_preserves_first_error_test), KUNIT_CASE(mt_scan_quiesce_retirement_test), + KUNIT_CASE(mt_scan_connect_settle_test), KUNIT_CASE(mt_rf_failure_during_publication_test), + KUNIT_CASE(mt_rf_failure_at_move_start_test), {} }; diff --git a/drivers/net/wireless/mediatek/mt7932/scan.h b/drivers/net/wireless/mediatek/mt7932/scan.h index 3be04716da001..572762fbb34e2 100644 --- a/drivers/net/wireless/mediatek/mt7932/scan.h +++ b/drivers/net/wireless/mediatek/mt7932/scan.h @@ -121,9 +121,13 @@ static inline unsigned int mt7932_scan_discovery_timeout(const u8 body[0x4d4]) } /* CID03 carries up to four directed SSIDs in 36-byte slots (le32 length, - * 32 bytes) at offset 8; selector 8 marks a directed request and byte 4 - * counts the populated slots. Wildcard entries occupy no slot. A directed - * request carries no probe IEs. + * 32 bytes) at offset 8, and byte 4 counts the populated slots. Byte 3 is + * the SSID type: 4 asks for the listed SSIDs and the wildcard together, as + * the other connac firmware takes it. Type 8 matched results against the + * firmware's remembered SSID for each BSSID, so a BSSID that changed its + * SSID was not reported to a directed scan until a wildcard scan refreshed + * it. Wildcard entries occupy no slot. A directed request carries no probe + * IEs. */ #define MT7932_SCAN_SSIDS 4 @@ -138,7 +142,7 @@ static inline int mt7932_scan_ssid_slot(u8 body[0x4d4], const u8 *ssid, return -EINVAL; if (slot >= MT7932_SCAN_SSIDS) return -ENOSPC; - body[3] = 8; + body[3] = 4; put_unaligned_le32(ssid_length, body + 8 + 36 * slot); memcpy(body + 12 + 36 * slot, ssid, ssid_length); body[4] = slot + 1; diff --git a/drivers/net/wireless/mediatek/mt7932/six-test.c b/drivers/net/wireless/mediatek/mt7932/six-test.c index 59430b8483c45..a8c88aa94e5c6 100644 --- a/drivers/net/wireless/mediatek/mt7932/six-test.c +++ b/drivers/net/wireless/mediatek/mt7932/six-test.c @@ -102,7 +102,7 @@ static void mt7932_band_scan_test(struct kunit *test) channels, 1), 0x4d4); KUNIT_EXPECT_EQ(test, mt7932_scan_options(body, (const u8 *)"hidden", 6, NULL, 0), 0); KUNIT_EXPECT_EQ(test, body[2], 1); - KUNIT_EXPECT_EQ(test, body[3], 8); + KUNIT_EXPECT_EQ(test, body[3], 4); KUNIT_EXPECT_EQ(test, body[4], 1); /* Further directed SSIDs fill the next slots; wildcards take none. */ KUNIT_EXPECT_EQ(test, mt7932_scan_ssid_slot(body, NULL, 0), 0); diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 9bc5db8ed8031..0a27e86995f99 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -715,8 +715,8 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request channel->flags |= IEEE80211_CHAN_DISABLED; continue; } - /* Radar channels carry cfg80211's RADAR flag: the firmware may - * listen there, but the station never probes or joins them. + /* Radar channels carry cfg80211's RADAR flag: the station never + * probes them, and joins only an AP it has heard there. */ put_unaligned_le16(channel->hw_value, reg.domain + 12 + count * 8); /* Driver bandwidth limits are already in channel->flags. Adding @@ -735,12 +735,14 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request spin_unlock_irqrestore(&m->response_lock, irqflags); return; } - if (!m->reg_pending && m->connected && !m->disconnecting && + if (!m->reg_pending && + ((m->connected && !m->disconnecting) || m->roam_moving) && mt7932_reg_only_relaxes(&m->reg_desired, ®) && memcmp(®, &m->reg_desired, sizeof(reg))) { - /* A pass would end the connection only to permit more than the - * firmware now allows. Keep the stricter applied domain and run - * the pass once the connection has ended. + /* A pass would end the connection, or the move to another AP + * under way, only to permit more than the firmware now allows. + * Keep the stricter applied domain and run the pass once the + * connection has ended. */ m->reg_deferred = reg; m->reg_deferral = true; diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index d9e0955fa1fc3..9ad89bba8415c 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2350,11 +2350,62 @@ static int apple_pcie_config_write(struct pci_bus *bus, unsigned int devfn, return 0; } +/* + * The tunnel wait covers only the root port's link. A switch Downstream Port + * with a slot but no hot-plug capability (an eGPU behind an ASM2464PD) trains + * its own link later, yet its bus is scanned once: a card that answers after + * that scan is never enumerated with the tunnel, whose bus numbers and windows + * are then fixed. Give such a link the same time before its bus is scanned, + * plus 100 ms (PCIe r7.0 sec 6.6.1 requires it after training above 5 GT/s; + * it is waited for at every speed here, which is conservative). + * Hot-plug capable ports are left to pciehp. + */ +static int apple_pcie_add_bus(struct pci_bus *bus) +{ + struct pci_config_window *cfg = bus->sysdata; + struct pci_dev *port = bus->self; + unsigned long start = jiffies; + unsigned long timeout = start + + msecs_to_jiffies(APPLE_PCIE_TUNNEL_READY_MS); + u16 lnksta, sltsta; + + if (!port || pci_pcie_type(port) != PCI_EXP_TYPE_DOWNSTREAM || + !(pcie_caps_reg(port) & PCI_EXP_FLAGS_SLOT) || + port->is_hotplug_bridge || !port->link_active_reporting || + !apple_pcie_lookup(cfg->parent)->hw->tunneled) + return 0; + + if (pcie_capability_read_word(port, PCI_EXP_LNKSTA, &lnksta) || + (lnksta & PCI_EXP_LNKSTA_DLLLA)) + return 0; + + if (pcie_capability_read_word(port, PCI_EXP_SLTSTA, &sltsta)) + return 0; + pci_info(port, "slot link down before scan (SltSta %#06x), waiting up to %d ms\n", + sltsta, APPLE_PCIE_TUNNEL_READY_MS); + do { + msleep(10); + if (pcie_capability_read_word(port, PCI_EXP_LNKSTA, &lnksta)) + return 0; + if (lnksta & PCI_EXP_LNKSTA_DLLLA) { + pci_info(port, "slot link up after %u ms\n", + jiffies_to_msecs(jiffies - start)); + msleep(PCIE_RESET_CONFIG_WAIT_MS); + return 0; + } + } while (time_before(jiffies, timeout)); + + pci_info(port, "slot link still down after %d ms\n", + APPLE_PCIE_TUNNEL_READY_MS); + return 0; +} + static const struct pci_ecam_ops apple_pcie_cfg_ecam_ops = { .init = apple_pcie_init, .enable_device = apple_pcie_enable_device, .disable_device = apple_pcie_disable_device, .pci_ops = { + .add_bus = apple_pcie_add_bus, .map_bus = apple_pcie_map_bus, .read = pci_generic_config_read, .write = apple_pcie_config_write, diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index e8b6fd640a1f3..2429a5637c2a9 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -1532,6 +1532,9 @@ struct apple_pmp_report_entry { u32 id; /* The PMP does not acknowledge this request (a started PMP only). */ bool no_ack; + /* Withdrawn for system sleep (apple,release-for-sleep), and now so. */ + bool sleep_release; + bool released; /* * With @dvfs, the SOC-DEV-DVFS entry this soc-device's performance * request lives in, and the request word written while powered. @@ -1582,6 +1585,61 @@ static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id, 50000); } +/* Find and check the report before taking the GPU vote; optionally update it. */ +static int apple_pmp_report_t6030_gfx(struct device *supplier, const bool *enable) +{ + struct apple_pmp_report *rep, *found = NULL; + struct device_node *pmp; + const u8 *table; + int len = 0, ret = -ENODEV; + + if (!IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) || + !of_machine_is_compatible("apple,t6030") || + !of_device_is_compatible(supplier->of_node, "apple,t6030-pmp-v2")) + return -ENODEV; + + /* T6030's ordered soc-device ID 5 is the acknowledged AGX request. */ + table = of_get_property(supplier->of_node, "apple,tunable-soc-device", &len); + if (!apple_pmp_devices_valid(table, len, BIT_ULL(4), BIT_ULL(4), true) || + memcmp(table + 4 * PMP_SOC_DEVICE_SIZE + 116, "AGX\0\0\0\0\0", 8)) + return -EINVAL; + + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->offsets != &apple_pmp_offsets_t6030) + continue; + pmp = of_parse_phandle(rep->dev->of_node, "apple,pmp", 0); + if (pmp == supplier->of_node) { + of_node_put(pmp); + if (found) + goto out; + found = rep; + } else { + of_node_put(pmp); + } + } + if (found) { + if (readq(found->base + found->offsets->status) != PMP_REPORT_READY) + ret = -EAGAIN; + else + ret = enable ? apple_pmp_report_set_state(found, 4, *enable, false) : 0; + } +out: + rcu_read_unlock(); + return ret; +} +int apple_pmp_report_check_t6030_gfx(struct device *supplier) +{ + return apple_pmp_report_t6030_gfx(supplier, NULL); +} +EXPORT_SYMBOL_GPL(apple_pmp_report_check_t6030_gfx); + +/* The RTKit AGX vote alone does not enable PMP's GPU bandwidth controllers. */ +int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable) +{ + return apple_pmp_report_t6030_gfx(supplier, &enable); +} +EXPORT_SYMBOL_GPL(apple_pmp_report_set_t6030_gfx); /* SOC-DEV-DVFS request words carry a valid bit; level 0 is the floor. */ #define PMP_DVFS_VALID BIT_ULL(61) @@ -1620,6 +1678,80 @@ static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, boo return ret; } +/* + * System sleep. An external display processor's request keeps the PMP's + * external display state up while nothing is shown, about 0.25 W in + * suspend-to-idle on J514S. By the noirq phase every display processor has + * suspended, its displays off and its links released or idle: withdraw such a + * request if it is acknowledged, with one replacement write as macOS releases + * its display requests, and assert it again before the processors resume. + */ +static int apple_pmp_report_entry_sleep(struct apple_pmp_report_entry *ent, bool release) +{ + struct apple_pmp_report *rep; + int ret = -ENODEV; + + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node != ent->report_node) + continue; + if (release && (!(readq(rep->base + rep->offsets->status) & PMP_REPORT_READY) || + !(readq(rep->base + rep->offsets->actual) & BIT_ULL(ent->id)))) + ret = -EALREADY; + else + ret = apple_pmp_report_set_state(rep, ent->id, !release, ent->no_ack); + break; + } + rcu_read_unlock(); + return ret; +} + +static int apple_pmp_report_entry_suspend_noirq(struct device *dev) +{ + struct apple_pmp_report_entry *ent = dev_get_drvdata(dev); + int ret; + + if (!ent || !ent->sleep_release || READ_ONCE(ent->removed)) + return 0; + ret = apple_pmp_report_entry_sleep(ent, true); + if (ret == -EALREADY) + return 0; + /* A withdrawal that was not acknowledged is asserted again all the same. */ + if (ret) + dev_warn(dev, "request %u not withdrawn for sleep: %d\n", ent->id, ret); + ent->released = true; + return 0; +} + +static int apple_pmp_report_entry_resume_noirq(struct device *dev) +{ + struct apple_pmp_report_entry *ent = dev_get_drvdata(dev); + int ret, tries = 0; + + if (!ent || !ent->released) + return 0; + ent->released = false; + /* + * Without the request the external display stays dark until reboot: + * write it again, three times at most, while the PMP does not + * acknowledge it in time. + */ + do { + ret = apple_pmp_report_entry_sleep(ent, false); + } while (ret == -ETIMEDOUT && ++tries < 3); + if (ret) + dev_err(dev, "request %u not asserted after sleep: %d\n", ent->id, ret); + else if (tries) + dev_warn(dev, "request %u asserted after sleep on write %d\n", ent->id, + tries + 1); + return ret; +} + +static const struct dev_pm_ops apple_pmp_report_entry_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_pmp_report_entry_suspend_noirq, + apple_pmp_report_entry_resume_noirq) +}; + static int apple_pmp_report_entry_power_on(struct generic_pm_domain *genpd) { return apple_pmp_report_entry_set_state(genpd, true); @@ -1692,6 +1824,7 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) } ent->no_ack = of_property_read_bool(node, "apple,no-ack"); } + ent->sleep_release = of_property_read_bool(node, "apple,release-for-sleep"); if (of_property_present(node, "apple,dvfs")) { u32 dvfs[2]; @@ -1794,6 +1927,7 @@ static struct platform_driver apple_pmp_report_entry_driver = { .driver = { .name = "apple-pmp-report-entry", .of_match_table = apple_pmp_report_entry_of_match, + .pm = pm_sleep_ptr(&apple_pmp_report_entry_pm_ops), /* Power-domain providers are not intended for manual unbinding. */ .suppress_bind_attrs = true, }, diff --git a/drivers/rtc/rtc-macsmc.c b/drivers/rtc/rtc-macsmc.c index a28dd7c577bde..aed6940cc61c0 100644 --- a/drivers/rtc/rtc-macsmc.c +++ b/drivers/rtc/rtc-macsmc.c @@ -5,11 +5,14 @@ */ #include +#include #include #include +#include #include #include #include +#include #include #include #include @@ -24,12 +27,26 @@ /* Offset kept as a 64-bit little-endian count of whole seconds */ #define RTC_OFFSET_SECS_BYTES 8 +/* + * PMU alarm control. The alarm compares a 32-bit count of whole seconds of + * the free-running counter, i.e. CLKM >> RTC_SEC_SHIFT, without the offset. + */ +#define RTC_CTRL_MONITOR BIT(0) +#define RTC_CTRL_ALARM_EVENT BIT(1) +#define RTC_CTRL_ALARM_EN BIT(6) + struct macsmc_rtc { struct device *dev; struct apple_smc *smc; struct rtc_device *rtc_dev; struct nvmem_cell *rtc_offset; bool offset_in_seconds; + /* Alarm, if the PMU's alarm registers and interrupt are described */ + struct nvmem_cell *alarm_ctrl; + struct nvmem_cell *alarm_value; + struct nvmem_cell *alarm_mask; + struct mutex alarm_lock; + u32 alarm_secs; }; static int macsmc_rtc_get_time_secs(struct macsmc_rtc *rtc, u64 ctr, @@ -131,15 +148,268 @@ static int macsmc_rtc_set_time(struct device *dev, struct rtc_time *tm) return nvmem_cell_write(rtc->rtc_offset, &off, RTC_BYTES); } + +static int macsmc_rtc_read_u8(struct nvmem_cell *cell, u8 *val) +{ + size_t len; + void *p; + + p = nvmem_cell_read(cell, &len); + if (IS_ERR(p)) + return PTR_ERR(p); + if (len < 1) { + kfree(p); + return -EIO; + } + *val = *(u8 *)p; + kfree(p); + + return 0; +} + +static int macsmc_rtc_write_u8(struct nvmem_cell *cell, u8 val) +{ + int ret = nvmem_cell_write(cell, &val, sizeof(val)); + + return ret < 0 ? ret : 0; +} + +/* The wall-clock offset in whole seconds: wall time = counter seconds + offset */ +static int macsmc_rtc_offset_secs(struct macsmc_rtc *rtc, s64 *secs) +{ + u64 off = 0; + size_t len; + void *p; + + p = nvmem_cell_read(rtc->rtc_offset, &len); + if (IS_ERR(p)) + return PTR_ERR(p); + + if (rtc->offset_in_seconds) { + if (len < RTC_OFFSET_SECS_BYTES) { + kfree(p); + return -EIO; + } + *secs = (s64)get_unaligned_le64(p); + } else { + if (len < RTC_BYTES) { + kfree(p); + return -EIO; + } + memcpy(&off, p, RTC_BYTES); + *secs = sign_extend64(off, RTC_BITS - 1) >> RTC_SEC_SHIFT; + } + kfree(p); + + return 0; +} + +/* + * Arm the alarm at counter second @secs, or disarm it if @secs is 0. The + * other bits of the control byte are kept. A latched event is cleared by + * writing it back, so the event bit is written as 1 only to clear it. + */ +static int macsmc_rtc_program_alarm(struct macsmc_rtc *rtc, u32 secs) +{ + __le32 val = cpu_to_le32(secs); + u8 ctrl; + int ret; + + ret = macsmc_rtc_read_u8(rtc->alarm_ctrl, &ctrl); + if (ret) + return ret; + if (ctrl & RTC_CTRL_ALARM_EVENT) { + ret = macsmc_rtc_write_u8(rtc->alarm_ctrl, ctrl); + if (ret) + return ret; + } + ctrl &= ~RTC_CTRL_ALARM_EVENT; + + if (!secs) { + ret = macsmc_rtc_write_u8(rtc->alarm_mask, 1); + if (ret) + return ret; + return macsmc_rtc_write_u8(rtc->alarm_ctrl, + (ctrl & ~RTC_CTRL_ALARM_EN) | + RTC_CTRL_MONITOR); + } + + ret = nvmem_cell_write(rtc->alarm_value, &val, sizeof(val)); + if (ret < 0) + return ret; + ret = macsmc_rtc_write_u8(rtc->alarm_mask, 0); + if (ret) + return ret; + return macsmc_rtc_write_u8(rtc->alarm_ctrl, ctrl | RTC_CTRL_ALARM_EN | + RTC_CTRL_MONITOR); +} + +static int macsmc_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm) +{ + struct macsmc_rtc *rtc = dev_get_drvdata(dev); + size_t len; + s64 off; + u32 secs; + void *p; + u8 ctrl; + int ret; + + ret = macsmc_rtc_offset_secs(rtc, &off); + if (ret) + return ret; + + mutex_lock(&rtc->alarm_lock); + ret = macsmc_rtc_read_u8(rtc->alarm_ctrl, &ctrl); + if (ret) + goto out; + p = nvmem_cell_read(rtc->alarm_value, &len); + if (IS_ERR(p)) { + ret = PTR_ERR(p); + goto out; + } + secs = len >= sizeof(u32) ? get_unaligned_le32(p) : 0; + kfree(p); + + rtc_time64_to_tm((time64_t)secs + off, &alrm->time); + alrm->enabled = !!(ctrl & RTC_CTRL_ALARM_EN); + alrm->pending = !!(ctrl & RTC_CTRL_ALARM_EVENT); +out: + mutex_unlock(&rtc->alarm_lock); + return ret; +} + +static int macsmc_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm) +{ + struct macsmc_rtc *rtc = dev_get_drvdata(dev); + s64 off, secs; + int ret; + + ret = macsmc_rtc_offset_secs(rtc, &off); + if (ret) + return ret; + + /* + * With a tick offset, off is rounded down to whole seconds but the + * time is floor((ctr + off) / 2^15), so the alarm fires at the set + * second or up to one second after it. + */ + secs = rtc_tm_to_time64(&alrm->time) - off; + if (secs <= 0 || secs > U32_MAX) + return -ERANGE; + + mutex_lock(&rtc->alarm_lock); + rtc->alarm_secs = secs; + ret = macsmc_rtc_program_alarm(rtc, alrm->enabled ? rtc->alarm_secs : 0); + mutex_unlock(&rtc->alarm_lock); + + return ret; +} + +static int macsmc_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled) +{ + struct macsmc_rtc *rtc = dev_get_drvdata(dev); + int ret; + + mutex_lock(&rtc->alarm_lock); + ret = macsmc_rtc_program_alarm(rtc, enabled ? rtc->alarm_secs : 0); + mutex_unlock(&rtc->alarm_lock); + + return ret; +} + +static irqreturn_t macsmc_rtc_alarm_irq(int irq, void *data) +{ + struct macsmc_rtc *rtc = data; + u8 ctrl; + + mutex_lock(&rtc->alarm_lock); + if (macsmc_rtc_read_u8(rtc->alarm_ctrl, &ctrl) || + !(ctrl & RTC_CTRL_ALARM_EVENT)) { + mutex_unlock(&rtc->alarm_lock); + return IRQ_NONE; + } + /* One-shot: clear the event and disarm until the alarm is set again */ + macsmc_rtc_program_alarm(rtc, 0); + mutex_unlock(&rtc->alarm_lock); + + rtc_update_irq(rtc->rtc_dev, 1, RTC_AF | RTC_IRQF); + + return IRQ_HANDLED; +} + static const struct rtc_class_ops macsmc_rtc_ops = { .read_time = macsmc_rtc_get_time, .set_time = macsmc_rtc_set_time, + .read_alarm = macsmc_rtc_read_alarm, + .set_alarm = macsmc_rtc_set_alarm, + .alarm_irq_enable = macsmc_rtc_alarm_irq_enable, }; +/* A missing cell means no alarm, not an error */ +static struct nvmem_cell *macsmc_rtc_cell_optional(struct device *dev, + const char *id) +{ + struct nvmem_cell *cell = devm_nvmem_cell_get(dev, id); + + if (IS_ERR(cell) && PTR_ERR(cell) == -ENOENT) + return NULL; + return cell; +} + +/* The alarm is optional: it needs the PMU's alarm registers and interrupt. */ +static int macsmc_rtc_init_alarm(struct platform_device *pdev, + struct macsmc_rtc *rtc) +{ + struct device *dev = &pdev->dev; + int irq, ret; + + rtc->alarm_ctrl = macsmc_rtc_cell_optional(dev, "rtc_ctrl"); + rtc->alarm_value = macsmc_rtc_cell_optional(dev, "rtc_alarm"); + rtc->alarm_mask = macsmc_rtc_cell_optional(dev, "rtc_alarm_mask"); + if (IS_ERR(rtc->alarm_ctrl)) + return PTR_ERR(rtc->alarm_ctrl); + if (IS_ERR(rtc->alarm_value)) + return PTR_ERR(rtc->alarm_value); + if (IS_ERR(rtc->alarm_mask)) + return PTR_ERR(rtc->alarm_mask); + if (!rtc->alarm_ctrl || !rtc->alarm_value || !rtc->alarm_mask) + return -ENODEV; + + irq = platform_get_irq_optional(pdev, 0); + if (irq == -EPROBE_DEFER) + return irq; + if (irq <= 0) + return -ENODEV; + + /* + * Firmware can leave the alarm armed or its event latched. Neither is + * an alarm Linux set, so disarm it before the interrupt is requested. + */ + ret = macsmc_rtc_program_alarm(rtc, 0); + if (ret) + return ret; + + ret = devm_request_threaded_irq(dev, irq, NULL, macsmc_rtc_alarm_irq, + IRQF_ONESHOT, dev_name(dev), rtc); + if (ret) + return ret; + + if (of_property_read_bool(dev->of_node, "wakeup-source")) { + ret = devm_device_init_wakeup(dev); + if (!ret) + ret = devm_pm_set_wake_irq(dev, irq); + if (ret) + dev_warn(dev, "alarm cannot wake the system: %d\n", ret); + } + + return 0; +} + static int macsmc_rtc_probe(struct platform_device *pdev) { struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); struct macsmc_rtc *rtc; + int ret; /* * MFD will probe this device even without a node in the device tree, @@ -155,6 +425,8 @@ static int macsmc_rtc_probe(struct platform_device *pdev) rtc->dev = &pdev->dev; rtc->smc = smc; + mutex_init(&rtc->alarm_lock); + platform_set_drvdata(pdev, rtc); rtc->rtc_offset = devm_nvmem_cell_get(&pdev->dev, "rtc_offset"); if (IS_ERR(rtc->rtc_offset)) @@ -169,6 +441,14 @@ static int macsmc_rtc_probe(struct platform_device *pdev) "apple,rtc-offset-seconds"); rtc->rtc_dev->ops = &macsmc_rtc_ops; + ret = macsmc_rtc_init_alarm(pdev, rtc); + if (ret == -EPROBE_DEFER) + return ret; + if (ret) { + if (ret != -ENODEV) + dev_warn(&pdev->dev, "no alarm: %d\n", ret); + clear_bit(RTC_FEATURE_ALARM, rtc->rtc_dev->features); + } if (rtc->offset_in_seconds) { rtc->rtc_dev->range_min = 0; rtc->rtc_dev->range_max = RTC_TIMESTAMP_END_9999; @@ -177,8 +457,6 @@ static int macsmc_rtc_probe(struct platform_device *pdev) rtc->rtc_dev->range_max = S64_MAX >> (RTC_SEC_SHIFT + (64 - RTC_BITS)); } - platform_set_drvdata(pdev, rtc); - return devm_rtc_register_device(rtc->rtc_dev); } diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index ba8b498786414..7a99bd1be95a7 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -274,9 +274,14 @@ config APPLE_DISPLAY_GATE nomodeset. It also serves T6031 (M3 Max) device trees with apple,t6031-handoff, - in the same way, but only for a boot with - apple_t6031_display.enable=1 on the kernel command line, and only - with the PMP values the boot loader passes. + in the same way and only with the PMP values the boot loader passes. + There the native display is on by default on the J516C, unless + apple_t6031_display.enable=0; the other T6031 boards need + apple_t6031_display.enable=1 on the kernel command line. The T6031 + GPU driver starts its firmware only after this native display is + up, so apple_t6031_display.enable=0, or a native display that does + not come up, also keeps the GPU off for that boot. It is not a + display-only switch. On T6041 (M4 Pro) device trees it enables the internal display nodes only when the boot loader has handed the running display coprocessor @@ -303,10 +308,11 @@ config APPLE_DISPLAY_GATE_KUNIT_TEST default KUNIT_ALL_TESTS help Runs the display gate on test device trees with the T6030, T6031 - and T8122 display nodes, without Apple hardware. On T6031: off by - default, enabled only by apple_t6031_display.enable=1, only with the - boot loader's handoff marker and this Mac's PMP values, and with - nothing changed when a check fails. On T6030: on by default, as + and T8122 display nodes, without Apple hardware. On T6031: on by + default only on the default board (the J516C), otherwise only with + apple_t6031_display.enable=1, off with apple_t6031_display.enable=0, + only with the boot loader's handoff marker and this Mac's PMP values, + and with nothing changed when a check fails. On T6030: on by default, as before, whatever the T6031 option says. On T8122: on by default, and nothing changed when the handoff or the device tree's own PMP topology does not check out. diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index 1201ee1ff1278..25b28c0522fe7 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -136,6 +136,7 @@ struct apple_clpc { u32 saved_floor; unsigned long samples; unsigned long update_errors; + unsigned long gpu_read_errors; struct dentry *debugfs; struct regmap *gpu_pmgr; struct apple_clpc_gpu *gpu; @@ -422,6 +423,12 @@ static bool apple_clpc_gpu_streaming(struct apple_clpc *clpc, unsigned long now) clpc->gpu_sampled = now; err = regmap_read(clpc->gpu_pmgr, clpc->gpu_offset, &status); + if (err) { + clpc->gpu_read_errors++; + dev_warn_once(clpc->dev, + "graphics power state read failed (%d); holding the GPU floor\n", + err); + } if (!err && (status & GENMASK(7, 4)) != GENMASK(7, 4)) { clpc->gpu_on = false; return false; @@ -647,6 +654,7 @@ static int apple_clpc_status_show(struct seq_file *s, void *unused) clpc->gpu_floor_releases, clpc->gpu_floor_returns); seq_printf(s, "gpu floor samples: streaming %lu dense %lu\n", clpc->gpu_streaming_samples, clpc->gpu_dense_samples); + seq_printf(s, "gpu read errors: %lu\n", clpc->gpu_read_errors); } for (i = 0; i < clpc->nr_clusters; i++) seq_printf(s, "cluster cpu%u: busy %u%% level %u%% -> dcs %u soc %u\n", @@ -711,6 +719,7 @@ static void apple_clpc_stop(void *data) static int apple_clpc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct device_node *gpu = NULL; struct apple_clpc *clpc; unsigned int rail, i; int ret; @@ -737,19 +746,18 @@ static int apple_clpc_probe(struct platform_device *pdev) if (ret) return ret; } - if (clpc->hw->gpu_demand) { - struct device_node *gpu = of_parse_phandle(dev->of_node, - "apple,gpu-power-state", 0); - - if (!gpu) - return dev_err_probe(dev, -EINVAL, "missing graphics power state\n"); + /* The property is optional: without it, CPU demand alone sets the floors. */ + if (clpc->hw->gpu_demand) + gpu = of_parse_phandle(dev->of_node, "apple,gpu-power-state", 0); + if (gpu) { ret = of_property_read_u32(gpu, "reg", &clpc->gpu_offset); clpc->gpu_pmgr = syscon_node_to_regmap(gpu->parent); of_node_put(gpu); if (ret) - return ret; + return dev_err_probe(dev, ret, "graphics power state has no reg\n"); if (IS_ERR(clpc->gpu_pmgr)) - return PTR_ERR(clpc->gpu_pmgr); + return dev_err_probe(dev, PTR_ERR(clpc->gpu_pmgr), + "no regmap for the graphics power state\n"); } if (!apple_pmc_rail_count(clpc->pmc)) return dev_err_probe(dev, -ENODEV, "parent is not a PMC\n"); diff --git a/drivers/soc/apple/display-gate-test.c b/drivers/soc/apple/display-gate-test.c index 73b36e5a722b6..c88a338fdc2cd 100644 --- a/drivers/soc/apple/display-gate-test.c +++ b/drivers/soc/apple/display-gate-test.c @@ -7,9 +7,11 @@ * - T6030 (t6030-gate-test.dtso): on by default, as before, and * the T6031 switch does not change it. Run first: the T6031 and T8122 * trees describe a PMP, and the T6030 gate refuses to add a second one. - * - T6031 (t6031-gate-test.dtso): off by default, enabled only by - * apple_t6031_display.enable=1 with the full handoff, and nothing changed - * when a check fails. + * - T6031 (t6031-gate-test.dtso): on by default only on the default board + * (the J516C; the cases point it at the test tree's own root to run that + * path), otherwise only with apple_t6031_display.enable=1, never with + * apple_t6031_display.enable=0; always with the full handoff, and nothing + * changed when a check fails. * - T8122 (t8122-gate-test.dtso): on by default, with the full handoff, * and nothing changed when a check of the handoff or of the reused PMP * topology fails. @@ -143,6 +145,8 @@ struct gate_test { const struct gt_soc *s; struct of_changeset cs; /* this case's own edits to the test tree */ bool requested, t6031_requested; + int t6031_switch; + const char *t6031_board; }; static struct device_node *__init gt_node(struct kunit *test, const char *path) @@ -392,6 +396,8 @@ static int __init gt_init(struct kunit *test, const struct gt_soc *s) /* As the boot left them: the test kernel's command line sets neither. */ t->requested = gate_requested; t->t6031_requested = gate_t6031_requested; + t->t6031_switch = gate_t6031_switch; + t->t6031_board = gate_t6031_board; of_changeset_init(&t->cs); gate_soc = s->soc; gate_pmp_values = NULL; @@ -399,6 +405,7 @@ static int __init gt_init(struct kunit *test, const struct gt_soc *s) gate_pmp_reused = false; gate_requested = true; gate_t6031_requested = false; + gate_t6031_switch = -1; if (gt_live_soc) kunit_skip(test, "the booted device tree has its own /soc"); return 0; @@ -444,6 +451,8 @@ static void __init gt_exit(struct kunit *test) gate_pmp_values = NULL; gate_requested = t->requested; gate_t6031_requested = t->t6031_requested; + gate_t6031_switch = t->t6031_switch; + gate_t6031_board = t->t6031_board; gate_soc = NULL; } @@ -455,29 +464,82 @@ static void __init gate_t6031_param(struct kunit *test) struct gate_test *t = test->priv; KUNIT_EXPECT_FALSE(test, t->t6031_requested); + KUNIT_EXPECT_EQ(test, t->t6031_switch, -1); + KUNIT_EXPECT_STREQ(test, t->t6031_board, "apple,j516c"); KUNIT_EXPECT_TRUE(test, t->requested); KUNIT_EXPECT_EQ(test, gate_setup_t6031("1"), 0); - KUNIT_EXPECT_TRUE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 1); KUNIT_EXPECT_EQ(test, gate_setup_t6031("0"), 0); - KUNIT_EXPECT_FALSE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 0); + /* A rejected value sets nothing: neither an explicit 1 nor a reset to unset. */ KUNIT_EXPECT_NE(test, gate_setup_t6031("maybe"), 0); - KUNIT_EXPECT_FALSE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 0); KUNIT_EXPECT_NE(test, gate_setup_t6031(NULL), 0); - KUNIT_EXPECT_FALSE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 0); + gate_t6031_switch = -1; + KUNIT_EXPECT_NE(test, gate_setup_t6031("maybe"), 0); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, -1); /* The T6030 and T8122 escapes do not touch it, nor it them. */ KUNIT_EXPECT_EQ(test, gate_setup("0"), 0); KUNIT_EXPECT_FALSE(test, gate_requested); - KUNIT_EXPECT_FALSE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, -1); KUNIT_EXPECT_EQ(test, gate_setup_t6031("1"), 0); KUNIT_EXPECT_FALSE(test, gate_requested); KUNIT_EXPECT_EQ(test, gate_setup_t8122("1"), 0); KUNIT_EXPECT_TRUE(test, gate_requested); - KUNIT_EXPECT_TRUE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 1); +} + +/* An explicit 0 or 1 wins; unset, only the default board asks for it. */ +static void __init gate_t6031_resolution(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, gate_t6031_wanted(-1, true)); + KUNIT_EXPECT_FALSE(test, gate_t6031_wanted(-1, false)); + KUNIT_EXPECT_FALSE(test, gate_t6031_wanted(0, true)); + KUNIT_EXPECT_FALSE(test, gate_t6031_wanted(0, false)); + KUNIT_EXPECT_TRUE(test, gate_t6031_wanted(1, true)); + KUNIT_EXPECT_TRUE(test, gate_t6031_wanted(1, false)); +} + +/* Points the default board at the test tree's root, so gate_run() takes that path. */ +static void __init gt_default_board(struct kunit *test) +{ + const char *root = NULL; + + if (of_property_read_string(of_root, "compatible", &root) || !root) + kunit_skip(test, "the test tree's root has no compatible"); + gate_t6031_board = root; +} + +/* On the default board, the full handoff and no switch: enabled. */ +static void __init gate_t6031_default_board_on(struct kunit *test) +{ + gt_default_board(test); + gt_run(test); + gt_expect_enabled(test); +} + +/* apple_t6031_display.enable=0 still wins on the default board. */ +static void __init gate_t6031_default_board_switched_off(struct kunit *test) +{ + gt_default_board(test); + KUNIT_ASSERT_EQ(test, gate_setup_t6031("0"), 0); + gt_run(test); + gt_expect_untouched(test); +} + +/* The default asks for the display; it does not stand in for the handoff. */ +static void __init gate_t6031_default_board_no_marker(struct kunit *test) +{ + gt_default_board(test); + gt_remove(test, gt_t6031.dcp, "apple,t6031-handoff"); + gt_run(test); + gt_expect_untouched(test); } -/* With a complete handoff and no command line switch, nothing changes. */ +/* Off the default board, with a complete handoff and no switch, nothing changes. */ static void __init gate_t6031_off_by_default(struct kunit *test) { gt_run(test); @@ -826,7 +888,11 @@ static void __init gate_t8122_enabled_again(struct kunit *test) /* The case and suite structs outlive init: only the functions are init code. */ static struct kunit_case gate_t6031_cases[] __refdata = { KUNIT_CASE(gate_t6031_param), + KUNIT_CASE(gate_t6031_resolution), KUNIT_CASE(gate_t6031_off_by_default), + KUNIT_CASE(gate_t6031_default_board_on), + KUNIT_CASE(gate_t6031_default_board_switched_off), + KUNIT_CASE(gate_t6031_default_board_no_marker), KUNIT_CASE(gate_t6031_switched_off), KUNIT_CASE(gate_t6031_enabled), KUNIT_CASE(gate_t6031_no_dcp_marker), diff --git a/drivers/soc/apple/display-gate.c b/drivers/soc/apple/display-gate.c index deb84564bc2bc..6976cec647e64 100644 --- a/drivers/soc/apple/display-gate.c +++ b/drivers/soc/apple/display-gate.c @@ -53,9 +53,13 @@ * option. * * On T6031 (M3 Max) the gate serves the same handoff, marked - * apple,t6031-handoff, but only when apple_t6031_display.enable=1 is on the - * kernel command line: the native display is off by default there until it - * has been seen to work. The PMP values must come from the boot loader, in + * apple,t6031-handoff. Its default is scoped to the J516C: there the native + * display is on unless apple_t6031_display.enable=0; the other T6031 boards + * need apple_t6031_display.enable=1. The T6031 GPU starts its firmware only + * after this native display is up, so enable=0 (or a display that never + * comes up) keeps the GPU off for that boot too: it is not a display-only + * switch. The default changes no check of the + * handoff or the PMP below. The PMP values must come from the boot loader, in * /chosen/asahi,t6031-pmp; there is no fallback. Its PMP topology is * reused from the device tree, as on T8122. The T6031 display power * states nest the other way round (the DCP's DISP_FE under DISP_CPU under @@ -160,9 +164,21 @@ static const char gate_t6031_pmp_uuid[] __initconst = "07FF4C5B-CB2E-3DE6-95FF-3 */ static bool gate_requested __initdata = true; -/* apple_t6031_display.enable: the T6031 native display is off unless it is 1. */ +/* + * apple_t6031_display.enable: -1 while it is not on the command line, else 0 + * or 1. Unset, the T6031 native display is on only on gate_t6031_board. + */ +static int gate_t6031_switch __initdata = -1; +static const char *gate_t6031_board __initdata = "apple,j516c"; +/* The resolved request, set by gate_run() on a T6031. */ static bool gate_t6031_requested __initdata; +/* Whether the T6031 native display is requested: an explicit 0 or 1 wins. */ +static bool __init gate_t6031_wanted(int sw, bool default_board) +{ + return sw < 0 ? default_board : sw; +} + /* One SoC's internal display handoff, as the gate checks and completes it */ struct gate_soc { const char *machine; /* root compatible */ @@ -259,8 +275,10 @@ static const struct gate_soc gate_t8122 __initconst = { }; /* - * T6031 (M3 Max): the T6030 handoff at the T6031 addresses, off unless - * apple_t6031_display.enable=1. The PMP values must come from the boot + * T6031 (M3 Max): the T6030 handoff at the T6031 addresses, on by default on + * the J516C and off unless apple_t6031_display.enable=1 on the other boards; + * apple_t6031_display.enable=0 keeps it, and with it the GPU, off everywhere. + * The PMP values must come from the boot * loader: there is no built-in fallback. The PMP image is T6031's own; * any other, the T6030 image included, is refused. The report driver checks * the image's SRAM layout before it starts the PMP. The internal display's @@ -368,10 +386,18 @@ static int __init gate_setup_t8122(char *arg) } early_param("apple_t8122_display.enable", gate_setup_t8122); -/* Its own switch: the T6030 and T8122 options do not change it, nor it them. */ +/* + * Its own switch: the T6030 and T8122 options do not change it, nor it them. + * A value kstrtobool() rejects sets nothing, so the board default stays. + */ static int __init gate_setup_t6031(char *arg) { - return kstrtobool(arg, &gate_t6031_requested); + bool on; + int ret = kstrtobool(arg, &on); + + if (!ret) + gate_t6031_switch = on; + return ret; } early_param("apple_t6031_display.enable", gate_setup_t6031); @@ -1487,6 +1513,9 @@ static int __init gate_ext_apply(struct gate_ext *ext, bool native) /* The driver must wait for this request's acknowledgement before RUN. */ if (!ret) ret = gate_set_u32(cs, ext->dcp, "apple,pmp-report", ext->entry->phandle); + /* No external display is attached in system sleep: the PMP may drop it. */ + if (!ret) + ret = gate_add_bool(cs, ext->entry, "apple,release-for-sleep"); if (!ret) ret = of_changeset_update_prop_string(cs, ext->entry, "status", "okay"); if (!ret) @@ -1993,8 +2022,17 @@ static void __init gate_run(void) if (gate_soc->describe) gate_describe(); + if (gate_soc == &gate_t6031) { + bool board = of_machine_is_compatible(gate_t6031_board); + + gate_t6031_requested = gate_t6031_wanted(gate_t6031_switch, board); + if (gate_t6031_switch < 0 && board) + gate_info("native display on by default on %s; apple_t6031_display.enable=0 keeps the boot framebuffer\n", + gate_t6031_board); + } + if (!*gate_soc->requested) { - if (gate_soc->opt_in) + if (gate_soc->opt_in && gate_t6031_switch < 0) gate_info("native display not requested (%s is not on the command line), display stays on the boot framebuffer\n", gate_soc->opt_in); else diff --git a/drivers/soc/apple/pmp-export.c b/drivers/soc/apple/pmp-export.c index 120c4f32ae2b3..41091755a77c6 100644 --- a/drivers/soc/apple/pmp-export.c +++ b/drivers/soc/apple/pmp-export.c @@ -58,6 +58,7 @@ EXPORT_SYMBOL_GPL(apple_pmp_link_device); int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) { + bool report_gfx; int ret; if ((command != 0x0e && command != 0x0f) || device_id > 15 || enabled > 1) @@ -69,6 +70,43 @@ int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) ret = apple_pmp_report_wait_supplier_ready(pmp_device, 5000); if (ret) return ret; - return apple_pmp_send_power_command(pmp_data, command, device_id, enabled); + /* T6030 needs both the RTKit power vote and the PTD bandwidth report. */ + report_gfx = IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) && + command == 0x0f && device_id == 5 && + of_machine_is_compatible("apple,t6030") && + of_device_is_compatible(pmp_device->of_node, "apple,t6030-pmp-v2"); + if (report_gfx) { + ret = apple_pmp_report_check_t6030_gfx(pmp_device); + if (ret) + return ret; + } + if (report_gfx && !enabled) { + ret = apple_pmp_report_set_t6030_gfx(pmp_device, false); + if (ret) { + /* Clearing was written before the ACK timed out. Restore it. */ + apple_pmp_report_set_t6030_gfx(pmp_device, true); + return ret; + } + } + ret = apple_pmp_send_power_command(pmp_data, command, device_id, enabled); + /* Power is unknown after a lost reply; assert the report conservatively. */ + if (report_gfx && !enabled && ret) + apple_pmp_report_set_t6030_gfx(pmp_device, true); + return ret; } EXPORT_SYMBOL_GPL(apple_pmp_set_device_power); + +/* Reassert after startup, including a report left set by an earlier failure. */ +int apple_pmp_t6030_gpu_ready(void) +{ + int ret; + + guard(mutex)(&pmp_lock); + if (!pmp_data) + return -ENODEV; + ret = apple_pmp_report_set_t6030_gfx(pmp_device, false); + if (ret) + return ret; + return apple_pmp_report_set_t6030_gfx(pmp_device, true); +} +EXPORT_SYMBOL_GPL(apple_pmp_t6030_gpu_ready); diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index e56017fbb9af2..fbdb9ba0e7f2a 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -314,7 +314,12 @@ static struct irq_chip apple_spmi_irq_chip = { .irq_unmask = apple_spmi_irq_unmask, .irq_ack = apple_spmi_irq_ack, .irq_set_type = apple_spmi_irq_set_type, - .flags = IRQCHIP_ONESHOT_SAFE, + /* + * The controller sits in an always-on block and its parent interrupt is + * chained, so it stays live through suspend: a wake-armed peripheral + * interrupt needs nothing more from the chip to wake the system. + */ + .flags = IRQCHIP_ONESHOT_SAFE | IRQCHIP_SKIP_SET_WAKE, }; static int apple_spmi_irq_domain_map(struct irq_domain *irqd, diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h index 3f2812743a7eb..6d51bc37b5fd7 100644 --- a/include/drm/ttm/ttm_resource.h +++ b/include/drm/ttm/ttm_resource.h @@ -236,6 +236,7 @@ struct ttm_resource_manager { * @addr: mapped virtual address * @offset: physical addr * @is_iomem: is this io memory ? + * @is_nonposted: require non-posted accesses to this I/O memory * @caching: See enum ttm_caching * * Structure indicating the bus placement of an object. @@ -244,6 +245,7 @@ struct ttm_bus_placement { void *addr; phys_addr_t offset; bool is_iomem; + bool is_nonposted; enum ttm_caching caching; }; diff --git a/include/linux/dma/apple-sio.h b/include/linux/dma/apple-sio.h new file mode 100644 index 0000000000000..8665b511659b6 --- /dev/null +++ b/include/linux/dma/apple-sio.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _LINUX_DMA_APPLE_SIO_H +#define _LINUX_DMA_APPLE_SIO_H + +#include + +/** + * struct apple_sio_idle - idle transfer of an Apple SIO channel + * @addr: DMA address of the buffer + * @len: buffer length; 0 removes the channel's idle transfer + * @period_len: length of each of its descriptors, not zero and at most @len + * + * Passed as &dma_slave_config.peripheral_config in a dmaengine_slave_config() + * call of its own, which sets or removes the idle transfer and nothing else: + * the channel's other settings take a separate call. + * + * While a channel has an idle transfer, it plays that buffer cyclically + * whenever no other transfer runs, and goes from one to the other in + * descriptor order: terminating a transfer hands over to the idle one instead + * of stopping the channel, and a transfer issued later follows the idle one's + * issued descriptors. The peripheral sees no gap in between. + * + * dmaengine_synchronize() returns once the descriptors of a terminated + * transfer have completed. If they do not in time, it stops the channel and + * waits for them first; the idle transfer then starts again, after a gap. + * Only an acknowledged TERMINATE counts as stopped. If the channel cannot be + * confirmed stopped, dmaengine_synchronize() warns, as it cannot fail. + * Removing the idle transfer stops the channel if nothing else runs, and + * returns 0 once its descriptors completed. If the channel cannot be + * confirmed stopped, it returns an error and the buffer is still in use. + * + * Setting an idle transfer fails with -EBUSY while the channel has one, or + * while it cannot take over the channel's issued descriptors. Setting and + * removing it, terminating and synchronizing must not run concurrently on a + * channel: its client serializes them. A channel that never had an idle + * transfer works as before. + */ +struct apple_sio_idle { + dma_addr_t addr; + size_t len; + size_t period_len; +}; + +#endif diff --git a/include/linux/soc/apple/display.h b/include/linux/soc/apple/display.h index 8978d95d40484..eb71ce91e0919 100644 --- a/include/linux/soc/apple/display.h +++ b/include/linux/soc/apple/display.h @@ -6,5 +6,6 @@ struct device; /* Initial native display admission, before starting retained G15 firmware. */ int apple_drm_t6030_link_device(struct device *consumer); +int apple_drm_t6031_link_device(struct device *consumer); #endif diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h index 5e819681ba106..dd3373517bf90 100644 --- a/include/linux/soc/apple/pmp-report.h +++ b/include/linux/soc/apple/pmp-report.h @@ -18,6 +18,9 @@ int apple_pmp_report_ane_available(const struct device_node *np); int apple_pmp_report_ane_perf(const struct device_node *np, struct device *owner, unsigned int level, bool *held); int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int timeout_ms); +/* Caller holds the PMP owner's vote lock and keeps the supplier bound. */ +int apple_pmp_report_check_t6030_gfx(struct device *supplier); +int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable); int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout); int apple_pmp_report_fast_die_effort(const struct device_node *np, unsigned int lane, u8 *effort); @@ -40,6 +43,16 @@ static inline int apple_pmp_report_wait_supplier_ready(struct device *supplier, return -ENODEV; } +static inline int apple_pmp_report_check_t6030_gfx(struct device *supplier) +{ + return -ENODEV; +} + +static inline int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable) +{ + return -ENODEV; +} + static inline int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout) { diff --git a/include/linux/soc/apple/pmp.h b/include/linux/soc/apple/pmp.h index 2317ed46f42de..8b329bf5d4dd5 100644 --- a/include/linux/soc/apple/pmp.h +++ b/include/linux/soc/apple/pmp.h @@ -8,6 +8,8 @@ struct device; #if IS_REACHABLE(CONFIG_APPLE_PMP) int apple_pmp_link_device(struct device *consumer); int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled); +/* After the acknowledged AGX vote and successful GPU firmware startup. */ +int apple_pmp_t6030_gpu_ready(void); #else static inline int apple_pmp_link_device(struct device *consumer) { @@ -17,6 +19,10 @@ static inline int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enab { return -ENODEV; } +static inline int apple_pmp_t6030_gpu_ready(void) +{ + return -ENODEV; +} #endif /* Registration is private to the PMP Rust driver and its C bridge. */ int apple_pmp_register(struct device *dev, const void *data); diff --git a/include/uapi/drm/ane_accel.h b/include/uapi/drm/ane_accel.h index 94109d1eead35..30d2ae73b8a70 100644 --- a/include/uapi/drm/ane_accel.h +++ b/include/uapi/drm/ane_accel.h @@ -21,6 +21,9 @@ extern "C" { * - ane_t6021 (H14 family: T6020, T6021, T6022, T8112) implements ABI * version %DRM_ANE_ABI_V2: GET_CAPS, BO_INIT, BO_FREE, PROG_LOAD, * PROC_CREATE and EXEC. + * - ane_t6021 on H16 (T6041) and H17 (T8140) implements ABI version + * %DRM_ANE_ABI_V3: GET_CAPS, H16_LOAD and H16_EXEC. The node exists only + * after the staged opt-in reached its program stage. * * The DRM_IOCTL_VERSION major and &drm_ane_get_caps.abi_version both give * the ABI version. The two versions share the ioctl numbers and the @@ -33,9 +36,12 @@ extern "C" { */ #define DRM_ANE_ABI_V1 1 #define DRM_ANE_ABI_V2 2 +#define DRM_ANE_ABI_V3 3 #define DRM_ANE_CHIP_H13 13 #define DRM_ANE_CHIP_H14 14 +#define DRM_ANE_CHIP_H16 16 +#define DRM_ANE_CHIP_H17 17 /* ABI version 1: entries of &drm_ane_submit.handles */ #define DRM_ANE_TILE_COUNT 0x20 @@ -56,13 +62,41 @@ extern "C" { #define DRM_ANE_PROG_LOAD 0x04 #define DRM_ANE_PROC_CREATE 0x05 #define DRM_ANE_EXEC 0x06 +#define DRM_ANE_H16_LOAD 0x07 +#define DRM_ANE_H16_EXEC 0x08 + +/* ABI version 3: host-built program sections that H16_LOAD places */ +#define DRM_ANE_H16_SECTION_GENERIC 1 +#define DRM_ANE_H16_SECTION_OPERATION 2 +#define DRM_ANE_H16_SECTION_PROCEDURE 3 +#define DRM_ANE_H16_SECTION_TEXT_PROPERTIES 4 +#define DRM_ANE_H16_SECTIONS 4 +/* ABI version 3: smallest and largest host-built section, in bytes */ +#define DRM_ANE_H16_SECTION_MIN 0x28 +#define DRM_ANE_H16_SECTION_MAX 0x10000 + +/* + * ABI version 3: firmware resource indices that the kernel gives the LOAD + * slots. A host-built section that names a slot uses these values. In + * particular, a program with a kernel section tags its operation record with + * %DRM_ANE_H16_RESOURCE_KERNEL. + */ +#define DRM_ANE_H16_RESOURCE_GENERIC 1002 +#define DRM_ANE_H16_RESOURCE_OPERATION 1003 +#define DRM_ANE_H16_RESOURCE_PROCEDURE 1004 +#define DRM_ANE_H16_RESOURCE_TEXT_PROPERTIES 1005 +#define DRM_ANE_H16_RESOURCE_TEXT 1006 +#define DRM_ANE_H16_RESOURCE_KERNEL 1007 +/* ABI version 3: most surfaces of one program */ +#define DRM_ANE_H16_SURFACES 4 /** * struct drm_ane_get_caps - DRM_IOCTL_ANE_GET_CAPS argument * @flags: In. No flags are defined: must be zero. * @size: Out. Bytes of this structure that the driver filled in. - * @abi_version: Out. %DRM_ANE_ABI_V1 or %DRM_ANE_ABI_V2. - * @chip_family: Out. %DRM_ANE_CHIP_H13 or %DRM_ANE_CHIP_H14. + * @abi_version: Out. %DRM_ANE_ABI_V1, %DRM_ANE_ABI_V2 or %DRM_ANE_ABI_V3. + * @chip_family: Out. %DRM_ANE_CHIP_H13, %DRM_ANE_CHIP_H14, + * %DRM_ANE_CHIP_H16 or %DRM_ANE_CHIP_H17. * @section_size: Out. Size of the &drm_ane_section entries that PROG_LOAD * reads, or 0 if the driver has no PROG_LOAD. * @bind_size: Out. Size of the &drm_ane_generic_bind entries that PROG_LOAD @@ -241,6 +275,82 @@ struct drm_ane_exec { __u64 io_ptr; }; +/** + * struct drm_ane_h16_section - one host-built program section (ABI version 3) + * @kind: %DRM_ANE_H16_SECTION_GENERIC to + * %DRM_ANE_H16_SECTION_TEXT_PROPERTIES, each exactly once. + * @size: Bytes at @data_ptr, %DRM_ANE_H16_SECTION_MIN to + * %DRM_ANE_H16_SECTION_MAX and a multiple of 4. + * @data_ptr: Pointer to the section bytes. The kernel copies them. + */ +struct drm_ane_h16_section { + __u32 kind; + __u32 size; + __u64 data_ptr; +}; + +/** + * struct drm_ane_h16_load - DRM_IOCTL_ANE_H16_LOAD argument (ABI version 3) + * @hwx_ptr: In. Pointer to the original HWX file. + * @hwx_size: In. Size of the HWX file in bytes. + * @sections_ptr: In. Pointer to struct drm_ane_h16_section[@section_count]. + * @section_count: In. Exactly %DRM_ANE_H16_SECTIONS. + * @flags: In. No flags are defined: must be zero. + * @surface_count: Out. Surfaces of the program, inputs and the one output in + * the HWX segment order that H16_EXEC uses. A program's __DATA scratch + * is kernel-owned and not a surface. + * @pad: Must be zero. + * @surface_size: Out. Bytes of each surface, for @surface_count entries. + * @surface_direction: Out. 1 for an input, 2 for the output. + * + * The kernel validates the HWX, places every section in its own private + * mapping, then loads the program and creates its process. The program + * belongs to this file and is unloaded when the file closes. The HWX must + * be compiled for the engine that GET_CAPS reports. On H17 a firmware boot + * accepts any number of programs, one per file. On H16 a firmware boot + * accepts one program: a second H16_LOAD fails with -EBUSY, also after the + * first program closed, and a reboot starts a new firmware boot. + */ +struct drm_ane_h16_load { + __u64 hwx_ptr; + __u64 hwx_size; + __u64 sections_ptr; + __u32 section_count; + __u32 flags; + __u32 surface_count; + __u32 pad; + __u64 surface_size[DRM_ANE_H16_SURFACES]; + __u32 surface_direction[DRM_ANE_H16_SURFACES]; +}; + +/** + * struct drm_ane_h16_buffer - one surface of an H16_EXEC (ABI version 3) + * @ptr: In. Pointer to the bytes: read for an input, written for the output. + * @size: In. Must equal the surface size that H16_LOAD reported. + */ +struct drm_ane_h16_buffer { + __u64 ptr; + __u64 size; +}; + +/** + * struct drm_ane_h16_exec - DRM_IOCTL_ANE_H16_EXEC argument (ABI version 3) + * @buffers_ptr: In. Pointer to struct drm_ane_h16_buffer[@count], one entry + * per surface in the order that H16_LOAD reported. + * @count: In. The surface count of the loaded program. + * @flags: In. No flags are defined: must be zero. + * + * A successful H16_EXEC returns after the firmware reported the end of the + * request, with the output copied to its buffer. The driver keeps all + * firmware-visible memory private: inputs are copied in before the request + * and the output is copied out after it. + */ +struct drm_ane_h16_exec { + __u64 buffers_ptr; + __u32 count; + __u32 flags; +}; + #define DRM_IOCTL_ANE_GET_CAPS \ DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_GET_CAPS, struct drm_ane_get_caps) #define DRM_IOCTL_ANE_BO_INIT \ @@ -255,6 +365,10 @@ struct drm_ane_exec { DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_PROC_CREATE, struct drm_ane_proc_create) #define DRM_IOCTL_ANE_EXEC \ DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_EXEC, struct drm_ane_exec) +#define DRM_IOCTL_ANE_H16_LOAD \ + DRM_IOWR(DRM_COMMAND_BASE + DRM_ANE_H16_LOAD, struct drm_ane_h16_load) +#define DRM_IOCTL_ANE_H16_EXEC \ + DRM_IOW(DRM_COMMAND_BASE + DRM_ANE_H16_EXEC, struct drm_ane_h16_exec) #if defined(__cplusplus) } diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 145711b2ccf00..6a8789909bbc8 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index d873eb4ef6384..8b40308767619 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -761,24 +761,38 @@ impl GpuVmInnerGuard<'_, T> { /// node or operation list is allocated. Call [`GpuVm::deferred_cleanup`] after releasing this /// guard, including when a callback failed after earlier mappings were removed. pub fn unmap_all(&mut self, unmap: impl FnMut(&mut T, &GpuVa) -> Result) -> Result { - self.unmap_matching(None, unmap) + self.unmap_matching(None, |_| true, unmap) } /// Allocation-free whole-node removal for one GEM object in an immediate-mode VM. /// /// Uses the same translation-removal and deferred-put contract as [`Self::unmap_all`]. This - /// scans the VM list and is intended as a low-memory cleanup fallback. + /// scans the VM list and can be used when the per-object operation list is unavailable. pub fn unmap_object( &mut self, object: &T::Object, unmap: impl FnMut(&mut T, &GpuVa) -> Result, ) -> Result { - self.unmap_matching(Some(object), unmap) + self.unmap_object_where(object, |_| true, unmap) + } + + /// Remove only mappings of an object selected by `matches`, without allocating. + /// + /// This scans the whole VM and uses the same translation-removal and deferred-put contract as + /// [`Self::unmap_all`]. If a callback fails, mappings already visited remain removed. + pub fn unmap_object_where( + &mut self, + object: &T::Object, + matches: impl FnMut(&GpuVa) -> bool, + unmap: impl FnMut(&mut T, &GpuVa) -> Result, + ) -> Result { + self.unmap_matching(Some(object), matches, unmap) } fn unmap_matching( &mut self, object: Option<&T::Object>, + mut matches: impl FnMut(&GpuVa) -> bool, mut unmap: impl FnMut(&mut T, &GpuVa) -> Result, ) -> Result { let vm = self.gpuvm.as_raw(); @@ -802,7 +816,11 @@ impl GpuVmInnerGuard<'_, T> { if object.is_some_and(|object| (*va).gem.obj != object.as_raw()) { continue; } - unmap(&mut *self.gpuvm.data.get(), GpuVa::::from_raw(va))?; + let typed_va = GpuVa::::from_raw(va); + if !matches(typed_va) { + continue; + } + unmap(&mut *self.gpuvm.data.get(), typed_va)?; // Remove it as `OpUnmap::remove()` does. let op = bindings::drm_gpuva_op_unmap { va, keep: false }; bindings::drm_gpuva_unmap(&op); diff --git a/sound/soc/codecs/tas2770.c b/sound/soc/codecs/tas2770.c index a69aba06a4b10..f0dbc2947ecda 100644 --- a/sound/soc/codecs/tas2770.c +++ b/sound/soc/codecs/tas2770.c @@ -948,8 +948,9 @@ static int tas2770_i2c_probe(struct i2c_client *client) if (client->dev.of_node) { result = tas2770_parse_dt(&client->dev, tas2770); if (result) { - dev_err(tas2770->dev, "%s: Failed to parse devicetree\n", - __func__); + if (result != -EPROBE_DEFER) + dev_err(tas2770->dev, "%s: Failed to parse devicetree\n", + __func__); return result; } } diff --git a/tools/asahi/asahi-stats-raw-tests.py b/tools/asahi/asahi-stats-raw-tests.py new file mode 100755 index 0000000000000..1dc45655eb548 --- /dev/null +++ b/tools/asahi/asahi-stats-raw-tests.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the raw firmware statistics snapshot and its export format. + +The snapshot keeps the firmware words verbatim: no unit is assumed, nothing is +scaled and no metric is derived. This harness pins that contract, because a +consumer parses the exported text. +""" + +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +stats = (ROOT / "drivers/gpu/drm/asahi/stats_raw.rs").read_text() + + +def block(marker): + """Extract the item starting at `marker`, up to its matching brace.""" + start = stats.index(marker) + index = stats.index("{", start) + 1 + depth = 1 + while depth: + depth += (stats[index] == "{") - (stats[index] == "}") + index += 1 + return stats[start:index] + +format_tag = re.search(r"^pub\(crate\) const STATS_FORMAT: &str = \".*\";$", stats, re.MULTILINE) +assert format_tag is not None + + +source = "\n".join([ + "#![allow(dead_code, unused_variables)]", + "use std::fmt::Write;", + format_tag.group(), + block("#[derive(Copy, Clone, PartialEq, Eq)]"), + block("impl StatsKind"), + block("#[derive(Copy, Clone, Default)]"), + block("impl RawStatsSnapshot"), + block("pub(crate) fn format_snapshot"), + r''' +fn render(snap: &RawStatsSnapshot) -> String { + let mut out = String::new(); + format_snapshot(snap, 1000, "testdev", &mut out).unwrap(); + out +} + +const EMPTY: &str = "\ +asahi_stats_v1 +device=testdev +now_ns=1000 +messages count=0 +power count=0 +power_on count=0 +power_off count=0 +utilization count=0 +avg_power count=0 +temperature count=0 +power_state count=0 +fw_busy count=0 +pstate count=0 +temp_sensor count=0 +"; + +#[test] +fn missing_samples_are_explicit() { + let snap = RawStatsSnapshot::default(); + assert_eq!(render(&snap), EMPTY); + // A kind that was never received carries no payload at all, so a missing + // sample can never be read as a zero value. + for line in EMPTY.lines() { + if line.ends_with("count=0") { + assert_eq!(line.split_whitespace().count(), 2, "{}", line); + } + } +} + +#[test] +fn raw_samples_are_preserved_verbatim() { + let mut snap = RawStatsSnapshot::default(); + snap.note_message(); + snap.set_power(0x1234, 11); + snap.note_message(); + snap.set_power_on(0x2222, 12); + snap.note_message(); + snap.set_power_off(0x3333, 13); + snap.note_message(); + snap.set_utilization(0x4444, [1, 2, 3, 4], 14); + snap.note_message(); + snap.set_avg_power(0x5555, [6, 7, 8], 9, 15); + snap.note_message(); + snap.set_temperature(30000, 1000, 0, 100000, 16); + snap.note_message(); + snap.set_power_state(0x6666, 0x7777, 1, 0, 2, 5, 3, 4, 17); + snap.note_message(); + snap.set_fw_busy(0x8888, 1, 18); + snap.note_message(); + snap.set_pstate(0, 1, 7, 2, 19); + snap.note_message(); + snap.set_temp_sensor(2, 30000, 1000, 0, 100000, 20); + + assert_eq!(render(&snap), "\ +asahi_stats_v1 +device=testdev +now_ns=1000 +messages count=10 +power count=1 captured_ns=11 power=4660 +power_on count=1 captured_ns=12 off_time=8738 +power_off count=1 captured_ns=13 on_time=13107 +utilization count=1 captured_ns=14 timestamp=17476 util1=1 util2=2 util3=3 util4=4 +avg_power count=1 captured_ns=15 active_cs=21845 unk2=6 unk3=7 unk4=8 avg_power=9 +temperature count=1 captured_ns=16 raw_value=30000 scale=1000 tmin=0 tmax=100000 +power_state count=1 captured_ns=17 timestamp=26214 last_busy_ts=30583 active=1 poweroff=0 unk1=2 pstate=5 unk2=3 unk3=4 +fw_busy count=1 captured_ns=18 timestamp=34952 busy=1 +pstate count=1 captured_ns=19 ps_min=0 unk1=1 ps_max=7 unk2=2 +temp_sensor count=1 captured_ns=20 sensor_id=2 raw_value=30000 scale=1000 tmin=0 tmax=100000 +"); +} + +#[test] +fn counters_and_capture_times_track_samples() { + let mut snap = RawStatsSnapshot::default(); + snap.set_fw_busy(1, 1, 100); + snap.set_fw_busy(2, 0, 200); + let text = render(&snap); + assert!(text.contains("fw_busy count=2 captured_ns=200 timestamp=2 busy=0\n"), "{}", text); + // The sample count of a kind that was never received stays at zero. + assert!(text.contains("power count=0\n"), "{}", text); + assert!(text.contains("messages count=0\n"), "{}", text); +} + +#[test] +fn no_metric_semantics_are_invented() { + let mut snap = RawStatsSnapshot::default(); + snap.set_fw_busy(1, 1, 100); + snap.set_utilization(1, [10, 20, 30, 40], 100); + let text = render(&snap); + // The firmware busy flag is not a GPU utilisation figure and the raw + // counters keep their own names; no scaled or renamed metric appears. + for forbidden in ["gpu_busy", "util_pct", "utilization_pct", "power_mw", "temp_c", "freq_mhz"] { + assert!(!text.contains(forbidden), "{} in {}", forbidden, text); + } + assert!(text.contains("fw_busy count=1 captured_ns=100 timestamp=1 busy=1\n"), "{}", text); +} +''', +]) + +with tempfile.TemporaryDirectory(prefix="asahi-stats-export-") as tmp: + source_path = Path(tmp) / "test.rs" + binary = Path(tmp) / "test" + source_path.write_text(source) + subprocess.run(["rustc", "--edition=2021", "--test", str(source_path), + "-o", str(binary)], check=True) + subprocess.run([str(binary)], check=True) diff --git a/tools/asahi/avd-codec-tail-tests.py b/tools/asahi/avd-codec-tail-tests.py index a1d57aa2e763c..80cf67b85a4d0 100755 --- a/tools/asahi/avd-codec-tail-tests.py +++ b/tools/asahi/avd-codec-tail-tests.py @@ -189,6 +189,10 @@ def main(): print(json.dumps({"passed": ok, "cases": len(cases), "variants": [{k: x[k] for k in ("name", "failed_cases", "verdict")} for x in results]})) + # Keep production AV1 bank/dependency coverage in the registered host suite. + if ok and not args.ref and not args.mutations: + subprocess.run([sys.executable, "-B", str(ROOT / "drivers/media/platform/apple/avd/tests/av1-test.py"), + str(args.output / "av1-banks")], check=True) return 0 if ok else 1 diff --git a/tools/asahi/clpc-gpu-demand-tests.py b/tools/asahi/clpc-gpu-demand-tests.py index 66f0175802e5a..b10dd91882371 100644 --- a/tools/asahi/clpc-gpu-demand-tests.py +++ b/tools/asahi/clpc-gpu-demand-tests.py @@ -58,6 +58,8 @@ def main(): #define time_after(a,b) ((long)((a)-(b)) > 0) #define time_before(a,b) ((long)((a)-(b)) < 0) #define msecs_to_jiffies(ms) (ms) +static int warned; +#define dev_warn_once(d, ...) ((void)(d), warned++) enum apple_clpc_rail { APPLE_CLPC_DCS, APPLE_CLPC_SOC, APPLE_CLPC_RAILS }; ''' + types + r''' struct apple_clpc_gpu { bool active; u64 sequence; }; @@ -67,7 +69,8 @@ def main(): struct apple_clpc_cluster clusters[1]; struct apple_clpc_vote vote[APPLE_CLPC_RAILS]; unsigned int nr_clusters, agent; - unsigned long samples, next_scan, update_errors; + unsigned long samples, next_scan, update_errors, gpu_read_errors; + void *dev; void *covered, *pmc, *gpu_pmgr; u32 gpu_offset, mask, last_floor, last_reason; bool gpu_on, gpu_limited, gpu_floor_released; @@ -110,9 +113,12 @@ def main(): assert(published==0x35 && c.last_reason==16); jiffies=132; apple_clpc_sample(&c); assert(published==0 && queued_idle); // A failed power-state read still protects an active graphics consumer. - jiffies=200; pmgr_status=0xe0; read_error=-EIO; + jiffies=200; pmgr_status=0xe0; read_error=-EIO; warned=0; apple_clpc_sample(&c); assert(published==0x35); + // The failure is counted for debugfs and warned about once. + assert(c.gpu_read_errors==1 && warned==1); jiffies=232; read_error=0; apple_clpc_sample(&c); assert(published==0); + assert(c.gpu_read_errors==1); } static void check_t8140(void) { @@ -199,6 +205,7 @@ def main(): ('dense-ignores-late', code.replace('if (!threshold || late || submissions < threshold)', 'if (!threshold || (late && false) || submissions < threshold)')), ('uncounted-floor-return', code.replace('clpc->gpu_floor_returns++;', 'clpc->gpu_floor_returns += 0;')), + ('uncounted-read-error', code.replace('clpc->gpu_read_errors++;', 'clpc->gpu_read_errors += 0;')), ('omit-pmp-backoff', code.replace('bool released = apple_clpc_gpu_limited(clpc, now);', 'bool released = (apple_clpc_gpu_limited(clpc, now), false);')), ('release-too-early', code.replace('msecs_to_jiffies(APPLE_CLPC_HOLD_MS)', '0')), diff --git a/tools/asahi/display26-tests.py b/tools/asahi/display26-tests.py index 8d2e497fdcf64..2366766042dc2 100644 --- a/tools/asahi/display26-tests.py +++ b/tools/asahi/display26-tests.py @@ -96,6 +96,7 @@ def main(): #include #include #include +#include #include struct device_node { int id; }; struct device { struct device_node *of_node; bool bound; int refs; }; struct device_link { int unused; }; @@ -117,12 +118,17 @@ def main(): static struct device *get_device(struct device*d){d->refs++;return d;} static void put_device(struct device*d){d->refs--;} static struct device_link *device_link_add(struct device*c,struct device*s,unsigned flags){static struct device_link l;assert(flags==1);return link_fail?NULL:&l;} +/* This harness exercises the J613/25G83 bridge, not the T6030 GFX report. */ +static bool of_machine_is_compatible(const char *name){return !strcmp(name,"apple,t8132");} +static bool of_device_is_compatible(const struct device_node*n,const char*name){return false;} ''') write(tmp, 'bridge.c', '''#include #include "drivers/soc/apple/pmp-export.c" static int ready_result,send_result,calls; int apple_pmp_report_wait_supplier_ready(struct device*d,unsigned timeout){assert(d==pmp_device);assert(timeout==5000);return ready_result;} int apple_pmp_send_power_command(const void*data,u8 command,u16 id,u32 enabled){assert(data==pmp_data);calls++;return send_result;} +int apple_pmp_report_check_t6030_gfx(struct device*d){assert(!"unexpected T6030 report on J613");return -EIO;} +int apple_pmp_report_set_t6030_gfx(struct device*d,bool enabled){assert(!"unexpected T6030 report on J613");return -EIO;} int main(void){ struct device_node n={1},wrong={2}; struct device supplier={&n,true,0},consumer={NULL,false,0}; int data; assert(apple_pmp_set_device_power(0xf,8,1)==-ENODEV); diff --git a/tools/asahi/g15-display-order-tests.py b/tools/asahi/g15-display-order-tests.py new file mode 100644 index 0000000000000..a6666efdf2776 --- /dev/null +++ b/tools/asahi/g15-display-order-tests.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run the actual G15 native-display start-order bodies on host models. + +The publication slots, the shared link helper and both SoC wrappers are taken +from drivers/gpu/drm/apple/apple_drv.c, the T6031 glue from g15/glue.c and +admit() from g15/boot.rs, unchanged. OF lookups, device references, links and +the admission steps are host models; no device or firmware is touched. Each +mutant must build and fail an intended assertion. +""" +import argparse +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +APPLE = "drivers/gpu/drm/apple/apple_drv.c" +GLUE = "drivers/gpu/drm/asahi/g15/glue.c" +BOOT = "drivers/gpu/drm/asahi/g15/boot.rs" +MODULE = "drivers/gpu/drm/asahi/g15/mod.rs" + + +def item(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def c_source(root): + apple = (root / APPLE).read_text() + glue = (root / GLUE).read_text() + state = [line for line in apple.splitlines() + if line in ("static DEFINE_MUTEX(apple_g15_display_lock);", + "static struct device *apple_t6030_display;", + "static struct device *apple_t6031_display;")] + assert len(state) == 3, f"publication state not found: {state}" + bodies = [item(apple, marker) for marker in ( + "\nstatic struct device **apple_g15_display_slot(struct device *dev)\n{", + "\nstatic int apple_g15_display_publish(struct device *dev)\n{", + "\nstatic void apple_g15_display_withdraw(struct device *dev)\n{", + "\nstatic int apple_g15_link_device(", + "\nint apple_drm_t6030_link_device(struct device *consumer)\n{", + "\nint apple_drm_t6031_link_device(struct device *consumer)\n{", + )] + bodies.append(item(glue, "\nint asahi_g15_display_prepare(struct device *dev)\n{")) + return "\n".join(state) + "\n\n" + "\n\n".join(bodies) + "\n" + + +C_PREFIX = r''' +#include +#include +#include +#include +#include + +#define ENODEV 19 +#define EBUSY 16 +#define EINVAL 22 +#define EPROBE_DEFER 517 +#define DL_FLAG_AUTOREMOVE_CONSUMER 1u +#define EXPORT_SYMBOL_GPL(sym) extern int sym##_exported + +struct device_node { const char *compatible; bool available; int refs; }; +struct device { struct device_node *of_node; int refs; }; +struct device_link { struct device *consumer, *supplier; unsigned int flags; }; +struct mutex { bool held; }; + +static const char *machine_compat[2]; +static struct device_node *nodes[4]; +static int links_made, link_fail, infos; +static struct device_link last_link; + +static int of_machine_is_compatible(const char *compat) +{ + for (int i = 0; i < 2; i++) + if (machine_compat[i] && !strcmp(machine_compat[i], compat)) + return 1; + return 0; +} +static int of_device_is_compatible(const struct device_node *np, const char *compat) +{ + return np && !strcmp(np->compatible, compat); +} +static struct device_node *of_find_compatible_node(struct device_node *from, const char *type, + const char *compat) +{ + (void)from; (void)type; + for (int i = 0; i < 4; i++) + if (nodes[i] && !strcmp(nodes[i]->compatible, compat)) { + nodes[i]->refs++; + return nodes[i]; + } + return NULL; +} +static bool of_device_is_available(const struct device_node *np) { return np && np->available; } +static void of_node_put(struct device_node *np) { if (np) np->refs--; } +static struct device *get_device(struct device *dev) { dev->refs++; return dev; } +static void put_device(struct device *dev) { if (dev) dev->refs--; } +static struct device_link *device_link_add(struct device *consumer, struct device *supplier, + unsigned int flags) +{ + if (link_fail) + return NULL; + links_made++; + last_link = (struct device_link){ consumer, supplier, flags }; + return &last_link; +} +#define dev_info_once(dev, ...) do { static bool once_; (void)(dev); if (!once_) { once_ = true; infos++; } } while (0) + +static void unlock_(struct mutex **m) { if (*m) { assert((*m)->held); (*m)->held = false; } } +static struct mutex *lock_(struct mutex *m) { assert(!m->held); m->held = true; return m; } +#define DEFINE_MUTEX(name) struct mutex name = { false } +#define guard(type) struct mutex *guard_ __attribute__((cleanup(unlock_))) = lock_ +#define scoped_guard(type, m) for (struct mutex *sg_ __attribute__((cleanup(unlock_))) = lock_(m), *done_ = NULL; !done_; done_ = (struct mutex *)1) +''' + +C_MAIN = r''' +static const char *scenario; +#define CHECK(c) do { if (!(c)) { fprintf(stderr, "%s:%d: %s\n", scenario, __LINE__, #c); exit(1); } } while (0) + +static struct device_node gpu31 = { "apple,agx-t6031", true }, gpu30 = { "apple,agx-t6030", true }; +static struct device_node disp31 = { "apple,t6031-display-subsystem", false }; +static struct device_node disp30 = { "apple,t6030-display-subsystem", true }; +static struct device_node other = { "apple,display-subsystem", true }; +static struct device g31 = { &gpu31 }, g30 = { &gpu30 }, none = { NULL }; +static struct device d31 = { &disp31 }, d31b = { &disp31 }, d30 = { &disp30 }, dother = { &other }; + +static void machine(const char *soc, const char *board) +{ + machine_compat[0] = board; + machine_compat[1] = soc; +} +static void clean(void) +{ + CHECK(!apple_g15_display_lock.held); + for (int i = 0; i < 4; i++) + CHECK(!nodes[i] || nodes[i]->refs == 0); +} +static int prepare(struct device *dev) { int ret = asahi_g15_display_prepare(dev); clean(); return ret; } +static int link30(struct device *dev) { int ret = apple_drm_t6030_link_device(dev); clean(); return ret; } +static int link31(struct device *dev) { int ret = apple_drm_t6031_link_device(dev); clean(); return ret; } +static int publish(struct device *dev) { int ret = apple_g15_display_publish(dev); clean(); return ret; } +static void withdraw(struct device *dev) { apple_g15_display_withdraw(dev); clean(); } + +int main(int argc, char **argv) +{ + scenario = argv[1]; + machine("apple,t6031", "apple,j516c"); + nodes[0] = &gpu31; + nodes[1] = &disp31; + nodes[2] = &disp30; + nodes[3] = &other; + if (!strcmp(scenario, "disabled")) { + /* The gate left the display disabled: no start, no success, said once. */ + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1); + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1 && links_made == 0); + } else if (!strcmp(scenario, "absent")) { + nodes[1] = NULL; + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1 && links_made == 0); + } else if (!strcmp(scenario, "pending")) { + /* Enabled but not published: no start, no success, said once. */ + disp31.available = true; + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1 && links_made == 0); + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1 && links_made == 0); + } else if (!strcmp(scenario, "ready")) { + disp31.available = true; + CHECK(publish(&d31) == 0 && d31.refs == 1); + CHECK(prepare(&g31) == 0 && links_made == 1 && infos == 0); + CHECK(last_link.consumer == &g31 && last_link.supplier == &d31 && + last_link.flags == DL_FLAG_AUTOREMOVE_CONSUMER); + link_fail = 1; + CHECK(prepare(&g31) == -EINVAL); + link_fail = 0; + CHECK(publish(&d31) == 0 && d31.refs == 1); + CHECK(publish(&d31b) == -EBUSY && d31b.refs == 0); + withdraw(&d31b); + CHECK(prepare(&g31) == 0); + withdraw(&d31); + CHECK(d31.refs == 0 && prepare(&g31) == -EPROBE_DEFER); + } else if (!strcmp(scenario, "isolation")) { + disp31.available = true; + CHECK(publish(&d30) == 0 && d30.refs == 1); + CHECK(publish(&dother) == 0 && dother.refs == 0); + withdraw(&dother); + CHECK(prepare(&g31) == -EPROBE_DEFER && links_made == 0); + } else if (!strcmp(scenario, "foreign")) { + disp31.available = true; + CHECK(publish(&d31) == 0); + CHECK(prepare(&g30) == -ENODEV && link31(&g30) == -ENODEV); + CHECK(prepare(&none) == -ENODEV && link31(&none) == -ENODEV); + machine("apple,t6030", "apple,j516s"); + CHECK(link31(&g31) == -ENODEV && links_made == 0); + } else if (!strcmp(scenario, "t6030")) { + machine("apple,t6030", "apple,j516s"); + CHECK(link30(&g30) == -EPROBE_DEFER); + disp31.available = true; + CHECK(publish(&d31) == 0 && link30(&g30) == -EPROBE_DEFER); + CHECK(publish(&d30) == 0 && link30(&g30) == 0 && last_link.supplier == &d30); + withdraw(&d30); + CHECK(d30.refs == 0 && link30(&g30) == -EPROBE_DEFER); + CHECK(link30(&g31) == -ENODEV); + } else { + CHECK(!"unknown scenario"); + } + return 0; +} +''' + +C_SCENARIOS = ["disabled", "absent", "pending", "ready", "isolation", "foreign", "t6030"] + +C_MUTANTS = { + "unpublished-succeeds": ("if (!*display)\n\t\treturn -EPROBE_DEFER;", "if (!*display)\n\t\treturn 0;"), + "disabled-display-skipped": (re.compile(r"\t\tdev_info_once\(dev,[^;]*;\n"), "\t\t{ of_node_put(np); return 0; }\n"), + "pending-display-silent": (re.compile(r"\telse\n\t\tdev_info_once\(dev,[^;]*;\n"), ""), + "t6031-reads-t6030-slot": ("&apple_t6031_display);", "&apple_t6030_display);"), + "t6031-published-as-t6030": ("return &apple_t6031_display;", "return &apple_t6030_display;"), + "no-machine-check": ("!of_machine_is_compatible(machine) ||", ""), + "withdraw-keeps-display": ("*slot = NULL;", ";"), + "publish-without-reference": ("*slot = get_device(dev);", "*slot = dev;"), +} + +RS_PREFIX = r''' +#![allow(dead_code, unused_variables, unused_imports, unused_macros, non_snake_case)] +use std::cell::RefCell; +use std::marker::PhantomData; +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +struct Error(i32); +impl Error { fn from_errno(errno: i32) -> Self { Self(errno) } } +type Result = std::result::Result; +const ENODEV: Error = Error(-19); +const EINVAL: Error = Error(-22); +const EIO: Error = Error(-5); +const EPROBE_DEFER: Error = Error(-517); +macro_rules! dev_err { ($($args:tt)*) => {{}}; } +macro_rules! dev_info { ($($args:tt)*) => {{}}; } +macro_rules! dev_warn { ($($args:tt)*) => {{ warned(); }}; } +struct State { board: bool, machine: bool, pmp: i32, display: i32, sleep_hold: bool, asc_fail: bool } +thread_local! { + static CALLS: RefCell> = const { RefCell::new(Vec::new()) }; + static STATE: RefCell = const { RefCell::new(State { board: true, machine: true, pmp: 0, display: 0, sleep_hold: true, asc_fail: false }) }; + static HOLD: RefCell> = const { RefCell::new(None) }; + static WARNINGS: RefCell = const { RefCell::new(0) }; +} +fn warned() { WARNINGS.with(|w| *w.borrow_mut() += 1); } +mod params { + pub fn sleep_hold() -> bool { super::STATE.with(|s| s.borrow().sleep_hold) } +} +fn call(name: &'static str) { CALLS.with(|c| c.borrow_mut().push(name)); } +mod device { + pub struct Device; + impl Device { pub fn as_raw(&self) -> *mut u8 { std::ptr::null_mut() } } +} +mod platform { + pub struct Device(pub super::device::Device, pub C); + impl Device { pub fn as_ref(&self) -> &super::device::Device { &self.0 } } +} +struct Core<'a>(PhantomData<&'a ()>); +struct AsahiDevice; +mod hw { + #[derive(Copy, Clone, PartialEq, Eq)] pub enum Platform { AdtWithPmp, DeviceTree } + #[derive(Copy, Clone, PartialEq, Eq)] pub enum Board { None, Templates, Blob } +} +struct Config { platform: hw::Platform, machine: &'static str, name: &'static str, firmware: (), board: hw::Board } +mod adt { + pub struct GpuAdt; + pub fn read(_: &super::device::Device, _: &super::Config) -> super::Result { super::call("adt"); Ok(GpuAdt) } +} +mod identity { + pub fn firmware(_: &super::device::Device, _: &super::adt::GpuAdt, _: ()) -> super::Result { super::call("identity"); Ok(()) } +} +mod initdata { + pub fn prepare_board(_: &super::device::Device, _: &super::Config, _: &super::adt::GpuAdt) -> super::Result { + super::call("board_images"); Ok(1) + } +} +mod regs { + pub struct Regs; + impl Regs { + pub fn new(_: &super::platform::Device) -> super::Result { super::call("regs"); Ok(Self) } + pub fn asc_control(&self) -> super::Result { + if super::STATE.with(|s| s.borrow().asc_fail) { Err(super::EIO) } else { Ok(0) } + } + } +} +mod glue { + use super::{call, STATE}; + pub fn board() -> bool { call("board"); STATE.with(|s| s.borrow().board) } + pub fn machine(_: &str) -> bool { call("machine"); STATE.with(|s| s.borrow().machine) } + pub unsafe fn asahi_g15_pmp_prepare(_: *mut core::ffi::c_void) -> core::ffi::c_int { call("pmp"); STATE.with(|s| s.borrow().pmp) } + pub unsafe fn asahi_g15_display_prepare(_: *mut core::ffi::c_void) -> core::ffi::c_int { call("display"); STATE.with(|s| s.borrow().display) } + pub fn dma_coherent(_: &super::device::Device) -> bool { call("coherent"); false } +@HOLD_POWER@ +} +struct Admitted { regs: regs::Regs, adt: adt::GpuAdt, board_images: u32 } +''' + +RS_MAIN = r''' +static T6031: Config = Config { platform: hw::Platform::DeviceTree, machine: "apple,j516c", name: "T6031 G15C", firmware: (), board: hw::Board::Blob }; +static T6030: Config = Config { platform: hw::Platform::AdtWithPmp, machine: "apple,t6030", name: "T6030 G15S", firmware: (), board: hw::Board::Templates }; +const TAIL: [&str; 6] = ["adt", "identity", "coherent", "board_images", "regs", "hold"]; +fn run(cfg: &'static Config, machine: bool, pmp: i32, display: i32) -> (Result<()>, Vec<&'static str>) { + CALLS.with(|c| c.borrow_mut().clear()); + STATE.with(|s| *s.borrow_mut() = State { board: true, machine, pmp, display, sleep_hold: true, asc_fail: false }); + HOLD.with(|h| *h.borrow_mut() = None); + WARNINGS.with(|w| *w.borrow_mut() = 0); + let pdev = platform::Device(device::Device, Core(PhantomData)); + let ret = admit(&pdev, &AsahiDevice, cfg).map(|_| ()); + (ret, CALLS.with(|c| c.borrow().clone())) +} +fn with_tail(head: &[&'static str]) -> Vec<&'static str> { head.iter().chain(TAIL.iter()).copied().collect() } +@SLEEP_HOLD_RUN@ +fn main() { + assert_eq!(run(&T6031, true, 0, -517), (Err(EPROBE_DEFER), vec!["machine", "display"]), "T6031 defers before firmware"); + assert_eq!(run(&T6031, true, 0, -19), (Err(ENODEV), vec!["machine", "display"]), "T6031 refusal stops admission"); + assert_eq!(run(&T6031, true, 0, 0), (Ok(()), with_tail(&["machine", "display"])), "T6031 admitted after its display"); + assert_eq!(run(&T6031, false, 0, 0), (Err(ENODEV), vec!["machine"]), "the machine check comes first"); + assert_eq!(run(&T6030, true, 0, -517), (Ok(()), with_tail(&["board", "pmp"])), "T6030 path unchanged"); + assert_eq!(run(&T6030, true, -517, 0), (Err(EPROBE_DEFER), vec!["board", "pmp"]), "T6030 still defers on its PMP link"); +@SLEEP_HOLD_CHECKS@ +} +''' + +HOLD_PLAIN = ' pub fn hold_power(_: &super::device::Device) { call("hold"); }' +HOLD_SLEEP = (' pub fn hold_power(_: &super::device::Device, sleep_hold: bool) {\n' + ' call("hold"); super::HOLD.with(|h| *h.borrow_mut() = Some(sleep_hold));\n }') +SLEEP_HOLD_RUN = r''' +fn run_hold(cfg: &'static Config, sleep_hold: bool, asc_fail: bool) -> (Result<()>, Vec<&'static str>, Option, u32) { + CALLS.with(|c| c.borrow_mut().clear()); + STATE.with(|s| *s.borrow_mut() = State { board: true, machine: true, pmp: 0, display: 0, sleep_hold, asc_fail }); + HOLD.with(|h| *h.borrow_mut() = None); + WARNINGS.with(|w| *w.borrow_mut() = 0); + let pdev = platform::Device(device::Device, Core(PhantomData)); + let ret = admit(&pdev, &AsahiDevice, cfg).map(|_| ()); + (ret, CALLS.with(|c| c.borrow().clone()), HOLD.with(|h| *h.borrow()), WARNINGS.with(|w| *w.borrow())) +} +fn before_hold(head: &[&'static str]) -> Vec<&'static str> { head.iter().chain(TAIL[..TAIL.len() - 1].iter()).copied().collect() } +''' +SLEEP_HOLD_CHECKS = r''' + // The system-sleep hold: T6031 only, and only while g15_sleep_hold is on; off is warned about. + assert_eq!(run_hold(&T6031, true, false), (Ok(()), with_tail(&["machine", "display"]), Some(true), 0), "T6031 holds gfx through sleep"); + assert_eq!(run_hold(&T6031, false, false), (Ok(()), with_tail(&["machine", "display"]), Some(false), 1), "g15_sleep_hold=0: no sleep hold, one warning"); + assert_eq!(run_hold(&T6030, true, false), (Ok(()), with_tail(&["board", "pmp"]), Some(false), 0), "T6030 never holds through sleep"); + // The ASC read comes before the hold: a failure there leaves no hold behind. + assert_eq!(run_hold(&T6031, true, true), (Err(EIO), before_hold(&["machine", "display"]), None, 0), "T6031: ASC failure takes no hold"); + assert_eq!(run_hold(&T6030, true, true), (Err(EIO), before_hold(&["board", "pmp"]), None, 0), "T6030: ASC failure takes no hold"); +''' +SLEEP_HOLD_MUTANTS = { + "sleep-hold-any-platform": ("let sleep_hold = t6031 && params::sleep_hold();", "let sleep_hold = params::sleep_hold();"), + "sleep-hold-ignores-parameter": ("let sleep_hold = t6031 && params::sleep_hold();", "let sleep_hold = t6031;"), + "no-warning-when-off": ("} else if t6031 {", "} else if false {"), + "asc-read-after-hold": (re.compile(r"(\n\s*)let asc = regs\.asc_control\(\)\?;(.*?glue::hold_power\(dev, sleep_hold\);)", re.S), + r"\2\1let asc = regs.asc_control()?;"), +} + +RS_MUTANTS = { + "display-refusal-ignored": ("dev_err!(dev, \"native display not admitted: {:?}\\n\", err);\n }\n return Err(err);", + "dev_err!(dev, \"native display not admitted: {:?}\\n\", err);\n }"), + "display-order-skipped": ("let ret = unsafe { glue::asahi_g15_display_prepare(dev.as_raw().cast()) };", "let ret = 0;"), +} + + +def mutate(source, change): + old, new = change + if isinstance(old, re.Pattern): + mutated, count = old.subn(new, source, count=1) + else: + count = source.count(old) + mutated = source.replace(old, new, 1) + assert count >= 1, f"mutation anchor not found: {old!r}" + return mutated + + +def compile_c(work, source, name): + src = work / f"{name}.c" + exe = work / name + src.write_text(C_PREFIX + source + C_MAIN) + result = subprocess.run(["gcc", "-std=gnu11", "-Wall", "-Werror", "-Wno-unused-function", + "-o", str(exe), str(src)], capture_output=True, text=True) + if result.returncode: + raise SystemExit(f"{name}: does not build:\n{result.stderr}") + return exe + + +def c_failures(exe): + return [s for s in C_SCENARIOS if subprocess.run([str(exe), s], capture_output=True).returncode] + + +def rs_model(admit): + """The prelude and main for this admit(): with the system-sleep hold argument (g15_sleep_hold) or without.""" + sleep = "glue::hold_power(dev, sleep_hold);" in admit + prefix = RS_PREFIX.replace("@HOLD_POWER@", HOLD_SLEEP if sleep else HOLD_PLAIN) + main = RS_MAIN.replace("@SLEEP_HOLD_RUN@", SLEEP_HOLD_RUN if sleep else "").replace("@SLEEP_HOLD_CHECKS@", SLEEP_HOLD_CHECKS if sleep else "") + return sleep, prefix, main + + +def compile_rs(work, rustc, source, name, model): + src = work / f"{name}.rs" + exe = work / name + src.write_text(model[1] + source + model[2]) + result = subprocess.run([rustc, "--edition=2021", "-o", str(exe), str(src)], capture_output=True, text=True) + if result.returncode: + raise SystemExit(f"{name}: does not build with {rustc}:\n{result.stderr}") + return exe + + +def static_checks(root): + apple = (root / APPLE).read_text() + module = (root / MODULE).read_text() + glue = (root / GLUE).read_text() + compat = re.compile(r'of::DeviceId::new\(c_str!\("([^"]+)"\)\)') + start = module.index("kernel::of_device_table!(\n AUTOLOAD_OF_TABLE,") + autoload = module[start:module.index("\n);", start)] + start = module.index("const OF_TABLE:") + matched = module[start:module.index("]);", start)] + assert compat.findall(autoload) == ["apple,agx-t6030", "apple,agx-t6031"], "autoload table" + assert compat.findall(matched) == ["apple,agx-t6030", "apple,agx-t6031"], "match table" + init = item(apple, "\nstatic int apple_drm_init(struct device *dev)\n{") + order = [init.index(s) for s in ("drm_dev_register(", "drm_client_setup_with_fourcc(", "apple_g15_display_publish(dev)")] + assert order == sorted(order), "publication must follow DRM registration and client setup" + uninit = item(apple, "\nstatic void apple_drm_uninit(struct device *dev)\n{") + assert uninit.index("apple_g15_display_withdraw(dev)") < uninit.index("component_unbind_all("), "withdraw before unbind" + assert "apple_t6030_display_publish" not in apple and "apple_t6030_display_withdraw" not in apple + pmp = item(glue, "\nint asahi_g15_pmp_prepare(struct device *dev)\n{") + assert pmp.index("apple_drm_t6030_link_device(dev)") < pmp.index("apple_pmp_link_device(dev)"), "T6030 order" + return 5 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--rustc", action="append", default=[], help="rustc to build the admit() model with") + args = parser.parse_args() + rustcs = args.rustc or ["rustc"] + checks = static_checks(args.root) + source = c_source(args.root) + admit = item((args.root / BOOT).read_text(), "pub(crate) fn admit(") + model = rs_model(admit) + rs_mutants = dict(RS_MUTANTS, **(SLEEP_HOLD_MUTANTS if model[0] else {})) + with tempfile.TemporaryDirectory(prefix="g15-display-order-", dir=os.environ.get("TMPDIR")) as tmp: + work = Path(tmp) + failed = c_failures(compile_c(work, source, "control")) + assert not failed, f"control scenarios failed: {failed}" + for name, change in C_MUTANTS.items(): + failed = c_failures(compile_c(work, mutate(source, change), f"mutant-{name}")) + assert failed, f"C mutant {name} survived" + print(f"mutant {name}: caught by {', '.join(failed)}") + for rustc in rustcs: + exe = compile_rs(work, rustc, admit, "admit", model) + run = subprocess.run([str(exe)], capture_output=True, text=True) + assert run.returncode == 0, f"admit() control failed with {rustc}:\n{run.stderr}" + for name, change in rs_mutants.items(): + exe = compile_rs(work, rustc, mutate(admit, change), f"admit-{name}", model) + run = subprocess.run([str(exe)], capture_output=True, text=True) + assert run.returncode, f"admit() mutant {name} survived with {rustc}" + print(f"mutant {name} ({rustc}): caught") + print(f"PASS: {checks} static checks, {len(C_SCENARIOS)} C scenarios, " + f"{len(C_MUTANTS)} C mutants, admit() on {len(rustcs)} rustc, {len(rs_mutants)} admit() mutants" + + (", system-sleep hold checks" if model[0] else "")) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/asahi/g15-input-error-tests.py b/tools/asahi/g15-input-error-tests.py index 809411963fd17..2dba24aa49204 100644 --- a/tools/asahi/g15-input-error-tests.py +++ b/tools/asahi/g15-input-error-tests.py @@ -2,8 +2,10 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT """Actual G15 job dependency completion and G17 status/policy bodies on host services. -DMA fence lookup, engine submission, queue-fence ordering and kernel mutexes are -models. No kernel scheduling, GPU access or firmware execution is exercised. +DMA fence lookup, engine submission, queue-fence ordering, VM slot allocation and +kernel mutexes are models. Production job binding/completion and prefix admission +bodies run against them. No kernel scheduling, GPU access or firmware execution +is exercised. """ import argparse import hashlib @@ -33,6 +35,7 @@ def extract(text, marker): type Result = std::result::Result; const EIO:Error=Error(-5);const ENODEV:Error=Error(-19);const ETIMEDOUT:Error=Error(-110); const ECANCELED:Error=Error(-125);const EINVAL:Error=Error(-22);const ENOMEM:Error=Error(-12);const ENOENT:Error=Error(-2); +const ENOSPC:Error=Error(-28);const EAGAIN:Error=Error(-11); macro_rules! pr_warn { ($($arg:tt)*)=>{{ let _=format_args!($($arg)*); }} } macro_rules! mod_pr_debug { ($($arg:tt)*)=>{{ let _=format_args!($($arg)*); }} } struct Mutex(StdMutex);impl Mutex{fn lock(&self)->MutexGuard<'_,T>{self.0.lock().unwrap()}} @@ -45,8 +48,17 @@ def extract(text, marker): mod g17 {use super::*;__POLICY__} mod bindings {pub unsafe fn dma_fence_get_status(p:*const i32)->i32{unsafe{*p}}} struct Fence(i32);impl Fence{fn raw(&self)->*const i32{&self.0}} -struct Vm{status:VmStatus} -struct Progress{signaled:AtomicBool} +struct Binding{slot:u32,drops:Arc} +impl Binding{fn slot(&self)->u32{self.slot}} +impl Drop for Binding{fn drop(&mut self){self.drops.fetch_add(1,Ordering::SeqCst);}} +struct Vm{status:VmStatus,refuse_binding:AtomicBool,next_slot:AtomicU32,allocations:AtomicU32,binding_drops:Arc} +impl Vm{fn try_bind(&self)->Result{ + if self.refuse_binding.load(Ordering::SeqCst){return Err(ENOSPC);} + self.allocations.fetch_add(1,Ordering::SeqCst); + Ok(Binding{slot:self.next_slot.load(Ordering::SeqCst),drops:self.binding_drops.clone()}) +}} +struct Progress{signaled:AtomicBool,next:AtomicU32,sent:AtomicU32} +impl Progress{fn next(&self)->u32{self.next.load(Ordering::SeqCst)}fn sent(&self)->u32{self.sent.load(Ordering::SeqCst)}} struct Guard{quarantines:AtomicU32}impl Guard{fn quarantine(&self){self.quarantines.fetch_add(1,Ordering::SeqCst);}} struct JobFence{vm:Arc,signals:AtomicU32,result:AtomicI32,saw_sticky:AtomicI32,saw_report:AtomicI32,saw_mirror:AtomicI32} impl JobFence{ @@ -61,12 +73,25 @@ def extract(text, marker): } struct Shared{crashed:AtomicBool,submits:AtomicU32,submit_result:Result} impl Shared{fn submit(&self,_:Arc)->Result{self.submits.fetch_add(1,Ordering::SeqCst);self.submit_result}} -struct Job{started:AtomicBool,vm:Arc,progress:Progress,_guard:Guard,fence:JobFence} +struct Job{started:AtomicBool,vm:Arc,progress:Progress,_guard:Guard,fence:JobFence,context:AtomicU32,binding:Mutex>} impl Job {__JOB__} +type Completions=Vec; +fn complete(done:&mut Completions,_job:&Arc,result:Result){done.push(result);} +struct Flight{claims:(),group:Vec>} +impl Flight{fn new(_job:Arc)->Self{Self{claims:(),group:vec![]}}} +struct Slots{releases:u32} +impl Slots{fn release(&mut self,_claims:&mut ()){self.releases+=1;}} +struct Engine{slot_wait:bool} +struct HostFirmware{engine:Engine,slots:Slots,publish_result:Result,publishes:u32} +impl HostFirmware{ + fn new(result:Result)->Self{Self{engine:Engine{slot_wait:false},slots:Slots{releases:0},publish_result:result,publishes:0}} + fn publish_prefix_into(&mut self,_job:&Arc,_flight:&mut Flight)->Result{self.publishes+=1;self.publish_result} + __PREFIX__ +} fn make_job(own:Option)->(Arc,Box){ - let mut mirror=Box::new(AtomicI32::new(0));let vm=Arc::new(Vm{status:VmStatus::new((&mut *mirror as *mut AtomicI32).cast())}); + let mut mirror=Box::new(AtomicI32::new(0));let vm=Arc::new(Vm{status:VmStatus::new((&mut *mirror as *mut AtomicI32).cast()),refuse_binding:AtomicBool::new(false),next_slot:AtomicU32::new(7),allocations:AtomicU32::new(0),binding_drops:Arc::new(AtomicU32::new(0))}); if let Some(error)=own{vm.status.record(error);} - let job=Arc::new(Job{started:AtomicBool::new(false),vm:vm.clone(),progress:Progress{signaled:AtomicBool::new(false)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm,signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)}}); + let job=Arc::new(Job{started:AtomicBool::new(false),vm:vm.clone(),progress:Progress{signaled:AtomicBool::new(false),next:AtomicU32::new(0),sent:AtomicU32::new(0)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm,signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)},context:AtomicU32::new(0),binding:Mutex(StdMutex::new(None))}); (job,mirror) } fn shared(crashed:bool,result:Result)->Arc{Arc::new(Shared{crashed:AtomicBool::new(crashed),submits:AtomicU32::new(0),submit_result:result})} @@ -90,7 +115,7 @@ def extract(text, marker): assert_eq!(job.fence.signals.load(Ordering::SeqCst),1,"dependency completion must signal once"); assert_eq!(job.vm.status.get(),0,"duplicate completion must not poison VM"); // A later job on the same VM is admitted despite the reported client error. - let later=Arc::new(Job{started:AtomicBool::new(false),vm:job.vm.clone(),progress:Progress{signaled:AtomicBool::new(false)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm:job.vm.clone(),signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)}}); + let later=Arc::new(Job{started:AtomicBool::new(false),vm:job.vm.clone(),progress:Progress{signaled:AtomicBool::new(false),next:AtomicU32::new(0),sent:AtomicU32::new(0)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm:job.vm.clone(),signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)},context:AtomicU32::new(0),binding:Mutex(StdMutex::new(None))}); later.start(&device,&[Fence(1)]);assert_eq!(device.submits.load(Ordering::SeqCst),1,"successful dependencies after report must submit"); } for own in [EIO,EINVAL]{for input in [ECANCELED,EIO,ENODEV,ETIMEDOUT,EINVAL]{ @@ -111,8 +136,70 @@ def extract(text, marker): } // An engine-owned fault still quarantines before completion. let(job,_mirror)=make_job(None);job.complete(Err(EIO));assert_eq!(job._guard.quarantines.load(Ordering::SeqCst),1);assert_eq!(job.vm.status.get(),EIO.to_errno()); + // Actual bind/release bodies: a job keeps its slot until explicitly released. + { + let(job,_mirror)=make_job(None); + job.vm.refuse_binding.store(true,Ordering::SeqCst); + assert_eq!(job.bind(),Err(ENOSPC));assert!(job.binding.lock().is_none()); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),0); + job.vm.refuse_binding.store(false,Ordering::SeqCst); + job.bind().unwrap();assert_eq!(job.slot(),7,"binding must publish its slot"); + job.vm.next_slot.store(9,Ordering::SeqCst); + job.bind().unwrap();assert_eq!(job.slot(),7,"repeated bind must keep existing binding"); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),1); + job.release_binding();job.release_binding(); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),1,"release_binding must drop exactly once"); + job.bind().unwrap();assert_eq!(job.slot(),9,"released job may acquire a different slot"); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),2); + } + // An engine failure must not recycle an ASID while retained job references live. + for result in [Ok(()),Err(EIO),Err(ETIMEDOUT),Err(ECANCELED)]{ + let(job,_mirror)=make_job(None);job.bind().unwrap();let drops=job.vm.binding_drops.clone(); + job.complete(result); + if result.is_err(){ + assert!(job.binding.lock().is_some(),"failed engine completion must retain binding"); + assert_eq!(drops.load(Ordering::SeqCst),0); + job.complete(Ok(()));assert!(job.binding.lock().is_some(),"duplicate completion must retain failed binding"); + let retained=job.clone();drop(job); + assert_eq!(drops.load(Ordering::SeqCst),0,"a retained reference must keep the binding"); + drop(retained);assert_eq!(drops.load(Ordering::SeqCst),1,"last reference releases failed binding"); + }else{ + assert!(job.binding.lock().is_none(),"successful engine completion releases binding"); + assert_eq!(drops.load(Ordering::SeqCst),1);drop(job);assert_eq!(drops.load(Ordering::SeqCst),1); + } + } + // Run the real prefix admission wrapper; only firmware publication is modeled. + { + let(job,_mirror)=make_job(None);let mut fw=HostFirmware::new(Ok(()));let mut done=vec![]; + job.vm.refuse_binding.store(true,Ordering::SeqCst); + assert_eq!(fw.publish_prefix(&job,&mut done),Err(EAGAIN)); + assert!(fw.engine.slot_wait,"slot exhaustion must request retry"); + assert_eq!(fw.publishes,0);assert!(done.is_empty());assert_eq!(job.vm.status.get(),0); + job.vm.refuse_binding.store(false,Ordering::SeqCst); + assert_eq!(fw.publish_prefix(&job,&mut done),Ok(()));assert_eq!(fw.publishes,1); + } + for published in [false,true]{ + let(job,_mirror)=make_job(None);job.bind().unwrap(); + job.progress.sent.store(u32::from(published),Ordering::SeqCst); + let mut fw=HostFirmware::new(Err(ENOMEM));let mut done=vec![]; + assert_eq!(fw.publish_prefix(&job,&mut done),Err(ENOMEM)); + assert_eq!(fw.slots.releases,1);assert!(done.is_empty()); + if published{ + assert!(job.binding.lock().is_some(),"published prefix must retain binding on append failure"); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),0); + }else{ + assert!(job.binding.lock().is_none(),"unpublished failure must release binding"); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),1); + } + } + { + let(job,_mirror)=make_job(None);let mut fw=HostFirmware::new(Err(EIO));let mut done=vec![]; + assert_eq!(fw.publish_prefix(&job,&mut done),Err(EIO)); + assert!(job.binding.lock().is_some(),"firmware-visible failure must retain binding"); + assert_eq!(fw.slots.releases,0); + } assert_eq!(unrelated.vm.status.get(),0,"unrelated VM untouched");assert_eq!(unrelated.vm.status.reported_error(),0);assert_eq!(other_mirror.load(Ordering::SeqCst),0);assert_eq!(unrelated.fence.signals.load(Ordering::SeqCst),0); - println!("PASS: actual Job::start/completion and VmStatus/policy: healthy/failed VMs, mirror ordering, one-shot completion, successful input, device faults and isolation"); + println!("PASS: production job/prefix/status bodies: input errors, binding reuse/release/retention, exhaustion retry, published-prefix lifetime and isolation"); } ''' @@ -130,17 +217,25 @@ def main(): current = (ROOT / job_path).read_text() status = (ROOT / "drivers/gpu/drm/asahi/g17/status.rs").read_text() policy = (ROOT / "drivers/gpu/drm/asahi/g17/dependency.rs").read_text() + engine = (ROOT / "drivers/gpu/drm/asahi/g15/engine.rs").read_text() methods = [extract(job, " pub(crate) fn complete("), extract(job, " fn finish("), extract(job, " fn start(self: &Arc")] # The old control needs the current helper for the duplicate-completion test; # it still runs its actual unmodified start, which reproduces the lost input error. methods.append(extract(job if "fn finish_dependency(" in job else current, " fn finish_dependency(")) + # Historical removed-fix controls predate publication bindings. Keep their + # actual start/completion bodies and use current binding helpers alongside. + binding_source = job if "fn release_binding(" in job else current + methods.extend(extract(binding_source, marker) for marker in [ + " pub(crate) fn slot(", " pub(crate) fn bind(", + " pub(crate) fn release_binding("]) bodies = "\n".join(methods) code = PRELUDE.replace("__JOB__", bodies).replace("__STATUS__", "\n".join( extract(status, marker) for marker in [" pub(crate) fn get(", " pub(crate) fn record(", " pub(crate) fn reported_error(", " pub(crate) fn report_failure(", " fn unobserved_teardown("])).replace( - "__POLICY__", extract(policy, "pub(crate) fn inherited_error(")) + "__POLICY__", extract(policy, "pub(crate) fn inherited_error(")).replace( + "__PREFIX__", extract(engine, " fn publish_prefix(")) variants = {"control": code} expected = {} if args.mutations: @@ -154,6 +249,11 @@ def main(): "success-refused": ("if let Some(status) = failed {", "if let Some(status) = failed.or(Some(-125)) {", "successful dependencies after report must submit"), "dependency-completes-twice": (" fn finish_dependency(&self, error: Error) {\n if self.progress.signaled.swap(true, Ordering::AcqRel) {\n return;\n }", " fn finish_dependency(&self, error: Error) {", "dependency completion must signal once"), "mirror-not-published": ("unsafe { AtomicI32::from_ptr(mirror.word) }.store(error, Ordering::SeqCst);", "let _ = mirror;", "mirror must precede fence signal"), + "failed-engine-releases-binding": ("if result.is_ok() || !engine {", "if true {", "failed engine completion must retain binding"), + "binding-release-leaks": ("drop(self.binding.lock().take());", "let _ = self;", "release_binding must drop exactly once"), + "bound-job-rebinds": ("if slot.is_none() {", "if true {", "repeated bind must keep existing binding"), + "slot-exhaustion-not-retried": ("self.engine.slot_wait = true;", "self.engine.slot_wait = false;", "slot exhaustion must request retry"), + "published-prefix-releases-binding": ("if job.progress.next() == job.progress.sent() {", "if true {", "published prefix must retain binding on append failure"), } for name, (old, new, message) in changes.items(): if code.count(old) != 1: @@ -162,7 +262,8 @@ def main(): expected[name] = message variants["control-after"] = code receipt = {"scope": __doc__, "ref": args.ref, "sources": {name: hashlib.sha256(text.encode()).hexdigest() - for name, text in [(job_path, job), ("g17/status.rs", status), ("g17/dependency.rs", policy)]}, "variants": {}} + for name, text in [(job_path, job), ("g15/engine.rs", engine), + ("g17/status.rs", status), ("g17/dependency.rs", policy)]}, "variants": {}} for name, source in variants.items(): path = args.output / (name + ".rs"); path.write_text(source) binary = args.output / name diff --git a/tools/asahi/g15-maintained-handoff-tests.py b/tools/asahi/g15-maintained-handoff-tests.py index 4dd2961d3a8ac..fdafd799645cb 100644 --- a/tools/asahi/g15-maintained-handoff-tests.py +++ b/tools/asahi/g15-maintained-handoff-tests.py @@ -137,10 +137,12 @@ def item(s, marker): OWNER_CALLS.set(0); let bytes = fixture(kind); assert!(parse(&dev, &Adt { data: &bytes }).is_err()); assert_eq!(OWNER_CALLS.get(), 0); } - assert_eq!(BUILDS.len(), 2, "maintained B0 firmware row must not require an absent profile symbol"); - assert_eq!(BUILDS[1].machine.unwrap().to_bytes(), b"apple,j516s"); - assert_eq!((BUILDS[1].text_size, BUILDS[1].data_size, BUILDS[1].marker), (0x5c000, 0x114000, 0x58e88)); - assert_eq!(BUILDS[1].text_sha256, [0xda,0x26,0xec,0xca,0xb1,0xf8,0x59,0x8d,0xa8,0x0c,0x85,0x5e,0xca,0x91,0x65,0x20,0x40,0x6d,0x1d,0x06,0x70,0xf1,0xd4,0x70,0xe7,0x9b,0xb9,0xb0,0x22,0x09,0xdb,0x53]); + assert_eq!(BUILDS.len(), 3, "maintained B0 firmware row must not require an absent profile symbol"); + for (row, machine) in [(1, &b"apple,j516s"[..]), (2, &b"apple,j514s"[..])] { + assert_eq!(BUILDS[row].machine.unwrap().to_bytes(), machine); + assert_eq!((BUILDS[row].text_size, BUILDS[row].data_size, BUILDS[row].marker), (0x5c000, 0x114000, 0x58e88)); + assert_eq!(BUILDS[row].text_sha256, [0xda,0x26,0xec,0xca,0xb1,0xf8,0x59,0x8d,0xa8,0x0c,0x85,0x5e,0xca,0x91,0x65,0x20,0x40,0x6d,0x1d,0x06,0x70,0xf1,0xd4,0x70,0xe7,0x9b,0xb9,0xb0,0x22,0x09,0xdb,0x53]); + } for error in [0, -517, -12] { INIT_ERROR.set(error); let shared = Arc::new(Shared); let result = Rtkit::new(&dev, &shared); assert_eq!(result.is_ok(), error == 0); @@ -156,13 +158,16 @@ def main(): p = argparse.ArgumentParser(description=__doc__); p.add_argument('--output', type=Path); p.add_argument('--mutations', action='store_true'); a=p.parse_args() out=(a.output or Path(tempfile.mkdtemp(prefix='g15-maintained-handoff-'))).resolve(); out.mkdir(parents=True, exist_ok=True) adt=(G15/'adt.rs').read_text(); boot=(G15/'boot.rs').read_text(); hw=(G15/'hw.rs').read_text() + record=(G15/'board/record.rs').read_text() helpers='\n'.join(item(adt,m) for m in ["pub(crate) struct GpuAdt", "struct Adt<'a>", 'fn le32(', 'fn le64(', 'fn strncmp_eq(', "impl Adt<'_>", 'fn collect_perf(', 'fn parse(', 'fn admit_regions(']) constants='\n'.join(re.search(r'pub\(crate\) const '+n+r':[^;]+;', hw).group(0) for n in ['PAGE','PA_MASK','TTBS_SIZE','TABLES_SIZE','HANDOFF_SIZE','SHARED_L2_SIZE','UPPER']) + # GpuAdt carries the T6031 sensor rows, sized by the record builder's production constant. + record_constants=re.search(r'pub\(crate\) const SENSOR_ROW_COUNT:[^;]+;', record).group(0) start=hw.index(' firmware: &['); start=hw.index('[',start); end=start+1; depth=1 while depth: depth+=(hw[end]=='[')-(hw[end]==']'); end+=1 firmware='static BUILDS: &[FirmwareBuild] = &'+hw[start:end]+';\n'+item(hw,'pub(crate) struct FirmwareBuild') constructor=item(boot,' fn new(dev: &device::Device, shared: &Arc)') - source=PREFIX+'\nmod hw {\n'+constants+'\n}\n'+helpers+'\n'+firmware+'\nimpl Rtkit {\n'+constructor+'\n}\n'+CASES + source=PREFIX+'\nmod hw {\n'+constants+'\n}\nmod record {\n'+record_constants+'\n}\n'+helpers+'\n'+firmware+'\nimpl Rtkit {\n'+constructor+'\n}\n'+CASES variants={'actual':source} if a.mutations: variants['skip-owner-check']=source.replace('if !glue::reserved_uat(i as u32, r.start, r.end - r.start) {','if false {') @@ -170,7 +175,7 @@ def main(): variants['drop-reserved-ram']=source.replace('Some(dram_base..dram_base + dram_size), true)?;', 'Some(dram_base..dram_base + dram_size), false)?;') variants['require-obsolete-b0-profile']=source.replace(' FirmwareBuild {\n name: "J516S B0', ' #[cfg(CONFIG_SILICONFOREST_NATIVE_GPU)]\n FirmwareBuild {\n name: "J516S B0') variants['named-mailbox']=source.replace('cookie, core::ptr::null(), 0, &RTKIT_OPS', 'cookie, c"mbox".as_ptr(), 0, &RTKIT_OPS') - receipt={'scope':__doc__,'sources':{p:hashlib.sha256(t.encode()).hexdigest() for p,t in [('adt.rs',adt),('boot.rs',boot),('hw.rs',hw)]},'variants':{}} + receipt={'scope':__doc__,'sources':{p:hashlib.sha256(t.encode()).hexdigest() for p,t in [('adt.rs',adt),('boot.rs',boot),('hw.rs',hw),('board/record.rs',record)]},'variants':{}} for name,text in variants.items(): if name!='actual' and text==source: raise SystemExit('NOT_APPLIED '+name) src=out/(name+'.rs');src.write_text(text); binary=out/name diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index 12ee4f16a4955..a1f2d67d7a327 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -3,16 +3,18 @@ """Run the G15 PB growth answer and its on-demand backing, actual bodies, against a model. The harness extracts from drivers/gpu/drm/asahi/g15/pools.rs Firmware::answer_growth(), -answer_request(), map_pb(), running_vm() and back_pb(), Pools::soft_max() and backed(), -PbChunk and Pools::new's PB backing statements; pb_max_blocks() from params.rs; the PB sizes, -tuning and PB leaf word from hw.rs; Prot::from_pte() and is_cached_noncoherent() from -pgtable.rs; and the parameter's default from asahi_core.rs. It includes pools/growth.rs, -fw/buffer.rs and fw/channels.rs whole. The bodies run against a model of the firmware objects, -the GEM backing allocation, the shared layer's map_at() and append_driver_mappings(), and the -DEVCTL wait, which logs every allocation, alias map, append, unmap, free, block record, -barrier, producer store, reply and log line in order. Allocation, alias map and append -failures can be injected. Source checks: publish_prefix_into() starts with the PB catch-up, -and pools.rs asserts the PB leaf GPU-only at compile time. +answer_request(), map_pb(), refill_backing(), note_idle(), reclaim_check(), free_backing() and +reclaim_delay_ns(), Pools::soft_max(), map_heap(), free_chunks() and back_chunk(), PbChunk, +chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from vm.rs VmState::map_one(), +map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks() from params.rs, whose +pb_blocks(), pb_reclaim(), pb_idle_ns() and pb_reclaim_blocks() must return the fixed hw.rs tuning +values; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and +is_cached_noncoherent() from pgtable.rs; and the g15_pb_max_blocks default from asahi_core.rs. +It includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies run against a model of the firmware objects, the GEM backing allocation, the shared layer's map_at() +and the DEVCTL wait, which logs every allocation, alias map, unmap, free, block record, barrier, +producer store, reply and log line in order. Allocation and alias map failures can be injected. +Source checks: publish_prefix_into() starts with the PB catch-up, and pools.rs asserts the PB leaf +GPU-only at compile time. What it shows: - a poll pass answers only the requests queued when it starts, in order; one the reply's wait @@ -21,22 +23,42 @@ offset, each producer store follows its records and a barrier, each reply follows its producer store and a barrier, and every producer is in 1..=4095, ending at 4095; - at the maximum two refusals are answered; further requests get no reply, count as bad and - warn once until the ledger is made again, as init() does; + warn once until the ledger is reset, as init() does; - the soft maximum is the parameter clamped to 64..=4095 and then to the ledger's initial..=max; growth answers above it are logged at info level, and the one that reaches it exactly is not; - invalid requests (type, slot, stale counter, epoch, no live render) get no reply, and requests rejected so allocate nothing; -- the backing: the initial blocks only when the GPU starts; chunks of at most 4 MiB, allocated - and held by the running VM before any record or producer names their blocks, never past the - maximum, at most growth::max_chunks(); a failed allocation, alias map or append posts and - answers nothing, leaves the request counter and the VM's count alone and drops what is not - held, and the next request is answered; a failed reply keeps everything; -- a VM's count always equals the aliases it holds, contiguous from the heap's start; a VM that - runs after another grew the backing maps it first (at publication, and again before it grows); - a partial catch-up counts only held aliases; -- the one-VM rule: a catch-up with a flight of another VM, or a count past the chunks, and a - grant with flights of two VMs, are refused (EPROTO) before anything is mapped or posted; +- the backing: the chunks of the initial posting and one reserve chunk when the GPU starts; + chunks of 16 MiB, backed after the replies of a pass (never between them) while fewer than a + reserve chunk remain, mapped into the VM of the flights before any record or producer names + their blocks, never past the heap; with the reserve dry a request waits at the head of the + queue, keeps its counter and is answered once a chunk is backed; a chunk that cannot be mapped + into the running VM is dropped again; a pending re-initialization backs nothing while no + render is published, and still backs for a render published before it; a failed reply keeps + everything; +- a VM's aliases are the chunks, contiguous from the heap's start; a VM that runs after another + grew the backing maps it first (at publication); a partial catch-up keeps the aliases made; a + chunk freed and backed again (an idle reclaim) is unmapped and mapped again at publication; +- the one-VM rule: a catch-up with a flight of another VM is refused (EPROTO) before anything + is mapped; +- the idle reclaim, actual reclaim_check(), free_backing() and note_idle() over a clock: a heavy + client that exits with nothing rendered after it is re-initialized after 50 ms with the backing + kept, and the kept backing is freed after the long idle, its posted count kept until the InitBM; + at reclaim level 1 the backing stays; a recovery's pending re-initialization frees nothing; + driven as the idle poll work arms it (reclaim_delay_ns()), the same exit wakes at 50 ms and at + the long idle and arms nothing after; an idle reclaim arms nothing once its re-initialization + is pending; +- nothing grown since the initialization (an initial posting above half the maximum, or a + reclaim floor below it): no re-initialization and nothing armed; +- the actual reclaim keeps the posted count until the next render's InitBM (a model of the + publication: map_pb() first, then for a render prefix with a re-initialization pending the + ledger reset of Pools::init()): the same client rendering again after 60 ms re-posts the + initial blocks and grows again from the kept backing with nothing allocated; after 1.1 s it + grows again over freshly backed chunks, each mapped before it is posted; a compute-only + submission between the reclaim and the InitBM maps, backs and posts nothing; +- a reclaim whose walk of the VMs fails counts nothing freed; the VM's next publication drops + its stale aliases and releases the pages; - the ledger reset of Pools::init() re-posts from the backing without allocating; - the PB leaf decodes as GPU-only, not cached-noncoherent: dropping an alias is a plain unmap; - Ledger::new() is Ledger::new() as of dba2a110de00 (the carried upstream constructor), capped at @@ -47,8 +69,7 @@ refusal or an unanswered request. The model's VMs have no open/closed state: that a closed VM whose jobs still run keeps its aliases comes from the VM lifetime source (Job and VmJobGuard hold the VM), not from these tests. The ledger reset is not a recovery. The shared layer's -mapping and its rollback are modelled, not run. Running only running_vm()'s or only map_pb()'s -one-VM check on a grant is not distinguishable: each covers it. +mapping is modelled, not run. With --mutations, each mutation below must fail its intended test with the expected message, and the control (a driver maximum of the whole 4096-block heap) must pass, held by the ledger's cap. @@ -75,6 +96,7 @@ HW = f"{G15}/hw.rs" CORE = "drivers/gpu/drm/asahi/asahi_core.rs" ENGINE = f"{G15}/engine.rs" +VM = f"{G15}/vm.rs" PGTABLE = "drivers/gpu/drm/asahi/pgtable.rs" OLD_REF = "dba2a110de00743eeecd41edf23e54ede4491d5e" @@ -97,14 +119,16 @@ } }''' -# Pools::new's PB backing, run verbatim by the model. -POOLS_NEW_BACKING = (' // Backing for the initial blocks; growth adds the rest.\n', ' chunks.push(') +# Pools::new's PB backing, run verbatim by the model (up to its log line). +POOLS_NEW_BACKING = (' // Backing for the initial blocks and the reserve; growth adds the rest.\n', + ' dev_info!(\n') # The compile-time proof that a PB alias drop is a plain unmap. PB_LEAF_ASSERT = 'static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent());' -# The two statements of Pools::init() that start a new ledger. -LEDGER_RESET = ['self.ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS);', - 'self.exhausted_logged = false;'] +# The two statements of Pools::init() that reset the ledger. +LEDGER_RESET = ['self.ledger.reinit();', + 'self.exhausted_logged = false;', + 'self.reclaim_pending = false;'] MUTATIONS = { # Answer until the queue is empty, including requests queued by the replies' waits. @@ -112,9 +136,9 @@ # Leave the last request queued before the pass for later. 'skip-last-queued': (POOLS, 'for _ in 0..queued {', 'for _ in 0..queued.saturating_sub(1) {'), # End the pass at a request that cannot be processed. - 'return-after-request-error': (POOLS, ' self.pools.stats.bad += 1;\n dev_err!(', - ' self.pools.stats.bad += 1;\n return Ok(());\n' - ' dev_err!('), + 'return-after-request-error': (POOLS, ' self.pools.stats.bad += 1;\n dev_err!(', + ' self.pools.stats.bad += 1;\n return Ok(());\n' + ' dev_err!('), # Answer three refusals at the maximum. 'three-refusals': (GROWTH, 'pub(crate) const MAX_REFUSALS: u32 = 2;', 'pub(crate) const MAX_REFUSALS: u32 = 3;'), # Answer refusals for ever. @@ -140,42 +164,115 @@ 'let nb = clamp(blocks, MIN_BLOCKS, MAX_BLOCKS);\n' ' (nb, clamp(max_blocks, nb, MAX_BLOCKS))\n' ' } else {\n (MAX_BLOCKS, MAX_BLOCKS)'), - # Post blocks without backing them first. - 'grant-without-backing': (POOLS, ' self.back_pb(end, ctx)?;\n let (first, n) = self.pools.ledger.accept(request);', - ' let (first, n) = self.pools.ledger.accept(request);'), - # Back the blocks only after the request is accepted. - 'back-after-accept': (POOLS, ' self.back_pb(end, ctx)?;\n let (first, n) = self.pools.ledger.accept(request);', - ' let (first, n) = self.pools.ledger.accept(request);\n self.back_pb(end, ctx)?;'), - # Count a catch-up alias before the VM holds it. - 'catch-up-cursor-before-append': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n mapped += 1;\n' - ' vm.pb_chunks.store(mapped, Ordering::Relaxed);', - ' mapped += 1;\n vm.pb_chunks.store(mapped, Ordering::Relaxed);\n' - ' vm.vm.append_driver_mappings(aliases)?;'), - # Count a growth chunk's alias before the VM holds it. - 'growth-cursor-before-append': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.', - ' vm.pb_chunks.store(self.pools.chunks.len() + 1, Ordering::Relaxed);\n' - ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.'), - # Keep a chunk whose alias the VM never held. - 'keep-unappended-chunk': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.\n' - ' self.pools\n .chunks\n .push_within_capacity(chunk)\n' - ' .map_err(|_| EIO)?;', - ' // Within the capacity checked above.\n self.pools\n .chunks\n' - ' .push_within_capacity(chunk)\n .map_err(|_| EIO)?;\n' - ' vm.vm.append_driver_mappings(aliases)?;'), - # Grow without catching the running VM up first. - 'no-growth-catch-up': (POOLS, ' let vm = self.running_vm(ctx)?;\n self.map_pb(&vm)?;\n', - ' let vm = self.running_vm(ctx)?;\n'), + # Grant without the backing: neither wait nor clip. + 'grant-without-backing': [(GROWTH, ' if room != 0 && self.spare() == 0 {\n return Answer::Wait;\n }\n', ''), + (GROWTH, 'count: GROW_STEP.min(room).min(self.spare()),', 'count: GROW_STEP.min(room),')], + # Allocate before the pass's answers. + 'refill-before-answers': [(POOLS, ' self.refill_backing();\n Ok(())', ' Ok(())'), + (POOLS, ' let queued = self.events.grow.len();\n', + ' self.refill_backing();\n let queued = self.events.grow.len();\n')], + # Back a chunk without mapping it into the running VM. + 'refill-maps-no-flight': (POOLS, ' flight.job.vm.map_chunk(i, gem.clone())?;\n', + ' let _ = (flight, i, gem.clone());\n'), + # Drop a request that waits for backing. + 'no-wait-requeue': (POOLS, ' if self.events.grow.insert_within_capacity(0, request).is_err() {\n' + ' self.pools.stats.bad += 1;\n }\n', ''), + # Keep a chunk that the running VM could not map. + 'refill-failure-keeps-chunk': (POOLS, ' self.pools.chunks.truncate(backed);\n', ''), + # Back chunks while a re-initialization is pending. + 'refill-during-reinit': (POOLS, ' if self.recovery.reinit && !render {\n return;\n }\n', ''), + # Skip the refill on any pending re-initialization, also with a live render (a stall). + 'refill-skip-with-live-render': (POOLS, 'if self.recovery.reinit && !render {', 'if self.recovery.reinit {'), + # Refill for any live flight, also compute-only ones that cannot ask for growth. + 'refill-for-any-flight': (POOLS, 'if self.recovery.reinit && !render {', + 'if self.recovery.reinit && self.engine.flights.is_empty() {'), + # Keep a VM's alias of a chunk that was freed and backed again. + 'map-without-identity': (VM, ' if core::ptr::eq(&*have.gem, &*gem) {', ' if true {'), + # Stop the reclaim check on a reclaim's pending re-initialization (the kept backing stays). + 'pending-reinit-stops-free': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + # Free the kept backing at reclaim level 1. + 'pending-free-at-level-1': (POOLS, 'if level >= 2 && ledger.pending_free_due(idle_ns, min_idle) {', + 'if ledger.pending_free_due(idle_ns, min_idle) {'), + # Do not record that the pending re-initialization is a reclaim's. + 'reclaim-pending-not-set': (POOLS, ' self.pools.reclaim_pending = true;\n', ''), + # Free for any pending re-initialization, also a recovery's. + 'free-on-recovery-reinit': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + # Both checks stop on any pending re-initialization: the code before the kept-backing free. + 'old-reinit-guards': [(POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n')], + # Arm nothing while a reclaim's re-initialization is pending. + 'pending-reinit-arms-nothing': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n'), + # Arm the poll for the expired 50 ms instead of the rest of the long idle. + 'pending-delay-from-drain': (GROWTH, 'Some(min_idle_ns.saturating_sub(idle_ns))', + 'Some(DRAIN_IDLE_NS.saturating_sub(idle_ns))'), + # Re-initialize at an idle with nothing grown (posted above the floor only). + 'reclaim-due-nothing-grown': (GROWTH, 'self.posted > self.initial && self.posted > floor && idle_ns >= min_idle_ns', + 'self.posted > floor && idle_ns >= min_idle_ns'), + # Re-initialize at a client boundary with nothing grown (initial above half the maximum). + 'drain-due-nothing-grown': (GROWTH, 'self.posted > self.initial && self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS', + 'self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS'), + # Arm the poll with nothing grown. + 'delay-nothing-grown': (GROWTH, ' let grown = self.posted > self.initial;\n', ' let grown = true;\n'), + # Reset the posted count at the idle reclaim instead of at the next render's InitBM. + 'reinit-at-idle-reclaim': (POOLS, ' if level >= 2 && idle {\n freed = self.free_backing();\n', + ' if level >= 2 && idle {\n self.pools.ledger.reinit();\n freed = self.free_backing();\n'), + # Reset the posted count at any reclaim instead of at the next render's InitBM. + 'reinit-at-any-reclaim': (POOLS, ' self.pools.reclaim_pending = true;\n', + ' self.pools.reclaim_pending = true;\n self.pools.ledger.reinit();\n'), + # Count the chunks freed although the VMs could not be walked. + 'freed-without-walk': (POOLS, ' );\n 0\n }\n };\n', + ' );\n dropped\n }\n };\n'), # Catch up without the one-VM check. - 'no-activation-guard': (POOLS, '.any(|f| !core::ptr::eq(&*f.job.vm, vm))', '.any(|_| false)'), - # Neither the catch-up's nor the grant's one-VM check (each alone covers a grant). - 'no-same-vm-checks': [(POOLS, '.any(|f| !core::ptr::eq(&*f.job.vm, vm))', '.any(|_| false)'), - (POOLS, 'if vm.slot != ctx || flights.iter().any(|f| !Arc::ptr_eq(&f.job.vm, &vm)) {', - 'if vm.slot != ctx {')], + 'no-activation-guard': (POOLS, '.filter(|f| !core::ptr::eq(&*f.job.vm, vm))', '.filter(|_| false)'), # Back the whole heap when the GPU starts. - 'initial-backs-whole-heap': (POOLS, 'chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?;', - 'chunks.push(PbChunk::new(dev, 0, buffer::MAX_BLOCKS)?, GFP_KERNEL)?;'), - # Growth chunks of 8 MiB. - 'chunks-above-4mib': (GROWTH, 'pub(crate) const CHUNK_BLOCKS: u32 = 32;', 'pub(crate) const CHUNK_BLOCKS: u32 = 64;'), + 'initial-backs-whole-heap': (POOLS, 'for _ in 0..Ledger::chunks_for(ledger.initial) {', + 'for _ in 0..growth::MAX_CHUNKS {'), + # Growth chunks of 32 MiB. + 'chunks-above-16mib': (GROWTH, 'pub(crate) const CHUNK_BLOCKS: u32 = 128;', 'pub(crate) const CHUNK_BLOCKS: u32 = 256;'), # A firmware-visible cached PB leaf (its drop would ask the firmware to flush). 'pb-leaf-fw-visible': (HW, 'pub(crate) const LEAF_CACHED: u64 = 0x00c0000000000c83;', 'pub(crate) const LEAF_CACHED: u64 = 0x00c0000000000c03;'), @@ -183,28 +280,43 @@ # The test each mutation must fail, and a piece of its failure message. INTENDED = { - 'drain-until-empty': ('queued_snapshot_keeps_order', 'unbounded answering in one poll pass'), + 'drain-until-empty': ('queued_snapshot_keeps_order', 'replies in the first pass'), 'skip-last-queued': ('queued_snapshot_keeps_order', 'replies in the first pass'), 'return-after-request-error': ('unprocessed_request_does_not_end_the_pass', 'requests taken in the pass'), 'three-refusals': ('default_growth_to_exhaustion', 'want Some(Log("warn"'), 'never-exhausted': ('default_growth_to_exhaustion', 'growth did not end'), 'warn-every-time': ('default_growth_to_exhaustion', 'later request after exhaustion'), - 'no-record-barrier': ('default_growth_to_exhaustion', 'got Some(Producer(1034)), want Some(Sync)'), + 'no-record-barrier': ('default_growth_to_exhaustion', 'got Some(Producer(74)), want Some(Sync)'), 'no-reply-barrier': ('default_growth_to_exhaustion', 'got Some(Reply([7, 1, 0, 5, 0, 256, 3, 1, 0, 0, 0, 0, 0, 0])), want Some(Sync)'), 'soft-boundary-inclusive': ('soft_max_boundaries', 'got Some(Log("info", "")), want Some(Log("dbg", ""))'), 'soft-max-unclamped': ('soft_max_bounds', 'soft_max, param 0'), 'uncapped-ledger': ('constructor_is_the_old_one_capped', 'new(true, '), - 'grant-without-backing': ('growth_backs_before_posting', 'record of block'), - 'back-after-accept': ('allocation_failure_posts_nothing', 'ledger after a backing failure'), - 'catch-up-cursor-before-append': ('partial_catch_up_counts_only_held_aliases', 'count after a failed append'), - 'growth-cursor-before-append': ('append_failure_posts_nothing', 'count differs from held aliases'), - 'keep-unappended-chunk': ('append_failure_posts_nothing', 'chunks after a backing failure'), - 'no-growth-catch-up': ('growth_catches_the_running_vm_up_first', 'running VM not caught up'), + 'grant-without-backing': ('dry_reserve_waits_and_keeps_order', 'record of block'), + 'refill-before-answers': ('allocation_follows_the_replies', 'allocation before a reply'), + 'refill-maps-no-flight': ('growth_backs_before_posting', 'record of block'), + 'no-wait-requeue': ('dry_reserve_waits_and_keeps_order', 'waiting request lost'), + 'refill-failure-keeps-chunk': ('refill_map_failure_drops_the_chunk', 'chunk kept after a failed map'), + 'refill-during-reinit': ('pending_reinit_without_flights_backs_nothing', 'backed during a pending re-initialization'), + 'refill-skip-with-live-render': ('pending_reinit_with_a_live_render_refills', 'growth request stranded'), + 'refill-for-any-flight': ('pending_reinit_with_compute_only_backs_nothing', 'backed for a compute-only flight'), + 'map-without-identity': ('freed_chunks_are_mapped_again', 'alias of another object'), + 'pending-reinit-stops-free': ('client_boundary_reinit_arms_the_long_idle', 'poll woke in a loop'), + 'old-reinit-guards': ('client_boundary_reinit_arms_the_long_idle', 'kept backing not freed'), + 'pending-reinit-arms-nothing': ('client_boundary_reinit_arms_the_long_idle', 'kept backing not freed'), + 'pending-delay-from-drain': ('client_boundary_reinit_arms_the_long_idle', 'poll woke in a loop'), + 'pending-free-at-level-1': ('client_boundary_reinit_at_level_1_keeps_the_backing', 'freed at level 1'), + 'reclaim-pending-not-set': ('client_boundary_reinit_frees_without_a_render', 'kept backing not freed'), + 'free-on-recovery-reinit': ('recovery_reinit_frees_nothing', 'freed for a recovery'), + 'reclaim-due-nothing-grown': ('nothing_grown_reinitializes_nothing', 're-initialization with nothing grown (64 blocks, floor 32)'), + 'drain-due-nothing-grown': ('nothing_grown_reinitializes_nothing', 're-initialization with nothing grown (3000 blocks'), + 'delay-nothing-grown': ('nothing_grown_arms_nothing', 'poll woke in a loop'), + 'reinit-at-idle-reclaim': ('same_client_render_after_1_1s', 'posted count reset before the InitBM (1500)'), + 'reinit-at-any-reclaim': ('same_client_render_after_60ms', 'posted count reset before the InitBM'), + 'freed-without-walk': ('failed_vm_walk_counts_nothing_freed', 'freed counted with the VM walk failed'), 'no-activation-guard': ('catch_up_refuses_flights_of_another_vm', 'catch-up with a flight of another VM'), - 'no-same-vm-checks': ('grant_refuses_flights_of_two_vms', 'grant with flights of two VMs'), 'initial-backs-whole-heap': ('startup_backs_the_initial_blocks', 'startup allocation'), - 'chunks-above-4mib': ('growth_backs_before_posting', 'a growth chunk above 4 MiB'), + 'chunks-above-16mib': ('growth_backs_before_posting', 'a growth chunk above 16 MiB'), 'pb-leaf-fw-visible': ('pb_leaf_is_gpu_only', 'PB leaf not GPU-only'), } assert set(INTENDED) == set(MUTATIONS) @@ -292,6 +404,26 @@ def consts(text, names): pub fn last(&self) -> Option<&T> { self.0.last() } + pub fn truncate(&mut self, n: usize) { + self.0.truncate(n) + } + pub fn reserve(&mut self, n: usize, _: Flags) -> Result { + self.0.reserve(n); + Ok(()) + } + pub fn iter(&self) -> std::slice::Iter<'_, T> { + self.0.iter() + } +} +/// A kernel mutex, single-threaded here. +pub struct Mutex(pub RefCell); +impl Mutex { + pub fn new(value: T) -> Self { + Mutex(RefCell::new(value)) + } + pub fn lock(&self) -> std::cell::RefMut<'_, T> { + self.0.borrow_mut() + } } #[macro_export] macro_rules! static_assert { @@ -311,20 +443,27 @@ def consts(text, names): /// Backing object `id` of this many bytes allocated, or freed (its last reference gone). Alloc(u32, usize), Free(u32), - /// A PB alias of object `id` at GVA in the VM with this context, made, held by the VM - /// (appended to its driver mappings), or unmapped. + /// A PB alias of object `id` at GVA in the VM with this context, made or unmapped. Map(u32, u64, usize, u32), - Append(u32, u64), Unmap(u32, u64), } thread_local! { pub static LOG: RefCell> = RefCell::new(Vec::new()); pub static PARAM: Cell = Cell::new(PARAM_DEFAULT); - /// Failures to inject: the next this many allocations, alias maps, appends fail. + pub static PARAM_BLOCKS: Cell = Cell::new(PB_BLOCKS_DEFAULT); + /// The initial posting and the reclaim values (hw::tuning PB_BLOCKS, PB_RECLAIM, + /// PB_IDLE_MS and PB_RECLAIM_BLOCKS), which a test may vary from the driver's. + pub static PARAM_RECLAIM: Cell = Cell::new(PB_RECLAIM_DEFAULT); + pub static PARAM_IDLE_MS: Cell = Cell::new(PB_IDLE_MS_DEFAULT); + pub static PARAM_RECLAIM_BLOCKS: Cell = Cell::new(PB_RECLAIM_BLOCKS_DEFAULT); + /// The monotonic clock (engine::now_ns()). + pub static NOW: Cell = Cell::new(1_000_000_000); + /// The VM registry walk (vms.open()) fails this many times. + pub static FAIL_OPEN: Cell = Cell::new(0); + /// Failures to inject: the next this many allocations or alias maps fail. pub static FAIL_ALLOC: Cell = Cell::new(0); pub static FAIL_MAP: Cell = Cell::new(0); pub static MAP_OK_BEFORE_FAIL: Cell = Cell::new(0); - pub static FAIL_APPEND: Cell = Cell::new(0); pub static NEXT_ID: Cell = Cell::new(1); } /// Takes one injected failure of `which`. @@ -391,7 +530,7 @@ def consts(text, names): } } -/// The shared layer's VM: kernel aliases at fixed addresses, held by the VM once appended. +/// The shared layer's VM: kernel aliases at fixed addresses. pub mod mmu { use crate::prelude::*; pub struct Prot(pub u64); @@ -414,11 +553,10 @@ def consts(text, names): } pub struct Vm { pub slot: u32, - pub held: std::cell::RefCell>, } impl Vm { pub fn new(slot: u32) -> Self { - Vm { slot, held: Default::default() } + Vm { slot } } pub fn map_at( &self, @@ -439,24 +577,9 @@ def consts(text, names): if !skip && crate::fail(&crate::FAIL_MAP) { return Err(ENOMEM); } - // Leaves already in use: EBUSY (prepare_map). - let end = addr + size as u64; - if self.held.borrow().iter().any(|m| m.gva < end && addr < m.gva + m.size as u64) { - return Err(EBUSY); - } crate::ev(crate::Ev::Map(self.slot, addr, size, gem.id)); Ok(KernelMapping { slot: self.slot, gva: addr, size, prot: prot.0, gem: gem.clone() }) } - pub fn append_driver_mappings(&self, mappings: KVec) -> Result { - if crate::fail(&crate::FAIL_APPEND) { - return Err(ENOMEM); // the mappings drop here - } - for m in mappings.0 { - crate::ev(crate::Ev::Append(self.slot, m.gva)); - self.held.borrow_mut().push(m); - } - Ok(()) - } } } ''' @@ -479,29 +602,74 @@ def consts(text, names): pub struct Init { pub control: Option>, } + /// vm.rs's chunk mapping, its fields as the driver names them. + pub struct HeapMapping { + pub gem: ARef, + pub _mapping: mmu::KernelMapping, + } + pub struct HeapMappings { + pub list: Mutex>, + } pub struct VmState { + pub id: u64, pub slot: u32, pub vm: mmu::Vm, - pub pb_chunks: AtomicUsize, + pub heap: Arc, } impl VmState { pub fn new(slot: u32) -> Arc { - Arc::new(VmState { slot, vm: mmu::Vm::new(slot), pb_chunks: AtomicUsize::new(0) }) + Arc::new(VmState { + // Keep VM identity distinct from the modeled hardware slot. + id: 0x10000 + u64::from(slot), + slot, + vm: mmu::Vm::new(slot), + heap: Arc::new(HeapMappings { list: Mutex::new(KVec(Vec::new())) }), + }) } } pub struct Job { pub vm: Arc, } + impl Job { + pub fn slot(&self) -> u32 { + self.vm.slot + } + } pub struct Flight { pub job: Job, } pub struct Engine { pub flights: Vec, + /// Jobs accepted and not fully published. + pub pending: Vec, + pub frozen: bool, + } + pub struct Recovery { + pub reinit: bool, + pub armed: Option<(usize, u64)>, + pub failed: Option, + } + impl Recovery { + pub fn failed(&self) -> Option { + self.failed + } + } + /// vm.rs's Registry: the VMs whose client handle may still be open. + pub struct Registry { + pub vms: Vec>, + } + impl Registry { + pub fn open(&self) -> Result>> { + if crate::fail(&crate::FAIL_OPEN) { + return Err(ENOMEM); + } + Ok(KVec(self.vms.clone())) + } } pub struct Receipts { pub renders: u32, } - /// A firmware object: the block list, block_ctl or the heap. + /// A firmware object: the block list or block_ctl. pub struct FwObject { pub gva: Option, pub size: usize, @@ -534,6 +702,10 @@ def consts(text, names): pub grown: u64, pub refused: u64, pub bad: u64, + pub waits: u64, + pub refills: u64, + pub reclaims: u64, + pub freed: u64, } pub struct Live { pub block_ctl: FwObject, @@ -546,6 +718,8 @@ def consts(text, names): pub live: Live, pub drm: ARef, pub chunks: KVec, + pub idle: growth::IdleClock, + pub reclaim_pending: bool, } impl Pools { pub fn chunk_count(&self) -> usize { @@ -554,6 +728,10 @@ def consts(text, names): pub fn chunk_capacity(&self) -> usize { self.chunks.capacity() } + /// The backing objects' ids, in heap order. + pub fn chunk_ids(&self) -> Vec { + self.chunks.iter().map(|c| c.backing.gem.id).collect() + } } pub struct Queue(pub std::collections::VecDeque<[u32; 14]>); impl Queue { @@ -567,6 +745,11 @@ def consts(text, names): assert_eq!(i, 0); self.0.pop_front().ok_or(()) } + pub fn insert_within_capacity(&mut self, i: usize, q: [u32; 14]) -> core::result::Result<(), [u32; 14]> { + assert_eq!(i, 0); + self.0.push_front(q); + Ok(()) + } } pub struct Events { pub grow: Queue, @@ -599,6 +782,8 @@ def consts(text, names): pub pools: Pools, pub init: Init, pub engine: Engine, + pub recovery: Recovery, + pub vms: Registry, pub renders: Vec, pub fw: FwModel, } @@ -610,11 +795,12 @@ def consts(text, names): let vm = VmState::new(5); let mut f = Self::without_flights(); f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); + f.vms.vms.push(vm.clone()); f.map_pb(&vm).unwrap(); crate::take(); f } - /// Pools::new's PB backing: the initial chunk, room for every chunk the ledger needs. + /// Pools::new's PB backing: the initial chunks and the reserve, room for the heap. pub fn without_flights() -> Self { let drm = ARef::new(AsahiDevice); let (ledger, chunks) = pools_new_backing(&drm).unwrap(); @@ -623,6 +809,8 @@ def consts(text, names): pools: Pools { ledger, exhausted_logged: false, + idle: growth::IdleClock::default(), + reclaim_pending: false, stats: PbStats::default(), drm, chunks, @@ -636,7 +824,9 @@ def consts(text, names): }, }, init: Init { control: Some(std::sync::Arc::new(Ctl { epoch: 0x1_0000_0003 })) }, - engine: Engine { flights: Vec::new() }, + engine: Engine { flights: Vec::new(), pending: Vec::new(), frozen: false }, + recovery: Recovery { reinit: false, armed: None, failed: None }, + vms: Registry { vms: Vec::new() }, renders: vec![1], fw: FwModel { ctx: 5, @@ -693,10 +883,10 @@ def consts(text, names): use crate::g15::firmware::{Firmware, VmState, Flight, Job, HEAP}; use crate::g15::hw::{client, pte}; use crate::g15::fw::buffer::{BLOCK_BYTES, MAX_BLOCKS}; - use crate::g15::pools::growth::{max_chunks, CHUNK_BLOCKS}; + use crate::g15::pools::growth::{CHUNK_BLOCKS, MAX_CHUNKS}; + use crate::g15::pools::{chunk_gva, CHUNK_BYTES}; use crate::g15::pgtable::Prot; - use crate::{EPROTO, FAIL_ALLOC, FAIL_APPEND, FAIL_MAP}; - use core::sync::atomic::Ordering; + use crate::{EPROTO, FAIL_ALLOC, FAIL_MAP}; use std::sync::Arc; use crate::g15::fw::buffer::{self, grow_reply, BLOCK_LIST_RECORDS, GROW_STEP}; use crate::g15::fw::channels::devctl; @@ -777,7 +967,7 @@ def consts(text, names): fn levels_only(evs: Vec) -> Vec { evs.into_iter() - .filter(|e| !is_backing(e)) + .filter(|e| !is_backing(e) && !matches!(e, Ev::Log(_, m) if m.starts_with("PB heap:"))) .map(|e| match e { Ev::Log(l, _) => Ev::Log(l, String::new()), e => e, @@ -787,6 +977,7 @@ def consts(text, names): /// Growth to the maximum with the firmware asking after every reply, one pass at a time. fn run_to_exhaustion(param: u32) { + defaults(); PARAM.with(|p| p.set(param)); let mut f = Firmware::new(); let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); @@ -850,9 +1041,11 @@ def consts(text, names): #[test] fn driver_values() { - let l = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + defaults(); + let l = Ledger::new(tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); assert!(tuning::PB_GROW); - assert_eq!((l.initial, l.max, l.posted), (1024, 4095, 1024)); + assert_eq!(crate::PB_BLOCKS_DEFAULT, 64); + assert_eq!((l.initial, l.max, l.posted), (64, 4095, 64)); assert_eq!(PARAM_DEFAULT, 3072); PARAM.with(|p| p.set(PARAM_DEFAULT)); assert_eq!(params::pb_max_blocks(), 3072); @@ -861,12 +1054,21 @@ def consts(text, names): #[test] fn soft_max_bounds() { + defaults(); let f = Firmware::new(); for param in [0, 1, 63, 64, 65, 1023, 1024, 1025, 3071, 3072, 3073, 4094, 4095, 4096, 8192, u32::MAX] { PARAM.with(|p| p.set(param)); assert_eq!(params::pb_max_blocks(), clamp(param, 64, tuning::PB_MAX_BLOCKS), "pb_max_blocks, param {param}"); - // Below the initial 1024 the firmware still gets 1024. - assert_eq!(f.pools.soft_max(), clamp(param, 1024, CAP), "soft_max, param {param}"); + // Below the initial 64 the firmware still gets 64. + assert_eq!(f.pools.soft_max(), clamp(param, 64, CAP), "soft_max, param {param}"); + } + // With a larger initial posting the firmware gets at least that. + crate::PARAM_BLOCKS.with(|b| b.set(1024)); + let f = Firmware::new(); + assert_eq!(f.pools.ledger.initial, 1024); + for param in [0, 64, 1023, 1024, 3072, 4095, u32::MAX] { + PARAM.with(|p| p.set(param)); + assert_eq!(f.pools.soft_max(), clamp(param, 1024, CAP), "soft_max, param {param}, 1024 initial"); } } @@ -877,7 +1079,7 @@ def consts(text, names): #[test] fn soft_max_boundaries() { - // 3074 is reached exactly (1024 + 205 * 10): that answer is not past it. + // 3074 is reached exactly (64 + 301 * 10): that answer is not past it. for param in [0, 1024, 3072, 3074, 3075, 4094, 4095, u32::MAX] { run_to_exhaustion(param); } @@ -890,6 +1092,7 @@ def consts(text, names): f.queue(); f.queue(); take(); + let mapped = f.pools.ledger.backed as u64; f.pass(); let log = take(); let replies: Vec = log @@ -933,7 +1136,7 @@ def consts(text, names): assert_eq!(f.pools.stats.requests, 2, "requests taken in the pass"); assert_eq!(f.pools.stats.bad, 1); assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -5")))); - assert!(log.contains(&Ev::Producer(1024 + GROW_STEP))); + assert!(log.contains(&Ev::Producer(64 + GROW_STEP))); assert!(f.events.grow.is_empty()); } @@ -947,7 +1150,7 @@ def consts(text, names): take(); f.pass(); let log = take(); - assert_eq!(f.pools.ledger.posted, 1024 + 3 * GROW_STEP); + assert_eq!(f.pools.ledger.posted, 64 + 3 * GROW_STEP); assert_eq!((f.pools.stats.grown, f.pools.stats.bad), (3 * GROW_STEP as u64, 0)); // Each failed reply is logged at info level. let info = log.iter().filter(|e| matches!(e, Ev::Log("info", m) if m.contains("DEVCTL 7: -5"))).count(); @@ -980,7 +1183,9 @@ def consts(text, names): assert!(log.iter().all(|e| matches!(e, Ev::Log("warn", _))), "{log:?}"); assert_eq!(log.len(), 4); assert_eq!(f.pools.stats.bad, 4); - assert_eq!(f.pools.ledger, Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS)); + let mut fresh = Ledger::new(tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); + fresh.backed = Ledger::chunks_for(fresh.initial) * CHUNK_BLOCKS; + assert_eq!(f.pools.ledger, fresh); // No render published: not live. f.renders[0] = 0; f.events.grow.0.push_back(good); @@ -998,54 +1203,43 @@ def consts(text, names): } fn is_backing(e: &Ev) -> bool { - matches!(e, Ev::Alloc(..) | Ev::Free(_) | Ev::Map(..) | Ev::Append(..) | Ev::Unmap(..)) - } - - /// A VM's PB aliases: its count, and the aliases it holds, from block 0 in order. - fn assert_cursor(vm: &VmState, chunks: usize) { - let held = vm.vm.held.borrow(); - assert_eq!(vm.pb_chunks.load(Ordering::Relaxed), held.len(), "count differs from held aliases"); - assert!(held.len() <= chunks); - let mut gva = client::PB_HEAP; - for m in held.iter() { - assert_eq!(m.gva, gva, "aliases not contiguous from the heap's start"); - assert_eq!(m.prot, pte::LEAF_CACHED); - gva += m.size as u64; + matches!(e, Ev::Alloc(..) | Ev::Free(_) | Ev::Map(..) | Ev::Unmap(..)) + } + + /// A VM's PB aliases: exactly the first `chunks` chunks, in order, each the whole chunk at + /// its GVA with the PB leaf, each of the chunk's object. + fn assert_cursor(f: &Firmware, vm: &VmState, chunks: usize) { + let heap = vm.heap.list.lock(); + assert_eq!(heap.len(), chunks, "VM aliases differ from the chunks"); + let ids = f.pools.chunk_ids(); + for (i, m) in heap.iter().enumerate() { + assert_eq!(m._mapping.gva, chunk_gva(i), "aliases not contiguous from the heap's start"); + assert_eq!(m._mapping.size, CHUNK_BYTES); + assert_eq!(m._mapping.prot, pte::LEAF_CACHED); + assert_eq!(m.gem.id, ids[i], "alias of another object"); } } /// In the order of `log`: no block record and no producer past the blocks the VM with - /// context `slot` holds aliases for at that point. + /// context `slot` has mapped at that point (`start` blocks before the log). fn assert_posted_backed(log: &[Ev], slot: u32, start: u64) { let mut mapped = start; for e in log { match e { - Ev::Append(s, gva) if *s == slot => { - assert_eq!(*gva, client::PB_HEAP + mapped * BLOCK_BYTES, "append out of order"); + Ev::Map(s, gva, size, _) if *s == slot => { + assert_eq!(*gva, client::PB_HEAP + mapped * BLOCK_BYTES, "map out of order"); + mapped += *size as u64 / BLOCK_BYTES; } - Ev::Map(s, gva, size, _) if *s == slot && *gva == client::PB_HEAP + mapped * BLOCK_BYTES => { - // Counted at its append. - let _ = size; + Ev::Unmap(s, gva) if *s == slot => { + mapped = mapped.min((gva - client::PB_HEAP) / BLOCK_BYTES); } - _ => {} - } - if let Ev::Append(s, gva) = e { - if *s == slot { - let size = log - .iter() - .find_map(|m| match m { - Ev::Map(ms, mg, size, _) if ms == s && mg == gva => Some(*size as u64), - _ => None, - }) - .unwrap(); - mapped += size / BLOCK_BYTES; + Ev::Record(off, _) => { + assert!((*off as u64 / 8) < mapped, "record of block {} with {} mapped", off / 8, mapped); } - } - if let Ev::Record(off, _) = e { - assert!((*off as u64 / 8) < mapped, "record of block {} with {} mapped", off / 8, mapped); - } - if let Ev::Producer(p) = e { - assert!(*p as u64 <= mapped, "producer {} with {} mapped", p, mapped); + Ev::Producer(p) => { + assert!(*p as u64 <= mapped, "producer {} with {} mapped", p, mapped); + } + _ => {} } } } @@ -1071,17 +1265,20 @@ def consts(text, names): let mut f = Firmware::without_flights(); let log = take(); let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); - assert_eq!(log, vec![Ev::Alloc(1, initial as usize * BLOCK_BYTES as usize)], "startup allocation"); - assert_eq!((f.pools.backed(), f.pools.chunk_count()), (initial, 1)); - assert_eq!(f.pools.chunk_capacity(), max_chunks(initial, max)); - assert_eq!(max_chunks(initial, max), 97); + let chunks = Ledger::chunks_for(initial) as usize; + assert_eq!(chunks, 2); + let want: Vec = (1..=chunks as u32).map(|id| Ev::Alloc(id, CHUNK_BYTES)).collect(); + assert_eq!(log, want, "startup allocation"); + assert_eq!((f.pools.ledger.backed, f.pools.chunk_count()), (chunks as u32 * CHUNK_BLOCKS, chunks)); + assert_eq!(f.pools.chunk_capacity(), MAX_CHUNKS as usize); + assert!(max <= MAX_CHUNKS * CHUNK_BLOCKS); // The first prefix of a VM maps the backing, nothing else. let vm = VmState::new(7); f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); f.map_pb(&vm).unwrap(); - let size = initial as usize * BLOCK_BYTES as usize; - assert_eq!(take(), vec![Ev::Map(7, client::PB_HEAP, size, 1), Ev::Append(7, client::PB_HEAP)]); - assert_cursor(&vm, 1); + let want: Vec = (0..chunks).map(|i| Ev::Map(7, chunk_gva(i), CHUNK_BYTES, i as u32 + 1)).collect(); + assert_eq!(take(), want); + assert_cursor(&f, &vm, chunks); // Mapping again maps nothing. f.map_pb(&vm).unwrap(); assert!(take().is_empty()); @@ -1095,8 +1292,10 @@ def consts(text, names): #[test] fn growth_backs_before_posting() { - PARAM.with(|p| p.set(PARAM_DEFAULT)); + defaults(); let mut f = Firmware::new(); + let start = f.pools.ledger.backed as u64; + let first_chunks = f.pools.chunk_count(); f.fw.ask_after_reply = true; f.queue(); take(); @@ -1107,77 +1306,179 @@ def consts(text, names): assert!(passes <= 1000, "growth did not end"); } let log = take(); - let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); - assert_posted_backed(&log, 5, initial as u64); - // Chunks of at most 32 blocks, allocated only as needed: up to the maximum, no further. + assert_posted_backed(&log, 5, start); + // 16 MiB chunks, backed as the reserve needs them: up to the whole heap, no further. let allocs: Vec = log .iter() .filter_map(|e| if let Ev::Alloc(_, b) = e { Some(*b) } else { None }) .collect(); - assert_eq!(allocs.len() + 1, max_chunks(initial, max)); - assert!(allocs.iter().all(|b| *b <= 4 << 20), "a growth chunk above 4 MiB"); - assert_eq!(CHUNK_BLOCKS as u64 * BLOCK_BYTES, 4 << 20); - assert_eq!(f.pools.backed(), max); + assert!(allocs.iter().all(|b| *b <= 16 << 20), "a growth chunk above 16 MiB"); + assert_eq!(CHUNK_BLOCKS as u64 * BLOCK_BYTES, 16 << 20); + assert_eq!(allocs.len() + first_chunks, MAX_CHUNKS as usize); + assert_eq!(f.pools.ledger.backed, MAX_BLOCKS); + assert_eq!(f.pools.stats.waits, 0); + assert_eq!(f.pools.stats.refills, allocs.len() as u64); assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); - assert_cursor(&f.engine.flights[0].job.vm, f.pools.chunk_count()); + assert_cursor(&f, &f.engine.flights[0].job.vm, f.pools.chunk_count()); } - /// A failure while backing posts nothing and answers nothing; the request counter and the - /// VM's count stay; the next request is answered. - fn failure_posts_nothing(which: &'static std::thread::LocalKey>) -> Vec { + #[test] + fn allocation_follows_the_replies() { let mut f = Firmware::new(); - which.with(|c| c.set(1)); + // Below the reserve line: the pass's answers come first, then the chunk. + f.pools.ledger.posted = f.pools.ledger.backed - growth_reserve() + 1; f.queue(); - f.pass(); - let log = take(); - assert_eq!(count(&log, |e| matches!(e, Ev::Record(..) | Ev::Producer(_) | Ev::Reply(_) | Ev::Sync)), 0, - "posted after a backing failure: {log:?}"); - assert_eq!((f.pools.ledger.posted, f.pools.ledger.next_request), (1024, 0), "ledger after a backing failure"); - assert_eq!((f.pools.stats.bad, f.pools.chunk_count()), (1, 1), "chunks after a backing failure"); - assert_cursor(&f.engine.flights[0].job.vm, 1); - assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -12")))); f.queue(); + take(); f.pass(); - let retry = take(); - assert!(retry.contains(&Ev::Producer(1034))); - assert_eq!(f.pools.chunk_count(), 2); - assert_cursor(&f.engine.flights[0].job.vm, 2); - log + let log = take(); + let last_reply = log.iter().rposition(|e| matches!(e, Ev::Reply(_))).expect("replies"); + let first_alloc = log.iter().position(|e| matches!(e, Ev::Alloc(..))).expect("a refill"); + assert!(first_alloc > last_reply, "allocation before a reply: {log:?}"); + assert_eq!(count(&log, |e| matches!(e, Ev::Reply(_))), 2); + assert_eq!(f.pools.stats.refills, 1); + } + + fn growth_reserve() -> u32 { + crate::g15::pools::growth::RESERVE_BLOCKS } #[test] - fn allocation_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_ALLOC); - assert_eq!(count(&log, is_backing), 0); + fn dry_reserve_waits_and_keeps_order() { + let mut f = Firmware::new(); + let start = f.pools.ledger.backed as u64; + FAIL_ALLOC.with(|c| c.set(1000)); + let mut log = Vec::new(); + let mut waited = None; + for _ in 0..40 { + f.queue(); + f.pass(); + let pass = take(); + let replied = pass.iter().any(|e| matches!(e, Ev::Reply(_))); + log.extend(pass); + if !replied { + waited = Some(f.fw.next_counter - 1); + break; + } + } + assert_posted_backed(&log, 5, start); + let waited = waited.expect("no request waited"); + assert_eq!(f.pools.ledger.posted, f.pools.ledger.backed); + let queued: Vec = f.events.grow.0.iter().map(|q| q[buffer::grow_request::COUNTER]).collect(); + assert_eq!(queued, vec![waited], "waiting request lost"); + assert_eq!(f.pools.ledger.next_request, waited); + assert!(f.pools.stats.waits >= 1); + // The allocator recovers: this pass still waits, then backs a chunk; the next answers. + FAIL_ALLOC.with(|c| c.set(0)); + f.pass(); + let log = take(); + assert!(!log.iter().any(|e| matches!(e, Ev::Reply(_)))); + assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 1); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 1); + let mapped = f.pools.ledger.backed as u64; + f.pass(); + let log = take(); + let replies: Vec = log + .iter() + .filter_map(|e| if let Ev::Reply(r) = e { Some(r[grow_reply::COUNTER]) } else { None }) + .collect(); + assert_eq!(replies, vec![waited]); + assert!(f.events.grow.is_empty()); + assert_posted_backed(&log, 5, mapped); } #[test] - fn map_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_MAP); + fn refill_map_failure_drops_the_chunk() { + let mut f = Firmware::new(); + let chunks = f.pools.chunk_count(); + let backed = f.pools.ledger.backed; + f.pools.ledger.posted = backed - growth_reserve() + 1; + FAIL_MAP.with(|c| c.set(1)); + take(); + f.pass(); + let log = take(); + assert_eq!(f.pools.chunk_count(), chunks, "chunk kept after a failed map"); let backing: Vec<&Ev> = log.iter().filter(|e| is_backing(e)).collect(); - assert!(matches!(backing[..], [Ev::Alloc(2, _), Ev::Free(2)]), "{backing:?}"); + assert!(matches!(backing[..], [Ev::Alloc(id, _), Ev::Free(free)] if id == free), "{backing:?}"); + assert_eq!(f.pools.ledger.backed, backed); + assert!(log.iter().any(|e| matches!(e, Ev::Log("warn", m) if m.contains("backing chunk")))); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks); + // The next pass backs it. + f.pass(); + assert_eq!(f.pools.chunk_count(), chunks + 1); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks + 1); } + /// A pending re-initialization with nothing published (a reclaim, or a recovery that + /// failed the flights): the posted count is about to drop, so nothing is backed. #[test] - fn append_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_APPEND); - let backing: Vec<&Ev> = log.iter().filter(|e| is_backing(e)).collect(); + fn pending_reinit_without_flights_backs_nothing() { + let mut f = Firmware::without_flights(); + f.pools.ledger.posted = f.pools.ledger.backed - growth_reserve() + 1; + f.recovery.reinit = true; + take(); + f.pass(); + assert_eq!(count(&take(), is_backing), 0, "backed during a pending re-initialization"); + f.recovery.reinit = false; + f.pass(); + assert_eq!(f.pools.stats.refills, 1); + } + + /// A pending re-initialization with only compute published (no render can ask for + /// growth): nothing is backed. + #[test] + fn pending_reinit_with_compute_only_backs_nothing() { + let mut f = Firmware::new(); + f.renders[0] = 0; + f.pools.ledger.posted = f.pools.ledger.backed; + f.recovery.reinit = true; + take(); + for _ in 0..4 { + f.pass(); + } + assert_eq!(count(&take(), is_backing), 0, "backed for a compute-only flight"); + assert_eq!(f.pools.stats.refills, 0); + } + + /// A render published before the re-initialization was set (the flights behind a fragment + /// kill) still asks for growth: with its reserve used up, the refill must still back + /// chunks, or the request waits until the deadline. + #[test] + fn pending_reinit_with_a_live_render_refills() { + let mut f = Firmware::new(); // one live render + let start = f.pools.ledger.backed as u64; + f.pools.ledger.posted = f.pools.ledger.backed; + f.recovery.reinit = true; + f.queue(); + take(); + for _ in 0..4 { + f.pass(); + } + let s = &f.pools.stats; assert!( - matches!(backing[..], [Ev::Alloc(2, _), Ev::Map(5, _, _, 2), Ev::Unmap(5, _), Ev::Free(2)]), - "{backing:?}" + f.events.grow.is_empty(), + "growth request stranded: posted={} backed={} waits={} refills={}", + f.pools.ledger.posted, + f.pools.ledger.backed, + s.waits, + s.refills ); + // The request waits one pass for the chunk; the grant's pass tops the reserve up. + assert_eq!((s.waits, s.refills, s.requests), (1, 2, 1)); + assert_posted_backed(&take(), 5, start); } #[test] fn transport_failure_keeps_everything() { let mut f = Firmware::new(); + let chunks = f.pools.chunk_count(); f.fw.reply_fails = true; f.queue(); f.pass(); let log = take(); - assert!(log.contains(&Ev::Producer(1034))); - assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (1034, 2)); - assert_cursor(&f.engine.flights[0].job.vm, 2); + assert!(log.contains(&Ev::Producer(74))); + assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (74, chunks)); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks); assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); } @@ -1185,9 +1486,10 @@ def consts(text, names): fn other_vms_catch_up_before_they_run() { let mut f = Firmware::new(); let a = f.engine.flights[0].job.vm.clone(); - grow_to(&mut f, 1100); + grow_to(&mut f, 200); take(); - assert_eq!(f.pools.chunk_count(), 4); + let chunks = f.pools.chunk_count(); + assert_eq!(chunks, 3); // A's work retires; a new VM runs next: it maps every chunk first. f.engine.flights.clear(); let b = VmState::new(6); @@ -1195,84 +1497,53 @@ def consts(text, names): f.fw.ctx = 6; f.map_pb(&b).unwrap(); let log = take(); - assert_eq!(count(&log, |e| matches!(e, Ev::Append(6, _))), 4); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(6, ..))), chunks); assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 0); - assert_cursor(&b, 4); + assert_cursor(&f, &b, chunks); // B grows the backing; A, behind, catches up when it runs again. - grow_to(&mut f, 1200); + grow_to(&mut f, 300); let log = take(); - assert_posted_backed(&log, 6, 1120); + assert_posted_backed(&log, 6, chunks as u64 * CHUNK_BLOCKS as u64); assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 0); + assert_eq!(f.pools.chunk_count(), chunks + 1); f.engine.flights.clear(); f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); f.map_pb(&a).unwrap(); - assert_cursor(&a, f.pools.chunk_count()); - assert_eq!(a.pb_chunks.load(Ordering::Relaxed), f.pools.chunk_count()); + assert_cursor(&f, &a, f.pools.chunk_count()); } #[test] - fn growth_catches_the_running_vm_up_first() { + fn partial_catch_up_keeps_the_aliases_made() { let mut f = Firmware::new(); - let a = f.engine.flights[0].job.vm.clone(); - // B grows the backing while A is idle. - f.engine.flights.clear(); - let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - f.map_pb(&b).unwrap(); - f.fw.ctx = 6; - grow_to(&mut f, 1100); - // A runs again; its growth request maps what it lacks before backing more. - f.engine.flights.clear(); - f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); - f.fw.ctx = 5; - take(); - grow_to(&mut f, 1130); - let log = take(); - assert_eq!(a.vm.held.borrow().len(), f.pools.chunk_count(), "running VM not caught up"); - assert_cursor(&a, f.pools.chunk_count()); - assert_posted_backed(&log, 5, 1024); - } - - #[test] - fn partial_catch_up_counts_only_held_aliases() { - let mut f = Firmware::new(); - grow_to(&mut f, 1100); + grow_to(&mut f, 200); + let chunks = f.pools.chunk_count(); f.engine.flights.clear(); let b = VmState::new(6); f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); take(); // The third alias cannot be made: two held. - FAIL_MAP.with(|c| c.set(0)); crate::MAP_OK_BEFORE_FAIL.with(|c| c.set(2)); FAIL_MAP.with(|c| c.set(1)); assert_eq!(f.map_pb(&b), Err(ENOMEM)); - assert_cursor(&b, 4); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2); - // The third alias is made but cannot be held: dropped, still two. - FAIL_APPEND.with(|c| c.set(1)); - assert_eq!(f.map_pb(&b), Err(ENOMEM)); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2, "count after a failed append"); - assert!(take().iter().any(|e| matches!(e, Ev::Unmap(6, _)))); - assert_cursor(&b, 4); + assert_cursor(&f, &b, 2); f.map_pb(&b).unwrap(); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 4); - assert_cursor(&b, 4); + assert_cursor(&f, &b, chunks); } /// The ledger reset of Pools::init(), not a full recovery. #[test] fn ledger_reset_reuses_the_backing() { let mut f = Firmware::new(); - grow_to(&mut f, 1300); + grow_to(&mut f, 300); let chunks = f.pools.chunk_count(); take(); f.reset_ledger(); - assert_eq!(f.pools.ledger.posted, 1024); - grow_to(&mut f, 1300); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 300); let log = take(); assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..) | Ev::Map(..))), 0); assert_eq!(f.pools.chunk_count(), chunks); - assert_posted_backed(&log, 5, f.pools.backed() as u64); + assert_posted_backed(&log, 5, f.pools.ledger.backed as u64); } #[test] @@ -1287,36 +1558,421 @@ def consts(text, names): assert_eq!(f.pools.stats.bad, 1); } + /// Every PB parameter at its default. libtest gives each test its own thread, but a test + /// that sets one starts from the defaults all the same. + fn defaults() { + PARAM.with(|p| p.set(PARAM_DEFAULT)); + crate::PARAM_BLOCKS.with(|b| b.set(crate::PB_BLOCKS_DEFAULT)); + crate::PARAM_RECLAIM.with(|r| r.set(crate::PB_RECLAIM_DEFAULT)); + crate::PARAM_IDLE_MS.with(|r| r.set(crate::PB_IDLE_MS_DEFAULT)); + crate::PARAM_RECLAIM_BLOCKS.with(|r| r.set(crate::PB_RECLAIM_BLOCKS_DEFAULT)); + } + + /// A prefix of `vm` published as engine.rs does it: publish_prefix_into() first catches the + /// VM up (map_pb(), before the flight is added), and a render prefix with a + /// re-initialization pending does the InitBM (Pools::init()'s ledger reset) and clears + /// `reinit` at its publication. `render`: the prefix has renders. + fn publish(f: &mut Firmware, vm: &Arc, render: bool) { + f.map_pb(vm).unwrap(); + if render && f.recovery.reinit { + f.reset_ledger(); + f.recovery.reinit = false; + } + f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); + let i = f.engine.flights.len() - 1; + if f.renders.len() <= i { + f.renders.resize(i + 1, 0); + } + f.renders[i] = render as u32; + f.fw.ctx = vm.slot; + f.note_idle(); + } + + /// What a reclaim leaves behind in a VM that was not walked (a closed VM no longer in the + /// registry, whose accepted jobs may still run): its aliases of freed chunks go, and chunks + /// backed again at the same GVA are mapped from their new objects at its next publication. + /// The reclaims are the actual idle reclaim; the posted count stays until the InitBM. #[test] - fn catch_up_refuses_flights_of_another_vm() { + fn freed_chunks_are_mapped_again() { + defaults(); let mut f = Firmware::new(); let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 1100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + assert!(chunks > 2); + // A's work retires and A leaves the registry; the idle reclaim frees the chunks above + // the kept two without walking A. + f.engine.flights.clear(); + f.vms.vms.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); + take(); + // B runs: its first render re-initializes; it grows the backing again, new objects at + // the old GVAs. let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - assert_eq!(f.map_pb(&b), Err(EPROTO), "catch-up with a flight of another VM"); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 0); - assert_eq!(count(&take(), is_backing), 0); - // A count past the chunks. - f.engine.flights.pop(); - a.pb_chunks.store(f.pools.chunk_count() + 1, Ordering::Relaxed); - assert_eq!(f.map_pb(&a), Err(EPROTO), "catch-up with a count past the chunks"); + f.vms.vms.push(b.clone()); + publish(&mut f, &b, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 1100); + assert_eq!(f.pools.chunk_count(), chunks); + // A runs again: its stale aliases are replaced by the new objects'. + f.engine.flights.clear(); + take(); + publish(&mut f, &a, true); + let log = take(); + assert_cursor(&f, &a, chunks); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), chunks - 2); + // Another idle reclaim walks B, not A: with fewer chunks than A maps, A's aliases past + // the backing go at its next catch-up. + f.engine.flights.clear(); + let t1 = now(); + check_at(&mut f, t1); + check_at(&mut f, t1 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + assert_cursor(&f, &b, 2); + f.map_pb(&a).unwrap(); + assert_cursor(&f, &a, 2); } + /// A reclaim whose walk of the VM registry fails: the chunks leave the backing, but a VM's + /// aliases still hold their pages, so nothing is counted freed; the VM drops its stale + /// aliases at its next publication, and only then are the pages released. #[test] - fn grant_refuses_flights_of_two_vms() { + fn failed_vm_walk_counts_nothing_freed() { + defaults(); let mut f = Firmware::new(); - let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - f.renders.push(1); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 1500); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + take(); + crate::FAIL_OPEN.with(|c| c.set(1)); + check_at(&mut f, t0 + 1_000_000_000); + let log = take(); + assert_eq!(f.pools.chunk_count(), 2); + assert_eq!(f.pools.stats.freed, 0, "freed counted with the VM walk failed"); + assert!(log.iter().any(|e| matches!(e, Ev::Log("warn", m) if m.contains("could not walk the VMs")))); + assert!(log.iter().any(|e| matches!(e, Ev::Log("info", m) if m.contains(" 0 chunks freed")))); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), 0, "pages released while aliases hold them"); + assert_eq!(a.heap.list.lock().len(), chunks); + // A's next publication drops the stale aliases: the pages go. + publish(&mut f, &a, true); + let log = take(); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + assert_cursor(&f, &a, 2); + } + + /// One client idles 60 ms after a heavy scene (more than half the maximum posted) and + /// renders again: the client-boundary re-initialization was set at 50 ms with the backing + /// kept, the posted count stays until the render's InitBM, which posts the initial blocks + /// again, and the scene grows again from the kept backing with nothing allocated or mapped. + #[test] + fn same_client_render_after_60ms() { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 60_000_000); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!( + (f.pools.ledger.posted, f.pools.chunk_count()), + (posted, chunks), + "posted count reset before the InitBM" + ); + take(); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + assert!(!f.recovery.reinit && !f.pools.reclaim_pending); + grow_to(&mut f, 2100); + let log = take(); + assert_eq!(count(&log, is_backing), 0, "backing changed: {:?}", log.iter().filter(|e| is_backing(e)).collect::>()); + assert_posted_backed(&log, 5, f.pools.ledger.backed as u64); + } + + /// One client idles 1.1 s and renders again, after a scene above the reclaim floor (the + /// idle reclaim at 1 s) or above half the maximum (the client boundary at 50 ms, then the + /// free at 1 s): the backing above the kept chunks was freed and unmapped from the client's + /// VM, the posted count stays until the render's InitBM, and the scene grows again with + /// every chunk backed and mapped before a record or producer names it. + #[test] + fn same_client_render_after_1_1s() { + for target in [1500, 2100] { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, target); + let posted = f.pools.ledger.posted; + f.engine.flights.clear(); + let t0 = now(); + for dt in [0, 50_000_000, 1_000_000_000, 1_100_000_000] { + check_at(&mut f, t0 + dt); + } + assert_eq!(f.pools.chunk_count(), 2, "not freed ({target})"); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM ({target})"); + assert_cursor(&f, &a, 2); + take(); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, target); + let log = take(); + assert_posted_backed(&log, 5, 2 * CHUNK_BLOCKS as u64); + assert_cursor(&f, &a, f.pools.chunk_count()); + } + } + + /// A compute-only submission between a level-2 reclaim and the next render's InitBM: its + /// publication maps nothing, its passes back and post nothing (the posted count is still + /// the old one, above the kept backing), and a growth request naming its context is left + /// unanswered; once it retires, the render after it re-initializes as usual. + #[test] + fn compute_between_reclaim_and_initbm() { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 1500); + let posted = f.pools.ledger.posted; + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + let bad = f.pools.stats.bad; + take(); + publish(&mut f, &a, false); f.queue(); - f.pass(); + for _ in 0..4 { + f.pass(); + } + let log = take(); + let touched: Vec<&Ev> = log + .iter() + .filter(|e| is_backing(e) || matches!(e, Ev::Record(..) | Ev::Producer(_) | Ev::Reply(_))) + .collect(); + assert!(touched.is_empty(), "compute-only flight backed or posted: {touched:?}"); + assert!(f.recovery.reinit && f.pools.ledger.posted == posted); + assert_eq!(f.pools.stats.bad, bad + 1); + // The compute retires; the check finds nothing more to do; the next render re-inits. + f.engine.flights.clear(); + check_at(&mut f, t0 + 2_000_000_000); + assert_eq!(count(&take(), is_backing), 0); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 300); + assert_posted_backed(&take(), 5, 2 * CHUNK_BLOCKS as u64); + } + + fn now() -> u64 { + crate::NOW.with(|n| n.get()) + } + + /// The idle clock at `t` and the reclaim check at that time, as a job arrival or an idle + /// poll pass runs it. + fn check_at(f: &mut Firmware, t: u64) { + crate::NOW.with(|n| n.set(t)); + f.note_idle(); + f.reclaim_check(); + } + + /// A heavy client (more than half the maximum posted) exits and nothing renders after it: + /// the 50 ms client-boundary re-initialization keeps the backing, and a check after the + /// long idle frees the backing above the kept chunks. The posted count stays until the next + /// render's InitBM. + #[test] + fn client_boundary_reinit_frees_without_a_render() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + assert!(posted > 2047 && chunks > 2); + // The pass that retires the last flight notes the idle. + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + take(); + check_at(&mut f, t0 + 50_000_000); + assert!(f.recovery.reinit && f.pools.chunk_count() == chunks, "client-boundary re-initialization"); + check_at(&mut f, t0 + 999_999_999); + assert_eq!(f.pools.chunk_count(), chunks, "freed before the long idle"); + check_at(&mut f, t0 + 1_000_000_000); + let log = take(); + assert_eq!(f.pools.chunk_count(), 2, "kept backing not freed: {} chunks", f.pools.chunk_count()); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!(f.pools.stats.reclaims, 1); + assert_eq!(f.pools.ledger.backed, 2 * CHUNK_BLOCKS); + assert_eq!(f.pools.stats.freed, (chunks - 2) as u64); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + // Later checks find nothing more to do. + check_at(&mut f, t0 + 100_000_000_000); + assert!(take().is_empty()); + } + + /// With a reclaim level of 1 (tuning::PB_RECLAIM is 2) the client-boundary re-initialization keeps the backing for good. + #[test] + fn client_boundary_reinit_at_level_1_keeps_the_backing() { + defaults(); + crate::PARAM_RECLAIM.with(|r| r.set(1)); + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 50_000_000); + assert!(f.recovery.reinit); + assert_eq!(f.reclaim_delay_ns(), None, "armed at level 1"); + check_at(&mut f, t0 + 5_000_000_000); + assert_eq!(f.pools.chunk_count(), chunks, "freed at level 1"); + assert_eq!(f.pools.stats.freed, 0); + } + + /// Nothing grown since the initialization: an initial posting above half the maximum + /// (an initial posting of 3000), or a floor below it (a reclaim floor of 32): no + /// re-initialization at any idle. + #[test] + fn nothing_grown_reinitializes_nothing() { + for (blocks, floor) in [(64, 32), (3000, 1024)] { + defaults(); + crate::PARAM_BLOCKS.with(|b| b.set(blocks)); + crate::PARAM_RECLAIM_BLOCKS.with(|b| b.set(floor)); + let mut f = Firmware::new(); + assert_eq!(f.pools.ledger.posted, blocks); + f.engine.flights.clear(); + let t0 = now(); + for dt in [0, 50_000_000, 1_000_000_000, 5_000_000_000] { + check_at(&mut f, t0 + dt); + } + assert!( + !f.recovery.reinit && f.pools.stats.reclaims == 0, + "re-initialization with nothing grown ({blocks} blocks, floor {floor})" + ); + } + } + + /// A recovery's re-initialization (not a reclaim's) is left alone: nothing freed. + #[test] + fn recovery_reinit_frees_nothing() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + f.recovery.reinit = true; + let t0 = now(); + check_at(&mut f, t0); + assert_eq!(f.reclaim_delay_ns(), None, "armed for a recovery's re-initialization"); + check_at(&mut f, t0 + 5_000_000_000); + assert_eq!(f.pools.chunk_count(), chunks, "freed for a recovery's re-initialization"); + assert_eq!(f.pools.stats.reclaims, 0); + } + + /// The idle poll work as poll.rs drives it: a pass with nothing published notes the idle + /// and runs the reclaim check; an idle pass is re-armed only for reclaim_delay_ns(), + /// rounded up to whole ms, plus one 4 ms jiffy. Returns the wake times. + fn idle_polls(f: &mut Firmware, limit: usize) -> Vec { + let mut wakes = Vec::new(); + loop { + f.note_idle(); + f.reclaim_check(); + let Some(ns) = f.reclaim_delay_ns() else { + return wakes; + }; + assert!(wakes.len() < limit, "poll woke in a loop: {} wakes, next in {ns} ns", wakes.len()); + let t = now() + ns.div_ceil(1_000_000) * 1_000_000 + 4_000_000; + crate::NOW.with(|n| n.set(t)); + wakes.push(t); + } + } + + /// The same exit with the idle poll work: the poll is armed for the 50 ms + /// re-initialization, then for the rest of the long idle (not the expired 50 ms), where the + /// kept backing is freed; then nothing is armed. + #[test] + fn client_boundary_reinit_arms_the_long_idle() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + assert!(posted > 2047 && chunks > 2); + // The pass that retires the last flight notes the idle. + f.engine.flights.clear(); + f.note_idle(); + let t0 = now(); + take(); + let wakes = idle_polls(&mut f, 8); let log = take(); assert_eq!( - count(&log, |e| is_backing(e) || matches!(e, Ev::Reply(_) | Ev::Producer(_) | Ev::Record(..))), - 0, - "grant with flights of two VMs" + f.pools.chunk_count(), + 2, + "kept backing not freed: {} chunks, wakes {:?} ms", + f.pools.chunk_count(), + wakes.iter().map(|t| (t - t0) / 1_000_000).collect::>() ); - assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -71")))); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!(f.pools.stats.reclaims, 1); + assert_eq!(f.pools.ledger.backed, 2 * CHUNK_BLOCKS); + assert_eq!(f.pools.stats.freed, (chunks - 2) as u64); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); + // Two wakes: the re-initialization after 50 ms, the free after the long idle. + let ms: Vec = wakes.iter().map(|t| (t - t0) / 1_000_000).collect(); + assert_eq!(ms, vec![54, 1004], "wakes (ms after the idle began)"); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + // Idle much longer, or a job arriving: nothing more is due. + crate::NOW.with(|n| n.set(t0 + 100_000_000_000)); + f.reclaim_check(); + assert_eq!(f.reclaim_delay_ns(), None); + assert!(take().is_empty()); + } + + /// Nothing grown: the idle poll work arms nothing. + #[test] + fn nothing_grown_arms_nothing() { + for (blocks, floor) in [(64, 32), (3000, 1024)] { + defaults(); + crate::PARAM_BLOCKS.with(|b| b.set(blocks)); + crate::PARAM_RECLAIM_BLOCKS.with(|b| b.set(floor)); + let mut f = Firmware::new(); + f.engine.flights.clear(); + let wakes = idle_polls(&mut f, 8); + assert!(wakes.is_empty(), "armed with nothing grown ({blocks} blocks, floor {floor}): {} wakes", wakes.len()); + assert!(!f.recovery.reinit); + } + } + + /// An idle reclaim below half the maximum frees at the long idle; with its re-initialization + /// pending nothing more is armed. + #[test] + fn idle_reclaim_arms_nothing_after() { + let mut f = Firmware::new(); + grow_to(&mut f, 1500); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + f.note_idle(); + let t0 = now(); + let wakes = idle_polls(&mut f, 8); + let ms: Vec = wakes.iter().map(|t| (t - t0) / 1_000_000).collect(); + assert_eq!(ms, vec![1004]); + assert_eq!((f.pools.chunk_count(), f.pools.stats.freed), (2, (chunks - 2) as u64)); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + } + + #[test] + fn catch_up_refuses_flights_of_another_vm() { + let mut f = Firmware::new(); + let b = VmState::new(6); + f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); + assert_eq!(f.map_pb(&b), Err(EPROTO), "catch-up with a flight of another VM"); + assert_eq!(b.heap.list.lock().len(), 0); + assert_eq!(count(&take(), is_backing), 0); } #[test] @@ -1348,7 +2004,7 @@ def consts(text, names): // Only inputs the old constructor took to the whole 4096-block heap changed. assert!(changed > 0 && changed < cases); // The driver's values are below the cap, so its ledger is the old one. - let d = (tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let d = (tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); let (old, new) = (OldLedger::new(d.0, d.1, d.2), Ledger::new(d.0, d.1, d.2)); if d.2 <= CAP { assert_eq!((old.initial, old.max, old.posted), (new.initial, new.max, new.posted)); @@ -1397,7 +2053,7 @@ def read(path, ref=None): raise SystemExit('embedded Ledger::new() differs from ' + OLD_REF) def run(mutation): - paths = (POOLS, GROWTH, BUFFER, CHANNELS, INITDATA, PARAMS, HW, CORE, ENGINE, PGTABLE) + paths = (POOLS, GROWTH, BUFFER, CHANNELS, INITDATA, PARAMS, HW, CORE, ENGINE, PGTABLE, VM) src = {path: read(path) for path in paths} if mutation: spec = {**MUTATIONS, **CONTROLS}[mutation] @@ -1407,15 +2063,28 @@ def run(mutation): src[path] = src[path].replace(old, new) pools, hw = src[POOLS], src[HW] bodies = [item(pools, s) for s in (' pub(crate) fn answer_growth(&mut self) -> Result {', - ' fn answer_request(&mut self, request: &[u32; 14]) -> Result {', + ' fn answer_request(&mut self, request: &[u32; 14]) -> Result {', ' pub(crate) fn map_pb(&mut self, vm: &VmState) -> Result {', - ' fn running_vm(&self, ctx: u32) -> Result> {', - ' fn back_pb(&mut self, end: u32, ctx: u32) -> Result {')] + ' pub(crate) fn refill_backing(&mut self) {', + ' pub(crate) fn note_idle(&mut self) {', + ' pub(crate) fn reclaim_check(&mut self) {', + ' fn free_backing(&mut self) -> usize {', + ' pub(crate) fn reclaim_delay_ns(&self) -> Option {')] soft_max = item(pools, ' pub(crate) fn soft_max(&self) -> u32 {') - backed = item(pools, ' pub(crate) fn backed(&self) -> u32 {') + pools_methods = (item(pools, ' fn map_heap(&self, vm: &VmState) -> Result {') + '\n' + + item(pools, ' fn free_chunks(&mut self, keep: usize) -> usize {') + '\n' + + item(pools, ' fn back_chunk(&mut self) -> Result> {')) chunk = item(pools, 'struct PbChunk {') + '\n' + item(pools, 'impl PbChunk {') + chunk_geometry = (re.search(r'^pub\(crate\) const CHUNK_BYTES: usize = [^;]+;', pools, re.M).group(0) + '\n' + + item(pools, 'pub(crate) const fn chunk_gva(i: usize) -> u64 {')) + vm_src = src[VM] + vm_methods = '\n'.join(item(vm_src, s) for s in ( + ' fn map_one(&self, heap: &mut KVec, i: usize, gem: ARef) -> Result {', + ' pub(crate) fn map_heap(&self, gems: impl Iterator>) -> Result {', + ' pub(crate) fn map_chunk(&self, i: usize, gem: ARef) -> Result {', + ' pub(crate) fn unmap_heap_above(&self, keep: usize) {')) start = pools.index(POOLS_NEW_BACKING[0]) - end = pools.index('\n', pools.index(POOLS_NEW_BACKING[1], start)) + 1 + end = pools.index(POOLS_NEW_BACKING[1], start) pools_new_backing = pools[start:end] if pools.count(PB_LEAF_ASSERT) != 1: raise SystemExit('pools.rs does not assert the PB leaf GPU-only') @@ -1437,32 +2106,64 @@ def run(mutation): if init.count(line) != 1: raise SystemExit('Pools::init() does not hold: ' + line) reset = '\n'.join('p.' + line[len('self.'):] for line in LEDGER_RESET) - pb_max_blocks = item(src[PARAMS], 'pub(crate) fn pb_max_blocks() -> u32 {') + pb_max_blocks = item(src[PARAMS], 'pub(crate) fn pb_max_blocks() -> ') + # The initial posting and the reclaim values are fixed driver values (hw::tuning), not + # module parameters: production's accessors must return exactly those constants. The + # harness's accessors read cells that start at them, so a test can vary them. + fixed = {'pb_blocks': ('u32', 'super::hw::tuning::PB_BLOCKS', 'PARAM_BLOCKS'), + 'pb_reclaim': ('u32', 'super::hw::tuning::PB_RECLAIM', 'PARAM_RECLAIM'), + 'pb_idle_ns': ('u64', 'super::hw::tuning::PB_IDLE_MS as u64 * 1_000_000', 'PARAM_IDLE_MS'), + 'pb_reclaim_blocks': ('u32', 'super::hw::tuning::PB_RECLAIM_BLOCKS', 'PARAM_RECLAIM_BLOCKS')} + for name, (ty, body, cell) in fixed.items(): + got = item(src[PARAMS], f'pub(crate) fn {name}() -> ') + want = f'pub(crate) fn {name}() -> {ty} {{\n {body}\n}}' + if got.strip() != want: + raise SystemExit(f'params.rs {name}() is not the fixed tuning value: {got!r}') + value = f'crate::{cell}.with(|p| p.get())' + if name == 'pb_idle_ns': + value = f'{value} as u64 * 1_000_000' + pb_max_blocks += f'\npub(crate) fn {name}() -> {ty} {{\n {value}\n}}\n' + tuning_hw = item(hw, 'pub(crate) mod tuning {') + reclaim_defaults = {} + for name, const in (('PB_RECLAIM', 'PB_RECLAIM_DEFAULT'), ('PB_IDLE_MS', 'PB_IDLE_MS_DEFAULT'), + ('PB_RECLAIM_BLOCKS', 'PB_RECLAIM_BLOCKS_DEFAULT')): + m = re.search(rf'pub\(crate\) const {name}: u32 = (\d+);', tuning_hw) + if not m: + raise SystemExit(f'tuning::{name} not found') + reclaim_defaults[const] = int(m.group(1)) m = re.search(r'g15_pb_max_blocks: u32 \{\s*default: (\d+),', src[CORE]) if not m: raise SystemExit('g15_pb_max_blocks default not found') param_default = int(m.group(1)) + m = re.search(r'pub\(crate\) const PB_BLOCKS: u32 = (\d+);', tuning_hw) + if not m: + raise SystemExit('tuning::PB_BLOCKS not found') + blocks_default = int(m.group(1)) sizes = consts(item(hw, 'pub(crate) mod sizes {'), ['PB_BLOCKS']) - tuning = consts(item(hw, 'pub(crate) mod tuning {'), ['PB_GROW', 'PB_BLOCKS', 'PB_MAX_BLOCKS']) + tuning = consts(item(hw, 'pub(crate) mod tuning {'), ['PB_GROW', 'PB_MAX_BLOCKS']) client = consts(item(hw, 'pub(crate) mod client {'), ['PB_HEAP', 'PB_BLOCK_LIST']) pte = consts(item(hw, 'pub(crate) mod pte {'), ['LEAF_CACHED']) control = consts(item(src[INITDATA], 'pub(crate) mod control {'), ['HALT_COUNT']) growth = src[GROWTH] clamp_fn = item(growth, 'const fn clamp(v: u32, lo: u32, hi: u32) -> u32 {') - tested = (''.join(bodies) + soft_max + backed + chunk + pools_new_backing + pb_max_blocks + growth + + tested = (''.join(bodies) + soft_max + pools_methods + chunk + chunk_geometry + vm_methods + + pools_new_backing + pb_max_blocks + growth + src[BUFFER] + sizes + tuning + pte + prot) dir = out / (mutation or 'actual') dir.mkdir(exist_ok=True) for path in (GROWTH, BUFFER, CHANNELS): (dir / Path(path).name).write_text(src[path]) code = (PREFIX + f'pub const PARAM_DEFAULT: u32 = {param_default};\n' + + f'pub const PB_BLOCKS_DEFAULT: u32 = {blocks_default};\n' + + ''.join(f'pub const {k}: u32 = {v};\n' for k, v in reclaim_defaults.items()) + 'pub mod g15 {\n' ' pub mod fw {\n' f' #[path = "{dir / "channels.rs"}"] pub mod channels;\n' f' #[path = "{dir / "buffer.rs"}"] pub mod buffer;\n' ' pub mod initdata { pub mod control {\n' + control + '\n} }\n' ' }\n' - f' pub mod pools {{ #[path = "{dir / "growth.rs"}"] pub mod growth; }}\n' + f' pub mod pools {{ #[path = "{dir / "growth.rs"}"] pub mod growth;\n' + ' use super::fw::buffer;\n use super::hw::client;\n' + chunk_geometry + '\n }\n' ' pub mod hw {\n' ' pub mod sizes {\n' + sizes + '\n}\n' ' pub mod tuning {\n' + tuning + '\n}\n' @@ -1471,6 +2172,7 @@ def run(mutation): ' }\n' ' pub mod pgtable {\n' + pg_consts + '\n' + prot + '\n }\n' ' pub mod boot {\n pub fn eproto() -> crate::Error { crate::EPROTO }\n }\n' + ' pub mod engine {\n pub fn now_ns() -> u64 { crate::NOW.with(|n| n.get()) }\n }\n' ' /// The checked backing allocation, failing where a test injects it.\n' ' pub mod object {\n' ' use crate::prelude::*;\n' @@ -1495,19 +2197,20 @@ def run(mutation): ' use super::fw::buffer::{self, block_ctl};\n' ' use super::fw::channels::DevCtl;\n' ' use super::fw::initdata::control;\n' - ' use super::hw::{client, pte, tuning};\n' - ' use super::pools::growth::{self, Ledger};\n' + ' use super::hw::{self, client, pte, tuning};\n' + ' use super::pools::growth::{self, Answer, Ledger};\n' + ' use super::pools::{chunk_gva, CHUNK_BYTES};\n' ' use super::object::{self, HeldPages};\n' ' use crate::driver::AsahiDevice;\n' - ' use crate::{gem, mmu, ARef};\n' - ' use core::sync::atomic::{AtomicUsize, Ordering};\n' + ' use crate::{gem, mmu, ARef, Mutex};\n' ' use std::sync::Arc;\n' + MODEL.replace('p.RESET', reset).replace('POOLS_NEW_BACKING', '') + chunk + '\n' ' /// Pools::new\'s PB backing, verbatim.\n' ' pub fn pools_new_backing(dev: &AsahiDevice) -> Result<(Ledger, KVec)> {\n' + pools_new_backing + ' Ok((ledger, chunks))\n }\n' - ' impl Pools {\n' + soft_max + '\n' + backed + '\n }\n' + ' impl Pools {\n' + soft_max + '\n' + pools_methods + '\n }\n' + ' impl VmState {\n' + vm_methods + '\n }\n' ' impl Firmware {\n' + '\n'.join(bodies) + '\n }\n' ' }\n' '}\n' + TESTS) @@ -1547,7 +2250,8 @@ def no_core(): result = run(a.mutation) receipt = {'compiler': compiler, 'rustc_version': rustc_version, 'ref': a.ref, 'old_ref': OLD_REF, 'old_reference_checked_against_git': old_git is not None, **result, - 'scope': 'Actual answer_growth/answer_request/soft_max/pb_max_blocks, pools/growth.rs, ' + 'scope': 'Actual answer_growth/answer_request/map_pb/refill_backing/soft_max/pb_max_blocks, ' + 'the PB chunk and VM heap-mapping bodies, pools/growth.rs, ' 'fw/buffer.rs and fw/channels.rs over a model of the firmware objects with an ' 'ordered access log, and Ledger::new against its body at ' f'{OLD_REF[:12]}. Not kernel, firmware or GPU execution, timing, or recovery.'} diff --git a/tools/asahi/g15-probe-init-tests.py b/tools/asahi/g15-probe-init-tests.py index 65cc9c0b69c9f..e7617fba021c2 100644 --- a/tools/asahi/g15-probe-init-tests.py +++ b/tools/asahi/g15-probe-init-tests.py @@ -193,6 +193,8 @@ def item(text, marker): mod boot { pub mod glue { pub fn retained() -> bool { crate::RETAINED.with(core::cell::Cell::get) } pub unsafe fn asahi_g15_reprobe() { crate::activation_reprobe(); } + // Reads the device tree and logs; no driver state. + pub unsafe fn asahi_g15_report_loader_gpu() {} }} ''' diff --git a/tools/asahi/g15-retention-tests.py b/tools/asahi/g15-retention-tests.py index 129567ab373e0..f5c8c8dce3294 100644 --- a/tools/asahi/g15-retention-tests.py +++ b/tools/asahi/g15-retention-tests.py @@ -34,6 +34,8 @@ def item(text, marker): use std::{cell::{Cell, RefCell, RefMut}, mem::MaybeUninit, ops::{Deref, DerefMut}, sync::{Arc, atomic::{AtomicBool, Ordering}}}; type Error = i32; +trait FromErrno { fn from_errno(value: i32) -> Self; } +impl FromErrno for Error { fn from_errno(value: i32) -> Self { value } } type Result = std::result::Result; const GFP_KERNEL: u32 = 0; const EINVAL: Error = -22; @@ -43,6 +45,7 @@ def item(text, marker): static FAIL: Cell = const { Cell::new(0) }; static OWNER_DROPS: Cell = const { Cell::new(0) }; static BUFFER_DROPS: Cell = const { Cell::new(0) }; + static PMP_CALLS: Cell = const { Cell::new(0) }; } fn step(site: u32) -> Result { if FAIL.get() & (1 << site) != 0 { Err(-100 - site as i32) } else { Ok(()) } @@ -72,8 +75,13 @@ def item(text, marker): impl DerefMut for KBox { fn deref_mut(&mut self) -> &mut T { &mut self.0 } } struct AsahiDevice; impl AsahiDevice { fn as_ref(&self) -> &Self { self } } -struct Config; -static CONFIG: Config = Config; +mod hw { + #[derive(PartialEq)] + pub enum Platform { Adt, AdtWithPmp } +} +struct Config { platform: hw::Platform } +static CONFIG: Config = Config { platform: hw::Platform::Adt }; +static PMP_CONFIG: Config = Config { platform: hw::Platform::AdtWithPmp }; mod mmu { pub struct Uat; } mod regs { pub struct Identity; @@ -152,6 +160,12 @@ def item(text, marker): } mod boot { use super::*; + pub mod glue { + pub unsafe fn asahi_g15_pmp_ready() -> i32 { + super::PMP_CALLS.set(super::PMP_CALLS.get() + 1); + match super::step(13) { Ok(()) => 0, Err(e) => e } + } + } use super::{object::Objects, initdata::InitData, channel::Channels, event::Events}; pub struct Session { _owner: Owner, uat: Arc, regs: regs::Regs } impl Session { @@ -185,7 +199,9 @@ def item(text, marker): let boot::Booted { session, init, objects, channels, events, .. } = booted(); boot::Orphan { session, _objects: objects, channels: Some(channels), _init: Some(init), events } } -fn reset(fail: u32) { FAIL.set(fail); OWNER_DROPS.set(0); BUFFER_DROPS.set(0); } +fn reset(fail: u32) { + FAIL.set(fail); OWNER_DROPS.set(0); BUFFER_DROPS.set(0); PMP_CALLS.set(0); +} fn retained(label: &str) { assert_eq!(OWNER_DROPS.get(), 0, "{label}: live session owner dropped"); assert_eq!(BUFFER_DROPS.get(), 0, "{label}: firmware buffer dropped"); @@ -221,17 +237,32 @@ def item(text, marker): std::mem::forget(shared); } } - reset(0); + for orphan_oom in [false, true] { + reset((1 << 13) | if orphan_oom { 1 << 7 } else { 0 }); + let shared = Shared::new(); + let result = Firmware::start_backend(&AsahiDevice, &PMP_CONFIG, booted(), &shared); + assert!(matches!(result, Err(-113)), "PMP readiness error changed"); + assert_eq!(PMP_CALLS.get(), 1, "PMP readiness was not reported"); + retained("PMP readiness failure"); + assert_eq!(shared.orphan.lock().is_some(), !orphan_oom); + std::mem::forget(shared); + } + for cfg in [&CONFIG, &PMP_CONFIG] { + // A pending PMP error must be irrelevant on a platform without PMP. + reset(if cfg.platform == hw::Platform::Adt { 1 << 13 } else { 0 }); let shared = Shared::new(); let booted = boot::start(&shared).expect("normal boot"); assert_eq!(booted.max_frequency_khz, 1234); let uat: Arc = booted.session.uat(); - let firmware = Firmware::start_backend(&AsahiDevice, &CONFIG, booted, &shared).expect("normal backend"); + let firmware = Firmware::start_backend(&AsahiDevice, cfg, booted, &shared).expect("normal backend"); + assert_eq!(PMP_CALLS.get(), u32::from(cfg.platform == hw::Platform::AdtWithPmp), + "PMP readiness used on the wrong platform"); assert!(Arc::ptr_eq(&uat, &firmware.session.uat())); assert!(!shared.orphaned.load(Ordering::Acquire)); retained("normal success"); std::mem::forget((shared, firmware)); - println!("PASS 23 retention cases: orphan OOM, six backend failure sites, four boot-completion sites and normal success"); + } + println!("PASS 26 retention cases: orphan OOM, six backend failure sites, four boot-completion sites, PMP readiness failure and both platform success paths"); println!("normal success owner_drops={} buffer_drops={}", OWNER_DROPS.get(), BUFFER_DROPS.get()); } ''' @@ -272,11 +303,16 @@ def main(): cwd=ROOT, text=True)) variants["consume-orphan-before-allocation"] = source.replace(orphan, item(old, " pub(crate) fn orphan(")) variants["unguarded-registry-allocation"] = source.replace(backend, item(old, " fn start_backend(")) + variants["ignore-pmp-readiness-error"] = source.replace( + "let ret = unsafe { boot::glue::asahi_g15_pmp_ready() };", + "let _ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; let ret = 0;") + variants["pmp-on-other-platform"] = source.replace( + "if cfg.platform != hw::Platform::AdtWithPmp {", "if false {") receipt = {"scope": __doc__, "sources": { name: hashlib.sha256(text.encode()).hexdigest() for name, text in [("mod.rs", mod), ("boot.rs", boot), ("kbox.rs", kbox)]}, "rustc": subprocess.check_output([os.environ.get("HOSTRUSTC", "rustc"), "--version"], text=True).strip(), - "cases": 23, "public_old_source_control": CONTROL_REF, "variants": {}} + "cases": 26, "public_old_source_control": CONTROL_REF, "variants": {}} if args.mutations: receipt["old_control_sha256"] = hashlib.sha256(old.encode()).hexdigest() for name, text in variants.items(): @@ -295,14 +331,20 @@ def main(): raise RuntimeError(result.stderr) else: expected = {"consume-orphan-before-allocation": "orphan allocation: live session owner dropped", - "unguarded-registry-allocation": "backend failure: live session owner dropped"}[name] - if result.returncode == 0 or expected not in result.stderr or "left: 5" not in result.stderr: + "unguarded-registry-allocation": "backend failure: live session owner dropped", + "ignore-pmp-readiness-error": "PMP readiness error changed", + "pmp-on-other-platform": "normal backend: -113"}[name] + if result.returncode == 0 or expected not in result.stderr: raise RuntimeError("INVALID mutant outcome " + name + "\n" + result.stderr) - receipt["variants"][name]["intended_owner_drops"] = 5 + if name in ("consume-orphan-before-allocation", "unguarded-registry-allocation"): + if "left: 5" not in result.stderr: + raise RuntimeError("INVALID owner-drop count " + name) + receipt["variants"][name]["intended_owner_drops"] = 5 + receipt["variants"][name]["intended_assertion"] = expected after = subprocess.run([str(out / "actual")], capture_output=True, text=True, check=True) receipt["actual_after_mutants"] = {"exit": after.returncode, "stdout": after.stdout, "stderr": after.stderr} receipt["fixed_invariants"] = {"live_session_owner_drops": 0, "firmware_buffer_drops": 0, - "checked_cases": 23} + "checked_cases": 26} (out / "result.json").write_text(json.dumps(receipt, indent=2) + "\n") print(json.dumps(receipt, indent=2)) diff --git a/tools/asahi/g15-zero-words-tests.py b/tools/asahi/g15-zero-words-tests.py index 5b728e95fc03e..c6e32d76cb415 100644 --- a/tools/asahi/g15-zero-words-tests.py +++ b/tools/asahi/g15-zero-words-tests.py @@ -1,12 +1,27 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Test the actual J516C record builder's narrow zero-word exemption. +"""Test the actual J516C record builder's narrow zero-word exemption and its word sources. Uses synthetic public data and temporary host copies of production Rust modules. No GPU, firmware file, private fixture, kernel build or hardware access. The golden T6030 inputs below are from SiliconForest 9f09c85f8d3b, except the full -hw.rs file refreshed at 709f0f08316a for the reviewed shared PB allocator changes. -Its T6030 configuration module is independently pinned to the original bytes. +hw.rs file refreshed at 709f0f08316a for the reviewed shared PB allocator changes +and again when the initial PB posting became a module parameter (the tuning +constant PB_BLOCKS removed), and when the T6031 board source became the +device tree with reference words instead of a board file (the Board enum; +the T6030 tables are unchanged), and for publication-bound slots (removal of +MAX_VMS and its lifetime-slot comment only), and when Board::DeviceTree was +gated on CONFIG_DRM_ASAHI_G15 for the T8122 runtime without the G15 backend +(one #[cfg] line; the tables are unchanged), and when the initial PB posting +and the idle reclaim became fixed tuning constants instead of module +parameters (four constants; the tables are unchanged), and when the two stamp +renumbering margins became tuning constants (the tables are unchanged). Its T6030 +configuration module is +independently pinned to the original bytes, +except for the J514S B0 FirmwareBuild row added to its firmware table: the +module is the original pinned bytes (0079ae08) plus that one 15-line row, and +the whole hw.rs pin was refreshed with it. prepare_board() is pinned too; +it was refreshed for the same Board change (its T6030 arm is unchanged). """ import argparse @@ -22,13 +37,13 @@ GOLDEN_FILES = { "board/data.rs": "7c5ff1077ca3296cbfc9e7144294162b6dea78a3774c38eeed4b0f59a14d7a5c", "board/template.rs": "286872f5474a67534f214e83e659253299f85143f3dc59035fa80894022e2549", - "hw.rs": "e4619590e670ed88d2a9bb76b03853081c615d3d0ad1c6a68321ce8e615ef8a0", + "hw.rs": "c54a4f8342fb466f82f4b9cfed6a137ee1ca1f4d80ede4e1b7f397d39223558e", "power/idle.rs": "4928ecf7699b3ca85ff5143eb6e996efd40d47321e29a238a02c8873bc9f102c", } GOLDEN_FUNCTIONS = { - ("hw.rs", "pub(crate) mod t6030 {"): "0079ae08fbe0e51cbf14e48711981a81048d1a1d3fc5495ab8d5c376283a760c", + ("hw.rs", "pub(crate) mod t6030 {"): "cab592b5d776be9ede93b96b1b0526c1d838ef2948a5e4a86caa4ec71f71c2ca", ("initdata.rs", "fn templates("): "c3115c9926d3d7d93317cb8510e93b04f30b0c256811d7137568e822dd93b8ee", - ("initdata.rs", "pub(crate) fn prepare_board("): "c59f968f125397efc63f75a49d3a8543c15c1fd4d50362f15683563398e64bc6", + ("initdata.rs", "pub(crate) fn prepare_board("): "ebef64ab87ef7b1f994eb7fa2a2424d05bc4fc9e88cbfbf18e51570bdfc4222d", ("board.rs", "pub(crate) fn from_adt("): "cecd0b28db5bd94d3b75c1bbb5f35204e8bf0699f74e5b3d08228c59025f6ac2", } @@ -49,7 +64,7 @@ def function(source, marker): use super::*; // Frozen pre-5131caff FROM_FILE offsets from SiliconForest 9f09c85f8d3b. - // This list is deliberately independent of both new lists. + // This list is deliberately independent of the three new lists. const OLD: [usize; 74] = [ 224,232,244,248,252,256,260,264,268,728,732,736,740,744,748, 992,1000,1012,1016,1024,1036,1496,1500,1504,1508,1512,1516, @@ -60,17 +75,32 @@ def function(source, marker): 45908,45912,45932,45940,45948,45956, ]; + const REF: Reference = Reference { + max_power_mw: 54321, + relative_power: [0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007], + fast_die_mask: 0x2111_2222_2333_2444, + alarm_mask: 0x2555_2666_2777_2888, + }; + + fn rows() -> [[u8; 12]; SENSOR_ROW_COUNT] { + [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"].map(|k| { + let mut r = [0x24u8; 12]; + r[..4].copy_from_slice(k); + r + }) + } + + fn row_words() -> Vec { + (SENSOR_ROWS..SENSOR_ROWS + 12 * SENSOR_ROW_COUNT).step_by(4).collect() + } + fn synthetic() -> (Vec, Vec, [Opp; 1], [u32; PROPS.len()]) { let mut file = vec![0u8; SIZE]; for (n, &at) in OLD.iter().enumerate() { put32(&mut file, at, 0x13570000 | (n as u32 + 1)).unwrap(); } - put32(&mut file, MAX_POWER, 54321).unwrap(); - for (n, key) in [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"].iter().enumerate() { - put32(&mut file, SENSOR_ROWS + 12*n, u32::from_le_bytes(**key)).unwrap(); - } // Public, synthetic OPPs: state0 off; eight voltage groups, five paired. - let mut main = vec![Opp {mhz:0, mv:[125;4], cells:4, mw:0}]; + let mut main = vec![Opp {mhz:0, mv:[125;4], cells:4}]; for (mv, freqs) in [ (600, &[300u32][..]), (650, &[500][..]), (700, &[700][..]), (750, &[800,900][..]), (800, &[950,1000][..]), @@ -78,10 +108,10 @@ def function(source, marker): (950, &[1250,1300][..]), ] { for &mhz in freqs { - main.push(Opp {mhz, mv:[mv;4], cells:4, mw:mhz*40}); + main.push(Opp {mhz, mv:[mv;4], cells:4}); } } - let cs = [Opp {mhz:24, mv:[635,0,0,0], cells:1, mw:0}]; + let cs = [Opp {mhz:24, mv:[635,0,0,0], cells:1}]; let mut props = [1u32; PROPS.len()]; props[PERIOD_MS] = 8; props[AVG_TC_MS] = 32; @@ -98,14 +128,16 @@ def function(source, marker): fn inputs<'a>(main: &'a [Opp], cs: &'a [Opp], props: [u32; PROPS.len()]) -> Inputs<'a> { Inputs {props, zones:&[], main, cs, afr:cs, slots:&[], empty_writable:&[], ttbat:0x100000, usc_base:0x1000000000, user_end:0x2000000000, - timestamps:0xfffffc2ffc000000, max_power_mw:54321} + timestamps:0xfffffc2ffc000000, sensor_rows: rows(), reference: &REF} } #[test] fn old74_is_exact_disjoint_partition() { - assert_eq!(FROM_FILE.len(), 32); + assert_eq!(REFERENCE_WORDS.len(), 14); + assert_eq!(row_words().len(), 18); assert_eq!(UNNEEDED.len(), 42); - let mut union = FROM_FILE.to_vec(); + let mut union = REFERENCE_WORDS.to_vec(); + union.extend(row_words()); union.extend_from_slice(&UNNEEDED); union.sort_unstable(); assert!(union.windows(2).all(|w| w[0] < w[1]), "duplicate or overlapping offsets"); @@ -115,30 +147,30 @@ def function(source, marker): } #[test] - fn actual_from_file_copies_only_retained32() { - let (file, _, _, _) = synthetic(); - let mut out = vec![0u8; SIZE]; - from_file(&mut out, &file).unwrap(); - for at in (0..SIZE).step_by(4) { - let expected = if FROM_FILE.contains(&at) {get32(&file, at).unwrap()} else {0}; - assert_eq!(get32(&out, at).unwrap(), expected, "copy offset +{at:#x}"); - } - } - - #[test] - fn actual_builder_leaves42_zero_and_preserves32() { + fn actual_builder_leaves42_zero_and_writes_the_32_from_their_sources() { let (file, main, cs, props) = synthetic(); - let mut out = vec![0u8; SIZE]; - from_file(&mut out, &file).unwrap(); + // Whatever the buffer held is not carried into the record. + let mut out = file.clone(); build(&mut out, &inputs(&main, &cs, props)).unwrap(); for &at in UNNEEDED.iter() { assert_ne!(get32(&file, at).unwrap(), 0, "fixture must exercise a real difference"); assert_eq!(get32(&out, at).unwrap(), 0, "built offset +{at:#x}"); } - for &at in FROM_FILE.iter() { - assert_eq!(get32(&out, at).unwrap(), get32(&file, at).unwrap(), "retained +{at:#x}"); + let mut reference = Vec::new(); + for r in REF.relative_power { reference.push(r); } + reference.push(REF.max_power_mw); + for m in [REF.fast_die_mask, REF.fast_die_mask, REF.alarm_mask] { + reference.push(m as u32); + reference.push((m >> 32) as u32); + } + for (&at, &want) in REFERENCE_WORDS.iter().zip(reference.iter()) { + assert_eq!(get32(&out, at).unwrap(), want, "reference +{at:#x}"); + } + let flat: Vec = rows().concat(); + for (k, &at) in row_words().iter().enumerate() { + assert_eq!(&out[at..at + 4], &flat[4 * k..4 * k + 4], "sensor row +{at:#x}"); } - // Model a matching board file with nonzero legacy values only at the42. + // Model a matching machine record with nonzero legacy values only at the42. let mut matching = out.clone(); for &at in UNNEEDED.iter() { put32(&mut matching, at, get32(&file, at).unwrap()).unwrap(); @@ -168,7 +200,7 @@ def function(source, marker): MUTANTS = { "broad-cross-check-exemption": ("cross_check", "UNNEEDED.contains(&at)", "true"), "nonzero-built-exempted": ("cross_check", "b == 0 && UNNEEDED.contains(&at)", "UNNEEDED.contains(&at)"), - "legacy-copy-reintroduced": ("from_file", "FROM_FILE.iter()", "FROM_FILE.iter().chain(UNNEEDED.iter())"), + "stale-buffer-kept": ("build", "out.fill(0);", ""), } @@ -246,7 +278,8 @@ def run(name, record): mutant = run(name, changed) if not mutant.returncode: raise RuntimeError("mutant survived: " + name) - regression = ("actual_from_file_copies_only_retained32" if func == "from_file" + regression = ("actual_builder_leaves42_zero_and_writes_the_32_from_their_sources" + if func == "build" else "actual_cross_check_exempts_only_zero_at_exact42") if regression + " ... FAILED" not in mutant.stdout: raise RuntimeError("new regression did not catch mutant: " + name) diff --git a/tools/asahi/mt7932-connect-result-tests.py b/tools/asahi/mt7932-connect-result-tests.py index 87054ccfcef80..61972b54619c5 100755 --- a/tools/asahi/mt7932-connect-result-tests.py +++ b/tools/asahi/mt7932-connect-result-tests.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only -"""Compile the connect worker's report path, its log-level helper and the -complete nl80211 encoder.""" +"""Compile the connect worker's report path, its log-level helper, the move +report helpers and the complete nl80211 encoder.""" from pathlib import Path import argparse import hashlib @@ -11,6 +11,7 @@ ROOT = Path(__file__).resolve().parents[2] LINK = Path('drivers/net/wireless/mediatek/mt7932/link.c') +HEADER = Path('drivers/net/wireless/mediatek/mt7932/mt7932.h') PREFIX = r''' #include @@ -24,6 +25,7 @@ #define WLAN_STATUS_SUCCESS 0 #define WLAN_STATUS_UNSPECIFIED_FAILURE 1 #define WLAN_REASON_UNSPECIFIED 1 +#define WLAN_REASON_DEAUTH_LEAVING 3 #define NL80211_TIMEOUT_UNSPECIFIED 0 #define GFP_ATOMIC 0 #define GFP_KERNEL 0 @@ -33,6 +35,7 @@ #define NLA_HDRLEN 4 #define BIT(n) (1U<<(n)) #define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) #define rcu_dereference(x) (x) #define rcu_read_lock() ((void)0) #define rcu_read_unlock() ((void)0) @@ -76,11 +79,12 @@ static int wiphy_net(int*w){return 0;} static void genlmsg_multicast_netns(int*f,int n,struct sk_buff*m,int p,int g,gfp_t t){} ENCODER -struct mt7932 {bool link_failed,stopping,connect_open,assoc_request_seen,connecting,connected,disconnecting,connect_cancelled,connect_publishing,connect_lost,rf_ready,connect_pmk_valid; +struct mt7932 {bool link_failed,stopping,connect_open,assoc_request_seen,connecting,connected,disconnecting,connect_cancelled,connect_publishing,connect_lost,rf_ready,connect_pmk_valid,roam_moving,roam_landed; int connect_error,response_lock,command_mutex,data_lock,wiphy,power_work;u16 disconnect_reason;struct net_device *netdev;struct cfg80211_bss *connect_bss;u8 connect_bssid[6],connect_pmk[32],connect_sae_pwd[63],connect_sae_pwd_length,assoc_request_ies[512];unsigned assoc_request_ie_len; struct {struct {bool ht_supported;}ht_cap;}band2;bool power_tim,data_ready,station_signal_valid,station_rx_rate_valid,station_tx_rate_valid;unsigned long station_tx_bytes,station_rx_bytes,station_tx_packets,station_rx_packets,station_connected;}; static unsigned long jiffies=1; -static unsigned info_logs,err_logs,notifications,disconnections,retirements,releases,authorizations,carrier,work_count; +static unsigned info_logs,err_logs,notifications,disconnections,retirements,releases,authorizations,carrier,work_count,carrier_offs,deferred; +static u16 last_reason;static bool last_local; static struct cfg80211_connect_resp_params saved; static bool mt_rf_allowed(struct mt7932*m){return m->rf_ready;} static void mt_transport_snapshot(struct mt7932*m){} @@ -95,11 +99,15 @@ static void cfg80211_connect_done(struct net_device*n,struct cfg80211_connect_resp_params*p,gfp_t f){struct cfg80211_registered_device r={};saved=*p;notifications++;nl80211_send_connect_result(&r,n,p,f);} static void schedule_work(int*w){work_count++;} static void netif_carrier_on(struct net_device*n){carrier++;} +static void netif_carrier_off(struct net_device*n){carrier_offs++;} +static void mt_reg_apply_deferred_locked(struct mt7932*m){deferred++;} +MOVE_RULES +static void mt_report_disconnected(struct mt7932*m,u16 r,bool local,gfp_t f){disconnections++;last_reason=r;last_local=local;} static void netif_wake_queue(struct net_device*n){} static void cfg80211_port_authorized(struct net_device*n,const u8*b,const u8*p,size_t l,gfp_t f){authorizations++;} HELPER static void report_worker(struct mt7932 *m,int ret,bool retirement_attempted,bool auth_submitted){ - struct cfg80211_connect_resp_params response={};unsigned long flags;bool submitted=true,peer_owned=true,cancelled;u16 reason; + struct cfg80211_connect_resp_params response={};unsigned long flags;bool submitted=true,peer_owned=true,cancelled;bool roam __attribute__((unused));struct mt7932_connect_report outcome __attribute__((unused));u16 reason; REPORT int main(void){struct net_device n={1};struct cfg80211_bss b={}; int errors[]={0,-ETIMEDOUT,-ECONNREFUSED,-EPROTO,-ECANCELED,-ECONNRESET,-EIO}; @@ -144,7 +152,30 @@ // Already-known IEEE AP status is preserved by the complete netlink encoder. struct cfg80211_registered_device r={};struct cfg80211_connect_resp_params ap={.status=17}; nl80211_send_connect_result(&r,&n,&ap,0);assert(encoded.values[NL80211_ATTR_STATUS_CODE]==17&&!encoded.seen[NL80211_ATTR_TIMED_OUT]); - puts("PASS 12 connect report, info-vs-error log level and nl80211 timeout/rejection/cancellation controls");} + // A cancelled connect reports the cancel's reason and applies no deferral. + m=(struct mt7932){.connecting=true,.rf_ready=true,.netdev=&n,.connect_bss=&b,.connect_cancelled=true,.connect_error=-ECANCELED,.disconnect_reason=8}; + notifications=disconnections=retirements=authorizations=carrier=carrier_offs=deferred=0; + report_worker(&m,0,false,true);assert(notifications==0&&disconnections==1&&last_reason==8&&last_local&&!deferred); + // A move that lands takes over the connection through one connect result. + m=(struct mt7932){.connecting=true,.rf_ready=true,.netdev=&n,.connect_bss=&b,.data_ready=true,.roam_moving=true}; + notifications=disconnections=retirements=authorizations=carrier=carrier_offs=deferred=0;info_logs=err_logs=0; + report_worker(&m,0,false,true);assert(notifications==1&&saved.status==0&&!disconnections&&m.connected&&m.roam_landed&&!m.roam_moving); + assert(carrier==1&&!carrier_offs&&authorizations==1&&!deferred&&!retirements&&!info_logs&&!err_logs); + // A move that fails reports its failed result, then ends the old connection. + m=(struct mt7932){.connecting=true,.rf_ready=true,.netdev=&n,.connect_bss=&b,.data_ready=true,.roam_moving=true,.roam_landed=true,.disconnect_reason=8}; + notifications=disconnections=retirements=authorizations=carrier=carrier_offs=deferred=0;info_logs=err_logs=0; + report_worker(&m,-ETIMEDOUT,false,true);assert(notifications==1&&saved.status==-1&&disconnections==1&&last_reason==WLAN_REASON_DEAUTH_LEAVING&&last_local); + assert(!m.connected&&!m.roam_landed&&!m.roam_moving&&carrier_offs==1&&deferred==1&&retirements==1&&!carrier&&!authorizations&&info_logs==1); + // A cancelled move reports its failed result, then the cancel's reason. + m=(struct mt7932){.connecting=true,.rf_ready=true,.netdev=&n,.connect_bss=&b,.roam_moving=true,.connect_cancelled=true,.connect_error=-ECANCELED,.disconnect_reason=8}; + notifications=disconnections=retirements=authorizations=carrier=carrier_offs=deferred=0; + report_worker(&m,0,false,true);assert(notifications==1&&saved.status==1&&disconnections==1&&last_reason==8&&last_local); + assert(!m.connected&&!m.roam_landed&&!m.roam_moving&&carrier_offs==1&&deferred==1&&!authorizations); + // An ordinary connect after a landed move clears the mark. + m=(struct mt7932){.connecting=true,.rf_ready=true,.netdev=&n,.connect_bss=&b,.data_ready=true,.roam_landed=true}; + notifications=disconnections=retirements=authorizations=carrier=carrier_offs=deferred=0; + report_worker(&m,0,false,true);assert(notifications==1&&!disconnections&&m.connected&&!m.roam_landed&&!carrier_offs&&!deferred); + puts("PASS 17 connect report, info-vs-error log level, nl80211 timeout/rejection/cancellation and move report controls");} ''' @@ -158,6 +189,17 @@ def peer_helper(source): return source[start:source.index('\n}\n', start) + 3] +def move_helpers(header): + """The move report helpers from mt7932.h; empty for a driver without + them.""" + marker = 'struct mt7932_connect_report {' + if marker not in header: + return '' + start = header.index(marker) + last = header.index('static inline bool mt7932_disconnect_local(', start) + return header[start:header.index('\n}\n', last) + 3] + + def main(): parser = argparse.ArgumentParser() parser.add_argument('--source-root', type=Path, default=ROOT) @@ -170,6 +212,7 @@ def main(): end = source.index('\n/* Narrow ordinary OPEN', start) report = source[start:end] helper = peer_helper(source) + move = move_helpers((root/HEADER).read_text()) net = (root/'net/wireless/nl80211.c').read_text() start = net.index('void nl80211_send_connect_result(') end = net.index('\nvoid nl80211_send_roamed(', start) @@ -188,7 +231,7 @@ def main(): results = {} for name, body in cases.items(): c = out/(name+'.c') - c.write_text(PREFIX.replace('ENCODER', encoder).replace('HELPER', helpers[name]) + c.write_text(PREFIX.replace('MOVE_RULES', move).replace('ENCODER', encoder).replace('HELPER', helpers[name]) .replace('REPORT', body)) binary = out/name subprocess.run(['cc','-std=gnu11','-Wall','-Werror','-Wno-unused-parameter', diff --git a/tools/asahi/mt7932-scan-tests.py b/tools/asahi/mt7932-scan-tests.py index a47c7b938af1b..2442e86d173fa 100644 --- a/tools/asahi/mt7932-scan-tests.py +++ b/tools/asahi/mt7932-scan-tests.py @@ -130,7 +130,7 @@ def function(source, name): m.connected=true;m.scan_home_channel=5;m.scan_home_band=NL80211_BAND_6GHZ; request.flags=request.n_6ghz_params=0; assert(mt_scan_submit_next(&m)==0); - assert(bodies[0][2]==1&&bodies[0][3]==8&&bodies[0][4]==1); + assert(bodies[0][2]==1&&bodies[0][3]==4&&bodies[0][4]==1); assert(!memcmp(bodies[0]+12,"hidden",6)); assert(bodies[0][0x4cc]==5&&bodies[0][0x4cd]==3); /* NO_IR remains passive, even for an explicit legacy SSID. */ diff --git a/tools/asahi/neo-stability-tests.py b/tools/asahi/neo-stability-tests.py index 16256c1843fd8..a8319e71b33d5 100644 --- a/tools/asahi/neo-stability-tests.py +++ b/tools/asahi/neo-stability-tests.py @@ -132,7 +132,7 @@ def function(source, name): static void rtnl_unlock(void){assert(rtnl_depth--==1);} static void wiphy_lock(struct wiphy*w){assert(rtnl_depth&& !wiphy_depth++);} static void wiphy_unlock(struct wiphy*w){assert(wiphy_depth--==1);} -struct mt7932 {struct ieee80211_channel channels[MT7932_CHANNELS_2G],channels5[MT7932_CHANNELS_5G],channels6[MT7932_CHANNELS_6G];struct wiphy*wiphy;struct net_device*netdev;struct {struct device dev;}*pdev;struct mt7932_reg_snapshot reg_desired,reg_deferred;unsigned reg_generation,response_lock,startup_work;bool reg_pending,stopping,interface_registered,connecting,connected,disconnecting,reg_deferral,data_ready,six_ap_valid;int data_lock;int connect_error,assoc_start,assoc_done,discovery_done;uint64_t policy_disabled[(MT7932_CHANNELS+63)/64];}; +struct mt7932 {struct ieee80211_channel channels[MT7932_CHANNELS_2G],channels5[MT7932_CHANNELS_5G],channels6[MT7932_CHANNELS_6G];struct wiphy*wiphy;struct net_device*netdev;struct {struct device dev;}*pdev;struct mt7932_reg_snapshot reg_desired,reg_deferred;unsigned reg_generation,response_lock,startup_work;bool reg_pending,stopping,interface_registered,connecting,connected,disconnecting,reg_deferral,data_ready,six_ap_valid,roam_moving;int data_lock;int connect_error,assoc_start,assoc_done,discovery_done;uint64_t policy_disabled[(MT7932_CHANNELS+63)/64];}; #define spin_lock(l) ((void)(l)) #define spin_unlock(l) ((void)(l)) static void mt_data_kick_locked(struct mt7932*m){assert(0 && "unexpected TX boundary");} @@ -245,6 +245,10 @@ def function(source, name): assert(m.reg_generation==applied&&!m.reg_pending&&m.reg_deferral); assert(get_unaligned_le32(m.reg_desired.domain+16)==IEEE80211_CHAN_NO_IR); assert(get_unaligned_le32(m.reg_deferred.domain+16)==0); + /* A move to another AP carries the connection on: still deferred. */ + m.connected=false;m.roam_moving=true;m.reg_deferral=false;mt_regulatory_notify(&w,&request); + assert(m.reg_generation==applied&&!m.reg_pending&&m.reg_deferral&&get_unaligned_le32(m.reg_desired.domain+16)==IEEE80211_CHAN_NO_IR); + m.roam_moving=false; m.stopping=true;unsigned n=m.reg_generation;request.alpha2[0]='U';mt_regulatory_notify(&w,&request);assert(m.reg_generation==n); } puts("PASS");return 0; diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 318f9e892505c..a075e2a1dab20 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -14,6 +14,9 @@ Negative controls read older revisions with `git show`, so the checkout needs the commits (a full or treeless clone, not a depth-1 one). +A test that reads device trees gets '{dtc}', this tree's scripts/dtc/dtc, built +with the kconfig tool unless --dtc names one. + Usage: tools/asahi/run-host-tests.py [--jobs N] [--work DIR] [--only SUBSTR] """ import argparse @@ -61,6 +64,7 @@ def mutations(flag, names, base, expect=KILL): 'apple-pwm-pm-tests.py': [([], PASS), (['--negative-control'], PASS)], 'apple-dart-fw-tests.py': [(out(), PASS)], 'apple-dart-range-tests.py': [([], PASS)], + 'asahi-stats-raw-tests.py': [([], PASS)], 'atc-usb3-tests.py': [(out(), PASS), (out() + ['--mutations'], PASS)], 'avd-av1-tail-tests.py': [(out(), PASS)] + [(out() + ['--case', c], PASS) for c in ('copy', 'seed', 'aligned')] @@ -89,6 +93,7 @@ def mutations(flag, names, base, expect=KILL): 'display26-tests.py': [([], PASS)], # The old revision publishes the fd before copy_to_user() succeeds. 'dma-heap-fd-tests.py': [(out() + ['--old-ref', 'cb732d027aa18e1fcf9d2797f47d20b179ebc59c'], PASS)], + 'g15-display-order-tests.py': [([], PASS)], 'g15-input-error-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS), (out('--output') + ['--ref', 'ede99b3c1990b4a4a8c045fdf627a2bd0f14446a'], PASS)], 'g15-maintained-handoff-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], @@ -136,6 +141,7 @@ def mutations(flag, names, base, expect=KILL): 't6030-clock-tests.py': [(out(), PASS)], 't6030-display-admission-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 't6030-display-linkage-tests.py': [(out('--output') + ['--conf', '{conf}'], PASS)], + 't6030-pmp-dt-tests.py': [(out('--output') + ['--dtc', '{dtc}'], PASS)], 't6030-pmp-graph-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 't6030-pmp-supplier-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 't6030-wiring-tests.py': [(out(), PASS), (out() + ['--old-classifier', 'tb'], PASS), @@ -168,7 +174,7 @@ def mutations(flag, names, base, expect=KILL): ] -def jobs(work, conf): +def jobs(work, conf, dtc): found = {p.name for p in ASAHI.iterdir() if re.search(r'-tests\.(py|rs|sh)$', p.name) and p.name != Path(__file__).name} known = set(ASAHI_TESTS) | set(ASAHI_RUST) | set(ASAHI_SHELL) @@ -190,9 +196,11 @@ def jobs(work, conf): for name, runs in ASAHI_TESTS.items(): for index, (args, expect) in enumerate(runs): label = name.removesuffix('.py') + ('' if not args or args == out() or args == out('--output') - else ' ' + ' '.join(a for a in args if a not in ('{out}', '--out', '--output', '{conf}', '--conf'))) + else ' ' + ' '.join(a for a in args if a not in ('{out}', '--out', '--output', '{conf}', '--conf', + '{dtc}', '--dtc'))) d = work / f'{name}.{index}' - cmd = ['python3', '-I', '-B', str(ASAHI / name)] + [a.replace('{out}', str(d / 'out')).replace('{conf}', str(conf)) for a in args] + cmd = ['python3', '-I', '-B', str(ASAHI / name)] + [a.replace('{out}', str(d / 'out')).replace('{conf}', str(conf)) + .replace('{dtc}', str(dtc)) for a in args] result.append((label, cmd, ASAHI, expect, d, {})) for name in ASAHI_RUST: d = work / name @@ -233,6 +241,7 @@ def main(): ap.add_argument('--jobs', type=int, default=os.cpu_count() or 2) ap.add_argument('--work', type=Path, help='keep logs and build products here (default: a temporary directory)') ap.add_argument('--conf', type=Path, help='a built scripts/kconfig/conf (default: build one)') + ap.add_argument('--dtc', type=Path, help='a built scripts/dtc/dtc (default: build one)') ap.add_argument('--only', help='run only jobs whose name contains this text') ap.add_argument('--timeout', type=int, default=900) args = ap.parse_args() @@ -240,13 +249,18 @@ def main(): work = args.work or Path(tempfile.mkdtemp(prefix='aurora-host-tests-')) work = work.resolve() work.mkdir(parents=True, exist_ok=True) - conf = args.conf - if conf is None: - build = work / 'kconfig' + conf, dtc = args.conf, args.dtc + build = work / 'kconfig' + if conf is None or dtc is None: subprocess.run(['make', '-s', '-C', str(ROOT), f'O={build}', 'ARCH=arm64', 'allnoconfig'], check=True, stdout=subprocess.DEVNULL) + if conf is None: conf = build / 'scripts/kconfig/conf' - todo = jobs(work, conf.resolve()) + if dtc is None: + subprocess.run(['make', '-s', '-C', str(ROOT), f'O={build}', 'ARCH=arm64', 'scripts_dtc'], + check=True, stdout=subprocess.DEVNULL) + dtc = build / 'scripts/dtc/dtc' + todo = jobs(work, conf.resolve(), dtc.resolve()) if args.only: todo = [j for j in todo if args.only in j[0]] print(f'{len(todo)} runs, {args.jobs} at a time, logs in {work}', flush=True) diff --git a/tools/asahi/t6030-display-admission-tests.c b/tools/asahi/t6030-display-admission-tests.c index e2c75eddb5ea5..ebf3e2e868324 100644 --- a/tools/asahi/t6030-display-admission-tests.c +++ b/tools/asahi/t6030-display-admission-tests.c @@ -22,12 +22,14 @@ struct device { struct device_link { int unused; }; -static pthread_mutex_t apple_t6030_display_lock = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t apple_g15_display_lock = PTHREAD_MUTEX_INITIALIZER; static struct device *apple_t6030_display; +static struct device *apple_t6031_display; static struct device_node display_node = { "apple,t6030-display-subsystem" }, gpu_node = { "apple,agx-t6030" }, - other_node = { "apple,t6031-display-subsystem" }; -static struct device display, duplicate, gpu; + other_node = { "apple,display-subsystem" }, + t6031_display_node = { "apple,t6031-display-subsystem" }; +static struct device display, duplicate, gpu, t6031_display; static const char *machine = "apple,t6030"; static int pmp_calls, link_calls, pmp_result; static bool link_ok = true; @@ -106,7 +108,7 @@ static int apple_pmp_link_device(struct device *dev) #include "production.h" static void reset(void) { - assert(!apple_t6030_display && !locked); + assert(!apple_t6030_display && !apple_t6031_display && !locked); display = (struct device){ .of_node = &display_node, .refs = 1 }; duplicate = (struct device){ .of_node = &display_node, .refs = 1 }; gpu = (struct device){ .of_node = &gpu_node, .refs = 1 }; @@ -127,7 +129,7 @@ static void *admit_thread(void *unused) static void *withdraw_thread(void *unused) { (void)unused; - apple_t6030_display_withdraw(&display); + apple_g15_display_withdraw(&display); atomic_store(&withdrew, true); return NULL; } @@ -140,18 +142,18 @@ int main(void) assert(asahi_g15_pmp_prepare(&gpu) == -EPROBE_DEFER && !pmp_calls); cases++; /* Component platform bind alone never calls this publication. Retry succeeds only afterward. */ - assert(apple_t6030_display_publish(&display) == 0 && + assert(apple_g15_display_publish(&display) == 0 && atomic_load(&display.refs) == 2); cases++; assert(asahi_g15_pmp_prepare(&gpu) == 0 && pmp_calls == 1); cases++; - assert(apple_t6030_display_publish(&display) == 0 && + assert(apple_g15_display_publish(&display) == 0 && atomic_load(&display.refs) == 2); cases++; - assert(apple_t6030_display_publish(&duplicate) == -EBUSY && + assert(apple_g15_display_publish(&duplicate) == -EBUSY && apple_t6030_display == &display); cases++; - apple_t6030_display_withdraw(&duplicate); + apple_g15_display_withdraw(&duplicate); assert(apple_t6030_display == &display && atomic_load(&display.refs) == 2); cases++; @@ -170,17 +172,27 @@ int main(void) assert(apple_drm_t6030_link_device(&gpu) == -ENODEV); cases++; gpu.of_node = &gpu_node; - apple_t6030_display_withdraw(&display); + apple_g15_display_withdraw(&display); assert(atomic_load(&display.refs) == 1 && !apple_t6030_display); cases++; assert(asahi_g15_pmp_prepare(&gpu) == -EPROBE_DEFER && pmp_calls == 2); cases++; display.of_node = &other_node; - assert(apple_t6030_display_publish(&display) == 0 && - !apple_t6030_display && atomic_load(&display.refs) == 1); + assert(apple_g15_display_publish(&display) == 0 && + !apple_t6030_display && !apple_t6031_display && + atomic_load(&display.refs) == 1); + cases++; + /* The T6031 display has its own slot: it never satisfies T6030 admission. */ + t6031_display = (struct device){ .of_node = &t6031_display_node, .refs = 1 }; + assert(apple_g15_display_publish(&t6031_display) == 0 && + apple_t6031_display == &t6031_display && !apple_t6030_display && + atomic_load(&t6031_display.refs) == 2); + assert(asahi_g15_pmp_prepare(&gpu) == -EPROBE_DEFER && pmp_calls == 2); + apple_g15_display_withdraw(&t6031_display); + assert(!apple_t6031_display && atomic_load(&t6031_display.refs) == 1); cases++; reset(); - assert(!apple_t6030_display_publish(&display)); + assert(!apple_g15_display_publish(&display)); atomic_store(&pause_link, true); pthread_t admit, withdraw; assert(!pthread_create(&admit, NULL, admit_thread, NULL)); diff --git a/tools/asahi/t6030-display-admission-tests.py b/tools/asahi/t6030-display-admission-tests.py index a1da987377f18..479c11b00fed0 100644 --- a/tools/asahi/t6030-display-admission-tests.py +++ b/tools/asahi/t6030-display-admission-tests.py @@ -2,9 +2,11 @@ # SPDX-License-Identifier: GPL-2.0-only """Actual T6030 display publisher and G15 admission, with pthread boundaries. -Tests initial admission and pointer lifetime, not safe shutdown of retained G15 -firmware after forced display unbind. Full DRM publication placement is checked -as source ordering separately from executable host boundary controls. +The publisher is the shared G15 display slot code; T6030 admission goes through +the shared link helper. Tests initial admission and pointer lifetime, not safe +shutdown of retained G15 firmware after forced display unbind. Full DRM +publication placement is checked as source ordering separately from executable +host boundary controls. """ import argparse import hashlib @@ -26,23 +28,26 @@ def main(): glue = (ROOT / 'drivers/gpu/drm/asahi/g15/glue.c').read_text() init = extract.function(drv, 'static int apple_drm_init(') uninit = extract.function(drv, 'static void apple_drm_uninit(') - assert init.index('ret = drm_dev_register(') < init.index('drm_client_setup_with_fourcc(') < init.index('ret = apple_t6030_display_publish(') - assert uninit.index('apple_t6030_display_withdraw(') < uninit.index('apple_drm_quiesce_connectors(') < uninit.index('drm_dev_unplug(') + assert init.index('ret = drm_dev_register(') < init.index('drm_client_setup_with_fourcc(') < init.index('ret = apple_g15_display_publish(') + assert uninit.index('apple_g15_display_withdraw(') < uninit.index('apple_drm_quiesce_connectors(') < uninit.index('drm_dev_unplug(') assert 'drm_dev_unregister(&apple->drm);\n\t\tgoto err_unbind;' in init - code = '\n'.join([extract.function(drv, 'static int apple_t6030_display_publish('), - extract.function(drv, 'static void apple_t6030_display_withdraw('), + code = '\n'.join([extract.function(drv, 'static struct device **apple_g15_display_slot('), + extract.function(drv, 'static int apple_g15_display_publish('), + extract.function(drv, 'static void apple_g15_display_withdraw('), + extract.function(drv, 'static int apple_g15_link_device('), extract.function(drv, 'int apple_drm_t6030_link_device('), extract.function(glue, 'int asahi_g15_pmp_prepare(struct device *dev)\n')]) variants = [('control', code)] if args.mutations: for name, old, new in [ - ('no-ready-gate', 'if (!apple_t6030_display)', 'if (false)'), - ('no-display-get', 'apple_t6030_display = get_device(dev);', 'apple_t6030_display = dev;'), + ('no-ready-gate', 'if (!*display)', 'if (false)'), + ('no-display-get', '*slot = get_device(dev);', '*slot = dev;'), ('no-display-put', 'put_device(old);', '(void)old;'), - ('replace-live-provider', 'return apple_t6030_display == dev ? 0 : -EBUSY;', 'apple_t6030_display = dev;'), + ('replace-live-provider', 'return *slot == dev ? 0 : -EBUSY;', '*slot = dev;'), ('ignore-display-error', 'if (ret)\n\t\treturn ret;', 'if (false)\n\t\treturn ret;'), - ('wrong-gpu', '!of_device_is_compatible(consumer->of_node, "apple,agx-t6030")', 'false'), - ('withdraw-wrong-provider', 'if (apple_t6030_display == dev)', 'if (((void)dev, true))'), + ('wrong-gpu', '!of_device_is_compatible(consumer->of_node, gpu)', '((void)gpu, false)'), + ('withdraw-wrong-provider', 'if (*slot == dev)', 'if (((void)dev, true))'), + ('shared-display-slot', 'return &apple_t6031_display;', 'return &apple_t6030_display;'), ]: assert code.count(old) == 1, (name, code.count(old)) variants.append((name, code.replace(old, new))) diff --git a/tools/asahi/t6030-pmp-dt-tests.py b/tools/asahi/t6030-pmp-dt-tests.py new file mode 100644 index 0000000000000..913e8d31aa7d7 --- /dev/null +++ b/tools/asahi/t6030-pmp-dt-tests.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Check static PMP ownership in the compiled T6030 device trees.""" + +import argparse +from pathlib import Path +import subprocess + + +ROOT = Path(__file__).resolve().parents[2] +BOARDS = ("t6030-j514s", "t6030-j516s") +PMP_NODES = { + "iommu@350300000", + "pmp_report@3503c0000", + "pmp@350500000", + "mbox@350c08000", +} +PMP_COMPATIBLES = {"apple,t6000-pmp-v2", "apple,t6030-pmp-v2-report"} + + +def check_default_compatibles(dtb, board): + pending = ["/"] + while pending: + node = pending.pop() + raw = subprocess.check_output( + ["fdtget", "-t", "bx", "-d", "", str(dtb), node, "compatible"], + text=True, + ) + # Match the gate's whole-tree OF lookup, including disabled nodes + # and every string in a compatible list. + compatible = bytes(int(value, 16) for value in raw.split()) + matches = PMP_COMPATIBLES.intersection( + value.decode("ascii").lower() for value in compatible.split(b"\0") + if value + ) + if matches: + raise AssertionError(f"{board} default: static PMP compatible " + f"{sorted(matches)} at {node}") + children = subprocess.check_output( + ["fdtget", "-l", str(dtb), node], text=True, + ) + pending.extend(node.rstrip("/") + "/" + child + for child in children.splitlines()) + + +def check_profile(tree, output, dtc, board, static): + profile = "static" if static else "default" + dtb = output / f"{board}-{profile}.dtb" + command = [ + "cpp", "-nostdinc", "-undef", "-D__DTS__", + "-I", str(tree / "scripts/dtc/include-prefixes"), + "-x", "assembler-with-cpp", + ] + if static: + command.append("-DAPPLE_USE_PMP") + command.append(str(tree / f"arch/arm64/boot/dts/apple/{board}.dts")) + source = subprocess.check_output(command) + subprocess.run([dtc, "-q", "-I", "dts", "-O", "dtb", "-o", str(dtb)], + input=source, check=True) + + children = subprocess.check_output(["fdtget", "-l", str(dtb), "/soc"], text=True) + present = PMP_NODES.intersection(children.split()) + expected = PMP_NODES if static else set() + if present != expected: + raise AssertionError(f"{board} {profile}: static PMP nodes " + f"{sorted(present)}, expected {sorted(expected)}") + if not static: + check_default_compatibles(dtb, board) + + alias = subprocess.run(["fdtget", "-t", "s", str(dtb), "/aliases", "pmp"], + capture_output=True, text=True) + if static: + if alias.returncode or alias.stdout.strip() != "/soc/pmp@350500000": + raise AssertionError(f"{board} {profile}: missing or incorrect PMP alias") + elif alias.returncode == 0: + raise AssertionError(f"{board} {profile}: unexpected static PMP alias") + print(f"{board} {profile}: PASS") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tree", type=Path, default=ROOT) + parser.add_argument("--dtc", required=True, help="this kernel's built scripts/dtc/dtc") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + for board in BOARDS: + for static in (False, True): + check_profile(args.tree.resolve(), args.output, args.dtc, board, static) + + +if __name__ == "__main__": + main() diff --git a/tools/asahi/t6030-pmp-graph-tests.c b/tools/asahi/t6030-pmp-graph-tests.c index 3746d2a3e3fdb..ca1cde04f1363 100644 --- a/tools/asahi/t6030-pmp-graph-tests.c +++ b/tools/asahi/t6030-pmp-graph-tests.c @@ -50,7 +50,7 @@ static void host_log(const char *format, ...) #define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) struct device_node { const char *name, *compatible; - bool available, always_on, no_ack, deferred, irq_safe, uuid, soc_record; + bool available, always_on, no_ack, deferred, irq_safe, uuid, soc_record, release_sleep; u32 reg; int refs; struct device_node *parent, *child, *sibling, *pmp; @@ -197,6 +197,7 @@ struct apple_pmp_report_entry { struct device_node *node, *report_node; u32 id; bool no_ack, dvfs, removed; + bool sleep_release, released; u32 dvfs_entry; u64 dvfs_on; }; @@ -398,11 +399,18 @@ static u64 readq(const void *a) { return *(const u64 *)a; } +/* The sleep cases: the PMP acknowledges the requests it is given, ack_late writes late. */ +static bool ack_follows; +static unsigned int ack_late; static void writeq(u64 v, void *a) { assert(a == (char *)regs + 0x108c0); *(u64 *)a = v; regs[0x1180 / 8] = v; + if (ack_follows && ack_late) + ack_late--; + else if (ack_follows) + regs[0x11c0 / 8] = v; } #define readq_poll_timeout(address, value, condition, delay, timeout) \ ({ \ @@ -467,6 +475,8 @@ static bool of_property_read_bool(const struct device_node *n, const char *p) return n->irq_safe; if (!strcmp(p, "apple,soc-device-record")) return n->soc_record; + if (!strcmp(p, "apple,release-for-sleep")) + return n->release_sleep; return false; } static bool of_property_present(const struct device_node *n, const char *p) @@ -843,6 +853,8 @@ static struct apple_pmp_report *setup(void) put32(devices + i * 0x7c + 8, 2); } ack_polls = 0; + ack_follows = false; + ack_late = 0; check_error = enable_error = 0; hw_binds = hw_ready = true; hw_ack = BIT_ULL(7); @@ -1041,6 +1053,49 @@ int main(void) cases++; cleanup(); #endif + +#ifdef HAVE_SLEEP_RELEASE + /* System sleep withdraws an acknowledged request marked for it, then asserts it again. */ + setup(); + entries[1].release_sleep = true; + assert(apple_pmp_report_probe(&report_dev) == 0); + assert(apple_pmp_report_wait_ready(&entries[0], 30000) == 0); + struct apple_pmp_report_entry *disp = entry_devs[0].dev.data; + struct apple_pmp_report_entry *ext = entry_devs[1].dev.data; + + assert(ext->sleep_release && !disp->sleep_release); + ack_follows = true; + regs[0x11c0 / 8] = regs[0x1180 / 8]; + u64 target = regs[0x1180 / 8]; + + assert(target & BIT_ULL(8)); + assert(apple_pmp_report_entry_suspend_noirq(&entry_devs[0].dev) == 0 && + !disp->released && regs[0x1180 / 8] == target); + assert(apple_pmp_report_entry_suspend_noirq(&entry_devs[1].dev) == 0); + assert(ext->released && regs[0x1180 / 8] == (target & ~BIT_ULL(8))); + assert(apple_pmp_report_entry_resume_noirq(&entry_devs[1].dev) == 0); + assert(!ext->released && regs[0x1180 / 8] == target); + cases++; + /* A request the PMP has not acknowledged is left as it is. */ + regs[0x11c0 / 8] &= ~BIT_ULL(8); + assert(apple_pmp_report_entry_suspend_noirq(&entry_devs[1].dev) == 0); + assert(!ext->released && regs[0x1180 / 8] == target); + assert(apple_pmp_report_entry_resume_noirq(&entry_devs[1].dev) == 0 && + regs[0x1180 / 8] == target); + cases++; + /* A re-assert the PMP acknowledges late is written again, three times at most. */ + regs[0x11c0 / 8] = target; + assert(apple_pmp_report_entry_suspend_noirq(&entry_devs[1].dev) == 0 && ext->released); + ack_late = 2; + assert(apple_pmp_report_entry_resume_noirq(&entry_devs[1].dev) == 0 && + !ack_late && regs[0x11c0 / 8] == target); + assert(apple_pmp_report_entry_suspend_noirq(&entry_devs[1].dev) == 0 && ext->released); + ack_late = 5; + assert(apple_pmp_report_entry_resume_noirq(&entry_devs[1].dev) == -ETIMEDOUT && + ack_late == 2 && regs[0x1180 / 8] == target); + cases++; + cleanup(); +#endif printf("PASS: %u actual overlay/probe/entry/DCP7/managed-start/supplier graph cases\n", cases); return 0; diff --git a/tools/asahi/t6030-pmp-graph-tests.py b/tools/asahi/t6030-pmp-graph-tests.py index ea83b1c743925..4c85001acf1a8 100644 --- a/tools/asahi/t6030-pmp-graph-tests.py +++ b/tools/asahi/t6030-pmp-graph-tests.py @@ -51,6 +51,13 @@ def main(): names.insert(0, 'static bool apple_pmp_report_running(') if 'static int apple_pmp_report_wait_t6030_bound(' in source: names.insert(names.index('static struct device_node *apple_pmp_report_t6030_entry('), 'static int apple_pmp_report_wait_t6030_bound(') + # System sleep (apple,release-for-sleep), where the source has it. + sleep = 'static int apple_pmp_report_entry_sleep(' in source + if sleep: + at = names.index('static int apple_pmp_report_entry_power_on(') + names[at:at] = ['static int apple_pmp_report_entry_sleep(', + 'static int apple_pmp_report_entry_suspend_noirq(', + 'static int apple_pmp_report_entry_resume_noirq('] offsets = re.search(r'static const struct apple_pmp_report_offsets apple_pmp_offsets_t6030 = \{.*?\n\};', source, re.S)[0] functions = '\n'.join(extract.function(source, n) for n in names) validation = re.sub(r'^#include .*\n', '', (ROOT / 'drivers/pmdomain/apple/pmp-report-validation.h').read_text(), flags=re.M) @@ -67,7 +74,8 @@ def main(): dram_attrs = ('#pragma GCC diagnostic push\n' '#pragma GCC diagnostic ignored "-Wunused-parameter"\n' + dram_attrs + '#pragma GCC diagnostic pop\n') - code = offsets + '\n' + validation + '\n' + dram_attrs + '\n' + functions + code = (('#define HAVE_SLEEP_RELEASE 1\n' if sleep else '') + offsets + '\n' + validation + + '\n' + dram_attrs + '\n' + functions) # The actual overlay must still describe a halted supplier and disabled DMA/mailbox. assert overlay.count('status = \"disabled\";') >= 3 (args.output / 'production.h').write_text(code) @@ -116,6 +124,13 @@ def main(): ('irq-safe-without-property', 'of_property_read_bool(node, "apple,irq-safe")', 'true'), ('skip-supplier-wait', 'apple_pmp_report_wait_ready(entry, msecs_to_jiffies(timeout_ms))', '((void)timeout_ms, 0)'), ] + if sleep: + controls += [ + ('sleep-releases-unacked', '!(readq(rep->base + rep->offsets->actual) & BIT_ULL(ent->id))', 'false'), + ('sleep-not-reasserted', 'ret = apple_pmp_report_entry_sleep(ent, false);', 'ret = 0;'), + ('sleep-ignores-mark', '!ent->sleep_release ||', ''), + ('sleep-not-rewritten', '++tries < 3', '++tries < 1'), + ] for name, old, new in controls: assert code.count(old) == 1, (name, code.count(old)) (args.output / 'production.h').write_text(code.replace(old, new)) diff --git a/tools/asahi/t6030-pmp-supplier-tests.c b/tools/asahi/t6030-pmp-supplier-tests.c index b581f3863a232..ecb6dcec9fff5 100644 --- a/tools/asahi/t6030-pmp-supplier-tests.c +++ b/tools/asahi/t6030-pmp-supplier-tests.c @@ -130,6 +130,8 @@ static int pmp_lock, ready_calls, add_calls, add_result; static int poll_calls, device_locked, power_calls, power_result, link_calls; static int find_calls, lockable, present_device, link_ok; static int case_count; +/* The report transaction itself is covered by the G15 PMP-vote selftest. */ +static int gfx_check_result, gfx_check_calls, gfx_set_calls; static struct device_node *of_node_get(struct device_node *node) { @@ -375,6 +377,19 @@ static void queue_work(void *wq, int *work) managed_start.result = ready_result; managed_start.started.done = true; } +static int apple_pmp_report_check_t6030_gfx(struct device *dev) +{ + assert(dev == pmp_device && pmp_lock == 1); + gfx_check_calls++; + return gfx_check_result; +} +static int apple_pmp_report_set_t6030_gfx(struct device *dev, bool enabled) +{ + assert(dev == pmp_device && pmp_lock == 1); + (void)enabled; + gfx_set_calls++; + return 0; +} #include "t6030-pmp-production.h" static void balanced(void) @@ -457,6 +472,7 @@ static void reset(void) pmp_data = (void *)1; ready_calls = add_calls = add_result = poll_calls = 0; power_calls = power_result = link_calls = find_calls = 0; + gfx_check_result = gfx_check_calls = gfx_set_calls = 0; lockable = present_device = link_ok = 1; status_word = PMP_REPORT_READY; } @@ -498,6 +514,9 @@ int main(void) balanced(); case_count++; RUN((void)0, 0, 0, 1); + assert(gfx_check_calls == 1 && gfx_set_calls == 0); + RUN(gfx_check_result = -EINVAL, -EINVAL, 0, 0); + assert(gfx_check_calls == 1 && gfx_set_calls == 0); RUN(report14.child = &entry7; entry7.sibling = &entry17; entry17.sibling = NULL, 0, 0, 1); RUN(apple_pmp_starts.first = NULL, -ENODEV, 0, 0); diff --git a/tools/asahi/t6030-pmp-supplier-tests.py b/tools/asahi/t6030-pmp-supplier-tests.py index d323dffbef728..269006b36aa4f 100755 --- a/tools/asahi/t6030-pmp-supplier-tests.py +++ b/tools/asahi/t6030-pmp-supplier-tests.py @@ -6,7 +6,9 @@ extracted. OF enumeration/property storage, device binding, MMIO polling, and locks are controlled host boundaries. The T6030 managed entry wait and start wait are extracted too; the firmware/workqueue boundary is controlled. -This models ownership/error handling, not real teardown concurrency or DMA. +The GFX-report service is modeled separately from supplier readiness; its +transaction has its own production-body PMP-vote selftest. This models +ownership/error handling, not real teardown concurrency or DMA. """ import argparse import hashlib @@ -139,6 +141,8 @@ def main(): ("gate-config", "IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) && gate_soc == &gate_t6030", "gate_soc == &gate_t6030"), ("gate-phandle-shape", 'of_property_count_u32_elems(gpu, "apple,pmp") != 1', "false"), ("serialized-command", "\tguard(mutex)(&pmp_lock);", "\t(void)&pmp_lock;"), + ("gfx-check-error", "ret = apple_pmp_report_check_t6030_gfx(pmp_device);", + "ret = (apple_pmp_report_check_t6030_gfx(pmp_device), 0);"), ] for name, old, new in mutations: assert code.count(old) == 1, (name, code.count(old)) diff --git a/tools/asahi/tbnet-refill-fixture.h b/tools/asahi/tbnet-refill-fixture.h index e88dfd62831db..d9dee1baab9b2 100644 --- a/tools/asahi/tbnet-refill-fixture.h +++ b/tools/asahi/tbnet-refill-fixture.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #define TBNET_RING_SIZE 256 #define TBNET_FRAME_SIZE 4096 @@ -92,7 +93,17 @@ static bool in_softirq(void){return fake_softirq;} static int smp_processor_id(void){return 0;} static void schedule_work(struct work_struct *w){} static bool queue_delayed_work(int wq,struct delayed_work *w,unsigned long delay){assert(delay==100);pthread_mutex_lock(&gate);bool new=!w->pending;if(new){w->pending=true;retry_queues++;retry_delay=delay;}pthread_mutex_unlock(&gate);return new;} -static void cancel_delayed_work_sync(struct delayed_work *w){pthread_mutex_lock(&gate);cancel_waiting=true;while(w->running)pthread_cond_wait(&cv,&gate);w->pending=false;cancel_waiting=false;pthread_mutex_unlock(&gate);} +static void cancel_delayed_work_sync(struct delayed_work *w) +{ + pthread_mutex_lock(&gate); + cancel_waiting = true; + pthread_cond_broadcast(&cv); + while (w->running) + pthread_cond_wait(&cv, &gate); + w->pending = false; + cancel_waiting = false; + pthread_mutex_unlock(&gate); +} static void fire_retry(struct delayed_work *w){pthread_mutex_lock(&gate);assert(w->pending&&!w->running);w->pending=false;w->running=true;pthread_mutex_unlock(&gate);w->work.fn(&w->work);pthread_mutex_lock(&gate);w->running=false;pthread_cond_broadcast(&cv);pthread_mutex_unlock(&gate);} static void nhi_ring_interrupt_mask(struct tb_ring *r,bool mask){r->masked=mask;} diff --git a/tools/asahi/tbnet-refill-tests.py b/tools/asahi/tbnet-refill-tests.py index f5a6e98e64175..641f8758bc927 100644 --- a/tools/asahi/tbnet-refill-tests.py +++ b/tools/asahi/tbnet-refill-tests.py @@ -56,6 +56,18 @@ def extract(name,text=source): static void *setup_thread(void *p){pthread_mutex_lock(&gate);setup_active=true;pthread_mutex_unlock(&gate);connect(p);pthread_mutex_lock(&gate);setup_active=false;pthread_cond_broadcast(&cv);pthread_mutex_unlock(&gate);return NULL;} static void *tear_thread(void *p){tbnet_tear_down(p,false);return NULL;} static void wait_paused(void){pthread_mutex_lock(&gate);while(!paused)pthread_cond_wait(&cv,&gate);pthread_mutex_unlock(&gate);} +static void wait_cancel(void) +{ + struct timespec deadline; + + assert(!clock_gettime(CLOCK_REALTIME, &deadline)); + deadline.tv_sec += 2; + pthread_mutex_lock(&gate); + while (!cancel_waiting) + assert(!pthread_cond_timedwait(&cv, &gate, &deadline)); + assert(ring_stops == 0 && pool_destroy_calls == 0); + pthread_mutex_unlock(&gate); +} static void release_poll(void){pthread_mutex_lock(&gate);resume_poll=1;pthread_cond_broadcast(&cv);pthread_mutex_unlock(&gate);} int main(int argc,char **argv){assert(argc==2);char *c=argv[1];struct tbnet n;struct net_device d;struct tb tb={0};struct tb_xdomain x={&tb};struct tb_service s={0};struct device dev={&s}; if(!strcmp(c,"pool-create-failure")){pool_error=1;memset(&n,0,sizeof(n));memset(&d,0,sizeof(d));d.priv=&n;n.dev=&d;n.xd=&x;n.svc=&s;pthread_mutex_init(&n.connection_lock,NULL);assert(tbnet_open(&d)==-ENOMEM);assert(!n.rx_ring.pool&&!n.rx_ring.ring&&!n.tx_ring.ring&&ring_frees==2&&enable_calls==0);return 0;} @@ -80,7 +92,7 @@ def extract(name,text=source): if(!strcmp(c,"retry-stop")||!strcmp(c,"retry-disconnect")||!strcmp(c,"retry-suspend")||!strcmp(c,"retry-shutdown")){exhaust(&n);int a=alloc_calls;if(!strcmp(c,"retry-stop"))tbnet_stop(&d);else if(!strcmp(c,"retry-suspend"))tbnet_suspend(&dev);else if(!strcmp(c,"retry-shutdown"))tbnet_shutdown(&s);else tbnet_tear_down(&n,false);assert(!n.rx_refill_work.pending&&!n.napi.enabled&&alloc_calls==a);assert(!n.rx_ring.ring||!n.rx_ring.ring->running);return 0;} if(!strcmp(c,"retry-reconnect")){exhaust(&n);tbnet_tear_down(&n,false);assert(!n.rx_refill_work.pending);alloc_limit=-1;n.login_sent=n.login_received=true;n.remote_transmit_path=12;connect(&n);assert(n.napi.enabled&&n.rx_ring.prod==256&&!n.rx_refill_work.pending);tbnet_stop(&d);return 0;} if(!strcmp(c,"retry-completion-tail")){alloc_limit=0;complete_frames(&n,1,2);pause_at=2;pthread_t a,b;assert(!pthread_create(&a,NULL,poll_thread,&n));wait_paused();assert(n.rx_refill_work.pending);assert(!pthread_create(&b,NULL,stop_thread,&n));usleep(30000);assert(ring_stops==0&&n.rx_refill_work.pending);release_poll();assert(!pthread_join(a,NULL)&&!pthread_join(b,NULL));assert(!n.rx_refill_work.pending&&ring_stops==2);return 0;} - if(!strcmp(c,"retry-running-teardown")){exhaust(&n);pause_at=6;pthread_t a,b;assert(!pthread_create(&a,NULL,retry_thread,&n));wait_paused();assert(!pthread_create(&b,NULL,stop_thread,&n));usleep(30000);assert(ring_stops==0&&pool_destroy_calls==0&&cancel_waiting);release_poll();assert(!pthread_join(a,NULL)&&!pthread_join(b,NULL));assert(!n.rx_refill_work.pending&&!n.rx_refill_work.running&&!n.napi.owner&&ring_stops==2&&pool_destroy_calls==1);return 0;} + if(!strcmp(c,"retry-running-teardown")){exhaust(&n);pause_at=6;pthread_t a,b;assert(!pthread_create(&a,NULL,retry_thread,&n));wait_paused();assert(!pthread_create(&b,NULL,stop_thread,&n));wait_cancel();release_poll();assert(!pthread_join(a,NULL)&&!pthread_join(b,NULL));assert(!n.rx_refill_work.pending&&!n.rx_refill_work.running&&!n.napi.owner&&ring_stops==2&&pool_destroy_calls==1);return 0;} if(!strcmp(c,"retry-already-polling")){exhaust(&n);tbnet_start_poll(&n);fire_retry(&n.rx_refill_work);int rearm=rearm_calls;assert(n.napi.missed&&tbnet_poll(&n.napi,64)==0&&n.napi.owner&&n.rx_refill_work.pending&&rearm_calls==rearm);alloc_limit=-1;assert(tbnet_poll(&n.napi,64)==0&&!n.napi.owner&&n.rx_ring.prod-n.rx_ring.cons==256);return 0;} if(!strcmp(c,"retry-disabled-worker")){exhaust(&n);napi_disable(&n.napi);int a=alloc_calls,sch=schedule_calls;fire_retry(&n.rx_refill_work);assert(!n.napi.owner&&schedule_calls==sch&&alloc_calls==a);return 0;} if(!strcmp(c,"retry-counter-wrap")){n.rx_ring.cons=0xffffff00;n.rx_ring.prod=0;exhaust(&n);alloc_limit=-1;retry_poll(&n);assert(n.rx_ring.cons==0&&n.rx_ring.prod==256&&!n.rx_refill_work.pending);return 0;} diff --git a/tools/testing/selftests/drivers/ane/Makefile b/tools/testing/selftests/drivers/ane/Makefile index 48712ae211d84..f1184d8987e8b 100644 --- a/tools/testing/selftests/drivers/ane/Makefile +++ b/tools/testing/selftests/drivers/ane/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 TEST_PROGS := h16-fixture.py -TEST_PROGS_EXTENDED := recovery.py make-hang-fixture.py +TEST_PROGS_EXTENDED := recovery.py make-hang-fixture.py h16-run.py h16-sections.py \ + ane-run.py h16-mobilenet.py include ../../lib.mk diff --git a/tools/testing/selftests/drivers/ane/ane-run.py b/tools/testing/selftests/drivers/ane/ane-run.py new file mode 100644 index 0000000000000..5a2369cc3d89b --- /dev/null +++ b/tools/testing/selftests/drivers/ane/ane-run.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Load one H16G or H17 HWX through the ABI 3 accel node and run cases. + +The program is loaded once. Each --case names the input files, one per +input surface in the order H16_LOAD reports, and optionally the expected +output. Before every run the output buffer is poisoned. With --expect the +output must match within --max-ulp on every element the expected file +covers, except elements it sets to 0xffff (padding lanes). With --out-dir each case's last output is saved as output_NN.bin. + + ane-run.py --hwx program.hwx --case in.bin:expected.bin --runs 20 + ane-run.py --hwx program.hwx --case a.bin,b.bin --out-dir out + +Requires a kernel whose ANE reached the program stage; skips otherwise. +""" + +import argparse +import ctypes +import fcntl +import glob +import importlib.util +import os +import pathlib +import statistics +import struct +import sys +import time + +KSFT_PASS, KSFT_FAIL, KSFT_SKIP = 0, 1, 4 + +DRM_COMMAND_BASE = 0x40 +DRM_ANE_ABI_V3 = 3 +CHIPS = (16, 17) +SECTIONS = ( + (1, "generic.bin"), + (2, "operation.bin"), + (3, "procedure.bin"), + (4, "text-properties.bin"), +) +GET_CAPS = struct.Struct("<8I") +SECTION = struct.Struct(" args.max_ulp: + result = KSFT_FAIL + if result != KSFT_PASS: + break + if args.out_dir: + (args.out_dir / f"output_{n:02d}.bin").write_bytes(buffers[output].raw) + if times: + times.sort() + p = lambda q: times[min(len(times) - 1, int(q * len(times)))] / 1e3 + print(f"host latency over {len(times)} runs: p50 {p(0.5):.1f} us, " + f"p90 {p(0.9):.1f} us, p99 {p(0.99):.1f} us, " + f"mean {statistics.mean(times) / 1e3:.1f} us") + os.close(fd) + print("PASS" if result == KSFT_PASS else "FAIL") + return result + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/testing/selftests/drivers/ane/h16-mobilenet.py b/tools/testing/selftests/drivers/ane/h16-mobilenet.py new file mode 100755 index 0000000000000..89ed82b01a9c5 --- /dev/null +++ b/tools/testing/selftests/drivers/ane/h16-mobilenet.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Run a linked H16G MobileNetV1 224 program through the ABI 3 accel node. + +The package directory holds linked/program-0.hwx (one program, 28 tasks, +DRAM scratch on BAR slot 3), refs/input_NN.bin (fp16 input surfaces), +refs/output_NN.bin (the CPU fp16-chain output surfaces) and refs/top1.json. +The program is loaded once; each image is one EXECUTE with a poisoned +output. The output surface is the 1001-class logit map as 1024 channels of +7x7 fp16 (row 64 bytes, plane 448 bytes); top-1 is the channel with the +largest 7x7 mean. PASS needs every image's top-1 to match top1.json. +The device output is not expected to equal the CPU chain bit for bit; the +exact-value count and largest difference are reported for information. +""" + +import argparse +import ctypes +import fcntl +import importlib.util +import json +import os +import pathlib +import statistics +import struct +import sys +import time + +KSFT_PASS, KSFT_FAIL, KSFT_SKIP = 0, 1, 4 +CLASSES, ROW, PLANE = 1001, 64, 448 +POISON = 0xff + + +def load_module(name, file): + path = pathlib.Path(__file__).resolve().parent / file + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def top1(surface): + best, best_mean = 0, None + for c in range(CLASSES): + total = 0.0 + for h in range(7): + total += sum(struct.unpack_from("<7e", surface, c * PLANE + h * ROW)) + if best_mean is None or total > best_mean: + best, best_mean = c, total + return best + + +def compare(got, want): + a = struct.unpack(f"<{len(got) // 2}e", got) + b = struct.unpack(f"<{len(want) // 2}e", want) + exact, worst = 0, 0.0 + for c in range(CLASSES): + for h in range(7): + for w in range(7): + k = (c * PLANE + h * ROW) // 2 + w + exact += a[k] == b[k] + worst = max(worst, abs(a[k] - b[k])) + return exact, CLASSES * 49, worst + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--package", type=pathlib.Path, required=True) + parser.add_argument("--device", help="accel node; default: the first ABI 3 H16 node") + parser.add_argument("--images", type=int, default=16) + parser.add_argument("--timing-runs", type=int, default=20, + help="extra EXECUTEs of image 0 for host latency") + args = parser.parse_args() + + run = load_module("h16_run", "h16-run.py") + hwx = (args.package / "linked/program-0.hwx").read_bytes() + refs = args.package / "refs" + want_top1 = json.loads((refs / "top1.json").read_text())["top1"][:args.images] + try: + built, ids, bar_refs, tasks = load_module("h16_sections", "h16-sections.py").build(hwx) + except ValueError as e: + print(f"FAIL: sections: {e}") + return KSFT_FAIL + print("sections built: buffer IDs", ids, "BAR refs", bar_refs, "tasks", tasks) + + errors = [] + path, fd = run.find_device(args.device, errors) + if fd is None and errors: + for error in errors: + print(f"FAIL: {error}") + return KSFT_FAIL + if fd is None: + print("SKIP: no ABI 3 H16 accel node (program stage not reached)") + return KSFT_SKIP + print(f"device {path}") + + keep = [] + hwx_buf = ctypes.create_string_buffer(hwx, len(hwx)) + table = bytearray() + for (kind, _), blob in zip(run.SECTIONS, built): + buf = ctypes.create_string_buffer(blob, len(blob)) + keep.append(buf) + table += run.SECTION.pack(kind, len(blob), ctypes.addressof(buf)) + table_buf = ctypes.create_string_buffer(bytes(table), len(table)) + load = bytearray(run.LOAD.pack(ctypes.addressof(hwx_buf), len(hwx), + ctypes.addressof(table_buf), len(built), 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0)) + try: + fcntl.ioctl(fd, run.IOCTL_H16_LOAD, load) + except OSError as e: + print(f"FAIL: H16_LOAD: {os.strerror(e.errno)} (errno {e.errno})") + return KSFT_FAIL + fields = run.LOAD.unpack(load) + count = fields[5] + sizes = list(fields[7:11][:count]) + directions = list(fields[11:15][:count]) + print(f"loaded: {count} surfaces, sizes {sizes}, directions {directions}") + if count != 2 or directions != [1, 2]: + print("FAIL: MobileNet must have one input and one output surface") + return KSFT_FAIL + in_size, out_size = sizes + buffers = [ctypes.create_string_buffer(in_size), ctypes.create_string_buffer(out_size)] + exec_table = b"".join(run.BUFFER.pack(ctypes.addressof(b), len(b)) for b in buffers) + exec_buf = ctypes.create_string_buffer(exec_table, len(exec_table)) + + def execute(image): + data = (refs / f"input_{image:02d}.bin").read_bytes() + if len(data) != in_size: + raise ValueError(f"input_{image:02d}.bin is {len(data)} bytes, surface {in_size}") + ctypes.memmove(buffers[0], data, in_size) + ctypes.memset(buffers[1], POISON, out_size) + start = time.perf_counter() + fcntl.ioctl(fd, run.IOCTL_H16_EXEC, run.EXEC.pack(ctypes.addressof(exec_buf), count, 0)) + return time.perf_counter() - start + + result, matched = KSFT_PASS, 0 + for image in range(args.images): + try: + seconds = execute(image) + except (OSError, ValueError) as e: + print(f"img{image}: FAIL: {e}") + result = KSFT_FAIL + break + got = buffers[1].raw + if got == bytes([POISON]) * out_size: + print(f"img{image}: FAIL: output untouched") + result = KSFT_FAIL + break + t = top1(got) + exact, total, worst = compare(got, (refs / f"output_{image:02d}.bin").read_bytes()) + ok = t == want_top1[image] + matched += ok + print(f"img{image}: ANE top1={t} ref={want_top1[image]} {'OK' if ok else 'DIFF'}; " + f"{exact}/{total} logits equal the CPU chain, max |diff| {worst:.4g}; " + f"{seconds * 1e3:.3f} ms") + if result == KSFT_PASS: + times = [] + for _ in range(args.timing_runs): + try: + times.append(execute(0)) + except OSError as e: + print(f"timing: FAIL: {e}") + result = KSFT_FAIL + break + if times: + times.sort() + print(f"host EXECUTE latency over {len(times)} runs: p50 " + f"{statistics.median(times) * 1e3:.3f} ms, " + f"p90 {times[int(len(times) * 0.9) - 1] * 1e3:.3f} ms") + if matched != args.images: + result = KSFT_FAIL + os.close(fd) + print(f"top-1 {matched}/{args.images}") + print("PASS" if result == KSFT_PASS else "FAIL") + return result + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/testing/selftests/drivers/ane/h16-run.py b/tools/testing/selftests/drivers/ane/h16-run.py new file mode 100755 index 0000000000000..bf5043bc61ee3 --- /dev/null +++ b/tools/testing/selftests/drivers/ane/h16-run.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Load the pinned H16G add-128 HWX through the ABI 3 accel node and run it. + +The program is loaded once and executed twice. Before each run the output +buffer is poisoned, and after each run it must equal expected.bin exactly. +Requires a kernel booted with the H16 program stage; skips otherwise. +""" + +import argparse +import ctypes +import fcntl +import glob +import hashlib +import importlib.util +import os +import pathlib +import struct +import sys + +KSFT_PASS, KSFT_FAIL, KSFT_SKIP = 0, 1, 4 + +DRM_COMMAND_BASE = 0x40 +DRM_ANE_ABI_V3 = 3 +DRM_ANE_CHIP_H16 = 16 +SECTIONS = ( + (1, "generic.bin"), + (2, "operation.bin"), + (3, "procedure.bin"), + (4, "text-properties.bin"), +) +GET_CAPS = struct.Struct("<8I") +SECTION = struct.Struct(" len(hwx): + raise HwxError("truncated load command") + if cmd == 0x19: + segment = cstr(hwx[pos + 8:pos + 24]) + vm, vmsize = u64(hwx, pos + 24), u64(hwx, pos + 32) + prot = u32(hwx, pos + 60) + for i in range(u32(hwx, pos + 64)): + at = pos + 72 + 80 * i + name = cstr(hwx[at:at + 16]) + svm, ssize, soff = u64(hwx, at + 32), u64(hwx, at + 40), u32(hwx, at + 48) + if segment == "__FVMLIB": + if prot not in (1, 2): + raise HwxError("surface segment is neither input nor output") + surfaces.append({"vm": svm, "allocation": vmsize, "direction": prot}) + elif segment == "__TEXT" and name == "__text": + text = (svm, ssize, soff) + elif segment == "__KERN_0" and name == "__kern_0": + kernel = (svm, ssize, soff) + elif segment == "__DATA" and name == "__bss": + if prot != 3: + raise HwxError("scratch segment is not read/write") + scratch = (svm, ssize) + elif cmd == 4 and u32(hwx, pos + 8) == 4: + descriptor = pos + elif cmd == 4 and u32(hwx, pos + 8) == 3: + shortoff = u32(hwx, pos + 0x7c) + n, batch = u32(hwx, pos + 0x28), u64(hwx, pos + 0x50) + tensors[cstr(hwx[pos + shortoff:pos + size])] = (u32(hwx, pos + 0x14), n * batch) + elif cmd == 0x40: + vm, name = u64(hwx, pos + 16), cstr(hwx[pos + 24:pos + size]) + for s in surfaces: + if s["vm"] == vm: + s["name"] = name + pos += size + if text is None or descriptor is None or len(surfaces) < 2: + raise HwxError("HWX lacks a task stream, program descriptor or surfaces") + for s in surfaces: + direction, span = tensors.get(s.get("name"), (None, None)) + if direction != s["direction"] or not span or span > s["allocation"]: + raise HwxError("surface %s has no matching tensor" % s.get("name")) + s["span"] = span + if sum(s["direction"] == 2 for s in surfaces) != 1: + raise HwxError("an H16 program has exactly one output") + return surfaces, text, kernel, scratch, descriptor + + +def register_valid(word): + return any(start <= word * 4 < end for start, end in REGISTERS) + + +def walk(stream, prologue): + """Count the tasks and collect the BAR slots their records reference.""" + if len(stream) < 16 or len(stream) % 4 or u32(stream, 0) != prologue or any(stream[4:16]): + raise HwxError("task stream lacks the prologue of its target") + pos, tasks, slots = 16, 0, set() + while pos < len(stream): + words = (u32(stream, pos) >> 16) & 0x7ff + if not words: + if any(stream[pos:pos + 16]): + raise HwxError("nonzero padding in the task stream") + pos += 16 + continue + end = pos + 4 * words + header = 10 if (u32(stream, pos + 32) & 3) == 3 else 9 + if words < header or end > len(stream): + raise HwxError("task overruns the stream") + rec = pos + 4 * header + while rec < end: + h = u32(stream, rec) + rec += 4 + if h & (1 << 31): + count = 1 + bin((h >> 15) & 0xffff).count("1") + else: + count = ((h >> 15) & 0x3f) + 1 + if h & (1 << 29): + slots.add((h >> 23) & 0x3f) + if not register_valid(h & 0x7fff) or rec + 4 * count > end: + raise HwxError("task record outside the register image") + rec += 4 * count + pos = (end + 15) & ~15 + tasks += 1 + if not tasks: + raise HwxError("task stream has no task") + return tasks, slots + + +def build(hwx): + """Return the four sections, in H16_LOAD kind order, and the buffer IDs.""" + surfaces, (text_vm, text_size, text_off), kernel, scratch, descriptor = parse(hwx) + stream = hwx[text_off:text_off + text_size] + tasks, slots = walk(stream, PROLOGUE[u32(hwx, 8)]) + + inputs = iter(range(FIRST_OUTPUT_ID + 1, 0x3d)) + for s in surfaces: + s["id"] = FIRST_OUTPUT_ID if s["direction"] == 2 else next(inputs) + entries = [(s["id"], s["direction"], s["allocation"]) for s in surfaces] + scratch_id = None + if scratch: + scratch_id = next(inputs) + entries.append((scratch_id, 1, scratch[1])) + + generic = bytearray(GENERIC_HEADER + GENERIC_ENTRY * len(entries)) + struct.pack_into(" OPERATION_REFS_MAX: + raise HwxError("program binds no buffer through a BAR slot") + operation = bytearray(4 + OPERATION_RECORD) + struct.pack_into("{t}" for s, t in refs), f"tasks {tasks}") + print("sizes", ", ".join(f"{n} {len(d):#x}" for n, d in zip(SECTION_NAMES, sections))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py b/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py index b7f4f7dd633dc..8a23ffbeb83ff 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py @@ -206,6 +206,14 @@ def brightness_takeover(root): 'static void dcp_backlight_sync_hint(', 'int dcp_backlight_register(', 'static int dcp_set_brightness(', 'static bool dcp_backlight_controls_device(')) + # The 14.7 panel's nits policy and class operations (an M3 internal + # panel) never run on the desktop panel. + if 'static bool dcp_backlight_presents_nits(' in source: + functions = ('static bool iomfb_v14_7_panel_nits(struct apple_dcp *d) ' + '{(void)d;return false;}\n' + 'static const int dcp_backlight_ops_v14;\n' + + block(source, 'static bool dcp_backlight_presents_nits(') + + '\n' + functions) return prefix + r''' #include #include diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 89dd1f31f954b..95043ab9afc36 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -483,6 +483,25 @@ def function(file, name): prefix += ('#define DCP_FIRMWARE_V_26_6_DESKTOP 100\n' 'static int __attribute__((unused)) dcp_desktop_dptx_connect(struct apple_dcp *d, u32 port)\n' '{ (void)d; (void)port; assert(!"desktop 26.6 connect in an H17P scenario"); return -EINVAL; }\n') + # Let the real helper classify an occupied display on newer trees, where + # -EBUSY means a transient mux/PHY failure rather than a taken pipeline. + if re.search(r'cd321x_display_taken\(int ret\)', pm): + assert shim.count('mode_error == -EBUSY') == 1 + assert tests.count('mode_error = -EBUSY;') == 2 + shim = shim.replace('mode_error == -EBUSY', 'cd321x_display_taken(mode_error)') + tests = tests.replace('mode_error = -EBUSY;', 'mode_error = -ENOSPC;') + busy = ' CHECK(sink.connected && !notices && !port.display_route_active);\n' + assert tests.count(busy) == 1 + tests = tests.replace(busy, busy + r''' + CHECK(roles == 1 && port.pm.phase == CD321X_PM_RUNNING); + /* A transient busy mux must retry before switching the USB role. */ + init(); port.display_route_active = false; role = USB_ROLE_NONE; + port.update_status.status = TPS_STATUS_PLUG_PRESENT; + port.update_status.data_status = TPS_DATA_STATUS_DP_CONNECTION | CD321X_DATA_STATUS_HPD_LEVEL; + mode_error = -EBUSY; cd321x_update_work(&port.update_work.work); + CHECK(!roles && port.pm.phase == CD321X_PM_REVALIDATE); + CHECK(sink.connected && !notices && !port.display_route_active); +''') # How long a held mux has been retried is measured on the driver's clock. if 'jiffies_to_msecs(jiffies)' in funcs: shim += ('\nstatic unsigned long jiffies;\n' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py b/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py index cb3bac542a88c..f11078c87b087 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/m4.py @@ -482,6 +482,12 @@ def blank(root): functions = '\n'.join(block(backlight, name) for name in ( 'bool dcp_backlight_active(', 'int dcp_backlight_dpms(', 'int dcp_backlight_prepare(', 'bool dcp_backlight_complete(')) + # The 14.7 panel's nits policy (an M3 internal panel) never runs here. + if 'static bool dcp_backlight_presents_nits(' in backlight: + functions = ('static bool iomfb_v14_7_panel_nits(struct apple_dcp *dcp) ' + '{ (void)dcp; return false; }\n' + + block(backlight, 'static bool dcp_backlight_presents_nits(') + + '\n' + functions) return production(root) + r''' #define min(x,y) ((x)<(y)?(x):(y)) static u64 div_u64(u64 x,u32 y) { return x/y; } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index faae722b9d8e1..37537310d333e 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -642,6 +642,19 @@ def function(text, name): } ''' + # Newer trees report an occupied pipeline as -ENOSPC on every SoC. + # Keep the old two-argument contract covered when testing an older tree. + if re.search(r'cd321x_display_taken\(int ret\)', pm): + old = ' assert(cd321x_display_taken(-ENOSPC,true)&&!cd321x_display_taken(-EBUSY,true)&&!cd321x_display_taken(0,true)&&cd321x_display_taken(-EBUSY,false)&&!cd321x_display_taken(-ENOSPC,false)&&!cd321x_display_taken(0,false));' + assert controls.count(old) == 1 + controls = controls.replace(old, r''' + /* An occupied display permits USB progress; transient/other errors retry. */ + assert(cd321x_display_taken(-ENOSPC)); + assert(!cd321x_display_taken(-EBUSY)&&!cd321x_display_taken(-EAGAIN)); + assert(!cd321x_display_taken(-ETIMEDOUT)&&!cd321x_display_taken(-EIO)); + assert(!cd321x_display_taken(0)); +''') + # A mux that holds a change up is retried outside the cable-state budget. if 'cd321x_pm_mux_held(' in pm: lost = ' puts("PASS actual lost events reopen a stale port; unacknowledged events keep their budget with capped backoff");\n' diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py index 52d7125f0c295..78e241892ca14 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -1105,6 +1105,65 @@ def publication(root): unit=re.sub(r'\blookup_mode\((&d,&[\w.\[\]]+)\)', r'lookup_mode(\1,NULL)', unit) return {**locals(), "block": block} + +def panel_dimensions(root): + n = publication(root) + unit, block = n['unit'], n['block'] + unit = unit.replace('bool modes_lock,modes_admitted,', + 'bool external_physically_absent,modes_lock,modes_admitted,', 1) + # The connector's physical size, which DRM keeps in display_info. + old = 'struct { struct { unsigned min_vfreq,max_vfreq; } monitor_range; } display_info;' + assert unit.count(old) == 1 + unit = unit.replace(old, 'struct { struct { unsigned min_vfreq,max_vfreq; } monitor_range; ' + 'int width_mm,height_mm; } display_info;') + dcp = (root / 'drivers/gpu/drm/apple/dcp.c').read_text() + unit = unit.replace(block(unit, 'static bool __attribute__((unused)) dcp_has_panel('), + block(dcp, 'bool dcp_has_panel(')) + unit = unit.replace(block(unit, 'static void dcp_update_connector_dimensions('), + block(dcp, 'void dcp_update_connector_dimensions(')) + # Ordinary panel modes do not take the 180 Hz exception's metadata lock. + unit = unit.replace('probing->dcpavserv.lock && probing->metadata.lock', + 'probing->dcpavserv.lock && ' + '(!probing->modes[0].automatic_rgb8_quirk || probing->metadata.lock)') + # Model DRM's display_info reset and the absence of an internal AV EDID. + unit = unit.replace(block(unit, 'static int drm_edid_connector_update('), r""" +static int drm_edid_connector_update(struct drm_connector *c,const struct drm_edid *e) { + assert(c->dev->mode_config.mutex); + c->display_info.width_mm=e ? 600 : 0; + c->display_info.height_mm=e ? 330 : 0; + return fail_update ? -EIO : 0; +} +""") + unit = unit.replace('if(!d->dcpavserv.enabled || !d->dcpavserv.service', + 'if(dcp_has_panel(d)) return ERR_PTR(-ENODEV);\n' + ' if(!d->dcpavserv.enabled || !d->dcpavserv.service') + checks = r""" +static void panel_dimension_probes(void) { + reset();callbacks(false);callbacks(true);timings();admit(0);probe(true); + assert(con.base.display_info.width_mm==600 && con.base.display_info.height_mm==330); + d.panel.width_mm=302;d.panel.height_mm=196;d.width_mm=0;d.height_mm=0; + d.modes[0].automatic_rgb8_quirk=false; + con.edid_invalidation_pending=true;probe(true); + assert(!con.drm_edid); + assert(con.base.display_info.width_mm==302 && con.base.display_info.height_mm==196); + con.edid_invalidation_pending=true;probe(true); + assert(con.base.display_info.width_mm==302 && con.base.display_info.height_mm==196); + d.width_mm=300;d.height_mm=190;con.edid_invalidation_pending=true;probe(true); + assert(con.base.display_info.width_mm==300 && con.base.display_info.height_mm==190); + d.width_mm=0;d.height_mm=190;con.edid_invalidation_pending=true;probe(true); + assert(con.base.display_info.width_mm==302 && con.base.display_info.height_mm==196); + cleanup(); + reset();callbacks(false);callbacks(true);timings();admit(0);probe(true); + assert(con.base.display_info.width_mm==600 && con.base.display_info.height_mm==330); + cleanup(); + puts("PASS: internal panel dimensions survive repeated EDID clears; firmware size and DT fallback; external EDID size preserved"); +} +""" + unit = unit.replace('int main(int argc,char **argv) {', + checks + 'int main(int argc,char **argv) {') + return unit.replace(' transport_edid=raw;', + ' transport_edid=raw;panel_dimension_probes();', 1) + def hotplug(root): n = publication(root) unit = n['unit'] diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index 3235ca5145866..07c4e9cd6139c 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -91,6 +91,7 @@ def main(): ('edid', 'ordered-rx', edid.rx_progress, ((),), ('-pthread',)), ('rate', 'parser-policy', lambda r: rate.policy(r)['unit'], ((),), ()), ('rate', 'publication-hpd', rate.hotplug, ((),), ()), + ('rate', 'panel-dimensions', rate.panel_dimensions, ((),), ()), ('rate', 'metadata-decoder', lambda r: rate.metadata(r, output), ((),), ('-Wno-sign-compare',)), ('rate', 'bounded-retry', rate.retry, ((),), ()), ('phy', 'pm-ownership', pm.ownership, ((),), ()), @@ -143,7 +144,7 @@ def main(): elif isinstance(unit, tuple): unit, cases = unit if unit is not None and name in ('provider-admission', 'ordered-rx', - 'publication-hpd', 'stale-crtc-poweroff', 'readiness'): + 'publication-hpd', 'panel-dimensions', 'stale-crtc-poweroff', 'readiness'): # These fixtures select older external-display backends. The M4 # transport is covered by the separate m4 group, not mocked here. prefix = ('#include \n#include \n' diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c new file mode 100644 index 0000000000000..806148f64d1bf --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Exercise the production PMP vote transaction with failed firmware replies. */ +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +#define CONFIG_APPLE_T6030_GPU_PMP_LINK 1 +#define IS_ENABLED(x) (x) +#define guard(kind) ignore_lock +#define dev_warn(...) ((void)0) +static int pmp_lock; +static void ignore_lock(void *p) { (void)p; } +struct device { void *of_node; }; +static struct device owner; +static struct device *pmp_device = &owner; +static const void *pmp_data = &owner; +static bool t6030, requested; +static int ready, check, power_result[2], report_result[3], powers, reports; +static char calls[16]; +static unsigned int used; +static void called(char c) { assert(used < sizeof(calls) - 1); calls[used++] = c; } +static bool of_machine_is_compatible(const char *s) { (void)s; return t6030; } +static bool of_device_is_compatible(void *p, const char *s) +{ + (void)p; (void)s; return t6030; +} +static int apple_pmp_report_wait_supplier_ready(struct device *d, unsigned int ms) +{ + assert(d == &owner && ms == 5000); called('W'); return ready; +} +static int apple_pmp_send_power_command(const void *p, u8 cmd, u16 id, u32 enabled) +{ + assert(p == &owner && cmd == 0x0f && id == 5); + called(enabled ? 'P' : 'p'); assert(powers < 2); return power_result[powers++]; +} +static int apple_pmp_report_check_t6030_gfx(struct device *d) +{ + assert(d == &owner); called('V'); return check; +} +static int apple_pmp_report_set_t6030_gfx(struct device *d, bool enabled) +{ + assert(d == &owner); called(enabled ? 'R' : 'r'); requested = enabled; + assert(reports < 3); return report_result[reports++]; +} +#include "pmp-vote-production.h" +static void reset(void) +{ + pmp_data = &owner; t6030 = true; requested = false; ready = check = 0; powers = reports = used = 0; + memset(calls, 0, sizeof(calls)); + memset(power_result, 0, sizeof(power_result)); + memset(report_result, 0, sizeof(report_result)); +} +static void result(int expected, const char *sequence, bool enabled) +{ + assert(apple_pmp_set_device_power(0x0f, 5, enabled) == expected); + assert(strcmp(calls, sequence) == 0); +} +int main(void) +{ + reset(); + assert(apple_pmp_set_device_power(0, 5, 1) == -EINVAL); + assert(apple_pmp_set_device_power(0x0f, 16, 1) == -EINVAL); + assert(apple_pmp_set_device_power(0x0f, 5, 2) == -EINVAL); + assert(!used); + pmp_data = NULL; result(-ENODEV, "", true); + reset(); ready = -EAGAIN; result(-EAGAIN, "W", true); + reset(); t6030 = false; result(0, "WP", true); + reset(); t6030 = false; result(0, "Wp", false); + reset(); check = -EINVAL; result(-EINVAL, "WV", true); + reset(); result(0, "WVP", true); + /* Power-on must not start bandwidth tracking before firmware READY. */ + assert(!reports); + assert(apple_pmp_t6030_gpu_ready() == 0); + assert(strcmp(calls, "WVPrR") == 0 && requested); + reset(); requested = true; + assert(apple_pmp_t6030_gpu_ready() == 0); + assert(strcmp(calls, "rR") == 0 && requested); + reset(); pmp_data = NULL; + assert(apple_pmp_t6030_gpu_ready() == -ENODEV && !used); + reset(); report_result[0] = -ETIMEDOUT; + assert(apple_pmp_t6030_gpu_ready() == -ETIMEDOUT); + assert(strcmp(calls, "r") == 0); + reset(); report_result[1] = -ETIMEDOUT; + assert(apple_pmp_t6030_gpu_ready() == -ETIMEDOUT); + assert(strcmp(calls, "rR") == 0); + reset(); result(0, "WVrp", false); + reset(); power_result[0] = -ETIMEDOUT; result(-ETIMEDOUT, "WVP", true); + reset(); report_result[0] = -ETIMEDOUT; result(-ETIMEDOUT, "WVrR", false); + reset(); report_result[0] = -ETIMEDOUT; report_result[1] = -EIO; + result(-ETIMEDOUT, "WVrR", false); + reset(); power_result[0] = -EIO; result(-EIO, "WVrpR", false); + puts("production PMP vote: scope, sequencing, reply errors and rollback: PASS"); +} diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index 35c96a7f36176..2ae5be4b57e56 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -102,6 +102,7 @@ pub type Result = std::result::Result; pub const EINVAL: Error = -22; pub const EFAULT: Error = -14; + pub const ENOSPC: Error = -28; pub use crate::{pr_warn, static_assert}; } #[macro_export] @@ -113,6 +114,7 @@ pub const UAT_PGSZ: usize = 0x4000; pub const UAT_PGMSK: usize = UAT_PGSZ - 1; pub struct Vm(pub Vec); + pub type VmReadGuard = Vm; impl Vm { pub fn read_user(&self, va: u64, out: &mut [u8]) -> Result { let start = usize::try_from(va).map_err(|_| EFAULT)?; @@ -124,8 +126,19 @@ } mod command { use crate::{mmu, prelude::*}; + use std::cell::Cell; pub const USC_WINDOW: u64 = 1 << 32; - pub fn cover(vm: &mmu::Vm, start: u64, end: u64, _: bool, _: bool) -> Result { + /// Addresses from here on are mapped and readable without backing in the mock VM. + pub const SYNTHETIC: u64 = 1 << 40; + thread_local! { + /// cover() calls: the page checks the walk made. + pub static COVERS: Cell = Cell::new(0); + } + pub fn cover(vm: &mut mmu::VmReadGuard, start: u64, end: u64, _: bool, _: bool) -> Result { + COVERS.with(|c| c.set(c.get() + 1)); + if start >= SYNTHETIC { + return Ok(()); + } let mut bytes = vec![0; usize::try_from(end - start).map_err(|_| EINVAL)?]; vm.read_user(start, &mut bytes) } @@ -144,28 +157,100 @@ } #[test] fn terminator_and_known_barrier() { - assert_eq!(gate::cdm(&vm(&[0x40000000]), 0, 0), Ok(())); - assert_eq!(gate::cdm(&vm(&[0x60000160, 0x40000000]), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&[0x40000000]), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&[0x60000160, 0x40000000]), 0, 0), Ok(())); } #[test] fn cyclic_links_are_rejected() { - assert_eq!(gate::cdm(&vm(&[0x20000000, 0]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&[0x20000000, 0]), 0, 0), Err(EINVAL)); } + /// Words of 4 bytes that fill the byte budget: the root block and one per link. + const BUDGET_WORDS: usize = 65 * 0x10000 / 4; #[test] - fn record_budget_requires_a_terminator() { - assert_eq!(gate::cdm(&vm(&vec![0x60000160; 8192]), 0, 0), Err(EINVAL)); + fn byte_budget_requires_a_terminator() { + assert_eq!(gate::cdm(&mut vm(&vec![0x60000160; BUDGET_WORDS + 1]), 0, 0), Err(EINVAL)); } #[test] fn final_record_may_terminate() { - let mut words = vec![0x60000160; 8192]; - words[8191] = 0x40000000; - assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + let mut words = vec![0x60000160; BUDGET_WORDS + 1]; + words[BUDGET_WORDS - 1] = 0x40000000; + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); + words[BUDGET_WORDS - 1] = 0x60000160; + words[BUDGET_WORDS] = 0x40000000; + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Err(EINVAL)); + } + #[test] + fn more_than_8192_records_are_admitted() { + // 1024 draws of ten records each, as a tessellated multi-draw writes them. + let mut words = vec![0x60000160; 10240]; + words.push(0x40000000); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); + } + #[test] + fn link_budget_still_bounds_the_walk() { + // 65 links, each to the next: the 65th is refused. + let mut words = Vec::new(); + for i in 0..65u32 { + words.extend([0x20000000, (i + 1) * 8]); + } + words.push(0x40000000); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Err(EINVAL)); + let mut words = Vec::new(); + for i in 0..64u32 { + words.extend([0x20000000, (i + 1) * 8]); + } + words.push(0x40000000); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); + } + /// The USC window of these cases and its first program, both in the mock's mapped range. + const USC: u64 = crate::command::SYNTHETIC; + /// A 24-byte indirect launch of the window's first program with its arguments at `args`. + fn indirect(args: u64) -> [u32; 6] { + [0x10080000, ((USC >> 38) as u32) << 22, (USC >> 6) as u32, + 0x40000000 | ((USC >> 40) & 0xff) as u32, (args >> 32) as u32 & 0xffff, args as u32] + } + /// `n` indirect launches, launch `i` with its arguments at `args(i)`, then TERMINATE. + fn launches(n: usize, args: impl Fn(usize) -> u64) -> Vec { + let mut words: Vec = (0..n).flat_map(|i| indirect(args(i))).collect(); + words.push(0x40000000); + words + } + fn covers(words: &[u32]) -> (Result, u32) { + crate::command::COVERS.with(|c| c.set(0)); + let r = gate::cdm(&mut vm(words), USC, 0); + (r, crate::command::COVERS.with(|c| c.get())) + } + const PAGE: u64 = mmu::UAT_PGSZ as u64; + #[test] + fn repeated_argument_pages_are_checked_once() { + // 1000 launches with their arguments in one page: the program page and that page. + let base = 2 * USC; + assert_eq!(covers(&launches(1000, |i| base + (i as u64 % 100) * 12)), (Ok(()), 2)); + // Alternating between two pages: one check per change of page. + assert_eq!(covers(&launches(10, |i| base + (i as u64 % 2) * PAGE)), (Ok(()), 11)); + // Arguments that cross a page are checked each time. + assert_eq!(covers(&launches(3, |_| base + PAGE - 8)), (Ok(()), 4)); + } + #[test] + fn page_check_budget_bounds_the_walk() { + let base = 2 * USC; + // The program page and 16383 argument pages: 16384 checks, admitted. + assert_eq!(covers(&launches(16383, |i| base + i as u64 * PAGE)), (Ok(()), 16384)); + // One more page: refused at the 16385th check, which is not made. + assert_eq!(covers(&launches(16384, |i| base + i as u64 * PAGE)), (Err(EINVAL), 16384)); + } + #[test] + fn a_large_stream_of_reused_pages_is_admitted() { + // 170000 launches (4 MB, inside the byte budget) over 16 argument pages. + let base = 2 * USC; + let words = launches(170000, |i| base + (i as u64 / 10625) * PAGE + (i as u64 % 1000) * 12); + assert_eq!(covers(&words), (Ok(()), 17)); } #[test] fn malformed_and_unreadable_streams_are_rejected() { - assert_eq!(gate::cdm(&vm(&[0xffffffff]), 0, 0), Err(EINVAL)); - assert_eq!(gate::cdm(&vm(&[0x20000000, 2]), 0, 0), Err(EINVAL)); - assert_eq!(gate::cdm(&mmu::Vm(vec![]), 0, 0), Err(EFAULT)); + assert_eq!(gate::cdm(&mut vm(&[0xffffffff]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&[0x20000000, 2]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut mmu::Vm(vec![]), 0, 0), Err(EFAULT)); } } ''' @@ -213,13 +298,44 @@ def main(): str(Path(__file__).with_name("pmp-bridge.c")), "-o", str(output)], check=True) subprocess.run([str(output)], check=True) + owner = (ROOT / "drivers/soc/apple/pmp-export.c").read_text() + start = owner.index("int apple_pmp_set_device_power(") + end = owner.index("\n}", start) + 2 + vote = owner[start:end] + start = owner.index("int apple_pmp_t6030_gpu_ready(") + end = owner.index("\n}", start) + 2 + (temp / "pmp-vote-production.h").write_text(vote + "\n" + owner[start:end]) + output = temp / "pmp-vote" + subprocess.run([os.environ.get("HOSTCC", "cc"), "-std=c11", "-Wall", "-Wextra", + "-Werror", "-I", str(temp), + str(Path(__file__).with_name("pmp-vote.c")), "-o", str(output)], check=True) + subprocess.run([str(output)], check=True) + # Native wire layout is a private per-backend facade. Its independent host test # checks that canonical normal/G17 UAPI remains unchanged; there is no global # Rust bindgen header selector in this integration. harness = RUST_PREFIX + f'\n#[path = "{G15 / "gate.rs"}"] mod gate;\n' + encode = (G15 / "encode.rs").read_text() + start = encode.index("pub(crate) const fn stamp_seed(") + end = encode.index("\n}", start) + 2 + harness += encode[start:end] + ''' +#[test] +fn firmware_stamp_wrap() { + for seed in [0xc1000000u32, 0x7a000000, 0x3d000000] { + let wrap = ((1u64 << 32) - seed as u64) / 0x100; + assert_eq!(stamp_seed(seed, 0), seed); + assert_eq!(stamp_seed(seed, wrap - 1), 0xffffff00); + assert_eq!(stamp_seed(seed, wrap), 0); + assert_eq!(stamp_seed(seed, wrap + 1), 0x100); + assert_eq!(stamp_seed(seed, 1 << 24), seed); + assert_eq!(stamp_seed(seed, u64::MAX), seed.wrapping_sub(0x100)); + } +} +''' harness += f'#[path = "{G15 / "recovery/logic.rs"}"] mod recovery;\n' harness += f'#[path = "{G15 / "power/idle.rs"}"] mod idle;\n' harness += f'#[path = "{G15 / "power/pstate.rs"}"] mod pstate;\n' + harness += f'#[path = "{G15 / "retain.rs"}"] mod retain;\n' harness += ("mod firmware_identity {\n" + f'#[path = "{G15 / "identity/matching.rs"}"] pub mod matching;\n' + f'#[path = "{G15 / "identity/lookup.rs"}"] pub mod lookup;\n' + "}\n") diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/uat-owner.c b/tools/testing/selftests/drivers/gpu/asahi-g15/uat-owner.c index 9909ac31e3a41..48e705c60b77f 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/uat-owner.c +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/uat-owner.c @@ -8,22 +8,58 @@ typedef uint32_t u32; typedef uint64_t u64; struct device_node { + const char *name; + struct device_node *child, *next; bool available, reusable, extra, bad_reg; + u64 start, size; + int refs; }; struct resource { u64 start, end; }; -static struct device_node node = { .available = true }; +static struct device_node nodes[5]; +static struct device_node rmem; +#define node nodes[0] static bool missing; static u64 start = 0x100000, size = 0x4000; -static int refs; +static inline struct device_node *of_node_get(struct device_node *np) +{ + if (np) + np->refs++; + return np; +} +static void of_node_put(struct device_node *np); static struct device_node *of_find_node_by_path(const char *path) { - assert(!strncmp(path, "/reserved-memory/uat-", 21)); + const char *prefix = "/reserved-memory/"; + if (missing) return NULL; - refs++; - return &node; + if (!strcmp(path, "/reserved-memory")) + return of_node_get(&rmem); + assert(!strncmp(path, prefix, strlen(prefix))); + for (struct device_node *np = rmem.child; np; np = np->next) + if (!strcmp(np->name, path + strlen(prefix))) + return of_node_get(np); + return NULL; +} +static inline struct device_node *of_get_next_child(struct device_node *parent, + struct device_node *prev) +{ + struct device_node *next = NULL; + + if (parent) + next = prev ? prev->next : parent->child; + of_node_put(prev); + return of_node_get(next); +} +#define for_each_child_of_node(parent, child) \ + for ((child) = of_get_next_child((parent), NULL); (child); \ + (child) = of_get_next_child((parent), (child))) +static inline bool of_node_name_eq(struct device_node *np, const char *name) +{ + return np && strlen(name) == strcspn(np->name, "@") && + !strncmp(np->name, name, strlen(name)); } static bool of_device_is_available(struct device_node *np) { @@ -44,8 +80,8 @@ static int of_address_to_resource(struct device_node *np, int index, struct resource *res) { assert(index == 0); - res->start = start; - res->end = start + size - 1; + res->start = np->start; + res->end = np->start + np->size - 1; return np->bad_reg ? -1 : 0; } static u64 resource_size(struct resource *res) @@ -55,14 +91,49 @@ static u64 resource_size(struct resource *res) static void of_node_put(struct device_node *np) { if (np) - refs--; + assert(np->refs-- > 0); } #include "uat-production.h" int main(void) { + static const char *const names[] = { + "uat-ttbs", "uat-pagetables", "uat-handoff", "uat-pagetables-l2" + }; + static const char *const unit_names[] = { + "uat-ttbs@10", "uat-pagetables@20", "uat-handoff@30", "uat-pagetables-l2@40" + }; + rmem.child = nodes; + for (u32 i = 0; i < 4; ++i) { + nodes[i].available = true; + nodes[i].start = start + i * size; + nodes[i].size = size; + nodes[i].name = names[i]; + nodes[i].next = i < 3 ? &nodes[i + 1] : NULL; + } + for (u32 i = 0; i < 4; ++i) + assert(asahi_g15_reserved_uat(i, nodes[i].start, size)); for (u32 i = 0; i < 4; ++i) - assert(asahi_g15_reserved_uat(i, start, size)); + nodes[i].name = unit_names[i]; + for (u32 i = 0; i < 4; ++i) + assert(asahi_g15_reserved_uat(i, nodes[i].start, size)); + nodes[3].next = &nodes[4]; + nodes[4].name = "uat-ttbs@99"; + nodes[4].available = true; + assert(!asahi_g15_reserved_uat(0, start, size)); + nodes[4].available = false; + assert(!asahi_g15_reserved_uat(0, start, size)); + nodes[3].next = NULL; + node.name = "uat-ttbs-other@10"; + assert(!asahi_g15_reserved_uat(0, start, size)); + node.name = unit_names[0]; + for (u32 i = 0; i < 4; ++i) { + const char *saved = nodes[i].name; + + nodes[i].name = "unrelated@10"; + assert(!asahi_g15_reserved_uat(i, nodes[i].start, size)); + nodes[i].name = saved; + } assert(!asahi_g15_reserved_uat(4, start, size)); assert(!asahi_g15_reserved_uat(0, start + 0x4000, size)); assert(!asahi_g15_reserved_uat(0, start, size + 0x4000)); @@ -80,6 +151,8 @@ int main(void) node.extra = false; node.bad_reg = true; assert(!asahi_g15_reserved_uat(0, start, size)); - assert(refs == 0); - puts("production UAT owner: exact ranges, unavailable/reusable/malformed/extra reservations: PASS"); + assert(rmem.refs == 0); + for (u32 i = 0; i < ARRAY_SIZE(nodes); ++i) + assert(nodes[i].refs == 0); + puts("production UAT owner: unit addresses, unique owners and exact ranges: PASS"); } diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py index 787992e9b4c24..124314ac83a5d 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py @@ -70,7 +70,7 @@ def body(source,marker): bindings=r''' #[derive(Clone,Copy)]struct Token(u32);impl Token{fn last_slot(&self)->u32{self.0}} struct Slot;impl Slot{fn changed(&self)->bool{true}fn slot(&self)->u32{3}fn token(&self)->Token{Token(3)}} -struct Allocator;impl Allocator{fn set_limit(&self,_l:Option){}fn get(&self,_t:Option)->Result{Ok(Slot)}} +struct Allocator;impl Allocator{fn set_limit(&self,_l:Option){}fn get(&self,_t:Option)->Result{Ok(Slot)}fn try_get(&self,_t:Option)->Result{Ok(Slot)}} struct VmBinding{active_users:usize,bind_token:Option,binding:Option,generation:u8,ttb:u64} #[derive(Clone)]struct Vm{id:u64,binding:Rc>} struct VmBind(Vm,u32); @@ -78,7 +78,8 @@ def body(source,marker): mod module_parameters{pub struct Param;impl Param{pub fn value(&self)->u32{0}}pub static robust_isolation:Param=Param;} ''' helpers='\n'.join(body(mm,m) for m in ['const fn binding_wire_generation(', 'const fn application_ttb1(']) -bind='impl Uat{'+body(mm,'pub(crate) fn bind(&self, vm: &Vm)')+'}\n' +# bind() is the waiting form of bind_inner(); G15's try_bind() is the other. +bind='impl Uat{'+body(mm,'pub(crate) fn bind(&self, vm: &Vm)')+'\n'+body(mm,'fn bind_inner(&self, vm: &Vm, wait: bool)')+'}\n' vm=r''' struct VmInner{id:u64,binding:Rc>,uat_inner:Rc,page_table:UatPageTable} impl VmInner{fn ttb(&self)->u64{self.page_table.ttb}}